ffi_gen 0.8 → 1.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Richard Musiol
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+ffi_gen - A Generator for Ruby FFI bindings
+===========================================
+
+*Author:* Richard Musiol  
+*Contributors:* Jeremy Voorhis (thanks for the initial idea)  
+*License:* MIT (see LICENSE)
+
+
+Features
+--------
+* Generation of FFI methods, structures, enums and callbacks
+* Generation of YARD documentation comments
+* Tested with headers of the following libraries:
+  * Clang
+  * LLVM
+  * OpenGL
+
+
+Requirements
+------------
+
+* Ruby 1.9
+* Clang 3.0 ([Download](http://llvm.org/releases/download.html#3.0), use the binaries or configure with ``--enable-shared``)
+
+*These requirements are only for running the generator. The generated files are Ruby 1.8 compatible and do not need Clang.*
+
+
+Example
+-------
+Use the following interface in a script or Rake task:
+
+    require "ffi_gen"
+    
+    FFIGen.generate(
+      ruby_module: "Clang",
+      ffi_lib:     "clang",
+      headers:     ["clang-c/Index.h"],
+      cflags:      `llvm-config --cflags`.split(" "),
+      prefixes:    ["clang_", "CX"],
+      blacklist:   ["clang_getExpansionLocation"],
+      output:      "clang.rb"
+    )
+
+Output: [clang.rb](https://github.com/neelance/ffi_gen/blob/master/lib/ffi_gen/clang.rb)
+
+
+Hints
+-----
+
+You may need to set additional include directories:
+
+    export CPATH=/usr/lib/gcc/x86_64-linux-gnu/4.6.1/include
+
+Your GCC include paths can be seen with:
+
+    `gcc -print-prog-name=cc1` -v
+
+
+Projects using ffi_gen
+----------------------
+
+* https://github.com/jvoorhis/ruby-llvm
+
+
+Roadmap
+-------
+
+* Support for more libraries:
+  * Cairo
+  * (Write me if you have a whish)
+* Automatic generation of object oriented wrappers
+* Polish YARD documentation comments some more
+
+
+Feedback
+--------
+Please use GitHub's issue tracker for problems or suggestions. Pull requests are welcome, too.

data/lib/ffi_gen.rb

@@ -24,6 +24,8 @@ class Clang::String
 end
 
 class FFIGen
+  RUBY_KEYWORDS = %w{alias allocate and begin break case class def defined do else elsif end ensure false for if in initialize module next nil not or redo rescue retry return self super then true undef unless until when while yield}
+
   class Enum
     attr_reader :constants
     
@@ -34,55 +36,63 @@ class FFIGen
       @constants = []
     end
     
-    def to_s
+    def write(writer)
       prefix_length = 0
       suffix_length = 0
       
       unless @constants.size < 2
-        search_pattern = @constants.all? { |constant| constant[0].include? "_" } ? /(?<=_)/ : /[A-Z]/
-        first_name = @constants.first[0]
+        search_pattern = @constants.all? { |constant| constant[:name].include? "_" } ? /(?<=_)/ : /[A-Z]/
+        first_name = @constants.first[:name]
         
         loop do
           position = first_name.index(search_pattern, prefix_length + 1) or break
           prefix = first_name[0...position]
-          break if not @constants.all? { |constant| constant[0].start_with? prefix }
+          break if not @constants.all? { |constant| constant[:name].start_with? prefix }
           prefix_length = position
         end
         
         loop do
           position = first_name.rindex(search_pattern, first_name.size - suffix_length - 1) or break
           prefix = first_name[position..-1]
-          break if not @constants.all? { |constant| constant[0].end_with? prefix }
+          break if not @constants.all? { |constant| constant[:name].end_with? prefix }
           suffix_length = first_name.size - position
         end
       end
       
-      symbols = []
-      definitions = []
-      symbol_descriptions = []
-      @constants.map do |(constant_name, constant_value, constant_comment)|
-        symbol = ":#{@generator.to_ruby_lowercase constant_name[prefix_length..(-1 - suffix_length)]}"
-        symbols << symbol
-        definitions << "    #{symbol}#{constant_value ? ", #{constant_value}" : ""}"
-        symbol_descriptions << "  # #{symbol} ::\n  #   #{@generator.create_description_comment(constant_comment, '  #   ', true)}\n"
+      @constants.each do |constant|
+        constant[:symbol] = ":#{@generator.to_ruby_lowercase constant[:name][prefix_length..(-1 - suffix_length)]}"
       end
       
-      str = ""
-      str << @generator.create_description_comment(@comment, '  # ')
-      str << "  # \n"
-      str << "  # === Options:\n#{symbol_descriptions.join}  #\n"
-      str << "  # @return [Array<Symbol>]\n"
-      str << "  def self.#{@generator.to_ruby_lowercase @name}_enum\n    [#{symbols.join(', ')}]\n  end\n"
-      str << "  enum :#{@generator.to_ruby_lowercase @name}, [\n#{definitions.join(",\n")}\n  ]"
-      str
+      writer.comment do
+        writer.write_description @comment
+        writer.puts "", "<em>This entry is only for documentation and no real method. The FFI::Enum can be accessed via #enum_type(:#{ruby_name}).</em>"
+        writer.puts "", "=== Options:"
+        @constants.each do |constant|
+          writer.puts "#{constant[:symbol]} ::"
+          writer.write_description constant[:comment], false, "  ", "  "
+        end
+        writer.puts "", "@method _enum_#{ruby_name}_", "@return [Symbol]", "@scope class"
+      end
+      
+      writer.puts "enum :#{ruby_name}, ["
+      writer.indent do
+        writer.write_array @constants, "," do |constant|
+          "#{constant[:symbol]}#{constant[:value] ? ", #{constant[:value]}" : ''}"
+        end
+      end
+      writer.puts "]", ""
+    end
+    
+    def ruby_name
+      @ruby_name ||= @generator.to_ruby_lowercase @name
     end
     
     def type_name(short)
-      short ? @name : "Symbol from #{@generator.to_ruby_lowercase @name}_enum"
+      short ? @name : "Symbol from _enum_#{ruby_name}_"
     end
     
     def reference
-      ":#{@generator.to_ruby_lowercase @name}"
+      ":#{ruby_name}"
     end
   end
   
@@ -96,31 +106,41 @@ class FFIGen
       @fields = []
     end
     
-    def to_s
-      field_definitions = []
-      field_descriptions = []
-      @fields.each do |(field_name, field_type, field_comment)|
-        symbol = ":#{@generator.to_ruby_lowercase field_name}"
-        field_definitions << "#{symbol}, #{@generator.to_ffi_type field_type}"
-        field_descriptions << "  # #{symbol} ::\n  #   (#{@generator.to_type_name field_type}) #{@generator.create_description_comment(field_comment, '  #   ', true)}\n"
+    def write(writer)
+      @fields.each do |field|
+        field[:symbol] = ":#{@generator.to_ruby_lowercase field[:name]}"
       end
       
-      str = ""
-      str << @generator.create_description_comment(@comment, '  # ')
-      str << "  # \n"
-      str << "  # = Fields:\n#{field_descriptions.join}  #\n"
-      str << "  class #{@generator.to_ruby_camelcase @name} < FFI::Struct\n"
-      str << "    layout #{field_definitions.join(",\n           ")}\n" unless @fields.empty?
-      str << "  end"
-      str
+      writer.comment do
+        writer.write_description @comment
+        unless @fields.empty?
+          writer.puts "", "= Fields:"
+          @fields.each do |field|
+            writer.puts "#{field[:symbol]} ::"
+            writer.write_description field[:comment], false, "  (#{@generator.to_type_name field[:type]}) ", "  "
+          end
+        end
+      end
+      
+      writer.puts "class #{ruby_name} < FFI::Struct"
+      writer.indent do
+        writer.write_array @fields, ",", "layout ", "       " do |field|
+          "#{field[:symbol]}, #{@generator.to_ffi_type field[:type]}"
+        end
+      end
+      writer.puts "end", ""
+    end
+    
+    def ruby_name
+      @ruby_name ||= @generator.to_ruby_camelcase @name
     end
     
     def type_name(short)
-      @generator.to_ruby_camelcase @name
+      ruby_name
     end
     
     def reference
-      "#{type_name(false)}.by_value"
+      "#{ruby_name}.by_value"
     end
   end
   
@@ -136,25 +156,22 @@ class FFIGen
       @comment = comment
     end
     
-    def to_s
-      ruby_name = @generator.to_ruby_lowercase @name
-      ruby_parameters = @parameters.map do |(name, type)|
-        ruby_param_type = @generator.to_type_name type
-        ruby_param_name = @generator.to_ruby_lowercase(name.empty? ? @generator.to_type_name(type, true) : name)
-        [ruby_param_name, ruby_param_type, []]
+    def write(writer)
+      @parameters.each do |parameter|
+        parameter[:ruby_type] = @generator.to_type_name parameter[:type]
+        parameter[:ruby_name] = @generator.to_ruby_lowercase(parameter[:name].empty? ? @generator.to_type_name(parameter[:type], true) : parameter[:name])
+        parameter[:description] = []
       end
       
-      ffi_signature = "[#{@parameters.map{ |(name, type)| @generator.to_ffi_type type }.join(', ')}], #{@generator.to_ffi_type @return_type}"
-      
       function_description = []
       return_value_description = []
       current_description = function_description
       @comment.split("\n").map do |line|
-        line = @generator.prepare_comment_line line
+        line = writer.prepare_comment_line line
         if line.gsub! /\\param (.*?) /, ''
-          index = @parameters.index { |(name, type)| name == $1 }
-          if index
-            current_description = ruby_parameters[index][2]
+          parameter = @parameters.find { |parameter| parameter[:name] == $1 }
+          if parameter
+            current_description = parameter[:description]
           else
             current_description << "#{$1}: "
           end
@@ -163,33 +180,102 @@ class FFIGen
         current_description << line
       end
       
-      str = ""
-      if @is_callback
-        str << "  # <em>This is no real method. This entry is only for documentation of the callback.</em>\n"
-        str << "  # \n"
-      end
-      str << @generator.create_description_comment(function_description, '  # ')
-      str << "  # \n"
-      str << "  # @method #{ruby_name}#{@is_callback ? '_callback' : ''}(#{ruby_parameters.map{ |(name, type, description)| name }.join(', ')})\n"
-      ruby_parameters.each do |(name, type, description)|
-        str << "  # @param [#{type}] #{name} #{@generator.create_description_comment(description, '  #   ', true)}\n"
+      writer.comment do
+        writer.write_description function_description
+        writer.puts "", "<em>This entry is only for documentation and no real method.</em>" if @is_callback
+        writer.puts "", "@method #{@is_callback ? "_callback_#{ruby_name}_" : ruby_name}(#{@parameters.map{ |parameter| parameter[:ruby_name] }.join(', ')})"
+        @parameters.each do |parameter|
+          writer.write_description parameter[:description], false, "@param [#{parameter[:ruby_type]}] #{parameter[:ruby_name]} ", "  "
+        end
+        writer.write_description return_value_description, false, "@return [#{@generator.to_type_name @return_type}] ", "  "
+        writer.puts "@scope class"
       end
-      str << "  # @return [#{@generator.to_type_name @return_type}] #{@generator.create_description_comment(return_value_description, '  #   ', true)}\n"
-      str << "  # @scope class\n"
+      
+      ffi_signature = "[#{@parameters.map{ |parameter| @generator.to_ffi_type parameter[:type] }.join(', ')}], #{@generator.to_ffi_type @return_type}"
       if @is_callback
-        str << "  callback :#{ruby_name}, #{ffi_signature}"
+        writer.puts "callback :#{ruby_name}, #{ffi_signature}", ""
       else
-        str << "  attach_function :#{ruby_name}, :#{@name}, #{ffi_signature}"
+        writer.puts "attach_function :#{ruby_name}, :#{@name}, #{ffi_signature}", ""
       end
-      str
+    end
+    
+    def ruby_name
+      @ruby_name ||= @generator.to_ruby_lowercase @name, true
     end
     
     def type_name(short)
-      "Proc(#{@generator.to_ruby_lowercase @name}_callback)"
+      "Proc(_callback_#{ruby_name}_)"
     end
     
     def reference
-      ":#{@generator.to_ruby_lowercase @name}"
+      ":#{ruby_name}"
+    end
+  end
+  
+  class Constant
+    def initialize(generator, name, value)
+      @generator = generator
+      @name = name
+      @value = value
+    end
+    
+    def write(writer)
+      writer.puts "#{@generator.to_ruby_lowercase(@name, true).upcase} = #{@value}", ""
+    end
+  end
+  
+  class Writer
+    attr_reader :output
+    
+    def initialize
+      @indentation = ""
+      @output = ""
+    end
+    
+    def indent(prefix = "  ")
+      previous_indentation = @indentation
+      @indentation += prefix
+      yield
+      @indentation = previous_indentation
+    end
+    
+    def comment(&block)
+      indent "# ", &block
+    end
+    
+    def puts(*lines)
+      lines.each do |line|
+        @output << "#{@indentation}#{line}\n"
+      end
+    end
+    
+    def write_array(array, separator = "", first_line_prefix = "", other_lines_prefix = "")
+      array.each_with_index do |entry, index|
+        entry = yield entry if block_given?
+        puts "#{index == 0 ? first_line_prefix : other_lines_prefix}#{entry}#{index < array.size - 1 ? separator : ''}"
+      end
+    end
+    
+    def prepare_comment_line(line)
+      line = line.dup
+      line.sub! /\ ?\*+\/\s*$/, ''
+      line.sub! /^\s*\/?\*+ ?/, ''
+      line.gsub! /\\(brief|determine) /, ''
+      line.gsub! '[', '('
+      line.gsub! ']', ')'
+      line
+    end
+    
+    def write_description(description, not_documented_message = true, first_line_prefix = "", other_lines_prefix = "")
+      if description.is_a? String
+        description = description.split("\n").map { |line| prepare_comment_line(line) }
+      end
+      
+      description.shift while not description.empty? and description.first.strip.empty?
+      description.pop while not description.empty? and description.last.strip.empty?
+      description << (not_documented_message ? "(Not documented)" : "") if description.empty?
+      
+      write_array description, "", first_line_prefix, other_lines_prefix
     end
   end
   
@@ -204,6 +290,9 @@ class FFIGen
     @blacklist   = options.fetch :blacklist, []
     @output      = options.fetch :output, $stdout
     
+    blacklist = @blacklist
+    @blacklist = lambda { |name| blacklist.include? name } if @blacklist.is_a? Array
+    
     @translation_unit = nil
     @declarations = nil
   end
@@ -221,7 +310,7 @@ class FFIGen
     args_ptr.write_array_of_pointer pointers
     
     index = Clang.create_index 0, 0
-    @translation_unit = Clang.parse_translation_unit index, File.join(File.dirname(__FILE__), "ffi_gen/empty.h"), args_ptr, args.size, nil, 0, 0
+    @translation_unit = Clang.parse_translation_unit index, File.join(File.dirname(__FILE__), "ffi_gen/empty.h"), args_ptr, args.size, nil, 0, Clang.enum_type(:translation_unit_flags)[:detailed_preprocessing_record]
     
     Clang.get_num_diagnostics(@translation_unit).times do |i|
       diag = Clang.get_diagnostic @translation_unit, i
@@ -257,7 +346,7 @@ class FFIGen
       next if not header_files.include? file
       
       name = Clang.get_cursor_spelling(declaration).to_s_and_dispose
-      next if blacklist.include? name
+      next if @blacklist[name]
       
       comment = extract_comment translation_unit, comment_range
       
@@ -274,7 +363,7 @@ class FFIGen
           next if function_child[:kind] != :parm_decl
           param_name = Clang.get_cursor_spelling(function_child).to_s_and_dispose
           param_type = Clang.get_cursor_type function_child
-          function.parameters << [param_name, param_type]
+          function.parameters << { name: param_name, type: param_type}
         end
       
       when :typedef_decl
@@ -290,10 +379,25 @@ class FFIGen
           typedef_children[1..-1].each do |param_decl|
             param_name = Clang.get_cursor_spelling(param_decl).to_s_and_dispose
             param_type = Clang.get_cursor_type param_decl
-            callback.parameters << [param_name, param_type]
+            callback.parameters << { name: param_name, type: param_type }
           end
         end
+      
+      when :macro_definition
+        tokens_ptr_ptr = FFI::MemoryPointer.new :pointer
+        num_tokens_ptr = FFI::MemoryPointer.new :uint
+        Clang.tokenize translation_unit, extent, tokens_ptr_ptr, num_tokens_ptr
+        num_tokens = num_tokens_ptr.read_uint
+        tokens_ptr = FFI::Pointer.new Clang::Token, tokens_ptr_ptr.read_pointer
         
+        if num_tokens == 3
+          token = Clang::Token.new tokens_ptr[1]
+          if Clang.get_token_kind(token) == :literal
+            value = Clang.get_token_spelling(translation_unit, token).to_s_and_dispose
+            @declarations[name] = Constant.new self, name, value
+          end 
+        end
+            
       end
     end
 
@@ -301,12 +405,20 @@ class FFIGen
   end
   
   def generate
-    content = "# Generated by ffi_gen. Please do not change this file by hand.\n\nrequire 'ffi'\n\nmodule #{@ruby_module}\n  extend FFI::Library\n  ffi_lib '#{@ffi_lib}'\n\n#{declarations.values.join("\n\n")}\n\nend"
+    writer = Writer.new
+    writer.puts "# Generated by ffi_gen. Please do not change this file by hand.", "", "require 'ffi'", "", "module #{@ruby_module}"
+    writer.indent do
+      writer.puts "extend FFI::Library", "ffi_lib '#{@ffi_lib}'", ""
+      declarations.each do |name, declaration|
+        declaration.write writer
+      end
+    end
+    writer.puts "end"
     if @output.is_a? String
-      File.open(@output, "w") { |file| file.write content }
+      File.open(@output, "w") { |file| file.write writer.output }
       puts "ffi_gen: #{@output}"
     else
-      @output.write content
+      @output.write writer.output
     end
   end
   
@@ -340,7 +452,7 @@ class FFIGen
         constant_comment = extract_comment translation_unit, constant_comment_range
         previous_constant_location = constant_location
         
-        enum.constants << [constant_name, constant_value, constant_comment]
+        enum.constants << { name: constant_name, value: constant_value, comment: constant_comment }
       end
       
     when :struct_decl
@@ -357,7 +469,7 @@ class FFIGen
         field_comment = extract_comment translation_unit, field_comment_range
         previous_field_location = field_location
         
-        struct.fields << [field_name, field_type, field_comment]
+        struct.fields << { name: field_name, type: field_type, comment: field_comment }
       end
     end
   end
@@ -389,6 +501,7 @@ class FFIGen
     when :u_int then ":uint"
     when :u_long then ":ulong"
     when :u_long_long then ":ulong_long"
+    when :char_s, :s_char then ":char"
     when :short then ":short"
     when :int then ":int"
     when :long then ":long"
@@ -416,7 +529,7 @@ class FFIGen
     case canonical_type[:kind]
     when :void then "nil"
     when :bool then "Boolean"
-    when :u_char, :u_short, :u_int, :u_long, :u_long_long, :short, :int, :long, :long_long then "Integer"
+    when :u_char, :u_short, :u_int, :u_long, :u_long_long, :char_s, :s_char, :short, :int, :long, :long_long then "Integer"
     when :float, :double then "Float"
     when :pointer
       pointee_type = Clang.get_pointee_type canonical_type
@@ -452,7 +565,7 @@ class FFIGen
     end
   end
   
-  def to_ruby_lowercase(str)
+  def to_ruby_lowercase(str, avoid_keywords = false)
     str = str.dup
     str.sub! /^(#{@prefixes.join('|')})/, '' # remove prefixes
     str.gsub! /([A-Z][a-z])/, '_\1' # add underscores before word beginnings
@@ -460,6 +573,8 @@ class FFIGen
     str.sub! /^_*/, '' # remove underscores at the beginning
     str.gsub! /__+/, '_' # replace multiple underscores by only one
     str.downcase!
+    str.sub! /^\d/, '_\1' # fix illegal beginnings
+    str = "_#{str}" if avoid_keywords and RUBY_KEYWORDS.include? str
     str
   end
   
@@ -469,34 +584,6 @@ class FFIGen
     str
   end
   
-  def prepare_comment_line(line)
-    line = line.dup
-    line.sub! /\ ?\*+\/\s*$/, ''
-    line.sub! /^\s*\/?\*+ ?/, ''
-    line.gsub! /\\(brief|determine) /, ''
-    line.gsub! '[', '('
-    line.gsub! ']', ')'
-    line
-  end
-  
-  def create_description_comment(description, line_prefix, inline_mode = false)
-    if description.is_a? String
-      description = description.split("\n").map { |line| prepare_comment_line(line) }
-    end
-    
-    description.shift while not description.empty? and description.first.strip.empty?
-    description.pop while not description.empty? and description.last.strip.empty?
-    description << "(Not documented)" if not inline_mode and description.empty?
-    
-    str = ""
-    description.each_with_index do |line, index|
-      str << line_prefix if not inline_mode or index > 0
-      str << line
-      str << "\n" if not inline_mode or index < description.size - 1
-    end
-    str
-  end
-  
   def self.generate(options = {})
     self.new(options).generate
   end