docurium 0.3.2 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 30fc3d67acaf76249929871ddf63e7e3d08d079b
+  data.tar.gz: d39f6a4acaf9c0dc2ce836db02969961650e1cd9
+SHA512:
+  metadata.gz: 40b673b51c990669d7dff887107b4a43f9318ffa19009b83e18490471bfbc09cfec2b987d69a890b0a566a05a620b5249e2155dab7bed6aaf4df5469f939683f
+  data.tar.gz: a90d06b292c137d45cb05f8dc64e98767335514c35af98f348dd4e3ff75ec9d127a163afe779137e8bb85d0137d7f9a125b14e34c996a4018aa1a90c33e22bbe

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+ - ruby-2.0
+ - ruby-2.1
+ - ruby-head
+ - rbx
+
+env:
+ - LLVM_CONFIG=llvm-config-3.3
+
+before_install:
+ - sudo apt-get update
+ - sudo apt-get install libclang-3.3-dev llvm-3.3

data/Gemfile

@@ -1,5 +1,11 @@
 source "http://rubygems.org"
 
+platforms :rbx do
+  gem 'rubysl', '~> 2.0'
+end
+
 gemspec
 
+gem 'ffi-clang', :git => 'https://github.com/ioquatix/ffi-clang.git'
+
 # vim:ft=ruby

data/Rakefile

@@ -1,7 +1,13 @@
 require 'rake/testtask'
+require 'rubygems'
+require 'rubygems/package_task'
 
 task :default => :test
 
+gemspec = Gem::Specification::load(File.expand_path('../docurium.gemspec', __FILE__))
+Gem::PackageTask.new(gemspec) do |pkg|
+end
+
 Rake::TestTask.new do |t|
   t.libs << 'libs' << 'test'
   t.pattern = 'test/**/*_test.rb'

data/docurium.gemspec

@@ -13,13 +13,15 @@ Gem::Specification.new do |s|
   s.description = s.summary
   s.license = 'MIT'
 
-  s.add_dependency "version_sorter", "~>1.1.0"
-  s.add_dependency "mustache", ">= 0.99.4"
+  s.add_dependency "version_sorter", "~>1.1"
+  s.add_dependency "mustache", "~> 0.99"
   s.add_dependency "rocco", "~>0.8"
   s.add_dependency "gli", "~>2.5"
-  s.add_dependency "rugged", "~>0.19"
+  s.add_dependency "rugged", "~>0.21"
   s.add_dependency "redcarpet", "~>3.0"
+  s.add_dependency "ffi-clang", "~> 0.2"
   s.add_development_dependency "bundler",   "~>1.0"
+  s.add_development_dependency "rake", "~> 10.1"
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/docurium.rb

@@ -4,10 +4,12 @@ require 'version_sorter'
 require 'rocco'
 require 'docurium/version'
 require 'docurium/layout'
-require 'docurium/cparser'
+require 'libdetect'
+require 'docurium/docparser'
 require 'pp'
 require 'rugged'
 require 'redcarpet'
+require 'thread'
 
 # Markdown expects the old redcarpet compat API, so let's tell it what
 # to use
@@ -20,19 +22,13 @@ class Docurium
     raise "You need to specify a config file" if !config_file
     raise "You need to specify a valid config file" if !valid_config(config_file)
     @sigs = {}
-    @groups = {}
-    if repo
-      @repo = repo
-    else
-      repo_path = Rugged::Repository.discover('.')
-      @repo = Rugged::Repository.new(repo_path)
-    end
-    clear_data
+    @repo = repo || Rugged::Repository.discover('.')
   end
 
-  def clear_data(version = 'HEAD')
-    @data = {:files => [], :functions => {}, :globals => {}, :types => {}, :prefix => ''}
-    @data[:prefix] = option_version(version, 'input', '')
+  def init_data(version = 'HEAD')
+    data = {:files => [], :functions => {}, :globals => {}, :types => {}, :prefix => ''}
+    data[:prefix] = option_version(version, 'input', '')
+    data
   end
 
   def option_version(version, option, default = nil)
@@ -48,81 +44,136 @@ class Docurium
     opt
   end
 
+  def format_examples!(data, version)
+    examples = []
+    if ex = option_version(version, 'examples')
+      if subtree = find_subtree(version, ex) # check that it exists
+        index = Rugged::Index.new
+        index.read_tree(subtree)
+
+        files = []
+        index.each do |entry|
+          next unless entry[:path].match(/\.c$/)
+          files << entry[:path]
+        end
+
+        files.each do |file|
+          # highlight, roccoize and link
+          rocco = Rocco.new(file, files, {:language => 'c'}) do
+            ientry = index[file]
+            blob = @repo.lookup(ientry[:oid])
+            blob.content
+          end
+          rocco_layout = Rocco::Layout.new(rocco, @tf)
+          rocco_layout.version = version
+          rf = rocco_layout.render
+
+          extlen = -(File.extname(file).length + 1)
+          rf_path = file[0..extlen] + '.html'
+          rel_path = "ex/#{version}/#{rf_path}"
+
+          # look for function names in the examples and link
+          id_num = 0
+          data[:functions].each do |f, fdata|
+            rf.gsub!(/#{f}([^\w])/) do |fmatch|
+              extra = $1
+              id_num += 1
+              name = f + '-' + id_num.to_s
+              # save data for cross-link
+              data[:functions][f][:examples] ||= {}
+              data[:functions][f][:examples][file] ||= []
+              data[:functions][f][:examples][file] << rel_path + '#' + name
+              "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
+            end
+          end
+
+          # write example to the repo
+          sha = @repo.write(rf, :blob)
+          examples << [rel_path, sha]
+
+          data[:examples] ||= []
+          data[:examples] << [file, rel_path]
+        end
+      end
+    end
+
+    examples
+  end
+
+  def generate_doc_for(version)
+    index = Rugged::Index.new
+    read_subtree(index, version, option_version(version, 'input', ''))
+    data = parse_headers(index, version)
+    data
+  end
+
   def generate_docs
-    out "* generating docs"
     output_index = Rugged::Index.new
     write_site(output_index)
+    @tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
     versions = get_versions
     versions << 'HEAD'
+    nversions = versions.size
+    output = Queue.new
+    pipes = {}
     versions.each do |version|
-      out "  - processing version #{version}"
-      index = @repo.index
-      index.clear
-      clear_data(version)
-      read_subtree(index, version, @data[:prefix])
-      parse_headers(index)
-      tally_sigs(version)
-
-      tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
-      if ex = option_version(version, 'examples')
-        if subtree = find_subtree(version, ex) # check that it exists
-          index.read_tree(subtree)
-          out "  - processing examples for #{version}"
-
-          files = []
-          index.each do |entry|
-            next unless entry[:path].match(/\.c$/)
-            files << entry[:path]
-          end
+      # We don't need to worry about joining since this process is
+      # going to die immediately
+      read, write = IO.pipe
+      pid = Process.fork do
+        read.close
 
-          files.each do |file|
-            out "    # #{file}"
+        data = generate_doc_for(version)
+        examples = format_examples!(data, version)
 
-            # highlight, roccoize and link
-            rocco = Rocco.new(file, files, {:language => 'c'}) do
-              ientry = index[file]
-              blob = @repo.lookup(ientry[:oid])
-              blob.content
-            end
-            rocco_layout = Rocco::Layout.new(rocco, tf)
-            rocco_layout.version = version
-            rf = rocco_layout.render
-
-            extlen = -(File.extname(file).length + 1)
-            rf_path = file[0..extlen] + '.html'
-            rel_path = "ex/#{version}/#{rf_path}"
-
-            # look for function names in the examples and link
-            id_num = 0
-            @data[:functions].each do |f, fdata|
-              rf.gsub!(/#{f}([^\w])/) do |fmatch|
-                extra = $1
-                id_num += 1
-                name = f + '-' + id_num.to_s
-                # save data for cross-link
-                @data[:functions][f][:examples] ||= {}
-                @data[:functions][f][:examples][file] ||= []
-                @data[:functions][f][:examples][file] << rel_path + '#' + name
-                "<a name=\"#{name}\" class=\"fnlink\" href=\"../../##{version}/group/#{fdata[:group]}/#{f}\">#{f}</a>#{extra}"
-              end
-            end
+        Marshal.dump([version, data, examples], write)
+        write.close
+      end
 
-            # write example to the repo
-            sha = @repo.write(rf, :blob)
-            output_index.add(:path => rel_path, :oid => sha, :mode => 0100644)
+      pipes[pid] = read
+      write.close
+    end
 
-            @data[:examples] ||= []
-            @data[:examples] << [file, rel_path]
-          end
-        end
+    print "Generating documentation [0/#{nversions}]\r"
+    head_data = nil
+
+    # This may seem odd, but we need to keep reading from the pipe or
+    # the buffer will fill and they'll block and never exit. Therefore
+    # we can't rely on Process.wait to tell us when the work is
+    # done. Instead read from all the pipes concurrently and send the
+    # ruby objects through the queue.
+    Thread.abort_on_exception = true
+    pipes.each do |pid, read|
+      Thread.new do
+        result = read.read
+        output << Marshal.load(result)
+      end
+    end
 
-        if version == 'HEAD'
-          show_warnings
-        end
+    for i in 1..nversions
+      version, data, examples = output.pop
+
+      # There's still some work we need to do serially
+      tally_sigs!(version, data)
+      sha = @repo.write(data.to_json, :blob)
+
+      print "Generating documentation [#{i}/#{nversions}]\r"
+
+      # Store it so we can show it at the end
+      if version == 'HEAD'
+        head_data = data
       end
 
-      sha = @repo.write(@data.to_json, :blob)
       output_index.add(:path => "#{version}.json", :oid => sha, :mode => 0100644)
+      examples.each do |path, id|
+        output_index.add(:path => path, :oid => id, :mode => 0100644)
+      end
+
+      if head_data
+        puts ''
+        show_warnings(data)
+      end
+
     end
 
     project = {
@@ -130,7 +181,6 @@ class Docurium
       :github   => @options['github'],
       :name     => @options['name'],
       :signatures => @sigs,
-      :groups   => @groups
     }
     sha = @repo.write(project.to_json, :blob)
     output_index.add(:path => "project.json", :oid => sha, :mode => 0100644)
@@ -140,7 +190,7 @@ class Docurium
     refname = "refs/heads/#{br}"
     tsha = output_index.write_tree(@repo)
     puts "\twrote tree   #{tsha}"
-    ref = Rugged::Reference.lookup(@repo, refname)
+    ref = @repo.references[refname]
     user = { :name => @repo.config['user.name'], :email => @repo.config['user.email'], :time => Time.now }
     options = {}
     options[:tree] = tsha
@@ -154,12 +204,12 @@ class Docurium
     puts "\tupdated #{br}"
   end
 
-  def show_warnings
+  def show_warnings(data)
     out '* checking your api'
 
     # check for unmatched paramaters
     unmatched = []
-    @data[:functions].each do |f, fdata|
+    data[:functions].each do |f, fdata|
       unmatched << f if fdata[:comments] =~ /@param/
     end
     if unmatched.size > 0
@@ -181,27 +231,35 @@ class Docurium
   end
 
   def get_versions
-    tags = []
-    @repo.tags.each { |tag| tags << tag.gsub(%r(^refs/tags/), '') }
-    VersionSorter.sort(tags)
+    VersionSorter.sort(@repo.tags.map { |tag| tag.name.gsub(%r(^refs/tags/), '') })
   end
 
-  def parse_headers(index)
-    headers(index).each do |header|
-      records = parse_header(index, header)
-      update_globals(records)
+  def parse_headers(index, version)
+    headers = index.map { |e| e[:path] }.grep(/\.h$/)
+
+    files = headers.map do |file|
+      [file, @repo.lookup(index[file][:oid]).content]
     end
 
-    @data[:groups] = group_functions
-    @data[:types] = @data[:types].sort # make it an assoc array
-    find_type_usage
+    data = init_data(version)
+    parser = DocParser.new
+    headers.each do |header|
+      records = parser.parse_file(header, files)
+      update_globals!(data, records)
+    end
+
+    data[:groups] = group_functions!(data)
+    data[:types] = data[:types].sort # make it an assoc array
+    find_type_usage!(data)
+
+    data
   end
 
   private
 
-  def tally_sigs(version)
+  def tally_sigs!(version, data)
     @lastsigs ||= {}
-    @data[:functions].each do |fun_name, fun_data|
+    data[:functions].each do |fun_name, fun_data|
       if !@sigs[fun_name]
         @sigs[fun_name] ||= {:exists => [], :changes => {}}
       else
@@ -217,10 +275,10 @@ class Docurium
   def find_subtree(version, path)
     tree = nil
     if version == 'HEAD'
-      tree = @repo.lookup(@repo.head.target).tree
+      tree = @repo.head.target.tree
     else
-      trg = @repo.lookup(Rugged::Reference.lookup(@repo, "refs/tags/#{version}").target)
-      if(trg.class == Rugged::Tag)
+      trg = @repo.references["refs/tags/#{version}"].target
+      if(trg.kind_of? Rugged::Tag::Annotation)
         trg = trg.target
       end
 
@@ -249,9 +307,9 @@ class Docurium
     !!@options['branch']
   end
 
-  def group_functions
+  def group_functions!(data)
     func = {}
-    @data[:functions].each_pair do |key, value|
+    data[:functions].each_pair do |key, value|
       if @options['prefix']
         k = key.gsub(@options['prefix'], '')
       else
@@ -262,8 +320,7 @@ class Docurium
       if !rest
         group = value[:file].gsub('.h', '').gsub('/', '_')
       end
-      @data[:functions][key][:group] = group
-      @groups[key] = group
+      data[:functions][key][:group] = group
       func[group] ||= []
       func[group] << key
       func[group].sort!
@@ -272,45 +329,31 @@ class Docurium
     func.to_a.sort
   end
 
-  def headers(index = nil)
-    h = []
-    index.each do |entry|
-      next unless entry[:path].match(/\.h$/)
-      h << entry[:path]
-    end
-    h
-  end
-
-  def find_type_usage
+  def find_type_usage!(data)
     # go through all the functions and see where types are used and returned
     # store them in the types data
-    @data[:functions].each do |func, fdata|
-      @data[:types].each_with_index do |tdata, i|
+    data[:functions].each do |func, fdata|
+      data[:types].each_with_index do |tdata, i|
         type, typeData = tdata
-        @data[:types][i][1][:used] ||= {:returns => [], :needs => []}
+        data[:types][i][1][:used] ||= {:returns => [], :needs => []}
         if fdata[:return][:type].index(/#{type}[ ;\)\*]/)
-          @data[:types][i][1][:used][:returns] << func
-          @data[:types][i][1][:used][:returns].sort!
+          data[:types][i][1][:used][:returns] << func
+          data[:types][i][1][:used][:returns].sort!
         end
         if fdata[:argline].index(/#{type}[ ;\)\*]/)
-          @data[:types][i][1][:used][:needs] << func
-          @data[:types][i][1][:used][:needs].sort!
+          data[:types][i][1][:used][:needs] << func
+          data[:types][i][1][:used][:needs].sort!
         end
       end
     end
   end
 
-  def parse_header(index, path)
-    id = index[path][:oid]
-    blob = @repo.lookup(id)
-    parser = Docurium::CParser.new
-    parser.parse_text(path, blob.content)
-  end
+  def update_globals!(data, recs)
+    return if recs.empty?
 
-  def update_globals(recs)
     wanted = {
       :functions => %W/type value file line lineto args argline sig return group description comments/.map(&:to_sym),
-      :types => %W/type value file line lineto block tdef comments/.map(&:to_sym),
+      :types => %W/decl type value file line lineto block tdef description comments fields/.map(&:to_sym),
       :globals => %W/value file line comments/.map(&:to_sym),
       :meta => %W/brief defgroup ingroup comments/.map(&:to_sym),
     }
@@ -331,7 +374,7 @@ class Docurium
       # process this type of record
       case r[:type]
       when :function
-        @data[:functions][r[:name]] ||= {}
+        data[:functions][r[:name]] ||= {}
         wanted[:functions].each do |k|
           next unless r.has_key? k
           conents = nil
@@ -340,18 +383,18 @@ class Docurium
           else
             contents = r[k]
           end
-          @data[:functions][r[:name]][k] = contents
+          data[:functions][r[:name]][k] = contents
         end
         file_map[r[:file]][:functions] << r[:name]
 
       when :define, :macro
-        @data[:globals][r[:decl]] ||= {}
+        data[:globals][r[:decl]] ||= {}
         wanted[:globals].each do |k|
           next unless r.has_key? k
           if k == :description || k == :comments
-            @data[:globals][r[:decl]][k] = md.render r[k]
+            data[:globals][r[:decl]][k] = md.render r[k]
           else
-            @data[:globals][r[:decl]][k] = r[k]
+            data[:globals][r[:decl]][k] = r[k]
           end
         end
 
@@ -364,46 +407,52 @@ class Docurium
         if !r[:name]
           # Explode unnamed enum into multiple global defines
           r[:decl].each do |n|
-            @data[:globals][n] ||= {
+            data[:globals][n] ||= {
               :file => r[:file], :line => r[:line],
               :value => "", :comments => md.render(r[:comments]),
             }
             m = /#{Regexp.quote(n)}/.match(r[:body])
             if m
-              @data[:globals][n][:line] += m.pre_match.scan("\n").length
+              data[:globals][n][:line] += m.pre_match.scan("\n").length
               if m.post_match =~ /\s*=\s*([^,\}]+)/
-                @data[:globals][n][:value] = $1
+                data[:globals][n][:value] = $1
               end
             end
           end
         else # enum has name
-          @data[:types][r[:name]] ||= {}
+          data[:types][r[:name]] ||= {}
           wanted[:types].each do |k|
             next unless r.has_key? k
             contents = r[k]
             if k == :comments
               contents = md.render r[k]
             elsif k == :block
-              old_block = @data[:types][r[:name]][k]
+              old_block = data[:types][r[:name]][k]
               contents = old_block ? [old_block, r[k]].join("\n") : r[k]
+            elsif k == :fields
+              type = data[:types][r[:name]]
+              type[:fields] = []
+              r[:fields].each do |f|
+                f[:comments] = md.render(f[:comments])
+              end
             end
-            @data[:types][r[:name]][k] = contents
+            data[:types][r[:name]][k] = contents
           end
         end
 
       when :struct, :fnptr
-        @data[:types][r[:name]] ||= {}
+        data[:types][r[:name]] ||= {}
         r[:value] ||= r[:name]
         wanted[:types].each do |k|
           next unless r.has_key? k
           if k == :comments
-            @data[:types][r[:name]][k] = md.render r[k]
+            data[:types][r[:name]][k] = md.render r[k]
           else
-            @data[:types][r[:name]][k] = r[k]
+            data[:types][r[:name]][k] = r[k]
           end
         end
         if r[:type] == :fnptr
-          @data[:types][r[:name]][:type] = "function pointer"
+          data[:types][r[:name]][:type] = "function pointer"
         end
 
       else
@@ -412,7 +461,7 @@ class Docurium
 
     end
 
-    @data[:files] << file_map.values[0]
+    data[:files] << file_map.values[0]
   end
 
   def add_dir_to_index(index, prefix, dir)