indexer 0.1.0

data/lib/indexer/core_ext.rb

@@ -0,0 +1,4 @@
+require 'indexer/core_ext/hash/to_h'
+require 'indexer/core_ext/hash/rekey'
+require 'indexer/core_ext/kernel/cli'
+

data/lib/indexer/error.rb

@@ -0,0 +1,23 @@
+module Indexer
+
+  # Tag module for Metaspec Exceptions.
+  #
+  # Use this module to extend arbitrary errors raised by Metaspec,
+  # so they can be easily identified as Metaspec errors if need be. 
+  module Error
+    # Just catch the error and raise this instead.
+    def self.exception(msg=nil,orig=$!)
+      if Class === orig
+        orig = orig.new(msg)
+      elsif orig.nil?
+        orig = StandardError.new(msg)
+      else
+        orig = orig.exception(msg) if msg
+      end
+      orig.extend self
+      orig
+    end
+  end
+
+end
+

data/lib/indexer/gemfile.rb

@@ -0,0 +1,75 @@
+#module Indexer
+
+  # Make sure Indexer is loaded.
+  require 'indexer' unless defined?(Indexer)
+
+  require 'bundler'
+
+  # Bundler integration.
+  #
+  # This does not support Bundler's `:git` references
+  # or `:require` option (at least not yet).
+  #
+  module Bundler
+
+    # Mixin for Bundler::Dsl.
+    #
+    class Dsl
+
+      #
+      # Dynamically update a Gemfile from the `.index` file. Just call `index`
+      # from your Gemfile.
+      #
+      #     rubyfile
+      #
+      # This is analogous to the Gemfile's `gemspec` method.
+      #
+      def index
+        spec = Indexer::Metadata.open
+        spec.requirements.each do |req|
+          next if req.external?
+          gem(req.name, req.version, :group=>req.groups)
+        end
+      end
+
+      #
+      def metadata
+        @metadata ||= Indexer::Metadata.new
+      end
+
+      #
+      alias :_method_missing :method_missing
+
+      #
+      # Evaluating on the Builder instance, allows Ruby basic metadata
+      # to be built via this method.
+      #
+      def method_missing(s, *a, &b)
+        r = s.to_s.chomp('=')
+        case a.size
+        when 0
+          if metadata.respond_to?(s)
+            return metadata.__send__(s, &b)
+          end
+        when 1
+          if metadata.respond_to?("#{r}=")
+            return metadata.__send__("#{r}=", *a)
+          end
+        else
+          if metadata.respond_to?("#{r}=")
+            return metadata.__send__("#{r}=", a)
+          end
+        end
+
+        _method_missing(s, *a, &b)
+        #super(s, *a, &b)  # if cases don't match-up
+      end
+
+    end
+
+  end
+
+#end
+
+#::Bundler::Dsl.__send__(:include, Indexer::Bundler::Dsl)
+

data/lib/indexer/importer.rb

@@ -0,0 +1,144 @@
+module Indexer
+
+  # Import external sources into metadata.
+  #
+  def self.import(*sources)
+    Importer.import(*sources)
+  end
+
+  # Importer class takes disperate data sources and imports them
+  # into a Metadata instance.
+  #
+  # Mixins are used to inject import behavior by overriding the `#import` method.
+  # Any such  mixin's #import method must call `#super` if it's method doesn't
+  # apply, allowing the routine to fallback the other possible import methods.
+  #
+  class Importer
+
+    #
+    # Require all import mixins.
+    #
+    def self.require_importers
+      require_relative 'importer/file'
+      require_relative 'importer/ruby'
+      require_relative 'importer/yaml'
+      require_relative 'importer/html'
+      require_relative 'importer/markdown'
+      #require_relative 'importer/rdoc'
+      #require_relative 'importer/textile'
+      require_relative 'importer/gemspec'
+      require_relative 'importer/gemfile'
+      require_relative 'importer/version'
+    end
+
+    #
+    # Import metadata from external sources.
+    #
+    def self.import(*source)
+      options = (Hash === source.last ? source.pop : {})
+
+      require_importers
+
+      #metadata = nil
+
+      ## use source of current metadata if none given
+      ## TODO: Only search the current directory or search up to root?
+      if source.empty? 
+        if file = Dir[LOCK_FILE].first  #or `Metadata.exists?` ?
+          data   = YAML.load_file(file)
+          source = Array(data['source'])
+        end
+      end
+
+      if source.empty?
+        source = [USER_FILE]
+      end
+
+      source.each do |file|
+        unless File.exist?(file)
+          warn "metadata source file not found - `#{file}'"
+        end
+      end
+
+      importer = Importer.new #(metadata)
+
+      source.each do |src|
+        importer.import(src)
+      end
+
+      return importer.metadata
+    end
+
+    #
+    # Initialize importer.
+    #
+    def initialize(metadata=nil)
+      @metadata   = metadata || Metadata.new
+      @file_cache = {}
+    end
+
+    #
+    # Metadata being built.
+    #
+    attr :metadata
+
+    #
+    #
+    #
+    def import(source)
+      success = super(source) if defined?(super)
+      if success
+        metadata.sources << source unless metadata.sources.include?(source)
+      else
+        raise "metadata source not found or not a known type -- #{source}"
+      end
+    end
+
+    #
+    # Provides a file contents cache. This is used by the YAMLImportation
+    # script, for instance, to see if the file begins with `---`, in
+    # which case the file is taken to be YAML format, even if the 
+    # file's extension is not `.yml` or `.yaml`.
+    #
+    def read(file)
+      @file_cache[file] ||= File.read(file)
+    end
+
+    #
+    # Evaluating on the Importer instance, allows Ruby basic metadata
+    # to be built via this method.
+    #
+    def method_missing(s, *a, &b)
+      return if s == :import
+
+      r = s.to_s.chomp('=')
+      case a.size
+      when 0
+        if metadata.respond_to?(s)
+          return metadata.__send__(s, &b)
+        end
+      when 1
+        if metadata.respond_to?("#{r}=")
+          return metadata.__send__("#{r}=", *a)
+        end
+      else
+        if metadata.respond_to?("#{r}=")
+          return metadata.__send__("#{r}=", a)
+        end
+      end
+
+      super(s, *a, &b)  # if cases don't match-up
+    end
+
+    #
+    # Is `text` a YAML document? It detrmines this simply
+    # be checking for `---` at the top of the text.
+    #
+    # @todo Ignore top comments.
+    #
+    def yaml?(text)
+      text =~ /\A(---|%TAG|%YAML)/
+    end
+  end
+
+end

data/lib/indexer/importer/file.rb

@@ -0,0 +1,94 @@
+module Indexer
+
+  class Importer
+
+    # Import metadata from individual files.
+    #
+    module FileImportation
+
+      #
+      # Files import procedure.
+      #
+      def import(source)
+        if File.directory?(source)
+          load_directory(source)
+          true
+        else
+          super(source) if defined?(super)
+        end
+      end
+
+      #
+      # Import files from a given directory. This will only import files
+      # that have a name corresponding to a metadata attribute, unless
+      # the file is listed in a `.index_extra` file within the directory.
+      #
+      # However, files with an extension of `.yml` or `.yaml` will be loaded
+      # wholeclothe and not as a single attribute.
+      #
+      # @todo Subdirectories are simply omitted. Maybe do otherwise in future?
+      #
+      def load_directory(folder)
+        if File.directory?(folder)
+          extra = []
+          extra_file = File.join(folder, '.index_extra')
+          if File.exist?(extra_file)
+            extra = File.read(extra_file).split("\n")
+            extra = extra.collect{ |pattern| pattern.strip  }
+            extra = extra.reject { |pattern| pattern.empty? }
+            extra = extra.collect{ |pattern| Dir[File.join(folder, pattern)] }.flatten
+          end
+          files = Dir[File.join(folder, '*')]
+          files.each do |file|
+            next if File.directory?(file)
+            name = File.basename(file).downcase
+            next load_yaml(file) if %w{.yaml .yml}.include?(File.extname(file))
+            next load_field_file(file) if extra.include?(name)
+            next load_field_file(file) if metadata.attributes.include?(name.to_sym)
+          end
+        end
+      end
+
+      #
+      # Import a field setting from a file.
+      #
+      # TODO: Ultimately support JSON and maybe other types, and possibly
+      # use mime-types library to recognize them.
+      #
+      def load_field_file(file)
+        if File.directory?(file)
+          # ...
+        else
+          case File.extname(file).downcase
+          when '.yaml', '.yml'
+            name = File.basename(file).downcase
+            name = name.chomp('.yaml').chomp('.yml')
+            metadata[name] = YAML.load_file(file)
+            # TODO: should yaml files with explict extension by merged instead?
+            #metadata.merge!(YAML.load_file(file))
+          when '.text', '.txt'
+            name = File.basename(file).downcase
+            name = name.chomp('.text').chomp('.txt')
+            text = File.read(file)
+            metadata[name] = text.strip
+          else
+            text = File.read(file)
+            if /\A---/ =~ text
+              name = File.basename(file).downcase
+              metadata[name] = YAML.load(text)
+            else
+              name = File.basename(file).downcase
+              metadata[name] = text.strip
+            end
+          end
+        end
+      end
+
+    end
+
+    # Include FileImportation mixin into Builder class.
+    include FileImportation
+
+  end
+
+end

data/lib/indexer/importer/gemfile.rb

@@ -0,0 +1,27 @@
+module Indexer
+
+  class Importer
+
+    # Build mixin for Bundler's Gemfile.
+    #
+    module GemfileImportation
+      #
+      # If the source file is a Gemfile, import it.
+      #
+      def import(source)
+        case source
+        when 'Gemfile'
+          metadata.import_gemfile(source)
+          true
+        else
+          super(source) if defined?(super)
+        end
+      end
+    end
+
+    # Include GemfileImportation mixin into Builder class.
+    include GemfileImportation
+
+  end
+
+end

data/lib/indexer/importer/gemspec.rb

@@ -0,0 +1,43 @@
+module Indexer
+
+  class Importer
+
+    # It is not the recommended that a .gemspec be the usual source of metadata.
+    # Rather it is recommended that a the gemspec be produced from the metadata
+    # instead. (Rumber's metadata covers almost every aspect of a emspec, and
+    # a gemspec can be augmented where needed.) Nonetheless, a gemspec can serve
+    # as a good soruce for creating an initial metadata file.
+    #
+    module GemspecImportation
+
+      #
+      # If the source file is a gemspec, import it.
+      #
+      def import(source)
+        case File.extname(source)
+        when '.gemspec'
+          # TODO: handle YAML-based gemspecs
+          gemspec = ::Gem::Specification.load(source)
+          metadata.import_gemspec(gemspec)
+          true
+        else
+          super(source) if defined?(super)
+        end
+      end
+
+      #
+      #def local_files(root, glob, *flags)
+      #  bits = flags.map{ |f| File.const_get("FNM_#{f.to_s.upcase}") }
+      #  files = Dir.glob(File.join(root,glob), bits)
+      #  files = files.map{ |f| f.sub(root,'') }
+      #  files
+      #end
+    end
+
+    # Include GemspecImportation mixin into Builder class.
+    include GemspecImportation
+
+  end
+
+end
+

data/lib/indexer/importer/html.rb

@@ -0,0 +1,289 @@
+module Indexer
+
+  class Importer
+
+    # Import metadata from a HTML source using microformats.
+    #
+    # NOTE: The implementation using css selectors is fairly slow.
+    #       If we even think it important to speed up then we might
+    #       try traversing instead.
+    #
+    module HTMLImportation
+
+      #
+      # YAML import procedure.
+      #
+      def import(source)
+        if File.file?(source)
+          case File.extname(source)
+          when '.html'
+            load_html(source)
+            return true
+          end
+        end
+        super(source) if defined?(super)
+      end
+
+      #
+      # Import metadata from HTML file.
+      #
+      def load_html(file)
+        require 'nokogiri'
+
+        case file
+        when Nokogiri::XML::Document
+          doc = file
+        when File
+          doc = Nokogiri::HTML(file)
+        else
+          doc = Nokogiri::HTML(File.new(file))
+        end
+
+        data = {}
+
+        %w{version summary description created}.each do |field|
+          load_html_simple(field, doc, data)
+        end
+
+        load_html_name(doc, data)
+        load_html_title(doc, data)
+        load_html_authors(doc, data)
+        load_html_organizations(doc, data)
+        load_html_requirements(doc, data)
+        load_html_resources(doc, data)
+        load_html_repositories(doc, data)
+        load_html_copyrights(doc, data)
+        load_html_categories(doc, data)
+
+        metadata.merge!(data)
+      end
+
+      #
+      # Load a simple field value.
+      #
+      def load_html_simple(field, doc, data)
+        nodes = doc.css(".i#{field}")
+        return if (nodes.nil? or nodes.empty?)
+        text = nodes.first.content.strip
+        data[field] = text
+      end
+
+      #
+      # Load name, and use it for title too if not already set.
+      #
+      def load_html_name(doc, data)
+        nodes = doc.css(".iname")
+        return if (nodes.nil? or nodes.empty?)
+        text = nodes.first.content.strip
+
+        unless metadata.title
+          data['title'] = text.capitalize
+        end
+
+        data['name'] = text
+      end
+
+      #
+      # Load title, and use it for name too if not already set.
+      #
+      def load_html_title(doc, data)
+        nodes = doc.css(".ititle")
+        return if (nodes.nil? or nodes.empty?)
+        text = nodes.first.content.strip
+
+        unless metadata.name
+          data['name'] = text.downcase.gsub(/\s+/, '_')
+        end
+
+        data['title'] = text
+      end
+
+      #
+      #
+      #
+      def load_html_categories(doc, data)
+        nodes = doc.css('.icategory')
+        return if (nodes.nil? or nodes.empty?)
+
+        data['categories'] ||= []
+
+        nodes.each do |node|
+          entry = node.content.strip
+          data['categories'] << entry unless entry == ""
+        end
+      end
+
+      #
+      #
+      #
+      def load_html_resources(doc, data)
+        nodes = doc.css('.iresource')
+        return if (nodes.nil? or nodes.empty?)
+
+        data['resources'] ||= []
+
+        nodes.each do |node|
+          entry = {}
+
+          entry['uri']   = node.attr('href')
+          entry['type']  = node.attr('name') || node.attr('title')  # best choice for this?
+          entry['label'] = node.content.strip
+
+          data['resources'] << entry if entry['uri']
+        end
+      end
+
+      #
+      #
+      #
+      def load_html_requirements(doc, data)
+        nodes = doc.css('.irequirement')
+        return if (nodes.nil? or nodes.empty?)
+
+        data['requirements'] ||= []
+
+        nodes.each do |node|
+          entry = {}
+
+          if n = node.at_css('.name')
+            entry['name'] = n.content.strip
+          end
+
+          if n = node.at_css('.version')
+            entry['version'] = n.content.strip
+          end
+
+          if n = (node.at_css('.groups') || node.at_css('.group'))
+            text = n.content.strip
+            text = text.sub(/^[(]/, '').sub(/[)]$/, '').strip
+            entry['groups'] = text.split(/\s+/)
+
+            if %w{test build document development}.any?{ |g| entry['groups'].include?(g) }
+              entry['development'] = true
+            end
+          end
+
+          data['requirements'] << entry if entry['name']
+        end
+      end
+
+      #
+      # Class is `iauthor`.
+      #
+      def load_html_authors(doc, data)
+        nodes = doc.css('.iauthor')
+        return if (nodes.nil? or nodes.empty?)
+
+        data['authors'] ||= []
+
+        nodes.each do |node|
+          entry = {}
+
+          if n = (node.at_css('.name') || node.at_css('.nickname'))
+            entry['name'] = n.content.strip
+          end
+
+          if n = node.at_css('.email')
+            text = n.attr(:href) || n.content.strip
+            text = text.sub(/^mailto\:/i, '')
+            entry['email'] = text
+          end
+
+          if n = node.at_css('.website') || node.at_css('.uri') || node.at_css('.url')
+            text = n.attr(:href) || n.content.strip
+            entry['website'] = text
+          end
+
+          data['authors'] << entry if entry['name']
+        end
+      end
+
+      #
+      # Class is `iorg`.
+      #
+      def load_html_organizations(doc, data)
+        nodes = doc.css('.iorg')
+        return if (nodes.nil? or nodes.empty?)
+
+        data['organizations'] ||= []
+
+        nodes.each do |node|
+          entry = {}
+
+          if n = node.at_css('.name')
+            entry['name'] = n.content.strip
+          end
+
+          if n = node.at_css('.email')
+            text = n.attr(:href) || n.content.strip
+            text = text.sub(/^mailto\:/i, '')
+            entry['email'] = text
+          end
+
+          if n = node.at_css('.website') || node.at_css('.uri') || node.at_css('.url')
+            text = n.attr(:href) || n.content.strip
+            entry['website'] = text
+          end
+
+          data['organizations'] << entry if entry['name']
+        end
+      end
+
+      #
+      # Class is `irepo`.
+      #
+      def load_html_repositories(doc, data)
+        nodes = doc.css('.irepo')
+        return if (nodes.nil? or nodes.empty?)
+
+        data['repositories'] ||= []
+
+        nodes.each do |node|
+          entry = {}
+
+          entry['uri']   = node.attr('href')
+          entry['type']  = node.attr('name') || node.attr('title')  # best choice for this?
+          entry['label'] = node.content.strip
+
+          data['resources'] << entry if entry['uri']
+        end
+      end
+
+      #
+      #
+      #
+      def load_html_copyrights(doc, data)
+        nodes = doc.css('.icopyright')
+        return if (nodes.nil? or nodes.empty?)
+
+        data['copyrights'] ||= []
+
+        nodes.each do |node|
+          entry = {}
+
+          if n = node.at_css('.holder')
+            entry['holder'] = n.content.strip
+          end
+
+          if n = node.at_css('.year')
+            entry['year'] = n.content.strip
+          end
+
+          if n = node.at_css('.license')
+            text = n.content.strip
+            text = text.sub(/license$/i,'').strip
+            entry['license'] = text
+          end
+
+          data['copyrights'] << entry
+        end
+      end
+
+    end
+
+    # Include YAMLImportation mixin into Builder class.
+    include HTMLImportation
+
+  end
+
+end