indexer 0.1.0

data/lib/indexer/components/resource.rb

@@ -0,0 +1,209 @@
+module Indexer
+
+  # A resource encapsulates information about a URI related to a project.
+  #
+  # TODO: This model is a little tricky b/c it's difficult to settle on a standard
+  # set of types or labels. It is also difficult to determine if label's and types
+  # should in fact be infered or just left empty if not provided. Any insight welcome!
+  #
+  class Resource < Model
+
+    # Parse `data` returning a new Resource instance.
+    #
+    # @param data [String,Array<String>,Array<Hash>,Hash]
+    #   Resource information.
+    #
+    # @return [Resource] resource instance
+    def self.parse(data)
+      case data
+      when String
+        new(:uri=>data)
+      when Array
+        h, d = {}, data.dup  # TODO: data.rekey(&:to_s)
+        h.update(d.pop) while Hash === d.last
+        if x = d.find{ |e| Valid.uri?(e) }
+          h[:uri] = d.delete(x)
+        end
+        if x = d.find{ |e| Valid.word?(e) && e.downcase == e }
+          h[:type] = d.delete(x)
+        end
+        h[:label] = d.shift unless d.empty?
+        raise ValidationError, "malformed resource -- #{data.inspect}" unless d.empty?
+        new(h)
+      when Hash
+        new(data)
+      else
+        raise(ValidationError, "not a valid resource")
+      end
+    end
+
+    #
+    # Initialize new Resource instance.
+    #
+    def initialize(data={})
+      @data = {}
+
+      data = data.rekey(&:to_sym)
+
+      ## best to assign in this order
+      self.uri   = data.delete(:uri)
+      self.type  = data.delete(:type) if data.key?(:type)
+
+      merge!(data)
+    end
+
+    #
+    # A label that can be used to identify the purpose of a
+    # particular repository.
+    #
+    attr :label
+
+    #
+    # Set the resource name. This can be any brief one line description.
+    #
+    # Examples
+    #
+    #   resource.label = "Homepage"
+    #
+    def label=(label)
+      Valid.oneline!(label)  # should be word!
+      @data[:type]  = infer_type(label) unless @type
+      @data[:label] = label.to_str
+    end
+
+    #
+    # Alias for #label.
+    #
+    alias :name :label
+    alias :name= :label=
+
+    #
+    # The repository's URI.
+    #
+    attr :uri
+
+    #
+    # Set repository URI
+    #
+    def uri=(uri)
+      Valid.uri!(uri)  # TODO: any other limitations?
+      @data[:type] = infer_type(uri) unless @type
+      @data[:uri]  = uri
+    end
+
+    #
+    #
+    #
+    attr :type
+
+    #
+    # Set the type of the resource. The type is a single downcased word.
+    # It can be anything but generally recognized types are:
+    #
+    # * bugs, issues
+    # * mail, email
+    # * chat, irc
+    # * home, homepage, website
+    # * work, code
+    #
+    def type=(type)
+      Valid.word!(type)
+      @data[:label] = infer_label(type) unless @label
+      @data[:type]  = type.to_str.downcase
+    end
+
+    ##
+    ## Convert resource to Hash.
+    ##
+    ## @return [Hash]
+    ##
+    #def to_h
+    #  h = {}
+    #  h['uri']   = uri
+    #  h['label'] = label if label
+    #  h['type']  = type  if type
+    #  h
+    #end
+
+    #
+    # Recognized types and the default labels that do with them.
+    #
+    TYPE_LABELS = {
+      'work'     => "Development",
+      'dev'      => "Development",
+      'code'     => "Source Code",
+      'source'   => "Source Code",
+      'bugs'     => "Issue Tracker",
+      'issues'   => "Issue Tracker",
+      'mail'     => "Mailing List",
+      'list'     => "Mailing List",
+      'forum'    => "Support Forum",
+      'support'  => "Support Forum",
+      'faq'      => "Fact Sheet",
+      'home'     => "Homepage",
+      'homepage' => "Homepage",
+      'web'      => "Website",
+      'website'  => "Website",
+      'blog'     => "Blog",
+      'pkg'      => "Download",
+      'dist'     => "Download",
+      'api'      => "API Guide",
+      'irc'      => "IRC Channel",
+      'chat'     => "IRC Channel",
+      'doc'      => "Documentation",
+      'docs'     => "Documentation",
+      'wiki'     => "User Guide"
+    }
+
+    #
+    #
+    #
+    KNOWN_TYPES = TYPE_LABELS.keys
+
+  private
+
+    #
+    #
+    #
+    def infer_label(type)
+      TYPE_LABELS[type.to_s.downcase] 
+    end
+
+    #
+    # @todo Deciding on a set of recognized types and inferences for them is rather tricky.
+    #   Some sort of guiding design principle would be helpful.
+    #
+    def infer_type(string)
+      case string.to_s.downcase
+      when /work/i, /dev/i
+        'dev'  #'development'
+      when /code/i, /source/i
+        'code' #'source-code'
+      when /bugs/i, /issue/i, /tracker/i
+        'bugs' #'issue-tracker'
+      when /mail/i, /list/i
+        'mail' #'mailing-list'
+      when /chat/i, /irc/i
+        'chat' #'online-chat'
+      when /forum/i, /support/i, /q\&a/i
+        'help' #'support-forum'
+      when /wiki/i, /user/i
+        'wiki' #'wiki-wiki'
+      when /blog/i, /weblog/i
+        'blog' #'weblog'
+      when /home/i, /website/i
+        'home' #'homepage'
+      when /gem/i, /packge/i, /dist/i, /download/i
+        'dist' #'package' or 'distribution' ?
+      when /api/i, /reference/i, /guide/i
+        'api'  #'reference' ???
+      when /doc/i
+        'doc' #'documentation'
+      else
+        nil
+      end
+    end
+
+  end
+
+end

data/lib/indexer/conversion.rb

@@ -0,0 +1,14 @@
+module Indexer
+
+  # Conversion module provides routines for converting
+  # other metadata sources to and from index format.
+  #
+  module Conversion
+  end
+
+end
+
+require_relative 'conversion/gemspec'
+require_relative 'conversion/gemspec_exporter'
+require_relative 'conversion/gemfile'
+

data/lib/indexer/conversion/gemfile.rb

@@ -0,0 +1,44 @@
+module Indexer
+
+  module Conversion
+
+    #
+    # Update Spec with a Gemfile.
+    #
+    # @param [nil,String,::Bundler::DSL,::Bundler::Definition]
+    #   Gemfile path, Bundler::Dsl or Bundler::Definition instance.
+    #
+    def import_gemfile(file=nil)
+      require 'bundler'
+
+      case file
+      when String
+        # FIXME: Is this the correct way fot load a gemfile?
+        bundle = ::Bundler::Dsl.new
+        bundle.eval_gemfile(file)
+      when NilClass
+        bundle = ::Bundler.definition
+      end
+
+      begin
+        merge!(bundle.metadata.to_h)
+      rescue
+      end
+
+      bundle.dependencies.each do |d|
+        add_requirement(d.name,
+          :version     => d.requirement.to_s,  # may need to parse this
+          :groups      => d.groups,
+          :development => (d.type == :development)
+          #:engines    => d.platforms ?
+          #:platforms  => ?
+        )
+      end
+    end
+
+    alias_method :gemfile, :import_gemfile
+
+  end
+
+end
+

data/lib/indexer/conversion/gemspec.rb

@@ -0,0 +1,114 @@
+module Indexer
+
+  module Conversion
+
+    #
+    # Create a Gem::Specification from Metadata.
+    #
+    # Becuase the specificaiton is extensive, a Gem::Specification
+    # can be created that is sufficient for most needs. However, there
+    # are a few points where the two do not intersect. In these cases
+    # the remaining gemspec fields need to be provided post conversion.
+    #
+    # Gem::Specification fields that the specification can't provide include:
+    #
+    # * platform
+    #
+    # In addition, some information can only be provided if a project's `root`
+    # directory is given. In these cases the most common project conventions
+    # are utilized in determining field values. Gem::Specification fields that
+    # draw from project files include:
+    #
+    # * files
+    # * bindir
+    # * extensions
+    # * rdoc_options
+    # * rdoc_extra_files
+    #
+    # Any or all of which can be reassigned post conversion, if need be.
+    #
+    # @param [Hash] options
+    #   Convertion options.
+    #
+    # @option options [NilClass,String] :root
+    #   project root directory
+    #
+    def to_gemspec(options={})
+      options[:data] = self.to_h
+      GemspecExporter.new(options).to_gemspec
+    end
+
+    #
+    # Import a Gem::Specification into Spec. This is intended to make it
+    # fairly easy to build the metadata from a pre-existing `.gemspec`.
+    #
+    # By making this an instance method, it is possible to import other
+    # resources into the Spec prior to a `.gemspec`.
+    #
+    # @todo Ensure all data possible is gathered from the gemspec.
+    #
+    def import_gemspec(gemspec)
+      require 'rubygems'
+
+      if not Gem::Specification === gemspec
+        # TODO: YAML-based gem specs ?
+        gemspec = Gem::Specification.load(gemspec)
+      end
+
+      # TODO: ensure this is robust
+      authors = (
+        zip = [gemspec.authors].flatten.zip([gemspec.email].flatten)
+        zip.map do |(name, email)|
+          email ? {:name=>name, :email=>email} : {:name=>name}
+        end
+      )
+
+      # TODO: how to handle license(s) ?
+
+      self.name         = gemspec.name
+      self.version      = gemspec.version.to_s
+      self.date         = gemspec.date
+      self.title        = gemspec.name.capitalize
+      self.summary      = gemspec.summary
+      self.description  = gemspec.description || gemspec.summary
+      self.authors      = authors
+      self.load_path    = gemspec.require_paths
+      self.homepage     = gemspec.homepage
+
+      #self.engines      = gemspec.platform
+      #self.extensions   = gemspec.extensions
+
+      # TODO: Spec currently doesn't support multiple constraints for requirements.
+      #       Probably 99.999% of the time it doesn't matter.
+      gemspec.dependencies.each do |d|
+        if d.type == :runtime
+          add_requirement(d.name, :versions=>d.requirements_list.first)
+        else
+          add_requirement(d.name, :versions=>d.requirements_list.first, :development=>true)
+        end
+      end
+    end
+
+  end
+
+end
+
+
+=begin
+
+      case gemspec
+      when ::Gem::Specification
+        spec = gemspec
+      else
+        file = Dir[root + "{*,}.gemspec"].first
+        return unless file
+
+        text = File.read(file)
+        if text =~ /\A---/
+          spec = ::Gem::Specification.from_yaml(text)
+        else
+          spec = ::Gem::Specification.load(file)
+        end
+      end
+
+=end

data/lib/indexer/conversion/gemspec_exporter.rb

@@ -0,0 +1,304 @@
+# encoding: utf-8
+
+require 'yaml'
+require 'pathname'
+
+module Indexer
+
+  # Convert index data into a gemspec.
+  #
+  # Notes:
+  #   * Assumes all executables are in bin/.
+  #   * Does not yet handle default_executable setting.
+  #   * Does not yet handle platform setting.
+  #   * Does not yet handle required_ruby_version.
+  #   * Support for rdoc entries is weak.
+  #
+  class GemspecExporter
+
+    # File globs to include in package --unless a manifest file exists.
+    FILES = ".index .yardopts alt bin data demo ext features lib man spec test try* [A-Z]*.*" unless defined?(FILES)
+
+    # File globs to omit from FILES.
+    OMIT = "Config.rb" unless defined?(OMIT)
+
+    # Standard file patterns.
+    PATTERNS = {
+      :root => '{.index,Gemfile}',
+      :bin  => 'bin/*',
+      :lib  => 'lib/{**/}*', #.rb',
+      :ext  => 'ext/{**/}extconf.rb',
+      :doc  => '*.{txt,rdoc,md,markdown,tt,textile}',
+      :test => '{test,spec}/{**/}*.rb'
+    } unless defined?(PATTERNS)
+
+    # For which revision of indexer spec is this converter intended?
+    REVISION = 2013 unless defined?(REVISION)
+
+    #
+    def self.gemspec
+      new.to_gemspec
+    end
+
+    #
+    attr :metadata
+
+    #
+    def initialize(metadata=nil)
+      @root_check = false
+
+      if metadata
+        root_dir = metadata.delete(:root)
+        if root_dir
+          @root = root_dir
+          @root_check = true
+        end
+        metadata = nil if metadata.empty?
+      end
+
+      @metadata = metadata || YAML.load_file(root + '.index')
+
+      if @metadata['revision'].to_i != REVISION
+        warn "This gemspec exporter was not designed for this revision of index metadata."
+      end
+    end
+
+    #
+    def has_root?
+      root ? true : false
+    end
+
+    #
+    def root
+      return @root if @root || @root_check
+      @root_check = true
+      @root = find_root
+    end
+
+    #
+    def manifest
+      return nil unless root
+      @manifest ||= Dir.glob(root + 'manifest{,.txt}', File::FNM_CASEFOLD).first
+    end
+
+    #
+    def scm
+      return nil unless root
+      @scm ||= %w{git hg}.find{ |m| (root + ".#{m}").directory? }.to_sym
+    end
+
+    #
+    def files
+      return [] unless root
+      @files ||= \
+        if manifest
+          File.readlines(manifest).
+            map{ |line| line.strip }.
+            reject{ |line| line.empty? || line[0,1] == '#' }
+        else
+          list = []
+          Dir.chdir(root) do
+            FILES.split(/\s+/).each do |pattern|
+              list.concat(glob(pattern))
+            end
+            OMIT.split(/\s+/).each do |pattern|
+              list = list - glob(pattern)
+            end
+          end
+          list
+        end.select{ |path| File.file?(path) }.uniq
+    end
+
+    #
+    def glob_files(pattern)
+      return [] unless root
+      Dir.chdir(root) do
+        Dir.glob(pattern).select do |path|
+          File.file?(path) && files.include?(path)
+        end
+      end
+    end
+
+    def patterns
+      PATTERNS
+    end
+
+    def executables
+      @executables ||= \
+        glob_files(patterns[:bin]).map do |path|
+          File.basename(path)
+        end
+    end
+
+    def extensions
+      @extensions ||= \
+        glob_files(patterns[:ext]).map do |path|
+          File.basename(path)
+        end
+    end
+
+    def name
+      metadata['name'] || metadata['title'].downcase.gsub(/\W+/,'_')
+    end
+
+    def homepage
+      page = (
+        metadata['resources'].find{ |r| r['type'] =~ /^home/i } ||
+        metadata['resources'].find{ |r| r['name'] =~ /^home/i } ||
+        metadata['resources'].find{ |r| r['name'] =~ /^web/i }
+      )
+      page ? page['uri'] : false
+    end
+
+    def licenses
+      metadata['copyrights'].map{ |c| c['license'] }.compact
+    end
+
+    def require_paths
+      metadata['load_path'] || ['lib']
+    end
+
+    #
+    # Convert to gemnspec.
+    #
+    def to_gemspec
+      if has_root?
+        Gem::Specification.new do |gemspec|
+          to_gemspec_data(gemspec)
+          to_gemspec_paths(gemspec)
+        end
+      else
+        Gem::Specification.new do |gemspec|
+          to_gemspec_data(gemspec)
+          to_gemspec_paths(gemspec)
+        end
+      end
+    end
+
+    #
+    # Convert pure data settings.
+    #
+    def to_gemspec_data(gemspec)
+      gemspec.name        = name
+      gemspec.version     = metadata['version']
+      gemspec.summary     = metadata['summary']
+      gemspec.description = metadata['description']
+
+      metadata['authors'].each do |author|
+        gemspec.authors << author['name']
+
+        if author.has_key?('email')
+          if gemspec.email
+            gemspec.email << author['email']
+          else
+            gemspec.email = [author['email']]
+          end
+        end
+      end
+
+      gemspec.licenses = licenses
+
+      requirements = metadata['requirements'] || []
+      requirements.each do |req|
+        next if req['optional']
+        next if req['external']
+
+        name    = req['name']
+        groups  = req['groups'] || []
+
+        version = gemify_version(req['version'])
+
+        if groups.empty? or groups.include?('runtime')
+          # populate runtime dependencies  
+          if gemspec.respond_to?(:add_runtime_dependency)
+            gemspec.add_runtime_dependency(name,*version)
+          else
+            gemspec.add_dependency(name,*version)
+          end
+        else
+          # populate development dependencies
+          if gemspec.respond_to?(:add_development_dependency)
+            gemspec.add_development_dependency(name,*version)
+          else
+            gemspec.add_dependency(name,*version)
+          end
+        end
+      end
+
+      # convert external dependencies into gemspec requirements
+      requirements.each do |req|
+        next unless req['external']
+        gemspec.requirements << ("%s-%s" % req.values_at('name', 'version'))
+      end
+
+      gemspec.homepage = homepage
+      gemspec.require_paths = require_paths
+      gemspec.post_install_message = metadata['install_message']
+    end
+
+    #
+    # Set gemspec settings that require a root directory path.
+    #
+    def to_gemspec_paths(gemspec)
+      gemspec.files       = files
+      gemspec.extensions  = extensions
+      gemspec.executables = executables
+
+      if Gem::VERSION < '1.7.'
+        gemspec.default_executable = gemspec.executables.first
+      end
+
+      gemspec.test_files = glob_files(patterns[:test])
+
+      unless gemspec.files.include?('.document')
+        gemspec.extra_rdoc_files = glob_files(patterns[:doc])
+      end
+    end
+
+    #
+    # Return a copy of this file. This is used to generate a local
+    # .gemspec file that can automatically read the index file.
+    #
+    def self.source_code
+      File.read(__FILE__)
+    end
+
+  private
+
+    def find_root
+      root_files = patterns[:root]
+      if Dir.glob(root_files).first
+        Pathname.new(Dir.pwd)
+      elsif Dir.glob("../#{root_files}").first
+        Pathname.new(Dir.pwd).parent
+      else
+        #raise "Can't find root of project containing `#{root_files}'."
+        warn "Can't find root of project containing `#{root_files}'."
+        nil
+      end
+    end
+
+    def glob(pattern)
+      if File.directory?(pattern)
+        Dir.glob(File.join(pattern, '**', '*'))
+      else
+        Dir.glob(pattern)
+      end
+    end
+
+    def gemify_version(version)
+      case version
+      when /^(.*?)\+$/
+        ">= #{$1}"
+      when /^(.*?)\-$/
+        "< #{$1}"
+      when /^(.*?)\~$/
+        "~> #{$1}"
+      else
+        version
+      end
+    end
+
+  end
+
+end