fig 0.1.51 → 0.1.52

data/lib/fig/package.rb

@@ -1,16 +1,21 @@
 require 'fig/logging'
 require 'fig/packageerror'
-require 'fig/package/archive'
-require 'fig/package/configuration'
-require 'fig/package/resource'
-require 'fig/package/retrieve'
+require 'fig/statement/archive'
+require 'fig/statement/configuration'
+require 'fig/statement/resource'
+require 'fig/statement/retrieve'
 
 module Fig; end
 
 # The parsed representation of a configuration file.  Contains the statement
 # objects.
 class Fig::Package
-  attr_reader :package_name, :version_name, :directory, :statements
+  include Comparable
+
+  UNPUBLISHED     = '<unpublished>'
+  DEFAULT_CONFIG  = 'default'
+
+  attr_reader   :package_name, :version_name, :directory, :statements
   attr_accessor :backtrace
 
   def initialize(package_name, version_name, directory, statements)
@@ -18,59 +23,170 @@ class Fig::Package
     @version_name = version_name
     @directory = directory
     @statements = statements
+    @applied_config_names = []
     @backtrace = nil
   end
 
   def [](config_name)
     @statements.each do |stmt|
-      return stmt if stmt.is_a?(Configuration) && stmt.name == config_name
+      return stmt if stmt.is_a?(Fig::Statement::Configuration) && stmt.name == config_name
     end
-    message = "Configuration not found: #{@package_name}/#{@version_name}:#{config_name}"
-    raise Fig::PackageError.new message
+
+    message =
+      'Configuration not found: ' +
+      (@package_name || '<empty>')  +
+      '/'                         +
+      (@version_name || '<empty>')  +
+      ':'                         +
+      (config_name || '<empty>')
+
+    raise Fig::PackageError.new(message)
+  end
+
+  def <=>(other)
+    compared = compare_components(package_name, other.package_name)
+    return compared if compared != 0
+
+    return compare_components(version_name, other.version_name)
   end
 
   def configs
-    @statements.select { |statement| statement.is_a?(Configuration) }
+    return @statements.select { |statement| statement.is_a?(Fig::Statement::Configuration) }
+  end
+
+  def config_names
+    return configs.collect { |statement| statement.name }
   end
 
   def retrieves
     retrieves = {}
-    statements.each { |statement| retrieves[statement.var] = statement.path if statement.is_a?(Retrieve) }
-    retrieves
+
+    statements.each do
+      |statement|
+
+      retrieves[statement.var] = statement.path if statement.is_a?(Fig::Statement::Retrieve)
+    end
+
+    return retrieves
   end
 
   def archive_urls
-    @statements.select{|s| s.is_a?(Archive)}.map{|s|s.url}
+    return @statements.select{|s| s.is_a?(Fig::Statement::Archive)}.map{|s| s.url}
   end
 
   def resource_urls
-    @statements.select{|s| s.is_a?(Resource)}.map{|s|s.url}
+    return @statements.select{|s| s.is_a?(Fig::Statement::Resource)}.map{|s|s.url}
+  end
+
+  def applied_config_names()
+    return @applied_config_names.clone
+  end
+
+  def add_applied_config_name(name)
+    @applied_config_names << name
+  end
+
+  def primary_config_name()
+    return @applied_config_names.first
+  end
+
+  # Returns an array of PackageDescriptors
+  def package_dependencies(config_name)
+    descriptors = []
+
+    self[config_name || DEFAULT_CONFIG].walk_statements do
+      |statement|
+
+      if statement.is_a?(Fig::Statement::Include)
+        descriptors << statement.resolved_dependency_descriptor(self)
+      end
+    end
+
+    return descriptors
   end
 
+  # Block will receive a Statement.
   def walk_statements(&block)
     @statements.each do |statement|
       yield statement
       statement.walk_statements &block
     end
+
+    return
+  end
+
+  # Block will receive a Package and a Statement.
+  def walk_statements_following_package_dependencies(repository, &block)
+    @statements.each do |statement|
+      yield self, statement
+      statement.walk_statements_following_package_dependencies(
+        repository, self, &block
+      )
+    end
+
+    return
   end
 
   def unparse
-    @statements.map { |statement| statement.unparse('') }.join("\n")
+    return @statements.map { |statement| statement.unparse('') }.join("\n")
   end
 
   def ==(other)
     return false if other.nil?
-    @package_name == other.package_name && @version_name == other.version_name && @statements.to_yaml == other.statements.to_yaml
+
+    return @package_name == other.package_name &&
+           @version_name == other.version_name &&
+           @statements.to_yaml == other.statements.to_yaml
   end
 
   def to_s
-    package_name ||= 'uninitialized'
-    version_name ||= 'uninitialized'
-    package_name + '/' + version_name
+    package_name = @package_name || '<empty>'
+    version_name = @version_name || '<empty>'
+    return package_name + '/' + version_name
+  end
+
+  def to_s_with_config(config_name)
+    string = nil
+
+    if package_name.nil?
+      string = UNPUBLISHED
+    else
+      string = to_s
+    end
+
+    if not config_name.nil? and config_name != DEFAULT_CONFIG
+      string += ":#{config_name}"
+    end
+
+    return string
+  end
+
+  def to_s_with_primary_config()
+    return to_s_with_config(primary_config_name)
+  end
+
+  private
+
+  def compare_components(mine, others)
+    if mine.nil?
+      if others.nil?
+        return 0
+      end
+
+      return 1
+    end
+
+    if others.nil?
+      return -1
+    end
+
+    return mine <=> others
   end
 end
 
+# TODO: get this out of the global namespace
 def unparse_statements(indent, prefix, statements, suffix)
   body = @statements.map { |statement| statement.unparse(indent+'  ') }.join("\n")
+
   return ["\n#{indent}#{prefix}", body, "#{indent}#{suffix}"].join("\n")
 end

data/lib/fig/packagecache.rb

@@ -0,0 +1,36 @@
+module Fig; end
+
+# Simple double-level cache of Packages.
+class Fig::PackageCache
+  def initialize()
+    @packages = {}
+  end
+
+  def add_package(package)
+    versions = @packages[package.package_name]
+    if not versions
+      versions = {}
+      @packages[package.package_name] = versions
+    end
+
+    versions[package.version_name] = package
+
+    return
+  end
+
+  def get_package(name, version)
+    versions = @packages[name]
+    return if not versions
+
+    return versions[version]
+  end
+
+  def remove_package(name, version)
+    versions = @packages[name]
+    return if not versions
+
+    versions.delete(version)
+
+    return
+  end
+end

data/lib/fig/packagedescriptor.rb

@@ -0,0 +1,46 @@
+module Fig; end
+
+# Parsed representation of a package name:config/version.
+class Fig::PackageDescriptor
+  include Comparable
+
+  DEFAULT_CONFIG = 'default'
+
+  attr_reader :name, :version, :config
+
+  def self.parse(raw_string)
+    self.new(
+      raw_string =~ %r< ^ ( [^:/]+ ) >x ? $1 : nil,
+      raw_string =~ %r< / ( [^:/]+ ) >x ? $1 : nil,
+      raw_string =~ %r< : ( [^:/]+ ) >x ? $1 : nil
+    )
+  end
+
+  def initialize(name, version, config)
+    @name     = name
+    @version  = version
+    @config   = config
+  end
+
+  def to_string(use_default_config = false)
+    string = @name || ''
+
+    if @version
+      string += '/'
+      string += @version
+    end
+
+    if @config
+      string += ':'
+      string += @config
+    elsif use_default_config
+      string += ':default'
+    end
+
+    return string
+  end
+
+  def <=>(other)
+    return to_string() <=> other.to_string()
+  end
+end

data/lib/fig/parser.rb

@@ -32,12 +32,12 @@ module Fig
     def find_multiple_command_statements(package)
       command_processed = false
       package.walk_statements do |statement|
-        if statement.is_a?(Package::Command)
+        if statement.is_a?(Statement::Command)
           if command_processed == true
             raise UserInputError.new("Multiple command statements cannot be processed.")
           end
           command_processed = true
-        elsif statement.is_a?(Package::Configuration)
+        elsif statement.is_a?(Statement::Configuration)
           command_processed = false
         end
       end

data/lib/fig/repository.rb

@@ -1,9 +1,10 @@
 require 'fig/logging'
 require 'fig/notfounderror'
-require 'fig/package/archive'
-require 'fig/package/resource'
+require 'fig/packagecache'
 require 'fig/parser'
 require 'fig/repositoryerror'
+require 'fig/statement/archive'
+require 'fig/statement/resource'
 require 'fig/urlaccesserror'
 
 module Fig
@@ -14,36 +15,31 @@ module Fig
       not (/ftp:\/\/|http:\/\/|file:\/\/|ssh:\/\// =~ url).nil?
     end
 
-    def initialize(os, local_repository_dir, remote_repository_url, application_config, remote_repository_user=nil, update=false, update_if_missing=true)
-      @os = os
+    def initialize(
+      os,
+      local_repository_dir,
+      remote_repository_url,
+      application_config,
+      remote_repository_user  = nil,
+      update                  = false,
+      update_if_missing       = true
+    )
+      @operating_system = os
       @local_repository_dir = local_repository_dir
       @remote_repository_url = remote_repository_url
       @remote_repository_user = remote_repository_user
-      @application_config = application_config
       @update = update
       @update_if_missing = update_if_missing
-      @parser = Parser.new(@application_config)
-
-      @overrides = {}
-      if File.exist?('fig.properties')
-        File.readlines('fig.properties').each do |line|
-          descriptor, path = line.strip.split('=')
-          @overrides[descriptor] = path
-        end
-      end
-    end
 
-    def clean(package_name, version_name)
-      dir = File.join(@local_repository_dir, package_name)
-      dir = File.join(dir, version_name) if version_name
-      FileUtils.rm_rf(dir)
+      @parser = Parser.new(@application_config)
+      @packages = PackageCache.new()
     end
 
     def list_packages
       results = []
       if File.exist?(@local_repository_dir)
-        @os.list(@local_repository_dir).each do |package_name|
-          @os.list(File.join(@local_repository_dir, package_name)).each do |version_name|
+        @operating_system.list(@local_repository_dir).each do |package_name|
+          @operating_system.list(File.join(@local_repository_dir, package_name)).each do |version_name|
             results << "#{package_name}/#{version_name}"
           end
         end
@@ -52,84 +48,142 @@ module Fig
     end
 
     def list_remote_packages
-      @os.download_list(@remote_repository_url)
+      @operating_system.download_list(@remote_repository_url)
+    end
+
+    def get_package(package_name, version_name, disable_updating = false)
+      package = @packages.get_package(package_name, version_name)
+      return package if package
+
+      Logging.debug "Considering #{package_name}/#{version_name}."
+
+      if should_update?(package_name, version_name, disable_updating)
+        update_package(package_name, version_name)
+      end
+
+      return read_local_package(package_name, version_name)
+    end
+
+    def clean(package_name, version_name)
+      @packages.remove_package(package_name, version_name)
+
+      dir = File.join(@local_repository_dir, package_name)
+      dir = File.join(dir, version_name) if version_name
+
+      FileUtils.rm_rf(dir)
+
+      return
     end
 
-    def publish_package(package_statements, package_name, version_name, local_only)
+    def publish_package(
+      package_statements, package_name, version_name, local_only
+    )
       temp_dir = temp_dir_for_package(package_name, version_name)
-      @os.clear_directory(temp_dir)
+      @operating_system.clear_directory(temp_dir)
       local_dir = local_dir_for_package(package_name, version_name)
-      @os.clear_directory(local_dir)
+      @operating_system.clear_directory(local_dir)
       fig_file = File.join(temp_dir, '.fig')
-      content = bundle_resources(package_statements).map do |statement|
-        if statement.is_a?(Package::Publish)
-          nil
-        elsif statement.is_a?(Package::Archive) || statement.is_a?(Package::Resource)
-          if statement.is_a?(Package::Resource) && !Repository.is_url?(statement.url)
-            archive_name = statement.url
-            archive_remote = "#{remote_dir_for_package(package_name, version_name)}/#{statement.url}"
-          else
-            archive_name = statement.url.split('/').last
-            archive_remote = "#{remote_dir_for_package(package_name, version_name)}/#{archive_name}"
-          end
-          if Repository.is_url?(statement.url)
-            archive_local = File.join(temp_dir, archive_name)
-            @os.download(statement.url, archive_local)
-          else
-            archive_local = statement.url
-          end
-          @os.upload(archive_local, archive_remote, @remote_repository_user) unless local_only
-          @os.copy(archive_local, local_dir + '/' + archive_name)
-          if statement.is_a?(Package::Archive)
-            @os.unpack_archive(local_dir, archive_name)
-          end
-          statement.class.new(archive_name).unparse('')
-        else
-          statement.unparse('')
-        end
-      end.select {|s|not s.nil?}
-      @os.write(fig_file, content.join("\n").strip)
-      @os.upload(fig_file, remote_fig_file_for_package(package_name, version_name), @remote_repository_user) unless local_only
-      @os.copy(fig_file, local_fig_file_for_package(package_name, version_name))
+      content = derive_package_content(
+        package_statements, package_name, version_name, local_dir, local_only
+      )
+      @operating_system.write(fig_file, content.join("\n").strip)
+      @operating_system.upload(
+        fig_file,
+        remote_fig_file_for_package(package_name, version_name),
+        @remote_repository_user
+      ) unless local_only
+      @operating_system.copy(
+        fig_file, local_fig_file_for_package(package_name, version_name)
+      )
+
+      FileUtils.rm_rf(temp_dir)
+    end
+
+    def updating?
+      return @update || @update_if_missing
+    end
+
+    private
+
+    def should_update?(package_name, version_name, disable_updating)
+      return false if disable_updating
+
+      return true if @update
+
+      return @update_if_missing &&
+        package_missing?(package_name, version_name)
+    end
+
+    def read_local_package(package_name, version_name)
+      dir = local_dir_for_package(package_name, version_name)
+      return read_package_from_directory(dir, package_name, version_name)
     end
 
     def bundle_resources(package_statements)
       resources = []
       new_package_statements = package_statements.reject do |statement|
-        if statement.is_a?(Package::Resource) && !Repository.is_url?(statement.url)
+        if (
+          statement.is_a?(Statement::Resource) &&
+          ! Repository.is_url?(statement.url)
+        )
           resources << statement.url
           true
         else
           false
         end
       end
+
       if resources.size > 0
         resources = expand_globs_from(resources)
         file = 'resources.tar.gz'
-        @os.create_archive(file, resources)
-        new_package_statements.unshift(Package::Archive.new(file))
+        @operating_system.create_archive(file, resources)
+        new_package_statements.unshift(Statement::Archive.new(file))
         at_exit { File.delete(file) }
       end
-      new_package_statements
-    end
 
-    def load_package(package_name, version_name)
-      Logging.debug "Considering #{package_name}/#{version_name}."
-      if @update || (@update_if_missing && package_missing?(package_name, version_name))
-        update_package(package_name, version_name)
-      end
-      read_local_package(package_name, version_name)
+      return new_package_statements
     end
 
-    def updating?
-      return @update || @update_if_missing
+    def install_package(package_name, version_name)
+      temp_dir = nil
+
+      begin
+        package = read_local_package(package_name, version_name)
+        temp_dir = temp_dir_for_package(package_name, version_name)
+        @operating_system.clear_directory(temp_dir)
+        package.archive_urls.each do |archive_url|
+          if not Repository.is_url?(archive_url)
+            archive_url = remote_dir_for_package(package_name, version_name) + '/' + archive_url
+          end
+          @operating_system.download_archive(archive_url, File.join(temp_dir))
+        end
+        package.resource_urls.each do |resource_url|
+          if not Repository.is_url?(resource_url)
+            resource_url = remote_dir_for_package(package_name, version_name) + '/' + resource_url
+          end
+          @operating_system.download_resource(resource_url, File.join(temp_dir))
+        end
+        local_dir = local_dir_for_package(package_name, version_name)
+        @operating_system.clear_directory(local_dir)
+        # some packages contain no files, only a fig file.
+        if not (package.archive_urls.empty? && package.resource_urls.empty?)
+          FileUtils.mv(Dir.glob(File.join(temp_dir, '*')), local_dir)
+        end
+        write_local_package(package_name, version_name, package)
+      rescue
+        Logging.fatal 'Install failed, cleaning up.'
+        delete_local_package(package_name, version_name)
+        raise RepositoryError.new
+      ensure
+        FileUtils.rm_rf(temp_dir)
+      end
     end
 
     def update_package(package_name, version_name)
       remote_fig_file = remote_fig_file_for_package(package_name, version_name)
       local_fig_file = local_fig_file_for_package(package_name, version_name)
       begin
-        if @os.download(remote_fig_file, local_fig_file)
+        if @operating_system.download(remote_fig_file, local_fig_file)
           install_package(package_name, version_name)
         end
       rescue NotFoundError
@@ -139,9 +193,11 @@ module Fig
       end
     end
 
-    def read_local_package(package_name, version_name)
-      dir = local_dir_for_package(package_name, version_name)
-      read_package_from_directory(dir, package_name, version_name)
+    # 'resources' is an Array of filenames: ['tmp/foo/file1', 'tmp/foo/*.jar']
+    def expand_globs_from(resources)
+      expanded_files = []
+      resources.each {|f| expanded_files.concat(Dir.glob(f))}
+      expanded_files
     end
 
     def read_package_from_directory(dir, package_name, version_name)
@@ -150,10 +206,11 @@ module Fig
         file = File.join(dir, 'package.fig')
       end
       if not File.exist?(file)
-        Logging.fatal "Fig file not found for package: #{file}"
+        Logging.fatal %Q<Fig file not found for package "#{package_name || '<unnamed>'}": #{file}>
         raise RepositoryError.new
       end
-      read_package_from_file(file, package_name, version_name)
+
+      return read_package_from_file(file, package_name, version_name)
     end
 
     def read_package_from_file(file_name, package_name, version_name)
@@ -162,58 +219,14 @@ module Fig
         raise RepositoryError.new
       end
       content = File.read(file_name)
-      return @parser.parse_package(package_name, version_name, File.dirname(file_name), content)
-    end
-
-    def local_dir_for_package(package_name, version_name)
-      descriptor = "#{package_name}/#{version_name}"
-      dir = @overrides[descriptor]
-      if dir
-        Logging.info "override: #{descriptor}=#{dir}"
-      else
-        dir = File.join(@local_repository_dir, package_name, version_name)
-      end
-      dir
-    end
 
-  private
+      package = @parser.parse_package(
+        package_name, version_name, File.dirname(file_name), content
+      )
 
-    def install_package(package_name, version_name)
-      begin
-        package = read_local_package(package_name, version_name)
-        temp_dir = temp_dir_for_package(package_name, version_name)
-        @os.clear_directory(temp_dir)
-        package.archive_urls.each do |archive_url|
-          if not Repository.is_url?(archive_url)
-            archive_url = remote_dir_for_package(package_name, version_name) + '/' + archive_url
-          end
-          @os.download_archive(archive_url, File.join(temp_dir))
-        end
-        package.resource_urls.each do |resource_url|
-          if not Repository.is_url?(resource_url)
-            resource_url = remote_dir_for_package(package_name, version_name) + '/' + resource_url
-          end
-          @os.download_resource(resource_url, File.join(temp_dir))
-        end
-        local_dir = local_dir_for_package(package_name, version_name)
-        @os.clear_directory(local_dir)
-        # some packages contain no files, only a fig file.
-        if not (package.archive_urls.empty? && package.resource_urls.empty?)
-          FileUtils.mv(Dir.glob(File.join(temp_dir, '*')), local_dir)
-        end
-        write_local_package(package_name, version_name, package)
-      rescue
-        Logging.fatal 'Install failed, cleaning up.'
-        delete_local_package(package_name, version_name)
-        raise RepositoryError.new
-      end
-    end
+      @packages.add_package(package)
 
-    # 'resources' is an Array of filenames: ['tmp/foo/file1', 'tmp/foo/*.jar']
-    def expand_globs_from(resources)
-      expanded_files = []
-      resources.each {|f| expanded_files.concat(Dir.glob(f))}
-      expanded_files
+      return package
     end
 
     def delete_local_package(package_name, version_name)
@@ -222,7 +235,7 @@ module Fig
 
     def write_local_package(package_name, version_name, package)
       file = local_fig_file_for_package(package_name, version_name)
-      @os.write(file, package.unparse)
+      @operating_system.write(file, package.unparse)
     end
 
     def remote_fig_file_for_package(package_name, version_name)
@@ -237,6 +250,10 @@ module Fig
       "#{@remote_repository_url}/#{package_name}/#{version_name}"
     end
 
+    def local_dir_for_package(package_name, version_name)
+      return File.join(@local_repository_dir, package_name, version_name)
+    end
+
     def temp_dir_for_package(package_name, version_name)
       File.join(@local_repository_dir, 'tmp')
     end
@@ -244,5 +261,37 @@ module Fig
     def package_missing?(package_name, version_name)
       not File.exist?(local_fig_file_for_package(package_name, version_name))
     end
+
+    def derive_package_content(
+      package_statements, package_name, version_name, local_dir, local_only
+    )
+      return bundle_resources(package_statements).map do |statement|
+        if statement.is_a?(Statement::Publish)
+          nil
+        elsif statement.is_a?(Statement::Archive) || statement.is_a?(Statement::Resource)
+          if statement.is_a?(Statement::Resource) && !Repository.is_url?(statement.url)
+            archive_name = statement.url
+            archive_remote = "#{remote_dir_for_package(package_name, version_name)}/#{statement.url}"
+          else
+            archive_name = statement.url.split('/').last
+            archive_remote = "#{remote_dir_for_package(package_name, version_name)}/#{archive_name}"
+          end
+          if Repository.is_url?(statement.url)
+            archive_local = File.join(temp_dir, archive_name)
+            @operating_system.download(statement.url, archive_local)
+          else
+            archive_local = statement.url
+          end
+          @operating_system.upload(archive_local, archive_remote, @remote_repository_user) unless local_only
+          @operating_system.copy(archive_local, local_dir + '/' + archive_name)
+          if statement.is_a?(Statement::Archive)
+            @operating_system.unpack_archive(local_dir, archive_name)
+          end
+          statement.class.new(archive_name).unparse('')
+        else
+          statement.unparse('')
+        end
+      end.select { |s|not s.nil? }
+    end
   end
 end