fig 0.1.24-universal-darwin9.0

data/lib/fig/environment.rb

@@ -0,0 +1,171 @@
+module Fig
+
+  # This class manages the program's state, including the value of all environment 
+  # variables, and which packages have already been applied
+  class Environment
+    DEFAULT_VERSION_NAME = "current"
+
+    def initialize(os, repository, variables)
+      @os = os
+      @repository = repository
+      @variables = variables
+      @retrieve_vars = {}
+      @packages = {}
+      @applied_configs = {}
+    end
+
+    # Returns the value of an envirionment variable
+    def [](name)
+      @variables[name]
+    end
+
+    # Indicates that the values from a particular envrionment variable path should
+    # be copied to a local directory 
+    def add_retrieve(name, path)
+      @retrieve_vars[name] = path
+    end
+
+    def register_package(package)
+      name = package.package_name
+      if @packages[name]
+        puts "Package already exists with name: #{name}"
+        exit 10
+      end
+      @packages[name] = package
+    end
+
+    def apply_config(package, config_name)
+      if (@applied_configs[package.package_name] ||= []).member?(config_name)
+        return
+      end
+      config = package[config_name]
+      config.statements.each { |stmt| apply_config_statement(package, stmt) }
+      @applied_configs[package.package_name] << config_name
+    end
+
+    def execute_shell(command)
+      with_environment do
+        yield command.map{|arg| expand_arg(arg)}
+      end
+    end
+
+    def execute_config(base_package, package_name, config_name, version_name, args)
+      package = lookup_package(package_name || base_package.package_name, version_name)
+      result = nil
+      commands = package[config_name || "default"].commands
+      with_environment do
+        # todo nil check
+        commands.each do |command|
+          result = yield expand_arg("#{command.command} #{args.join(' ')}").gsub("@",package.directory).split(" ")
+        end
+      end
+      result
+    end
+
+    def apply_config_statement(base_package, statement)
+      case statement
+      when Path
+        append_variable(base_package, statement.name, statement.value)
+      when Set
+        set_variable(base_package, statement.name, statement.value)
+      when Include
+        include_config(base_package, statement.package_name, statement.config_name, statement.version_name)
+      when Command
+        # ignore
+      else
+        fail "Unexpected statement: #{statement}"
+      end
+    end
+
+    def include_config(base_package, package_name, config_name, version_name)
+      package = lookup_package(package_name || base_package.package_name, version_name)
+      apply_config(package, config_name || "default")
+    end
+
+    def direct_retrieve(package_name, source_path, target_path)
+      package = lookup_package(package_name, nil)
+      FileUtils.mkdir_p(target_path)
+      FileUtils.cp_r(File.join(package.directory, source_path, '.'), target_path)
+    end
+
+    private 
+
+    def set_variable(base_package, name, value)
+      @variables[name] = expand_value(base_package, name, value)
+    end
+
+    def append_variable(base_package, name, value)
+      value = expand_value(base_package, name, value)
+      # TODO: converting all environment variables to upcase is not a robust
+      #       comparison. It also assumes all env vars will be in upcase
+      #       in package.fig
+      prev = nil
+      @variables.each do |key, val|
+        if key.upcase == name.upcase
+          name = key
+          prev = val
+        end
+      end
+      if prev
+        @variables[name] = value + File::PATH_SEPARATOR + prev
+      else
+        @variables[name] = value
+      end
+    end
+
+    def with_environment
+      old_env = {}
+      begin
+        @variables.each { |key,value| old_env[key] = ENV[key]; ENV[key] = value }
+        yield
+      ensure
+        old_env.each { |key,value| ENV[key] = value }
+      end
+    end
+
+    def lookup_package(package_name, version_name)
+      package = @packages[package_name]
+      if package.nil?
+        package = @repository.load_package(package_name, version_name || DEFAULT_VERSION_NAME)
+        @packages[package_name] = package
+      elsif version_name && version_name != package.version_name
+        puts "Version mismatch: #{package_name}" 
+        exit 10
+      end
+      package
+    end
+
+    # Replace @ symbol with the package's directory
+    def expand_value(base_package, name, value)
+      return value unless base_package && base_package.package_name
+      file = value.gsub(/\@/, base_package.directory)
+      if @retrieve_vars.member?(name)
+        # A '//' in the source file's path tells us to preserve path information
+        # after the '//' when doing a retrieve.
+        if file.split('//').size > 1
+          preserved_path = file.split('//').last
+          target = File.join(@retrieve_vars[name].gsub(/\[package\]/, base_package.package_name), preserved_path)
+        else
+          target = File.join(@retrieve_vars[name].gsub(/\[package\]/, base_package.package_name), File.basename(file))
+        end
+        unless @os.exist?(target) && @os.mtime(target) >= @os.mtime(file)
+          @os.log_info("retrieving #{target}")
+          @os.copy(file, target)
+        end
+        file = target
+      end
+      file
+    end
+
+    def expand_arg(arg)
+      arg.gsub(/\@([a-zA-Z0-9\-\.]+)/) do |match|
+        package = @packages[$1]
+        if package.nil?
+          puts "Package not found: #{$1}"
+          exit 10
+        end
+        package.directory
+      end
+    end
+  end
+end

data/lib/fig/grammar.treetop

@@ -0,0 +1,141 @@
+require 'fig/package'
+
+module Fig
+
+grammar Fig
+  rule package
+    ws statements:(package_statement*) {
+      def to_package(package_name, version_name, directory)
+        Package.new(package_name, version_name, directory, statements.elements.map { |statement| statement.to_package_statement })
+      end
+    }
+  end
+
+  rule package_statement
+    archive / resource / retrieve / config
+  end
+
+  rule archive
+    "archive" ws url {
+      def to_package_statement
+        Archive.new(url.value.text_value)
+      end
+    }
+  end
+
+  rule resource
+    "resource" ws url {
+      def to_package_statement
+        Resource.new(url.value.text_value)
+      end
+    }
+  end
+
+  rule retrieve
+    "retrieve" ws var:[@a-zA-Z0-9/\._]+ "->" path:[a-zA-Z0-9/\.\-\[\]]+ ws {
+      def to_package_statement
+        Retrieve.new(var.text_value, path.text_value)
+      end
+    }
+  end
+
+  rule install
+    "install" ws statements:config_statement* "end" ws {
+      def to_package_statement
+        Install.new(statements.elements.map { |statement| statement.to_config_statement })
+      end
+    }
+  end
+
+  rule config
+    "config" ws config_name ws statements:config_statement* "end" ws {
+      def to_package_statement
+        Configuration.new(config_name.text_value, statements.elements.map { |statement| statement.to_config_statement })
+      end
+    }
+  end
+
+  rule config_statement
+    include / command / path / set
+  end
+
+  rule include
+    "include" ws descriptor {
+      def to_config_statement
+        package = descriptor.respond_to?(:package) ? descriptor.package.text_value : nil
+        config = descriptor.get_config
+        version = descriptor.get_version
+        Include.new(package, config, version)
+      end
+    }
+  end
+
+  rule path
+    ("append" / "path" / "add") ws name:[a-zA-Z0-9_]+ "=" value:[@a-zA-Z0-9/\-\\._]+ ws {
+      def to_config_statement
+        Path.new(name.text_value, value.text_value)
+      end
+    }
+  end
+
+  rule set
+    "set" ws name:[a-zA-Z0-9_]+ "=" value:[@a-zA-Z0-9/\-\\._]+ ws {
+      def to_config_statement
+        Set.new(name.text_value, value.text_value)
+      end
+    }
+  end
+
+  rule command
+    "command" ws string {
+      def to_config_statement
+        Command.new(string.value.text_value)
+      end
+    }
+  end
+
+  rule string
+    '"' value:(!'"' . )* '"' ws
+  end
+
+  rule descriptor
+    ((package:[a-zA-Z0-9.-]+ ("/" version:[a-zA-Z0-9_\-.]+)? (":" config:config_name)? ws) / 
+    (":" config:config_name ws)) {
+      def get_version
+        elements.each do |element|
+          if element.respond_to?(:version)
+            return element.version.text_value
+          end
+        end
+        nil
+      end
+      def get_config
+        return self.config.text_value if self.respond_to?(:config)
+        elements.each do |element|
+          if element.respond_to?(:config)
+            return element.config.text_value
+          end
+        end
+        nil
+      end
+    }
+  end
+
+  rule config_name
+    [a-zA-Z0-9_\-.]+
+  end
+
+  rule name
+    value:[a-zA-Z0-9]+ ws
+  end
+
+  rule url
+    (value:[a-zA-Z0-9:/\-\\._\*]+ ws) / ('"' value:[a-zA-Z0-9:/\-\\._]+ '"' ws)
+  end
+
+  rule ws
+    [ \n\r\t]+
+  end
+end
+
+end

data/lib/fig/options.rb

@@ -0,0 +1,98 @@
+require 'optparse'
+require 'fig/package'
+
+module Fig
+  def parse_descriptor(descriptor)
+    # todo should use treetop for these:
+    package_name = descriptor =~ /^([^:\/]+)/ ? $1 : nil
+    config_name = descriptor =~ /:([^:\/]+)/ ? $1 : nil
+    version_name = descriptor =~ /\/([^:\/]+)/ ? $1 : nil  
+    return package_name, config_name, version_name
+  end
+
+  def parse_options(argv)
+    options = {}
+
+    parser = OptionParser.new do |opts|
+      opts.banner = "Usage: fig [--debug] [--update] [--config <config>] [--get <var> | --list | <package> | - <command>]"
+
+      opts.on('-?', '-h','--help','display this help text') do
+        puts opts
+	exit 1
+      end 
+
+      options[:debug] = false
+      opts.on('-d', '--debug', 'print debug info') { options[:debug] = true }
+
+      options[:update] = false
+      opts.on('-u', '--update', 'check remote repository for updates') { options[:update] = true; options[:retrieve] = true }
+
+      options[:update_if_missing] = false
+      opts.on('-m', '--update-if-missing', 'check for updates only if package is missing locally') { options[:update_if_missing] = true; options[:retrieve] = true }
+
+      options[:config] = "default"
+      opts.on('-c', '--config CFG', 'name of configuration to apply') { |config| options[:config] = config }
+
+      options[:echo] = nil
+      opts.on('-g', '--get VAR', 'print value of environment variable') { |echo| options[:echo] = echo }
+
+      options[:publish] = nil
+      opts.on('--publish PKG', 'install package in local and remote repositories') { |publish| options[:publish] = publish }
+
+      options[:publish_local] = nil
+      opts.on('--publish-local PKG', 'install package in local repositorie only') { |publish_local| options[:publish_local] = publish_local }
+
+      options[:force] = nil
+      opts.on('--force', 'force overwriting of an existing remote package version') { |force| options[:force] = force }
+
+      options[:resources] =[]
+      opts.on('--resource PATH', 'resource to include in package (when using --publish)') do |path| 
+        options[:resources] << Resource.new(path) 
+      end
+
+      options[:archives] =[]
+      opts.on('--archive PATH', 'archive to include in package (when using --publish)') do |path| 
+        options[:archives] << Archive.new(path)
+      end
+
+      options[:list] = false
+      opts.on('--list', 'list packages in local repository') { options[:list] = true }
+
+      options[:list_remote] = false
+      opts.on('--list-remote', 'list packages in remote repository') { options[:list_remote] = true }
+
+      options[:list_configs] = []
+      opts.on('--list-configs PKG', 'list configurations in package') { |descriptor| options[:list_configs] << descriptor }
+
+      options[:cleans] = []
+      opts.on('--clean PKG', 'remove  package from local repository') { |descriptor| options[:cleans] <<  descriptor }
+
+      options[:modifiers] = []
+
+      opts.on('-i', '--include PKG', 'include package in environment') do |descriptor| 
+        package_name, config_name, version_name = parse_descriptor(descriptor)
+        options[:modifiers] << Include.new(package_name, config_name, version_name) 
+      end
+
+      opts.on('-s', '--set VAR=VAL', 'set environment variable') do |var_val| 
+        var, val = var_val.split('=')
+        options[:modifiers] << Set.new(var, val) 
+      end
+
+      opts.on('-p', '--append VAR=VAL', 'append environment variable') do |var_val| 
+        var, val = var_val.split('=')
+        options[:modifiers] << Path.new(var, val)
+      end
+
+      options[:input] = nil
+      opts.on('--file FILE', 'fig file to read (use - for stdin)') { |path| options[:input] = path }
+      opts.on('--no-file', 'ignore package.fig file in current directory') { |path| options[:input] = :none }
+
+      options[:home] = ENV['FIG_HOME'] || File.expand_path("~/.fighome")
+    end
+
+    parser.parse!(argv)
+
+    return options, argv
+  end
+end

data/lib/fig/os.rb

@@ -0,0 +1,325 @@
+require 'fileutils'
+# Must specify absolute path of ::Archive when using
+# this module to avoid conflicts with Fig::Package::Archive
+require 'libarchive_ruby' unless RUBY_PLATFORM == 'java'
+require 'uri'
+require 'net/http'
+require 'net/ssh'
+require 'net/sftp'
+require 'tempfile'
+
+module Fig
+  class NotFoundException < Exception
+  end
+
+  class OS
+    def list(dir)
+      Dir.entries(dir) - ['.','..']
+    end
+    
+    def exist?(path)
+      File.exist?(path)
+    end
+    
+    def mtime(path)
+      File.mtime(path)
+    end
+    
+    def read(path)
+      File.read(path)
+    end
+    
+    def write(path, content)
+      File.open(path, "wb") { |f| f.binmode; f << content }
+    end
+    
+    SUCCESS = 0
+    NOT_MODIFIED = 3
+    NOT_FOUND = 4
+    
+    def download_list(url)
+      begin
+        uri = URI.parse(url)
+      rescue 
+        puts "Unable to parse url: '#{url}'"
+        exit 10
+      end
+      case uri.scheme
+      when "ftp"
+        ftp = Net::FTP.new(uri.host)
+        ftp.login
+        ftp.chdir(uri.path)
+        packages = []
+        ftp.retrlines('LIST -R .') do |line|
+          parts = line.gsub(/\\/, '/').sub(/^\.\//, '').sub(/:$/, '').split('/')
+          packages << parts.join('/') if parts.size == 2
+        end
+        ftp.close
+        packages
+      when "ssh"
+        packages = []
+        Net::SSH.start(uri.host, uri.user) do |ssh|
+          ls = ssh.exec!("[ -d #{uri.path} ] && find #{uri.path}")
+          if not ls.nil?
+            ls = ls.gsub(uri.path + "/", "").gsub(uri.path, "")
+            ls.each do |line|
+              parts = line.gsub(/\\/, '/').sub(/^\.\//, '').sub(/:$/, '').chomp().split('/')
+              packages << parts.join('/') if parts.size == 2
+            end
+          end
+        end
+        packages
+      else
+        puts "Protocol not supported: #{url}"
+        exit 10
+      end
+    end
+    
+    def download(url, path)
+      FileUtils.mkdir_p(File.dirname(path))
+      uri = URI.parse(url)
+      case uri.scheme
+      when "ftp"
+        ftp = Net::FTP.new(uri.host)
+        ftp.login
+        begin
+          if File.exist?(path) && ftp.mtime(uri.path) <= File.mtime(path)
+            return false
+          else 
+            puts "downloading #{url}"
+            ftp.getbinaryfile(uri.path, path, 256*1024)
+            return true
+          end
+        rescue Net::FTPPermError
+          raise NotFoundException.new
+        end
+      when "http"
+        http = Net::HTTP.new(uri.host)
+        puts "downloading #{url}"
+        File.open(path, "wb") do |file|
+          file.binmode
+          http.get(uri.path) do |block|
+            file.write(block)
+          end
+        end
+      when "ssh"
+        # TODO need better way to do conditional download
+        #       timestamp = `ssh #{uri.user + '@' if uri.user}#{uri.host} "ruby -e 'puts File.mtime(\\"#{uri.path}\\").to_i'"`.to_i
+        timestamp = File.exist?(path) ? File.mtime(path).to_i : 0 
+        cmd = `which fig-download`.strip + " #{timestamp} #{uri.path}"
+        ssh_download(uri.user, uri.host, path, cmd)
+      else
+        puts "Unknown protocol: #{url}"
+        exit 10
+      end
+   end
+    
+    def download_resource(url, dir)
+      FileUtils.mkdir_p(dir)
+      download(url, File.join(dir, URI.parse(url).path.split('/').last))
+    end
+    
+    def download_archive(url, dir)
+      FileUtils.mkdir_p(dir)
+      basename = URI.parse(url).path.split('/').last
+      path = File.join(dir, basename)
+      download(url, path)
+      case basename
+      when /\.tar\.gz$/
+        unpack_archive(dir, path)
+      when /\.tgz$/
+        unpack_archive(dir, path)
+      when /\.tar\.bz2$/
+        unpack_archive(dir, path)
+      when /\.zip$/
+        unpack_archive(dir, path)
+      else
+        puts "Unknown archive type: #{basename}"
+        exit 10
+      end
+    end
+    
+    def upload(local_file, remote_file, user)
+      puts "uploading #{local_file} to #{remote_file}"
+      uri = URI.parse(remote_file)
+      case uri.scheme
+      when "ssh"
+        ssh_upload(uri.user, uri.host, local_file, remote_file)
+      when "ftp"
+        #      fail unless system "curl -T #{local_file} --create-dirs --ftp-create-dirs #{remote_file}"
+       require 'net/ftp'
+        ftp_uri = URI.parse(ENV["FIG_REMOTE_URL"])
+        ftp_root_path = ftp_uri.path
+        ftp_root_dirs = ftp_uri.path.split("/")
+        remote_publish_path = uri.path[0, uri.path.rindex("/")]
+        remote_publish_dirs = remote_publish_path.split("/")
+        # Use array subtraction to deduce which project/version folder to upload to,
+        # i.e. [1,2,3] - [2,3,4] = [1]
+        remote_project_dirs = remote_publish_dirs - ftp_root_dirs
+        Net::FTP.open(uri.host) do |ftp|
+          ftp.login
+          # Assume that the FIG_REMOTE_URL path exists.
+          ftp.chdir(ftp_root_path)
+          remote_project_dirs.each do |dir|
+            # Can't automatically create parent directories, so do it manually.
+           if ftp.nlst().index(dir).nil?
+             ftp.mkdir(dir)
+             ftp.chdir(dir)
+           else
+             ftp.chdir(dir)
+           end
+          end
+          ftp.putbinaryfile(local_file)
+        end
+      end
+    end
+    
+    def clear_directory(dir)
+      FileUtils.rm_rf(dir)
+      FileUtils.mkdir_p(dir)
+    end
+
+    def exec(dir,command)
+      Dir.chdir(dir) {
+        unless system command
+          puts "Command failed"
+          exit 10
+        end
+      }
+    end
+    
+    def copy(source, target)
+      FileUtils.mkdir_p(File.dirname(target))
+      FileUtils.cp_r(source, target)
+    end
+
+    def move_file(dir, from, to)
+      Dir.chdir(dir) { FileUtils.mv(from, to, :force => true) }
+    end
+    
+    def log_info(msg)
+      $stderr.puts msg
+    end
+
+    # Expects files_to_archive as an Array of filenames.
+    def create_archive(archive_name, files_to_archive)
+      if OS.java?
+        `tar czvf #{archive_name} #{files_to_archive.join(' ')}`
+      else
+        # TODO: Need to verify files_to_archive exists.
+        ::Archive.write_open_filename(archive_name, ::Archive::COMPRESSION_GZIP, ::Archive::FORMAT_TAR) do |ar|
+          files_to_archive.each do |fn|
+            ar.new_entry do |entry|
+              entry.copy_stat(fn)
+              entry.pathname = fn
+              ar.write_header(entry)
+              if !entry.directory?
+                ar.write_data(open(fn) {|f| f.binmode; f.read })
+              end
+            end
+          end
+        end
+      end
+    end
+
+    # This method can handle the following archive types:
+    # .tar.bz2
+    # .tar.gz
+    # .tgz
+    # .zip
+    def unpack_archive(dir, file)
+      Dir.chdir(dir) do
+        if OS.java?
+          `tar xzvf #{file}`
+        else
+          ::Archive.read_open_filename(file) do |ar|
+            while entry = ar.next_header
+              ar.extract(entry)
+            end
+          end
+        end
+      end
+    end
+
+    def self.windows?
+      Config::CONFIG['host_os'] =~ /mswin|mingw/
+    end
+
+    def self.java?
+      RUBY_PLATFORM == 'java'
+    end
+
+    def self.unix?
+      !windows?
+    end
+
+    def shell_exec(cmd)
+      if OS.windows?
+        Windows.shell_exec_windows(cmd)
+      else
+        shell_exec_unix(cmd)
+      end
+    end
+
+    private
+
+    def shell_exec_unix(cmd)
+      Kernel.exec(ENV['SHELL'], '-c', cmd.join(' '))
+    end
+
+    def shell_exec_windows(cmd)
+      #command = ["C:/WINDOWS/system32/cmd.exe", "/C", "call"] + cmd
+      command = ["cmd.exe", "/C"] + cmd
+      command = command.join(' ')
+      Kernel.exec(command)
+    end
+
+    # path = The local path the file should be downloaded to.
+    # cmd = The command to be run on the remote host.
+    def ssh_download(user, host, path, cmd)
+      return_code = nil
+      tempfile = Tempfile.new("tmp")
+      Net::SSH.start(host, user) do |ssh|
+        ssh.open_channel do |channel|
+          channel.exec(cmd)
+          channel.on_data() { |ch, data| tempfile << data }
+          channel.on_extended_data() { |ch, type, data| $stderr.puts "SSH Download ERROR: #{data}" }
+          channel.on_request("exit-status") { |ch, request|
+            return_code = request.read_long
+          }
+        end
+      end
+
+      tempfile.close()
+
+      case return_code
+      when NOT_MODIFIED
+        tempfile.delete
+        return false
+      when NOT_FOUND
+        tempfile.delete
+        puts "File not found: #{path}"
+        exit 10
+      when SUCCESS
+        FileUtils.mv(tempfile.path, path)
+        return true
+      else
+        tempfile.delete
+        $stderr.puts "Unable to download file: #{return_code}"
+        exit 1
+      end
+    end
+
+    def ssh_upload(user, host, local_file, remote_file)
+      uri = URI.parse(remote_file)
+      dir = uri.path[0, uri.path.rindex('/')]
+      Net::SSH.start(host, user) do |ssh|
+        ssh.exec!("mkdir -p #{dir}")
+      end
+      Net::SFTP.start(host, user) do |sftp|
+        sftp.upload!(local_file, uri.path)
+      end
+    end
+
+  end
+end