fig 1.0.0 → 1.1.0

data/lib/fig/command/options.rb

@@ -9,6 +9,8 @@ require 'fig/command/action/list_configs'
 require 'fig/command/action/list_dependencies'
 require 'fig/command/action/list_dependencies/all_configs'
 require 'fig/command/action/list_dependencies/default'
+require 'fig/command/action/list_dependencies/graphviz'
+require 'fig/command/action/list_dependencies/graphviz_all_configs'
 require 'fig/command/action/list_dependencies/tree'
 require 'fig/command/action/list_dependencies/tree_all_configs'
 require 'fig/command/action/list_local'
@@ -16,6 +18,8 @@ require 'fig/command/action/list_remote'
 require 'fig/command/action/list_variables'
 require 'fig/command/action/list_variables/all_configs'
 require 'fig/command/action/list_variables/default'
+require 'fig/command/action/list_variables/graphviz'
+require 'fig/command/action/list_variables/graphviz_all_configs'
 require 'fig/command/action/list_variables/tree'
 require 'fig/command/action/list_variables/tree_all_configs'
 require 'fig/command/action/options'
@@ -163,6 +167,10 @@ class Fig::Command::Options
     return @list_tree
   end
 
+  def graphviz?()
+    return @graphviz
+  end
+
   def strip_shell_command(argv)
     argv.each_with_index do |arg, i|
       case arg
@@ -308,6 +316,13 @@ class Fig::Command::Options
       @list_tree = true
     end
 
+    @parser.on(
+      '--graphviz',
+      'for listings, output DOT (http://graphviz.org/content/dot-language)'
+    ) do
+      @graphviz = true
+    end
+
     @parser.on(
       '--list-all-configs',
       'for listings, follow all configurations of the base package'
@@ -523,7 +538,7 @@ class Fig::Command::Options
     end
 
     @parser.on(
-      '-l', '--login', 'login to remote repo as a non-anonymous user'
+      '-l', '--login', 'login to FTP repo as a non-anonymous user'
     ) do
       @login = true
     end
@@ -678,6 +693,12 @@ class Fig::Command::Options
         raise Fig::Command::OptionError.new(
           'Cannot use --suppress-all-includes/--suppress-cross-package-includes with --list-tree.'
         )
+      elsif graphviz?
+        # Not conceptually incompatible, just not implemented (would need to
+        # handle in command/action/role/list_*)
+        raise Fig::Command::OptionError.new(
+          'Cannot use --suppress-all-includes/--suppress-cross-package-includes with --graphviz.'
+        )
       elsif list_all_configs?
         # Not conceptually incompatible, just not implemented (would need to
         # handle in command/action/role/list_*)
@@ -696,17 +717,27 @@ class Fig::Command::Options
         )
       end
     elsif list_tree?
-      if ! @base_action.list_dependencies? && ! @base_action.list_variables?
-        raise Fig::Command::OptionError.new(
-          %q<The --list-tree option isn't useful without --list-dependencies/--list-variables.>
-        )
-      end
+      validate_list_option '--list-tree'
+    elsif graphviz?
+      validate_list_option '--graphviz'
     elsif list_all_configs?
-      if ! @base_action.list_dependencies? && ! @base_action.list_variables?
-        raise Fig::Command::OptionError.new(
-          %q<The --list-all-configs option isn't useful without --list-dependencies/--list-variables.>
-        )
-      end
+      validate_list_option '--list-all-configs'
+    end
+
+    if list_tree? && graphviz?
+      raise Fig::Command::OptionError.new(
+        'Cannot use --list-tree and --graphviz at the same time.'
+      )
+    end
+
+    return
+  end
+
+  def validate_list_option(option)
+    if ! @base_action.list_dependencies? && ! @base_action.list_variables?
+      raise Fig::Command::OptionError.new(
+        %Q<The #{option} option isn't useful without --list-dependencies/--list-variables.>
+      )
     end
 
     return
@@ -720,6 +751,8 @@ class Fig::Command::Options
       sub_action_name = :Default
       if list_tree?
         sub_action_name = list_all_configs? ? :TreeAllConfigs : :Tree
+      elsif graphviz?
+        sub_action_name = list_all_configs? ? :GraphvizAllConfigs : :Graphviz
       elsif list_all_configs?
         sub_action_name = :AllConfigs
       end

data/lib/fig/command/options/parser.rb

@@ -63,13 +63,18 @@ Querying:
 
   fig {--list-local | --list-remote}                                      [...]
   fig {-g | --get} VARIABLE                                  [DESCRIPTOR] [...]
-  fig --list-dependencies [--list-tree] [--list-all-configs] [DESCRIPTOR] [...]
-  fig --list-variables    [--list-tree] [--list-all-configs] [DESCRIPTOR] [...]
+  fig --list-dependencies [...list options...]               [DESCRIPTOR] [...]
+  fig --list-variables    [...list options...]               [DESCRIPTOR] [...]
   fig --list-configs                                         [DESCRIPTOR] [...]
   fig --dump-package-definition-text                         [DESCRIPTOR] [...]
   fig --dump-package-definition-parsed                       [DESCRIPTOR] [...]
   fig --dump-package-definition-for-command-line             [DESCRIPTOR] [...]
 
+List options (represented as "[...list options...]" above):
+
+      [--list-tree | --graphviz]
+      [--list-all-configs]
+
 Standard options (represented as "[...]" above):
 
       [-u | --update | -m | --update-if-missing]

data/lib/fig/figrc.rb

@@ -17,7 +17,7 @@ class Fig::FigRC
   def self.find(
     override_path,
     specified_repository_url,
-    login,
+    operating_system,
     fig_home,
     disable_figrc = false
   )
@@ -33,7 +33,7 @@ class Fig::FigRC
     configuration.remote_repository_url = repository_url
 
     handle_repository_configuration(
-      configuration, repository_url, login, fig_home
+      configuration, repository_url, operating_system, fig_home
     )
 
     return configuration
@@ -77,7 +77,7 @@ class Fig::FigRC
   end
 
   def self.handle_repository_configuration(
-    configuration, repository_url, login, fig_home
+    configuration, repository_url, operating_system, fig_home
   )
     return if repository_url.nil?
 
@@ -85,11 +85,9 @@ class Fig::FigRC
     repo_figrc_path =
       File.expand_path(File.join(fig_home, REPOSITORY_CONFIGURATION))
 
-    os = Fig::OperatingSystem.new(login)
-
     repo_config_exists = nil
     begin
-      os.download( figrc_url, repo_figrc_path )
+      operating_system.download figrc_url, repo_figrc_path
       repo_config_exists = true
     rescue Fig::FileNotFoundError
       repo_config_exists = false

data/lib/fig/operating_system.rb

@@ -1,24 +1,19 @@
 require 'cgi'
 require 'fileutils'
-require 'find'
 # Must specify absolute path of ::Archive when using
 # this module to avoid conflicts with Fig::Statement::Archive
 require 'libarchive_ruby'
-require 'net/http'
-require 'net/ssh'
-require 'net/sftp'
-require 'net/netrc'
 require 'rbconfig'
-require 'tempfile'
-
-require 'highline/import'
 
 require 'fig/at_exit'
 require 'fig/environment_variables/case_insensitive'
 require 'fig/environment_variables/case_sensitive'
-require 'fig/file_not_found_error'
 require 'fig/logging'
 require 'fig/network_error'
+require 'fig/protocol/file'
+require 'fig/protocol/ftp'
+require 'fig/protocol/http'
+require 'fig/protocol/sftp'
 require 'fig/repository_error'
 require 'fig/url'
 require 'fig/user_input_error'
@@ -64,33 +59,11 @@ class Fig::OperatingSystem
   end
 
   def initialize(login)
-    @login = login
-    @username = ENV['FIG_USERNAME']
-    @password = ENV['FIG_PASSWORD']
-  end
-
-  def get_username()
-    # #ask() comes from highline
-    @username ||= ask('Username: ') { |q| q.echo = true }
-  end
-
-  def get_password()
-    # #ask() comes from highline
-    @password ||= ask('Password: ') { |q| q.echo = false }
-  end
-
-  def ftp_login(ftp, host)
-    if @login
-      rc = Net::Netrc.locate(host)
-      if rc
-        @username = rc.login
-        @password = rc.password
-      end
-      ftp.login(get_username, get_password)
-    else
-      ftp.login()
-    end
-    ftp.passive = true
+    @protocols = {}
+    @protocols['file'] = Fig::Protocol::File.new
+    @protocols['ftp']  = Fig::Protocol::FTP.new login
+    @protocols['http'] = Fig::Protocol::HTTP.new
+    @protocols['sftp'] = Fig::Protocol::SFTP.new
   end
 
   def list(dir)
@@ -105,55 +78,11 @@ class Fig::OperatingSystem
     File.open(path, 'wb') { |f| f.binmode; f << content }
   end
 
-  def strip_paths_for_list(ls_output, packages, path)
-    if not ls_output.nil?
-      ls_output = ls_output.gsub(path + '/', '').gsub(path, '').split("\n")
-      ls_output.each do |line|
-        parts = line.gsub(/\\/, '/').sub(/^\.\//, '').sub(/:$/, '').chomp().split('/')
-        packages << parts.join('/') if parts.size == 2
-      end
-    end
-  end
-
   def download_list(url)
     begin
-      uri = Fig::URL.parse(url)
-    rescue
-      Fig::Logging.fatal %Q<Unable to parse url: "#{url}">
-      raise Fig::NetworkError.new
-    end
+      protocol, uri = decode_protocol url
 
-    begin
-      case uri.scheme
-      when 'ftp'
-        ftp = Net::FTP.new(uri.host)
-        ftp_login(ftp, uri.host)
-        ftp.chdir(uri.path)
-        dirs = ftp.nlst
-        ftp.close
-
-        download_ftp_list(uri, dirs)
-      when 'ssh'
-        packages = []
-        Net::SSH.start(uri.host, uri.user) do |ssh|
-          ls = ssh.exec!("[ -d #{uri.path} ] && find #{uri.path}")
-          strip_paths_for_list(ls, packages, uri.path)
-        end
-        packages
-      when 'file'
-        packages = []
-        unescaped_path = CGI.unescape uri.path
-        return packages if ! File.exist?(unescaped_path)
-
-        ls = ''
-        Find.find(unescaped_path) { |file| ls << file.to_s; ls << "\n" }
-
-        strip_paths_for_list(ls, packages, unescaped_path)
-        return packages
-      else
-        Fig::Logging.fatal "Protocol not supported: #{url}"
-        raise Fig::NetworkError.new "Protocol not supported: #{url}"
-      end
+      return protocol.download_list uri
     rescue SocketError => error
       Fig::Logging.debug error.message
       raise Fig::NetworkError.new "#{url}: #{error.message}"
@@ -163,150 +92,23 @@ class Fig::OperatingSystem
     end
   end
 
-  def download_ftp_list(uri, dirs)
-    # Run a bunch of these in parallel since they're slow as hell
-    num_threads = (ENV['FIG_FTP_THREADS'] || '16').to_i
-    threads = []
-    all_packages = []
-    (0..num_threads-1).each { |num| all_packages[num] = [] }
-    (0..num_threads-1).each do |num|
-      threads << Thread.new do
-        packages = all_packages[num]
-        ftp = Net::FTP.new(uri.host)
-        ftp_login(ftp, uri.host)
-        ftp.chdir(uri.path)
-        pos = num
-        while pos < dirs.length
-          pkg = dirs[pos]
-          begin
-            ftp.nlst(dirs[pos]).each do |ver|
-              packages << pkg + '/' + ver
-            end
-          rescue Net::FTPPermError
-            # Ignore this error because it's indicative of the FTP library
-            # encountering a file or directory that it does not have
-            # permission to open.  Fig needs to be able to have secure
-            # repos/packages and there is no way easy way to deal with the
-            # permissions issues other than consuming these errors.
-            #
-            # Actually, with FTP, you can't tell the difference between a
-            # file not existing and not having permission to access it (which
-            # is probably a good thing).
-          end
-          pos += num_threads
-        end
-        ftp.close
-      end
-    end
-    threads.each { |thread| thread.join }
-    all_packages.flatten.sort
-  end
-
   # Determine whether we need to update something.  Returns nil to indicate
   # "don't know".
   def path_up_to_date?(url, path)
     return false if ! File.exist? path
 
-    uri = Fig::URL.parse(url)
-    case uri.scheme
-    when 'ftp'
-      begin
-        ftp = Net::FTP.new(uri.host)
-        ftp_login(ftp, uri.host)
-
-        if ftp.mtime(uri.path) <= File.mtime(path)
-          return true
-        end
-
-        return false
-      rescue Net::FTPPermError => error
-        Fig::Logging.debug error.message
-        raise Fig::FileNotFoundError.new error.message, url
-      rescue SocketError => error
-        Fig::Logging.debug error.message
-        raise Fig::FileNotFoundError.new error.message, url
-      end
-    when 'http'
-      return nil # Not implemented
-    when 'ssh'
-    when 'file'
-      begin
-        unescaped_path = CGI.unescape uri.path
-        if File.mtime(unescaped_path) <= File.mtime(path)
-          return true
-        end
-
-        return false
-      rescue Errno::ENOENT => error
-        raise Fig::FileNotFoundError.new error.message, url
-      end
-    else
-      raise_unknown_protocol(url)
-    end
+    protocol, uri = decode_protocol url
+    return protocol.path_up_to_date? uri, path
   end
 
   # Returns whether the file was not downloaded because the file already
   # exists and is already up-to-date.
   def download(url, path)
-    FileUtils.mkdir_p(File.dirname(path))
-    uri = Fig::URL.parse(url)
-    case uri.scheme
-    when 'ftp'
-      begin
-        ftp = Net::FTP.new(uri.host)
-        ftp_login(ftp, uri.host)
-
-        if File.exist?(path) && ftp.mtime(uri.path) <= File.mtime(path)
-          Fig::Logging.debug "#{path} is up to date."
-          return false
-        else
-          log_download(url, path)
-          ftp.getbinaryfile(uri.path, path, 256*1024)
-          return true
-        end
-      rescue Net::FTPPermError => error
-        Fig::Logging.debug error.message
-        raise Fig::FileNotFoundError.new error.message, url
-      rescue SocketError => error
-        Fig::Logging.debug error.message
-        raise Fig::FileNotFoundError.new error.message, url
-      rescue Errno::ETIMEDOUT => error
-        Fig::Logging.debug error.message
-        raise Fig::FileNotFoundError.new error.message, url
-      end
-    when 'http'
-      log_download(url, path)
-      File.open(path, 'wb') do |file|
-        file.binmode
-
-        begin
-          download_via_http_get(url, file)
-        rescue SystemCallError => error
-          Fig::Logging.debug error.message
-          raise Fig::FileNotFoundError.new error.message, url
-        rescue SocketError => error
-          Fig::Logging.debug error.message
-          raise Fig::FileNotFoundError.new error.message, url
-        end
-      end
-    when 'ssh'
-      # TODO need better way to do conditional download
-      timestamp = File.exist?(path) ? File.mtime(path).to_i : 0
-      # Requires that remote installation of fig be at the same location as the local machine.
-      command = `which fig-download`.strip + " #{timestamp} #{uri.path}"
-      log_download(url, path)
-      ssh_download(uri.user, uri.host, path, command)
-    when 'file'
-      begin
-        unescaped_path = CGI.unescape uri.path
-        FileUtils.cp(unescaped_path, path)
-        return true
-      rescue Errno::ENOENT => error
-        raise Fig::FileNotFoundError.new error.message, url
-      end
-    else
-      raise_unknown_protocol(url)
-    end
+    protocol, uri = decode_protocol url
+
+    FileUtils.mkdir_p(File.dirname path)
+
+    return protocol.download uri, path
   end
 
   # Returns the basename and full path to the download.
@@ -343,43 +145,12 @@ class Fig::OperatingSystem
 
   def upload(local_file, remote_file)
     Fig::Logging.debug "Uploading #{local_file} to #{remote_file}."
-    uri = Fig::URL.parse(remote_file)
-    case uri.scheme
-    when 'ssh'
-      ssh_upload(uri.user, uri.host, local_file, remote_file)
-    when 'ftp'
-      #      fail unless system "curl -T #{local_file} --create-dirs --ftp-create-dirs #{remote_file}"
-      require 'net/ftp'
-      ftp_uri = Fig::URL.parse(ENV['FIG_REMOTE_URL'])
-      ftp_root_path = ftp_uri.path
-      ftp_root_dirs = ftp_uri.path.split('/')
-      remote_publish_path = uri.path[0, uri.path.rindex('/')]
-      remote_publish_dirs = remote_publish_path.split('/')
-      # Use array subtraction to deduce which project/version folder to upload
-      # to, i.e. [1,2,3] - [2,3,4] = [1]
-      remote_project_dirs = remote_publish_dirs - ftp_root_dirs
-      Net::FTP.open(uri.host) do |ftp|
-        ftp_login(ftp, uri.host)
-        # Assume that the FIG_REMOTE_URL path exists.
-        ftp.chdir(ftp_root_path)
-        remote_project_dirs.each do |dir|
-          # Can't automatically create parent directories, so do it manually.
-          if ftp.nlst().index(dir).nil?
-            ftp.mkdir(dir)
-            ftp.chdir(dir)
-          else
-            ftp.chdir(dir)
-          end
-        end
-        ftp.putbinaryfile(local_file)
-      end
-    when 'file'
-      unescaped_path = CGI.unescape uri.path
-      FileUtils.mkdir_p(File.dirname(unescaped_path))
-      FileUtils.cp(local_file, unescaped_path)
-    else
-      raise_unknown_protocol(uri)
-    end
+
+
+    protocol, uri = decode_protocol remote_file
+    protocol.upload local_file, uri
+
+    return
   end
 
   def delete_and_recreate_directory(dir)
@@ -397,7 +168,7 @@ class Fig::OperatingSystem
       end
     else
       if ! File.exist?(target) || File.mtime(source) != File.mtime(target)
-        log_info "#{msg} #{target}" if msg
+        Fig::Logging.info "#{msg} #{target}" if msg
         FileUtils.mkdir_p(File.dirname(target))
         FileUtils.cp(source, target)
         File.utime(File.atime(source), File.mtime(source), target)
@@ -409,10 +180,6 @@ class Fig::OperatingSystem
     Dir.chdir(directory) { FileUtils.mv(from, to, :force => true) }
   end
 
-  def log_info(msg)
-    Fig::Logging.info msg
-  end
-
   # Expects files_to_archive as an Array of filenames.
   def create_archive(archive_name, files_to_archive)
     # TODO: Need to verify files_to_archive exists.
@@ -509,9 +276,13 @@ class Fig::OperatingSystem
 
   private
 
-  SUCCESS = 0
-  NOT_MODIFIED = 3
-  NOT_FOUND = 4
+  def decode_protocol(url)
+    uri = Fig::URL.parse(url)
+    protocol = @protocols[uri.scheme]
+    raise_unknown_protocol(url) if protocol.nil?
+
+    return protocol, uri
+  end
 
   def check_archive_entry_for_windows(entry, archive_path)
     bad_type = nil
@@ -536,77 +307,6 @@ class Fig::OperatingSystem
     return
   end
 
-  # path = The local path the file should be downloaded to.
-  # command = The command to be run on the remote host.
-  def ssh_download(user, host, path, command)
-    return_code = nil
-    tempfile = Tempfile.new('tmp')
-    Net::SSH.start(host, user) do |ssh|
-      ssh.open_channel do |channel|
-        channel.exec(command)
-        channel.on_data() { |ch, data| tempfile << data }
-        channel.on_extended_data() { |ch, type, data| Fig::Logging.error "SSH Download ERROR: #{data}" }
-        channel.on_request('exit-status') { |ch, request|
-          return_code = request.read_long
-        }
-      end
-    end
-
-    tempfile.close()
-
-    case return_code
-    when NOT_MODIFIED
-      tempfile.delete
-      return false
-    when NOT_FOUND
-      tempfile.delete
-      raise Fig::FileNotFoundError.new 'Remote path not found', path
-    when SUCCESS
-      FileUtils.mv(tempfile.path, path)
-      return true
-    else
-      tempfile.delete
-      Fig::Logging.fatal "Unable to download file #{path}: #{return_code}"
-      raise Fig::NetworkError.new("Unable to download file #{path}: #{return_code}")
-    end
-  end
-
-  def ssh_upload(user, host, local_file, remote_file)
-    uri = Fig::URL.parse(remote_file)
-    dir = uri.path[0, uri.path.rindex('/')]
-    Net::SSH.start(host, user) do |ssh|
-      ssh.exec!("mkdir -p #{dir}")
-    end
-    Net::SFTP.start(host, user) do |sftp|
-      sftp.upload!(local_file, uri.path)
-    end
-  end
-
-  def download_via_http_get(uri_string, file, redirection_limit = 10)
-    if redirection_limit < 1
-      Fig::Logging.debug 'Too many HTTP redirects.'
-      raise Fig::FileNotFoundError.new 'Too many HTTP redirects.', uri_string
-    end
-
-    response = Net::HTTP.get_response(URI(uri_string))
-
-    case response
-    when Net::HTTPSuccess then
-      file.write(response.body)
-    when Net::HTTPRedirection then
-      location = response['location']
-      Fig::Logging.debug "Redirecting to #{location}."
-      download_via_http_get(location, file, redirection_limit - 1)
-    else
-      Fig::Logging.debug "Download failed: #{response.code} #{response.message}."
-      raise Fig::FileNotFoundError.new(
-        "Download failed: #{response.code} #{response.message}.", uri_string
-      )
-    end
-
-    return
-  end
-
   def raise_unknown_protocol(url)
     Fig::Logging.fatal %Q<Don't know how to handle the protocol in "#{url}".>
     raise Fig::NetworkError.new(
@@ -615,8 +315,4 @@ class Fig::OperatingSystem
 
     return
   end
-
-  def log_download(url, path)
-    Fig::Logging.debug "Downloading #{url} to #{path}."
-  end
 end