fig 1.0.0 → 1.1.0

data/lib/fig/package_descriptor.rb

@@ -1,5 +1,4 @@
 require 'fig/package_descriptor_parse_error'
-require 'fig/statement'
 
 module Fig; end
 
@@ -7,6 +6,8 @@ module Fig; end
 class Fig::PackageDescriptor
   include Comparable
 
+  COMPONENT_PATTERN = / \A (?! [.]{1,2} $) [a-zA-Z0-9_.-]+ \z /x
+
   attr_reader :name, :version, :config, :original_string, :description
 
   def self.format(
@@ -117,7 +118,7 @@ class Fig::PackageDescriptor
   def validate_component_format(value, name, options)
     return if value.nil?
 
-    return if value =~ / \A [a-zA-Z0-9_.-]+ \z /x
+    return if value =~ COMPONENT_PATTERN
 
     raise Fig::PackageDescriptorParseError.new(
       %Q<Invalid #{name} ("#{value}")#{standard_exception_suffix(options)}>,

data/lib/fig/protocol.rb

@@ -0,0 +1,47 @@
+require 'fig/logging'
+require 'fig/network_error'
+
+module Fig; end
+
+# File transfers.
+module Fig::Protocol
+  def download_list(uri)
+    Fig::Logging.fatal "Protocol not supported: #{uri}"
+    raise Fig::NetworkError.new "Protocol not supported: #{uri}"
+  end
+
+  # Determine whether we need to update something.  Returns nil to indicate
+  # "don't know".
+  def path_up_to_date?(uri, path)
+    return nil # Not implemented
+  end
+
+  # Returns whether the file was not downloaded because the file already
+  # exists and is already up-to-date.
+  def download(uri, path)
+    Fig::Logging.fatal "Protocol not supported: #{uri}"
+    raise Fig::NetworkError.new "Protocol not supported: #{uri}"
+  end
+
+  def upload(local_file, uri)
+    Fig::Logging.fatal "Protocol not supported: #{uri}"
+    raise Fig::NetworkError.new "Protocol not supported: #{uri}"
+  end
+
+  private
+
+  def strip_paths_for_list(ls_output, packages, path)
+    if not ls_output.nil?
+      ls_output = ls_output.gsub(path + '/', '').gsub(path, '').split("\n")
+      ls_output.each do |line|
+        parts =
+          line.gsub(/\\/, '/').sub(/^\.\//, '').sub(/:$/, '').chomp().split('/')
+        packages << parts.join('/') if parts.size == 2
+      end
+    end
+  end
+
+  def log_download(uri, path)
+    Fig::Logging.debug "Downloading #{uri} to #{path}."
+  end
+end

data/lib/fig/protocol/file.rb

@@ -0,0 +1,64 @@
+require 'cgi'
+require 'fileutils'
+require 'find'
+
+require 'fig/file_not_found_error'
+require 'fig/logging'
+require 'fig/protocol'
+
+module Fig; end
+module Fig::Protocol; end
+
+# File transfers for the local filesystem.
+class Fig::Protocol::File
+  include Fig::Protocol
+
+  def download_list(uri)
+    packages = []
+    unescaped_path = CGI.unescape uri.path
+    return packages if ! ::File.exist?(unescaped_path)
+
+    ls = ''
+    Find.find(unescaped_path) { |file| ls << file.to_s; ls << "\n" }
+
+    strip_paths_for_list(ls, packages, unescaped_path)
+
+    return packages
+  end
+
+  # Determine whether we need to update something.  Returns nil to indicate
+  # "don't know".
+  def path_up_to_date?(uri, path)
+    begin
+      unescaped_path = CGI.unescape uri.path
+      if ::File.mtime(unescaped_path) <= ::File.mtime(path)
+        return true
+      end
+
+      return false
+    rescue Errno::ENOENT => error
+      raise Fig::FileNotFoundError.new error.message, uri
+    end
+  end
+
+  # Returns whether the file was not downloaded because the file already
+  # exists and is already up-to-date.
+  def download(uri, path)
+    begin
+      unescaped_path = CGI.unescape uri.path
+      FileUtils.cp(unescaped_path, path)
+
+      return true
+    rescue Errno::ENOENT => error
+      raise Fig::FileNotFoundError.new error.message, uri
+    end
+  end
+
+  def upload(local_file, uri)
+    unescaped_path = CGI.unescape uri.path
+    FileUtils.mkdir_p(::File.dirname(unescaped_path))
+    FileUtils.cp(local_file, unescaped_path)
+
+    return
+  end
+end

data/lib/fig/protocol/ftp.rb

@@ -0,0 +1,162 @@
+require 'net/ftp'
+
+require 'fig/file_not_found_error'
+require 'fig/logging'
+require 'fig/network_error'
+require 'fig/protocol'
+require 'fig/protocol/netrc_enabled'
+require 'fig/url'
+
+module Fig; end
+module Fig::Protocol; end
+
+# File transfers via FTP
+class Fig::Protocol::FTP
+  include Fig::Protocol
+  include Fig::Protocol::NetRCEnabled
+
+  def initialize(login)
+    @login = login
+  end
+
+  def download_list(uri)
+    ftp = Net::FTP.new(uri.host)
+    ftp_login(ftp, uri.host)
+    ftp.chdir(uri.path)
+    dirs = ftp.nlst
+    ftp.close
+
+    download_ftp_list(uri, dirs)
+  end
+
+  # Determine whether we need to update something.  Returns nil to indicate
+  # "don't know".
+  def path_up_to_date?(uri, path)
+    begin
+      ftp = Net::FTP.new(uri.host)
+      ftp_login(ftp, uri.host)
+
+      if ftp.mtime(uri.path) <= ::File.mtime(path)
+        return true
+      end
+
+      return false
+    rescue Net::FTPPermError => error
+      Fig::Logging.debug error.message
+      raise Fig::FileNotFoundError.new error.message, uri
+    rescue SocketError => error
+      Fig::Logging.debug error.message
+      raise Fig::FileNotFoundError.new error.message, uri
+    end
+  end
+
+  # Returns whether the file was not downloaded because the file already
+  # exists and is already up-to-date.
+  def download(uri, path)
+    begin
+      ftp = Net::FTP.new(uri.host)
+      ftp_login(ftp, uri.host)
+
+      if ::File.exist?(path) && ftp.mtime(uri.path) <= ::File.mtime(path)
+        Fig::Logging.debug "#{path} is up to date."
+        return false
+      else
+        log_download(uri, path)
+        ftp.getbinaryfile(uri.path, path, 256*1024)
+        return true
+      end
+    rescue Net::FTPPermError => error
+      Fig::Logging.debug error.message
+      raise Fig::FileNotFoundError.new error.message, uri
+    rescue SocketError => error
+      Fig::Logging.debug error.message
+      raise Fig::FileNotFoundError.new error.message, uri
+    rescue Errno::ETIMEDOUT => error
+      Fig::Logging.debug error.message
+      raise Fig::FileNotFoundError.new error.message, uri
+    end
+  end
+
+  def upload(local_file, uri)
+    ftp_uri = Fig::URL.parse(ENV['FIG_REMOTE_URL'])
+    ftp_root_path = ftp_uri.path
+    ftp_root_dirs = ftp_uri.path.split('/')
+    remote_publish_path = uri.path[0, uri.path.rindex('/')]
+    remote_publish_dirs = remote_publish_path.split('/')
+    # Use array subtraction to deduce which project/version folder to upload
+    # to, i.e. [1,2,3] - [2,3,4] = [1]
+    remote_project_dirs = remote_publish_dirs - ftp_root_dirs
+    Net::FTP.open(uri.host) do |ftp|
+      ftp_login(ftp, uri.host)
+      # Assume that the FIG_REMOTE_URL path exists.
+      ftp.chdir(ftp_root_path)
+      remote_project_dirs.each do |dir|
+        # Can't automatically create parent directories, so do it manually.
+        if ftp.nlst().index(dir).nil?
+          ftp.mkdir(dir)
+          ftp.chdir(dir)
+        else
+          ftp.chdir(dir)
+        end
+      end
+      ftp.putbinaryfile(local_file)
+    end
+  end
+
+  private
+
+  def ftp_login(ftp, host)
+    begin
+      if @login
+        load_authentication_for host
+        ftp.login get_username, get_password
+      else
+        ftp.login
+      end
+      ftp.passive = true
+    rescue Net::FTPPermError => error
+      raise Fig::NetworkError.new "Could not log in: #{error.message}"
+    end
+
+    return
+  end
+
+  def download_ftp_list(uri, dirs)
+    # Run a bunch of these in parallel since they're slow as hell
+    num_threads = (ENV['FIG_FTP_THREADS'] || '16').to_i
+    threads = []
+    all_packages = []
+    (0..num_threads-1).each { |num| all_packages[num] = [] }
+    (0..num_threads-1).each do |num|
+      threads << Thread.new do
+        packages = all_packages[num]
+        ftp = Net::FTP.new(uri.host)
+        ftp_login(ftp, uri.host)
+        ftp.chdir(uri.path)
+        pos = num
+        while pos < dirs.length
+          pkg = dirs[pos]
+          begin
+            ftp.nlst(dirs[pos]).each do |ver|
+              packages << pkg + '/' + ver
+            end
+          rescue Net::FTPPermError
+            # Ignore this error because it's indicative of the FTP library
+            # encountering a file or directory that it does not have
+            # permission to open.  Fig needs to be able to have secure
+            # repos/packages and there is no way easy way to deal with the
+            # permissions issues other than consuming these errors.
+            #
+            # Actually, with FTP, you can't tell the difference between a
+            # file not existing and not having permission to access it (which
+            # is probably a good thing).
+          end
+          pos += num_threads
+        end
+        ftp.close
+      end
+    end
+    threads.each { |thread| thread.join }
+    all_packages.flatten.sort
+  end
+end

data/lib/fig/protocol/http.rb

@@ -0,0 +1,61 @@
+require 'net/http'
+require 'uri'
+
+require 'fig/file_not_found_error'
+require 'fig/logging'
+require 'fig/network_error'
+require 'fig/protocol'
+
+module Fig; end
+module Fig::Protocol; end
+
+# File transfers via HTTP.
+class Fig::Protocol::HTTP
+  include Fig::Protocol
+
+  # Returns whether the file was not downloaded because the file already
+  # exists and is already up-to-date.
+  def download(uri, path)
+    log_download(uri, path)
+    ::File.open(path, 'wb') do |file|
+      file.binmode
+
+      begin
+        download_via_http_get(uri, file)
+      rescue SystemCallError => error
+        Fig::Logging.debug error.message
+        raise Fig::FileNotFoundError.new error.message, uri
+      rescue SocketError => error
+        Fig::Logging.debug error.message
+        raise Fig::FileNotFoundError.new error.message, uri
+      end
+    end
+  end
+
+  private
+
+  def download_via_http_get(uri_string, file, redirection_limit = 10)
+    if redirection_limit < 1
+      Fig::Logging.debug 'Too many HTTP redirects.'
+      raise Fig::FileNotFoundError.new 'Too many HTTP redirects.', uri_string
+    end
+
+    response = Net::HTTP.get_response(URI(uri_string))
+
+    case response
+    when Net::HTTPSuccess then
+      file.write(response.body)
+    when Net::HTTPRedirection then
+      location = response['location']
+      Fig::Logging.debug "Redirecting to #{location}."
+      download_via_http_get(location, file, redirection_limit - 1)
+    else
+      Fig::Logging.debug "Download failed: #{response.code} #{response.message}."
+      raise Fig::FileNotFoundError.new(
+        "Download failed: #{response.code} #{response.message}.", uri_string
+      )
+    end
+
+    return
+  end
+end

data/lib/fig/protocol/netrc_enabled.rb

@@ -0,0 +1,42 @@
+require 'highline'
+require 'net/netrc'
+
+require 'fig/user_input_error'
+
+module Fig; end
+module Fig::Protocol; end
+
+# Login information acquisition via .netrc.
+module Fig::Protocol::NetRCEnabled
+  private
+
+  def get_username()
+    @username ||= HighLine.new.ask('Username: ') { |q| q.echo = true }
+    return @username
+  end
+
+  def get_password()
+    @password ||= HighLine.new.ask('Password: ') { |q| q.echo = false }
+    return @password
+  end
+
+  def load_authentication_for(host)
+    return if @username || @password
+
+    @username ||= ENV['FIG_USERNAME']
+    @password ||= ENV['FIG_PASSWORD']
+    return if @username || @password
+
+    begin
+      login_data = Net::Netrc.locate host
+      if login_data
+        @username = login_data.login
+        @password = login_data.password
+      end
+    rescue SecurityError => error
+      raise Fig::UserInputError.new error.message
+    end
+
+    return
+  end
+end

data/lib/fig/protocol/sftp.rb

@@ -0,0 +1,150 @@
+require 'net/sftp'
+
+require 'fig/logging'
+require 'fig/network_error'
+require 'fig/package_descriptor'
+require 'fig/protocol'
+require 'fig/protocol/netrc_enabled'
+
+module Fig; end
+module Fig::Protocol; end
+
+# File transfers via SFTP
+class Fig::Protocol::SFTP
+  include Fig::Protocol
+  include Fig::Protocol::NetRCEnabled
+
+  def download_list(uri)
+    package_versions = []
+
+    sftp_run(uri) do
+      |connection|
+
+      connection.dir.foreach uri.path do
+        |package_directory|
+
+        if package_directory.directory?
+          package_name = package_directory.name
+
+          if package_name =~ Fig::PackageDescriptor::COMPONENT_PATTERN
+            connection.dir.foreach "#{uri.path}/#{package_name}" do
+              |version_directory|
+
+              if version_directory.directory?
+                version_name = version_directory.name
+
+                if version_name =~ Fig::PackageDescriptor::COMPONENT_PATTERN
+                  package_versions << "#{package_name}/#{version_name}"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    return package_versions
+  end
+
+  # Determine whether we need to update something.  Returns nil to indicate
+  # "don't know".
+  def path_up_to_date?(uri, path)
+    sftp_run(uri) do
+      |connection|
+
+      return connection.stat!(uri.path).mtime <= ::File.mtime(path)
+    end
+
+    return nil
+  end
+
+  # Returns whether the file was not downloaded because the file already
+  # exists and is already up-to-date.
+  def download(uri, path)
+    sftp_run(uri) do
+      |connection|
+
+      begin
+        # *sigh* Always call #stat!(), even if the local file does not exist
+        # because #download!() throws Strings and not proper exception objects
+        # when the remote path does not exist.
+        stat = connection.stat!(uri.path)
+
+        if ::File.exist?(path) && stat.mtime <= ::File.mtime(path)
+          Fig::Logging.debug "#{path} is up to date."
+          return false
+        else
+          log_download uri, path
+          connection.download! uri.path, path
+
+          return true
+        end
+      rescue Net::SFTP::StatusException => error
+        if error.code == Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE
+          raise Fig::FileNotFoundError.new(error.message, uri)
+        end
+        raise error
+      end
+    end
+
+    return
+  end
+
+  def upload(local_file, uri)
+    sftp_run(uri) do
+      |connection|
+
+      ensure_directory_exists connection, ::File.dirname(uri.path)
+      connection.upload! local_file, uri.path
+    end
+
+    return
+  end
+
+  private
+
+  def sftp_run(uri, &block)
+    host = uri.host
+
+    load_authentication_for host
+
+    begin
+      options = {:password => get_password}
+      port = uri.port
+      if port
+        options[:port] = port
+      end
+
+      Net::SFTP.start(host, get_username, options, &block)
+    rescue Net::SSH::Exception => error
+      raise Fig::NetworkError.new error.message
+    rescue Net::SFTP::Exception => error
+      raise Fig::NetworkError.new error.message
+    end
+
+    return
+  end
+
+  def ensure_directory_exists(connection, path)
+    begin
+      connection.lstat!(path)
+      return
+    rescue Net::SFTP::StatusException => error
+      if error.code != Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE
+        raise Fig::NetworkError.new(
+          "Could not stat #{path}: #{response.message} (#{response.code})"
+        )
+      end
+    end
+
+    if path == '/'
+      raise Fig::NetworkError.new 'Root path does not exist.'
+    end
+
+    ensure_directory_exists connection, ::File.dirname(path)
+
+    connection.mkdir! path
+
+    return
+  end
+end