fpm 1.8.1 → 1.9.1

data/lib/fpm/package/gem.rb

@@ -44,9 +44,27 @@ class FPM::Package::Gem < FPM::Package
   option "--disable-dependency", "gem_name",
     "The gem name to remove from dependency list",
     :multivalued => true, :attribute_name => :gem_disable_dependencies
+  option "--embed-dependencies", :flag, "Should the gem dependencies " \
+    "be installed?", :default => false
 
   option "--version-bins", :flag, "Append the version to the bins", :default => false
 
+  option "--stagingdir", "STAGINGDIR",
+    "The directory where fpm installs the gem temporarily before conversion. " \
+    "Normally a random subdirectory of workdir."
+
+  # Override parent method
+  def staging_path(path=nil)
+    @gem_staging_path ||= attributes[:gem_stagingdir] || Stud::Temporary.directory("package-#{type}-staging")
+    @staging_path = @gem_staging_path
+
+    if path.nil?
+      return @staging_path
+    else
+      return File.join(@staging_path, path)
+    end
+  end # def staging_path
+
   def input(gem)
     # 'arg'  is the name of the rubygem we should unpack.
     path_to_gem = download_if_necessary(gem, version)
@@ -144,7 +162,7 @@ class FPM::Package::Gem < FPM::Package
     # composing multiple packages, it's best to explicitly include it in the provides list.
     self.provides << "#{self.name} = #{self.version}"
 
-    if !attributes[:no_auto_depends?]
+    if !attributes[:no_auto_depends?] && !attributes[:gem_embed_dependencies?]
       spec.runtime_dependencies.map do |dep|
         # rubygems 1.3.5 doesn't have 'Gem::Dependency#requirement'
         if dep.respond_to?(:requirement)
@@ -181,7 +199,12 @@ class FPM::Package::Gem < FPM::Package
     ::FileUtils.mkdir_p(installdir)
     # TODO(sissel): Allow setting gem tool path
     args = [attributes[:gem_gem], "install", "--quiet", "--no-ri", "--no-rdoc",
-       "--no-user-install", "--install-dir", installdir, "--ignore-dependencies"]
+       "--no-user-install", "--install-dir", installdir]
+
+    if !attributes[:gem_embed_dependencies?]
+      args += ["--ignore-dependencies"]
+    end
+
     if attributes[:gem_env_shebang?]
       args += ["-E"]
     end
@@ -231,6 +254,21 @@ class FPM::Package::Gem < FPM::Package
         FileUtils.mv("#{bin_path}/#{bin}", "#{bin_path}/#{bin}-#{self.version}")
       end
     end
+
+    if attributes[:source_date_epoch_from_changelog?]
+      detect_source_date_from_changelog(installdir)
+    end
+
+    # Remove generated Makefile and gem_make.out files, if any; they
+    # are not needed, and may contain generated paths that cause
+    # different output on successive runs.
+    Find.find(installdir) do |path|
+      if path =~ /.*(gem_make.out|Makefile|mkmf.log)$/
+        logger.info("Removing no longer needed file %s to reduce nondeterminism" % path)
+        File.unlink(path)
+      end
+    end
+
   end # def install_to_staging
 
   # Sanitize package name.
@@ -239,5 +277,75 @@ class FPM::Package::Gem < FPM::Package
   def fix_name(name)
     return [attributes[:gem_package_name_prefix], name].join("-")
   end # def fix_name
+
+  # Regular expression to accept a gem changelog line, and store date & version, if any, in named capture groups.
+  # Supports formats suggested by http://keepachangelog.com and https://github.com/tech-angels/vandamme
+  # as well as other similar formats that actually occur in the wild.
+  # Build it in pieces for readability, and allow version and date in either order.
+  # Whenever you change this, add a row to the test case in spec/fpm/package/gem_spec.rb.
+  # Don't even try to handle dates that lack four-digit years.
+  # Building blocks:
+  P_RE_LEADIN    = '^[#=]{0,3}\s?'
+  P_RE_VERSION_  = '[\w\.-]+\.[\w\.-]+[a-zA-Z0-9]'
+  P_RE_SEPARATOR = '\s[-=/(]?\s?'
+  P_RE_DATE1     = '\d{4}-\d{2}-\d{2}'
+  P_RE_DATE2     = '\w+ \d{1,2}(?:st|nd|rd|th)?,\s\d{4}'
+  P_RE_DATE3     = '\w+\s+\w+\s+\d{1,2},\s\d{4}'
+  P_RE_DATE      = "(?<date>#{P_RE_DATE1}|#{P_RE_DATE2}|#{P_RE_DATE3})"
+  P_RE_URL       = '\(https?:[-\w/.%]*\)'    # In parens, per markdown
+  P_RE_GTMAGIC   = '\[\]'                    # github magic version diff, per chandler
+  P_RE_VERSION   = "\\[?(?:Version |v)?(?<version>#{P_RE_VERSION_})\\]?(?:#{P_RE_URL}|#{P_RE_GTMAGIC})?"
+  # The final RE's:
+  P_RE_VERSION_DATE = "#{P_RE_LEADIN}#{P_RE_VERSION}#{P_RE_SEPARATOR}#{P_RE_DATE}"
+  P_RE_DATE_VERSION = "#{P_RE_LEADIN}#{P_RE_DATE}#{P_RE_SEPARATOR}#{P_RE_VERSION}"
+
+  # Detect release date, if found, store in attributes[:source_date_epoch]
+  def detect_source_date_from_changelog(installdir)
+    name = self.name.sub("rubygem-", "") + "-" + self.version
+    changelog = nil
+    datestr = nil
+    r1 = Regexp.new(P_RE_VERSION_DATE)
+    r2 = Regexp.new(P_RE_DATE_VERSION)
+
+    # Changelog doesn't have a standard name, so check all common variations
+    # Sort this list using LANG=C, i.e. caps first
+    [
+      "CHANGELIST",
+      "CHANGELOG", "CHANGELOG.asciidoc", "CHANGELOG.md", "CHANGELOG.rdoc", "CHANGELOG.rst", "CHANGELOG.txt",
+      "CHANGES",   "CHANGES.md",   "CHANGES.txt",
+      "ChangeLog", "ChangeLog.md", "ChangeLog.txt",
+      "Changelog", "Changelog.md", "Changelog.txt",
+      "changelog", "changelog.md", "changelog.txt",
+    ].each do |changelogname|
+      path = File.join(installdir, "gems", name, changelogname)
+      if File.exist?(path)
+        changelog = path
+        File.open path do |file|
+          file.each_line do |line|
+            if line =~ /#{self.version}/
+              [r1, r2].each do |r|
+                if r.match(line)
+                  datestr = $~[:date]
+                  break
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+    if datestr
+      date = Date.parse(datestr)
+      sec = date.strftime("%s")
+      attributes[:source_date_epoch] = sec
+      logger.debug("Gem %s has changelog date %s, setting source_date_epoch to %s" % [name, datestr, sec])
+    elsif changelog
+      logger.debug("Gem %s changelog %s did not have recognizable date for release %s" % [name, changelog, self.version])
+    else
+      logger.debug("Gem %s did not have changelog with recognized name" % [name])
+      # FIXME: check rubygems.org?
+    end
+  end # detect_source_date_from_changelog
+
   public(:input, :output)
 end # class FPM::Package::Gem

data/lib/fpm/package/pleaserun.rb

@@ -10,7 +10,7 @@ require "pleaserun/cli"
 # This does not currently support 'output'
 class FPM::Package::PleaseRun < FPM::Package
   # TODO(sissel): Implement flags.
-  
+
   require "pleaserun/platform/systemd"
   require "pleaserun/platform/upstart"
   require "pleaserun/platform/launchd"
@@ -28,6 +28,10 @@ class FPM::Package::PleaseRun < FPM::Package
       ::PleaseRun::Platform::Launchd.new("10.9"), # OS X
       ::PleaseRun::Platform::SYSV.new("lsb-3.1") # Ancient stuff
     ]
+    pleaserun_attributes = [ "chdir", "user", "group", "umask", "chroot", "nice", "limit_coredump",
+                             "limit_cputime", "limit_data", "limit_file_size", "limit_locked_memory",
+                             "limit_open_files", "limit_user_processes", "limit_physical_memory", "limit_stack_size",
+                             "log_directory", "log_file_stderr", "log_file_stdout"]
 
     attributes[:pleaserun_name] ||= File.basename(command.first)
     attributes[:prefix] ||= "/usr/share/pleaserun/#{attributes[:pleaserun_name]}"
@@ -37,12 +41,18 @@ class FPM::Package::PleaseRun < FPM::Package
       platform.program = command.first
       platform.name = attributes[:pleaserun_name]
       platform.args = command[1..-1]
-      platform.chdir = attributes[:pleaserun_chdir] if attributes[:pleaserun_chdir]
       platform.description = if attributes[:description_given?]
         attributes[:description]
       else
         platform.name
       end
+      pleaserun_attributes.each do |attribute_name|
+        attribute = "pleaserun_#{attribute_name}".to_sym
+        if attributes.has_key?(attribute) and not attributes[attribute].nil?
+          platform.send("#{attribute_name}=", attributes[attribute])
+        end
+      end
+
       base = staging_path(File.join(attributes[:prefix], "#{platform.platform}/#{platform.target_version || "default"}"))
       target = File.join(base, "files")
       actions_script = File.join(base, "install_actions.sh")

data/lib/fpm/package/python.rb

@@ -73,6 +73,10 @@ class FPM::Package::Python < FPM::Package
     "The python package name to remove from dependency list",
     :multivalued => true, :attribute_name => :python_disable_dependency,
     :default => []
+  option "--setup-py-arguments", "setup_py_argument",
+    "Arbitrary argument(s) to be passed to setup.py",
+    :multivalued => true, :attribute_name => :python_setup_py_arguments,
+    :default => []
 
   private
 
@@ -310,6 +314,13 @@ class FPM::Package::Python < FPM::Package
         flags += [ "build_scripts", "--executable", attributes[:python_scripts_executable] ]
       end
 
+      if !attributes[:python_setup_py_arguments].nil? and !attributes[:python_setup_py_arguments].empty?
+        # Add optional setup.py arguments
+        attributes[:python_setup_py_arguments].each do |a|
+          flags += [ a ]
+        end
+      end
+
       safesystem(attributes[:python_bin], "setup.py", "install", *flags)
     end
   end # def install_to_staging

data/lib/fpm/package/rpm.rb

@@ -472,6 +472,15 @@ class FPM::Package::RPM < FPM::Package
       self.directories = alldirs
     end
 
+    # include external config files
+    (attributes[:config_files] or []).each do |conf|
+      path = conf
+      dest_conf = File.join(staging_path, path)
+      FileUtils.mkdir_p(File.dirname(dest_conf))
+      FileUtils.cp_r conf, dest_conf
+      File.chmod(0755, dest_conf)
+    end
+
     # scan all conf file paths for files and add them
     allconfigs = []
     self.config_files.each do |path|

data/lib/fpm/package/tar.rb

@@ -50,15 +50,7 @@ class FPM::Package::Tar < FPM::Package
     output_check(output_path)
 
     # Write the scripts, too.
-    scripts_path = File.join(staging_path, ".scripts")
-    ::Dir.mkdir(scripts_path)
-    [:before_install, :after_install, :before_remove, :after_remove].each do |name|
-      next unless script?(name)
-      out = File.join(scripts_path, name.to_s)
-      logger.debug("Writing script", :source => name, :target => out)
-      File.write(out, script(name))
-      File.chmod(0755, out)
-    end
+    write_scripts
 
     # Unpack the tarball to the staging path
     args = ["-cf", output_path, "-C", staging_path]

data/lib/fpm/package/virtualenv.rb

@@ -15,8 +15,11 @@ class FPM::Package::Virtualenv < FPM::Package
   option "--package-name-prefix", "PREFIX", "Name to prefix the package " \
   "name with.", :default => "virtualenv"
 
-  option "--install-location", "DIRECTORY", "Location to which to " \
-  "install the virtualenv by default.", :default => "/usr/share/python" do |path|
+  option "--install-location", "DIRECTORY", "DEPRECATED: Use --prefix instead." \
+    "  Location to which to install the virtualenv by default.",
+    :default => "/usr/share/python" do |path|
+    logger.warn("Using deprecated flag: --install-location. Please use " \
+                  "--prefix instead.")
     File.expand_path(path)
   end
 
@@ -37,6 +40,12 @@ class FPM::Package::Virtualenv < FPM::Package
   option "--system-site-packages", :flag, "Give the virtual environment access to the "\
   "global site-packages"
 
+  option "--find-links", "PIP_FIND_LINKS", "If a url or path to an html file, then parse for "\
+    "links to archives. If a local path or file:// url that's a directory, then look "\
+    "for archives in the directory listing.",
+    :multivalued => true, :attribute_name => :virtualenv_find_links_urls,
+    :default => nil
+
   private
 
   # Input a package.
@@ -76,9 +85,14 @@ class FPM::Package::Virtualenv < FPM::Package
                    self.name].join("-")
     end
 
+    # prefix wins over previous virtual_install_location behaviour
     virtualenv_folder =
-      File.join(installdir,
-                virtualenv_name)
+      if self.attributes[:prefix]
+        self.attributes[:prefix]
+      else
+        File.join(installdir,
+                  virtualenv_name)
+      end
 
     virtualenv_build_folder = build_path(virtualenv_folder)
 
@@ -107,6 +121,13 @@ class FPM::Package::Virtualenv < FPM::Package
       end
     end
 
+    find_links_url_args = []
+    if attributes[:virtualenv_find_links_urls]
+      attributes[:virtualenv_find_links_urls].each do |links_url|
+        find_links_url_args << "--find-links" << links_url
+      end
+    end
+
     target_args = []
     if is_requirements_file
       target_args << "-r" << package
@@ -114,7 +135,7 @@ class FPM::Package::Virtualenv < FPM::Package
       target_args << package
     end
 
-    pip_args = [python_exe, pip_exe, "install", "-i", attributes[:virtualenv_pypi]] << extra_index_url_args << target_args
+    pip_args = [python_exe, pip_exe, "install", "-i", attributes[:virtualenv_pypi]] << extra_index_url_args << find_links_url_args << target_args
     safesystem(*pip_args.flatten)
 
     if attributes[:virtualenv_setup_install?]
@@ -155,7 +176,8 @@ class FPM::Package::Virtualenv < FPM::Package
     # use dir to set stuff up properly, mainly so I don't have to reimplement
     # the chdir/prefix stuff special for tar.
     dir = convert(FPM::Package::Dir)
-
+    # don't double prefix the files
+    dir.attributes[:prefix] = nil
     if attributes[:chdir]
       dir.attributes[:chdir] = File.join(build_path, attributes[:chdir])
     else

data/lib/fpm/package/zip.rb

@@ -36,28 +36,12 @@ class FPM::Package::Zip < FPM::Package
     dir.cleanup_build
   end # def input
 
-  # Output a tarball.
-  #
-  # If the output path ends predictably (like in .tar.gz) it will try to obey
-  # the compression type.
+  # Output a zipfile.
   def output(output_path)
     output_check(output_path)
-
-    files = Find.find(staging_path).to_a
-    safesystem("zip", output_path, *files)
-  end # def output
-
-  # Generate the proper tar flags based on the path name.
-  def tar_compression_flag(path)
-    case path
-      when /\.tar\.bz2$/
-        return "-j"
-      when /\.tar\.gz$|\.tgz$/
-        return "-z"
-      when /\.tar\.xz$/
-        return "-J"
-      else
-        return nil
+    realpath = Pathname.new(output_path).realdirpath.to_s
+    ::Dir.chdir(staging_path) do
+      safesystem("zip", "-9r", realpath, ".")
     end
-  end # def tar_compression_flag
+  end # def output
 end # class FPM::Package::Tar

data/lib/fpm/util.rb

@@ -1,6 +1,7 @@
 require "fpm/namespace"
 require "childprocess"
 require "ffi"
+require "fileutils"
 
 # Some utility functions
 module FPM::Util
@@ -190,9 +191,14 @@ module FPM::Util
     if args.size == 1
       args = [ default_shell, "-c", args[0] ]
     end
-    program = args[0]
 
-    exit_code = execmd(args)
+    if args[0].kind_of?(Hash)
+      env = args.shift()
+      exit_code = execmd(env, args)
+    else
+      exit_code = execmd(args)
+    end
+    program = args[0]
     success = (exit_code == 0)
 
     if !success
@@ -226,26 +232,90 @@ module FPM::Util
     return stdout_r_str
   end # def safesystemout
 
+  # Get an array containing the recommended 'ar' command for this platform
+  # and the recommended options to quickly create/append to an archive
+  # without timestamps or uids (if possible).
+  def ar_cmd
+    return @@ar_cmd if defined? @@ar_cmd
+
+    @@ar_cmd_deterministic = false
+
+    # FIXME: don't assume current directory writeable
+    FileUtils.touch(["fpm-dummy.tmp"])
+    ["ar", "gar"].each do |ar|
+      ["-qc", "-qcD"].each do |ar_create_opts|
+        FileUtils.rm_f(["fpm-dummy.ar.tmp"])
+        # Return this combination if it creates archives without uids or timestamps.
+        # Exitstatus will be nonzero if the archive can't be created,
+        # or its table of contents doesn't match the regular expression.
+        # Be extra-careful about locale and timezone when matching output.
+        system("#{ar} #{ar_create_opts} fpm-dummy.ar.tmp fpm-dummy.tmp 2>/dev/null && env TZ=UTC LANG=C LC_TIME=C #{ar} -tv fpm-dummy.ar.tmp | grep '0/0.*1970' > /dev/null 2>&1")
+        if $?.exitstatus == 0
+           @@ar_cmd = [ar, ar_create_opts]
+           @@ar_cmd_deterministic = true
+           return @@ar_cmd
+        end
+      end
+    end
+    # If no combination of ar and options omits timestamps, fall back to default.
+    @@ar_cmd = ["ar", "-qc"]
+    return @@ar_cmd
+  ensure
+    # Clean up
+    FileUtils.rm_f(["fpm-dummy.ar.tmp", "fpm-dummy.tmp"])
+  end # def ar_cmd
+
+  # Return whether the command returned by ar_cmd can create deterministic archives
+  def ar_cmd_deterministic?
+    ar_cmd if not defined? @@ar_cmd_deterministic
+    return @@ar_cmd_deterministic
+  end
+
   # Get the recommended 'tar' command for this platform.
   def tar_cmd
-    # Rely on gnu tar for solaris and OSX.
-    case %x{uname -s}.chomp
-    when "SunOS"
-      return "gtar"
-    when "Darwin"
-      # Try running gnutar, it was renamed(??) in homebrew to 'gtar' at some point, I guess? I don't know.
-      ["gnutar", "gtar"].each do |tar|
-        system("#{tar} > /dev/null 2> /dev/null")
-        return tar unless $?.exitstatus == 127
+    return @@tar_cmd if defined? @@tar_cmd
+
+    # FIXME: don't assume current directory writeable
+    FileUtils.touch(["fpm-dummy.tmp"])
+
+    # Prefer tar that supports more of the features we want, stop if we find tar of our dreams
+    best="tar"
+    bestscore=0
+    @@tar_cmd_deterministic = false
+    # GNU Tar, if not the default, is usually on the path as gtar, but
+    # Mac OS X 10.8 and earlier shipped it as /usr/bin/gnutar
+    ["tar", "gtar", "gnutar"].each do |tar|
+      opts=[]
+      score=0
+      ["--sort=name", "--mtime=@0"].each do |opt|
+        system("#{tar} #{opt} -cf fpm-dummy.tar.tmp fpm-dummy.tmp > /dev/null 2>&1")
+        if $?.exitstatus == 0
+          opts << opt
+          score += 1
+        end
+      end
+      if score > bestscore
+        best=tar
+        bestscore=score
+        if score == 2
+          @@tar_cmd_deterministic = true
+          break
+        end
       end
-    when "FreeBSD"
-      # use gnutar instead
-      return "gtar"
-    else
-      return "tar"
     end
+    @@tar_cmd = best
+    return @@tar_cmd
+  ensure
+    # Clean up
+    FileUtils.rm_f(["fpm-dummy.tar.tmp", "fpm-dummy.tmp"])
   end # def tar_cmd
 
+  # Return whether the command returned by tar_cmd can create deterministic archives
+  def tar_cmd_supports_sort_names_and_set_mtime?
+    tar_cmd if not defined? @@tar_cmd_deterministic
+    return @@tar_cmd_deterministic
+  end
+
   # wrapper around mknod ffi calls
   def mknod_w(path, mode, dev)
     rc = -1
@@ -298,8 +368,8 @@ module FPM::Util
       if known_entry
         FileUtils.ln(known_entry, dst)
       else
-        FileUtils.copy_entry(src, dst, preserve=preserve,
-                             remove_destination=remove_destination)
+        FileUtils.copy_entry(src, dst, preserve, false,
+                             remove_destination)
         copied_entries[[st.dev, st.ino]] = dst
       end
     end # else...