fpm 1.8.1 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2053d568639fe148fe3d955eb70e27c41bfb9d6b
-  data.tar.gz: 1a4d6ac0ddd23b851d74e8cdbe5c57d3c5378ef7
+  metadata.gz: b9fb4eab81222c77d6790e9e06c8348744a020e7
+  data.tar.gz: a29942bc9fcc9dcbe6bf0f4d1fbfa431d202eb2b
 SHA512:
-  metadata.gz: dbf0ccf583f7833316f65e28494ab6fc761ec937518af0f4bdc2812591f2a7205b60490e997b38e98a8ff44d5c5d31b95b3834f538ae90c9e783e1f6e1da462f
-  data.tar.gz: 1d9316d61d328cd8978b2bc5bc3e50e7f08bd6b30e2fe29c82d3096aea7731d963920555d0db6f37407b98ed5798540881fecd80b400d13505e8d10d6a43df77
+  metadata.gz: c97a69b63e17f684c7f481bda43e479d34a030e253b0a7294de356eaf1a35dc83b230deec5cf77270ff17756b98b8336016588be3ea41317ec354bd56649c1e8
+  data.tar.gz: a3708d970c1d1b6d4d384893f7c596b5fad2ea6d4eb01e0baa42bec1de3bbb85286f3d5acab9823928cf00bb04f59c98247d307269f04fe01bae824f387e59cf

data/CHANGELOG.rst

@@ -1,6 +1,45 @@
 Release Notes and Change Log
 ============================
 
+1.9.1 (July 28, 2017) happy sysadmin day!
+^^^^^^^^^^^^^^^^^^^^^
+
+* Documentation improvements: `#1291`_; Pablo Castellano. `#1321`_; ge-fa. `#1309`_; jesusbagpuss. `#1349`_; Perry Stole. `#1352`_, Jordan Sissel. `#1384`_; Justin Kolberg.
+* Testing improvements: `#1320`_; Rob Young. `#1266`_; Ryan Parman. `#1374`_; Thiago Figueiró.
+* Fix bug so fpm can now copy symlinks correctly (`#1348`_; ServiusHack)
+* apk: Improve performance (`#1358`_; Jan Delgado)
+* cpan: Fix crash when CPAN query returns a version value that was a number and fpm was expecting a string. (`#1344`_, `#1343`_; liger1978)
+* cpan: Fix MetaCPAN searches to use v1 of MetaCPAN's API. The v0 API is no longer provided by MetaCPAN. (`#1341`_, `#1339`; Bob Bell)
+* cpan: Have perl modules implicitly "provide" (`--provides`) capabilities. (`#1340`_; Bob Bell. `#1345`_; liger1978)
+* cpan: Now transforms perl version values like "5.008001" to "5.8.1" (`#1342`_; Bob Bell)
+* cpan: Use `>=` ("this version or newer") for package dependencies instead of `=` ("exactly this version"). (`#1338`_; Bob Bell)
+* deb: Add `--deb-after-purge` flag for running a script after `apt-get purge` is run. (Alexander Weidinger)
+* deb: fix bug when using `--deb-upstart` would use the wrong file name (`#1325`_, `#1287`_; vbakayev)
+* deb: New flags `--deb-interest-noawait` and `--deb-activate-nowait`. (`#1225`_, `#1359`_; Philippe Poilbarbe)
+* dir: Remove a debug statement that would put fpm into a debug prompt (`#1293`_, `#1259`_; Joseph Anthony Pasquale Holsten)
+* dir: When using `path mapping`_ (`a=b` syntax), and `a` is a symlink, use the path `b` as the symlink, not `b/a` (`#1253`_, Nemanja Boric)
+* gem: Can now make reproducible_builds_ when building a deb (`-s gem -t deb`). See the `Deterministic output`_ docs.
+* gem: Add `--gem-embed-dependencies` flag to include in the output package all dependent gems of the target. For example, `fpm -s gem -t rpm --gem-embed-dependencies rails` will create a single `rails` rpm that includes active_support, active_record, etc.
+* pleaserun: Add more flags (`--pleaserun-chdir`, `--pleaserun-user`, etc) to allow more customization of pleaserun services. (`#1311`_; Paulo Sousa)
+* python: Add `--python-setup-py-arguments` flag for passing arbitrary flags to `python setup.py install` (`#1120`_, `#1376`_; Ward Vandewege, Joseph Anthony Pasquale Holsten)
+* rpm: --config-files can now copy files from outside of the package source. This means you can do things like `fpm -s gem -t rpm --config-files etc/my/config` and have `etc/my/config` come from the local filesystem. (`#860`_, `#1379`_; jakerobinson, Joseph Anthony Pasquale Holsten)
+* tar: Only create `.scripts` directory if there are scripts to include (`#1123`_, `#1374`_; Thiago Figueiró)
+* virtualenv: Add `--virtualenv-find-links` flag which appends `--find-links` to the `pip install` command.
+* virtualenv: documentation improvements (Nick Griffiths)
+* virtualenv: Make `--prefix` useful and deprecate `--virtualenv-install-location` (`#1262`_; Nick Griffiths)
+* zip: fix bug in output where the temporary directory would be included in the file listing (`#1313`_, `#1314`_; Bob Vincent)
+* Other: Remove unused archive-tar-minitar as a dependency of fpm (`#1355`_; Diego Martins)
+* Other: Add stud as a runtime dependency (`#1354`_; Elan Ruusamäe)
+
+.. _reproducible_builds:: https://reproducible-builds.org/
+.. _path mapping:: http://fpm.readthedocs.io/en/latest/source/dir.html#path-mapping
+.. _Deterministic output:: http://fpm.readthedocs.io/en/latest/source/gem.html
+
+1.9.0 (July 28, 2017)
+^^^^^^^^^^^^^^^^^^^^^
+
+Yanked offline. I forgot some dependency changes. Hi.
+
 1.8.1 (February 7, 2017)
 ^^^^^^^^^^^^^^^^^^^^^^^^
 * Pin archive-tar-minitar library to version 0.5.2 to work around a problem breaking `gem install fpm`

data/lib/fpm/command.rb

@@ -233,6 +233,20 @@ class FPM::Command < Clamp::Command
     "copying, downloading, etc. Roughly any scratch space fpm needs to build " \
     "your package.", :default => Dir.tmpdir
 
+  option "--source-date-epoch-from-changelog", :flag,
+    "Use release date from changelog as timestamp on generated files to reduce nondeterminism. " \
+    "Experimental; only implemented for gem so far. ",
+    :default => false
+
+  option "--source-date-epoch-default", "SOURCE_DATE_EPOCH_DEFAULT",
+    "If no release date otherwise specified, use this value as timestamp on generated files to reduce nondeterminism. " \
+    "Reproducible build environments such as dpkg-dev and rpmbuild set this via envionment variable SOURCE_DATE_EPOCH " \
+    "variable to the integer unix timestamp to use in generated archives, " \
+    "and expect tools like fpm to use it as a hint to avoid nondeterministic output. " \
+    "This is a Unix timestamp, i.e. number of seconds since 1 Jan 1970 UTC. " \
+    "See https://reproducible-builds.org/specs/source-date-epoch ",
+    :environment_variable => "SOURCE_DATE_EPOCH"
+
   parameter "[ARGS] ...",
     "Inputs to the source package type. For the 'dir' type, this is the files" \
     " and directories you want to include in the package. For others, like " \

data/lib/fpm/package.rb

@@ -175,8 +175,8 @@ class FPM::Package
     @directories = []
     @attrs = {}
 
-    staging_path
     build_path
+    # Dont' initialize staging_path just yet, do it lazily so subclass can get a word in.
   end # def initialize
 
   # Get the 'type' for this instance.
@@ -492,6 +492,22 @@ class FPM::Package
     return scripts.include?(name)
   end # def script?
 
+  # write all scripts to .scripts (tar and dir)
+  def write_scripts
+    scripts_path = File.join(staging_path, ".scripts")
+    target_scripts = [:before_install, :after_install, :before_remove, :after_remove]
+    if target_scripts.any? {|name| script?(name)}
+      ::Dir.mkdir(scripts_path)
+      target_scripts.each do |name|
+        next unless script?(name)
+        out = File.join(scripts_path, name.to_s)
+        logger.debug('Writing script', :source => name, :target => out)
+        File.write(out, script(name))
+        File.chmod(0755, out)
+      end
+    end
+  end
+  
   # Get the contents of the script by a given name.
   #
   # If template_scripts? is set in attributes (often by the --template-scripts

data/lib/fpm/package/apk.rb

@@ -238,7 +238,7 @@ class FPM::Package::APK< FPM::Package
         record_length = determine_record_length(record_length)
 
         until(data.length == record_length)
-          data += file.read(TAR_CHUNK_SIZE)
+          data << file.read(TAR_CHUNK_SIZE)
         end
 
         # Clear ownership fields

data/lib/fpm/package/cpan.rb

@@ -99,6 +99,12 @@ class FPM::Package::CPAN < FPM::Package
       self.name = fix_name(metadata["name"])
     end
 
+    unless metadata["module"].nil?
+      metadata["module"].each do |m|
+        self.provides << cap_name(m["name"]) + " = #{self.version}"
+      end
+    end
+
     # author is not always set or it may be a string instead of an array
     self.vendor = case metadata["author"]
       when String; metadata["author"]
@@ -150,16 +156,16 @@ class FPM::Package::CPAN < FPM::Package
        found_dependencies.each do |dep_name, version|
           # Special case for representing perl core as a version.
           if dep_name == "perl"
+            m = version.match(/^(\d)\.(\d{3})(\d{3})$/)
+            if m
+               version = m[1] + '.' + m[2].sub(/^0*/, '') + '.' + m[3].sub(/^0*/, '')
+            end
             self.dependencies << "#{dep_name} >= #{version}"
             next
           end
           dep = search(dep_name)
 
-          if dep.include?("distribution")
-            name = fix_name(dep["distribution"])
-          else
-            name = fix_name(dep_name)
-          end
+          name = cap_name(dep_name)
 
           if version.to_s == "0"
             # Assume 'Foo = 0' means any version?
@@ -167,12 +173,14 @@ class FPM::Package::CPAN < FPM::Package
           else
             # The 'version' string can be something complex like:
             #   ">= 0, != 1.0, != 1.2"
+            # If it is not specified explicitly, require the given
+            # version or newer, as that is all CPAN itself enforces
             if version.is_a?(String)
               version.split(/\s*,\s*/).each do |v|
                 if v =~ /\s*[><=]/
                   self.dependencies << "#{name} #{v}"
                 else
-                  self.dependencies << "#{name} = #{v}"
+                  self.dependencies << "#{name} >= #{v}"
                 end
               end
             else
@@ -259,7 +267,7 @@ class FPM::Package::CPAN < FPM::Package
                       :path => path.gsub(staging_path, ""))
         File.unlink(path)
       end
-      
+
       # Remove useless .packlist files and their empty parent folders
       # https://github.com/jordansissel/fpm/issues/1179
       ::Dir.glob(File.join(staging_path, glob_prefix, "**/.packlist")).each do |path|
@@ -315,13 +323,10 @@ class FPM::Package::CPAN < FPM::Package
     else
       self.version = "#{cpan_version}"
     end
-    
-    # remove 'v' prefix from version if it is there
-    self.version.sub!(/^v/, '')
-    
+
     # Search metacpan to get download URL for this version of the module
-    metacpan_search_url = "http://api.metacpan.org/v0/release/_search"
-    metacpan_search_query = '{"query":{"match_all":{}},"filter":{"term":{"release.name":"' + "#{distribution}-#{self.version}" + '"}}}'
+    metacpan_search_url = "https://fastapi.metacpan.org/v1/release/_search"
+    metacpan_search_query = '{"fields":["download_url"],"filter":{"term":{"name":"' + "#{distribution}-#{self.version}" + '"}}}'
     begin
       search_response = httppost(metacpan_search_url,metacpan_search_query)
     rescue Net::HTTPServerException => e
@@ -333,7 +338,7 @@ class FPM::Package::CPAN < FPM::Package
     data = search_response.body
     release_metadata = JSON.parse(data)
 
-    download_url = release_metadata['hits']['hits'][0]['_source']['download_url']
+    download_url = release_metadata['hits']['hits'][0]['fields']['download_url']
     download_path = URI.parse(download_url).path
     tarball = File.basename(download_path)
 
@@ -361,7 +366,7 @@ class FPM::Package::CPAN < FPM::Package
 
   def search(package)
     logger.info("Asking metacpan about a module", :module => package)
-    metacpan_url = "http://api.metacpan.org/v0/module/" + package
+    metacpan_url = "https://fastapi.metacpan.org/v1/module/" + package
     begin
       response = httpfetch(metacpan_url)
     rescue Net::HTTPServerException => e
@@ -379,6 +384,10 @@ class FPM::Package::CPAN < FPM::Package
     return metadata
   end # def metadata
 
+  def cap_name(name)
+    return "perl(" + name.gsub("-", "::") + ")"
+  end # def cap_name
+
   def fix_name(name)
     case name
       when "perl"; return "perl"
@@ -394,6 +403,7 @@ class FPM::Package::CPAN < FPM::Package
     else
       http = Net::HTTP.new(uri.host, uri.port)
     end
+    http.use_ssl = uri.scheme == 'https'
     response = http.request(Net::HTTP::Get.new(uri.request_uri))
     case response
       when Net::HTTPSuccess; return response
@@ -410,6 +420,7 @@ class FPM::Package::CPAN < FPM::Package
     else
       http = Net::HTTP.new(uri.host, uri.port)
     end
+    http.use_ssl = uri.scheme == 'https'
     response = http.post(uri.request_uri, body)
     case response
       when Net::HTTPSuccess; return response

data/lib/fpm/package/deb.rb

@@ -18,6 +18,7 @@ class FPM::Package::Deb < FPM::Package
     :after_install      => "postinst",
     :before_remove      => "prerm",
     :after_remove       => "postrm",
+    :after_purge        => "postrm",
   } unless defined?(SCRIPT_MAP)
 
   # The list of supported compression types. Default is gz (gzip)
@@ -121,6 +122,18 @@ class FPM::Package::Deb < FPM::Package
     next @activated_triggers
   end
 
+  option "--interest-noawait", "EVENT", "Package is interested in EVENT trigger without awaiting" do |event|
+    @interested_noawait_triggers ||= []
+    @interested_noawait_triggers << event
+    next @interested_noawait_triggers
+  end
+
+  option "--activate-noawait", "EVENT", "Package activates EVENT trigger" do |event|
+    @activated_noawait_triggers ||= []
+    @activated_noawait_triggers << event
+    next @activated_noawait_triggers
+  end
+
   option "--field", "'FIELD: VALUE'", "Add custom field to the control file" do |fv|
     @custom_fields ||= {}
     field, value = fv.split(/: */, 2)
@@ -164,6 +177,12 @@ class FPM::Package::Deb < FPM::Package
 
   option "--systemd-restart-after-upgrade", :flag , "Restart service after upgrade", :default => true
 
+  option "--after-purge", "FILE",
+    "A script to be run after package removal to purge remaining (config) files " \
+    "(a.k.a. postrm purge within apt-get purge)" do |val|
+    File.expand_path(val) # Get the full path to the script
+  end # --after-purge
+
   def initialize(*args)
     super(*args)
     attributes[:deb_priority] = "extra"
@@ -243,7 +262,7 @@ class FPM::Package::Deb < FPM::Package
     build_path("control").tap do |path|
       FileUtils.mkdir(path) if !File.directory?(path)
       # Unpack the control tarball
-      safesystem("ar p #{package} control.tar.gz | tar -zxf - -C #{path}")
+      safesystem(ar_cmd[0] + " p #{package} control.tar.gz | tar -zxf - -C #{path}")
 
       control = File.read(File.join(path, "control"))
 
@@ -340,7 +359,7 @@ class FPM::Package::Deb < FPM::Package
 
   def extract_files(package)
     # Find out the compression type
-    compression = `ar t #{package}`.split("\n").grep(/data.tar/).first.split(".").last
+    compression = `#{ar_cmd[0]} t #{package}`.split("\n").grep(/data.tar/).first.split(".").last
     case compression
       when "gz"
         datatar = "data.tar.gz"
@@ -358,7 +377,7 @@ class FPM::Package::Deb < FPM::Package
     end
 
     # unpack the data.tar.{gz,bz2,xz} from the deb package into staging_path
-    safesystem("ar p #{package} #{datatar} " \
+    safesystem(ar_cmd[0] + " p #{package} #{datatar} " \
                "| tar #{compression} -xf - -C #{staging_path}")
   end # def extract_files
 
@@ -387,6 +406,22 @@ class FPM::Package::Deb < FPM::Package
       end
     end
 
+    if attributes[:source_date_epoch].nil? and not attributes[:source_date_epoch_default].nil?
+      attributes[:source_date_epoch] = attributes[:source_date_epoch_default]
+    end
+    if attributes[:source_date_epoch] == "0"
+      logger.error("Alas, ruby's Zlib::GzipWriter does not support setting an mtime of zero.  Aborting.")
+      raise "#{name}: source_date_epoch of 0 not supported."
+    end
+    if not attributes[:source_date_epoch].nil? and not ar_cmd_deterministic?
+      logger.error("Alas, could not find an ar that can handle -D option. Try installing recent gnu binutils. Aborting.")
+      raise "#{name}: ar is insufficient to support source_date_epoch."
+    end
+    if not attributes[:source_date_epoch].nil? and not tar_cmd_supports_sort_names_and_set_mtime?
+      logger.error("Alas, could not find a tar that can set mtime and sort.  Try installing recent gnu tar. Aborting.")
+      raise "#{name}: tar is insufficient to support source_date_epoch."
+    end
+
     attributes.fetch(:deb_systemd_list, []).each do |systemd|
       name = File.basename(systemd, ".service")
       dest_systemd = staging_path("lib/systemd/system/#{name}.service")
@@ -412,24 +447,9 @@ class FPM::Package::Deb < FPM::Package
       if script?(:after_remove)
         scripts[:after_remove] = template("deb/postrm_upgrade.sh.erb").result(binding)
       end
-    end
-
-    write_control_tarball
-
-    # Tar up the staging_path into data.tar.{compression type}
-    case self.attributes[:deb_compression]
-      when "gz", nil
-        datatar = build_path("data.tar.gz")
-        compression = "-z"
-      when "bzip2"
-        datatar = build_path("data.tar.bz2")
-        compression = "-j"
-      when "xz"
-        datatar = build_path("data.tar.xz")
-        compression = "-J"
-      else
-        raise FPM::InvalidPackageConfiguration,
-          "Unknown compression type '#{self.attributes[:deb_compression]}'"
+      if script?(:after_purge)
+        scripts[:after_purge] = template("deb/postrm_upgrade.sh.erb").result(binding)
+      end
     end
 
     # There are two changelogs that may appear:
@@ -442,6 +462,9 @@ class FPM::Package::Deb < FPM::Package
     mkdir_p(File.dirname(dest_changelog))
     File.new(dest_changelog, "wb", 0644).tap do |changelog|
       Zlib::GzipWriter.new(changelog, Zlib::BEST_COMPRESSION).tap do |changelog_gz|
+        if not attributes[:source_date_epoch].nil?
+          changelog_gz.mtime = attributes[:source_date_epoch].to_i
+        end
         if attributes[:deb_changelog]
           logger.info("Writing user-specified changelog", :source => attributes[:deb_changelog])
           File.new(attributes[:deb_changelog]).tap do |fd|
@@ -461,6 +484,9 @@ class FPM::Package::Deb < FPM::Package
     if attributes[:deb_upstream_changelog]
       File.new(dest_upstream_changelog, "wb", 0644).tap do |changelog|
         Zlib::GzipWriter.new(changelog, Zlib::BEST_COMPRESSION).tap do |changelog_gz|
+            if not attributes[:source_date_epoch].nil?
+              changelog_gz.mtime = attributes[:source_date_epoch].to_i
+            end
             logger.info("Writing user-specified upstream changelog", :source => attributes[:deb_upstream_changelog])
             File.new(attributes[:deb_upstream_changelog]).tap do |fd|
               chunk = nil
@@ -494,6 +520,7 @@ class FPM::Package::Deb < FPM::Package
 
     attributes.fetch(:deb_upstart_list, []).each do |upstart|
       name = File.basename(upstart, ".upstart")
+      dest_init = staging_path("etc/init.d/#{name}")
       name = "#{name}.conf" if !(name =~ /\.conf$/)
       dest_upstart = staging_path("etc/init/#{name}")
       mkdir_p(File.dirname(dest_upstart))
@@ -501,7 +528,6 @@ class FPM::Package::Deb < FPM::Package
       File.chmod(0644, dest_upstart)
 
       # Install an init.d shim that calls upstart
-      dest_init = staging_path("etc/init.d/#{name}")
       mkdir_p(File.dirname(dest_init))
       FileUtils.ln_s("/lib/init/upstart-job", dest_init)
     end
@@ -533,13 +559,19 @@ class FPM::Package::Deb < FPM::Package
     end
 
     args = [ tar_cmd, "-C", staging_path, compression ] + data_tar_flags + [ "-cf", datatar, "." ]
+    if tar_cmd_supports_sort_names_and_set_mtime? and not attributes[:source_date_epoch].nil?
+      # Use gnu tar options to force deterministic file order and timestamp
+      args += ["--sort=name", ("--mtime=@%s" % attributes[:source_date_epoch])]
+      # gnu tar obeys GZIP environment variable with options for gzip; -n = forget original filename and date
+      args.unshift({"GZIP" => "-9n"})
+    end
     safesystem(*args)
 
     # pack up the .deb, which is just an 'ar' archive with 3 files
     # the 'debian-binary' file has to be first
     File.expand_path(output_path).tap do |output_path|
       ::Dir.chdir(build_path) do
-        safesystem("ar", "-qc", output_path, "debian-binary", "control.tar.gz", datatar)
+        safesystem(*ar_cmd, output_path, "debian-binary", "control.tar.gz", datatar)
       end
     end
   end # def output
@@ -693,6 +725,12 @@ class FPM::Package::Deb < FPM::Package
 
       args = [ tar_cmd, "-C", control_path, "-zcf", controltar,
         "--owner=0", "--group=0", "--numeric-owner", "." ]
+      if tar_cmd_supports_sort_names_and_set_mtime? and not attributes[:source_date_epoch].nil?
+        # Force deterministic file order and timestamp
+        args += ["--sort=name", ("--mtime=@%s" % attributes[:source_date_epoch])]
+        # gnu tar obeys GZIP environment variable with options for gzip; -n = forget original filename and date
+        args.unshift({"GZIP" => "-9n"})
+      end
       safesystem(*args)
     end
 
@@ -865,7 +903,10 @@ class FPM::Package::Deb < FPM::Package
 
   def write_triggers
     lines = [['interest', :deb_interest],
-             ['activate', :deb_activate]].map { |label, attr|
+             ['activate', :deb_activate],
+             ['interest-noawait', :deb_interest_noawait],
+             ['activate-noawait', :deb_activate_noawait],
+             ].map { |label, attr|
       (attributes[attr] || []).map { |e| "#{label} #{e}\n" }
     }.flatten.join('')
 

data/lib/fpm/package/dir.rb

@@ -101,18 +101,7 @@ class FPM::Package::Dir < FPM::Package
     end
 
     # Write the scripts, too.
-    scripts_path = File.join(output_path, ".scripts")
-    ::Dir.mkdir(scripts_path)
-    [:before_install, :after_install, :before_remove, :after_remove].each do |name|
-      next unless script?(name)
-      out = File.join(scripts_path, name.to_s)
-      logger.debug("Writing script", :source => name, :target => out)
-      File.write(out, script(name))
-      require "pry"
-      binding.pry
-      File.chmod(0755, out)
-    end
-
+    write_scripts
   ensure
     logger.remove("method")
   end # def output
@@ -147,14 +136,19 @@ class FPM::Package::Dir < FPM::Package
 
     # For single file copies, permit file destinations
     fileinfo = File.lstat(source)
-    if fileinfo.file? && !File.directory?(destination)
+    destination_is_directory = File.directory?(destination)
+    if fileinfo.file? && !destination_is_directory
       if destination[-1,1] == "/"
         copy(source, File.join(destination, source))
       else
         copy(source, destination)
       end
     elsif fileinfo.symlink?
-      copy(source, File.join(destination, source))
+      if destination_is_directory
+        copy(source, File.join(destination, source))
+      else
+        copy(source, destination)
+      end
     else
       # Copy all files from 'path' into staging_path
       Find.find(source) do |path|