fpm-fry 0.1.3

data/lib/fpm/fry/source/dir.rb

@@ -0,0 +1,56 @@
+require 'fpm/fry/source'
+require 'fileutils'
+require 'digest'
+module FPM; module Fry ; module Source
+  class Dir
+
+    REGEX = %r!\A(?:file:|/|\.)!
+
+    def self.guess( url )
+      Source::guess_regex(REGEX, url)
+    end
+
+    class Cache < Struct.new(:package, :dir)
+      extend Forwardable
+
+      def_delegators :package, :url, :logger, :file_map
+
+      def tar_io
+        cmd = ['tar','-c','.']
+        logger.debug("Running tar",cmd: cmd, dir: dir)
+        ::Dir.chdir(dir) do
+          return IO.popen(cmd)
+        end
+      end
+
+      def copy_to(dst)
+        children = ::Dir.new(dir).select{|x| x[0...1] != "." }.map{|x| File.join(dir,x) }
+        FileUtils.cp_r(children, dst)
+      end
+
+      def cachekey
+        dig = Digest::SHA2.new
+        tar_io.each(1024) do |block|
+          dig << block
+        end
+        return dig.hexdigest
+      end
+    end
+
+    attr :url, :logger, :file_map
+
+    def initialize( url, options = {} )
+      @url = URI(url)
+      if @url.relative?
+        @url.path = File.expand_path(@url.path)
+      end
+      @logger = options.fetch(:logger){ Cabin::Channel.get }
+      @file_map = options.fetch(:file_map){ {'' => ''} }
+    end
+
+    def build_cache(_)
+      Cache.new(self, url.path)
+    end
+  end
+end ; end ; end
+

data/lib/fpm/fry/source/git.rb

@@ -0,0 +1,90 @@
+require 'fileutils'
+require 'forwardable'
+require 'open3'
+require 'fpm/fry/source'
+module FPM; module Fry ; module Source
+  class Git
+
+    REGEX = %r!\A(?:git:|\S+@\S+:\S+\.git\z|https?:.*\.git\z|ssh:.*\.git\z)!
+
+    def self.guess( url )
+      Source::guess_regex(REGEX, url)
+    end
+
+    class Cache < Struct.new(:package, :tempdir)
+      extend Forwardable
+
+      def_delegators :package, :url, :rev, :logger, :file_map
+
+      def update
+        begin
+          if !File.exists? repodir
+            if git('init', '--bare') != 0
+              raise "Initializing git repository failed"
+            end
+          end
+          if git('fetch','--depth=1', url.to_s, rev) != 0
+            raise "Failed to fetch from remote #{url.to_s} ( #{rev} )"
+          end
+          return self
+        rescue Errno::ENOENT
+          raise "Cannot find git binary. Is it installed?"
+        end
+      end
+
+      def tar_io
+        cmd = [package.git, "--git-dir=#{repodir}",'archive','--format=tar','FETCH_HEAD']
+        logger.debug("Running git",cmd: cmd)
+        IO.popen(cmd)
+      end
+
+      def copy_to(dst)
+        cmd = [package.git, "--git-dir=#{repodir}", "--work-tree=#{dst}",'checkout','FETCH_HEAD','--','*']
+        logger.debug("Running git",cmd: cmd)
+        system(*cmd, chdir: dst)
+      end
+
+      def cachekey
+        cmd = [package.git, "--git-dir=#{repodir}",'rev-parse','FETCH_HEAD^{tree}']
+        logger.debug("Running git",cmd: cmd)
+        return IO.popen(cmd).read.chomp
+      end
+    private
+      def repodir
+        File.join(tempdir,File.basename(url.path))
+      end
+
+      def git(*args)
+        cmd = [package.git, "--git-dir=#{repodir}",*args]
+        logger.debug("Running git",cmd: cmd.join(' '))
+        Open3.popen3(*cmd) do |sin, out, err, thr|
+          sin.close
+          out.each_line do |line|
+            logger.debug(line.chomp)
+          end
+          err.each_line do |line|
+            logger.debug(line.chomp)
+          end
+          return thr.value
+        end
+      end
+
+    end
+
+    attr :logger, :git, :rev, :file_map, :url
+
+    def initialize( url, options = {} )
+      url = url.sub(/\A(\S+@\S+):(\S+\.git)\z/,'ssh://\1/\2')
+      @url = URI(url)
+      @logger = options.fetch(:logger){ Cabin::Channel.get }
+      @rev = options[:branch] || options[:tag] || options[:rev] || 'HEAD'
+      @file_map = options.fetch(:file_map){ {'' => ''} }
+      @git = options[:git] || 'git'
+    end
+
+    def build_cache(tempdir)
+      Cache.new(self, tempdir).update
+    end
+
+  end
+end ; end ; end

data/lib/fpm/fry/source/package.rb

@@ -0,0 +1,202 @@
+require 'uri'
+require 'digest'
+require 'net/http'
+require 'forwardable'
+require 'zlib'
+require 'fpm/fry/source'
+module FPM; module Fry ; module Source
+  class Package
+
+    REGEX = %r!\Ahttps?:!
+
+    def self.guess( url )
+      Source::guess_regex(REGEX, url)
+    end
+
+    class RedirectError < CacheFailed
+    end
+
+    class Cache < Struct.new(:package,:tempdir)
+      extend Forwardable
+
+      def_delegators :package, :url, :checksum, :checksum_algorithm, :agent, :logger, :file_map
+
+      def initialize(*_)
+        super
+        if !checksum
+          update!
+        end
+      end
+
+      def cache_valid?
+        c = @observed_checksum || checksum
+        begin
+          checksum_algorithm.file(tempfile).hexdigest == c
+        rescue Errno::ENOENT
+          return false
+        end
+      end
+
+      def update!
+        if cache_valid?
+          logger.debug("Found valid cache", url: url, tempfile: tempfile)
+          return
+        end
+        d = checksum_algorithm.new
+        f = nil
+        fetch_url(url) do |resp|
+          begin
+            f = File.new(tempfile,'w')
+            resp.read_body do | chunk |
+              d.update(chunk)
+              f.write(chunk)
+            end
+          rescue => e
+            raise CacheFailed, e
+          ensure
+            f.close
+          end
+        end
+
+        @observed_checksum = d.hexdigest
+        logger.debug("got checksum", checksum: @observed_checksum)
+        if checksum
+          if d.hexdigest != checksum
+            raise CacheFailed.new("Checksum failed",given: d.hexdigest, expected: checksum)
+          end
+        else
+          return true
+        end
+      end
+
+      def fetch_url( url, redirs = 3, &block)
+        url = URI(url.to_s) unless url.kind_of? URI
+        Net::HTTP.get_response(url) do |resp|
+          case(resp)
+          when Net::HTTPRedirection
+            if redirs == 0
+              raise RedirectError, "Too many redirects"
+            end
+            logger.debug("Following redirect", url: url.to_s , location: resp['location'])
+            return fetch_url( resp['location'], redirs - 1, &block)
+          when Net::HTTPSuccess
+            return block.call(resp)
+          else
+            raise CacheFailed.new('Unable to fetch file',url: url.to_s, http_code: resp.code, http_message: resp.message)
+          end
+        end
+      end
+
+      def tempfile
+        File.join(tempdir,File.basename(url.path))
+      end
+
+      def cachekey
+        @observed_checksum || checksum
+      end
+
+    end
+
+    class TarCache < Cache
+
+      def tar_io
+        update!
+        ioclass.open(tempfile)
+      end
+
+      def copy_to(dst)
+        update!
+        cmd = ['tar','-xf',tempfile,'-C',dst]
+        logger.debug("Running tar",cmd: cmd)
+        system(*cmd)
+      end
+
+    protected
+      def ioclass
+        File
+      end
+    end
+
+    class TarGzCache < TarCache
+    protected
+
+      def ioclass
+        Zlib::GzipReader
+      end
+    end
+
+    class ZipCache < Cache
+
+      def tar_io
+        if !::File.directory?( unpacked_tmpdir )
+          workdir = unpacked_tmpdir + '.tmp'
+          begin
+            FileUtils.mkdir(workdir)
+          rescue Errno::EEXIST
+            FileUtils.rm_rf(workdir)
+            FileUtils.mkdir(workdir)
+          end
+          copy_to( workdir )
+          File.rename(workdir, unpacked_tmpdir)
+        end
+        cmd = ['tar','-c','.']
+        logger.debug("Running tar",cmd: cmd, dir: unpacked_tmpdir)
+        ::Dir.chdir(unpacked_tmpdir) do
+          return IO.popen(cmd)
+        end
+      end
+
+      def copy_to(dst)
+        update!
+        cmd = ['unzip', tempfile, '-d', dst ]
+        logger.debug("Running unzip",cmd: cmd)
+        system(*cmd, out: '/dev/null')
+      end
+
+      def unpacked_tmpdir
+        File.join(tempdir, cachekey)
+      end
+    end
+
+    CACHE_CLASSES = {
+      '.tar' => TarCache,
+      '.tar.gz' => TarGzCache,
+      '.tgz' => TarGzCache,
+      '.zip' => ZipCache
+    }
+
+    attr :file_map, :data, :url, :extension, :checksum, :checksum_algorithm, :agent, :logger
+
+    def initialize( url, options = {} )
+      @url = URI(url)
+      @extension = options.fetch(:extension){
+        CACHE_CLASSES.keys.find{|ext|
+          @url.path.end_with?(ext)
+        }
+      }
+      @logger = options.fetch(:logger){ Cabin::Channel.get }
+      @checksum = options[:checksum]
+      @checksum_algorithm = guess_checksum_algorithm(options[:checksum])
+      @file_map = options.fetch(:file_map){ {'' => ''} }
+    end
+
+    def build_cache(tempdir)
+      CACHE_CLASSES.fetch(extension).new(self, tempdir)
+    end
+  private
+
+    def guess_checksum_algorithm( checksum )
+      case(checksum)
+      when nil
+        return Digest::SHA256
+      when /\A(sha256:)?[0-9a-f]{64}\z/ then
+        return Digest::SHA256
+      when /\A(sha1:)?[0-9a-f]{40}\z/ then
+        return Digest::SHA1
+      else
+        raise "Unknown checksum algorithm"
+      end
+    end
+
+  end
+end end end

data/lib/fpm/fry/source/patched.rb

@@ -0,0 +1,118 @@
+require 'fpm/fry/tar'
+require 'digest'
+module FPM; module Fry ; module Source
+  class Patched
+
+    class Cache < Struct.new(:package, :tmpdir)
+      extend Forwardable
+
+      def_delegators :package, :logger, :file_map
+
+      attr :inner
+
+      def initialize(*_)
+        @updated = false
+        super
+        @inner = package.inner.build_cache(tmpdir)
+      end
+
+      def update!
+        @updated ||= begin
+          if !File.directory?(unpacked_tmpdir)
+            workdir = unpacked_tmpdir + '.tmp'
+            begin
+              FileUtils.mkdir(workdir)
+            rescue Errno::EEXIST
+              FileUtils.rm_rf(workdir)
+              FileUtils.mkdir(workdir)
+            end
+            if inner.respond_to? :copy_to
+              inner.copy_to(workdir)
+            else
+              ex = Tar::Extractor.new(logger: logger)
+              tio = inner.tar_io
+              begin
+                ex.extract(workdir, ::Gem::Package::TarReader.new(tio), chown: false)
+              ensure
+                tio.close
+              end
+            end
+            package.patches.each do |patch|
+              cmd = ['patch','-p1','-i',patch[:file]]
+              chdir = File.expand_path(patch.fetch(:chdir,'.'),workdir)
+              logger.debug("Running patch",cmd: cmd, dir: chdir )
+              system(*cmd, chdir: chdir, out: :close)
+            end
+            File.rename(workdir, unpacked_tmpdir)
+          end
+          true
+        end
+      end
+      private :update!
+
+      def tar_io
+        update!
+        cmd = ['tar','-c','.']
+        logger.debug("Running tar",cmd: cmd, dir: unpacked_tmpdir)
+        # IO.popen( ..., chdir: ... ) doesn't work on older ruby
+        ::Dir.chdir(unpacked_tmpdir) do
+          return IO.popen(cmd)
+        end
+      end
+
+      def cachekey
+        dig = Digest::SHA2.new
+        dig << inner.cachekey << "\x00"
+        package.patches.each do |patch|
+          dig.file(patch[:file])
+          dig << "\x00"
+        end
+        return dig.hexdigest
+      end
+
+      def unpacked_tmpdir
+        File.join(tmpdir, cachekey)
+      end
+
+    end
+
+    attr :inner, :patches
+
+    extend Forwardable
+
+    def_delegators :inner, :logger, :file_map
+
+    def initialize( inner , options = {})
+      @inner = inner
+      @patches = Array(options[:patches]).map do |file|
+        if file.kind_of? String
+          options = {file: file}
+        elsif file.kind_of? Hash
+          options = file.dup
+        else
+          raise ArgumentError, "Expected a Hash or a String, got #{file.inspect}"
+        end
+        options[:file] = File.expand_path(options[:file])
+        if !File.exists?(options[:file])
+          raise ArgumentError, "File doesn't exist: #{options[:file]}"
+        end
+        options
+      end
+    end
+
+    def build_cache(tmpdir)
+      Cache.new(self,tmpdir)
+    end
+
+    def self.decorate(options)
+      if options.key?(:patches) && Array(options[:patches]).size > 0
+        p = options.delete(:patches)
+        return new( yield(options), patches: p )
+      else
+        return yield options
+      end
+    end
+  end
+
+end ; end ; end
+

data/lib/fpm/fry/source.rb

@@ -0,0 +1,47 @@
+module FPM; module Fry ; module Source
+
+  class CacheFailed < StandardError
+
+    attr :options
+
+    def initialize(e, opts = {})
+      if e.kind_of? Exception
+        @options = {reason: e}.merge opts
+        super(e.message)
+      else
+        @options = opts.dup
+        super(e.to_s)
+      end
+    end
+  end
+
+  module Null
+
+    module Cache
+      def self.tar_io
+        StringIO.new("\x00"*1024)
+      end
+      def self.file_map
+        return {}
+      end
+      def self.cachekey
+        return '0' * 32
+      end
+    end
+
+    def self.build_cache(*_)
+      return Cache
+    end
+
+  end
+
+  class << self
+
+    def guess_regex(rx, url)
+      if m = rx.match(url.to_s)
+        return m[0].size
+      end
+    end
+
+  end
+end ; end ; end

data/lib/fpm/fry/stream_parser.rb

@@ -0,0 +1,98 @@
+require 'excon/middlewares/base'
+module FPM; module Fry
+  class StreamParser
+
+    class ShortRead < EOFError
+    end
+
+    class Instance < Excon::Middleware::Base
+
+      def initialize(stack, parser)
+        super(stack)
+        @parser = parser
+      end
+
+      def response_call(datum)
+        if datum[:response]
+          # probably mocked
+          if datum[:response][:body]
+            @parser.parse(StringIO.new(datum[:response][:body]))
+          end
+          return @stack.response_call(datum)
+        else
+          socket = datum[:connection].send(:socket)
+          begin
+            line = socket.readline
+            match = /^HTTP\/\d+\.\d+\s(\d{3})\s/.match(line)
+          end while !match
+          status = match[1].to_i
+
+          datum[:response] = {
+            :body          => '',
+            :headers       => Excon::Headers.new,
+            :status        => status,
+            :remote_ip     => socket.respond_to?(:remote_ip) && socket.remote_ip,
+          }
+          Excon::Response.parse_headers(socket, datum)
+
+          @parser.parse(socket)
+          return @stack.response_call(datum)
+        end
+      end
+
+    end
+
+    attr :out, :err
+
+    def initialize(out, err)
+      @out, @err = out, err
+      @state = :null
+      @left = 0
+    end
+
+    def new(stack)
+      Instance.new(stack, self)
+    end
+
+    def parse(socket)
+      left  = 0
+      streams = {1 => out, 2 => err}
+      loop do
+        type = read_exactly(socket,4){|part|
+          if part.bytesize == 0
+            return
+          else
+            raise ShortRead
+          end
+        }.unpack("c".freeze)[0]
+        stream = streams.fetch(type){ raise ArgumentError, "Wrong stream type: #{type}"}
+        len  = read_exactly(socket,4).unpack('I>')[0]
+        while len > 0
+          chunk = socket.read([64,len].min)
+          raise ShortRead if chunk.nil?
+          len -= chunk.bytesize
+          stream.write(chunk)
+        end
+      end
+    end
+
+    def read_exactly(socket, len)
+      buf = ""
+      left = len
+      while left != 0
+        read = socket.read(left)
+        if read.nil?
+          if block_given?
+            yield buf
+          else
+            raise ShortRead
+          end
+        end
+        buf << read
+        left = len - buf.bytesize
+      end
+      return buf
+    end
+
+  end
+end ; end

data/lib/fpm/fry/tar.rb

@@ -0,0 +1,71 @@
+module FPM; module Fry; end ; end
+
+class FPM::Fry::Tar
+
+  class Extractor
+
+    def initialize( options = {} )
+      @logger = options.fetch(:logger){ Cabin::Channel.get }
+    end 
+
+    def extract(destdir, reader, options = {})
+      reader.each do |entry|
+        extract_entry(File.join(destdir, entry.full_name), entry, options)
+      end
+    end
+
+    def extract_entry(dest, entry, options = {})
+      full_name = entry.full_name
+      mode = entry.header.mode
+
+      destdir = File.dirname(dest)
+      uid = map_user(entry.header.uid, entry.header.uname)
+      gid = map_group(entry.header.gid, entry.header.gname)
+
+      @logger.debug('Extracting','file' => dest, 'uid'=> uid, 'gid' => gid, 'entry.fullname' => full_name, 'entry.mode' => mode )
+
+      case(entry.header.typeflag)
+      when "5" # Directory
+        FileUtils.mkdir_p(dest, :mode => mode)
+      when "2" # Symlink
+        destdir = File.dirname(dest)
+        FileUtils.mkdir_p(destdir, :mode => 0755)
+        File.symlink( entry.header.linkname, dest )
+      when "0" # File
+        destdir = File.dirname(dest)
+        FileUtils.mkdir_p(destdir, :mode => 0755)
+        File.open(dest, "wb", entry.header.mode) do |os|
+          loop do
+            data = entry.read(4096)
+            break unless data
+            os.write(data)
+          end
+          os.fsync
+        end
+      else
+        @logger.warn('Ignoring unknown tar entry',name: full_name)
+        return
+      end
+      FileUtils.chmod(entry.header.mode, dest)
+      chown( uid, gid, dest ) if options.fetch(:chown,true)
+    end
+
+    def chown( uid, gid, path )
+      FileUtils.chown( uid, gid, path )
+    rescue Errno::EPERM
+      @logger.warn('Unable to chown file', 'file' => path, 'uid' => uid, 'gid' => gid)
+    end
+
+    def map_user( uid, _ )
+      return uid
+    end
+
+    def map_group( gid, _ )
+      return gid
+    end
+
+  end
+
+end
+
+

data/lib/fpm/fry/templates/debian/after_install.erb

@@ -0,0 +1,9 @@
+#!/bin/bash
+case "$1" in
+  configure)
+  <%= configure.join("\n") %>
+  ;;
+  *)
+  exit 1
+  ;;
+esac

data/lib/fpm/fry/templates/debian/before_install.erb

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+case "$1" in
+  install)
+  <%= install.join("\n") %>
+  ;;
+  upgrade)
+  <%= upgrade.join("\n") %>
+  ;;
+  *)
+  exit 1
+  ;;
+esac