fpm-fry 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6d10781e6b5740b80f68e6a7130acd844c2b1d5c
+  data.tar.gz: 5dc23f36f547bd2359b433424ae8ef091dd2cf53
+SHA512:
+  metadata.gz: ba95b38a5652f87439105db3c7430505695e5429ef90fdfadcb1b6f517556259c8add886a2c0b0cca72e0d7c7683a4584883d3e1263266ad5f4e162fa911b9f0
+  data.tar.gz: 9ded152eb45fbab21bdef04a651bee26689f0dd8a45a10d096ece7fe5cbc646f727fbc082e353f49bd62b6a303887d6ab86b6bb0e245625667d0529779f39f3e

data/bin/fpm-fry

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+$: << File.expand_path(File.join( "..", "lib"),File.dirname(__FILE__))
+require "fpm"
+require "fpm/command"
+require "fpm/package/docker"
+require "fpm/fry/command"
+
+exit(FPM::Fry::Command.run || 0)

data/lib/cabin/nice_output.rb

@@ -0,0 +1,70 @@
+require 'cabin'
+class Cabin::NiceOutput
+
+  CODEMAP = {
+    :normal => "\e[0m",
+    :red => "\e[1;31m",
+    :green => "\e[1;32m",
+    :yellow => "\e[0;33m",
+    :white => "\e[0;37m"
+  }
+
+  DIM_CODEMAP = {
+    red:   "\e[0;31m",
+    green: "\e[0;32m",
+    white: "\e[1;30m",
+    yellow: "\e[33m"
+  }
+
+  LEVELMAP = {
+    :fatal => :red,
+    :error => :red,
+    :warn => :yellow,
+    :info => :green,
+    :debug => :white,
+  }
+
+  attr :io
+
+  def initialize(io)
+    @io = io
+  end
+
+  def <<(event)
+    data = event.clone
+    data.delete(:line)
+    data.delete(:file)
+    level = data.delete(:level) || :normal
+    data.delete(:message)
+    ts = data.delete(:timestamp)
+
+    color = data.delete(:color)
+    # :bold is expected to be truthy
+    bold = data.delete(:bold) ? :bold : nil
+
+    backtrace = data.delete(:backtrace)
+
+    # Make 'error' and other log levels have color
+    if color.nil?
+      color = LEVELMAP[level]
+    end
+
+    message = [event[:level] ? '====> ' : '      ',event[:message]]
+    message.unshift(CODEMAP[color.to_sym]) if !color.nil?
+    message << DIM_CODEMAP[color] if !color.nil?
+    if data.any?
+      message << "\n" <<  pp(data)
+    end
+    if backtrace
+      message << "\n\t--backtrace---------------\n\t" << backtrace.join("\n\t")
+    end
+    message << CODEMAP[:normal]  if !color.nil?
+    @io.puts(message.join(""))
+    @io.flush
+  end
+
+  def pp(hash)
+    hash.map{|k,v| '      '+k.to_s + ": " + v.inspect }.join("\n")
+  end
+
+end

data/lib/fpm/fry/block_enumerator.rb

@@ -0,0 +1,25 @@
+module FPM; module Fry
+  class BlockEnumerator < Struct.new(:io, :blocksize)
+    include Enumerable
+
+    def initialize(_, blocksize = 128)
+      super
+    end
+
+    def each
+      return to_enum unless block_given?
+      # Reading bigger chunks is far more efficient that eaching over the
+      while chunk = io.read(blocksize)
+        yield chunk
+      end
+    end
+
+    def call
+      while x = io.read(blocksize)
+        next if x == ""
+        return x
+      end
+      return ""
+    end
+  end
+end ; end

data/lib/fpm/fry/build_output_parser.rb

@@ -0,0 +1,22 @@
+require 'json'
+module FPM; module Fry
+  class BuildOutputParser < Struct.new(:out)
+
+    attr :images
+
+    def initialize(*_)
+      super
+      @images = []
+    end
+
+    def call(chunk, *_)
+      json = JSON.parse(chunk)
+      stream = json['stream']
+      if /\ASuccessfully built (\w+)\Z/.match(stream)
+        images << $1
+      end
+      out << stream
+    end
+
+  end
+end ; end

data/lib/fpm/fry/client.rb

@@ -0,0 +1,162 @@
+require 'excon'
+require 'rubygems/package'
+require 'json'
+require 'fileutils'
+require 'forwardable'
+require 'fpm/fry/tar'
+
+module FPM; module Fry; end ; end
+
+class FPM::Fry::Client
+
+  class LogInstrumentor < Struct.new(:logger)
+
+    def instrument(event, data = {})
+      if block_given?
+        logger.debug('Requesting HTTP', filtered(data))
+        r = yield
+        return r
+      else
+        logger.debug('Getting HTTP response', filtered(data))
+      end
+    end
+
+    def filtered(data)
+      filtered = {}
+      filtered[:path] = data[:path] if data[:path]
+      filtered[:verb] = data[:method] if data[:method]
+      filtered[:status] = data[:status] if data[:status]
+      filtered[:body] = data[:body][0..500] if data[:body]
+      filtered[:headers] = data[:headers] 
+      return filtered
+    end
+
+  end
+
+  class FileNotFound < StandardError
+  end
+
+  extend Forwardable
+  def_delegators :agent, :post, :get, :delete
+
+  attr :docker_url, :logger, :tls
+
+  def initialize(options = {})
+    @docker_url = options.fetch(:docker_url){ self.class.docker_url }
+    @logger = options[:logger]
+    if @logger.nil?
+      @logger = Cabin::Channel.get
+    end
+    if options[:tls].nil? ? docker_url =~ %r!(\Ahttps://|:2376\z)! : options[:tls]
+      # enable tls
+      @tls = {
+        client_cert: File.join(self.class.docker_cert_path,'cert.pem'),
+        client_key: File.join(self.class.docker_cert_path, 'key.pem'),
+        ssl_ca_file: File.join(self.class.docker_cert_path, 'ca.pem'),
+        ssl_verify_peer: options.fetch(:tlsverify){ false }
+      }
+      [:client_cert, :client_key, :ssl_ca_file].each do |k|
+        if !File.exists?(@tls[k])
+          raise ArgumentError.new("#{k} #{@tls[k]} doesn't exist. Did you set DOCKER_CERT_PATH correctly?")
+        end
+      end
+    else
+      @tls = {}
+    end
+  end
+
+  def server_version
+    @server_version ||= begin
+      res = agent.get(
+        expects: [200],
+        path: '/version'
+      )
+      JSON.parse(res.body)
+    end
+  end
+
+  def self.docker_cert_path
+    ENV.fetch('DOCKER_CERT_PATH',File.join(Dir.home, '.docker'))
+  end
+
+  def self.docker_url
+    ENV.fetch('DOCKER_HOST'.freeze, 'unix:///var/run/docker.sock')
+  end
+
+  def tls?
+    tls.any?
+  end
+
+
+  def url(*path)
+    ['', "v"+server_version['ApiVersion'],*path].join('/')
+  end
+
+  def read(name, resource)
+    return to_enum(:read, name, resource) unless block_given?
+    body = JSON.generate({'Resource' => resource})
+    res = agent.post(
+      path: url('containers',name,'copy'),
+      headers: { 'Content-Type' => 'application/json' },
+      body: body,
+      expects: [200,500]
+    )
+    if res.status == 500
+      raise FileNotFound, "File #{resource.inspect} not found: #{res.body}"
+    end
+    sio = StringIO.new(res.body)
+    tar = ::Gem::Package::TarReader.new( sio )
+    tar.each do |entry|
+      yield entry
+    end
+  end
+
+  def copy(name, resource, map, options = {})
+    ex = FPM::Fry::Tar::Extractor.new(logger: @logger)
+    base = File.dirname(resource)
+    read(name, resource) do | entry |
+      file = File.join(base, entry.full_name).chomp('/')
+      file = file.sub(%r"\A\./",'')
+      to = map[file]
+      next unless to
+      @logger.debug("Copy",name: file, to: to)
+      ex.extract_entry(to, entry, options)
+    end
+  end
+
+  def changes(name)
+    res = agent.get(path: url('containers',name,'changes'))
+    raise res.reason if res.status != 200
+    return JSON.parse(res.body)
+  end
+
+  def agent
+    @agent ||= agent_for(docker_url, tls)
+  end
+
+  def broken_symlinks?
+    return true
+  end
+
+  def agent_for( uri, tls )
+    proto, address = uri.split('://',2)
+    options = {
+      instrumentor: LogInstrumentor.new(logger),
+      read_timeout: 10000
+    }.merge( tls )
+    case(proto)
+    when 'unix'
+      uri = "unix:///"
+      options[:socket] = address
+    when 'tcp'
+      if tls.any?
+        return agent_for("https://#{address}", tls)
+      else
+        return agent_for("http://#{address}", tls)
+      end
+    when 'http', 'https'
+    end
+    logger.debug("Creating Agent", options.merge(uri: uri))
+    return Excon.new(uri, options)
+  end
+end

data/lib/fpm/fry/command/cook.rb

@@ -0,0 +1,370 @@
+require 'fpm/fry/command'
+module FPM; module Fry
+  class Command::Cook < Command
+
+    option '--distribution', 'distribution', 'Distribution like ubuntu-12.04'
+    option '--keep', :flag, 'Keep the container after build'
+    option '--overwrite', :flag, 'Overwrite package', default: true
+
+    UPDATE_VALUES = ['auto','never','always']
+    option '--update',"<#{UPDATE_VALUES.join('|')}>", 'Update image before installing packages ( only apt currently )',attribute_name: 'update', default: 'auto' do |value|
+      if !UPDATE_VALUES.include? value
+        raise "Unknown value for --update: #{value.inspect}\nPossible values are #{UPDATE_VALUES.join(', ')}"
+      else
+        value
+      end
+    end
+
+    parameter 'image', 'Docker image to build from'
+    parameter '[recipe]', 'Recipe file to cook', default: 'recipe.rb'
+
+    attr :ui
+    extend Forwardable
+    def_delegators :ui, :out, :err, :logger, :tmpdir
+
+    def initialize(invocation_path, ctx = {}, parent_attribute_values = {})
+      @tls = nil
+      require 'digest'
+      require 'fileutils'
+      require 'fpm/fry/recipe'
+      require 'fpm/fry/recipe/builder'
+      require 'fpm/fry/detector'
+      require 'fpm/fry/docker_file'
+      require 'fpm/fry/stream_parser'
+      require 'fpm/fry/os_db'
+      require 'fpm/fry/block_enumerator'
+      require 'fpm/fry/build_output_parser'
+      super
+      @ui = ctx.fetch(:ui){ UI.new }
+      if debug?
+        ui.logger.level = :debug
+      end
+    end
+
+    def detector
+      @detector || begin
+        if distribution
+          d = Detector::String.new(distribution)
+        else
+          d = Detector::Image.new(client, image)
+        end
+        self.detector=d
+      end
+    end
+
+    def detector=(d)
+      begin
+        unless d.detect!
+          raise "Unable to detect distribution from given image"
+        end
+      rescue Excon::Errors::NotFound
+        raise "Image not found"
+      end
+      @detector = d
+    end
+
+    def flavour
+      @flavour ||= OsDb.fetch(detector.distribution,{flavour: "unknown"})[:flavour]
+    end
+    attr_writer :flavour
+
+    def output_class
+      @output_class ||= begin
+        logger.info("Autodetecting package type",flavour: flavour)
+        case(flavour)
+        when 'debian'
+          require 'fpm/package/deb'
+          FPM::Package::Deb
+        when 'redhat'
+          require 'fpm/package/rpm'
+          FPM::Package::RPM
+        else
+          raise "Cannot auto-detect package type."
+        end
+      end
+    end
+    attr_writer :output_class
+
+    def builder
+      @builder ||= begin
+        vars = {
+          distribution: detector.distribution,
+          distribution_version: detector.version,
+          flavour: flavour
+        }
+        logger.info("Loading recipe",variables: vars, recipe: recipe)
+        b = Recipe::Builder.new(vars, Recipe.new, logger: ui.logger)
+        b.load_file( recipe )
+        b
+      end
+    end
+    attr_writer :builder
+
+    def cache
+      @cache ||= builder.recipe.source.build_cache(tmpdir)
+    end
+    attr_writer :cache
+
+    def lint_output_class!
+
+    end
+
+    def lint_recipe_file!
+      File.exists?(recipe) || raise(Recipe::NotFound)
+    end
+
+    def lint_recipe!
+      problems = builder.recipe.lint
+      if problems.any?
+        problems.each do |p|
+          logger.error(p)
+        end
+        raise
+      end
+    end
+
+    def image_id
+      @image_id ||= begin
+        res = client.get(
+          expects: [200],
+          path: client.url("images/#{image}/json")
+        )
+        body = JSON.parse(res.body)
+        body.fetch('id'){ body.fetch('Id') }
+      end
+    end
+    attr_writer :image_id
+
+    def build_image
+      @build_image ||= begin
+        sum = Digest::SHA256.hexdigest( image_id + "\0" + cache.cachekey )
+        cachetag = "fpm-fry:#{sum[0..30]}"
+        res = client.get(
+          expects: [200,404],
+          path: client.url("images/#{cachetag}/json")
+        )
+        if res.status == 404
+          df = DockerFile::Source.new(builder.variables.merge(image: image_id),cache)
+          client.post(
+            headers: {
+              'Content-Type'=>'application/tar'
+            },
+            expects: [200],
+            path: client.url("build?rm=1&t=#{cachetag}"),
+            request_block: BlockEnumerator.new(df.tar_io)
+          )
+        end
+
+        df = DockerFile::Build.new(cachetag, builder.variables.dup,builder.recipe, update: update?)
+        parser = BuildOutputParser.new(out)
+        res = client.post(
+          headers: {
+            'Content-Type'=>'application/tar'
+          },
+          expects: [200],
+          path: client.url('build?rm=1'),
+          request_block: BlockEnumerator.new(df.tar_io),
+          response_block: parser
+        )
+        if parser.images.none?
+          raise "Unable to detect build image"
+        end
+        image = parser.images.last
+        logger.debug("Detected build image", image: image)
+        image
+      end
+    end
+    attr_writer :build_image
+
+    def update?
+      if flavour == 'debian'
+        case(update)
+        when 'auto'
+          body = JSON.generate({"Image" => image, "Cmd" => "exit 0"})
+          res = client.post( path: client.url('containers','create'),
+                             headers: {'Content-Type' => 'application/json'},
+                             body: body,
+                             expects: [201]
+                           )
+          body = JSON.parse(res.body)
+          container = body.fetch('Id')
+          begin
+            client.read( container, '/var/lib/apt/lists') do |file|
+              next if file.header.name == 'lists/'
+              logger.info("/var/lib/apt/lists is not empty, no update is required ( force update with --apt-update=always )")
+              return false
+            end
+          ensure
+            client.delete(path: client.url('containers',container))
+          end
+          logger.info("/var/lib/apt/lists is empty, update is required ( disable update with --apt-update=never )")
+          return true
+        when 'always'
+          return true
+        when 'never'
+          return false
+        end
+      else
+        return false
+      end
+    end
+
+    def build!
+      res = client.post(
+         headers: {
+          'Content-Type' => 'application/json'
+         },
+         path: client.url('containers','create'),
+         expects: [201],
+         body: JSON.generate({"Image" => build_image})
+      )
+
+      body = JSON.parse(res.body)
+      container = body['Id']
+      begin
+        client.post(
+          headers: {
+            'Content-Type' => 'application/json'
+          },
+          path: client.url('containers',container,'start'),
+          expects: [204],
+          body: JSON.generate({})
+        )
+
+        client.post(
+          path: client.url('containers',container,'attach?stderr=1&stdout=1&stream=1'),
+          body: '',
+          expects: [200],
+          middlewares: [
+            StreamParser.new(out,err),
+            Excon::Middleware::Expects,
+            Excon::Middleware::Instrumentor,
+            Excon::Middleware::Mock
+          ]
+        )
+
+        res = client.post(
+          path: client.url('containers',container,'wait'),
+          expects: [200],
+          body: ''
+        )
+        json = JSON.parse(res.body)
+        if json["StatusCode"] != 0
+          raise "Build failed"
+        end
+        return yield container
+      ensure
+        unless keep?
+          client.delete(path: client.url('containers',container))
+        end
+      end
+    end
+
+    def input_package(container)
+      input = FPM::Package::Docker.new(logger: logger, client: client)
+      builder.recipe.apply_input(input)
+      begin
+        input.input(container)
+        return yield(input)
+      ensure
+        input.cleanup_staging
+        input.cleanup_build
+      end
+    end
+
+    def write_output!(output)
+      package_file = File.expand_path(output.to_s(nil))
+      FileUtils.mkdir_p(File.dirname(package_file))
+      tmp_package_file = package_file + '.tmp'
+      begin
+        FileUtils.rm_rf tmp_package_file
+      rescue Errno::ENOENT
+      end
+
+      output.output(tmp_package_file)
+
+      begin
+        FileUtils.rm_rf package_file
+      rescue Errno::ENOENT
+      end
+      File.rename tmp_package_file, package_file
+
+      logger.info("Created package", :path => package_file)
+    end
+
+    def packages
+      dir_map = []
+      out_map = {}
+
+      package_map = builder.recipe.packages.map do | package |
+        output = output_class.new
+        output.instance_variable_set(:@logger,logger)
+        package.files.each do | pattern |
+          dir_map << [ pattern, output.staging_path ]
+        end
+        out_map[ output ] = package
+      end
+
+      dir_map = Hash[ dir_map.reverse ]
+
+      yield dir_map
+
+      out_map.each do |output, package|
+        package.apply_output(output)
+      end
+
+      out_map.each do |output, _|
+        write_output!(output)
+      end
+
+    ensure
+
+      out_map.each do |output, _|
+        output.cleanup_staging
+        output.cleanup_build
+      end
+
+    end
+
+
+  public
+
+    def execute
+      # force some eager loading
+      lint_recipe_file!
+      detector
+      flavour
+      output_class
+      lint_output_class!
+      builder
+      lint_recipe!
+      cache
+
+      image_id
+      build_image
+
+      packages do | dir_map |
+
+        build! do |container|
+          input_package(container) do |input|
+            input.split( container, dir_map )
+          end
+        end
+
+      end
+
+      return 0
+    rescue Recipe::NotFound => e
+      logger.error("Recipe not found", recipe: recipe, exeception: e)
+      return 1
+    rescue => e
+      logger.error(e)
+      return 1
+    end
+
+  end
+
+  class Command
+    subcommand 'cook', 'Cooks a package', Cook
+  end
+end ; end