baps 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 26742e6633e6fd17a446096d5205ea9d845fa4f0
+  data.tar.gz: 89c927289e60d5e8d017dfa929431a03d4b24088
+SHA512:
+  metadata.gz: 63acc006bf8feec859a0edc6d407e59556daefc73446d655d45fe7f39afeb2be96337378fc16a0e2f0aecc4352d6198512450fdbd5286cc9474ec8a8e6a53c93
+  data.tar.gz: 08977962a992b220517cba693876ae2b0847d97d18a5aa2f16bdeef27bb9c3476c6911ec32aa3d2b54975c42efbece0a32e79df3386c5c8f14ac947aec69e4a5

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.ruby-version

@@ -0,0 +1 @@
+2.3.1

data/Dockerfile

@@ -0,0 +1,8 @@
+FROM ruby:2.3
+
+# throw errors if Gemfile has been modified since Gemfile.lock
+RUN bundle config --global frozen 1
+
+COPY . /app
+WORKDIR /app
+RUN bundle install

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test, :development do
+  gem 'pry'
+end

data/README.md

@@ -0,0 +1,32 @@
+# BAPS - Build and Packaging System
+Define your build using provided DSL and `baps` will take care of building
+the given software in a clean room environment, package it as `deb` and push to packagecloud.
+
+## Usage
+`exe/baps -p ./definitions`
+This will go through every definition file in the given `./definitions` folder and if there's at least
+one definition that needs to be executed `baps` will pull prebuilt Docker image, copy the content of `./definitions`
+folder to it and fire a container based on the image for each definition needed to be executed. Inside the container
+`baps` will execute the given definition using `exe/baps -p ./definitions/path_to_definition_file.rb`
+
+If you don't want a clean room environment i.e not execute your definition in Docker you can directly use `exe/baps --no-docker -p ./definitions/path_to_definition_file.rb`.
+
+## Example
+An example definition to build `pmtud` from a remote git repository:
+```
+deb do
+  name 'pmtud'
+  version '0.7'
+  dependencies %w(libpcap0.8 libnetfilter-log1)
+  description 'Path MTU daemon - broadcast lost ICMP packets on ECMP networks'
+
+  build do
+    source git: 'https://github.com/Shopify/pmtud.git', branch: "v#{@version}"
+    dependencies %w(libnetfilter-log-dev libnfnetlink-dev libpcap-dev)
+    run 'make pmtud BUILD=release CC=gcc'
+    run "install -d 0700 #{deb_dir}/usr/local/bin"
+    run "install pmtud #{deb_dir}/usr/local/bin"
+  end
+end
+```
+More examples can be found at https://github.com/Shopify/shopify-open-source-builds

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/baps.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'baps/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "baps"
+  spec.version       = Baps::VERSION
+  spec.authors       = ["Elvin Efendi"]
+  spec.email         = ["elvin.efendiyev@gmail.com"]
+
+  spec.summary       = %q{Build and Packaging System Edit.}
+  spec.description   = %q{Define your build using provided DSL and baps will take care of building the given software in a clean room environment, package it as deb and push to packagecloud.}
+  spec.homepage      = ""
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'fpm'
+  spec.add_runtime_dependency 'packagecloud-ruby'
+  spec.add_runtime_dependency 'docker-api'
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'bundler/setup'
+require 'baps'
+require 'pry'
+
+binding.pry

data/circle.yml

@@ -0,0 +1,15 @@
+machine:
+  services:
+    - docker
+
+test:
+  override:
+    - bundle exec rake test
+
+deployment:
+  production:
+    branch: master
+    commands:
+      - docker build --rm=false -t elvinefendi/baps .
+      - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - docker push elvinefendi/baps

data/exe/baps

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'bundler/setup'
+require 'baps'
+require 'baps/executor'
+require 'optparse'
+
+options = { use_docker: true }
+OptionParser.new do |opts|
+  opts.banner = 'Usage: exe/baps [options]. Either -d or -f has to be provided.'
+
+  opts.on('-p', '--path PATH', 'Path to a directory with definitions or absolute path of definition file') do |v|
+    options[:path] = v
+  end
+
+  opts.on("--no-docker", "Path to a definition file") do
+    options[:use_docker] = false
+  end
+end.parse!
+
+executor = Baps::Executor.new(options)
+executor.execute

data/lib/baps.rb

@@ -0,0 +1,24 @@
+require 'baps/build'
+require 'baps/command'
+require 'baps/deb'
+require 'baps/definition'
+require 'baps/executor'
+require 'baps/packagecloud_api'
+require 'baps/version'
+
+require 'docker'
+
+module Baps
+  extend self
+
+  TIMEOUT = 5*60
+  Excon.defaults[:write_timeout] = TIMEOUT
+  Excon.defaults[:read_timeout] = TIMEOUT
+
+  class ChecksumError < StandardError; end
+  class ExecutorError < StandardError; end
+
+  def packagecloud
+    @@packagecloud ||= PackagecloudAPI.new('shopify', ENV['PACKAGECLOUD_TOKEN'])
+  end
+end

data/lib/baps/build.rb

@@ -0,0 +1,82 @@
+require 'tmpdir'
+
+module Baps
+  class Build
+    def initialize(name, version, &block)
+      raise StandardError.new('Nothing defined') unless block_given?
+      @name = name
+      @version = version
+      @commands = []
+      instance_eval(&block)
+      @commands.unshift(Command.new("apt-get install -y -q #{dependencies.join(' ')}")) if dependencies
+    end
+
+    def start
+      Dir.chdir(dir) do
+        unpack_source
+        @commands.each(&:execute)
+      end
+    end
+
+    def deb_dir
+      @deb_dir ||= File.join(dir, "debian-#{Time.now.utc.to_i}")
+    end
+
+    def dir
+      @dir ||= Dir.mktmpdir("baps-#{@name}-#{@version}-")
+    end
+
+    private
+
+    def unpack_source
+      return unless source
+      if source[:git]
+        Command.new("git clone #{source[:git]} ./").execute
+        if source[:branch]
+          Command.new("git reset --hard #{source[:branch]}").execute
+        end
+      elsif source[:remote_tar]
+        path = "#{@name}-#{@version}.tar.gz"
+        IO.copy_stream(open(source[:remote_tar]), path)
+        content = File.read(path)
+        if sha256sum
+          raise ChecksumError, 'sha256sum cheksum did not match.' unless Digest::SHA256.hexdigest(content) == sha256sum
+        end
+        if md5sum
+          raise ChecksumError, 'md5sum cheksum did not match.' unless Digest::MD5.hexdigest(content) == md5sum
+        end
+        unpack_tar(path)
+      elsif source[:local_tar]
+        unpack_tar(File.join('baps-definitions', "#{@name}-#{@version}.tar.gz"))
+      end
+    end
+
+    def run(cmd)
+      @commands << Command.new(cmd)
+    end
+
+    def unpack_tar(path)
+      cmd = Command.new("tar -tf #{path}")
+      cmd.execute
+      dirs = cmd.stdout.split("\n")
+      have_root_dir = true
+      i = 0
+      while have_root_dir && i < dirs.size - 1
+        curr_dir = dirs[i].split('/').first
+        next_dir = dirs[i+1].split('/').first
+        have_root_dir = (curr_dir == next_dir)
+        i +=1
+      end
+      Command.new("tar -xzf #{path} -C . --strip #{have_root_dir ? 1 : 0}").execute
+    ensure
+      FileUtils.rm(path) if File.exist?(path)
+    end
+
+    %i(source dependencies sha256sum md5sum).each do |method|
+      define_method(method) do |value = nil|
+        return instance_variable_get("@#{method}") if value.nil?
+        instance_variable_set("@#{method}", value)
+      end
+    end
+  end
+end

data/lib/baps/command.rb

@@ -0,0 +1,53 @@
+require 'open3'
+
+module Baps
+  class Command
+    attr_reader :command, :exit_status, :stdout, :stderr
+
+    def initialize(command)
+      @command = command
+      @stdout, @stderr = String.new, String.new
+    end
+
+    def on_stdout(data)
+      @stdout += data
+    end
+
+    def on_stderr(data)
+      @stderr += data
+    end
+
+    def exit_status=(value)
+      @exit_status = value
+      if @exit_status > 0
+        message = ''
+        message += "#{command} exit status: #{@exit_status}\n"
+        message += "#{command} stdout: " + (stdout.strip.empty? ? "Nothing written" : stdout.strip) + "\n"
+        message += "#{command} stderr: " + (stderr.strip.empty? ? 'Nothing written' : stderr.strip) + "\n"
+        raise StandardError, message
+      end
+    end
+
+    def execute
+      puts "Executing: #{command}"
+      Open3.popen3(command) do |_stdin, stdout, stderr, wait_thr|
+        stdout_thread = Thread.new do
+          while (line = stdout.gets) do
+            on_stdout(line)
+          end
+        end
+
+        stderr_thread = Thread.new do
+          while (line = stderr.gets) do
+            on_stderr(line)
+          end
+        end
+
+        stdout_thread.join
+        stderr_thread.join
+
+        self.exit_status = wait_thr.value.to_i
+      end
+    end
+  end
+end

data/lib/baps/deb.rb

@@ -0,0 +1,49 @@
+module Baps
+  class Deb
+    def initialize(&block)
+      raise StandardError.new('Nothing defined') unless block_given?
+      instance_eval(&block)
+    end
+    
+    def already_exist?
+      Baps.packagecloud.package_exist?(name: name, version: version)
+    end
+
+    def create
+      build.start
+      package
+      publish
+    end
+
+    private
+
+    def publish
+      file = Dir['*.deb'].first
+      Baps.packagecloud.publish_package(file)
+    end
+
+    def package
+      command = "fpm -s dir -C #{build.deb_dir} -v #{version} -t deb "\
+        "--description '#{description}' --vendor '' "\
+        "-n #{name} --deb-priority #{priority or 'optional'} --deb-compression bzip2"
+      command << " #{dependencies.map { |d| "-d '#{d}'" }.join(' ')}" if dependencies
+      command << " --replaces '#{replaces}'" if replaces
+      command << " #{targets.join(' ')}" if targets
+      # TODO use programming API
+      # as described in https://github.com/jordansissel/fpm/blob/master/examples/api/dir-to-deb-rpm-with-init.rb
+      Command.new(command).execute
+    end
+
+    def build(&block)
+      return @build unless block_given?
+      @build = Build.new(name, version, &block)
+    end
+
+    %i(name version dependencies description targets priority replaces).each do |method|
+      define_method(method) do |value = nil|
+        return instance_variable_get("@#{method}") if value.nil?
+        instance_variable_set("@#{method}", value)
+      end
+    end
+  end
+end

data/lib/baps/definition.rb

@@ -0,0 +1,26 @@
+module Baps
+  class Definition
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+      instance_eval(File.read(@path), @path)
+    end
+
+    def execute
+      puts "Executing #{@path}"
+      deb.create
+    end
+
+    def executed?
+      deb.already_exist?
+    end
+
+    private
+
+    def deb(&block)
+      return @deb unless block_given?
+      @deb = Deb.new(&block)
+    end
+  end
+end

data/lib/baps/executor.rb

@@ -0,0 +1,66 @@
+module Baps
+  class Executor
+    DOCKER_IMAGE_ID = 'elvinefendi/baps'
+
+    def initialize(path:, use_docker: true)
+      raise ArgumentError.new('Given path does not exist.') unless File.exist?(path)
+      @path = File.expand_path(path)
+      @use_docker = use_docker
+      @definitions = definitions_from_directory if File.directory?(@path)
+      @definitions = [Definition.new(@path)] if File.file?(@path)
+    end
+
+
+    def execute
+      if !@definitions || @definitions.empty?
+        puts 'No definition to execute'
+        return
+      end
+
+      return @definitions.each(&:execute) unless @use_docker
+
+      errors = {}
+      @definitions.each do |definition|
+        container = base_container.run("exe/baps -p #{path_in_docker} --no-docker")
+        container.attach { |stream, chunk| puts "#{stream}: #{chunk}" }
+        if (code = container.wait(Baps::TIMEOUT)['StatusCode']) > 0
+          errors[definition] = code
+        end
+      end
+      unless errors.empty?
+        raise ExecutorError, errors.map { |d, c| "#{d.path} exited with #{c}" }.join("\n")
+      end
+    end
+
+    private
+
+    def base_container
+      return @base_container if @base_container
+      @base_container = image.run('apt-get update -y -q', 'Env' => ["PACKAGECLOUD_TOKEN=#{ENV['PACKAGECLOUD_TOKEN']}"])
+      @base_container.attach { |stream, chunk| puts "#{stream}: #{chunk}" }
+      @base_container
+    end
+
+    def image
+      return @image if @image
+      begin
+        @image = Docker::Image.get(DOCKER_IMAGE_ID)
+      rescue Docker::Error::NotFoundError
+        @image = Docker::Image.create('fromImage' => DOCKER_IMAGE_ID)
+      end
+      @image = @image.insert_local('localPath' => @path, 'outputPath' => path_in_docker)
+    end
+
+    def path_in_docker
+      "/baps-definitions/#{File.basename(@path)}"
+    end
+
+    def definitions_from_directory
+      Dir[File.join(@path, '**/*.rb')].map do |path|
+        definition = Definition.new(path)
+        next if definition.executed?
+        definition
+      end.compact
+    end
+  end
+end

data/lib/baps/packagecloud_api.rb

@@ -0,0 +1,28 @@
+require 'packagecloud'
+
+module Baps
+  class PackagecloudAPI
+    def initialize(username, token)
+      connection = Packagecloud::Connection.new('https', 'packages.shopify.io')
+      credentials = Packagecloud::Credentials.new(username, token)
+      @client = Packagecloud::Client.new(credentials, "packagecloud-ruby #{Packagecloud::VERSION}", connection)
+    end
+
+    def publish_package(file)
+      package = Packagecloud::Package.new(file: file)
+      result = @client.put_package('public', package, 'ubuntu/trusty')
+      if result.succeeded
+        puts "Package #{file} has been pushed to packagecloud"
+      else
+        puts "Could not push #{file} to packagecoud: #{result.response}"
+      end
+    end
+
+    def package_exist?(name:, version:, arch: 'amd64', type: 'deb')
+      url = "/api/v1/repos/shopify/public/package/#{type}/ubuntu/trusty/#{name}/#{arch}/#{version}/stats/downloads/count.json"
+      json = @client.send(:get, url)
+      result = @client.send(:parsed_json_result, json)
+      result.succeeded
+    end
+  end
+end

data/lib/baps/version.rb

@@ -0,0 +1,3 @@
+module Baps
+  VERSION = '0.1.1'
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: baps
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Elvin Efendi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-01-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fpm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: packagecloud-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Define your build using provided DSL and baps will take care of building
+  the given software in a clean room environment, package it as deb and push to packagecloud.
+email:
+- elvin.efendiyev@gmail.com
+executables:
+- baps
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- baps.gemspec
+- bin/console
+- circle.yml
+- exe/baps
+- lib/baps.rb
+- lib/baps/build.rb
+- lib/baps/command.rb
+- lib/baps/deb.rb
+- lib/baps/definition.rb
+- lib/baps/executor.rb
+- lib/baps/packagecloud_api.rb
+- lib/baps/version.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Build and Packaging System Edit.
+test_files: []