linecook-gem 0.6.10 → 0.7.0

data/lib/linecook-gem/config.rb

@@ -0,0 +1,45 @@
+require 'yaml'
+require 'json'
+
+module Linecook
+  def self.config
+    config = Config.config
+    config
+  end
+
+
+  module Config
+    extend self
+
+    CONFIG_PATH = File.join(Dir.pwd, 'linecook.yml').freeze
+    SECRETS_PATH = File.join(Dir.pwd, 'secrets.ejson').freeze
+    LINECOOK_HOME = File.expand_path('~/.linecook').freeze
+    DEFAULT_CONFIG_PATH = File.join(LINECOOK_HOME, 'config.yml').freeze
+
+    def config
+      @config ||= begin
+        config_path = ENV['LINECOOK_CONFIG_PATH'] || CONFIG_PATH
+        config = {}
+        config ||= YAML.load(File.read(DEFAULT_CONFIG_PATH)) if File.exist?(DEFAULT_CONFIG_PATH)
+        config.deep_merge!(YAML.load(File.read(config_path))) if File.exist?(config_path)
+        (config || {}).deep_symbolize_keys!
+        config.deep_merge!(secrets)
+      end
+    end
+
+  private
+    def secrets
+      @secrets ||= begin
+        secrets_path = ENV['LINECOOK_SECRETS_PATH'] || SECRETS_PATH
+        if File.exists?(secrets_path)
+          ejson_path = File.join(Gem::Specification.find_by_name('ejson').gem_dir, 'build', 'linux-amd64', 'ejson' )
+          command = "#{ejson_path} decrypt #{secrets_path}"
+          secrets = JSON.load(`sudo #{command}`)
+          secrets.deep_symbolize_keys
+        else
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/linecook-gem/image.rb

@@ -0,0 +1,77 @@
+require 'fileutils'
+require 'tmpdir'
+
+require 'linecook-gem/image/s3'
+require 'linecook-gem/image/crypt'
+require 'linecook-gem/util/downloader'
+
+module Linecook
+  class Image
+
+    include Downloader
+    include Crypto
+
+    IMAGE_PATH = File.join(Config::LINECOOK_HOME, 'images').freeze
+
+    attr_reader :path, :id, :name, :group, :tag
+
+
+    def initialize(name, group, tag)
+      @name = name
+      @group = group ? "#{name}-#{group.gsub(/-|\//,'_')}" : name
+      @tag = tag
+      @id = if @tag == 'latest'
+        id = File.basename(latest).split('.')[0]
+        @tag = id.split('-').last
+        id
+      elsif tag
+        "#{@group}-#{@tag}"
+      else
+        @group
+      end
+      @path = image_path
+    end
+
+    def fetch
+      return if File.exists?(@path)
+
+      Dir.mktmpdir("#{@id}-download") do |tmpdir|
+        tmppath = File.join(tmpdir, File.basename(@path))
+        download(url, tmppath)
+        FileUtils.mkdir_p(File.dirname(@path))
+        decrypt(tmppath, dest: @path)
+        FileUtils.rm_f(tmppath)
+      end
+
+    end
+
+    def upload
+      puts "Encrypting and uploading image #{@path}"
+      encrypted = encrypt(@path)
+      provider.upload(encrypted, group: @group)
+      FileUtils.rm_f(encrypted)
+    end
+
+    def url
+      provider.url(@id, group: @group)
+    end
+
+    def list
+      provider.list(group: @group)
+    end
+
+    def latest
+      provider.latest(@group)
+    end
+
+  private
+
+    def image_path
+      File.join([IMAGE_PATH, @group, "#{@id}.tar.xz"].compact)
+    end
+
+    def provider
+      S3Manager
+    end
+  end
+end

data/lib/linecook-gem/image/crypt.rb

@@ -1,60 +1,27 @@
-require 'tempfile'
-
 require 'rbnacl/libsodium'
 
-require 'linecook-gem/image/manager'
-require 'linecook-gem/util/executor'
-require 'linecook-gem/util/config'
-
 module Linecook
-  class Crypto
-    include Executor
-
-    def initialize(remote: nil)
-      @remote = remote
-      load_key
-    end
+  module Crypto
 
-    def encrypt_image(image)
-      image_path = File.join(Linecook::ImageManager::IMAGE_PATH,File.basename(image))
-      encrypt_file(image_path)
+    def self.keygen
+       RbNaCl::Random.random_bytes(RbNaCl::SecretBox.key_bytes).unpack('H*').first
     end
 
-    def encrypt_file(source, dest: nil)
+    def encrypt(source, dest: nil)
       dest ||= "/tmp/#{File.basename(source)}"
       File.write(dest, box.encrypt(IO.binread(source)))
       dest
     end
 
-    def decrypt_file(source, dest: nil)
+    def decrypt(source, dest: nil)
       dest ||= "/tmp/#{File.basename(source)}-decrypted"
-      if @remote
-        script = "/tmp/decrypt-#{SecureRandom.hex(4)}"
-        @remote.upload(decryptor_script(source, dest), script)
-        @remote.run("bash #{script}")
-        @remote.run("rm #{script}")
-      else
-        File.write(dest, box.decrypt(IO.binread(source)))
-      end
+      File.write(dest, box.decrypt(IO.binread(source)))
       dest
     end
 
-    def self.keygen
-       RbNaCl::Random.random_bytes(RbNaCl::SecretBox.key_bytes).unpack('H*').first
-    end
-
   private
-
-    def decryptor_script(source, dest)
-      "ruby -e \"require 'rbnacl/libsodium'; box = RbNaCl::SimpleBox.from_secret_key(['#{@secret_key}'].pack('H*')); File.write('#{dest}', box.decrypt(File.read('#{source}')))\""
-    end
-
     def box
-      @box ||= RbNaCl::SimpleBox.from_secret_key([@secret_key].pack('H*'))
-    end
-
-    def load_key
-      @secret_key = Linecook.config[:imagekey]
+      @box ||= RbNaCl::SimpleBox.from_secret_key([Linecook.config[:imagekey]].pack('H*'))
     end
   end
 end

data/lib/linecook-gem/image/s3.rb

@@ -5,29 +5,29 @@ module Linecook
   module S3Manager
     extend self
     EXPIRY = 20
-    PREFIX = 'builds'
+    PREFIX = 'built-images'
 
-    def url(name, type: nil)
+    def url(id, group: nil)
       client
       s3 = Aws::S3::Resource.new
-      obj = s3.bucket(Linecook.config[:aws][:s3][:bucket]).object(File.join([PREFIX, type, name].compact))
+      obj = s3.bucket(Linecook.config[:aws][:s3][:bucket]).object(File.join([PREFIX, group, "#{id}.tar.xz"].compact))
       obj.presigned_url(:get, expires_in: EXPIRY * 60)
     end
 
-    def list(type: nil)
-      list_objects(type: type).map{ |x| x.key if x.key =~ /squashfs$/ }.compact
+    def list(group: nil)
+      list_objects(group: group).map{ |x| x.key if x.key =~ /\.tar\.xz/ }.compact
     end
 
-    def latest(type)
-      objects = list_objects(type: type).sort! { |a,b| a.last_modified <=> b.last_modified }
+    def latest(group)
+      objects = list_objects(group: group).sort! { |a,b| a.last_modified <=> b.last_modified }
       key = objects.last ? objects.last.key : nil
     end
 
-    def upload(path, type: nil)
+    def upload(path, group: nil)
       File.open(path, 'rb') do |file|
-        fname = File.basename(path)
-        pbar = ProgressBar.create(title: fname, total: file.size)
-        common_opts = { bucket: Linecook.config[:aws][:s3][:bucket], key: File.join([PREFIX, type, fname].compact) }
+        fid = File.basename(path)
+        pbar = ProgressBar.create(title: fid, total: file.size)
+        common_opts = { bucket: Linecook.config[:aws][:s3][:bucket], key: File.join([PREFIX, group, fid].compact) }
         resp = client.create_multipart_upload(storage_class: 'REDUCED_REDUNDANCY', server_side_encryption: 'AES256', **common_opts)
         id = resp.upload_id
         part = 0
@@ -39,7 +39,7 @@ module Linecook
           parts << { etag: resp.etag, part_number: part }
           total += content.length
           pbar.progress = total
-          pbar.title = "#{fname} - (#{((total.to_f/file.size.to_f)*100.0).round(2)}%)"
+          pbar.title = "#{fid} - (#{((total.to_f/file.size.to_f)*100.0).round(2)}%)"
         end
         client.complete_multipart_upload(upload_id: id, multipart_upload: { parts: parts }, **common_opts)
       end
@@ -47,8 +47,8 @@ module Linecook
 
   private
 
-    def list_objects(type: nil)
-      client.list_objects(bucket: Linecook.config[:aws][:s3][:bucket], prefix: File.join([PREFIX, type].compact)).contents
+    def list_objects(group: nil)
+      client.list_objects(bucket: Linecook.config[:aws][:s3][:bucket], prefix: File.join([PREFIX, group].compact)).contents
     end
 
     def client

data/lib/linecook-gem/packager.rb

@@ -0,0 +1,30 @@
+
+require 'linecook-gem/image'
+require 'linecook-gem/packager/packer'
+require 'linecook-gem/packager/squashfs'
+require 'kitchen/configurable'
+
+
+module Linecook
+  module Packager
+    extend self
+
+    def package(image, name: 'packer')
+      image.fetch
+      provider(name.to_sym).package(image)
+    end
+
+  private
+    def provider(name)
+      config = Linecook.config[:packager][name]
+      case name
+      when :packer
+        Linecook::AmiPacker.new(**config)
+      when :squashfs
+        Linecook::Squashfs.new(**config)
+      else
+        fail "No packager implemented for for #{name}"
+      end
+    end
+  end
+end

data/lib/linecook-gem/packager/packer.rb

@@ -0,0 +1,249 @@
+require 'json'
+require 'mkmf'
+require 'fileutils'
+require 'open-uri'
+require 'tempfile'
+require 'tmpdir'
+require 'pty'
+
+require 'linecook-gem/image'
+require 'linecook-gem/util/downloader'
+require 'linecook-gem/packager/route53'
+
+module Linecook
+  class AmiPacker
+
+    include Downloader
+
+    SOURCE_URL = 'https://releases.hashicorp.com/packer/'
+    PACKER_VERSION = '0.8.6'
+    PACKER_PATH = File.join(Linecook::Config::LINECOOK_HOME, 'bin', 'packer')
+
+    BUILDER_CONFIG = {
+      type: 'amazon-chroot',
+      access_key: '{{ user `aws_access_key` }}',
+      secret_key: '{{ user `aws_secret_key` }}',
+      source_ami: "{{user `source_ami`}}",
+      ami_name: 'packer-image.{{ user `image_name` }} {{timestamp}}',
+      device_path: '/dev/{{ user `ebs_device` }}'
+    }.freeze
+
+    PROVISIONER_COMMANDS = [
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/proc/sys/fs/binfmt_misc',
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/proc',
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/sys',
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/dev/pts',
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/dev',
+      'mv /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/etc/network /tmp/{{ user `ebs_device`}}-network',
+      'umount /dev/{{ user `ebs_device` }}1',
+      'mkfs.ext4 /dev/{{ user `ebs_device` }}1',
+      'tune2fs -L cloudimg-rootfs /dev/{{ user `ebs_device` }}1',
+      'mount /dev/{{ user `ebs_device` }}1 /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}',
+      'tar -C /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }} -xpf {{ user `source_image_path` }}',
+      'cp /etc/resolv.conf /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/etc',
+      'echo "LABEL=cloudimg-rootfs   /        ext4   defaults,discard        0 0" > /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/etc/fstab',
+      'mount -t proc none /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/proc',
+      'mount -o bind /sys /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/sys',
+      'mount -o bind /dev /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/dev',
+      # Sadly we need to install grub inside the image, and this implementation is Ubunut specific. This can be patched eventually when needed
+      'chroot /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }} apt-get update',
+      'chroot /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }} apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y --force-yes --no-upgrade install grub-pc grub-legacy-ec2',
+      'chroot /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }} update-grub',
+      'grub-install --root-directory=/mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }} /dev/{{ user `ebs_device` }}',
+      'rm -rf /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/etc/network',
+      'mv /tmp/{{ user `ebs_device`}}-network /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/etc/network',
+      'rm -f /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/etc/init/fake-container-events.conf', # HACK
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/proc',
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/sys',
+      'umount /mnt/packer-amazon-chroot-volumes/{{ user `ebs_device` }}/dev'
+    ]
+
+    def initialize(config)
+      system("#{packer_path} --version")
+      @hvm = config[:hvm] || true
+      @root_size = config[:root_size] || 10
+      @region = config[:region] || 'us-east-1'
+      @copy_regions = config[:copy_regions] || []
+      @accounts = config[:account_ids] || []
+      @source_ami = config[:source_ami] || find_ami
+      @write_txt = Linecook.config[:packager] && Linecook.config[:packager][:ami] && Linecook.config[:packager][:ami][:update_txt]
+    end
+
+    def package(image)
+      @image = image
+      conf_file = Tempfile.new("#{@image.id}-packer.json")
+      config = generate_config
+      conf_file.write(config)
+      conf_file.close
+      output = []
+      PTY.spawn("sudo #{PACKER_PATH} build -machine-readable #{conf_file.path}") do |stdout, _, _|
+        begin
+          stdout.each do |line|
+            output << line if line =~ /artifact/
+            tokens = line.split(',')
+            if tokens.length > 4
+              out = tokens[4].gsub('%!(PACKER_COMMA)', ',')
+              time = DateTime.strptime(tokens[0], '%s').strftime('%c')
+              puts "#{time} | #{out}"
+            else
+              puts "unexpected output format"
+              puts tokens
+            end
+          end
+        rescue Errno::EIO
+          puts "Packer finshed executing"
+        end
+      end
+      extract_amis_from_output(output)
+    ensure
+      conf_file.close
+      conf_file.unlink
+    end
+
+  private
+
+    # TO DO:
+    # support for multiple accounts, multiple regions
+    # code to extract ami name(s) from output
+    #    amis = `grep "amazon-ebs,artifact,0,id" packer.log`.chomp.split(',')[5].split('%!(PACKER_COMMA)')
+    # route53 TXT record integration
+
+    def generate_config
+      config = {
+        variables: {
+          aws_access_key: Linecook.config[:aws][:access_key],
+          aws_secret_key: Linecook.config[:aws][:secret_key],
+          ebs_device: free_device,
+          source_ami: @source_ami,
+          image_name: "linecook-#{@image.id}",
+          source_image_path: @image.path
+        },
+        builders: [
+          BUILDER_CONFIG.merge(
+            ami_users: @accounts,
+            ami_regions: @copy_regions,
+            ami_virtualization_type: virt_type,
+            root_volume_size: @root_size
+          )
+        ],
+        provisioners: build_provisioner(PROVISIONER_COMMANDS)
+      }
+      JSON.pretty_generate(config)
+    end
+
+    def build_provisioner(commands)
+      provisioner = []
+      commands.each do |cmd|
+        provisioner << {
+          type: 'shell-local',
+          command: cmd
+        }
+      end
+      provisioner
+    end
+
+    def packer_path
+      @path ||= begin
+        found = File.exists?(PACKER_PATH) ? PACKER_PATH : find_executable('packer')
+        path = if found
+          version = `#{found} --version`
+          Gem::Version.new(version) >= Gem::Version.new(PACKER_VERSION) ? found : nil
+        end
+        path ||= get_packer
+      end
+    end
+
+    def get_packer
+      puts "packer too old (<#{PACKER_VERSION}) or not present, getting latest packer"
+      arch = 1.size == 8 ? 'amd64' : '386'
+      path = File.join(File.dirname(PACKER_PATH), 'packer.zip')
+      url = File.join(SOURCE_URL, PACKER_VERSION, "packer_#{PACKER_VERSION}_linux_#{arch}.zip")
+      download(url, path)
+      unzip(path)
+      PACKER_PATH
+    end
+
+    def free_device
+      lock('device_scan')
+      free = nil
+      prefix = device_prefix
+      ('f'..'zzz').to_a.each do |suffix|
+        device = "#{prefix}#{suffix}"
+        if free_device?(device)
+          lock(device)
+          at_exit do
+            clear_lock(device)
+          end
+          free = device
+          break
+        end
+      end
+      unlock('device_scan')
+      return free
+    end
+
+    def free_device?(device)
+      !File.exists?("/dev/#{device}") && !File.exists?(lock_path(device))
+    end
+
+    def lock(name)
+      lockfile(name).flock(File::LOCK_EX)
+    end
+
+    def unlock(name)
+      lockfile(name).flock(File::LOCK_UN)
+      lockfile(name).close
+    end
+
+    def clear_lock(name)
+      unlock(name)
+      FileUtils.rm_f(lockfile(name))
+    end
+
+    def lockfile(suffix)
+      @locks ||= {}
+      path = lock_path(suffix)
+      @locks[path] = @locks[path] || File.open(path, File::RDWR|File::CREAT, 0644)
+    end
+
+    def lock_path(suffix)
+      "/tmp/free_device_lock_#{suffix.gsub('/','_')}"
+    end
+
+    def device_prefix
+      prefixes = ['xvd', 'sd']
+      `sudo ls -1 /sys/block`.lines.each do |dev| # FIXME
+        prefixes.each do |prefix|
+          return prefix if dev =~ /^#{prefix}/
+        end
+      end
+      return prefixes.first
+    end
+
+    def extract_amis_from_output(output)
+      amis = output.grep(/amazon-chroot,artifact,0,id/).first.chomp.split(',')[5].split('%!(PACKER_COMMA)')
+      amis.each do |info_str|
+        ami_info = info_str.split(':')
+        ami_region = ami_info[0]
+        ami_id = ami_info[1]
+        puts "Built #{ami_id} for #{ami_region}"
+        Linecook::Route53.upsert_record(@image.name, ami_id, ami_region) if @write_txt
+      end
+    end
+
+    def virt_type
+      @hvm ? 'hvm' : 'paravirtual'
+    end
+
+    def find_ami
+      url = "http://uec-images.ubuntu.com/query/trusty/server/released.current.txt"
+      data = open(url).read.split("\n").map{|l| l.split}.detect do |ary|
+        ary[4] == 'ebs' &&
+          ary[5] == 'amd64' &&
+          ary[6] == @region &&
+          ary.last == virt_type
+      end
+      data[7]
+    end
+  end
+end