linecook-gem 0.7.35 → 0.7.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3cf921c31e786054ae969f8e6c1a90353d29425
-  data.tar.gz: dd9d3643c179ea74141cb2758b62274caff13d8b
+  metadata.gz: 4c609eabae7f16453cf3a38d0de4637acc199f84
+  data.tar.gz: 6406d3fe9102552f5ee59886f38cb55514245015
 SHA512:
-  metadata.gz: 530ee97b09e4364cd13f05c09dbd68e8e0f2e55025a9e0473ae2e3aebccc45548f7f31400897385cc0f4299c8e71724e1ed193c19cd813d74af1cabeee1e2b06
-  data.tar.gz: ff3bfd89ba139e6e3d0f43a3b5368e897d6e53d8cf4e57e3eb431f4b58591694dfa1c8d0279c9c4d5500fad6788e9f01af42af8ad43fedaed924fac0087ddd2e
+  metadata.gz: f3f8e661860c24c3933737251b06c2477646f184ca2d78af8b84acb2d208de4845793a8587875d7be5ef3172c37d03ed45d9ab0552d1dd535a05cce6b5ae1f5f
+  data.tar.gz: 6e1584143c306beb4d32f211080fd08d2ae70301f72e39087b6249d29d9220322b07806513695d7350933d72f93543f22b2eb561da25b2a095b3a9c8eb649d71

data/lib/linecook-gem/image/s3.rb

@@ -39,7 +39,7 @@ module Linecook
         fid = File.basename(path)
         pbar = ProgressBar.create(title: fid, total: file.size)
         common_opts = { bucket: Linecook.config[:aws][:s3][:bucket], key: File.join([PREFIX, group, fid].compact) }
-        resp = client.create_multipart_upload(storage_class: 'REDUCED_REDUNDANCY', server_side_encryption: 'AES256', **common_opts)
+        resp = client.create_multipart_upload(storage_class: 'STANDARD', server_side_encryption: 'AES256', **common_opts)
         id = resp.upload_id
         part = 0
         total = 0

data/lib/linecook-gem/packager/packer.rb

@@ -61,7 +61,10 @@ module Linecook
       'update-grub',
       'rm -f /etc/init/fake-container-events.conf', # HACK
       'mkdir -p /run/resolvconf/interface',
-      'DEBIAN_FRONTEND=noninteractive dpkg-reconfigure resolvconf' # re-linkify resolvconf
+      'echo "resolvconf resolvconf/linkify-resolvconf   boolean true" | debconf-set-selections', # write debconf
+      'dpkg-reconfigure -f noninteractive resolvconf', # re-linkify resolvconf
+      'truncate --size 0 /etc/resolv.conf', # clear build resolvconf config
+      'truncate --size 0 /etc/resolvconf/resolv.conf.d/original' # clear build resolvconf config
     ]
 
     def initialize(config)

data/lib/linecook-gem/packager/packer/ami.rb

@@ -0,0 +1,89 @@
+require 'linecook-gem/packager/packer/base'
+require 'linecook-gem/packager/route53'
+
+module Linecook
+  class AmiPacker < Linecook::Packer
+
+    ROOT_DEVICE_MAP = {
+      device_name: 'xvda',
+      delete_on_termination: true
+    }
+
+    BUILDER_CONFIG = {
+      type: 'amazon-chroot',
+      access_key: '{{ user `aws_access_key` }}',
+      secret_key: '{{ user `aws_secret_key` }}',
+      ami_name: 'packer-image.{{ user `image_name` }} {{timestamp}}',
+      from_scratch: true,
+      root_device_name: ROOT_DEVICE_MAP[:device_name],
+      ami_block_device_mappings: [ ROOT_DEVICE_MAP ],
+      pre_mount_commands: PRE_MOUNT_COMMANDS,
+      post_mount_commands: POST_MOUNT_COMMANDS,
+    }.freeze
+
+
+    def initialize(config)
+      system("#{packer_path} --version")
+      @hvm = config[:hvm] || true
+      @root_size = config[:root_size] || 10
+      @region = config[:region] || 'us-east-1'
+      @copy_regions = config[:copy_regions] || []
+      @accounts = config[:account_ids] || []
+      @write_txt = Linecook.config[:packager] && Linecook.config[:packager][:ami] && Linecook.config[:packager][:ami][:update_txt]
+    end
+
+    def package(image, directory)
+      super(image, directory)
+      extract_amis_from_output
+    end
+  private
+
+    # TO DO:
+    # support for multiple accounts, multiple regions
+    # code to extract ami name(s) from output
+    #    amis = `grep "amazon-ebs,artifact,0,id" packer.log`.chomp.split(',')[5].split('%!(PACKER_COMMA)')
+    # route53 TXT record integration
+
+    def generate_config(packager)
+      packager ||= {}
+      config = {
+        variables: {
+          aws_access_key: Linecook.config[:aws][:access_key],
+          aws_secret_key: Linecook.config[:aws][:secret_key],
+          image_name: "linecook-#{@image.id}",
+          source_image_path: @image.path
+        },
+        builders: [
+          BUILDER_CONFIG.merge(
+            ami_users: @accounts,
+            ami_regions: @copy_regions,
+            ami_virtualization_type: virt_type,
+            root_volume_size: @root_size
+          ).deep_merge(packager[:builder] || {})
+        ],
+        provisioners: build_provisioner(CHROOT_COMMANDS)
+      }
+
+      unless config[:builders].first[:ami_block_device_mappings].find { |x| x[:device_name] == ROOT_DEVICE_MAP[:device_name] }
+        config[:builders].first[:ami_block_device_mappings].prepend ROOT_DEVICE_MAP
+      end
+      JSON.pretty_generate(config)
+    end
+
+    def extract_amis_from_output
+      amis = @output.grep(/amazon-chroot,artifact,0,id/).first.chomp.split(',')[5].split('%!(PACKER_COMMA)')
+      amis.each do |info_str|
+        ami_info = info_str.split(':')
+        ami_region = ami_info[0]
+        ami_id = ami_info[1]
+        puts "Built #{ami_id} for #{ami_region}"
+        Linecook::Route53.upsert_record(@image.name, ami_id, ami_region) if @write_txt
+      end
+    end
+
+    def virt_type
+      @hvm ? 'hvm' : 'paravirtual'
+    end
+
+  end
+end

data/lib/linecook-gem/packager/packer/base.rb

@@ -0,0 +1,126 @@
+require 'json'
+require 'mkmf'
+require 'fileutils'
+require 'open-uri'
+require 'tempfile'
+require 'tmpdir'
+require 'pty'
+
+require 'linecook-gem/image'
+require 'linecook-gem/util/downloader'
+require 'linecook-gem/util/locking'
+require 'linecook-gem/util/common'
+
+module Linecook
+  class Packer
+
+    include Downloader
+    include Locking
+
+    SOURCE_URL = 'https://releases.hashicorp.com/packer/'
+    PACKER_VERSION = '0.12.0'
+    PACKER_PATH = File.join(Linecook::Config::LINECOOK_HOME, 'bin', 'packer')
+
+    PRE_MOUNT_COMMANDS = [
+      'parted -s {{.Device}} mklabel msdos',
+      'parted -s {{.Device}} mkpart primary ext2 0% 100%',
+      'mkfs.ext4 {{.Device}}1',
+      'tune2fs -L cloudimg-rootfs {{.Device}}1',
+    ]
+
+    POST_MOUNT_COMMANDS = [
+      'tar -C {{.MountPath}} -xpf {{ user `source_image_path` }}',
+      'cp /etc/resolv.conf {{.MountPath}}/etc',
+      'echo "LABEL=cloudimg-rootfs   /        ext4   defaults,discard        0 0" > {{.MountPath}}/etc/fstab',
+      'grub-install --root-directory={{.MountPath}} {{.Device}}',
+      'rm -rf {{.MountPath}}/etc/network',
+      'cp -r /etc/network {{.MountPath}}/etc/',
+    ]
+
+    CHROOT_COMMANDS = [
+      'apt-get update',
+      'DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y --force-yes --no-upgrade install grub-pc grub-legacy-ec2', # FIXME - ec2 specific
+      'update-grub',
+      'rm -f /etc/init/fake-container-events.conf', # HACK
+      'mkdir -p /run/resolvconf/interface',
+      'DEBIAN_FRONTEND=noninteractive dpkg-reconfigure resolvconf' # re-linkify resolvconf
+    ]
+
+    def initialize
+      raise 'Not implemented'
+    end
+
+    def package(image, directory)
+      @image = image
+      kitchen_config = load_config(directory).send(:data).instance_variable_get(:@data)
+      image_config = kitchen_config[:suites].find{ |x| x[:name] == image.name }
+      if image_config && image_config[:packager]
+        packager = image_config[:packager] || {}
+      end
+      conf_file = Tempfile.new("#{@image.id}-packer.json")
+      config = generate_config(packager)
+      conf_file.write(config)
+      conf_file.close
+      @output = []
+      PTY.spawn("sudo #{PACKER_PATH} build -machine-readable #{conf_file.path}") do |stdout, _, _|
+        begin
+          stdout.each do |line|
+            @output << line if line =~ /artifact/
+            tokens = line.split(',')
+            if tokens.length > 4
+              out = tokens[4].gsub('%!(PACKER_COMMA)', ',')
+              time = DateTime.strptime(tokens[0], '%s').strftime('%c')
+              puts "#{time} | #{out}"
+            else
+              puts "unexpected output format"
+              puts tokens
+            end
+          end
+        rescue Errno::EIO
+          puts "Packer finshed executing"
+        end
+      end
+    ensure
+      conf_file.close
+      conf_file.unlink
+    end
+
+  private
+
+    def generate_config(packager)
+      raise 'Not implemented'
+    end
+
+    def build_provisioner(chroot_commands)
+      provisioner = [
+        {
+          type: 'shell',
+          inline: chroot_commands
+        }
+      ]
+    end
+
+    def packer_path
+      @path ||= begin
+        found = File.exists?(PACKER_PATH) ? PACKER_PATH : find_executable('packer')
+        path = if found
+          version = `#{found} --version`
+          Gem::Version.new(version) >= Gem::Version.new(PACKER_VERSION) ? found : nil
+        end
+        path ||= get_packer
+      end
+    end
+
+    def get_packer
+      puts "packer too old (<#{PACKER_VERSION}) or not present, getting latest packer"
+      arch = 1.size == 8 ? 'amd64' : '386'
+
+      FileUtils.rm_f(Dir[File.join(File.dirname(PACKER_PATH), "*")])
+      path = File.join(File.dirname(PACKER_PATH), 'packer.zip')
+      url = File.join(SOURCE_URL, PACKER_VERSION, "packer_#{PACKER_VERSION}_linux_#{arch}.zip")
+      download(url, path)
+      unzip(path)
+      PACKER_PATH
+    end
+  end
+end

data/lib/linecook-gem/packager/packer/gcp.rb

@@ -0,0 +1,118 @@
+require 'fog/google'
+
+require 'linecook-gem/packager/packer/base'
+
+module Linecook
+  class GCPPacker < Linecook::Packer
+
+    BUILDER_CONFIG = {
+      type: 'null',
+    }.freeze
+
+    # run with 'shell-local'
+    PRE_MOUNT_COMMANDS = [
+
+      # create a sparse image with dd
+      "dd if=/dev/zero of=#{image_path} bs=1 count=0 seek=#{image_size}",
+      "parted -s $(losetup | grep #{image_path} | awk '{print $1}') mklabel msdos",
+      "parted -s $(losetup | grep #{image_path} | awk '{print $1}') mkpart primary ext4 0% 100%",
+      "mkfs.ext4 $(losetup | grep #{image_path} | awk '{print $1}')p1",
+      "tune2fs -L cloudimg-rootfs $(losetup | grep #{image_path} | awk '{print $1}')p1",
+      "mount $(losetup | grep #{image_path} | awk '{print $1}')p1 /mnt/#{build_uuid}"
+    ]
+
+    CHROOT_SETUP = [
+#      tar -C "${MOUNT_PATH}" -xpf "${TAR_PATH}"
+#
+#mount --bind /dev "${MOUNT_PATH}"/dev
+#mount --bind /proc "${MOUNT_PATH}"/proc
+#mount --bind /sys "${MOUNT_PATH}"/sys
+#
+#echo "LABEL=cloudimg-rootfs   /        ext4   defaults,discard        0 0" >> "${MOUNT_PATH}"/etc/fstab
+    ]
+
+    CHROOT_COMMNANDS_EXTRA = [
+#APT_PACKAGES=(
+#  grub-pc
+#  cloud-init
+#  cloud-initramfs-growroot
+#  gce-daemon
+#  gce-startup-scripts
+#  gce-image-bundle
+#  gce-cloud-config
+#)
+#
+#DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
+#  -o Dpkg::Options::="--force-confold" -y --force-yes --no-upgrade install "${APT_PACKAGES}"
+#
+#rm -f /etc/hostname
+#ln -s /usr/share/google/set-hostname /etc/dhcp/dhclient-exit-hooks.d/
+#
+#echo "169.254.169.254 metadata.google.internal metadata" >> /etc/hosts
+#echo "127.0.0.1 localhost" >> /etc/hosts
+    ]
+
+   CHROOT_UNMOUNT = [
+     "umount /mnt/#{build_uuid}/dev",
+     "umount /mnt/#{build_uuid}/sys",
+     "umount /mnt/#{build_uuid}/proc",
+     "umount /mnt/#{build_uuid}"
+   ]
+
+   COMPRESS_IMAGE = [
+     "losetup -d $(losetup | grep #{image_path} | awk '{print $1}')",
+     "mkdir -p /tmp/#{build_uuid}",
+     "mv #{image_path} /tmp/#{build_uuid}/disk.raw", #must be called disk.raw
+     "cd /tmp/#{build_uuid} && tar -czvf #{build_uuid}.tar.gz disk.raw",
+     "rm /tmp#{build_uuid}/disk.raw"
+   ]
+
+
+  private
+
+    def upload
+      connection = Fog::Storage::Google.new
+
+      image_source = File.open("/tmp/#{build_uuid}/#{build_uuid}.tar.gz")
+      connection.put_object("bucket", "#{image_name}-#{build_uuid}.tar.gz", image_source)
+      image_source.close
+      image_source.unlink
+
+    end
+
+    def create_gcp_image
+      connection = Fog::Compute.new(:provider => "Google")
+
+      rawdisk = {
+        source: "gs://#{bucket}/#{image_slug}",
+        container_type: "TAR"
+      }
+
+
+      img = connection.images.create(name: "test-image", # FIXME - from config
+                                     description: "Test image (via fog)", # FIXME - from config
+                                     raw_disk: rawdisk)
+
+      img.reload # will raise if image was not saved correctly
+    end
+
+    # run with 'shell-local' and "execute_command"
+    #  https://www.packer.io/docs/provisioners/shell-local.html
+    # to wrap everything with chroot
+
+# Rather than use the 'googlecloud' provider in packer, which would require us to spin up a VM
+# We post-process the image locally, and upload it to google cloud. Google is able to deal with tarballs
+# directly, so there's no need for a fancy chroot builder.
+#
+# Gist of how it works:
+# 1. Use a null builder to post-process a linecook image into an acceptable format for google
+# 2. Snapshot that image as a tarball
+# 3. Upload that image to google storage
+# 4. Create a google compute image from the tarball URL.  https://cloud.google.com/compute/docs/images/import-existing-image
+
+
+# https://github.com/Shopify/google-cloud-images/blob/master/deps/chroot-commands.sh
+# https://github.com/Shopify/google-cloud-images/blob/master/deps/generate-image.sh
+
+  end
+end

data/lib/linecook-gem/version.rb

@@ -1,3 +1,3 @@
 module Linecook
-  VERSION = '0.7.35'
+  VERSION = '0.7.36'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: linecook-gem
 version: !ruby/object:Gem::Version
-  version: 0.7.35
+  version: 0.7.36
 platform: ruby
 authors:
 - Dale Hamel
@@ -283,6 +283,9 @@ files:
 - lib/linecook-gem/image/s3.rb
 - lib/linecook-gem/packager.rb
 - lib/linecook-gem/packager/packer.rb
+- lib/linecook-gem/packager/packer/ami.rb
+- lib/linecook-gem/packager/packer/base.rb
+- lib/linecook-gem/packager/packer/gcp.rb
 - lib/linecook-gem/packager/route53.rb
 - lib/linecook-gem/packager/squashfs.rb
 - lib/linecook-gem/util/common.rb