linecook-gem 0.0.3 → 0.0.4

data/lib/linecook/packager/ebs.rb

@@ -0,0 +1,270 @@
+require 'net/http'
+require 'open-uri'
+require 'timeout'
+require 'tmpdir'
+
+require 'linecook/util/ssh'
+require 'linecook/util/executor'
+
+require 'aws-sdk'
+
+module Linecook
+  # Installs a linecook image on a target device
+  module Packager
+    class EBS
+      include Executor
+
+      def initialize(hvm: true, size: 10, region: 'us-east-1')
+        @hvm = hvm
+        @size = size
+        @region = region
+      end
+
+      def package(image)
+        @image = image
+        setup_remote unless instance_id
+        prepare
+        execute("tar -C #{@mountpoint} -cpf - . | sudo tar -C #{@root} -xpf -")
+        finalize
+        snapshot
+      end
+
+    private
+
+      def prepare
+        @mountpoint = Dir.mktmpdir
+        execute("mkdir -p #{@mountpoint}")
+        execute("mount #{@image} #{@mountpoint}")
+        create_volume
+        format_and_mount
+      end
+
+      def finalize
+        execute("echo \"UUID=\\\"$(blkid -o value -s UUID #{@rootdev})\\\" /               ext4            defaults  1 2\" > /tmp/fstab")
+        execute("mv /tmp/fstab #{@root}/etc/fstab")
+        chroot_exec('apt-get update')
+        chroot_exec('apt-get install -y --force-yes grub-pc grub-legacy-ec2')
+        chroot_exec('update-grub')
+        execute("grub-install --root-directory=#{@root} $(echo #{@rootdev} | sed \"s/[0-9]*//g\")") if @hvm
+      end
+
+
+      def chroot_exec(command)
+        execute("mount -o bind /dev #{@root}/dev")
+        execute("mount -o bind /sys #{@root}/sys")
+        execute("mount -t proc none #{@root}/proc")
+        execute("cp /etc/resolv.conf #{@root}/etc")
+        execute("chroot #{@root} #{command}")
+        execute("umount #{@root}/dev")
+        execute("umount #{@root}/sys")
+        execute("umount #{@root}/proc")
+      end
+
+      def partition
+        execute("parted -s #{@rootdev} mklabel msdos")
+        execute("parted -s #{@rootdev} mkpart primary ext2 0% 100%")
+        @rootdev = "#{@rootdev}1"
+      end
+
+      def format_and_mount
+        partition if @hvm
+        execute("mkfs.ext4 #{@rootdev}")
+        @root = Dir.mktmpdir
+        execute("mkdir -p #{@root}")
+        execute("mount #{@rootdev} #{@root}")
+      end
+
+      def instance_id
+        @instance_id ||= metadata('instance-id')
+      end
+
+      def availability_zone
+        @availability_zone ||= metadata('placement/availability-zone')
+      end
+
+      def client
+        @client ||= begin
+          credentials = Aws::Credentials.new(Linecook::Config.secrets['aws_access_key'], Linecook::Config.secrets['aws_secret_key'])
+          Aws::EC2::Client.new(region: @region, credentials: credentials)
+        end
+      end
+
+      def create_volume
+        resp = client.create_volume({
+          size: @size,
+          availability_zone: availability_zone, # required
+          volume_type: "standard", # accepts standard, io1, gp2
+        })
+
+        @volume_id = resp.volume_id
+        rootdev = free_device
+
+        puts "Waiting for volume to become available"
+        wait_for_state('available', 120) do
+          client.describe_volumes(volume_ids: [@volume_id]).volumes.first.state
+        end
+
+        resp = client.attach_volume({
+          volume_id: @volume_id,
+          instance_id: instance_id,
+          device: rootdev,
+        })
+
+        puts "Waiting for volume to attach"
+        wait_for_state('attached', 120) do
+          client.describe_volumes(volume_ids: [@volume_id]).volumes.first.attachments.first.state
+        end
+        @rootdev = "/dev/#{rootdev}"
+      end
+
+      def snapshot
+        execute("umount #{@root}")
+        puts 'Creating snapshot'
+        resp = client.detach_volume(volume_id: @volume_id)
+        wait_for_state('available', 120) do
+          client.describe_volumes(volume_ids: [@volume_id]).volumes.first.state
+        end
+        resp = client.create_snapshot(volume_id: @volume_id, description: "Snapshot of #{File.basename(@image)}")
+        tag(resp.snapshot_id, Name: 'Linecook snapshot', image: File.basename(@image), hvm: @hvm.to_s)
+        client.delete_volume(volume_id: @volume_id)
+      end
+
+      def free_device
+        prefix = device_prefix
+        ('f'..'zzz').to_a.each do |suffix|
+          device = "#{prefix}#{suffix}"
+          if free_device?(device)
+            lock_device(device)
+            return device
+          end
+        end
+        return nil
+      end
+
+      def free_device?(device)
+        test("[ ! -e /dev/#{device} ]") && test("[ ! -e /run/lock/linecook-#{device} ]")
+      end
+
+      def lock_device(device)
+        execute("echo #{Process.pid} > /run/lock/linecook-#{device}")
+      end
+
+      def unlock_device(device)
+        execute("rm /run/lock/linecook-#{device}")
+      end
+
+      def device_prefix
+        prefixes = ['xvd', 'sd']
+        capture('ls -1 /sys/block').lines.each do |dev|
+          prefixes.each do |prefix|
+            return prefix if dev =~ /^#{prefix}/
+          end
+        end
+        return nil
+      end
+
+      def setup_remote
+        start_node
+        path = "/tmp/#{File.basename(@image)}"
+        @remote.run("wget '#{Linecook::ImageManager.url(File.basename(@image))}' -nv -O #{path}")
+        @image = Linecook::Crypto.new(remote: @remote).decrypt_file(path)
+      end
+
+      def start_node
+        resp = client.run_instances(
+          image_id: find_ami,
+          min_count: 1,
+          max_count: 1,
+          instance_type: 'c4.large',
+          instance_initiated_shutdown_behavior: 'terminate',
+          security_groups: [security_group],
+          key_name: key_pair
+        )
+        @instance_id = resp.instances.first.instance_id
+        @availability_zone = resp.instances.first.placement.availability_zone
+
+        puts 'Waiting for temporary instance to come online'
+        wait_for_state('running', 300) do
+          client.describe_instances(instance_ids: [@instance_id]).reservations.first.instances.first.state.name
+        end
+        tag(@instance_id, Name: 'linecook-temporary-installer-node')
+        @remote = Linecook::SSH.new(instance_ip, username: 'ubuntu', keyfile: Linecook::SSH.private_key)
+        @remote.upload("exec shutdown -h 60 'Delayed shutdown started'", '/tmp/delay-shutdown')
+        execute('mv /tmp/delay-shutdown /etc/init/delay-shutdown.conf') # ubuntism is ok, since the temporary host can always be ubuntu
+        execute('start delay-shutdown')
+      end
+
+      def find_ami
+        url = "http://uec-images.ubuntu.com/query/trusty/server/released.current.txt"
+        type = @hvm ? 'hvm' : 'paravirtual'
+        data = open(url).read.split("\n").map{|l| l.split}.detect do |ary|
+          ary[4] == 'ebs' &&
+            ary[5] == 'amd64' &&
+            ary[6] == @region &&
+            ary.last == type
+        end
+        data[7]
+      end
+
+      def instance_ip
+        client.describe_instances(instance_ids: [@instance_id]).reservations.first.instances.first.public_ip_address
+      end
+
+      def security_group
+        group_name = 'linecook-global-ssh'
+        resp = client.describe_security_groups(filters: [{name: 'group-name', values: [group_name]}])
+        if resp.security_groups.length < 1
+          resp = client.create_security_group({
+            group_name: group_name,
+            description: "Allow global ssh for linecook temporary builder instances",
+          })
+
+          resp = client.authorize_security_group_ingress({
+            group_name: group_name,
+            ip_protocol: "tcp",
+            from_port: 22,
+            to_port: 22,
+            cidr_ip: "0.0.0.0/0",
+          })
+        end
+        group_name
+      end
+
+      def tag(id, **kwargs)
+        resp = client.create_tags(resources: [id], tags: kwargs.map{ |k,v| {key: k, value: v } })
+      end
+
+      def key_pair
+        pubkey = Linecook::SSH.public_key
+        resp = client.describe_key_pairs({
+          filters: [ { name: 'fingerprint', values: [Linecook::SSH.sshv2_fingerprint(pubkey)] } ]
+        })
+
+        if resp.key_pairs.length >= 1
+          return resp.key_pairs.first.key_name
+        else
+          keyname = "linecook-#{SecureRandom.uuid}"
+          resp = client.import_key_pair({
+            key_name: keyname,
+            public_key_material: pubkey,
+          })
+          return keyname
+        end
+      end
+
+      def metadata(key)
+       (Timeout::timeout(1) { Net::HTTP.get(URI(File.join("http://169.254.169.254/latest/meta-data", key))) } rescue nil)
+      end
+
+      def wait_for_state(desired, timeout)
+        attempts = 0
+        state = nil
+        while attempts < timeout && state != desired
+          state = yield
+          attempts += 1
+          sleep(1)
+        end
+      end
+    end
+  end
+end

data/lib/linecook/packager/manager.rb

@@ -0,0 +1,23 @@
+require 'linecook/packager/ebs'
+
+module Linecook
+  module Packager
+    extend self
+
+    def package(image)
+      provider.package(image)
+    end
+
+  private
+    def provider
+      name = Linecook::Config.load_config[:packager][:provider]
+      config = Linecook::Config.load_config[:packager][name]
+      case name
+      when :ebs
+        Linecook::Packager::EBS.new(**config)
+      else
+        fail "No packager implemented for for #{name}"
+      end
+    end
+  end
+end

data/lib/linecook/{chef.rb → provisioner/chef-zero.rb}

@@ -8,13 +8,14 @@ module Linecook
   module Chef
     extend self
 
-    def provision(build)
+    def provision(build, role)
       chef_config = setup
+      role_config = Linecook::Config.load_config[:roles][role.to_sym]
       script = ChefProvisioner::Bootstrap.generate(
         node_name: chef_config[:node_name],
         chef_version: chef_config[:version] || nil,
         first_boot: {
-          run_list: chef_config[:run_list]
+          run_list: role_config[:run_list]
         }
       )
 

data/lib/linecook/provisioner/manager.rb

@@ -0,0 +1,34 @@
+require 'securerandom'
+
+require 'linecook/builder/build'
+require 'linecook/provisioner/chef-zero'
+require 'linecook/provisioner/packer'
+
+module Linecook
+  module Baker
+    extend self
+
+    def bake(name: nil, image: nil, snapshot: nil, upload: nil, package: nil, build: nil)
+      build_agent = Linecook::Build.new(name, image: image)
+      provider(name).provision(build_agent, name) if build
+      snapshot = build_agent.snapshot(save: true) if snapshot ||  upload || package
+      Linecook::ImageManager.upload(snapshot) if upload || package
+      Linecook::Packager.package(snapshot) if package
+      build_agent.stop
+    end
+
+  private
+
+    def provider(name)
+      provisioner = Linecook::Config.load_config[:roles][name.to_sym][:provisioner] || Linecook::Config.load_config[:provisioner][:default_provider]
+      case provisioner
+      when :chefzero
+        Linecook::Chef
+      when :packer
+        Linecook::Packer
+      else
+        fail "Unsupported provisioner #{provisioner}"
+      end
+    end
+  end
+end

data/lib/linecook/provisioner/packer.rb

@@ -0,0 +1,82 @@
+require 'json'
+require 'tempfile'
+require 'mkmf'
+require 'fileutils'
+
+require 'linecook/util/executor'
+
+
+module Linecook
+  module Packer
+    def self.provision(build, role)
+      Runner.new(build, role).run
+    end
+
+    class Runner
+      SOURCE_URL = 'https://releases.hashicorp.com/packer/'
+      PACKER_VERSION = '0.8.6'
+      PACKER_PATH = File.join(Linecook::Config::LINECOOK_HOME, 'bin', 'packer')
+
+      include Executor
+
+      def initialize(build, role)
+        role_config = Linecook::Config.load_config[:roles][role.to_sym]
+        @packer = packer_path
+        @build = build
+        @template = role_config[:template_path]
+      end
+
+      def run
+        @build.start
+        packer_template = inject_builder
+        Tempfile.open('packer-linecook') do |template|
+          template.write(JSON.dump(packer_template))
+          template.flush
+          execute("#{@packer} build #{template.path}", sudo: false)
+        end
+      end
+
+    private
+
+      def packer_path
+        found = File.exists?(PACKER_PATH) ? PACKER_PATH : find_executable('packer')
+        path = if found
+          version = execute("#{found} --version", sudo: false, capture: true)
+          Gem::Version.new(version) >= Gem::Version.new(PACKER_VERSION) ? found : nil
+        end
+
+        path ||= get_packer
+      end
+
+      def get_packer
+        puts "packer too old (<#{PACKER_VERSION}) or not present, getting latest packer"
+        arch = 1.size == 8 ? 'amd64' : '386'
+        path = File.join(File.dirname(PACKER_PATH), 'packer.zip')
+        url = File.join(SOURCE_URL, PACKER_VERSION, "packer_#{PACKER_VERSION}_#{Linecook::Config.platform}_#{arch}.zip")
+        Linecook::Downloader.download(url, path)
+        Linecook::Downloader.unzip(path)
+        PACKER_PATH
+      end
+
+      def inject_builder
+        packer_template = JSON.load(File.read(@template)).symbolize_keys
+        packer_template.merge(builders: null_builder)
+      end
+
+      def null_builder
+        [ communicator.merge(type: 'null') ]
+      end
+
+      def communicator
+        {
+          ssh_host: @build.ssh.hostname,
+          ssh_username: @build.ssh.username,
+          ssh_private_key_file: @build.ssh.keyfile,
+          ssh_bastion_host: Linecook::Builder.ssh.hostname,
+          ssh_bastion_username: Linecook::Builder.ssh.username,
+          ssh_bastion_private_key_file: Linecook::Builder.ssh.keyfile,
+        }
+      end
+    end
+  end
+end

data/lib/linecook/{config.rb → util/config.rb}

@@ -1,4 +1,5 @@
 require 'yaml'
+require 'json'
 require 'fileutils'
 
 require 'xhyve'
@@ -19,13 +20,52 @@ module Linecook
         username: 'ubuntu',
         password: 'ubuntu'
       },
-      images: {
-        live_iso: 'livesys.iso',
-        live_image: 'livesys.squashfs',
-        base_image: 'ubuntu-base.squashfs'
+      provisioner: {
+        default_provider: :chefzero,
+        default_image: :base_image,
+      },
+      image: {
+        provider: {
+          public: :github,
+          private: :s3,
+        },
+        images: {
+          live_iso: {
+            name: 'livesys.iso',
+            profile: :public,
+          },
+          live_image: {
+            name: 'livesys.squashfs',
+            profile: :public,
+          },
+          base_image: {
+            name: 'ubuntu-base.squashfs',
+            profile: :public,
+          }
+        }
+      },
+      packager: {
+        provider: :ebs,
+        ebs: {
+          hvm: true,
+          size: 10,
+          region: 'us-east-1'
+        }
+      },
+      roles: {
       }
     }
 
+    def secrets
+      @secrets ||= begin
+        if File.exists?('secrets.ejson')
+          JSON.load(`ejson decrypt secrets.ejson`)
+        else
+          {}
+        end
+      end
+    end
+
     def setup
       FileUtils.mkdir_p(LINECOOK_HOME)
       File.write(DEFAULT_CONFIG_PATH, YAML.dump(DEFAULT_CONFIG)) unless File.exist?(DEFAULT_CONFIG_PATH)

data/lib/linecook/util/downloader.rb

@@ -0,0 +1,35 @@
+require 'open-uri'
+require 'fileutils'
+
+require 'zip'
+require 'ruby-progressbar'
+
+module Linecook
+  module Downloader
+    def self.download(url, path)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.open(path, 'w') do |f|
+        pbar = ProgressBar.create(title: File.basename(path), total: nil)
+        IO.copy_stream(open(url,
+                            content_length_proc: lambda do|t|
+                              pbar.total = t if t && 0 < t
+                            end,
+                            progress_proc: lambda do|s|
+                              pbar.progress = s
+                            end), f)
+      end
+    end
+
+    def self.unzip(source, dest: nil)
+      puts "Extracting #{source}..."
+      dest ||= File.dirname(source)
+      Zip::File.open(source) do |zip_file|
+        zip_file.each do |f|
+          file_path = File.join(dest, f.name)
+          FileUtils.mkdir_p(File.dirname(file_path))
+          zip_file.extract(f, file_path)
+        end
+      end
+    end
+  end
+end

data/lib/linecook/util/executor.rb

@@ -0,0 +1,33 @@
+module Linecook
+  module Executor
+    def capture(command, sudo: true)
+      execute(command, sudo: sudo, capture: true)
+    end
+
+    def test(check)
+      if @remote
+        return @remote.test(check)
+      else
+        `#{check}`
+        return $?.exitstatus == 0
+      end
+    end
+
+    def execute(command, sudo: true, capture: false)
+      command = "sudo #{command}" if sudo
+      if @remote
+        if capture
+          return @remote.capture(command)
+        else
+          @remote.run(command)
+        end
+      else
+        if capture
+          return `#{command}`
+        else
+          system(command)
+        end
+      end
+    end
+  end
+end

data/lib/linecook/{ssh.rb → util/ssh.rb}

@@ -1,9 +1,11 @@
+require 'openssl'
+
 require 'sshkit'
 require 'sshkit/dsl'
 require 'net/ssh'
 require 'net/ssh/proxy/command'
 
-require 'linecook/config'
+require 'linecook/util/config'
 
 module Linecook
   class SSHKit::Formatter::Linecook < SSHKit::Formatter::Pretty
@@ -48,14 +50,40 @@ module Linecook
   end
 
   class SSH
+    MAX_RETRIES = 5
+    attr_reader :username, :password, :hostname, :keyfile
+
+    def self.private_key
+      userkey = File.expand_path("~/.ssh/id_rsa")
+      dedicated_key = File.join(Linecook::Config::LINECOOK_HOME, 'linecook_ssh.pem')
+      unless File.exists?(dedicated_key)
+        File.write(dedicated_key, SSHKey.generate.private_key)
+        FileUtils.chmod(0600, dedicated_key)
+      end
+      File.exists?(userkey) ? userkey : dedicated_key
+    end
+
+    def self.public_key(keyfile: nil)
+      SSHKey.new(File.read(keyfile || private_key)).ssh_public_key
+    end
+
+    # Generate a fingerprint for an SSHv2 key used by amazon, yes, this is ugly
+    def self.sshv2_fingerprint(key)
+      _, blob = key.split(/ /)
+      blob = blob.unpack("m*").first
+      reader = Net::SSH::Buffer.new(blob)
+      k=reader.read_key
+      OpenSSL::Digest.new('md5',k.to_der).hexdigest.scan(/../).join(":")
+    end
 
-    attr_reader :username, :hostname
     def initialize(hostname, username: 'ubuntu', password: nil, keyfile: nil, proxy: nil)
       @username = username
       @password = password
       @hostname = hostname
       @keyfile = keyfile
       @proxy = proxy_command(proxy) if proxy
+      wait_for_connection
+      setup_ssh_key if @keyfile
     end
 
     def forward(local, remote:nil)
@@ -107,8 +135,37 @@ module Linecook
       end
     end
 
+    def download(path, local: nil)
+      on linecook_host do |_host|
+        download! path, local || File.basename(path)
+      end
+    end
+
     private
 
+    def wait_for_connection
+      puts "Waiting for SSH connection"
+      attempts = 0
+      while attempts < MAX_RETRIES
+        begin
+          run("echo connected")
+          return
+        rescue SSHKit::Runner::ExecuteError
+          puts "Retrying SSH connection"
+          sleep(5)
+          attempts += 1
+        end
+      end
+    end
+
+    def setup_ssh_key
+      pubkey = Linecook::SSH.public_key(keyfile: @keyfile)
+      config = Linecook::Config.load_config[:builder]
+      run("mkdir -p /home/#{config[:username]}/.ssh")
+      upload(pubkey, "/home/#{config[:username]}/.ssh/authorized_keys")
+    end
+
+
     def linecook_host
       @host ||= begin
         host = SSHKit::Host.new(user: @username, hostname: @hostname)

data/lib/linecook/version.rb

@@ -1,3 +1,3 @@
 module Linecook
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/lib/linecook.rb

@@ -2,7 +2,8 @@ $LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
 
 require 'active_support/all'
 require 'linecook/version'
-require 'linecook/config'
-require 'linecook/build'
-require 'linecook/bake'
-require 'linecook/image'
+require 'linecook/util/config'
+require 'linecook/image/manager'
+require 'linecook/builder/manager'
+require 'linecook/provisioner/manager'
+require 'linecook/packager/manager'