linecook-gem 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f3d23161644b325da20922d5a50a9915344b80d
-  data.tar.gz: bafc625040aa05abc282487ab7ae5f61c7206f3e
+  metadata.gz: e6a9328562fa6cf23fc9c9d97500f929b07299e3
+  data.tar.gz: 70c988ff74e983fed3ae8e74c20c3cd59ea6baaa
 SHA512:
-  metadata.gz: d06ea091b4b2df1f79e5f61f75b232fa29af7230ad06bb60daa2caf41e497fa549f3de272e1d990e0143f578ebcedc4675d77e0678ec225da9cab9f6386294db
-  data.tar.gz: cbd070f9ce4eabaf39d9be7bcb03c66254e02dbc1e4591368950722f4653e3fea85abef8bc41d3041ff86d1e4c9257a17172cda0e0e054eaf45d70c01407b326
+  metadata.gz: c7d3c9ce2562d88673b7bed970b8411c1a560d394e8c5401f982b5608464ca9a2c38026fbf825bf55b6097c5145c305bc3489cd21ac9e2a934d56770d513a9ea
+  data.tar.gz: 5856d130e60480ca800ed090ef5e59b016e627225ca4f4ba6b00a9d9e6b6498218f0fd3737ea026802cb1b38efb2b73f60810199a521632d8189929043309542

data/lib/linecook/builder/build.rb

@@ -0,0 +1,28 @@
+require 'forwardable'
+require 'linecook/builder/manager'
+
+module Linecook
+  class Build
+    extend Forwardable
+
+    def_instance_delegators :@container, :stop, :start, :ip, :info
+
+    def initialize(name, image: nil)
+      Linecook::Builder.start
+      @name = name
+      @image = image || Linecook::Config.load_config[:provisioner][:default_image]
+      @container = Linecook::Lxc::Container.new(name: @name, image: @image, remote: Linecook::Builder.ssh)
+    end
+
+    def ssh
+      @ssh ||= Linecook::SSH.new(@container.ip, username: 'ubuntu', password: 'ubuntu', proxy: Linecook::Builder.ssh, keyfile: Linecook::SSH.private_key)
+    end
+
+    def snapshot(save: false)
+      path = "/tmp/#{@name}-#{Time.now.to_i}.squashfs"
+      Linecook::Builder.ssh.run("sudo mksquashfs #{@container.root} #{path} -wildcards -e 'usr/src' 'var/lib/apt/lists/archive*' 'var/cache/apt/archives'") # FIXME make these excludes dynamic based on OS
+      Linecook::Builder.ssh.download(path, local: File.join(Linecook::ImageManager::IMAGE_PATH, File.basename(path))) if save
+      path
+    end
+  end
+end

data/lib/linecook/{darwin_backend.rb → builder/darwin_backend.rb}

@@ -1,6 +1,9 @@
 require 'securerandom'
-require 'linecook/image'
+require 'linecook/image/manager'
 # FIXME: read config values from config file
+#  - create cache loopback image
+#   - dd, based on config file.
+
 
 module Linecook
   module OSXBuilder

data/lib/linecook/{linux_backend.rb → builder/linux_backend.rb}

@@ -1,13 +1,13 @@
 module Linecook
   module LinuxBuilder
     extend self
-    LXC_MIN_VERSION = '1.0.7'
+    LXC_MIN_VERSION = '1.1.4'
 
     def backend
       check_lxc_version
       config = Linecook::Config.load_config[:builder]
-      images = Linecook::Config.load_config[:images]
-      Linecook::Lxc::Container.new(name: config[:name], home: config[:home], image: images[config[:image]])
+      images = Linecook::Config.load_config[:image][:images]
+      Linecook::Lxc::Container.new(name: config[:name], home: config[:home], image: config[:image], bridge: true)
     end
 
     private

data/lib/linecook/{lxc.rb → builder/lxc.rb}

@@ -1,34 +1,37 @@
-require 'linecook/ssh'
-require 'linecook/image'
-require 'linecook/config'
+require 'linecook/image/manager'
+require 'linecook/util/ssh'
+require 'linecook/util/config'
+require 'linecook/util/executor'
 require 'tempfile'
 require 'ipaddress'
 
 module Linecook
   module Lxc
     class Container
+
+      include Executor
       MAX_WAIT = 60
-      attr_reader :config
-      def initialize(name: 'linecook', home: '/u/lxc', image: nil, remote: :local)
-        @remote = remote == :local ? false : remote
-        config = { utsname: name, rootfs: File.join(home, name, 'rootfs') }
-        config.merge!(network: { type: 'veth', flags: 'up', link: 'lxcbr0' }) # FIXME
-        @config = Linecook::Lxc::Config.generate(config) # FIXME read link from config
-        @source_image = image || Linecook::Config.load_config[:images][:base_image]
+      attr_reader :config, :root
+      def initialize(name: 'linecook', home: '/u/lxc', image: nil, remote: :local, bridge: false)
         @name = name
         @home = home
+        @bridge = bridge
+        @remote = remote == :local ? false : remote
+        @root = File.join(@home, @name, 'rootfs')
+        config = { utsname: name, rootfs: @root }
+        @config = Linecook::Lxc::Config.generate(config) # FIXME read link from config
+        @source_image = image || :base_image
       end
 
       def start
+        return if running?
         setup_image
         setup_dirs
         mount_all
         write_config
         execute("lxc-start #{container_str} -d")
         wait_running
-        # Don't start a cgmanager if we're already in a container
-        execute('[ -f /etc/init/cgmanager.conf ] && sudo status cgmanager | grep -q running && sudo stop cgmanager || true') if lxc?
-        setup_bridge unless @remote
+        setup_bridge if @bridge
         wait_ssh
         unmount unless running?
       end
@@ -59,7 +62,7 @@ module Linecook
 
       def info
         @info = {}
-        capture("lxc-info #{container_str}").lines.each do |line|
+        capture("lxc-info #{container_str} || true").lines.each do |line|
           k, v = line.strip.split(/:\s+/)
           key = k.downcase.to_sym
           @info[key] = @info[key] ? [@info[key]].flatten << v : v
@@ -95,9 +98,8 @@ module Linecook
         @upper_base = tmpdir(label: 'upper')
         @upper_dir = File.join(@upper_base, '/upper')
         @work_dir = File.join(@upper_base, '/work')
-        @overlay = File.join(@home, @name, '/rootfs')
         @socket_dirs = []
-        (Linecook::Config.load_config[:socket_dirs] ||[]).each{ |sock| @socket_dirs << File.join(@overlay, sock) }
+        (Linecook::Config.load_config[:socket_dirs] ||[]).each{ |sock| @socket_dirs << File.join(@root, sock) }
       end
 
       def write_config
@@ -115,14 +117,14 @@ module Linecook
 
       def mount_all
         # Prepare an overlayfs
-        execute("mkdir -p #{@overlay}")
+        execute("mkdir -p #{@root}")
         execute("mkdir -p #{@lower_dir}")
         execute("mkdir -p #{@upper_base}")
         mount(@image_path, @lower_dir, options: '-o loop')
         mount('tmpfs', @upper_base, type: '-t tmpfs', options:'-o noatime') # FIXME: - don't always be tmpfs
         execute("mkdir -p #{@work_dir}")
         execute("mkdir -p #{@upper_dir}")
-        mount('overlay', @overlay, type: '-t overlay', options: "-o lowerdir=#{@lower_dir},upperdir=#{@upper_dir},workdir=#{@work_dir}")
+        mount('overlay', @root, type: '-t overlay', options: "-o lowerdir=#{@lower_dir},upperdir=#{@upper_dir},workdir=#{@work_dir}")
         # Overlayfs doesn't support unix domain sockets
         @socket_dirs.each do |sock|
           execute("mkdir -p #{sock}")
@@ -136,7 +138,7 @@ module Linecook
 
       def unmount
         @socket_dirs.each { |sock| execute("umount #{sock}") }
-        execute("umount #{@overlay}")
+        execute("umount #{@root}")
         execute("umount #{@upper_base}")
         execute("umount #{@lower_dir}")
         execute("rmdir #{@lower_dir}")
@@ -164,17 +166,17 @@ eos
         interfaces = Tempfile.new('interfaces')
         interfaces.write(bridge_config)
         interfaces.close
-        execute("mv #{interfaces.path} #{File.join(@home, @name, 'rootfs', 'etc', 'network', 'interfaces')}")
+        execute("mv #{interfaces.path} #{File.join(@root, 'etc', 'network', 'interfaces')}")
 
         execute("lxc-attach -n #{@name} -P #{@home} ifup lxcbr0")
       end
 
 
       def setup_image
-        @source_path = Linecook::ImageFetcher.fetch(@source_image)
+        @source_path = Linecook::ImageManager.fetch(@source_image)
         if @remote
           dest = "#{File.basename(@source_path)}"
-          @remote.upload(@source_path, dest) unless @remote.test("[ -f #{dest} ]")
+          @remote.upload(@source_path, dest) unless test("[ -f #{dest} ]")
           @image_path = dest
         else
           @image_path = @source_path
@@ -189,26 +191,6 @@ eos
         "-n #{@name} -P #{@home}"
       end
 
-      def capture(command, sudo: true)
-        execute(command, sudo: sudo, capture: true)
-      end
-
-      def execute(command, sudo: true, capture: false)
-        command = "sudo #{command}" if sudo
-        if @remote
-          if capture
-            return @remote.capture(command)
-          else
-            @remote.run(command)
-          end
-        else
-          if capture
-            return `#{command}`
-          else
-            system(command)
-          end
-        end
-      end
     end
 
     module Config
@@ -222,7 +204,7 @@ eos
         network: {
           type: 'veth',
           flags: 'up',
-          link: 'br0'
+          link: 'lxcbr0'
         },
         mount: {
           auto: 'cgroup'

data/lib/linecook/builder/manager.rb

@@ -0,0 +1,56 @@
+require 'forwardable'
+require 'sshkey'
+
+require 'linecook/builder/lxc'
+require 'linecook/builder/darwin_backend'
+require 'linecook/builder/linux_backend'
+require 'linecook/builder/build'
+
+module Linecook
+  module Builder
+    extend self
+    extend Forwardable
+    BUILD_HOME = '/u/lxc'
+
+    def_instance_delegators :backend, :stop, :ip, :info, :running?
+
+    def backend
+      @backend ||= backend_for_platform
+    end
+
+    def start
+      return if running?
+      backend.start
+    end
+
+    def ssh
+      config = Linecook::Config.load_config[:builder]
+      @ssh ||= SSH.new(ip, username: config[:username], password: config[:password], keyfile: Linecook::SSH.private_key)
+    end
+
+    def builds
+      ssh.test("[ -d #{BUILD_HOME} ]") ? ssh.capture("find  #{BUILD_HOME} -maxdepth 1 -mindepth 1 -type d -printf \"%f\n\"").delete(';').lines : []
+    end
+
+    def build_info
+      info = {}
+      builds.each do |build|
+        info[build] = Linecook::Build.new(build, nil).info
+      end
+      info
+    end
+
+    private
+
+    def backend_for_platform
+      case Config.platform
+      when 'linux'
+        LinuxBuilder.backend
+      when 'darwin'
+        OSXBuilder.backend
+      else
+        fail "Cannot find supported backend for #{Config.platform}"
+      end
+    end
+  end
+end

data/lib/linecook/cli.rb

@@ -1,6 +1,56 @@
 require 'thor'
 require 'linecook'
 
+class Crypto < Thor
+  desc 'keygen', 'Generate AES key for securing images'
+  def keygen
+    puts Linecook::Crypto.keygen
+  end
+
+  desc 'decrypt IMAGE', ''
+  def decrypt(image)
+    puts Linecook::Crypto.new.decrypt_file(image)
+  end
+
+  desc 'encrypt IMAGE', ''
+  def encrypt(image)
+    puts Linecook::Crypto.new.encrypt_file(image)
+  end
+end
+
+class Image < Thor
+  desc 'crypto SUBCOMMAND', 'Manage image encryption'
+  subcommand 'crypto', Crypto
+
+  desc 'list', 'List images' # add remote flag
+  def list
+  end
+
+  desc 'fetch IMAGE_NAME', 'Fetch an image by name'
+  method_options name: :string
+  def fetch(name)
+    Linecook::ImageManager.fetch(name)
+  end
+
+  desc 'upload IMAGE', 'Upload an image'
+  method_options name: :string
+  def upload(image)
+    Linecook::ImageManager.upload(image)
+  end
+
+  desc 'url IMAGE', 'Get URL for image'
+  method_options image: :string
+  def url(image)
+    puts Linecook::ImageManager.url(image)
+  end
+
+  desc 'package IMAGE', 'Package image'
+  method_options image: :string
+  def package(image)
+    Linecook::Packager.package(image)
+  end
+end
+
 class Builder < Thor
   desc 'info', 'Show builder info'
   def info
@@ -29,27 +79,42 @@ class Build < Thor
     puts Linecook::Builder.builds
   end
 
-  desc 'info', 'Show build info' # FIXME: accept the build name
+  desc 'info', 'Show build info'
   def info
-    puts Linecook::Builder.build_info
+  end
+
+  desc 'ip', 'Show IP address for build'
+  def ip
+  end
+
+  desc 'snapshot', 'Take a snapshot of the build with NAME'
+  method_option :name, type: :string, required: true, banner: 'ROLE_NAME', desc: 'Name of the role to build', aliases: '-n'
+  def snapshot
+    build = Linecook::Build.new(name, '')
+    build.snapshot(save: true)
   end
 end
 
 class Linecook::CLI < Thor
-  desc 'builder SUBCOMMAND', 'Manage builders'
+
+  desc 'image SUBCOMMAND', 'Manage linecook images.'
+  subcommand 'image', Image
+
+  desc 'builder SUBCOMMAND', 'Manage the builder.'
   subcommand 'builder', Builder
 
-  desc 'build SUBCOMMAND', 'Manage builds'
+  desc 'build SUBCOMMAND', 'Manage running and completed builds.'
   subcommand 'build', Build
 
-  desc 'bake', 'Bake a new image'
+  desc 'bake', 'Bake a new image.'
+  method_option :name, type: :string, required: true, banner: 'ROLE_NAME', desc: 'Name of the role to build', aliases: '-n'
+  method_option :image, type: :string,  banner: 'SOURCE_IMAGE', desc: 'Source image to seed the build.', aliases: '-i'
+  method_option :build, type: :boolean, default: true, desc: 'Build the image', aliases: '-b'
+  method_option :snapshot, type: :boolean, default: false, desc: 'Snapshot the resulting build to create an image', aliases: '-s'
+  method_option :upload, type: :boolean, default: false, desc: 'Upload the resulting build. Implies --snapshot and --encrypt.', aliases: '-u'
+  method_option :package, type: :boolean, default: false, desc: 'Package the resulting image. Implies --upload, --snapshot and --encrypt.', aliases: '-p'
   def bake
-    Linecook::Baker.bake
-  end
-
-  desc 'fetch IMAGE_NAME', 'Fetch an image by name'
-  method_options name: :string
-  def fetch(name)
-    Linecook::ImageFetcher.fetch(name)
+    opts = options.symbolize_keys
+    Linecook::Baker.bake(**opts)
   end
 end

data/lib/linecook/image/crypt.rb

@@ -0,0 +1,49 @@
+require 'base64'
+require 'encryptor'
+
+require 'linecook/image/manager'
+require 'linecook/util/executor'
+require 'linecook/util/config'
+
+module Linecook
+  class Crypto
+    include Executor
+    CIPHER = 'aes-256-cbc'
+    KEY_BYTES = 32 # 256 bits
+    attr_reader :iv, :secret_key
+
+    def initialize(remote: nil)
+      @remote = remote
+      load_key
+    end
+
+    def encrypt_image(image)
+      image_path = File.join(Linecook::ImageManager::IMAGE_PATH,File.basename(image))
+      encrypt_file(image_path)
+    end
+
+    def encrypt_file(source, dest: nil, keypath: nil)
+      dest ||= "/tmp/#{File.basename(source)}"
+      capture("openssl enc -#{CIPHER} -out #{dest} -in #{source} -K #{@secret_key} -iv #{@iv}")
+      dest
+    end
+
+    def decrypt_file(source, dest: nil, keypath: nil)
+      dest ||= "/tmp/#{File.basename(source)}-decrypted"
+      capture("openssl enc -#{CIPHER} -out #{dest} -in #{source} -K #{@secret_key} -iv #{@iv} -d")
+      dest
+    end
+
+    def self.keygen
+      iv = OpenSSL::Cipher::Cipher.new(CIPHER).random_iv.unpack('H*').first
+      secret_key = Base64.encode64(OpenSSL::Random.random_bytes(KEY_BYTES)).unpack('H*').first
+      "[:IV:#{iv}:KY:#{secret_key}]"
+    end
+
+  private
+
+    def load_key
+      @iv, @secret_key = Linecook::Config.secrets['aeskey'].match(/\[:IV:(.+):KY:(.+)\]/m).captures
+    end
+  end
+end

data/lib/linecook/image/github.rb

@@ -0,0 +1,27 @@
+require 'octokit'
+
+require 'linecook/util/config'
+
+module Linecook
+  module GithubManager
+    extend self
+
+    def url(name)
+      latest[:assets].find { |a| a[:name] =~ /#{name}/ }[:browser_download_url]
+    end
+
+  private
+
+    def client
+      @client ||= Octokit::Client.new
+    end
+
+    def source
+      @source ||= (Config.load_config['source_repo'] || 'dalehamel/lxb')
+    end
+
+    def latest
+      client.releases(source).sort_by { |r| r[:published_at] }.last
+    end
+  end
+end

data/lib/linecook/image/manager.rb

@@ -0,0 +1,43 @@
+require 'linecook/image/crypt'
+require 'linecook/image/github'
+require 'linecook/image/s3'
+require 'linecook/util/downloader'
+
+module Linecook
+  module ImageManager
+    IMAGE_PATH = File.join(Config::LINECOOK_HOME, 'images').freeze
+    extend self
+
+    def fetch(name, upgrade:false, profile: :private)
+      image_name = Linecook::Config.load_config[:image][:images][name][:name]
+      path = File.join(IMAGE_PATH, image_name)
+      url = provider(profile).url(name) unless File.exist?(path) || upgrade# FIXME
+      Linecook::Downloader.download(url, path) unless File.exist?(path) || upgrade
+      path
+    end
+
+    def upload(image, profile: :private)
+      path = File.join(IMAGE_PATH, File.basename(image))
+      puts "Encrypting and uploading image #{path}"
+      provider(profile).upload(Linecook::Crypto.new.encrypt_file(path))
+    end
+
+    def url(image, profile: :private)
+      provider(profile).url("builds/#{image}")
+    end
+
+  private
+
+    def provider(image_profile)
+      profile = Linecook::Config.load_config[:image][:provider][image_profile]
+      case profile
+      when :s3
+        S3Manager
+      when :github
+        GithubManager
+      else
+        fail "No provider implemented for for #{profile}"
+      end
+    end
+  end
+end

data/lib/linecook/image/s3.rb

@@ -0,0 +1,47 @@
+require 'ruby-progressbar'
+require 'aws-sdk'
+
+module Linecook
+  module S3Manager
+    extend self
+    EXPIRY = 20
+
+    def url(name)
+      client
+      s3 = Aws::S3::Resource.new
+      obj = s3.bucket(Linecook::Config.secrets['bucket']).object(name)
+      obj.presigned_url(:get, expires_in: EXPIRY * 60)
+    end
+
+    def upload(path)
+      File.open(path, 'rb') do |file|
+        fname = File.basename(path)
+        pbar = ProgressBar.create(title: fname, total: file.size)
+        common_opts = { bucket: Linecook::Config.secrets['bucket'], key: File.join('builds', fname) }
+        resp = client.create_multipart_upload(storage_class: 'REDUCED_REDUNDANCY', server_side_encryption: 'AES256', **common_opts)
+        id = resp.upload_id
+        part = 0
+        total = 0
+        parts = []
+        while content = file.read(1048576 * 20)
+          part += 1
+          resp = client.upload_part(body: content, content_length: content.length, part_number: part, upload_id: id, **common_opts)
+          parts << { etag: resp.etag, part_number: part }
+          total += content.length
+          pbar.progress = total
+          pbar.title = "#{fname} - (#{((total.to_f/file.size.to_f)*100.0).round(2)}%)"
+        end
+        client.complete_multipart_upload(upload_id: id, multipart_upload: { parts: parts }, **common_opts)
+      end
+    end
+
+  private
+    def client
+      @client ||= begin
+        Aws.config[:credentials] = Aws::Credentials.new(Linecook::Config.secrets['aws_access_key'], Linecook::Config.secrets['aws_secret_key'])
+        Aws.config[:region] = 'us-east-1'
+        Aws::S3::Client.new
+      end
+    end
+  end
+end