linecook-gem 0.6.10 → 0.7.0

data/lib/linecook-gem/builder/linux_backend.rb

@@ -1,11 +0,0 @@
-module Linecook
-  module LinuxBuilder
-    extend self
-
-    def backend
-      config = Linecook.config[:builder]
-      images = Linecook.config[:image][:images]
-      Linecook::Lxc::Container.new(name: config[:name], home: config[:home], image: config[:image], bridge: true, tmp: Linecook.config[:tmp])
-    end
-  end
-end

data/lib/linecook-gem/builder/lxc.rb

@@ -1,286 +0,0 @@
-require 'linecook-gem/image/manager'
-require 'linecook-gem/util/ssh'
-require 'linecook-gem/util/config'
-require 'linecook-gem/util/executor'
-require 'tempfile'
-require 'ipaddress'
-
-module Linecook
-  module Lxc
-    class Container
-
-      include Executor
-      MAX_WAIT = 60
-      attr_reader :config, :root
-      def initialize(name: 'linecook', home: '/u/lxc', image: nil, remote: :local, bridge: false, tmp: nil)
-        @name = name
-        @home = home
-        @bridge = bridge
-        @remote = remote == :local ? false : remote
-        @root = File.join(@home, @name, 'rootfs')
-        @tmp = tmp
-        @tmpmount = "linecook_#{name}"
-        config = { utsname: name, rootfs: @root, mount: {} }
-        config[:mount][:entry] ||= []
-        config[:mount][:entry] << '/sys/fs/cgroup sys/fs/cgroup none bind 0 0'
-        config[:mount][:entry] << "#{capture("mktemp -d --tmpdir=/#{@tmp} #{@tmpmount}XXXXXXX").strip} #{@tmp} none bind,create=dir 0 0" if @tmp
-
-        @config = Linecook::Lxc::Config.generate(config) # FIXME read link from config
-        @source_image = image || :base_image
-      end
-
-      def start
-        return if running?
-        setup_image
-        setup_dirs
-        mount_all
-        write_config
-        execute("lxc-start #{container_str} -d")
-        wait_running
-        setup_bridge if @bridge
-        wait_ssh
-        unmount unless running?
-      end
-
-      def resume
-        execute("lxc-start #{container_str} -d") unless running?
-      end
-
-      def stop(clean: false, destroy: true)
-        setup_dirs
-        if running?
-          cexec("sudo userdel -r -f #{Linecook::Build::USERNAME}") if @remote
-          execute("lxc-stop #{container_str} -k") if running?
-        end
-        unmount(clean: clean) if destroy
-      end
-
-      def ip
-        attempts = 10
-        attempt = 0
-        until info[:ip] || attempt >= attempts
-          attempt += 1
-          sleep(1)
-        end
-        info[:ip].is_a?(Array) ? info[:ip].find { |ip| bridge_network.include?(IPAddress(ip)) } : info[:ip]
-      end
-
-      def running?
-        info[:state] == 'RUNNING'
-      end
-
-      def pid
-        info[:pid]
-      end
-
-      def info
-        @info = {}
-        capture("lxc-info #{container_str} || true").lines.each do |line|
-          k, v = line.strip.split(/:\s+/)
-          key = k.downcase.to_sym
-          @info[key] = @info[key] ? [@info[key]].flatten << v : v
-        end
-        @info
-      end
-
-      private
-
-      def cexec(command)
-        execute("lxc-attach #{container_str} -- #{command}")
-      end
-
-      def wait_running
-        wait_for { running? }
-      end
-
-      def wait_ssh
-        if @remote
-          user = Linecook::Build::USERNAME
-          cexec("useradd -m -G sudo #{user} || true")
-          cexec("mkdir -p /home/#{user}/.ssh")
-          Linecook::Builder.ssh.upload(Linecook::SSH.public_key, "/tmp/#{@name}-pubkey")
-          Linecook::Builder.ssh.run("sudo mv /tmp/#{@name}-pubkey #{@root}/home/#{user}/.ssh/authorized_keys")
-          cexec("chown -R #{user} /home/#{user}/.ssh")
-        end
-        wait_for { capture("lxc-attach -n #{@name} -P #{@home} status ssh || true") =~ /running/ }
-      end
-
-      def wait_for
-        attempts = 0
-        until attempts > MAX_WAIT
-          break if yield
-          attempts += 1
-          sleep(1)
-        end
-      end
-
-      def lxc?
-        namespaces = capture('cat /proc/1/cgroup').lines.map{ |l| l.strip.split(':').last }.uniq
-        namespaces.length != 1 || namespaces.first != '/'
-      end
-
-      def setup_dirs
-        @lower_dir = tmpdir(label: 'lower')
-        @upper_base = tmpdir(label: 'upper')
-        @upper_dir = File.join(@upper_base, '/upper')
-        @work_dir = File.join(@upper_base, '/work')
-        @socket_dirs = []
-        (Linecook.config[:socket_dirs] ||[]).each{ |sock| @socket_dirs << File.join(@root, sock) }
-      end
-
-      def write_config
-        path = if @remote
-                 file = "/tmp/lxc-config-#{SecureRandom.hex(4)}"
-                 @remote.upload(@config, file)
-                 file
-               else
-                 file = Tempfile.new('lxc-config')
-                 file.write(@config)
-                 file.close
-                 file.path
-        end
-        execute("mv #{path} #{File.join(@home, @name, 'config')}")
-      end
-
-      def mount_all
-        # Prepare an overlayfs
-        execute("mkdir -p #{@root}")
-        execute("mkdir -p #{@lower_dir}")
-        execute("mkdir -p #{@upper_base}")
-        mount(@image_path, @lower_dir, options: '-o loop')
-        mount('tmpfs', @upper_base, type: '-t tmpfs', options:'-o noatime') # FIXME: - don't always be tmpfs
-        execute("mkdir -p #{@work_dir}")
-        execute("mkdir -p #{@upper_dir}")
-        mount('overlay', @root, type: '-t overlay', options: "-o lowerdir=#{@lower_dir},upperdir=#{@upper_dir},workdir=#{@work_dir}")
-        # Overlayfs doesn't support unix domain sockets
-        @socket_dirs.each do |sock|
-          execute("mkdir -p #{sock}")
-          mount('tmpfs', sock, type: '-t tmpfs', options:'-o noatime')
-        end
-      end
-
-      def mount(source, dest, type: '', options:'')
-        execute("grep -q #{dest} /etc/mtab || sudo mount #{type} #{options} #{source} #{dest}")
-      end
-
-      def unmount(clean: false)
-        @socket_dirs.each { |sock| execute("umount #{sock}") }
-        source = capture("mount | grep #{@lower_dir} | grep squashfs | awk '{print $1}'") if clean
-        execute("umount #{@root}")
-        execute("umount #{@upper_base}")
-        execute("umount #{@lower_dir}")
-        execute("rmdir #{@lower_dir}")
-        execute("rmdir #{@upper_base}")
-        execute("rm -rf /#{@tmp}/#{@tmpmount}*") if @tmp && !@tmp.empty? && @tmp != '/'
-
-        # Clean up the source image, but only if it's not mounted elsewhre
-        FileUtils.rm_f(source) if clean && capture("mount | grep #{source} || true").strip.empty?
-      end
-
-      def bridge_network
-        broad_ip = Socket.getifaddrs.find do |a|
-          a.name == 'lxcbr0' && a.broadaddr && a.broadaddr.afamily == 2 && !a.broadaddr.inspect_sockaddr.start_with?('169.254')
-        end.broadaddr.inspect_sockaddr
-        IPAddress("#{broad_ip.to_s}/24")
-      end
-
-      def setup_bridge
-        bridge_config = <<-eos
-auto lo
-iface lo inet loopback
-
-auto lxcbr0
-iface lxcbr0 inet dhcp
-  bridge_ports eth0
-  bridge_fd 0
-  bridge_maxwait 0
-eos
-        interfaces = Tempfile.new('interfaces')
-        interfaces.write(bridge_config)
-        interfaces.close
-        execute("mv #{interfaces.path} #{File.join(@root, 'etc', 'network', 'interfaces')}")
-
-        execute("lxc-attach -n #{@name} -P #{@home} ifup lxcbr0")
-      end
-
-      def setup_image
-        @source_path = Linecook::ImageManager.fetch(@source_image, profile: :public)
-        if @remote
-          name = File.basename(@source_path)
-          dest = "/u/linecook/images/#{name}"
-          unless test("[ -f #{dest} ]") && capture("shasum #{dest}").split.first == `shasum #{@source_path}`.split.first
-            tmp = "/tmp/#{name}-#{SecureRandom.hex(4)}"
-            @remote.run("sudo mkdir -p #{File.dirname(dest)}")
-            @remote.upload(@source_path, tmp)
-            @remote.run("sudo mv #{tmp} #{dest}")
-          end
-          @image_path = dest
-        else
-          @image_path = @source_path
-        end
-      end
-
-      def tmpdir(label: 'tmp')
-        "/tmp/#{@name}-#{label}"
-      end
-
-      def container_str
-        "-n #{@name} -P #{@home}"
-      end
-
-    end
-
-    module Config
-      extend self
-      DEFAULT_LXC_CONFIG = {
-        include: '/usr/share/lxc/config/ubuntu.common.conf',
-        aa_profile: 'unconfined', #'lxc-container-default-with-nesting',
-        arch: 'x86.64',
-        utsname: 'linecook',
-        rootfs: '/u/lxc/linecook/rootfs',
-        network: {
-          type: 'veth',
-          flags: 'up',
-          link: 'lxcbr0'
-        },
-        mount: {
-          auto: ['cgroup', 'proc:rw', 'sys:rw']
-        },
-        cap: {
-          drop: ''
-        },
-        cgroup: {
-          devices: {
-            allow: 'a'
-          }
-        }
-      }.freeze
-
-      def generate(**kwargs)
-        cfg = []
-        flatten(DEFAULT_LXC_CONFIG.deep_merge(kwargs || {})).each do |k, v|
-          [v].flatten.each do |val|
-            cfg << "lxc.#{k}=#{val}"
-          end
-        end
-        cfg.join("\n")
-      end
-
-      private
-
-      def flatten(hash)
-        flattened = {}
-        hash.each do |k, v|
-          if v.is_a?(Hash)
-            flatten(v).each do |key, val|
-              flattened["#{k}.#{key}"] = val
-            end
-          else
-            flattened[k] = v
-          end
-        end
-        flattened
-      end
-    end
-  end
-end

data/lib/linecook-gem/builder/manager.rb

@@ -1,79 +0,0 @@
-require 'forwardable'
-require 'sshkey'
-
-require 'linecook-gem/builder/lxc'
-require 'linecook-gem/builder/darwin_backend'
-require 'linecook-gem/builder/linux_backend'
-require 'linecook-gem/builder/build'
-
-module Linecook
-  module Builder
-    extend self
-    extend Forwardable
-    BUILD_HOME = '/u/lxc'
-
-    def_instance_delegators :backend, :stop, :ip, :info, :running?
-
-    def backend
-      @backend ||= backend_for_platform
-    end
-
-    def start
-      return if running?
-      backend.start
-      increase_loop_devices
-    end
-
-    def ssh
-      config = Linecook.config[:builder]
-      @ssh ||= SSH.new(ip, username: config[:username], password: config[:password], keyfile: Linecook::SSH.private_key)
-    end
-
-    def builds
-      ssh.test("[ -d #{BUILD_HOME} ]") ? ssh.capture("find  #{BUILD_HOME} -maxdepth 1 -mindepth 1 -type d -printf \"%f\n\"").delete(';').lines : []
-    end
-
-    def build_info
-      info = {}
-      builds.each do |build|
-        info[build] = Linecook::Build.new(build, nil).info
-      end
-      info
-    end
-
-    private
-
-    def backend_for_platform
-      case Config.platform
-      when 'linux'
-        LinuxBuilder.backend
-      when 'darwin'
-        OSXBuilder.backend
-      else
-        fail "Cannot find supported backend for #{Config.platform}"
-      end
-    end
-
-    def increase_loop_devices
-      kparams = {}
-
-      ssh.capture('cat /proc/cmdline').split(/\s+/).each do |param|
-        k,v = param.split('=')
-        kparams[k] = v
-      end
-
-      if loops = kparams['max_loop']
-        current =  ssh.capture('ls -1 /dev | grep loop')
-        last_loop = current.lines.length
-        to_create = loops.to_i - last_loop
-
-        to_create.times do |count|
-          index = last_loop + count
-          ssh.run("[ ! -e /dev/loop#{index} ] && sudo mknod -m660 /dev/loop#{index} b 7 #{index}")
-          ssh.run("sudo chown root:disk /dev/loop#{index}")
-        end
-      end
-    end
-
-  end
-end

data/lib/linecook-gem/image/github.rb

@@ -1,27 +0,0 @@
-require 'octokit'
-
-require 'linecook-gem/util/config'
-
-module Linecook
-  module GithubManager
-    extend self
-
-    def url(name)
-      latest[:assets].find { |a| a[:name] =~ /#{name}/ }[:browser_download_url]
-    end
-
-  private
-
-    def client
-      @client ||= Octokit::Client.new
-    end
-
-    def source
-      @source ||= (Linecook.config['source_repo'] || 'dalehamel/lxb')
-    end
-
-    def latest
-      client.releases(source).sort_by { |r| r[:published_at] }.last
-    end
-  end
-end

data/lib/linecook-gem/image/manager.rb

@@ -1,73 +0,0 @@
-require 'fileutils'
-
-require 'linecook-gem/image/s3'
-require 'linecook-gem/image/github'
-require 'linecook-gem/image/crypt'
-
-module Linecook
-  module ImageManager
-    IMAGE_PATH = File.join(Config::LINECOOK_HOME, 'images').freeze
-    extend self
-
-    def fetch(image, upgrade:false, profile: :private, type: nil, encrypted: false)
-      url_path = if image.is_a?(Symbol)
-        image_name = Linecook.config[:image][:images][image][:name]
-        path = File.join(IMAGE_PATH, image_name)
-        provider(profile).url(image_name) unless File.exist?(path) || upgrade
-      elsif image.is_a?(Hash)
-        profile = :private
-        encrypted = true
-        name = image[:name] == :latest ? File.basename(latest(image[:type])) : image[:name]
-        path = File.join([IMAGE_PATH, image[:type], name].compact)
-        provider(profile).url(name, type: image[:type])
-      else
-        puts "#{image} is invalid"
-      end
-
-      Linecook::Downloader.download(url_path, path, encrypted: encrypted) unless File.exist?(path) || upgrade
-      path
-    end
-
-    def clean(type: nil)
-      Dir["#{File.join([IMAGE_PATH, type].compact)}/**/*"].each do |image|
-        FileUtils.rm_f(image) unless `mount`.index(image)
-      end
-    end
-
-    def upload(image, profile: :private, type: nil)
-      path = File.join(IMAGE_PATH, File.basename(image))
-      puts "Encrypting and uploading image #{path}"
-      encrypted = Linecook::Crypto.new.encrypt_file(path)
-      provider(profile).upload(encrypted, type: type)
-      FileUtils.rm_f(encrypted)
-    end
-
-    def url(image, profile: :private, type: nil)
-      provider(profile).url(image, type: type)
-    end
-
-    def list(type: nil, profile: :private)
-      profile = profile.to_sym
-      provider(profile).list(type: type)
-    end
-
-    def latest(type, profile: :private)
-      profile = profile.to_sym
-      provider(profile).latest(type)
-    end
-
-  private
-
-    def provider(image_profile)
-      profile = Linecook.config[:image][:provider][image_profile]
-      case profile
-      when :s3
-        S3Manager
-      when :github
-        GithubManager
-      else
-        fail "No provider implemented for for #{profile}"
-      end
-    end
-  end
-end