linecook-gem 0.6.10 → 0.7.0

data/lib/linecook-gem/packager/route53.rb

@@ -0,0 +1,62 @@
+require 'aws-sdk'
+
+module Linecook
+   module Route53
+     extend self
+
+     def upsert_record(name, ami, region)
+       ami_config = Linecook.config[:packager][:ami]
+
+       zone = [ami_config[:zone], ami_config[:regions][region.to_sym], ami_config[:domain]].compact.join('.')
+       record = "#{name}.#{zone}"
+
+       resp = client.list_hosted_zones_by_name({
+         dns_name: zone,
+         max_items: 1,
+       })
+
+       if resp.hosted_zones.size < 1
+         puts "Failed to find dns zone: #{resp.dns_name}"
+         return false
+       end
+
+
+       zone_id = resp.hosted_zones[0].id
+       client.change_resource_record_sets({
+         hosted_zone_id: zone_id,
+         change_batch: {
+           comment: "create #{ami}",
+           changes: [
+             {
+               action: "UPSERT",
+               resource_record_set: {
+                 name: record,
+                 type: 'TXT',
+                 ttl: 1,
+                 resource_records: [
+                   {
+                     value: "\"#{ami}\"",
+                   },
+                 ],
+               },
+             },
+           ],
+         },
+       })
+       puts "Saved #{ami} to #{record}"
+     rescue Aws::Route53::Errors::ServiceError => e
+       puts "AWS Error: #{e.code} #{e.context.http_response.body_contents}"
+       return false
+     end
+
+  private
+
+    def client
+      @client ||= begin
+        Aws.config[:credentials] = Aws::Credentials.new(Linecook.config[:aws][:access_key], Linecook.config[:aws][:secret_key])
+        Aws.config[:region] = 'us-east-1'
+        Aws::Route53::Client.new
+      end
+    end
+  end
+end

data/lib/linecook-gem/packager/squashfs.rb

@@ -0,0 +1,51 @@
+require 'mkmf'
+require 'fileutils'
+require 'tmpdir'
+
+require 'linecook-gem/image'
+
+module Linecook
+  class Squashfs
+
+    EXCLUDE_PROFILES = {
+      common: [
+        'dev/*',
+        'sys/*',
+        'proc/*',
+        'run/*',
+        'tmp/*',
+        'home/kitchen',
+        'etc/sudoers.d/kitchen',
+        '.docker*',
+      ],
+      ubuntu: [
+        'usr/src',
+        'var/lib/apt/lists/archive*',
+        'var/cache/apt/archives*'
+      ]
+    }.freeze
+
+    def initialize(config)
+      @excludes = []
+      @excludes << EXCLUDE_PROFILES[:common]
+      @excludes << EXCLUDE_PROFILES[config[:distro]] if config[:distro]
+      @excludes << config[:excludes] if config[:excludes]
+      @excludes.flatten!
+      @outdir = config[:outdir] || Dir.pwd
+    end
+
+    def package(image)
+      FileUtils.mkdir_p(@outdir)
+      tmpdir = Dir.mktmpdir("#{image.id}-squashfs")
+      outfile = File.join(@outdir, "#{image.id}.squashfs")
+      puts "Extracting #{image.id} to temporary directory #{tmpdir}..."
+      system("sudo tar -C #{tmpdir} -xpf #{image.path}")
+      system("sudo mksquashfs #{tmpdir} #{outfile} -noappend -wildcards -e #{@excludes.map { |e| "'#{e}'" }.join(' ')}")
+      puts "Squashed image is at #{outfile}"
+    ensure
+      puts "Cleaning up #{tmpdir}..."
+      system("sudo rm -rf #{tmpdir}")
+    end
+
+  end
+end

data/lib/linecook-gem/util/downloader.rb

@@ -5,18 +5,14 @@ require 'securerandom'
 require 'zip'
 require 'ruby-progressbar'
 
-require 'linecook-gem/image/crypt'
-
 module Linecook
   module Downloader
     LOCK_WAIT_TIMEOUT = 180
 
-    def self.download(url, path, encrypted: false)
-      acquire_lock(path)
+    def download(url, path)
       FileUtils.mkdir_p(File.dirname(path))
       cryptfile = "#{File.basename(path)}-encrypted-#{SecureRandom.hex(4)}"
-      destination = encrypted ? File.join('/tmp', cryptfile) : path
-      File.open(destination, 'w') do |f|
+      File.open(path, 'w') do |f|
         pbar = ProgressBar.create(title: File.basename(path), total: nil)
         IO.copy_stream(open(url,
                             content_length_proc: lambda do|t|
@@ -26,17 +22,9 @@ module Linecook
                               pbar.progress = s
                             end), f)
       end
-
-      if encrypted
-        Linecook::Crypto.new.decrypt_file(destination, dest: path)
-        FileUtils.rm_f(destination)
-      end
-    ensure
-      unlock(path)
     end
 
-
-    def self.unzip(source, dest: nil)
+    def unzip(source, dest: nil)
       puts "Extracting #{source}..."
       dest ||= File.dirname(source)
       Zip::File.open(source) do |zip_file|
@@ -47,32 +35,5 @@ module Linecook
         end
       end
     end
-
-  private
-
-    def self.acquire_lock(path)
-      attempts = 0
-      while attempts < LOCK_WAIT_TIMEOUT
-        return lock(path) unless locked?(path)
-        attempts += 1
-        sleep(1)
-      end
-    end
-
-    def self.locked?(path)
-      File.exists?(lockfile(path)) && (true if Process.kill(0, File.read(lockfile(path))) rescue false)
-    end
-
-    def self.lock(path)
-      File.write(lockfile(path), Process.pid.to_s)
-    end
-
-    def self.unlock(path)
-      FileUtils.rm_f(lockfile(path))
-    end
-
-    def self.lockfile(path)
-      "/tmp/#{File.basename(path)}-flock"
-    end
   end
 end

data/lib/linecook-gem/util/secrets.rb

@@ -0,0 +1,47 @@
+require 'json'
+
+require 'kitchen/configurable'
+
+# FIXME - overhaul linecook config
+# be able to read kitchen configs
+
+module Linecook
+  class Secrets
+
+    SECRETS_PATH = File.join(Dir.pwd, 'secrets.ejson').freeze
+
+    include Configurable
+
+    def initialize(config = {})
+      init_config(config)
+    end
+
+#    CONFIG_PATH = File.join(Dir.pwd, 'linecook.yml').freeze # File.expand_path('../../../config/config.yml', __FILE__)
+#    LINECOOK_HOME = File.expand_path('~/.linecook').freeze
+#    DEFAULT_CONFIG = {
+#      packager: {
+#        provider: :ebs,
+#        ebs: {
+#          hvm: true,
+#          size: 10,
+#          region: 'us-east-1',
+#          copy_regions: [],
+#          account_ids: []
+#        }
+#      },
+#    }
+
+  def secrets
+    @secrets ||= begin
+      secrets_path = ENV['LINECOOK_SECRETS_PATH'] || SECRETS_PATH
+      if File.exists?(secrets_path)
+        ejson_path = File.join(Gem::Specification.find_by_name('ejson').gem_dir, 'build', "#{Linecook::Config.platform}-amd64", 'ejson' )
+        command = "#{ejson_path} decrypt #{secrets_path}"
+        secrets = JSON.load(`sudo #{command}`)
+        secrets.deep_symbolize_keys
+      else
+        {}
+      end
+    end
+  end
+end

data/lib/linecook-gem/version.rb

@@ -1,3 +1,3 @@
 module Linecook
-  VERSION = '0.6.10'
+  VERSION = '0.7.0'
 end

data/man/LINECOOK.1

@@ -1,139 +1,170 @@
-.TH LINECOOK 1 "December 2015" Unix "User Manuals"
+.TH LINECOOK 1 "June 2016" Unix "User Manuals"
 .SH NAME
 .PP
-linecook \- Linux system image builder
+linecook \- Linux system image builder based on test kitchen
 .SH SYNOPSIS
 .PP
-linecook setup \- interactive setup
-.PP
 linecook help [\fB\fCCOMMAND\fR]\- for specific command help
 .SH DESCRIPTION
 .PP
-Linecook builds system images utilizing overlayfs, squashfs, and linux containers via LXC. Currently, linecook only natively supports chef for provisioning, but using packer with a null resource, any of the mechanisms supported by packer are also supported by linecook.
+Linecook is a workflow tool that allows you to use test kitchen to build generic system images. Linecook works with arbitrary test\-kitchen provisioners, and generations a neutral format that can be packaged into specific output formats.
+.PP
+Currently, linecook supports the following test kitchen drivers:
+.RS
+.IP \(bu 2
+kitchen\-docker
+.RE
+.PP
+And linecook uses packer to generate output. It currently supports:
+.RS
+.IP \(bu 2
+AMIs via packer's ebs_chroot builder.
+.RS
+.IP \(bu 2
+The amis may also update a TXT record in a route53 zone once the build is complete
+.RE
+.IP \(bu 2
+squashfs, a provider neutral format.
+.RE
 .PP
-Linecook is intended to serve 3 main purposes:
+Linecook builds may be saved and annotated according to the folliwng convention:
 .RS
 .IP \(bu 2
-Providing a simple, portable image building process that is useable both in baremetal and cloud deployments.
+name \- a descriptive name for the build, based on the name of the test kitchen suite.
 .IP \(bu 2
-Enabling a means of simple local image development with high production efficacy on Linux and OS X.
+group \- an arbitrary grouping of suites, intended to group builds by branches.
 .IP \(bu 2
-Simplifying continuous integration and testing of linux systems.
+tag \- a numeric tag for a build, intended to increment. If 'latest' is specified when resolving a build, the latest uploaded build is used.
 .RE
+.PP
+These three attributes are composed to make a build id in a very simple manor:
+.RS
+.IP \(bu 2
+name is always required
+.IP \(bu 2
+if group is specified, it will be joined with name using a '\-' character.
+.IP \(bu 2
+if tag is specified, it will be joined with the name and the gorup using a '\-' character.
+.IP \(bu 2
+For example, the resulting id for a base build on the master group, with id of '5' would be 'base\-master\-5'. If this is the latest build for the base\-master group, 'base\-master\-latest' will resolve to this.
+.RE
+.PP
+If linecook uploads a build, it will always encrypt it using rbnacl.
 .SH USAGE
 .PP
+To test linecook builds locally, it is best to use test kitchen directly:
+.PP
 .RS
 .nf
-linecook bake SPEC [\-n \-\-name `NAME`] [\-s \-\-snapshot]
-  \-\-name \- The name
-  \-\-snapshot \- Snapshot the resulting image for later use
-  \-\-encrypt \- Encrypt the snapshot using the configured key. Implies snapshot.
-  \-\-upload \- Upload the resulting image to the configured destination. Implies snapshot.
-  \-\-all \- Snapshot, encrypt, and upload the resulting image.
-  Build a linecook image defined by SPEC, with an optional name to help identify it. The default will be the SPEC name
-linecook builder
-  start \- start a new builder
-  stop \- stop a running builder
-  info \- show the info about the builder
-  ip \- show the builder's ip
-linecook config
-  setup
-  check \- validate config
-  show
-linecook build
-  list
-  info NAME
-  ip NAME
-  stop NAME
-linecook image
-  list
-  keygen \- generate a new secret key for image encryption
-  fetch
-  find [`REGEX`] \- list available remote images filtered by an optional regex
-linecook ami [`image`] [\-r \-\-region `REGION1,REGION2`] [\-x \-\-xen\-type `PV|HVM`] [\-r \-\-root\-size GIGABYTES] \- create an AMI (Amazon Machine Image) from a snapshot.
+bundle exec kitchen converge [SUITE NAME]
 .fi
 .RE
-.SH CONFIGURATION
 .PP
-Describe config file here once it's been determined
-.SH PROVISIONERS
+For more specific usage, use \fIlinecook help\fP
+.SH CONFIGURATION
 .PP
-Linecook includes an embedded chef\-zero server, and uses the chef\-provisioner
-\[la]https://rubygems.org/gems/chef-provisioner\[ra] and chefdepartie
-\[la]https://rubygems.org/gems/chefdepartie\[ra] gems to have first\-class support for local chef\-zero builds.
+See test kitchen's documentation for configuring suites and provisioners.
+.SH KITCHEN EXTENSIONS
 .PP
-However, if you're not using chef or don't want to use chef\-zero, linecook can be used seamlessly with packer
-\[la]https://www.packer.io\[ra], and supports any of the Linux\-based provisioners. This includes:
+kitchen.yml is extended to support the following additional attributes:
+.TP
+\fBinherit\fP
+Inherit from a previous linecook build. This saves time if there are several builds based on the same ancestor.
 .RS
 .IP \(bu 2
-Chef\-solo
+name \- the name of the build to inherit
 .IP \(bu 2
-Chef\-client (with a real chef server)
+group \- the group / branch of the build to inherit
 .IP \(bu 2
-Ansible
+tag \- the explicit tag, or 'latest' to discovery the latest tag.
+.RE
+.SH PACKAGER
+.PP
+Right now there are two packagers supported. The interface may change.
+.TP
+\fBsquashfs\fP
+Package the resulting build as a squashfs image.
+.RS
 .IP \(bu 2
-Puppet (masterless or server)
+\fIexcludes\fP \- a list of glob expressions to exclude from the archive
 .IP \(bu 2
-Salt
+\fIdistro\fP \- inherit a specific set of presets for paths to exclude by distro. Currently only ubuntu is supported.
 .IP \(bu 2
-Plain old shell scripts
+outdir \- the output directory for the image
 .RE
-.PP
-See the packer documentation for how to configure these provisioners.
-.PP
-To use a packerfile with linecook, just leave out the 'builder' section, or have the builder section be an empty array. You need only specify the 'provisioner' section, and other sections may cause errors. Linecook will automatically insert a null builder with the appropriate connection string for you.
-.PP
-Linecook with packer is a powerful combination, as it allows you to leverage packer's 'null builder' to take advantage of all of the provisioners packer already has really good support for. The result is an intermediate image format (squashfs) that can be easily applied to any target.
-.SH FILES
 .TP
-\fI\&./linecook.yml\fP
-Local config file. Gets deep merged over the system config file. If not explicitly specified, found from current directory if exists.
-.TP
-\fI~/linecook/config.yml\fP
-The system wide configuration file, base or 'common' configuration. Other configurations get deep merged on top of this.
-.SH DEPENDENCIES
-.PP
-Ruby 2.0 or greater, gem, and bundler.
-.PP
-Does not work and will never work on Windows.
-.SS Linux
-.PP
-Only tested on Gentoo and Ubuntu
+\fBpacker\fP
+Package an AMI using packer. Currently only AMIs are supported, but any packer builder could be implemented relatively easily.
 .RS
 .IP \(bu 2
-lxc >= 1.0.7
+hvm \- build an HVM instance (defaults to true).
 .IP \(bu 2
-brutils
+root_size \- the size of the root volume to snapshot for the AMI (in GB).
 .IP \(bu 2
-dnsmasq
+region \- the region to build the AMI in.
 .IP \(bu 2
-iptables with masquerade support
+copy_regions \- additional regions to copy the AMI to.
 .IP \(bu 2
-Linux 3.19 or greater with support for cgroups, and netfilter as described by lxc and iptables for NAT.
+account_ids \- a list of account ids that are permitted to launch this AMI.
+.IP \(bu 2
+ami \- details for storing the AMI ID in a DNS TXT record on route53.
+.RS
+.IP \(bu 2
+update_txt \- should a TXT record be written? (true or false)
+.IP \(bu 2
+regions \- a dictionary of regional aliases
+.IP \(bu 2
+domain \- the route53 domain to write to
+.IP \(bu 2
+zone \- the zone within the domain.
+.RE
 .RE
-.SS OS X
+.SH SECRETS
+.PP
+Linecook will look for secrets in config.ejson. In particular:
+.TP
+\fBimagekey\fP
+the key to use when encrypting images. Generate one with the \fIimage keygen\fP command.
+.TP
+\fBaws\fP
+This is used access to S3, as well as to create EBS based AMIs and update TXT records on route53. A sample IAM policy is provided in the github repo.
 .RS
 .IP \(bu 2
-OS X 10.10 or later (Hypervisor.framework required for Xhyve)
+{ "s3": { "bucket" : "name" } } can be used to set the name of the bucket
+.IP \(bu 2
+{ "access_key" : "ACCESS_KEY" } can be used to set the access key for the IAM user associated with the profile with the necessary access.
+.IP \(bu 2
+{ "secret_key" : "ACCESS_KEY" } can be used to set the secret key for the IAM user associated with the profile with the necessary access.
 .RE
-.SH QUIRKS
-.SS Xhyve
+.TP
+\fBchef\fP
+To decrypt data bags securely, you can set the \fIencrypted_data_bag_secret\fP here. Make sure any newlines are replaced with \[rs]n.
+.SH PROVISIONERS
+.PP
+Currently only the docker driver is supported for provisioning. You must have docker installed to use this provisioner.
+.SH DEPENDENCIES
+.PP
+\fBCommon\fP
 .RS
 .IP \(bu 2
-Xhyve requires root privileges until 
-\[la]https://github.com/mist64/xhyve/issues/60\[ra] is resolved. Linecook will setuid on the xhyve binary.
+Ruby 2.0 or greater, gem, and bundler.
 .RE
-.SS Overlayfs
+.PP
+\fBLinux\fP
 .RS
 .IP \(bu 2
-Overlayfs doesn't support unix domain sockets (yet), so anything using a unix domain socket outside of the /run tree should do manually symlink to /run.
+mksquashfs \- to generate squashfs output.
+.RE
+.PP
+\fBOS X\fP
+.RS
 .IP \(bu 2
-Config file will allow you to explicitly mount tmpfs over things that don't do /run if you need to create unix domain sockets
+docker for mac
 .RE
 .SH BUGS
 .PP
-Report bugs against github.com/dalehamel/linecook
+Report bugs against github.com/shopify/linecook\-gem
 .SH AUTHOR
 .PP
 Dale Hamel 
-\[la]dale.hamel@srvthe.net\[ra]
+\[la]dale.hamel@shopify.com\[ra]