bento-ya 0.0.1

data/lib/bento/build_remote.rb

@@ -0,0 +1,112 @@
+require 'buildkit'
+
+class BuildRemoteRunner
+  include Common
+
+  attr_reader :bento_version, :dry_run, :platforms, :queue, :token, :s3_endpoint, :s3_bucket
+
+  def initialize(opts)
+    @dry_run = opts.dry_run
+    @bento_version = opts.override_version
+    @platforms = opts.platforms
+    @org = ENV['BUILDKITE_ORG']
+    @queue = ENV['BUILDKITE_QUEUE']
+    @token = ENV['BUILDKITE_TOKEN']
+    @s3_endpoint ||= ENV['BENTO_S3_ENDPOINT'] # 'https://s3.amazonaws.com''
+    @s3_bucket ||= ENV['BENTO_S3_BUCKET'] # 'opscode-vm-bento''
+  end
+
+  def start
+    msg_pre = dry_run ? "DRY RUN: " : ""
+
+    banner("#{msg_pre}Scheduling builds for Bento Version: #{bento_version}")
+    time = Benchmark.measure do
+      builds['public'].each do |platform, versions|
+        versions.each do |version, archs|
+          archs.each do |arch|
+            builds['providers'].each do |provider|
+              build(platform, version, arch, provider, bento_version, dry_run)
+            end
+          end
+        end
+      end
+    end
+    banner("#{msg_pre}Scheduling finished in #{duration(time.real)}.")
+  end
+
+  private
+
+  def bk_uri
+    "v1/organizations/#{org}"
+  end
+
+  def build(platform, version, arch, provider, bento_version, dry_run)
+    plat =  platform.include?('omnios') ? "#{platform}-#{version}" : "#{platform}-#{version}-#{arch}"
+    atlas_name = /(.*)64/.match(arch) ? plat.chomp("-#{arch}") : plat
+    no_shared = builds['no_shared_folder'].include?("#{platform}-#{version}-#{arch}-#{provider}") || builds['no_shared_folder'].include?("#{platform}-#{version}-#{arch}-all")
+    test_shared = no_shared ? "0" : "1"
+    unless builds['broken'].include?("#{plat}-#{provider}") || builds['broken'].include?("#{plat}-all")
+      build = {
+        "commit"=> "HEAD",
+        "branch"=> "master",
+        "message"=> "#{plat}-#{provider}",
+        "env"=> {
+          "ATLAS_ORG" => atlas_org,
+          "ATLAS_TOKEN" => atlas_token,
+          "ATLAS_NAME" => atlas_name,
+          "AWS_ACCESS_KEY_ID" => ENV['AWS_ACCESS_KEY_ID'],
+          "AWS_SECRET_ACCESS_KEY" => ENV['AWS_SECRET_ACCESS_KEY'],
+          "AWS_REGION" => 'us-east-1',
+          "BENTO_TEST_SHARED_FOLDER" => test_shared,
+          "BENTO_UPLOAD" => bento_upload,
+          "BENTO_VERSION" => bento_version,
+          "BENTO_S3_ENDPOINT" => s3_endpoint,
+          "BENTO_S3_BUCKET" => s3_bucket,
+          "BENTO_PROVIDERS" => provider,
+          "PLATFORM" => plat,
+        }
+     }
+     puts "  - #{plat}-#{provider}"
+     client.post("/#{bk_uri}/projects/bento-#{provider.chomp('-iso')}/builds", build) unless dry_run
+    end
+  end
+
+  def client
+    Buildkit.new(token: token)
+  end
+
+  def create_project
+    project = {
+      "name"=> project,
+      "repository"=>"git@github.com:chef/bento.git",
+      "env"=>{"BUILDKITE_BIN_PATH" => "/usr/local/opt/buildkite-agent/bin"},
+      "steps"=>
+      [{"type"=>"script",
+        "name"=>"build",
+        "command"=>"build.sh",
+        "agent_query_rules"=>["queue=#{queue}"]
+       }
+      ]
+    }
+
+    puts "Creating Project #{project['name']}"
+    client.post("/#{bk_uri}/projects", project)
+  end
+
+  def delete_project
+    banner("Deleting Project #{project['name']}")
+    client.delete("/#{bk_uri}/projects/#{project}")
+  end
+
+  def get_builders(provider)
+    organization = client.organization(org)
+    agents = organization.rels[:agents].get.data
+
+    banner("Available Builders for provider: #{provider}")
+    agents.each do |agent|
+      if agent[:meta_data].include? "#{provider.chomp('-iso')}=true"
+        puts "  - #{agent[:name]}"
+      end
+    end
+  end
+end

data/lib/bento/buildmetadata.rb

@@ -0,0 +1,70 @@
+require 'bento/common'
+
+class BuildMetadata
+  include Common
+
+  def initialize(template, build_timestamp, override_version)
+    @template = template
+    @build_timestamp = build_timestamp
+    @override_version = override_version
+  end
+
+  def read
+    {
+      name:             name,
+      version:          version,
+      build_timestamp:  build_timestamp,
+      git_revision:     git_revision,
+      #git_status:       git_clean? ? "clean" : "dirty"
+      box_basename:     box_basename,
+      template:         template_vars.fetch("template", UNKNOWN),
+      cpus:             cpus.to_s,
+      memory:           memory.to_s,
+    }
+  end
+
+  private
+
+  UNKNOWN = "__unknown__".freeze
+
+  attr_reader :template, :build_timestamp, :override_version
+
+  def box_basename
+    "#{name.gsub("/", "__")}-#{version}"
+  end
+
+  def git_revision
+    sha = %x{git rev-parse HEAD}.strip
+  end
+
+  def git_clean?
+    %x{git status --porcelain}.strip.empty?
+  end
+
+  def merged_vars
+    @merged_vars ||= begin
+      if File.exist?("#{template}.variables.json")
+        template_vars.merge(JSON.load(IO.read("#{template}.variables.json")))
+      else
+        template_vars
+      end
+    end
+  end
+
+  def name
+    merged_vars.fetch("name", template)
+  end
+
+  def template_vars
+    @template_vars ||= JSON.load(IO.read("#{template}.json")).fetch("variables")
+  end
+
+  def version
+    if override_version
+       override_version
+    else
+    merged_vars.fetch("version", "#{UNKNOWN}.TIMESTAMP").
+      rpartition(".").first.concat(".#{build_timestamp}")
+    end
+  end
+end

data/lib/bento/cli.rb

@@ -0,0 +1,228 @@
+require 'optparse'
+require 'ostruct'
+
+require 'bento/common'
+require 'bento/build'
+require 'bento/build_remote'
+require 'bento/delete'
+require 'bento/normalize'
+require 'bento/release'
+require 'bento/revoke'
+require 'bento/test'
+require 'bento/upload'
+
+
+class Options
+
+  NAME = File.basename($0).freeze
+
+  def self.parse(args)
+    options = OpenStruct.new
+    options.templates = calculate_templates("*.json")
+
+    global = OptionParser.new do |opts|
+      opts.banner = "Usage: #{NAME} [SUBCOMMAND [options]]"
+      opts.separator ""
+      opts.separator <<-COMMANDS.gsub(/^ {8}/, "")
+        build        :   build one or more templates
+        build_remote :   build one or more templates via buildkite
+        help         :   prints this help message
+        list         :   list all templates in project
+        normalize    :   normalize one or more templates
+        test         :   test one or more builds with kitchen
+        upload       :   upload one or more builds to Atlas and S3
+        release      :   release a version of a box on Atlas
+        revoke       :   revoke a version of a box on Atlas
+        delete       :   delete a version of a box from Atlas
+      COMMANDS
+    end
+
+    platforms_argv_proc = proc { |options|
+      options.platforms = builds['public'] unless args.empty?
+    }
+
+    templates_argv_proc = proc { |options|
+      options.templates = calculate_templates(args) unless args.empty?
+
+      options.templates.each do |t|
+        if !File.exists?("#{t}.json")
+          $stderr.puts "File #{t}.json does not exist for template '#{t}'"
+          exit(1)
+        end
+      end
+    }
+
+    box_version_argv_proc = proc { |options|
+      options.box = ARGV[0]
+      options.version = ARGV[1]
+    }
+
+    subcommand = {
+      help: {
+        parser: OptionParser.new {},
+        argv: proc { |options|
+          puts global
+          exit(0)
+        }
+      },
+      build: {
+        class: BuildRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} build [options] TEMPLATE[ TEMPLATE ...]"
+
+          opts.on("-n", "--dry-run", "Dry run (what would happen)") do |opt|
+            options.dry_run = opt
+          end
+
+          opts.on("-d", "--[no-]debug", "Run packer with debug output") do |opt|
+            options.debug = opt
+          end
+
+          opts.on("-o BUILDS", "--only BUILDS", "Only build some Packer builds") do |opt|
+            options.builds = opt
+          end
+
+          opts.on("-e BUILDS", "--except BUILDS", "Build all Packer builds except these") do |opt|
+            options.except = opt
+          end
+
+          opts.on("-m MIRROR", "--mirror MIRROR", "Look for isos at MIRROR") do |opt|
+            options.mirror = opt
+          end
+
+          opts.on("-H", "--headless", "Run providers as headless") do |opt|
+            options.headless = opt
+          end
+
+          opts.on("-v VERSION", "--version VERSION", "Override the version set in the template") do |opt|
+            options.override_version = opt
+          end
+        },
+        argv: templates_argv_proc
+      },
+      build_remote: {
+        class: BuildRemoteRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} build_remote [options] [PLATFORM ...]"
+
+          opts.on("-v VERSION", "--version VERSION", "Override the version set in the template") do |opt|
+            options.override_version = opt
+          end
+
+          opts.on("--dry-run", "Show me what you got") do |opt|
+            options.dry_run = opt
+          end
+        },
+        argv: platforms_argv_proc
+      },
+      list: {
+        class: ListRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} list [TEMPLATE ...]"
+        },
+        argv: templates_argv_proc
+      },
+      normalize: {
+        class: NormalizeRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} normalize TEMPLATE[ TEMPLATE ...]"
+
+          opts.on("-d", "--[no-]debug", "Run packer with debug output") do |opt|
+            options.debug = opt
+          end
+        },
+        argv: templates_argv_proc
+      },
+      test: {
+        class: TestRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} test [options]"
+
+          opts.on("-f", "--shared-folder", "Enable shared folder") do |opt|
+            options.shared_folder = opt
+          end
+
+          opts.on("-p", "--provisioner PROVISIONER", "Use a specfic provisioner") do |opt|
+            options.provisioner = opt
+          end
+        },
+        argv: Proc.new {}
+      },
+      upload: {
+        class: UploadRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} upload"
+        },
+        argv: box_version_argv_proc
+      },
+      release: {
+        class: ReleaseRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} release BOX VERSION"
+        },
+        argv: box_version_argv_proc
+      },
+      revoke: {
+        class: RevokeRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} revoke BOX VERSION"
+        },
+        argv: box_version_argv_proc
+      },
+      delete: {
+        class: DeleteRunner,
+        parser: OptionParser.new { |opts|
+          opts.banner = "Usage: #{NAME} delete BOX VERSION"
+        },
+        argv: box_version_argv_proc
+      }
+    }
+
+    global.order!
+    command = args.empty? ? :help : ARGV.shift.to_sym
+    subcommand.fetch(command).fetch(:parser).order!
+    subcommand.fetch(command).fetch(:argv).call(options)
+
+    options.command = command
+    options.klass = subcommand.fetch(command).fetch(:class)
+
+    options
+  end
+
+  def self.calculate_templates(globs)
+    Array(globs).
+      map { |glob| result = Dir.glob(glob); result.empty? ? glob : result }.
+      flatten.
+      sort.
+      delete_if { |file| file =~ /\.variables\./ }.
+      map { |template| template.sub(/\.json$/, '') }
+  end
+end
+
+class ListRunner
+
+  include Common
+
+  attr_reader :templates
+
+  def initialize(opts)
+    @templates = opts.templates
+  end
+
+  def start
+    templates.each { |template| puts template }
+  end
+end
+
+class Runner
+
+  attr_reader :options
+
+  def initialize(options)
+    @options = options
+  end
+
+  def start
+    options.klass.new(options).start
+  end
+end

data/lib/bento/common.rb

@@ -0,0 +1,160 @@
+require 'benchmark'
+require 'fileutils'
+require 'json'
+require 'tempfile'
+require 'yaml'
+
+MEGABYTE = 1024.0 * 1024.0
+
+module Common
+
+  def banner(msg)
+    puts "==> #{msg}"
+  end
+
+  def info(msg)
+    puts "    #{msg}"
+  end
+
+  def warn(msg)
+    puts ">>> #{msg}"
+  end
+
+  def duration(total)
+    total = 0 if total.nil?
+    minutes = (total / 60).to_i
+    seconds = (total - (minutes * 60))
+    format("%dm%.2fs", minutes, seconds)
+  end
+
+  def box_metadata(metadata_file)
+    metadata = Hash.new
+    file = File.read(metadata_file)
+    json = JSON.parse(file)
+
+    # metadata needed for upload:  boxname, version, provider, box filename
+    metadata['name'] = json['name']
+    metadata['version'] = json['version']
+    metadata['box_basename'] = json['box_basename']
+    metadata['tool_versions'] = json['box_basename']
+    metadata['providers'] = Hash.new
+    json['providers'].each do |provider|
+      metadata['providers'][provider['name']] = provider.reject { |k, _| k == 'name' }
+    end
+    metadata
+  end
+
+  def metadata_files
+    @metadata_files ||= compute_metadata_files
+  end
+
+  def compute_metadata_files
+    `ls builds/*.json`.split("\n")
+  end
+
+  def atlas_api
+    @atlas_api ||= 'https://atlas.hashicorp.com/api/v1'
+  end
+
+  def atlas_org
+    @atlas_org ||= ENV['ATLAS_ORG']
+  end
+
+  def atlas_token
+    @atlas_token ||= ENV['ATLAS_TOKEN']
+  end
+
+  def bento_upload
+    "1"
+  end
+
+  def bento_version
+    @bento_version ||= ENV['BENTO_VERSION']
+  end
+
+  def cpus
+    1
+  end
+
+  def memory
+    1024
+  end
+
+  def builds
+    YAML.load(File.read("builds.yml"))
+  end
+
+  def private_box?(boxname)
+    proprietary_os_list = %w(macosx sles solaris windows)
+    proprietary_os_list.any? { |p| boxname.include?(p) }
+  end
+
+  def tool_versions
+    tool_versions = Hash.new
+    path = File.join('/Applications/VMware\ Fusion.app/Contents/Library')
+    fusion_cmd = File.join(path, "vmware-vmx -v")
+    ver_hash = {
+      packer: "packer --version",
+      vagrant: "vagrant --version",
+      parallels: "prlctl --version",
+      virtualbox: "VBoxManage --version",
+      vmware_fusion: fusion_cmd
+    }
+    ver_hash.each do |tool, command|
+      cmd = Mixlib::ShellOut.new(command)
+      cmd.run_command
+      case tool
+      when /parallels/
+        ver = cmd.stdout.split(' ')[2]
+        # hash = cmd.stdout.split(' ')[3]
+        # ver = "#{semver} #{hash}"
+      when /fusion/
+        ver = cmd.stderr.split(' ')[5]
+        # hash = cmd.stderr.split(' ')[6].gsub(/^build-/,'')
+        # ver = "#{semver} \(#{hash}\)"
+      when /vagrant/
+        semver = cmd.stdout.split(' ')[1]
+        ver = "#{semver}"
+      when /virtualbox/
+        ver = cmd.stdout.split('r')[0]
+        # hash = cmd.stdout.split('r')[1].gsub(/\n/,'')
+        # ver = "#{semver} \(#{hash}\)"
+      else
+        ver = cmd.stdout.split("\n")[0]
+      end
+      tool_versions[tool] = ver
+    end
+    tool_versions
+  end
+end
+
+# module PackerExec
+
+#   def for_packer_run_with(template)
+#     Tempfile.open("#{template}-metadata.json") do |md_file|
+#       Tempfile.open("#{template}-metadata-var-file") do |var_file|
+#         write_box_metadata(template, md_file)
+#         yield md_file, var_file
+#       end
+#     end
+#   end
+
+#   def write_box_metadata(template, io)
+#     md = BuildMetadata.new(template, build_timestamp, override_version).read
+#     io.write(JSON.pretty_generate(md))
+#     io.close
+#   end
+
+#   # def write_var_file(template, md_file, io)
+#   #   md = BuildMetadata.new(template, build_timestamp, override_version).read
+
+#   #   io.write(JSON.pretty_generate({
+#   #     box_basename:     md[:box_basename],
+#   #     build_timestamp:  md[:build_timestamp],
+#   #     git_revision:     md[:git_revision],
+#   #     metadata:         md_file.path,
+#   #     version:          md[:version],
+#   #   }))
+#   #   io.close
+#   # end
+# end