bosh_cli 0.16 → 0.17

data/lib/cli/commands/blob_management.rb

@@ -0,0 +1,47 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli::Command
+  class BlobManagement < Base
+
+    # Prints out blobs status
+    def status
+      blob_manager.print_status
+    end
+
+    # Adds blob to managed blobs
+    # @param [String] local_path Local file path
+    # @param [optional, String] blob_dir Directory to store blob in, relative
+    #     to blobs dir
+    def add(local_path, blob_dir = nil)
+      blob_path = File.basename(local_path)
+      if blob_dir
+        # We don't need about blobs prefix,
+        # but it might be handy for people who rely on auto-completion
+        if blob_dir[0..5] == "blobs/"
+          blob_dir = blob_dir[6..-1]
+        end
+        blob_path = File.join(blob_dir, blob_path)
+      end
+      blob_manager.add_blob(local_path, blob_path)
+    end
+
+    # Uploads all blobs that need to be uploaded
+    def upload
+      blob_manager.print_status
+
+      blob_manager.blobs_to_upload.each do |blob|
+        nl
+        if confirmed?("Upload blob #{blob.yellow}?")
+          blob_manager.upload_blob(blob)
+        end
+      end
+    end
+
+    # Syncs blobs with blobstore
+    def sync
+      blob_manager.sync
+      blob_manager.print_status
+    end
+
+  end
+end

data/lib/cli/commands/release.rb

@@ -8,8 +8,12 @@ module Bosh::Cli::Command
     include Bosh::Cli::VersionCalc
 
     def init(base=nil, *options)
-      flags = options.inject({}) { |h, option| h[option] = true; h }
-      git = flags.delete("--git")
+      if base[0..0] == "-"
+        # TODO: need to add some option parsing helpers to avoid that
+        options.unshift(base)
+        base = nil
+      end
+      git = options.include?("--git")
 
       if base
         FileUtils.mkdir_p(base) unless Dir.exist?(base)
@@ -17,17 +21,15 @@ module Bosh::Cli::Command
       end
 
       err("Release already initialized") if in_release_dir?
-
       git_init if git
 
-      %w[jobs packages src blobs].each do |dir|
+      %w[config jobs packages src blobs].each do |dir|
         FileUtils.mkdir(dir)
       end
 
-      # initialize an empty blob_index file
-      blobs = {}
-      File.open("blob_index.yml", "w") do |f|
-        YAML.dump(blobs, f)
+      # Initialize an empty blobs index
+      File.open(File.join("config", "blobs.yml"), "w") do |f|
+        YAML.dump({}, f)
       end
 
       say("Release directory initialized".green)
@@ -39,11 +41,23 @@ module Bosh::Cli::Command
         say("error running 'git init':\n#{out}")
       else
         File.open(".gitignore", "w") do |f|
-          f.puts("config/dev.yml\nconfig/private.yml")
-          f.puts("blobs\nreleases/*.tgz")
-          f.puts("dev_releases\n.dev_builds")
-          f.puts(".final_builds/jobs/**/*.tgz")
-          f.puts(".final_builds/packages/**/*.tgz")
+          f << <<-EOS.gsub(/^\s{10}/, '')
+          config/dev.yml
+          config/private.yml
+          releases/*.tgz
+          dev_releases
+          .blobs
+          blobs
+          .dev_builds
+          .idea
+          .DS_Store
+          .final_builds/jobs/**/*.tgz
+          .final_builds/packages/**/*.tgz
+          *.swp
+          *~
+          *#
+          #*
+          EOS
         end
       end
     rescue Errno::ENOENT
@@ -202,11 +216,18 @@ module Bosh::Cli::Command
         exit(1)
       end
 
-      if !force
-        check_dirty_blobs
-        check_if_dirty_state
+      blob_manager.sync
+      if blob_manager.dirty?
+        blob_manager.print_status
+        if force
+          say("Proceeding with dirty blobs as '--force' is given".red)
+        else
+          err("Please use '--force' or upload new blobs")
+        end
       end
 
+      check_if_dirty_state unless force
+
       confirmation = "Are you sure you want to " +
                      "generate #{'final'.red} version? "
 

data/lib/cli/runner.rb

@@ -537,24 +537,28 @@ module Bosh::Cli
         route  :cloud_check, :perform
       end
 
-      command :upload_blob do
-        usage  "upload blob <blobs>"
-        desc   "Upload given blob to the blobstore"
-        option "--force", "bypass duplicate checking"
-        route  :blob, :upload_blob
+      command :add_blob do
+        usage  "add blob <local_path> [<blob_dir>]"
+        desc   "Add a local file as BOSH blob"
+        route  :blob_management, :add
+      end
+
+      command :upload_blobs do
+        usage  "upload blobs"
+        desc   "Upload new and updated blobs to the blobstore"
+        route  :blob_management, :upload
       end
 
       command :sync_blobs do
         usage "sync blobs"
         desc  "Sync blob with the blobstore"
-        option "--force", "overwrite all local copies with the remote blob"
-        route :blob, :sync_blobs
+        route :blob_management, :sync
       end
 
-      command :blobs do
+      command :blobs_status do
         usage "blobs"
-        desc  "Print blob status"
-        route :blob, :blobs_info
+        desc  "Print current blobs status"
+        route :blob_management, :status
       end
 
       def define_plugin_commands

data/lib/cli/templates/help_message.erb

@@ -70,6 +70,7 @@ Remote access
 <%= command_usage(:ssh_cleanup) %>
 
 Blob
-<%= command_usage(:upload_blob) %>
+<%= command_usage(:add_blob) %>
+<%= command_usage(:upload_blobs) %>
 <%= command_usage(:sync_blobs) %>
-<%= command_usage(:blobs) %>
+<%= command_usage(:blobs_status) %>

data/lib/cli/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module Cli
-    VERSION = "0.16"
+    VERSION = "0.17"
   end
 end

data/spec/unit/blob_manager_spec.rb

@@ -0,0 +1,291 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "spec_helper"
+
+describe Bosh::Cli::BlobManager do
+
+  def make_manager(release)
+    Bosh::Cli::BlobManager.new(release)
+  end
+
+  before(:each) do
+    @blobstore = mock("blobstore")
+    @dir = Dir.mktmpdir
+    @src_dir = FileUtils.mkdir(File.join(@dir, "src"))
+    @config_dir = File.join(@dir, "config")
+    FileUtils.mkdir(@config_dir)
+    @blobs_dir = File.join(@dir, "blobs")
+    @release = mock("release", :dir => @dir, :blobstore => @blobstore)
+  end
+
+  describe "initialization" do
+    it "fails if 'src' directory is missing" do
+      FileUtils.rm_rf(@src_dir)
+      expect {
+        make_manager(@release)
+      }.to raise_error("`src' directory is missing")
+    end
+
+    it "fails if blobstore is not configured" do
+      @release.stub!(:blobstore).and_return(nil)
+      expect {
+        make_manager(@release)
+      }.to raise_error("Blobstore is not configured")
+    end
+
+    it "creates necessary directories in release dir" do
+      make_manager(@release)
+      File.directory?(File.join(@dir, "blobs")).should be_true
+      File.directory?(File.join(@dir, ".blobs")).should be_true
+    end
+
+    it "has dirty flag cleared and upload list empty" do
+      manager = make_manager(@release)
+      manager.dirty?.should be_false
+      manager.blobs_to_upload.should == []
+    end
+
+    it "doesn't like bad index file'" do
+      File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
+        YAML.dump("string", f)
+      end
+      expect {
+        make_manager(@release)
+      }.to raise_error(/Incorrect file format/)
+    end
+
+    it "migrates legacy index file" do
+      legacy_file = File.join(@release.dir, "blob_index.yml")
+      test_hash = { "foo" => "bar" }
+
+      File.open(legacy_file, "w") do |f|
+        YAML.dump({ "foo" => "bar" }, f)
+      end
+
+      make_manager(@release)
+      File.exists?(legacy_file).should be_false
+      YAML.load_file(File.join(@config_dir, "blobs.yml")).should == test_hash
+    end
+  end
+
+  describe "adding a blob" do
+    before(:each) do
+      @manager = make_manager(@release)
+      @blob = Tempfile.new("blob")
+      @blob.write("blob contents")
+      @blob.close
+    end
+
+    it "cannot add non-existing file" do
+      expect {
+        @manager.add_blob("tmp/foobar.tgz", "test")
+      }.to raise_error("File `tmp/foobar.tgz' not found")
+    end
+
+    it "cannot add directory" do
+      tmp_dir = Dir.mktmpdir
+      expect {
+        @manager.add_blob(tmp_dir, "test")
+      }.to raise_error("`#{tmp_dir}' is a directory")
+    end
+
+    it "cannot use absolute path as blob destination" do
+      expect {
+        @manager.add_blob(@blob.path, "/test")
+      }.to raise_error("Blob path should be a relative path")
+    end
+
+    it "cannot use 'blobs' prefix for blob destination" do
+      expect {
+        @manager.add_blob(@blob.path, "blobs/foo/bar")
+      }.to raise_error("Blob path should not start with `blobs/'")
+    end
+
+    it "cannot use directory as blob destination" do
+      foo_dir = File.join(@blobs_dir, "foo")
+      FileUtils.mkdir(foo_dir)
+      expect {
+        @manager.add_blob(@blob.path, "foo")
+      }.to raise_error("`#{foo_dir}' is a directory, " +
+                       "please pick a different path")
+    end
+
+    it "adds blob to a blobs directory" do
+      blob_dst = File.join(@blobs_dir, "foo", "blob")
+      @manager.add_blob(@blob.path, "foo/blob")
+      File.exists?(blob_dst).should be_true
+      File.read(blob_dst).should == "blob contents"
+      File.symlink?(blob_dst).should be_false
+      File.stat(blob_dst).mode.to_s(8)[-4..-1].should == "0644"
+      File.exists?(@blob.path).should be_true # original still exists
+
+      @manager.process_blobs_directory
+      @manager.dirty?.should be_true
+      @manager.new_blobs.should == %w(foo/blob)
+      @manager.updated_blobs.should == []
+    end
+
+    it "prevents double adds of the same file" do
+      @manager.add_blob(@blob.path, "foo/blob")
+      expect {
+        @manager.add_blob(@blob.path, "foo/blob")
+      }.to raise_error(/Already tracking/)
+    end
+
+    it "updates blob" do
+      new_blob = Tempfile.new("new-blob")
+      new_blob.write("foobar")
+      new_blob.close
+      blob_dst = File.join(@blobs_dir, "foo", "blob")
+      @manager.add_blob(@blob.path, "foo/blob")
+      File.read(blob_dst).should == "blob contents"
+      @manager.add_blob(new_blob.path, "foo/blob")
+      File.read(blob_dst).should == "foobar"
+    end
+  end
+
+  describe "downloading a blob" do
+    it "cannot download blob if path is not in index" do
+      @manager = make_manager(@release)
+
+      expect {
+        @manager.download_blob("foo")
+      }.to raise_error(/Unknown blob path/)
+    end
+
+    it "downloads blob from blobstore" do
+      index = {
+        "foo" => {
+          "size" => "1000",
+          "object_id" => "deadbeef",
+          "sha" => Digest::SHA1.hexdigest("blob contents")
+        }
+      }
+
+      tmp_file = Tempfile.new("mock-blob")
+      Tempfile.stub!(:new).with("bosh-blob").and_return(tmp_file)
+
+      File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
+        YAML.dump(index, f)
+      end
+
+      @manager = make_manager(@release)
+      @blobstore.should_receive(:get).with("deadbeef",
+                                           an_instance_of(Tempfile)).
+        and_return { tmp_file.write("blob contents"); tmp_file.close }
+
+      path = @manager.download_blob("foo")
+      File.read(path).should == "blob contents"
+    end
+  end
+
+  describe "uploading a blob" do
+    before(:each) do
+      @manager = make_manager(@release)
+    end
+
+    it "needs blob path to exist" do
+      expect {
+        @manager.upload_blob("foo")
+      }.to raise_error(/doesn't exist/)
+    end
+
+    it "doesn't follow symlinks" do
+      FileUtils.touch(File.join(@dir, "blob"))
+      FileUtils.ln_s(File.join(@dir, "blob"), File.join(@blobs_dir, "foo"))
+      expect {
+        @manager.upload_blob("foo")
+      }.to raise_error(/is a symlink/)
+    end
+
+    it "uploads file to a blobstore, updates index and symlinks blob" do
+      new_blob = File.join(@dir, "blob")
+      File.open(new_blob, "w") { |f| f.write("test blob") }
+      @manager.add_blob(new_blob, "foo")
+
+      @blobstore.should_receive(:create).and_return("deadbeef")
+      @manager.upload_blob("foo").should == "deadbeef"
+
+      blob_dst = File.join(@blobs_dir, "foo")
+      checksum = Digest::SHA1.hexdigest("test blob")
+
+      File.symlink?(blob_dst).should be_true
+      File.readlink(blob_dst).should == File.join(@dir, ".blobs", checksum)
+      File.read(blob_dst).should == "test blob"
+    end
+  end
+
+  describe "syncing blobs" do
+    it "includes several steps" do
+      @manager = make_manager(@release)
+      @manager.should_receive(:remove_symlinks).ordered
+      @manager.should_receive(:process_blobs_directory).ordered
+      @manager.should_receive(:process_index).ordered
+      @manager.sync
+    end
+
+    it "processes blobs directory" do
+      @manager = make_manager(@release)
+      @blobstore.stub!(:create).and_return("new-object-id")
+
+      new_blob = Tempfile.new("new-blob")
+      new_blob.write("test")
+      new_blob.close
+
+      @manager.add_blob(new_blob.path, "foo")
+      @manager.process_blobs_directory
+      @manager.new_blobs.should == %w(foo)
+
+      @manager.add_blob(new_blob.path, "bar")
+      @manager.process_blobs_directory
+      @manager.new_blobs.sort.should == %w(bar foo)
+
+      @manager.upload_blob("bar")
+
+      new_blob.open
+      new_blob.write("stuff")
+      new_blob.close
+
+      @manager.add_blob(new_blob.path, "bar")
+      @manager.process_blobs_directory
+      @manager.new_blobs.sort.should == %w(foo)
+      @manager.updated_blobs.sort.should == %w(bar)
+    end
+
+    it "downloads missing blobs" do
+      index = {
+        "foo" => {
+          "size" => 1000,
+          "sha" => Digest::SHA1.hexdigest("foo"),
+          "object_id" => "da-foo"
+        },
+        "bar" => {
+          "size" => 500,
+          "sha" => Digest::SHA1.hexdigest("bar"),
+          "object_id" => "da-bar"
+        }
+      }
+
+      File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
+        YAML.dump(index, f)
+      end
+
+      foo = Tempfile.new("foo")
+      foo.write("foo")
+      foo.close
+
+      bar = Tempfile.new("bar")
+      bar.write("bar")
+      bar.close
+
+      @manager = make_manager(@release)
+      @manager.should_receive(:download_blob).with("foo").and_return(foo.path)
+      @manager.should_receive(:download_blob).with("bar").and_return(bar.path)
+
+      @manager.process_index
+
+      File.read(File.join(@blobs_dir, "foo")).should == "foo"
+      File.read(File.join(@blobs_dir, "bar")).should == "bar"
+    end
+  end
+end