bosh-gen 0.17.0 → 0.17.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NmY5YjI3MTFiNTQ5YWVlNzZiODVjNTkzMjc5ZTUyOWI3YjcwOGZiNA==
+    N2JjZDA3YzU4NWRmYmE2YmNlZjYxZDU5YzZjNWU5NzQwODRlOWIxNA==
   data.tar.gz: !binary |-
-    N2UxN2QxZTQ2MjNhMTY0YjNlMWRhZjRjYTg0ZWJmMjE5ZmQxYmZiYw==
+    MjU1MGQzNDJjYTRkYjYzNjc5OTYwYTg5NGJhZTRjYjgzMDMxZGMwZQ==
 SHA512:
   metadata.gz: !binary |-
-    MWMwN2NmNTc5OTYwYTYwMjMxODdkODMwNTRlOGE1NjhjYmNkMjViZTJhZDQx
-    NjUxOWVlNzZhMDAyMWMyOTFjNGFjN2Q4YmExNGVlZmQzNGQwMmE1MjY1OGFi
-    Njg4YTE5ODAxMWNlNGE2ZDEwZmQ2ZTA4ZmI4YWNkNWVjNDhlMzk=
+    MTZmMDA4OTQxYWMxYjk0MDk1NzIzZDViNTUxNDQxYTI1ODI2ZWUyZTBmZTNm
+    ZTRiYTliNWVlY2IxNDlhNWJkOWZjMWY1Y2U2YjAzN2E2NTJiZWRhNTZkNGJj
+    ZWZmYWY0ZjUzZDJjZGQxOTRmNTMwMTM0ODhlN2MxMTg1OTE4ZWI=
   data.tar.gz: !binary |-
-    YjBmODA3NzM3MzVhYmI4MTg3ZDYyMzQ3ZWUyMzZjODMwNGRmNGEyMjRmMzNj
-    NmRmNGZlODk2Zjk4Njc3OTllMzJjYjY1NTAyMjEyYjAyNmM1ZjZiMzM5NjU3
-    OGJhNDE3OGM2MmM2NDc5YzcyYTFiODY1YTQ4MWUwZWQ0NzM5NTg=
+    MWFhMjE5ZDZjYTMwYzkyMWQ2MDQ3ZmQ4NjAwZGIwZGZjZmJlMmM2MTRkOWVk
+    ZDY4ZjJkYTZmMTIyOTA2NWRjNWIxZWFkOTFjMGU2NGJjOTNiY2VlZTczOTdj
+    NmE0ZmYxZDU5MmJhMTRkZWI1OGNhNjdkNWU5OTY4ZDkyOTQ3ZjI=

data/ChangeLog.md

@@ -4,7 +4,9 @@ Change Log
 v0.17.0
 -------
 
-- `new --s3` - no more read-only credentials; no more encryption keys
+- `new` - no more read-only credentials; no more encryption keys
+- `new` - no more --s3/--swift flags; prompts instead
+- `share release` - easily upload a final release and get a public URL to share (v0.17.1)
 
 v0.16.0
 -------

data/README.md

@@ -1,9 +1,13 @@
 BOSH Generators
 ===============
 
-Generators for creating BOSH releases.
+Generators for creating and sharing BOSH releases.
 
-New in 0.17: Creates blobstore/bucket when creating new release.
+New in 0.17: Creates blobstore/bucket when creating new release. AWS bucket is publicly readable.
+
+If you would like to share your BOSH release with the world, you can use the [BOSH Community AWS S3 account](#share-bosh-releases).
+
+![example](http://f.cl.ly/items/3v2F43020a3N0Q1g3Z0E/bosh-gen-new.gif)
 
 Installation
 ------------
@@ -210,6 +214,66 @@ bosh upload release
 
 When you create a final release, you will first need to setup your AWS credentials in `config/final.yml`
 
+Share BOSH releases
+-------------------
+
+To share your BOSH release with other BOSH users you need ONLY:
+
+- Use a public blobstore (such as AWS S3)
+- Use a public source control repository
+- Optionally, publicly share pre-created final release tarballs via a HTTP URL.
+
+### Share release tarballs via HTTP
+
+bosh-gen includes a BOSH CLI plugin to upload dev or final tarballs to your blobstore, and get a public URL back (if supported by your blobstore)
+
+```
+$ bosh create release --with-tarball
+$ bosh share release releases/my-project-1.tgz
+https://my-project-boshrelease.s3.amazonaws.com/boshrelease-my-project-1.tgz
+```
+
+The URL is displayed and can be given to other users and uploaded directly to their BOSH:
+
+```
+$ bosh upload release https://my-project-boshrelease.s3.amazonaws.com/boshrelease-my-project-1.tgz
+```
+
+They no longer require your BOSH release repo to access the BOSH release.
+
+### BOSH community facilities
+
+You are welcome to re-use the BOSH user community facilities:
+
+- Use the shared AWS S3 account (currently over 30 BOSH release blobstores).
+- Place your release git repository in the [@cloudfoundry-community](https://github.com/cloudfoundry-community) GitHub account (over 50 people have access).
+
+One time only, please email [Dr Nic Williams](mailto:drnicwilliams@gmail.com) and he will set you up with access:
+
+- Read/write credentials to the AWS S3 account for your BOSH release blobstores/buckets
+- Access to create [@cloudfoundry-community](https://github.com/cloudfoundry-community) GitHub repositories for your BOSH releases
+
+When he gives you the AWS S3 credentials, place them in the `~/.fog` file and you'll easily be able to reuse them for each new BOSH release:
+
+```yaml
+:community:
+  :aws_access_key_id:     ACCESS
+  :aws_secret_access_key: SECRET
+```
+
+Then for your next BOSH release:
+
+```
+$ bosh-gen new my-project
+      create  
+Auto-detected infrastructure API credentials at ~/.fog (override with $FOG)
+1. AWS (community)
+2. Alternate credentials
+Choose an auto-detected infrastructure: 2
+```
+
+You'll only need to do this once. Yes, it would be awesome if there was some public service to do this nicely. Want to build it?
+
 Contributing
 ------------
 

data/bosh-gen.gemspec

@@ -18,8 +18,9 @@ Gem::Specification.new do |gem|
   gem.add_dependency "thor"
   gem.add_dependency "bosh_cli"
   gem.add_dependency "bosh_common"
+  gem.add_dependency "progressbar"
 
-  gem.add_dependency "cyoi", "~> 0.10.0"
+  gem.add_dependency "cyoi", "~> 0.10"
   gem.add_dependency "fog", "~> 1.11"
   gem.add_dependency "readwritesettings", "~> 3.0"
 

data/lib/bosh/cli/commands/share_release.rb

@@ -0,0 +1,78 @@
+require 'bosh/gen/utils/file_with_progress_bar'
+
+module Bosh::Cli::Command
+  class ShareRelease < Base
+    usage "share release"
+    desc  "upload release tarball to blobstore"
+    def share_release(tarball_path)
+      # need to be in release dir to get blobstore credentials
+      check_if_release_dir
+
+      unless File.exist?(tarball_path)
+        err("Release tarball file doesn't exist")
+      end
+
+      tarball = Bosh::Cli::ReleaseTarball.new(tarball_path)
+      say("\nVerifying release...")
+      tarball.validate(:allow_sparse => true)
+      nl
+
+      unless tarball.valid?
+        err('Release is invalid, please fix, verify and upload again')
+      end
+
+      upload_name = "boshrelease-#{tarball.release_name}-#{tarball.version}.tgz"
+
+      f = ::FileWithProgressBar.open(tarball_path, 'r')
+      f.out = Bosh::Cli::Config.output
+
+      raw_blobstore_client = unwrap_blobstore_client(blobstore)
+      bucket_name = raw_blobstore_client.instance_variable_get("@bucket_name")
+
+      fog = fog_storage(raw_blobstore_client)
+      dir = fog.directories.get(bucket_name)
+
+      say("\nUploading release...")
+      if file = dir.files.new(key: upload_name, body: f)
+        file.public = true
+        file.save
+        nl
+        say(file.public_url)
+      else
+        err('Failed to upload file to blobstore')
+      end
+    end
+
+    private
+    def unwrap_blobstore_client(blobstore)
+      if blobstore.is_a?(Bosh::Blobstore::RetryableBlobstoreClient)
+        unwrap_blobstore_client(blobstore.instance_variable_get("@client"))
+      elsif blobstore.is_a?(Bosh::Blobstore::Sha1VerifiableBlobstoreClient)
+        unwrap_blobstore_client(blobstore.instance_variable_get("@client"))
+      else
+        blobstore
+      end
+    end
+
+    def fog_storage(blobstore)
+      blobstore_options = blobstore.instance_variable_get("@options")
+      if blobstore.is_a?(Bosh::Blobstore::S3BlobstoreClient)
+        require "fog/aws"
+        return Fog::Storage.new(
+          provider: 'AWS',
+          aws_access_key_id: blobstore_options[:access_key_id],
+          aws_secret_access_key: blobstore_options[:secret_access_key],
+        )
+      # elsif blobstore.is_a?(Bosh::Blobstore::SwiftBlobstoreClient)
+      #   require "fog/openstack"
+      #   return Fog::Storage.new(
+      #     provider: 'OpenStack',
+      #     aws_access_key_id: blobstore_options[:access_key_id],
+      #     aws_secret_access_key: blobstore_options[:secret_access_key],
+      #   )
+      else
+        err('Not yet implemented for #{blobstore.class} blobstore')
+      end
+    end
+  end
+end

data/lib/bosh/gen/utils/file_with_progress_bar.rb

@@ -0,0 +1,62 @@
+require "progressbar"
+# f = FileWithProgressBar.open(file, 'r')
+# f.out = $stdout
+#
+# Then pass f like any File object that invokes f.read
+#
+# To upload with Fog:
+# d = Storage[:aws].directories.create(key: 'drnic-test-upload')
+# d.files.create(key: 'test.tgz', body: f)
+# redis-2.8.12.: 100% |oooooooooooooo|   1.2MB 243.4KB/s Time: 00:00:04
+class FileWithProgressBar < ::File
+
+  def out=(out)
+    @out = out
+  end
+
+  def progress_bar
+    return @progress_bar if @progress_bar
+    @out ||= StringIO.new
+    @progress_bar = ProgressBar.new(file_name, size, @out)
+    @progress_bar.file_transfer_mode
+    @progress_bar
+  end
+
+  def file_name
+    File.basename(self.path)
+  end
+
+  def stop_progress_bar
+    progress_bar.halt unless progress_bar.finished?
+  end
+
+  def size
+    @size || File.size(self.path)
+  end
+
+  def size=(size)
+    @size=size
+  end
+
+  def read(*args)
+    result = super(*args)
+
+    if result && result.size > 0
+      progress_bar.inc(result.size)
+    else
+      progress_bar.finish
+    end
+
+    result
+  end
+
+  def write(*args)
+    count = super(*args)
+    if count
+      progress_bar.inc(count)
+    else
+      progress_bar.finish
+    end
+    count
+  end
+end

data/lib/bosh/gen/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Gen
-    VERSION = "0.17.0"
+    VERSION = "0.17.1"
   end
 end

data/spec/generators/new_release_generator_spec.rb

@@ -4,9 +4,31 @@ require "bosh/gen/generators/new_release_generator"
 describe Bosh::Gen::Generators::NewReleaseGenerator do
   include Bosh::Gen::Settings
 
-  it "generates with s3 blobstore" do
+  def cyoi_provider
+    provider = instance_double("Cyoi::Cli::Provider")
+    expect(provider).to receive(:execute!)
+    expect(Cyoi::Cli::Provider).to receive(:new).with([settings_dir]).and_return(provider)
+    provider
+  end
+
+  def cyoi_blobstore(blobstore_name)
+    blobstore = instance_double("Cyoi::Cli::Blobstore")
+    expect(blobstore).to receive(:execute!)
+    expect(Cyoi::Cli::Blobstore).to receive(:new).with([blobstore_name, settings_dir]).and_return(blobstore)
+    blobstore
+  end
+
+  before do
     in_home_folder do
       self.settings_dir = "redis-boshrelease/config"
+      cyoi_provider
+      cyoi_blobstore "redis-boshrelease"
+    end
+  end
+
+  it "generates with s3 blobstore" do
+    in_home_folder do
+
       setting "provider.name", "aws"
       setting "provider.region", "us-west-2"
       setting "provider.credentials.aws_access_key_id", "ACCESS"
@@ -32,8 +54,7 @@ describe Bosh::Gen::Generators::NewReleaseGenerator do
     end
   end
 
-  # TODO: Fog::Errors::MockNotImplemented: Contributions welcome!
-  xit "generates with swift blobstore" do
+  it "generates with swift blobstore" do
     in_home_folder do
       setting "provider.name", "openstack"
       setting "provider.credentials.openstack_auth_url", "http://10.0.0.2:5000/v2.0/tokens"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-gen
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.17.1
 platform: ruby
 authors:
 - Dr Nic Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-29 00:00:00.000000000 Z
+date: 2014-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -52,20 +52,34 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: cyoi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
@@ -141,6 +155,7 @@ files:
 - Rakefile
 - bin/bosh-gen
 - bosh-gen.gemspec
+- lib/bosh/cli/commands/share_release.rb
 - lib/bosh/gen.rb
 - lib/bosh/gen/cli.rb
 - lib/bosh/gen/generators/deployment_manifest_generator.rb
@@ -210,6 +225,7 @@ files:
 - lib/bosh/gen/models/deployment_manifest.rb
 - lib/bosh/gen/models/release_detection.rb
 - lib/bosh/gen/settings.rb
+- lib/bosh/gen/utils/file_with_progress_bar.rb
 - lib/bosh/gen/version.rb
 - spec/fixtures/bosh_config/multiple_boshes.yml
 - spec/fixtures/deployment_manifests/2_jobs_1_ip_8196_disk.yml
@@ -285,7 +301,6 @@ files:
 - spec/models/deployment_manifest_spec.rb
 - spec/models/release_detection_spec.rb
 - spec/spec_helper.rb
-- spec/support/fog.rb
 - spec/support/generator_spec_helper.rb
 homepage: https://github.com/drnic/bosh-gen
 licenses: []
@@ -385,5 +400,4 @@ test_files:
 - spec/models/deployment_manifest_spec.rb
 - spec/models/release_detection_spec.rb
 - spec/spec_helper.rb
-- spec/support/fog.rb
 - spec/support/generator_spec_helper.rb

data/spec/support/fog.rb

@@ -1,3 +0,0 @@
-require "fog"
-
-Fog.mock!