mamiya 0.0.1.alpha10 → 0.0.1.alpha11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 759f3b4c1591ed1aadc5ec73dfea2ec3fb262d52
-  data.tar.gz: b28cb4415f67b1a49c1c6f77deac0324eb165283
+  metadata.gz: 72fe2e5902f669e48c73a5b62ed1aec2f4b55e2e
+  data.tar.gz: 1135c8729f2c67847a9a736f447d2bf77e386e44
 SHA512:
-  metadata.gz: fac824c20946f70f0ca4223a97db53b7868186ef7470ec7dcc99d9fa3ca51e672aa479dd53955a139f984373a13ae231c37f433de3c74cc23a81746ff1572780
-  data.tar.gz: c694781fe55e32e16f1fdb8caaa9c7b9180e7fcf994efc517bbbd8ad18c1bfd2bb0e8ef5467beefdebeb4163b6cac1558c320aa8f1bbd39a4c7e6c821182bea9
+  metadata.gz: ace24e5310616ea9bc045d9a7dc74498bec6caef2425c0f29d6641488e036976a12d6766b1e3c33f67700a7a0b8035f0b0842315430418d684633cdab7224a56
+  data.tar.gz: ddad6f309a018a1db82f0f04e2db9478bcfefb04adc1592331d117f0d64c4373c8d7541e64d95378156333b18f170c94537fc7a39b6ae26e9f1340b40fe67d3f

data/lib/mamiya/storages/s3.rb

@@ -95,11 +95,15 @@ module Mamiya
       end
 
       def self.initiate_s3_with_config(config) # :nodoc:
-        s3_config = config.dup
-        s3_config.delete(:bucket)
-        s3_config.delete(:application)
-        s3_config.delete(:type)
-        Aws::S3.new(s3_config)
+        Aws::S3.new(s3_config(config))
+      end
+
+      def self.s3_config(base) # :nodoc:
+        base.dup.tap do |c|
+          c.delete(:bucket)
+          c.delete(:application)
+          c.delete(:type)
+        end
       end
 
       private

data/lib/mamiya/storages/s3_proxy.rb

@@ -0,0 +1,93 @@
+require 'mamiya/package'
+require 'mamiya/storages/s3'
+
+module Mamiya
+  module Storages
+    # Because there's no S3 endpoint in Amazon VPC, fetching from instances
+    # with no public IP may consume your NAT instances' bandwidth. This
+    # basically uses Amazon S3 but use s3_proxy.gem for fetching from specific
+    # host to avoid heavy bandwidth load.
+    #
+    # Note: s3_proxy is a simple Rack app that proxies HTTP GET requests to
+    # Amazon S3 GetObject.  You can use it with puma + nginx proxy_cache
+    # to avoid heavy bandwidth load.
+    class S3Proxy < Mamiya::Storages::S3
+      def fetch(package_name, dir)
+        package_key, meta_key = package_and_meta_key_for(package_name)
+
+        package_path = File.join(dir, File.basename(package_key))
+        meta_path = File.join(dir, File.basename(meta_key))
+
+        if File.exists?(package_path) || File.exists?(meta_path)
+          raise AlreadyFetched
+        end
+
+        open(package_path, 'wb+') do |io|
+          proxy_get(package_key, io)
+        end
+        open(meta_path, 'wb+') do |io|
+          proxy_get(meta_key, io)
+        end
+
+        return Mamiya::Package.new(package_path)
+      rescue Exception => e
+        File.unlink package_path if package_path && File.exists?(package_path)
+        File.unlink meta_path if meta_path && File.exists?(meta_path)
+
+        raise e
+      end
+
+      def meta(package_name)
+        _, meta_key = package_and_meta_key_for(package_name)
+        JSON.parse(proxy_get(meta_key, nil, &:body))
+      rescue NotFound
+        return nil
+      end
+
+      def self.s3_config(base) # :nodoc:
+        superclass.s3_config(base).tap do |c|
+          c.delete(:proxy_host)
+          c.delete(:proxy_ssl_verify_none)
+        end
+      end
+
+      private
+
+      def proxy_host_uri
+        @proxy_host_uri ||= URI.parse(@config[:proxy_host])
+      end
+
+      def connect_proxy(&block)
+        Net::HTTP.new(proxy_host_uri.host, proxy_host_uri.port).tap do |http|
+          http.use_ssl = (proxy_host_uri.scheme == 'https')
+          if @config[:proxy_ssl_verify_none] == 'none'
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          end
+
+          break http.start(&block)
+        end
+      end
+
+      def proxy_get(key, target)
+        connect_proxy do |http|
+          response = http.get("#{proxy_host_uri.path}/#{@config[:bucket]}/#{key}")
+          response.value
+
+          if block_given?
+            yield(response)
+          else
+            response.read_body do |chunk|
+              target.write chunk
+            end
+          end
+        end
+      rescue Net::HTTPServerException => e
+        if e.message.start_with?('404 ')
+          raise NotFound
+        else
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/version.rb

@@ -1,3 +1,3 @@
 module Mamiya
-  VERSION = "0.0.1.alpha10"
+  VERSION = "0.0.1.alpha11"
 end

data/spec/storages/s3_proxy_spec.rb

@@ -0,0 +1,139 @@
+require 'spec_helper'
+require 'aws-sdk-core'
+require 'mamiya/package'
+require 'mamiya/storages/abstract'
+require 'mamiya/storages/s3_proxy'
+require 'mamiya/storages/s3'
+require 'tmpdir'
+require 'fileutils'
+require 'stringio'
+
+describe Mamiya::Storages::S3Proxy do
+  let(:bucket) { 'testbucket' }
+
+  let(:config) do
+    {
+      application: 'myapp',
+      bucket: bucket,
+      foo: :bar,
+      access_key_id: 'AKI',
+      secret_access_key: 'secret',
+      region: 'ap-northeast-1',
+      proxy_host: 'http://my-proxy:8080/_',
+    }
+  end
+  subject(:storage) { described_class.new(config) }
+
+  let(:http) do
+    double('http').tap do |http|
+      allow(http).to receive(:use_ssl=).with(false)
+      allow(http).to receive(:start).and_yield(http)
+
+      allow(http).to receive(:get).with('/_/testbucket/myapp/test.tar.gz').and_return(
+        double('tarball response').tap do |resp|
+          allow(resp).to receive(:value).and_return(200)
+          allow(resp).to receive(:read_body).and_yield("{}\n")
+        end
+      )
+
+      allow(http).to receive(:get).with('/_/testbucket/myapp/test.json').and_return(
+        double('json response').tap do |resp|
+          allow(resp).to receive(:value).and_return(200)
+          allow(resp).to receive(:read_body).and_yield("{}\n")
+        end
+      )
+
+      allow(http).to receive(:get).with('/_/testbucket/myapp/not-found.tar.gz').and_return(
+        double('tarball 404 response').tap do |resp|
+          allow(resp).to receive(:value).and_raise(Net::HTTPServerException.new('404 "Not Found"',''))
+        end
+      )
+
+      allow(http).to receive(:get).with('/_/testbucket/myapp/not-found.json').and_return(
+        double('json 404 response').tap do |resp|
+          allow(resp).to receive(:value).and_raise(Net::HTTPServerException.new('404 "Not Found"',''))
+        end
+      )
+    end
+  end
+
+  before do
+    expect(Aws::S3).not_to receive(:new)
+    allow(Net::HTTP).to receive(:new).with('my-proxy', 8080).and_return(http)
+  end
+
+  it "inherits S3 storage" do
+    expect(described_class.ancestors).to include(Mamiya::Storages::S3)
+  end
+
+  describe "#fetch(package_name, dir)" do
+    let!(:tmpdir) { Dir.mktmpdir("mamiya-package-spec") }
+    after { FileUtils.remove_entry_secure tmpdir }
+
+    let(:tarball) { File.join(tmpdir, 'test.tar.gz') }
+    let(:metafile) { File.join(tmpdir, 'test.json') }
+
+    let(:package_name) { 'test' }
+    subject(:fetch) { storage.fetch(package_name, tmpdir) }
+
+    it "retrieves package from S3" do
+      expect(fetch).to be_a_kind_of(Mamiya::Package)
+      expect(File.realpath(fetch.path)).to eq File.realpath(tarball)
+
+      expect(File.read(tarball)).to eq "{}\n"
+      expect(File.read(metafile)).to eq "{}\n"
+    end
+
+
+    context "when not found" do
+      let(:package_name) { 'not-found' }
+
+      it "raises error" do
+        expect {
+          fetch
+        }.to raise_error(Mamiya::Storages::Abstract::NotFound)
+      end
+    end
+
+    context "when meta already exists" do
+      before do
+        File.write metafile, "\n"
+      end
+
+      it "raises error" do
+        expect {
+          fetch
+        }.to raise_error(Mamiya::Storages::Abstract::AlreadyFetched)
+      end
+    end
+
+    context "when tarball already exists" do
+      before do
+        File.write tarball, "\n"
+      end
+
+      it "raises error" do
+        expect {
+          fetch
+        }.to raise_error(Mamiya::Storages::Abstract::AlreadyFetched)
+      end
+    end
+  end
+
+  #describe "#meta(package_name)" do
+  #  let(:package_name) { 'test' }
+  #  subject(:meta) { storage.meta(package_name) }
+
+  #  it "retrieves meta JSON from S3" do
+  #    expect(meta).to eq({})
+  #  end
+
+  #  context "when not found" do
+  #    let(:package_name) { 'not-found' }
+
+  #    it "returns nil" do
+  #      expect(meta).to be_nil
+  #    end
+  #  end
+  #end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mamiya
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha10
+  version: 0.0.1.alpha11
 platform: ruby
 authors:
 - Shota Fukumori (sora_h)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-05 00:00:00.000000000 Z
+date: 2014-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -198,6 +198,7 @@ files:
 - lib/mamiya/storages/abstract.rb
 - lib/mamiya/storages/mock.rb
 - lib/mamiya/storages/s3.rb
+- lib/mamiya/storages/s3_proxy.rb
 - lib/mamiya/util/label_matcher.rb
 - lib/mamiya/version.rb
 - mamiya.gemspec
@@ -227,6 +228,7 @@ files:
 - spec/steps/fetch_spec.rb
 - spec/steps/push_spec.rb
 - spec/storages/abstract_spec.rb
+- spec/storages/s3_proxy_spec.rb
 - spec/storages/s3_spec.rb
 - spec/storages_spec.rb
 - spec/support/dummy_serf.rb
@@ -281,6 +283,7 @@ test_files:
 - spec/steps/fetch_spec.rb
 - spec/steps/push_spec.rb
 - spec/storages/abstract_spec.rb
+- spec/storages/s3_proxy_spec.rb
 - spec/storages/s3_spec.rb
 - spec/storages_spec.rb
 - spec/support/dummy_serf.rb