tumugi-plugin-google_cloud_storage 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a54e93d2ae132b0372b36c6f3d1974c1fbb7ef92
-  data.tar.gz: 6c1c6aa1fe96b655ab3f8f9b49176ef34dc9858f
+  metadata.gz: d867db4d8e41e26c1aeccc2ae435882def2eaedf
+  data.tar.gz: 1e1a67c26142bf89ffb1c2c2f3b8c571031f835e
 SHA512:
-  metadata.gz: 83efe8fafc377ac90a8fc103bcee1195fb80296330a50b8a28d6c029037e19b1b29f0e15337a8443313f6fa643311ccc42a9b9747bf4308bc183bb454bacd4ed
-  data.tar.gz: f2ce008443eaf65118d94650698ec6ccf4b2ebf07d6079ae11a2ac7d28da8992cb683dae22cb20f6558dcc7603a41e0d2b1843a804f7bb2545e8e164aac13a71
+  metadata.gz: 5f167627c6079a672936b132ef4980b2d0d0db217d47c67e956a7a2b0b3f5714b6f8f10978ec848539bfc60205ad6f4d37870a7752573482c2d1d6125af939c1
+  data.tar.gz: 8e9f630ce8bfbd0c12ca2061e0593122e251bbed638b04ff2a1e0559f691c7b1dce2ec8021681062a4bc4b953422c1e81a0d29d1ca319c907a09763b3ace2501

data/.travis.yml

@@ -1,13 +1,8 @@
 language: ruby
 cache: bundler
 rvm:
-  - 2.1.10
-  - 2.2.5
+  - 2.1
   - 2.3.1
-  - ruby-head
-  - jruby-9.0.5.0
+  - jruby-9.1.2.0
 before_install:
   - gem install bundler
-matrix:
-  allow_failures:
-    - rvm: ruby-head

data/CHANGELOG.md

@@ -0,0 +1,37 @@
+# Change Log
+
+## [0.2.0](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/tree/0.2.0) (2016-06-06)
+[Full Changelog](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/compare/v0.1.0...0.2.0)
+
+**Implemented enhancements:**
+
+- Change tumugi dependency version [\#14](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/14) ([hakobera](https://github.com/hakobera))
+- Support custom filesystem for GoogleCloudStorageFileTarget [\#11](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/11) ([hakobera](https://github.com/hakobera))
+
+**Merged pull requests:**
+
+- Remove unneeded log [\#16](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/16) ([hakobera](https://github.com/hakobera))
+- Prepare release for 0.2.0 [\#15](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/15) ([hakobera](https://github.com/hakobera))
+- Change test directory [\#13](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/13) ([hakobera](https://github.com/hakobera))
+- Change ruby version on travis [\#12](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/12) ([hakobera](https://github.com/hakobera))
+
+## [v0.1.0](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/tree/v0.1.0) (2016-05-30)
+**Closed issues:**
+
+- Release v0.1.0 [\#8](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/issues/8)
+
+**Merged pull requests:**
+
+- Preapre release for v0.1.0 [\#10](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/10) ([hakobera](https://github.com/hakobera))
+- Support JSON private key file for authentication [\#9](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/9) ([hakobera](https://github.com/hakobera))
+- Update README [\#7](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/7) ([hakobera](https://github.com/hakobera))
+- Standardize Google API Call error handling [\#6](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/6) ([hakobera](https://github.com/hakobera))
+- Fix railse\_if\_exist option not working when copy and move [\#5](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/5) ([hakobera](https://github.com/hakobera))
+- Rename gem and namespace [\#4](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/4) ([hakobera](https://github.com/hakobera))
+- Add tests [\#3](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/3) ([hakobera](https://github.com/hakobera))
+- Update tumugi to v0.5.1 [\#2](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/2) ([hakobera](https://github.com/hakobera))
+- Fix test failed on travis [\#1](https://github.com/tumugi/tumugi-plugin-google_cloud_storage/pull/1) ([hakobera](https://github.com/hakobera))
+
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/lib/tumugi/plugin/google_cloud_storage/atomic_file.rb

@@ -4,13 +4,13 @@ module Tumugi
   module Plugin
     module GoogleCloudStorage
       class AtomicFile < Tumugi::AtomicFile
-        def initialize(path, client)
+        def initialize(path, fs)
           super(path)
-          @client = client
+          @fs = fs
         end
 
         def move_to_final_destination(temp_file)
-          @client.upload(temp_file, path)
+          @fs.upload(temp_file, path)
         end
       end
     end

data/lib/tumugi/plugin/google_cloud_storage/file_system.rb

@@ -1,6 +1,8 @@
 require 'uri'
 require 'json'
+require 'googleauth/service_account'
 require 'google/apis/storage_v1'
+require 'google/apis/drive_v3'
 require 'tumugi/file_system'
 
 module Tumugi
@@ -10,7 +12,7 @@ module Tumugi
         attr_reader :client
 
         def initialize(config)
-          @client = create_client(config)
+          save_config(config)
         end
 
         #######################################################################
@@ -33,16 +35,16 @@ module Tumugi
           raise Tumugi::FileSystemError.new("Cannot delete root of bucket at path '#{path}'") if root?(key)
 
           if obj_exist?(bucket, key)
-            @client.delete_object(bucket, key)
+            client.delete_object(bucket, key, options: request_options)
             wait_until { !obj_exist?(bucket, key) }
             true
           elsif directory?(path)
             raise Tumugi::FileSystemError.new("Path '#{path}' is a directory. Must use recursive delete") if !recursive
 
             objs = entries(path).map(&:name)
-            @client.batch do |client|
+            client.batch do |client|
               objs.each do |obj|
-                client.delete_object(bucket, obj)
+                client.delete_object(bucket, obj, options: request_options)
               end
             end
             wait_until { !directory?(path) }
@@ -80,7 +82,7 @@ module Tumugi
               true
             else
               # Any objects with this prefix
-              objects = @client.list_objects(bucket, prefix: obj, max_results: 20)
+              objects = client.list_objects(bucket, prefix: obj, max_results: 20, options: request_options)
               !!(objects.items && objects.items.size > 0)
             end
           end
@@ -95,7 +97,7 @@ module Tumugi
           next_page_token = ''
 
           until next_page_token.nil?
-            objects = @client.list_objects(bucket, prefix: obj, page_token: next_page_token)
+            objects = client.list_objects(bucket, prefix: obj, page_token: next_page_token, options: request_options)
             if objects && objects.items
               results.concat(objects.items)
               next_page_token = objects.next_page_token
@@ -120,7 +122,7 @@ module Tumugi
         def upload(media, path, content_type: nil)
           bucket, key = path_to_bucket_and_key(path)
           obj = Google::Apis::StorageV1::Object.new(bucket: bucket, name: key)
-          @client.insert_object(bucket, obj, upload_source: media, content_type: content_type)
+          client.insert_object(bucket, obj, upload_source: media, content_type: content_type, options: request_options)
           wait_until { obj_exist?(bucket, key) }
         rescue
           process_error($!)
@@ -131,7 +133,7 @@ module Tumugi
           if download_path.nil?
             download_path = Tempfile.new('tumugi_gcs_file_system').path
           end
-          @client.get_object(bucket, key, download_dest: download_path)
+          client.get_object(bucket, key, download_dest: download_path, options: request_options)
           wait_until { File.exist?(download_path) }
 
           if block_given?
@@ -164,13 +166,13 @@ module Tumugi
             copied_objs = []
             entries(src_path).each do |entry|
               suffix = entry.name[src_prefix.length..-1]
-              @client.copy_object(src_bucket, src_prefix + suffix,
-                                  dest_bucket, dest_prefix + suffix)
+              client.copy_object(src_bucket, src_prefix + suffix,
+                                  dest_bucket, dest_prefix + suffix, options: request_options)
               copied_objs << (dest_prefix + suffix)
             end
             wait_until { copied_objs.all? {|obj| obj_exist?(dest_bucket, obj)} }
           else
-            @client.copy_object(src_bucket, src_key, dest_bucket, dest_key)
+            client.copy_object(src_bucket, src_key, dest_bucket, dest_key, options: request_options)
             wait_until { obj_exist?(dest_bucket, dest_key) }
           end
         rescue
@@ -186,7 +188,7 @@ module Tumugi
         def create_bucket(bucket)
           unless bucket_exist?(bucket)
             b = Google::Apis::StorageV1::Bucket.new(name: bucket)
-            @client.insert_bucket(@project_id, b)
+            client.insert_bucket(@project_id, b, options: request_options)
             true
           else
             false
@@ -197,7 +199,7 @@ module Tumugi
 
         def remove_bucket(bucket)
           if bucket_exist?(bucket)
-            @client.delete_bucket(bucket)
+            client.delete_bucket(bucket, options: request_options)
             true
           else
             false
@@ -207,7 +209,7 @@ module Tumugi
         end
 
         def bucket_exist?(bucket)
-          @client.get_bucket(bucket)
+          client.get_bucket(bucket, options: request_options)
           true
         rescue => e
           return false if e.status_code == 404
@@ -217,7 +219,7 @@ module Tumugi
         private
 
         def obj_exist?(bucket, key)
-          @client.get_object(bucket, key)
+          client.get_object(bucket, key, options: request_options)
           true
         rescue => e
           return false if e.status_code == 404
@@ -236,7 +238,7 @@ module Tumugi
           end
         end
 
-        def create_client(config)
+        def save_config(config)
           if config.private_key_file.nil?
             @project_id = config.project_id
             client_email = config.client_email
@@ -247,28 +249,39 @@ module Tumugi
             client_email = json['client_email']
             private_key = json['private_key']
           end
+          @key = {
+            client_email: client_email,
+            private_key: private_key
+          }
+        end
+
+        def client
+          return @cached_client if @cached_client && @cached_client_expiration > Time.now
 
-          # https://cloud.google.com/storage/docs/authentication
-          scope = "https://www.googleapis.com/auth/devstorage.read_write"
-
-          if client_email and private_key
-            auth = Signet::OAuth2::Client.new(
-              token_credential_uri: "https://accounts.google.com/o/oauth2/token",
-              audience: "https://accounts.google.com/o/oauth2/token",
-              scope: scope,
-              issuer: client_email,
-              signing_key: OpenSSL::PKey.read(private_key))
-            # MEMO: signet-0.6.1 depend on Farady.default_connection
-            Faraday.default_connection.options.timeout = 60
-            auth.fetch_access_token!
+          client = Google::Apis::StorageV1::StorageService.new
+          scope = Google::Apis::StorageV1::AUTH_DEVSTORAGE_READ_WRITE
+
+          if @key[:client_email] and @key[:private_key]
+            options = {
+              json_key_io: StringIO.new(JSON.generate(@key)),
+              scope: scope
+            }
+            auth = Google::Auth::ServiceAccountCredentials.make_creds(options)
           else
             auth = Google::Auth.get_application_default([scope])
-            auth.fetch_access_token!
           end
-
-          client = Google::Apis::StorageV1::StorageService.new
+          auth.fetch_access_token!
           client.authorization = auth
-          client
+
+          @cached_client_expiration = Time.now + (auth.expires_in / 2)
+          @cached_client = client
+        end
+
+        def request_options
+          {
+            retries: 5,
+            timeout_sec: 60
+          }
         end
 
         def wait_until(&block)

data/lib/tumugi/plugin/target/google_cloud_storage_file.rb

@@ -12,11 +12,11 @@ module Tumugi
 
       attr_reader :bucket, :key, :path
 
-      def initialize(bucket:, key:)
+      def initialize(bucket:, key:, fs: nil)
         @bucket = bucket
         @key = key
         @path = "gs://#{File.join(bucket, key)}"
-        log "bucket='#{bucket}, key='#{key}'"
+        @fs = fs unless fs.nil?
       end
 
       def fs

data/tumugi-plugin-google_cloud_storage.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "tumugi-plugin-google_cloud_storage"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["Kazuyuki Honda"]
   spec.email         = ["hakobera@gmail.com"]
 
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.1'
 
-  spec.add_runtime_dependency "tumugi", "~> 0.5.1"
+  spec.add_runtime_dependency "tumugi", ">= 0.5.1"
   spec.add_runtime_dependency "google-api-client", "~> 0.9.3"
 
   spec.add_development_dependency "bundler", "~> 1.11"

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: tumugi-plugin-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-30 00:00:00.000000000 Z
+date: 2016-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tumugi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.1
 - !ruby/object:Gem::Dependency
@@ -131,6 +131,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - README.md
 - Rakefile