tumugi-plugin-google_cloud_storage 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a54e93d2ae132b0372b36c6f3d1974c1fbb7ef92
+  data.tar.gz: 6c1c6aa1fe96b655ab3f8f9b49176ef34dc9858f
+SHA512:
+  metadata.gz: 83efe8fafc377ac90a8fc103bcee1195fb80296330a50b8a28d6c029037e19b1b29f0e15337a8443313f6fa643311ccc42a9b9747bf4308bc183bb454bacd4ed
+  data.tar.gz: f2ce008443eaf65118d94650698ec6ccf4b2ebf07d6079ae11a2ac7d28da8992cb683dae22cb20f6558dcc7603a41e0d2b1843a804f7bb2545e8e164aac13a71

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.env

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+cache: bundler
+rvm:
+  - 2.1.10
+  - 2.2.5
+  - 2.3.1
+  - ruby-head
+  - jruby-9.0.5.0
+before_install:
+  - gem install bundler
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tumugi-plugin-gcs.gemspec
+gemspec

data/README.md

@@ -0,0 +1,87 @@
+[![Build Status](https://travis-ci.org/tumugi/tumugi-plugin-google_cloud_storage.svg?branch=master)](https://travis-ci.org/tumugi/tumugi-plugin-google_cloud_storage) [![Code Climate](https://codeclimate.com/github/tumugi/tumugi-plugin-google_cloud_storage/badges/gpa.svg)](https://codeclimate.com/github/tumugi/tumugi-plugin-google_cloud_storage) [![Coverage Status](https://coveralls.io/repos/github/tumugi/tumugi-plugin-google_cloud_storage/badge.svg?branch=master)](https://coveralls.io/github/tumugi/tumugi-plugin-google_cloud_storage?branch=master) [![Gem Version](https://badge.fury.io/rb/tumugi-plugin-google_cloud_storage.svg)](https://badge.fury.io/rb/tumugi-plugin-google_cloud_storage)
+
+# tumugi-plugin-google_cloud_storage
+
+[tumugi](https://github.com/tumugi/tumugi) plugin for Google Cloud Storage.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'tumugi-plugin-google_cloud_storage'
+```
+
+And then execute:
+
+```sh
+$ bundle
+```
+
+Or install it yourself as:
+
+```sh
+$ gem install tumugi-plugin-google_cloud_storage
+```
+
+## Component
+
+### Tumugi::Plugin::GoogleCloudStorageFileTarget
+
+This target represent file or directory on Googl Cloud Storage.
+This target has 2 parameters, `bucket` and `key`.
+
+Tumugi workflow file using this target is like this:
+
+```rb
+task :task1 do
+  param :bucket, type: :string, auto_bind: true, required: true
+  param :day, type: :time, auto_bind: true, required: true
+  output do
+    target(:google_cloud_storage_file, bucket: bucket, key: "test_#{day.strftime('%Y%m%d')}.txt")
+  end
+  run do
+    log 'task1#run'
+    output.open('w') {|f| f.puts('done') }
+  end
+end
+```
+
+### Config Section
+
+tumugi-plugin-google_cloud_storage provide config section named "google_cloud_storage" which can specified Google Cloud Storage autenticaion info.
+
+#### Authenticate by client_email and private_key
+
+```rb
+Tumugi.config do |config|
+  config.section("google_cloud_storage") do |section|
+    section.project_id = "xxx"
+    section.client_email = "yyy@yyy.iam.gserviceaccount.com"
+    section.private_key = "zzz"
+  end
+end
+```
+
+#### Authenticate by JSON key file
+
+```rb
+Tumugi.configure do |config|
+  config.section("google_cloud_storage") do |section|
+    section.private_key_file = "/path/to/key.json"
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/tumugi/ttumugi-plugin-google_cloud_storage
+
+## License
+
+The gem is available as open source under the terms of the [Apache License
+Version 2.0](http://www.apache.org/licenses/).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "tumugi/plugin/gcs"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/example.rb

@@ -0,0 +1,11 @@
+task :task1 do
+  param :bucket, type: :string, auto_bind: true, required: true
+  param :day, type: :time, auto_bind: true, required: true
+  output do
+    target(:google_cloud_storage_file, bucket: bucket, key: "test_#{day.strftime('%Y%m%d')}.txt")
+  end
+  run do
+    log 'task1#run'
+    output.open('w') {|f| f.puts('done') }
+  end
+end

data/examples/tumugi_config_example.rb

@@ -0,0 +1,7 @@
+Tumugi.configure do |config|
+  config.section('google_cloud_storage') do |section|
+    section.project_id = ENV["PROJECT_ID"]
+    section.client_email = ENV["CLIENT_EMAIL"]
+    section.private_key = ENV["PRIVATE_KEY"].gsub(/\\n/, "\n")
+  end
+end

data/lib/tumugi/plugin/google_cloud_storage/atomic_file.rb

@@ -0,0 +1,18 @@
+require 'tumugi/atomic_file'
+
+module Tumugi
+  module Plugin
+    module GoogleCloudStorage
+      class AtomicFile < Tumugi::AtomicFile
+        def initialize(path, client)
+          super(path)
+          @client = client
+        end
+
+        def move_to_final_destination(temp_file)
+          @client.upload(temp_file, path)
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/google_cloud_storage/file_system.rb

@@ -0,0 +1,299 @@
+require 'uri'
+require 'json'
+require 'google/apis/storage_v1'
+require 'tumugi/file_system'
+
+module Tumugi
+  module Plugin
+    module GoogleCloudStorage
+      class FileSystem < Tumugi::FileSystem
+        attr_reader :client
+
+        def initialize(config)
+          @client = create_client(config)
+        end
+
+        #######################################################################
+        # FileSystem interfaces
+        #######################################################################
+
+        def exist?(path)
+          bucket, key = path_to_bucket_and_key(path)
+          if obj_exist?(bucket, key)
+            true
+          else
+            directory?(path)
+          end
+        rescue
+          process_error($!)
+        end
+
+        def remove(path, recursive: true)
+          bucket, key = path_to_bucket_and_key(path)
+          raise Tumugi::FileSystemError.new("Cannot delete root of bucket at path '#{path}'") if root?(key)
+
+          if obj_exist?(bucket, key)
+            @client.delete_object(bucket, key)
+            wait_until { !obj_exist?(bucket, key) }
+            true
+          elsif directory?(path)
+            raise Tumugi::FileSystemError.new("Path '#{path}' is a directory. Must use recursive delete") if !recursive
+
+            objs = entries(path).map(&:name)
+            @client.batch do |client|
+              objs.each do |obj|
+                client.delete_object(bucket, obj)
+              end
+            end
+            wait_until { !directory?(path) }
+            true
+          else
+            false
+          end
+        rescue
+          process_error($!)
+        end
+
+        def mkdir(path, parents: true, raise_if_exist: false)
+          if exist?(path)
+            if raise_if_exist
+              raise Tumugi::FileAlreadyExistError.new("Path #{path} is already exist")
+            elsif !directory?(path)
+              raise Tumugi::NotADirectoryError.new("Path #{path} is not a directory")
+            end
+            false
+          else
+            put_string("", add_path_delimiter(path))
+            true
+          end
+        rescue
+          process_error($!)
+        end
+
+        def directory?(path)
+          bucket, key = path_to_bucket_and_key(path)
+          if root?(key)
+            bucket_exist?(bucket)
+          else
+            obj = add_path_delimiter(key)
+            if obj_exist?(bucket, obj)
+              true
+            else
+              # Any objects with this prefix
+              objects = @client.list_objects(bucket, prefix: obj, max_results: 20)
+              !!(objects.items && objects.items.size > 0)
+            end
+          end
+        rescue
+          process_error($!)
+        end
+
+        def entries(path)
+          bucket, key = path_to_bucket_and_key(path)
+          obj = add_path_delimiter(key)
+          results = []
+          next_page_token = ''
+
+          until next_page_token.nil?
+            objects = @client.list_objects(bucket, prefix: obj, page_token: next_page_token)
+            if objects && objects.items
+              results.concat(objects.items)
+              next_page_token = objects.next_page_token
+            else
+              next_page_token = nil
+            end
+          end
+          results
+        rescue
+          process_error($!)
+        end
+
+        def move(src_path, dest_path, raise_if_exist: false)
+          copy(src_path, dest_path, raise_if_exist: raise_if_exist)
+          remove(src_path)
+        end
+
+        #######################################################################
+        # Specific methods
+        #######################################################################
+
+        def upload(media, path, content_type: nil)
+          bucket, key = path_to_bucket_and_key(path)
+          obj = Google::Apis::StorageV1::Object.new(bucket: bucket, name: key)
+          @client.insert_object(bucket, obj, upload_source: media, content_type: content_type)
+          wait_until { obj_exist?(bucket, key) }
+        rescue
+          process_error($!)
+        end
+
+        def download(path, download_path: nil, mode: 'r', &block)
+          bucket, key = path_to_bucket_and_key(path)
+          if download_path.nil?
+            download_path = Tempfile.new('tumugi_gcs_file_system').path
+          end
+          @client.get_object(bucket, key, download_dest: download_path)
+          wait_until { File.exist?(download_path) }
+
+          if block_given?
+            File.open(download_path, mode, &block)
+          else
+            File.open(download_path, mode)
+          end
+        rescue
+          process_error($!)
+        end
+
+        def put_string(contents, path, content_type: 'text/plain')
+          media = StringIO.new(contents)
+          upload(media, path, content_type: content_type)
+        end
+
+        def copy(src_path, dest_path, raise_if_exist: false)
+          if raise_if_exist && exist?(dest_path)
+            raise Tumugi::FileAlreadyExistError.new("Path #{dest_path} is already exist")
+          end
+
+          src_bucket, src_key = path_to_bucket_and_key(src_path)
+          dest_bucket, dest_key = path_to_bucket_and_key(dest_path)
+
+          if directory?(src_path)
+            src_prefix = add_path_delimiter(src_key)
+            dest_prefix = add_path_delimiter(dest_key)
+
+            src_path = add_path_delimiter(src_path)
+            copied_objs = []
+            entries(src_path).each do |entry|
+              suffix = entry.name[src_prefix.length..-1]
+              @client.copy_object(src_bucket, src_prefix + suffix,
+                                  dest_bucket, dest_prefix + suffix)
+              copied_objs << (dest_prefix + suffix)
+            end
+            wait_until { copied_objs.all? {|obj| obj_exist?(dest_bucket, obj)} }
+          else
+            @client.copy_object(src_bucket, src_key, dest_bucket, dest_key)
+            wait_until { obj_exist?(dest_bucket, dest_key) }
+          end
+        rescue
+          process_error($!)
+        end
+
+        def path_to_bucket_and_key(path)
+          uri = URI.parse(path)
+          raise Tumugi::FileSystemError.new("URI scheme must be 'gs' but '#{uri.scheme}'") unless uri.scheme == 'gs'
+          [ uri.host, uri.path[1..-1] ]
+        end
+
+        def create_bucket(bucket)
+          unless bucket_exist?(bucket)
+            b = Google::Apis::StorageV1::Bucket.new(name: bucket)
+            @client.insert_bucket(@project_id, b)
+            true
+          else
+            false
+          end
+        rescue
+          process_error($!)
+        end
+
+        def remove_bucket(bucket)
+          if bucket_exist?(bucket)
+            @client.delete_bucket(bucket)
+            true
+          else
+            false
+          end
+        rescue
+          process_error($!)
+        end
+
+        def bucket_exist?(bucket)
+          @client.get_bucket(bucket)
+          true
+        rescue => e
+          return false if e.status_code == 404
+          process_error(e)
+        end
+
+        private
+
+        def obj_exist?(bucket, key)
+          @client.get_object(bucket, key)
+          true
+        rescue => e
+          return false if e.status_code == 404
+          process_error(e)
+        end
+
+        def root?(key)
+          key.nil? || key == ''
+        end
+
+        def add_path_delimiter(key)
+          if key.end_with?('/')
+            key
+          else
+            "#{key}/"
+          end
+        end
+
+        def create_client(config)
+          if config.private_key_file.nil?
+            @project_id = config.project_id
+            client_email = config.client_email
+            private_key = config.private_key
+          else
+            json = JSON.parse(File.read(config.private_key_file))
+            @project_id = json['project_id']
+            client_email = json['client_email']
+            private_key = json['private_key']
+          end
+
+          # https://cloud.google.com/storage/docs/authentication
+          scope = "https://www.googleapis.com/auth/devstorage.read_write"
+
+          if client_email and private_key
+            auth = Signet::OAuth2::Client.new(
+              token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+              audience: "https://accounts.google.com/o/oauth2/token",
+              scope: scope,
+              issuer: client_email,
+              signing_key: OpenSSL::PKey.read(private_key))
+            # MEMO: signet-0.6.1 depend on Farady.default_connection
+            Faraday.default_connection.options.timeout = 60
+            auth.fetch_access_token!
+          else
+            auth = Google::Auth.get_application_default([scope])
+            auth.fetch_access_token!
+          end
+
+          client = Google::Apis::StorageV1::StorageService.new
+          client.authorization = auth
+          client
+        end
+
+        def wait_until(&block)
+          while not block.call
+            sleep 1
+          end
+        end
+
+        def process_error(err)
+          if err.respond_to?(:body)
+            begin
+              jobj = JSON.parse(err.body)
+              error = jobj["error"]
+              reason = error["errors"].map{|e| e["reason"]}.join(",")
+              errors = error["errors"].map{|e| e["message"] }.join("\n")
+            rescue JSON::ParserError
+              reason = err.status_code.to_s
+              errors = "HTTP Status: #{err.status_code}\nHeaders: #{err.header.inspect}\nBody:\n#{err.body}"
+            end
+            raise Tumugi::FileSystemError.new(errors, reason)
+          else
+            raise err
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/target/google_cloud_storage_file.rb

@@ -0,0 +1,41 @@
+require 'tumugi/config'
+require 'tumugi/plugin'
+require 'tumugi/plugin/file_system_target'
+require 'tumugi/plugin/google_cloud_storage/atomic_file'
+require 'tumugi/plugin/google_cloud_storage/file_system'
+
+module Tumugi
+  module Plugin
+    class GoogleCloudStorageFileTarget < Tumugi::Plugin::FileSystemTarget
+      Tumugi::Plugin.register_target('google_cloud_storage_file', self)
+      Tumugi::Config.register_section('google_cloud_storage', :project_id, :client_email, :private_key, :private_key_file)
+
+      attr_reader :bucket, :key, :path
+
+      def initialize(bucket:, key:)
+        @bucket = bucket
+        @key = key
+        @path = "gs://#{File.join(bucket, key)}"
+        log "bucket='#{bucket}, key='#{key}'"
+      end
+
+      def fs
+        @fs ||= Tumugi::Plugin::GoogleCloudStorage::FileSystem.new(Tumugi.config.section('google_cloud_storage'))
+      end
+
+      def open(mode="r", &block)
+        if mode.include? 'r'
+          fs.download(path, mode: mode, &block)
+        elsif mode.include? 'w'
+          Tumugi::Plugin::GoogleCloudStorage::AtomicFile.new(path, fs).open(&block)
+        else
+          raise Tumugi::TumugiError.new('Invalid mode: #{mode}')
+        end
+      end
+
+      def to_s
+        path
+      end
+    end
+  end
+end

data/tumugi-plugin-google_cloud_storage.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "tumugi-plugin-google_cloud_storage"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Kazuyuki Honda"]
+  spec.email         = ["hakobera@gmail.com"]
+
+  spec.summary       = "Tumugi plugin for Google Cloud Storage"
+  spec.homepage      = "https://github.com/tumugi/tumugi-plugin-google_cloud_storage"
+  spec.license       = "Apache License Version 2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.1'
+
+  spec.add_runtime_dependency "tumugi", "~> 0.5.1"
+  spec.add_runtime_dependency "google-api-client", "~> 0.9.3"
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", "~> 3.1"
+  spec.add_development_dependency "test-unit-rr"
+  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "github_changelog_generator"
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: tumugi-plugin-google_cloud_storage
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kazuyuki Honda
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tumugi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: github_changelog_generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- hakobera@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/example.rb
+- examples/tumugi_config_example.rb
+- lib/tumugi/plugin/google_cloud_storage/atomic_file.rb
+- lib/tumugi/plugin/google_cloud_storage/file_system.rb
+- lib/tumugi/plugin/target/google_cloud_storage_file.rb
+- tumugi-plugin-google_cloud_storage.gemspec
+homepage: https://github.com/tumugi/tumugi-plugin-google_cloud_storage
+licenses:
+- Apache License Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Tumugi plugin for Google Cloud Storage
+test_files: []