tumugi-plugin-google_drive 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 47062dacaeadccdc1a602bc81aa2cc4ca7291c59
+  data.tar.gz: 25690c27283aadbd29843ddd2bb6dcc59e3edbe4
+SHA512:
+  metadata.gz: f5b8763001af3a4c6cc5a60872aed270cac5108695627be3d45057961910b4bb12800d9ba95d8dcc20c11fe98db67ff7201bbbabf550255f0991f906bf7fec16
+  data.tar.gz: 27706dd1077923a595ca29c28ae1030adc519a300c24862fa354204c828c45af2cc388ac0c9313061868b7eda7fcf2487674553542fb389e73d5993dc4909fd7

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.env

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+cache: bundler
+rvm:
+  - 2.1
+  - 2.3.1
+  - jruby-9.1.2.0
+before_install:
+  - gem install bundler

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tumugi-plugin-google_drive.gemspec
+gemspec

data/README.md

@@ -0,0 +1,123 @@
+[![Build Status](https://travis-ci.org/tumugi/tumugi-plugin-google_drive.svg?branch=master)](https://travis-ci.org/tumugi/tumugi-plugin-google_drive) [![Code Climate](https://codeclimate.com/github/tumugi/tumugi-plugin-google_drive/badges/gpa.svg)](https://codeclimate.com/github/tumugi/tumugi-plugin-google_drive) [![Coverage Status](https://coveralls.io/repos/github/tumugi/tumugi-plugin-google_drive/badge.svg?branch=master)](https://coveralls.io/github/tumugi/tumugi-plugin-google_drive?branch=master) [![Gem Version](https://badge.fury.io/rb/tumugi-plugin-google_drive.svg)](https://badge.fury.io/rb/tumugi-plugin-google_drive)
+
+# tumugi-plugin-google_drive
+
+[tumugi](https://github.com/tumugi/tumugi) plugin for Google Drive.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'tumugi-plugin-google_drive'
+```
+
+And then execute:
+
+```sh
+$ bundle
+```
+
+Or install it yourself as:
+
+```sh
+$ gem install tumugi-plugin-google_drive
+```
+
+## Component
+
+### Tumugi::Plugin::GoogleDriveFileTarget
+
+This target represent file on Googl Drive.
+This target has following parameters.
+
+- name (required)
+  - Filename **string**
+- file_id
+  - File ID **string**
+- parents
+  - Parent folder ID **string** or **array of string**
+
+Tumugi workflow file using this target is like this:
+
+```rb
+task :task1 do
+  param :day, type: :time, auto_bind: true, required: true
+  output do
+    target(:google_drive_file,
+          name: "test_#{day.strftime('%Y%m%d')}.txt",
+          parents: "xyz")
+  end
+  run do
+    log 'task1#run'
+    output.open('w') {|f| f.puts('done') }
+  end
+end
+```
+
+### Tumugi::Plugin::GoogleDriveFolderTarget
+
+This target represent folder on Googl Drive.
+This target has following parameters.
+
+- name (required)
+  - Folder name **string**
+- folder_id
+  - Folder ID **string**
+- parents
+  - Parent folder ID **string** or **array of string**
+
+### Tumugi::Plugin::GoogleDriveFolderTask
+
+This task create a folder on Googl Drive.
+Tumugi workflow file using this task is like this:
+
+```rb
+task :task1, type: :google_drive_folder do
+  param :day, type: :time, auto_bind: true, required: true
+  output do
+    target(:google_drive_folder,
+          name: "test_#{day.strftime('%Y%m%d')}.txt",
+          parents: "xyz")
+  end
+end
+```
+
+### Config Section
+
+tumugi-plugin-google_drive provide config section named "google_drive" which can specified Google Drive autenticaion info.
+
+#### Authenticate by client_email and private_key
+
+```rb
+Tumugi.config do |config|
+  config.section("google_drive") do |section|
+    section.project_id = "xxx"
+    section.client_email = "yyy@yyy.iam.gserviceaccount.com"
+    section.private_key = "zzz"
+  end
+end
+```
+
+#### Authenticate by JSON key file
+
+```rb
+Tumugi.configure do |config|
+  config.section("google_drive") do |section|
+    section.private_key_file = "/path/to/key.json"
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/tumugi/ttumugi-plugin-google_drive
+
+## License
+
+The gem is available as open source under the terms of the [Apache License
+Version 2.0](http://www.apache.org/licenses/).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "tumugi/plugin/google_drive"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/example.rb

@@ -0,0 +1,17 @@
+task :task1 do
+  requires :folder
+  param :day, type: :time, auto_bind: true, required: true
+  output do
+    target(:google_drive_file,
+          name: "test_#{day.strftime('%Y%m%d')}.txt",
+          parents: input.folder_id)
+  end
+  run do
+    log 'task1#run'
+    output.open('w') {|f| f.puts('done') }
+  end
+end
+
+task :folder, type: :google_drive_folder do
+  param_set :name, 'xyz'
+end

data/examples/tumgui_config_example.rb

@@ -0,0 +1,7 @@
+Tumugi.configure do |config|
+  config.section('google_drive') do |section|
+    section.project_id = ENV["PROJECT_ID"]
+    section.client_email = ENV["CLIENT_EMAIL"]
+    section.private_key = ENV["PRIVATE_KEY"].gsub(/\\n/, "\n")
+  end
+end

data/lib/tumugi/plugin/google_drive/atomic_file.rb

@@ -0,0 +1,22 @@
+require 'tumugi/atomic_file'
+
+module Tumugi
+  module Plugin
+    module GoogleDrive
+      class AtomicFile < Tumugi::AtomicFile
+        attr_reader :id
+
+        def initialize(path, fs, file_id: nil, parents: nil)
+          super(path)
+          @fs = fs
+          @parents = parents
+          @id = (file_id.nil? ? @fs.generate_file_id : file_id)
+        end
+
+        def move_to_final_destination(temp_file)
+          @fs.upload(temp_file, path, file_id: @id, parents: @parents)
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/google_drive/file_system.rb

@@ -0,0 +1,194 @@
+require 'google/apis/drive_v3'
+require 'tumugi/error'
+
+Tumugi::Config.register_section('google_drive', :project_id, :client_email, :private_key, :private_key_file)
+
+module Tumugi
+  module Plugin
+    module GoogleDrive
+      class FileSystem
+
+        MIME_TYPE_FOLDER = 'application/vnd.google-apps.folder'
+
+        def initialize(config)
+          save_config(config)
+        end
+
+        def exist?(file_id)
+          client.get_file(file_id, options: request_options)
+          true
+        rescue => e
+          return false if e.respond_to?(:status_code) && e.status_code == 404
+          process_error(e)
+        end
+
+        def remove(file_id)
+          return unless exist?(file_id)
+
+          client.delete_file(file_id, options: request_options)
+          wait_until { !exist?(file_id) }
+          file_id
+        rescue
+          process_error($!)
+        end
+
+        def mkdir(name, folder_id: nil, parents: nil)
+          file_metadata = Google::Apis::DriveV3::File.new({
+            name: name,
+            mime_type: MIME_TYPE_FOLDER,
+            parents: parents,
+            id: folder_id
+          })
+          file = client.create_file(file_metadata, options: request_options)
+          wait_until { exist?(file.id) }
+          file
+        rescue
+          process_error($!)
+        end
+
+        def directory?(file_id)
+          file = client.get_file(file_id, options: request_options)
+          file.mime_type == MIME_TYPE_FOLDER
+        rescue
+          process_error($!)
+        end
+
+        def move(src_file_id, dest_name, dest_parents: nil)
+          file = copy(src_file_id, dest_name, dest_parents: dest_parents)
+          remove(src_file_id)
+          file
+        end
+
+        def copy(src_file_id, dest_name, dest_parents: nil)
+          dest_parents = [dest_parents] if dest_parents.is_a?(String)
+          dest_file_metadata = Google::Apis::DriveV3::File.new({
+            name: dest_name,
+            parents: dest_parents
+          })
+          file = client.copy_file(src_file_id, dest_file_metadata, options: request_options)
+          wait_until { exist?(file.id) }
+          file
+        rescue
+          process_error($!)
+        end
+
+        def upload(media, name, content_type: nil, file_id: nil, parents: nil)
+          parents = [parents] if parents.is_a?(String)
+          file_metadata = Google::Apis::DriveV3::File.new({
+            id: file_id,
+            name: name,
+            parents: parents
+          })
+          file = client.create_file(file_metadata, upload_source: media, content_type: content_type, options: request_options)
+          wait_until { exist?(file.id) }
+          file
+        rescue
+          process_error($!)
+        end
+
+        def put_string(contents, name, content_type: 'text/plain', file_id: nil, parents: nil)
+          media = StringIO.new(contents)
+          upload(media, name, content_type: content_type, file_id: file_id, parents: parents)
+        end
+
+        def download(file_id, download_path: nil, mode: 'r', &block)
+          if download_path.nil?
+            download_path = Tempfile.new('tumugi_google_drive_file_system').path
+          end
+          client.get_file(file_id, download_dest: download_path, options: request_options)
+          wait_until { File.exist?(download_path) }
+
+          if block_given?
+            File.open(download_path, mode, &block)
+          else
+            File.open(download_path, mode)
+          end
+        rescue
+          process_error($!)
+        end
+
+        def generate_file_id
+          client.generate_file_ids(count: 1, options: request_options).ids.first
+        rescue
+          process_error($!)
+        end
+
+        private
+
+        def save_config(config)
+          if config.private_key_file.nil?
+            @project_id = config.project_id
+            client_email = config.client_email
+            private_key = config.private_key
+          elsif config.private_key_file
+            json = JSON.parse(File.read(config.private_key_file))
+            @project_id = json['project_id']
+            client_email = json['client_email']
+            private_key = json['private_key']
+          end
+          @key = {
+            client_email: client_email,
+            private_key: private_key
+          }
+        end
+
+        def client
+          return @cached_client if @cached_client && @cached_client_expiration > Time.now
+
+          client = Google::Apis::DriveV3::DriveService.new
+          scope = Google::Apis::DriveV3::AUTH_DRIVE
+
+          if @key[:client_email] && @key[:private_key]
+            options = {
+              json_key_io: StringIO.new(JSON.generate(@key)),
+              scope: scope
+            }
+            auth = Google::Auth::ServiceAccountCredentials.make_creds(options)
+          else
+            auth = Google::Auth.get_application_default([scope])
+          end
+          auth.fetch_access_token!
+          client.authorization = auth
+
+          @cached_client_expiration = Time.now + (auth.expires_in / 2)
+          @cached_client = client
+        end
+
+        def wait_until(&block)
+          while not block.call
+            sleep 3
+          end
+        end
+
+        def process_error(err)
+          if err.respond_to?(:body)
+            begin
+              if err.body.nil?
+                reason = err.status_code.to_s
+                errors = "HTTP Status: #{err.status_code}\nHeaders: #{err.header.inspect}"
+              else
+                jobj = JSON.parse(err.body)
+                error = jobj["error"]
+                reason = error["errors"].map{|e| e["reason"]}.join(",")
+                errors = error["errors"].map{|e| e["message"] }.join("\n")
+              end
+            rescue JSON::ParserError
+              reason = err.status_code.to_s
+              errors = "HTTP Status: #{err.status_code}\nHeaders: #{err.header.inspect}\nBody:\n#{err.body}"
+            end
+            raise Tumugi::FileSystemError.new(errors, reason)
+          else
+            raise err
+          end
+        end
+
+        def request_options
+          {
+            retries: 5,
+            timeout_sec: 60
+          }
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/target/google_drive_file.rb

@@ -0,0 +1,49 @@
+require 'tumugi/config'
+require 'tumugi/plugin'
+require 'tumugi/plugin/file_system_target'
+require 'tumugi/plugin/google_drive/atomic_file'
+require 'tumugi/plugin/google_drive/file_system'
+
+module Tumugi
+  module Plugin
+    class GoogleDriveFileTarget < Tumugi::Plugin::FileSystemTarget
+      Tumugi::Plugin.register_target('google_drive_file', self)
+
+      attr_reader :file_id, :name, :parents
+
+      def initialize(file_id: nil, name: nil, parents: nil, fs: nil)
+        @fs = fs unless fs.nil?
+        @file_id = file_id || self.fs.generate_file_id
+        @name = name
+        @parents = parents
+        super(@file_id)
+      end
+
+      def fs
+        @fs ||= Tumugi::Plugin::GoogleDrive::FileSystem.new(Tumugi.config.section('google_drive'))
+      end
+
+      def open(mode="r", &block)
+        if mode.include? 'r'
+          fs.download(file_id, mode: mode, &block)
+        elsif mode.include? 'w'
+          file = Tumugi::Plugin::GoogleDrive::AtomicFile.new(name, fs, file_id: @file_id, parents: @parents)
+          file.open(&block)
+          @file_id = file.id
+        else
+          raise Tumugi::TumugiError.new('Invalid mode: #{mode}')
+        end
+      end
+
+      def exist?
+        fs.exist?(file_id)
+      end
+
+      def to_s
+        s = "file_id: #{file_id}, name: #{name}"
+        s += ", parents: #{parents}" if parents
+        s
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/target/google_drive_folder.rb

@@ -0,0 +1,34 @@
+require 'tumugi/config'
+require 'tumugi/plugin'
+require 'tumugi/plugin/google_drive/file_system'
+
+module Tumugi
+  module Plugin
+    class GoogleDriveFolderTarget < Tumugi::Target
+      Tumugi::Plugin.register_target('google_drive_folder', self)
+
+      attr_reader :folder_id, :name, :parents
+
+      def initialize(folder_id: nil, name: nil, parents: nil, fs: nil)
+        @fs = fs unless fs.nil?
+        @folder_id = folder_id || self.fs.generate_file_id
+        @name = name
+        @parents = parents
+      end
+
+      def fs
+        @fs ||= Tumugi::Plugin::GoogleDrive::FileSystem.new(Tumugi.config.section('google_drive'))
+      end
+
+      def exist?
+        fs.exist?(folder_id)
+      end
+
+      def to_s
+        s = "folder_id: #{folder_id}, name: #{name}"
+        s += ", parents: #{parents}" if parents
+        s
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/task/google_drive_folder.rb

@@ -0,0 +1,27 @@
+require 'tumugi'
+require_relative '../target/google_drive_folder'
+
+module Tumugi
+  module Plugin
+    class GoogleDriveFolderTask < Tumugi::Task
+      Tumugi::Plugin.register_task('google_drive_folder', self)
+
+      param :name, type: :string, required: true
+      param :folder_id, type: :string
+      param :parent, type: :string
+
+      def output
+        @output ||= Tumugi::Plugin::GoogleDriveFolderTarget.new(folder_id: folder_id, name: name, parents: parent.nil? ? nil : [parent])
+      end
+
+      def run
+        if output.exist?
+          log "skip: #{output} is already exists"
+        else
+          log "create folder: #{output}"
+          output.fs.mkdir(name, folder_id: output.folder_id, parents: output.parents)
+        end
+      end
+    end
+  end
+end

data/tumugi-plugin-google_drive.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "tumugi-plugin-google_drive"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Kazuyuki Honda"]
+  spec.email         = ["hakobera@gmail.com"]
+
+  spec.summary       = "Tumugi plugin for Google Drive"
+  spec.homepage      = "https://github.com/tumugi/tumugi-plugin-gcs"
+  spec.license       = "Apache License Version 2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.1'
+
+  spec.add_runtime_dependency "tumugi", ">= 0.5.1"
+  spec.add_runtime_dependency "google-api-client", "~> 0.9.3"
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", "~> 3.1"
+  spec.add_development_dependency "test-unit-rr"
+  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "github_changelog_generator"
+end

metadata

@@ -0,0 +1,171 @@
+--- !ruby/object:Gem::Specification
+name: tumugi-plugin-google_drive
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kazuyuki Honda
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tumugi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.3
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: github_changelog_generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- hakobera@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/example.rb
+- examples/tumgui_config_example.rb
+- lib/tumugi/plugin/google_drive/atomic_file.rb
+- lib/tumugi/plugin/google_drive/file_system.rb
+- lib/tumugi/plugin/target/google_drive_file.rb
+- lib/tumugi/plugin/target/google_drive_folder.rb
+- lib/tumugi/plugin/task/google_drive_folder.rb
+- tumugi-plugin-google_drive.gemspec
+homepage: https://github.com/tumugi/tumugi-plugin-gcs
+licenses:
+- Apache License Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Tumugi plugin for Google Drive
+test_files: []