tumugi-plugin-bigquery 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3ab49f7c9e04361951e1f7ab9eff4019d09c8829
+  data.tar.gz: 1b5ed8b53e77fb413a98ce6cfe4a682611b33f62
+SHA512:
+  metadata.gz: ea2e037bc46885c0e7cec71d085a5883dd9508f597d6e98d275b11464d2a9f27596a26cfd4a9141f51090b9ab2c9a96df63eb27c59ebba12113c00e2cac4d239
+  data.tar.gz: 1e18b4578eda83a38d200896cf0b8e22a85726b08f956bd20e9e794d81322df18416004d58f977e78b053b3609e559064d04cfe683ab1ecc60d32e8f89f34067

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+tumugi_config.rb

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+cache: bundler
+rvm:
+- 2.1.10
+- 2.2.5
+- 2.3.1
+- jruby-9.0.5.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tumugi-plugin-bigquery.gemspec
+gemspec

data/README.md

@@ -0,0 +1,126 @@
+[![Build Status](https://travis-ci.org/tumugi/tumugi-plugin-bigquery.svg?branch=master)](https://travis-ci.org/tumugi/tumugi-plugin-bigquery) [![Code Climate](https://codeclimate.com/github/tumugi/tumugi-plugin-bigquery/badges/gpa.svg)](https://codeclimate.com/github/tumugi/tumugi-plugin-bigquery) [![Coverage Status](https://coveralls.io/repos/github/tumugi/tumugi-plugin-bigquery/badge.svg?branch=master)](https://coveralls.io/github/tumugi/tumugi-plugin-bigquery)
+
+# tumugi-plugin-bigquery
+
+tumugi-plugin-bigquery is a plugin for integrate [Google BigQuery](https://cloud.google.com/bigquery/) and [Tumugi](https://github.com/tumugi/tumugi).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'tumugi-plugin-bigquery'
+```
+
+And then execute:
+
+```sh
+$ bundle
+```
+
+Or install it yourself as:
+
+```sb
+$ gem install tumugi-plugin-bigquery
+```
+
+## Target
+
+### Tumugi::Plugin::BigqueryDatasetTarget
+
+`Tumugi::Plugin::BigqueryDatasetTarget` is target for BigQuery dataset.
+
+#### Tumugi::Plugin::BigqueryTableTarget
+
+`Tumugi::Plugin::BigqueryDatasetTarget` is target for BigQuery table.
+
+## Task
+
+### Tumugi::Plugin::BigqueryDatasetTask
+
+`Tumugi::Plugin::BigqueryDatasetTask` is task to create a dataset.
+
+#### Usage
+
+```rb
+task :task1, type: :bigquery_dataset do
+  param_set :dataset_id, 'test'
+end
+```
+
+### Tumugi::Plugin::BigqueryQueryTask
+
+`Tumugi::Plugin::BigqueryQueryTask` is task to run `query` and save the result into the table which specified by parameter.
+
+#### Usage
+
+```rb
+task :task1, type: :bigquery_query do
+  param_set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  param_set :dataset_id, 'test'
+  param_set :table_id, "dest_table#{Time.now.to_i}"
+end
+```
+
+### Tumugi::Plugin::BigqueryCopyTask
+
+`Tumugi::Plugin::BigqueryCopyTask` is task to copy table which specified by parameter.
+
+#### Usage
+
+```rb
+task :task1, type: :bigquery_copy do
+  param_set :src_dataset_id, 'test'
+  param_set :src_table_id, 'src_table'
+  param_set :dest_dataset_id, 'test'
+  param_set :dest_table_id, 'dest_table'
+end
+```
+
+### Config Section
+
+tumugi-plugin-bigquery provide config section named "bigquery" which can specified BigQuery autenticaion info.
+
+#### Authenticate by client_email and private_key
+
+```rb
+Tumugi.config do |config|
+  config.section("bigquery") do |section|
+    section.project_id = "xxx"
+    section.client_email = "yyy@yyy.iam.gserviceaccount.com"
+    section.private_key = "zzz"
+  end
+end
+```
+
+#### Authenticate by JSON key file
+
+```rb
+Tumugi.config do |config|
+  config.section("bigquery") do |section|
+    section.private_key_file = "/path/to/key.json"
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies.
+Then, export [Google Cloud Platform Service Accounts](https://cloud.google.com/iam/docs/service-accounts) key as following,
+
+```sh
+export PROJECT_ID="xxx"
+export CLIENT_EMAIL="yyy@yyy.iam.gserviceaccount.com"
+export PRIVATE_KEY="zzz"
+```
+
+Then run `bundle exec rake test` to run the tests.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/tumugi/tumugi-plugin-bigquery.
+
+## License
+
+The gem is available as open source under the terms of the [Apache License
+Version 2.0](http://www.apache.org/licenses/).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/copy.rb

@@ -0,0 +1,21 @@
+task :task1, type: :bigquery_copy do
+  param_set :src_project_id, ->{ input.project_id }
+  param_set :src_dataset_id, ->{ input.dataset_id }
+  param_set :src_table_id, ->{ input.table_id }
+  param_set :dest_dataset_id, "test"
+  param_set :dest_table_id, ->{ "dest_table_#{Time.now.to_i}" }
+
+  requires :task2
+end
+
+task :task2, type: :bigquery_query do
+  param_set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  param_set :dataset_id, "test" #->{ input.dataset_id }
+  param_set :table_id, "dest_#{Time.now.to_i}"
+
+  requires :task3
+end
+
+task :task3, type: :bigquery_dataset do
+  param_set :dataset_id, "test"
+end

data/examples/dataset.rb

@@ -0,0 +1,10 @@
+task :task1 do
+  requires :task2
+  run do
+    log input.dataset_id
+  end
+end
+
+task :task2, type: :bigquery_dataset do
+  param_set :dataset_id, 'test'
+end

data/examples/query.rb

@@ -0,0 +1,12 @@
+task :task1 do
+  requires :task2
+  run do
+    log input.table_name
+  end
+end
+
+task :task2, type: :bigquery_query do
+  param_set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  param_set :dataset_id, 'test'
+  param_set :table_id, "dest_#{Time.now.to_i}"
+end

data/examples/tumugi_config_example.rb

@@ -0,0 +1,7 @@
+Tumugi.config do |c|
+  c.section('bigquery') do |s|
+    s.project_id = ENV["PROJECT_ID"]
+    s.client_email = ENV["CLIENT_EMAIL"]
+    s.private_key = ENV["PRIVATE_KEY"].gsub(/\\n/, "\n")
+  end
+end

data/lib/tumugi/plugin/bigquery/client.rb

@@ -0,0 +1,288 @@
+require 'kura'
+require_relative './error'
+
+Tumugi::Config.register_section('bigquery', :project_id, :client_email, :private_key, :private_key_file)
+
+module Tumugi
+  module Plugin
+    module Bigquery
+      class Client
+        attr_reader :project_id
+
+        def initialize(project_id: nil, client_email: nil, private_key: nil)
+          config = Tumugi.config.section('bigquery')
+          @project_id = project_id || config.project_id
+          @client_email = client_email || config.client_email
+          @private_key = private_key || config.private_key
+          @client = Kura.client(@project_id, @client_email, @private_key)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def projects(limit: 1000, &blk)
+          @client.projects(limit: limit, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def datasets(project_id: nil, all: false, limit: 1000, &blk)
+          @client.datasets(project_id: project_id || @project_id, all: all, limit: limit, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def dataset(dataset_id, project_id: nil, &blk)
+          @client.dataset(dataset_id, project_id: project_id || @project_id, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def dataset_exist?(dataset_id, project_id: nil)
+          !@client.dataset(dataset_id, project_id: project_id || @project_id).nil?
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def insert_dataset(dataset_id, project_id: nil, &blk)
+          @client.insert_dataset(dataset_id, project_id: project_id || @project_id, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def delete_dataset(dataset_id, project_id: nil, delete_contents: true, &blk)
+          @client.delete_dataset(dataset_id, project_id: project_id || @project_id, delete_contents: delete_contents, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def patch_dataset(dataset_id, project_id: nil,
+                          access: nil,
+                          description: :na,
+                          default_table_expiration_ms: :na,
+                          friendly_name: :na,
+                          &blk)
+          @client.patch_dataset(dataset_id, project_id: project_id || @project_id,
+                                access: access,
+                                description: description,
+                                default_table_expiration_ms: default_table_expiration_ms,
+                                friendly_name: friendly_name,
+                                &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def tables(dataset_id, project_id: nil, limit: 1000, &blk)
+          @client.tables(dataset_id, project_id: project_id || @project_id, limit: limit, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def table_exist?(dataset_id, table_id, project_id: nil)
+          !@client.table(dataset_id, table_id, project_id: project_id || @project_id).nil?
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def insert_table(dataset_id, table_id,
+                          project_id: nil,
+                          expiration_time: nil,
+                          friendly_name: nil,
+                          schema: nil,
+                          description: nil,
+                          query: nil,
+                          external_data_configuration: nil,
+                          &blk)
+          @client.insert_table(dataset_id, table_id,
+                                project_id: project_id || @project_id,
+                                expiration_time: expiration_time,
+                                friendly_name: friendly_name,
+                                schema: schema,
+                                description: description,
+                                query: query,
+                                external_data_configuration: external_data_configuration,
+                                &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def delete_table(dataset_id, table_id, project_id: nil, &blk)
+          @client.delete_table(dataset_id, table_id, project_id: project_id || @project_id, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def patch_table(dataset_id, table_id,
+                        project_id: nil,
+                        expiration_time: :na,
+                        friendly_name: :na,
+                        description: :na,
+                        &blk)
+          @client.patch_table(dataset_id, table_id,
+                              project_id: project_id || @project_id,
+                              expiration_time: expiration_time,
+                              friendly_name: friendly_name,
+                              description: description,
+                              &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def list_tabledata(dataset_id, table_id, project_id: nil, start_index: 0, max_result: 100, page_token: nil, schema: nil, &blk)
+          @client.list_tabledata(dataset_id, table_id,
+                                  project_id: project_id || @project_id,
+                                  start_index: start_index,
+                                  max_result: max_result,
+                                  page_token: page_token,
+                                  schema: schema,
+                                  &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def insert_tabledata(dataset_id, table_id, rows, project_id: nil, ignore_unknown_values: false, skip_invalid_rows: false, template_suffix: nil)
+          @client.insert_tabledata(dataset_id, table_id, rows,
+                                  project_id: project_id || @project_id,
+                                  ignore_unknown_values: ignore_unknown_values,
+                                  skip_invalid_rows: skip_invalid_rows,
+                                  template_suffix: template_suffix)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def insert_job(configuration, job_id: nil, project_id: nil, media: nil, wait: nil, &blk)
+          @client.insert_job(configuration, job_id: job_id, project_id: project_id || @project_id, media: media, wait: wait, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def query(sql, mode: :truncate,
+                  dataset_id: nil, table_id: nil,
+                  allow_large_results: true,
+                  flatten_results: true,
+                  priority: "INTERACTIVE",
+                  use_query_cache: true,
+                  user_defined_function_resources: nil,
+                  project_id: nil,
+                  job_id: nil,
+                  wait: nil,
+                  dry_run: false,
+                  &blk)
+          @client.query(sql, mode: mode,
+                        dataset_id: dataset_id, table_id: table_id,
+                        allow_large_results: allow_large_results,
+                        flatten_results: flatten_results,
+                        priority: priority,
+                        use_query_cache: use_query_cache,
+                        user_defined_function_resources: user_defined_function_resources,
+                        project_id: project_id || @project_id,
+                        job_project_id: project_id || @project_id,
+                        job_id: job_id,
+                        wait: wait,
+                        dry_run: dry_run,
+                        &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def load(dataset_id, table_id, source_uris=nil,
+                 schema: nil, delimiter: ",", field_delimiter: delimiter, mode: :append,
+                 allow_jagged_rows: false, max_bad_records: 0,
+                 ignore_unknown_values: false,
+                 allow_quoted_newlines: false,
+                 quote: '"', skip_leading_rows: 0,
+                 source_format: "CSV",
+                 project_id: nil,
+                 job_id: nil,
+                 file: nil, wait: nil,
+                 dry_run: false,
+                 &blk)
+          @client.load(dataset_id, table_id, source_uris=source_uris,
+                       schema: schema, delimiter: delimiter, field_delimiter: field_delimiter, mode: mode,
+                       allow_jagged_rows: allow_jagged_rows, max_bad_records: max_bad_records,
+                       ignore_unknown_values: ignore_unknown_values,
+                       allow_quoted_newlines: allow_quoted_newlines,
+                       quote: quote, skip_leading_rows: skip_leading_rows,
+                       source_format: source_format,
+                       project_id: project_id || @project_id,
+                       job_project_id: project_id || @project_id,
+                       job_id: job_id,
+                       file: file, wait: wait,
+                       dry_run: dry_run,
+                       &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def extract(dataset_id, table_id, dest_uris,
+                    compression: "NONE",
+                    destination_format: "CSV",
+                    field_delimiter: ",",
+                    print_header: true,
+                    project_id: nil,
+                    job_id: nil,
+                    wait: nil,
+                    dry_run: false,
+                    &blk)
+          @client.extract(dataset_id, table_id, dest_uris,
+                          compression: compression,
+                          destination_format: destination_format,
+                          field_delimiter: field_delimiter,
+                          print_header: print_header,
+                          project_id: project_id || @project_id,
+                          job_project_id: project_id || @project_id,
+                          job_id: job_id,
+                          wait: wait,
+                          dry_run: dry_run,
+                          &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def copy(src_dataset_id, src_table_id, dest_dataset_id, dest_table_id,
+                 mode: :truncate,
+                 src_project_id: nil,
+                 dest_project_id: nil,
+                 job_id: nil,
+                 wait: nil,
+                 dry_run: false,
+                 &blk)
+          @client.copy(src_dataset_id, src_table_id, dest_dataset_id, dest_table_id,
+                       mode: mode,
+                       src_project_id: src_project_id || @project_id,
+                       dest_project_id: dest_project_id || @project_id,
+                       job_project_id: dest_project_id || @project_id,
+                       job_id: job_id,
+                       wait: wait,
+                       dry_run: dry_run,
+                       &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def job(job_id, project_id: nil, &blk)
+          @client.job(job_id, project_id: project_id || @project_id, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def cancel_job(job_id, project_id: nil, &blk)
+          @client.cancel_job(job_id, project_id: project_id || @project_id, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        def wait_job(job, timeout=60*10, project_id: nil, &blk)
+          @client.wait_job(job, timeout, project_id: project_id || @project_id, &blk)
+        rescue Kura::ApiError => e
+          process_error(e)
+        end
+
+        private
+
+        def process_error(e)
+          raise Tumugi::Plugin::Bigquery::BigqueryError.new(e.reason, e.message)
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/bigquery/dataset.rb

@@ -0,0 +1,26 @@
+module Tumugi
+  module Plugin
+    module Bigquery
+      class Dataset
+        attr_reader :project_id, :dataset_id
+
+        def initialize(project_id:, dataset_id:)
+          @project_id = project_id
+          @dataset_id = dataset_id
+        end
+
+        def dataset_name
+          "#{dataset_id}"
+        end
+
+        def dataset_full_name
+          "#{project_id}:#{dataset_id}"
+        end
+
+        def to_s
+          "bq://#{project_id}/#{dataset_id}"
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/bigquery/error.rb

@@ -0,0 +1,15 @@
+require 'tumugi/error'
+
+module Tumugi
+  module Plugin
+    module Bigquery
+      class BigqueryError < Tumugi::TumugiError
+        def initialize(reason, message)
+          @reason = reason
+          super(message)
+        end
+        attr_reader :reason
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/bigquery/table.rb

@@ -0,0 +1,33 @@
+require_relative './dataset'
+
+module Tumugi
+  module Plugin
+    module Bigquery
+      class Table
+        attr_reader :project_id, :dataset_id, :table_id
+
+        def initialize(project_id:, dataset_id:, table_id:)
+          @project_id = project_id
+          @dataset_id = dataset_id
+          @table_id = table_id
+        end
+
+        def dataset
+          Tumugi::Plugin::Bigquery::Dataset.new(project_id: @project_id, dataset_id: @dataset_id)
+        end
+
+        def table_name
+          "#{dataset_id}.#{table_id}"
+        end
+
+        def table_full_name
+          "#{project_id}:#{dataset_id}.#{table_id}"
+        end
+
+        def to_s
+          "bq://#{project_id}/#{dataset_id}/#{table_id}"
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/bigquery/version.rb

@@ -0,0 +1,7 @@
+module Tumugi
+  module Plugin
+    module Bigquery
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/tumugi/plugin/target/bigquery_dataset.rb

@@ -0,0 +1,29 @@
+require 'forwardable'
+require 'tumugi'
+require_relative '../bigquery/client'
+require_relative '../bigquery/dataset'
+
+module Tumugi
+  module Plugin
+    class BigqueryDatasetTarget
+      Tumugi::Plugin.register_target('bigquery_dataset', self)
+
+      attr_reader :project_id, :dataset_id, :client
+
+      extend Forwardable
+      def_delegators :@dataset, :dataset_name, :dataset_full_name, :to_s
+
+      def initialize(project_id: nil, dataset_id:, client: nil)
+        cfg = Tumugi.config.section('bigquery')
+        @project_id = project_id || cfg.project_id
+        @dataset_id = dataset_id
+        @client = client || Tumugi::Plugin::Bigquery::Client.new(project_id: @project_id)
+        @dataset = Tumugi::Plugin::Bigquery::Dataset.new(project_id: @project_id, dataset_id: @dataset_id)
+      end
+
+      def exist?
+        @client.dataset_exist?(@dataset_id, project_id: @project_id)
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/target/bigquery_table.rb

@@ -0,0 +1,30 @@
+require 'forwardable'
+require 'tumugi'
+require_relative '../bigquery/client'
+require_relative '../bigquery/table'
+
+module Tumugi
+  module Plugin
+    class BigqueryTableTarget
+      Tumugi::Plugin.register_target('bigquery_table', self)
+
+      extend Forwardable
+      def_delegators :@table, :table_name, :table_full_name, :to_s
+
+      attr_reader :project_id, :dataset_id, :table_id, :client
+
+      def initialize(project_id: nil, dataset_id:, table_id:, client: nil)
+        cfg = Tumugi.config.section('bigquery')
+        @project_id = project_id || cfg.project_id
+        @dataset_id = dataset_id
+        @table_id = table_id
+        @client = client || Tumugi::Plugin::Bigquery::Client.new(project_id: @project_id)
+        @table = Tumugi::Plugin::Bigquery::Table.new(project_id: @project_id, dataset_id: @dataset_id, table_id: @table_id)
+      end
+
+      def exist?
+        @client.table_exist?(@dataset_id, @table_id, project_id: @project_id)
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/task/bigquery_copy.rb

@@ -0,0 +1,35 @@
+require 'tumugi'
+require_relative '../target/bigquery_table'
+
+module Tumugi
+  module Plugin
+    class BigqueryCopyTask < Tumugi::Task
+      Tumugi::Plugin.register_task('bigquery_copy', self)
+
+      param :src_project_id, type: :string
+      param :src_dataset_id, type: :string, required: true
+      param :src_table_id, type: :string, required: true
+      param :dest_project_id, type: :string
+      param :dest_dataset_id, type: :string, required: true
+      param :dest_table_id, type: :string, required: true
+      param :wait, type: :int, default: 60
+
+      def output
+        opts = { dataset_id: dest_dataset_id, table_id: dest_table_id }
+        opts[:project_id] = dest_project_id if dest_project_id
+        Tumugi::Plugin::BigqueryTableTarget.new(opts)
+      end
+
+      def run
+        log "Source: bq://#{src_project_id}/#{src_dataset_id}/#{src_table_id}"
+        log "Destination: #{output}"
+
+        bq_client = output.client
+        opts = { wait: wait }
+        opts[:src_project_id] = src_project_id if src_project_id
+        opts[:dest_project_id] = dest_project_id if dest_project_id
+        bq_client.copy(src_dataset_id, src_table_id, dest_dataset_id, dest_table_id, opts)
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/task/bigquery_dataset.rb

@@ -0,0 +1,28 @@
+require 'tumugi'
+require_relative '../target/bigquery_dataset'
+
+module Tumugi
+  module Plugin
+    class BigqueryDatasetTask < Tumugi::Task
+      Tumugi::Plugin.register_task('bigquery_dataset', self)
+
+      param :project_id, type: :string
+      param :dataset_id, type: :string, required: true
+
+      def output
+        Tumugi::Plugin::BigqueryDatasetTarget.new(project_id: project_id, dataset_id: dataset_id)
+      end
+
+      def run
+        log "Dataset: #{output}"
+        if output.exist?
+          log "skip: #{output} is already exists"
+        else
+          bq_client = output.client
+          bq_client.insert_dataset(dataset_id, project_id: project_id)
+          log "run: #{output}"
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/task/bigquery_query.rb

@@ -0,0 +1,29 @@
+require 'tumugi'
+require_relative '../target/bigquery_table'
+
+module Tumugi
+  module Plugin
+    class BigqueryQueryTask < Tumugi::Task
+      Tumugi::Plugin.register_task('bigquery_query', self)
+
+      param :query, type: :string, required: true
+      param :project_id, type: :string
+      param :dataset_id, type: :string, required: true
+      param :table_id, type: :string, required: true
+      param :wait, type: :int, default: 60
+
+      def output
+        Tumugi::Plugin::BigqueryTableTarget.new(project_id: project_id, dataset_id: dataset_id, table_id: table_id)
+      end
+
+      def run
+        log "Launching Query"
+        log "Query: #{query}"
+        log "Query destination: #{output}"
+
+        bq_client = output.client
+        bq_client.query(query, project_id: project_id, dataset_id: output.dataset_id, table_id: output.table_id, wait: wait)
+      end
+    end
+  end
+end

data/tumugi-plugin-bigquery.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tumugi/plugin/bigquery/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "tumugi-plugin-bigquery"
+  spec.version       = Tumugi::Plugin::Bigquery::VERSION
+  spec.authors       = ["Kazuyuki Honda"]
+  spec.email         = ["hakobera@gmail.com"]
+
+  spec.summary       = "Tumugi plugin for Google BigQuery"
+  spec.homepage      = "https://github.com/tumugi/tumugi-plugin-bigquery"
+  spec.license       = "Apache License Version 2.0"
+
+  spec.required_ruby_version = '>= 2.1'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "tumugi", "~> 0.4.5"
+  spec.add_runtime_dependency "kura", "0.2.16"
+
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'test-unit', '~> 3.1'
+  spec.add_development_dependency 'test-unit-rr'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'github_changelog_generator'
+end

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification
+name: tumugi-plugin-bigquery
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kazuyuki Honda
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tumugi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.5
+- !ruby/object:Gem::Dependency
+  name: kura
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.16
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.16
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: github_changelog_generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- hakobera@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/setup
+- examples/copy.rb
+- examples/dataset.rb
+- examples/query.rb
+- examples/tumugi_config_example.rb
+- lib/tumugi/plugin/bigquery/client.rb
+- lib/tumugi/plugin/bigquery/dataset.rb
+- lib/tumugi/plugin/bigquery/error.rb
+- lib/tumugi/plugin/bigquery/table.rb
+- lib/tumugi/plugin/bigquery/version.rb
+- lib/tumugi/plugin/target/bigquery_dataset.rb
+- lib/tumugi/plugin/target/bigquery_table.rb
+- lib/tumugi/plugin/task/bigquery_copy.rb
+- lib/tumugi/plugin/task/bigquery_dataset.rb
+- lib/tumugi/plugin/task/bigquery_query.rb
+- tumugi-plugin-bigquery.gemspec
+homepage: https://github.com/tumugi/tumugi-plugin-bigquery
+licenses:
+- Apache License Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Tumugi plugin for Google BigQuery
+test_files: []