embulk-input-bigquery 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8993211376746215694e0c70cee753dd6751444a
-  data.tar.gz: 70d9952fbd03e1567e6fec62fa8410e02212095f
+  metadata.gz: b7f8b54f5c0bf602236407494d3c0bf35257e04d
+  data.tar.gz: bd1116a46dc6016dc58cc536d69e1b71c0f80242
 SHA512:
-  metadata.gz: 8e8c18cdbc6bcda320cc998ea594e6c20d2679d1c8aebcd85a7557ace7de3d7ce0f04e47685dee947bd591f9a7072ab3900c2c332d1764a9ce859c8ffed69385
-  data.tar.gz: 2692842d7386fc9d40026a12635017eb477495064ae0d3ecfe1f645645245232f82dc00670cb2f5c305679f4a7eb01e5915c86f60574a75fb00aebb40d504b2f
+  metadata.gz: dfcfe921546bc8d89e2df091056247f3a1b1a1eb17590eea4f512d7eb3f8856fe5f778c071b281f75e8349a4af93e2a8a3131aeadbf5b8893acb2bbea93fb547
+  data.tar.gz: 9f00dd42dc3219ab1f75a445609cc73419f36c87c4ef5ab98e00954bc990c8560f42f3fc75208037d542ef3767c54b6356932f9bdb68ff45286c0007d4b05e85

data/README.md

@@ -4,17 +4,9 @@ This is Embulk input plugin from Bigquery.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+install it yourself as:
 
-    gem 'embulk-input-bigquery'
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install embulk-input-bigquery
+    $ embulk gem install embulk-input-bigquery
 
 ## Usage
 
@@ -30,3 +22,20 @@ in:
 out:
   type: stdout
 ```
+
+If, table name is changeable, then
+
+```
+in:
+  type: bigquery
+  project: 'project-name'
+  keyfile: '/home/hogehoge/bigquery-keyfile.json'
+  sql: 'SELECT price,category_id FROM [ecsite.products_<%= params["date"].strftime("%Y%m")  %>] GROUP BY category_id'
+  erb_params:
+    date: "require 'date'; (Date.today - 1)"
+  columns:
+    - {name: price, type: long}
+    - {name: category_id, type: long}
+    - {name: month, type: timestamp, format: '%Y-%m', eval: 'require "time"; Time.parse(params["date"]).to_i'}
+```
+

data/lib/embulk/input/bigquery.rb

@@ -1,5 +1,6 @@
 require "embulk/input/bigquery/version"
 require "google/cloud/bigquery"
+require 'erb'
 
 module Embulk
   module Input
@@ -7,8 +8,27 @@ module Embulk
 			Plugin.register_input('bigquery', self)
 
 			def self.transaction(config, &control)
+				sql = config[:sql]
+				params = {}
+				unless sql
+					sql_erb = config[:sql_erb]
+					erb = ERB.new(sql_erb)
+					erb_params = config[:erb_params]
+					erb_params.each do |k, v|
+						params[k] = eval(v)
+					end
+
+					sql = erb.result(binding)
+				end
+
+				task = {
+					project: config[:project],
+					keyfile: config[:keyfile],
+					sql: sql,
+					columns: config[:columns],
+					params: params
+				}
 
-				task = {project: config[:project], keyfile: config[:keyfile], sql: config[:sql], columns: config[:columns]}
 				columns = []
 				config[:columns].each_with_index do |c, i|
 					columns << Column.new(i, c['name'], c['type'].to_sym)
@@ -21,12 +41,18 @@ module Embulk
 
 			def run
 				bq = Google::Cloud::Bigquery.new(project: @task[:project], keyfile: @task[:keyfile])
+				params = @task[:params]
 				rows = bq.query(@task[:sql])
 				rows.each do |row|
 					columns = []
 					@task[:columns].each do |c|
-						columns << row[c['name']]
+						val = row[c['name']]
+						if c['eval']
+							val = eval(c['eval'], binding)
+						end
+						columns << val
 					end
+
 					@page_builder.add(columns)
 				end
 				@page_builder.finish

data/lib/embulk/input/bigquery/version.rb

@@ -1,7 +1,7 @@
 module Embulk
   module Input
     module Bigquery
-      VERSION = "0.0.1"
+      VERSION = "0.0.2"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Takeru Narita