embulk-input-bigquery 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +19 -10
- data/lib/embulk/input/bigquery.rb +28 -2
- data/lib/embulk/input/bigquery/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7f8b54f5c0bf602236407494d3c0bf35257e04d
|
4
|
+
data.tar.gz: bd1116a46dc6016dc58cc536d69e1b71c0f80242
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dfcfe921546bc8d89e2df091056247f3a1b1a1eb17590eea4f512d7eb3f8856fe5f778c071b281f75e8349a4af93e2a8a3131aeadbf5b8893acb2bbea93fb547
|
7
|
+
data.tar.gz: 9f00dd42dc3219ab1f75a445609cc73419f36c87c4ef5ab98e00954bc990c8560f42f3fc75208037d542ef3767c54b6356932f9bdb68ff45286c0007d4b05e85
|
data/README.md
CHANGED
@@ -4,17 +4,9 @@ This is Embulk input plugin from Bigquery.
|
|
4
4
|
|
5
5
|
## Installation
|
6
6
|
|
7
|
-
|
7
|
+
install it yourself as:
|
8
8
|
|
9
|
-
gem
|
10
|
-
|
11
|
-
And then execute:
|
12
|
-
|
13
|
-
$ bundle
|
14
|
-
|
15
|
-
Or install it yourself as:
|
16
|
-
|
17
|
-
$ gem install embulk-input-bigquery
|
9
|
+
$ embulk gem install embulk-input-bigquery
|
18
10
|
|
19
11
|
## Usage
|
20
12
|
|
@@ -30,3 +22,20 @@ in:
|
|
30
22
|
out:
|
31
23
|
type: stdout
|
32
24
|
```
|
25
|
+
|
26
|
+
If, table name is changeable, then
|
27
|
+
|
28
|
+
```
|
29
|
+
in:
|
30
|
+
type: bigquery
|
31
|
+
project: 'project-name'
|
32
|
+
keyfile: '/home/hogehoge/bigquery-keyfile.json'
|
33
|
+
sql: 'SELECT price,category_id FROM [ecsite.products_<%= params["date"].strftime("%Y%m") %>] GROUP BY category_id'
|
34
|
+
erb_params:
|
35
|
+
date: "require 'date'; (Date.today - 1)"
|
36
|
+
columns:
|
37
|
+
- {name: price, type: long}
|
38
|
+
- {name: category_id, type: long}
|
39
|
+
- {name: month, type: timestamp, format: '%Y-%m', eval: 'require "time"; Time.parse(params["date"]).to_i'}
|
40
|
+
```
|
41
|
+
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "embulk/input/bigquery/version"
|
2
2
|
require "google/cloud/bigquery"
|
3
|
+
require 'erb'
|
3
4
|
|
4
5
|
module Embulk
|
5
6
|
module Input
|
@@ -7,8 +8,27 @@ module Embulk
|
|
7
8
|
Plugin.register_input('bigquery', self)
|
8
9
|
|
9
10
|
def self.transaction(config, &control)
|
11
|
+
sql = config[:sql]
|
12
|
+
params = {}
|
13
|
+
unless sql
|
14
|
+
sql_erb = config[:sql_erb]
|
15
|
+
erb = ERB.new(sql_erb)
|
16
|
+
erb_params = config[:erb_params]
|
17
|
+
erb_params.each do |k, v|
|
18
|
+
params[k] = eval(v)
|
19
|
+
end
|
20
|
+
|
21
|
+
sql = erb.result(binding)
|
22
|
+
end
|
23
|
+
|
24
|
+
task = {
|
25
|
+
project: config[:project],
|
26
|
+
keyfile: config[:keyfile],
|
27
|
+
sql: sql,
|
28
|
+
columns: config[:columns],
|
29
|
+
params: params
|
30
|
+
}
|
10
31
|
|
11
|
-
task = {project: config[:project], keyfile: config[:keyfile], sql: config[:sql], columns: config[:columns]}
|
12
32
|
columns = []
|
13
33
|
config[:columns].each_with_index do |c, i|
|
14
34
|
columns << Column.new(i, c['name'], c['type'].to_sym)
|
@@ -21,12 +41,18 @@ module Embulk
|
|
21
41
|
|
22
42
|
def run
|
23
43
|
bq = Google::Cloud::Bigquery.new(project: @task[:project], keyfile: @task[:keyfile])
|
44
|
+
params = @task[:params]
|
24
45
|
rows = bq.query(@task[:sql])
|
25
46
|
rows.each do |row|
|
26
47
|
columns = []
|
27
48
|
@task[:columns].each do |c|
|
28
|
-
|
49
|
+
val = row[c['name']]
|
50
|
+
if c['eval']
|
51
|
+
val = eval(c['eval'], binding)
|
52
|
+
end
|
53
|
+
columns << val
|
29
54
|
end
|
55
|
+
|
30
56
|
@page_builder.add(columns)
|
31
57
|
end
|
32
58
|
@page_builder.finish
|