embulk-output-bigquery 0.2.3 → 0.3.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +6 -12
- data/CHANGELOG.md +18 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +20 -0
- data/README.md +165 -39
- data/Rakefile +11 -0
- data/embulk-output-bigquery.gemspec +20 -0
- data/example/config_client_options.yml +33 -0
- data/example/config_csv.yml +30 -0
- data/example/config_delete_in_advance.yml +29 -0
- data/example/config_expose_errors.yml +30 -0
- data/example/config_guess_from_embulk_schema.yml +29 -0
- data/example/config_guess_with_column_options.yml +40 -0
- data/example/config_gzip.yml +30 -0
- data/example/config_jsonl.yml +30 -0
- data/example/config_mode_append.yml +30 -0
- data/example/config_mode_append_direct.yml +30 -0
- data/example/config_payload_column.yml +20 -0
- data/example/config_payload_column_index.yml +20 -0
- data/example/config_prevent_duplicate_insert.yml +30 -0
- data/example/config_replace.yml +30 -0
- data/example/config_replace_backup.yml +32 -0
- data/example/config_skip_file_generation.yml +32 -0
- data/example/config_table_strftime.yml +30 -0
- data/example/config_template_table.yml +21 -0
- data/example/config_uncompressed.yml +30 -0
- data/example/config_with_rehearsal.yml +32 -0
- data/example/example.csv +17 -0
- data/example/example.jsonl +16 -0
- data/example/example.yml +30 -0
- data/example/json_key.json +12 -0
- data/example/nested_example.jsonl +16 -0
- data/example/schema.json +30 -0
- data/example/schema_expose_errors.json +30 -0
- data/lib/embulk/output/bigquery.rb +388 -3
- data/lib/embulk/output/bigquery/bigquery_client.rb +396 -0
- data/lib/embulk/output/bigquery/file_writer.rb +103 -0
- data/lib/embulk/output/bigquery/helper.rb +78 -0
- data/lib/embulk/output/bigquery/value_converter_factory.rb +292 -0
- data/test/helper.rb +13 -0
- data/test/test_bigquery_client.rb +166 -0
- data/test/test_configure.rb +254 -0
- data/test/test_example.rb +34 -0
- data/test/test_file_writer.rb +129 -0
- data/test/test_helper.rb +103 -0
- data/test/test_transaction.rb +129 -0
- data/test/test_value_converter_factory.rb +316 -0
- metadata +114 -45
- data/build.gradle +0 -80
- data/config/checkstyle/checkstyle.xml +0 -128
- data/config/checkstyle/default.xml +0 -108
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +0 -6
- data/gradlew +0 -164
- data/gradlew.bat +0 -90
- data/settings.gradle +0 -2
- data/src/main/java/org/embulk/output/BigqueryAuthentication.java +0 -117
- data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java +0 -508
- data/src/main/java/org/embulk/output/BigqueryWriter.java +0 -575
- data/src/test/java/org/embulk/output/TestBigqueryAuthentication.java +0 -5
- data/src/test/java/org/embulk/output/TestBigqueryOutputPlugin.java +0 -5
- data/src/test/java/org/embulk/output/TestBigqueryWriter.java +0 -5
@@ -0,0 +1,129 @@
|
|
1
|
+
require_relative './helper'
|
2
|
+
require 'embulk/output/bigquery'
|
3
|
+
|
4
|
+
Bigquery = Embulk::Output::Bigquery unless defined?(Bigquery)
|
5
|
+
|
6
|
+
module Embulk
|
7
|
+
class Output::Bigquery
|
8
|
+
class TestTransaction < Test::Unit::TestCase
|
9
|
+
def least_config
|
10
|
+
DataSource.new({
|
11
|
+
'project' => 'your_project_name',
|
12
|
+
'dataset' => 'your_dataset_name',
|
13
|
+
'table' => 'your_table_name',
|
14
|
+
'p12_keyfile' => __FILE__, # fake
|
15
|
+
})
|
16
|
+
end
|
17
|
+
|
18
|
+
def schema
|
19
|
+
Schema.new([
|
20
|
+
Column.new({index: 0, name: 'boolean', type: :boolean}),
|
21
|
+
Column.new({index: 1, name: 'long', type: :long}),
|
22
|
+
Column.new({index: 2, name: 'double', type: :double}),
|
23
|
+
Column.new({index: 3, name: 'string', type: :string}),
|
24
|
+
Column.new({index: 4, name: 'timestamp', type: :timestamp}),
|
25
|
+
Column.new({index: 5, name: 'json', type: :json}),
|
26
|
+
])
|
27
|
+
end
|
28
|
+
|
29
|
+
def processor_count
|
30
|
+
1
|
31
|
+
end
|
32
|
+
|
33
|
+
def control
|
34
|
+
Proc.new {|task| task_reports = [] }
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_append
|
38
|
+
config = least_config.merge('mode' => 'append', 'temp_table' => 'temp_table')
|
39
|
+
any_instance_of(BigqueryClient) do |obj|
|
40
|
+
mock(obj).get_dataset(config['dataset'])
|
41
|
+
mock(obj).create_table(config['temp_table'])
|
42
|
+
mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
|
43
|
+
mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND')
|
44
|
+
mock(obj).delete_table(config['temp_table'])
|
45
|
+
end
|
46
|
+
Bigquery.transaction(config, schema, processor_count, &control)
|
47
|
+
end
|
48
|
+
|
49
|
+
sub_test_case "append_direct" do
|
50
|
+
def test_append_direct
|
51
|
+
config = least_config.merge('mode' => 'append_direct')
|
52
|
+
any_instance_of(BigqueryClient) do |obj|
|
53
|
+
mock(obj).get_dataset(config['dataset'])
|
54
|
+
mock(obj).get_table(config['table'])
|
55
|
+
mock(obj).load_in_parallel(anything, config['table']) { [] }
|
56
|
+
end
|
57
|
+
Bigquery.transaction(config, schema, processor_count, &control)
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_append_direct_with_auto_create
|
61
|
+
config = least_config.merge('mode' => 'append_direct', 'auto_create_dataset' => true, 'auto_create_table' => true)
|
62
|
+
any_instance_of(BigqueryClient) do |obj|
|
63
|
+
mock(obj).create_dataset(config['dataset'])
|
64
|
+
mock(obj).create_table(config['table'])
|
65
|
+
mock(obj).load_in_parallel(anything, config['table']) { [] }
|
66
|
+
end
|
67
|
+
Bigquery.transaction(config, schema, processor_count, &control)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def test_delete_in_advance
|
72
|
+
config = least_config.merge('mode' => 'delete_in_advance')
|
73
|
+
any_instance_of(BigqueryClient) do |obj|
|
74
|
+
mock(obj).get_dataset(config['dataset'])
|
75
|
+
mock(obj).delete_table(config['table'])
|
76
|
+
mock(obj).create_table(config['table'])
|
77
|
+
mock(obj).load_in_parallel(anything, config['table']) { [] }
|
78
|
+
end
|
79
|
+
Bigquery.transaction(config, schema, processor_count, &control)
|
80
|
+
end
|
81
|
+
|
82
|
+
def test_replace
|
83
|
+
config = least_config.merge('mode' => 'replace', 'temp_table' => 'temp_table')
|
84
|
+
any_instance_of(BigqueryClient) do |obj|
|
85
|
+
mock(obj).get_dataset(config['dataset'])
|
86
|
+
mock(obj).create_table(config['temp_table'])
|
87
|
+
mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
|
88
|
+
mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
|
89
|
+
mock(obj).delete_table(config['temp_table'])
|
90
|
+
end
|
91
|
+
Bigquery.transaction(config, schema, processor_count, &control)
|
92
|
+
end
|
93
|
+
|
94
|
+
sub_test_case "replace_backup" do
|
95
|
+
def test_replace_backup
|
96
|
+
config = least_config.merge('mode' => 'replace_backup', 'dataset_old' => 'dataset_old', 'table_old' => 'table_old', 'temp_table' => 'temp_table')
|
97
|
+
any_instance_of(BigqueryClient) do |obj|
|
98
|
+
mock(obj).get_dataset(config['dataset'])
|
99
|
+
mock(obj).get_dataset(config['dataset_old'])
|
100
|
+
mock(obj).create_table(config['temp_table'])
|
101
|
+
mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
|
102
|
+
|
103
|
+
mock(obj).copy(config['table'], config['table_old'], config['dataset_old'])
|
104
|
+
|
105
|
+
mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
|
106
|
+
mock(obj).delete_table(config['temp_table'])
|
107
|
+
end
|
108
|
+
Bigquery.transaction(config, schema, processor_count, &control)
|
109
|
+
end
|
110
|
+
|
111
|
+
def test_replace_backup_auto_create_dataset
|
112
|
+
config = least_config.merge('mode' => 'replace_backup', 'dataset_old' => 'dataset_old', 'table_old' => 'table_old', 'temp_table' => 'temp_table', 'auto_create_dataset' => true)
|
113
|
+
any_instance_of(BigqueryClient) do |obj|
|
114
|
+
mock(obj).create_dataset(config['dataset'])
|
115
|
+
mock(obj).create_dataset(config['dataset_old'], reference: config['dataset'])
|
116
|
+
mock(obj).create_table(config['temp_table'])
|
117
|
+
mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
|
118
|
+
|
119
|
+
mock(obj).copy(config['table'], config['table_old'], config['dataset_old'])
|
120
|
+
|
121
|
+
mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
|
122
|
+
mock(obj).delete_table(config['temp_table'])
|
123
|
+
end
|
124
|
+
Bigquery.transaction(config, schema, processor_count, &control)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
@@ -0,0 +1,316 @@
|
|
1
|
+
require_relative './helper'
|
2
|
+
require 'embulk/output/bigquery/value_converter_factory'
|
3
|
+
|
4
|
+
module Embulk
|
5
|
+
class Output::Bigquery
|
6
|
+
class TestValueConverterFactory < Test::Unit::TestCase
|
7
|
+
|
8
|
+
class TestCreateConverters < Test::Unit::TestCase
|
9
|
+
def test_create_default_converter
|
10
|
+
schema = Schema.new([
|
11
|
+
Column.new({index: 0, name: 'boolean', type: :boolean}),
|
12
|
+
Column.new({index: 1, name: 'long', type: :long}),
|
13
|
+
Column.new({index: 2, name: 'double', type: :double}),
|
14
|
+
Column.new({index: 3, name: 'string', type: :string}),
|
15
|
+
Column.new({index: 4, name: 'timestamp', type: :timestamp}),
|
16
|
+
Column.new({index: 5, name: 'json', type: :json}),
|
17
|
+
])
|
18
|
+
converters = ValueConverterFactory.create_converters({}, schema)
|
19
|
+
assert_equal schema.size, converters.size
|
20
|
+
# Check correct converters are created
|
21
|
+
# Proc can not have names, so we have to execute to check...
|
22
|
+
assert_equal true, converters[0].call(true)
|
23
|
+
assert_equal 1, converters[1].call(1)
|
24
|
+
assert_equal 1.1, converters[2].call(1.1)
|
25
|
+
assert_equal 'foo', converters[3].call('foo')
|
26
|
+
timestamp = Time.parse("2016-02-26 00:00:00.100000 UTC")
|
27
|
+
assert_equal 1456444800.1, converters[4].call(timestamp)
|
28
|
+
assert_equal %Q[{"foo":"foo"}], converters[5].call({'foo'=>'foo'})
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_create_custom_converter
|
32
|
+
schema = Schema.new([
|
33
|
+
Column.new({index: 0, name: 'boolean', type: :boolean}),
|
34
|
+
Column.new({index: 1, name: 'long', type: :long}),
|
35
|
+
Column.new({index: 2, name: 'double', type: :double}),
|
36
|
+
Column.new({index: 3, name: 'string', type: :string}),
|
37
|
+
Column.new({index: 4, name: 'timestamp', type: :timestamp}),
|
38
|
+
Column.new({index: 5, name: 'json', type: :json}),
|
39
|
+
])
|
40
|
+
task = {
|
41
|
+
'column_options' => [
|
42
|
+
{'name' => 'boolean', 'type' => 'STRING'},
|
43
|
+
{'name' => 'long', 'type' => 'STRING'},
|
44
|
+
{'name' => 'double', 'type' => 'STRING'},
|
45
|
+
{'name' => 'string', 'type' => 'INTEGER'},
|
46
|
+
{'name' => 'timestamp', 'type' => 'INTEGER'},
|
47
|
+
{'name' => 'json', 'type' => 'RECORD'},
|
48
|
+
],
|
49
|
+
}
|
50
|
+
converters = ValueConverterFactory.create_converters(task, schema)
|
51
|
+
assert_equal schema.size, converters.size
|
52
|
+
# Check correct converters are created
|
53
|
+
# Proc can not have names, so we have to execute to check...
|
54
|
+
assert_equal 'true', converters[0].call(true)
|
55
|
+
assert_equal '1', converters[1].call(1)
|
56
|
+
assert_equal '1.1', converters[2].call(1.1)
|
57
|
+
assert_equal 1, converters[3].call('1')
|
58
|
+
timestamp = Time.parse("2016-02-26 00:00:00.100000 UTC")
|
59
|
+
assert_equal 1456444800, converters[4].call(timestamp)
|
60
|
+
assert_equal({'foo'=>'foo'}, converters[5].call({'foo'=>'foo'}))
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
class TestBooleanConverter < Test::Unit::TestCase
|
65
|
+
SCHEMA_TYPE = :boolean
|
66
|
+
|
67
|
+
def test_boolean
|
68
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'BOOLEAN').create_converter
|
69
|
+
assert_equal nil, converter.call(nil)
|
70
|
+
assert_equal true, converter.call(true)
|
71
|
+
assert_equal false, converter.call(false)
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_integer
|
75
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'INTEGER').create_converter }
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_float
|
79
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'FLOAT').create_converter }
|
80
|
+
end
|
81
|
+
|
82
|
+
def test_string
|
83
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'STRING').create_converter
|
84
|
+
assert_equal nil, converter.call(nil)
|
85
|
+
assert_equal "true", converter.call(true)
|
86
|
+
assert_equal "false", converter.call(false)
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_timestamp
|
90
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_record
|
94
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
class TestLongConverter < Test::Unit::TestCase
|
99
|
+
SCHEMA_TYPE = :long
|
100
|
+
|
101
|
+
def test_boolean
|
102
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'BOOLEAN').create_converter
|
103
|
+
assert_equal nil, converter.call(nil)
|
104
|
+
assert_equal true, converter.call(1)
|
105
|
+
assert_equal false, converter.call(0)
|
106
|
+
assert_raise { converter.call(2) }
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_integer
|
110
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'INTEGER').create_converter
|
111
|
+
assert_equal nil, converter.call(nil)
|
112
|
+
assert_equal 1, converter.call(1)
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_float
|
116
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'FLOAT').create_converter
|
117
|
+
assert_equal nil, converter.call(nil)
|
118
|
+
assert_equal 1.0, converter.call(1)
|
119
|
+
end
|
120
|
+
|
121
|
+
def test_string
|
122
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'STRING').create_converter
|
123
|
+
assert_equal nil, converter.call(nil)
|
124
|
+
assert_equal "1", converter.call(1)
|
125
|
+
end
|
126
|
+
|
127
|
+
def test_timestamp
|
128
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter
|
129
|
+
assert_equal nil, converter.call(nil)
|
130
|
+
assert_equal 1408452095, converter.call(1408452095)
|
131
|
+
end
|
132
|
+
|
133
|
+
def test_record
|
134
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
class TestDoubleConverter < Test::Unit::TestCase
|
139
|
+
SCHEMA_TYPE = :double
|
140
|
+
|
141
|
+
def test_boolean
|
142
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'BOOLEAN').create_converter }
|
143
|
+
end
|
144
|
+
|
145
|
+
def test_integer
|
146
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'INTEGER').create_converter
|
147
|
+
assert_equal nil, converter.call(nil)
|
148
|
+
assert_equal 1, converter.call(1.1)
|
149
|
+
end
|
150
|
+
|
151
|
+
def test_float
|
152
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'FLOAT').create_converter
|
153
|
+
assert_equal nil, converter.call(nil)
|
154
|
+
assert_equal 1.1, converter.call(1.1)
|
155
|
+
end
|
156
|
+
|
157
|
+
def test_string
|
158
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'STRING').create_converter
|
159
|
+
assert_equal nil, converter.call(nil)
|
160
|
+
assert_equal "1.1", converter.call(1.1)
|
161
|
+
end
|
162
|
+
|
163
|
+
def test_timestamp
|
164
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter
|
165
|
+
assert_equal nil, converter.call(nil)
|
166
|
+
assert_equal 1408452095.188766, converter.call(1408452095.188766)
|
167
|
+
end
|
168
|
+
|
169
|
+
def test_record
|
170
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
class TestStringConverter < Test::Unit::TestCase
|
175
|
+
SCHEMA_TYPE = :string
|
176
|
+
|
177
|
+
def test_boolean
|
178
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'BOOLEAN').create_converter
|
179
|
+
assert_equal nil, converter.call(nil)
|
180
|
+
assert_equal true, converter.call('true')
|
181
|
+
assert_equal false, converter.call('false')
|
182
|
+
assert_raise { converter.call('foo') }
|
183
|
+
end
|
184
|
+
|
185
|
+
def test_integer
|
186
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'INTEGER').create_converter
|
187
|
+
assert_equal nil, converter.call(nil)
|
188
|
+
assert_equal 1, converter.call('1')
|
189
|
+
assert_raise { converter.call('1.1') }
|
190
|
+
end
|
191
|
+
|
192
|
+
def test_float
|
193
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'FLOAT').create_converter
|
194
|
+
assert_equal nil, converter.call(nil)
|
195
|
+
assert_equal 1.1, converter.call('1.1')
|
196
|
+
assert_raise { converter.call('foo') }
|
197
|
+
end
|
198
|
+
|
199
|
+
def test_string
|
200
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'STRING').create_converter
|
201
|
+
assert_equal nil, converter.call(nil)
|
202
|
+
assert_equal "foo", converter.call("foo")
|
203
|
+
end
|
204
|
+
|
205
|
+
def test_timestamp
|
206
|
+
converter = ValueConverterFactory.new(
|
207
|
+
SCHEMA_TYPE, 'TIMESTAMP',
|
208
|
+
timestamp_format: '%Y-%m-%d', timezone: 'Asia/Tokyo'
|
209
|
+
).create_converter
|
210
|
+
assert_equal nil, converter.call(nil)
|
211
|
+
assert_equal 1456412400.0, converter.call("2016-02-26")
|
212
|
+
|
213
|
+
# Users must care of BQ timestamp format by themselves with no timestamp_format
|
214
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter
|
215
|
+
assert_equal nil, converter.call(nil)
|
216
|
+
assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
|
217
|
+
end
|
218
|
+
|
219
|
+
def test_record
|
220
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
|
221
|
+
assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
|
222
|
+
assert_raise { converter.call('foo') }
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
class TestTimestampConverter < Test::Unit::TestCase
|
227
|
+
SCHEMA_TYPE = :timestamp
|
228
|
+
|
229
|
+
def test_boolean
|
230
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'BOOLEAN').create_converter }
|
231
|
+
end
|
232
|
+
|
233
|
+
def test_integer
|
234
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'INTEGER').create_converter
|
235
|
+
assert_equal nil, converter.call(nil)
|
236
|
+
expected = 1456444800
|
237
|
+
assert_equal expected, converter.call(Time.at(expected))
|
238
|
+
end
|
239
|
+
|
240
|
+
def test_float
|
241
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'FLOAT').create_converter
|
242
|
+
assert_equal nil, converter.call(nil)
|
243
|
+
expected = 1456444800.100000
|
244
|
+
assert_equal expected, converter.call(Time.at(expected))
|
245
|
+
end
|
246
|
+
|
247
|
+
def test_string
|
248
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'STRING').create_converter
|
249
|
+
assert_equal nil, converter.call(nil)
|
250
|
+
timestamp = Time.parse("2016-02-26 00:00:00.100000 UTC")
|
251
|
+
expected = "2016-02-26 00:00:00.100000"
|
252
|
+
assert_equal expected, converter.call(timestamp)
|
253
|
+
|
254
|
+
converter = ValueConverterFactory.new(
|
255
|
+
SCHEMA_TYPE, 'STRING',
|
256
|
+
timestamp_format: '%Y-%m-%d', timezone: 'Asia/Tokyo'
|
257
|
+
).create_converter
|
258
|
+
timestamp = Time.parse("2016-02-25 15:00:00.100000 UTC")
|
259
|
+
expected = "2016-02-26"
|
260
|
+
assert_equal expected, converter.call(timestamp)
|
261
|
+
end
|
262
|
+
|
263
|
+
def test_timestamp
|
264
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter
|
265
|
+
assert_equal nil, converter.call(nil)
|
266
|
+
expected = 1456444800.100000
|
267
|
+
assert_equal expected, converter.call(Time.at(expected))
|
268
|
+
end
|
269
|
+
|
270
|
+
def test_record
|
271
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
class TestJsonConverter < Test::Unit::TestCase
|
276
|
+
SCHEMA_TYPE = :json
|
277
|
+
|
278
|
+
def test_boolean
|
279
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'BOOLEAN').create_converter }
|
280
|
+
end
|
281
|
+
|
282
|
+
def test_integer
|
283
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'INTEGER').create_converter }
|
284
|
+
end
|
285
|
+
|
286
|
+
def test_float
|
287
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'FLOAT').create_converter }
|
288
|
+
end
|
289
|
+
|
290
|
+
def test_string
|
291
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'STRING').create_converter
|
292
|
+
assert_equal nil, converter.call(nil)
|
293
|
+
assert_equal(%Q[{"foo":"foo"}], converter.call({'foo'=>'foo'}))
|
294
|
+
end
|
295
|
+
|
296
|
+
def test_timestamp
|
297
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
|
298
|
+
end
|
299
|
+
|
300
|
+
def test_record
|
301
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
|
302
|
+
assert_equal nil, converter.call(nil)
|
303
|
+
assert_equal({'foo'=>'foo'}, converter.call({'foo'=>'foo'}))
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
def test_strict_false
|
308
|
+
converter = ValueConverterFactory.new(:string, 'BOOLEAN', strict: false).create_converter
|
309
|
+
assert_equal nil, converter.call('foo')
|
310
|
+
|
311
|
+
converter = ValueConverterFactory.new(:string, 'INTEGER', strict: false).create_converter
|
312
|
+
assert_equal nil, converter.call('foo')
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|
metadata
CHANGED
@@ -1,83 +1,144 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0.pre1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
8
|
+
- Naotoshi Seo
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2016-
|
12
|
+
date: 2016-03-14 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
15
|
+
name: google-api-client
|
16
|
+
version_requirements: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
14
21
|
requirement: !ruby/object:Gem::Requirement
|
15
22
|
requirements:
|
16
|
-
- -
|
23
|
+
- - ">="
|
17
24
|
- !ruby/object:Gem::Version
|
18
|
-
version: '
|
19
|
-
name: bundler
|
25
|
+
version: '0'
|
20
26
|
prerelease: false
|
21
|
-
type: :
|
27
|
+
type: :runtime
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: tzinfo
|
22
30
|
version_requirements: !ruby/object:Gem::Requirement
|
23
31
|
requirements:
|
24
|
-
- -
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
25
38
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
39
|
+
version: '0'
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
27
42
|
- !ruby/object:Gem::Dependency
|
43
|
+
name: embulk
|
44
|
+
version_requirements: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: 0.8.2
|
28
49
|
requirement: !ruby/object:Gem::Requirement
|
29
50
|
requirements:
|
30
|
-
- -
|
51
|
+
- - ">="
|
31
52
|
- !ruby/object:Gem::Version
|
32
|
-
version:
|
33
|
-
|
53
|
+
version: 0.8.2
|
54
|
+
prerelease: false
|
55
|
+
type: :development
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: bundler
|
58
|
+
version_requirements: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: 1.10.6
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: 1.10.6
|
34
68
|
prerelease: false
|
35
69
|
type: :development
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rake
|
36
72
|
version_requirements: !ruby/object:Gem::Requirement
|
37
73
|
requirements:
|
38
|
-
- -
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '10.0'
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
39
80
|
- !ruby/object:Gem::Version
|
40
81
|
version: '10.0'
|
82
|
+
prerelease: false
|
83
|
+
type: :development
|
41
84
|
description: Embulk plugin that insert records to Google BigQuery.
|
42
85
|
email:
|
43
86
|
- satoshiakama@gmail.com
|
87
|
+
- sonots@gmail.com
|
44
88
|
executables: []
|
45
89
|
extensions: []
|
46
90
|
extra_rdoc_files: []
|
47
91
|
files:
|
48
|
-
- .gitignore
|
92
|
+
- ".gitignore"
|
49
93
|
- CHANGELOG.md
|
94
|
+
- Gemfile
|
95
|
+
- LICENSE.txt
|
50
96
|
- README.md
|
51
|
-
-
|
52
|
-
-
|
53
|
-
-
|
54
|
-
-
|
55
|
-
-
|
56
|
-
-
|
57
|
-
-
|
97
|
+
- Rakefile
|
98
|
+
- embulk-output-bigquery.gemspec
|
99
|
+
- example/config_client_options.yml
|
100
|
+
- example/config_csv.yml
|
101
|
+
- example/config_delete_in_advance.yml
|
102
|
+
- example/config_expose_errors.yml
|
103
|
+
- example/config_guess_from_embulk_schema.yml
|
104
|
+
- example/config_guess_with_column_options.yml
|
105
|
+
- example/config_gzip.yml
|
106
|
+
- example/config_jsonl.yml
|
107
|
+
- example/config_mode_append.yml
|
108
|
+
- example/config_mode_append_direct.yml
|
109
|
+
- example/config_payload_column.yml
|
110
|
+
- example/config_payload_column_index.yml
|
111
|
+
- example/config_prevent_duplicate_insert.yml
|
112
|
+
- example/config_replace.yml
|
113
|
+
- example/config_replace_backup.yml
|
114
|
+
- example/config_skip_file_generation.yml
|
115
|
+
- example/config_table_strftime.yml
|
116
|
+
- example/config_template_table.yml
|
117
|
+
- example/config_uncompressed.yml
|
118
|
+
- example/config_with_rehearsal.yml
|
119
|
+
- example/example.csv
|
120
|
+
- example/example.jsonl
|
121
|
+
- example/example.yml
|
122
|
+
- example/json_key.json
|
123
|
+
- example/nested_example.jsonl
|
124
|
+
- example/schema.json
|
125
|
+
- example/schema_expose_errors.json
|
58
126
|
- lib/embulk/output/bigquery.rb
|
59
|
-
-
|
60
|
-
-
|
61
|
-
-
|
62
|
-
-
|
63
|
-
-
|
64
|
-
-
|
65
|
-
-
|
66
|
-
-
|
67
|
-
-
|
68
|
-
-
|
69
|
-
-
|
70
|
-
-
|
71
|
-
- classpath/google-http-client-1.20.0.jar
|
72
|
-
- classpath/google-http-client-jackson2-1.20.0.jar
|
73
|
-
- classpath/google-oauth-client-1.20.0.jar
|
74
|
-
- classpath/guava-jdk5-13.0.jar
|
75
|
-
- classpath/httpclient-4.0.1.jar
|
76
|
-
- classpath/httpcore-4.0.1.jar
|
77
|
-
- classpath/jsr305-1.3.9.jar
|
127
|
+
- lib/embulk/output/bigquery/bigquery_client.rb
|
128
|
+
- lib/embulk/output/bigquery/file_writer.rb
|
129
|
+
- lib/embulk/output/bigquery/helper.rb
|
130
|
+
- lib/embulk/output/bigquery/value_converter_factory.rb
|
131
|
+
- test/helper.rb
|
132
|
+
- test/test_bigquery_client.rb
|
133
|
+
- test/test_configure.rb
|
134
|
+
- test/test_example.rb
|
135
|
+
- test/test_file_writer.rb
|
136
|
+
- test/test_helper.rb
|
137
|
+
- test/test_transaction.rb
|
138
|
+
- test/test_value_converter_factory.rb
|
78
139
|
homepage: https://github.com/embulk/embulk-output-bigquery
|
79
140
|
licenses:
|
80
|
-
-
|
141
|
+
- MIT
|
81
142
|
metadata: {}
|
82
143
|
post_install_message:
|
83
144
|
rdoc_options: []
|
@@ -85,18 +146,26 @@ require_paths:
|
|
85
146
|
- lib
|
86
147
|
required_ruby_version: !ruby/object:Gem::Requirement
|
87
148
|
requirements:
|
88
|
-
- -
|
149
|
+
- - ">="
|
89
150
|
- !ruby/object:Gem::Version
|
90
151
|
version: '0'
|
91
152
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
153
|
requirements:
|
93
|
-
- -
|
154
|
+
- - ">"
|
94
155
|
- !ruby/object:Gem::Version
|
95
|
-
version:
|
156
|
+
version: 1.3.1
|
96
157
|
requirements: []
|
97
158
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
159
|
+
rubygems_version: 2.4.8
|
99
160
|
signing_key:
|
100
161
|
specification_version: 4
|
101
162
|
summary: Google BigQuery output plugin for Embulk
|
102
|
-
test_files:
|
163
|
+
test_files:
|
164
|
+
- test/helper.rb
|
165
|
+
- test/test_bigquery_client.rb
|
166
|
+
- test/test_configure.rb
|
167
|
+
- test/test_example.rb
|
168
|
+
- test/test_file_writer.rb
|
169
|
+
- test/test_helper.rb
|
170
|
+
- test/test_transaction.rb
|
171
|
+
- test/test_value_converter_factory.rb
|