embulk-output-bigquery 0.2.3 → 0.3.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +6 -12
  3. data/CHANGELOG.md +18 -0
  4. data/Gemfile +8 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.md +165 -39
  7. data/Rakefile +11 -0
  8. data/embulk-output-bigquery.gemspec +20 -0
  9. data/example/config_client_options.yml +33 -0
  10. data/example/config_csv.yml +30 -0
  11. data/example/config_delete_in_advance.yml +29 -0
  12. data/example/config_expose_errors.yml +30 -0
  13. data/example/config_guess_from_embulk_schema.yml +29 -0
  14. data/example/config_guess_with_column_options.yml +40 -0
  15. data/example/config_gzip.yml +30 -0
  16. data/example/config_jsonl.yml +30 -0
  17. data/example/config_mode_append.yml +30 -0
  18. data/example/config_mode_append_direct.yml +30 -0
  19. data/example/config_payload_column.yml +20 -0
  20. data/example/config_payload_column_index.yml +20 -0
  21. data/example/config_prevent_duplicate_insert.yml +30 -0
  22. data/example/config_replace.yml +30 -0
  23. data/example/config_replace_backup.yml +32 -0
  24. data/example/config_skip_file_generation.yml +32 -0
  25. data/example/config_table_strftime.yml +30 -0
  26. data/example/config_template_table.yml +21 -0
  27. data/example/config_uncompressed.yml +30 -0
  28. data/example/config_with_rehearsal.yml +32 -0
  29. data/example/example.csv +17 -0
  30. data/example/example.jsonl +16 -0
  31. data/example/example.yml +30 -0
  32. data/example/json_key.json +12 -0
  33. data/example/nested_example.jsonl +16 -0
  34. data/example/schema.json +30 -0
  35. data/example/schema_expose_errors.json +30 -0
  36. data/lib/embulk/output/bigquery.rb +388 -3
  37. data/lib/embulk/output/bigquery/bigquery_client.rb +396 -0
  38. data/lib/embulk/output/bigquery/file_writer.rb +103 -0
  39. data/lib/embulk/output/bigquery/helper.rb +78 -0
  40. data/lib/embulk/output/bigquery/value_converter_factory.rb +292 -0
  41. data/test/helper.rb +13 -0
  42. data/test/test_bigquery_client.rb +166 -0
  43. data/test/test_configure.rb +254 -0
  44. data/test/test_example.rb +34 -0
  45. data/test/test_file_writer.rb +129 -0
  46. data/test/test_helper.rb +103 -0
  47. data/test/test_transaction.rb +129 -0
  48. data/test/test_value_converter_factory.rb +316 -0
  49. metadata +114 -45
  50. data/build.gradle +0 -80
  51. data/config/checkstyle/checkstyle.xml +0 -128
  52. data/config/checkstyle/default.xml +0 -108
  53. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  54. data/gradle/wrapper/gradle-wrapper.properties +0 -6
  55. data/gradlew +0 -164
  56. data/gradlew.bat +0 -90
  57. data/settings.gradle +0 -2
  58. data/src/main/java/org/embulk/output/BigqueryAuthentication.java +0 -117
  59. data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java +0 -508
  60. data/src/main/java/org/embulk/output/BigqueryWriter.java +0 -575
  61. data/src/test/java/org/embulk/output/TestBigqueryAuthentication.java +0 -5
  62. data/src/test/java/org/embulk/output/TestBigqueryOutputPlugin.java +0 -5
  63. data/src/test/java/org/embulk/output/TestBigqueryWriter.java +0 -5
data/example/config_delete_in_advance.yml ADDED
@@ -0,0 +1,29 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: delete_in_advance
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ source_format: NEWLINE_DELIMITED_JSON
27
+ auto_create_dataset: true
28
+ auto_create_table: true
29
+ schema_file: example/schema.json
data/example/config_expose_errors.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ source_format: NEWLINE_DELIMITED_JSON
27
+ compression: NONE
28
+ auto_create_dataset: true
29
+ auto_create_table: true
30
+ schema_file: example/schema_expose_errors.json
data/example/config_guess_from_embulk_schema.yml ADDED
@@ -0,0 +1,29 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ compression: GZIP
27
+ source_format: NEWLINE_DELIMITED_JSON
28
+ auto_create_dataset: true
29
+ auto_create_table: true
data/example/config_guess_with_column_options.yml ADDED
@@ -0,0 +1,40 @@
1
+ # embulk gem install embulk-parser-jsonl
2
+ in:
3
+ type: file
4
+ path_prefix: example/nested_example.jsonl
5
+ parser:
6
+ type: jsonl
7
+ columns:
8
+ - {name: date, type: string}
9
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
10
+ - {name: "null", type: string}
11
+ - {name: long, type: long}
12
+ - {name: string, type: string}
13
+ - {name: double, type: double}
14
+ - {name: json, type: json}
15
+ - {name: boolean, type: boolean}
16
+ out:
17
+ type: bigquery
18
+ mode: replace
19
+ auth_method: json_key
20
+ json_keyfile: /tmp/your-project-000.json
21
+ dataset: your_dataset_name
22
+ table: your_table_name
23
+ compression: GZIP
24
+ source_format: NEWLINE_DELIMITED_JSON
25
+ auto_create_dataset: true
26
+ auto_create_table: true
27
+ column_options:
28
+ - {name: date, type: TIMESTAMP, timestamp_format: "%Y-%m-%d", timezone: "+09:00"}
29
+ - {name: timestamp, type: STRING, timestamp_format: "%Y-%m-%d", timezone: "+09:00"}
30
+ - {name: long, type: STRING}
31
+ - {name: string, type: STRING}
32
+ - {name: double, type: STRING}
33
+ - {name: boolean, type: STRING}
34
+ - name: json
35
+ type: RECORD
36
+ fields:
37
+ - {name: k1, type: STRING}
38
+ - {name: k2, type: STRING}
39
+ # 2015-07-13
40
+ # 2015-07-12 15:00:00
data/example/config_gzip.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ source_format: CSV
27
+ compression: GZIP
28
+ auto_create_dataset: true
29
+ auto_create_table: true
30
+ schema_file: example/schema.json
data/example/config_jsonl.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ source_format: NEWLINE_DELIMITED_JSON
27
+ compression: NONE
28
+ auto_create_dataset: true
29
+ auto_create_table: true
30
+ schema_file: example/schema.json
data/example/config_mode_append.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ null_string: 'NULL'
8
+ skip_header_lines: 1
9
+ comment_line_marker: '#'
10
+ columns:
11
+ - {name: date, type: string}
12
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
13
+ - {name: "null", type: string}
14
+ - {name: long, type: long}
15
+ - {name: string, type: string}
16
+ - {name: double, type: double}
17
+ - {name: boolean, type: boolean}
18
+ out:
19
+ type: bigquery
20
+ mode: append
21
+ auth_method: json_key
22
+ json_keyfile: /tmp/your-project-000.json
23
+ dataset: your_dataset_name
24
+ table: your_table_name
25
+ compression: GZIP
26
+ source_format: CSV
27
+ auto_create_dataset: true
28
+ auto_create_table: true
29
+ schema_file: example/schema.json
30
+ delete_from_local_when_job_end: false
data/example/config_mode_append_direct.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ null_string: 'NULL'
8
+ skip_header_lines: 1
9
+ comment_line_marker: '#'
10
+ columns:
11
+ - {name: date, type: string}
12
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
13
+ - {name: "null", type: string}
14
+ - {name: long, type: long}
15
+ - {name: string, type: string}
16
+ - {name: double, type: double}
17
+ - {name: boolean, type: boolean}
18
+ out:
19
+ type: bigquery
20
+ mode: append_direct
21
+ auth_method: json_key
22
+ json_keyfile: /tmp/your-project-000.json
23
+ dataset: your_dataset_name
24
+ table: your_table_name
25
+ compression: GZIP
26
+ source_format: CSV
27
+ auto_create_dataset: true
28
+ auto_create_table: true
29
+ schema_file: example/schema.json
30
+ delete_from_local_when_job_end: false
data/example/config_payload_column.yml ADDED
@@ -0,0 +1,20 @@
1
+ # embulk gem install embulk-parser-none
2
+ in:
3
+ type: file
4
+ path_prefix: example/example.jsonl
5
+ parser:
6
+ type: none
7
+ column_name: payload
8
+ out:
9
+ type: bigquery
10
+ mode: replace
11
+ auth_method: json_key
12
+ json_keyfile: /tmp/your-project-000.json
13
+ dataset: your_dataset_name
14
+ table: your_table_name
15
+ compression: GZIP
16
+ source_format: NEWLINE_DELIMITED_JSON
17
+ auto_create_dataset: true
18
+ auto_create_table: true
19
+ schema_file: example/schema.json
20
+ payload_column: payload
data/example/config_payload_column_index.yml ADDED
@@ -0,0 +1,20 @@
1
+ # embulk gem install embulk-parser-none
2
+ in:
3
+ type: file
4
+ path_prefix: example/example.jsonl
5
+ parser:
6
+ type: none
7
+ column_name: payload
8
+ out:
9
+ type: bigquery
10
+ mode: replace
11
+ auth_method: json_key
12
+ json_keyfile: /tmp/your-project-000.json
13
+ dataset: your_dataset_name
14
+ table: your_table_name
15
+ compression: GZIP
16
+ source_format: NEWLINE_DELIMITED_JSON
17
+ auto_create_dataset: true
18
+ auto_create_table: true
19
+ schema_file: example/schema.json
20
+ payload_column_index: 0
data/example/config_prevent_duplicate_insert.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ source_format: NEWLINE_DELIMITED_JSON
27
+ auto_create_dataset: true
28
+ auto_create_table: true
29
+ schema_file: example/schema.json
30
+ prevent_duplicate_insert: true
data/example/config_replace.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ source_format: NEWLINE_DELIMITED_JSON
27
+ compression: NONE
28
+ auto_create_dataset: true
29
+ auto_create_table: true
30
+ schema_file: example/schema.json
data/example/config_replace_backup.yml ADDED
@@ -0,0 +1,32 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace_backup
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ dataset_old: your_dataset_name_old
27
+ table_old: your_table_name_old
28
+ source_format: NEWLINE_DELIMITED_JSON
29
+ auto_create_dataset: true
30
+ auto_create_table: true
31
+ schema_file: example/schema.json
32
+ skip_load: true # for debug
data/example/config_skip_file_generation.yml ADDED
@@ -0,0 +1,32 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name
26
+ source_format: NEWLINE_DELIMITED_JSON
27
+ auto_create_dataset: true
28
+ auto_create_table: true
29
+ schema_file: example/schema.json
30
+ path_prefix: example/example
31
+ file_ext: .jsonl
32
+ skip_file_generation: true
data/example/config_table_strftime.yml ADDED
@@ -0,0 +1,30 @@
1
+ in:
2
+ type: file
3
+ path_prefix: example/example.csv
4
+ parser:
5
+ type: csv
6
+ charset: UTF-8
7
+ newline: CRLF
8
+ null_string: 'NULL'
9
+ skip_header_lines: 1
10
+ comment_line_marker: '#'
11
+ columns:
12
+ - {name: date, type: string}
13
+ - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
+ - {name: "null", type: string}
15
+ - {name: long, type: long}
16
+ - {name: string, type: string}
17
+ - {name: double, type: double}
18
+ - {name: boolean, type: boolean}
19
+ out:
20
+ type: bigquery
21
+ mode: replace
22
+ auth_method: json_key
23
+ json_keyfile: /tmp/your-project-000.json
24
+ dataset: your_dataset_name
25
+ table: your_table_name_%Y%m%d
26
+ source_format: NEWLINE_DELIMITED_JSON
27
+ auto_create_dataset: true
28
+ auto_create_table: true
29
+ schema_file: example/schema.json
30
+ skip_load: true # for debug