embulk-output-bigquery 0.6.0 → 0.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +7 -3
  3. data/README.md +12 -7
  4. data/embulk-output-bigquery.gemspec +4 -2
  5. data/lib/embulk/output/bigquery.rb +3 -3
  6. metadata +2 -46
  7. data/example/config_append_direct_schema_update_options.yml +0 -31
  8. data/example/config_client_options.yml +0 -33
  9. data/example/config_csv.yml +0 -30
  10. data/example/config_delete_in_advance.yml +0 -29
  11. data/example/config_delete_in_advance_field_partitioned_table.yml +0 -33
  12. data/example/config_delete_in_advance_partitioned_table.yml +0 -33
  13. data/example/config_expose_errors.yml +0 -30
  14. data/example/config_gcs.yml +0 -32
  15. data/example/config_guess_from_embulk_schema.yml +0 -29
  16. data/example/config_guess_with_column_options.yml +0 -40
  17. data/example/config_gzip.yml +0 -1
  18. data/example/config_jsonl.yml +0 -1
  19. data/example/config_max_threads.yml +0 -34
  20. data/example/config_min_ouput_tasks.yml +0 -34
  21. data/example/config_mode_append.yml +0 -30
  22. data/example/config_mode_append_direct.yml +0 -30
  23. data/example/config_nested_record.yml +0 -1
  24. data/example/config_payload_column.yml +0 -20
  25. data/example/config_payload_column_index.yml +0 -20
  26. data/example/config_progress_log_interval.yml +0 -31
  27. data/example/config_replace.yml +0 -30
  28. data/example/config_replace_backup.yml +0 -32
  29. data/example/config_replace_backup_field_partitioned_table.yml +0 -34
  30. data/example/config_replace_backup_partitioned_table.yml +0 -34
  31. data/example/config_replace_field_partitioned_table.yml +0 -33
  32. data/example/config_replace_partitioned_table.yml +0 -33
  33. data/example/config_replace_schema_update_options.yml +0 -33
  34. data/example/config_skip_file_generation.yml +0 -32
  35. data/example/config_table_strftime.yml +0 -30
  36. data/example/config_template_table.yml +0 -21
  37. data/example/config_uncompressed.yml +0 -1
  38. data/example/config_with_rehearsal.yml +0 -33
  39. data/example/example.csv +0 -17
  40. data/example/example.yml +0 -1
  41. data/example/example2_1.csv +0 -1
  42. data/example/example2_2.csv +0 -1
  43. data/example/example4_1.csv +0 -1
  44. data/example/example4_2.csv +0 -1
  45. data/example/example4_3.csv +0 -1
  46. data/example/example4_4.csv +0 -1
  47. data/example/json_key.json +0 -12
  48. data/example/nested_example.jsonl +0 -16
  49. data/example/schema.json +0 -30
  50. data/example/schema_expose_errors.json +0 -30
data/example/config_replace_partitioned_table.yml DELETED
@@ -1,33 +0,0 @@
1
- in:
2
- type: file
3
- path_prefix: example/example.csv
4
- parser:
5
- type: csv
6
- charset: UTF-8
7
- newline: CRLF
8
- null_string: 'NULL'
9
- skip_header_lines: 1
10
- comment_line_marker: '#'
11
- columns:
12
- - {name: date, type: string}
13
- - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
- - {name: "null", type: string}
15
- - {name: long, type: long}
16
- - {name: string, type: string}
17
- - {name: double, type: double}
18
- - {name: boolean, type: boolean}
19
- out:
20
- type: bigquery
21
- mode: replace
22
- auth_method: service_account
23
- json_keyfile: example/your-project-000.json
24
- dataset: your_dataset_name
25
- table: your_partitioned_table_name$20160929
26
- source_format: NEWLINE_DELIMITED_JSON
27
- compression: NONE
28
- auto_create_dataset: true
29
- auto_create_table: true
30
- schema_file: example/schema.json
31
- time_partitioning:
32
- type: 'DAY'
33
- expiration_ms: 100
data/example/config_replace_schema_update_options.yml DELETED
@@ -1,33 +0,0 @@
1
- in:
2
- type: file
3
- path_prefix: example/example.csv
4
- parser:
5
- type: csv
6
- charset: UTF-8
7
- newline: CRLF
8
- null_string: 'NULL'
9
- skip_header_lines: 1
10
- comment_line_marker: '#'
11
- columns:
12
- - {name: date, type: string}
13
- - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
- - {name: "null", type: string}
15
- - {name: long, type: long}
16
- - {name: string, type: string}
17
- - {name: double, type: double}
18
- - {name: boolean, type: boolean}
19
- out:
20
- type: bigquery
21
- mode: replace
22
- auth_method: service_account
23
- json_keyfile: example/your-project-000.json
24
- dataset: your_dataset_name
25
- table: your_partitioned_table_name$20160929
26
- source_format: NEWLINE_DELIMITED_JSON
27
- compression: NONE
28
- auto_create_dataset: true
29
- auto_create_table: true
30
- schema_file: example/schema.json
31
- time_partitioning:
32
- type: 'DAY'
33
- expiration_ms: 100
data/example/config_skip_file_generation.yml DELETED
@@ -1,32 +0,0 @@
1
- in:
2
- type: file
3
- path_prefix: example/example.csv
4
- parser:
5
- type: csv
6
- charset: UTF-8
7
- newline: CRLF
8
- null_string: 'NULL'
9
- skip_header_lines: 1
10
- comment_line_marker: '#'
11
- columns:
12
- - {name: date, type: string}
13
- - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
- - {name: "null", type: string}
15
- - {name: long, type: long}
16
- - {name: string, type: string}
17
- - {name: double, type: double}
18
- - {name: boolean, type: boolean}
19
- out:
20
- type: bigquery
21
- mode: replace
22
- auth_method: service_account
23
- json_keyfile: example/your-project-000.json
24
- dataset: your_dataset_name
25
- table: your_table_name
26
- source_format: NEWLINE_DELIMITED_JSON
27
- auto_create_dataset: true
28
- auto_create_table: true
29
- schema_file: example/schema.json
30
- path_prefix: example/example
31
- file_ext: .jsonl
32
- skip_file_generation: true
data/example/config_table_strftime.yml DELETED
@@ -1,30 +0,0 @@
1
- in:
2
- type: file
3
- path_prefix: example/example.csv
4
- parser:
5
- type: csv
6
- charset: UTF-8
7
- newline: CRLF
8
- null_string: 'NULL'
9
- skip_header_lines: 1
10
- comment_line_marker: '#'
11
- columns:
12
- - {name: date, type: string}
13
- - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
- - {name: "null", type: string}
15
- - {name: long, type: long}
16
- - {name: string, type: string}
17
- - {name: double, type: double}
18
- - {name: boolean, type: boolean}
19
- out:
20
- type: bigquery
21
- mode: replace
22
- auth_method: service_account
23
- json_keyfile: example/your-project-000.json
24
- dataset: your_dataset_name
25
- table: your_table_name_%Y%m%d
26
- source_format: NEWLINE_DELIMITED_JSON
27
- auto_create_dataset: true
28
- auto_create_table: true
29
- schema_file: example/schema.json
30
- skip_load: true # for debug
data/example/config_template_table.yml DELETED
@@ -1,21 +0,0 @@
1
- # embulk gem install embulk-parser-none
2
- in:
3
- type: file
4
- path_prefix: example/example.jsonl
5
- parser:
6
- type: none
7
- column_name: payload
8
- out:
9
- type: bigquery
10
- mode: replace
11
- auth_method: service_account
12
- json_keyfile: example/your-project-000.json
13
- dataset: your_dataset_name
14
- table: your_table_name_%Y%m%d
15
- compression: GZIP
16
- source_format: NEWLINE_DELIMITED_JSON
17
- auto_create_dataset: true
18
- auto_create_table: true
19
- template_table: your_table_name
20
- payload_column: payload
21
- skip_load: true # for debug
data/example/config_uncompressed.yml DELETED
@@ -1 +0,0 @@
1
- example/config_replace.yml
data/example/config_with_rehearsal.yml DELETED
@@ -1,33 +0,0 @@
1
- in:
2
- type: file
3
- path_prefix: example/example.csv
4
- parser:
5
- type: csv
6
- charset: UTF-8
7
- newline: CRLF
8
- null_string: 'NULL'
9
- skip_header_lines: 1
10
- comment_line_marker: '#'
11
- columns:
12
- - {name: date, type: string}
13
- - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
14
- - {name: "null", type: string}
15
- - {name: long, type: long}
16
- - {name: string, type: string}
17
- - {name: double, type: double}
18
- - {name: boolean, type: boolean}
19
- out:
20
- type: bigquery
21
- mode: replace
22
- auth_method: service_account
23
- json_keyfile: example/your-project-000.json
24
- dataset: your_dataset_name
25
- table: your_table_name
26
- source_format: NEWLINE_DELIMITED_JSON
27
- auto_create_dataset: true
28
- auto_create_table: true
29
- schema_file: example/schema.json
30
- with_rehearsal: true
31
- rehearsal_counts: 1
32
- skip_load: true # for debug
33
- compression: GZIP
data/example/example.csv DELETED
@@ -1,17 +0,0 @@
1
- date,timestamp,null,long,string,double,boolean
2
- 2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,true
3
- 2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,true
4
- 2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,true
5
- 2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,true
6
- 2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,true
7
- 2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,true
8
- 2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,true
9
- 2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,true
10
- 2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,false
11
- 2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,false
12
- 2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,false
13
- 2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,false
14
- 2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,false
15
- 2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,false
16
- 2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,false
17
- 2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,false
data/example/example.yml DELETED
@@ -1 +0,0 @@
1
- example/config_replace.yml
data/example/example2_1.csv DELETED
@@ -1 +0,0 @@
1
- example/example.csv
data/example/example2_2.csv DELETED
@@ -1 +0,0 @@
1
- example/example.csv
data/example/example4_1.csv DELETED
@@ -1 +0,0 @@
1
- example/example.csv
data/example/example4_2.csv DELETED
@@ -1 +0,0 @@
1
- example/example.csv
data/example/example4_3.csv DELETED
@@ -1 +0,0 @@
1
- example/example.csv
data/example/example4_4.csv DELETED
@@ -1 +0,0 @@
1
- example/example.csv
data/example/json_key.json DELETED
@@ -1,12 +0,0 @@
1
- {
2
- "type": "service_account",
3
- "project_id": "your_project_name",
4
- "private_key_id": "your_private_key_id",
5
- "private_key": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n",
6
- "client_email": "your_service_account_email",
7
- "client_id": "your_client_id",
8
- "auth_uri": "https://accounts.google.com/o/oauth2/auth",
9
- "token_uri": "https://accounts.google.com/o/oauth2/token",
10
- "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
11
- "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/account-3%40your_project_name.iam.gserviceaccount.com"
12
- }
data/example/nested_example.jsonl DELETED
@@ -1,16 +0,0 @@
1
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":90,"string":"l6lTsvxd","double":903.4,"json":{"k1":"v1","k2":"v2"},"boolean":true}
2
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":91,"string":"XoALSEQg","double":394.5,"json":{"k1":"v1","k2":"v2"},"boolean":true}
3
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":92,"string":"0hgDRI_m","double":810.9,"json":{"k1":"v1","k2":"v2"},"boolean":true}
4
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":93,"string":"KjCRAc-A","double":477.4,"json":{"k1":"v1","k2":"v2"},"boolean":true}
5
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":94,"string":"fyQVGlT8","double":725.3,"json":{"k1":"v1","k2":"v2"},"boolean":true}
6
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":95,"string":"FpBYRPWK","double":316.6,"json":{"k1":"v1","k2":"v2"},"boolean":true}
7
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":96,"string":"9ikvnUqp","double":369.5,"json":{"k1":"v1","k2":"v2"},"boolean":true}
8
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":97,"string":"RRNYDAzK","double":506.5,"json":{"k1":"v1","k2":"v2"},"boolean":true}
9
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":90,"string":"l6lTsvxd","double":903.4,"json":{"k1":"v1","k2":"v2"},"boolean":false}
10
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":91,"string":"XoALSEQg","double":394.5,"json":{"k1":"v1","k2":"v2"},"boolean":false}
11
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":92,"string":"0hgDRI_m","double":810.9,"json":{"k1":"v1","k2":"v2"},"boolean":false}
12
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":93,"string":"KjCRAc-A","double":477.4,"json":{"k1":"v1","k2":"v2"},"boolean":false}
13
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":94,"string":"fyQVGlT8","double":725.3,"json":{"k1":"v1","k2":"v2"},"boolean":false}
14
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":95,"string":"FpBYRPWK","double":316.6,"json":{"k1":"v1","k2":"v2"},"boolean":false}
15
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":96,"string":"9ikvnUqp","double":369.5,"json":{"k1":"v1","k2":"v2"},"boolean":false}
16
- {"date":"2015-07-13","timestamp":"2015-07-13 00:00:00.100000","null":null,"long":97,"string":"RRNYDAzK","double":506.5,"json":{"k1":"v1","k2":"v2"},"boolean":false}
data/example/schema.json DELETED
@@ -1,30 +0,0 @@
1
- [
2
- {
3
- "name":"date",
4
- "type":"STRING"
5
- },
6
- {
7
- "name":"timestamp",
8
- "type":"TIMESTAMP"
9
- },
10
- {
11
- "name":"null",
12
- "type":"STRING"
13
- },
14
- {
15
- "name":"long",
16
- "type":"INTEGER"
17
- },
18
- {
19
- "name":"string",
20
- "type":"STRING"
21
- },
22
- {
23
- "name":"double",
24
- "type":"FLOAT"
25
- },
26
- {
27
- "name":"boolean",
28
- "type":"BOOLEAN"
29
- }
30
- ]
data/example/schema_expose_errors.json DELETED
@@ -1,30 +0,0 @@
1
- [
2
- {
3
- "name":"dat",
4
- "type":"STRING"
5
- },
6
- {
7
- "name":"timestamp",
8
- "type":"TIMESTAMP"
9
- },
10
- {
11
- "name":"null",
12
- "type":"STRING"
13
- },
14
- {
15
- "name":"long",
16
- "type":"INTEGER"
17
- },
18
- {
19
- "name":"string",
20
- "type":"STRING"
21
- },
22
- {
23
- "name":"double",
24
- "type":"FLOAT"
25
- },
26
- {
27
- "name":"boolean",
28
- "type":"BOOLEAN"
29
- }
30
- ]