bigquery_migration 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +11 -0
- data/lib/bigquery_migration/bigquery_wrapper.rb +5 -5
- data/lib/bigquery_migration/schema.rb +2 -2
- data/lib/bigquery_migration/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7d3ea2055c104bf0217847c2f8d1efedbfdc5ebd
|
4
|
+
data.tar.gz: 531d0a9fd96ed38a099ab4bdae85682d844c617c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fa9676f4b90a977a81d1b3faeaa96fcabac9fd014b185d9e2108914650b4a902609d6b9b6680ac9b52fb5d5a9e50d54e5d47dccedcbd93aee4a1a1791c22f209
|
7
|
+
data.tar.gz: d01485a18936f412c3b59185d0c41df62b49f28da130853cec76f205169959988316550c252ba6aa593e0414ef674d2d025c26fee32a23d27ea1f2ceb3a89556
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -76,6 +76,17 @@ migrator.migrate_table(columns: columns)
|
|
76
76
|
# migrator.migrate_table(schema_file: '/path/to/schema.json')
|
77
77
|
```
|
78
78
|
|
79
|
+
## LIMITATIONS
|
80
|
+
|
81
|
+
There are serveral limitations because of BigQuery API limitations:
|
82
|
+
|
83
|
+
* Can not handle `mode: REPEATED` columns
|
84
|
+
* Can add only `mode: NULLABLE` columns
|
85
|
+
* Columns become `mode: NULLABLE` after type changing
|
86
|
+
* Will be charged because a query is issued (If only adding columns, it is not charged because it uses patch_table API)
|
87
|
+
|
88
|
+
This tool has an advantage that it is **faster** than reloading data entirely.
|
89
|
+
|
79
90
|
## Further Details
|
80
91
|
|
81
92
|
* See [BigQueryテーブルのスキーマを変更する - sonots:blog](http://blog.livedoor.jp/sonots/archives/47294596.html) (Japanese)
|
@@ -584,18 +584,18 @@ class BigqueryMigration
|
|
584
584
|
if status == "DONE"
|
585
585
|
logger.info {
|
586
586
|
"#{kind} job completed... " \
|
587
|
-
"
|
587
|
+
"job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
|
588
588
|
}
|
589
589
|
break
|
590
590
|
elsif elapsed.to_i > max_polling_time
|
591
|
-
message = "
|
592
|
-
"
|
591
|
+
message = "#{kind} job checking... " \
|
592
|
+
"job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[TIMEOUT]"
|
593
593
|
logger.info { message }
|
594
594
|
raise JobTimeoutError.new(message)
|
595
595
|
else
|
596
596
|
logger.info {
|
597
|
-
"
|
598
|
-
"
|
597
|
+
"#{kind} job checking... " \
|
598
|
+
"job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
|
599
599
|
}
|
600
600
|
sleep wait_interval
|
601
601
|
_response = client.get_job(project, job_id)
|
@@ -84,13 +84,13 @@ class BigqueryMigration
|
|
84
84
|
end
|
85
85
|
|
86
86
|
def validate_type!(type)
|
87
|
-
unless ALLOWED_FIELD_TYPES.include?(type)
|
87
|
+
unless ALLOWED_FIELD_TYPES.include?(type.upcase)
|
88
88
|
raise ConfigError, "Column type `#{type}` is not allowed type"
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
92
|
def validate_mode!(mode)
|
93
|
-
unless ALLOWED_FIELD_MODES.include?(mode)
|
93
|
+
unless ALLOWED_FIELD_MODES.include?(mode.upcase)
|
94
94
|
raise ConfigError, "Column mode `#{mode}` is not allowed mode"
|
95
95
|
end
|
96
96
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bigquery_migration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naotoshi Seo
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: google-api-client
|