bigquery_migration 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68a48ec86057ec0242c55beb90f6dfc8db688e6073d634fbcc9a03e0066b3357
-  data.tar.gz: 2a4358172e12aea071e5b260aa9d8f2dd57ec4d9a29d01ba2926f563ab58beb4
+  metadata.gz: dbaba64f116082e53c68531eda4252832447fbdc722539be390145fbfb0f30bf
+  data.tar.gz: a16286041654a619a9b41d1954e610812c3e9fa956446d848e5ac2bbbcc503a0
 SHA512:
-  metadata.gz: 1df093bcd1f22dc01cd6e09e8ed10d6ad7b85182ecb0d268b8afac48d46489eb5ed7c1f5ff3d5f564b0d7f15017aaf7b02f55fa1bef23618d4f4fde044bb5f67
-  data.tar.gz: 9af8c78d3ed6c9dfd3a269bde24e016f7cf4b63b2ac8a3e2f39ac513a1e035657329c8709ed7ee8ef0f71f5c91881d330804a5af7652bf8b54e0338c50ad24c2
+  metadata.gz: fcafd619bea25e1c79e2cd841a0a4e0ca7b4bec474ec403b73ee8d2fac041fd621afd8a8c302876c0317196dccfc9926b31b557aae98bddeeb141e545accacfb
+  data.tar.gz: cd1e6d6a427fa43e5b58dc063608f9fd3aa10affec688043a711e431d8b9cebe0c37d5069cb689c70a24baa3beb3a308f0997110c0c538c137c41f1630f10baf

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.2 (2019/04/29)
+
+Enhancements:
+
+* Support clustered table
+
 # 0.3.1 (2018/05/23)
 
 Enhancements:

data/bigquery_migration.gemspec

@@ -29,5 +29,4 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "test-unit-rr"
-  spec.add_development_dependency "test-unit-power_assert"
 end

data/example/migrate_clustered_table.yml

@@ -0,0 +1,28 @@
+bigquery: &bigquery
+  credentials_file: example/your-project-000.json
+  dataset: your_dataset_name
+  table: your_clustered_table_name
+  clustering:
+    fields:
+      - timestamp
+      - integer
+
+actions:
+- action: create_dataset
+  <<: *bigquery
+- action: migrate_partitioned_table
+  <<: *bigquery
+  columns:
+    - { name: 'timestamp', type: 'TIMESTAMP' }
+    - { name: 'integer', type: 'INTEGER' }
+    - name: 'record'
+      type: 'RECORD'
+      fields:
+        - { name: 'string', type: 'STRING' }
+        - { name: 'integer', type: 'INTEGER' }
+        - { name: 'bytes', type: 'BYTES' }
+- action: migrate_partitioned_table
+  <<: *bigquery
+  schema_file: example/schema.json
+- action: delete_table
+  <<: *bigquery

data/lib/bigquery_migration/bigquery_wrapper.rb

@@ -88,7 +88,7 @@ class BigqueryMigration
       dataset ||= self.dataset
       begin
         logger.info { "Get dataset... #{project}:#{dataset}" }
-        client.get_dataset(project, dataset)
+        response = client.get_dataset(project, dataset)
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
         if e.status_code == 404
           raise NotFoundError, "Dataset #{project}:#{dataset} is not found"
@@ -191,6 +191,12 @@ class BigqueryMigration
           }
         end
 
+        if clustering && clustering[:fields]
+          body[:clustering] = {
+            fields: clustering[:fields]
+          }
+        end
+
         opts = {}
         logger.debug { "#{head}insert_table(#{project}, #{dataset}, #{body}, #{opts})" }
         unless dry_run?
@@ -729,7 +735,14 @@ class BigqueryMigration
           when String
             return HashUtil.deep_symbolize_keys(JSON.parse(File.read(json_keyfile)))
           when Hash
-            return json_keyfile[:content]
+            case json_keyfile[:content]
+            when String
+              return HashUtil.deep_symbolize_keys(JSON.parse(json_keyfile[:content]))
+            when Hash
+              return json_keyfile[:content]
+            else
+              raise ConfigError.new "Unsupported json_keyfile type"
+            end
           else
             raise ConfigError.new "Unsupported json_keyfile type"
           end
@@ -830,6 +843,10 @@ class BigqueryMigration
       config[:location]
     end
 
+    def clustering
+      config[:clustering]
+    end
+
     def job_status_polling_interval
       @job_status_polling_interval ||= config[:job_status_polling_interval] || 5
     end

data/lib/bigquery_migration/version.rb

@@ -1,3 +1,3 @@
 class BigqueryMigration
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bigquery_migration
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-22 00:00:00.000000000 Z
+date: 2019-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -138,20 +138,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: test-unit-power_assert
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Migrate BigQuery table schema.
 email:
 - sonots@gmail.com
@@ -179,6 +165,7 @@ files:
 - example/copy_table.yml
 - example/example.yml
 - example/insert_select.yml
+- example/migrate_clustered_table.yml
 - example/migrate_partitioned_table.yml
 - example/migrate_table.yml
 - example/schema.json
@@ -217,8 +204,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Migrate BigQuery table schema