bigquery_migration 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85f03f29febe10514da940bbd913562c67746211
-  data.tar.gz: c2d37cfceb775b429f5c5eac8bfe4ccac4db7785
+  metadata.gz: 0fc953326453253906c139453f2bd5b7b95e09fd
+  data.tar.gz: e8e2d924b197c6f2b6ddbd4dc162c765396d13ff
 SHA512:
-  metadata.gz: b21c0a5be60c12a92df59eedcaa874ba054e9ea7bd6391ec90267a1c47f0f7d8520b4db124be9a9254ca18b034fc15507fc2168090ba6dbbb78407ee5f644442
-  data.tar.gz: 8b26abdeff14898c3d815ec8b37f976c7eb4dfdff51f5c1ad57ac4bc9cdd9cc5ddd36fded4c348be6cddf5a70760b2110323f44cbd5e2892eb940b85d0352665
+  metadata.gz: 4e207d801d7b5a20e241e95c76d391ddc8863cf42c319016ed84e97e226cf2247ad4bd0a452c09ec342725fdd1c92a39353032adb282ca218b2eb1cd4e211663
+  data.tar.gz: a6a6c3932945426f65641eebbc93cb0bda24150a33fd4456a5ebe4b372ab985312131e92c1eab5d9e122db4165682233f0a3f9341ceb8e57112fdf91f1418021

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.1.3 (2016/04/22)
+
+Enhancements:
+
+* Support new BYTES types
+* Add exe/bq-migrate as an alias to exe/bq_migrate
+
 # 0.1.2 (2016/04/14)
 
 Changes:

data/README.md

@@ -76,6 +76,10 @@ migrator.migrate_table(columns: columns)
 # migrator.migrate_table(schema_file: '/path/to/schema.json')
 ```
 
+## Further Details
+
+* See [BigQueryテーブルのスキーマを変更する - sonots:blog](http://blog.livedoor.jp/sonots/archives/47294596.html) (Japanese)
+
 ## Development
 
 ### Run example:

data/example/migrate_table.yml

@@ -15,6 +15,7 @@ actions:
       fields:
         - { name: 'string', type: 'STRING' }
         - { name: 'integer', type: 'INTEGER' }
+        - { name: 'bytes', type: 'BYTES' }
 - action: migrate_table
   <<: *bigquery
   schema_file: example/schema.json

data/example/table_info.yml

@@ -0,0 +1,23 @@
+bigquery: &bigquery
+  json_keyfile: example/your-project-000.json
+  dataset: your_dataset_name
+  table: your_table_name
+
+actions:
+- action: create_dataset
+  <<: *bigquery
+- action: migrate_table
+  <<: *bigquery
+  columns:
+    - { name: 'timestamp', type: 'TIMESTAMP' }
+    - name: 'record'
+      type: 'RECORD'
+      fields:
+        - { name: 'string', type: 'STRING' }
+        - { name: 'integer', type: 'INTEGER' }
+- action: table_info
+  table: your_table_name
+  <<: *bigquery
+- action: table_info
+  prefix: your_table_name
+  <<: *bigquery

data/exe/bq-migrate

@@ -0,0 +1 @@
+bq_migrate

data/lib/bigquery_migration/action.rb

@@ -41,6 +41,7 @@ class BigqueryMigration
         preview
         insert_select
         copy_table
+        table_info
       ])
     end
 
@@ -102,5 +103,23 @@ class BigqueryMigration
         write_disposition: config[:write_disposition],
       )
     end
+
+    def table_info
+      if config[:prefix]
+        tables = client.list_tables[:tables].select {|table| table.start_with?(config[:prefix]) }
+        table_infos = tables.map do |table|
+          result = client.get_table(table: table)
+          result.delete(:responses)
+          result
+        end
+        result = {
+          sum_num_bytes: table_infos.map {|info| info[:num_bytes].to_i }.inject(:+),
+          sum_num_rows: table_infos.map {|info| info[:num_rows].to_i }.inject(:+),
+          table_infos: table_infos,
+        }
+      else
+        client.get_table
+      end
+    end
   end
 end

data/lib/bigquery_migration/bigquery_wrapper.rb

@@ -182,8 +182,20 @@ class BigqueryMigration
         response = {status_code: e.status_code, message: e.message, error_class: e.class}
         raise Error, "Failed to get_table(#{project}, #{dataset}, #{table}), response:#{response}"
       end
-     
-      { responses: { get_table: response } }
+
+      result = {}
+      if response
+        result = {
+          table_id: response.id,
+          creation_time: response.creation_time.to_i, # millisec
+          last_modified_time: response.last_modified_time.to_i, # millisec
+          location: response.location,
+          num_bytes: response.num_bytes.to_i,
+          num_rows: response.num_rows.to_i,
+        }
+      end
+
+      result.merge!({ responses: { get_table: response } })
     end
 
     def insert_table(dataset: nil, table: nil, columns: )
@@ -262,7 +274,7 @@ class BigqueryMigration
         end
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
         if e.status_code == 404 && /Not found:/ =~ e.message
-          railse NotFoundError, "Dataset #{project}:#{dataset} is not found"
+          raise NotFoundError, "Dataset #{project}:#{dataset} is not found"
         end
 
         response = {status_code: e.status_code, message: e.message, error_class: e.class}

data/lib/bigquery_migration/schema.rb

@@ -4,7 +4,7 @@ require_relative 'error'
 
 class BigqueryMigration
   class Schema < ::Array
-    ALLOWED_FIELD_TYPES = Set.new(['STRING', 'INTEGER', 'FLOAT', 'BOOLEAN', 'RECORD', 'TIMESTAMP'])
+    ALLOWED_FIELD_TYPES = Set.new(['STRING', 'INTEGER', 'FLOAT', 'BOOLEAN', 'RECORD', 'TIMESTAMP', 'BYTES'])
     ALLOWED_FIELD_MODES = Set.new(['NULLABLE', 'REQUIRED', 'REPEATED'])
 
     def initialize(columns = [])

data/lib/bigquery_migration/version.rb

@@ -1,3 +1,3 @@
 class BigqueryMigration
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bigquery_migration
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-14 00:00:00.000000000 Z
+date: 2016-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -126,6 +126,7 @@ description: Migrate BigQuery table schema.
 email:
 - sonots@gmail.com
 executables:
+- bq-migrate
 - bq_migrate
 extensions: []
 extra_rdoc_files: []
@@ -147,6 +148,8 @@ files:
 - example/insert_select.yml
 - example/migrate_table.yml
 - example/schema.json
+- example/table_info.yml
+- exe/bq-migrate
 - exe/bq_migrate
 - lib/bigquery_migration.rb
 - lib/bigquery_migration/action.rb