bigquery_migration 0.1.7 → 0.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c113ced051e1bf7c9b8a78123b56e06cb750251f
-  data.tar.gz: 3d9f5f9c8ea1af31753b7ea92eb77a8a8a612ef5
+  metadata.gz: 619d8bcaa5756501fbe6e9a07726343024ff0bc5
+  data.tar.gz: 931f57eb0f381ad91a1cd2ab6c39c9fe645aad35
 SHA512:
-  metadata.gz: d98d9fac6a2466fba7bddadf38fedccf11829782c549bf7b2ac3709bf7836fc42725cdd385e34c3cf56da262501d35315f4cd516a826309ff5b2b2236a47ca65
-  data.tar.gz: a9e14ad54d9144d6414af8d0c09922821905dbde2738dfec1eb5fd17a9c67a403b970e4ca8b46b193cd9aea9be3650ff8082cefb4a9d9ef417dec87fe69db7d9
+  metadata.gz: cfd6a985186c5cb426a3f09796764cfda1697aeeb21a531260062ec37b38b3ed835460d9ecf07abda544ddd7edc072252efbf1e79281d5c8ba8a2e4e0a9f0c2c
+  data.tar.gz: 789b6000dc10f4ea0554243bbff1158505831768595b1e6bd85b9226731f3e950ff61bc431ac404293589b8c3b962ad4e7e0affbbee1d4eba863e0bd88af0bb3

data/CHANGELOG.md

@@ -1,12 +1,8 @@
-# 0.1.7 (2016/09/16)
-
-Fixes:
-
-* migrate_table should create table only if a table does not exist
+# 0.2.0 (2016/10/03)
 
 Enhancements:
 
-* create_table should not create a table with empty columns
+* Support migrate_partitioned_table
 
 # 0.1.6 (2016/07/26)
 

data/example/migrate_partitioned_table.yml

@@ -0,0 +1,23 @@
+bigquery: &bigquery
+  json_keyfile: example/your-project-000.json
+  dataset: your_dataset_name
+  table: your_table_name
+
+actions:
+- action: create_dataset
+  <<: *bigquery
+- action: migrate_partitioned_table
+  <<: *bigquery
+  columns:
+    - { name: 'timestamp', type: 'TIMESTAMP' }
+    - name: 'record'
+      type: 'RECORD'
+      fields:
+        - { name: 'string', type: 'STRING' }
+        - { name: 'integer', type: 'INTEGER' }
+        - { name: 'bytes', type: 'BYTES' }
+- action: migrate_partitioned_table
+  <<: *bigquery
+  schema_file: example/schema.json
+- action: delete_table
+  <<: *bigquery

data/lib/bigquery_migration/action.rb

@@ -42,6 +42,7 @@ class BigqueryMigration
         insert_select
         copy_table
         table_info
+        migrate_partitioned_table
       ])
     end
 
@@ -77,6 +78,13 @@ class BigqueryMigration
       )
     end
 
+    def migrate_partitioned_table
+      client.migrate_partitioned_table(
+        schema_file: config[:schema_file],
+        columns: config[:columns],
+      )
+    end
+
     def insert
       client.insert_all_table_data(rows: config[:rows])
     end

data/lib/bigquery_migration/bigquery_wrapper.rb

@@ -199,7 +199,7 @@ class BigqueryMigration
       result.merge!({ responses: { get_table: response } })
     end
 
-    def insert_table(dataset: nil, table: nil, columns: )
+    def insert_table(dataset: nil, table: nil, columns:, options: {})
       dataset ||= self.dataset
       table ||= self.table
       raise Error, "columns is empty" if columns.empty?
@@ -215,6 +215,14 @@ class BigqueryMigration
             fields: schema,
           }
         }
+
+        if options['time_partitioning']
+          body[:time_partitioning] = {
+            type: options['time_partitioning']['type'],
+            expiration_ms: options['time_partitioning']['expiration_ms'],
+          }
+        end
+
         opts = {}
         logger.debug { "#{head}insert_table(#{project}, #{dataset}, #{body}, #{opts})" }
         unless dry_run?
@@ -234,6 +242,12 @@ class BigqueryMigration
     end
     alias :create_table :insert_table
 
+    def insert_partitioned_table(dataset: nil, table: nil, columns:, options: {})
+      options['time_partitioning'] = {'type'=>'DAY'}
+      insert_table(dataset: dataset, table: table, columns: columns, options: options)
+    end
+    alias :create_partitioned_table :insert_partitioned_table
+
     def delete_table(dataset: nil, table: nil)
       dataset ||= self.dataset
       table ||= self.table
@@ -684,5 +698,47 @@ class BigqueryMigration
 
       result.merge!( before_columns: before_columns, after_columns: after_columns )
     end
+
+    # creates a table with time_partitioning option
+    # this version only uses patch table API (no query job) because querying partitioned table should cost lots
+    def migrate_partitioned_table(table: nil, schema_file: nil, columns: nil, options: {})
+      table ||= self.table
+
+      if schema_file.nil? and columns.nil?
+        raise ArgumentError, '`schema_file` or `columns` is required'
+      end
+      if schema_file
+        columns = HashUtil.deep_symbolize_keys(JSON.parse(File.read(schema_file)))
+      end
+      Schema.validate_columns!(columns)
+
+      result = {}
+      begin
+        get_table
+      rescue NotFoundError
+        before_columns = []
+        result = create_partitioned_table(table: table, columns: columns, options: options)
+      else
+        before_columns = existing_columns
+        add_columns  = Schema.diff_columns(before_columns, columns)
+        drop_columns = Schema.diff_columns(columns, before_columns)
+
+        if !drop_columns.empty? || !add_columns.empty?
+          Schema.make_nullable!(drop_columns) # drop columns will be NULLABLE columns
+          Schema.reverse_merge!(columns, patch_columns = drop_columns)
+          Schema.reverse_merge!(patch_columns, patch_columns = add_columns)
+          patch_table(table: table, columns: patch_columns)
+        end
+      end
+
+      after_columns = existing_columns
+
+      if after_columns.empty? and !dry_run?
+        raise Error, "after_columns is empty. " \
+          "before_columns: #{before_columns}, after_columns: #{after_columns}, columns: #{columns}"
+      end
+
+      result.merge!( before_columns: before_columns, after_columns: after_columns )
+    end
   end
 end

data/lib/bigquery_migration/schema.rb

@@ -70,7 +70,6 @@ class BigqueryMigration
       self.class.build_query_fields(source_columns, self)
     end
 
-
     class << self
       # The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
       # and must start with a letter or underscore. The maximum length is 128 characters.
@@ -383,6 +382,17 @@ class BigqueryMigration
           end
         end
       end
+
+      def make_nullable!(columns)
+        columns.each do |column|
+          if column[:fields]
+            make_nullable!(column[:fields])
+          else
+            column[:mode] = 'NULLABLE'
+          end
+        end
+        columns
+      end
     end
   end
 end

data/lib/bigquery_migration/version.rb

@@ -1,3 +1,3 @@
 class BigqueryMigration
-  VERSION = "0.1.7"
+  VERSION = "0.2.0.pre1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bigquery_migration
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0.pre1
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-16 00:00:00.000000000 Z
+date: 2016-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -146,6 +146,7 @@ files:
 - example/copy_table.yml
 - example/example.yml
 - example/insert_select.yml
+- example/migrate_partitioned_table.yml
 - example/migrate_table.yml
 - example/schema.json
 - example/table_info.yml
@@ -179,9 +180,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1