bigquery_migration 0.1.7 → 0.2.0.pre1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c113ced051e1bf7c9b8a78123b56e06cb750251f
4
- data.tar.gz: 3d9f5f9c8ea1af31753b7ea92eb77a8a8a612ef5
3
+ metadata.gz: 619d8bcaa5756501fbe6e9a07726343024ff0bc5
4
+ data.tar.gz: 931f57eb0f381ad91a1cd2ab6c39c9fe645aad35
5
5
  SHA512:
6
- metadata.gz: d98d9fac6a2466fba7bddadf38fedccf11829782c549bf7b2ac3709bf7836fc42725cdd385e34c3cf56da262501d35315f4cd516a826309ff5b2b2236a47ca65
7
- data.tar.gz: a9e14ad54d9144d6414af8d0c09922821905dbde2738dfec1eb5fd17a9c67a403b970e4ca8b46b193cd9aea9be3650ff8082cefb4a9d9ef417dec87fe69db7d9
6
+ metadata.gz: cfd6a985186c5cb426a3f09796764cfda1697aeeb21a531260062ec37b38b3ed835460d9ecf07abda544ddd7edc072252efbf1e79281d5c8ba8a2e4e0a9f0c2c
7
+ data.tar.gz: 789b6000dc10f4ea0554243bbff1158505831768595b1e6bd85b9226731f3e950ff61bc431ac404293589b8c3b962ad4e7e0affbbee1d4eba863e0bd88af0bb3
@@ -1,12 +1,8 @@
1
- # 0.1.7 (2016/09/16)
2
-
3
- Fixes:
4
-
5
- * migrate_table should create table only if a table does not exist
1
+ # 0.2.0 (2016/10/03)
6
2
 
7
3
  Enhancements:
8
4
 
9
- * create_table should not create a table with empty columns
5
+ * Support migrate_partitioned_table
10
6
 
11
7
  # 0.1.6 (2016/07/26)
12
8
 
@@ -0,0 +1,23 @@
1
+ bigquery: &bigquery
2
+ json_keyfile: example/your-project-000.json
3
+ dataset: your_dataset_name
4
+ table: your_table_name
5
+
6
+ actions:
7
+ - action: create_dataset
8
+ <<: *bigquery
9
+ - action: migrate_partitioned_table
10
+ <<: *bigquery
11
+ columns:
12
+ - { name: 'timestamp', type: 'TIMESTAMP' }
13
+ - name: 'record'
14
+ type: 'RECORD'
15
+ fields:
16
+ - { name: 'string', type: 'STRING' }
17
+ - { name: 'integer', type: 'INTEGER' }
18
+ - { name: 'bytes', type: 'BYTES' }
19
+ - action: migrate_partitioned_table
20
+ <<: *bigquery
21
+ schema_file: example/schema.json
22
+ - action: delete_table
23
+ <<: *bigquery
@@ -42,6 +42,7 @@ class BigqueryMigration
42
42
  insert_select
43
43
  copy_table
44
44
  table_info
45
+ migrate_partitioned_table
45
46
  ])
46
47
  end
47
48
 
@@ -77,6 +78,13 @@ class BigqueryMigration
77
78
  )
78
79
  end
79
80
 
81
+ def migrate_partitioned_table
82
+ client.migrate_partitioned_table(
83
+ schema_file: config[:schema_file],
84
+ columns: config[:columns],
85
+ )
86
+ end
87
+
80
88
  def insert
81
89
  client.insert_all_table_data(rows: config[:rows])
82
90
  end
@@ -199,7 +199,7 @@ class BigqueryMigration
199
199
  result.merge!({ responses: { get_table: response } })
200
200
  end
201
201
 
202
- def insert_table(dataset: nil, table: nil, columns: )
202
+ def insert_table(dataset: nil, table: nil, columns:, options: {})
203
203
  dataset ||= self.dataset
204
204
  table ||= self.table
205
205
  raise Error, "columns is empty" if columns.empty?
@@ -215,6 +215,14 @@ class BigqueryMigration
215
215
  fields: schema,
216
216
  }
217
217
  }
218
+
219
+ if options['time_partitioning']
220
+ body[:time_partitioning] = {
221
+ type: options['time_partitioning']['type'],
222
+ expiration_ms: options['time_partitioning']['expiration_ms'],
223
+ }
224
+ end
225
+
218
226
  opts = {}
219
227
  logger.debug { "#{head}insert_table(#{project}, #{dataset}, #{body}, #{opts})" }
220
228
  unless dry_run?
@@ -234,6 +242,12 @@ class BigqueryMigration
234
242
  end
235
243
  alias :create_table :insert_table
236
244
 
245
+ def insert_partitioned_table(dataset: nil, table: nil, columns:, options: {})
246
+ options['time_partitioning'] = {'type'=>'DAY'}
247
+ insert_table(dataset: dataset, table: table, columns: columns, options: options)
248
+ end
249
+ alias :create_partitioned_table :insert_partitioned_table
250
+
237
251
  def delete_table(dataset: nil, table: nil)
238
252
  dataset ||= self.dataset
239
253
  table ||= self.table
@@ -684,5 +698,47 @@ class BigqueryMigration
684
698
 
685
699
  result.merge!( before_columns: before_columns, after_columns: after_columns )
686
700
  end
701
+
702
+ # creates a table with time_partitioning option
703
+ # this version only uses patch table API (no query job) because querying partitioned table should cost lots
704
+ def migrate_partitioned_table(table: nil, schema_file: nil, columns: nil, options: {})
705
+ table ||= self.table
706
+
707
+ if schema_file.nil? and columns.nil?
708
+ raise ArgumentError, '`schema_file` or `columns` is required'
709
+ end
710
+ if schema_file
711
+ columns = HashUtil.deep_symbolize_keys(JSON.parse(File.read(schema_file)))
712
+ end
713
+ Schema.validate_columns!(columns)
714
+
715
+ result = {}
716
+ begin
717
+ get_table
718
+ rescue NotFoundError
719
+ before_columns = []
720
+ result = create_partitioned_table(table: table, columns: columns, options: options)
721
+ else
722
+ before_columns = existing_columns
723
+ add_columns = Schema.diff_columns(before_columns, columns)
724
+ drop_columns = Schema.diff_columns(columns, before_columns)
725
+
726
+ if !drop_columns.empty? || !add_columns.empty?
727
+ Schema.make_nullable!(drop_columns) # drop columns will be NULLABLE columns
728
+ Schema.reverse_merge!(columns, patch_columns = drop_columns)
729
+ Schema.reverse_merge!(patch_columns, patch_columns = add_columns)
730
+ patch_table(table: table, columns: patch_columns)
731
+ end
732
+ end
733
+
734
+ after_columns = existing_columns
735
+
736
+ if after_columns.empty? and !dry_run?
737
+ raise Error, "after_columns is empty. " \
738
+ "before_columns: #{before_columns}, after_columns: #{after_columns}, columns: #{columns}"
739
+ end
740
+
741
+ result.merge!( before_columns: before_columns, after_columns: after_columns )
742
+ end
687
743
  end
688
744
  end
@@ -70,7 +70,6 @@ class BigqueryMigration
70
70
  self.class.build_query_fields(source_columns, self)
71
71
  end
72
72
 
73
-
74
73
  class << self
75
74
  # The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
76
75
  # and must start with a letter or underscore. The maximum length is 128 characters.
@@ -383,6 +382,17 @@ class BigqueryMigration
383
382
  end
384
383
  end
385
384
  end
385
+
386
+ def make_nullable!(columns)
387
+ columns.each do |column|
388
+ if column[:fields]
389
+ make_nullable!(column[:fields])
390
+ else
391
+ column[:mode] = 'NULLABLE'
392
+ end
393
+ end
394
+ columns
395
+ end
386
396
  end
387
397
  end
388
398
  end
@@ -1,3 +1,3 @@
1
1
  class BigqueryMigration
2
- VERSION = "0.1.7"
2
+ VERSION = "0.2.0.pre1"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bigquery_migration
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.7
4
+ version: 0.2.0.pre1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-09-16 00:00:00.000000000 Z
11
+ date: 2016-10-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: google-api-client
@@ -146,6 +146,7 @@ files:
146
146
  - example/copy_table.yml
147
147
  - example/example.yml
148
148
  - example/insert_select.yml
149
+ - example/migrate_partitioned_table.yml
149
150
  - example/migrate_table.yml
150
151
  - example/schema.json
151
152
  - example/table_info.yml
@@ -179,9 +180,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
179
180
  version: '0'
180
181
  required_rubygems_version: !ruby/object:Gem::Requirement
181
182
  requirements:
182
- - - ">="
183
+ - - ">"
183
184
  - !ruby/object:Gem::Version
184
- version: '0'
185
+ version: 1.3.1
185
186
  requirements: []
186
187
  rubyforge_project:
187
188
  rubygems_version: 2.5.1