bigquery_migration 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7d3ea2055c104bf0217847c2f8d1efedbfdc5ebd
-  data.tar.gz: 531d0a9fd96ed38a099ab4bdae85682d844c617c
+  metadata.gz: b64d00c827e85ec40f7c37bc2153ee2c06ce80c1
+  data.tar.gz: 9ff2ebad940ec9f46739d5c4a934f7c773f9f5ea
 SHA512:
-  metadata.gz: fa9676f4b90a977a81d1b3faeaa96fcabac9fd014b185d9e2108914650b4a902609d6b9b6680ac9b52fb5d5a9e50d54e5d47dccedcbd93aee4a1a1791c22f209
-  data.tar.gz: d01485a18936f412c3b59185d0c41df62b49f28da130853cec76f205169959988316550c252ba6aa593e0414ef674d2d025c26fee32a23d27ea1f2ceb3a89556
+  metadata.gz: 075e151988a8566dbcff6c9a20debd29fb9642fecc8bfe952a3fe0bae33f2a63a6f8e49d2a186f39a4943abd2de29d8692ab4985fea397cb4ede5b25a797493b
+  data.tar.gz: 1eeb968a5161bb89089a38bf5aacee696c07672f5f1a977bf00bcf10f2a5d7eba87038f86048dedb00ec746e725255f57507c1dae49690584b71c39caeffa538

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.5 (2016/07/25)
+
+Enhancements:
+
+* Support record type and repeated mode for list table data
+
 # 0.1.4 (2016/07/12)
 
 Fixes:
@@ -26,4 +32,3 @@ Changes:
 # 0.1.0 (2016/04/08)
 
 Initial release
-

data/lib/bigquery_migration/bigquery_wrapper.rb

@@ -1,6 +1,7 @@
 require 'csv'
 require 'json'
 require_relative 'schema'
+require_relative 'table_data'
 require_relative 'error'
 require_relative 'time_with_zone'
 require_relative 'hash_util'
@@ -399,45 +400,28 @@ class BigqueryMigration
         raise Error, "Failed to list table_data #{project}:#{dataset}.#{table}, response:#{response}"
       end
 
-      flattened_columns = Schema.new(existing_columns).flattened_columns.map do |name, column|
+      columns = existing_columns
+      flattened_columns = Schema.new(columns).flattened_columns.map do |name, column|
         {name: name}.merge!(column)
       end
       if rows = response.to_h[:rows]
-        flattened_values = flatten_values(rows)
+        values = TableData.new(columns, rows).generate_values
       end
 
       {
         total_rows: response.total_rows,
         columns: flattened_columns,
-        values: flattened_values,
+        values: values,
         responses: {
           list_table_data: response,
         }
       }
     end
 
-    private def flatten_values(rows)
-      rows.map do |r|
-        if r.key?(:f)
-          r[:f].map do |f|
-            if f[:v].respond_to?(:key?) && f[:v].key?(:f)
-              flatten_values(f[:v][:f])
-            elsif f[:v].is_a?(Hash) && f[:v].empty? # nil is converted into {} by to_h
-              nil
-            else
-              f[:v]
-            end
-          end.flatten
-        else
-          r[:v]
-        end
-      end
-    end
-
     def patch_table(dataset: nil, table: nil, columns: nil, add_columns: nil)
       dataset ||= self.dataset
       table ||= self.table
-      
+
       if columns.nil? and add_columns.nil?
         raise ArgumentError, 'patch_table: `columns` or `add_columns` is required'
       end

data/lib/bigquery_migration/table_data.rb

@@ -0,0 +1,118 @@
+# Convert from BigQuery Web console's JavaScript
+require_relative 'error'
+
+class BigqueryMigration
+  class TableData
+    attr_reader :rows
+    attr_reader :columns
+
+    def logger
+      BigqueryMigration.logger
+    end
+
+    def initialize(columns, rows)
+      @columns = columns || raise(ConfigError, '`columns` is required.')
+      @rows = rows || raise(ConfigError, '`rows` is required.')
+    end
+
+    def generate_values
+      rows = @rows.map do |row|
+        values = []
+        max_repeated_count = calculate_repeated_count(columns: @columns, rows: row).max
+        max_repeated_count.times do |count|
+          values.push(generate_value(columns: @columns, rows: row, count: count))
+        end
+        values
+      end
+      # For backword compatibility
+      max_row_count = (rows.map(&:length) || []).max
+      max_row_count > 1 ? rows : rows.map(&:flatten)
+    end
+
+    # This method called recursively.
+    # So, rows must be a hash and hash has key f:.
+    private def calculate_repeated_count(columns: nil, rows: nil)
+      logger.info { "calculate_repeated_count(columns: #{columns}, rows: #{rows})" }
+      return [1] if rows.nil?
+      validate_rows!(rows)
+      rows[:f].zip(columns).map do |row, column|
+        if column[:type] == 'RECORD'
+          if column[:mode] == 'REPEATED'
+            if row[:v].length == 0
+              1
+            else
+              recursive_repeated_counts = row[:v].map do |v|
+                _repeated_counts = calculate_repeated_count(columns: column[:fields], rows: v[:v])
+                repeated_count = _repeated_counts.inject(0) { |acc, n| [acc, n].max }
+                v[:repeated_count] = repeated_count
+              end
+              recursive_repeated_counts.inject(0) { |acc, n| acc + n }
+            end
+          else
+            _repeated_counts = calculate_repeated_count(columns: column[:fields], rows: row[:v])
+            _repeated_counts.inject(0) { |acc, n| [acc, n].max }
+          end
+        elsif column[:mode] == 'REPEATED'
+          [(row[:v] || []).length, 1].max
+        else
+          1
+        end
+      end
+    end
+
+    # This method called recursively.
+    # So, rows must be a hash and hash has key f:.
+    private def generate_value(columns: nil, rows: nil, count: nil)
+      logger.info { "generate_value(columns: #{columns}, rows: #{rows}, count: #{count})" }
+      value = []
+      return value if rows.nil?
+      validate_rows!(rows)
+      rows[:f].zip(columns).each do |row, column|
+        if column[:type] == 'RECORD'
+          if column[:mode] == 'REPEATED'
+            recursive = false
+            # Fixme: would like to avoid using the index counter
+            current = 0
+            row[:v].each do |v|
+              repeated_count = v[:repeated_count]
+              if current <= count && count < (current + repeated_count)
+                generated_values = generate_value(columns: column[:fields], rows: v[:v], count: count - current)
+                value.concat(generated_values)
+                recursive = true
+              end
+              current = current + repeated_count
+            end
+            unless recursive
+              nil_count = generate_nil_count(column[:fields])
+              value.concat(Array.new(nil_count))
+            end
+          elsif row[:v].nil?
+            nil_count = generate_nil_count(column[:fields])
+            value.concat(Array.new(nil_count))
+          else
+            generated_values = generate_value(columns: column[:fields], rows: row[:v], count: count)
+            value.concat(generated_values)
+          end
+        elsif column[:mode] == 'REPEATED'
+          v = row[:v]
+          count < v.length ? value.push(v[count][:v]) : value.push(nil)
+        elsif count == 0
+          value.push(row[:v])
+        else
+          value.push(nil)
+        end
+      end
+      value
+    end
+
+    private def generate_nil_count(fields)
+      fields.inject(0) do |acc, f|
+        f[:type] == 'RECORD' ? acc + generate_nil_count(f[:fields]) : acc + 1
+      end
+    end
+
+    private def validate_rows!(rows)
+      raise ConfigError, '`rows` must be a hash and hash has key `:f`.' if !rows.is_a?(Hash) || !rows.has_key?(:f)
+    end
+  end
+end

data/lib/bigquery_migration/version.rb

@@ -1,3 +1,3 @@
 class BigqueryMigration
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bigquery_migration
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-12 00:00:00.000000000 Z
+date: 2016-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -161,6 +161,7 @@ files:
 - lib/bigquery_migration/hash_util.rb
 - lib/bigquery_migration/logger.rb
 - lib/bigquery_migration/schema.rb
+- lib/bigquery_migration/table_data.rb
 - lib/bigquery_migration/time_with_zone.rb
 - lib/bigquery_migration/version.rb
 homepage: https://github.com/sonots/bigquery_migration