bigquery_migration 0.2.0.pre2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8e9acd181e25a8e935168d4a99213c52bca11c9
-  data.tar.gz: a5f4f71c2a04fbc453656bff29b9b961d56b16dd
+  metadata.gz: ee06c260e388082cdec0d3e4f3a72577952acfb8
+  data.tar.gz: 225387ac8df6a0fda1b723bc4f10ca6955fd9025
 SHA512:
-  metadata.gz: d4cbc58db0ca399e5edae6fc21d9fab708db9edebf0e9fadb8f3fdf7f3098d595de662fb92355bbea0bebffa1ffab0e426f89fc7708226917ede3832378572f4
-  data.tar.gz: 3fa2a4a0313eebca630b9648e8a8c5b747ea596ccfc57aa5e477ad5cbb935bb32c626128df8b1f71d9b7a09752ee5f18c02e7e86032eaea6c39894947321c97a
+  metadata.gz: 77326f487936158701a52e0a74f0d2241d2375ca5f96cb6803e7b85cec1b3206fa7728a699eae10a6b351adfb595242fa83ac742d257179128df157ec082b49f
+  data.tar.gz: a1b0e5a6418c43383b3c8b4d36deed62d9af9f823d9b1e6e333c9066f4e879d09214850b00a85238c0b8826392744da92dd9794edca97a67f80a8d794069445d

data/.gitignore

@@ -9,3 +9,4 @@
 /tmp/
 your-project-000.json
 .tags
+.ruby-version

data/CHANGELOG.md

@@ -4,11 +4,22 @@ Enhancements:
 
 * Support migrate_partitioned_table
 
+Fixes:
+
+* Fix list_table_data for when a value is an empty hash
+
+# 0.1.7 (2016/09/17)
+
+Fixes:
+
+* Prohibit to create a table with empty columns
+* Create a table only if a table does not exist
+
 # 0.1.6 (2016/07/26)
 
 Fixes:
 
-* Fix empty hash to nil
+* Fix empty hash to nil for list table data
 
 # 0.1.5 (2016/07/25)
 

data/lib/bigquery_migration/bigquery_wrapper.rb

@@ -420,7 +420,7 @@ class BigqueryMigration
         {name: name}.merge!(column)
       end
       if rows = response.to_h[:rows]
-        values = TableData.new(columns, rows).generate_values
+        values = TableData.new(columns, rows).values
       end
 
       {

data/lib/bigquery_migration/table_data.rb

@@ -1,39 +1,76 @@
-# Convert from BigQuery Web console's JavaScript
+# This codes are translated from BigQuery Web console's JavaScript
 require_relative 'error'
 
 class BigqueryMigration
   class TableData
-    attr_reader :rows
-    attr_reader :columns
+    attr_reader :rows, :columns
 
     def logger
       BigqueryMigration.logger
     end
 
     def initialize(columns, rows)
-      @columns = columns || raise(ConfigError, '`columns` is required.')
-      @rows = rows || raise(ConfigError, '`rows` is required.')
+      @columns = columns || raise(Error, '`columns` is required.')
+      @rows = rows || raise(Error, '`rows` is required.')
     end
 
-    def generate_values
-      rows = @rows.map do |row|
-        values = []
-        max_repeated_count = calculate_repeated_count(columns: @columns, rows: row).max
-        max_repeated_count.times do |count|
-          values.push(generate_value(columns: @columns, rows: row, count: count))
+    # format list_table_data response rows which is like
+    #
+    # [
+    #   { f: [
+    #     { v: "foo" },
+    #     { v: "1" },
+    #     { v: [] },
+    #     { v: "1.1" },
+    #     { v: "true" },
+    #     { v: "1.444435200E9" }
+    #   ] },
+    #   { f: [
+    #     { v: "foo" },
+    #     { v: "2" },
+    #     { v: [
+    #       { v: "foo" },
+    #       { v: "bar" }
+    #     ] },
+    #     { v: "2.2" },
+    #     { v: "false" },
+    #     { v: "1.444435200E9" }
+    #   ] }
+    # ]
+    #
+    # into
+    #
+    # [
+    #   # first row
+    #   [
+    #     [ "foo", "1", nil, "1.1", "true", "1.444435200E9" ]
+    #   ],
+    #   # second row
+    #   [
+    #     [ "foo", "2", "foo", "2.2", "false", "1.444435200E9" ],
+    #     [ nil, nil, "bar", nil, nil, nil ],
+    #   ],
+    # ]
+    def values
+      values = @rows.map do |row|
+        repeated_count = repeated_count(columns: @columns, rows: row)
+        formatted_row = []
+        repeated_count.times do |count|
+          formatted_row << format_row(columns: @columns, rows: row, count: count)
         end
-        values
+        formatted_row
       end
-      # For backword compatibility
-      max_row_count = (rows.map(&:length) || []).max
-      max_row_count > 1 ? rows : rows.map(&:flatten)
+      # flattern if there is no repeated column for backward compatibility
+      values.map(&:length).max > 1 ? values : values.flatten(1)
     end
 
-    # This method called recursively.
-    # So, rows must be a hash and hash has key f:.
-    private def calculate_repeated_count(columns: nil, rows: nil)
-      # logger.info { "calculate_repeated_count(columns: #{columns}, rows: #{rows})" }
-      return [1] if (rows.nil? || rows.empty?)
+    private
+
+    # Count maximum number of rows on repeated columns
+    #
+    # This method called recursively, rows must be a hash and hash has key f:
+    def repeated_count(columns: nil, rows: nil)
+      return 1 if (rows.nil? || rows.empty?)
       validate_rows!(rows)
       rows[:f].zip(columns).map do |row, column|
         if column[:type] == 'RECORD'
@@ -41,84 +78,76 @@ class BigqueryMigration
             if row[:v].length == 0
               1
             else
-              recursive_repeated_counts = row[:v].map do |v|
-                _repeated_counts = calculate_repeated_count(columns: column[:fields], rows: v[:v])
-                repeated_count = _repeated_counts.inject(0) { |acc, n| [acc, n].max }
-                v[:repeated_count] = repeated_count
-              end
-              recursive_repeated_counts.inject(0) { |acc, n| acc + n }
+              row[:v].map do |v|
+                v[:repeated_count] = repeated_count(columns: column[:fields], rows: v[:v])
+              end.inject(:+)
             end
           else
-            _repeated_counts = calculate_repeated_count(columns: column[:fields], rows: row[:v])
-            _repeated_counts.inject(0) { |acc, n| [acc, n].max }
+            repeated_count(columns: column[:fields], rows: row[:v])
           end
         elsif column[:mode] == 'REPEATED'
           [(row[:v] || []).length, 1].max
         else
           1
         end
-      end
+      end.max
     end
 
     # This method called recursively.
     # So, rows must be a hash and hash has key f:.
-    private def generate_value(columns: nil, rows: nil, count: nil)
-      # logger.info { "generate_value(columns: #{columns}, rows: #{rows}, count: #{count})" }
-      value = []
+    def format_row(columns: nil, rows: nil, count: nil)
+      formatted_row = []
       return [nil] if (rows.nil? || rows.empty?)
       validate_rows!(rows)
       rows[:f].zip(columns).each do |row, column|
         if column[:type] == 'RECORD'
           if column[:mode] == 'REPEATED'
             recursive = false
-            # Fixme: would like to avoid using the index counter
             current = 0
             row[:v].each do |v|
               repeated_count = v[:repeated_count]
               if current <= count && count < (current + repeated_count)
-                generated_values = generate_value(columns: column[:fields], rows: v[:v], count: count - current)
-                value.concat(generated_values)
+                formatted_row.concat format_row(columns: column[:fields], rows: v[:v], count: count - current)
                 recursive = true
               end
               current = current + repeated_count
             end
             unless recursive
-              nil_count = generate_nil_count(column[:fields])
-              value.concat(Array.new(nil_count))
+              nil_count = get_nil_count(column[:fields])
+              formatted_row.concat(Array.new(nil_count))
             end
           elsif row[:v].nil?
-            nil_count = generate_nil_count(column[:fields])
-            value.concat(Array.new(nil_count))
+            nil_count = get_nil_count(column[:fields])
+            formatted_row.concat(Array.new(nil_count))
           else
-            generated_values = generate_value(columns: column[:fields], rows: row[:v], count: count)
-            value.concat(generated_values)
+            formatted_row.concat format_row(columns: column[:fields], rows: row[:v], count: count)
           end
         elsif column[:mode] == 'REPEATED'
           v = row[:v]
-          count < v.length ? value.push(normalize_value(v[count][:v])) : value.push(nil)
+          count < v.length ? formatted_row.push(normalize_value(v[count][:v])) : formatted_row.push(nil)
         elsif count == 0
-          value.push((normalize_value(row[:v])))
+          formatted_row.push((normalize_value(row[:v])))
         else
-          value.push(nil)
+          formatted_row.push(nil)
         end
       end
-      value
+      formatted_row
     end
 
     # special treatment empty hash.
     # nil is converted into {} by to_h
-    private def normalize_value(v)
+    def normalize_value(v)
       v.is_a?(Hash) && v.empty? ? nil : v
     end
 
-    private def generate_nil_count(fields)
+    def get_nil_count(fields)
       fields.inject(0) do |acc, f|
-        f[:type] == 'RECORD' ? acc + generate_nil_count(f[:fields]) : acc + 1
+        f[:type] == 'RECORD' ? acc + get_nil_count(f[:fields]) : acc + 1
       end
     end
 
-    private def validate_rows!(rows)
-      raise ConfigError, '`rows` must be a hash and hash has key `:f`.' if !rows.is_a?(Hash) || !rows.has_key?(:f)
+    def validate_rows!(rows)
+      raise Error, '`rows` must be a hash and hash has key `:f`.' if !rows.is_a?(Hash) || !rows.has_key?(:f)
     end
   end
 end

data/lib/bigquery_migration/version.rb

@@ -1,3 +1,3 @@
 class BigqueryMigration
-  VERSION = "0.2.0.pre2"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bigquery_migration
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre2
+  version: 0.2.0
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -180,9 +180,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1