sequel-bigquery 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c37139c40fd486391d9d16ab147430352b67a1a2b5a55f7eed4189c368b5e87a
-  data.tar.gz: e3753842a9727cf53451ff80ac9abc250343a719d4215ab368c5a93a9d2b4830
+  metadata.gz: 6c15547bd0bbe53a384a987286cda32f9e1ad624c2c9e5339ac412322e810653
+  data.tar.gz: 26d87c1ff3d3aecf9d5401de88ad9048a239dd65bc3a682cfbb9d9d726675b9f
 SHA512:
-  metadata.gz: 6189f0b53a6edd9d66c915316ce8e9e8c836181609405b54e246c0dbe5451bce995c6f5b73e7b320624c35b62bfd66874757734f839ef7f9023dbc099271b2a4
-  data.tar.gz: 715b5adbdb6e225ca9af37363fbc30b59bde54a0b4602d91afcca93506cac541d252b31767571605ee555f2fdaad8d316ccf67a083354bfdcc9cd5f925c51646
+  metadata.gz: daa23d28fc8359540d61e57a5f4087f76da68045e6535d864f711c51d8fd0517bf9d74e332ea9cd3a5399ece489beb06df08dfaf409caf8877a1cf6aaa8140a3
+  data.tar.gz: eb73f36dbe0317d42e5b1fd9a774dd7d5440d36a8ddebd33c5683a67723efaf5538d4b915a4b518e49a6165854b42152762d57934a57dd050efee0f735563c4f

data/README.md

@@ -17,7 +17,9 @@ Beyond migrations, I'm unsure how useful this gem is. I haven't yet tested what
   - [Creating tables with column defaults](#creating-tables-with-column-defaults)
   - [Transactions](#transactions)
   - [Update statements without `WHERE`](#update-statements-without-where)
+  - [Combining statements](#combining-statements)
   - [Alter table](#alter-table)
+  - [Column recreation](#column-recreation)
 - [Installation](#installation)
 - [Usage](#usage)
 - [Contributing](#contributing)
@@ -65,9 +67,19 @@ BigQuery doesn't support transactions where the statements are executed individu
 
 BigQuery requires all `UPDATE` statement to have a `WHERE` clause. As a workaround, statements which lack one have `where 1 = 1` appended automatically (crudely).
 
+### Combining statements
+
+When combining multiple statements into one query (with `;`), and the final statement is not a `SELECT`, the `google-cloud-bigquery` gem has a [bug](https://github.com/googleapis/google-cloud-ruby/issues/9617) which causes an exception. Note that all the statements have been executed when this happens. A workaround is to append `; SELECT 1`.
+
 ### Alter table
 
-We've found that the `google-cloud-bigquery` gem seems to have a bug where an internal lack of result value results in a `NoMethodError` on nil for `fields` within `from_gapi_json`. [See issue #6](https://github.com/ZimbiX/sequel-bigquery/issues/6#issuecomment-968523731). As a workaround, all generated statements within `alter_table` are joined together with `;` and executed only at the end of the block. A `select 1` is also appended to try to ensure we have a result to avoid the aforementioned exception. A bonus of batching the queries is that the latency should be somewhat reduced.
+BigQuery [rate-limits alter table statements](https://cloud.google.com/bigquery/quotas#dataset_limits) to 10 per second. This is mitigated somewhat by Sequel combining `ALTER TABLE` statements whenever possible, and BigQuery having extremely high latency (\~2 seconds per query); but you may still run into this limitation.
+
+We've also noticed a bug with `google-cloud-bigquery` where an `ALTER TABLE` statement resulted in a `NoMethodError` on nil for `fields` within `from_gapi_json`. We're not yet sure what caused this.
+
+### Column recreation
+
+Be careful when deleting a column which you might want to re-add. BigQuery reserves the name of a deleted column for up to the time travel duration - which is [*seven days*](https://cloud.google.com/bigquery/docs/time-travel). Re-creating the entire dataset is a painful workaround.
 
 ## Installation
 

data/lib/sequel-bigquery.rb

@@ -78,9 +78,10 @@ module Sequel
           warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
         end
 
+        sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
+
         synchronize(opts[:server]) do |conn|
           results = log_connection_yield(sql, conn) do
-            sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
             conn.query(sql_to_execute)
           end
           log_each(:debug, results.awesome_inspect)
@@ -89,7 +90,19 @@ module Sequel
           else
             results
           end
-        rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
+        rescue Google::Cloud::InvalidArgumentError, Google::Cloud::PermissionDeniedError => e
+          if e.message.include?('too many table update operations for this table')
+            warn('Triggered rate limit of table update operations for this table. For more information, see https://cloud.google.com/bigquery/docs/troubleshoot-quotas')
+            if retryable_query?(sql_to_execute)
+              warn('Detected retryable query - re-running query after a 1 second sleep')
+              sleep 1
+              retry
+            else
+              log_each(:error, "Query not detected as retryable; can't automatically recover from being rate-limited")
+            end
+          end
+          raise_error(e)
+        rescue ArgumentError => e
           raise_error(e)
         end # rubocop:disable Style/MultilineBlockChain
           .tap do
@@ -210,11 +223,20 @@ module Sequel
         sql
       end
 
-      # Batch the alter table queries and make sure something is returned to avoid an error related to the return value
-      def apply_alter_table(name, ops)
-        sqls = alter_table_sql_list(name, ops)
-        sqls_joined = (sqls + ['select 1']).join(";\n")
-        execute_ddl(sqls_joined)
+      def supports_combining_alter_table_ops?
+        true
+      end
+
+      def retryable_query?(sql)
+        single_statement_query?(sql) && alter_table_query?(sql)
+      end
+
+      def single_statement_query?(sql)
+        !sql.rstrip.chomp(';').include?(';')
+      end
+
+      def alter_table_query?(sql)
+        sql.match?(/\Aalter table /i)
       end
     end
 

data/lib/sequel_bigquery/version.rb

@@ -2,6 +2,6 @@
 
 module Sequel
   module Bigquery
-    VERSION = '0.5.0'
+    VERSION = '0.6.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sequel-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Brendan Weibrecht