sequel-bigquery 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 288cbe303c3db264ccc9e71f7c39c63185428056bfebfe67e1cb520ddeef5243
-  data.tar.gz: 583b3d835d03762ef0ba1ba821d6545032db97202a672b4cd08be9cd50e134d9
+  metadata.gz: e444c2d4a5c6b54b5a1bb0efde17e1dd83761ec53f1b796f5e959646bd37f315
+  data.tar.gz: 41875a2325c00eb26786fbd3ab8430e9880b93d795a18b000bff3dd75ed0ea47
 SHA512:
-  metadata.gz: 2691185b162d701bc1c8d78acbb0ea7624716a10612eedbbcf5c4feb0d04f34b6ef5f7c1e8936052d3ab28a6196a75613c8c09aaa7713245105ddc989f0a1266
-  data.tar.gz: e642c036248215c810e2c066b37f5864d753fa0ad52cb0352538c661e7ec41b475cf43cb5a0b7220d0410402e6b2100e02b1b4411ce4c2c8fd7511b810a7c928
+  metadata.gz: 725fce3447e58e75c5cf817d736322c567661752d1b2d2f4f45aa0ce712825e8792214d1f56313272d8a84facdadff51f63ee4a6b3bda1b8c55d92a80a826798
+  data.tar.gz: 3aa8900e8fba7400049e8f83879998b246296e1705d01b25f2a1cf20855cf8faeea09b9d7e4cb7280b2783296d0a27f61ed295275089d6de311c256c5f7ea763

data/README.md

@@ -4,6 +4,10 @@
 
 A Sequel adapter for [Google's BigQuery](https://cloud.google.com/bigquery).
 
+This gem was created in order to manage schema migrations of a BigQuery dataset at GreenSync. At the time of writing, we couldn't find any good tools in any language to manage changes to the schema as a set of migrations.
+
+Beyond migrations, I'm unsure how useful this gem is. I haven't yet tested what the performance would be for data interactions vs. directly using the `google-cloud-bigquery` gem's native facilities. If you're inserting a bunch of data, it's probably a better idea to use an [inserter from that gem](https://googleapis.dev/ruby/google-cloud-bigquery/latest/Google/Cloud/Bigquery/Dataset.html#insert_async-instance_method) rather than going through SQL.
+
 ## Contents
 
 <!-- MarkdownTOC autolink=true -->
@@ -68,6 +72,7 @@ Connect to BigQuery:
 
 ```
 require 'sequel-bigquery'
+require 'logger'
 
 db = Sequel.connect(
   adapter: :bigquery,
@@ -80,6 +85,8 @@ db = Sequel.connect(
 
 And use Sequel like normal.
 
+Note that it is important to supply a logger that will at least output warning messages so you know when your queries are being modifed or buffered, which may be unexpected behaviour.
+
 ## Contributing
 
 Pull requests welcome! =)

data/lib/sequel-bigquery.rb

@@ -4,6 +4,7 @@ require 'delegate'
 require 'time'
 
 require 'google/cloud/bigquery'
+require 'amazing_print'
 require 'paint'
 require 'sequel'
 
@@ -16,39 +17,33 @@ module Sequel
     class Database < Sequel::Database # rubocop:disable Metrics/ClassLength
       set_adapter_scheme :bigquery
 
-      def initialize(*args, **kawrgs)
-        puts '.new'
-        @orig_opts = kawrgs.fetch(:orig_opts)
+      def initialize(*args, **kwargs)
+        @bigquery_config = kwargs.fetch(:orig_opts)
         @sql_buffer = []
         @sql_buffering = false
         super
       end
 
       def connect(*_args)
-        puts '#connect'
-        config = @orig_opts.dup
-        config.delete(:adapter)
-        config.delete(:logger)
-        location = config.delete(:location)
-        bq_dataset_name = config.delete(:dataset) || config.delete(:database)
-        @bigquery = Google::Cloud::Bigquery.new(config)
+        log_each(:debug, '#connect')
+        get_or_create_bigquery_dataset
+          .tap { log_each(:debug, '#connect end') }
+      end
+
+      def bigquery
         # ObjectSpace.each_object(HTTPClient).each { |c| c.debug_dev = STDOUT }
-        @bigquery.dataset(bq_dataset_name) || begin
-          @loggers[0].debug('BigQuery dataset %s does not exist; creating it' % bq_dataset_name)
-          @bigquery.create_dataset(bq_dataset_name, location: location)
-        end
-          .tap { puts '#connect end' }
+        @bigquery ||= Google::Cloud::Bigquery.new(google_cloud_bigquery_gem_config)
       end
 
       def disconnect_connection(_c)
-        puts '#disconnect_connection'
+        log_each(:debug, '#disconnect_connection')
         # c.disconnect
       end
 
       def drop_datasets(*dataset_names_to_drop)
         dataset_names_to_drop.each do |dataset_name_to_drop|
-          puts "Dropping dataset #{dataset_name_to_drop.inspect}"
-          dataset_to_drop = @bigquery.dataset(dataset_name_to_drop)
+          log_each(:debug, "Dropping dataset #{dataset_name_to_drop.inspect}")
+          dataset_to_drop = bigquery.dataset(dataset_name_to_drop)
           next unless dataset_to_drop
           dataset_to_drop.tables.each(&:delete)
           dataset_to_drop.delete
@@ -57,7 +52,7 @@ module Sequel
       alias drop_dataset drop_datasets
 
       def execute(sql, opts = OPTS) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        puts '#execute'
+        log_each(:debug, '#execute')
         log_query(sql)
 
         # require 'pry'; binding.pry if sql =~ /CREATE TABLE IF NOT EXISTS/i
@@ -88,15 +83,12 @@ module Sequel
             sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
             conn.query(sql_to_execute)
           end
-          require 'amazing_print'
-          ap results
+          log_each(:debug, results.awesome_inspect)
           if block_given?
             yield results
           else
             results
           end
-        # TODO
-        # rescue ::ODBC::Error, ArgumentError => e
         rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
           raise_error(e)
         end # rubocop:disable Style/MultilineBlockChain
@@ -124,6 +116,33 @@ module Sequel
 
       private
 
+      attr_reader :bigquery_config
+
+      def google_cloud_bigquery_gem_config
+        bigquery_config.dup.tap do |config|
+          %i[
+            adapter
+            database
+            dataset
+            location
+            logger
+          ].each do |option|
+            config.delete(option)
+          end
+        end
+      end
+
+      def get_or_create_bigquery_dataset # rubocop:disable Naming/AccessorMethodName
+        bigquery.dataset(bigquery_dataset_name) || begin
+          log_each(:debug, 'BigQuery dataset %s does not exist; creating it' % bigquery_dataset_name)
+          bigquery.create_dataset(bigquery_dataset_name, location: bigquery_config[:location])
+        end
+      end
+
+      def bigquery_dataset_name
+        bigquery_config[:dataset] || bigquery_config[:database] || (raise ArgumentError, 'BigQuery dataset must be specified')
+      end
+
       def connection_execute_method
         :query
       end
@@ -138,9 +157,9 @@ module Sequel
       end
 
       def schema_parse_table(_table_name, _opts)
-        logger.debug(Paint['schema_parse_table', :red, :bold])
+        log_each(:debug, Paint['schema_parse_table', :red, :bold])
         # require 'pry'; binding.pry
-        @bigquery.datasets.map do |dataset|
+        bigquery.datasets.map do |dataset|
           [
             dataset.dataset_id,
             {},
@@ -155,13 +174,12 @@ module Sequel
 
       # Padded to horizontally align with post-execution log message which includes the execution time
       def log_query(sql)
-        pad = '                                                                '
-        puts Paint[pad + sql, :cyan, :bold]
-        # @loggers[0]&.debug('            ' + sql)
+        pad = ' ' * 12
+        log_each(:debug, Paint[pad + sql, :cyan, :bold])
       end
 
       def warn(msg)
-        @loggers[0].warn(Paint[msg, '#FFA500', :bold])
+        log_each(:warn, Paint[msg, '#FFA500', :bold])
       end
 
       def warn_default_removal(sql)
@@ -195,7 +213,7 @@ module Sequel
 
     class Dataset < Sequel::Dataset
       def fetch_rows(sql, &block)
-        puts '#fetch_rows'
+        db.send(:log_each, :debug, '#fetch_rows')
 
         execute(sql) do |bq_result|
           self.columns = bq_result.fields.map { |field| field.name.to_sym }

data/lib/sequel_bigquery/version.rb

@@ -2,6 +2,6 @@
 
 module Sequel
   module Bigquery
-    VERSION = '0.4.1'
+    VERSION = '0.4.2'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Brendan Weibrecht
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-28 00:00:00.000000000 Z
+date: 2021-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print