sequel-bigquery 0.2.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13d55b1e469dca990e67e526e1f9774c0e86ce9ddee8619531c6f5ad3c671f7b
-  data.tar.gz: 10b66d4878ca799922ae38e2c847b5d4a29ade0d1b65be098744210bc9c12d51
+  metadata.gz: e444c2d4a5c6b54b5a1bb0efde17e1dd83761ec53f1b796f5e959646bd37f315
+  data.tar.gz: 41875a2325c00eb26786fbd3ab8430e9880b93d795a18b000bff3dd75ed0ea47
 SHA512:
-  metadata.gz: 2004fdac1976b084a361524147411451f514810bf1d8c0ff50dc8f2161bb63d620a79357a70a5b761cffd8995f9b144023f64cefb40330b36a4961844b6aefbd
-  data.tar.gz: b131f9e89bea9e6839c65220961e9d7b157b5baf6cb7761f26872d7c35750a9995d115e99454830cd90b2fe8df705b0ee90e05c8de0ce0d115ef1ff92ad8340f
+  metadata.gz: 725fce3447e58e75c5cf817d736322c567661752d1b2d2f4f45aa0ce712825e8792214d1f56313272d8a84facdadff51f63ee4a6b3bda1b8c55d92a80a826798
+  data.tar.gz: 3aa8900e8fba7400049e8f83879998b246296e1705d01b25f2a1cf20855cf8faeea09b9d7e4cb7280b2783296d0a27f61ed295275089d6de311c256c5f7ea763

data/README.md

@@ -4,6 +4,10 @@
 
 A Sequel adapter for [Google's BigQuery](https://cloud.google.com/bigquery).
 
+This gem was created in order to manage schema migrations of a BigQuery dataset at GreenSync. At the time of writing, we couldn't find any good tools in any language to manage changes to the schema as a set of migrations.
+
+Beyond migrations, I'm unsure how useful this gem is. I haven't yet tested what the performance would be for data interactions vs. directly using the `google-cloud-bigquery` gem's native facilities. If you're inserting a bunch of data, it's probably a better idea to use an [inserter from that gem](https://googleapis.dev/ruby/google-cloud-bigquery/latest/Google/Cloud/Bigquery/Dataset.html#insert_async-instance_method) rather than going through SQL.
+
 ## Contents
 
 <!-- MarkdownTOC autolink=true -->
@@ -31,6 +35,7 @@ Features:
 - Updating rows, with automatic addition of `where 1 = 1` to statements (since BigQuery requires a `where` clause)
 - Querying
 - Transactions (buffered since BigQuery only supports them when you execute the whole transaction at once)
+- Table partitioning
 - Ruby types:
   + String
   + Integer
@@ -39,6 +44,7 @@ Features:
   + Date
   + Float
   + BigDecimal
+- Selecting the BigQuery server location
 
 ## Installation
 
@@ -66,17 +72,21 @@ Connect to BigQuery:
 
 ```
 require 'sequel-bigquery'
+require 'logger'
 
 db = Sequel.connect(
   adapter: :bigquery,
   project: 'your-gcp-project',
   database: 'your_bigquery_dataset_name',
+  location: 'australia-southeast2',
   logger: Logger.new(STDOUT),
 )
 ```
 
 And use Sequel like normal.
 
+Note that it is important to supply a logger that will at least output warning messages so you know when your queries are being modifed or buffered, which may be unexpected behaviour.
+
 ## Contributing
 
 Pull requests welcome! =)

data/lib/sequel-bigquery.rb

@@ -4,6 +4,7 @@ require 'delegate'
 require 'time'
 
 require 'google/cloud/bigquery'
+require 'amazing_print'
 require 'paint'
 require 'sequel'
 
@@ -16,38 +17,34 @@ module Sequel
     class Database < Sequel::Database # rubocop:disable Metrics/ClassLength
       set_adapter_scheme :bigquery
 
-      def initialize(*args, **kawrgs)
-        puts '.new'
-        @orig_opts = kawrgs.fetch(:orig_opts)
+      def initialize(*args, **kwargs)
+        @bigquery_config = kwargs.fetch(:orig_opts)
         @sql_buffer = []
         @sql_buffering = false
         super
       end
 
       def connect(*_args)
-        puts '#connect'
-        config = @orig_opts.dup
-        config.delete(:adapter)
-        config.delete(:logger)
-        bq_dataset_name = config.delete(:dataset) || config.delete(:database)
-        @bigquery = Google::Cloud::Bigquery.new(config)
+        log_each(:debug, '#connect')
+        get_or_create_bigquery_dataset
+          .tap { log_each(:debug, '#connect end') }
+      end
+
+      def bigquery
         # ObjectSpace.each_object(HTTPClient).each { |c| c.debug_dev = STDOUT }
-        @bigquery.dataset(bq_dataset_name) || begin
-          @loggers[0].debug('BigQuery dataset %s does not exist; creating it' % bq_dataset_name)
-          @bigquery.create_dataset(bq_dataset_name)
-        end
-          .tap { puts '#connect end' }
+        @bigquery ||= Google::Cloud::Bigquery.new(google_cloud_bigquery_gem_config)
       end
 
       def disconnect_connection(_c)
-        puts '#disconnect_connection'
+        log_each(:debug, '#disconnect_connection')
         # c.disconnect
       end
 
       def drop_datasets(*dataset_names_to_drop)
         dataset_names_to_drop.each do |dataset_name_to_drop|
-          puts "Dropping dataset #{dataset_name_to_drop.inspect}"
-          dataset_to_drop = @bigquery.dataset(dataset_name_to_drop)
+          log_each(:debug, "Dropping dataset #{dataset_name_to_drop.inspect}")
+          dataset_to_drop = bigquery.dataset(dataset_name_to_drop)
+          next unless dataset_to_drop
           dataset_to_drop.tables.each(&:delete)
           dataset_to_drop.delete
         end
@@ -55,7 +52,7 @@ module Sequel
       alias drop_dataset drop_datasets
 
       def execute(sql, opts = OPTS) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        puts '#execute'
+        log_each(:debug, '#execute')
         log_query(sql)
 
         # require 'pry'; binding.pry if sql =~ /CREATE TABLE IF NOT EXISTS/i
@@ -86,15 +83,12 @@ module Sequel
             sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
             conn.query(sql_to_execute)
           end
-          require 'amazing_print'
-          ap results
+          log_each(:debug, results.awesome_inspect)
           if block_given?
             yield results
           else
             results
           end
-        # TODO
-        # rescue ::ODBC::Error, ArgumentError => e
         rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
           raise_error(e)
         end # rubocop:disable Style/MultilineBlockChain
@@ -122,6 +116,33 @@ module Sequel
 
       private
 
+      attr_reader :bigquery_config
+
+      def google_cloud_bigquery_gem_config
+        bigquery_config.dup.tap do |config|
+          %i[
+            adapter
+            database
+            dataset
+            location
+            logger
+          ].each do |option|
+            config.delete(option)
+          end
+        end
+      end
+
+      def get_or_create_bigquery_dataset # rubocop:disable Naming/AccessorMethodName
+        bigquery.dataset(bigquery_dataset_name) || begin
+          log_each(:debug, 'BigQuery dataset %s does not exist; creating it' % bigquery_dataset_name)
+          bigquery.create_dataset(bigquery_dataset_name, location: bigquery_config[:location])
+        end
+      end
+
+      def bigquery_dataset_name
+        bigquery_config[:dataset] || bigquery_config[:database] || (raise ArgumentError, 'BigQuery dataset must be specified')
+      end
+
       def connection_execute_method
         :query
       end
@@ -136,9 +157,9 @@ module Sequel
       end
 
       def schema_parse_table(_table_name, _opts)
-        logger.debug(Paint['schema_parse_table', :red, :bold])
+        log_each(:debug, Paint['schema_parse_table', :red, :bold])
         # require 'pry'; binding.pry
-        @bigquery.datasets.map do |dataset|
+        bigquery.datasets.map do |dataset|
           [
             dataset.dataset_id,
             {},
@@ -153,13 +174,12 @@ module Sequel
 
       # Padded to horizontally align with post-execution log message which includes the execution time
       def log_query(sql)
-        pad = '                                                                '
-        puts Paint[pad + sql, :cyan, :bold]
-        # @loggers[0]&.debug('            ' + sql)
+        pad = ' ' * 12
+        log_each(:debug, Paint[pad + sql, :cyan, :bold])
       end
 
       def warn(msg)
-        @loggers[0].warn(Paint[msg, '#FFA500', :bold])
+        log_each(:warn, Paint[msg, '#FFA500', :bold])
       end
 
       def warn_default_removal(sql)
@@ -173,11 +193,27 @@ module Sequel
           'Note that no result data is returned while the transaction is open.',
         )
       end
+
+      # SQL for creating a table with BigQuery specific options
+      def create_table_sql(name, generator, options)
+        "#{super}#{create_table_suffix_sql(name, options)}"
+      end
+
+      # Handle BigQuery specific table extensions (i.e. partitioning)
+      def create_table_suffix_sql(_name, options)
+        sql = +''
+
+        if (partition_by = options[:partition_by])
+          sql << " PARTITION BY #{literal(Array(partition_by))}"
+        end
+
+        sql
+      end
     end
 
     class Dataset < Sequel::Dataset
       def fetch_rows(sql, &block)
-        puts '#fetch_rows'
+        db.send(:log_each, :debug, '#fetch_rows')
 
         execute(sql) do |bq_result|
           self.columns = bq_result.fields.map { |field| field.name.to_sym }
@@ -203,7 +239,7 @@ module Sequel
 
       # Like MySQL, BigQuery uses the nonstandard ` (backtick) for quoting identifiers.
       def quoted_identifier_append(sql, c)
-        sql << '`%s`' % c
+        sql << ('`%s`' % c)
       end
 
       def input_identifier(v)

data/lib/sequel_bigquery/version.rb

@@ -2,6 +2,6 @@
 
 module Sequel
   module Bigquery
-    VERSION = '0.2.0'
+    VERSION = '0.4.2'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Brendan Weibrecht
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-26 00:00:00.000000000 Z
+date: 2021-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print