sequel-bigquery 0.1.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e110842bf3d9ef623ca308976da97240474929d5598771f16d773e634e5e9981
-  data.tar.gz: 50f4606adfcbd90786cf0ae758fb632564b2456d02144ad8cdc1fd78b6d23eda
+  metadata.gz: 3e8459abb689482e387bc4b0cd7dac2c582a40a9c449d5cd5bad5c8da57fffdb
+  data.tar.gz: 4409fa1b5704c03afcd068915311c4944bdaec1326a21ba6df3ed47d9ea118a2
 SHA512:
-  metadata.gz: 445496592bf4d61fd2caf0365fd1a07635c6e9da41b528aae7dcf847cd42aa4d31522e555a3c350004d958ba82319729e52c6f27f695bb1f96902b9760e4de01
-  data.tar.gz: e792871b3615d5605fbb4d2ea803d9042e3963f5db0778750612775dddc6c0951c0416a1e0cd4f50ec548cafc1440a372d31ffac69e8a18e222923828a120556
+  metadata.gz: 8573f4d5d1f46d63fd062d97efa1bf19ffc7d2e9a3dd24433cdd079554380f7639e2d767657285a9cc1eea829a300c08aa40cfd3a7e24eb9f0f229c0f13e0e8c
+  data.tar.gz: 51371f99aba0a7799a0c8c34f9d3541c86c76113d4acda78badb420375abded01cdd456f184753793505be70f92eba1d9f0a99f35abdd1f766806023f3860979

data/README.md

@@ -1,2 +1,115 @@
 # sequel-bigquery
-A Sequel adapter for Google's BigQuery
+
+[![Gem Version](https://badge.fury.io/rb/sequel-bigquery.svg)](https://rubygems.org/gems/sequel-bigquery)
+
+A Sequel adapter for [Google's BigQuery](https://cloud.google.com/bigquery).
+
+## Contents
+
+<!-- MarkdownTOC autolink=true -->
+
+- [Intro](#intro)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Contributing](#contributing)
+- [Development](#development)
+  - [Pre-push hook](#pre-push-hook)
+  - [Release](#release)
+
+<!-- /MarkdownTOC -->
+
+## Intro
+
+**Be warned: Given I was unable to find Sequel documentation covering how to write a database adapter, this was put together by reading Sequel's source and hacking at things until they worked. There are probably a lot of rough edges.**
+
+Features:
+
+- Connecting
+- Migrating
+- Table creation, with automatic removal of defaults from statements (since BigQuery doesn't support it)
+- Inserting rows
+- Updating rows, with automatic addition of `where 1 = 1` to statements (since BigQuery requires a `where` clause)
+- Querying
+- Transactions (buffered since BigQuery only supports them when you execute the whole transaction at once)
+- Table partitioning
+- Ruby types:
+  + String
+  + Integer
+  + _Boolean_ (`TrueClass`/`FalseClass`)
+  + DateTime (note that BigQuery does not persist timezone)
+  + Date
+  + Float
+  + BigDecimal
+- Selecting the BigQuery server location
+
+## Installation
+
+Add it to the `Gemfile` of your project:
+
+```ruby
+gem 'sequel-bigquery'
+```
+
+and install all your gems:
+
+```bash
+bundle install
+```
+
+Or you can install it to your system directly using:
+
+```bash
+gem install sequel-bigquery
+```
+
+## Usage
+
+Connect to BigQuery:
+
+```
+require 'sequel-bigquery'
+
+db = Sequel.connect(
+  adapter: :bigquery,
+  project: 'your-gcp-project',
+  database: 'your_bigquery_dataset_name',
+  location: 'australia-southeast2',
+  logger: Logger.new(STDOUT),
+)
+```
+
+And use Sequel like normal.
+
+## Contributing
+
+Pull requests welcome! =)
+
+## Development
+
+### Pre-push hook
+
+This hook runs style checks and tests.
+
+To set up the pre-push hook:
+
+```bash
+echo -e "#\!/bin/bash\n\$(dirname \$0)/../../auto/pre-push-hook" > .git/hooks/pre-push
+chmod +x .git/hooks/pre-push
+```
+
+### Release
+
+To release a new version:
+
+```bash
+auto/release/update-version && auto/release/tag && auto/release/publish
+```
+
+This takes care of the whole process:
+
+- Incrementing the version number (the patch number by default)
+- Tagging & pushing commits
+- Publishing the gem to RubyGems
+- Creating a draft GitHub release
+
+To increment the minor or major versions instead of the patch number, run `auto/release/update-version` with `--minor` or `--major`.

data/lib/sequel-bigquery.rb

@@ -11,9 +11,9 @@ module Sequel
   module Bigquery
     # Contains procs keyed on subadapter type that extend the
     # given database object so it supports the correct database type.
-    DATABASE_SETUP = {}
-      
-    class Database < Sequel::Database
+    DATABASE_SETUP = {}.freeze
+
+    class Database < Sequel::Database # rubocop:disable Metrics/ClassLength
       set_adapter_scheme :bigquery
 
       def initialize(*args, **kawrgs)
@@ -26,28 +26,36 @@ module Sequel
 
       def connect(*_args)
         puts '#connect'
-        # self.input_identifier_meth = nil
-        # self.identifier_output_method = nil
-
         config = @orig_opts.dup
         config.delete(:adapter)
         config.delete(:logger)
+        location = config.delete(:location)
         bq_dataset_name = config.delete(:dataset) || config.delete(:database)
         @bigquery = Google::Cloud::Bigquery.new(config)
         # ObjectSpace.each_object(HTTPClient).each { |c| c.debug_dev = STDOUT }
         @bigquery.dataset(bq_dataset_name) || begin
           @loggers[0].debug('BigQuery dataset %s does not exist; creating it' % bq_dataset_name)
-          @bigquery.create_dataset(bq_dataset_name)
+          @bigquery.create_dataset(bq_dataset_name, location: location)
         end
           .tap { puts '#connect end' }
       end
 
-      def disconnect_connection(c)
+      def disconnect_connection(_c)
         puts '#disconnect_connection'
         # c.disconnect
       end
 
-      def execute(sql, opts=OPTS)
+      def drop_datasets(*dataset_names_to_drop)
+        dataset_names_to_drop.each do |dataset_name_to_drop|
+          puts "Dropping dataset #{dataset_name_to_drop.inspect}"
+          dataset_to_drop = @bigquery.dataset(dataset_name_to_drop)
+          dataset_to_drop.tables.each(&:delete)
+          dataset_to_drop.delete
+        end
+      end
+      alias drop_dataset drop_datasets
+
+      def execute(sql, opts = OPTS) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         puts '#execute'
         log_query(sql)
 
@@ -60,44 +68,37 @@ module Sequel
 
         if sql =~ /^update/i && sql !~ / where /i
           warn("Warning: Appended 'where 1 = 1' to query since BigQuery requires UPDATE statements to include a WHERE clause")
-          sql = sql + ' where 1 = 1'
+          sql += ' where 1 = 1'
         end
 
-        if sql =~ /^begin/i
+        if /^begin/i.match?(sql)
           warn_transaction
           @sql_buffering = true
         end
 
         if @sql_buffering
           @sql_buffer << sql
-          if sql =~ /^commit/i
-            warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
-          else
-            return []
-          end
+          return [] unless /^commit/i.match?(sql)
+          warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
         end
 
         synchronize(opts[:server]) do |conn|
-          begin
-            results = log_connection_yield(sql, conn) do
-              sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
-              conn.query(sql_to_execute)
-              # raw_result = conn.query(sql_to_execute)
-              # BQResult.new(raw_result)
-            end
-            require 'amazing_print'
-            ap results
-            if block_given?
-              yield results
-            else
-              results
-            end
-          # TODO
-          # rescue ::ODBC::Error, ArgumentError => e
-          rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
-            raise_error(e)
+          results = log_connection_yield(sql, conn) do
+            sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
+            conn.query(sql_to_execute)
           end
-        end
+          require 'amazing_print'
+          ap results
+          if block_given?
+            yield results
+          else
+            results
+          end
+        # TODO
+        # rescue ::ODBC::Error, ArgumentError => e
+        rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
+          raise_error(e)
+        end # rubocop:disable Style/MultilineBlockChain
           .tap do
             @sql_buffer = []
             @sql_buffering = false
@@ -116,34 +117,11 @@ module Sequel
         end
       end
 
-      # def supports_transactional_ddl?
-      #   false
-      # end
-      
-      # def execute_dui(sql, opts=OPTS)
-      # end
-
-      # def execute_dui(sql, opts=OPTS)
-      #   # require 'pry'; binding.pry
-      #   synchronize(opts[:server]) do |conn|
-      #     begin
-      #       log_connection_yield(sql, conn){conn.do(sql)}
-      #     # TODO:
-      #     # rescue ::ODBC::Error, ArgumentError => e
-      #     rescue ArgumentError => e
-      #       raise_error(e)
-      #     end
-      #   end
-      # end
+      def type_literal_generic_float(_column)
+        :float64
+      end
 
       private
-      
-      def adapter_initialize
-        puts '#adapter_initialize'
-        self.extension(:identifier_mangling)
-        self.identifier_input_method = nil
-        self.quote_identifiers = false
-      end
 
       def connection_execute_method
         :query
@@ -158,18 +136,18 @@ module Sequel
         Dataset
       end
 
-      def schema_parse_table(table_name, opts)
+      def schema_parse_table(_table_name, _opts)
         logger.debug(Paint['schema_parse_table', :red, :bold])
         # require 'pry'; binding.pry
         @bigquery.datasets.map do |dataset|
           [
             dataset.dataset_id,
-            {}
+            {},
           ]
         end
       end
 
-      def disconnect_error?(e, opts)
+      def disconnect_error?(e, opts) # rubocop:disable Lint/UselessMethodDefinition
         # super || (e.is_a?(::ODBC::Error) && /\A08S01/.match(e.message))
         super
       end
@@ -190,87 +168,64 @@ module Sequel
       end
 
       def warn_transaction
-        warn('Warning: Transaction detected. This only supported on BigQuery in a script or session. Commencing buffering to run the whole transaction at once as a script upon commit. Note that no result data is returned while the transaction is open.')
+        warn(
+          'Warning: Transaction detected. This only supported on BigQuery in a script or session. '\
+          'Commencing buffering to run the whole transaction at once as a script upon commit. ' \
+          'Note that no result data is returned while the transaction is open.',
+        )
       end
-    end
 
-    # class BQResult < SimpleDelegator
+      # SQL for creating a table with BigQuery specific options
+      def create_table_sql(name, generator, options)
+        "#{super}#{create_table_suffix_sql(name, options)}"
+      end
+
+      # Handle BigQuery specific table extensions (i.e. partitioning)
+      def create_table_suffix_sql(_name, options)
+        sql = +''
+
+        if (partition_by = options[:partition_by])
+          sql << " PARTITION BY #{literal(Array(partition_by))}"
+        end
+
+        sql
+      end
+    end
 
-    # end
-    
     class Dataset < Sequel::Dataset
-      def fetch_rows(sql)
+      def fetch_rows(sql, &block)
         puts '#fetch_rows'
-        # execute(sql) do |s|
-        #   i = -1
-        #   cols = s.columns(true).map{|c| [output_identifier(c.name), c.type, i+=1]}
-        #   columns = cols.map{|c| c[0]}
-        #   self.columns = columns
-        #   s.each do |row|
-        #     hash = {}
-        #     cols.each{|n,t,j| hash[n] = convert_odbc_value(row[j], t)}
-        #     yield hash
-        #   end
-        # end
-        # self
 
         execute(sql) do |bq_result|
           self.columns = bq_result.fields.map { |field| field.name.to_sym }
-          bq_result.each do |row|
-            yield row
-          end
+          bq_result.each(&block)
         end
 
-        # execute(sql).each do |row|
-        #   yield row
-        # end
         self
       end
 
-      # def columns
-      #   fields.map { |field| field.name.to_sym }
-      # end
-      
       private
 
-      # def convert_odbc_value(v, t)
-      #   # When fetching a result set, the Ruby ODBC driver converts all ODBC
-      #   # SQL types to an equivalent Ruby type; with the exception of
-      #   # SQL_TYPE_DATE, SQL_TYPE_TIME and SQL_TYPE_TIMESTAMP.
-      #   #
-      #   # The conversions below are consistent with the mappings in
-      #   # ODBCColumn#mapSqlTypeToGenericType and Column#klass.
-      #   case v
-      #   when ::ODBC::TimeStamp
-      #     db.to_application_timestamp([v.year, v.month, v.day, v.hour, v.minute, v.second, v.fraction])
-      #   when ::ODBC::Time
-      #     Sequel::SQLTime.create(v.hour, v.minute, v.second)
-      #   when ::ODBC::Date
-      #     Date.new(v.year, v.month, v.day)
-      #   else
-      #     if t == ::ODBC::SQL_BIT
-      #       v == 1
-      #     else
-      #       v
-      #     end
-      #   end
-      # end
-
       def literal_time(v)
         "'#{v.iso8601}'"
       end
 
-      # def literal_date(v)
-      #   v.strftime("{d '%Y-%m-%d'}")
-      # end
-      
       def literal_false
         'false'
       end
-      
+
       def literal_true
         'true'
       end
+
+      # Like MySQL, BigQuery uses the nonstandard ` (backtick) for quoting identifiers.
+      def quoted_identifier_append(sql, c)
+        sql << '`%s`' % c
+      end
+
+      def input_identifier(v)
+        v.to_s
+      end
     end
   end
 end

data/lib/sequel_bigquery/version.rb

@@ -2,6 +2,6 @@
 
 module Sequel
   module Bigquery
-    VERSION = '0.1.0'
+    VERSION = '0.4.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Brendan Weibrecht
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-08 00:00:00.000000000 Z
+date: 2021-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -98,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.16
 signing_key:
 specification_version: 4
 summary: A Sequel adapter for Google's BigQuery