sequel-bigquery 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e110842bf3d9ef623ca308976da97240474929d5598771f16d773e634e5e9981
+  data.tar.gz: 50f4606adfcbd90786cf0ae758fb632564b2456d02144ad8cdc1fd78b6d23eda
+SHA512:
+  metadata.gz: 445496592bf4d61fd2caf0365fd1a07635c6e9da41b528aae7dcf847cd42aa4d31522e555a3c350004d958ba82319729e52c6f27f695bb1f96902b9760e4de01
+  data.tar.gz: e792871b3615d5605fbb4d2ea803d9042e3963f5db0778750612775dddc6c0951c0416a1e0cd4f50ec548cafc1440a372d31ffac69e8a18e222923828a120556

data/README.md

@@ -0,0 +1,2 @@
+# sequel-bigquery
+A Sequel adapter for Google's BigQuery

data/lib/sequel-bigquery.rb

@@ -0,0 +1,276 @@
+# frozen-string-literal: true
+
+require 'delegate'
+require 'time'
+
+require 'google/cloud/bigquery'
+require 'paint'
+require 'sequel'
+
+module Sequel
+  module Bigquery
+    # Contains procs keyed on subadapter type that extend the
+    # given database object so it supports the correct database type.
+    DATABASE_SETUP = {}
+      
+    class Database < Sequel::Database
+      set_adapter_scheme :bigquery
+
+      def initialize(*args, **kawrgs)
+        puts '.new'
+        @orig_opts = kawrgs.fetch(:orig_opts)
+        @sql_buffer = []
+        @sql_buffering = false
+        super
+      end
+
+      def connect(*_args)
+        puts '#connect'
+        # self.input_identifier_meth = nil
+        # self.identifier_output_method = nil
+
+        config = @orig_opts.dup
+        config.delete(:adapter)
+        config.delete(:logger)
+        bq_dataset_name = config.delete(:dataset) || config.delete(:database)
+        @bigquery = Google::Cloud::Bigquery.new(config)
+        # ObjectSpace.each_object(HTTPClient).each { |c| c.debug_dev = STDOUT }
+        @bigquery.dataset(bq_dataset_name) || begin
+          @loggers[0].debug('BigQuery dataset %s does not exist; creating it' % bq_dataset_name)
+          @bigquery.create_dataset(bq_dataset_name)
+        end
+          .tap { puts '#connect end' }
+      end
+
+      def disconnect_connection(c)
+        puts '#disconnect_connection'
+        # c.disconnect
+      end
+
+      def execute(sql, opts=OPTS)
+        puts '#execute'
+        log_query(sql)
+
+        # require 'pry'; binding.pry if sql =~ /CREATE TABLE IF NOT EXISTS/i
+
+        sql = sql.gsub(/\sdefault \S+/i) do
+          warn_default_removal(sql)
+          ''
+        end
+
+        if sql =~ /^update/i && sql !~ / where /i
+          warn("Warning: Appended 'where 1 = 1' to query since BigQuery requires UPDATE statements to include a WHERE clause")
+          sql = sql + ' where 1 = 1'
+        end
+
+        if sql =~ /^begin/i
+          warn_transaction
+          @sql_buffering = true
+        end
+
+        if @sql_buffering
+          @sql_buffer << sql
+          if sql =~ /^commit/i
+            warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
+          else
+            return []
+          end
+        end
+
+        synchronize(opts[:server]) do |conn|
+          begin
+            results = log_connection_yield(sql, conn) do
+              sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
+              conn.query(sql_to_execute)
+              # raw_result = conn.query(sql_to_execute)
+              # BQResult.new(raw_result)
+            end
+            require 'amazing_print'
+            ap results
+            if block_given?
+              yield results
+            else
+              results
+            end
+          # TODO
+          # rescue ::ODBC::Error, ArgumentError => e
+          rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
+            raise_error(e)
+          end
+        end
+          .tap do
+            @sql_buffer = []
+            @sql_buffering = false
+          end
+      end
+
+      def supports_create_table_if_not_exists?
+        true
+      end
+
+      def type_literal_generic_string(column)
+        if column[:size]
+          "string(#{column[:size]})"
+        else
+          :string
+        end
+      end
+
+      # def supports_transactional_ddl?
+      #   false
+      # end
+      
+      # def execute_dui(sql, opts=OPTS)
+      # end
+
+      # def execute_dui(sql, opts=OPTS)
+      #   # require 'pry'; binding.pry
+      #   synchronize(opts[:server]) do |conn|
+      #     begin
+      #       log_connection_yield(sql, conn){conn.do(sql)}
+      #     # TODO:
+      #     # rescue ::ODBC::Error, ArgumentError => e
+      #     rescue ArgumentError => e
+      #       raise_error(e)
+      #     end
+      #   end
+      # end
+
+      private
+      
+      def adapter_initialize
+        puts '#adapter_initialize'
+        self.extension(:identifier_mangling)
+        self.identifier_input_method = nil
+        self.quote_identifiers = false
+      end
+
+      def connection_execute_method
+        :query
+      end
+
+      def database_error_classes
+        # [::ODBC::Error]
+        # TODO
+      end
+
+      def dataset_class_default
+        Dataset
+      end
+
+      def schema_parse_table(table_name, opts)
+        logger.debug(Paint['schema_parse_table', :red, :bold])
+        # require 'pry'; binding.pry
+        @bigquery.datasets.map do |dataset|
+          [
+            dataset.dataset_id,
+            {}
+          ]
+        end
+      end
+
+      def disconnect_error?(e, opts)
+        # super || (e.is_a?(::ODBC::Error) && /\A08S01/.match(e.message))
+        super
+      end
+
+      # Padded to horizontally align with post-execution log message which includes the execution time
+      def log_query(sql)
+        pad = '                                                                '
+        puts Paint[pad + sql, :cyan, :bold]
+        # @loggers[0]&.debug('            ' + sql)
+      end
+
+      def warn(msg)
+        @loggers[0].warn(Paint[msg, '#FFA500', :bold])
+      end
+
+      def warn_default_removal(sql)
+        warn("Warning: Default removed from below query as it's not supported on BigQuery:\n%s" % sql)
+      end
+
+      def warn_transaction
+        warn('Warning: Transaction detected. This only supported on BigQuery in a script or session. Commencing buffering to run the whole transaction at once as a script upon commit. Note that no result data is returned while the transaction is open.')
+      end
+    end
+
+    # class BQResult < SimpleDelegator
+
+    # end
+    
+    class Dataset < Sequel::Dataset
+      def fetch_rows(sql)
+        puts '#fetch_rows'
+        # execute(sql) do |s|
+        #   i = -1
+        #   cols = s.columns(true).map{|c| [output_identifier(c.name), c.type, i+=1]}
+        #   columns = cols.map{|c| c[0]}
+        #   self.columns = columns
+        #   s.each do |row|
+        #     hash = {}
+        #     cols.each{|n,t,j| hash[n] = convert_odbc_value(row[j], t)}
+        #     yield hash
+        #   end
+        # end
+        # self
+
+        execute(sql) do |bq_result|
+          self.columns = bq_result.fields.map { |field| field.name.to_sym }
+          bq_result.each do |row|
+            yield row
+          end
+        end
+
+        # execute(sql).each do |row|
+        #   yield row
+        # end
+        self
+      end
+
+      # def columns
+      #   fields.map { |field| field.name.to_sym }
+      # end
+      
+      private
+
+      # def convert_odbc_value(v, t)
+      #   # When fetching a result set, the Ruby ODBC driver converts all ODBC
+      #   # SQL types to an equivalent Ruby type; with the exception of
+      #   # SQL_TYPE_DATE, SQL_TYPE_TIME and SQL_TYPE_TIMESTAMP.
+      #   #
+      #   # The conversions below are consistent with the mappings in
+      #   # ODBCColumn#mapSqlTypeToGenericType and Column#klass.
+      #   case v
+      #   when ::ODBC::TimeStamp
+      #     db.to_application_timestamp([v.year, v.month, v.day, v.hour, v.minute, v.second, v.fraction])
+      #   when ::ODBC::Time
+      #     Sequel::SQLTime.create(v.hour, v.minute, v.second)
+      #   when ::ODBC::Date
+      #     Date.new(v.year, v.month, v.day)
+      #   else
+      #     if t == ::ODBC::SQL_BIT
+      #       v == 1
+      #     else
+      #       v
+      #     end
+      #   end
+      # end
+
+      def literal_time(v)
+        "'#{v.iso8601}'"
+      end
+
+      # def literal_date(v)
+      #   v.strftime("{d '%Y-%m-%d'}")
+      # end
+      
+      def literal_false
+        'false'
+      end
+      
+      def literal_true
+        'true'
+      end
+    end
+  end
+end

data/lib/sequel_bigquery/version.rb

@@ -0,0 +1,7 @@
+# frozen-string-literal: true
+
+module Sequel
+  module Bigquery
+    VERSION = '0.1.0'
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: sequel-bigquery
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Brendan Weibrecht
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-09-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: amazing_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: google-cloud-bigquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.35'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.35'
+- !ruby/object:Gem::Dependency
+  name: paint
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.48'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.48'
+description:
+email:
+- brendan@weibrecht.net.au
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/sequel-bigquery.rb
+- lib/sequel_bigquery/version.rb
+homepage: https://github.com/ZimbiX/sequel-bigquery
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/ZimbiX/sequel-bigquery
+  source_code_uri: https://github.com/ZimbiX/sequel-bigquery
+  changelog_uri: https://github.com/ZimbiX/sequel-bigquery/releases
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: A Sequel adapter for Google's BigQuery
+test_files: []