sequel-bigquery 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: e110842bf3d9ef623ca308976da97240474929d5598771f16d773e634e5e9981
4
+ data.tar.gz: 50f4606adfcbd90786cf0ae758fb632564b2456d02144ad8cdc1fd78b6d23eda
5
+ SHA512:
6
+ metadata.gz: 445496592bf4d61fd2caf0365fd1a07635c6e9da41b528aae7dcf847cd42aa4d31522e555a3c350004d958ba82319729e52c6f27f695bb1f96902b9760e4de01
7
+ data.tar.gz: e792871b3615d5605fbb4d2ea803d9042e3963f5db0778750612775dddc6c0951c0416a1e0cd4f50ec548cafc1440a372d31ffac69e8a18e222923828a120556
data/README.md ADDED
@@ -0,0 +1,2 @@
1
+ # sequel-bigquery
2
+ A Sequel adapter for Google's BigQuery
@@ -0,0 +1,276 @@
1
+ # frozen-string-literal: true
2
+
3
+ require 'delegate'
4
+ require 'time'
5
+
6
+ require 'google/cloud/bigquery'
7
+ require 'paint'
8
+ require 'sequel'
9
+
10
+ module Sequel
11
+ module Bigquery
12
+ # Contains procs keyed on subadapter type that extend the
13
+ # given database object so it supports the correct database type.
14
+ DATABASE_SETUP = {}
15
+
16
+ class Database < Sequel::Database
17
+ set_adapter_scheme :bigquery
18
+
19
+ def initialize(*args, **kawrgs)
20
+ puts '.new'
21
+ @orig_opts = kawrgs.fetch(:orig_opts)
22
+ @sql_buffer = []
23
+ @sql_buffering = false
24
+ super
25
+ end
26
+
27
+ def connect(*_args)
28
+ puts '#connect'
29
+ # self.input_identifier_meth = nil
30
+ # self.identifier_output_method = nil
31
+
32
+ config = @orig_opts.dup
33
+ config.delete(:adapter)
34
+ config.delete(:logger)
35
+ bq_dataset_name = config.delete(:dataset) || config.delete(:database)
36
+ @bigquery = Google::Cloud::Bigquery.new(config)
37
+ # ObjectSpace.each_object(HTTPClient).each { |c| c.debug_dev = STDOUT }
38
+ @bigquery.dataset(bq_dataset_name) || begin
39
+ @loggers[0].debug('BigQuery dataset %s does not exist; creating it' % bq_dataset_name)
40
+ @bigquery.create_dataset(bq_dataset_name)
41
+ end
42
+ .tap { puts '#connect end' }
43
+ end
44
+
45
+ def disconnect_connection(c)
46
+ puts '#disconnect_connection'
47
+ # c.disconnect
48
+ end
49
+
50
+ def execute(sql, opts=OPTS)
51
+ puts '#execute'
52
+ log_query(sql)
53
+
54
+ # require 'pry'; binding.pry if sql =~ /CREATE TABLE IF NOT EXISTS/i
55
+
56
+ sql = sql.gsub(/\sdefault \S+/i) do
57
+ warn_default_removal(sql)
58
+ ''
59
+ end
60
+
61
+ if sql =~ /^update/i && sql !~ / where /i
62
+ warn("Warning: Appended 'where 1 = 1' to query since BigQuery requires UPDATE statements to include a WHERE clause")
63
+ sql = sql + ' where 1 = 1'
64
+ end
65
+
66
+ if sql =~ /^begin/i
67
+ warn_transaction
68
+ @sql_buffering = true
69
+ end
70
+
71
+ if @sql_buffering
72
+ @sql_buffer << sql
73
+ if sql =~ /^commit/i
74
+ warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
75
+ else
76
+ return []
77
+ end
78
+ end
79
+
80
+ synchronize(opts[:server]) do |conn|
81
+ begin
82
+ results = log_connection_yield(sql, conn) do
83
+ sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
84
+ conn.query(sql_to_execute)
85
+ # raw_result = conn.query(sql_to_execute)
86
+ # BQResult.new(raw_result)
87
+ end
88
+ require 'amazing_print'
89
+ ap results
90
+ if block_given?
91
+ yield results
92
+ else
93
+ results
94
+ end
95
+ # TODO
96
+ # rescue ::ODBC::Error, ArgumentError => e
97
+ rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
98
+ raise_error(e)
99
+ end
100
+ end
101
+ .tap do
102
+ @sql_buffer = []
103
+ @sql_buffering = false
104
+ end
105
+ end
106
+
107
+ def supports_create_table_if_not_exists?
108
+ true
109
+ end
110
+
111
+ def type_literal_generic_string(column)
112
+ if column[:size]
113
+ "string(#{column[:size]})"
114
+ else
115
+ :string
116
+ end
117
+ end
118
+
119
+ # def supports_transactional_ddl?
120
+ # false
121
+ # end
122
+
123
+ # def execute_dui(sql, opts=OPTS)
124
+ # end
125
+
126
+ # def execute_dui(sql, opts=OPTS)
127
+ # # require 'pry'; binding.pry
128
+ # synchronize(opts[:server]) do |conn|
129
+ # begin
130
+ # log_connection_yield(sql, conn){conn.do(sql)}
131
+ # # TODO:
132
+ # # rescue ::ODBC::Error, ArgumentError => e
133
+ # rescue ArgumentError => e
134
+ # raise_error(e)
135
+ # end
136
+ # end
137
+ # end
138
+
139
+ private
140
+
141
+ def adapter_initialize
142
+ puts '#adapter_initialize'
143
+ self.extension(:identifier_mangling)
144
+ self.identifier_input_method = nil
145
+ self.quote_identifiers = false
146
+ end
147
+
148
+ def connection_execute_method
149
+ :query
150
+ end
151
+
152
+ def database_error_classes
153
+ # [::ODBC::Error]
154
+ # TODO
155
+ end
156
+
157
+ def dataset_class_default
158
+ Dataset
159
+ end
160
+
161
+ def schema_parse_table(table_name, opts)
162
+ logger.debug(Paint['schema_parse_table', :red, :bold])
163
+ # require 'pry'; binding.pry
164
+ @bigquery.datasets.map do |dataset|
165
+ [
166
+ dataset.dataset_id,
167
+ {}
168
+ ]
169
+ end
170
+ end
171
+
172
+ def disconnect_error?(e, opts)
173
+ # super || (e.is_a?(::ODBC::Error) && /\A08S01/.match(e.message))
174
+ super
175
+ end
176
+
177
+ # Padded to horizontally align with post-execution log message which includes the execution time
178
+ def log_query(sql)
179
+ pad = ' '
180
+ puts Paint[pad + sql, :cyan, :bold]
181
+ # @loggers[0]&.debug(' ' + sql)
182
+ end
183
+
184
+ def warn(msg)
185
+ @loggers[0].warn(Paint[msg, '#FFA500', :bold])
186
+ end
187
+
188
+ def warn_default_removal(sql)
189
+ warn("Warning: Default removed from below query as it's not supported on BigQuery:\n%s" % sql)
190
+ end
191
+
192
+ def warn_transaction
193
+ warn('Warning: Transaction detected. This only supported on BigQuery in a script or session. Commencing buffering to run the whole transaction at once as a script upon commit. Note that no result data is returned while the transaction is open.')
194
+ end
195
+ end
196
+
197
+ # class BQResult < SimpleDelegator
198
+
199
+ # end
200
+
201
+ class Dataset < Sequel::Dataset
202
+ def fetch_rows(sql)
203
+ puts '#fetch_rows'
204
+ # execute(sql) do |s|
205
+ # i = -1
206
+ # cols = s.columns(true).map{|c| [output_identifier(c.name), c.type, i+=1]}
207
+ # columns = cols.map{|c| c[0]}
208
+ # self.columns = columns
209
+ # s.each do |row|
210
+ # hash = {}
211
+ # cols.each{|n,t,j| hash[n] = convert_odbc_value(row[j], t)}
212
+ # yield hash
213
+ # end
214
+ # end
215
+ # self
216
+
217
+ execute(sql) do |bq_result|
218
+ self.columns = bq_result.fields.map { |field| field.name.to_sym }
219
+ bq_result.each do |row|
220
+ yield row
221
+ end
222
+ end
223
+
224
+ # execute(sql).each do |row|
225
+ # yield row
226
+ # end
227
+ self
228
+ end
229
+
230
+ # def columns
231
+ # fields.map { |field| field.name.to_sym }
232
+ # end
233
+
234
+ private
235
+
236
+ # def convert_odbc_value(v, t)
237
+ # # When fetching a result set, the Ruby ODBC driver converts all ODBC
238
+ # # SQL types to an equivalent Ruby type; with the exception of
239
+ # # SQL_TYPE_DATE, SQL_TYPE_TIME and SQL_TYPE_TIMESTAMP.
240
+ # #
241
+ # # The conversions below are consistent with the mappings in
242
+ # # ODBCColumn#mapSqlTypeToGenericType and Column#klass.
243
+ # case v
244
+ # when ::ODBC::TimeStamp
245
+ # db.to_application_timestamp([v.year, v.month, v.day, v.hour, v.minute, v.second, v.fraction])
246
+ # when ::ODBC::Time
247
+ # Sequel::SQLTime.create(v.hour, v.minute, v.second)
248
+ # when ::ODBC::Date
249
+ # Date.new(v.year, v.month, v.day)
250
+ # else
251
+ # if t == ::ODBC::SQL_BIT
252
+ # v == 1
253
+ # else
254
+ # v
255
+ # end
256
+ # end
257
+ # end
258
+
259
+ def literal_time(v)
260
+ "'#{v.iso8601}'"
261
+ end
262
+
263
+ # def literal_date(v)
264
+ # v.strftime("{d '%Y-%m-%d'}")
265
+ # end
266
+
267
+ def literal_false
268
+ 'false'
269
+ end
270
+
271
+ def literal_true
272
+ 'true'
273
+ end
274
+ end
275
+ end
276
+ end
@@ -0,0 +1,7 @@
1
+ # frozen-string-literal: true
2
+
3
+ module Sequel
4
+ module Bigquery
5
+ VERSION = '0.1.0'
6
+ end
7
+ end
metadata ADDED
@@ -0,0 +1,105 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sequel-bigquery
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Brendan Weibrecht
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2021-09-08 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: amazing_print
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.3'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: google-cloud-bigquery
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.35'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.35'
41
+ - !ruby/object:Gem::Dependency
42
+ name: paint
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '2.2'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '2.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: sequel
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '5.48'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '5.48'
69
+ description:
70
+ email:
71
+ - brendan@weibrecht.net.au
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - README.md
77
+ - lib/sequel-bigquery.rb
78
+ - lib/sequel_bigquery/version.rb
79
+ homepage: https://github.com/ZimbiX/sequel-bigquery
80
+ licenses:
81
+ - MIT
82
+ metadata:
83
+ homepage_uri: https://github.com/ZimbiX/sequel-bigquery
84
+ source_code_uri: https://github.com/ZimbiX/sequel-bigquery
85
+ changelog_uri: https://github.com/ZimbiX/sequel-bigquery/releases
86
+ post_install_message:
87
+ rdoc_options: []
88
+ require_paths:
89
+ - lib
90
+ required_ruby_version: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: 2.6.0
95
+ required_rubygems_version: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - ">="
98
+ - !ruby/object:Gem::Version
99
+ version: '0'
100
+ requirements: []
101
+ rubygems_version: 3.1.6
102
+ signing_key:
103
+ specification_version: 4
104
+ summary: A Sequel adapter for Google's BigQuery
105
+ test_files: []