sequel-bigquery 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +2 -0
- data/lib/sequel-bigquery.rb +276 -0
- data/lib/sequel_bigquery/version.rb +7 -0
- metadata +105 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: e110842bf3d9ef623ca308976da97240474929d5598771f16d773e634e5e9981
|
4
|
+
data.tar.gz: 50f4606adfcbd90786cf0ae758fb632564b2456d02144ad8cdc1fd78b6d23eda
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 445496592bf4d61fd2caf0365fd1a07635c6e9da41b528aae7dcf847cd42aa4d31522e555a3c350004d958ba82319729e52c6f27f695bb1f96902b9760e4de01
|
7
|
+
data.tar.gz: e792871b3615d5605fbb4d2ea803d9042e3963f5db0778750612775dddc6c0951c0416a1e0cd4f50ec548cafc1440a372d31ffac69e8a18e222923828a120556
|
data/README.md
ADDED
@@ -0,0 +1,276 @@
|
|
1
|
+
# frozen-string-literal: true
|
2
|
+
|
3
|
+
require 'delegate'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
require 'google/cloud/bigquery'
|
7
|
+
require 'paint'
|
8
|
+
require 'sequel'
|
9
|
+
|
10
|
+
module Sequel
|
11
|
+
module Bigquery
|
12
|
+
# Contains procs keyed on subadapter type that extend the
|
13
|
+
# given database object so it supports the correct database type.
|
14
|
+
DATABASE_SETUP = {}
|
15
|
+
|
16
|
+
class Database < Sequel::Database
|
17
|
+
set_adapter_scheme :bigquery
|
18
|
+
|
19
|
+
def initialize(*args, **kawrgs)
|
20
|
+
puts '.new'
|
21
|
+
@orig_opts = kawrgs.fetch(:orig_opts)
|
22
|
+
@sql_buffer = []
|
23
|
+
@sql_buffering = false
|
24
|
+
super
|
25
|
+
end
|
26
|
+
|
27
|
+
def connect(*_args)
|
28
|
+
puts '#connect'
|
29
|
+
# self.input_identifier_meth = nil
|
30
|
+
# self.identifier_output_method = nil
|
31
|
+
|
32
|
+
config = @orig_opts.dup
|
33
|
+
config.delete(:adapter)
|
34
|
+
config.delete(:logger)
|
35
|
+
bq_dataset_name = config.delete(:dataset) || config.delete(:database)
|
36
|
+
@bigquery = Google::Cloud::Bigquery.new(config)
|
37
|
+
# ObjectSpace.each_object(HTTPClient).each { |c| c.debug_dev = STDOUT }
|
38
|
+
@bigquery.dataset(bq_dataset_name) || begin
|
39
|
+
@loggers[0].debug('BigQuery dataset %s does not exist; creating it' % bq_dataset_name)
|
40
|
+
@bigquery.create_dataset(bq_dataset_name)
|
41
|
+
end
|
42
|
+
.tap { puts '#connect end' }
|
43
|
+
end
|
44
|
+
|
45
|
+
def disconnect_connection(c)
|
46
|
+
puts '#disconnect_connection'
|
47
|
+
# c.disconnect
|
48
|
+
end
|
49
|
+
|
50
|
+
def execute(sql, opts=OPTS)
|
51
|
+
puts '#execute'
|
52
|
+
log_query(sql)
|
53
|
+
|
54
|
+
# require 'pry'; binding.pry if sql =~ /CREATE TABLE IF NOT EXISTS/i
|
55
|
+
|
56
|
+
sql = sql.gsub(/\sdefault \S+/i) do
|
57
|
+
warn_default_removal(sql)
|
58
|
+
''
|
59
|
+
end
|
60
|
+
|
61
|
+
if sql =~ /^update/i && sql !~ / where /i
|
62
|
+
warn("Warning: Appended 'where 1 = 1' to query since BigQuery requires UPDATE statements to include a WHERE clause")
|
63
|
+
sql = sql + ' where 1 = 1'
|
64
|
+
end
|
65
|
+
|
66
|
+
if sql =~ /^begin/i
|
67
|
+
warn_transaction
|
68
|
+
@sql_buffering = true
|
69
|
+
end
|
70
|
+
|
71
|
+
if @sql_buffering
|
72
|
+
@sql_buffer << sql
|
73
|
+
if sql =~ /^commit/i
|
74
|
+
warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
|
75
|
+
else
|
76
|
+
return []
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
synchronize(opts[:server]) do |conn|
|
81
|
+
begin
|
82
|
+
results = log_connection_yield(sql, conn) do
|
83
|
+
sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
|
84
|
+
conn.query(sql_to_execute)
|
85
|
+
# raw_result = conn.query(sql_to_execute)
|
86
|
+
# BQResult.new(raw_result)
|
87
|
+
end
|
88
|
+
require 'amazing_print'
|
89
|
+
ap results
|
90
|
+
if block_given?
|
91
|
+
yield results
|
92
|
+
else
|
93
|
+
results
|
94
|
+
end
|
95
|
+
# TODO
|
96
|
+
# rescue ::ODBC::Error, ArgumentError => e
|
97
|
+
rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
|
98
|
+
raise_error(e)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
.tap do
|
102
|
+
@sql_buffer = []
|
103
|
+
@sql_buffering = false
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def supports_create_table_if_not_exists?
|
108
|
+
true
|
109
|
+
end
|
110
|
+
|
111
|
+
def type_literal_generic_string(column)
|
112
|
+
if column[:size]
|
113
|
+
"string(#{column[:size]})"
|
114
|
+
else
|
115
|
+
:string
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
# def supports_transactional_ddl?
|
120
|
+
# false
|
121
|
+
# end
|
122
|
+
|
123
|
+
# def execute_dui(sql, opts=OPTS)
|
124
|
+
# end
|
125
|
+
|
126
|
+
# def execute_dui(sql, opts=OPTS)
|
127
|
+
# # require 'pry'; binding.pry
|
128
|
+
# synchronize(opts[:server]) do |conn|
|
129
|
+
# begin
|
130
|
+
# log_connection_yield(sql, conn){conn.do(sql)}
|
131
|
+
# # TODO:
|
132
|
+
# # rescue ::ODBC::Error, ArgumentError => e
|
133
|
+
# rescue ArgumentError => e
|
134
|
+
# raise_error(e)
|
135
|
+
# end
|
136
|
+
# end
|
137
|
+
# end
|
138
|
+
|
139
|
+
private
|
140
|
+
|
141
|
+
def adapter_initialize
|
142
|
+
puts '#adapter_initialize'
|
143
|
+
self.extension(:identifier_mangling)
|
144
|
+
self.identifier_input_method = nil
|
145
|
+
self.quote_identifiers = false
|
146
|
+
end
|
147
|
+
|
148
|
+
def connection_execute_method
|
149
|
+
:query
|
150
|
+
end
|
151
|
+
|
152
|
+
def database_error_classes
|
153
|
+
# [::ODBC::Error]
|
154
|
+
# TODO
|
155
|
+
end
|
156
|
+
|
157
|
+
def dataset_class_default
|
158
|
+
Dataset
|
159
|
+
end
|
160
|
+
|
161
|
+
def schema_parse_table(table_name, opts)
|
162
|
+
logger.debug(Paint['schema_parse_table', :red, :bold])
|
163
|
+
# require 'pry'; binding.pry
|
164
|
+
@bigquery.datasets.map do |dataset|
|
165
|
+
[
|
166
|
+
dataset.dataset_id,
|
167
|
+
{}
|
168
|
+
]
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def disconnect_error?(e, opts)
|
173
|
+
# super || (e.is_a?(::ODBC::Error) && /\A08S01/.match(e.message))
|
174
|
+
super
|
175
|
+
end
|
176
|
+
|
177
|
+
# Padded to horizontally align with post-execution log message which includes the execution time
|
178
|
+
def log_query(sql)
|
179
|
+
pad = ' '
|
180
|
+
puts Paint[pad + sql, :cyan, :bold]
|
181
|
+
# @loggers[0]&.debug(' ' + sql)
|
182
|
+
end
|
183
|
+
|
184
|
+
def warn(msg)
|
185
|
+
@loggers[0].warn(Paint[msg, '#FFA500', :bold])
|
186
|
+
end
|
187
|
+
|
188
|
+
def warn_default_removal(sql)
|
189
|
+
warn("Warning: Default removed from below query as it's not supported on BigQuery:\n%s" % sql)
|
190
|
+
end
|
191
|
+
|
192
|
+
def warn_transaction
|
193
|
+
warn('Warning: Transaction detected. This only supported on BigQuery in a script or session. Commencing buffering to run the whole transaction at once as a script upon commit. Note that no result data is returned while the transaction is open.')
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# class BQResult < SimpleDelegator
|
198
|
+
|
199
|
+
# end
|
200
|
+
|
201
|
+
class Dataset < Sequel::Dataset
|
202
|
+
def fetch_rows(sql)
|
203
|
+
puts '#fetch_rows'
|
204
|
+
# execute(sql) do |s|
|
205
|
+
# i = -1
|
206
|
+
# cols = s.columns(true).map{|c| [output_identifier(c.name), c.type, i+=1]}
|
207
|
+
# columns = cols.map{|c| c[0]}
|
208
|
+
# self.columns = columns
|
209
|
+
# s.each do |row|
|
210
|
+
# hash = {}
|
211
|
+
# cols.each{|n,t,j| hash[n] = convert_odbc_value(row[j], t)}
|
212
|
+
# yield hash
|
213
|
+
# end
|
214
|
+
# end
|
215
|
+
# self
|
216
|
+
|
217
|
+
execute(sql) do |bq_result|
|
218
|
+
self.columns = bq_result.fields.map { |field| field.name.to_sym }
|
219
|
+
bq_result.each do |row|
|
220
|
+
yield row
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
# execute(sql).each do |row|
|
225
|
+
# yield row
|
226
|
+
# end
|
227
|
+
self
|
228
|
+
end
|
229
|
+
|
230
|
+
# def columns
|
231
|
+
# fields.map { |field| field.name.to_sym }
|
232
|
+
# end
|
233
|
+
|
234
|
+
private
|
235
|
+
|
236
|
+
# def convert_odbc_value(v, t)
|
237
|
+
# # When fetching a result set, the Ruby ODBC driver converts all ODBC
|
238
|
+
# # SQL types to an equivalent Ruby type; with the exception of
|
239
|
+
# # SQL_TYPE_DATE, SQL_TYPE_TIME and SQL_TYPE_TIMESTAMP.
|
240
|
+
# #
|
241
|
+
# # The conversions below are consistent with the mappings in
|
242
|
+
# # ODBCColumn#mapSqlTypeToGenericType and Column#klass.
|
243
|
+
# case v
|
244
|
+
# when ::ODBC::TimeStamp
|
245
|
+
# db.to_application_timestamp([v.year, v.month, v.day, v.hour, v.minute, v.second, v.fraction])
|
246
|
+
# when ::ODBC::Time
|
247
|
+
# Sequel::SQLTime.create(v.hour, v.minute, v.second)
|
248
|
+
# when ::ODBC::Date
|
249
|
+
# Date.new(v.year, v.month, v.day)
|
250
|
+
# else
|
251
|
+
# if t == ::ODBC::SQL_BIT
|
252
|
+
# v == 1
|
253
|
+
# else
|
254
|
+
# v
|
255
|
+
# end
|
256
|
+
# end
|
257
|
+
# end
|
258
|
+
|
259
|
+
def literal_time(v)
|
260
|
+
"'#{v.iso8601}'"
|
261
|
+
end
|
262
|
+
|
263
|
+
# def literal_date(v)
|
264
|
+
# v.strftime("{d '%Y-%m-%d'}")
|
265
|
+
# end
|
266
|
+
|
267
|
+
def literal_false
|
268
|
+
'false'
|
269
|
+
end
|
270
|
+
|
271
|
+
def literal_true
|
272
|
+
'true'
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
276
|
+
end
|
metadata
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sequel-bigquery
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Brendan Weibrecht
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-09-08 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: amazing_print
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.3'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.3'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: google-cloud-bigquery
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.35'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.35'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: paint
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '2.2'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '2.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: sequel
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '5.48'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '5.48'
|
69
|
+
description:
|
70
|
+
email:
|
71
|
+
- brendan@weibrecht.net.au
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- README.md
|
77
|
+
- lib/sequel-bigquery.rb
|
78
|
+
- lib/sequel_bigquery/version.rb
|
79
|
+
homepage: https://github.com/ZimbiX/sequel-bigquery
|
80
|
+
licenses:
|
81
|
+
- MIT
|
82
|
+
metadata:
|
83
|
+
homepage_uri: https://github.com/ZimbiX/sequel-bigquery
|
84
|
+
source_code_uri: https://github.com/ZimbiX/sequel-bigquery
|
85
|
+
changelog_uri: https://github.com/ZimbiX/sequel-bigquery/releases
|
86
|
+
post_install_message:
|
87
|
+
rdoc_options: []
|
88
|
+
require_paths:
|
89
|
+
- lib
|
90
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 2.6.0
|
95
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - ">="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '0'
|
100
|
+
requirements: []
|
101
|
+
rubygems_version: 3.1.6
|
102
|
+
signing_key:
|
103
|
+
specification_version: 4
|
104
|
+
summary: A Sequel adapter for Google's BigQuery
|
105
|
+
test_files: []
|