sequel-bigquery 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +111 -1
- data/lib/sequel-bigquery.rb +47 -119
- data/lib/sequel_bigquery/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c5943f6638cf7c58e4eeee545d11bc8730f37a4724cda51e6f1000d496f6c49f
|
4
|
+
data.tar.gz: 3ccaff19ff790f4840a747efa8cb1c4b7289c0a7b60bed6fb998aa989be1fcf9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7e9b48c5f94af9dbd49048eb8e7ee6dc81d8927f6c62d2806c12111caf23e9f868e7bf571ad2444d81efc2c61bf6778daf872d4ff928e285e042ccad0c1aeaa2
|
7
|
+
data.tar.gz: be45afe3bedabb2888f3149553acee7d7ebd3e4ee3cafbe8ebbb40594df8c8bc43775937d5440e116c51ace191909192d88ad801d0bf9c62cf97568f42d6b769
|
data/README.md
CHANGED
@@ -1,2 +1,112 @@
|
|
1
1
|
# sequel-bigquery
|
2
|
-
|
2
|
+
|
3
|
+
[](https://rubygems.org/gems/sequel-bigquery)
|
4
|
+
|
5
|
+
A Sequel adapter for [Google's BigQuery](https://cloud.google.com/bigquery).
|
6
|
+
|
7
|
+
## Contents
|
8
|
+
|
9
|
+
<!-- MarkdownTOC autolink=true -->
|
10
|
+
|
11
|
+
- [Intro](#intro)
|
12
|
+
- [Installation](#installation)
|
13
|
+
- [Usage](#usage)
|
14
|
+
- [Contributing](#contributing)
|
15
|
+
- [Development](#development)
|
16
|
+
- [Pre-push hook](#pre-push-hook)
|
17
|
+
- [Release](#release)
|
18
|
+
|
19
|
+
<!-- /MarkdownTOC -->
|
20
|
+
|
21
|
+
## Intro
|
22
|
+
|
23
|
+
**Be warned: Given I was unable to find Sequel documentation covering how to write a database adapter, this was put together by reading Sequel's source and hacking at things until they worked. There are probably a lot of rough edges.**
|
24
|
+
|
25
|
+
Features:
|
26
|
+
|
27
|
+
- Connecting
|
28
|
+
- Migrating
|
29
|
+
- Table creation, with automatic removal of defaults from statements (since BigQuery doesn't support it)
|
30
|
+
- Inserting rows
|
31
|
+
- Updating rows, with automatic addition of `where 1 = 1` to statements (since BigQuery requires a `where` clause)
|
32
|
+
- Querying
|
33
|
+
- Transactions (buffered since BigQuery only supports them when you execute the whole transaction at once)
|
34
|
+
- Ruby types:
|
35
|
+
+ String
|
36
|
+
+ Integer
|
37
|
+
+ _Boolean_ (`TrueClass`/`FalseClass`)
|
38
|
+
+ DateTime (note that BigQuery does not persist timezone)
|
39
|
+
+ Date
|
40
|
+
+ Float
|
41
|
+
+ BigDecimal
|
42
|
+
|
43
|
+
## Installation
|
44
|
+
|
45
|
+
Add it to the `Gemfile` of your project:
|
46
|
+
|
47
|
+
```ruby
|
48
|
+
gem 'sequel-bigquery'
|
49
|
+
```
|
50
|
+
|
51
|
+
and install all your gems:
|
52
|
+
|
53
|
+
```bash
|
54
|
+
bundle install
|
55
|
+
```
|
56
|
+
|
57
|
+
Or you can install it to your system directly using:
|
58
|
+
|
59
|
+
```bash
|
60
|
+
gem install sequel-bigquery
|
61
|
+
```
|
62
|
+
|
63
|
+
## Usage
|
64
|
+
|
65
|
+
Connect to BigQuery:
|
66
|
+
|
67
|
+
```
|
68
|
+
require 'sequel-bigquery'
|
69
|
+
|
70
|
+
db = Sequel.connect(
|
71
|
+
adapter: :bigquery,
|
72
|
+
project: 'your-gcp-project',
|
73
|
+
database: 'your_bigquery_dataset_name',
|
74
|
+
logger: Logger.new(STDOUT),
|
75
|
+
)
|
76
|
+
```
|
77
|
+
|
78
|
+
And use Sequel like normal.
|
79
|
+
|
80
|
+
## Contributing
|
81
|
+
|
82
|
+
Pull requests welcome! =)
|
83
|
+
|
84
|
+
## Development
|
85
|
+
|
86
|
+
### Pre-push hook
|
87
|
+
|
88
|
+
This hook runs style checks and tests.
|
89
|
+
|
90
|
+
To set up the pre-push hook:
|
91
|
+
|
92
|
+
```bash
|
93
|
+
echo -e "#\!/bin/bash\n\$(dirname \$0)/../../auto/pre-push-hook" > .git/hooks/pre-push
|
94
|
+
chmod +x .git/hooks/pre-push
|
95
|
+
```
|
96
|
+
|
97
|
+
### Release
|
98
|
+
|
99
|
+
To release a new version:
|
100
|
+
|
101
|
+
```bash
|
102
|
+
auto/release/update-version && auto/release/tag && auto/release/publish
|
103
|
+
```
|
104
|
+
|
105
|
+
This takes care of the whole process:
|
106
|
+
|
107
|
+
- Incrementing the version number (the patch number by default)
|
108
|
+
- Tagging & pushing commits
|
109
|
+
- Publishing the gem to RubyGems
|
110
|
+
- Creating a draft GitHub release
|
111
|
+
|
112
|
+
To increment the minor or major versions instead of the patch number, run `auto/release/update-version` with `--minor` or `--major`.
|
data/lib/sequel-bigquery.rb
CHANGED
@@ -11,9 +11,9 @@ module Sequel
|
|
11
11
|
module Bigquery
|
12
12
|
# Contains procs keyed on subadapter type that extend the
|
13
13
|
# given database object so it supports the correct database type.
|
14
|
-
DATABASE_SETUP = {}
|
15
|
-
|
16
|
-
class Database < Sequel::Database
|
14
|
+
DATABASE_SETUP = {}.freeze
|
15
|
+
|
16
|
+
class Database < Sequel::Database # rubocop:disable Metrics/ClassLength
|
17
17
|
set_adapter_scheme :bigquery
|
18
18
|
|
19
19
|
def initialize(*args, **kawrgs)
|
@@ -26,9 +26,6 @@ module Sequel
|
|
26
26
|
|
27
27
|
def connect(*_args)
|
28
28
|
puts '#connect'
|
29
|
-
# self.input_identifier_meth = nil
|
30
|
-
# self.identifier_output_method = nil
|
31
|
-
|
32
29
|
config = @orig_opts.dup
|
33
30
|
config.delete(:adapter)
|
34
31
|
config.delete(:logger)
|
@@ -42,12 +39,12 @@ module Sequel
|
|
42
39
|
.tap { puts '#connect end' }
|
43
40
|
end
|
44
41
|
|
45
|
-
def disconnect_connection(
|
42
|
+
def disconnect_connection(_c)
|
46
43
|
puts '#disconnect_connection'
|
47
44
|
# c.disconnect
|
48
45
|
end
|
49
46
|
|
50
|
-
def execute(sql, opts=OPTS)
|
47
|
+
def execute(sql, opts = OPTS) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
51
48
|
puts '#execute'
|
52
49
|
log_query(sql)
|
53
50
|
|
@@ -60,44 +57,37 @@ module Sequel
|
|
60
57
|
|
61
58
|
if sql =~ /^update/i && sql !~ / where /i
|
62
59
|
warn("Warning: Appended 'where 1 = 1' to query since BigQuery requires UPDATE statements to include a WHERE clause")
|
63
|
-
sql
|
60
|
+
sql += ' where 1 = 1'
|
64
61
|
end
|
65
62
|
|
66
|
-
if
|
63
|
+
if /^begin/i.match?(sql)
|
67
64
|
warn_transaction
|
68
65
|
@sql_buffering = true
|
69
66
|
end
|
70
67
|
|
71
68
|
if @sql_buffering
|
72
69
|
@sql_buffer << sql
|
73
|
-
|
74
|
-
|
75
|
-
else
|
76
|
-
return []
|
77
|
-
end
|
70
|
+
return [] unless /^commit/i.match?(sql)
|
71
|
+
warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
|
78
72
|
end
|
79
73
|
|
80
74
|
synchronize(opts[:server]) do |conn|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
conn.query(sql_to_execute)
|
85
|
-
# raw_result = conn.query(sql_to_execute)
|
86
|
-
# BQResult.new(raw_result)
|
87
|
-
end
|
88
|
-
require 'amazing_print'
|
89
|
-
ap results
|
90
|
-
if block_given?
|
91
|
-
yield results
|
92
|
-
else
|
93
|
-
results
|
94
|
-
end
|
95
|
-
# TODO
|
96
|
-
# rescue ::ODBC::Error, ArgumentError => e
|
97
|
-
rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
|
98
|
-
raise_error(e)
|
75
|
+
results = log_connection_yield(sql, conn) do
|
76
|
+
sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
|
77
|
+
conn.query(sql_to_execute)
|
99
78
|
end
|
100
|
-
|
79
|
+
require 'amazing_print'
|
80
|
+
ap results
|
81
|
+
if block_given?
|
82
|
+
yield results
|
83
|
+
else
|
84
|
+
results
|
85
|
+
end
|
86
|
+
# TODO
|
87
|
+
# rescue ::ODBC::Error, ArgumentError => e
|
88
|
+
rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
|
89
|
+
raise_error(e)
|
90
|
+
end # rubocop:disable Style/MultilineBlockChain
|
101
91
|
.tap do
|
102
92
|
@sql_buffer = []
|
103
93
|
@sql_buffering = false
|
@@ -116,34 +106,11 @@ module Sequel
|
|
116
106
|
end
|
117
107
|
end
|
118
108
|
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
# def execute_dui(sql, opts=OPTS)
|
124
|
-
# end
|
125
|
-
|
126
|
-
# def execute_dui(sql, opts=OPTS)
|
127
|
-
# # require 'pry'; binding.pry
|
128
|
-
# synchronize(opts[:server]) do |conn|
|
129
|
-
# begin
|
130
|
-
# log_connection_yield(sql, conn){conn.do(sql)}
|
131
|
-
# # TODO:
|
132
|
-
# # rescue ::ODBC::Error, ArgumentError => e
|
133
|
-
# rescue ArgumentError => e
|
134
|
-
# raise_error(e)
|
135
|
-
# end
|
136
|
-
# end
|
137
|
-
# end
|
109
|
+
def type_literal_generic_float(_column)
|
110
|
+
:float64
|
111
|
+
end
|
138
112
|
|
139
113
|
private
|
140
|
-
|
141
|
-
def adapter_initialize
|
142
|
-
puts '#adapter_initialize'
|
143
|
-
self.extension(:identifier_mangling)
|
144
|
-
self.identifier_input_method = nil
|
145
|
-
self.quote_identifiers = false
|
146
|
-
end
|
147
114
|
|
148
115
|
def connection_execute_method
|
149
116
|
:query
|
@@ -158,18 +125,18 @@ module Sequel
|
|
158
125
|
Dataset
|
159
126
|
end
|
160
127
|
|
161
|
-
def schema_parse_table(
|
128
|
+
def schema_parse_table(_table_name, _opts)
|
162
129
|
logger.debug(Paint['schema_parse_table', :red, :bold])
|
163
130
|
# require 'pry'; binding.pry
|
164
131
|
@bigquery.datasets.map do |dataset|
|
165
132
|
[
|
166
133
|
dataset.dataset_id,
|
167
|
-
{}
|
134
|
+
{},
|
168
135
|
]
|
169
136
|
end
|
170
137
|
end
|
171
138
|
|
172
|
-
def disconnect_error?(e, opts)
|
139
|
+
def disconnect_error?(e, opts) # rubocop:disable Lint/UselessMethodDefinition
|
173
140
|
# super || (e.is_a?(::ODBC::Error) && /\A08S01/.match(e.message))
|
174
141
|
super
|
175
142
|
end
|
@@ -190,87 +157,48 @@ module Sequel
|
|
190
157
|
end
|
191
158
|
|
192
159
|
def warn_transaction
|
193
|
-
warn(
|
160
|
+
warn(
|
161
|
+
'Warning: Transaction detected. This only supported on BigQuery in a script or session. '\
|
162
|
+
'Commencing buffering to run the whole transaction at once as a script upon commit. ' \
|
163
|
+
'Note that no result data is returned while the transaction is open.',
|
164
|
+
)
|
194
165
|
end
|
195
166
|
end
|
196
167
|
|
197
|
-
# class BQResult < SimpleDelegator
|
198
|
-
|
199
|
-
# end
|
200
|
-
|
201
168
|
class Dataset < Sequel::Dataset
|
202
|
-
def fetch_rows(sql)
|
169
|
+
def fetch_rows(sql, &block)
|
203
170
|
puts '#fetch_rows'
|
204
|
-
# execute(sql) do |s|
|
205
|
-
# i = -1
|
206
|
-
# cols = s.columns(true).map{|c| [output_identifier(c.name), c.type, i+=1]}
|
207
|
-
# columns = cols.map{|c| c[0]}
|
208
|
-
# self.columns = columns
|
209
|
-
# s.each do |row|
|
210
|
-
# hash = {}
|
211
|
-
# cols.each{|n,t,j| hash[n] = convert_odbc_value(row[j], t)}
|
212
|
-
# yield hash
|
213
|
-
# end
|
214
|
-
# end
|
215
|
-
# self
|
216
171
|
|
217
172
|
execute(sql) do |bq_result|
|
218
173
|
self.columns = bq_result.fields.map { |field| field.name.to_sym }
|
219
|
-
bq_result.each
|
220
|
-
yield row
|
221
|
-
end
|
174
|
+
bq_result.each(&block)
|
222
175
|
end
|
223
176
|
|
224
|
-
# execute(sql).each do |row|
|
225
|
-
# yield row
|
226
|
-
# end
|
227
177
|
self
|
228
178
|
end
|
229
179
|
|
230
|
-
# def columns
|
231
|
-
# fields.map { |field| field.name.to_sym }
|
232
|
-
# end
|
233
|
-
|
234
180
|
private
|
235
181
|
|
236
|
-
# def convert_odbc_value(v, t)
|
237
|
-
# # When fetching a result set, the Ruby ODBC driver converts all ODBC
|
238
|
-
# # SQL types to an equivalent Ruby type; with the exception of
|
239
|
-
# # SQL_TYPE_DATE, SQL_TYPE_TIME and SQL_TYPE_TIMESTAMP.
|
240
|
-
# #
|
241
|
-
# # The conversions below are consistent with the mappings in
|
242
|
-
# # ODBCColumn#mapSqlTypeToGenericType and Column#klass.
|
243
|
-
# case v
|
244
|
-
# when ::ODBC::TimeStamp
|
245
|
-
# db.to_application_timestamp([v.year, v.month, v.day, v.hour, v.minute, v.second, v.fraction])
|
246
|
-
# when ::ODBC::Time
|
247
|
-
# Sequel::SQLTime.create(v.hour, v.minute, v.second)
|
248
|
-
# when ::ODBC::Date
|
249
|
-
# Date.new(v.year, v.month, v.day)
|
250
|
-
# else
|
251
|
-
# if t == ::ODBC::SQL_BIT
|
252
|
-
# v == 1
|
253
|
-
# else
|
254
|
-
# v
|
255
|
-
# end
|
256
|
-
# end
|
257
|
-
# end
|
258
|
-
|
259
182
|
def literal_time(v)
|
260
183
|
"'#{v.iso8601}'"
|
261
184
|
end
|
262
185
|
|
263
|
-
# def literal_date(v)
|
264
|
-
# v.strftime("{d '%Y-%m-%d'}")
|
265
|
-
# end
|
266
|
-
|
267
186
|
def literal_false
|
268
187
|
'false'
|
269
188
|
end
|
270
|
-
|
189
|
+
|
271
190
|
def literal_true
|
272
191
|
'true'
|
273
192
|
end
|
193
|
+
|
194
|
+
# Like MySQL, BigQuery uses the nonstandard ` (backtick) for quoting identifiers.
|
195
|
+
def quoted_identifier_append(sql, c)
|
196
|
+
sql << '`%s`' % c
|
197
|
+
end
|
198
|
+
|
199
|
+
def input_identifier(v)
|
200
|
+
v.to_s
|
201
|
+
end
|
274
202
|
end
|
275
203
|
end
|
276
204
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sequel-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brendan Weibrecht
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: amazing_print
|