sequel-bigquery 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e110842bf3d9ef623ca308976da97240474929d5598771f16d773e634e5e9981
4
- data.tar.gz: 50f4606adfcbd90786cf0ae758fb632564b2456d02144ad8cdc1fd78b6d23eda
3
+ metadata.gz: c5943f6638cf7c58e4eeee545d11bc8730f37a4724cda51e6f1000d496f6c49f
4
+ data.tar.gz: 3ccaff19ff790f4840a747efa8cb1c4b7289c0a7b60bed6fb998aa989be1fcf9
5
5
  SHA512:
6
- metadata.gz: 445496592bf4d61fd2caf0365fd1a07635c6e9da41b528aae7dcf847cd42aa4d31522e555a3c350004d958ba82319729e52c6f27f695bb1f96902b9760e4de01
7
- data.tar.gz: e792871b3615d5605fbb4d2ea803d9042e3963f5db0778750612775dddc6c0951c0416a1e0cd4f50ec548cafc1440a372d31ffac69e8a18e222923828a120556
6
+ metadata.gz: 7e9b48c5f94af9dbd49048eb8e7ee6dc81d8927f6c62d2806c12111caf23e9f868e7bf571ad2444d81efc2c61bf6778daf872d4ff928e285e042ccad0c1aeaa2
7
+ data.tar.gz: be45afe3bedabb2888f3149553acee7d7ebd3e4ee3cafbe8ebbb40594df8c8bc43775937d5440e116c51ace191909192d88ad801d0bf9c62cf97568f42d6b769
data/README.md CHANGED
@@ -1,2 +1,112 @@
1
1
  # sequel-bigquery
2
- A Sequel adapter for Google's BigQuery
2
+
3
+ [![Gem Version](https://badge.fury.io/rb/sequel-bigquery.svg)](https://rubygems.org/gems/sequel-bigquery)
4
+
5
+ A Sequel adapter for [Google's BigQuery](https://cloud.google.com/bigquery).
6
+
7
+ ## Contents
8
+
9
+ <!-- MarkdownTOC autolink=true -->
10
+
11
+ - [Intro](#intro)
12
+ - [Installation](#installation)
13
+ - [Usage](#usage)
14
+ - [Contributing](#contributing)
15
+ - [Development](#development)
16
+ - [Pre-push hook](#pre-push-hook)
17
+ - [Release](#release)
18
+
19
+ <!-- /MarkdownTOC -->
20
+
21
+ ## Intro
22
+
23
+ **Be warned: Given I was unable to find Sequel documentation covering how to write a database adapter, this was put together by reading Sequel's source and hacking at things until they worked. There are probably a lot of rough edges.**
24
+
25
+ Features:
26
+
27
+ - Connecting
28
+ - Migrating
29
+ - Table creation, with automatic removal of defaults from statements (since BigQuery doesn't support it)
30
+ - Inserting rows
31
+ - Updating rows, with automatic addition of `where 1 = 1` to statements (since BigQuery requires a `where` clause)
32
+ - Querying
33
+ - Transactions (buffered since BigQuery only supports them when you execute the whole transaction at once)
34
+ - Ruby types:
35
+ + String
36
+ + Integer
37
+ + _Boolean_ (`TrueClass`/`FalseClass`)
38
+ + DateTime (note that BigQuery does not persist timezone)
39
+ + Date
40
+ + Float
41
+ + BigDecimal
42
+
43
+ ## Installation
44
+
45
+ Add it to the `Gemfile` of your project:
46
+
47
+ ```ruby
48
+ gem 'sequel-bigquery'
49
+ ```
50
+
51
+ and install all your gems:
52
+
53
+ ```bash
54
+ bundle install
55
+ ```
56
+
57
+ Or you can install it to your system directly using:
58
+
59
+ ```bash
60
+ gem install sequel-bigquery
61
+ ```
62
+
63
+ ## Usage
64
+
65
+ Connect to BigQuery:
66
+
67
+ ```
68
+ require 'sequel-bigquery'
69
+
70
+ db = Sequel.connect(
71
+ adapter: :bigquery,
72
+ project: 'your-gcp-project',
73
+ database: 'your_bigquery_dataset_name',
74
+ logger: Logger.new(STDOUT),
75
+ )
76
+ ```
77
+
78
+ And use Sequel like normal.
79
+
80
+ ## Contributing
81
+
82
+ Pull requests welcome! =)
83
+
84
+ ## Development
85
+
86
+ ### Pre-push hook
87
+
88
+ This hook runs style checks and tests.
89
+
90
+ To set up the pre-push hook:
91
+
92
+ ```bash
93
+ echo -e "#\!/bin/bash\n\$(dirname \$0)/../../auto/pre-push-hook" > .git/hooks/pre-push
94
+ chmod +x .git/hooks/pre-push
95
+ ```
96
+
97
+ ### Release
98
+
99
+ To release a new version:
100
+
101
+ ```bash
102
+ auto/release/update-version && auto/release/tag && auto/release/publish
103
+ ```
104
+
105
+ This takes care of the whole process:
106
+
107
+ - Incrementing the version number (the patch number by default)
108
+ - Tagging & pushing commits
109
+ - Publishing the gem to RubyGems
110
+ - Creating a draft GitHub release
111
+
112
+ To increment the minor or major versions instead of the patch number, run `auto/release/update-version` with `--minor` or `--major`.
@@ -11,9 +11,9 @@ module Sequel
11
11
  module Bigquery
12
12
  # Contains procs keyed on subadapter type that extend the
13
13
  # given database object so it supports the correct database type.
14
- DATABASE_SETUP = {}
15
-
16
- class Database < Sequel::Database
14
+ DATABASE_SETUP = {}.freeze
15
+
16
+ class Database < Sequel::Database # rubocop:disable Metrics/ClassLength
17
17
  set_adapter_scheme :bigquery
18
18
 
19
19
  def initialize(*args, **kawrgs)
@@ -26,9 +26,6 @@ module Sequel
26
26
 
27
27
  def connect(*_args)
28
28
  puts '#connect'
29
- # self.input_identifier_meth = nil
30
- # self.identifier_output_method = nil
31
-
32
29
  config = @orig_opts.dup
33
30
  config.delete(:adapter)
34
31
  config.delete(:logger)
@@ -42,12 +39,12 @@ module Sequel
42
39
  .tap { puts '#connect end' }
43
40
  end
44
41
 
45
- def disconnect_connection(c)
42
+ def disconnect_connection(_c)
46
43
  puts '#disconnect_connection'
47
44
  # c.disconnect
48
45
  end
49
46
 
50
- def execute(sql, opts=OPTS)
47
+ def execute(sql, opts = OPTS) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
51
48
  puts '#execute'
52
49
  log_query(sql)
53
50
 
@@ -60,44 +57,37 @@ module Sequel
60
57
 
61
58
  if sql =~ /^update/i && sql !~ / where /i
62
59
  warn("Warning: Appended 'where 1 = 1' to query since BigQuery requires UPDATE statements to include a WHERE clause")
63
- sql = sql + ' where 1 = 1'
60
+ sql += ' where 1 = 1'
64
61
  end
65
62
 
66
- if sql =~ /^begin/i
63
+ if /^begin/i.match?(sql)
67
64
  warn_transaction
68
65
  @sql_buffering = true
69
66
  end
70
67
 
71
68
  if @sql_buffering
72
69
  @sql_buffer << sql
73
- if sql =~ /^commit/i
74
- warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
75
- else
76
- return []
77
- end
70
+ return [] unless /^commit/i.match?(sql)
71
+ warn("Warning: Will now execute entire buffered transaction:\n" + @sql_buffer.join("\n"))
78
72
  end
79
73
 
80
74
  synchronize(opts[:server]) do |conn|
81
- begin
82
- results = log_connection_yield(sql, conn) do
83
- sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
84
- conn.query(sql_to_execute)
85
- # raw_result = conn.query(sql_to_execute)
86
- # BQResult.new(raw_result)
87
- end
88
- require 'amazing_print'
89
- ap results
90
- if block_given?
91
- yield results
92
- else
93
- results
94
- end
95
- # TODO
96
- # rescue ::ODBC::Error, ArgumentError => e
97
- rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
98
- raise_error(e)
75
+ results = log_connection_yield(sql, conn) do
76
+ sql_to_execute = @sql_buffer.any? ? @sql_buffer.join("\n") : sql
77
+ conn.query(sql_to_execute)
99
78
  end
100
- end
79
+ require 'amazing_print'
80
+ ap results
81
+ if block_given?
82
+ yield results
83
+ else
84
+ results
85
+ end
86
+ # TODO
87
+ # rescue ::ODBC::Error, ArgumentError => e
88
+ rescue Google::Cloud::InvalidArgumentError, ArgumentError => e
89
+ raise_error(e)
90
+ end # rubocop:disable Style/MultilineBlockChain
101
91
  .tap do
102
92
  @sql_buffer = []
103
93
  @sql_buffering = false
@@ -116,34 +106,11 @@ module Sequel
116
106
  end
117
107
  end
118
108
 
119
- # def supports_transactional_ddl?
120
- # false
121
- # end
122
-
123
- # def execute_dui(sql, opts=OPTS)
124
- # end
125
-
126
- # def execute_dui(sql, opts=OPTS)
127
- # # require 'pry'; binding.pry
128
- # synchronize(opts[:server]) do |conn|
129
- # begin
130
- # log_connection_yield(sql, conn){conn.do(sql)}
131
- # # TODO:
132
- # # rescue ::ODBC::Error, ArgumentError => e
133
- # rescue ArgumentError => e
134
- # raise_error(e)
135
- # end
136
- # end
137
- # end
109
+ def type_literal_generic_float(_column)
110
+ :float64
111
+ end
138
112
 
139
113
  private
140
-
141
- def adapter_initialize
142
- puts '#adapter_initialize'
143
- self.extension(:identifier_mangling)
144
- self.identifier_input_method = nil
145
- self.quote_identifiers = false
146
- end
147
114
 
148
115
  def connection_execute_method
149
116
  :query
@@ -158,18 +125,18 @@ module Sequel
158
125
  Dataset
159
126
  end
160
127
 
161
- def schema_parse_table(table_name, opts)
128
+ def schema_parse_table(_table_name, _opts)
162
129
  logger.debug(Paint['schema_parse_table', :red, :bold])
163
130
  # require 'pry'; binding.pry
164
131
  @bigquery.datasets.map do |dataset|
165
132
  [
166
133
  dataset.dataset_id,
167
- {}
134
+ {},
168
135
  ]
169
136
  end
170
137
  end
171
138
 
172
- def disconnect_error?(e, opts)
139
+ def disconnect_error?(e, opts) # rubocop:disable Lint/UselessMethodDefinition
173
140
  # super || (e.is_a?(::ODBC::Error) && /\A08S01/.match(e.message))
174
141
  super
175
142
  end
@@ -190,87 +157,48 @@ module Sequel
190
157
  end
191
158
 
192
159
  def warn_transaction
193
- warn('Warning: Transaction detected. This only supported on BigQuery in a script or session. Commencing buffering to run the whole transaction at once as a script upon commit. Note that no result data is returned while the transaction is open.')
160
+ warn(
161
+ 'Warning: Transaction detected. This only supported on BigQuery in a script or session. '\
162
+ 'Commencing buffering to run the whole transaction at once as a script upon commit. ' \
163
+ 'Note that no result data is returned while the transaction is open.',
164
+ )
194
165
  end
195
166
  end
196
167
 
197
- # class BQResult < SimpleDelegator
198
-
199
- # end
200
-
201
168
  class Dataset < Sequel::Dataset
202
- def fetch_rows(sql)
169
+ def fetch_rows(sql, &block)
203
170
  puts '#fetch_rows'
204
- # execute(sql) do |s|
205
- # i = -1
206
- # cols = s.columns(true).map{|c| [output_identifier(c.name), c.type, i+=1]}
207
- # columns = cols.map{|c| c[0]}
208
- # self.columns = columns
209
- # s.each do |row|
210
- # hash = {}
211
- # cols.each{|n,t,j| hash[n] = convert_odbc_value(row[j], t)}
212
- # yield hash
213
- # end
214
- # end
215
- # self
216
171
 
217
172
  execute(sql) do |bq_result|
218
173
  self.columns = bq_result.fields.map { |field| field.name.to_sym }
219
- bq_result.each do |row|
220
- yield row
221
- end
174
+ bq_result.each(&block)
222
175
  end
223
176
 
224
- # execute(sql).each do |row|
225
- # yield row
226
- # end
227
177
  self
228
178
  end
229
179
 
230
- # def columns
231
- # fields.map { |field| field.name.to_sym }
232
- # end
233
-
234
180
  private
235
181
 
236
- # def convert_odbc_value(v, t)
237
- # # When fetching a result set, the Ruby ODBC driver converts all ODBC
238
- # # SQL types to an equivalent Ruby type; with the exception of
239
- # # SQL_TYPE_DATE, SQL_TYPE_TIME and SQL_TYPE_TIMESTAMP.
240
- # #
241
- # # The conversions below are consistent with the mappings in
242
- # # ODBCColumn#mapSqlTypeToGenericType and Column#klass.
243
- # case v
244
- # when ::ODBC::TimeStamp
245
- # db.to_application_timestamp([v.year, v.month, v.day, v.hour, v.minute, v.second, v.fraction])
246
- # when ::ODBC::Time
247
- # Sequel::SQLTime.create(v.hour, v.minute, v.second)
248
- # when ::ODBC::Date
249
- # Date.new(v.year, v.month, v.day)
250
- # else
251
- # if t == ::ODBC::SQL_BIT
252
- # v == 1
253
- # else
254
- # v
255
- # end
256
- # end
257
- # end
258
-
259
182
  def literal_time(v)
260
183
  "'#{v.iso8601}'"
261
184
  end
262
185
 
263
- # def literal_date(v)
264
- # v.strftime("{d '%Y-%m-%d'}")
265
- # end
266
-
267
186
  def literal_false
268
187
  'false'
269
188
  end
270
-
189
+
271
190
  def literal_true
272
191
  'true'
273
192
  end
193
+
194
+ # Like MySQL, BigQuery uses the nonstandard ` (backtick) for quoting identifiers.
195
+ def quoted_identifier_append(sql, c)
196
+ sql << '`%s`' % c
197
+ end
198
+
199
+ def input_identifier(v)
200
+ v.to_s
201
+ end
274
202
  end
275
203
  end
276
204
  end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Sequel
4
4
  module Bigquery
5
- VERSION = '0.1.0'
5
+ VERSION = '0.1.1'
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sequel-bigquery
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Brendan Weibrecht
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-09-08 00:00:00.000000000 Z
11
+ date: 2021-10-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: amazing_print