sequelizer 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +14 -0
- data/Gemfile +4 -0
- data/lib/sequel/extensions/make_readyable.rb +108 -0
- data/lib/sequel/extensions/sqls.rb +31 -0
- data/lib/sequel/extensions/usable.rb +15 -0
- data/lib/sequelizer/version.rb +1 -1
- data/lib/sequelizer.rb +3 -3
- data/test/lib/sequel/extensions/test_make_readyable.rb +88 -0
- data/test/lib/sequel/extensions/test_usable.rb +12 -0
- metadata +11 -4
- data/.travis.yml +0 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 734691cabf0f5799023e6f921c222e771ef04285462903ae109376a364970573
|
4
|
+
data.tar.gz: 86ea46b200a4cfa33d2adf3fb8ddbbc770b1f0d5a64041d61c19a139735e1406
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7ff33a0b6c7722d1cd5ff704916af17e70a819936c308b44fbb8031883d6e50cfe2399f42484d392d09070086ecf3dcf8db4e8678a1b9bb10a9f213dc163bf22
|
7
|
+
data.tar.gz: 180e2a43051c492d36f4be93802b0be3fb0b3eea65bc6457f354b96ad93b724862cccec83e7ebead01f4d2206ddeb329e1e1401f37bc217bf6c1a61240f6072f
|
@@ -0,0 +1,14 @@
|
|
1
|
+
name: Run Tests
|
2
|
+
on: [push, pull_request]
|
3
|
+
jobs:
|
4
|
+
Run-Tests:
|
5
|
+
runs-on: ubuntu-latest
|
6
|
+
steps:
|
7
|
+
- uses: actions/checkout@v3
|
8
|
+
- uses: ruby/setup-ruby@v1
|
9
|
+
with:
|
10
|
+
ruby-version: '3.2'
|
11
|
+
bundler-cache: true
|
12
|
+
-
|
13
|
+
name: Run Tests
|
14
|
+
run: bundle exec rake test
|
data/Gemfile
CHANGED
@@ -0,0 +1,108 @@
|
|
1
|
+
module Sequel
|
2
|
+
module MakeReadyable
|
3
|
+
##
|
4
|
+
# This method is primarily geared towards Spark SQL-based databases.
|
5
|
+
#
|
6
|
+
# Given some options, prepares a set of views to represent a set
|
7
|
+
# of tables across a collection of different schemas and external,
|
8
|
+
# unmanaged tables.
|
9
|
+
#
|
10
|
+
# DB.make_ready(use_schema: :schema)
|
11
|
+
# # => USE `schema`
|
12
|
+
#
|
13
|
+
# When using search_path, tables from previous schema override tables
|
14
|
+
# from the next schema. This is analogous to the way Unix searches
|
15
|
+
# the PATH variable for programs.
|
16
|
+
#
|
17
|
+
# Assuming the following tables: schema1.a, schema2.a, schema2.b
|
18
|
+
#
|
19
|
+
# DB.make_ready(search_path: [:schema1, :schema2])
|
20
|
+
# # => CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a;`
|
21
|
+
# # => CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b;`
|
22
|
+
#
|
23
|
+
# When using Pathnames, the extension on the file becomes the format
|
24
|
+
# to try to read from the file.
|
25
|
+
#
|
26
|
+
# DB.make_ready(search_path: [Pathname.new("c.parquet"), Pathname.new("d.orc")])
|
27
|
+
# # => CREATE TEMPORARY VIEW `c` USING parquet OPTIONS ('path'='c.parquet')
|
28
|
+
# # => CREATE TEMPORARY VIEW `d` USING orc OPTIONS ('path'='d.orc')
|
29
|
+
#
|
30
|
+
# @param [Hash] opts the options used to prepare the database
|
31
|
+
# @option opts [String] :use_schema The schema to be used as the primary schema
|
32
|
+
# @option opts [Array] :search_path A set of sympbols (to represent schemas) or Pathnames (to represent externally managed data files)
|
33
|
+
def make_ready(opts = {})
|
34
|
+
ReadyMaker.new(self, opts).run
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
class ReadyMaker
|
40
|
+
attr_reader :db, :opts
|
41
|
+
|
42
|
+
def initialize(db, opts)
|
43
|
+
@db = db
|
44
|
+
@opts = opts
|
45
|
+
end
|
46
|
+
|
47
|
+
def run
|
48
|
+
if opts[:use_schema]
|
49
|
+
db.extension :usable
|
50
|
+
db.use(opts[:use_schema])
|
51
|
+
end
|
52
|
+
only_tables = Array(opts[:only])
|
53
|
+
created_views = (Array(opts[:except]) || [])
|
54
|
+
(opts[:search_path] || []).each do |schema|
|
55
|
+
schema = schema.is_a?(Pathname) ? schema : schema.to_sym
|
56
|
+
source = get_source(db, schema)
|
57
|
+
tables = source.tables(schema: schema) - created_views
|
58
|
+
tables &= only_tables unless only_tables.empty?
|
59
|
+
tables.each do |table|
|
60
|
+
create_view(source, table, schema)
|
61
|
+
created_views << table
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def create_view(source, table, schema)
|
67
|
+
if schema.to_s =~ %r{/}
|
68
|
+
source.create_view(table, temp: true)
|
69
|
+
else
|
70
|
+
source.create_view(table, db[Sequel.qualify(schema, table)], temp: true)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def get_source(db, schema)
|
75
|
+
if schema.to_s =~ %r{/}
|
76
|
+
FileSourcerer.new(db, Pathname.new(schema))
|
77
|
+
else
|
78
|
+
db
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
class FileSourcerer
|
83
|
+
attr_reader :db, :schema
|
84
|
+
def initialize(db, schema)
|
85
|
+
@db = db
|
86
|
+
@schema = schema
|
87
|
+
end
|
88
|
+
|
89
|
+
def tables(opts = {})
|
90
|
+
[schema.basename(".*").to_s.to_sym]
|
91
|
+
end
|
92
|
+
|
93
|
+
def create_view(table, opts = {})
|
94
|
+
db.create_view(table, {
|
95
|
+
temp: true,
|
96
|
+
using: format,
|
97
|
+
options: { path: schema.expand_path }
|
98
|
+
}.merge(opts))
|
99
|
+
end
|
100
|
+
|
101
|
+
def format
|
102
|
+
schema.extname[1..-1]
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
Database.register_extension(:make_readyable, MakeReadyable)
|
108
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen-string-literal: true
|
2
|
+
#
|
3
|
+
# The sqls extension will record each SQL statement sent to the
|
4
|
+
# database
|
5
|
+
#
|
6
|
+
# DB.extension :sqls
|
7
|
+
# DB[:table]
|
8
|
+
# DB.sqls # => ["SELECT * FROM table LIMIT 1"]
|
9
|
+
#
|
10
|
+
# Related module: Sequel::Sqls
|
11
|
+
|
12
|
+
module Sequel
|
13
|
+
module Sqls
|
14
|
+
attr_reader :sqls
|
15
|
+
|
16
|
+
# Record SQL statements when logging query.
|
17
|
+
def log_connection_yield(sql, conn, args=nil)
|
18
|
+
@sqls_mutex.synchronize{sqls.push(sql)}
|
19
|
+
super
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.extended(db)
|
23
|
+
db.instance_exec do
|
24
|
+
@sqls_mutex ||= Mutex.new
|
25
|
+
@sqls ||= []
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
Database.register_extension(:sqls, Sqls)
|
31
|
+
end
|
data/lib/sequelizer/version.rb
CHANGED
data/lib/sequelizer.rb
CHANGED
@@ -30,11 +30,11 @@ module Sequelizer
|
|
30
30
|
def new_db(options = {})
|
31
31
|
cached = find_cached(options)
|
32
32
|
return cached if cached && !options[:force_new]
|
33
|
-
@
|
33
|
+
@_sequelizer_cache[options] = ConnectionMaker.new(options).connection
|
34
34
|
end
|
35
35
|
|
36
36
|
def find_cached(options)
|
37
|
-
@
|
38
|
-
@
|
37
|
+
@_sequelizer_cache ||= {}
|
38
|
+
@_sequelizer_cache[options]
|
39
39
|
end
|
40
40
|
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require_relative "../../../test_helper"
|
2
|
+
require "fileutils"
|
3
|
+
require "pathname"
|
4
|
+
require "sequel"
|
5
|
+
require "sequel/extensions/make_readyable"
|
6
|
+
|
7
|
+
class TestUsable < Minitest::Test
|
8
|
+
def setup
|
9
|
+
# These features are mostly intended for Spark, but sqlite is a close enough
|
10
|
+
# mock that we'll just roll with it
|
11
|
+
@db = Sequel.mock(host: :spark)
|
12
|
+
@db.extension :make_readyable
|
13
|
+
def @db.tables(opts = {})
|
14
|
+
case opts[:schema]
|
15
|
+
when :schema1
|
16
|
+
[:a]
|
17
|
+
when :schema2
|
18
|
+
[:a, :b]
|
19
|
+
when :schema3
|
20
|
+
[:a, :b]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_should_call_use_schema
|
26
|
+
@db.make_ready(use_schema: :some_schema)
|
27
|
+
assert_equal(["USE `some_schema`"], @db.sqls)
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_should_create_views_based_on_tables_in_search_paths
|
31
|
+
@db.make_ready(search_path: [:schema1, :schema2, :schema3])
|
32
|
+
assert_equal([
|
33
|
+
"CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`",
|
34
|
+
"CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
|
35
|
+
], @db.sqls)
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_should_create_views_based_on_tables_in_search_paths_passed_as_strings
|
39
|
+
@db.make_ready(search_path: ["schema1", "schema2", "schema3"])
|
40
|
+
assert_equal([
|
41
|
+
"CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`",
|
42
|
+
"CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
|
43
|
+
], @db.sqls)
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_should_create_views_based_on_tables_in_search_paths_accepts_except
|
47
|
+
@db.make_ready(search_path: [:schema1, :schema2, :schema3], except: :a)
|
48
|
+
assert_equal([
|
49
|
+
"CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
|
50
|
+
], @db.sqls)
|
51
|
+
end
|
52
|
+
|
53
|
+
def test_should_create_views_based_on_tables_in_search_paths_accepts_only
|
54
|
+
@db.make_ready(search_path: [:schema1, :schema2, :schema3], only: :b)
|
55
|
+
assert_equal([
|
56
|
+
"CREATE TEMPORARY VIEW `b` AS SELECT * FROM `schema2`.`b`"
|
57
|
+
], @db.sqls)
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_should_create_views_based_on_path
|
61
|
+
dir = Pathname.new(Dir.mktmpdir)
|
62
|
+
a_file = dir + "a.parquet"
|
63
|
+
b_file = dir + "b.parquet"
|
64
|
+
FileUtils.touch(a_file.to_s)
|
65
|
+
FileUtils.touch(b_file.to_s)
|
66
|
+
|
67
|
+
@db.make_ready(search_path: [:schema1, a_file, b_file, :schema2])
|
68
|
+
sqls = @db.sqls.dup
|
69
|
+
assert_equal("CREATE TEMPORARY VIEW `a` AS SELECT * FROM `schema1`.`a`", sqls[0])
|
70
|
+
assert_match(%r{CREATE TEMPORARY VIEW `b` USING parquet OPTIONS \('path'='/tmp/[^/]+/b.parquet'\)}, sqls[1])
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_should_create_views_format_based_on_path
|
74
|
+
dir = Pathname.new(Dir.mktmpdir)
|
75
|
+
a_file = dir + "a.parquet"
|
76
|
+
b_file = dir + "b.delta"
|
77
|
+
c_file = dir + "c.csv"
|
78
|
+
FileUtils.touch(a_file.to_s)
|
79
|
+
FileUtils.touch(b_file.to_s)
|
80
|
+
FileUtils.touch(c_file.to_s)
|
81
|
+
|
82
|
+
@db.make_ready(search_path: [a_file, b_file, c_file])
|
83
|
+
sqls = @db.sqls.dup
|
84
|
+
assert_match(%r{CREATE TEMPORARY VIEW `a` USING parquet OPTIONS \('path'='/tmp/[^/]+/a.parquet'\)}, sqls[0])
|
85
|
+
assert_match(%r{CREATE TEMPORARY VIEW `b` USING delta OPTIONS \('path'='/tmp/[^/]+/b.delta'\)}, sqls[1])
|
86
|
+
assert_match(%r{CREATE TEMPORARY VIEW `c` USING csv OPTIONS \('path'='/tmp/[^/]+/c.csv'\)}, sqls[2])
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require_relative '../../../test_helper'
|
2
|
+
require 'sequel'
|
3
|
+
require 'sequel/extensions/usable'
|
4
|
+
|
5
|
+
class TestUsable < Minitest::Test
|
6
|
+
def test_should_call_use
|
7
|
+
db = Sequel.mock(host: :sqlite)
|
8
|
+
db.extension :usable
|
9
|
+
db.use(:some_schema)
|
10
|
+
assert_equal(db.sqls, ["USE `some_schema`"])
|
11
|
+
end
|
12
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sequelizer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Duryea
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-04-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -145,8 +145,8 @@ executables:
|
|
145
145
|
extensions: []
|
146
146
|
extra_rdoc_files: []
|
147
147
|
files:
|
148
|
+
- ".github/workflows/test.yml"
|
148
149
|
- ".gitignore"
|
149
|
-
- ".travis.yml"
|
150
150
|
- CHANGELOG.md
|
151
151
|
- Gemfile
|
152
152
|
- Guardfile
|
@@ -158,7 +158,10 @@ files:
|
|
158
158
|
- examples/database.yml
|
159
159
|
- examples/dot_env.txt
|
160
160
|
- lib/sequel/extensions/db_opts.rb
|
161
|
+
- lib/sequel/extensions/make_readyable.rb
|
161
162
|
- lib/sequel/extensions/settable.rb
|
163
|
+
- lib/sequel/extensions/sqls.rb
|
164
|
+
- lib/sequel/extensions/usable.rb
|
162
165
|
- lib/sequelizer.rb
|
163
166
|
- lib/sequelizer/cli.rb
|
164
167
|
- lib/sequelizer/connection_maker.rb
|
@@ -171,6 +174,8 @@ files:
|
|
171
174
|
- lib/sequelizer/yaml_config.rb
|
172
175
|
- sequelizer.gemspec
|
173
176
|
- test/lib/sequel/extensions/test_db_opts.rb
|
177
|
+
- test/lib/sequel/extensions/test_make_readyable.rb
|
178
|
+
- test/lib/sequel/extensions/test_usable.rb
|
174
179
|
- test/lib/sequelizer/test_connection_maker.rb
|
175
180
|
- test/lib/sequelizer/test_env_config.rb
|
176
181
|
- test/lib/sequelizer/test_gemfile_modifier.rb
|
@@ -196,12 +201,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
196
201
|
- !ruby/object:Gem::Version
|
197
202
|
version: '0'
|
198
203
|
requirements: []
|
199
|
-
rubygems_version: 3.
|
204
|
+
rubygems_version: 3.4.6
|
200
205
|
signing_key:
|
201
206
|
specification_version: 4
|
202
207
|
summary: Sequel database connections via config/database.yml or .env
|
203
208
|
test_files:
|
204
209
|
- test/lib/sequel/extensions/test_db_opts.rb
|
210
|
+
- test/lib/sequel/extensions/test_make_readyable.rb
|
211
|
+
- test/lib/sequel/extensions/test_usable.rb
|
205
212
|
- test/lib/sequelizer/test_connection_maker.rb
|
206
213
|
- test/lib/sequelizer/test_env_config.rb
|
207
214
|
- test/lib/sequelizer/test_gemfile_modifier.rb
|
data/.travis.yml
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
rvm:
|
3
|
-
- 2.4
|
4
|
-
sudo: false
|
5
|
-
env:
|
6
|
-
secure: xFZ00J+axPQADZ2Dqc61Ljo56vT8/uwVzOFw9zxKQs3ThMfI8+OvlI7LV8sf0XPC5Cp+BObYCxzsqIOY8M8yo+NfknlNmlrQaPqi3lf+3tKvEFeFiQZ/jvbGReIdxRdPViVls1W3zEDdRwe9zUAiz7C+xBYCRfZRoGjfm7gx/4c=
|
7
|
-
before_install:
|
8
|
-
- gem install bundler
|
9
|
-
- bundle
|
10
|
-
script: bundle exec rake test
|
11
|
-
jobs:
|
12
|
-
include:
|
13
|
-
- stage: deploy
|
14
|
-
before_install: gem install tping
|
15
|
-
install: true
|
16
|
-
script:
|
17
|
-
- tping --token $TRAVIS_PRO_TOKEN --user outcomesinsights --repo t_shank --pro --branch $TRAVIS_BRANCH
|
18
|
-
- tping --token $TRAVIS_PRO_TOKEN --user outcomesinsights --repo jigsaw-diagram-editor --pro --branch $TRAVIS_BRANCH
|
19
|
-
notifications:
|
20
|
-
slack:
|
21
|
-
secure: b+ao+3BuBtM5nj/m1gP5AbrrTIdQiV/HkT1deXvY4gg5xZQDheDeLmOI7wSFP1o67BrwrAY7rpcwIP7S/99fudrU3rKI3+GLn8KoefdAv78Z4tsMs9rodJJ3Z3ZmnEdMK2i2+hCLJ1pzZ9Ae3e+GDHsBPkTz4+TNE1lrOPxDIUo=
|