purview 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -2
- data/.travis.yml +4 -10
- data/Gemfile +8 -6
- data/README.md +33 -36
- data/TODO +54 -89
- data/lib/purview/connections.rb +2 -0
- data/lib/purview/connections/sqlite.rb +11 -0
- data/lib/purview/databases.rb +2 -0
- data/lib/purview/databases/base.rb +7 -7
- data/lib/purview/databases/mssql.rb +180 -0
- data/lib/purview/databases/postgresql.rb +4 -0
- data/lib/purview/databases/sqlite.rb +173 -0
- data/lib/purview/dialects.rb +1 -0
- data/lib/purview/dialects/sqlite.rb +25 -0
- data/lib/purview/loaders.rb +2 -0
- data/lib/purview/loaders/mssql.rb +85 -0
- data/lib/purview/loaders/sqlite.rb +85 -0
- data/lib/purview/pullers.rb +1 -0
- data/lib/purview/pullers/sqlite.rb +15 -0
- data/lib/purview/raw_connections.rb +5 -2
- data/lib/purview/raw_connections/jdbc/mysql.rb +1 -1
- data/lib/purview/raw_connections/jdbc/postgres.rb +1 -1
- data/lib/purview/raw_connections/jdbc/sqlite3.rb +21 -0
- data/lib/purview/raw_connections/sqlite3.rb +34 -0
- data/lib/purview/structs/base.rb +2 -1
- data/lib/purview/version.rb +1 -1
- data/purview.gemspec +3 -2
- metadata +17 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ba79aba1227a4d2427c31df3f0915e4ca579cc2f
|
4
|
+
data.tar.gz: 8c1bc2ce35e5f9d96dbfcc6460856fc5c9725c98
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0f4f52d8dd55a5d59f73e7022c6007fcfaa4b2c66506cb3f73982ade552e3d9fdaa45a00987965056a20a646f78c0f3c9def2aade3820025a6bf782a0b5cab3
|
7
|
+
data.tar.gz: 43dd4c3d045d9cd242cd046768cc98d384ba8ad76706640143ebc55d581c132e774653cecdeb18a935a11ebfd1d9c98d4899372fa0618309af7a5f6a8ac9c6d5
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
@@ -2,13 +2,15 @@ source 'https://rubygems.org'
|
|
2
2
|
|
3
3
|
group :development do
|
4
4
|
if defined?(JRUBY_VERSION)
|
5
|
-
gem 'jdbc-jtds', '~> 1.
|
6
|
-
gem 'jdbc-mysql', '~> 5.
|
7
|
-
gem 'jdbc-postgres', '~> 9.
|
5
|
+
gem 'jdbc-jtds', '~> 1.0'
|
6
|
+
gem 'jdbc-mysql', '~> 5.0'
|
7
|
+
gem 'jdbc-postgres', '~> 9.0'
|
8
|
+
gem 'jdbc-sqlite3', '~> 3.0'
|
8
9
|
else
|
9
|
-
gem 'mysql2', '~> 0.
|
10
|
-
gem 'pg', '~> 0.
|
11
|
-
gem '
|
10
|
+
gem 'mysql2', '~> 0.4'
|
11
|
+
gem 'pg', '~> 0.20'
|
12
|
+
gem 'sqlite3', '~> 1.0'
|
13
|
+
gem 'tiny_tds', '~> 1.0'
|
12
14
|
end
|
13
15
|
end
|
14
16
|
|
data/README.md
CHANGED
@@ -21,7 +21,7 @@ Or install it yourself as:
|
|
21
21
|
|
22
22
|
## Usage
|
23
23
|
|
24
|
-
Load the `MySQL` client (for `MSSQL`
|
24
|
+
Load the `MySQL` client (for `MSSQL` simply change 'mysql2' to 'tiny_tds'; for
|
25
25
|
`PostgreSQL` simply change 'mysql2' to 'pg' -- when using this gem in a JRuby
|
26
26
|
environment the 'jdbc/jtds', 'jdbc/mysql' and/or 'jdbc/postgres'library must be
|
27
27
|
installed/available)
|
@@ -39,11 +39,11 @@ Define the `Column(s)` (available column-types: `Boolean`, `CreatedTimestamp`,
|
|
39
39
|
`UpdatedTimestamp` & `UUID` -- the `Id`, `CreatedTimestamp` & `UpdatedTimestamp`
|
40
40
|
columns are required for all `BaseSyncable` tables)
|
41
41
|
```ruby
|
42
|
-
id_column = Purview::Columns::Id.new(:id)
|
43
|
-
name_column = Purview::Columns::String.new(:name, :nullable => false)
|
44
|
-
email_column = Purview::Columns::String.new(:email, :nullable => false, :limit => 100)
|
45
|
-
created_at_column = Purview::Columns::CreatedTimestamp.new(:created_at)
|
46
|
-
updated_at_column = Purview::Columns::UpdatedTimestamp.new(:updated_at)
|
42
|
+
id_column = Purview::Columns::Id.new(:id)
|
43
|
+
name_column = Purview::Columns::String.new(:name, :nullable => false)
|
44
|
+
email_column = Purview::Columns::String.new(:email, :nullable => false, :limit => 100)
|
45
|
+
created_at_column = Purview::Columns::CreatedTimestamp.new(:created_at)
|
46
|
+
updated_at_column = Purview::Columns::UpdatedTimestamp.new(:updated_at)
|
47
47
|
|
48
48
|
columns = [
|
49
49
|
id_column,
|
@@ -58,32 +58,24 @@ Define the `Indices` (availble index-types: `Composite` & `Simple`). By default
|
|
58
58
|
`Indices` will be added for the required column-types (`CreatedTimestamp` &
|
59
59
|
`UpdatedTimestamp`)
|
60
60
|
```ruby
|
61
|
-
indices = [
|
62
|
-
Purview::Indices::Simple.new(email_column, :unique => true),
|
63
|
-
]
|
61
|
+
indices = [Purview::Indices::Simple.new(email_column, :unique => true)]
|
64
62
|
```
|
65
63
|
|
66
64
|
Configure the `Puller` (available puller-types: `MSSQL`, `MySQL`, `PostgreSQL` &
|
67
65
|
`URI`)
|
68
66
|
```ruby
|
69
|
-
puller_opts = {
|
70
|
-
:type => Purview::Pullers::URI,
|
71
|
-
:uri => 'http://feed.test.com/users',
|
72
|
-
}
|
67
|
+
puller_opts = {:type => Purview::Pullers::URI, :uri => 'http://feed.test.com/users'}
|
73
68
|
```
|
74
69
|
|
75
70
|
Configure the `Parser` (available parser-types: `CSV`, `SQL` & `TSV`)
|
76
71
|
```ruby
|
77
|
-
parser_opts = {
|
78
|
-
:type => Purview::Parsers::TSV,
|
79
|
-
}
|
72
|
+
parser_opts = {:type => Purview::Parsers::TSV}
|
80
73
|
```
|
81
74
|
|
82
|
-
Configure the `Loader` (for
|
75
|
+
Configure the `Loader` (for a different loader type simply change `MySQL` to
|
76
|
+
`MSSQL`, `PostgreSQL` or `SQLite`)
|
83
77
|
```ruby
|
84
|
-
loader_opts = {
|
85
|
-
:type => Purview::Loaders::MySQL,
|
86
|
-
}
|
78
|
+
loader_opts = {:type => Purview::Loaders::MySQL}
|
87
79
|
```
|
88
80
|
|
89
81
|
Combine all the configuration options and instantiate the `Table`
|
@@ -96,10 +88,7 @@ table_opts = {
|
|
96
88
|
:puller => puller_opts,
|
97
89
|
}
|
98
90
|
|
99
|
-
table = Purview::Tables::Raw.new(
|
100
|
-
table_name,
|
101
|
-
table_opts
|
102
|
-
)
|
91
|
+
table = Purview::Tables::Raw.new(table_name, table_opts)
|
103
92
|
```
|
104
93
|
|
105
94
|
Set the database-name (this can be anything, but it must exist)
|
@@ -107,22 +96,19 @@ Set the database-name (this can be anything, but it must exist)
|
|
107
96
|
database_name = :data_warehouse_raw
|
108
97
|
```
|
109
98
|
|
110
|
-
Combine all the configuration options and instantiate the `Database` (for
|
111
|
-
|
99
|
+
Combine all the configuration options and instantiate the `Database` (for a
|
100
|
+
different database engine simply change `MySQL` to `MSSQL`, `PostgreSQL` or
|
101
|
+
`SQLite`)
|
112
102
|
```ruby
|
113
|
-
database_opts = {
|
114
|
-
:tables => [table],
|
115
|
-
}
|
103
|
+
database_opts = {:tables => [table]}
|
116
104
|
|
117
|
-
database = Purview::Databases::MySQL.new(
|
118
|
-
database_name,
|
119
|
-
database_opts
|
120
|
-
)
|
105
|
+
database = Purview::Databases::MySQL.new(database_name, database_opts)
|
121
106
|
```
|
122
107
|
|
123
108
|
Create the `Table` (in the DB). Recommended for testing purposes *only*. For
|
124
109
|
production environments you will likely want an external process to manage the
|
125
|
-
schema (for `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable
|
110
|
+
schema (for `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`,
|
111
|
+
for `SQLite` simply change `Mysql2::Error` to `SQLite3::SQLException`, etc.)
|
126
112
|
```ruby
|
127
113
|
begin
|
128
114
|
database.create_table(table)
|
@@ -135,9 +121,11 @@ Initialize the `Table` (in the DB). This process sets the `max_timestamp_pulled`
|
|
135
121
|
value in the `table_metadata` table and is used by the candidate `Table`
|
136
122
|
selection algorithm to determine which `Table` should be synchronized next (the
|
137
123
|
least recently synchronized `Table` will be selected). This value is also used
|
138
|
-
as the high-water mark for records pulled from its source
|
124
|
+
as the high-water mark for records pulled from its source. Unless a `timestamp`
|
125
|
+
is specified, as the second argument, the high-water mark will default to
|
126
|
+
`Time.now.utc`
|
139
127
|
```ruby
|
140
|
-
database.initialize_table(table
|
128
|
+
database.initialize_table(table)
|
141
129
|
```
|
142
130
|
|
143
131
|
Baseline the `Table`. This process will quickly get the state of the `Table` as
|
@@ -156,6 +144,15 @@ to determine the pool of `Table(s)` available for synchronization (to remove a
|
|
156
144
|
database.enable_table(table)
|
157
145
|
```
|
158
146
|
|
147
|
+
Disable the `Table` (in the DB). This process clears the `enabled_at` value in
|
148
|
+
the `table_metadata` table which will remove the table from the candidate `Table`
|
149
|
+
selection algorithm used to determine the pool of `Table(s)` available for
|
150
|
+
synchronization (to add a `Table` back into the pool, simply execute
|
151
|
+
`enable_table`)
|
152
|
+
```ruby
|
153
|
+
database.disable_table(table)
|
154
|
+
```
|
155
|
+
|
159
156
|
Sync the `Table`. This process will pull data from its [remote-]source and
|
160
157
|
reconcile the new data against the main-table (e.g. perform 'INSERT', 'UPDATE'
|
161
158
|
and 'DELETE' operations).
|
data/TODO
CHANGED
@@ -1,89 +1,54 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
* Add comments where appropriate
|
56
|
-
* DRY/Refactor
|
57
|
-
|
58
|
-
... OPEN TASKS ....
|
59
|
-
|
60
|
-
* Add COPY `def copy(sql)` support to PostgreSQL connection
|
61
|
-
* Leverage COPY support for PostgreSQL pulls
|
62
|
-
* Further encapsulate `Dialect` logic (in order to DRY `Database` classes)
|
63
|
-
* Add support for MSSQL
|
64
|
-
|
65
|
-
* Build out change-log tables
|
66
|
-
* Build out canonical, fact and aggregate tables (and related transforms)
|
67
|
-
* Configurable re-pull window (do this automatically once up to current?)
|
68
|
-
* Add schema management capabilities (detect schema-deltas and suggestion
|
69
|
-
modifications)
|
70
|
-
* Consider refactoring mutator methods to be globally available (e.g. quoted ->
|
71
|
-
Quote, etc.)
|
72
|
-
|
73
|
-
... CLOSED QUESTIONS ...
|
74
|
-
|
75
|
-
* Where is the best place to create the connection?
|
76
|
-
* Immediately? Then it's open during the pull, etc.
|
77
|
-
* Lazily? Makes for connection logic all over the place
|
78
|
-
* Perhaps it can be centralized entirely into the `Database` class?
|
79
|
-
* Consider using ISO-8601 timestamps in query-string (not sure on this one, it
|
80
|
-
is slightly more fragile because of URL-encoding, etc.)
|
81
|
-
|
82
|
-
... OPEN QUESTIONS ...
|
83
|
-
|
84
|
-
* Parallel pulls from the same "source"? Can this be done in one request?
|
85
|
-
|
86
|
-
... INTEGRATION CONSIDERATIONS ...
|
87
|
-
|
88
|
-
* Deployment
|
89
|
-
* Scheduling (Daemon? CRON?)
|
1
|
+
a 1450000000 1450000000 Add COPY `def copy(sql)` support to PostgreSQL connection
|
2
|
+
a 1450000000 1450000000 Leverage COPY support for PostgreSQL pulls
|
3
|
+
a 1450000000 1450000000 Build out change-log tables
|
4
|
+
a 1450000000 1450000000 Build out canonical, fact and aggregate tables (and related transforms)
|
5
|
+
a 1450000000 1450000000 Configurable re-pull window (do this automatically once up to current?)
|
6
|
+
a 1450000000 1450000000 Add schema management capabilities (detect schema-deltas and suggestion modifications)
|
7
|
+
a 1450000000 1450000000 Consider refactoring mutator methods to be globally available (e.g. quoted -> Quote, etc.)
|
8
|
+
a 1450000000 1450000000 Support parallel pulls?
|
9
|
+
a 1450000000 1450000000 Come up w/ a deployment recommendation
|
10
|
+
r 1450000000 1450000000 Add support for MSSQL
|
11
|
+
r 1450000000 1495070870 Further encapsulate `Dialect` logic (in order to DRY `Database` classes)
|
12
|
+
d 1494850714 1495046648 Finish SQLite3 raw-connections
|
13
|
+
d 1495046665 1495046689 Fix JDBC driver configuration
|
14
|
+
d 1495051831 1495070248 Code-review recent changes and merge
|
15
|
+
d 1450000000 1450000000 Handle blank values a bit more intelligently (all blanks -> nil)
|
16
|
+
d 1450000000 1450000000 Deal w/ NULL values
|
17
|
+
d 1450000000 1450000000 Deal w/ BOOL values
|
18
|
+
d 1450000000 1450000000 Build out MVP for PostgreSQL loader
|
19
|
+
d 1450000000 1450000000 Add windowing to pulls (simulate w/ 2 different data-files if necessary)
|
20
|
+
d 1450000000 1450000000 Include [window] timestamps in query-string for pull[er]
|
21
|
+
d 1450000000 1450000000 Figure out an intelligent way to solve the chicken <-> egg problem for `table_metadata`
|
22
|
+
d 1450000000 1450000000 Fail fast if there are no candidate tables (`next_table`)
|
23
|
+
d 1450000000 1450000000 Ensure that the puller does not try to go into the future (`next_window`)
|
24
|
+
d 1450000000 1450000000 Add in a primitive logging facility (can be temporary)
|
25
|
+
d 1450000000 1450000000 Enhance `next_table_sql`
|
26
|
+
d 1450000000 1450000000 Lock tables (feeds) during pull, parse and load
|
27
|
+
d 1450000000 1450000000 Verify table-locking mechanism behaves as expected
|
28
|
+
d 1450000000 1450000000 Ability to drop a table
|
29
|
+
d 1450000000 1450000000 Reorder args: "table, table_name" to: "table_name, table"
|
30
|
+
d 1450000000 1450000000 Reorder args: "table, index_name, *columns" to "index_name, table, *columns"
|
31
|
+
d 1450000000 1450000000 Ability to create an index
|
32
|
+
d 1450000000 1450000000 Ability to drop an index
|
33
|
+
d 1450000000 1450000000 Indices on `created_at` and `updated_at` columns
|
34
|
+
d 1450000000 1450000000 Gracefully handle table {,un}locking in case of error
|
35
|
+
d 1450000000 1450000000 Add debugging code
|
36
|
+
d 1450000000 1450000000 Raise no-window & no-table exceptions (making them catchable upstream)
|
37
|
+
d 1450000000 1450000000 Minimize race-condition around table-locking (use where on UPDATE, raise if zero rows are updated)
|
38
|
+
d 1450000000 1450000000 Log number of delete, inserts and updates in `loader`
|
39
|
+
d 1450000000 1450000000 `load_temporary_table` should bomb if there is data outside the window
|
40
|
+
d 1450000000 1450000000 Delete from outside window where id in temporary table
|
41
|
+
d 1450000000 1450000000 Rename `data` to `rows` in `execute`
|
42
|
+
d 1450000000 1450000000 Rename `Postgres` to `PostgreSQL`
|
43
|
+
d 1450000000 1450000000 Finish PostgreSQL support (requires `pg` gem)
|
44
|
+
d 1450000000 1450000000 Finish MySQL support (requires `mysql2` gem)
|
45
|
+
d 1450000000 1450000000 Usage/configuration examples in README.md
|
46
|
+
d 1450000000 1450000000 Create SQL-puller(s)
|
47
|
+
d 1450000000 1450000000 Introduce [SQL] dialect(s)
|
48
|
+
d 1450000000 1450000000 Fix JRuby bundle/build
|
49
|
+
d 1450000000 1450000000 Fall back to logged in user if no [database-]username is provided
|
50
|
+
d 1450000000 1450000000 Create class to encapulate `table_metadata` (in order to clean up the logic in the `Database` class(es))
|
51
|
+
d 1450000000 1450000000 Rename columns during pull/parse `source_name` & `target_name` perhaps? Maybe create a new type of column?
|
52
|
+
d 1450000000 1495070826 Add support for SQLite3
|
53
|
+
d 1494850696 1495070857 Add TODOs from $HOME\purview-TODO
|
54
|
+
d 1495070065 1495072292 Finish MSSQL built-out
|
data/lib/purview/connections.rb
CHANGED
data/lib/purview/databases.rb
CHANGED
@@ -13,7 +13,7 @@ module Purview
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
-
def baseline_table(table)
|
16
|
+
def baseline_table(table, timestamp=Time.now.utc)
|
17
17
|
ensure_table_valid_for_database(table)
|
18
18
|
raise Purview::Exceptions::CouldNotBaselineTable.new(table) \
|
19
19
|
unless table_initialized?(table)
|
@@ -84,7 +84,7 @@ module Purview
|
|
84
84
|
table_name
|
85
85
|
end
|
86
86
|
|
87
|
-
def disable_table(table)
|
87
|
+
def disable_table(table, timestamp=Time.now.utc)
|
88
88
|
ensure_table_valid_for_database(table)
|
89
89
|
table_name = table_name(table)
|
90
90
|
with_context_logging("`disable_table` for: #{table_name}") do
|
@@ -142,7 +142,7 @@ module Purview
|
|
142
142
|
table_name
|
143
143
|
end
|
144
144
|
|
145
|
-
def enable_table(table, timestamp=
|
145
|
+
def enable_table(table, timestamp=Time.now.utc)
|
146
146
|
ensure_table_valid_for_database(table)
|
147
147
|
table_name = table_name(table)
|
148
148
|
with_context_logging("`enable_table` for: #{table_name}") do
|
@@ -156,7 +156,7 @@ module Purview
|
|
156
156
|
table_name
|
157
157
|
end
|
158
158
|
|
159
|
-
def initialize_table(table, timestamp=
|
159
|
+
def initialize_table(table, timestamp=Time.now.utc)
|
160
160
|
ensure_table_valid_for_database(table)
|
161
161
|
table_name = table_name(table)
|
162
162
|
with_context_logging("`initialize_table` for: #{table_name}") do
|
@@ -170,7 +170,7 @@ module Purview
|
|
170
170
|
table_name
|
171
171
|
end
|
172
172
|
|
173
|
-
def lock_table(table, timestamp=
|
173
|
+
def lock_table(table, timestamp=Time.now.utc)
|
174
174
|
ensure_table_valid_for_database(table)
|
175
175
|
table_name = table_name(table)
|
176
176
|
with_context_logging("`lock_table` for: #{table_name}") do
|
@@ -194,7 +194,7 @@ module Purview
|
|
194
194
|
end
|
195
195
|
end
|
196
196
|
|
197
|
-
def sync_table(table)
|
197
|
+
def sync_table(table, timestamp=Time.now.utc)
|
198
198
|
ensure_table_valid_for_database(table)
|
199
199
|
raise Purview::Exceptions::CouldNotSyncTable.new(table) \
|
200
200
|
unless table_initialized?(table) && table_enabled?(table)
|
@@ -211,7 +211,7 @@ module Purview
|
|
211
211
|
table_name = table_name(table)
|
212
212
|
with_context_logging("`table_metadata` for: #{table_name}") do
|
213
213
|
with_new_connection do |connection|
|
214
|
-
Purview::Structs::TableMetadata.new(
|
214
|
+
table_metadata = Purview::Structs::TableMetadata.new(
|
215
215
|
table_metadata_table.columns.reduce({}) do |memo, column|
|
216
216
|
memo[column.name] = get_table_metadata_value(
|
217
217
|
connection,
|
@@ -0,0 +1,180 @@
|
|
1
|
+
module Purview
|
2
|
+
module Databases
|
3
|
+
class MSSQL < Base
|
4
|
+
private
|
5
|
+
|
6
|
+
def connection_type
|
7
|
+
Purview::Connections::MSSQL
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
11
|
+
'CREATE%sINDEX %s ON %s (%s)' % [
|
12
|
+
index.unique? ? ' UNIQUE ' : ' ',
|
13
|
+
index_name,
|
14
|
+
table_name,
|
15
|
+
column_names(index).join(', '),
|
16
|
+
]
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_table_sql(table_name, table, table_opts={})
|
20
|
+
'CREATE TABLE %s (%s)' % [
|
21
|
+
table_name,
|
22
|
+
column_definitions(table).join(', '),
|
23
|
+
]
|
24
|
+
end
|
25
|
+
|
26
|
+
def create_temporary_table_sql(table_name, table, table_opts={})
|
27
|
+
'CREATE TEMPORARY TABLE %s (%s)' % [
|
28
|
+
table_name,
|
29
|
+
column_definitions(table).join(', '),
|
30
|
+
]
|
31
|
+
end
|
32
|
+
|
33
|
+
def dialect_type
|
34
|
+
Purview::Dialects::MSSQL
|
35
|
+
end
|
36
|
+
|
37
|
+
def disable_table_sql(table)
|
38
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
39
|
+
table_metadata_table.name,
|
40
|
+
table_metadata_table.enabled_at_column.name,
|
41
|
+
null_value,
|
42
|
+
table_metadata_table.table_name_column.name,
|
43
|
+
quoted(table.name),
|
44
|
+
table_metadata_table.enabled_at_column.name,
|
45
|
+
]
|
46
|
+
end
|
47
|
+
|
48
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
49
|
+
'DROP INDEX %s' % [
|
50
|
+
index_name,
|
51
|
+
]
|
52
|
+
end
|
53
|
+
|
54
|
+
def drop_table_sql(table_name, table, table_opts={})
|
55
|
+
'DROP TABLE %s' % [
|
56
|
+
table_name,
|
57
|
+
]
|
58
|
+
end
|
59
|
+
|
60
|
+
def enable_table_sql(table, timestamp)
|
61
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
62
|
+
table_metadata_table.name,
|
63
|
+
table_metadata_table.enabled_at_column.name,
|
64
|
+
quoted(timestamp),
|
65
|
+
table_metadata_table.table_name_column.name,
|
66
|
+
quoted(table.name),
|
67
|
+
table_metadata_table.enabled_at_column.name,
|
68
|
+
]
|
69
|
+
end
|
70
|
+
|
71
|
+
def ensure_table_metadata_absent_for_table_sql(table)
|
72
|
+
'DELETE FROM %s WHERE %s = %s' % [
|
73
|
+
table_metadata_table.name,
|
74
|
+
table_metadata_table.table_name_column.name,
|
75
|
+
quoted(table.name),
|
76
|
+
]
|
77
|
+
end
|
78
|
+
|
79
|
+
def ensure_table_metadata_exists_for_table_sql(table)
|
80
|
+
'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
|
81
|
+
table_metadata_table.name,
|
82
|
+
table_metadata_table.table_name_column.name,
|
83
|
+
quoted(table.name),
|
84
|
+
table_metadata_table.name,
|
85
|
+
table_metadata_table.table_name_column.name,
|
86
|
+
quoted(table.name),
|
87
|
+
]
|
88
|
+
end
|
89
|
+
|
90
|
+
def ensure_table_metadata_table_exists_sql
|
91
|
+
'CREATE TABLE IF NOT EXISTS %s (%s)' % [
|
92
|
+
table_metadata_table.name,
|
93
|
+
column_definitions(table_metadata_table).join(', '),
|
94
|
+
]
|
95
|
+
end
|
96
|
+
|
97
|
+
def initialize_table_sql(table, timestamp)
|
98
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
99
|
+
table_metadata_table.name,
|
100
|
+
table_metadata_table.max_timestamp_pulled_column.name,
|
101
|
+
quoted(timestamp),
|
102
|
+
table_metadata_table.table_name_column.name,
|
103
|
+
quoted(table.name),
|
104
|
+
table_metadata_table.max_timestamp_pulled_column.name,
|
105
|
+
]
|
106
|
+
end
|
107
|
+
|
108
|
+
def get_table_metadata_value_sql(table, column)
|
109
|
+
'SELECT %s FROM %s WHERE %s = %s' % [
|
110
|
+
column.name,
|
111
|
+
table_metadata_table.name,
|
112
|
+
table_metadata_table.table_name_column.name,
|
113
|
+
quoted(table.name),
|
114
|
+
]
|
115
|
+
end
|
116
|
+
|
117
|
+
def limit_map
|
118
|
+
super.merge(Purview::Types::String => 255)
|
119
|
+
end
|
120
|
+
|
121
|
+
def limitless_types
|
122
|
+
super + [
|
123
|
+
Purview::Types::Money,
|
124
|
+
Purview::Types::UUID,
|
125
|
+
]
|
126
|
+
end
|
127
|
+
|
128
|
+
def lock_table_sql(table, timestamp)
|
129
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
130
|
+
table_metadata_table.name,
|
131
|
+
table_metadata_table.locked_at_column.name,
|
132
|
+
quoted(timestamp),
|
133
|
+
table_metadata_table.table_name_column.name,
|
134
|
+
quoted(table.name),
|
135
|
+
table_metadata_table.locked_at_column.name,
|
136
|
+
]
|
137
|
+
end
|
138
|
+
|
139
|
+
def next_table_sql(timestamp)
|
140
|
+
'SELECT TOP 1 %s FROM %s WHERE %s IS NOT NULL AND %s IS NOT NULL AND %s IS NULL ORDER BY (CASE WHEN %s IS NULL THEN 0 ELSE 1 END), %s' % [
|
141
|
+
table_metadata_table.table_name_column.name,
|
142
|
+
table_metadata_table.name,
|
143
|
+
table_metadata_table.enabled_at_column.name,
|
144
|
+
table_metadata_table.max_timestamp_pulled_column.name,
|
145
|
+
table_metadata_table.locked_at_column.name,
|
146
|
+
table_metadata_table.last_pulled_at_column.name,
|
147
|
+
table_metadata_table.last_pulled_at_column.name,
|
148
|
+
]
|
149
|
+
end
|
150
|
+
|
151
|
+
def set_table_metadata_value_sql(table, column, value)
|
152
|
+
'UPDATE %s SET %s = %s WHERE %s = %s' % [
|
153
|
+
table_metadata_table.name,
|
154
|
+
column.name,
|
155
|
+
quoted(value),
|
156
|
+
table_metadata_table.table_name_column.name,
|
157
|
+
quoted(table.name),
|
158
|
+
]
|
159
|
+
end
|
160
|
+
|
161
|
+
def type_map
|
162
|
+
super.merge(
|
163
|
+
Purview::Types::Money => 'money',
|
164
|
+
Purview::Types::UUID => 'uniqueidentifier',
|
165
|
+
)
|
166
|
+
end
|
167
|
+
|
168
|
+
def unlock_table_sql(table)
|
169
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
170
|
+
table_metadata_table.name,
|
171
|
+
table_metadata_table.locked_at_column.name,
|
172
|
+
null_value,
|
173
|
+
table_metadata_table.table_name_column.name,
|
174
|
+
quoted(table.name),
|
175
|
+
table_metadata_table.locked_at_column.name,
|
176
|
+
]
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|