purview 1.5.0 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +4 -2
- data/.travis.yml +4 -10
- data/Gemfile +8 -6
- data/README.md +33 -36
- data/TODO +54 -89
- data/lib/purview/connections.rb +2 -0
- data/lib/purview/connections/sqlite.rb +11 -0
- data/lib/purview/databases.rb +2 -0
- data/lib/purview/databases/base.rb +7 -7
- data/lib/purview/databases/mssql.rb +180 -0
- data/lib/purview/databases/postgresql.rb +4 -0
- data/lib/purview/databases/sqlite.rb +173 -0
- data/lib/purview/dialects.rb +1 -0
- data/lib/purview/dialects/sqlite.rb +25 -0
- data/lib/purview/loaders.rb +2 -0
- data/lib/purview/loaders/mssql.rb +85 -0
- data/lib/purview/loaders/sqlite.rb +85 -0
- data/lib/purview/pullers.rb +1 -0
- data/lib/purview/pullers/sqlite.rb +15 -0
- data/lib/purview/raw_connections.rb +5 -2
- data/lib/purview/raw_connections/jdbc/mysql.rb +1 -1
- data/lib/purview/raw_connections/jdbc/postgres.rb +1 -1
- data/lib/purview/raw_connections/jdbc/sqlite3.rb +21 -0
- data/lib/purview/raw_connections/sqlite3.rb +34 -0
- data/lib/purview/structs/base.rb +2 -1
- data/lib/purview/version.rb +1 -1
- data/purview.gemspec +3 -2
- metadata +17 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ba79aba1227a4d2427c31df3f0915e4ca579cc2f
|
4
|
+
data.tar.gz: 8c1bc2ce35e5f9d96dbfcc6460856fc5c9725c98
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0f4f52d8dd55a5d59f73e7022c6007fcfaa4b2c66506cb3f73982ade552e3d9fdaa45a00987965056a20a646f78c0f3c9def2aade3820025a6bf782a0b5cab3
|
7
|
+
data.tar.gz: 43dd4c3d045d9cd242cd046768cc98d384ba8ad76706640143ebc55d581c132e774653cecdeb18a935a11ebfd1d9c98d4899372fa0618309af7a5f6a8ac9c6d5
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
@@ -2,13 +2,15 @@ source 'https://rubygems.org'
|
|
2
2
|
|
3
3
|
group :development do
|
4
4
|
if defined?(JRUBY_VERSION)
|
5
|
-
gem 'jdbc-jtds', '~> 1.
|
6
|
-
gem 'jdbc-mysql', '~> 5.
|
7
|
-
gem 'jdbc-postgres', '~> 9.
|
5
|
+
gem 'jdbc-jtds', '~> 1.0'
|
6
|
+
gem 'jdbc-mysql', '~> 5.0'
|
7
|
+
gem 'jdbc-postgres', '~> 9.0'
|
8
|
+
gem 'jdbc-sqlite3', '~> 3.0'
|
8
9
|
else
|
9
|
-
gem 'mysql2', '~> 0.
|
10
|
-
gem 'pg', '~> 0.
|
11
|
-
gem '
|
10
|
+
gem 'mysql2', '~> 0.4'
|
11
|
+
gem 'pg', '~> 0.20'
|
12
|
+
gem 'sqlite3', '~> 1.0'
|
13
|
+
gem 'tiny_tds', '~> 1.0'
|
12
14
|
end
|
13
15
|
end
|
14
16
|
|
data/README.md
CHANGED
@@ -21,7 +21,7 @@ Or install it yourself as:
|
|
21
21
|
|
22
22
|
## Usage
|
23
23
|
|
24
|
-
Load the `MySQL` client (for `MSSQL`
|
24
|
+
Load the `MySQL` client (for `MSSQL` simply change 'mysql2' to 'tiny_tds'; for
|
25
25
|
`PostgreSQL` simply change 'mysql2' to 'pg' -- when using this gem in a JRuby
|
26
26
|
environment the 'jdbc/jtds', 'jdbc/mysql' and/or 'jdbc/postgres'library must be
|
27
27
|
installed/available)
|
@@ -39,11 +39,11 @@ Define the `Column(s)` (available column-types: `Boolean`, `CreatedTimestamp`,
|
|
39
39
|
`UpdatedTimestamp` & `UUID` -- the `Id`, `CreatedTimestamp` & `UpdatedTimestamp`
|
40
40
|
columns are required for all `BaseSyncable` tables)
|
41
41
|
```ruby
|
42
|
-
id_column = Purview::Columns::Id.new(:id)
|
43
|
-
name_column = Purview::Columns::String.new(:name, :nullable => false)
|
44
|
-
email_column = Purview::Columns::String.new(:email, :nullable => false, :limit => 100)
|
45
|
-
created_at_column = Purview::Columns::CreatedTimestamp.new(:created_at)
|
46
|
-
updated_at_column = Purview::Columns::UpdatedTimestamp.new(:updated_at)
|
42
|
+
id_column = Purview::Columns::Id.new(:id)
|
43
|
+
name_column = Purview::Columns::String.new(:name, :nullable => false)
|
44
|
+
email_column = Purview::Columns::String.new(:email, :nullable => false, :limit => 100)
|
45
|
+
created_at_column = Purview::Columns::CreatedTimestamp.new(:created_at)
|
46
|
+
updated_at_column = Purview::Columns::UpdatedTimestamp.new(:updated_at)
|
47
47
|
|
48
48
|
columns = [
|
49
49
|
id_column,
|
@@ -58,32 +58,24 @@ Define the `Indices` (availble index-types: `Composite` & `Simple`). By default
|
|
58
58
|
`Indices` will be added for the required column-types (`CreatedTimestamp` &
|
59
59
|
`UpdatedTimestamp`)
|
60
60
|
```ruby
|
61
|
-
indices = [
|
62
|
-
Purview::Indices::Simple.new(email_column, :unique => true),
|
63
|
-
]
|
61
|
+
indices = [Purview::Indices::Simple.new(email_column, :unique => true)]
|
64
62
|
```
|
65
63
|
|
66
64
|
Configure the `Puller` (available puller-types: `MSSQL`, `MySQL`, `PostgreSQL` &
|
67
65
|
`URI`)
|
68
66
|
```ruby
|
69
|
-
puller_opts = {
|
70
|
-
:type => Purview::Pullers::URI,
|
71
|
-
:uri => 'http://feed.test.com/users',
|
72
|
-
}
|
67
|
+
puller_opts = {:type => Purview::Pullers::URI, :uri => 'http://feed.test.com/users'}
|
73
68
|
```
|
74
69
|
|
75
70
|
Configure the `Parser` (available parser-types: `CSV`, `SQL` & `TSV`)
|
76
71
|
```ruby
|
77
|
-
parser_opts = {
|
78
|
-
:type => Purview::Parsers::TSV,
|
79
|
-
}
|
72
|
+
parser_opts = {:type => Purview::Parsers::TSV}
|
80
73
|
```
|
81
74
|
|
82
|
-
Configure the `Loader` (for
|
75
|
+
Configure the `Loader` (for a different loader type simply change `MySQL` to
|
76
|
+
`MSSQL`, `PostgreSQL` or `SQLite`)
|
83
77
|
```ruby
|
84
|
-
loader_opts = {
|
85
|
-
:type => Purview::Loaders::MySQL,
|
86
|
-
}
|
78
|
+
loader_opts = {:type => Purview::Loaders::MySQL}
|
87
79
|
```
|
88
80
|
|
89
81
|
Combine all the configuration options and instantiate the `Table`
|
@@ -96,10 +88,7 @@ table_opts = {
|
|
96
88
|
:puller => puller_opts,
|
97
89
|
}
|
98
90
|
|
99
|
-
table = Purview::Tables::Raw.new(
|
100
|
-
table_name,
|
101
|
-
table_opts
|
102
|
-
)
|
91
|
+
table = Purview::Tables::Raw.new(table_name, table_opts)
|
103
92
|
```
|
104
93
|
|
105
94
|
Set the database-name (this can be anything, but it must exist)
|
@@ -107,22 +96,19 @@ Set the database-name (this can be anything, but it must exist)
|
|
107
96
|
database_name = :data_warehouse_raw
|
108
97
|
```
|
109
98
|
|
110
|
-
Combine all the configuration options and instantiate the `Database` (for
|
111
|
-
|
99
|
+
Combine all the configuration options and instantiate the `Database` (for a
|
100
|
+
different database engine simply change `MySQL` to `MSSQL`, `PostgreSQL` or
|
101
|
+
`SQLite`)
|
112
102
|
```ruby
|
113
|
-
database_opts = {
|
114
|
-
:tables => [table],
|
115
|
-
}
|
103
|
+
database_opts = {:tables => [table]}
|
116
104
|
|
117
|
-
database = Purview::Databases::MySQL.new(
|
118
|
-
database_name,
|
119
|
-
database_opts
|
120
|
-
)
|
105
|
+
database = Purview::Databases::MySQL.new(database_name, database_opts)
|
121
106
|
```
|
122
107
|
|
123
108
|
Create the `Table` (in the DB). Recommended for testing purposes *only*. For
|
124
109
|
production environments you will likely want an external process to manage the
|
125
|
-
schema (for `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable
|
110
|
+
schema (for `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`,
|
111
|
+
for `SQLite` simply change `Mysql2::Error` to `SQLite3::SQLException`, etc.)
|
126
112
|
```ruby
|
127
113
|
begin
|
128
114
|
database.create_table(table)
|
@@ -135,9 +121,11 @@ Initialize the `Table` (in the DB). This process sets the `max_timestamp_pulled`
|
|
135
121
|
value in the `table_metadata` table and is used by the candidate `Table`
|
136
122
|
selection algorithm to determine which `Table` should be synchronized next (the
|
137
123
|
least recently synchronized `Table` will be selected). This value is also used
|
138
|
-
as the high-water mark for records pulled from its source
|
124
|
+
as the high-water mark for records pulled from its source. Unless a `timestamp`
|
125
|
+
is specified, as the second argument, the high-water mark will default to
|
126
|
+
`Time.now.utc`
|
139
127
|
```ruby
|
140
|
-
database.initialize_table(table
|
128
|
+
database.initialize_table(table)
|
141
129
|
```
|
142
130
|
|
143
131
|
Baseline the `Table`. This process will quickly get the state of the `Table` as
|
@@ -156,6 +144,15 @@ to determine the pool of `Table(s)` available for synchronization (to remove a
|
|
156
144
|
database.enable_table(table)
|
157
145
|
```
|
158
146
|
|
147
|
+
Disable the `Table` (in the DB). This process clears the `enabled_at` value in
|
148
|
+
the `table_metadata` table which will remove the table from the candidate `Table`
|
149
|
+
selection algorithm used to determine the pool of `Table(s)` available for
|
150
|
+
synchronization (to add a `Table` back into the pool, simply execute
|
151
|
+
`enable_table`)
|
152
|
+
```ruby
|
153
|
+
database.disable_table(table)
|
154
|
+
```
|
155
|
+
|
159
156
|
Sync the `Table`. This process will pull data from its [remote-]source and
|
160
157
|
reconcile the new data against the main-table (e.g. perform 'INSERT', 'UPDATE'
|
161
158
|
and 'DELETE' operations).
|
data/TODO
CHANGED
@@ -1,89 +1,54 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
* Add comments where appropriate
|
56
|
-
* DRY/Refactor
|
57
|
-
|
58
|
-
... OPEN TASKS ....
|
59
|
-
|
60
|
-
* Add COPY `def copy(sql)` support to PostgreSQL connection
|
61
|
-
* Leverage COPY support for PostgreSQL pulls
|
62
|
-
* Further encapsulate `Dialect` logic (in order to DRY `Database` classes)
|
63
|
-
* Add support for MSSQL
|
64
|
-
|
65
|
-
* Build out change-log tables
|
66
|
-
* Build out canonical, fact and aggregate tables (and related transforms)
|
67
|
-
* Configurable re-pull window (do this automatically once up to current?)
|
68
|
-
* Add schema management capabilities (detect schema-deltas and suggestion
|
69
|
-
modifications)
|
70
|
-
* Consider refactoring mutator methods to be globally available (e.g. quoted ->
|
71
|
-
Quote, etc.)
|
72
|
-
|
73
|
-
... CLOSED QUESTIONS ...
|
74
|
-
|
75
|
-
* Where is the best place to create the connection?
|
76
|
-
* Immediately? Then it's open during the pull, etc.
|
77
|
-
* Lazily? Makes for connection logic all over the place
|
78
|
-
* Perhaps it can be centralized entirely into the `Database` class?
|
79
|
-
* Consider using ISO-8601 timestamps in query-string (not sure on this one, it
|
80
|
-
is slightly more fragile because of URL-encoding, etc.)
|
81
|
-
|
82
|
-
... OPEN QUESTIONS ...
|
83
|
-
|
84
|
-
* Parallel pulls from the same "source"? Can this be done in one request?
|
85
|
-
|
86
|
-
... INTEGRATION CONSIDERATIONS ...
|
87
|
-
|
88
|
-
* Deployment
|
89
|
-
* Scheduling (Daemon? CRON?)
|
1
|
+
a 1450000000 1450000000 Add COPY `def copy(sql)` support to PostgreSQL connection
|
2
|
+
a 1450000000 1450000000 Leverage COPY support for PostgreSQL pulls
|
3
|
+
a 1450000000 1450000000 Build out change-log tables
|
4
|
+
a 1450000000 1450000000 Build out canonical, fact and aggregate tables (and related transforms)
|
5
|
+
a 1450000000 1450000000 Configurable re-pull window (do this automatically once up to current?)
|
6
|
+
a 1450000000 1450000000 Add schema management capabilities (detect schema-deltas and suggestion modifications)
|
7
|
+
a 1450000000 1450000000 Consider refactoring mutator methods to be globally available (e.g. quoted -> Quote, etc.)
|
8
|
+
a 1450000000 1450000000 Support parallel pulls?
|
9
|
+
a 1450000000 1450000000 Come up w/ a deployment recommendation
|
10
|
+
r 1450000000 1450000000 Add support for MSSQL
|
11
|
+
r 1450000000 1495070870 Further encapsulate `Dialect` logic (in order to DRY `Database` classes)
|
12
|
+
d 1494850714 1495046648 Finish SQLite3 raw-connections
|
13
|
+
d 1495046665 1495046689 Fix JDBC driver configuration
|
14
|
+
d 1495051831 1495070248 Code-review recent changes and merge
|
15
|
+
d 1450000000 1450000000 Handle blank values a bit more intelligently (all blanks -> nil)
|
16
|
+
d 1450000000 1450000000 Deal w/ NULL values
|
17
|
+
d 1450000000 1450000000 Deal w/ BOOL values
|
18
|
+
d 1450000000 1450000000 Build out MVP for PostgreSQL loader
|
19
|
+
d 1450000000 1450000000 Add windowing to pulls (simulate w/ 2 different data-files if necessary)
|
20
|
+
d 1450000000 1450000000 Include [window] timestamps in query-string for pull[er]
|
21
|
+
d 1450000000 1450000000 Figure out an intelligent way to solve the chicken <-> egg problem for `table_metadata`
|
22
|
+
d 1450000000 1450000000 Fail fast if there are no candidate tables (`next_table`)
|
23
|
+
d 1450000000 1450000000 Ensure that the puller does not try to go into the future (`next_window`)
|
24
|
+
d 1450000000 1450000000 Add in a primitive logging facility (can be temporary)
|
25
|
+
d 1450000000 1450000000 Enhance `next_table_sql`
|
26
|
+
d 1450000000 1450000000 Lock tables (feeds) during pull, parse and load
|
27
|
+
d 1450000000 1450000000 Verify table-locking mechanism behaves as expected
|
28
|
+
d 1450000000 1450000000 Ability to drop a table
|
29
|
+
d 1450000000 1450000000 Reorder args: "table, table_name" to: "table_name, table"
|
30
|
+
d 1450000000 1450000000 Reorder args: "table, index_name, *columns" to "index_name, table, *columns"
|
31
|
+
d 1450000000 1450000000 Ability to create an index
|
32
|
+
d 1450000000 1450000000 Ability to drop an index
|
33
|
+
d 1450000000 1450000000 Indices on `created_at` and `updated_at` columns
|
34
|
+
d 1450000000 1450000000 Gracefully handle table {,un}locking in case of error
|
35
|
+
d 1450000000 1450000000 Add debugging code
|
36
|
+
d 1450000000 1450000000 Raise no-window & no-table exceptions (making them catchable upstream)
|
37
|
+
d 1450000000 1450000000 Minimize race-condition around table-locking (use where on UPDATE, raise if zero rows are updated)
|
38
|
+
d 1450000000 1450000000 Log number of delete, inserts and updates in `loader`
|
39
|
+
d 1450000000 1450000000 `load_temporary_table` should bomb if there is data outside the window
|
40
|
+
d 1450000000 1450000000 Delete from outside window where id in temporary table
|
41
|
+
d 1450000000 1450000000 Rename `data` to `rows` in `execute`
|
42
|
+
d 1450000000 1450000000 Rename `Postgres` to `PostgreSQL`
|
43
|
+
d 1450000000 1450000000 Finish PostgreSQL support (requires `pg` gem)
|
44
|
+
d 1450000000 1450000000 Finish MySQL support (requires `mysql2` gem)
|
45
|
+
d 1450000000 1450000000 Usage/configuration examples in README.md
|
46
|
+
d 1450000000 1450000000 Create SQL-puller(s)
|
47
|
+
d 1450000000 1450000000 Introduce [SQL] dialect(s)
|
48
|
+
d 1450000000 1450000000 Fix JRuby bundle/build
|
49
|
+
d 1450000000 1450000000 Fall back to logged in user if no [database-]username is provided
|
50
|
+
d 1450000000 1450000000 Create class to encapulate `table_metadata` (in order to clean up the logic in the `Database` class(es))
|
51
|
+
d 1450000000 1450000000 Rename columns during pull/parse `source_name` & `target_name` perhaps? Maybe create a new type of column?
|
52
|
+
d 1450000000 1495070826 Add support for SQLite3
|
53
|
+
d 1494850696 1495070857 Add TODOs from $HOME\purview-TODO
|
54
|
+
d 1495070065 1495072292 Finish MSSQL built-out
|
data/lib/purview/connections.rb
CHANGED
data/lib/purview/databases.rb
CHANGED
@@ -13,7 +13,7 @@ module Purview
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
-
def baseline_table(table)
|
16
|
+
def baseline_table(table, timestamp=Time.now.utc)
|
17
17
|
ensure_table_valid_for_database(table)
|
18
18
|
raise Purview::Exceptions::CouldNotBaselineTable.new(table) \
|
19
19
|
unless table_initialized?(table)
|
@@ -84,7 +84,7 @@ module Purview
|
|
84
84
|
table_name
|
85
85
|
end
|
86
86
|
|
87
|
-
def disable_table(table)
|
87
|
+
def disable_table(table, timestamp=Time.now.utc)
|
88
88
|
ensure_table_valid_for_database(table)
|
89
89
|
table_name = table_name(table)
|
90
90
|
with_context_logging("`disable_table` for: #{table_name}") do
|
@@ -142,7 +142,7 @@ module Purview
|
|
142
142
|
table_name
|
143
143
|
end
|
144
144
|
|
145
|
-
def enable_table(table, timestamp=
|
145
|
+
def enable_table(table, timestamp=Time.now.utc)
|
146
146
|
ensure_table_valid_for_database(table)
|
147
147
|
table_name = table_name(table)
|
148
148
|
with_context_logging("`enable_table` for: #{table_name}") do
|
@@ -156,7 +156,7 @@ module Purview
|
|
156
156
|
table_name
|
157
157
|
end
|
158
158
|
|
159
|
-
def initialize_table(table, timestamp=
|
159
|
+
def initialize_table(table, timestamp=Time.now.utc)
|
160
160
|
ensure_table_valid_for_database(table)
|
161
161
|
table_name = table_name(table)
|
162
162
|
with_context_logging("`initialize_table` for: #{table_name}") do
|
@@ -170,7 +170,7 @@ module Purview
|
|
170
170
|
table_name
|
171
171
|
end
|
172
172
|
|
173
|
-
def lock_table(table, timestamp=
|
173
|
+
def lock_table(table, timestamp=Time.now.utc)
|
174
174
|
ensure_table_valid_for_database(table)
|
175
175
|
table_name = table_name(table)
|
176
176
|
with_context_logging("`lock_table` for: #{table_name}") do
|
@@ -194,7 +194,7 @@ module Purview
|
|
194
194
|
end
|
195
195
|
end
|
196
196
|
|
197
|
-
def sync_table(table)
|
197
|
+
def sync_table(table, timestamp=Time.now.utc)
|
198
198
|
ensure_table_valid_for_database(table)
|
199
199
|
raise Purview::Exceptions::CouldNotSyncTable.new(table) \
|
200
200
|
unless table_initialized?(table) && table_enabled?(table)
|
@@ -211,7 +211,7 @@ module Purview
|
|
211
211
|
table_name = table_name(table)
|
212
212
|
with_context_logging("`table_metadata` for: #{table_name}") do
|
213
213
|
with_new_connection do |connection|
|
214
|
-
Purview::Structs::TableMetadata.new(
|
214
|
+
table_metadata = Purview::Structs::TableMetadata.new(
|
215
215
|
table_metadata_table.columns.reduce({}) do |memo, column|
|
216
216
|
memo[column.name] = get_table_metadata_value(
|
217
217
|
connection,
|
@@ -0,0 +1,180 @@
|
|
1
|
+
module Purview
|
2
|
+
module Databases
|
3
|
+
class MSSQL < Base
|
4
|
+
private
|
5
|
+
|
6
|
+
def connection_type
|
7
|
+
Purview::Connections::MSSQL
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
11
|
+
'CREATE%sINDEX %s ON %s (%s)' % [
|
12
|
+
index.unique? ? ' UNIQUE ' : ' ',
|
13
|
+
index_name,
|
14
|
+
table_name,
|
15
|
+
column_names(index).join(', '),
|
16
|
+
]
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_table_sql(table_name, table, table_opts={})
|
20
|
+
'CREATE TABLE %s (%s)' % [
|
21
|
+
table_name,
|
22
|
+
column_definitions(table).join(', '),
|
23
|
+
]
|
24
|
+
end
|
25
|
+
|
26
|
+
def create_temporary_table_sql(table_name, table, table_opts={})
|
27
|
+
'CREATE TEMPORARY TABLE %s (%s)' % [
|
28
|
+
table_name,
|
29
|
+
column_definitions(table).join(', '),
|
30
|
+
]
|
31
|
+
end
|
32
|
+
|
33
|
+
def dialect_type
|
34
|
+
Purview::Dialects::MSSQL
|
35
|
+
end
|
36
|
+
|
37
|
+
def disable_table_sql(table)
|
38
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
39
|
+
table_metadata_table.name,
|
40
|
+
table_metadata_table.enabled_at_column.name,
|
41
|
+
null_value,
|
42
|
+
table_metadata_table.table_name_column.name,
|
43
|
+
quoted(table.name),
|
44
|
+
table_metadata_table.enabled_at_column.name,
|
45
|
+
]
|
46
|
+
end
|
47
|
+
|
48
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
49
|
+
'DROP INDEX %s' % [
|
50
|
+
index_name,
|
51
|
+
]
|
52
|
+
end
|
53
|
+
|
54
|
+
def drop_table_sql(table_name, table, table_opts={})
|
55
|
+
'DROP TABLE %s' % [
|
56
|
+
table_name,
|
57
|
+
]
|
58
|
+
end
|
59
|
+
|
60
|
+
def enable_table_sql(table, timestamp)
|
61
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
62
|
+
table_metadata_table.name,
|
63
|
+
table_metadata_table.enabled_at_column.name,
|
64
|
+
quoted(timestamp),
|
65
|
+
table_metadata_table.table_name_column.name,
|
66
|
+
quoted(table.name),
|
67
|
+
table_metadata_table.enabled_at_column.name,
|
68
|
+
]
|
69
|
+
end
|
70
|
+
|
71
|
+
def ensure_table_metadata_absent_for_table_sql(table)
|
72
|
+
'DELETE FROM %s WHERE %s = %s' % [
|
73
|
+
table_metadata_table.name,
|
74
|
+
table_metadata_table.table_name_column.name,
|
75
|
+
quoted(table.name),
|
76
|
+
]
|
77
|
+
end
|
78
|
+
|
79
|
+
def ensure_table_metadata_exists_for_table_sql(table)
|
80
|
+
'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
|
81
|
+
table_metadata_table.name,
|
82
|
+
table_metadata_table.table_name_column.name,
|
83
|
+
quoted(table.name),
|
84
|
+
table_metadata_table.name,
|
85
|
+
table_metadata_table.table_name_column.name,
|
86
|
+
quoted(table.name),
|
87
|
+
]
|
88
|
+
end
|
89
|
+
|
90
|
+
def ensure_table_metadata_table_exists_sql
|
91
|
+
'CREATE TABLE IF NOT EXISTS %s (%s)' % [
|
92
|
+
table_metadata_table.name,
|
93
|
+
column_definitions(table_metadata_table).join(', '),
|
94
|
+
]
|
95
|
+
end
|
96
|
+
|
97
|
+
def initialize_table_sql(table, timestamp)
|
98
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
99
|
+
table_metadata_table.name,
|
100
|
+
table_metadata_table.max_timestamp_pulled_column.name,
|
101
|
+
quoted(timestamp),
|
102
|
+
table_metadata_table.table_name_column.name,
|
103
|
+
quoted(table.name),
|
104
|
+
table_metadata_table.max_timestamp_pulled_column.name,
|
105
|
+
]
|
106
|
+
end
|
107
|
+
|
108
|
+
def get_table_metadata_value_sql(table, column)
|
109
|
+
'SELECT %s FROM %s WHERE %s = %s' % [
|
110
|
+
column.name,
|
111
|
+
table_metadata_table.name,
|
112
|
+
table_metadata_table.table_name_column.name,
|
113
|
+
quoted(table.name),
|
114
|
+
]
|
115
|
+
end
|
116
|
+
|
117
|
+
def limit_map
|
118
|
+
super.merge(Purview::Types::String => 255)
|
119
|
+
end
|
120
|
+
|
121
|
+
def limitless_types
|
122
|
+
super + [
|
123
|
+
Purview::Types::Money,
|
124
|
+
Purview::Types::UUID,
|
125
|
+
]
|
126
|
+
end
|
127
|
+
|
128
|
+
def lock_table_sql(table, timestamp)
|
129
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
130
|
+
table_metadata_table.name,
|
131
|
+
table_metadata_table.locked_at_column.name,
|
132
|
+
quoted(timestamp),
|
133
|
+
table_metadata_table.table_name_column.name,
|
134
|
+
quoted(table.name),
|
135
|
+
table_metadata_table.locked_at_column.name,
|
136
|
+
]
|
137
|
+
end
|
138
|
+
|
139
|
+
def next_table_sql(timestamp)
|
140
|
+
'SELECT TOP 1 %s FROM %s WHERE %s IS NOT NULL AND %s IS NOT NULL AND %s IS NULL ORDER BY (CASE WHEN %s IS NULL THEN 0 ELSE 1 END), %s' % [
|
141
|
+
table_metadata_table.table_name_column.name,
|
142
|
+
table_metadata_table.name,
|
143
|
+
table_metadata_table.enabled_at_column.name,
|
144
|
+
table_metadata_table.max_timestamp_pulled_column.name,
|
145
|
+
table_metadata_table.locked_at_column.name,
|
146
|
+
table_metadata_table.last_pulled_at_column.name,
|
147
|
+
table_metadata_table.last_pulled_at_column.name,
|
148
|
+
]
|
149
|
+
end
|
150
|
+
|
151
|
+
def set_table_metadata_value_sql(table, column, value)
|
152
|
+
'UPDATE %s SET %s = %s WHERE %s = %s' % [
|
153
|
+
table_metadata_table.name,
|
154
|
+
column.name,
|
155
|
+
quoted(value),
|
156
|
+
table_metadata_table.table_name_column.name,
|
157
|
+
quoted(table.name),
|
158
|
+
]
|
159
|
+
end
|
160
|
+
|
161
|
+
def type_map
|
162
|
+
super.merge(
|
163
|
+
Purview::Types::Money => 'money',
|
164
|
+
Purview::Types::UUID => 'uniqueidentifier',
|
165
|
+
)
|
166
|
+
end
|
167
|
+
|
168
|
+
def unlock_table_sql(table)
|
169
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
170
|
+
table_metadata_table.name,
|
171
|
+
table_metadata_table.locked_at_column.name,
|
172
|
+
null_value,
|
173
|
+
table_metadata_table.table_name_column.name,
|
174
|
+
quoted(table.name),
|
175
|
+
table_metadata_table.locked_at_column.name,
|
176
|
+
]
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|