sneaql 0.0.15-java → 0.0.16-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/sneaql +0 -14
- data/lib/sneaql.rb +20 -40
- data/lib/sneaql_lib/step_manager.rb +0 -46
- metadata +2 -9
- data/lib/sneaql_lib/database_manager.rb +0 -79
- data/lib/sneaql_lib/database_prefs/redshift.rb +0 -21
- data/lib/sneaql_lib/database_prefs/sqlite.rb +0 -12
- data/lib/sneaql_lib/database_prefs/vertica.rb +0 -21
- data/lib/sneaql_lib/lock_manager.rb +0 -176
- data/lib/sneaql_lib/standard.rb +0 -30
- data/lib/sneaql_lib/standard_db_objects.rb +0 -232
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9a88464715cbbb4c6efbadb8191725f3aeb3e7fa
|
4
|
+
data.tar.gz: a06d84d005827d7e072b1f582fc98debed7b0cf5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f42467e68a8c13215d3c36e2940ae51601a767ec253814a04180697fbdb94ab96086b65306e92442af35137c7a79cffe6b7da0b091f2dc7436ead25dd9d03f3f
|
7
|
+
data.tar.gz: 0b54cda35436015a39e72f328539b18f7c9373de8f0c888db6717a22fb365a6b45fd3e46fdba8d89e2cd1709ea2eacec9f74416e3c599eb46ab7abca7b219f9f
|
data/bin/sneaql
CHANGED
@@ -142,12 +142,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
142
142
|
|
143
143
|
logger.debug("available sneaql resources: #{Sneaql::Core.class_map}")
|
144
144
|
|
145
|
-
database = Sneaql::Core.database_type(jdbc_url)
|
146
|
-
unless database
|
147
|
-
raise 'database specified in jdbc url is not supported, or malformed jdbc url'
|
148
|
-
exit 1
|
149
|
-
end
|
150
|
-
|
151
145
|
begin
|
152
146
|
require_relative jdbc_driver_jar
|
153
147
|
java_import jdbc_driver_class
|
@@ -161,7 +155,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
161
155
|
transform_name: base_path,
|
162
156
|
repo_base_dir: base_path,
|
163
157
|
repo_type: 'local',
|
164
|
-
database: database,
|
165
158
|
jdbc_url: jdbc_url,
|
166
159
|
db_user: db_user,
|
167
160
|
db_pass: db_pass,
|
@@ -263,12 +256,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
263
256
|
|
264
257
|
logger.debug("available sneaql resources: #{Sneaql::Core.class_map}")
|
265
258
|
|
266
|
-
database = Sneaql::Core.database_type(jdbc_url)
|
267
|
-
unless database
|
268
|
-
raise 'database specified in jdbc url is not supported, or malformed jdbc url'
|
269
|
-
exit 1
|
270
|
-
end
|
271
|
-
|
272
259
|
begin
|
273
260
|
require_relative jdbc_driver_jar
|
274
261
|
java_import jdbc_driver_class
|
@@ -282,7 +269,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
282
269
|
transform_name: base_path,
|
283
270
|
repo_base_dir: base_path,
|
284
271
|
repo_type: 'local',
|
285
|
-
database: database,
|
286
272
|
jdbc_url: jdbc_url,
|
287
273
|
db_user: db_user,
|
288
274
|
db_pass: db_pass,
|
data/lib/sneaql.rb
CHANGED
@@ -1,17 +1,13 @@
|
|
1
1
|
require 'jdbc_helpers'
|
2
2
|
require 'logger'
|
3
3
|
|
4
|
-
#require_relative 'sneaql_lib/logging.rb'
|
5
4
|
require_relative 'sneaql_lib/exceptions.rb'
|
6
5
|
require_relative 'sneaql_lib/core.rb'
|
7
|
-
require_relative 'sneaql_lib/lock_manager.rb'
|
8
6
|
require_relative 'sneaql_lib/repo_manager.rb'
|
9
7
|
require_relative 'sneaql_lib/step_manager.rb'
|
10
8
|
require_relative 'sneaql_lib/parser.rb'
|
11
9
|
require_relative 'sneaql_lib/expressions.rb'
|
12
10
|
require_relative 'sneaql_lib/recordset.rb'
|
13
|
-
require_relative 'sneaql_lib/database_manager.rb'
|
14
|
-
require_relative 'sneaql_lib/standard_db_objects.rb'
|
15
11
|
require_relative 'sneaql_lib/docker.rb'
|
16
12
|
require_relative 'sneaql_lib/tokenizer.rb'
|
17
13
|
|
@@ -50,7 +46,6 @@ module Sneaql
|
|
50
46
|
# transform_name: 'test-transform',
|
51
47
|
# repo_base_dir: "test/fixtures/test-transform",
|
52
48
|
# repo_type: 'local',
|
53
|
-
# database: 'sqlite',
|
54
49
|
# jdbc_url: 'jdbc:sqlite:memory',
|
55
50
|
# db_user: 'dbuser',
|
56
51
|
# db_pass: 'password',
|
@@ -62,34 +57,30 @@ module Sneaql
|
|
62
57
|
# @param [Hash] params various parameters are passed to define the transform
|
63
58
|
# @param [Logger] logger customer logger if provided (otherwise default logger is created)
|
64
59
|
def initialize(params, logger = nil)
|
60
|
+
# initialzing and basic parameter stuff
|
65
61
|
@logger = logger ? logger : Logger.new(STDOUT)
|
66
|
-
|
67
62
|
@start_time = Time.new.utc
|
68
|
-
|
69
63
|
current_status(:initializing)
|
70
64
|
@params = params
|
71
|
-
|
72
65
|
@exit_code = 0
|
73
|
-
|
74
66
|
@transform_name = @params[:transform_name]
|
75
|
-
@transform_lock_id = @params[:transform_lock_id]
|
76
67
|
@jdbc_url = @params[:jdbc_url]
|
77
68
|
@db_user = @params[:db_user]
|
78
69
|
@db_pass = @params[:db_pass]
|
79
|
-
|
80
|
-
|
70
|
+
|
71
|
+
# build fancy objects for processing the transform
|
81
72
|
@expression_handler = create_expression_handler
|
82
73
|
@recordset_manager = create_recordset_manager
|
83
74
|
@exception_manager = create_exception_manager
|
84
|
-
|
75
|
+
@repo_manager = create_repo_manager
|
76
|
+
@steps = create_metadata_manager
|
77
|
+
@parsed_steps = create_parsed_steps(@steps)
|
78
|
+
|
85
79
|
run if @params[:run] == true
|
86
80
|
end
|
87
81
|
|
88
82
|
# validate the transform.
|
89
83
|
def validate
|
90
|
-
@repo_manager = create_repo_manager
|
91
|
-
@steps = create_metadata_manager
|
92
|
-
@parsed_steps = create_parsed_steps(@steps)
|
93
84
|
current_status(:validating)
|
94
85
|
validate_parsed_steps(@parsed_steps)
|
95
86
|
rescue Sneaql::Exceptions::TransformIsLocked => e
|
@@ -119,17 +110,12 @@ module Sneaql
|
|
119
110
|
|
120
111
|
# Runs the actual transform.
|
121
112
|
def run
|
122
|
-
|
123
|
-
@lock_manager = create_lock_manager if @params[:locked_transform] == true
|
124
|
-
@steps = create_metadata_manager
|
125
|
-
@parsed_steps = create_parsed_steps(@steps)
|
113
|
+
current_status(:validating)
|
126
114
|
validate_parsed_steps(@parsed_steps)
|
115
|
+
current_status(:connecting_to_database)
|
127
116
|
@jdbc_connection = create_jdbc_connection
|
128
117
|
current_status(:running)
|
129
118
|
iterate_steps_and_statements
|
130
|
-
rescue Sneaql::Exceptions::TransformIsLocked => e
|
131
|
-
@transform_error = e
|
132
|
-
@logger.info(e.message)
|
133
119
|
rescue Sneaql::Exceptions::SQLTestExitCondition => e
|
134
120
|
@transform_error = nil
|
135
121
|
@logger.info(e.message)
|
@@ -140,7 +126,6 @@ module Sneaql
|
|
140
126
|
@logger.error(e.message)
|
141
127
|
e.backtrace { |b| @logger.error b }
|
142
128
|
ensure
|
143
|
-
@lock_manager.remove_lock if @params[:locked_transform] == true
|
144
129
|
@jdbc_connection.close if @jdbc_connection
|
145
130
|
@end_time = Time.new.utc
|
146
131
|
|
@@ -172,16 +157,6 @@ module Sneaql
|
|
172
157
|
Sneaql::Core.find_class(:repo_manager, @params[:repo_type]).new(@params, @logger)
|
173
158
|
end
|
174
159
|
|
175
|
-
# Creates a TransformLockManager object
|
176
|
-
# The actual object returns depends upon params[:locked_transform] provided at initialize.
|
177
|
-
# @return [Sneaql::Core::RepoDownloadManager]
|
178
|
-
def create_lock_manager
|
179
|
-
# create a lock manager for this transform (uses a separate connection)
|
180
|
-
lock_manager = Sneaql::TransformLockManager.new(@params, @logger)
|
181
|
-
raise Sneaql::Exceptions::TransformIsLocked unless lock_manager.acquire_lock == true
|
182
|
-
lock_manager
|
183
|
-
end
|
184
|
-
|
185
160
|
# Creates a StepMetadataManager object
|
186
161
|
# The actual object returns depends upon params[:step_metadata_manager_type] provided at initialize.
|
187
162
|
# @return [Sneaql::Core::StepMetadataManager]
|
@@ -243,10 +218,9 @@ module Sneaql
|
|
243
218
|
# behavior for a connection that closes before a commit.
|
244
219
|
def iterate_steps_and_statements
|
245
220
|
@parsed_steps.each_with_index do |this_step|
|
246
|
-
#
|
247
|
-
|
248
|
-
|
249
|
-
break if exit_step == true
|
221
|
+
# raise any lingering errors not handled in previous step
|
222
|
+
raise @exception_manager.pending_error if @exception_manager.pending_error != nil
|
223
|
+
|
250
224
|
# set this so that other processes can poll the state
|
251
225
|
@current_step = this_step[:step_number]
|
252
226
|
# within a step... iterate through each statement
|
@@ -254,6 +228,7 @@ module Sneaql
|
|
254
228
|
# set this so that other processes can poll the state
|
255
229
|
@current_statement = stmt_index + 1
|
256
230
|
|
231
|
+
# log the pending error
|
257
232
|
@exception_manager.output_pending_error
|
258
233
|
|
259
234
|
# log some useful info
|
@@ -280,7 +255,13 @@ module Sneaql
|
|
280
255
|
@expression_handler.evaluate_all_expressions(this_stmt),
|
281
256
|
@logger
|
282
257
|
)
|
283
|
-
|
258
|
+
|
259
|
+
if @exception_manager.pending_error != nil
|
260
|
+
unless c.class == Sneaql::Core::Commands::SneaqlOnError
|
261
|
+
raise @exception_manager.pending_error
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
284
265
|
c.action(*this_cmd[:arguments])
|
285
266
|
|
286
267
|
# check if there was an error from the action
|
@@ -294,7 +275,6 @@ module Sneaql
|
|
294
275
|
end
|
295
276
|
|
296
277
|
rescue Sneaql::Exceptions::SQLTestStepExitCondition => e
|
297
|
-
exit_step = true
|
298
278
|
@logger.info e.message
|
299
279
|
break
|
300
280
|
end
|
@@ -24,51 +24,5 @@ module Sneaql
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
27
|
-
|
28
|
-
# source step metadata from a standardized table in the target database
|
29
|
-
class TransformStepTableManager < Sneaql::Core::StepMetadataManager
|
30
|
-
Sneaql::Core::RegisterMappedClass.new(
|
31
|
-
:step_metadata_manager,
|
32
|
-
'transform_steps_table',
|
33
|
-
Sneaql::StepManagers::TransformStepTableManager
|
34
|
-
)
|
35
|
-
|
36
|
-
# Manages steps based in a standardized table.
|
37
|
-
def manage_steps
|
38
|
-
jdbc_connection = JDBCHelpers::ConnectionFactory.new(
|
39
|
-
@params[:jdbc_url],
|
40
|
-
@params[:db_user],
|
41
|
-
@params[:db_pass]
|
42
|
-
).connection
|
43
|
-
|
44
|
-
@steps = JDBCHelpers::QueryResultsToArray.new(
|
45
|
-
jdbc_connection,
|
46
|
-
steps_sql
|
47
|
-
).results
|
48
|
-
|
49
|
-
@steps.map! do |s|
|
50
|
-
{
|
51
|
-
step_number: s['transform_step'],
|
52
|
-
step_file: s['sql_file_path_in_repo']
|
53
|
-
}
|
54
|
-
end
|
55
|
-
|
56
|
-
jdbc_connection.close
|
57
|
-
end
|
58
|
-
|
59
|
-
def steps_sql
|
60
|
-
%(select
|
61
|
-
transform_step
|
62
|
-
,sql_file_path_in_repo
|
63
|
-
from
|
64
|
-
#{@params[:transform_steps_table]}
|
65
|
-
where
|
66
|
-
transform_name='#{@params[:transform_name]}'
|
67
|
-
and
|
68
|
-
is_active=#{@params[:database_manager].has_boolean ? 'true' : 1}
|
69
|
-
order by
|
70
|
-
transform_step asc;)
|
71
|
-
end
|
72
|
-
end
|
73
27
|
end
|
74
28
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sneaql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.16
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- jeremy winters
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -119,19 +119,12 @@ files:
|
|
119
119
|
- lib/sneaql.rb
|
120
120
|
- lib/sneaql_lib/base.rb
|
121
121
|
- lib/sneaql_lib/core.rb
|
122
|
-
- lib/sneaql_lib/database_manager.rb
|
123
|
-
- lib/sneaql_lib/database_prefs/redshift.rb
|
124
|
-
- lib/sneaql_lib/database_prefs/sqlite.rb
|
125
|
-
- lib/sneaql_lib/database_prefs/vertica.rb
|
126
122
|
- lib/sneaql_lib/docker.rb
|
127
123
|
- lib/sneaql_lib/exceptions.rb
|
128
124
|
- lib/sneaql_lib/expressions.rb
|
129
|
-
- lib/sneaql_lib/lock_manager.rb
|
130
125
|
- lib/sneaql_lib/parser.rb
|
131
126
|
- lib/sneaql_lib/recordset.rb
|
132
127
|
- lib/sneaql_lib/repo_manager.rb
|
133
|
-
- lib/sneaql_lib/standard.rb
|
134
|
-
- lib/sneaql_lib/standard_db_objects.rb
|
135
128
|
- lib/sneaql_lib/step_manager.rb
|
136
129
|
- lib/sneaql_lib/tokenizer.rb
|
137
130
|
homepage: https://www.full360.com
|
@@ -1,79 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# returns the database type based upon the jdbc url
|
4
|
-
# @param [String]
|
5
|
-
def self.database_type(jdbc_url)
|
6
|
-
Sneaql::Core.class_map[:database].each do |d|
|
7
|
-
return d[:text] if jdbc_url.match(d[:text])
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
# Manages preferences for a specific RDBMS implementation.
|
12
|
-
class DatabaseManager
|
13
|
-
attr_reader(
|
14
|
-
:has_boolean,
|
15
|
-
:autocommit_off_statement,
|
16
|
-
:supports_transactions,
|
17
|
-
:supports_table_locking,
|
18
|
-
:begin_statement,
|
19
|
-
:commit_statement,
|
20
|
-
:rollback_statement
|
21
|
-
)
|
22
|
-
|
23
|
-
# @param [Hash] options values to override defaults
|
24
|
-
def initialize(options = {})
|
25
|
-
@has_boolean = options.fetch(:has_boolean, default_has_boolean)
|
26
|
-
@autocommit_off_statement = options.fetch(:autocommit_off_statement, default_autocommit_off_statement)
|
27
|
-
@supports_transactions = options.fetch(:supports_transactions, default_supports_transactions)
|
28
|
-
@supports_table_locking = options.fetch(:supports_table_locking, default_supports_table_locking)
|
29
|
-
@begin_statement = options.fetch(:begin_statement, default_begin_statement)
|
30
|
-
@commit_statement = options.fetch(:commit_statement, default_commit_statement)
|
31
|
-
@rollback_statement = options.fetch(:rollback_statement, default_rollback_statement)
|
32
|
-
end
|
33
|
-
|
34
|
-
# @return [Boolean]
|
35
|
-
def default_has_boolean
|
36
|
-
false
|
37
|
-
end
|
38
|
-
|
39
|
-
# @return [String]
|
40
|
-
def default_autocommit_off_statement
|
41
|
-
nil
|
42
|
-
end
|
43
|
-
|
44
|
-
# @return [Boolean]
|
45
|
-
def default_supports_transactions
|
46
|
-
true
|
47
|
-
end
|
48
|
-
|
49
|
-
# @return [Boolean]
|
50
|
-
def default_supports_table_locking
|
51
|
-
false
|
52
|
-
end
|
53
|
-
|
54
|
-
# @return [String] begin statement
|
55
|
-
def default_begin_statement
|
56
|
-
"begin;"
|
57
|
-
end
|
58
|
-
|
59
|
-
# @return [String] commit statement
|
60
|
-
def default_commit_statement
|
61
|
-
"commit;"
|
62
|
-
end
|
63
|
-
|
64
|
-
# @return [String] rollback statement
|
65
|
-
def default_rollback_statement
|
66
|
-
"rollback;"
|
67
|
-
end
|
68
|
-
|
69
|
-
# @param [String] table_name
|
70
|
-
# @return [String] rollback statement
|
71
|
-
def lock_table_statement(table_name)
|
72
|
-
"lock table #{table_name};"
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
# Require all tested RDBMS
|
79
|
-
Dir.glob("#{File.dirname(__FILE__)}/database_prefs/*.rb").each { |f| require File.expand_path(f) }
|
@@ -1,21 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# Settings for interaction with Amazon Redshift
|
4
|
-
class RedshiftDatabaseManager < Sneaql::Core::DatabaseManager
|
5
|
-
Sneaql::Core::RegisterMappedClass.new(
|
6
|
-
:database,
|
7
|
-
'redshift',
|
8
|
-
Sneaql::Core::RedshiftDatabaseManager
|
9
|
-
)
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
super(
|
13
|
-
{
|
14
|
-
has_boolean: true,
|
15
|
-
autocommit_off_statement: 'set autocommit=off;'
|
16
|
-
}
|
17
|
-
)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
@@ -1,12 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# Settings for interacting with SQLite
|
4
|
-
class SqliteDatabaseManager < Sneaql::Core::DatabaseManager
|
5
|
-
Sneaql::Core::RegisterMappedClass.new(
|
6
|
-
:database,
|
7
|
-
'sqlite',
|
8
|
-
Sneaql::Core::SqliteDatabaseManager
|
9
|
-
)
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# Settings for interacting with HP Vertica
|
4
|
-
class VerticaDatabaseManager < Sneaql::Core::DatabaseManager
|
5
|
-
Sneaql::Core::RegisterMappedClass.new(
|
6
|
-
:database,
|
7
|
-
'vertica',
|
8
|
-
Sneaql::Core::VerticaDatabaseManager
|
9
|
-
)
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
super(
|
13
|
-
{
|
14
|
-
has_boolean: true,
|
15
|
-
autocommit_off_statement: 'set session autocommit to off;'
|
16
|
-
}
|
17
|
-
)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
@@ -1,176 +0,0 @@
|
|
1
|
-
require 'jdbc_helpers'
|
2
|
-
|
3
|
-
module Sneaql
|
4
|
-
module Core
|
5
|
-
# manages transform locking operations using a standardized
|
6
|
-
# table for storing the locks.
|
7
|
-
class TransformLockManager
|
8
|
-
# set instance variables that will be used to manage the locks
|
9
|
-
def initialize(params, logger = nil)
|
10
|
-
@logger = logger ? logger : Logger.new(STDOUT)
|
11
|
-
@transform_name = params[:transform_name]
|
12
|
-
@transform_lock_id = params[:transform_lock_id]
|
13
|
-
@transform_lock_table = params[:transform_lock_table]
|
14
|
-
@jdbc_url = params[:jdbc_url]
|
15
|
-
@db_user = params[:db_user]
|
16
|
-
@db_pass = params[:db_pass]
|
17
|
-
@database_manager = Sneaql::Core.find_class(
|
18
|
-
:database,
|
19
|
-
params[:database]
|
20
|
-
).new
|
21
|
-
rescue => e
|
22
|
-
@logger.error e.message
|
23
|
-
e.backtrace.each { |b| @logger.error b}
|
24
|
-
end
|
25
|
-
|
26
|
-
# Creates a connection in the current JDBC context
|
27
|
-
def create_jdbc_connection
|
28
|
-
JDBCHelpers::ConnectionFactory.new(
|
29
|
-
@jdbc_url,
|
30
|
-
@db_user,
|
31
|
-
@db_pass,
|
32
|
-
@logger
|
33
|
-
).connection
|
34
|
-
end
|
35
|
-
|
36
|
-
# Checks to see if the current transform is locked
|
37
|
-
# @return [Boolean]
|
38
|
-
def acquire_lock
|
39
|
-
# check to see if this transform is locked by
|
40
|
-
# another transform returns true if locked
|
41
|
-
jdbc_connection = create_jdbc_connection
|
42
|
-
|
43
|
-
# initialize lock value
|
44
|
-
lock_value = false
|
45
|
-
|
46
|
-
if @database_manager.supports_transactions == true
|
47
|
-
JDBCHelpers::Execute.new(
|
48
|
-
jdbc_connection,
|
49
|
-
@database_manager.begin_statement,
|
50
|
-
@logger
|
51
|
-
)
|
52
|
-
end
|
53
|
-
|
54
|
-
if @database_manager.supports_table_locking == true
|
55
|
-
JDBCHelpers::Execute.new(
|
56
|
-
jdbc_connection,
|
57
|
-
@database_manager.lock_table_statement(@transform_lock_table),
|
58
|
-
@logger
|
59
|
-
)
|
60
|
-
end
|
61
|
-
|
62
|
-
# query the number of rows which match the condition...
|
63
|
-
# should be 1 or 0... 1 indicating a lock
|
64
|
-
r = JDBCHelpers::SingleValueFromQuery.new(
|
65
|
-
jdbc_connection,
|
66
|
-
%(select
|
67
|
-
count(*)
|
68
|
-
from
|
69
|
-
#{@transform_lock_table}
|
70
|
-
where
|
71
|
-
transform_name='#{@transform_name}'
|
72
|
-
and
|
73
|
-
transform_lock_id!=#{@transform_lock_id};),
|
74
|
-
@logger
|
75
|
-
).result
|
76
|
-
|
77
|
-
# table is unlocked
|
78
|
-
if r == 0
|
79
|
-
l = JDBCHelpers::Execute.new(
|
80
|
-
jdbc_connection,
|
81
|
-
%{insert into #{@transform_lock_table}
|
82
|
-
(
|
83
|
-
transform_lock_id,
|
84
|
-
transform_name,
|
85
|
-
transform_lock_time
|
86
|
-
)
|
87
|
-
values
|
88
|
-
(
|
89
|
-
#{@transform_lock_id},
|
90
|
-
'#{@transform_name}',
|
91
|
-
current_timestamp
|
92
|
-
);},
|
93
|
-
@logger
|
94
|
-
)
|
95
|
-
|
96
|
-
if @database_manager.supports_transactions == true
|
97
|
-
JDBCHelpers::Execute.new(
|
98
|
-
jdbc_connection,
|
99
|
-
@database_manager.commit_statement,
|
100
|
-
@logger
|
101
|
-
)
|
102
|
-
end
|
103
|
-
|
104
|
-
lock_value = true
|
105
|
-
else
|
106
|
-
if @database_manager.supports_transactions == true
|
107
|
-
JDBCHelpers::Execute.new(
|
108
|
-
jdbc_connection,
|
109
|
-
@database_manager.rollback_statement,
|
110
|
-
@logger
|
111
|
-
)
|
112
|
-
end
|
113
|
-
lock_value = false
|
114
|
-
end
|
115
|
-
|
116
|
-
if lock_value == true
|
117
|
-
@logger.info("#{@transform_name} transform lock acquired;")
|
118
|
-
else
|
119
|
-
@logger.info("#{@transform_name} is locked by another process")
|
120
|
-
end
|
121
|
-
ensure
|
122
|
-
# close this connection
|
123
|
-
jdbc_connection.close
|
124
|
-
|
125
|
-
lock_value
|
126
|
-
end
|
127
|
-
|
128
|
-
# Removes transform lock if it's present.
|
129
|
-
def remove_lock
|
130
|
-
# get a fresh jdbc connection...
|
131
|
-
# to avoid committing the main transform unnecessarily
|
132
|
-
jdbc_connection = create_jdbc_connection
|
133
|
-
|
134
|
-
if @database_manager.supports_transactions == true
|
135
|
-
JDBCHelpers::Execute.new(
|
136
|
-
jdbc_connection,
|
137
|
-
@database_manager.begin_statement,
|
138
|
-
@logger
|
139
|
-
)
|
140
|
-
end
|
141
|
-
|
142
|
-
if @database_manager.supports_table_locking == true
|
143
|
-
JDBCHelpers::Execute.new(
|
144
|
-
jdbc_connection,
|
145
|
-
@database_manager.lock_table_statement(@transform_lock_table),
|
146
|
-
@logger
|
147
|
-
)
|
148
|
-
end
|
149
|
-
|
150
|
-
# delete the lock record and commit
|
151
|
-
JDBCHelpers::Execute.new(
|
152
|
-
jdbc_connection,
|
153
|
-
%(delete from #{@transform_lock_table}
|
154
|
-
where transform_name='#{@transform_name}'
|
155
|
-
and transform_lock_id=#{@transform_lock_id};),
|
156
|
-
@logger
|
157
|
-
)
|
158
|
-
|
159
|
-
c = JDBCHelpers::Execute.new(
|
160
|
-
jdbc_connection,
|
161
|
-
@database_manager.commit_statement,
|
162
|
-
@logger
|
163
|
-
)
|
164
|
-
ensure
|
165
|
-
jdbc_connection.close
|
166
|
-
|
167
|
-
return true
|
168
|
-
end
|
169
|
-
|
170
|
-
# TBD
|
171
|
-
def lock_all_available_transforms
|
172
|
-
# undefined at this time
|
173
|
-
end
|
174
|
-
end
|
175
|
-
end
|
176
|
-
end
|
data/lib/sneaql_lib/standard.rb
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
# Classes to manage DB objects for standard SneaQL deployment.
|
3
|
-
module Standard
|
4
|
-
# Manages execution of a standard transform
|
5
|
-
class StandardTransform
|
6
|
-
|
7
|
-
attr_accessor {
|
8
|
-
:jdbc_url,
|
9
|
-
:db_user,
|
10
|
-
:db_pass
|
11
|
-
}
|
12
|
-
|
13
|
-
def initialize()
|
14
|
-
# set parameters
|
15
|
-
|
16
|
-
# determine database type
|
17
|
-
|
18
|
-
end
|
19
|
-
|
20
|
-
def execute_all_transforms()
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
def execute_single_transform(transform_name)
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
@@ -1,232 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
# Classes to manage DB objects for standard SneaQL deployment.
|
3
|
-
module Standard
|
4
|
-
# Facilitate creation/recreation of database table objects
|
5
|
-
# for standard SneaQL deployment.
|
6
|
-
class DBObjectCreator
|
7
|
-
attr_accessor :jdbc_connection
|
8
|
-
|
9
|
-
# @param [Object] jdbc_connection
|
10
|
-
# @param [Sneaql::Core::DatabaseManager] database_manager
|
11
|
-
# @param [Logger] logger
|
12
|
-
def initialize(jdbc_connection, database_manager, logger = nil)
|
13
|
-
@logger = logger ? logger : Logger.new(STDOUT)
|
14
|
-
@jdbc_connection = jdbc_connection
|
15
|
-
@database_manager = database_manager
|
16
|
-
end
|
17
|
-
|
18
|
-
# Drops and recreates the primary transform table.
|
19
|
-
# @param [String] transform_table_name fully qualified name for this table
|
20
|
-
# @return [Boolean]
|
21
|
-
def recreate_transforms_table(transform_table_name)
|
22
|
-
JDBCHelpers::Execute.new(
|
23
|
-
@jdbc_connection,
|
24
|
-
"drop table if exists #{ transform_table_name };",
|
25
|
-
@logger
|
26
|
-
)
|
27
|
-
create_transforms_table(transform_table_name)
|
28
|
-
return true
|
29
|
-
end
|
30
|
-
|
31
|
-
# Creates the primary transform table.
|
32
|
-
# @param [String] transform_table_name fully qualified name for this table
|
33
|
-
# @return [Boolean]
|
34
|
-
def create_transforms_table(transform_table_name)
|
35
|
-
JDBCHelpers::Execute.new(
|
36
|
-
@jdbc_connection,
|
37
|
-
transforms_table_create_statement(transform_table_name),
|
38
|
-
@logger
|
39
|
-
)
|
40
|
-
return true
|
41
|
-
end
|
42
|
-
|
43
|
-
# Coerces a boolean to the appropriate value for the database type.
|
44
|
-
# May return a 0 or 1 in RDBMS where boolean is not supported.
|
45
|
-
# @param [Boolean] boolean_value
|
46
|
-
# @return [Boolean, Fixnum]
|
47
|
-
def coerce_boolean(boolean_value)
|
48
|
-
if @database_manager.has_boolean
|
49
|
-
boolean_value
|
50
|
-
else
|
51
|
-
boolean_value == true ? 1 : 0
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
# Create table statement for primary transform table.
|
56
|
-
# @param [String] transform_table_name fully qualified name for this table
|
57
|
-
# @return [String]
|
58
|
-
def transforms_table_create_statement(transform_table_name)
|
59
|
-
%{create table if not exists #{ transform_table_name }
|
60
|
-
(
|
61
|
-
transform_name varchar(255) not null
|
62
|
-
,sql_repository varchar(255)
|
63
|
-
,sql_repository_branch varchar(255)
|
64
|
-
,is_active #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
65
|
-
,notify_on_success #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
66
|
-
,notify_on_non_precondition_failure #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
67
|
-
,notify_on_precondition_failure #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
68
|
-
,updated_ts timestamp
|
69
|
-
);}
|
70
|
-
end
|
71
|
-
|
72
|
-
# Creates a record in the transforms table.
|
73
|
-
# @param [String] transform_table_name
|
74
|
-
# @param [Hash] params Hash of parameters with symbols matching column names
|
75
|
-
def create_transform(transform_table_name, params)
|
76
|
-
JDBCHelpers::Execute.new(
|
77
|
-
@jdbc_connection,
|
78
|
-
create_transform_statement(transform_table_name, params),
|
79
|
-
@logger
|
80
|
-
)
|
81
|
-
end
|
82
|
-
|
83
|
-
# @param [String] transform_table_name
|
84
|
-
# @param [Hash] params Hash of parameters with symbols matching column names
|
85
|
-
def create_transform_statement(transform_table_name, params)
|
86
|
-
%{insert into #{transform_table_name}
|
87
|
-
(
|
88
|
-
transform_name
|
89
|
-
,sql_repository
|
90
|
-
,sql_repository_branch
|
91
|
-
,is_active
|
92
|
-
,notify_on_success
|
93
|
-
,notify_on_non_precondition_failure
|
94
|
-
,notify_on_precondition_failure
|
95
|
-
,updated_ts
|
96
|
-
)
|
97
|
-
values
|
98
|
-
(
|
99
|
-
'#{params[:transform_name]}'
|
100
|
-
,'#{params[:sql_repository]}'
|
101
|
-
,'#{params[:sql_repository_branch]}'
|
102
|
-
,#{coerce_boolean(params[:is_active])}
|
103
|
-
,#{coerce_boolean(params[:notify_on_success])}
|
104
|
-
,#{coerce_boolean(params[:notify_on_non_precondition_failure])}
|
105
|
-
,#{coerce_boolean(params[:notify_on_precondition_failure])}
|
106
|
-
,current_timestamp
|
107
|
-
);}
|
108
|
-
end
|
109
|
-
|
110
|
-
# Drops and recreates the transform steps table.
|
111
|
-
# @param [String] transform_steps_table_name fully qualified name for this table
|
112
|
-
# @return [Boolean]
|
113
|
-
def recreate_transform_steps_table(transform_steps_table_name)
|
114
|
-
JDBCHelpers::Execute.new(
|
115
|
-
@jdbc_connection,
|
116
|
-
"drop table if exists #{ transform_steps_table_name };",
|
117
|
-
@logger
|
118
|
-
)
|
119
|
-
create_transform_steps_table(transform_steps_table_name)
|
120
|
-
return true
|
121
|
-
end
|
122
|
-
|
123
|
-
# Creates the transform steps table.
|
124
|
-
# @param [String] transform_steps_table_name fully qualified name for this table
|
125
|
-
# @return [Boolean]
|
126
|
-
def create_transform_steps_table(transform_steps_table_name)
|
127
|
-
JDBCHelpers::Execute.new(
|
128
|
-
@jdbc_connection,
|
129
|
-
transform_steps_table_create_statement(transform_steps_table_name),
|
130
|
-
@logger
|
131
|
-
)
|
132
|
-
return true
|
133
|
-
end
|
134
|
-
|
135
|
-
# Create table statement for transform steps table.
|
136
|
-
# @param [String] transform_steps_table_name fully qualified name for this table
|
137
|
-
# @return [String]
|
138
|
-
def transform_steps_table_create_statement(transform_steps_table_name)
|
139
|
-
%{create table if not exists #{ transform_steps_table_name }
|
140
|
-
(
|
141
|
-
transform_name varchar(255) not null
|
142
|
-
,transform_step integer not null
|
143
|
-
,sql_file_path_in_repo varchar(1024)
|
144
|
-
,is_active #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
145
|
-
,is_precondition #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
146
|
-
,updated_ts timestamp
|
147
|
-
);}
|
148
|
-
end
|
149
|
-
|
150
|
-
# Drops and recreates the transform lock table.
|
151
|
-
# @param [String] transform_lock_table_name fully qualified name for this table
|
152
|
-
# @return [Boolean]
|
153
|
-
def recreate_transform_lock_table(transform_lock_table_name)
|
154
|
-
JDBCHelpers::Execute.new(
|
155
|
-
@jdbc_connection,
|
156
|
-
"drop table if exists #{ transform_lock_table_name };",
|
157
|
-
@logger
|
158
|
-
)
|
159
|
-
create_transform_lock_table(transform_lock_table_name)
|
160
|
-
return true
|
161
|
-
end
|
162
|
-
|
163
|
-
# Creates the transform lock table.
|
164
|
-
# @param [String] transform_lock_table_name fully qualified name for this table
|
165
|
-
# @return [Boolean]
|
166
|
-
def create_transform_lock_table(transform_lock_table_name)
|
167
|
-
JDBCHelpers::Execute.new(
|
168
|
-
@jdbc_connection,
|
169
|
-
transform_lock_table_create_statement(transform_lock_table_name),
|
170
|
-
@logger
|
171
|
-
)
|
172
|
-
return true
|
173
|
-
end
|
174
|
-
|
175
|
-
# Create table statement for transform lock table.
|
176
|
-
# @param [String] transform_lock_table_name fully qualified name for this table
|
177
|
-
# @return [String]
|
178
|
-
def transform_lock_table_create_statement(transform_lock_table_name)
|
179
|
-
%{create table if not exists #{ transform_lock_table_name }
|
180
|
-
(
|
181
|
-
transform_lock_id bigint
|
182
|
-
,transform_name varchar(255)
|
183
|
-
,transform_lock_time timestamp
|
184
|
-
);}
|
185
|
-
end
|
186
|
-
|
187
|
-
# Drops and recreates the transform log table.
|
188
|
-
# @param [String] transform_log_table_name fully qualified name for this table
|
189
|
-
# @return [Boolean]
|
190
|
-
def recreate_transform_log_table(transform_log_table_name)
|
191
|
-
JDBCHelpers::Execute.new(
|
192
|
-
@jdbc_connection,
|
193
|
-
"drop table if exists #{ transform_log_table_name };",
|
194
|
-
@logger
|
195
|
-
)
|
196
|
-
create_transform_log_table(transform_log_table_name)
|
197
|
-
return true
|
198
|
-
end
|
199
|
-
|
200
|
-
# Creates the transform log table.
|
201
|
-
# @param [String] transform_log_table_name fully qualified name for this table
|
202
|
-
# @return [Boolean]
|
203
|
-
def create_transform_log_table(transform_log_table_name)
|
204
|
-
JDBCHelpers::Execute.new(
|
205
|
-
@jdbc_connection,
|
206
|
-
transform_log_table_create_statement(transform_log_table_name),
|
207
|
-
@logger
|
208
|
-
)
|
209
|
-
return true
|
210
|
-
end
|
211
|
-
|
212
|
-
# Create table statement for transform log table.
|
213
|
-
# @param [String] transform_log_table_name fully qualified name for this table
|
214
|
-
# @return [String]
|
215
|
-
def transform_log_table_create_statement(transform_log_table_name)
|
216
|
-
%{create table if not exists #{ transform_log_table_name }
|
217
|
-
(
|
218
|
-
transform_run_id bigint
|
219
|
-
,transform_lock_id bigint
|
220
|
-
,transform_name varchar(255)
|
221
|
-
,transform_step integer
|
222
|
-
,transform_statement integer
|
223
|
-
,all_steps_complete #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
224
|
-
,failed_in_precondition #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
225
|
-
,message varchar(65000)
|
226
|
-
,transform_start_time timestamp
|
227
|
-
,transform_end_time timestamp
|
228
|
-
);}
|
229
|
-
end
|
230
|
-
end
|
231
|
-
end
|
232
|
-
end
|