sneaql 0.0.15-java → 0.0.16-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/sneaql +0 -14
- data/lib/sneaql.rb +20 -40
- data/lib/sneaql_lib/step_manager.rb +0 -46
- metadata +2 -9
- data/lib/sneaql_lib/database_manager.rb +0 -79
- data/lib/sneaql_lib/database_prefs/redshift.rb +0 -21
- data/lib/sneaql_lib/database_prefs/sqlite.rb +0 -12
- data/lib/sneaql_lib/database_prefs/vertica.rb +0 -21
- data/lib/sneaql_lib/lock_manager.rb +0 -176
- data/lib/sneaql_lib/standard.rb +0 -30
- data/lib/sneaql_lib/standard_db_objects.rb +0 -232
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9a88464715cbbb4c6efbadb8191725f3aeb3e7fa
|
4
|
+
data.tar.gz: a06d84d005827d7e072b1f582fc98debed7b0cf5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f42467e68a8c13215d3c36e2940ae51601a767ec253814a04180697fbdb94ab96086b65306e92442af35137c7a79cffe6b7da0b091f2dc7436ead25dd9d03f3f
|
7
|
+
data.tar.gz: 0b54cda35436015a39e72f328539b18f7c9373de8f0c888db6717a22fb365a6b45fd3e46fdba8d89e2cd1709ea2eacec9f74416e3c599eb46ab7abca7b219f9f
|
data/bin/sneaql
CHANGED
@@ -142,12 +142,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
142
142
|
|
143
143
|
logger.debug("available sneaql resources: #{Sneaql::Core.class_map}")
|
144
144
|
|
145
|
-
database = Sneaql::Core.database_type(jdbc_url)
|
146
|
-
unless database
|
147
|
-
raise 'database specified in jdbc url is not supported, or malformed jdbc url'
|
148
|
-
exit 1
|
149
|
-
end
|
150
|
-
|
151
145
|
begin
|
152
146
|
require_relative jdbc_driver_jar
|
153
147
|
java_import jdbc_driver_class
|
@@ -161,7 +155,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
161
155
|
transform_name: base_path,
|
162
156
|
repo_base_dir: base_path,
|
163
157
|
repo_type: 'local',
|
164
|
-
database: database,
|
165
158
|
jdbc_url: jdbc_url,
|
166
159
|
db_user: db_user,
|
167
160
|
db_pass: db_pass,
|
@@ -263,12 +256,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
263
256
|
|
264
257
|
logger.debug("available sneaql resources: #{Sneaql::Core.class_map}")
|
265
258
|
|
266
|
-
database = Sneaql::Core.database_type(jdbc_url)
|
267
|
-
unless database
|
268
|
-
raise 'database specified in jdbc url is not supported, or malformed jdbc url'
|
269
|
-
exit 1
|
270
|
-
end
|
271
|
-
|
272
259
|
begin
|
273
260
|
require_relative jdbc_driver_jar
|
274
261
|
java_import jdbc_driver_class
|
@@ -282,7 +269,6 @@ the above will execute the transform in the current directory. note that sneaql
|
|
282
269
|
transform_name: base_path,
|
283
270
|
repo_base_dir: base_path,
|
284
271
|
repo_type: 'local',
|
285
|
-
database: database,
|
286
272
|
jdbc_url: jdbc_url,
|
287
273
|
db_user: db_user,
|
288
274
|
db_pass: db_pass,
|
data/lib/sneaql.rb
CHANGED
@@ -1,17 +1,13 @@
|
|
1
1
|
require 'jdbc_helpers'
|
2
2
|
require 'logger'
|
3
3
|
|
4
|
-
#require_relative 'sneaql_lib/logging.rb'
|
5
4
|
require_relative 'sneaql_lib/exceptions.rb'
|
6
5
|
require_relative 'sneaql_lib/core.rb'
|
7
|
-
require_relative 'sneaql_lib/lock_manager.rb'
|
8
6
|
require_relative 'sneaql_lib/repo_manager.rb'
|
9
7
|
require_relative 'sneaql_lib/step_manager.rb'
|
10
8
|
require_relative 'sneaql_lib/parser.rb'
|
11
9
|
require_relative 'sneaql_lib/expressions.rb'
|
12
10
|
require_relative 'sneaql_lib/recordset.rb'
|
13
|
-
require_relative 'sneaql_lib/database_manager.rb'
|
14
|
-
require_relative 'sneaql_lib/standard_db_objects.rb'
|
15
11
|
require_relative 'sneaql_lib/docker.rb'
|
16
12
|
require_relative 'sneaql_lib/tokenizer.rb'
|
17
13
|
|
@@ -50,7 +46,6 @@ module Sneaql
|
|
50
46
|
# transform_name: 'test-transform',
|
51
47
|
# repo_base_dir: "test/fixtures/test-transform",
|
52
48
|
# repo_type: 'local',
|
53
|
-
# database: 'sqlite',
|
54
49
|
# jdbc_url: 'jdbc:sqlite:memory',
|
55
50
|
# db_user: 'dbuser',
|
56
51
|
# db_pass: 'password',
|
@@ -62,34 +57,30 @@ module Sneaql
|
|
62
57
|
# @param [Hash] params various parameters are passed to define the transform
|
63
58
|
# @param [Logger] logger customer logger if provided (otherwise default logger is created)
|
64
59
|
def initialize(params, logger = nil)
|
60
|
+
# initialzing and basic parameter stuff
|
65
61
|
@logger = logger ? logger : Logger.new(STDOUT)
|
66
|
-
|
67
62
|
@start_time = Time.new.utc
|
68
|
-
|
69
63
|
current_status(:initializing)
|
70
64
|
@params = params
|
71
|
-
|
72
65
|
@exit_code = 0
|
73
|
-
|
74
66
|
@transform_name = @params[:transform_name]
|
75
|
-
@transform_lock_id = @params[:transform_lock_id]
|
76
67
|
@jdbc_url = @params[:jdbc_url]
|
77
68
|
@db_user = @params[:db_user]
|
78
69
|
@db_pass = @params[:db_pass]
|
79
|
-
|
80
|
-
|
70
|
+
|
71
|
+
# build fancy objects for processing the transform
|
81
72
|
@expression_handler = create_expression_handler
|
82
73
|
@recordset_manager = create_recordset_manager
|
83
74
|
@exception_manager = create_exception_manager
|
84
|
-
|
75
|
+
@repo_manager = create_repo_manager
|
76
|
+
@steps = create_metadata_manager
|
77
|
+
@parsed_steps = create_parsed_steps(@steps)
|
78
|
+
|
85
79
|
run if @params[:run] == true
|
86
80
|
end
|
87
81
|
|
88
82
|
# validate the transform.
|
89
83
|
def validate
|
90
|
-
@repo_manager = create_repo_manager
|
91
|
-
@steps = create_metadata_manager
|
92
|
-
@parsed_steps = create_parsed_steps(@steps)
|
93
84
|
current_status(:validating)
|
94
85
|
validate_parsed_steps(@parsed_steps)
|
95
86
|
rescue Sneaql::Exceptions::TransformIsLocked => e
|
@@ -119,17 +110,12 @@ module Sneaql
|
|
119
110
|
|
120
111
|
# Runs the actual transform.
|
121
112
|
def run
|
122
|
-
|
123
|
-
@lock_manager = create_lock_manager if @params[:locked_transform] == true
|
124
|
-
@steps = create_metadata_manager
|
125
|
-
@parsed_steps = create_parsed_steps(@steps)
|
113
|
+
current_status(:validating)
|
126
114
|
validate_parsed_steps(@parsed_steps)
|
115
|
+
current_status(:connecting_to_database)
|
127
116
|
@jdbc_connection = create_jdbc_connection
|
128
117
|
current_status(:running)
|
129
118
|
iterate_steps_and_statements
|
130
|
-
rescue Sneaql::Exceptions::TransformIsLocked => e
|
131
|
-
@transform_error = e
|
132
|
-
@logger.info(e.message)
|
133
119
|
rescue Sneaql::Exceptions::SQLTestExitCondition => e
|
134
120
|
@transform_error = nil
|
135
121
|
@logger.info(e.message)
|
@@ -140,7 +126,6 @@ module Sneaql
|
|
140
126
|
@logger.error(e.message)
|
141
127
|
e.backtrace { |b| @logger.error b }
|
142
128
|
ensure
|
143
|
-
@lock_manager.remove_lock if @params[:locked_transform] == true
|
144
129
|
@jdbc_connection.close if @jdbc_connection
|
145
130
|
@end_time = Time.new.utc
|
146
131
|
|
@@ -172,16 +157,6 @@ module Sneaql
|
|
172
157
|
Sneaql::Core.find_class(:repo_manager, @params[:repo_type]).new(@params, @logger)
|
173
158
|
end
|
174
159
|
|
175
|
-
# Creates a TransformLockManager object
|
176
|
-
# The actual object returns depends upon params[:locked_transform] provided at initialize.
|
177
|
-
# @return [Sneaql::Core::RepoDownloadManager]
|
178
|
-
def create_lock_manager
|
179
|
-
# create a lock manager for this transform (uses a separate connection)
|
180
|
-
lock_manager = Sneaql::TransformLockManager.new(@params, @logger)
|
181
|
-
raise Sneaql::Exceptions::TransformIsLocked unless lock_manager.acquire_lock == true
|
182
|
-
lock_manager
|
183
|
-
end
|
184
|
-
|
185
160
|
# Creates a StepMetadataManager object
|
186
161
|
# The actual object returns depends upon params[:step_metadata_manager_type] provided at initialize.
|
187
162
|
# @return [Sneaql::Core::StepMetadataManager]
|
@@ -243,10 +218,9 @@ module Sneaql
|
|
243
218
|
# behavior for a connection that closes before a commit.
|
244
219
|
def iterate_steps_and_statements
|
245
220
|
@parsed_steps.each_with_index do |this_step|
|
246
|
-
#
|
247
|
-
|
248
|
-
|
249
|
-
break if exit_step == true
|
221
|
+
# raise any lingering errors not handled in previous step
|
222
|
+
raise @exception_manager.pending_error if @exception_manager.pending_error != nil
|
223
|
+
|
250
224
|
# set this so that other processes can poll the state
|
251
225
|
@current_step = this_step[:step_number]
|
252
226
|
# within a step... iterate through each statement
|
@@ -254,6 +228,7 @@ module Sneaql
|
|
254
228
|
# set this so that other processes can poll the state
|
255
229
|
@current_statement = stmt_index + 1
|
256
230
|
|
231
|
+
# log the pending error
|
257
232
|
@exception_manager.output_pending_error
|
258
233
|
|
259
234
|
# log some useful info
|
@@ -280,7 +255,13 @@ module Sneaql
|
|
280
255
|
@expression_handler.evaluate_all_expressions(this_stmt),
|
281
256
|
@logger
|
282
257
|
)
|
283
|
-
|
258
|
+
|
259
|
+
if @exception_manager.pending_error != nil
|
260
|
+
unless c.class == Sneaql::Core::Commands::SneaqlOnError
|
261
|
+
raise @exception_manager.pending_error
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
284
265
|
c.action(*this_cmd[:arguments])
|
285
266
|
|
286
267
|
# check if there was an error from the action
|
@@ -294,7 +275,6 @@ module Sneaql
|
|
294
275
|
end
|
295
276
|
|
296
277
|
rescue Sneaql::Exceptions::SQLTestStepExitCondition => e
|
297
|
-
exit_step = true
|
298
278
|
@logger.info e.message
|
299
279
|
break
|
300
280
|
end
|
@@ -24,51 +24,5 @@ module Sneaql
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
27
|
-
|
28
|
-
# source step metadata from a standardized table in the target database
|
29
|
-
class TransformStepTableManager < Sneaql::Core::StepMetadataManager
|
30
|
-
Sneaql::Core::RegisterMappedClass.new(
|
31
|
-
:step_metadata_manager,
|
32
|
-
'transform_steps_table',
|
33
|
-
Sneaql::StepManagers::TransformStepTableManager
|
34
|
-
)
|
35
|
-
|
36
|
-
# Manages steps based in a standardized table.
|
37
|
-
def manage_steps
|
38
|
-
jdbc_connection = JDBCHelpers::ConnectionFactory.new(
|
39
|
-
@params[:jdbc_url],
|
40
|
-
@params[:db_user],
|
41
|
-
@params[:db_pass]
|
42
|
-
).connection
|
43
|
-
|
44
|
-
@steps = JDBCHelpers::QueryResultsToArray.new(
|
45
|
-
jdbc_connection,
|
46
|
-
steps_sql
|
47
|
-
).results
|
48
|
-
|
49
|
-
@steps.map! do |s|
|
50
|
-
{
|
51
|
-
step_number: s['transform_step'],
|
52
|
-
step_file: s['sql_file_path_in_repo']
|
53
|
-
}
|
54
|
-
end
|
55
|
-
|
56
|
-
jdbc_connection.close
|
57
|
-
end
|
58
|
-
|
59
|
-
def steps_sql
|
60
|
-
%(select
|
61
|
-
transform_step
|
62
|
-
,sql_file_path_in_repo
|
63
|
-
from
|
64
|
-
#{@params[:transform_steps_table]}
|
65
|
-
where
|
66
|
-
transform_name='#{@params[:transform_name]}'
|
67
|
-
and
|
68
|
-
is_active=#{@params[:database_manager].has_boolean ? 'true' : 1}
|
69
|
-
order by
|
70
|
-
transform_step asc;)
|
71
|
-
end
|
72
|
-
end
|
73
27
|
end
|
74
28
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sneaql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.16
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- jeremy winters
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -119,19 +119,12 @@ files:
|
|
119
119
|
- lib/sneaql.rb
|
120
120
|
- lib/sneaql_lib/base.rb
|
121
121
|
- lib/sneaql_lib/core.rb
|
122
|
-
- lib/sneaql_lib/database_manager.rb
|
123
|
-
- lib/sneaql_lib/database_prefs/redshift.rb
|
124
|
-
- lib/sneaql_lib/database_prefs/sqlite.rb
|
125
|
-
- lib/sneaql_lib/database_prefs/vertica.rb
|
126
122
|
- lib/sneaql_lib/docker.rb
|
127
123
|
- lib/sneaql_lib/exceptions.rb
|
128
124
|
- lib/sneaql_lib/expressions.rb
|
129
|
-
- lib/sneaql_lib/lock_manager.rb
|
130
125
|
- lib/sneaql_lib/parser.rb
|
131
126
|
- lib/sneaql_lib/recordset.rb
|
132
127
|
- lib/sneaql_lib/repo_manager.rb
|
133
|
-
- lib/sneaql_lib/standard.rb
|
134
|
-
- lib/sneaql_lib/standard_db_objects.rb
|
135
128
|
- lib/sneaql_lib/step_manager.rb
|
136
129
|
- lib/sneaql_lib/tokenizer.rb
|
137
130
|
homepage: https://www.full360.com
|
@@ -1,79 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# returns the database type based upon the jdbc url
|
4
|
-
# @param [String]
|
5
|
-
def self.database_type(jdbc_url)
|
6
|
-
Sneaql::Core.class_map[:database].each do |d|
|
7
|
-
return d[:text] if jdbc_url.match(d[:text])
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
# Manages preferences for a specific RDBMS implementation.
|
12
|
-
class DatabaseManager
|
13
|
-
attr_reader(
|
14
|
-
:has_boolean,
|
15
|
-
:autocommit_off_statement,
|
16
|
-
:supports_transactions,
|
17
|
-
:supports_table_locking,
|
18
|
-
:begin_statement,
|
19
|
-
:commit_statement,
|
20
|
-
:rollback_statement
|
21
|
-
)
|
22
|
-
|
23
|
-
# @param [Hash] options values to override defaults
|
24
|
-
def initialize(options = {})
|
25
|
-
@has_boolean = options.fetch(:has_boolean, default_has_boolean)
|
26
|
-
@autocommit_off_statement = options.fetch(:autocommit_off_statement, default_autocommit_off_statement)
|
27
|
-
@supports_transactions = options.fetch(:supports_transactions, default_supports_transactions)
|
28
|
-
@supports_table_locking = options.fetch(:supports_table_locking, default_supports_table_locking)
|
29
|
-
@begin_statement = options.fetch(:begin_statement, default_begin_statement)
|
30
|
-
@commit_statement = options.fetch(:commit_statement, default_commit_statement)
|
31
|
-
@rollback_statement = options.fetch(:rollback_statement, default_rollback_statement)
|
32
|
-
end
|
33
|
-
|
34
|
-
# @return [Boolean]
|
35
|
-
def default_has_boolean
|
36
|
-
false
|
37
|
-
end
|
38
|
-
|
39
|
-
# @return [String]
|
40
|
-
def default_autocommit_off_statement
|
41
|
-
nil
|
42
|
-
end
|
43
|
-
|
44
|
-
# @return [Boolean]
|
45
|
-
def default_supports_transactions
|
46
|
-
true
|
47
|
-
end
|
48
|
-
|
49
|
-
# @return [Boolean]
|
50
|
-
def default_supports_table_locking
|
51
|
-
false
|
52
|
-
end
|
53
|
-
|
54
|
-
# @return [String] begin statement
|
55
|
-
def default_begin_statement
|
56
|
-
"begin;"
|
57
|
-
end
|
58
|
-
|
59
|
-
# @return [String] commit statement
|
60
|
-
def default_commit_statement
|
61
|
-
"commit;"
|
62
|
-
end
|
63
|
-
|
64
|
-
# @return [String] rollback statement
|
65
|
-
def default_rollback_statement
|
66
|
-
"rollback;"
|
67
|
-
end
|
68
|
-
|
69
|
-
# @param [String] table_name
|
70
|
-
# @return [String] rollback statement
|
71
|
-
def lock_table_statement(table_name)
|
72
|
-
"lock table #{table_name};"
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
# Require all tested RDBMS
|
79
|
-
Dir.glob("#{File.dirname(__FILE__)}/database_prefs/*.rb").each { |f| require File.expand_path(f) }
|
@@ -1,21 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# Settings for interaction with Amazon Redshift
|
4
|
-
class RedshiftDatabaseManager < Sneaql::Core::DatabaseManager
|
5
|
-
Sneaql::Core::RegisterMappedClass.new(
|
6
|
-
:database,
|
7
|
-
'redshift',
|
8
|
-
Sneaql::Core::RedshiftDatabaseManager
|
9
|
-
)
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
super(
|
13
|
-
{
|
14
|
-
has_boolean: true,
|
15
|
-
autocommit_off_statement: 'set autocommit=off;'
|
16
|
-
}
|
17
|
-
)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
@@ -1,12 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# Settings for interacting with SQLite
|
4
|
-
class SqliteDatabaseManager < Sneaql::Core::DatabaseManager
|
5
|
-
Sneaql::Core::RegisterMappedClass.new(
|
6
|
-
:database,
|
7
|
-
'sqlite',
|
8
|
-
Sneaql::Core::SqliteDatabaseManager
|
9
|
-
)
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
module Core
|
3
|
-
# Settings for interacting with HP Vertica
|
4
|
-
class VerticaDatabaseManager < Sneaql::Core::DatabaseManager
|
5
|
-
Sneaql::Core::RegisterMappedClass.new(
|
6
|
-
:database,
|
7
|
-
'vertica',
|
8
|
-
Sneaql::Core::VerticaDatabaseManager
|
9
|
-
)
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
super(
|
13
|
-
{
|
14
|
-
has_boolean: true,
|
15
|
-
autocommit_off_statement: 'set session autocommit to off;'
|
16
|
-
}
|
17
|
-
)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
@@ -1,176 +0,0 @@
|
|
1
|
-
require 'jdbc_helpers'
|
2
|
-
|
3
|
-
module Sneaql
|
4
|
-
module Core
|
5
|
-
# manages transform locking operations using a standardized
|
6
|
-
# table for storing the locks.
|
7
|
-
class TransformLockManager
|
8
|
-
# set instance variables that will be used to manage the locks
|
9
|
-
def initialize(params, logger = nil)
|
10
|
-
@logger = logger ? logger : Logger.new(STDOUT)
|
11
|
-
@transform_name = params[:transform_name]
|
12
|
-
@transform_lock_id = params[:transform_lock_id]
|
13
|
-
@transform_lock_table = params[:transform_lock_table]
|
14
|
-
@jdbc_url = params[:jdbc_url]
|
15
|
-
@db_user = params[:db_user]
|
16
|
-
@db_pass = params[:db_pass]
|
17
|
-
@database_manager = Sneaql::Core.find_class(
|
18
|
-
:database,
|
19
|
-
params[:database]
|
20
|
-
).new
|
21
|
-
rescue => e
|
22
|
-
@logger.error e.message
|
23
|
-
e.backtrace.each { |b| @logger.error b}
|
24
|
-
end
|
25
|
-
|
26
|
-
# Creates a connection in the current JDBC context
|
27
|
-
def create_jdbc_connection
|
28
|
-
JDBCHelpers::ConnectionFactory.new(
|
29
|
-
@jdbc_url,
|
30
|
-
@db_user,
|
31
|
-
@db_pass,
|
32
|
-
@logger
|
33
|
-
).connection
|
34
|
-
end
|
35
|
-
|
36
|
-
# Checks to see if the current transform is locked
|
37
|
-
# @return [Boolean]
|
38
|
-
def acquire_lock
|
39
|
-
# check to see if this transform is locked by
|
40
|
-
# another transform returns true if locked
|
41
|
-
jdbc_connection = create_jdbc_connection
|
42
|
-
|
43
|
-
# initialize lock value
|
44
|
-
lock_value = false
|
45
|
-
|
46
|
-
if @database_manager.supports_transactions == true
|
47
|
-
JDBCHelpers::Execute.new(
|
48
|
-
jdbc_connection,
|
49
|
-
@database_manager.begin_statement,
|
50
|
-
@logger
|
51
|
-
)
|
52
|
-
end
|
53
|
-
|
54
|
-
if @database_manager.supports_table_locking == true
|
55
|
-
JDBCHelpers::Execute.new(
|
56
|
-
jdbc_connection,
|
57
|
-
@database_manager.lock_table_statement(@transform_lock_table),
|
58
|
-
@logger
|
59
|
-
)
|
60
|
-
end
|
61
|
-
|
62
|
-
# query the number of rows which match the condition...
|
63
|
-
# should be 1 or 0... 1 indicating a lock
|
64
|
-
r = JDBCHelpers::SingleValueFromQuery.new(
|
65
|
-
jdbc_connection,
|
66
|
-
%(select
|
67
|
-
count(*)
|
68
|
-
from
|
69
|
-
#{@transform_lock_table}
|
70
|
-
where
|
71
|
-
transform_name='#{@transform_name}'
|
72
|
-
and
|
73
|
-
transform_lock_id!=#{@transform_lock_id};),
|
74
|
-
@logger
|
75
|
-
).result
|
76
|
-
|
77
|
-
# table is unlocked
|
78
|
-
if r == 0
|
79
|
-
l = JDBCHelpers::Execute.new(
|
80
|
-
jdbc_connection,
|
81
|
-
%{insert into #{@transform_lock_table}
|
82
|
-
(
|
83
|
-
transform_lock_id,
|
84
|
-
transform_name,
|
85
|
-
transform_lock_time
|
86
|
-
)
|
87
|
-
values
|
88
|
-
(
|
89
|
-
#{@transform_lock_id},
|
90
|
-
'#{@transform_name}',
|
91
|
-
current_timestamp
|
92
|
-
);},
|
93
|
-
@logger
|
94
|
-
)
|
95
|
-
|
96
|
-
if @database_manager.supports_transactions == true
|
97
|
-
JDBCHelpers::Execute.new(
|
98
|
-
jdbc_connection,
|
99
|
-
@database_manager.commit_statement,
|
100
|
-
@logger
|
101
|
-
)
|
102
|
-
end
|
103
|
-
|
104
|
-
lock_value = true
|
105
|
-
else
|
106
|
-
if @database_manager.supports_transactions == true
|
107
|
-
JDBCHelpers::Execute.new(
|
108
|
-
jdbc_connection,
|
109
|
-
@database_manager.rollback_statement,
|
110
|
-
@logger
|
111
|
-
)
|
112
|
-
end
|
113
|
-
lock_value = false
|
114
|
-
end
|
115
|
-
|
116
|
-
if lock_value == true
|
117
|
-
@logger.info("#{@transform_name} transform lock acquired;")
|
118
|
-
else
|
119
|
-
@logger.info("#{@transform_name} is locked by another process")
|
120
|
-
end
|
121
|
-
ensure
|
122
|
-
# close this connection
|
123
|
-
jdbc_connection.close
|
124
|
-
|
125
|
-
lock_value
|
126
|
-
end
|
127
|
-
|
128
|
-
# Removes transform lock if it's present.
|
129
|
-
def remove_lock
|
130
|
-
# get a fresh jdbc connection...
|
131
|
-
# to avoid committing the main transform unnecessarily
|
132
|
-
jdbc_connection = create_jdbc_connection
|
133
|
-
|
134
|
-
if @database_manager.supports_transactions == true
|
135
|
-
JDBCHelpers::Execute.new(
|
136
|
-
jdbc_connection,
|
137
|
-
@database_manager.begin_statement,
|
138
|
-
@logger
|
139
|
-
)
|
140
|
-
end
|
141
|
-
|
142
|
-
if @database_manager.supports_table_locking == true
|
143
|
-
JDBCHelpers::Execute.new(
|
144
|
-
jdbc_connection,
|
145
|
-
@database_manager.lock_table_statement(@transform_lock_table),
|
146
|
-
@logger
|
147
|
-
)
|
148
|
-
end
|
149
|
-
|
150
|
-
# delete the lock record and commit
|
151
|
-
JDBCHelpers::Execute.new(
|
152
|
-
jdbc_connection,
|
153
|
-
%(delete from #{@transform_lock_table}
|
154
|
-
where transform_name='#{@transform_name}'
|
155
|
-
and transform_lock_id=#{@transform_lock_id};),
|
156
|
-
@logger
|
157
|
-
)
|
158
|
-
|
159
|
-
c = JDBCHelpers::Execute.new(
|
160
|
-
jdbc_connection,
|
161
|
-
@database_manager.commit_statement,
|
162
|
-
@logger
|
163
|
-
)
|
164
|
-
ensure
|
165
|
-
jdbc_connection.close
|
166
|
-
|
167
|
-
return true
|
168
|
-
end
|
169
|
-
|
170
|
-
# TBD
|
171
|
-
def lock_all_available_transforms
|
172
|
-
# undefined at this time
|
173
|
-
end
|
174
|
-
end
|
175
|
-
end
|
176
|
-
end
|
data/lib/sneaql_lib/standard.rb
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
# Classes to manage DB objects for standard SneaQL deployment.
|
3
|
-
module Standard
|
4
|
-
# Manages execution of a standard transform
|
5
|
-
class StandardTransform
|
6
|
-
|
7
|
-
attr_accessor {
|
8
|
-
:jdbc_url,
|
9
|
-
:db_user,
|
10
|
-
:db_pass
|
11
|
-
}
|
12
|
-
|
13
|
-
def initialize()
|
14
|
-
# set parameters
|
15
|
-
|
16
|
-
# determine database type
|
17
|
-
|
18
|
-
end
|
19
|
-
|
20
|
-
def execute_all_transforms()
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
def execute_single_transform(transform_name)
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
@@ -1,232 +0,0 @@
|
|
1
|
-
module Sneaql
|
2
|
-
# Classes to manage DB objects for standard SneaQL deployment.
|
3
|
-
module Standard
|
4
|
-
# Facilitate creation/recreation of database table objects
|
5
|
-
# for standard SneaQL deployment.
|
6
|
-
class DBObjectCreator
|
7
|
-
attr_accessor :jdbc_connection
|
8
|
-
|
9
|
-
# @param [Object] jdbc_connection
|
10
|
-
# @param [Sneaql::Core::DatabaseManager] database_manager
|
11
|
-
# @param [Logger] logger
|
12
|
-
def initialize(jdbc_connection, database_manager, logger = nil)
|
13
|
-
@logger = logger ? logger : Logger.new(STDOUT)
|
14
|
-
@jdbc_connection = jdbc_connection
|
15
|
-
@database_manager = database_manager
|
16
|
-
end
|
17
|
-
|
18
|
-
# Drops and recreates the primary transform table.
|
19
|
-
# @param [String] transform_table_name fully qualified name for this table
|
20
|
-
# @return [Boolean]
|
21
|
-
def recreate_transforms_table(transform_table_name)
|
22
|
-
JDBCHelpers::Execute.new(
|
23
|
-
@jdbc_connection,
|
24
|
-
"drop table if exists #{ transform_table_name };",
|
25
|
-
@logger
|
26
|
-
)
|
27
|
-
create_transforms_table(transform_table_name)
|
28
|
-
return true
|
29
|
-
end
|
30
|
-
|
31
|
-
# Creates the primary transform table.
|
32
|
-
# @param [String] transform_table_name fully qualified name for this table
|
33
|
-
# @return [Boolean]
|
34
|
-
def create_transforms_table(transform_table_name)
|
35
|
-
JDBCHelpers::Execute.new(
|
36
|
-
@jdbc_connection,
|
37
|
-
transforms_table_create_statement(transform_table_name),
|
38
|
-
@logger
|
39
|
-
)
|
40
|
-
return true
|
41
|
-
end
|
42
|
-
|
43
|
-
# Coerces a boolean to the appropriate value for the database type.
|
44
|
-
# May return a 0 or 1 in RDBMS where boolean is not supported.
|
45
|
-
# @param [Boolean] boolean_value
|
46
|
-
# @return [Boolean, Fixnum]
|
47
|
-
def coerce_boolean(boolean_value)
|
48
|
-
if @database_manager.has_boolean
|
49
|
-
boolean_value
|
50
|
-
else
|
51
|
-
boolean_value == true ? 1 : 0
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
# Create table statement for primary transform table.
|
56
|
-
# @param [String] transform_table_name fully qualified name for this table
|
57
|
-
# @return [String]
|
58
|
-
def transforms_table_create_statement(transform_table_name)
|
59
|
-
%{create table if not exists #{ transform_table_name }
|
60
|
-
(
|
61
|
-
transform_name varchar(255) not null
|
62
|
-
,sql_repository varchar(255)
|
63
|
-
,sql_repository_branch varchar(255)
|
64
|
-
,is_active #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
65
|
-
,notify_on_success #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
66
|
-
,notify_on_non_precondition_failure #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
67
|
-
,notify_on_precondition_failure #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
68
|
-
,updated_ts timestamp
|
69
|
-
);}
|
70
|
-
end
|
71
|
-
|
72
|
-
# Creates a record in the transforms table.
|
73
|
-
# @param [String] transform_table_name
|
74
|
-
# @param [Hash] params Hash of parameters with symbols matching column names
|
75
|
-
def create_transform(transform_table_name, params)
|
76
|
-
JDBCHelpers::Execute.new(
|
77
|
-
@jdbc_connection,
|
78
|
-
create_transform_statement(transform_table_name, params),
|
79
|
-
@logger
|
80
|
-
)
|
81
|
-
end
|
82
|
-
|
83
|
-
# @param [String] transform_table_name
|
84
|
-
# @param [Hash] params Hash of parameters with symbols matching column names
|
85
|
-
def create_transform_statement(transform_table_name, params)
|
86
|
-
%{insert into #{transform_table_name}
|
87
|
-
(
|
88
|
-
transform_name
|
89
|
-
,sql_repository
|
90
|
-
,sql_repository_branch
|
91
|
-
,is_active
|
92
|
-
,notify_on_success
|
93
|
-
,notify_on_non_precondition_failure
|
94
|
-
,notify_on_precondition_failure
|
95
|
-
,updated_ts
|
96
|
-
)
|
97
|
-
values
|
98
|
-
(
|
99
|
-
'#{params[:transform_name]}'
|
100
|
-
,'#{params[:sql_repository]}'
|
101
|
-
,'#{params[:sql_repository_branch]}'
|
102
|
-
,#{coerce_boolean(params[:is_active])}
|
103
|
-
,#{coerce_boolean(params[:notify_on_success])}
|
104
|
-
,#{coerce_boolean(params[:notify_on_non_precondition_failure])}
|
105
|
-
,#{coerce_boolean(params[:notify_on_precondition_failure])}
|
106
|
-
,current_timestamp
|
107
|
-
);}
|
108
|
-
end
|
109
|
-
|
110
|
-
# Drops and recreates the transform steps table.
|
111
|
-
# @param [String] transform_steps_table_name fully qualified name for this table
|
112
|
-
# @return [Boolean]
|
113
|
-
def recreate_transform_steps_table(transform_steps_table_name)
|
114
|
-
JDBCHelpers::Execute.new(
|
115
|
-
@jdbc_connection,
|
116
|
-
"drop table if exists #{ transform_steps_table_name };",
|
117
|
-
@logger
|
118
|
-
)
|
119
|
-
create_transform_steps_table(transform_steps_table_name)
|
120
|
-
return true
|
121
|
-
end
|
122
|
-
|
123
|
-
# Creates the transform steps table.
|
124
|
-
# @param [String] transform_steps_table_name fully qualified name for this table
|
125
|
-
# @return [Boolean]
|
126
|
-
def create_transform_steps_table(transform_steps_table_name)
|
127
|
-
JDBCHelpers::Execute.new(
|
128
|
-
@jdbc_connection,
|
129
|
-
transform_steps_table_create_statement(transform_steps_table_name),
|
130
|
-
@logger
|
131
|
-
)
|
132
|
-
return true
|
133
|
-
end
|
134
|
-
|
135
|
-
# Create table statement for transform steps table.
|
136
|
-
# @param [String] transform_steps_table_name fully qualified name for this table
|
137
|
-
# @return [String]
|
138
|
-
def transform_steps_table_create_statement(transform_steps_table_name)
|
139
|
-
%{create table if not exists #{ transform_steps_table_name }
|
140
|
-
(
|
141
|
-
transform_name varchar(255) not null
|
142
|
-
,transform_step integer not null
|
143
|
-
,sql_file_path_in_repo varchar(1024)
|
144
|
-
,is_active #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
145
|
-
,is_precondition #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
146
|
-
,updated_ts timestamp
|
147
|
-
);}
|
148
|
-
end
|
149
|
-
|
150
|
-
# Drops and recreates the transform lock table.
|
151
|
-
# @param [String] transform_lock_table_name fully qualified name for this table
|
152
|
-
# @return [Boolean]
|
153
|
-
def recreate_transform_lock_table(transform_lock_table_name)
|
154
|
-
JDBCHelpers::Execute.new(
|
155
|
-
@jdbc_connection,
|
156
|
-
"drop table if exists #{ transform_lock_table_name };",
|
157
|
-
@logger
|
158
|
-
)
|
159
|
-
create_transform_lock_table(transform_lock_table_name)
|
160
|
-
return true
|
161
|
-
end
|
162
|
-
|
163
|
-
# Creates the transform lock table.
|
164
|
-
# @param [String] transform_lock_table_name fully qualified name for this table
|
165
|
-
# @return [Boolean]
|
166
|
-
def create_transform_lock_table(transform_lock_table_name)
|
167
|
-
JDBCHelpers::Execute.new(
|
168
|
-
@jdbc_connection,
|
169
|
-
transform_lock_table_create_statement(transform_lock_table_name),
|
170
|
-
@logger
|
171
|
-
)
|
172
|
-
return true
|
173
|
-
end
|
174
|
-
|
175
|
-
# Create table statement for transform lock table.
|
176
|
-
# @param [String] transform_lock_table_name fully qualified name for this table
|
177
|
-
# @return [String]
|
178
|
-
def transform_lock_table_create_statement(transform_lock_table_name)
|
179
|
-
%{create table if not exists #{ transform_lock_table_name }
|
180
|
-
(
|
181
|
-
transform_lock_id bigint
|
182
|
-
,transform_name varchar(255)
|
183
|
-
,transform_lock_time timestamp
|
184
|
-
);}
|
185
|
-
end
|
186
|
-
|
187
|
-
# Drops and recreates the transform log table.
|
188
|
-
# @param [String] transform_log_table_name fully qualified name for this table
|
189
|
-
# @return [Boolean]
|
190
|
-
def recreate_transform_log_table(transform_log_table_name)
|
191
|
-
JDBCHelpers::Execute.new(
|
192
|
-
@jdbc_connection,
|
193
|
-
"drop table if exists #{ transform_log_table_name };",
|
194
|
-
@logger
|
195
|
-
)
|
196
|
-
create_transform_log_table(transform_log_table_name)
|
197
|
-
return true
|
198
|
-
end
|
199
|
-
|
200
|
-
# Creates the transform log table.
|
201
|
-
# @param [String] transform_log_table_name fully qualified name for this table
|
202
|
-
# @return [Boolean]
|
203
|
-
def create_transform_log_table(transform_log_table_name)
|
204
|
-
JDBCHelpers::Execute.new(
|
205
|
-
@jdbc_connection,
|
206
|
-
transform_log_table_create_statement(transform_log_table_name),
|
207
|
-
@logger
|
208
|
-
)
|
209
|
-
return true
|
210
|
-
end
|
211
|
-
|
212
|
-
# Create table statement for transform log table.
|
213
|
-
# @param [String] transform_log_table_name fully qualified name for this table
|
214
|
-
# @return [String]
|
215
|
-
def transform_log_table_create_statement(transform_log_table_name)
|
216
|
-
%{create table if not exists #{ transform_log_table_name }
|
217
|
-
(
|
218
|
-
transform_run_id bigint
|
219
|
-
,transform_lock_id bigint
|
220
|
-
,transform_name varchar(255)
|
221
|
-
,transform_step integer
|
222
|
-
,transform_statement integer
|
223
|
-
,all_steps_complete #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
224
|
-
,failed_in_precondition #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
|
225
|
-
,message varchar(65000)
|
226
|
-
,transform_start_time timestamp
|
227
|
-
,transform_end_time timestamp
|
228
|
-
);}
|
229
|
-
end
|
230
|
-
end
|
231
|
-
end
|
232
|
-
end
|