waterworks 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/waterworks +30 -0
- data/lib/waterworks/actions/sns_alarm.rb +22 -0
- data/lib/waterworks/actions/terminate.rb +18 -0
- data/lib/waterworks/activities/copy_activity.rb +38 -0
- data/lib/waterworks/activities/emr_activity.rb +43 -0
- data/lib/waterworks/activities/hadoop_activity.rb +44 -0
- data/lib/waterworks/activities/hive_activity.rb +49 -0
- data/lib/waterworks/activities/hive_copy_activity.rb +43 -0
- data/lib/waterworks/activities/pig_activity.rb +48 -0
- data/lib/waterworks/activities/redshift_copy_activity.rb +42 -0
- data/lib/waterworks/activities/shell_command_activity.rb +44 -0
- data/lib/waterworks/activities/sql_activity.rb +43 -0
- data/lib/waterworks/containers/pipeline_definition.rb +66 -0
- data/lib/waterworks/data_formats/csv.rb +20 -0
- data/lib/waterworks/data_formats/custom.rb +21 -0
- data/lib/waterworks/data_formats/dynamo_db_data_format.rb +19 -0
- data/lib/waterworks/data_formats/dynamo_db_export_data_format.rb +19 -0
- data/lib/waterworks/data_formats/reg_ex.rb +21 -0
- data/lib/waterworks/data_formats/tsv.rb +22 -0
- data/lib/waterworks/data_nodes/dynamo_db_data_node.rb +41 -0
- data/lib/waterworks/data_nodes/my_sql_data_node.rb +42 -0
- data/lib/waterworks/data_nodes/redshift_data_node.rb +41 -0
- data/lib/waterworks/data_nodes/s3_data_node.rb +42 -0
- data/lib/waterworks/data_nodes/sql_data_node.rb +42 -0
- data/lib/waterworks/databases/jdbc_database.rb +25 -0
- data/lib/waterworks/databases/rds_database.rb +25 -0
- data/lib/waterworks/databases/redshift_database.rb +25 -0
- data/lib/waterworks/other/default.rb +22 -0
- data/lib/waterworks/pipeline_object.rb +205 -0
- data/lib/waterworks/preconditions/dynamo_db_data_exists.rb +31 -0
- data/lib/waterworks/preconditions/dynamo_db_table_exists.rb +31 -0
- data/lib/waterworks/preconditions/exists.rb +29 -0
- data/lib/waterworks/preconditions/s3_key_exists.rb +31 -0
- data/lib/waterworks/preconditions/s3_prefix_not_empty.rb +31 -0
- data/lib/waterworks/preconditions/shell_command_precondition.rb +34 -0
- data/lib/waterworks/resources/ec2_resource.rb +57 -0
- data/lib/waterworks/resources/emr_cluster.rb +68 -0
- data/lib/waterworks/resources/http_proxy.rb +25 -0
- data/lib/waterworks/schedule/schedule.rb +23 -0
- data/lib/waterworks/util.rb +27 -0
- data/lib/waterworks/utilities/emr_configuration.rb +21 -0
- data/lib/waterworks/utilities/property.rb +20 -0
- data/lib/waterworks/utilities/shell_script_config.rb +20 -0
- data/lib/waterworks.rb +1 -0
- metadata +62 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8761403bf118f39291c4eccbf1df1eb2c3044bc6
|
4
|
+
data.tar.gz: 9250556d67c9b6488c303405406f484ed6002c56
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2ac32443b3f1f9e60276cbe9ccd0056250965406b9c14989ecf94e64215916d9eaa3f003b74719315e89ee4f5c608287deb6190f5192d2dbe4ee0a239b1fd788
|
7
|
+
data.tar.gz: 4c1913ed2e0001b00ee1d7a7e6fa53a20c0456b79b87b4c994837a1d67c0423d754d4c0cc645cae5a317770a6b1a20f6509110e58b4ef61e3d2d8680b8e7b0d3
|
data/bin/waterworks
CHANGED
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'waterworks'
|
4
|
+
require 'thor'
|
5
|
+
|
6
|
+
include Waterworks
|
7
|
+
class Runner < Thor
|
8
|
+
desc 'parse <file>',
|
9
|
+
'Parse JSON exported from AWS and print compatible source. STDIN if no file specified'
|
10
|
+
def parse(json_file = nil)
|
11
|
+
jdata = if json_file.nil?
|
12
|
+
JSON.parse(STDIN.read)
|
13
|
+
else
|
14
|
+
JSON.parse(File.read(json_file))
|
15
|
+
end
|
16
|
+
jdata['objects'].each do |jobject|
|
17
|
+
type = jobject['type']
|
18
|
+
if type.nil?
|
19
|
+
print PipelineObject.from_hash(jobject, Default.new).sourceify
|
20
|
+
else
|
21
|
+
clazz = Object.const_get(type).new
|
22
|
+
raise "Unknown Pipeline #{type}" unless clazz.is_a? PipelineObject
|
23
|
+
print PipelineObject.from_hash(jobject, clazz).sourceify
|
24
|
+
end
|
25
|
+
puts
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
Runner.start(ARGV)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class SnsAlarm < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
parent: :ref,
|
8
|
+
message: :string,
|
9
|
+
role: :string,
|
10
|
+
subject: :string,
|
11
|
+
topicArn: :string,
|
12
|
+
type: :string,
|
13
|
+
}.merge superclass.safe_fields
|
14
|
+
end
|
15
|
+
|
16
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
17
|
+
|
18
|
+
def initialize(id = nil, name = nil)
|
19
|
+
super(id, name).set_attrs(type: 'SnsAlarm')
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class Terminate < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
parent: :ref,
|
8
|
+
type: :string,
|
9
|
+
}.merge superclass.safe_fields
|
10
|
+
end
|
11
|
+
|
12
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
13
|
+
|
14
|
+
def initialize(id = nil, name = nil)
|
15
|
+
super(id, name).set_attrs(type: 'Terminate')
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class CopyActivity < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
dependsOn: :ref,
|
10
|
+
failureAndRerunMode: :string,
|
11
|
+
input: :ref,
|
12
|
+
lateAfterTimeout: :string,
|
13
|
+
maxActiveInstances: :string,
|
14
|
+
maximumRetries: :string,
|
15
|
+
onFail: :ref,
|
16
|
+
onLateAction: :ref,
|
17
|
+
onSuccess: :ref,
|
18
|
+
output: :ref,
|
19
|
+
parent: :ref,
|
20
|
+
pipelineLogUri: :string,
|
21
|
+
precondition: :ref,
|
22
|
+
reportProgressTimeout: :string,
|
23
|
+
retryDelay: :string,
|
24
|
+
scheduleType: :string,
|
25
|
+
schedule: :ref,
|
26
|
+
runsOn: :ref,
|
27
|
+
workerGroup: :string,
|
28
|
+
type: :string,
|
29
|
+
}.merge superclass.safe_fields
|
30
|
+
end
|
31
|
+
|
32
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
33
|
+
|
34
|
+
def initialize(id = nil, name = nil)
|
35
|
+
super(id, name).set_attrs(type: 'CopyActivity')
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class EmrActivity < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
dependsOn: :ref,
|
10
|
+
failureAndRerunMode: :string,
|
11
|
+
input: :ref,
|
12
|
+
lateAfterTimeout: :string,
|
13
|
+
maxActiveInstances: :string,
|
14
|
+
maximumRetries: :string,
|
15
|
+
onFail: :ref,
|
16
|
+
onLateAction: :ref,
|
17
|
+
onSuccess: :ref,
|
18
|
+
output: :ref,
|
19
|
+
parent: :ref,
|
20
|
+
pipelineLogUri: :string,
|
21
|
+
postStepCommand: :string,
|
22
|
+
precondition: :ref,
|
23
|
+
preStepCommand: :string,
|
24
|
+
reportProgressTimeout: :string,
|
25
|
+
resizeClusterBeforeRunning: :string,
|
26
|
+
resizeClusterMaxInstances: :string,
|
27
|
+
retryDelay: :string,
|
28
|
+
scheduleType: :string,
|
29
|
+
step: :string,
|
30
|
+
schedule: :ref,
|
31
|
+
runsOn: :ref,
|
32
|
+
workerGroup: :string,
|
33
|
+
type: :string,
|
34
|
+
}.merge superclass.safe_fields
|
35
|
+
end
|
36
|
+
|
37
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
38
|
+
|
39
|
+
def initialize(id = nil, name = nil)
|
40
|
+
super(id, name).set_attrs(type: 'EmrActivity')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class HadoopActivity < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
argument: :string,
|
8
|
+
attemptStatus: :string,
|
9
|
+
attemptTimeout: :string,
|
10
|
+
dependsOn: :ref,
|
11
|
+
failureAndRerunMode: :string,
|
12
|
+
hadoopQueue: :string,
|
13
|
+
input: :ref,
|
14
|
+
lateAfterTimeout: :string,
|
15
|
+
mainClass: :string,
|
16
|
+
maxActiveInstances: :string,
|
17
|
+
maximumRetries: :string,
|
18
|
+
onFail: :ref,
|
19
|
+
onLateAction: :ref,
|
20
|
+
onSuccess: :ref,
|
21
|
+
output: :ref,
|
22
|
+
parent: :ref,
|
23
|
+
pipelineLogUri: :string,
|
24
|
+
postActivityTaskConfig: :ref,
|
25
|
+
preActivityTaskConfig: :ref,
|
26
|
+
precondition: :ref,
|
27
|
+
reportProgressTimeout: :string,
|
28
|
+
retryDelay: :string,
|
29
|
+
scheduleType: :string,
|
30
|
+
jarUri: :string,
|
31
|
+
schedule: :ref,
|
32
|
+
runsOn: :ref,
|
33
|
+
workerGroup: :string,
|
34
|
+
type: :string,
|
35
|
+
}.merge superclass.safe_fields
|
36
|
+
end
|
37
|
+
|
38
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
39
|
+
|
40
|
+
def initialize(id = nil, name = nil)
|
41
|
+
super(id, name).set_attrs(type: 'HadoopActivity')
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class HiveActivity < PipelineObject
|
5
|
+
# rubocop:disable Metrics/MethodLength
|
6
|
+
def self.safe_fields
|
7
|
+
{
|
8
|
+
attemptStatus: :string,
|
9
|
+
attemptTimeout: :string,
|
10
|
+
dependsOn: :ref,
|
11
|
+
failureAndRerunMode: :string,
|
12
|
+
hadoopQueue: :string,
|
13
|
+
input: :ref,
|
14
|
+
lateAfterTimeout: :string,
|
15
|
+
maxActiveInstances: :string,
|
16
|
+
maximumRetries: :string,
|
17
|
+
onFail: :ref,
|
18
|
+
onLateAction: :ref,
|
19
|
+
onSuccess: :ref,
|
20
|
+
output: :ref,
|
21
|
+
parent: :ref,
|
22
|
+
pipelineLogUri: :string,
|
23
|
+
postActivityTaskConfig: :ref,
|
24
|
+
preActivityTaskConfig: :ref,
|
25
|
+
precondition: :ref,
|
26
|
+
reportProgressTimeout: :string,
|
27
|
+
resizeClusterBeforeRunning: :string,
|
28
|
+
resizeClusterMaxInstances: :string,
|
29
|
+
retryDelay: :string,
|
30
|
+
scheduleType: :string,
|
31
|
+
scriptVariable: :string,
|
32
|
+
stage: :string,
|
33
|
+
schedule: :ref,
|
34
|
+
hiveScript: :string,
|
35
|
+
scriptUri: :string,
|
36
|
+
runsOn: :ref,
|
37
|
+
workerGroup: :string,
|
38
|
+
type: :string,
|
39
|
+
}.merge superclass.safe_fields
|
40
|
+
end
|
41
|
+
# rubocop:enable Metrics/MethodLength
|
42
|
+
|
43
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
44
|
+
|
45
|
+
def initialize(id = nil, name = nil)
|
46
|
+
super(id, name).set_attrs(type: 'HiveActivity')
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class HiveCopyActivity < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
dependsOn: :ref,
|
10
|
+
failureAndRerunMode: :string,
|
11
|
+
filterSql: :string,
|
12
|
+
input: :ref,
|
13
|
+
lateAfterTimeout: :string,
|
14
|
+
maxActiveInstances: :string,
|
15
|
+
maximumRetries: :string,
|
16
|
+
onFail: :ref,
|
17
|
+
onLateAction: :ref,
|
18
|
+
onSuccess: :ref,
|
19
|
+
output: :ref,
|
20
|
+
parent: :ref,
|
21
|
+
pipelineLogUri: :string,
|
22
|
+
postActivityTaskConfig: :ref,
|
23
|
+
preActivityTaskConfig: :ref,
|
24
|
+
precondition: :ref,
|
25
|
+
reportProgressTimeout: :string,
|
26
|
+
resizeClusterBeforeRunning: :string,
|
27
|
+
resizeClusterMaxInstances: :string,
|
28
|
+
retryDelay: :string,
|
29
|
+
scheduleType: :string,
|
30
|
+
schedule: :ref,
|
31
|
+
runsOn: :ref,
|
32
|
+
workerGroup: :string,
|
33
|
+
type: :string,
|
34
|
+
}.merge superclass.safe_fields
|
35
|
+
end
|
36
|
+
|
37
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
38
|
+
|
39
|
+
def initialize(id = nil, name = nil)
|
40
|
+
super(id, name).set_attrs(type: 'HiveCopyActivity')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class PigActivity < PipelineObject
|
5
|
+
# rubocop:disable Metrics/MethodLength
|
6
|
+
def self.safe_fields
|
7
|
+
{
|
8
|
+
attemptStatus: :string,
|
9
|
+
attemptTimeout: :string,
|
10
|
+
dependsOn: :ref,
|
11
|
+
failureAndRerunMode: :string,
|
12
|
+
input: :ref,
|
13
|
+
lateAfterTimeout: :string,
|
14
|
+
maxActiveInstances: :string,
|
15
|
+
maximumRetries: :string,
|
16
|
+
onFail: :ref,
|
17
|
+
onLateAction: :ref,
|
18
|
+
onSuccess: :ref,
|
19
|
+
output: :ref,
|
20
|
+
parent: :ref,
|
21
|
+
pipelineLogUri: :string,
|
22
|
+
postActivityTaskConfig: :ref,
|
23
|
+
preActivityTaskConfig: :ref,
|
24
|
+
precondition: :ref,
|
25
|
+
reportProgressTimeout: :string,
|
26
|
+
resizeClusterBeforeRunning: :string,
|
27
|
+
resizeClusterMaxInstances: :string,
|
28
|
+
retryDelay: :string,
|
29
|
+
scheduleType: :string,
|
30
|
+
scriptVariable: :string,
|
31
|
+
stage: :string,
|
32
|
+
schedule: :ref,
|
33
|
+
script: :string,
|
34
|
+
scriptUri: :string,
|
35
|
+
runsOn: :ref,
|
36
|
+
workerGroup: :string,
|
37
|
+
type: :string,
|
38
|
+
}.merge superclass.safe_fields
|
39
|
+
end
|
40
|
+
# rubocop:enable Metrics/MethodLength
|
41
|
+
|
42
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
43
|
+
|
44
|
+
def initialize(id = nil, name = nil)
|
45
|
+
super(id, name).set_attrs(type: 'PigActivity')
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class RedshiftCopyActivity < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
commandOptions: :string,
|
10
|
+
dependsOn: :ref,
|
11
|
+
failureAndRerunMode: :string,
|
12
|
+
input: :ref,
|
13
|
+
lateAfterTimeout: :string,
|
14
|
+
maxActiveInstances: :string,
|
15
|
+
maximumRetries: :string,
|
16
|
+
onFail: :ref,
|
17
|
+
onLateAction: :ref,
|
18
|
+
onSuccess: :ref,
|
19
|
+
output: :ref,
|
20
|
+
parent: :ref,
|
21
|
+
pipelineLogUri: :string,
|
22
|
+
precondition: :ref,
|
23
|
+
queue: :string,
|
24
|
+
reportProgressTimeout: :string,
|
25
|
+
retryDelay: :string,
|
26
|
+
scheduleType: :string,
|
27
|
+
transformSql: :string,
|
28
|
+
insertMode: :string,
|
29
|
+
schedule: :ref,
|
30
|
+
runsOn: :ref,
|
31
|
+
workerGroup: :string,
|
32
|
+
type: :string,
|
33
|
+
}.merge superclass.safe_fields
|
34
|
+
end
|
35
|
+
|
36
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
37
|
+
|
38
|
+
def initialize(id = nil, name = nil)
|
39
|
+
super(id, name).set_attrs(type: 'RedshiftCopyActivity')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class ShellCommandActivity < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
dependsOn: :ref,
|
10
|
+
failureAndRerunMode: :string,
|
11
|
+
input: :ref,
|
12
|
+
lateAfterTimeout: :string,
|
13
|
+
maxActiveInstances: :string,
|
14
|
+
maximumRetries: :string,
|
15
|
+
onFail: :ref,
|
16
|
+
onLateAction: :ref,
|
17
|
+
onSuccess: :ref,
|
18
|
+
output: :ref,
|
19
|
+
parent: :ref,
|
20
|
+
pipelineLogUri: :string,
|
21
|
+
precondition: :ref,
|
22
|
+
reportProgressTimeout: :string,
|
23
|
+
retryDelay: :string,
|
24
|
+
scheduleType: :string,
|
25
|
+
scriptArgument: :string,
|
26
|
+
stage: :string,
|
27
|
+
stderr: :string,
|
28
|
+
stdout: :string,
|
29
|
+
schedule: :ref,
|
30
|
+
command: :string,
|
31
|
+
scriptUri: :string,
|
32
|
+
runsOn: :ref,
|
33
|
+
workerGroup: :string,
|
34
|
+
type: :string,
|
35
|
+
}.merge superclass.safe_fields
|
36
|
+
end
|
37
|
+
|
38
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
39
|
+
|
40
|
+
def initialize(id = nil, name = nil)
|
41
|
+
super(id, name).set_attrs(type: 'ShellCommandActivity')
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class SqlActivity < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
dependsOn: :ref,
|
10
|
+
failureAndRerunMode: :string,
|
11
|
+
input: :ref,
|
12
|
+
lateAfterTimeout: :string,
|
13
|
+
maxActiveInstances: :string,
|
14
|
+
maximumRetries: :string,
|
15
|
+
onFail: :ref,
|
16
|
+
onLateAction: :ref,
|
17
|
+
onSuccess: :ref,
|
18
|
+
output: :ref,
|
19
|
+
parent: :ref,
|
20
|
+
pipelineLogUri: :string,
|
21
|
+
precondition: :ref,
|
22
|
+
queue: :string,
|
23
|
+
reportProgressTimeout: :string,
|
24
|
+
retryDelay: :string,
|
25
|
+
scheduleType: :string,
|
26
|
+
scriptArgument: :string,
|
27
|
+
database: :ref,
|
28
|
+
schedule: :ref,
|
29
|
+
script: :string,
|
30
|
+
scriptUri: :string,
|
31
|
+
runsOn: :ref,
|
32
|
+
workerGroup: :string,
|
33
|
+
type: :string,
|
34
|
+
}.merge superclass.safe_fields
|
35
|
+
end
|
36
|
+
|
37
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
38
|
+
|
39
|
+
def initialize(id = nil, name = nil)
|
40
|
+
super(id, name).set_attrs(type: 'SqlActivity')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class PipelineDefinition
|
5
|
+
attr_accessor :uid, :name, :pipeline_id, :description, :tags, :pipeline_objects
|
6
|
+
|
7
|
+
def initialize(uid, name, client = nil)
|
8
|
+
@uid = uid
|
9
|
+
@name = name
|
10
|
+
client = Aws::DataPipeline::Client.new if client.nil?
|
11
|
+
@client = client
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_pipeline!
|
15
|
+
resp = @client.create_pipeline(
|
16
|
+
name: @name,
|
17
|
+
unique_id: @uid,
|
18
|
+
description: @description,
|
19
|
+
tags: [@tags].flatten.map(&:to_hash)
|
20
|
+
)
|
21
|
+
@pipeline_id = resp.pipeline_id
|
22
|
+
resp
|
23
|
+
end
|
24
|
+
|
25
|
+
def put_pipeline!
|
26
|
+
@client.put_pipeline_definition(
|
27
|
+
pipeline_id: @pipeline_id,
|
28
|
+
pipeline_objects: @pipeline_objects.map(&:to_fhash)
|
29
|
+
)
|
30
|
+
end
|
31
|
+
|
32
|
+
def activate_pipeline!
|
33
|
+
@client.activate_pipeline(
|
34
|
+
pipeline_id: @pipeline_id
|
35
|
+
)
|
36
|
+
end
|
37
|
+
|
38
|
+
def go_baby_go!
|
39
|
+
responses = {}
|
40
|
+
responses[:create] = create_pipeline!
|
41
|
+
responses[:put] = put_pipeline!
|
42
|
+
responses[:activate] = activate_pipeline!
|
43
|
+
responses
|
44
|
+
end
|
45
|
+
|
46
|
+
def validate!
|
47
|
+
@uid && @name && @pipeline_object.any?
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class Tag
|
52
|
+
attr_accessor :key, :value
|
53
|
+
|
54
|
+
def initialize(key, value)
|
55
|
+
@key = key
|
56
|
+
@value = value
|
57
|
+
end
|
58
|
+
|
59
|
+
def to_hash
|
60
|
+
{
|
61
|
+
key: @key,
|
62
|
+
value: @value
|
63
|
+
}
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class CSV < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
column: :string,
|
8
|
+
escapeChar: :string,
|
9
|
+
parent: :ref,
|
10
|
+
type: :string,
|
11
|
+
}.merge superclass.safe_fields
|
12
|
+
end
|
13
|
+
|
14
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
15
|
+
|
16
|
+
def initialize(id = nil, name = nil)
|
17
|
+
super(id, name).set_attrs(type: 'CSV')
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class Custom < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
column: :string,
|
8
|
+
parent: :ref,
|
9
|
+
recordSeparator: :string,
|
10
|
+
columnSeparator: :string,
|
11
|
+
type: :string,
|
12
|
+
}.merge superclass.safe_fields
|
13
|
+
end
|
14
|
+
|
15
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
16
|
+
|
17
|
+
def initialize(id = nil, name = nil)
|
18
|
+
super(id, name).set_attrs(type: 'Custom')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class DynamoDBDataFormat < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
column: :string,
|
8
|
+
parent: :ref,
|
9
|
+
type: :string,
|
10
|
+
}.merge superclass.safe_fields
|
11
|
+
end
|
12
|
+
|
13
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
14
|
+
|
15
|
+
def initialize(id = nil, name = nil)
|
16
|
+
super(id, name).set_attrs(type: 'DynamoDBDataFormat')
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class DynamoDBExportDataFormat < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
column: :string,
|
8
|
+
parent: :ref,
|
9
|
+
type: :string,
|
10
|
+
}.merge superclass.safe_fields
|
11
|
+
end
|
12
|
+
|
13
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
14
|
+
|
15
|
+
def initialize(id = nil, name = nil)
|
16
|
+
super(id, name).set_attrs(type: 'DynamoDBExportDataFormat')
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class RegEx < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
column: :string,
|
8
|
+
inputRegEx: :string,
|
9
|
+
outputFormat: :string,
|
10
|
+
parent: :ref,
|
11
|
+
type: :string,
|
12
|
+
}.merge superclass.safe_fields
|
13
|
+
end
|
14
|
+
|
15
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
16
|
+
|
17
|
+
def initialize(id = nil, name = nil)
|
18
|
+
super(id, name).set_attrs(type: 'RegEx')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class TSV < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
column: :string,
|
8
|
+
columnSeparator: :string,
|
9
|
+
escapeChar: :string,
|
10
|
+
parent: :ref,
|
11
|
+
recordSeparator: :string,
|
12
|
+
type: :string,
|
13
|
+
}.merge superclass.safe_fields
|
14
|
+
end
|
15
|
+
|
16
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
17
|
+
|
18
|
+
def initialize(id = nil, name = nil)
|
19
|
+
super(id, name).set_attrs(type: 'TSV')
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|