waterworks 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/waterworks +30 -0
- data/lib/waterworks/actions/sns_alarm.rb +22 -0
- data/lib/waterworks/actions/terminate.rb +18 -0
- data/lib/waterworks/activities/copy_activity.rb +38 -0
- data/lib/waterworks/activities/emr_activity.rb +43 -0
- data/lib/waterworks/activities/hadoop_activity.rb +44 -0
- data/lib/waterworks/activities/hive_activity.rb +49 -0
- data/lib/waterworks/activities/hive_copy_activity.rb +43 -0
- data/lib/waterworks/activities/pig_activity.rb +48 -0
- data/lib/waterworks/activities/redshift_copy_activity.rb +42 -0
- data/lib/waterworks/activities/shell_command_activity.rb +44 -0
- data/lib/waterworks/activities/sql_activity.rb +43 -0
- data/lib/waterworks/containers/pipeline_definition.rb +66 -0
- data/lib/waterworks/data_formats/csv.rb +20 -0
- data/lib/waterworks/data_formats/custom.rb +21 -0
- data/lib/waterworks/data_formats/dynamo_db_data_format.rb +19 -0
- data/lib/waterworks/data_formats/dynamo_db_export_data_format.rb +19 -0
- data/lib/waterworks/data_formats/reg_ex.rb +21 -0
- data/lib/waterworks/data_formats/tsv.rb +22 -0
- data/lib/waterworks/data_nodes/dynamo_db_data_node.rb +41 -0
- data/lib/waterworks/data_nodes/my_sql_data_node.rb +42 -0
- data/lib/waterworks/data_nodes/redshift_data_node.rb +41 -0
- data/lib/waterworks/data_nodes/s3_data_node.rb +42 -0
- data/lib/waterworks/data_nodes/sql_data_node.rb +42 -0
- data/lib/waterworks/databases/jdbc_database.rb +25 -0
- data/lib/waterworks/databases/rds_database.rb +25 -0
- data/lib/waterworks/databases/redshift_database.rb +25 -0
- data/lib/waterworks/other/default.rb +22 -0
- data/lib/waterworks/pipeline_object.rb +205 -0
- data/lib/waterworks/preconditions/dynamo_db_data_exists.rb +31 -0
- data/lib/waterworks/preconditions/dynamo_db_table_exists.rb +31 -0
- data/lib/waterworks/preconditions/exists.rb +29 -0
- data/lib/waterworks/preconditions/s3_key_exists.rb +31 -0
- data/lib/waterworks/preconditions/s3_prefix_not_empty.rb +31 -0
- data/lib/waterworks/preconditions/shell_command_precondition.rb +34 -0
- data/lib/waterworks/resources/ec2_resource.rb +57 -0
- data/lib/waterworks/resources/emr_cluster.rb +68 -0
- data/lib/waterworks/resources/http_proxy.rb +25 -0
- data/lib/waterworks/schedule/schedule.rb +23 -0
- data/lib/waterworks/util.rb +27 -0
- data/lib/waterworks/utilities/emr_configuration.rb +21 -0
- data/lib/waterworks/utilities/property.rb +20 -0
- data/lib/waterworks/utilities/shell_script_config.rb +20 -0
- data/lib/waterworks.rb +1 -0
- metadata +62 -5
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class DynamoDbDataNode < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
dataFormat: :ref,
|
10
|
+
dependsOn: :ref,
|
11
|
+
failureAndRerunMode: :string,
|
12
|
+
lateAfterTimeout: :string,
|
13
|
+
maxActiveInstances: :string,
|
14
|
+
maximumRetries: :string,
|
15
|
+
onFail: :ref,
|
16
|
+
onLateAction: :ref,
|
17
|
+
onSuccess: :ref,
|
18
|
+
parent: :ref,
|
19
|
+
pipelineLogUri: :string,
|
20
|
+
precondition: :ref,
|
21
|
+
readThroughputPercent: :string,
|
22
|
+
region: :string,
|
23
|
+
reportProgressTimeout: :string,
|
24
|
+
retryDelay: :string,
|
25
|
+
runsOn: :ref,
|
26
|
+
scheduleType: :string,
|
27
|
+
workerGroup: :string,
|
28
|
+
writeThroughputPercent: :string,
|
29
|
+
tableName: :string,
|
30
|
+
schedule: :ref,
|
31
|
+
type: :string,
|
32
|
+
}.merge superclass.safe_fields
|
33
|
+
end
|
34
|
+
|
35
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
36
|
+
|
37
|
+
def initialize(id = nil, name = nil)
|
38
|
+
super(id, name).set_attrs(type: 'DynamoDbDataNode')
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class MySqlDataNode < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
createTableSql: :string,
|
10
|
+
database: :ref,
|
11
|
+
dependsOn: :ref,
|
12
|
+
failureAndRerunMode: :string,
|
13
|
+
insertQuery: :string,
|
14
|
+
lateAfterTimeout: :string,
|
15
|
+
maxActiveInstances: :string,
|
16
|
+
maximumRetries: :string,
|
17
|
+
onFail: :ref,
|
18
|
+
onLateAction: :ref,
|
19
|
+
onSuccess: :ref,
|
20
|
+
parent: :ref,
|
21
|
+
pipelineLogUri: :string,
|
22
|
+
precondition: :ref,
|
23
|
+
reportProgressTimeout: :string,
|
24
|
+
retryDelay: :string,
|
25
|
+
runsOn: :ref,
|
26
|
+
scheduleType: :string,
|
27
|
+
schemaName: :string,
|
28
|
+
selectQuery: :string,
|
29
|
+
workerGroup: :string,
|
30
|
+
table: :string,
|
31
|
+
schedule: :ref,
|
32
|
+
type: :string,
|
33
|
+
}.merge superclass.safe_fields
|
34
|
+
end
|
35
|
+
|
36
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
37
|
+
|
38
|
+
def initialize(id = nil, name = nil)
|
39
|
+
super(id, name).set_attrs(type: 'MySqlDataNode')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class RedshiftDataNode < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
createTableSql: :string,
|
10
|
+
dependsOn: :ref,
|
11
|
+
failureAndRerunMode: :string,
|
12
|
+
lateAfterTimeout: :string,
|
13
|
+
maxActiveInstances: :string,
|
14
|
+
maximumRetries: :string,
|
15
|
+
onFail: :ref,
|
16
|
+
onLateAction: :ref,
|
17
|
+
onSuccess: :ref,
|
18
|
+
parent: :ref,
|
19
|
+
pipelineLogUri: :string,
|
20
|
+
precondition: :ref,
|
21
|
+
primaryKeys: :string,
|
22
|
+
reportProgressTimeout: :string,
|
23
|
+
retryDelay: :string,
|
24
|
+
runsOn: :ref,
|
25
|
+
scheduleType: :string,
|
26
|
+
schemaName: :string,
|
27
|
+
workerGroup: :string,
|
28
|
+
database: :ref,
|
29
|
+
tableName: :string,
|
30
|
+
schedule: :ref,
|
31
|
+
type: :string,
|
32
|
+
}.merge superclass.safe_fields
|
33
|
+
end
|
34
|
+
|
35
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
36
|
+
|
37
|
+
def initialize(id = nil, name = nil)
|
38
|
+
super(id, name).set_attrs(type: 'RedshiftDataNode')
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class S3DataNode < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
compression: :string,
|
10
|
+
dataFormat: :ref,
|
11
|
+
dependsOn: :ref,
|
12
|
+
directoryPath: :string,
|
13
|
+
failureAndRerunMode: :string,
|
14
|
+
filePath: :string,
|
15
|
+
lateAfterTimeout: :string,
|
16
|
+
manifestFilePath: :string,
|
17
|
+
maxActiveInstances: :string,
|
18
|
+
maximumRetries: :string,
|
19
|
+
onFail: :ref,
|
20
|
+
onLateAction: :ref,
|
21
|
+
onSuccess: :ref,
|
22
|
+
parent: :ref,
|
23
|
+
pipelineLogUri: :string,
|
24
|
+
precondition: :ref,
|
25
|
+
reportProgressTimeout: :string,
|
26
|
+
retryDelay: :string,
|
27
|
+
runsOn: :ref,
|
28
|
+
s3EncryptionType: :string,
|
29
|
+
scheduleType: :string,
|
30
|
+
workerGroup: :string,
|
31
|
+
schedule: :ref,
|
32
|
+
type: :string,
|
33
|
+
}.merge superclass.safe_fields
|
34
|
+
end
|
35
|
+
|
36
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
37
|
+
|
38
|
+
def initialize(id = nil, name = nil)
|
39
|
+
super(id, name).set_attrs(type: 'S3DataNode')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class SqlDataNode < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
createTableSql: :string,
|
10
|
+
database: :ref,
|
11
|
+
dependsOn: :ref,
|
12
|
+
failureAndRerunMode: :string,
|
13
|
+
insertQuery: :string,
|
14
|
+
lateAfterTimeout: :string,
|
15
|
+
maxActiveInstances: :string,
|
16
|
+
maximumRetries: :string,
|
17
|
+
onFail: :ref,
|
18
|
+
onLateAction: :ref,
|
19
|
+
onSuccess: :ref,
|
20
|
+
parent: :ref,
|
21
|
+
pipelineLogUri: :string,
|
22
|
+
precondition: :ref,
|
23
|
+
reportProgressTimeout: :string,
|
24
|
+
retryDelay: :string,
|
25
|
+
runsOn: :ref,
|
26
|
+
scheduleType: :string,
|
27
|
+
schemaName: :string,
|
28
|
+
selectQuery: :string,
|
29
|
+
workerGroup: :string,
|
30
|
+
table: :string,
|
31
|
+
schedule: :ref,
|
32
|
+
type: :string,
|
33
|
+
}.merge superclass.safe_fields
|
34
|
+
end
|
35
|
+
|
36
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
37
|
+
|
38
|
+
def initialize(id = nil, name = nil)
|
39
|
+
super(id, name).set_attrs(type: 'SqlDataNode')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class JdbcDatabase < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
databaseName: :string,
|
8
|
+
jdbcDriverJarUri: :string,
|
9
|
+
jdbcProperties: :string,
|
10
|
+
parent: :ref,
|
11
|
+
connectionString: :string,
|
12
|
+
jdbcDriverClass: :string,
|
13
|
+
password: :string,
|
14
|
+
username: :string,
|
15
|
+
type: :string,
|
16
|
+
}.merge superclass.safe_fields
|
17
|
+
end
|
18
|
+
|
19
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
20
|
+
|
21
|
+
def initialize(id = nil, name = nil)
|
22
|
+
super(id, name).set_attrs(type: 'JdbcDatabase')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class RdsDatabase < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
databaseName: :string,
|
8
|
+
jdbcDriverJarUri: :string,
|
9
|
+
jdbcProperties: :string,
|
10
|
+
parent: :ref,
|
11
|
+
region: :string,
|
12
|
+
password: :string,
|
13
|
+
rdsInstanceId: :string,
|
14
|
+
username: :string,
|
15
|
+
type: :string,
|
16
|
+
}.merge superclass.safe_fields
|
17
|
+
end
|
18
|
+
|
19
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
20
|
+
|
21
|
+
def initialize(id = nil, name = nil)
|
22
|
+
super(id, name).set_attrs(type: 'RdsDatabase')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class RedshiftDatabase < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
databaseName: :string,
|
8
|
+
jdbcProperties: :string,
|
9
|
+
parent: :ref,
|
10
|
+
region: :string,
|
11
|
+
password: :string,
|
12
|
+
username: :string,
|
13
|
+
clusterId: :string,
|
14
|
+
connectionString: :string,
|
15
|
+
type: :string,
|
16
|
+
}.merge superclass.safe_fields
|
17
|
+
end
|
18
|
+
|
19
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
20
|
+
|
21
|
+
def initialize(id = nil, name = nil)
|
22
|
+
super(id, name).set_attrs(type: 'RedshiftDatabase')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class Default < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
schedule: :ref,
|
8
|
+
failureAndRerunMode: :string,
|
9
|
+
resourceRole: :string,
|
10
|
+
role: :string,
|
11
|
+
pipelineLogUri: :string,
|
12
|
+
scheduleType: :string,
|
13
|
+
}.merge superclass.safe_fields
|
14
|
+
end
|
15
|
+
|
16
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
17
|
+
|
18
|
+
def initialize(id = 'Default', name = 'Default')
|
19
|
+
super(id, name)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,205 @@
|
|
1
|
+
require_relative 'util.rb'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Waterworks
|
5
|
+
class PipelineObject
|
6
|
+
def self.base_attrs
|
7
|
+
[:id, :name]
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.safe_fields
|
11
|
+
{} # used by subclass
|
12
|
+
end
|
13
|
+
base_attrs.each { |attr| attr_accessor attr }
|
14
|
+
|
15
|
+
def initialize(id = nil, name = nil)
|
16
|
+
@id = id
|
17
|
+
@name = name
|
18
|
+
@fields = []
|
19
|
+
self
|
20
|
+
end
|
21
|
+
|
22
|
+
def set_attrs(attrs)
|
23
|
+
raise 'InvalidAttributeHash: set_attr requires a Hash' unless attrs.is_a? Hash
|
24
|
+
modifiable = self.class.safe_fields.keys.concat self.class.base_attrs
|
25
|
+
attrs.each do |attr, value|
|
26
|
+
if modifiable.include? attr
|
27
|
+
instance_variable_set("@#{attr}", value)
|
28
|
+
else
|
29
|
+
raise "Attempted to set invalid attribute #{attr} on #{self.class}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
self
|
33
|
+
end
|
34
|
+
|
35
|
+
def to_json
|
36
|
+
JSON.generate(to_hash)
|
37
|
+
end
|
38
|
+
|
39
|
+
def to_hash
|
40
|
+
collect_attrs!
|
41
|
+
output = {
|
42
|
+
id: @id,
|
43
|
+
name: @name,
|
44
|
+
}
|
45
|
+
@fields.each { |f| output.merge!(f.to_hash) }
|
46
|
+
output
|
47
|
+
end
|
48
|
+
|
49
|
+
def collect_attrs!
|
50
|
+
@fields.clear
|
51
|
+
self.class.safe_fields.map do |field, type|
|
52
|
+
next unless instance_variable_defined? "@#{field}"
|
53
|
+
val = instance_variable_get "@#{field}"
|
54
|
+
if type == :string
|
55
|
+
add_string_field(field, val)
|
56
|
+
elsif type == :ref
|
57
|
+
add_ref_field(field, val)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def to_fhash
|
63
|
+
collect_attrs!
|
64
|
+
{
|
65
|
+
id: @id,
|
66
|
+
name: @name,
|
67
|
+
fields: @fields.map(&:to_fhash)
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
def sourceify
|
72
|
+
collect_attrs!
|
73
|
+
base = "#{self.class}.new('#{@id}', '#{@name}').set_attrs({\n"
|
74
|
+
@fields.each do |field|
|
75
|
+
base << " #{field.sourceify_as_attr}, \n" unless field.key == :type
|
76
|
+
end
|
77
|
+
base + '})'
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.from_hash(hash, pipe_object = nil)
|
81
|
+
hash = Util.indifferentify(hash)
|
82
|
+
pipe_object = PipelineObject.new if pipe_object.nil?
|
83
|
+
|
84
|
+
hash.each do |key, value|
|
85
|
+
if key == 'id'
|
86
|
+
pipe_object.id = value
|
87
|
+
elsif key == 'name'
|
88
|
+
pipe_object.name = value
|
89
|
+
else
|
90
|
+
key = Util.deasterisk(key)
|
91
|
+
if value.is_a? Hash
|
92
|
+
Util.indifferentify(value)
|
93
|
+
pipe_object.set_attrs(key.to_sym => value[:ref])
|
94
|
+
else
|
95
|
+
pipe_object.set_attrs(key.to_sym => value)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
pipe_object
|
100
|
+
end
|
101
|
+
|
102
|
+
def self.from_json(blob)
|
103
|
+
from_hash(JSON.parse(blob))
|
104
|
+
end
|
105
|
+
|
106
|
+
def <=>(obj)
|
107
|
+
id <=> obj.id
|
108
|
+
end
|
109
|
+
|
110
|
+
def add_string_field(key, value)
|
111
|
+
key = Util.asterisk_if_needed(key)
|
112
|
+
[value].flatten.each do |val|
|
113
|
+
@fields << PipelineObject::Field.new(key, string_value: val)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def add_ref_field(key, value)
|
118
|
+
value = value.id if value.is_a? PipelineObject
|
119
|
+
@fields << PipelineObject::Field.new(key, ref_value: value)
|
120
|
+
self
|
121
|
+
end
|
122
|
+
|
123
|
+
private :add_string_field, :add_ref_field
|
124
|
+
|
125
|
+
class Field
|
126
|
+
attr_accessor :key, :string_value, :ref_value
|
127
|
+
|
128
|
+
def initialize(key, string_value: nil, ref_value: nil)
|
129
|
+
@key = key
|
130
|
+
@string_value = string_value unless string_value.nil?
|
131
|
+
@ref_value = ref_value unless ref_value.nil?
|
132
|
+
end
|
133
|
+
|
134
|
+
def set_string_value(val)
|
135
|
+
@string_value = val
|
136
|
+
self
|
137
|
+
end
|
138
|
+
|
139
|
+
def set_ref_value(val)
|
140
|
+
@ref_value = val
|
141
|
+
self
|
142
|
+
end
|
143
|
+
|
144
|
+
def get_value
|
145
|
+
return @string_value unless @string_value.nil?
|
146
|
+
@ref_value
|
147
|
+
end
|
148
|
+
|
149
|
+
def to_json
|
150
|
+
JSON.generate(to_hash)
|
151
|
+
end
|
152
|
+
|
153
|
+
def to_hash
|
154
|
+
value = @string_value if @string_value
|
155
|
+
value = { ref: @ref_value } if value.nil?
|
156
|
+
hash = {
|
157
|
+
@key => value
|
158
|
+
}
|
159
|
+
hash
|
160
|
+
end
|
161
|
+
|
162
|
+
def to_fhash
|
163
|
+
output = {
|
164
|
+
key: @key
|
165
|
+
}
|
166
|
+
output[:string_value] = @string_value if @string_value
|
167
|
+
output[:ref_value] = @ref_value if @ref_value
|
168
|
+
output
|
169
|
+
end
|
170
|
+
|
171
|
+
def validate
|
172
|
+
end
|
173
|
+
|
174
|
+
def <=>(obj)
|
175
|
+
key <=> obj.key
|
176
|
+
end
|
177
|
+
|
178
|
+
def self.from_hash(hash)
|
179
|
+
hash = Util.indifferentify(hash)
|
180
|
+
field = Field.new
|
181
|
+
field.key = hash[:key]
|
182
|
+
field.string_value = hash[:string_value] if hash[:string_value]
|
183
|
+
field.ref_value = hash[:ref_value] if hash[:ref_value]
|
184
|
+
end
|
185
|
+
|
186
|
+
def sourceify_as_attr
|
187
|
+
if @key == '*password'.to_sym
|
188
|
+
"'#{key}'.to_sym => \"#{get_value}\""
|
189
|
+
else
|
190
|
+
"#{key}: #{get_value.dump}"
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def sourceify
|
195
|
+
key = "PipelineObject::Field.new(\"#{@key}\")"
|
196
|
+
if @string_value.respond_to? :each
|
197
|
+
|
198
|
+
end
|
199
|
+
key << ".set_string_value(\"#{@string_value.gsub('#', '\\#')}\")" if @string_value
|
200
|
+
key << ".set_ref_value(\"#{@ref_value}\")" if @ref_value
|
201
|
+
key
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class DynamoDBDataExists < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
failureAndRerunMode: :string,
|
10
|
+
lateAfterTimeout: :string,
|
11
|
+
maximumRetries: :string,
|
12
|
+
onFail: :ref,
|
13
|
+
onLateAction: :ref,
|
14
|
+
onSuccess: :ref,
|
15
|
+
parent: :ref,
|
16
|
+
preconditionTimeout: :string,
|
17
|
+
reportProgressTimeout: :string,
|
18
|
+
retryDelay: :string,
|
19
|
+
role: :string,
|
20
|
+
tableName: :string,
|
21
|
+
type: :string,
|
22
|
+
}.merge superclass.safe_fields
|
23
|
+
end
|
24
|
+
|
25
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
26
|
+
|
27
|
+
def initialize(id = nil, name = nil)
|
28
|
+
super(id, name).set_attrs(type: 'DynamoDBDataExists')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class DynamoDBTableExists < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
failureAndRerunMode: :string,
|
10
|
+
lateAfterTimeout: :string,
|
11
|
+
maximumRetries: :string,
|
12
|
+
onFail: :ref,
|
13
|
+
onLateAction: :ref,
|
14
|
+
onSuccess: :ref,
|
15
|
+
parent: :ref,
|
16
|
+
preconditionTimeout: :string,
|
17
|
+
reportProgressTimeout: :string,
|
18
|
+
retryDelay: :string,
|
19
|
+
role: :string,
|
20
|
+
tableName: :string,
|
21
|
+
type: :string,
|
22
|
+
}.merge superclass.safe_fields
|
23
|
+
end
|
24
|
+
|
25
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
26
|
+
|
27
|
+
def initialize(id = nil, name = nil)
|
28
|
+
super(id, name).set_attrs(type: 'DynamoDBTableExists')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class Exists < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
failureAndRerunMode: :string,
|
10
|
+
lateAfterTimeout: :string,
|
11
|
+
maximumRetries: :string,
|
12
|
+
onFail: :ref,
|
13
|
+
onLateAction: :ref,
|
14
|
+
onSuccess: :ref,
|
15
|
+
parent: :ref,
|
16
|
+
preconditionTimeout: :string,
|
17
|
+
reportProgressTimeout: :string,
|
18
|
+
retryDelay: :string,
|
19
|
+
type: :string,
|
20
|
+
}.merge superclass.safe_fields
|
21
|
+
end
|
22
|
+
|
23
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
24
|
+
|
25
|
+
def initialize(id = nil, name = nil)
|
26
|
+
super(id, name).set_attrs(type: 'Exists')
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class S3KeyExists < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
failureAndRerunMode: :string,
|
10
|
+
lateAfterTimeout: :string,
|
11
|
+
maximumRetries: :string,
|
12
|
+
onFail: :ref,
|
13
|
+
onLateAction: :ref,
|
14
|
+
onSuccess: :ref,
|
15
|
+
parent: :ref,
|
16
|
+
preconditionTimeout: :string,
|
17
|
+
reportProgressTimeout: :string,
|
18
|
+
retryDelay: :string,
|
19
|
+
role: :string,
|
20
|
+
s3Key: :string,
|
21
|
+
type: :string,
|
22
|
+
}.merge superclass.safe_fields
|
23
|
+
end
|
24
|
+
|
25
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
26
|
+
|
27
|
+
def initialize(id = nil, name = nil)
|
28
|
+
super(id, name).set_attrs(type: 'S3KeyExists')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require_relative '../pipeline_object.rb'
|
2
|
+
|
3
|
+
module Waterworks
|
4
|
+
class S3PrefixNotEmpty < PipelineObject
|
5
|
+
def self.safe_fields
|
6
|
+
{
|
7
|
+
attemptStatus: :string,
|
8
|
+
attemptTimeout: :string,
|
9
|
+
failureAndRerunMode: :string,
|
10
|
+
lateAfterTimeout: :string,
|
11
|
+
maximumRetries: :string,
|
12
|
+
onFail: :ref,
|
13
|
+
onLateAction: :ref,
|
14
|
+
onSuccess: :ref,
|
15
|
+
parent: :ref,
|
16
|
+
preconditionTimeout: :string,
|
17
|
+
reportProgressTimeout: :string,
|
18
|
+
retryDelay: :string,
|
19
|
+
role: :string,
|
20
|
+
s3Prefix: :string,
|
21
|
+
type: :string,
|
22
|
+
}.merge superclass.safe_fields
|
23
|
+
end
|
24
|
+
|
25
|
+
safe_fields.keys.each { |attr| attr_accessor attr }
|
26
|
+
|
27
|
+
def initialize(id = nil, name = nil)
|
28
|
+
super(id, name).set_attrs(type: 'S3PrefixNotEmpty')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|