waterworks 0.0.0 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/bin/waterworks +30 -0
  3. data/lib/waterworks/actions/sns_alarm.rb +22 -0
  4. data/lib/waterworks/actions/terminate.rb +18 -0
  5. data/lib/waterworks/activities/copy_activity.rb +38 -0
  6. data/lib/waterworks/activities/emr_activity.rb +43 -0
  7. data/lib/waterworks/activities/hadoop_activity.rb +44 -0
  8. data/lib/waterworks/activities/hive_activity.rb +49 -0
  9. data/lib/waterworks/activities/hive_copy_activity.rb +43 -0
  10. data/lib/waterworks/activities/pig_activity.rb +48 -0
  11. data/lib/waterworks/activities/redshift_copy_activity.rb +42 -0
  12. data/lib/waterworks/activities/shell_command_activity.rb +44 -0
  13. data/lib/waterworks/activities/sql_activity.rb +43 -0
  14. data/lib/waterworks/containers/pipeline_definition.rb +66 -0
  15. data/lib/waterworks/data_formats/csv.rb +20 -0
  16. data/lib/waterworks/data_formats/custom.rb +21 -0
  17. data/lib/waterworks/data_formats/dynamo_db_data_format.rb +19 -0
  18. data/lib/waterworks/data_formats/dynamo_db_export_data_format.rb +19 -0
  19. data/lib/waterworks/data_formats/reg_ex.rb +21 -0
  20. data/lib/waterworks/data_formats/tsv.rb +22 -0
  21. data/lib/waterworks/data_nodes/dynamo_db_data_node.rb +41 -0
  22. data/lib/waterworks/data_nodes/my_sql_data_node.rb +42 -0
  23. data/lib/waterworks/data_nodes/redshift_data_node.rb +41 -0
  24. data/lib/waterworks/data_nodes/s3_data_node.rb +42 -0
  25. data/lib/waterworks/data_nodes/sql_data_node.rb +42 -0
  26. data/lib/waterworks/databases/jdbc_database.rb +25 -0
  27. data/lib/waterworks/databases/rds_database.rb +25 -0
  28. data/lib/waterworks/databases/redshift_database.rb +25 -0
  29. data/lib/waterworks/other/default.rb +22 -0
  30. data/lib/waterworks/pipeline_object.rb +205 -0
  31. data/lib/waterworks/preconditions/dynamo_db_data_exists.rb +31 -0
  32. data/lib/waterworks/preconditions/dynamo_db_table_exists.rb +31 -0
  33. data/lib/waterworks/preconditions/exists.rb +29 -0
  34. data/lib/waterworks/preconditions/s3_key_exists.rb +31 -0
  35. data/lib/waterworks/preconditions/s3_prefix_not_empty.rb +31 -0
  36. data/lib/waterworks/preconditions/shell_command_precondition.rb +34 -0
  37. data/lib/waterworks/resources/ec2_resource.rb +57 -0
  38. data/lib/waterworks/resources/emr_cluster.rb +68 -0
  39. data/lib/waterworks/resources/http_proxy.rb +25 -0
  40. data/lib/waterworks/schedule/schedule.rb +23 -0
  41. data/lib/waterworks/util.rb +27 -0
  42. data/lib/waterworks/utilities/emr_configuration.rb +21 -0
  43. data/lib/waterworks/utilities/property.rb +20 -0
  44. data/lib/waterworks/utilities/shell_script_config.rb +20 -0
  45. data/lib/waterworks.rb +1 -0
  46. metadata +62 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e94d09fdbabaaa00b82c48ead62ea9422599e7a4
4
- data.tar.gz: 5493296d67beae39728ad3c235aa61a9aef3cc1e
3
+ metadata.gz: 8761403bf118f39291c4eccbf1df1eb2c3044bc6
4
+ data.tar.gz: 9250556d67c9b6488c303405406f484ed6002c56
5
5
  SHA512:
6
- metadata.gz: a8ce1d738461aa70a6df7abb6d56fc780515f0073575b001067111f2dbb6908b880916c7a5f1326559be8e5d4c1717dff2da758af0925dd207aeea537690b8c3
7
- data.tar.gz: 9b8cc25acc03f70eb3ba48cf1f55aa96a5ea44be72ec39f9a362d061a1f183d5254489414e90ac5e741341e00f931fb47e520305a298c40f38cd4495c0bfd3a1
6
+ metadata.gz: 2ac32443b3f1f9e60276cbe9ccd0056250965406b9c14989ecf94e64215916d9eaa3f003b74719315e89ee4f5c608287deb6190f5192d2dbe4ee0a239b1fd788
7
+ data.tar.gz: 4c1913ed2e0001b00ee1d7a7e6fa53a20c0456b79b87b4c994837a1d67c0423d754d4c0cc645cae5a317770a6b1a20f6509110e58b4ef61e3d2d8680b8e7b0d3
data/bin/waterworks CHANGED
@@ -0,0 +1,30 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'waterworks'
4
+ require 'thor'
5
+
6
+ include Waterworks
7
+ class Runner < Thor
8
+ desc 'parse <file>',
9
+ 'Parse JSON exported from AWS and print compatible source. STDIN if no file specified'
10
+ def parse(json_file = nil)
11
+ jdata = if json_file.nil?
12
+ JSON.parse(STDIN.read)
13
+ else
14
+ JSON.parse(File.read(json_file))
15
+ end
16
+ jdata['objects'].each do |jobject|
17
+ type = jobject['type']
18
+ if type.nil?
19
+ print PipelineObject.from_hash(jobject, Default.new).sourceify
20
+ else
21
+ clazz = Object.const_get(type).new
22
+ raise "Unknown Pipeline #{type}" unless clazz.is_a? PipelineObject
23
+ print PipelineObject.from_hash(jobject, clazz).sourceify
24
+ end
25
+ puts
26
+ end
27
+ end
28
+ end
29
+
30
+ Runner.start(ARGV)
@@ -0,0 +1,22 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class SnsAlarm < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ parent: :ref,
8
+ message: :string,
9
+ role: :string,
10
+ subject: :string,
11
+ topicArn: :string,
12
+ type: :string,
13
+ }.merge superclass.safe_fields
14
+ end
15
+
16
+ safe_fields.keys.each { |attr| attr_accessor attr }
17
+
18
+ def initialize(id = nil, name = nil)
19
+ super(id, name).set_attrs(type: 'SnsAlarm')
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,18 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class Terminate < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ parent: :ref,
8
+ type: :string,
9
+ }.merge superclass.safe_fields
10
+ end
11
+
12
+ safe_fields.keys.each { |attr| attr_accessor attr }
13
+
14
+ def initialize(id = nil, name = nil)
15
+ super(id, name).set_attrs(type: 'Terminate')
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,38 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class CopyActivity < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ attemptStatus: :string,
8
+ attemptTimeout: :string,
9
+ dependsOn: :ref,
10
+ failureAndRerunMode: :string,
11
+ input: :ref,
12
+ lateAfterTimeout: :string,
13
+ maxActiveInstances: :string,
14
+ maximumRetries: :string,
15
+ onFail: :ref,
16
+ onLateAction: :ref,
17
+ onSuccess: :ref,
18
+ output: :ref,
19
+ parent: :ref,
20
+ pipelineLogUri: :string,
21
+ precondition: :ref,
22
+ reportProgressTimeout: :string,
23
+ retryDelay: :string,
24
+ scheduleType: :string,
25
+ schedule: :ref,
26
+ runsOn: :ref,
27
+ workerGroup: :string,
28
+ type: :string,
29
+ }.merge superclass.safe_fields
30
+ end
31
+
32
+ safe_fields.keys.each { |attr| attr_accessor attr }
33
+
34
+ def initialize(id = nil, name = nil)
35
+ super(id, name).set_attrs(type: 'CopyActivity')
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,43 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class EmrActivity < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ attemptStatus: :string,
8
+ attemptTimeout: :string,
9
+ dependsOn: :ref,
10
+ failureAndRerunMode: :string,
11
+ input: :ref,
12
+ lateAfterTimeout: :string,
13
+ maxActiveInstances: :string,
14
+ maximumRetries: :string,
15
+ onFail: :ref,
16
+ onLateAction: :ref,
17
+ onSuccess: :ref,
18
+ output: :ref,
19
+ parent: :ref,
20
+ pipelineLogUri: :string,
21
+ postStepCommand: :string,
22
+ precondition: :ref,
23
+ preStepCommand: :string,
24
+ reportProgressTimeout: :string,
25
+ resizeClusterBeforeRunning: :string,
26
+ resizeClusterMaxInstances: :string,
27
+ retryDelay: :string,
28
+ scheduleType: :string,
29
+ step: :string,
30
+ schedule: :ref,
31
+ runsOn: :ref,
32
+ workerGroup: :string,
33
+ type: :string,
34
+ }.merge superclass.safe_fields
35
+ end
36
+
37
+ safe_fields.keys.each { |attr| attr_accessor attr }
38
+
39
+ def initialize(id = nil, name = nil)
40
+ super(id, name).set_attrs(type: 'EmrActivity')
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,44 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class HadoopActivity < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ argument: :string,
8
+ attemptStatus: :string,
9
+ attemptTimeout: :string,
10
+ dependsOn: :ref,
11
+ failureAndRerunMode: :string,
12
+ hadoopQueue: :string,
13
+ input: :ref,
14
+ lateAfterTimeout: :string,
15
+ mainClass: :string,
16
+ maxActiveInstances: :string,
17
+ maximumRetries: :string,
18
+ onFail: :ref,
19
+ onLateAction: :ref,
20
+ onSuccess: :ref,
21
+ output: :ref,
22
+ parent: :ref,
23
+ pipelineLogUri: :string,
24
+ postActivityTaskConfig: :ref,
25
+ preActivityTaskConfig: :ref,
26
+ precondition: :ref,
27
+ reportProgressTimeout: :string,
28
+ retryDelay: :string,
29
+ scheduleType: :string,
30
+ jarUri: :string,
31
+ schedule: :ref,
32
+ runsOn: :ref,
33
+ workerGroup: :string,
34
+ type: :string,
35
+ }.merge superclass.safe_fields
36
+ end
37
+
38
+ safe_fields.keys.each { |attr| attr_accessor attr }
39
+
40
+ def initialize(id = nil, name = nil)
41
+ super(id, name).set_attrs(type: 'HadoopActivity')
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,49 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class HiveActivity < PipelineObject
5
+ # rubocop:disable Metrics/MethodLength
6
+ def self.safe_fields
7
+ {
8
+ attemptStatus: :string,
9
+ attemptTimeout: :string,
10
+ dependsOn: :ref,
11
+ failureAndRerunMode: :string,
12
+ hadoopQueue: :string,
13
+ input: :ref,
14
+ lateAfterTimeout: :string,
15
+ maxActiveInstances: :string,
16
+ maximumRetries: :string,
17
+ onFail: :ref,
18
+ onLateAction: :ref,
19
+ onSuccess: :ref,
20
+ output: :ref,
21
+ parent: :ref,
22
+ pipelineLogUri: :string,
23
+ postActivityTaskConfig: :ref,
24
+ preActivityTaskConfig: :ref,
25
+ precondition: :ref,
26
+ reportProgressTimeout: :string,
27
+ resizeClusterBeforeRunning: :string,
28
+ resizeClusterMaxInstances: :string,
29
+ retryDelay: :string,
30
+ scheduleType: :string,
31
+ scriptVariable: :string,
32
+ stage: :string,
33
+ schedule: :ref,
34
+ hiveScript: :string,
35
+ scriptUri: :string,
36
+ runsOn: :ref,
37
+ workerGroup: :string,
38
+ type: :string,
39
+ }.merge superclass.safe_fields
40
+ end
41
+ # rubocop:enable Metrics/MethodLength
42
+
43
+ safe_fields.keys.each { |attr| attr_accessor attr }
44
+
45
+ def initialize(id = nil, name = nil)
46
+ super(id, name).set_attrs(type: 'HiveActivity')
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,43 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class HiveCopyActivity < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ attemptStatus: :string,
8
+ attemptTimeout: :string,
9
+ dependsOn: :ref,
10
+ failureAndRerunMode: :string,
11
+ filterSql: :string,
12
+ input: :ref,
13
+ lateAfterTimeout: :string,
14
+ maxActiveInstances: :string,
15
+ maximumRetries: :string,
16
+ onFail: :ref,
17
+ onLateAction: :ref,
18
+ onSuccess: :ref,
19
+ output: :ref,
20
+ parent: :ref,
21
+ pipelineLogUri: :string,
22
+ postActivityTaskConfig: :ref,
23
+ preActivityTaskConfig: :ref,
24
+ precondition: :ref,
25
+ reportProgressTimeout: :string,
26
+ resizeClusterBeforeRunning: :string,
27
+ resizeClusterMaxInstances: :string,
28
+ retryDelay: :string,
29
+ scheduleType: :string,
30
+ schedule: :ref,
31
+ runsOn: :ref,
32
+ workerGroup: :string,
33
+ type: :string,
34
+ }.merge superclass.safe_fields
35
+ end
36
+
37
+ safe_fields.keys.each { |attr| attr_accessor attr }
38
+
39
+ def initialize(id = nil, name = nil)
40
+ super(id, name).set_attrs(type: 'HiveCopyActivity')
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,48 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class PigActivity < PipelineObject
5
+ # rubocop:disable Metrics/MethodLength
6
+ def self.safe_fields
7
+ {
8
+ attemptStatus: :string,
9
+ attemptTimeout: :string,
10
+ dependsOn: :ref,
11
+ failureAndRerunMode: :string,
12
+ input: :ref,
13
+ lateAfterTimeout: :string,
14
+ maxActiveInstances: :string,
15
+ maximumRetries: :string,
16
+ onFail: :ref,
17
+ onLateAction: :ref,
18
+ onSuccess: :ref,
19
+ output: :ref,
20
+ parent: :ref,
21
+ pipelineLogUri: :string,
22
+ postActivityTaskConfig: :ref,
23
+ preActivityTaskConfig: :ref,
24
+ precondition: :ref,
25
+ reportProgressTimeout: :string,
26
+ resizeClusterBeforeRunning: :string,
27
+ resizeClusterMaxInstances: :string,
28
+ retryDelay: :string,
29
+ scheduleType: :string,
30
+ scriptVariable: :string,
31
+ stage: :string,
32
+ schedule: :ref,
33
+ script: :string,
34
+ scriptUri: :string,
35
+ runsOn: :ref,
36
+ workerGroup: :string,
37
+ type: :string,
38
+ }.merge superclass.safe_fields
39
+ end
40
+ # rubocop:enable Metrics/MethodLength
41
+
42
+ safe_fields.keys.each { |attr| attr_accessor attr }
43
+
44
+ def initialize(id = nil, name = nil)
45
+ super(id, name).set_attrs(type: 'PigActivity')
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,42 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class RedshiftCopyActivity < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ attemptStatus: :string,
8
+ attemptTimeout: :string,
9
+ commandOptions: :string,
10
+ dependsOn: :ref,
11
+ failureAndRerunMode: :string,
12
+ input: :ref,
13
+ lateAfterTimeout: :string,
14
+ maxActiveInstances: :string,
15
+ maximumRetries: :string,
16
+ onFail: :ref,
17
+ onLateAction: :ref,
18
+ onSuccess: :ref,
19
+ output: :ref,
20
+ parent: :ref,
21
+ pipelineLogUri: :string,
22
+ precondition: :ref,
23
+ queue: :string,
24
+ reportProgressTimeout: :string,
25
+ retryDelay: :string,
26
+ scheduleType: :string,
27
+ transformSql: :string,
28
+ insertMode: :string,
29
+ schedule: :ref,
30
+ runsOn: :ref,
31
+ workerGroup: :string,
32
+ type: :string,
33
+ }.merge superclass.safe_fields
34
+ end
35
+
36
+ safe_fields.keys.each { |attr| attr_accessor attr }
37
+
38
+ def initialize(id = nil, name = nil)
39
+ super(id, name).set_attrs(type: 'RedshiftCopyActivity')
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,44 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class ShellCommandActivity < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ attemptStatus: :string,
8
+ attemptTimeout: :string,
9
+ dependsOn: :ref,
10
+ failureAndRerunMode: :string,
11
+ input: :ref,
12
+ lateAfterTimeout: :string,
13
+ maxActiveInstances: :string,
14
+ maximumRetries: :string,
15
+ onFail: :ref,
16
+ onLateAction: :ref,
17
+ onSuccess: :ref,
18
+ output: :ref,
19
+ parent: :ref,
20
+ pipelineLogUri: :string,
21
+ precondition: :ref,
22
+ reportProgressTimeout: :string,
23
+ retryDelay: :string,
24
+ scheduleType: :string,
25
+ scriptArgument: :string,
26
+ stage: :string,
27
+ stderr: :string,
28
+ stdout: :string,
29
+ schedule: :ref,
30
+ command: :string,
31
+ scriptUri: :string,
32
+ runsOn: :ref,
33
+ workerGroup: :string,
34
+ type: :string,
35
+ }.merge superclass.safe_fields
36
+ end
37
+
38
+ safe_fields.keys.each { |attr| attr_accessor attr }
39
+
40
+ def initialize(id = nil, name = nil)
41
+ super(id, name).set_attrs(type: 'ShellCommandActivity')
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,43 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class SqlActivity < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ attemptStatus: :string,
8
+ attemptTimeout: :string,
9
+ dependsOn: :ref,
10
+ failureAndRerunMode: :string,
11
+ input: :ref,
12
+ lateAfterTimeout: :string,
13
+ maxActiveInstances: :string,
14
+ maximumRetries: :string,
15
+ onFail: :ref,
16
+ onLateAction: :ref,
17
+ onSuccess: :ref,
18
+ output: :ref,
19
+ parent: :ref,
20
+ pipelineLogUri: :string,
21
+ precondition: :ref,
22
+ queue: :string,
23
+ reportProgressTimeout: :string,
24
+ retryDelay: :string,
25
+ scheduleType: :string,
26
+ scriptArgument: :string,
27
+ database: :ref,
28
+ schedule: :ref,
29
+ script: :string,
30
+ scriptUri: :string,
31
+ runsOn: :ref,
32
+ workerGroup: :string,
33
+ type: :string,
34
+ }.merge superclass.safe_fields
35
+ end
36
+
37
+ safe_fields.keys.each { |attr| attr_accessor attr }
38
+
39
+ def initialize(id = nil, name = nil)
40
+ super(id, name).set_attrs(type: 'SqlActivity')
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,66 @@
1
+ require 'aws-sdk'
2
+
3
+ module Waterworks
4
+ class PipelineDefinition
5
+ attr_accessor :uid, :name, :pipeline_id, :description, :tags, :pipeline_objects
6
+
7
+ def initialize(uid, name, client = nil)
8
+ @uid = uid
9
+ @name = name
10
+ client = Aws::DataPipeline::Client.new if client.nil?
11
+ @client = client
12
+ end
13
+
14
+ def create_pipeline!
15
+ resp = @client.create_pipeline(
16
+ name: @name,
17
+ unique_id: @uid,
18
+ description: @description,
19
+ tags: [@tags].flatten.map(&:to_hash)
20
+ )
21
+ @pipeline_id = resp.pipeline_id
22
+ resp
23
+ end
24
+
25
+ def put_pipeline!
26
+ @client.put_pipeline_definition(
27
+ pipeline_id: @pipeline_id,
28
+ pipeline_objects: @pipeline_objects.map(&:to_fhash)
29
+ )
30
+ end
31
+
32
+ def activate_pipeline!
33
+ @client.activate_pipeline(
34
+ pipeline_id: @pipeline_id
35
+ )
36
+ end
37
+
38
+ def go_baby_go!
39
+ responses = {}
40
+ responses[:create] = create_pipeline!
41
+ responses[:put] = put_pipeline!
42
+ responses[:activate] = activate_pipeline!
43
+ responses
44
+ end
45
+
46
+ def validate!
47
+ @uid && @name && @pipeline_object.any?
48
+ end
49
+ end
50
+
51
+ class Tag
52
+ attr_accessor :key, :value
53
+
54
+ def initialize(key, value)
55
+ @key = key
56
+ @value = value
57
+ end
58
+
59
+ def to_hash
60
+ {
61
+ key: @key,
62
+ value: @value
63
+ }
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,20 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class CSV < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ column: :string,
8
+ escapeChar: :string,
9
+ parent: :ref,
10
+ type: :string,
11
+ }.merge superclass.safe_fields
12
+ end
13
+
14
+ safe_fields.keys.each { |attr| attr_accessor attr }
15
+
16
+ def initialize(id = nil, name = nil)
17
+ super(id, name).set_attrs(type: 'CSV')
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,21 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class Custom < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ column: :string,
8
+ parent: :ref,
9
+ recordSeparator: :string,
10
+ columnSeparator: :string,
11
+ type: :string,
12
+ }.merge superclass.safe_fields
13
+ end
14
+
15
+ safe_fields.keys.each { |attr| attr_accessor attr }
16
+
17
+ def initialize(id = nil, name = nil)
18
+ super(id, name).set_attrs(type: 'Custom')
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,19 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class DynamoDBDataFormat < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ column: :string,
8
+ parent: :ref,
9
+ type: :string,
10
+ }.merge superclass.safe_fields
11
+ end
12
+
13
+ safe_fields.keys.each { |attr| attr_accessor attr }
14
+
15
+ def initialize(id = nil, name = nil)
16
+ super(id, name).set_attrs(type: 'DynamoDBDataFormat')
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,19 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class DynamoDBExportDataFormat < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ column: :string,
8
+ parent: :ref,
9
+ type: :string,
10
+ }.merge superclass.safe_fields
11
+ end
12
+
13
+ safe_fields.keys.each { |attr| attr_accessor attr }
14
+
15
+ def initialize(id = nil, name = nil)
16
+ super(id, name).set_attrs(type: 'DynamoDBExportDataFormat')
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,21 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class RegEx < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ column: :string,
8
+ inputRegEx: :string,
9
+ outputFormat: :string,
10
+ parent: :ref,
11
+ type: :string,
12
+ }.merge superclass.safe_fields
13
+ end
14
+
15
+ safe_fields.keys.each { |attr| attr_accessor attr }
16
+
17
+ def initialize(id = nil, name = nil)
18
+ super(id, name).set_attrs(type: 'RegEx')
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,22 @@
1
+ require_relative '../pipeline_object.rb'
2
+
3
+ module Waterworks
4
+ class TSV < PipelineObject
5
+ def self.safe_fields
6
+ {
7
+ column: :string,
8
+ columnSeparator: :string,
9
+ escapeChar: :string,
10
+ parent: :ref,
11
+ recordSeparator: :string,
12
+ type: :string,
13
+ }.merge superclass.safe_fields
14
+ end
15
+
16
+ safe_fields.keys.each { |attr| attr_accessor attr }
17
+
18
+ def initialize(id = nil, name = nil)
19
+ super(id, name).set_attrs(type: 'TSV')
20
+ end
21
+ end
22
+ end