waterworks 0.0.0 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e94d09fdbabaaa00b82c48ead62ea9422599e7a4
-  data.tar.gz: 5493296d67beae39728ad3c235aa61a9aef3cc1e
+  metadata.gz: 8761403bf118f39291c4eccbf1df1eb2c3044bc6
+  data.tar.gz: 9250556d67c9b6488c303405406f484ed6002c56
 SHA512:
-  metadata.gz: a8ce1d738461aa70a6df7abb6d56fc780515f0073575b001067111f2dbb6908b880916c7a5f1326559be8e5d4c1717dff2da758af0925dd207aeea537690b8c3
-  data.tar.gz: 9b8cc25acc03f70eb3ba48cf1f55aa96a5ea44be72ec39f9a362d061a1f183d5254489414e90ac5e741341e00f931fb47e520305a298c40f38cd4495c0bfd3a1
+  metadata.gz: 2ac32443b3f1f9e60276cbe9ccd0056250965406b9c14989ecf94e64215916d9eaa3f003b74719315e89ee4f5c608287deb6190f5192d2dbe4ee0a239b1fd788
+  data.tar.gz: 4c1913ed2e0001b00ee1d7a7e6fa53a20c0456b79b87b4c994837a1d67c0423d754d4c0cc645cae5a317770a6b1a20f6509110e58b4ef61e3d2d8680b8e7b0d3

data/bin/waterworks

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'waterworks'
+require 'thor'
+
+include Waterworks
+class Runner < Thor
+  desc 'parse <file>',
+       'Parse JSON exported from AWS and print compatible source. STDIN if no file specified'
+  def parse(json_file = nil)
+    jdata = if json_file.nil?
+              JSON.parse(STDIN.read)
+            else
+              JSON.parse(File.read(json_file))
+            end
+    jdata['objects'].each do |jobject|
+      type = jobject['type']
+      if type.nil?
+        print PipelineObject.from_hash(jobject, Default.new).sourceify
+      else
+        clazz = Object.const_get(type).new
+        raise "Unknown Pipeline #{type}" unless clazz.is_a? PipelineObject
+        print PipelineObject.from_hash(jobject, clazz).sourceify
+      end
+      puts
+    end
+  end
+end
+
+Runner.start(ARGV)

data/lib/waterworks/actions/sns_alarm.rb

@@ -0,0 +1,22 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class SnsAlarm < PipelineObject
+    def self.safe_fields
+      {
+        parent: :ref,
+        message: :string,
+        role: :string,
+        subject: :string,
+        topicArn: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'SnsAlarm')
+    end
+  end
+end

data/lib/waterworks/actions/terminate.rb

@@ -0,0 +1,18 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class Terminate < PipelineObject
+    def self.safe_fields
+      {
+        parent: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'Terminate')
+    end
+  end
+end

data/lib/waterworks/activities/copy_activity.rb

@@ -0,0 +1,38 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class CopyActivity < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        schedule: :ref,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'CopyActivity')
+    end
+  end
+end

data/lib/waterworks/activities/emr_activity.rb

@@ -0,0 +1,43 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class EmrActivity < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        postStepCommand: :string,
+        precondition: :ref,
+        preStepCommand: :string,
+        reportProgressTimeout: :string,
+        resizeClusterBeforeRunning: :string,
+        resizeClusterMaxInstances: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        step: :string,
+        schedule: :ref,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'EmrActivity')
+    end
+  end
+end

data/lib/waterworks/activities/hadoop_activity.rb

@@ -0,0 +1,44 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class HadoopActivity < PipelineObject
+    def self.safe_fields
+      {
+        argument: :string,
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        hadoopQueue: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        mainClass: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        postActivityTaskConfig: :ref,
+        preActivityTaskConfig: :ref,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        jarUri: :string,
+        schedule: :ref,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'HadoopActivity')
+    end
+  end
+end

data/lib/waterworks/activities/hive_activity.rb

@@ -0,0 +1,49 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class HiveActivity < PipelineObject
+    # rubocop:disable Metrics/MethodLength
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        hadoopQueue: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        postActivityTaskConfig: :ref,
+        preActivityTaskConfig: :ref,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        resizeClusterBeforeRunning: :string,
+        resizeClusterMaxInstances: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        scriptVariable: :string,
+        stage: :string,
+        schedule: :ref,
+        hiveScript: :string,
+        scriptUri: :string,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'HiveActivity')
+    end
+  end
+end

data/lib/waterworks/activities/hive_copy_activity.rb

@@ -0,0 +1,43 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class HiveCopyActivity < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        filterSql: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        postActivityTaskConfig: :ref,
+        preActivityTaskConfig: :ref,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        resizeClusterBeforeRunning: :string,
+        resizeClusterMaxInstances: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        schedule: :ref,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'HiveCopyActivity')
+    end
+  end
+end

data/lib/waterworks/activities/pig_activity.rb

@@ -0,0 +1,48 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class PigActivity < PipelineObject
+    # rubocop:disable Metrics/MethodLength
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        postActivityTaskConfig: :ref,
+        preActivityTaskConfig: :ref,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        resizeClusterBeforeRunning: :string,
+        resizeClusterMaxInstances: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        scriptVariable: :string,
+        stage: :string,
+        schedule: :ref,
+        script: :string,
+        scriptUri: :string,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'PigActivity')
+    end
+  end
+end

data/lib/waterworks/activities/redshift_copy_activity.rb

@@ -0,0 +1,42 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class RedshiftCopyActivity < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        commandOptions: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        queue: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        transformSql: :string,
+        insertMode: :string,
+        schedule: :ref,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'RedshiftCopyActivity')
+    end
+  end
+end

data/lib/waterworks/activities/shell_command_activity.rb

@@ -0,0 +1,44 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class ShellCommandActivity < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        scriptArgument: :string,
+        stage: :string,
+        stderr: :string,
+        stdout: :string,
+        schedule: :ref,
+        command: :string,
+        scriptUri: :string,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'ShellCommandActivity')
+    end
+  end
+end

data/lib/waterworks/activities/sql_activity.rb

@@ -0,0 +1,43 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class SqlActivity < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        input: :ref,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        output: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        queue: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        scheduleType: :string,
+        scriptArgument: :string,
+        database: :ref,
+        schedule: :ref,
+        script: :string,
+        scriptUri: :string,
+        runsOn: :ref,
+        workerGroup: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'SqlActivity')
+    end
+  end
+end

data/lib/waterworks/containers/pipeline_definition.rb

@@ -0,0 +1,66 @@
+require 'aws-sdk'
+
+module Waterworks
+  class PipelineDefinition
+    attr_accessor :uid, :name, :pipeline_id, :description, :tags, :pipeline_objects
+
+    def initialize(uid, name, client = nil)
+      @uid = uid
+      @name = name
+      client = Aws::DataPipeline::Client.new if client.nil?
+      @client = client
+    end
+
+    def create_pipeline!
+      resp = @client.create_pipeline(
+        name: @name,
+        unique_id: @uid,
+        description: @description,
+        tags: [@tags].flatten.map(&:to_hash)
+      )
+      @pipeline_id = resp.pipeline_id
+      resp
+    end
+
+    def put_pipeline!
+      @client.put_pipeline_definition(
+        pipeline_id: @pipeline_id,
+        pipeline_objects: @pipeline_objects.map(&:to_fhash)
+      )
+    end
+
+    def activate_pipeline!
+      @client.activate_pipeline(
+        pipeline_id: @pipeline_id
+      )
+    end
+
+    def go_baby_go!
+      responses = {}
+      responses[:create] = create_pipeline!
+      responses[:put] = put_pipeline!
+      responses[:activate] = activate_pipeline!
+      responses
+    end
+
+    def validate!
+      @uid && @name && @pipeline_object.any?
+    end
+  end
+
+  class Tag
+    attr_accessor :key, :value
+
+    def initialize(key, value)
+      @key = key
+      @value = value
+    end
+
+    def to_hash
+      {
+        key: @key,
+        value: @value
+      }
+    end
+  end
+end

data/lib/waterworks/data_formats/csv.rb

@@ -0,0 +1,20 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class CSV < PipelineObject
+    def self.safe_fields
+      {
+        column: :string,
+        escapeChar: :string,
+        parent: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'CSV')
+    end
+  end
+end

data/lib/waterworks/data_formats/custom.rb

@@ -0,0 +1,21 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class Custom < PipelineObject
+    def self.safe_fields
+      {
+        column: :string,
+        parent: :ref,
+        recordSeparator: :string,
+        columnSeparator: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'Custom')
+    end
+  end
+end

data/lib/waterworks/data_formats/dynamo_db_data_format.rb

@@ -0,0 +1,19 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class DynamoDBDataFormat < PipelineObject
+    def self.safe_fields
+      {
+        column: :string,
+        parent: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'DynamoDBDataFormat')
+    end
+  end
+end

data/lib/waterworks/data_formats/dynamo_db_export_data_format.rb

@@ -0,0 +1,19 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class DynamoDBExportDataFormat < PipelineObject
+    def self.safe_fields
+      {
+        column: :string,
+        parent: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'DynamoDBExportDataFormat')
+    end
+  end
+end

data/lib/waterworks/data_formats/reg_ex.rb

@@ -0,0 +1,21 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class RegEx < PipelineObject
+    def self.safe_fields
+      {
+        column: :string,
+        inputRegEx: :string,
+        outputFormat: :string,
+        parent: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'RegEx')
+    end
+  end
+end

data/lib/waterworks/data_formats/tsv.rb

@@ -0,0 +1,22 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class TSV < PipelineObject
+    def self.safe_fields
+      {
+        column: :string,
+        columnSeparator: :string,
+        escapeChar: :string,
+        parent: :ref,
+        recordSeparator: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'TSV')
+    end
+  end
+end