waterworks 0.0.0 → 0.1.0

data/lib/waterworks/data_nodes/dynamo_db_data_node.rb

@@ -0,0 +1,41 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class DynamoDbDataNode < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        dataFormat: :ref,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        readThroughputPercent: :string,
+        region: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        runsOn: :ref,
+        scheduleType: :string,
+        workerGroup: :string,
+        writeThroughputPercent: :string,
+        tableName: :string,
+        schedule: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'DynamoDbDataNode')
+    end
+  end
+end

data/lib/waterworks/data_nodes/my_sql_data_node.rb

@@ -0,0 +1,42 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class MySqlDataNode < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        createTableSql: :string,
+        database: :ref,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        insertQuery: :string,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        runsOn: :ref,
+        scheduleType: :string,
+        schemaName: :string,
+        selectQuery: :string,
+        workerGroup: :string,
+        table: :string,
+        schedule: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'MySqlDataNode')
+    end
+  end
+end

data/lib/waterworks/data_nodes/redshift_data_node.rb

@@ -0,0 +1,41 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class RedshiftDataNode < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        createTableSql: :string,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        primaryKeys: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        runsOn: :ref,
+        scheduleType: :string,
+        schemaName: :string,
+        workerGroup: :string,
+        database: :ref,
+        tableName: :string,
+        schedule: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'RedshiftDataNode')
+    end
+  end
+end

data/lib/waterworks/data_nodes/s3_data_node.rb

@@ -0,0 +1,42 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class S3DataNode < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        compression: :string,
+        dataFormat: :ref,
+        dependsOn: :ref,
+        directoryPath: :string,
+        failureAndRerunMode: :string,
+        filePath: :string,
+        lateAfterTimeout: :string,
+        manifestFilePath: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        runsOn: :ref,
+        s3EncryptionType: :string,
+        scheduleType: :string,
+        workerGroup: :string,
+        schedule: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'S3DataNode')
+    end
+  end
+end

data/lib/waterworks/data_nodes/sql_data_node.rb

@@ -0,0 +1,42 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class SqlDataNode < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        createTableSql: :string,
+        database: :ref,
+        dependsOn: :ref,
+        failureAndRerunMode: :string,
+        insertQuery: :string,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        precondition: :ref,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        runsOn: :ref,
+        scheduleType: :string,
+        schemaName: :string,
+        selectQuery: :string,
+        workerGroup: :string,
+        table: :string,
+        schedule: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'SqlDataNode')
+    end
+  end
+end

data/lib/waterworks/databases/jdbc_database.rb

@@ -0,0 +1,25 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class JdbcDatabase < PipelineObject
+    def self.safe_fields
+      {
+        databaseName: :string,
+        jdbcDriverJarUri: :string,
+        jdbcProperties: :string,
+        parent: :ref,
+        connectionString: :string,
+        jdbcDriverClass: :string,
+        password: :string,
+        username: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'JdbcDatabase')
+    end
+  end
+end

data/lib/waterworks/databases/rds_database.rb

@@ -0,0 +1,25 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class RdsDatabase < PipelineObject
+    def self.safe_fields
+      {
+        databaseName: :string,
+        jdbcDriverJarUri: :string,
+        jdbcProperties: :string,
+        parent: :ref,
+        region: :string,
+        password: :string,
+        rdsInstanceId: :string,
+        username: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'RdsDatabase')
+    end
+  end
+end

data/lib/waterworks/databases/redshift_database.rb

@@ -0,0 +1,25 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class RedshiftDatabase < PipelineObject
+    def self.safe_fields
+      {
+        databaseName: :string,
+        jdbcProperties: :string,
+        parent: :ref,
+        region: :string,
+        password: :string,
+        username: :string,
+        clusterId: :string,
+        connectionString: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'RedshiftDatabase')
+    end
+  end
+end

data/lib/waterworks/other/default.rb

@@ -0,0 +1,22 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class Default < PipelineObject
+    def self.safe_fields
+      {
+        schedule: :ref,
+        failureAndRerunMode: :string,
+        resourceRole: :string,
+        role: :string,
+        pipelineLogUri: :string,
+        scheduleType: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = 'Default', name = 'Default')
+      super(id, name)
+    end
+  end
+end

data/lib/waterworks/pipeline_object.rb

@@ -0,0 +1,205 @@
+require_relative 'util.rb'
+require 'json'
+
+module Waterworks
+  class PipelineObject
+    def self.base_attrs
+      [:id, :name]
+    end
+
+    def self.safe_fields
+      {} # used by subclass
+    end
+    base_attrs.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      @id = id
+      @name = name
+      @fields = []
+      self
+    end
+
+    def set_attrs(attrs)
+      raise 'InvalidAttributeHash: set_attr requires a Hash' unless attrs.is_a? Hash
+      modifiable = self.class.safe_fields.keys.concat self.class.base_attrs
+      attrs.each do |attr, value|
+        if modifiable.include? attr
+          instance_variable_set("@#{attr}", value)
+        else
+          raise "Attempted to set invalid attribute #{attr} on #{self.class}"
+        end
+      end
+      self
+    end
+
+    def to_json
+      JSON.generate(to_hash)
+    end
+
+    def to_hash
+      collect_attrs!
+      output = {
+        id: @id,
+        name: @name,
+      }
+      @fields.each { |f| output.merge!(f.to_hash) }
+      output
+    end
+
+    def collect_attrs!
+      @fields.clear
+      self.class.safe_fields.map do |field, type|
+        next unless instance_variable_defined? "@#{field}"
+        val = instance_variable_get "@#{field}"
+        if type == :string
+          add_string_field(field, val)
+        elsif type == :ref
+          add_ref_field(field, val)
+        end
+      end
+    end
+
+    def to_fhash
+      collect_attrs!
+      {
+        id: @id,
+        name: @name,
+        fields: @fields.map(&:to_fhash)
+      }
+    end
+
+    def sourceify
+      collect_attrs!
+      base = "#{self.class}.new('#{@id}', '#{@name}').set_attrs({\n"
+      @fields.each do |field|
+        base << "  #{field.sourceify_as_attr}, \n" unless field.key == :type
+      end
+      base + '})'
+    end
+
+    def self.from_hash(hash, pipe_object = nil)
+      hash = Util.indifferentify(hash)
+      pipe_object = PipelineObject.new if pipe_object.nil?
+
+      hash.each do |key, value|
+        if key == 'id'
+          pipe_object.id = value
+        elsif key == 'name'
+          pipe_object.name = value
+        else
+          key = Util.deasterisk(key)
+          if value.is_a? Hash
+            Util.indifferentify(value)
+            pipe_object.set_attrs(key.to_sym => value[:ref])
+          else
+            pipe_object.set_attrs(key.to_sym => value)
+          end
+        end
+      end
+      pipe_object
+    end
+
+    def self.from_json(blob)
+      from_hash(JSON.parse(blob))
+    end
+
+    def <=>(obj)
+      id <=> obj.id
+    end
+
+    def add_string_field(key, value)
+      key = Util.asterisk_if_needed(key)
+      [value].flatten.each do |val|
+        @fields << PipelineObject::Field.new(key, string_value: val)
+      end
+    end
+
+    def add_ref_field(key, value)
+      value = value.id if value.is_a? PipelineObject
+      @fields << PipelineObject::Field.new(key, ref_value: value)
+      self
+    end
+
+    private :add_string_field, :add_ref_field
+
+    class Field
+      attr_accessor :key, :string_value, :ref_value
+
+      def initialize(key, string_value: nil, ref_value: nil)
+        @key = key
+        @string_value = string_value unless string_value.nil?
+        @ref_value = ref_value unless ref_value.nil?
+      end
+
+      def set_string_value(val)
+        @string_value = val
+        self
+      end
+
+      def set_ref_value(val)
+        @ref_value = val
+        self
+      end
+
+      def get_value
+        return @string_value unless @string_value.nil?
+        @ref_value
+      end
+
+      def to_json
+        JSON.generate(to_hash)
+      end
+
+      def to_hash
+        value = @string_value if @string_value
+        value = { ref: @ref_value } if value.nil?
+        hash = {
+          @key => value
+        }
+        hash
+      end
+
+      def to_fhash
+        output = {
+          key: @key
+        }
+        output[:string_value] = @string_value if @string_value
+        output[:ref_value] = @ref_value if @ref_value
+        output
+      end
+
+      def validate
+      end
+
+      def <=>(obj)
+        key <=> obj.key
+      end
+
+      def self.from_hash(hash)
+        hash = Util.indifferentify(hash)
+        field = Field.new
+        field.key = hash[:key]
+        field.string_value = hash[:string_value] if hash[:string_value]
+        field.ref_value = hash[:ref_value] if hash[:ref_value]
+      end
+
+      def sourceify_as_attr
+        if @key == '*password'.to_sym
+          "'#{key}'.to_sym => \"#{get_value}\""
+        else
+          "#{key}: #{get_value.dump}"
+        end
+      end
+
+      def sourceify
+        key = "PipelineObject::Field.new(\"#{@key}\")"
+        if @string_value.respond_to? :each
+
+        end
+        key << ".set_string_value(\"#{@string_value.gsub('#', '\\#')}\")" if @string_value
+        key << ".set_ref_value(\"#{@ref_value}\")" if @ref_value
+        key
+      end
+    end
+  end
+end

data/lib/waterworks/preconditions/dynamo_db_data_exists.rb

@@ -0,0 +1,31 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class DynamoDBDataExists < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        preconditionTimeout: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        role: :string,
+        tableName: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'DynamoDBDataExists')
+    end
+  end
+end

data/lib/waterworks/preconditions/dynamo_db_table_exists.rb

@@ -0,0 +1,31 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class DynamoDBTableExists < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        preconditionTimeout: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        role: :string,
+        tableName: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'DynamoDBTableExists')
+    end
+  end
+end

data/lib/waterworks/preconditions/exists.rb

@@ -0,0 +1,29 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class Exists < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        preconditionTimeout: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'Exists')
+    end
+  end
+end

data/lib/waterworks/preconditions/s3_key_exists.rb

@@ -0,0 +1,31 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class S3KeyExists < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        preconditionTimeout: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        role: :string,
+        s3Key: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'S3KeyExists')
+    end
+  end
+end

data/lib/waterworks/preconditions/s3_prefix_not_empty.rb

@@ -0,0 +1,31 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class S3PrefixNotEmpty < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        preconditionTimeout: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        role: :string,
+        s3Prefix: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'S3PrefixNotEmpty')
+    end
+  end
+end