waterworks 0.0.0 → 0.1.0

data/lib/waterworks/preconditions/shell_command_precondition.rb

@@ -0,0 +1,34 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class ShellCommandPrecondition < PipelineObject
+    def self.safe_fields
+      {
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        failureAndRerunMode: :string,
+        lateAfterTimeout: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        preconditionTimeout: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        scriptArgument: :string,
+        stderr: :string,
+        stdout: :string,
+        command: :string,
+        scriptUri: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'ShellCommandPrecondition')
+    end
+  end
+end

data/lib/waterworks/resources/ec2_resource.rb

@@ -0,0 +1,57 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class Ec2Resource < PipelineObject
+    # rubocop:disable Metrics/MethodLength
+    def self.safe_fields
+      {
+        actionOnResourceFailure: :string,
+        actionOnTaskFailure: :string,
+        associatePublicIpAddress: :string,
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        availabilityZone: :string,
+        failureAndRerunMode: :string,
+        httpProxy: :ref,
+        imageId: :string,
+        initTimeout: :string,
+        instanceCount: :string,
+        instanceType: :string,
+        keyPair: :string,
+        lateAfterTimeout: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        minInstanceCount: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        region: :string,
+        reportProgressTimeout: :string,
+        retryDelay: :string,
+        runAsUser: :string,
+        runsOn: :ref,
+        scheduleType: :string,
+        securityGroupIds: :string,
+        securityGroups: :string,
+        spotBidPrice: :string,
+        subnetId: :string,
+        terminateAfter: :string,
+        useOnDemandOnLastAttempt: :string,
+        workerGroup: :string,
+        resourceRole: :string,
+        role: :string,
+        schedule: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'Ec2Resource')
+    end
+  end
+end

data/lib/waterworks/resources/emr_cluster.rb

@@ -0,0 +1,68 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class EmrCluster < PipelineObject
+    # rubocop:disable Metrics/MethodLength
+    def self.safe_fields
+      {
+        actionOnResourceFailure: :string,
+        actionOnTaskFailure: :string,
+        additionalMasterSecurityGroupIds: :string,
+        additionalSlaveSecurityGroupIds: :string,
+        amiVersion: :string,
+        applications: :string,
+        attemptStatus: :string,
+        attemptTimeout: :string,
+        availabilityZone: :string,
+        bootstrapAction: :string,
+        configuration: :ref,
+        coreInstanceBidPrice: :string,
+        coreInstanceCount: :string,
+        coreInstanceType: :string,
+        emrManagedMasterSecurityGroupId: :string,
+        emrManagedSlaveSecurityGroupId: :string,
+        enableDebugging: :string,
+        failureAndRerunMode: :string,
+        hadoopSchedulerType: :string,
+        httpProxy: :ref,
+        initTimeout: :string,
+        keyPair: :string,
+        lateAfterTimeout: :string,
+        masterInstanceBidPrice: :string,
+        masterInstanceType: :string,
+        maxActiveInstances: :string,
+        maximumRetries: :string,
+        onFail: :ref,
+        onLateAction: :ref,
+        onSuccess: :ref,
+        parent: :ref,
+        pipelineLogUri: :string,
+        region: :string,
+        releaseLabel: :string,
+        reportProgressTimeout: :string,
+        resourceRole: :string,
+        retryDelay: :string,
+        role: :string,
+        runsOn: :ref,
+        scheduleType: :string,
+        subnetId: :string,
+        supportedProducts: :string,
+        taskInstanceBidPrice: :string,
+        taskInstanceCount: :string,
+        taskInstanceType: :string,
+        terminateAfter: :string,
+        useOnDemandOnLastAttempt: :string,
+        workerGroup: :string,
+        schedule: :ref,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'EmrCluster')
+    end
+  end
+end

data/lib/waterworks/resources/http_proxy.rb

@@ -0,0 +1,25 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class HttpProxy < PipelineObject
+    def self.safe_fields
+      {
+        parent: :ref,
+        password: :string,
+        s3NoProxy: :string,
+        username: :string,
+        windowsDomain: :string,
+        windowsWorkgroup: :string,
+        hostname: :string,
+        port: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'HttpProxy')
+    end
+  end
+end

data/lib/waterworks/schedule/schedule.rb

@@ -0,0 +1,23 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class Schedule < PipelineObject
+    def self.safe_fields
+      {
+        endDateTime: :string,
+        occurrences: :string,
+        parent: :ref,
+        period: :string,
+        startAt: :string,
+        startDateTime: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'Schedule')
+    end
+  end
+end

data/lib/waterworks/util.rb

@@ -0,0 +1,27 @@
+require 'set'
+module Waterworks
+  class Util
+    @asterisk_fields = Set.new(%i(password))
+    def self.indifferentify(hash)
+      hash.default_proc = proc { |h, k| h.key?(k.to_s) ? h[k.to_s] : nil }
+      hash
+    end
+
+    def self.asterisk_if_needed(field)
+      if @asterisk_fields.include? field.to_sym
+        "*#{field}".to_sym
+      else
+        field.to_sym
+      end
+    end
+
+    def self.deasterisk(key)
+      key = key[1..-1] if key.start_with? '*'
+      key
+    end
+
+    def time_expr(offset = '1', delim = '/')
+      "\#{format(minusDays(@scheduledStartTime,#{offset}),'YYYY#{delim}MM#{delim}dd')}"
+    end
+  end
+end

data/lib/waterworks/utilities/emr_configuration.rb

@@ -0,0 +1,21 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class EmrConfiguration < PipelineObject
+    def self.safe_fields
+      {
+        configuration: :ref,
+        parent: :ref,
+        property: :ref,
+        classification: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'EmrConfiguration')
+    end
+  end
+end

data/lib/waterworks/utilities/property.rb

@@ -0,0 +1,20 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class Property < PipelineObject
+    def self.safe_fields
+      {
+        parent: :ref,
+        key: :string,
+        value: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'Property')
+    end
+  end
+end

data/lib/waterworks/utilities/shell_script_config.rb

@@ -0,0 +1,20 @@
+require_relative '../pipeline_object.rb'
+
+module Waterworks
+  class ShellScriptConfig < PipelineObject
+    def self.safe_fields
+      {
+        parent: :ref,
+        scriptArgument: :string,
+        scriptUri: :string,
+        type: :string,
+      }.merge superclass.safe_fields
+    end
+
+    safe_fields.keys.each { |attr| attr_accessor attr }
+
+    def initialize(id = nil, name = nil)
+      super(id, name).set_attrs(type: 'ShellScriptConfig')
+    end
+  end
+end

data/lib/waterworks.rb

@@ -0,0 +1 @@
+Dir[File.dirname(__FILE__) + '/**/*.rb'].each { |file| require file }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterworks
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors:
 - Logan Akamatsu
@@ -24,17 +24,74 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.4'
-description: It will be mine. Oh yes. It will be mine.
-email: logan.akamatsu@gmail.com
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+description: Object representations for Amazon Data Pipelines
+email: 
 executables:
 - waterworks
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/waterworks
+- lib/waterworks.rb
+- lib/waterworks/actions/sns_alarm.rb
+- lib/waterworks/actions/terminate.rb
+- lib/waterworks/activities/copy_activity.rb
+- lib/waterworks/activities/emr_activity.rb
+- lib/waterworks/activities/hadoop_activity.rb
+- lib/waterworks/activities/hive_activity.rb
+- lib/waterworks/activities/hive_copy_activity.rb
+- lib/waterworks/activities/pig_activity.rb
+- lib/waterworks/activities/redshift_copy_activity.rb
+- lib/waterworks/activities/shell_command_activity.rb
+- lib/waterworks/activities/sql_activity.rb
+- lib/waterworks/containers/pipeline_definition.rb
+- lib/waterworks/data_formats/csv.rb
+- lib/waterworks/data_formats/custom.rb
+- lib/waterworks/data_formats/dynamo_db_data_format.rb
+- lib/waterworks/data_formats/dynamo_db_export_data_format.rb
+- lib/waterworks/data_formats/reg_ex.rb
+- lib/waterworks/data_formats/tsv.rb
+- lib/waterworks/data_nodes/dynamo_db_data_node.rb
+- lib/waterworks/data_nodes/my_sql_data_node.rb
+- lib/waterworks/data_nodes/redshift_data_node.rb
+- lib/waterworks/data_nodes/s3_data_node.rb
+- lib/waterworks/data_nodes/sql_data_node.rb
+- lib/waterworks/databases/jdbc_database.rb
+- lib/waterworks/databases/rds_database.rb
+- lib/waterworks/databases/redshift_database.rb
+- lib/waterworks/other/default.rb
+- lib/waterworks/pipeline_object.rb
+- lib/waterworks/preconditions/dynamo_db_data_exists.rb
+- lib/waterworks/preconditions/dynamo_db_table_exists.rb
+- lib/waterworks/preconditions/exists.rb
+- lib/waterworks/preconditions/s3_key_exists.rb
+- lib/waterworks/preconditions/s3_prefix_not_empty.rb
+- lib/waterworks/preconditions/shell_command_precondition.rb
+- lib/waterworks/resources/ec2_resource.rb
+- lib/waterworks/resources/emr_cluster.rb
+- lib/waterworks/resources/http_proxy.rb
+- lib/waterworks/schedule/schedule.rb
+- lib/waterworks/util.rb
+- lib/waterworks/utilities/emr_configuration.rb
+- lib/waterworks/utilities/property.rb
+- lib/waterworks/utilities/shell_script_config.rb
 homepage: http://rubygems.org/gems/waterworks
 licenses:
-- MIT
+- Apache 2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -55,5 +112,5 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Placeholder... gem to follow shortly
+summary: Lightweight AWS Data Pipeline SDK
 test_files: []