pangea-aws 0.1.0

data/lib/pangea/resources/aws_glue_catalog_table/types.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'pangea/resources/types'
+require_relative 'types/storage_descriptor'
+require_relative 'types/validators'
+require_relative 'types/table_helpers'
+require_relative 'types/format_helpers'
+
+module Pangea
+  module Resources
+    module AWS
+      module Types
+        # Type-safe attributes for AWS Glue Catalog Table resources
+        class GlueCatalogTableAttributes < Pangea::Resources::BaseAttributes
+          extend GlueCatalogTableFormatHelpers
+          extend GlueCatalogTableValidators
+          include GlueCatalogTableHelpers
+
+          # Table name (required)
+          attribute? :name, Resources::Types::String.optional
+
+          # Database name (required)
+          attribute? :database_name, Resources::Types::String.optional
+
+          # Catalog ID (optional, defaults to AWS account ID)
+          attribute? :catalog_id, Resources::Types::String.optional
+
+          # Table owner
+          attribute? :owner, Resources::Types::String.optional
+
+          # Table description
+          attribute? :description, Resources::Types::String.optional
+
+          # Table type
+          attribute? :table_type, Resources::Types::String.constrained(included_in: ["EXTERNAL_TABLE", "MANAGED_TABLE", "VIRTUAL_VIEW"]).optional
+
+          # Parameters for the table
+          attribute :parameters, Resources::Types::Hash.map(Resources::Types::String, Resources::Types::String).default({}.freeze)
+
+          # Storage descriptor
+          attribute? :storage_descriptor, GlueCatalogTableStorageDescriptor::StorageDescriptorSchema.optional
+
+          # Partition keys
+          attribute? :partition_keys, Resources::Types::Array.of(
+            Resources::Types::Hash.schema(
+              name: Resources::Types::String,
+              type: Resources::Types::String,
+              comment?: Resources::Types::String.optional
+            ).lax
+          ).default([].freeze)
+
+          # Retention period in days
+          attribute? :retention, Resources::Types::Integer.optional
+
+          # View information for VIRTUAL_VIEW tables
+          attribute? :view_original_text, Resources::Types::String.optional
+          attribute? :view_expanded_text, Resources::Types::String.optional
+
+          # Targeted column information
+          attribute? :target_table, Resources::Types::Hash.schema(
+            catalog_id?: Resources::Types::String.optional,
+            database_name?: Resources::Types::String.optional,
+            name?: Resources::Types::String.optional
+          ).lax.optional
+
+          # Custom validation
+          def self.new(attributes = {})
+            attrs = super(attributes)
+            validate_attributes(attrs)
+            attrs
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_catalog_table/types.rbs

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Pangea
+  module Resources
+    module AWS
+      class GlueCatalogTableAttributes < Dry::Struct
+        @attributes: Hash[Symbol, untyped]
+        
+        attr_reader name: String
+        attr_reader database_name: String
+        attr_reader catalog_id: String?
+        attr_reader owner: String?
+        attr_reader description: String?
+        attr_reader table_type: String?
+        attr_reader parameters: Hash[String, String]
+        attr_reader storage_descriptor: Hash[Symbol, untyped]?
+        attr_reader partition_keys: Array[Hash[Symbol, untyped]]
+        attr_reader retention: Integer?
+        attr_reader view_original_text: String?
+        attr_reader view_expanded_text: String?
+        attr_reader target_table: Hash[Symbol, String?]
+        
+        def self.new: (Hash[Symbol, untyped] attributes) -> GlueCatalogTableAttributes
+        
+        # Query methods
+        def is_partitioned?: () -> bool
+        def is_external?: () -> bool
+        def is_view?: () -> bool
+        
+        # Computed properties
+        def table_format: () -> String
+        def compression_type: () -> String?
+        def estimated_size_gb: () -> Float
+        def column_summary: () -> Hash[Symbol, Integer]
+        
+        # Helper class methods
+        def self.serde_info_for_format: (String | Symbol format) -> Hash[Symbol, untyped]
+        def self.input_output_format_for_type: (String | Symbol format) -> Hash[Symbol, String]
+        
+        def to_h: () -> Hash[Symbol, untyped]
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_job/resource.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require 'pangea/resources/base'
+require 'pangea/resources/reference'
+require 'pangea/resources/aws_glue_job/types'
+require 'pangea/resource_registry'
+
+module Pangea
+  module Resources
+    module AWS
+      # Create an AWS Glue Job with type-safe attributes
+      #
+      # @param name [Symbol] The resource name
+      # @param attributes [Hash] Glue Job attributes
+      # @return [ResourceReference] Reference object with outputs and computed properties
+      def aws_glue_job(name, attributes = {})
+        # Validate attributes using dry-struct
+        job_attrs = Types::GlueJobAttributes.new(attributes)
+        
+        # Generate terraform resource block via terraform-synthesizer
+        resource(:aws_glue_job, name) do
+          # Required attributes
+          job_name = job_attrs.name
+          role_arn job_attrs.role_arn
+          
+          # Description
+          description job_attrs.description if job_attrs.description
+          
+          # Glue version
+          glue_version job_attrs.glue_version if job_attrs.glue_version
+          
+          # Command configuration
+          command do
+            script_location job_attrs.command&.dig(:script_location)
+            name job_attrs.command&.dig(:name) if job_attrs.command&.dig(:name)
+            python_version job_attrs.command&.dig(:python_version) if job_attrs.command&.dig(:python_version)
+            runtime job_attrs.command&.dig(:runtime) if job_attrs.command&.dig(:runtime)
+          end
+          
+          # Default arguments
+          if job_attrs.default_arguments&.any?
+            default_arguments do
+              job_attrs.default_arguments.each do |key, value|
+                public_send(key.to_s.gsub(/[^a-zA-Z0-9_]/, '_').downcase, value)
+              end
+            end
+          end
+          
+          # Non-overridable arguments
+          if job_attrs.non_overridable_arguments&.any?
+            non_overridable_arguments do
+              job_attrs.non_overridable_arguments.each do |key, value|
+                public_send(key.to_s.gsub(/[^a-zA-Z0-9_]/, '_').downcase, value)
+              end
+            end
+          end
+          
+          # Connections
+          if job_attrs.connections&.any?
+            connections job_attrs.connections
+          end
+          
+          # Capacity configuration - use either max_capacity OR worker configuration
+          if job_attrs.uses_worker_configuration?
+            worker_type job_attrs.worker_type
+            number_of_workers job_attrs.number_of_workers
+          elsif job_attrs.max_capacity
+            max_capacity job_attrs.max_capacity
+          end
+          
+          # Job configuration
+          timeout job_attrs.timeout if job_attrs.timeout
+          max_retries job_attrs.max_retries if job_attrs.max_retries
+          security_configuration job_attrs.security_configuration if job_attrs.security_configuration
+          
+          # Notification properties
+          if job_attrs.notification_property
+            notification_property do
+              np = job_attrs.notification_property
+              notify_delay_after np[:notify_delay_after] if np[:notify_delay_after]
+            end
+          end
+          
+          # Execution properties
+          if job_attrs.execution_property&.any?
+            execution_property do
+              ep = job_attrs.execution_property
+              max_concurrent_runs ep[:max_concurrent_runs] if ep[:max_concurrent_runs]
+            end
+          end
+          
+          # Apply tags if present
+          if job_attrs.tags&.any?
+            tags do
+              job_attrs.tags.each do |key, value|
+                public_send(key, value)
+              end
+            end
+          end
+        end
+        
+        # Return resource reference with available outputs
+        ResourceReference.new(
+          type: 'aws_glue_job',
+          name: name,
+          resource_attributes: job_attrs.to_h,
+          outputs: {
+            id: "${aws_glue_job.#{name}.id}",
+            name: "${aws_glue_job.#{name}.name}",
+            arn: "${aws_glue_job.#{name}.arn}",
+            max_capacity: "${aws_glue_job.#{name}.max_capacity}",
+            worker_type: "${aws_glue_job.#{name}.worker_type}",
+            number_of_workers: "${aws_glue_job.#{name}.number_of_workers}"
+          },
+          computed_properties: {
+            uses_worker_configuration: job_attrs.uses_worker_configuration?,
+            is_streaming_job: job_attrs.is_streaming_job?,
+            is_python_shell_job: job_attrs.is_python_shell_job?,
+            is_etl_job: job_attrs.is_etl_job?,
+            effective_glue_version: job_attrs.effective_glue_version,
+            effective_python_version: job_attrs.effective_python_version,
+            estimated_dpu_capacity: job_attrs.estimated_dpu_capacity,
+            estimated_hourly_cost_usd: job_attrs.estimated_hourly_cost_usd,
+            configuration_warnings: job_attrs.configuration_warnings
+          }
+        )
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_job/types/attributes.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module Types
+        # Type-safe attributes for AWS Glue Job resources
+        class GlueJobAttributes < Pangea::Resources::BaseAttributes
+          extend GlueJobClassMethods
+          extend GlueJobValidation
+          include GlueJobInstanceMethods
+
+          # Job name (required)
+          attribute? :name, Resources::Types::String.optional
+
+          # IAM role ARN (required)
+          attribute? :role_arn, Resources::Types::String.optional
+
+          # Job description
+          attribute? :description, Resources::Types::String.optional
+
+          # Glue version
+          attribute? :glue_version, Resources::Types::String.constrained(included_in: ['0.9', '1.0', '2.0', '3.0', '4.0']).optional
+
+          # Job command
+          attribute? :command, Resources::Types::Hash.schema(
+            script_location: Resources::Types::String,
+            name?: Resources::Types::String.constrained(included_in: ['glueetl', 'gluestreaming', 'pythonshell']).optional,
+            python_version?: Resources::Types::String.constrained(included_in: ['2', '3', '3.6', '3.9']).optional,
+            runtime?: Resources::Types::String.optional
+          ).lax
+
+          # Default job arguments
+          attribute :default_arguments, Resources::Types::Hash.map(Resources::Types::String, Resources::Types::String).default({}.freeze)
+
+          # Non-overridable arguments
+          attribute :non_overridable_arguments, Resources::Types::Hash.map(Resources::Types::String, Resources::Types::String).default({}.freeze)
+
+          # Job connections
+          attribute :connections, Resources::Types::Array.of(Resources::Types::String).default([].freeze)
+
+          # Maximum capacity (DPUs)
+          attribute? :max_capacity, Resources::Types::Float.optional
+
+          # Worker configuration for Glue 2.0+
+          attribute? :worker_type, Resources::Types::String.constrained(included_in: ['Standard', 'G.1X', 'G.2X', 'G.025X', 'G.4X', 'G.8X', 'Z.2X']).optional
+          attribute? :number_of_workers, Resources::Types::Integer.optional
+
+          # Job timeout in minutes
+          attribute? :timeout, Resources::Types::Integer.optional
+
+          # Maximum retries
+          attribute? :max_retries, Resources::Types::Integer.optional
+
+          # Security configuration
+          attribute? :security_configuration, Resources::Types::String.optional
+
+          # Notification properties
+          attribute? :notification_property, Resources::Types::Hash.schema(
+            notify_delay_after?: Resources::Types::Integer.optional
+          ).lax.optional
+
+          # Execution properties
+          attribute? :execution_property, Resources::Types::Hash.schema(
+            max_concurrent_runs?: Resources::Types::Integer.constrained(gteq: 1, lteq: 1000).optional
+          ).lax.optional
+
+          # Tags
+          attribute :tags, Resources::Types::AwsTags.default({}.freeze)
+
+          # Custom validation
+          def self.new(attributes = {})
+            attrs = super(attributes)
+            validate_job_name(attrs.name)
+            validate_role_arn(attrs.role_arn)
+            validate_script_location(attrs.command)
+            validate_worker_configuration(attrs)
+            validate_timeout(attrs.timeout)
+            attrs
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_job/types/class_methods.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module Types
+        # Class methods for AWS Glue Job attributes
+        module GlueJobClassMethods
+          # Generate common default arguments based on job type
+          def default_arguments_for_job_type(job_type, options = {})
+            case job_type.to_s
+            when 'etl'
+              etl_default_arguments
+            when 'streaming'
+              streaming_default_arguments(options)
+            when 'pythonshell'
+              pythonshell_default_arguments(options)
+            else
+              base_default_arguments
+            end
+          end
+
+          # Generate worker configuration recommendations
+          def worker_recommendations_for_workload(workload_type, _data_size_gb = nil)
+            recommendations = {
+              'small_etl' => { worker_type: 'G.1X', number_of_workers: 2 },
+              'medium_etl' => { worker_type: 'G.1X', number_of_workers: 10 },
+              'large_etl' => { worker_type: 'G.2X', number_of_workers: 20 },
+              'memory_intensive' => { worker_type: 'Z.2X', number_of_workers: 10 },
+              'streaming' => { worker_type: 'G.1X', number_of_workers: 2 },
+              'python_shell' => {}
+            }
+            recommendations[workload_type.to_s] || { worker_type: 'G.1X', number_of_workers: 5 }
+          end
+
+          private
+
+          def base_default_arguments
+            {
+              '--job-language' => 'python',
+              '--enable-metrics' => '',
+              '--enable-continuous-cloudwatch-log' => 'true'
+            }
+          end
+
+          def etl_default_arguments
+            base_default_arguments.merge(
+              '--enable-job-insights' => 'true',
+              '--enable-auto-scaling' => 'true'
+            )
+          end
+
+          def streaming_default_arguments(options)
+            base_default_arguments.merge(
+              '--enable-metrics' => '',
+              '--continuous-log-logStream' => options[:log_stream] || 'glue-streaming-job',
+              '--window-size' => options[:window_size] || '100',
+              '--checkpoint-location' => options[:checkpoint_location] || 's3://bucket/checkpoints/'
+            )
+          end
+
+          def pythonshell_default_arguments(options)
+            {
+              '--job-language' => 'python',
+              '--python-modules-installer-option' => options[:python_modules] || ''
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_job/types/instance_methods.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module Types
+        # Instance methods for AWS Glue Job attributes
+        module GlueJobInstanceMethods
+          # Check if job uses modern worker configuration
+          def uses_worker_configuration?
+            !!(worker_type && number_of_workers)
+          end
+
+          # Check if job is streaming
+          def is_streaming_job?
+            command[:name] == 'gluestreaming'
+          end
+
+          # Check if job is Python shell
+          def is_python_shell_job?
+            command[:name] == 'pythonshell'
+          end
+
+          # Check if job is ETL
+          def is_etl_job?
+            command[:name].nil? || command[:name] == 'glueetl'
+          end
+
+          # Get effective Glue version (with defaults)
+          def effective_glue_version
+            glue_version || (is_python_shell_job? ? '1.0' : '2.0')
+          end
+
+          # Get effective Python version
+          def effective_python_version
+            command[:python_version] || (effective_glue_version >= '2.0' ? '3' : '2')
+          end
+
+          # Calculate estimated DPU capacity
+          def estimated_dpu_capacity
+            if uses_worker_configuration?
+              calculate_worker_dpu_capacity
+            elsif max_capacity
+              max_capacity
+            elsif is_python_shell_job?
+              0.0625 # Python shell default
+            else
+              2.0 # ETL job default
+            end
+          end
+
+          # Estimate hourly cost based on DPU usage
+          def estimated_hourly_cost_usd
+            dpu_capacity = estimated_dpu_capacity
+            # AWS Glue pricing (approximate, varies by region)
+            cost_per_dpu_hour = 0.44
+            (dpu_capacity * cost_per_dpu_hour).round(4)
+          end
+
+          # Check if job configuration is optimal
+          def configuration_warnings
+            warnings = []
+            warnings << 'Consider upgrading to Glue 2.0+ for better performance and features' if glue_version && glue_version < '2.0'
+            warnings << 'Consider specifying worker configuration for better resource control' if !uses_worker_configuration? && !max_capacity && !is_python_shell_job?
+            warnings << 'Very long timeout (>24h) may indicate job optimization opportunities' if timeout && timeout > 1440
+            warnings << 'Consider enabling CloudWatch metrics for streaming jobs' if is_streaming_job? && !default_arguments.key?('--enable-metrics')
+            warnings
+          end
+
+          private
+
+          def calculate_worker_dpu_capacity
+            dpu_multiplier = {
+              'Standard' => 1.0, 'G.1X' => 1.0, 'G.2X' => 2.0,
+              'G.025X' => 0.25, 'G.4X' => 4.0, 'G.8X' => 8.0, 'Z.2X' => 2.0
+            }
+            number_of_workers * (dpu_multiplier[worker_type] || 1.0)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_job/types/validation.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module Types
+        # Validation module for AWS Glue Job attributes
+        module GlueJobValidation
+          def validate_job_name(name)
+            unless name =~ /\A[a-zA-Z_][a-zA-Z0-9_-]*\z/
+              raise Dry::Struct::Error,
+                    'Job name must start with letter or underscore and contain only alphanumeric characters, underscores, and hyphens'
+            end
+            raise Dry::Struct::Error, 'Job name must be 255 characters or less' if name.length > 255
+          end
+
+          def validate_role_arn(role_arn)
+            unless role_arn.match(/\Aarn:aws:iam::\d{12}:role\//)
+              raise Dry::Struct::Error, 'Role ARN must be in format arn:aws:iam::account:role/role-name'
+            end
+          end
+
+          def validate_script_location(command)
+            script_location = command[:script_location]
+            unless script_location.match(/\As3:\/\//)
+              raise Dry::Struct::Error, 'Script location must be an S3 URL (s3://bucket/path)'
+            end
+          end
+
+          def validate_worker_configuration(attrs)
+            if attrs.worker_type && !attrs.number_of_workers
+              raise Dry::Struct::Error, 'number_of_workers is required when worker_type is specified'
+            end
+
+            if attrs.number_of_workers && !attrs.worker_type
+              raise Dry::Struct::Error, 'worker_type is required when number_of_workers is specified'
+            end
+
+            if attrs.max_capacity && (attrs.worker_type || attrs.number_of_workers)
+              raise Dry::Struct::Error, 'max_capacity cannot be used with worker_type/number_of_workers configuration'
+            end
+          end
+
+          def validate_timeout(timeout)
+            return if timeout.nil?
+            if timeout < 1 || timeout > 2880
+              raise Dry::Struct::Error, 'Timeout must be between 1 and 2880 minutes (48 hours)'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_job/types.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'pangea/resources/types'
+require_relative 'types/validation'
+require_relative 'types/instance_methods'
+require_relative 'types/class_methods'
+require_relative 'types/attributes'