pangea 0.0.45 → 0.0.46

data/lib/pangea/resources/aws_glue_job/CLAUDE.md

@@ -1,398 +0,0 @@
-# AWS Glue Job - Architecture Notes
-
-## Resource Purpose
-
-AWS Glue Job provides serverless, managed ETL processing for data lake architectures, enabling schema-aware transformations, real-time streaming, and Python-based data processing with automatic scaling and built-in integration with the Glue Data Catalog.
-
-## Key Architectural Patterns
-
-### Serverless ETL Pipeline Pattern
-- **Auto-scaling Execution**: Dynamic worker allocation based on workload requirements
-- **Schema-aware Processing**: Integration with Glue Catalog for automatic schema discovery
-- **Checkpoint Management**: Built-in support for incremental processing and fault tolerance
-- **Multi-format Support**: Native support for Parquet, ORC, Avro, JSON, CSV, and custom formats
-
-### Stream Processing Pattern
-- **Real-time Transformation**: Continuous processing of streaming data from Kinesis, Kafka, MSK
-- **Windowed Operations**: Time-based and count-based windowing for stream analytics
-- **State Management**: Checkpoint-based state management for exactly-once processing
-- **Late Data Handling**: Watermark-based handling of out-of-order data
-
-### Data Quality Orchestration Pattern
-- **Validation Workflows**: Python shell jobs for data quality validation and monitoring
-- **Rule Engine Integration**: Integration with Great Expectations and custom validation frameworks
-- **Notification Integration**: Automated alerting through SNS/SQS for data quality issues
-- **Audit Trail**: Comprehensive logging and metrics for compliance requirements
-
-## Architecture Integration Points
-
-### Data Lake ETL Architecture
-```ruby
-# Multi-stage data lake processing pipeline
-raw_processor = aws_glue_job(:raw_ingestion, {
-  name: "raw-data-ingestion",
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  description: "Initial raw data ingestion and cataloging",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/raw_ingestion.py",
-    name: "glueetl"
-  },
-  **GlueJobAttributes.worker_recommendations_for_workload("small_etl"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--source-connection" => "rds-source",
-    "--target-database" => "raw_data_lake",
-    "--bookmark-option" => "job-bookmark-enable",
-    "--enable-data-lineage" => "true"
-  }
-})
-
-transformation_processor = aws_glue_job(:data_transformation, {
-  name: "data-transformation-etl",
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  description: "Business logic transformation and enrichment",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/transform.py",
-    name: "glueetl"
-  },
-  **GlueJobAttributes.worker_recommendations_for_workload("large_etl"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--source-database" => raw_processor.computed_properties[:source_database],
-    "--target-database" => "processed_data_lake",
-    "--transformation-rules" => "s3://config/transform-rules.json",
-    "--enable-spark-ui" => "true"
-  },
-  execution_property: {
-    max_concurrent_runs: 2
-  }
-})
-
-analytics_aggregator = aws_glue_job(:analytics_aggregation, {
-  name: "analytics-aggregation-etl",
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  description: "Analytics aggregation and mart creation",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/aggregate.py",
-    name: "glueetl"
-  },
-  **GlueJobAttributes.worker_recommendations_for_workload("medium_etl"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--source-database" => "processed_data_lake",
-    "--target-database" => "analytics_mart",
-    "--aggregation-level" => "daily",
-    "--partition-strategy" => "date_based"
-  }
-})
-
-# Orchestration dependencies handled by Glue Trigger
-pipeline_trigger = aws_glue_trigger(:etl_pipeline, {
-  name: "data-lake-pipeline-trigger",
-  type: "CONDITIONAL",
-  actions: [
-    { job_name: raw_processor.outputs[:name] },
-    { job_name: transformation_processor.outputs[:name], depends_on: [raw_processor.outputs[:name]] },
-    { job_name: analytics_aggregator.outputs[:name], depends_on: [transformation_processor.outputs[:name]] }
-  ]
-})
-```
-
-### Real-time Analytics Architecture
-```ruby
-# Stream processing for real-time analytics
-stream_enricher = aws_glue_job(:stream_enrichment, {
-  name: "real-time-enrichment",
-  role_arn: "arn:aws:iam::123456789012:role/GlueStreamingRole",
-  description: "Real-time data enrichment from Kinesis streams",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://streaming-scripts/enrich_stream.py",
-    name: "gluestreaming"
-  },
-  worker_type: "G.1X",
-  number_of_workers: 4,
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("streaming", {
-      checkpoint_location: "s3://streaming-checkpoints/enrichment/"
-    }),
-    "--source-stream-arn" => "arn:aws:kinesis:region:account:stream/raw-events",
-    "--enrichment-database" => "reference_data",
-    "--target-stream-arn" => "arn:aws:kinesis:region:account:stream/enriched-events",
-    "--window-size" => "100",
-    "--watermark-delay" => "10 minutes"
-  },
-  timeout: 2880 # Long-running streaming job
-})
-
-stream_aggregator = aws_glue_job(:stream_aggregation, {
-  name: "real-time-aggregation",
-  role_arn: "arn:aws:iam::123456789012:role/GlueStreamingRole",
-  description: "Real-time metrics aggregation", 
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://streaming-scripts/aggregate_stream.py",
-    name: "gluestreaming"
-  },
-  worker_type: "G.2X",
-  number_of_workers: 6,
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("streaming", {
-      checkpoint_location: "s3://streaming-checkpoints/aggregation/"
-    }),
-    "--source-stream-arn" => "arn:aws:kinesis:region:account:stream/enriched-events",
-    "--aggregation-windows" => "1 minute,5 minutes,1 hour",
-    "--target-table" => "real_time_metrics",
-    "--output-mode" => "update"
-  }
-})
-
-# EMR cluster can consume the same streams for batch analytics
-emr_cluster = aws_emr_cluster(:analytics_cluster, {
-  name: "streaming-analytics-cluster",
-  applications: ["Spark", "Hadoop", "Hive"],
-  ec2_attributes: {
-    instance_profile: "EMR_EC2_DefaultRole",
-    key_name: "analytics-keypair"
-  },
-  master_instance_group: {
-    instance_type: "m5.xlarge"
-  },
-  core_instance_group: {
-    instance_type: "m5.large",
-    instance_count: 4
-  }
-})
-```
-
-### Data Quality and Governance Architecture
-```ruby
-# Data quality validation pipeline
-quality_profiler = aws_glue_job(:data_profiling, {
-  name: "data-quality-profiling",
-  role_arn: "arn:aws:iam::123456789012:role/GlueQualityRole",
-  description: "Automated data profiling and quality assessment",
-  glue_version: "3.0",
-  command: {
-    script_location: "s3://quality-scripts/profile_data.py",
-    name: "pythonshell",
-    python_version: "3.9"
-  },
-  max_capacity: 1.0,
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("pythonshell", {
-      python_modules: "great-expectations==0.15.0,pandas==1.5.0,boto3==1.26.0"
-    }),
-    "--target-database" => "processed_data_lake",
-    "--profile-output-location" => "s3://data-quality/profiles/",
-    "--quality-threshold" => "0.95",
-    "--notification-topic" => "arn:aws:sns:region:account:data-quality-alerts"
-  }
-})
-
-schema_validator = aws_glue_job(:schema_validation, {
-  name: "schema-drift-detection", 
-  role_arn: "arn:aws:iam::123456789012:role/GlueQualityRole",
-  description: "Schema drift detection and validation",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://quality-scripts/validate_schema.py",
-    name: "glueetl"
-  },
-  **GlueJobAttributes.worker_recommendations_for_workload("small_etl"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--catalog-database" => "processed_data_lake",
-    "--schema-registry" => "s3://schemas/registry/",
-    "--drift-detection-mode" => "strict",
-    "--auto-evolve-schema" => "false"
-  }
-})
-
-# Data lineage tracking job
-lineage_tracker = aws_glue_job(:lineage_tracking, {
-  name: "data-lineage-tracker",
-  role_arn: "arn:aws:iam::123456789012:role/GlueLineageRole",
-  description: "Data lineage tracking and impact analysis",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://governance-scripts/track_lineage.py",
-    name: "pythonshell",
-    python_version: "3.9"
-  },
-  max_capacity: 0.0625,
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("pythonshell"),
-    "--lineage-database" => "data_lineage",
-    "--impact-analysis" => "enabled",
-    "--governance-rules" => "s3://governance/rules.json"
-  }
-})
-```
-
-## Performance Optimization Patterns
-
-### Dynamic Scaling Architecture
-```ruby
-# Auto-scaling ETL job with performance monitoring
-scalable_etl = aws_glue_job(:scalable_processor, {
-  name: "auto-scaling-etl",
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/scalable_transform.py",
-    name: "glueetl"
-  },
-  worker_type: "G.1X",
-  number_of_workers: 2, # Minimum workers
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--enable-auto-scaling" => "true",
-    "--max-workers" => "50",
-    "--target-utilization" => "0.8",
-    "--scale-up-behavior" => "aggressive",
-    "--scale-down-behavior" => "conservative",
-    "--enable-spark-ui" => "true",
-    "--spark-event-logs-path" => "s3://spark-logs/"
-  }
-})
-
-# Memory-optimized job for large datasets  
-memory_intensive_job = aws_glue_job(:memory_processor, {
-  name: "memory-intensive-etl",
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/memory_intensive.py",
-    name: "glueetl"
-  },
-  **GlueJobAttributes.worker_recommendations_for_workload("memory_intensive"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--enable-spark-ui" => "true",
-    "--conf" => "spark.sql.adaptive.enabled=true",
-    "--conf" => "spark.sql.adaptive.coalescePartitions.enabled=true",
-    "--conf" => "spark.serializer=org.apache.spark.serializer.KryoSerializer"
-  }
-})
-```
-
-### Batch vs Streaming Hybrid Architecture
-```ruby
-# Batch job for historical data processing
-batch_processor = aws_glue_job(:batch_historical, {
-  name: "historical-data-processor",
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/batch_historical.py",
-    name: "glueetl"
-  },
-  **GlueJobAttributes.worker_recommendations_for_workload("large_etl"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--processing-mode" => "batch",
-    "--lookback-days" => "90",
-    "--partition-strategy" => "date_based",
-    "--enable-pushdown-predicate" => "true"
-  }
-})
-
-# Streaming job for real-time updates
-streaming_processor = aws_glue_job(:streaming_realtime, {
-  name: "real-time-processor",
-  role_arn: "arn:aws:iam::123456789012:role/GlueStreamingRole",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://streaming-scripts/realtime_updates.py",
-    name: "gluestreaming"
-  },
-  worker_type: "G.1X",
-  number_of_workers: 4,
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("streaming", {
-      checkpoint_location: "s3://streaming-checkpoints/realtime/"
-    }),
-    "--processing-mode" => "streaming",
-    "--trigger-interval" => "30 seconds",
-    "--watermark-delay" => "2 minutes"
-  }
-})
-```
-
-## Cost Optimization Patterns
-
-### Resource Right-sizing Strategy
-```ruby
-# Cost-optimized small workload job
-cost_optimized_job = aws_glue_job(:cost_optimized, {
-  name: "cost-optimized-etl",
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/cost_optimized.py",
-    name: "glueetl"
-  },
-  **GlueJobAttributes.worker_recommendations_for_workload("small_etl"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--enable-auto-scaling" => "true",
-    "--max-workers" => "10",
-    "--enable-spot-instances" => "true", # If supported
-    "--cost-optimization-mode" => "enabled"
-  },
-  timeout: 120, # Shorter timeout for cost control
-  execution_property: {
-    max_concurrent_runs: 1 # Limit concurrent executions
-  }
-})
-
-# Flexible capacity job with cost monitoring
-flexible_capacity_job = aws_glue_job(:flexible_capacity, {
-  name: "flexible-capacity-etl", 
-  role_arn: "arn:aws:iam::123456789012:role/GlueETLRole",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://etl-scripts/flexible_processing.py",
-    name: "glueetl"
-  },
-  worker_type: "G.025X", # Smallest workers
-  number_of_workers: 2,
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--cost-tracking" => "enabled",
-    "--cost-alert-threshold" => "50.00",
-    "--processing-priority" => "cost_optimized"
-  }
-})
-```
-
-## Security and Compliance Integration
-
-### Encrypted Data Processing
-```ruby
-secure_processor = aws_glue_job(:secure_etl, {
-  name: "secure-data-processor",
-  role_arn: "arn:aws:iam::123456789012:role/GlueSecureRole",
-  description: "Secure processing of sensitive data",
-  glue_version: "4.0",
-  command: {
-    script_location: "s3://secure-scripts/encrypted_processing.py",
-    name: "glueetl"
-  },
-  security_configuration: "secure-glue-config",
-  **GlueJobAttributes.worker_recommendations_for_workload("medium_etl"),
-  default_arguments: {
-    **GlueJobAttributes.default_arguments_for_job_type("etl"),
-    "--encryption-mode" => "SSE-KMS",
-    "--kms-key-id" => "arn:aws:kms:region:account:key/key-id",
-    "--audit-logging" => "enabled",
-    "--data-classification" => "sensitive"
-  }
-})
-```
-
-This job resource provides the core compute layer for data lake architectures, enabling sophisticated ETL pipelines, real-time streaming, and data quality workflows with built-in optimization and governance capabilities.

data/lib/pangea/resources/aws_glue_job/resource.rb

@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-# Copyright 2025 The Pangea Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-require 'pangea/resources/base'
-require 'pangea/resources/reference'
-require 'pangea/resources/aws_glue_job/types'
-require 'pangea/resource_registry'
-
-module Pangea
-  module Resources
-    module AWS
-      # Create an AWS Glue Job with type-safe attributes
-      #
-      # @param name [Symbol] The resource name
-      # @param attributes [Hash] Glue Job attributes
-      # @return [ResourceReference] Reference object with outputs and computed properties
-      def aws_glue_job(name, attributes = {})
-        # Validate attributes using dry-struct
-        job_attrs = Types::GlueJobAttributes.new(attributes)
-        
-        # Generate terraform resource block via terraform-synthesizer
-        resource(:aws_glue_job, name) do
-          # Required attributes
-          job_name = job_attrs.name
-          role_arn job_attrs.role_arn
-          
-          # Description
-          description job_attrs.description if job_attrs.description
-          
-          # Glue version
-          glue_version job_attrs.glue_version if job_attrs.glue_version
-          
-          # Command configuration
-          command do
-            script_location job_attrs.command[:script_location]
-            name job_attrs.command[:name] if job_attrs.command[:name]
-            python_version job_attrs.command[:python_version] if job_attrs.command[:python_version]
-            runtime job_attrs.command[:runtime] if job_attrs.command[:runtime]
-          end
-          
-          # Default arguments
-          if job_attrs.default_arguments.any?
-            default_arguments do
-              job_attrs.default_arguments.each do |key, value|
-                public_send(key.gsub(/[^a-zA-Z0-9_]/, '_').downcase, value)
-              end
-            end
-          end
-          
-          # Non-overridable arguments
-          if job_attrs.non_overridable_arguments.any?
-            non_overridable_arguments do
-              job_attrs.non_overridable_arguments.each do |key, value|
-                public_send(key.gsub(/[^a-zA-Z0-9_]/, '_').downcase, value)
-              end
-            end
-          end
-          
-          # Connections
-          if job_attrs.connections.any?
-            connections job_attrs.connections
-          end
-          
-          # Capacity configuration - use either max_capacity OR worker configuration
-          if job_attrs.uses_worker_configuration?
-            worker_type job_attrs.worker_type
-            number_of_workers job_attrs.number_of_workers
-          elsif job_attrs.max_capacity
-            max_capacity job_attrs.max_capacity
-          end
-          
-          # Job configuration
-          timeout job_attrs.timeout if job_attrs.timeout
-          max_retries job_attrs.max_retries if job_attrs.max_retries
-          security_configuration job_attrs.security_configuration if job_attrs.security_configuration
-          
-          # Notification properties
-          if job_attrs.notification_property
-            notification_property do
-              np = job_attrs.notification_property
-              notify_delay_after np[:notify_delay_after] if np[:notify_delay_after]
-            end
-          end
-          
-          # Execution properties
-          if job_attrs.execution_property.any?
-            execution_property do
-              ep = job_attrs.execution_property
-              max_concurrent_runs ep[:max_concurrent_runs] if ep[:max_concurrent_runs]
-            end
-          end
-          
-          # Apply tags if present
-          if job_attrs.tags.any?
-            tags do
-              job_attrs.tags.each do |key, value|
-                public_send(key, value)
-              end
-            end
-          end
-        end
-        
-        # Return resource reference with available outputs
-        ResourceReference.new(
-          type: 'aws_glue_job',
-          name: name,
-          resource_attributes: job_attrs.to_h,
-          outputs: {
-            id: "${aws_glue_job.#{name}.id}",
-            name: "${aws_glue_job.#{name}.name}",
-            arn: "${aws_glue_job.#{name}.arn}",
-            max_capacity: "${aws_glue_job.#{name}.max_capacity}",
-            worker_type: "${aws_glue_job.#{name}.worker_type}",
-            number_of_workers: "${aws_glue_job.#{name}.number_of_workers}"
-          },
-          computed_properties: {
-            uses_worker_configuration: job_attrs.uses_worker_configuration?,
-            is_streaming_job: job_attrs.is_streaming_job?,
-            is_python_shell_job: job_attrs.is_python_shell_job?,
-            is_etl_job: job_attrs.is_etl_job?,
-            effective_glue_version: job_attrs.effective_glue_version,
-            effective_python_version: job_attrs.effective_python_version,
-            estimated_dpu_capacity: job_attrs.estimated_dpu_capacity,
-            estimated_hourly_cost_usd: job_attrs.estimated_hourly_cost_usd,
-            configuration_warnings: job_attrs.configuration_warnings
-          }
-        )
-      end
-    end
-  end
-end
-
-# Auto-register this module when it's loaded
-Pangea::ResourceRegistry.register(:aws, Pangea::Resources::AWS)