pangea 0.0.42 → 0.0.45

data/lib/pangea/resources/aws_glue_catalog_database/types.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require 'pangea/resources/types'
+
+module Pangea
+  module Resources
+    module AWS
+      module Types
+      # Type-safe attributes for AWS Glue Catalog Database resources
+      class GlueCatalogDatabaseAttributes < Dry::Struct
+        # Database name (required)
+        attribute :name, Resources::Types::String
+        
+        # Catalog ID (optional, defaults to AWS account ID)
+        attribute :catalog_id, Resources::Types::String.optional
+        
+        # Database description
+        attribute :description, Resources::Types::String.optional
+        
+        # Location URI for external databases
+        attribute :location_uri, Resources::Types::String.optional
+        
+        # Parameters for the database
+        attribute :parameters, Resources::Types::Hash.map(Types::String, Types::String).default({}.freeze)
+        
+        # Database type
+        attribute :database_input, Resources::Types::Hash.schema(
+          description?: Types::String.optional,
+          location_uri?: Types::String.optional,
+          parameters?: Types::Hash.map(Types::String, Types::String).optional
+        ).optional
+        
+        # Permissions and access control
+        attribute :create_table_default_permission, Resources::Types::Array.of(
+          Types::Hash.schema(
+            permissions: Types::Array.of(Types::String.enum(
+              "ALL", "SELECT", "INSERT", "DELETE", "UPDATE", "CREATE_TABLE", "DROP_TABLE", "ALTER"
+            )),
+            principal: Types::Hash.schema(
+              data_lake_principal_identifier?: Types::String.optional,
+              data_lake_principal?: Types::String.optional
+            )
+          )
+        ).default([].freeze)
+        
+        # Tags
+        attribute :tags, Resources::Types::AwsTags.default({}.freeze)
+
+        # Custom validation
+        def self.new(attributes = {})
+          attrs = super(attributes)
+          
+          # Validate database name format
+          unless attrs.name =~ /\A[a-zA-Z_][a-zA-Z0-9_]*\z/
+            raise Dry::Struct::Error, "Database name must start with letter or underscore and contain only alphanumeric characters and underscores"
+          end
+          
+          # Validate database name length
+          if attrs.name.length > 128
+            raise Dry::Struct::Error, "Database name must be 128 characters or less"
+          end
+          
+          # Validate location URI format if provided
+          if attrs.location_uri && !attrs.location_uri.match(/\A(s3|hdfs|file):\/\//)
+            raise Dry::Struct::Error, "Location URI must start with s3://, hdfs://, or file://"
+          end
+
+          attrs
+        end
+
+        # Check if database is external
+        def is_external?
+          !location_uri.nil?
+        end
+
+        # Check if database has custom permissions
+        def has_custom_permissions?
+          create_table_default_permission.any?
+        end
+
+        # Get database type based on location
+        def database_type
+          return "glue" unless location_uri
+          
+          case location_uri
+          when /\As3:\/\//
+            "s3"
+          when /\Ahdfs:\/\//
+            "hdfs"
+          when /\Afile:\/\//
+            "file"
+          else
+            "external"
+          end
+        end
+
+        # Helper method to generate S3 path
+        def s3_path
+          return nil unless location_uri && location_uri.start_with?("s3://")
+          location_uri.sub("s3://", "").split("/").first
+        end
+
+        # Estimate storage requirements for catalog metadata
+        def estimated_catalog_size_kb
+          base_size = 1 # Base metadata
+          base_size += (description&.length || 0) / 1024.0
+          base_size += parameters.sum { |k, v| k.length + v.length } / 1024.0
+          
+          [base_size, 0.1].max.round(2)
+        end
+
+        # Generate default parameters for common database types
+        def self.default_parameters_for_type(type)
+          case type.to_s
+          when "data_lake"
+            {
+              "classification" => "data_lake",
+              "compressionType" => "gzip",
+              "typeOfData" => "structured"
+            }
+          when "analytics"
+            {
+              "classification" => "analytics",
+              "optimized_for" => "query_performance"
+            }
+          when "raw"
+            {
+              "classification" => "raw_data",
+              "retention_policy" => "long_term"
+            }
+          else
+            {}
+          end
+        end
+      end
+    end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_catalog_database/types.rbs

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Pangea
+  module Resources
+    module AWS
+      class GlueCatalogDatabaseAttributes < Dry::Struct
+        @attributes: Hash[Symbol, untyped]
+        
+        attr_reader name: String
+        attr_reader catalog_id: String?
+        attr_reader description: String?
+        attr_reader location_uri: String?
+        attr_reader parameters: Hash[String, String]
+        attr_reader database_input: Hash[Symbol, untyped]?
+        attr_reader create_table_default_permission: Array[Hash[Symbol, untyped]]
+        attr_reader tags: Hash[Symbol, String]
+        
+        def self.new: (Hash[Symbol, untyped] attributes) -> GlueCatalogDatabaseAttributes
+        
+        # Query methods
+        def is_external?: () -> bool
+        def has_custom_permissions?: () -> bool
+        
+        # Computed properties
+        def database_type: () -> String
+        def s3_path: () -> String?
+        def estimated_catalog_size_kb: () -> Float
+        
+        # Class methods for defaults
+        def self.default_parameters_for_type: (String | Symbol type) -> Hash[String, String]
+        
+        def to_h: () -> Hash[Symbol, untyped]
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_glue_catalog_table/CLAUDE.md

@@ -0,0 +1,387 @@
+# AWS Glue Catalog Table - Architecture Notes
+
+## Resource Purpose
+
+AWS Glue Catalog Table provides schema registry and metadata management for data lake architectures, enabling schema evolution, query optimization, and data governance across analytics engines like Athena, Spark, EMR, and Redshift Spectrum.
+
+## Key Architectural Patterns
+
+### Schema Registry Pattern
+- **Centralized Schema Management**: Single source of truth for data structure definitions
+- **Multi-Engine Compatibility**: Same schema accessible from Spark, Athena, Presto, Hive
+- **Schema Evolution**: Support for backward/forward compatible schema changes
+- **Type System Integration**: Rich type system supporting complex nested structures
+
+### Data Lake Organization Pattern
+- **Layered Table Structure**: Separate tables for raw, processed, and analytics-ready data
+- **Partition Strategy**: Time-based and dimensional partitioning for query optimization
+- **Format Optimization**: Different storage formats optimized for specific access patterns
+- **Lifecycle Management**: Automated table lifecycle through retention policies
+
+### Multi-Tenant Analytics Pattern
+- **Tenant Isolation**: Separate table namespaces per tenant with controlled access
+- **Shared Analytics**: Common analytical views across tenant data
+- **Cost Attribution**: Per-tenant cost tracking through table-level tagging
+- **Security Boundaries**: Fine-grained access control at table and column level
+
+## Architecture Integration Points
+
+### Data Pipeline Integration
+```ruby
+# Pipeline with schema evolution support
+raw_table = aws_glue_catalog_table(:raw_events, {
+  name: "raw_event_stream",
+  database_name: "ingestion",
+  table_type: "EXTERNAL_TABLE",
+  storage_descriptor: {
+    location: "s3://pipeline-raw/events/",
+    **GlueCatalogTableAttributes.input_output_format_for_type("json"),
+    serde_info: GlueCatalogTableAttributes.serde_info_for_format("json"),
+    columns: [
+      { name: "event_payload", type: "string" },
+      { name: "schema_version", type: "string" },
+      { name: "ingested_at", type: "timestamp" }
+    ]
+  },
+  partition_keys: [
+    { name: "date", type: "string" },
+    { name: "hour", type: "string" }
+  ]
+})
+
+processed_table = aws_glue_catalog_table(:processed_events, {
+  name: "structured_events",
+  database_name: "processed",
+  table_type: "EXTERNAL_TABLE", 
+  storage_descriptor: {
+    location: "s3://pipeline-processed/events/",
+    **GlueCatalogTableAttributes.input_output_format_for_type("parquet"),
+    serde_info: GlueCatalogTableAttributes.serde_info_for_format("parquet"),
+    columns: [
+      { name: "user_id", type: "string" },
+      { name: "event_type", type: "string" },
+      { name: "properties", type: "map<string,string>" },
+      { name: "timestamp", type: "timestamp" },
+      { name: "processed_at", type: "timestamp" }
+    ]
+  },
+  partition_keys: [{ name: "date", type: "string" }]
+})
+
+# ETL job references these table schemas
+etl_job = aws_glue_job(:schema_aware_etl, {
+  name: "raw_to_processed_etl",
+  script_location: "s3://scripts/transform.py",
+  default_arguments: {
+    "--source_table" => raw_table.outputs[:name],
+    "--target_table" => processed_table.outputs[:name],
+    "--source_database" => raw_table.resource_attributes[:database_name],
+    "--target_database" => processed_table.resource_attributes[:database_name]
+  }
+})
+```
+
+### Analytics Engine Integration
+```ruby
+# Analytics-optimized tables for different query patterns
+fact_table = aws_glue_catalog_table(:sales_facts, {
+  name: "sales_fact",
+  database_name: "analytics",
+  description: "Sales transaction facts",
+  table_type: "EXTERNAL_TABLE",
+  storage_descriptor: {
+    location: "s3://analytics-warehouse/sales_fact/",
+    **GlueCatalogTableAttributes.input_output_format_for_type("parquet"),
+    serde_info: GlueCatalogTableAttributes.serde_info_for_format("parquet"),
+    columns: [
+      { name: "transaction_id", type: "string" },
+      { name: "customer_key", type: "bigint" },
+      { name: "product_key", type: "bigint" },
+      { name: "time_key", type: "bigint" },
+      { name: "amount", type: "decimal(18,2)" },
+      { name: "quantity", type: "int" }
+    ],
+    sort_columns: [
+      { column: "time_key", sort_order: 1 },
+      { column: "customer_key", sort_order: 1 }
+    ]
+  },
+  partition_keys: [
+    { name: "year", type: "string" },
+    { name: "quarter", type: "string" }
+  ],
+  parameters: {
+    "optimization.target" => "analytics",
+    "compression.codec" => "snappy",
+    "projection.enabled" => "true",
+    "projection.year.type" => "integer",
+    "projection.year.range" => "2020,2030"
+  }
+})
+
+# Customer dimension with SCD Type 2 support
+dim_table = aws_glue_catalog_table(:customer_dim, {
+  name: "customer_dimension", 
+  database_name: "analytics",
+  table_type: "EXTERNAL_TABLE",
+  storage_descriptor: {
+    location: "s3://analytics-warehouse/customer_dim/",
+    **GlueCatalogTableAttributes.input_output_format_for_type("parquet"),
+    serde_info: GlueCatalogTableAttributes.serde_info_for_format("parquet"),
+    columns: [
+      { name: "customer_key", type: "bigint" },
+      { name: "customer_id", type: "string" },
+      { name: "name", type: "string" },
+      { name: "email", type: "string" },
+      { name: "segment", type: "string" },
+      { name: "effective_date", type: "date" },
+      { name: "expiry_date", type: "date" },
+      { name: "is_current", type: "boolean" }
+    ]
+  }
+})
+
+# Materialized view for complex analytics
+analytics_view = aws_glue_catalog_table(:customer_metrics, {
+  name: "customer_monthly_metrics",
+  database_name: "analytics",
+  table_type: "VIRTUAL_VIEW",
+  view_original_text: %{
+    SELECT 
+      c.customer_id,
+      c.segment,
+      DATE_TRUNC('month', f.transaction_date) as month,
+      SUM(f.amount) as total_revenue,
+      COUNT(*) as transaction_count,
+      AVG(f.amount) as avg_transaction_value
+    FROM sales_fact f
+    JOIN customer_dimension c ON f.customer_key = c.customer_key
+    WHERE c.is_current = true
+    GROUP BY c.customer_id, c.segment, DATE_TRUNC('month', f.transaction_date)
+  },
+  parameters: {
+    "view_type" => "materialized",
+    "refresh_schedule" => "daily",
+    "dependencies" => "sales_fact,customer_dimension"
+  }
+})
+```
+
+## Performance Optimization Patterns
+
+### Partition Strategy Design
+```ruby
+# Time-based partitioning for time-series data
+time_series_table = aws_glue_catalog_table(:iot_sensors, {
+  name: "iot_sensor_data",
+  database_name: "telemetry",
+  storage_descriptor: {
+    location: "s3://iot-data/sensors/",
+    columns: [
+      { name: "device_id", type: "string" },
+      { name: "sensor_type", type: "string" }, 
+      { name: "value", type: "double" },
+      { name: "timestamp", type: "timestamp" }
+    ]
+  },
+  partition_keys: [
+    { name: "year", type: "string" },
+    { name: "month", type: "string" },
+    { name: "day", type: "string" },
+    { name: "hour", type: "string" }
+  ],
+  parameters: {
+    "projection.enabled" => "true",
+    "projection.year.type" => "integer", 
+    "projection.year.range" => "2020,2030",
+    "projection.month.type" => "integer",
+    "projection.month.range" => "1,12", 
+    "projection.day.type" => "integer",
+    "projection.day.range" => "1,31",
+    "projection.hour.type" => "integer",
+    "projection.hour.range" => "0,23"
+  }
+})
+
+# Multi-dimensional partitioning for business data
+business_table = aws_glue_catalog_table(:orders, {
+  name: "order_transactions",
+  database_name: "business",
+  storage_descriptor: {
+    location: "s3://business-data/orders/",
+    columns: [
+      { name: "order_id", type: "string" },
+      { name: "customer_id", type: "string" },
+      { name: "amount", type: "decimal(12,2)" },
+      { name: "status", type: "string" }
+    ]
+  },
+  partition_keys: [
+    { name: "region", type: "string", comment: "Geographic region" },
+    { name: "order_date", type: "string", comment: "Date in YYYY-MM-DD format" }
+  ]
+})
+```
+
+### Format Optimization
+```ruby
+# High-frequency write table (streaming ingestion)
+streaming_table = aws_glue_catalog_table(:stream_events, {
+  name: "real_time_events",
+  database_name: "streaming",
+  storage_descriptor: {
+    location: "s3://streaming-data/events/",
+    **GlueCatalogTableAttributes.input_output_format_for_type("json"),
+    serde_info: GlueCatalogTableAttributes.serde_info_for_format("json"),
+    columns: [
+      { name: "event_id", type: "string" },
+      { name: "payload", type: "string" },
+      { name: "timestamp", type: "timestamp" }
+    ]
+  },
+  parameters: {
+    "write_pattern" => "high_frequency",
+    "compaction_schedule" => "hourly"
+  }
+})
+
+# Analytics-optimized table (batch processing)
+analytics_table = aws_glue_catalog_table(:analytics_events, {
+  name: "analytics_events", 
+  database_name: "analytics",
+  storage_descriptor: {
+    location: "s3://analytics-data/events/",
+    **GlueCatalogTableAttributes.input_output_format_for_type("parquet"),
+    serde_info: GlueCatalogTableAttributes.serde_info_for_format("parquet"),
+    compressed: true,
+    columns: [
+      { name: "event_id", type: "string" },
+      { name: "user_id", type: "string" },
+      { name: "event_type", type: "string" },
+      { name: "properties", type: "map<string,string>" },
+      { name: "timestamp", type: "timestamp" }
+    ],
+    sort_columns: [
+      { column: "timestamp", sort_order: 1 },
+      { column: "user_id", sort_order: 1 }
+    ]
+  },
+  parameters: {
+    "parquet.compression" => "SNAPPY",
+    "write.target-file-size-bytes" => "134217728" # 128MB
+  }
+})
+```
+
+## Schema Evolution Patterns
+
+### Backward Compatible Changes
+```ruby
+# Version 1 schema
+v1_table = aws_glue_catalog_table(:user_events_v1, {
+  name: "user_events_v1",
+  database_name: "events",
+  storage_descriptor: {
+    columns: [
+      { name: "user_id", type: "string" },
+      { name: "event_type", type: "string" },
+      { name: "timestamp", type: "timestamp" }
+    ]
+  },
+  parameters: {
+    "schema_version" => "1.0",
+    "compatibility_mode" => "backward"
+  }
+})
+
+# Version 2 schema (backward compatible - adds optional fields)
+v2_table = aws_glue_catalog_table(:user_events_v2, {
+  name: "user_events_v2",
+  database_name: "events",
+  storage_descriptor: {
+    columns: [
+      { name: "user_id", type: "string" },
+      { name: "event_type", type: "string" },
+      { name: "timestamp", type: "timestamp" },
+      # New optional fields
+      { name: "session_id", type: "string", comment: "Added in v2" },
+      { name: "properties", type: "map<string,string>", comment: "Added in v2" }
+    ]
+  },
+  parameters: {
+    "schema_version" => "2.0",
+    "compatibility_mode" => "backward",
+    "previous_version" => "1.0"
+  }
+})
+```
+
+### Complex Data Type Evolution
+```ruby
+# Evolving nested structures
+evolved_table = aws_glue_catalog_table(:complex_events, {
+  name: "complex_event_data",
+  database_name: "events",
+  storage_descriptor: {
+    columns: [
+      { name: "event_id", type: "string" },
+      { name: "user_profile", type: "struct<id:string,email:string,preferences:map<string,string>>" },
+      { name: "interaction_data", type: "array<struct<type:string,target:string,timestamp:timestamp,metadata:map<string,string>>>" },
+      { name: "context", type: "struct<session_id:string,device:struct<type:string,os:string,browser:string>,location:struct<country:string,region:string,city:string>>" }
+    ]
+  },
+  parameters: {
+    "schema_registry" => "enabled",
+    "type_evolution" => "additive_only"
+  }
+})
+```
+
+## Governance and Security Integration
+
+### Data Classification
+```ruby
+sensitive_table = aws_glue_catalog_table(:customer_pii, {
+  name: "customer_personal_data",
+  database_name: "sensitive",
+  description: "Customer PII data with classification",
+  storage_descriptor: {
+    columns: [
+      { name: "customer_id", type: "string", comment: "Non-sensitive identifier" },
+      { name: "ssn", type: "string", comment: "PII-SSN" },
+      { name: "email", type: "string", comment: "PII-EMAIL" },
+      { name: "phone", type: "string", comment: "PII-PHONE" },
+      { name: "address", type: "struct<street:string,city:string,state:string,zip:string>", comment: "PII-ADDRESS" }
+    ]
+  },
+  parameters: {
+    "classification" => "pii",
+    "encryption_required" => "true",
+    "retention_policy" => "7_years",
+    "access_control" => "restricted"
+  }
+})
+
+# Anonymized view for analytics
+anonymized_view = aws_glue_catalog_table(:customer_anonymous, {
+  name: "customer_analytics_view",
+  database_name: "analytics",
+  table_type: "VIRTUAL_VIEW",
+  view_original_text: %{
+    SELECT 
+      customer_id,
+      HASH(email) as email_hash,
+      SUBSTR(phone, 1, 3) as area_code,
+      address.city,
+      address.state,
+      address.zip
+    FROM customer_personal_data
+  },
+  parameters: {
+    "privacy_level" => "anonymized",
+    "source_classification" => "pii"
+  }
+})
+```
+
+This table resource provides the schema foundation that enables sophisticated data lake architectures with proper governance, performance optimization, and multi-engine analytics capabilities.