pangea-aws 0.1.0

data/lib/pangea/resources/aws_emr_cluster/resource/dsl_builder/cluster_settings.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module EmrCluster
+        class DSLBuilder
+          # Cluster settings building methods for EMR clusters
+          module ClusterSettings
+            def build_bootstrap_actions(ctx)
+              attrs.bootstrap_action.each do |bootstrap|
+                ctx.bootstrap_action do
+                  name bootstrap[:name]
+                  path bootstrap[:path]
+                  args bootstrap[:args] if bootstrap[:args]&.any?
+                end
+              end
+            end
+
+            def build_logging(ctx)
+              ctx.log_uri attrs.log_uri if attrs.log_uri
+              return unless attrs.log_encryption_kms_key_id
+
+              ctx.log_encryption_kms_key_id attrs.log_encryption_kms_key_id
+            end
+
+            def build_cluster_behavior(ctx)
+              ctx.termination_protection attrs.termination_protection
+              ctx.keep_job_flow_alive_when_no_steps attrs.keep_job_flow_alive_when_no_steps
+              ctx.visible_to_all_users attrs.visible_to_all_users
+            end
+
+            def build_auto_termination_policy(ctx)
+              return unless attrs.auto_termination_policy
+
+              atp = attrs.auto_termination_policy
+              ctx.auto_termination_policy do
+                idle_timeout atp[:idle_timeout] if atp[:idle_timeout]
+              end
+            end
+
+            def build_custom_ami(ctx)
+              ctx.custom_ami_id attrs.custom_ami_id if attrs.custom_ami_id
+            end
+
+            def build_ebs_root_volume(ctx)
+              ctx.ebs_root_volume_size attrs.ebs_root_volume_size if attrs.ebs_root_volume_size
+            end
+
+            def build_kerberos_attributes(ctx)
+              return unless attrs.kerberos_attributes
+
+              ka = attrs.kerberos_attributes
+              ctx.kerberos_attributes do
+                kdc_admin_password ka[:kdc_admin_password]
+                realm ka[:realm]
+                ad_domain_join_password ka[:ad_domain_join_password] if ka[:ad_domain_join_password]
+                ad_domain_join_user ka[:ad_domain_join_user] if ka[:ad_domain_join_user]
+                if ka[:cross_realm_trust_principal_password]
+                  cross_realm_trust_principal_password ka[:cross_realm_trust_principal_password]
+                end
+              end
+            end
+
+            def build_step_concurrency(ctx)
+              return unless attrs.step_concurrency_level
+
+              ctx.step_concurrency_level attrs.step_concurrency_level
+            end
+
+            def build_placement_group_configs(ctx)
+              attrs.placement_group_configs.each do |placement_config|
+                ctx.placement_group_configs do
+                  instance_role placement_config[:instance_role]
+                  placement_strategy placement_config[:placement_strategy] if placement_config[:placement_strategy]
+                end
+              end
+            end
+
+            def build_tags(ctx)
+              return unless attrs.tags.any?
+
+              ctx.tags do
+                attrs.tags.each { |key, value| public_send(key, value) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_emr_cluster/resource/dsl_builder/configurations.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module EmrCluster
+        class DSLBuilder
+          # Configuration building methods for EMR clusters
+          module Configurations
+            def build_applications(ctx)
+              return unless attrs.applications.any?
+
+              attrs.applications.each do |app|
+                ctx.applications app
+              end
+            end
+
+            def build_configurations(ctx)
+              attrs.configurations.each do |config|
+                build_single_configuration(ctx, config)
+              end
+            end
+
+            private
+
+            def build_single_configuration(ctx, config)
+              builder = self
+              ctx.configurations do
+                classification config[:classification]
+                builder.send(:build_nested_configurations, self, config[:configurations])
+                builder.send(:build_properties, self, config[:properties])
+              end
+            end
+
+            def build_nested_configurations(ctx, configurations)
+              return unless configurations
+
+              builder = self
+              configurations.each do |sub_config|
+                ctx.configurations do
+                  classification sub_config[:classification] if sub_config[:classification]
+                  builder.send(:build_properties, self, sub_config[:properties])
+                end
+              end
+            end
+
+            def build_properties(ctx, properties)
+              return unless properties&.any?
+
+              ctx.properties do
+                properties.each do |key, value|
+                  public_send(key.gsub(/[^a-zA-Z0-9_]/, '_').downcase, value)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_emr_cluster/resource/dsl_builder/ec2_attributes.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module EmrCluster
+        class DSLBuilder
+          # EC2 attributes building methods for EMR clusters
+          module Ec2Attributes
+            def build_ec2_attributes(ctx)
+              ec2_attrs = attrs.ec2_attributes
+              ctx.ec2_attributes do
+                instance_profile ec2_attrs[:instance_profile]
+                key_name ec2_attrs[:key_name] if ec2_attrs[:key_name]
+
+                if ec2_attrs[:emr_managed_master_security_group]
+                  emr_managed_master_security_group ec2_attrs[:emr_managed_master_security_group]
+                end
+                if ec2_attrs[:emr_managed_slave_security_group]
+                  emr_managed_slave_security_group ec2_attrs[:emr_managed_slave_security_group]
+                end
+                if ec2_attrs[:service_access_security_group]
+                  service_access_security_group ec2_attrs[:service_access_security_group]
+                end
+
+                if ec2_attrs[:additional_master_security_groups]&.any?
+                  additional_master_security_groups ec2_attrs[:additional_master_security_groups]
+                end
+                if ec2_attrs[:additional_slave_security_groups]&.any?
+                  additional_slave_security_groups ec2_attrs[:additional_slave_security_groups]
+                end
+
+                subnet_id ec2_attrs[:subnet_id] if ec2_attrs[:subnet_id]
+                subnet_ids ec2_attrs[:subnet_ids] if ec2_attrs[:subnet_ids]&.any?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_emr_cluster/resource/dsl_builder/instance_groups.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module EmrCluster
+        class DSLBuilder
+          # Instance group building methods for EMR clusters
+          module InstanceGroups
+            def build_master_instance_group(ctx)
+              mig = attrs.master_instance_group
+              ctx.master_instance_group do
+                instance_type mig[:instance_type]
+                instance_count mig[:instance_count] if mig[:instance_count]
+              end
+            end
+
+            def build_core_instance_group(ctx)
+              return unless attrs.core_instance_group
+
+              cig = attrs.core_instance_group
+              builder = self
+              ctx.core_instance_group do
+                instance_type cig[:instance_type]
+                instance_count cig[:instance_count] if cig[:instance_count]
+                bid_price cig[:bid_price] if cig[:bid_price]
+                builder.send(:build_ebs_config, self, cig[:ebs_config])
+              end
+            end
+
+            def build_task_instance_groups(ctx)
+              attrs.task_instance_groups.each do |task_group|
+                build_single_task_group(ctx, task_group)
+              end
+            end
+
+            private
+
+            def build_single_task_group(ctx, task_group)
+              builder = self
+              ctx.task_instance_groups do
+                name task_group[:name] if task_group[:name]
+                instance_role task_group[:instance_role]
+                instance_type task_group[:instance_type]
+                instance_count task_group[:instance_count]
+                bid_price task_group[:bid_price] if task_group[:bid_price]
+                builder.send(:build_ebs_config, self, task_group[:ebs_config])
+                builder.build_auto_scaling_policy(self, task_group[:auto_scaling_policy])
+              end
+            end
+
+            def build_ebs_config(ctx, ebs_config)
+              return unless ebs_config
+
+              builder = self
+              ctx.ebs_config do
+                ebs_optimized ebs_config[:ebs_optimized] unless ebs_config[:ebs_optimized].nil?
+                builder.send(:build_ebs_block_devices, self, ebs_config[:ebs_block_device_config])
+              end
+            end
+
+            def build_ebs_block_devices(ctx, device_configs)
+              return unless device_configs&.any?
+
+              device_configs.each do |device_config|
+                ctx.ebs_block_device_config do
+                  volumes_per_instance device_config[:volumes_per_instance] if device_config[:volumes_per_instance]
+                  build_volume_specification(self, device_config[:volume_specification])
+                end
+              end
+            end
+
+            def build_volume_specification(ctx, vol_spec)
+              return unless vol_spec
+
+              ctx.volume_specification do
+                volume_type vol_spec[:volume_type]
+                size_in_gb vol_spec[:size_in_gb]
+                iops vol_spec[:iops] if vol_spec[:iops]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_emr_cluster/resource/dsl_builder.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'dsl_builder/configurations'
+require_relative 'dsl_builder/ec2_attributes'
+require_relative 'dsl_builder/instance_groups'
+require_relative 'dsl_builder/auto_scaling'
+require_relative 'dsl_builder/cluster_settings'
+
+module Pangea
+  module Resources
+    module AWS
+      module EmrCluster
+        # DSL Builder for EMR Cluster resource
+        class DSLBuilder
+          include Configurations
+          include Ec2Attributes
+          include InstanceGroups
+          include AutoScaling
+          include ClusterSettings
+
+          attr_reader :attrs
+
+          def initialize(cluster_attrs)
+            @attrs = cluster_attrs
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_emr_cluster/resource.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'pangea/resources/base'
+require 'pangea/resources/reference'
+require 'pangea/resources/aws_emr_cluster/types'
+require 'pangea/resource_registry'
+require_relative 'resource/dsl_builder'
+
+module Pangea
+  module Resources
+    module AWS
+      # Create an AWS EMR Cluster with type-safe attributes
+      #
+      # @param name [Symbol] The resource name
+      # @param attributes [Hash] EMR Cluster attributes
+      # @return [ResourceReference] Reference object with outputs and computed properties
+      def aws_emr_cluster(name, attributes = {})
+        cluster_attrs = Types::EmrClusterAttributes.new(attributes)
+        builder = EmrCluster::DSLBuilder.new(cluster_attrs)
+
+        resource(:aws_emr_cluster, name) do
+          release_label cluster_attrs.release_label
+          service_role cluster_attrs.service_role
+
+          builder.build_applications(self)
+          builder.build_configurations(self)
+          builder.build_ec2_attributes(self)
+          builder.build_master_instance_group(self)
+          builder.build_core_instance_group(self)
+          builder.build_task_instance_groups(self)
+          builder.build_bootstrap_actions(self)
+          builder.build_logging(self)
+          builder.build_cluster_behavior(self)
+          builder.build_auto_termination_policy(self)
+          builder.build_custom_ami(self)
+          builder.build_ebs_root_volume(self)
+          builder.build_kerberos_attributes(self)
+          builder.build_step_concurrency(self)
+          builder.build_placement_group_configs(self)
+          builder.build_tags(self)
+        end
+
+        build_aws_emr_cluster_resource_reference(name, cluster_attrs)
+      end
+
+      private
+
+      def build_aws_emr_cluster_resource_reference(name, cluster_attrs)
+        ResourceReference.new(
+          type: 'aws_emr_cluster',
+          name: name,
+          resource_attributes: cluster_attrs.to_h,
+          outputs: emr_outputs(name),
+          computed_properties: emr_computed_properties(cluster_attrs)
+        )
+      end
+
+      def emr_outputs(name)
+        {
+          id: "${aws_emr_cluster.#{name}.id}",
+          name: "${aws_emr_cluster.#{name}.name}",
+          arn: "${aws_emr_cluster.#{name}.arn}",
+          cluster_state: "${aws_emr_cluster.#{name}.cluster_state}",
+          master_instance_group_id: "${aws_emr_cluster.#{name}.master_instance_group[0].id}",
+          core_instance_group_id: "${aws_emr_cluster.#{name}.core_instance_group[0].id}",
+          master_public_dns: "${aws_emr_cluster.#{name}.master_public_dns}",
+          log_uri: "${aws_emr_cluster.#{name}.log_uri}",
+          applications: "${aws_emr_cluster.#{name}.applications}"
+        }
+      end
+
+      def emr_computed_properties(cluster_attrs)
+        {
+          uses_spark: cluster_attrs.uses_spark?,
+          uses_hive: cluster_attrs.uses_hive?,
+          uses_presto: cluster_attrs.uses_presto?,
+          uses_ml_frameworks: cluster_attrs.uses_ml_frameworks?,
+          uses_notebooks: cluster_attrs.uses_notebooks?,
+          is_multi_az: cluster_attrs.is_multi_az?,
+          uses_spot_instances: cluster_attrs.uses_spot_instances?,
+          has_auto_scaling: cluster_attrs.has_auto_scaling?,
+          total_core_instances: cluster_attrs.total_core_instances,
+          total_task_instances: cluster_attrs.total_task_instances,
+          total_cluster_instances: cluster_attrs.total_cluster_instances,
+          estimated_hourly_cost_usd: cluster_attrs.estimated_hourly_cost_usd,
+          configuration_warnings: cluster_attrs.configuration_warnings
+        }
+      end
+    end
+  end
+end

data/lib/pangea/resources/aws_emr_cluster/types/attributes.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Copyright 2025 The Pangea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Pangea
+  module Resources
+    module AWS
+      module Types
+        # Type-safe attributes for AWS EMR Cluster resources
+        class EmrClusterAttributes < Pangea::Resources::BaseAttributes
+          extend EmrClusterClassMethods
+          include EmrClusterInstanceMethods
+          extend EmrClusterValidation
+
+          attribute? :name, Resources::Types::String.optional
+          attribute? :release_label, Resources::Types::String.optional
+          attribute? :applications, Resources::Types::Array.of(
+            Resources::Types::String.constrained(included_in: ['Hadoop', 'Spark', 'Hive', 'Presto', 'Trino', 'HBase', 'Phoenix', 'Pig',
+                               'Sqoop', 'Oozie', 'ZooKeeper', 'Tez', 'Ganglia', 'Flume', 'MXNet',
+                               'TensorFlow', 'JupyterHub', 'Livy', 'Zeppelin'])
+          ).default(%w[Hadoop Spark].freeze)
+          attribute? :service_role, Resources::Types::String.optional
+          attribute? :configurations, Resources::Types::Array.of(
+            Resources::Types::Hash.schema(classification: Resources::Types::String, configurations?: Resources::Types::Array.of(Resources::Types::Hash).optional,
+                               properties?: Resources::Types::Hash.map(Resources::Types::String, Resources::Types::String).lax.optional)
+          ).default([].freeze)
+
+          # EC2 attributes
+          attribute? :ec2_attributes, Resources::Types::Hash.schema(
+            key_name?: Resources::Types::String.optional, instance_profile: Resources::Types::String,
+            emr_managed_master_security_group?: Resources::Types::String.optional, emr_managed_slave_security_group?: Resources::Types::String.optional,
+            service_access_security_group?: Resources::Types::String.optional, additional_master_security_groups?: Resources::Types::Array.of(Resources::Types::String).optional,
+            additional_slave_security_groups?: Resources::Types::Array.of(Resources::Types::String).optional, subnet_id?: Resources::Types::String.optional,
+            subnet_ids?: Resources::Types::Array.of(Resources::Types::String).optional
+          ).lax
+
+          # Instance groups
+          attribute? :master_instance_group, Resources::Types::Hash.schema(instance_type: Resources::Types::String, instance_count?: Resources::Types::Integer.constrained(eql: 1).lax.optional)
+          attribute? :core_instance_group, Resources::Types::Hash.schema(
+            instance_type: Resources::Types::String, instance_count?: Resources::Types::Integer.constrained(gteq: 1).optional, bid_price?: Resources::Types::String.optional,
+            ebs_config?: Resources::Types::Hash.schema(ebs_block_device_config?: Resources::Types::Array.of(Resources::Types::Hash).lax.optional, ebs_optimized?: Resources::Types::Bool.optional).optional
+          ).optional
+          attribute :task_instance_groups, Resources::Types::Array.of(Resources::Types::Hash).default([].freeze)
+
+          # Bootstrap and logging
+          attribute? :bootstrap_action, Resources::Types::Array.of(
+            Resources::Types::Hash.schema(path: Resources::Types::String, name: Resources::Types::String, args?: Resources::Types::Array.of(Resources::Types::String).lax.optional)
+          ).default([].freeze)
+          attribute? :log_uri, Resources::Types::String.optional
+          attribute? :log_encryption_kms_key_id, Resources::Types::String.optional
+
+          # Cluster behavior
+          attribute :termination_protection, Resources::Types::Bool.default(false)
+          attribute :keep_job_flow_alive_when_no_steps, Resources::Types::Bool.default(true)
+          attribute :visible_to_all_users, Resources::Types::Bool.default(true)
+          attribute? :auto_termination_policy, Resources::Types::Hash.schema(idle_timeout?: Resources::Types::Integer.optional).lax.optional
+          attribute? :custom_ami_id, Resources::Types::String.optional
+          attribute? :ebs_root_volume_size, Resources::Types::Integer.optional
+
+          # Kerberos and advanced settings
+          attribute? :kerberos_attributes, Resources::Types::Hash.schema(
+            kdc_admin_password: Resources::Types::String, realm: Resources::Types::String, ad_domain_join_password?: Resources::Types::String.optional,
+            ad_domain_join_user?: Resources::Types::String.optional, cross_realm_trust_principal_password?: Resources::Types::String.optional
+          ).lax.optional
+          attribute? :step_concurrency_level, Resources::Types::Integer.constrained(gteq: 1, lteq: 256).optional
+          attribute? :placement_group_configs, Resources::Types::Array.of(
+            Resources::Types::Hash.schema(instance_role: Resources::Types::String.constrained(included_in: ['MASTER', 'CORE', 'TASK']).lax, placement_strategy?: Resources::Types::String.constrained(included_in: ['SPREAD', 'PARTITION', 'CLUSTER']).optional)
+          ).default([].freeze)
+          attribute :tags, Resources::Types::AwsTags.default({}.freeze)
+
+          def self.new(attributes = {})
+            attrs = super(attributes)
+            validate_cluster_name(attrs.name)
+            validate_release_label(attrs.release_label)
+            validate_service_role(attrs.service_role)
+            validate_instance_profile(attrs.ec2_attributes&.dig(:instance_profile))
+            validate_log_uri(attrs.log_uri)
+            validate_subnet_config(attrs.ec2_attributes)
+            attrs
+          end
+        end
+      end
+    end
+  end
+end