itamae-plugin-recipe-spark 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c89c752cc2d79e0c7514b2ef9ce5e4579355ba68e19a22c60f4b273181a6b97
-  data.tar.gz: eedd14729fd7fdb8a5afde967984adf09e003d46a1ec874e028ead143a65f4a1
+  metadata.gz: 0d4c843d9f1451d6574eb7dc53e04514a17f83c440300d87c9ee83059f407e5d
+  data.tar.gz: 04b68e26473e7d794ac5ddf095e7f6cdf2cc63462c417d6de48a83394ee579c1
 SHA512:
-  metadata.gz: e6a0b3361bb5be777eadcd1e811989b941d5c0754fca305fb230168e69fca936364a7e2231b9b16b86deb546750b2eea983ae03072640b7d07965eb5557d3a82
-  data.tar.gz: d1ddab8c7a5c5a96793578f993b234651e767dd9fdb5ec809678336f5a537830258edaac70a62af2fbc8873eede709e08c91477e34032e207035f1e8012b2a44
+  metadata.gz: 156ddc8e74c4eed60c6c8a9c9c631bac9ebadc4c734e8ee3b8b15e1f79aebc99e5adea177f0ab764ee09ac7e724564844899ffa585b2ed3af2c144316fb106c9
+  data.tar.gz: 3dfd725daaf86562d48847fd7ab9e51641a7319601b92e9d1d709e879b8768b7b82e041830581cbd82377c871dc6118857ccfd1c81a3f9a010f9a268bb19b7ee

data/lib/itamae/plugin/recipe/spark/install.rb

@@ -1,6 +1,6 @@
 version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
 hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
-hadoop_type = if Gem::Version.create(hadoop_version) == Gem::Version.create('3.3.3')
+hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.3.3')
                 '3'
               elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
                 '3.2'
@@ -25,6 +25,7 @@ minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
 redshift_jdbc_version = ENV['REDSHIFT_JDBC_VERSION'] || Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
 fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
 jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
+aws_secretsmanager_caching_java_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_CACHING_JAVA_VERSION
 aws_secretsmanager_jdbc_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_JDBC_VERSION
 execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
   cwd '/tmp'
@@ -42,6 +43,11 @@ execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
         <<-EOS
           wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
         EOS
+      elsif spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+        <<-EOS
+          wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-caching-java/#{aws_secretsmanager_caching_java_version}/aws-secretsmanager-caching-java-#{aws_secretsmanager_caching_java_version}.jar -O aws-secretsmanager-caching-java-#{aws_secretsmanager_caching_java_version}.jar
+          wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
+        EOS
       end}
   EOF
   
@@ -56,6 +62,62 @@ execute 'download aws-java-sdk' do
   not_if 'test -e /tmp/aws-java-sdk.zip'
 end
 
+
+aws_sdk_v2_version = ENV['AWS_SDK_V2_VERSION'] || Itamae::Plugin::Recipe::Spark::AWS_SDK_V2_VERSION
+aws_sdk_v2_jars = %w[
+  apache-client
+  arns
+  auth
+  aws-core
+  aws-query-protocol
+  aws-xml-protocol
+  checksums
+  checksums-spi
+  endpoints-spi
+  http-auth
+  http-auth-aws
+  http-auth-spi
+  http-client-spi
+  identity-spi
+  json-utils
+  metrics-spi
+  profiles
+  protocol-core
+  regions
+  retries
+  retries-spi
+  s3
+  s3-transfer-manager
+  sdk-core
+  third-party-jackson-core
+  utils
+]
+reactive_streams_version = Itamae::Plugin::Recipe::Spark::REACTIVE_STREAMS_VERSION
+if spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+  directory '/tmp/aws_java_sdk_v2' do
+    action :create
+  end
+
+  aws_sdk_v2_jars.each do |jar|
+    execute "download aws java sdk v2 #{jar} jar" do
+      cwd '/tmp/aws_java_sdk_v2'
+      command <<-EOF
+        wget -q https://repo1.maven.org/maven2/software/amazon/awssdk/#{jar}/#{aws_sdk_v2_version}/#{jar}-#{aws_sdk_v2_version}.jar
+      EOF
+      not_if "test -e /tmp/aws_java_sdk_v2/#{jar}-#{aws_sdk_v2_version}.jar"
+    end
+  end
+
+  execute "download reactive_streams jar" do
+    cwd '/tmp/aws_java_sdk_v2'
+    command <<-EOF
+      wget -q https://repo1.maven.org/maven2/org/reactivestreams/reactive-streams/#{reactive_streams_version}/reactive-streams-#{reactive_streams_version}.jar
+    EOF
+    not_if "test -e /tmp/aws_java_sdk_v2/reactive-streams-#{reactive_streams_version}.jar"
+  end
+end
+
+
 execute 'unzip aws-java-sdk' do
   cwd '/tmp'
   command <<-EOF
@@ -94,6 +156,28 @@ execute 'install aws java sdk jar' do
   not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/aws-java-sdk-*.jar"
 end
 
+if spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+  aws_sdk_v2_jars.each do |jar|
+    execute "install aws java sdk v2 #{jar} jar" do
+      cwd '/tmp/aws_java_sdk_v2'
+      command <<-EOF
+        cp -f #{jar}-#{aws_sdk_v2_version}.jar \
+            /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
+      EOF
+      not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/#{jar}-#{aws_sdk_v2_version}.jar"
+    end
+  end
+
+  execute "install reactive_streams jar" do
+    cwd '/tmp/aws_java_sdk_v2'
+    command <<-EOF
+      cp -f reactive-streams-#{reactive_streams_version}.jar \
+          /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
+    EOF
+    not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/reactive-streams-#{reactive_streams_version}.jar"
+  end
+end
+
 execute 'install hadoop aws jar' do
   cwd '/opt/hadoop/current'
   command <<-EOF
@@ -103,6 +187,36 @@ execute 'install hadoop aws jar' do
   not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/hadoop-aws-*.jar"
 end
 
+if spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+  source = '/opt/hadoop/current/share/hadoop/client'
+  destination = "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars"
+  jar_name = "hadoop-client-api-#{hadoop_version}.jar"
+  
+  execute "remove old hadoop client api jars" do
+    cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars"
+    command <<-EOF
+      for file in hadoop-client-api-*.jar; do
+        if [[ -e $file ]]; then
+          file_version=${file#hadoop-client-api-}
+          file_version=${file_version%.jar}
+          if [[ "$(printf '%s\n%s' "#{hadoop_version}" "${file_version}" | sort -V | head -n1)" != "#{hadoop_version}" ]]; then
+            rm -f $file
+          fi
+        fi
+      done
+    EOF
+    only_if "ls /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/hadoop-client-api-*.jar"
+  end
+  
+  execute "install hadoop client api jar" do
+    cwd source
+    command <<-EOF
+      cp -f #{jar_name} #{destination}
+    EOF
+    not_if "test -e #{destination}/#{jar_name}"
+  end
+end
+
 execute 'install spark-redshift jars' do
   cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
   command <<-EOF
@@ -128,6 +242,14 @@ execute 'install spark-redshift jars' do
         EOS
       elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
         <<-EOS
+          ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
+        EOS
+      elsif spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+        <<-EOS
+          ls -d $(find jars) | grep 'aws-secretsmanager-caching-java-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/aws-secretsmanager-caching-java-#{aws_secretsmanager_caching_java_version}.jar jars/
+
           ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
           cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
         EOS

data/lib/itamae/plugin/recipe/spark/spark-3.5.5_hadoop_3_sha256.txt

@@ -0,0 +1 @@
+8daa3f7fb0af2670fe11beb8a2ac79d908a534d7298353ec4746025b102d5e31  spark-3.5.5-bin-hadoop3.tgz

data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.4.3-spark_3.5_sha256.txt

@@ -0,0 +1,5 @@
+43d428ca25610fd3e7800c6d2270d1f27eacb6af1936c5e0641ad5d82abc65b9  spark-redshift_2.12-6.4.3-spark_3.5.jar
+16815c7ae6e68eb8f097c3987737689a20fb9cd8718f1c6cce72b76008a94d06  spark-avro_2.12-3.5.5.jar
+b30825a09c05293a79c8839bfef172901a9dcb7a46ec19559409d0e6eb7b5299  RedshiftJDBC42-2.1.0.29.jar
+efa849f894b83fbef1f2c27c72b8e4a581051f106338f94bd027810fc646ae85  aws-secretsmanager-caching-java-1.0.2.jar
+b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd  aws-secretsmanager-jdbc-1.0.12.jar

data/lib/itamae/plugin/recipe/spark/version.rb

@@ -2,21 +2,25 @@ module Itamae
   module Plugin
     module Recipe
       module Spark
-        VERSION = "0.1.11"
+        VERSION = "0.1.12"
 
         SPARK_VERSION = [
           SPARK_VERSION_MAJOR = '3',
-          SPARK_VERSION_MINOR = '4',
-          SPARK_VERSION_REVISION = '1'
+          SPARK_VERSION_MINOR = '5',
+          SPARK_VERSION_REVISION = '5'
         ].join('.')
 
-        SPARK_REDSHIFT_VERSION = '2.12-6.2.0-spark_3.4'
-        SPARK_AVRO_VERSION = '2.12-3.4.2'
+        SPARK_REDSHIFT_VERSION = '2.12-6.4.3-spark_3.5'
+        SPARK_AVRO_VERSION = '2.12-3.5.5'
         MINIMAL_JSON_VERSION = '0.9.4'
-        REDSHIFT_JDBC_VERSION = '2.1.0.24'
+        REDSHIFT_JDBC_VERSION = '2.1.0.29'
         FASTDOUBLEPARSER_VERSION = '0.8.0'
         JETS3T_VERSION = '0.9.4'
+        AWS_SECRETSMANAGER_CACHING_JAVA_VERSION = '1.0.2'
         AWS_SECRETSMANAGER_JDBC_VERSION = '1.0.12'
+
+        AWS_SDK_V2_VERSION = '2.31.78'
+        REACTIVE_STREAMS_VERSION = '1.0.4'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: itamae-plugin-recipe-spark
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - ichylinux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-17 00:00:00.000000000 Z
+date: 2025-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: itamae
@@ -107,8 +107,10 @@ files:
 - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.5.5_hadoop_3_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.4.3-spark_3.5_sha256.txt
 - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
 - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
 - lib/itamae/plugin/recipe/spark/version.rb
@@ -131,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: itamae recipe for apache spark installation