itamae-plugin-recipe-spark 0.1.9 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4a57bd0c4c43b94a896f87b20b4580c10ffe76e2995af89ae2a60e44963554c
-  data.tar.gz: f61d1f73264122bd76db928b6d85b261fec7ee754cdd06b94f683ec20a08f2f7
+  metadata.gz: 0d4c843d9f1451d6574eb7dc53e04514a17f83c440300d87c9ee83059f407e5d
+  data.tar.gz: 04b68e26473e7d794ac5ddf095e7f6cdf2cc63462c417d6de48a83394ee579c1
 SHA512:
-  metadata.gz: 49e7761e3f424ce810e3bd423e4ade4cf122122fbdff80dffc7d200573bea3d31b19a367e8648fac00bf80796ec23b34a171e24564d94f2579dd9fa64c95278f
-  data.tar.gz: 343ffa8a4f31b83bdf2543185c3a65832b785cb9734de40c10866cb7d38b7f851db1f7462f91872252dd59291376a7bc56578fc18aaf25d8769f94df0d72e4b9
+  metadata.gz: 156ddc8e74c4eed60c6c8a9c9c631bac9ebadc4c734e8ee3b8b15e1f79aebc99e5adea177f0ab764ee09ac7e724564844899ffa585b2ed3af2c144316fb106c9
+  data.tar.gz: 3dfd725daaf86562d48847fd7ab9e51641a7319601b92e9d1d709e879b8768b7b82e041830581cbd82377c871dc6118857ccfd1c81a3f9a010f9a268bb19b7ee

data/lib/itamae/plugin/recipe/spark/install.rb

@@ -1,6 +1,6 @@
 version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
 hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
-hadoop_type = if Gem::Version.create(hadoop_version) == Gem::Version.create('3.3.3')
+hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.3.3')
                 '3'
               elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
                 '3.2'
@@ -19,22 +19,38 @@ execute "download spark-#{version}" do
   not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
 end
 
-spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
-spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
+spark_redshift_version = ENV['SPARK_REDSHIFT_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
+spark_avro_version = ENV['SPARK_AVRO_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
 minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
-redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
+redshift_jdbc_version = ENV['REDSHIFT_JDBC_VERSION'] || Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
 fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
 jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
+aws_secretsmanager_caching_java_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_CACHING_JAVA_VERSION
+aws_secretsmanager_jdbc_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_JDBC_VERSION
 execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
   cwd '/tmp'
   command <<-EOF
-    wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
+    wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-', 2).first}/#{spark_redshift_version.split('-', 2).last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
     wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
-    wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
-    wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
-    wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
-    wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
+    wget -q https://repo1.maven.org/maven2/com/amazon/redshift/redshift-jdbc42/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
+    #{if spark_redshift_version.split('-', 2).last == '5.0.3'
+        <<-EOS
+          wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
+          wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
+          wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
+        EOS
+      elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
+        <<-EOS
+          wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
+        EOS
+      elsif spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+        <<-EOS
+          wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-caching-java/#{aws_secretsmanager_caching_java_version}/aws-secretsmanager-caching-java-#{aws_secretsmanager_caching_java_version}.jar -O aws-secretsmanager-caching-java-#{aws_secretsmanager_caching_java_version}.jar
+          wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
+        EOS
+      end}
   EOF
+  
   not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
 end
 
@@ -46,6 +62,62 @@ execute 'download aws-java-sdk' do
   not_if 'test -e /tmp/aws-java-sdk.zip'
 end
 
+
+aws_sdk_v2_version = ENV['AWS_SDK_V2_VERSION'] || Itamae::Plugin::Recipe::Spark::AWS_SDK_V2_VERSION
+aws_sdk_v2_jars = %w[
+  apache-client
+  arns
+  auth
+  aws-core
+  aws-query-protocol
+  aws-xml-protocol
+  checksums
+  checksums-spi
+  endpoints-spi
+  http-auth
+  http-auth-aws
+  http-auth-spi
+  http-client-spi
+  identity-spi
+  json-utils
+  metrics-spi
+  profiles
+  protocol-core
+  regions
+  retries
+  retries-spi
+  s3
+  s3-transfer-manager
+  sdk-core
+  third-party-jackson-core
+  utils
+]
+reactive_streams_version = Itamae::Plugin::Recipe::Spark::REACTIVE_STREAMS_VERSION
+if spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+  directory '/tmp/aws_java_sdk_v2' do
+    action :create
+  end
+
+  aws_sdk_v2_jars.each do |jar|
+    execute "download aws java sdk v2 #{jar} jar" do
+      cwd '/tmp/aws_java_sdk_v2'
+      command <<-EOF
+        wget -q https://repo1.maven.org/maven2/software/amazon/awssdk/#{jar}/#{aws_sdk_v2_version}/#{jar}-#{aws_sdk_v2_version}.jar
+      EOF
+      not_if "test -e /tmp/aws_java_sdk_v2/#{jar}-#{aws_sdk_v2_version}.jar"
+    end
+  end
+
+  execute "download reactive_streams jar" do
+    cwd '/tmp/aws_java_sdk_v2'
+    command <<-EOF
+      wget -q https://repo1.maven.org/maven2/org/reactivestreams/reactive-streams/#{reactive_streams_version}/reactive-streams-#{reactive_streams_version}.jar
+    EOF
+    not_if "test -e /tmp/aws_java_sdk_v2/reactive-streams-#{reactive_streams_version}.jar"
+  end
+end
+
+
 execute 'unzip aws-java-sdk' do
   cwd '/tmp'
   command <<-EOF
@@ -84,6 +156,28 @@ execute 'install aws java sdk jar' do
   not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/aws-java-sdk-*.jar"
 end
 
+if spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+  aws_sdk_v2_jars.each do |jar|
+    execute "install aws java sdk v2 #{jar} jar" do
+      cwd '/tmp/aws_java_sdk_v2'
+      command <<-EOF
+        cp -f #{jar}-#{aws_sdk_v2_version}.jar \
+            /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
+      EOF
+      not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/#{jar}-#{aws_sdk_v2_version}.jar"
+    end
+  end
+
+  execute "install reactive_streams jar" do
+    cwd '/tmp/aws_java_sdk_v2'
+    command <<-EOF
+      cp -f reactive-streams-#{reactive_streams_version}.jar \
+          /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
+    EOF
+    not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/reactive-streams-#{reactive_streams_version}.jar"
+  end
+end
+
 execute 'install hadoop aws jar' do
   cwd '/opt/hadoop/current'
   command <<-EOF
@@ -93,27 +187,73 @@ execute 'install hadoop aws jar' do
   not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/hadoop-aws-*.jar"
 end
 
+if spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+  source = '/opt/hadoop/current/share/hadoop/client'
+  destination = "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars"
+  jar_name = "hadoop-client-api-#{hadoop_version}.jar"
+  
+  execute "remove old hadoop client api jars" do
+    cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars"
+    command <<-EOF
+      for file in hadoop-client-api-*.jar; do
+        if [[ -e $file ]]; then
+          file_version=${file#hadoop-client-api-}
+          file_version=${file_version%.jar}
+          if [[ "$(printf '%s\n%s' "#{hadoop_version}" "${file_version}" | sort -V | head -n1)" != "#{hadoop_version}" ]]; then
+            rm -f $file
+          fi
+        fi
+      done
+    EOF
+    only_if "ls /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/hadoop-client-api-*.jar"
+  end
+  
+  execute "install hadoop client api jar" do
+    cwd source
+    command <<-EOF
+      cp -f #{jar_name} #{destination}
+    EOF
+    not_if "test -e #{destination}/#{jar_name}"
+  end
+end
+
 execute 'install spark-redshift jars' do
   cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
   command <<-EOF
     ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
-    cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
-        jars/
+    cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar jars/
+    
     ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
-    cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
-        jars/
-    ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
-    cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
-        jars/
+    cp -f /tmp/spark-avro_#{spark_avro_version}.jar jars/
+    
     ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
-    cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
-        jars/
-    ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
-    cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar \
-        jars/
-    ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
-    cp -f /tmp/jets3t-#{jets3t_version}.jar \
-        jars/
+    cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar jars/
+    
+    #{if spark_redshift_version.split('-', 2).last == '5.0.3'
+        <<-EOS
+          ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/minimal-json-#{minimal_json_version}.jar jars/
+      
+          ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar jars/
+      
+          ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/jets3t-#{jets3t_version}.jar jars/
+        EOS
+      elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
+        <<-EOS
+          ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
+        EOS
+      elsif spark_redshift_version.split('-', 2).last == '6.4.3-spark_3.5'
+        <<-EOS
+          ls -d $(find jars) | grep 'aws-secretsmanager-caching-java-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/aws-secretsmanager-caching-java-#{aws_secretsmanager_caching_java_version}.jar jars/
+
+          ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
+          cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
+        EOS
+      end}
   EOF
 end
 

data/lib/itamae/plugin/recipe/spark/spark-3.5.5_hadoop_3_sha256.txt

@@ -0,0 +1 @@
+8daa3f7fb0af2670fe11beb8a2ac79d908a534d7298353ec4746025b102d5e31  spark-3.5.5-bin-hadoop3.tgz

data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt

@@ -0,0 +1,4 @@
+97d829e5e47e0b0306bc54549fbfc67516bea15d249581eeb1152f155fdbee49  spark-redshift_2.12-6.2.0-spark_3.4.jar
+ed252b44d546a3df7778debf563eeb535ae07d74c3c18d4163fcbad88cd6ffbf  spark-avro_2.12-3.4.2.jar
+dd12dc400549fb055b64b186b7bb05f3e31d417f398fca6755a02394d0302bae  RedshiftJDBC42-2.1.0.24.jar
+b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd  aws-secretsmanager-jdbc-1.0.12.jar

data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.4.3-spark_3.5_sha256.txt

@@ -0,0 +1,5 @@
+43d428ca25610fd3e7800c6d2270d1f27eacb6af1936c5e0641ad5d82abc65b9  spark-redshift_2.12-6.4.3-spark_3.5.jar
+16815c7ae6e68eb8f097c3987737689a20fb9cd8718f1c6cce72b76008a94d06  spark-avro_2.12-3.5.5.jar
+b30825a09c05293a79c8839bfef172901a9dcb7a46ec19559409d0e6eb7b5299  RedshiftJDBC42-2.1.0.29.jar
+efa849f894b83fbef1f2c27c72b8e4a581051f106338f94bd027810fc646ae85  aws-secretsmanager-caching-java-1.0.2.jar
+b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd  aws-secretsmanager-jdbc-1.0.12.jar

data/lib/itamae/plugin/recipe/spark/version.rb

@@ -2,20 +2,25 @@ module Itamae
   module Plugin
     module Recipe
       module Spark
-        VERSION = "0.1.9"
+        VERSION = "0.1.12"
 
         SPARK_VERSION = [
           SPARK_VERSION_MAJOR = '3',
-          SPARK_VERSION_MINOR = '4',
-          SPARK_VERSION_REVISION = '1'
+          SPARK_VERSION_MINOR = '5',
+          SPARK_VERSION_REVISION = '5'
         ].join('.')
 
-        SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
-        SPARK_AVRO_VERSION = '2.12-3.0.1'
+        SPARK_REDSHIFT_VERSION = '2.12-6.4.3-spark_3.5'
+        SPARK_AVRO_VERSION = '2.12-3.5.5'
         MINIMAL_JSON_VERSION = '0.9.4'
-        REDSHIFT_JDBC_VERSION = '2.1.0.17'
+        REDSHIFT_JDBC_VERSION = '2.1.0.29'
         FASTDOUBLEPARSER_VERSION = '0.8.0'
         JETS3T_VERSION = '0.9.4'
+        AWS_SECRETSMANAGER_CACHING_JAVA_VERSION = '1.0.2'
+        AWS_SECRETSMANAGER_JDBC_VERSION = '1.0.12'
+
+        AWS_SDK_V2_VERSION = '2.31.78'
+        REACTIVE_STREAMS_VERSION = '1.0.4'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: itamae-plugin-recipe-spark
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.12
 platform: ruby
 authors:
 - ichylinux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-16 00:00:00.000000000 Z
+date: 2025-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: itamae
@@ -107,7 +107,10 @@ files:
 - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.5.5_hadoop_3_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.4.3-spark_3.5_sha256.txt
 - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
 - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
 - lib/itamae/plugin/recipe/spark/version.rb