itamae-plugin-recipe-spark 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a942c57a61e3be0a5d65916011b4c4a93ccce2665152f480cb4a5d79fd07a415
-  data.tar.gz: 1f8bf34d5120a62bedd9f245dff03a62956e31e17d90839cda8ec68df478b4e6
+  metadata.gz: 201de04bbae7c37fcb100e26b798361d35f7064b2809e01e1a34b1efb2e0768b
+  data.tar.gz: 92cf7328ac3cd04c97f948e6fb8ccb10e114040d03ac50bc8556ecb8b60e5d9b
 SHA512:
-  metadata.gz: d2a4069326bbcdccf2e57a6470299d5111576c43aa7614527b24ec89abd81939adf830a55033e227caeb4d429e6a26d93fd8382bfbe18919d92374f91530da15
-  data.tar.gz: 945aa06cf08dbf9523cbb9e339d8e33073127c66b0eff1abf230ebdbc33096bc93391f98a34ae4272dd1d3e4dda32eead921814a6f40322c1c8a4486be5ef69b
+  metadata.gz: 90e5650f5f509b8f29d2bde84b16df9f11278f8b9d2ff3f099e920f1728423c957dea0d29b698d229a8ec32dfe4196d4e3ae32aa8f5e53126929b1b6e6c0ad3f
+  data.tar.gz: 7f84c85db7a99f58fe1036483dc81157b2eba1f1f857b612ab26607110f9bf09cd75c94df74376c5c0f701ad830d2b38c987d5b4bfb6c97535780c8ffc0c0122

data/lib/itamae/plugin/recipe/spark/install.rb

@@ -1,13 +1,20 @@
 version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
 hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
+hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
+                '3.2'
+              elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('2.7')
+                '2.7'
+              else
+                raise "Hadoop version #{hadoop_version} is not supported."
+              end
 
 execute "download spark-#{version}" do
   cwd '/tmp'
   command <<-EOF
-    rm -f spark-#{version}-bin-hadoop2.7.tgz
-    wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop2.7.tgz
+    rm -f spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+    wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop#{hadoop_type}.tgz
   EOF
-  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_sha256.txt")).strip} | sha256sum -c"
+  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
 end
 
 directory '/opt/spark' do
@@ -20,37 +27,37 @@ end
 execute "install spark-#{version}" do
   cwd '/tmp'
   command <<-EOF
-    rm -Rf spark-#{version}-bin-hadoop2.7/
-    tar zxf spark-#{version}-bin-hadoop2.7.tgz
-    sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop2.7
-    sudo mv spark-#{version}-bin-hadoop2.7 /opt/spark/
-    sudo touch /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED
+    rm -Rf spark-#{version}-bin-hadoop#{hadoop_type}/
+    tar zxf spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+    sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}
+    sudo mv spark-#{version}-bin-hadoop#{hadoop_type} /opt/spark/
+    sudo touch /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED
   EOF
-  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED"
+  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED"
 end
 
 execute 'install hadoop aws jars' do
   cwd '/opt/hadoop/current'
   command <<-EOF
     cp -f share/hadoop/tools/lib/aws-java-sdk-*.jar \
-        /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+        /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
     cp -f share/hadoop/tools/lib/hadoop-aws-#{hadoop_version}.jar \
-        /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+        /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
   EOF
-  not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop2.7/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
+  not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
 end
 
-template "/opt/spark/spark-#{version}-bin-hadoop2.7/conf/spark-defaults.conf"
+template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/spark-defaults.conf"
 
 if ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
-  template "/opt/spark/spark-#{version}-bin-hadoop2.7/conf/hdfs-site.xml" do
+  template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/hdfs-site.xml" do
     variables aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
               aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
   end
 end
 
 link '/opt/spark/current' do
-  to "/opt/spark/spark-#{version}-bin-hadoop2.7"
+  to "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
   user 'root'
   force true
 end

data/lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt

@@ -0,0 +1 @@
+e2d05efa1c657dd5180628a83ea36c97c00f972b4aee935b7affa2e1058b0279  spark-3.0.1-bin-hadoop3.2.tgz

data/lib/itamae/plugin/recipe/spark/version.rb

@@ -2,7 +2,7 @@ module Itamae
   module Plugin
     module Recipe
       module Spark
-        VERSION = "0.1.4"
+        VERSION = "0.1.5"
         
         SPARK_VERSION = [
           SPARK_VERSION_MAJOR = '3',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: itamae-plugin-recipe-spark
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - ichylinux
@@ -92,16 +92,17 @@ files:
 - itamae-plugin-recipe-spark.gemspec
 - lib/itamae/plugin/recipe/spark.rb
 - lib/itamae/plugin/recipe/spark/install.rb
-- lib/itamae/plugin/recipe/spark/spark-2.2.0_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.3.1_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.3.2_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.0_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.3_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.4_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.5_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.6_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.7_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-3.0.1_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.2.0_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.3.1_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.3.2_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.0_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.3_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.4_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.5_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.6_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.7_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt
 - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
 - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
 - lib/itamae/plugin/recipe/spark/version.rb