itamae-plugin-recipe-spark 0.1.2 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 605132bc36830fea99471ce83f20ab11d186b10f5d83e6762b9b27090ef2417c
-  data.tar.gz: d65298d73321e36b7ff8a896f097eb48d7c33287f72598e1e677d87b2ed28d67
+  metadata.gz: 65e26a2e185e47252ef2e7bc92c7aab513cb19266ed2ad11d78764a2306ccb3d
+  data.tar.gz: fc71a105d5189e80525d25839d65bfacb6978e24a5878e809672522dcd10557e
 SHA512:
-  metadata.gz: 20d8e643ccbbb54a3a37d323f15a2c07521afdeaa5fd4e94cf470f77a7afae9aeda927fbce8784c76284d5bdcc913db2135b48970fbee35d36afb20a04c3f703
-  data.tar.gz: bd801b5977fcbaf3a6639a8587eccfd3388480727a69a94255701388b96914a15be4b7072b59b3998e7f2d7b92195c2f8a080e628b57e0c898e63aef65d87395
+  metadata.gz: 925e77e886ea6a4bf482a88e496ee5e408bfa1284d5b355178d1034a7df61e5ddee1a23b047660105768c194f01c3c0bf800f397ee3eceb2720996447c69ab2d
+  data.tar.gz: 2afc8f9b8f5fa3401f92513af200d38a4d3bc2839e3d973ad0663ab5f340f58514e3db44efb9a5735be5bc3399271f389b12584aa6b3a6cd9b3608b462527cb8

data/lib/itamae/plugin/recipe/spark/install.rb

@@ -1,13 +1,37 @@
 version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
 hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
+hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
+                '3.2'
+              elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('2.7')
+                '2.7'
+              else
+                raise "Hadoop version #{hadoop_version} is not supported."
+              end
 
 execute "download spark-#{version}" do
   cwd '/tmp'
   command <<-EOF
-    rm -f spark-#{version}-bin-hadoop2.7.tgz
-    wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop2.7.tgz
+    rm -f spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+    wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop#{hadoop_type}.tgz
   EOF
-  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_sha256.txt")).strip} | sha256sum -c"
+  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
+end
+
+spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
+spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
+minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
+redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
+jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
+execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
+  cwd '/tmp'
+  command <<-EOF
+    wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
+    wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
+    wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
+    wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/RedshiftJDBC42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
+    wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
+  EOF
+  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
 end
 
 directory '/opt/spark' do
@@ -20,37 +44,58 @@ end
 execute "install spark-#{version}" do
   cwd '/tmp'
   command <<-EOF
-    rm -Rf spark-#{version}-bin-hadoop2.7/
-    tar zxf spark-#{version}-bin-hadoop2.7.tgz
-    sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop2.7
-    sudo mv spark-#{version}-bin-hadoop2.7 /opt/spark/
-    sudo touch /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED
+    rm -Rf spark-#{version}-bin-hadoop#{hadoop_type}/
+    tar zxf spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+    sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}
+    sudo mv spark-#{version}-bin-hadoop#{hadoop_type} /opt/spark/
+    sudo touch /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED
   EOF
-  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED"
+  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED"
 end
 
 execute 'install hadoop aws jars' do
   cwd '/opt/hadoop/current'
   command <<-EOF
     cp -f share/hadoop/tools/lib/aws-java-sdk-*.jar \
-        /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+        /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
     cp -f share/hadoop/tools/lib/hadoop-aws-#{hadoop_version}.jar \
-        /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+        /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
+  EOF
+  not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
+end
+
+execute 'install spark-redshift jars' do
+  cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
+  command <<-EOF
+    ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
+    cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
+    cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
+    cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
+    cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
+    cp -f /tmp/jets3t-#{jets3t_version}.jar \
+        jars/
   EOF
-  not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop2.7/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
 end
 
-template "/opt/spark/spark-#{version}-bin-hadoop2.7/conf/spark-defaults.conf"
+template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/spark-defaults.conf"
 
 if ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
-  template "/opt/spark/spark-#{version}-bin-hadoop2.7/conf/hdfs-site.xml" do
+  template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/hdfs-site.xml" do
     variables aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
               aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
   end
 end
 
 link '/opt/spark/current' do
-  to "/opt/spark/spark-#{version}-bin-hadoop2.7"
+  to "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
   user 'root'
   force true
 end

data/lib/itamae/plugin/recipe/spark/spark-2.4.7_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+13098490936c9931beda3acc4c30cdc5ca707acd1415eebde1030b11903934fe  spark-2.4.7-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+61543e748d225d859e3e0d488362f9dc4b6496a104ef5a4b297d341a47a49a12  spark-3.0.1-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt

@@ -0,0 +1 @@
+e2d05efa1c657dd5180628a83ea36c97c00f972b4aee935b7affa2e1058b0279  spark-3.0.1-bin-hadoop3.2.tgz

data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+3a79e324d12f46de44d042641d9340ba03f8ccb3db6f2496a9ccb65431dbb593  spark-3.1.2-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt

@@ -0,0 +1 @@
+0d9cf9dbbb3b4215afebe7fa4748b012e406dd1f1ad2a61b993ac04adcb94eaa  spark-3.1.2-bin-hadoop3.2.tgz

data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt

@@ -0,0 +1,5 @@
+a4ded0f2806b50eef02885d2543b16d2f0654744e8f1f45dc913f74b22071b93  spark-redshift_2.12-5.0.3.jar
+b1ffe6dbfb9a6492d9e5ceed645ee49bb5f3ab2e48a3836ee8d91a892995436b  spark-avro_2.12-3.0.1.jar
+e1d280900c78f18ae2e00c14e7410a77ba19cf084154b386532846aa6dc81721  minimal-json-0.9.4.jar
+a27fd9e446021557c17ca3697d67d1c1857ce7e92a5c80c1038b767b835ad841  RedshiftJDBC42-1.2.37.1061.jar
+a03f0d25e810ecff30137e0d3148d1bf9afdcf62357ebd1b1af338f7f6115bbb  jets3t-0.9.4.jar

data/lib/itamae/plugin/recipe/spark/version.rb

@@ -2,14 +2,19 @@ module Itamae
   module Plugin
     module Recipe
       module Spark
-        VERSION = "0.1.2"
-        
+        VERSION = "0.1.6"
+
         SPARK_VERSION = [
-          SPARK_VERSION_MAJOR = '2',
-          SPARK_VERSION_MINOR = '4',
-          SPARK_VERSION_REVISION = '6'
+          SPARK_VERSION_MAJOR = '3',
+          SPARK_VERSION_MINOR = '1',
+          SPARK_VERSION_REVISION = '2'
         ].join('.')
 
+        SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
+        SPARK_AVRO_VERSION = '2.12-3.0.1'
+        MINIMAL_JSON_VERSION = '0.9.4'
+        REDSHIFT_JDBC_VERSION = '1.2.37.1061'
+        JETS3T_VERSION = '0.9.4'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: itamae-plugin-recipe-spark
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.6
 platform: ruby
 authors:
 - ichylinux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-24 00:00:00.000000000 Z
+date: 2022-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: itamae
@@ -92,14 +92,20 @@ files:
 - itamae-plugin-recipe-spark.gemspec
 - lib/itamae/plugin/recipe/spark.rb
 - lib/itamae/plugin/recipe/spark/install.rb
-- lib/itamae/plugin/recipe/spark/spark-2.2.0_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.3.1_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.3.2_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.0_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.3_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.4_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.5_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.6_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.2.0_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.3.1_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.3.2_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.0_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.3_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.4_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.5_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.6_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.7_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
 - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
 - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
 - lib/itamae/plugin/recipe/spark/version.rb
@@ -122,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.2.16
 signing_key: 
 specification_version: 4
 summary: itamae recipe for apache spark installation