itamae-plugin-recipe-spark 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 201de04bbae7c37fcb100e26b798361d35f7064b2809e01e1a34b1efb2e0768b
-  data.tar.gz: 92cf7328ac3cd04c97f948e6fb8ccb10e114040d03ac50bc8556ecb8b60e5d9b
+  metadata.gz: 65e26a2e185e47252ef2e7bc92c7aab513cb19266ed2ad11d78764a2306ccb3d
+  data.tar.gz: fc71a105d5189e80525d25839d65bfacb6978e24a5878e809672522dcd10557e
 SHA512:
-  metadata.gz: 90e5650f5f509b8f29d2bde84b16df9f11278f8b9d2ff3f099e920f1728423c957dea0d29b698d229a8ec32dfe4196d4e3ae32aa8f5e53126929b1b6e6c0ad3f
-  data.tar.gz: 7f84c85db7a99f58fe1036483dc81157b2eba1f1f857b612ab26607110f9bf09cd75c94df74376c5c0f701ad830d2b38c987d5b4bfb6c97535780c8ffc0c0122
+  metadata.gz: 925e77e886ea6a4bf482a88e496ee5e408bfa1284d5b355178d1034a7df61e5ddee1a23b047660105768c194f01c3c0bf800f397ee3eceb2720996447c69ab2d
+  data.tar.gz: 2afc8f9b8f5fa3401f92513af200d38a4d3bc2839e3d973ad0663ab5f340f58514e3db44efb9a5735be5bc3399271f389b12584aa6b3a6cd9b3608b462527cb8

data/lib/itamae/plugin/recipe/spark/install.rb

@@ -17,6 +17,23 @@ execute "download spark-#{version}" do
   not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
 end
 
+spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
+spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
+minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
+redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
+jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
+execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
+  cwd '/tmp'
+  command <<-EOF
+    wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
+    wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
+    wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
+    wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/RedshiftJDBC42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
+    wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
+  EOF
+  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
+end
+
 directory '/opt/spark' do
   user 'root'
   owner 'root'
@@ -47,6 +64,27 @@ execute 'install hadoop aws jars' do
   not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
 end
 
+execute 'install spark-redshift jars' do
+  cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
+  command <<-EOF
+    ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
+    cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
+    cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
+    cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
+    cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
+        jars/
+    ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
+    cp -f /tmp/jets3t-#{jets3t_version}.jar \
+        jars/
+  EOF
+end
+
 template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/spark-defaults.conf"
 
 if ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']

data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+3a79e324d12f46de44d042641d9340ba03f8ccb3db6f2496a9ccb65431dbb593  spark-3.1.2-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt

@@ -0,0 +1 @@
+0d9cf9dbbb3b4215afebe7fa4748b012e406dd1f1ad2a61b993ac04adcb94eaa  spark-3.1.2-bin-hadoop3.2.tgz

data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt

@@ -0,0 +1,5 @@
+a4ded0f2806b50eef02885d2543b16d2f0654744e8f1f45dc913f74b22071b93  spark-redshift_2.12-5.0.3.jar
+b1ffe6dbfb9a6492d9e5ceed645ee49bb5f3ab2e48a3836ee8d91a892995436b  spark-avro_2.12-3.0.1.jar
+e1d280900c78f18ae2e00c14e7410a77ba19cf084154b386532846aa6dc81721  minimal-json-0.9.4.jar
+a27fd9e446021557c17ca3697d67d1c1857ce7e92a5c80c1038b767b835ad841  RedshiftJDBC42-1.2.37.1061.jar
+a03f0d25e810ecff30137e0d3148d1bf9afdcf62357ebd1b1af338f7f6115bbb  jets3t-0.9.4.jar

data/lib/itamae/plugin/recipe/spark/version.rb

@@ -2,14 +2,19 @@ module Itamae
   module Plugin
     module Recipe
       module Spark
-        VERSION = "0.1.5"
-        
+        VERSION = "0.1.6"
+
         SPARK_VERSION = [
           SPARK_VERSION_MAJOR = '3',
-          SPARK_VERSION_MINOR = '0',
-          SPARK_VERSION_REVISION = '1'
+          SPARK_VERSION_MINOR = '1',
+          SPARK_VERSION_REVISION = '2'
         ].join('.')
 
+        SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
+        SPARK_AVRO_VERSION = '2.12-3.0.1'
+        MINIMAL_JSON_VERSION = '0.9.4'
+        REDSHIFT_JDBC_VERSION = '1.2.37.1061'
+        JETS3T_VERSION = '0.9.4'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: itamae-plugin-recipe-spark
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - ichylinux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-09 00:00:00.000000000 Z
+date: 2022-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: itamae
@@ -103,6 +103,9 @@ files:
 - lib/itamae/plugin/recipe/spark/spark-2.4.7_hadoop_2.7_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_2.7_sha256.txt
 - lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
 - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
 - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
 - lib/itamae/plugin/recipe/spark/version.rb
@@ -125,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.16
 signing_key: 
 specification_version: 4
 summary: itamae recipe for apache spark installation