itamae-plugin-recipe-spark 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68a1f138f9520f824e99226292606ce3155c30724103b7129f30420b78a564ae
-  data.tar.gz: e244bf5b5273e19e872d03f00f397d54cb6bd3763caa3ddbb20d00b5abfb89db
+  metadata.gz: 201de04bbae7c37fcb100e26b798361d35f7064b2809e01e1a34b1efb2e0768b
+  data.tar.gz: 92cf7328ac3cd04c97f948e6fb8ccb10e114040d03ac50bc8556ecb8b60e5d9b
 SHA512:
-  metadata.gz: 41ce3768627671b0fae643d7108e56396b2c6fd3cc2f8c358e0e6edf09fc1859903654b3b9aa1d6a014b6984532a2ed37e6700d1fae851a7a415f66314b36b4a
-  data.tar.gz: a3f5dc7c49516e8c88557e6650222e67f0fb9819cb874f5756564da9da1d6d8e6f8c7f77a362d49b5c5cc7ff9ad66f3b1f572171f45c8a54fd4b071a178954d8
+  metadata.gz: 90e5650f5f509b8f29d2bde84b16df9f11278f8b9d2ff3f099e920f1728423c957dea0d29b698d229a8ec32dfe4196d4e3ae32aa8f5e53126929b1b6e6c0ad3f
+  data.tar.gz: 7f84c85db7a99f58fe1036483dc81157b2eba1f1f857b612ab26607110f9bf09cd75c94df74376c5c0f701ad830d2b38c987d5b4bfb6c97535780c8ffc0c0122

data/.gitignore

@@ -45,7 +45,7 @@ build-iPhoneSimulator/
 
 # for a library or gem, you might want to ignore these files since the code is
 # intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
+Gemfile.lock
 # .ruby-version
 # .ruby-gemset
 

data/lib/itamae/plugin/recipe/spark/install.rb

@@ -1,13 +1,20 @@
 version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
 hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
+hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
+                '3.2'
+              elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('2.7')
+                '2.7'
+              else
+                raise "Hadoop version #{hadoop_version} is not supported."
+              end
 
 execute "download spark-#{version}" do
   cwd '/tmp'
   command <<-EOF
-    rm -f spark-#{version}-bin-hadoop2.7.tgz
-    wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop2.7.tgz
+    rm -f spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+    wget https://archive.apache.org/dist/spark/spark-#{version}/spark-#{version}-bin-hadoop#{hadoop_type}.tgz
   EOF
-  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_sha256.txt")).strip} | sha256sum -c"
+  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
 end
 
 directory '/opt/spark' do
@@ -20,35 +27,37 @@ end
 execute "install spark-#{version}" do
   cwd '/tmp'
   command <<-EOF
-    rm -Rf spark-#{version}-bin-hadoop2.7/
-    tar zxf spark-#{version}-bin-hadoop2.7.tgz
-    sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop2.7
-    sudo mv spark-#{version}-bin-hadoop2.7 /opt/spark/
-    sudo touch /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED
+    rm -Rf spark-#{version}-bin-hadoop#{hadoop_type}/
+    tar zxf spark-#{version}-bin-hadoop#{hadoop_type}.tgz
+    sudo rm -Rf /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}
+    sudo mv spark-#{version}-bin-hadoop#{hadoop_type} /opt/spark/
+    sudo touch /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED
   EOF
-  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop2.7/INSTALLED"
+  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED"
 end
 
 execute 'install hadoop aws jars' do
   cwd '/opt/hadoop/current'
   command <<-EOF
     cp -f share/hadoop/tools/lib/aws-java-sdk-*.jar \
-        /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+        /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
     cp -f share/hadoop/tools/lib/hadoop-aws-#{hadoop_version}.jar \
-        /opt/spark/spark-#{version}-bin-hadoop2.7/jars/
+        /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
   EOF
-  not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop2.7/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 2"
+  not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
 end
 
+template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/spark-defaults.conf"
+
 if ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
-  template "/opt/spark/spark-#{version}-bin-hadoop2.7/conf/hdfs-site.xml" do
+  template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/hdfs-site.xml" do
     variables aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
               aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
   end
 end
 
 link '/opt/spark/current' do
-  to "/opt/spark/spark-#{version}-bin-hadoop2.7"
+  to "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
   user 'root'
   force true
 end

data/lib/itamae/plugin/recipe/spark/spark-2.4.5_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+020be52524e4df366eb974d41a6e18fcb6efcaba9a51632169e917c74267dd81  spark-2.4.5-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-2.4.6_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+7174fee30057fbf226698ebfcfa7c2944a26197e149053aa29b26b250d1e1eba  spark-2.4.6-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-2.4.7_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+13098490936c9931beda3acc4c30cdc5ca707acd1415eebde1030b11903934fe  spark-2.4.7-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_2.7_sha256.txt

@@ -0,0 +1 @@
+61543e748d225d859e3e0d488362f9dc4b6496a104ef5a4b297d341a47a49a12  spark-3.0.1-bin-hadoop2.7.tgz

data/lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt

@@ -0,0 +1 @@
+e2d05efa1c657dd5180628a83ea36c97c00f972b4aee935b7affa2e1058b0279  spark-3.0.1-bin-hadoop3.2.tgz

data/lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb

@@ -0,0 +1,2 @@
+spark.serializer                                                org.apache.spark.serializer.KryoSerializer
+spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version    2

data/lib/itamae/plugin/recipe/spark/version.rb

@@ -2,12 +2,12 @@ module Itamae
   module Plugin
     module Recipe
       module Spark
-        VERSION = "0.1.0"
+        VERSION = "0.1.5"
         
         SPARK_VERSION = [
-          SPARK_VERSION_MAJOR = '2',
-          SPARK_VERSION_MINOR = '4',
-          SPARK_VERSION_REVISION = '4'
+          SPARK_VERSION_MAJOR = '3',
+          SPARK_VERSION_MINOR = '0',
+          SPARK_VERSION_REVISION = '1'
         ].join('.')
 
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: itamae-plugin-recipe-spark
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - ichylinux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-14 00:00:00.000000000 Z
+date: 2021-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: itamae
@@ -83,7 +83,6 @@ files:
 - ".project"
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - LICENSE.txt
 - README.md
@@ -93,14 +92,19 @@ files:
 - itamae-plugin-recipe-spark.gemspec
 - lib/itamae/plugin/recipe/spark.rb
 - lib/itamae/plugin/recipe/spark/install.rb
-- lib/itamae/plugin/recipe/spark/spark-2.1.1_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.2.0_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.3.1_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.3.2_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.0_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.3_sha256.txt
-- lib/itamae/plugin/recipe/spark/spark-2.4.4_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.2.0_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.3.1_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.3.2_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.0_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.3_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.4_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.5_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.6_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-2.4.7_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_2.7_sha256.txt
+- lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt
 - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
+- lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
 - lib/itamae/plugin/recipe/spark/version.rb
 homepage: https://github.com/maedadev/itamae-plugin-recipe-spark
 licenses:
@@ -121,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: itamae recipe for apache spark installation

data/Gemfile.lock

@@ -1,44 +0,0 @@
-PATH
-  remote: .
-  specs:
-    itamae-plugin-recipe-spark (0.1.0)
-      itamae (~> 1.10, >= 1.10.4)
-      itamae-plugin-recipe-hadoop (~> 0.1)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ansi (1.5.0)
-    hashie (4.1.0)
-    itamae (1.10.6)
-      ansi
-      hashie
-      schash (~> 0.1.0)
-      specinfra (>= 2.64.0, < 3.0.0)
-      thor
-    itamae-plugin-recipe-hadoop (0.1.0)
-      itamae (~> 1.10, >= 1.10.4)
-    net-scp (2.0.0)
-      net-ssh (>= 2.6.5, < 6.0.0)
-    net-ssh (5.2.0)
-    net-telnet (0.1.1)
-    rake (13.0.1)
-    schash (0.1.2)
-    sfl (2.3)
-    specinfra (2.82.12)
-      net-scp
-      net-ssh (>= 2.7)
-      net-telnet (= 0.1.1)
-      sfl
-    thor (1.0.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 2.0)
-  itamae-plugin-recipe-spark!
-  rake (>= 12.3.3)
-
-BUNDLED WITH
-   2.0.2

data/lib/itamae/plugin/recipe/spark/spark-2.1.1_sha256.txt

@@ -1 +0,0 @@
-372ac4f73221c07696793101007a4f19e31566d1f0d9bd0e5205b6fb5b45bfc2  spark-2.1.1-bin-hadoop2.7.tgz