itamae-plugin-recipe-spark 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/lib/itamae/plugin/recipe/spark/install.rb +38 -0
  3. data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt +1 -0
  4. data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt +1 -0
  5. data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt +5 -0
  6. data/lib/itamae/plugin/recipe/spark/version.rb +9 -4
  7. metadata +6 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 201de04bbae7c37fcb100e26b798361d35f7064b2809e01e1a34b1efb2e0768b
4
- data.tar.gz: 92cf7328ac3cd04c97f948e6fb8ccb10e114040d03ac50bc8556ecb8b60e5d9b
3
+ metadata.gz: 65e26a2e185e47252ef2e7bc92c7aab513cb19266ed2ad11d78764a2306ccb3d
4
+ data.tar.gz: fc71a105d5189e80525d25839d65bfacb6978e24a5878e809672522dcd10557e
5
5
  SHA512:
6
- metadata.gz: 90e5650f5f509b8f29d2bde84b16df9f11278f8b9d2ff3f099e920f1728423c957dea0d29b698d229a8ec32dfe4196d4e3ae32aa8f5e53126929b1b6e6c0ad3f
7
- data.tar.gz: 7f84c85db7a99f58fe1036483dc81157b2eba1f1f857b612ab26607110f9bf09cd75c94df74376c5c0f701ad830d2b38c987d5b4bfb6c97535780c8ffc0c0122
6
+ metadata.gz: 925e77e886ea6a4bf482a88e496ee5e408bfa1284d5b355178d1034a7df61e5ddee1a23b047660105768c194f01c3c0bf800f397ee3eceb2720996447c69ab2d
7
+ data.tar.gz: 2afc8f9b8f5fa3401f92513af200d38a4d3bc2839e3d973ad0663ab5f340f58514e3db44efb9a5735be5bc3399271f389b12584aa6b3a6cd9b3608b462527cb8
data/lib/itamae/plugin/recipe/spark/install.rb CHANGED
@@ -17,6 +17,23 @@ execute "download spark-#{version}" do
17
17
  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
18
18
  end
19
19
 
20
+ spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
21
+ spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
22
+ minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
23
+ redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
24
+ jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
25
+ execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
26
+ cwd '/tmp'
27
+ command <<-EOF
28
+ wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
29
+ wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
30
+ wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
31
+ wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/RedshiftJDBC42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
32
+ wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
33
+ EOF
34
+ not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
35
+ end
36
+
20
37
  directory '/opt/spark' do
21
38
  user 'root'
22
39
  owner 'root'
@@ -47,6 +64,27 @@ execute 'install hadoop aws jars' do
47
64
  not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
48
65
  end
49
66
 
67
+ execute 'install spark-redshift jars' do
68
+ cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
69
+ command <<-EOF
70
+ ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
71
+ cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
72
+ jars/
73
+ ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
74
+ cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
75
+ jars/
76
+ ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
77
+ cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
78
+ jars/
79
+ ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
80
+ cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
81
+ jars/
82
+ ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
83
+ cp -f /tmp/jets3t-#{jets3t_version}.jar \
84
+ jars/
85
+ EOF
86
+ end
87
+
50
88
  template "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/conf/spark-defaults.conf"
51
89
 
52
90
  if ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt ADDED
@@ -0,0 +1 @@
1
+ 3a79e324d12f46de44d042641d9340ba03f8ccb3db6f2496a9ccb65431dbb593 spark-3.1.2-bin-hadoop2.7.tgz
data/lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt ADDED
@@ -0,0 +1 @@
1
+ 0d9cf9dbbb3b4215afebe7fa4748b012e406dd1f1ad2a61b993ac04adcb94eaa spark-3.1.2-bin-hadoop3.2.tgz
data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt ADDED
@@ -0,0 +1,5 @@
1
+ a4ded0f2806b50eef02885d2543b16d2f0654744e8f1f45dc913f74b22071b93 spark-redshift_2.12-5.0.3.jar
2
+ b1ffe6dbfb9a6492d9e5ceed645ee49bb5f3ab2e48a3836ee8d91a892995436b spark-avro_2.12-3.0.1.jar
3
+ e1d280900c78f18ae2e00c14e7410a77ba19cf084154b386532846aa6dc81721 minimal-json-0.9.4.jar
4
+ a27fd9e446021557c17ca3697d67d1c1857ce7e92a5c80c1038b767b835ad841 RedshiftJDBC42-1.2.37.1061.jar
5
+ a03f0d25e810ecff30137e0d3148d1bf9afdcf62357ebd1b1af338f7f6115bbb jets3t-0.9.4.jar
data/lib/itamae/plugin/recipe/spark/version.rb CHANGED
@@ -2,14 +2,19 @@ module Itamae
2
2
  module Plugin
3
3
  module Recipe
4
4
  module Spark
5
- VERSION = "0.1.5"
6
-
5
+ VERSION = "0.1.6"
6
+
7
7
  SPARK_VERSION = [
8
8
  SPARK_VERSION_MAJOR = '3',
9
- SPARK_VERSION_MINOR = '0',
10
- SPARK_VERSION_REVISION = '1'
9
+ SPARK_VERSION_MINOR = '1',
10
+ SPARK_VERSION_REVISION = '2'
11
11
  ].join('.')
12
12
 
13
+ SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
14
+ SPARK_AVRO_VERSION = '2.12-3.0.1'
15
+ MINIMAL_JSON_VERSION = '0.9.4'
16
+ REDSHIFT_JDBC_VERSION = '1.2.37.1061'
17
+ JETS3T_VERSION = '0.9.4'
13
18
  end
14
19
  end
15
20
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: itamae-plugin-recipe-spark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - ichylinux
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-04-09 00:00:00.000000000 Z
11
+ date: 2022-01-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: itamae
@@ -103,6 +103,9 @@ files:
103
103
  - lib/itamae/plugin/recipe/spark/spark-2.4.7_hadoop_2.7_sha256.txt
104
104
  - lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_2.7_sha256.txt
105
105
  - lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt
106
+ - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt
107
+ - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
108
+ - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
106
109
  - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
107
110
  - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
108
111
  - lib/itamae/plugin/recipe/spark/version.rb
@@ -125,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
125
128
  - !ruby/object:Gem::Version
126
129
  version: '0'
127
130
  requirements: []
128
- rubygems_version: 3.1.4
131
+ rubygems_version: 3.2.16
129
132
  signing_key:
130
133
  specification_version: 4
131
134
  summary: itamae recipe for apache spark installation