itamae-plugin-recipe-spark 0.1.6 → 0.1.8

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/lib/itamae/plugin/recipe/spark/install.rb +38 -7
  3. data/lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt +1 -0
  4. data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt +2 -1
  5. data/lib/itamae/plugin/recipe/spark/version.rb +4 -3
  6. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 65e26a2e185e47252ef2e7bc92c7aab513cb19266ed2ad11d78764a2306ccb3d
4
- data.tar.gz: fc71a105d5189e80525d25839d65bfacb6978e24a5878e809672522dcd10557e
3
+ metadata.gz: 268f4460c423dae139b48c63cb8a09f3e4161ca3ff09f13b3d257af437e68519
4
+ data.tar.gz: 8ec5eea9f055fd2cf26170cf57015788453bdde32a5a01f259b1eac39bdc44af
5
5
  SHA512:
6
- metadata.gz: 925e77e886ea6a4bf482a88e496ee5e408bfa1284d5b355178d1034a7df61e5ddee1a23b047660105768c194f01c3c0bf800f397ee3eceb2720996447c69ab2d
7
- data.tar.gz: 2afc8f9b8f5fa3401f92513af200d38a4d3bc2839e3d973ad0663ab5f340f58514e3db44efb9a5735be5bc3399271f389b12584aa6b3a6cd9b3608b462527cb8
6
+ metadata.gz: c2b69df2d7e764fcd5bf4d6b9f4296ca71154e9b621e715956deefe84794fb330be6bd4868016122c136c0ebe77da4713facf7275b88115c5f42edc903979e35
7
+ data.tar.gz: e9199dad27e2413af0ce50ec65867fab010a815f429a1ac8c836fdafccdcf7bb017096cad4abb5ae2e758579b07d369e11cc18b56b6462b2a377653304b1ffe1
data/lib/itamae/plugin/recipe/spark/install.rb CHANGED
@@ -1,6 +1,8 @@
1
1
  version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
2
2
  hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
3
- hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
3
+ hadoop_type = if Gem::Version.create(hadoop_version) == Gem::Version.create('3.3.3')
4
+ '3'
5
+ elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
4
6
  '3.2'
5
7
  elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('2.7')
6
8
  '2.7'
@@ -21,6 +23,7 @@ spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
21
23
  spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
22
24
  minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
23
25
  redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
26
+ fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
24
27
  jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
25
28
  execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
26
29
  cwd '/tmp'
@@ -28,12 +31,31 @@ execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
28
31
  wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
29
32
  wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
30
33
  wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
31
- wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/RedshiftJDBC42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
34
+ wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
+ wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
32
36
  wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
33
37
  EOF
34
38
  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
35
39
  end
36
40
 
41
+ execute 'download aws-java-sdk' do
42
+ cwd '/tmp'
43
+ command <<-EOF
44
+ wget -q https://sdk-for-java.amazonwebservices.com/latest/aws-java-sdk.zip
45
+ EOF
46
+ not_if 'test -e /tmp/aws-java-sdk.zip'
47
+ end
48
+
49
+ execute 'unzip aws-java-sdk' do
50
+ cwd '/tmp'
51
+ command <<-EOF
52
+ unzip -o aws-java-sdk.zip aws-java-sdk-*/lib/aws-java-sdk-*.jar
53
+ rm -rf aws-java-sdk-*/lib/aws-java-sdk-*-javadoc.jar
54
+ rm -rf aws-java-sdk-*/lib/aws-java-sdk-*-sources.jar
55
+ EOF
56
+ not_if 'test -e /tmp/aws-java-sdk-*/lib/aws-java-sdk-*.jar'
57
+ end
58
+
37
59
  directory '/opt/spark' do
38
60
  user 'root'
39
61
  owner 'root'
@@ -53,15 +75,22 @@ execute "install spark-#{version}" do
53
75
  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED"
54
76
  end
55
77
 
56
- execute 'install hadoop aws jars' do
57
- cwd '/opt/hadoop/current'
78
+ execute 'install aws java sdk jar' do
79
+ cwd '/tmp'
58
80
  command <<-EOF
59
- cp -f share/hadoop/tools/lib/aws-java-sdk-*.jar \
81
+ cp -f aws-java-sdk-*/lib/aws-java-sdk-*.jar \
60
82
  /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
83
+ EOF
84
+ not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/aws-java-sdk-*.jar"
85
+ end
86
+
87
+ execute 'install hadoop aws jar' do
88
+ cwd '/opt/hadoop/current'
89
+ command <<-EOF
61
90
  cp -f share/hadoop/tools/lib/hadoop-aws-#{hadoop_version}.jar \
62
91
  /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
63
92
  EOF
64
- not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
93
+ not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/hadoop-aws-*.jar"
65
94
  end
66
95
 
67
96
  execute 'install spark-redshift jars' do
@@ -79,6 +108,9 @@ execute 'install spark-redshift jars' do
79
108
  ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
80
109
  cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
81
110
  jars/
111
+ ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
112
+ cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar \
113
+ jars/
82
114
  ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
83
115
  cp -f /tmp/jets3t-#{jets3t_version}.jar \
84
116
  jars/
@@ -99,4 +131,3 @@ link '/opt/spark/current' do
99
131
  user 'root'
100
132
  force true
101
133
  end
102
-
data/lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt ADDED
@@ -0,0 +1 @@
1
+ 4cd2396069fbe0f8efde2af4fd301bf46f8c6317e9dea1dd42a405de6a38380635d49b17972cb92c619431acece2c3af4c23bfdf193cedb3ea913ed69ded23a1 spark-3.3.2-bin-hadoop3.tgz
data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt CHANGED
@@ -1,5 +1,6 @@
1
1
  a4ded0f2806b50eef02885d2543b16d2f0654744e8f1f45dc913f74b22071b93 spark-redshift_2.12-5.0.3.jar
2
2
  b1ffe6dbfb9a6492d9e5ceed645ee49bb5f3ab2e48a3836ee8d91a892995436b spark-avro_2.12-3.0.1.jar
3
3
  e1d280900c78f18ae2e00c14e7410a77ba19cf084154b386532846aa6dc81721 minimal-json-0.9.4.jar
4
- a27fd9e446021557c17ca3697d67d1c1857ce7e92a5c80c1038b767b835ad841 RedshiftJDBC42-1.2.37.1061.jar
4
+ 1206f3b61fd0ce408c129eb96df01f8023ebf8c0e82ed2795734838c53906939 RedshiftJDBC42-2.1.0.17.jar
5
+ 10fe288fd7a2cdaf5175332b73529f9abf8fd54dcfff317d6967c0c35ffb133b fastdoubleparser-0.8.0.jar
5
6
  a03f0d25e810ecff30137e0d3148d1bf9afdcf62357ebd1b1af338f7f6115bbb jets3t-0.9.4.jar
data/lib/itamae/plugin/recipe/spark/version.rb CHANGED
@@ -2,18 +2,19 @@ module Itamae
2
2
  module Plugin
3
3
  module Recipe
4
4
  module Spark
5
- VERSION = "0.1.6"
5
+ VERSION = "0.1.8"
6
6
 
7
7
  SPARK_VERSION = [
8
8
  SPARK_VERSION_MAJOR = '3',
9
- SPARK_VERSION_MINOR = '1',
9
+ SPARK_VERSION_MINOR = '3',
10
10
  SPARK_VERSION_REVISION = '2'
11
11
  ].join('.')
12
12
 
13
13
  SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
14
14
  SPARK_AVRO_VERSION = '2.12-3.0.1'
15
15
  MINIMAL_JSON_VERSION = '0.9.4'
16
- REDSHIFT_JDBC_VERSION = '1.2.37.1061'
16
+ REDSHIFT_JDBC_VERSION = '2.1.0.17'
17
+ FASTDOUBLEPARSER_VERSION = '0.8.0'
17
18
  JETS3T_VERSION = '0.9.4'
18
19
  end
19
20
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: itamae-plugin-recipe-spark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.1.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - ichylinux
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-01-19 00:00:00.000000000 Z
11
+ date: 2023-08-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: itamae
@@ -105,6 +105,7 @@ files:
105
105
  - lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt
106
106
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt
107
107
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
108
+ - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
108
109
  - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
109
110
  - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
110
111
  - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
@@ -128,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
128
129
  - !ruby/object:Gem::Version
129
130
  version: '0'
130
131
  requirements: []
131
- rubygems_version: 3.2.16
132
+ rubygems_version: 3.3.26
132
133
  signing_key:
133
134
  specification_version: 4
134
135
  summary: itamae recipe for apache spark installation