itamae-plugin-recipe-spark 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/lib/itamae/plugin/recipe/spark/install.rb +38 -7
  3. data/lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt +1 -0
  4. data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt +2 -1
  5. data/lib/itamae/plugin/recipe/spark/version.rb +4 -3
  6. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 65e26a2e185e47252ef2e7bc92c7aab513cb19266ed2ad11d78764a2306ccb3d
4
- data.tar.gz: fc71a105d5189e80525d25839d65bfacb6978e24a5878e809672522dcd10557e
3
+ metadata.gz: 268f4460c423dae139b48c63cb8a09f3e4161ca3ff09f13b3d257af437e68519
4
+ data.tar.gz: 8ec5eea9f055fd2cf26170cf57015788453bdde32a5a01f259b1eac39bdc44af
5
5
  SHA512:
6
- metadata.gz: 925e77e886ea6a4bf482a88e496ee5e408bfa1284d5b355178d1034a7df61e5ddee1a23b047660105768c194f01c3c0bf800f397ee3eceb2720996447c69ab2d
7
- data.tar.gz: 2afc8f9b8f5fa3401f92513af200d38a4d3bc2839e3d973ad0663ab5f340f58514e3db44efb9a5735be5bc3399271f389b12584aa6b3a6cd9b3608b462527cb8
6
+ metadata.gz: c2b69df2d7e764fcd5bf4d6b9f4296ca71154e9b621e715956deefe84794fb330be6bd4868016122c136c0ebe77da4713facf7275b88115c5f42edc903979e35
7
+ data.tar.gz: e9199dad27e2413af0ce50ec65867fab010a815f429a1ac8c836fdafccdcf7bb017096cad4abb5ae2e758579b07d369e11cc18b56b6462b2a377653304b1ffe1
data/lib/itamae/plugin/recipe/spark/install.rb CHANGED
@@ -1,6 +1,8 @@
1
1
  version = ENV['SPARK_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_VERSION
2
2
  hadoop_version = ENV['HADOOP_VERSION'] || Itamae::Plugin::Recipe::Hadoop::HADOOP_VERSION
3
- hadoop_type = if Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
3
+ hadoop_type = if Gem::Version.create(hadoop_version) == Gem::Version.create('3.3.3')
4
+ '3'
5
+ elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('3.2')
4
6
  '3.2'
5
7
  elsif Gem::Version.create(hadoop_version) >= Gem::Version.create('2.7')
6
8
  '2.7'
@@ -21,6 +23,7 @@ spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
21
23
  spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
22
24
  minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
23
25
  redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
26
+ fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
24
27
  jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
25
28
  execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
26
29
  cwd '/tmp'
@@ -28,12 +31,31 @@ execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
28
31
  wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
29
32
  wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
30
33
  wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
31
- wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/RedshiftJDBC42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
34
+ wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
+ wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
32
36
  wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
33
37
  EOF
34
38
  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
35
39
  end
36
40
 
41
+ execute 'download aws-java-sdk' do
42
+ cwd '/tmp'
43
+ command <<-EOF
44
+ wget -q https://sdk-for-java.amazonwebservices.com/latest/aws-java-sdk.zip
45
+ EOF
46
+ not_if 'test -e /tmp/aws-java-sdk.zip'
47
+ end
48
+
49
+ execute 'unzip aws-java-sdk' do
50
+ cwd '/tmp'
51
+ command <<-EOF
52
+ unzip -o aws-java-sdk.zip aws-java-sdk-*/lib/aws-java-sdk-*.jar
53
+ rm -rf aws-java-sdk-*/lib/aws-java-sdk-*-javadoc.jar
54
+ rm -rf aws-java-sdk-*/lib/aws-java-sdk-*-sources.jar
55
+ EOF
56
+ not_if 'test -e /tmp/aws-java-sdk-*/lib/aws-java-sdk-*.jar'
57
+ end
58
+
37
59
  directory '/opt/spark' do
38
60
  user 'root'
39
61
  owner 'root'
@@ -53,15 +75,22 @@ execute "install spark-#{version}" do
53
75
  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED"
54
76
  end
55
77
 
56
- execute 'install hadoop aws jars' do
57
- cwd '/opt/hadoop/current'
78
+ execute 'install aws java sdk jar' do
79
+ cwd '/tmp'
58
80
  command <<-EOF
59
- cp -f share/hadoop/tools/lib/aws-java-sdk-*.jar \
81
+ cp -f aws-java-sdk-*/lib/aws-java-sdk-*.jar \
60
82
  /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
83
+ EOF
84
+ not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/aws-java-sdk-*.jar"
85
+ end
86
+
87
+ execute 'install hadoop aws jar' do
88
+ cwd '/opt/hadoop/current'
89
+ command <<-EOF
61
90
  cp -f share/hadoop/tools/lib/hadoop-aws-#{hadoop_version}.jar \
62
91
  /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/
63
92
  EOF
64
- not_if "test `ls -1 /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/ | egrep '(hadoop-)?aws-.*' | wc -l` = 4"
93
+ not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/jars/hadoop-aws-*.jar"
65
94
  end
66
95
 
67
96
  execute 'install spark-redshift jars' do
@@ -79,6 +108,9 @@ execute 'install spark-redshift jars' do
79
108
  ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
80
109
  cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
81
110
  jars/
111
+ ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
112
+ cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar \
113
+ jars/
82
114
  ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
83
115
  cp -f /tmp/jets3t-#{jets3t_version}.jar \
84
116
  jars/
@@ -99,4 +131,3 @@ link '/opt/spark/current' do
99
131
  user 'root'
100
132
  force true
101
133
  end
102
-
data/lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt ADDED
@@ -0,0 +1 @@
1
+ 4cd2396069fbe0f8efde2af4fd301bf46f8c6317e9dea1dd42a405de6a38380635d49b17972cb92c619431acece2c3af4c23bfdf193cedb3ea913ed69ded23a1 spark-3.3.2-bin-hadoop3.tgz
data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt CHANGED
@@ -1,5 +1,6 @@
1
1
  a4ded0f2806b50eef02885d2543b16d2f0654744e8f1f45dc913f74b22071b93 spark-redshift_2.12-5.0.3.jar
2
2
  b1ffe6dbfb9a6492d9e5ceed645ee49bb5f3ab2e48a3836ee8d91a892995436b spark-avro_2.12-3.0.1.jar
3
3
  e1d280900c78f18ae2e00c14e7410a77ba19cf084154b386532846aa6dc81721 minimal-json-0.9.4.jar
4
- a27fd9e446021557c17ca3697d67d1c1857ce7e92a5c80c1038b767b835ad841 RedshiftJDBC42-1.2.37.1061.jar
4
+ 1206f3b61fd0ce408c129eb96df01f8023ebf8c0e82ed2795734838c53906939 RedshiftJDBC42-2.1.0.17.jar
5
+ 10fe288fd7a2cdaf5175332b73529f9abf8fd54dcfff317d6967c0c35ffb133b fastdoubleparser-0.8.0.jar
5
6
  a03f0d25e810ecff30137e0d3148d1bf9afdcf62357ebd1b1af338f7f6115bbb jets3t-0.9.4.jar
data/lib/itamae/plugin/recipe/spark/version.rb CHANGED
@@ -2,18 +2,19 @@ module Itamae
2
2
  module Plugin
3
3
  module Recipe
4
4
  module Spark
5
- VERSION = "0.1.6"
5
+ VERSION = "0.1.8"
6
6
 
7
7
  SPARK_VERSION = [
8
8
  SPARK_VERSION_MAJOR = '3',
9
- SPARK_VERSION_MINOR = '1',
9
+ SPARK_VERSION_MINOR = '3',
10
10
  SPARK_VERSION_REVISION = '2'
11
11
  ].join('.')
12
12
 
13
13
  SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
14
14
  SPARK_AVRO_VERSION = '2.12-3.0.1'
15
15
  MINIMAL_JSON_VERSION = '0.9.4'
16
- REDSHIFT_JDBC_VERSION = '1.2.37.1061'
16
+ REDSHIFT_JDBC_VERSION = '2.1.0.17'
17
+ FASTDOUBLEPARSER_VERSION = '0.8.0'
17
18
  JETS3T_VERSION = '0.9.4'
18
19
  end
19
20
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: itamae-plugin-recipe-spark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.1.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - ichylinux
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-01-19 00:00:00.000000000 Z
11
+ date: 2023-08-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: itamae
@@ -105,6 +105,7 @@ files:
105
105
  - lib/itamae/plugin/recipe/spark/spark-3.0.1_hadoop_3.2_sha256.txt
106
106
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt
107
107
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
108
+ - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
108
109
  - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
109
110
  - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
110
111
  - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
@@ -128,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
128
129
  - !ruby/object:Gem::Version
129
130
  version: '0'
130
131
  requirements: []
131
- rubygems_version: 3.2.16
132
+ rubygems_version: 3.3.26
132
133
  signing_key:
133
134
  specification_version: 4
134
135
  summary: itamae recipe for apache spark installation