itamae-plugin-recipe-spark 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6c89c752cc2d79e0c7514b2ef9ce5e4579355ba68e19a22c60f4b273181a6b97
|
4
|
+
data.tar.gz: eedd14729fd7fdb8a5afde967984adf09e003d46a1ec874e028ead143a65f4a1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e6a0b3361bb5be777eadcd1e811989b941d5c0754fca305fb230168e69fca936364a7e2231b9b16b86deb546750b2eea983ae03072640b7d07965eb5557d3a82
|
7
|
+
data.tar.gz: d1ddab8c7a5c5a96793578f993b234651e767dd9fdb5ec809678336f5a537830258edaac70a62af2fbc8873eede709e08c91477e34032e207035f1e8012b2a44
|
@@ -19,22 +19,32 @@ execute "download spark-#{version}" do
|
|
19
19
|
not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
|
20
20
|
end
|
21
21
|
|
22
|
-
spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
|
23
|
-
spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
|
22
|
+
spark_redshift_version = ENV['SPARK_REDSHIFT_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
|
23
|
+
spark_avro_version = ENV['SPARK_AVRO_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
|
24
24
|
minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
|
25
|
-
redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
|
25
|
+
redshift_jdbc_version = ENV['REDSHIFT_JDBC_VERSION'] || Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
|
26
26
|
fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
|
27
27
|
jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
|
28
|
+
aws_secretsmanager_jdbc_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_JDBC_VERSION
|
28
29
|
execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
|
29
30
|
cwd '/tmp'
|
30
31
|
command <<-EOF
|
31
|
-
wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
|
32
|
+
wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-', 2).first}/#{spark_redshift_version.split('-', 2).last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
|
32
33
|
wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
|
33
|
-
wget -q https://repo1.maven.org/maven2/com/
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
wget -q https://repo1.maven.org/maven2/com/amazon/redshift/redshift-jdbc42/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
|
35
|
+
#{if spark_redshift_version.split('-', 2).last == '5.0.3'
|
36
|
+
<<-EOS
|
37
|
+
wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
|
38
|
+
wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
|
39
|
+
wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
|
40
|
+
EOS
|
41
|
+
elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
|
42
|
+
<<-EOS
|
43
|
+
wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
|
44
|
+
EOS
|
45
|
+
end}
|
37
46
|
EOF
|
47
|
+
|
38
48
|
not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
|
39
49
|
end
|
40
50
|
|
@@ -97,23 +107,31 @@ execute 'install spark-redshift jars' do
|
|
97
107
|
cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
|
98
108
|
command <<-EOF
|
99
109
|
ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
|
100
|
-
cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar
|
101
|
-
|
110
|
+
cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar jars/
|
111
|
+
|
102
112
|
ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
|
103
|
-
cp -f /tmp/spark-avro_#{spark_avro_version}.jar
|
104
|
-
|
105
|
-
ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
|
106
|
-
cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
|
107
|
-
jars/
|
113
|
+
cp -f /tmp/spark-avro_#{spark_avro_version}.jar jars/
|
114
|
+
|
108
115
|
ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
|
109
|
-
cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
116
|
+
cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar jars/
|
117
|
+
|
118
|
+
#{if spark_redshift_version.split('-', 2).last == '5.0.3'
|
119
|
+
<<-EOS
|
120
|
+
ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
|
121
|
+
cp -f /tmp/minimal-json-#{minimal_json_version}.jar jars/
|
122
|
+
|
123
|
+
ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
|
124
|
+
cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar jars/
|
125
|
+
|
126
|
+
ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
|
127
|
+
cp -f /tmp/jets3t-#{jets3t_version}.jar jars/
|
128
|
+
EOS
|
129
|
+
elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
|
130
|
+
<<-EOS
|
131
|
+
ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
|
132
|
+
cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
|
133
|
+
EOS
|
134
|
+
end}
|
117
135
|
EOF
|
118
136
|
end
|
119
137
|
|
@@ -0,0 +1,4 @@
|
|
1
|
+
97d829e5e47e0b0306bc54549fbfc67516bea15d249581eeb1152f155fdbee49 spark-redshift_2.12-6.2.0-spark_3.4.jar
|
2
|
+
ed252b44d546a3df7778debf563eeb535ae07d74c3c18d4163fcbad88cd6ffbf spark-avro_2.12-3.4.2.jar
|
3
|
+
dd12dc400549fb055b64b186b7bb05f3e31d417f398fca6755a02394d0302bae RedshiftJDBC42-2.1.0.24.jar
|
4
|
+
b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd aws-secretsmanager-jdbc-1.0.12.jar
|
@@ -2,7 +2,7 @@ module Itamae
|
|
2
2
|
module Plugin
|
3
3
|
module Recipe
|
4
4
|
module Spark
|
5
|
-
VERSION = "0.1.
|
5
|
+
VERSION = "0.1.11"
|
6
6
|
|
7
7
|
SPARK_VERSION = [
|
8
8
|
SPARK_VERSION_MAJOR = '3',
|
@@ -10,12 +10,13 @@ module Itamae
|
|
10
10
|
SPARK_VERSION_REVISION = '1'
|
11
11
|
].join('.')
|
12
12
|
|
13
|
-
SPARK_REDSHIFT_VERSION = '2.12-
|
14
|
-
SPARK_AVRO_VERSION = '2.12-3.
|
13
|
+
SPARK_REDSHIFT_VERSION = '2.12-6.2.0-spark_3.4'
|
14
|
+
SPARK_AVRO_VERSION = '2.12-3.4.2'
|
15
15
|
MINIMAL_JSON_VERSION = '0.9.4'
|
16
|
-
REDSHIFT_JDBC_VERSION = '2.1.0.
|
16
|
+
REDSHIFT_JDBC_VERSION = '2.1.0.24'
|
17
17
|
FASTDOUBLEPARSER_VERSION = '0.8.0'
|
18
18
|
JETS3T_VERSION = '0.9.4'
|
19
|
+
AWS_SECRETSMANAGER_JDBC_VERSION = '1.0.12'
|
19
20
|
end
|
20
21
|
end
|
21
22
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: itamae-plugin-recipe-spark
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ichylinux
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-04-
|
11
|
+
date: 2024-04-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: itamae
|
@@ -108,6 +108,7 @@ files:
|
|
108
108
|
- lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
|
109
109
|
- lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt
|
110
110
|
- lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
|
111
|
+
- lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt
|
111
112
|
- lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
|
112
113
|
- lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
|
113
114
|
- lib/itamae/plugin/recipe/spark/version.rb
|
@@ -130,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
130
131
|
- !ruby/object:Gem::Version
|
131
132
|
version: '0'
|
132
133
|
requirements: []
|
133
|
-
rubygems_version: 3.
|
134
|
+
rubygems_version: 3.4.19
|
134
135
|
signing_key:
|
135
136
|
specification_version: 4
|
136
137
|
summary: itamae recipe for apache spark installation
|