itamae-plugin-recipe-spark 0.1.8 → 0.1.11

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/lib/itamae/plugin/recipe/spark/install.rb +41 -23
  3. data/lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt +1 -0
  4. data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt +4 -0
  5. data/lib/itamae/plugin/recipe/spark/version.rb +7 -6
  6. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 268f4460c423dae139b48c63cb8a09f3e4161ca3ff09f13b3d257af437e68519
4
- data.tar.gz: 8ec5eea9f055fd2cf26170cf57015788453bdde32a5a01f259b1eac39bdc44af
3
+ metadata.gz: 6c89c752cc2d79e0c7514b2ef9ce5e4579355ba68e19a22c60f4b273181a6b97
4
+ data.tar.gz: eedd14729fd7fdb8a5afde967984adf09e003d46a1ec874e028ead143a65f4a1
5
5
  SHA512:
6
- metadata.gz: c2b69df2d7e764fcd5bf4d6b9f4296ca71154e9b621e715956deefe84794fb330be6bd4868016122c136c0ebe77da4713facf7275b88115c5f42edc903979e35
7
- data.tar.gz: e9199dad27e2413af0ce50ec65867fab010a815f429a1ac8c836fdafccdcf7bb017096cad4abb5ae2e758579b07d369e11cc18b56b6462b2a377653304b1ffe1
6
+ metadata.gz: e6a0b3361bb5be777eadcd1e811989b941d5c0754fca305fb230168e69fca936364a7e2231b9b16b86deb546750b2eea983ae03072640b7d07965eb5557d3a82
7
+ data.tar.gz: d1ddab8c7a5c5a96793578f993b234651e767dd9fdb5ec809678336f5a537830258edaac70a62af2fbc8873eede709e08c91477e34032e207035f1e8012b2a44
data/lib/itamae/plugin/recipe/spark/install.rb CHANGED
@@ -19,22 +19,32 @@ execute "download spark-#{version}" do
19
19
  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
20
20
  end
21
21
 
22
- spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
- spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
22
+ spark_redshift_version = ENV['SPARK_REDSHIFT_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
+ spark_avro_version = ENV['SPARK_AVRO_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
24
24
  minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
25
- redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
25
+ redshift_jdbc_version = ENV['REDSHIFT_JDBC_VERSION'] || Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
26
26
  fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
27
27
  jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
28
+ aws_secretsmanager_jdbc_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_JDBC_VERSION
28
29
  execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
29
30
  cwd '/tmp'
30
31
  command <<-EOF
31
- wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
+ wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-', 2).first}/#{spark_redshift_version.split('-', 2).last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
33
  wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
33
- wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
34
- wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
- wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
36
- wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
34
+ wget -q https://repo1.maven.org/maven2/com/amazon/redshift/redshift-jdbc42/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
36
+ <<-EOS
37
+ wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
38
+ wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
39
+ wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
40
+ EOS
41
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
42
+ <<-EOS
43
+ wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
44
+ EOS
45
+ end}
37
46
  EOF
47
+
38
48
  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
39
49
  end
40
50
 
@@ -97,23 +107,31 @@ execute 'install spark-redshift jars' do
97
107
  cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
98
108
  command <<-EOF
99
109
  ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
100
- cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
101
- jars/
110
+ cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar jars/
111
+
102
112
  ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
103
- cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
104
- jars/
105
- ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
106
- cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
107
- jars/
113
+ cp -f /tmp/spark-avro_#{spark_avro_version}.jar jars/
114
+
108
115
  ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
109
- cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
110
- jars/
111
- ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
112
- cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar \
113
- jars/
114
- ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
115
- cp -f /tmp/jets3t-#{jets3t_version}.jar \
116
- jars/
116
+ cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar jars/
117
+
118
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
119
+ <<-EOS
120
+ ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
121
+ cp -f /tmp/minimal-json-#{minimal_json_version}.jar jars/
122
+
123
+ ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
124
+ cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar jars/
125
+
126
+ ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
127
+ cp -f /tmp/jets3t-#{jets3t_version}.jar jars/
128
+ EOS
129
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
130
+ <<-EOS
131
+ ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
132
+ cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
133
+ EOS
134
+ end}
117
135
  EOF
118
136
  end
119
137
 
data/lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt ADDED
@@ -0,0 +1 @@
1
+ 5a21295b4c3d1d3f8fc85375c711c7c23e3eeb3ec9ea91778f149d8d321e3905e2f44cf19c69a28df693cffd536f7316706c78932e7e148d224424150f18b2c5 spark-3.4.1-bin-hadoop3.tgz
data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt ADDED
@@ -0,0 +1,4 @@
1
+ 97d829e5e47e0b0306bc54549fbfc67516bea15d249581eeb1152f155fdbee49 spark-redshift_2.12-6.2.0-spark_3.4.jar
2
+ ed252b44d546a3df7778debf563eeb535ae07d74c3c18d4163fcbad88cd6ffbf spark-avro_2.12-3.4.2.jar
3
+ dd12dc400549fb055b64b186b7bb05f3e31d417f398fca6755a02394d0302bae RedshiftJDBC42-2.1.0.24.jar
4
+ b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd aws-secretsmanager-jdbc-1.0.12.jar
data/lib/itamae/plugin/recipe/spark/version.rb CHANGED
@@ -2,20 +2,21 @@ module Itamae
2
2
  module Plugin
3
3
  module Recipe
4
4
  module Spark
5
- VERSION = "0.1.8"
5
+ VERSION = "0.1.11"
6
6
 
7
7
  SPARK_VERSION = [
8
8
  SPARK_VERSION_MAJOR = '3',
9
- SPARK_VERSION_MINOR = '3',
10
- SPARK_VERSION_REVISION = '2'
9
+ SPARK_VERSION_MINOR = '4',
10
+ SPARK_VERSION_REVISION = '1'
11
11
  ].join('.')
12
12
 
13
- SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
14
- SPARK_AVRO_VERSION = '2.12-3.0.1'
13
+ SPARK_REDSHIFT_VERSION = '2.12-6.2.0-spark_3.4'
14
+ SPARK_AVRO_VERSION = '2.12-3.4.2'
15
15
  MINIMAL_JSON_VERSION = '0.9.4'
16
- REDSHIFT_JDBC_VERSION = '2.1.0.17'
16
+ REDSHIFT_JDBC_VERSION = '2.1.0.24'
17
17
  FASTDOUBLEPARSER_VERSION = '0.8.0'
18
18
  JETS3T_VERSION = '0.9.4'
19
+ AWS_SECRETSMANAGER_JDBC_VERSION = '1.0.12'
19
20
  end
20
21
  end
21
22
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: itamae-plugin-recipe-spark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.8
4
+ version: 0.1.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - ichylinux
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-08-31 00:00:00.000000000 Z
11
+ date: 2024-04-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: itamae
@@ -106,7 +106,9 @@ files:
106
106
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt
107
107
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
108
108
  - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
109
+ - lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt
109
110
  - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
111
+ - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt
110
112
  - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
111
113
  - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
112
114
  - lib/itamae/plugin/recipe/spark/version.rb
@@ -129,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
129
131
  - !ruby/object:Gem::Version
130
132
  version: '0'
131
133
  requirements: []
132
- rubygems_version: 3.3.26
134
+ rubygems_version: 3.4.19
133
135
  signing_key:
134
136
  specification_version: 4
135
137
  summary: itamae recipe for apache spark installation