itamae-plugin-recipe-spark 0.1.8 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/lib/itamae/plugin/recipe/spark/install.rb +41 -23
  3. data/lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt +1 -0
  4. data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt +4 -0
  5. data/lib/itamae/plugin/recipe/spark/version.rb +7 -6
  6. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 268f4460c423dae139b48c63cb8a09f3e4161ca3ff09f13b3d257af437e68519
4
- data.tar.gz: 8ec5eea9f055fd2cf26170cf57015788453bdde32a5a01f259b1eac39bdc44af
3
+ metadata.gz: 6c89c752cc2d79e0c7514b2ef9ce5e4579355ba68e19a22c60f4b273181a6b97
4
+ data.tar.gz: eedd14729fd7fdb8a5afde967984adf09e003d46a1ec874e028ead143a65f4a1
5
5
  SHA512:
6
- metadata.gz: c2b69df2d7e764fcd5bf4d6b9f4296ca71154e9b621e715956deefe84794fb330be6bd4868016122c136c0ebe77da4713facf7275b88115c5f42edc903979e35
7
- data.tar.gz: e9199dad27e2413af0ce50ec65867fab010a815f429a1ac8c836fdafccdcf7bb017096cad4abb5ae2e758579b07d369e11cc18b56b6462b2a377653304b1ffe1
6
+ metadata.gz: e6a0b3361bb5be777eadcd1e811989b941d5c0754fca305fb230168e69fca936364a7e2231b9b16b86deb546750b2eea983ae03072640b7d07965eb5557d3a82
7
+ data.tar.gz: d1ddab8c7a5c5a96793578f993b234651e767dd9fdb5ec809678336f5a537830258edaac70a62af2fbc8873eede709e08c91477e34032e207035f1e8012b2a44
data/lib/itamae/plugin/recipe/spark/install.rb CHANGED
@@ -19,22 +19,32 @@ execute "download spark-#{version}" do
19
19
  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
20
20
  end
21
21
 
22
- spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
- spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
22
+ spark_redshift_version = ENV['SPARK_REDSHIFT_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
+ spark_avro_version = ENV['SPARK_AVRO_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
24
24
  minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
25
- redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
25
+ redshift_jdbc_version = ENV['REDSHIFT_JDBC_VERSION'] || Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
26
26
  fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
27
27
  jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
28
+ aws_secretsmanager_jdbc_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_JDBC_VERSION
28
29
  execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
29
30
  cwd '/tmp'
30
31
  command <<-EOF
31
- wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
+ wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-', 2).first}/#{spark_redshift_version.split('-', 2).last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
33
  wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
33
- wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
34
- wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
- wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
36
- wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
34
+ wget -q https://repo1.maven.org/maven2/com/amazon/redshift/redshift-jdbc42/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
36
+ <<-EOS
37
+ wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
38
+ wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
39
+ wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
40
+ EOS
41
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
42
+ <<-EOS
43
+ wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
44
+ EOS
45
+ end}
37
46
  EOF
47
+
38
48
  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
39
49
  end
40
50
 
@@ -97,23 +107,31 @@ execute 'install spark-redshift jars' do
97
107
  cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
98
108
  command <<-EOF
99
109
  ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
100
- cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
101
- jars/
110
+ cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar jars/
111
+
102
112
  ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
103
- cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
104
- jars/
105
- ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
106
- cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
107
- jars/
113
+ cp -f /tmp/spark-avro_#{spark_avro_version}.jar jars/
114
+
108
115
  ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
109
- cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
110
- jars/
111
- ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
112
- cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar \
113
- jars/
114
- ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
115
- cp -f /tmp/jets3t-#{jets3t_version}.jar \
116
- jars/
116
+ cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar jars/
117
+
118
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
119
+ <<-EOS
120
+ ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
121
+ cp -f /tmp/minimal-json-#{minimal_json_version}.jar jars/
122
+
123
+ ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
124
+ cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar jars/
125
+
126
+ ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
127
+ cp -f /tmp/jets3t-#{jets3t_version}.jar jars/
128
+ EOS
129
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
130
+ <<-EOS
131
+ ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
132
+ cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
133
+ EOS
134
+ end}
117
135
  EOF
118
136
  end
119
137
 
data/lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt ADDED
@@ -0,0 +1 @@
1
+ 5a21295b4c3d1d3f8fc85375c711c7c23e3eeb3ec9ea91778f149d8d321e3905e2f44cf19c69a28df693cffd536f7316706c78932e7e148d224424150f18b2c5 spark-3.4.1-bin-hadoop3.tgz
data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt ADDED
@@ -0,0 +1,4 @@
1
+ 97d829e5e47e0b0306bc54549fbfc67516bea15d249581eeb1152f155fdbee49 spark-redshift_2.12-6.2.0-spark_3.4.jar
2
+ ed252b44d546a3df7778debf563eeb535ae07d74c3c18d4163fcbad88cd6ffbf spark-avro_2.12-3.4.2.jar
3
+ dd12dc400549fb055b64b186b7bb05f3e31d417f398fca6755a02394d0302bae RedshiftJDBC42-2.1.0.24.jar
4
+ b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd aws-secretsmanager-jdbc-1.0.12.jar
data/lib/itamae/plugin/recipe/spark/version.rb CHANGED
@@ -2,20 +2,21 @@ module Itamae
2
2
  module Plugin
3
3
  module Recipe
4
4
  module Spark
5
- VERSION = "0.1.8"
5
+ VERSION = "0.1.11"
6
6
 
7
7
  SPARK_VERSION = [
8
8
  SPARK_VERSION_MAJOR = '3',
9
- SPARK_VERSION_MINOR = '3',
10
- SPARK_VERSION_REVISION = '2'
9
+ SPARK_VERSION_MINOR = '4',
10
+ SPARK_VERSION_REVISION = '1'
11
11
  ].join('.')
12
12
 
13
- SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
14
- SPARK_AVRO_VERSION = '2.12-3.0.1'
13
+ SPARK_REDSHIFT_VERSION = '2.12-6.2.0-spark_3.4'
14
+ SPARK_AVRO_VERSION = '2.12-3.4.2'
15
15
  MINIMAL_JSON_VERSION = '0.9.4'
16
- REDSHIFT_JDBC_VERSION = '2.1.0.17'
16
+ REDSHIFT_JDBC_VERSION = '2.1.0.24'
17
17
  FASTDOUBLEPARSER_VERSION = '0.8.0'
18
18
  JETS3T_VERSION = '0.9.4'
19
+ AWS_SECRETSMANAGER_JDBC_VERSION = '1.0.12'
19
20
  end
20
21
  end
21
22
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: itamae-plugin-recipe-spark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.8
4
+ version: 0.1.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - ichylinux
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-08-31 00:00:00.000000000 Z
11
+ date: 2024-04-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: itamae
@@ -106,7 +106,9 @@ files:
106
106
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_2.7_sha256.txt
107
107
  - lib/itamae/plugin/recipe/spark/spark-3.1.2_hadoop_3.2_sha256.txt
108
108
  - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
109
+ - lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt
109
110
  - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
111
+ - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt
110
112
  - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
111
113
  - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
112
114
  - lib/itamae/plugin/recipe/spark/version.rb
@@ -129,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
129
131
  - !ruby/object:Gem::Version
130
132
  version: '0'
131
133
  requirements: []
132
- rubygems_version: 3.3.26
134
+ rubygems_version: 3.4.19
133
135
  signing_key:
134
136
  specification_version: 4
135
137
  summary: itamae recipe for apache spark installation