itamae-plugin-recipe-spark 0.1.9 → 0.1.11

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/lib/itamae/plugin/recipe/spark/install.rb +41 -23
  3. data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt +4 -0
  4. data/lib/itamae/plugin/recipe/spark/version.rb +5 -4
  5. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c4a57bd0c4c43b94a896f87b20b4580c10ffe76e2995af89ae2a60e44963554c
4
- data.tar.gz: f61d1f73264122bd76db928b6d85b261fec7ee754cdd06b94f683ec20a08f2f7
3
+ metadata.gz: 6c89c752cc2d79e0c7514b2ef9ce5e4579355ba68e19a22c60f4b273181a6b97
4
+ data.tar.gz: eedd14729fd7fdb8a5afde967984adf09e003d46a1ec874e028ead143a65f4a1
5
5
  SHA512:
6
- metadata.gz: 49e7761e3f424ce810e3bd423e4ade4cf122122fbdff80dffc7d200573bea3d31b19a367e8648fac00bf80796ec23b34a171e24564d94f2579dd9fa64c95278f
7
- data.tar.gz: 343ffa8a4f31b83bdf2543185c3a65832b785cb9734de40c10866cb7d38b7f851db1f7462f91872252dd59291376a7bc56578fc18aaf25d8769f94df0d72e4b9
6
+ metadata.gz: e6a0b3361bb5be777eadcd1e811989b941d5c0754fca305fb230168e69fca936364a7e2231b9b16b86deb546750b2eea983ae03072640b7d07965eb5557d3a82
7
+ data.tar.gz: d1ddab8c7a5c5a96793578f993b234651e767dd9fdb5ec809678336f5a537830258edaac70a62af2fbc8873eede709e08c91477e34032e207035f1e8012b2a44
data/lib/itamae/plugin/recipe/spark/install.rb CHANGED
@@ -19,22 +19,32 @@ execute "download spark-#{version}" do
19
19
  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
20
20
  end
21
21
 
22
- spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
- spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
22
+ spark_redshift_version = ENV['SPARK_REDSHIFT_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
+ spark_avro_version = ENV['SPARK_AVRO_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
24
24
  minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
25
- redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
25
+ redshift_jdbc_version = ENV['REDSHIFT_JDBC_VERSION'] || Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
26
26
  fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
27
27
  jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
28
+ aws_secretsmanager_jdbc_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_JDBC_VERSION
28
29
  execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
29
30
  cwd '/tmp'
30
31
  command <<-EOF
31
- wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
+ wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-', 2).first}/#{spark_redshift_version.split('-', 2).last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
33
  wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
33
- wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
34
- wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
- wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
36
- wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
34
+ wget -q https://repo1.maven.org/maven2/com/amazon/redshift/redshift-jdbc42/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
36
+ <<-EOS
37
+ wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
38
+ wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
39
+ wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
40
+ EOS
41
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
42
+ <<-EOS
43
+ wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
44
+ EOS
45
+ end}
37
46
  EOF
47
+
38
48
  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
39
49
  end
40
50
 
@@ -97,23 +107,31 @@ execute 'install spark-redshift jars' do
97
107
  cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
98
108
  command <<-EOF
99
109
  ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
100
- cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
101
- jars/
110
+ cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar jars/
111
+
102
112
  ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
103
- cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
104
- jars/
105
- ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
106
- cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
107
- jars/
113
+ cp -f /tmp/spark-avro_#{spark_avro_version}.jar jars/
114
+
108
115
  ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
109
- cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
110
- jars/
111
- ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
112
- cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar \
113
- jars/
114
- ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
115
- cp -f /tmp/jets3t-#{jets3t_version}.jar \
116
- jars/
116
+ cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar jars/
117
+
118
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
119
+ <<-EOS
120
+ ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
121
+ cp -f /tmp/minimal-json-#{minimal_json_version}.jar jars/
122
+
123
+ ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
124
+ cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar jars/
125
+
126
+ ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
127
+ cp -f /tmp/jets3t-#{jets3t_version}.jar jars/
128
+ EOS
129
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
130
+ <<-EOS
131
+ ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
132
+ cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
133
+ EOS
134
+ end}
117
135
  EOF
118
136
  end
119
137
 
data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt ADDED
@@ -0,0 +1,4 @@
1
+ 97d829e5e47e0b0306bc54549fbfc67516bea15d249581eeb1152f155fdbee49 spark-redshift_2.12-6.2.0-spark_3.4.jar
2
+ ed252b44d546a3df7778debf563eeb535ae07d74c3c18d4163fcbad88cd6ffbf spark-avro_2.12-3.4.2.jar
3
+ dd12dc400549fb055b64b186b7bb05f3e31d417f398fca6755a02394d0302bae RedshiftJDBC42-2.1.0.24.jar
4
+ b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd aws-secretsmanager-jdbc-1.0.12.jar
data/lib/itamae/plugin/recipe/spark/version.rb CHANGED
@@ -2,7 +2,7 @@ module Itamae
2
2
  module Plugin
3
3
  module Recipe
4
4
  module Spark
5
- VERSION = "0.1.9"
5
+ VERSION = "0.1.11"
6
6
 
7
7
  SPARK_VERSION = [
8
8
  SPARK_VERSION_MAJOR = '3',
@@ -10,12 +10,13 @@ module Itamae
10
10
  SPARK_VERSION_REVISION = '1'
11
11
  ].join('.')
12
12
 
13
- SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
14
- SPARK_AVRO_VERSION = '2.12-3.0.1'
13
+ SPARK_REDSHIFT_VERSION = '2.12-6.2.0-spark_3.4'
14
+ SPARK_AVRO_VERSION = '2.12-3.4.2'
15
15
  MINIMAL_JSON_VERSION = '0.9.4'
16
- REDSHIFT_JDBC_VERSION = '2.1.0.17'
16
+ REDSHIFT_JDBC_VERSION = '2.1.0.24'
17
17
  FASTDOUBLEPARSER_VERSION = '0.8.0'
18
18
  JETS3T_VERSION = '0.9.4'
19
+ AWS_SECRETSMANAGER_JDBC_VERSION = '1.0.12'
19
20
  end
20
21
  end
21
22
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: itamae-plugin-recipe-spark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9
4
+ version: 0.1.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - ichylinux
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-04-16 00:00:00.000000000 Z
11
+ date: 2024-04-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: itamae
@@ -108,6 +108,7 @@ files:
108
108
  - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
109
109
  - lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt
110
110
  - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
111
+ - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt
111
112
  - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
112
113
  - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
113
114
  - lib/itamae/plugin/recipe/spark/version.rb
@@ -130,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
130
131
  - !ruby/object:Gem::Version
131
132
  version: '0'
132
133
  requirements: []
133
- rubygems_version: 3.3.26
134
+ rubygems_version: 3.4.19
134
135
  signing_key:
135
136
  specification_version: 4
136
137
  summary: itamae recipe for apache spark installation