itamae-plugin-recipe-spark 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/lib/itamae/plugin/recipe/spark/install.rb +41 -23
  3. data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt +4 -0
  4. data/lib/itamae/plugin/recipe/spark/version.rb +5 -4
  5. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c4a57bd0c4c43b94a896f87b20b4580c10ffe76e2995af89ae2a60e44963554c
4
- data.tar.gz: f61d1f73264122bd76db928b6d85b261fec7ee754cdd06b94f683ec20a08f2f7
3
+ metadata.gz: 6c89c752cc2d79e0c7514b2ef9ce5e4579355ba68e19a22c60f4b273181a6b97
4
+ data.tar.gz: eedd14729fd7fdb8a5afde967984adf09e003d46a1ec874e028ead143a65f4a1
5
5
  SHA512:
6
- metadata.gz: 49e7761e3f424ce810e3bd423e4ade4cf122122fbdff80dffc7d200573bea3d31b19a367e8648fac00bf80796ec23b34a171e24564d94f2579dd9fa64c95278f
7
- data.tar.gz: 343ffa8a4f31b83bdf2543185c3a65832b785cb9734de40c10866cb7d38b7f851db1f7462f91872252dd59291376a7bc56578fc18aaf25d8769f94df0d72e4b9
6
+ metadata.gz: e6a0b3361bb5be777eadcd1e811989b941d5c0754fca305fb230168e69fca936364a7e2231b9b16b86deb546750b2eea983ae03072640b7d07965eb5557d3a82
7
+ data.tar.gz: d1ddab8c7a5c5a96793578f993b234651e767dd9fdb5ec809678336f5a537830258edaac70a62af2fbc8873eede709e08c91477e34032e207035f1e8012b2a44
data/lib/itamae/plugin/recipe/spark/install.rb CHANGED
@@ -19,22 +19,32 @@ execute "download spark-#{version}" do
19
19
  not_if "test -e /opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}/INSTALLED || echo #{::File.read(::File.join(::File.dirname(__FILE__), "spark-#{version}_hadoop_#{hadoop_type}_sha256.txt")).strip} | sha256sum -c"
20
20
  end
21
21
 
22
- spark_redshift_version = Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
- spark_avro_version = Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
22
+ spark_redshift_version = ENV['SPARK_REDSHIFT_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_REDSHIFT_VERSION
23
+ spark_avro_version = ENV['SPARK_AVRO_VERSION'] || Itamae::Plugin::Recipe::Spark::SPARK_AVRO_VERSION
24
24
  minimal_json_version = Itamae::Plugin::Recipe::Spark::MINIMAL_JSON_VERSION
25
- redshift_jdbc_version = Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
25
+ redshift_jdbc_version = ENV['REDSHIFT_JDBC_VERSION'] || Itamae::Plugin::Recipe::Spark::REDSHIFT_JDBC_VERSION
26
26
  fastdoubleparser_version = Itamae::Plugin::Recipe::Spark::FASTDOUBLEPARSER_VERSION
27
27
  jets3t_version = Itamae::Plugin::Recipe::Spark::JETS3T_VERSION
28
+ aws_secretsmanager_jdbc_version = Itamae::Plugin::Recipe::Spark::AWS_SECRETSMANAGER_JDBC_VERSION
28
29
  execute "download spark-redshift-#{spark_redshift_version} and dependencies" do
29
30
  cwd '/tmp'
30
31
  command <<-EOF
31
- wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-').first}/#{spark_redshift_version.split('-').last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
+ wget -q https://repo1.maven.org/maven2/io/github/spark-redshift-community/spark-redshift_#{spark_redshift_version.split('-', 2).first}/#{spark_redshift_version.split('-', 2).last}/spark-redshift_#{spark_redshift_version}.jar -O spark-redshift_#{spark_redshift_version}.jar
32
33
  wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-avro_#{spark_avro_version.split('-').first}/#{spark_avro_version.split('-').last}/spark-avro_#{spark_avro_version}.jar -O spark-avro_#{spark_avro_version}.jar
33
- wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
34
- wget -q https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
- wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
36
- wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
34
+ wget -q https://repo1.maven.org/maven2/com/amazon/redshift/redshift-jdbc42/#{redshift_jdbc_version}/redshift-jdbc42-#{redshift_jdbc_version}.jar -O RedshiftJDBC42-#{redshift_jdbc_version}.jar
35
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
36
+ <<-EOS
37
+ wget -q https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/#{minimal_json_version}/minimal-json-#{minimal_json_version}.jar -O minimal-json-#{minimal_json_version}.jar
38
+ wget -q https://repo1.maven.org/maven2/ch/randelshofer/fastdoubleparser/#{fastdoubleparser_version}/fastdoubleparser-#{fastdoubleparser_version}.jar -O fastdoubleparser-#{fastdoubleparser_version}.jar
39
+ wget -q https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/#{jets3t_version}/jets3t-#{jets3t_version}.jar -O jets3t-#{jets3t_version}.jar
40
+ EOS
41
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
42
+ <<-EOS
43
+ wget -q https://repo1.maven.org/maven2/com/amazonaws/secretsmanager/aws-secretsmanager-jdbc/#{aws_secretsmanager_jdbc_version}/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar -O aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar
44
+ EOS
45
+ end}
37
46
  EOF
47
+
38
48
  not_if "sha256sum -c #{File.join(File.dirname(__FILE__), "spark-redshift_#{spark_redshift_version}_sha256.txt")}"
39
49
  end
40
50
 
@@ -97,23 +107,31 @@ execute 'install spark-redshift jars' do
97
107
  cwd "/opt/spark/spark-#{version}-bin-hadoop#{hadoop_type}"
98
108
  command <<-EOF
99
109
  ls -d $(find jars) | grep 'spark-redshift_[0-9.-]*.jar' | xargs rm -f
100
- cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar \
101
- jars/
110
+ cp -f /tmp/spark-redshift_#{spark_redshift_version}.jar jars/
111
+
102
112
  ls -d $(find jars) | grep 'spark-avro_[0-9.-]*.jar' | xargs rm -f
103
- cp -f /tmp/spark-avro_#{spark_avro_version}.jar \
104
- jars/
105
- ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
106
- cp -f /tmp/minimal-json-#{minimal_json_version}.jar \
107
- jars/
113
+ cp -f /tmp/spark-avro_#{spark_avro_version}.jar jars/
114
+
108
115
  ls -d $(find jars) | grep 'RedshiftJDBC42-[0-9.]*.jar' | xargs rm -f
109
- cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar \
110
- jars/
111
- ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
112
- cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar \
113
- jars/
114
- ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
115
- cp -f /tmp/jets3t-#{jets3t_version}.jar \
116
- jars/
116
+ cp -f /tmp/RedshiftJDBC42-#{redshift_jdbc_version}.jar jars/
117
+
118
+ #{if spark_redshift_version.split('-', 2).last == '5.0.3'
119
+ <<-EOS
120
+ ls -d $(find jars) | grep 'minimal-json-[0-9.]*.jar' | xargs rm -f
121
+ cp -f /tmp/minimal-json-#{minimal_json_version}.jar jars/
122
+
123
+ ls -d $(find jars) | grep 'fastdoubleparser-[0-9.]*.jar' | xargs rm -f
124
+ cp -f /tmp/fastdoubleparser-#{fastdoubleparser_version}.jar jars/
125
+
126
+ ls -d $(find jars) | grep 'jets3t-[0-9.]*.jar' | xargs rm -f
127
+ cp -f /tmp/jets3t-#{jets3t_version}.jar jars/
128
+ EOS
129
+ elsif spark_redshift_version.split('-', 2).last == '6.2.0-spark_3.4'
130
+ <<-EOS
131
+ ls -d $(find jars) | grep 'aws-secretsmanager-jdbc-[0-9.]*.jar' | xargs rm -f
132
+ cp -f /tmp/aws-secretsmanager-jdbc-#{aws_secretsmanager_jdbc_version}.jar jars/
133
+ EOS
134
+ end}
117
135
  EOF
118
136
  end
119
137
 
data/lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt ADDED
@@ -0,0 +1,4 @@
1
+ 97d829e5e47e0b0306bc54549fbfc67516bea15d249581eeb1152f155fdbee49 spark-redshift_2.12-6.2.0-spark_3.4.jar
2
+ ed252b44d546a3df7778debf563eeb535ae07d74c3c18d4163fcbad88cd6ffbf spark-avro_2.12-3.4.2.jar
3
+ dd12dc400549fb055b64b186b7bb05f3e31d417f398fca6755a02394d0302bae RedshiftJDBC42-2.1.0.24.jar
4
+ b90f9d51b6f34918cfdcbb100a486937fbc94eadd60862f2d2574716babe80bd aws-secretsmanager-jdbc-1.0.12.jar
data/lib/itamae/plugin/recipe/spark/version.rb CHANGED
@@ -2,7 +2,7 @@ module Itamae
2
2
  module Plugin
3
3
  module Recipe
4
4
  module Spark
5
- VERSION = "0.1.9"
5
+ VERSION = "0.1.11"
6
6
 
7
7
  SPARK_VERSION = [
8
8
  SPARK_VERSION_MAJOR = '3',
@@ -10,12 +10,13 @@ module Itamae
10
10
  SPARK_VERSION_REVISION = '1'
11
11
  ].join('.')
12
12
 
13
- SPARK_REDSHIFT_VERSION = '2.12-5.0.3'
14
- SPARK_AVRO_VERSION = '2.12-3.0.1'
13
+ SPARK_REDSHIFT_VERSION = '2.12-6.2.0-spark_3.4'
14
+ SPARK_AVRO_VERSION = '2.12-3.4.2'
15
15
  MINIMAL_JSON_VERSION = '0.9.4'
16
- REDSHIFT_JDBC_VERSION = '2.1.0.17'
16
+ REDSHIFT_JDBC_VERSION = '2.1.0.24'
17
17
  FASTDOUBLEPARSER_VERSION = '0.8.0'
18
18
  JETS3T_VERSION = '0.9.4'
19
+ AWS_SECRETSMANAGER_JDBC_VERSION = '1.0.12'
19
20
  end
20
21
  end
21
22
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: itamae-plugin-recipe-spark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9
4
+ version: 0.1.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - ichylinux
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-04-16 00:00:00.000000000 Z
11
+ date: 2024-04-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: itamae
@@ -108,6 +108,7 @@ files:
108
108
  - lib/itamae/plugin/recipe/spark/spark-3.3.2_hadoop_3_sha256.txt
109
109
  - lib/itamae/plugin/recipe/spark/spark-3.4.1_hadoop_3_sha256.txt
110
110
  - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-5.0.3_sha256.txt
111
+ - lib/itamae/plugin/recipe/spark/spark-redshift_2.12-6.2.0-spark_3.4_sha256.txt
111
112
  - lib/itamae/plugin/recipe/spark/templates/hdfs-site.xml.erb
112
113
  - lib/itamae/plugin/recipe/spark/templates/spark-defaults.conf.erb
113
114
  - lib/itamae/plugin/recipe/spark/version.rb
@@ -130,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
130
131
  - !ruby/object:Gem::Version
131
132
  version: '0'
132
133
  requirements: []
133
- rubygems_version: 3.3.26
134
+ rubygems_version: 3.4.19
134
135
  signing_key:
135
136
  specification_version: 4
136
137
  summary: itamae recipe for apache spark installation