adstax-spark-job-manager 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/adstax-spark-job-manager.gemspec +1 -1
  4. data/bin/adstax-spark-job-manager +3 -2
  5. metadata +16 -16
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cc80ab6b9302b75a919ebb704ea9aae8d9fdd4dd
4
- data.tar.gz: ea6fec36162de3c22244ae096674e2d6c4de0f13
3
+ metadata.gz: 0e1ced2b4cea6154d01614e03473512454f6d217
4
+ data.tar.gz: 49410948a5a9b2ff057c44f949e599115913bf06
5
5
  SHA512:
6
- metadata.gz: 3c368582d4553c96e46b7e405cf4c82be59e2448c3a40a1e7f3dc923bba0b3dc8286029ebaae71a8b7350d8a1d20da193af1c2f3bcf95312f09f0bbd419c43c5
7
- data.tar.gz: d7d00b40bbc6d4214805f3b39f0268ecaccf773e0891d5772f7ebbacdbea578275bc7821945821a2d246a5bec1342d2ef49e4b2b93ea5fd838aeadda7ace71d7
6
+ metadata.gz: 7ec013ecc7911db53d0a32c5704efbe40465d16d7f2574fcdad26a9a604499c60b359a7261f08bbbc18f07133027d3e28aad6ba64ad20daad5555f15f14ba08e
7
+ data.tar.gz: 5ae71b20673a5628b8c2ff28bdb74098c657fe765542100285a58486fce09d3db64a379a557e7d13084c6bce9013287e1629e741e498b0077e0e367b3fe0ba3f
data/README.md CHANGED
@@ -20,7 +20,7 @@ Clone this repo and build the gem:
20
20
 
21
21
  $ git clone git://github.com/ShiftForward/adstax-spark-job-manager.git
22
22
  $ gem build adstax-spark-job-manager.gemspec
23
- $ gem install adstax-spark-job-manager-0.0.1.gem
23
+ $ gem install adstax-spark-job-manager-0.2.0.gem
24
24
 
25
25
  ## Usage
26
26
 
data/adstax-spark-job-manager.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "adstax-spark-job-manager"
7
- spec.version = "0.1.0"
7
+ spec.version = "0.2.0"
8
8
  spec.authors = ["ShiftForward"]
9
9
  spec.email = ["info@shiftforward.eu"]
10
10
  spec.summary = "Manage Spark jobs running on an AdStax cluster."
data/bin/adstax-spark-job-manager CHANGED
@@ -12,7 +12,8 @@ require 'tempfile'
12
12
  # -----------------
13
13
 
14
14
  MAIN_CLASS = 'eu.shiftforward.adstax.spark.SparkJobRunner'
15
- SPARK_EXECUTOR_URI = 'https://s3.amazonaws.com/shiftforward-public/bin/spark/spark-2.0.0-SNAPSHOT-bin-2.4.0.tgz'
15
+ SPARK_EXECUTOR_URI = 'https://s3.amazonaws.com/shiftforward-public/bin/spark/spark-2.0.0-bin-hadoop2.4.tgz'
16
+ ELASTICSEARCH_SPARK_JAR = 'https://s3.amazonaws.com/shiftforward-public/bin/spark/elasticsearch-spark_2.11-2.3.4.jar'
16
17
  SPARK_SCALA_VERSION = '2.11' # TODO: Support other versions and use different executors
17
18
 
18
19
  # -----------------
@@ -150,7 +151,7 @@ def submit_job(jar, job)
150
151
  'SPARK_SCALA_VERSION' => SPARK_SCALA_VERSION
151
152
  },
152
153
  'sparkProperties' => {
153
- 'spark.jars' => $cli_args[:jar],
154
+ 'spark.jars' => "#{ELASTICSEARCH_SPARK_JAR},#{$cli_args[:jar]}",
154
155
  'spark.driver.supervise' => 'false',
155
156
  'spark.app.name' => MAIN_CLASS,
156
157
  'spark.es.port' => '49200',
metadata CHANGED
@@ -1,83 +1,83 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: adstax-spark-job-manager
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - ShiftForward
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-21 00:00:00.000000000 Z
11
+ date: 2016-08-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: file-tail
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: '1.1'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.1'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: json
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: '1.8'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.8'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: colorize
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ~>
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0.7'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ~>
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0.7'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: bundler
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ~>
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
61
  version: '1.7'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: '1.7'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: rake
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ~>
73
+ - - "~>"
74
74
  - !ruby/object:Gem::Version
75
75
  version: '10.0'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ~>
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: '10.0'
83
83
  description: Allow submitting, querying the status, outputting the log and killing
@@ -89,7 +89,7 @@ executables:
89
89
  extensions: []
90
90
  extra_rdoc_files: []
91
91
  files:
92
- - .gitignore
92
+ - ".gitignore"
93
93
  - Gemfile
94
94
  - LICENSE
95
95
  - README.md
@@ -106,17 +106,17 @@ require_paths:
106
106
  - lib
107
107
  required_ruby_version: !ruby/object:Gem::Requirement
108
108
  requirements:
109
- - - '>='
109
+ - - ">="
110
110
  - !ruby/object:Gem::Version
111
111
  version: 2.0.0
112
112
  required_rubygems_version: !ruby/object:Gem::Requirement
113
113
  requirements:
114
- - - '>='
114
+ - - ">="
115
115
  - !ruby/object:Gem::Version
116
116
  version: '0'
117
117
  requirements: []
118
118
  rubyforge_project:
119
- rubygems_version: 2.4.6
119
+ rubygems_version: 2.5.1
120
120
  signing_key:
121
121
  specification_version: 4
122
122
  summary: Manage Spark jobs running on an AdStax cluster.