adstax-spark-job-manager 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc80ab6b9302b75a919ebb704ea9aae8d9fdd4dd
+  data.tar.gz: ea6fec36162de3c22244ae096674e2d6c4de0f13
+SHA512:
+  metadata.gz: 3c368582d4553c96e46b7e405cf4c82be59e2448c3a40a1e7f3dc923bba0b3dc8286029ebaae71a8b7350d8a1d20da193af1c2f3bcf95312f09f0bbd419c43c5
+  data.tar.gz: d7d00b40bbc6d4214805f3b39f0268ecaccf773e0891d5772f7ebbacdbea578275bc7821945821a2d246a5bec1342d2ef49e4b2b93ea5fd838aeadda7ace71d7

data/.gitignore

@@ -0,0 +1,17 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.iml
+
+*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in adstax-spark-job-manager.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,15 @@
+This software is licensed under the Apache 2 license, quoted below.
+
+Copyright 2016 ShiftForward, S.A. [http://www.shiftforward.eu]
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
+
+    [http://www.apache.org/licenses/LICENSE-2.0]
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.

data/README.md

@@ -0,0 +1,101 @@
+# AdStax Spark Job Manager
+
+The AdStax Spark Job Manager is a gem to manager Spark jobs running in an AdStax
+cluster.
+
+## Installation
+
+### From RubyGems
+
+Make sure you have [ruby][ruby-install] (at least v2.0.0) installed, and just
+run:
+
+    $ gem install adstax-spark-job-manager
+
+[ruby-install]: https://www.ruby-lang.org/en/documentation/installation/
+
+### From source
+
+Clone this repo and build the gem:
+
+    $ git clone git://github.com/ShiftForward/adstax-spark-job-manager.git
+    $ gem build adstax-spark-job-manager.gemspec
+    $ gem install adstax-spark-job-manager-0.0.1.gem
+
+## Usage
+
+The AdStax Spark Job Manager publishes an `adstax-spark-job-manager` binary,
+which provides a set of utilities to submit, kill and query the status of Spark
+jobs running on an AdStax cluster. See the help for the command (running it with
+`-h`) for more details.
+
+The methods available are `submit`, `kill`, `status` or `log`. To submit a job,
+one has to provide the task with the `--adstax-host` parameter (pointing to
+where the AdStax instance is running), a `--jar` parameter pointing to a bundled
+jar with your application, all required dependencies, and which includes at
+least one implementation of `eu.shiftforward.adstax.spark.SparkJob`. Note that
+you don't need to bundle the `spark-core` dependency, as it will be provided at
+runtime. The `--job` parameter should be the fully qualified name of the class
+extending `eu.shiftforward.adstax.spark.SparkJob` and which is going to be used
+as the Spark job to run. Everything following the required parameters will be
+used as arguments for the `SparkJob`. For example, in order to submit the
+`SparkPI` example, one can use the following command:
+
+```
+$ adstax-spark-job-manager submit --adstax-host sample-adstax-instance.dev.adstax.io --jar http://s3.amazonaws.com/shiftforward-public/bin/spark/adstax-spark-examples-1.0.jar --job eu.shiftforward.adstax.spark.examples.SparkPi 100
+```
+
+This command should return information about the submission, for example:
+
+```
+{
+  "action" : "CreateSubmissionResponse",
+  "serverSparkVersion" : "2.0.0-SNAPSHOT",
+  "submissionId" : "driver-20160713161243-0002",
+  "success" : true
+}
+```
+
+You can now use the returned submission id to query the status of the job, as
+well as list its standard output. In order to query the status of the job, use
+the `status` command:
+
+```
+$ adstax-spark-job-manager status --adstax-host sample-adstax-instance.dev.adstax.io --submission-id driver-20160713161243-0002
+{
+  "action" : "SubmissionStatusResponse",
+  "driverState" : "FINISHED",
+  "message" : "task_id {\n  value: \"driver-20160713161243-0002\"\n}\nstate: TASK_FINISHED\nmessage: \"Command exited with status 0\"\nslave_id {\n  value: \"9f18159e-ebe9-4a70-89e1-9774adf2cdd6-S9\"\n}\ntimestamp: 1.468426400438861E9\nexecutor_id {\n  value: \"driver-20160713161243-0002\"\n}\nsource: SOURCE_EXECUTOR\n11: \"A\\371\\330\\365+\\027Ds\\237\\243\\\"\\317\\276\\353\\363\\367\"\n13: \"\\n\\036\\022\\f10.0.174.173*\\016\\022\\f10.0.174.173\"\n",
+  "serverSparkVersion" : "2.0.0-SNAPSHOT",
+  "submissionId" : "driver-20160713161243-0002",
+  "success" : true
+}
+```
+
+The `log` command allows you to output the stdout and stderr of the job's
+driver. You can hide the stderr with the `--hide-stderr` command and keep
+tailing the output with the `--follow` command:
+
+```
+$ adstax-spark-job-manager log --adstax-host sample-adstax-instance.dev.adstax.io --submission-id driver-20160713161243-0002 --hide-stderr --follow
+Registered executor on ec2-54-87-240-29.compute-1.amazonaws.com
+Starting task driver-20160713161243-0002
+Forked command at 22260
+sh -c 'cd spark-2*;  bin/spark-submit --name eu.shiftforward.adstax.spark.SparkJobRunner --master mesos://zk://zk.sample-adstax-instance.dev.adstax.io:2181/mesos --driver-cores 1.0 --driver-memory 1024M --class eu.shiftforward.adstax.spark.SparkJobRunner --conf spark.driver.supervise=false --conf spark.app.name=eu.shiftforward.adstax.spark.SparkJobRunner --conf spark.es.port=49200 --conf spark.es.nodes=localhost --conf spark.mesos.coarse=false --conf spark.executor.uri=https://s3.amazonaws.com/shiftforward-public/bin/spark/spark-2.0.0-SNAPSHOT-bin-2.4.0.tgz ../adstax-spark-examples-1.0.jar --job eu.shiftforward.adstax.spark.examples.SparkPi 100'
+Pi is roughly 3.1407
+Command exited with status 0 (pid: 22260)
+```
+
+The `kill` command allows you to cancel and kill an ongoing job. Killing already
+finished jobs has no effect:
+
+```
+$ adstax-spark-job-manager kill --adstax-host sample-adstax-instance.dev.adstax.io --submission-id driver-20160713161243-0002
+{
+  "action" : "KillSubmissionResponse",
+  "message" : "Driver already terminated",
+  "serverSparkVersion" : "2.0.0-SNAPSHOT",
+  "submissionId" : "driver-20160713161243-0002",
+  "success" : false
+}
+```

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+task "publish" do
+  gem_helper = Bundler::GemHelper.instance
+  built_gem_path = gem_helper.build_gem
+  Process.wait spawn("gem nexus #{built_gem_path}")
+  Bundler.ui.confirm "#{gem_helper.gemspec.name} (#{gem_helper.gemspec.version}) pushed to Nexus."
+end

data/adstax-spark-job-manager.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "adstax-spark-job-manager"
+  spec.version       = "0.1.0"
+  spec.authors       = ["ShiftForward"]
+  spec.email         = ["info@shiftforward.eu"]
+  spec.summary       = "Manage Spark jobs running on an AdStax cluster."
+  spec.description   = "Allow submitting, querying the status, outputting the log and killing Spark jobs on an AdStax cluster."
+
+  spec.licenses      = ['Apache-2.0']
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.0.0'
+
+  spec.add_runtime_dependency "file-tail", "~> 1.1"
+  spec.add_runtime_dependency "json", "~> 1.8"
+  spec.add_runtime_dependency "colorize", "~> 0.7"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/bin/adstax-spark-job-manager

@@ -0,0 +1,325 @@
+#!/usr/bin/env ruby
+
+require 'colorize'
+require 'file-tail'
+require 'json'
+require 'net/http'
+require 'optparse'
+require 'tempfile'
+
+# -----------------
+# Constants
+# -----------------
+
+MAIN_CLASS = 'eu.shiftforward.adstax.spark.SparkJobRunner'
+SPARK_EXECUTOR_URI = 'https://s3.amazonaws.com/shiftforward-public/bin/spark/spark-2.0.0-SNAPSHOT-bin-2.4.0.tgz'
+SPARK_SCALA_VERSION = '2.11' # TODO: Support other versions and use different executors
+
+# -----------------
+# CLI arguments parsing
+# -----------------
+
+$cli_args = {
+  follow: false,
+  show_stderr: true
+}
+
+ARGV << '-h' if ARGV.empty?
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} <action> --adstax-host <adstax_host> [<options>]"
+  opts.separator ''
+  opts.separator 'Submit, kill, query the status, or inspect the log of a Spark job running in an AdStax cluster.'
+  opts.separator "<action> is one of 'submit', 'kill', 'status' or 'log'."
+  opts.separator "Example: #{$PROGRAM_NAME} submit --adstax-host apollo.dev.adstax.io --jar http://s3.amazonaws.com/shiftforward-public/bin/spark/adstax-spark-examples-1.0.jar --job eu.shiftforward.adstax.spark.examples.SparkPi 1000"
+  opts.separator "Example: #{$PROGRAM_NAME} kill driver-20160420105830-0001"
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('--adstax-host STRING', 'Host suffix to the AdStax cluster services.') do |host_suffix|
+    $cli_args[:host_suffix] = host_suffix
+  end
+
+  opts.on('--jar STRING',
+          'Path to a bundled jar including your application and all dependencies.',
+          'The URL must be globally visible inside of your cluster.') do |jar|
+    $cli_args[:jar] = jar
+  end
+
+  opts.on('--job STRING',
+          'Fully qualified name of the class extending `eu.shiftforward.adstax.spark.SparkJob`.',
+          'The class will be used as the Spark job to run.') do |job|
+    $cli_args[:job] = job
+  end
+
+  opts.on('--submission-id STRING',
+          'Id of the submission (required for the kill, status and log actions).') do |submission_id|
+    $cli_args[:submission_id] = submission_id
+  end
+
+  opts.on('-f', '--follow',
+          "Enables following the file updates in the 'log' action.") do
+    $cli_args[:follow] = true
+  end
+
+  opts.on('--hide-stderr',
+          "Hides stderr output in the 'log' action.") do
+    $cli_args[:show_stderr] = false
+  end
+
+  opts.on_tail('-h', '--help', 'Show this message.') do
+    puts opts
+    exit
+  end
+end.parse!
+
+def warn_missing(name)
+  puts "Missing required argument: #{name}"
+  exit 1
+end
+
+def get_http(uri)
+  uri = URI.parse(uri)
+  Net::HTTP.new(uri.host, uri.port)
+end
+
+def get_task(state_response, task_id)
+  target_tasks = []
+  state_response['completed_frameworks'].concat(state_response['frameworks']).each do |framework|
+    framework['completed_tasks'].concat(framework['tasks']).each do |task|
+      target_tasks.push(task) if task['id'] == task_id
+    end
+  end
+  target_tasks[0]
+end
+
+def get_executor(state_response, task_id)
+  target_executors = []
+  state_response['completed_frameworks'].concat(state_response['frameworks']).each do |framework|
+    framework['completed_executors'].concat(framework['executors']).each do |executor|
+      target_executors.push(executor) if executor['id'] == task_id
+    end
+  end
+  target_executors[0]
+end
+
+def mesos_download(http, remote_file, local_file)
+  params = { path: remote_file }
+  encoded_params = URI.encode_www_form(params)
+  file_response = http.request(Net::HTTP::Get.new(['/files/download', encoded_params].join('?')))
+  unless file_response.class.body_permitted?
+    puts 'Unable to fetch file from slave'
+    exit 1
+  end
+  local_file.rewind
+  local_file.write(file_response.body)
+end
+
+def tail_file(file, output_method = Proc.new { |line| puts line })
+  Thread.new do
+    File.open(file.path) do |log|
+      log.extend(File::Tail)
+      log.interval = 1
+      log.backward(10)
+      begin
+        log.tail { |line| output_method.call(line) }
+      rescue Interrupt => e
+        exit 1
+      end
+    end
+  end
+end
+
+$action = ARGV.shift || begin
+  warn_missing('action')
+end
+
+warn_missing('--adstax-host') unless $cli_args[:host_suffix]
+$cluster_dispatcher_host = "http://spark-cluster-dispatcher.#{$cli_args[:host_suffix]}:7077"
+
+def submit_job(jar, job)
+  uri = URI.parse($cluster_dispatcher_host)
+  http = Net::HTTP.new(uri.host, uri.port)
+  payload = {
+    'action' => 'CreateSubmissionRequest',
+    'appArgs' => ['--job', job].concat(ARGV),
+    'appResource' => jar,
+    'mainClass' => MAIN_CLASS,
+    'clientSparkVersion' => '1.6.1',
+    'environmentVariables' => {
+      'SPARK_SCALA_VERSION' => SPARK_SCALA_VERSION
+    },
+    'sparkProperties' => {
+      'spark.jars' => $cli_args[:jar],
+      'spark.driver.supervise' => 'false',
+      'spark.app.name' => MAIN_CLASS,
+      'spark.es.port' => '49200',
+      'spark.es.nodes' => 'localhost',
+      'spark.submit.deployMode' => 'cluster',
+      'spark.mesos.coarse' => 'false',
+      'spark.master' => "mesos://spark-cluster-dispatcher.#{$cli_args[:host_suffix]}:7077",
+      'spark.executor.uri' => SPARK_EXECUTOR_URI
+    }
+  }.to_json
+  request = Net::HTTP::Post.new(
+    '/v1/submissions/create',
+    initheader = { 'Content-Type' => 'application/json' })
+  request.body = payload
+  http.request(request)
+end
+
+def kill_job(submission_id)
+  uri = URI.parse($cluster_dispatcher_host)
+  http = Net::HTTP.new(uri.host, uri.port)
+  request = Net::HTTP::Post.new("/v1/submissions/kill/#{submission_id}")
+  http.request(request)
+end
+
+def status_job(submission_id)
+  uri = URI.parse($cluster_dispatcher_host)
+  http = Net::HTTP.new(uri.host, uri.port)
+  request = Net::HTTP::Get.new("/v1/submissions/status/#{submission_id}")
+  http.request(request)
+end
+
+def log_job(submission_id, follow, show_stderr)
+  status_response = JSON.parse(status_job(submission_id).body)
+  if status_response['driverState'] == "NOT_FOUND"
+    puts "Unable to find submission with id #{submission_id}"
+    exit 1
+  end
+  if status_response['driverState'] == "QUEUED"
+    puts "Submission with id #{submission_id} is still queued for execution"
+    if follow
+      print "Waiting for submission with id #{submission_id} to start"
+      waiting_thread = Thread.new do
+        queued = true
+        while queued do
+          begin
+            sleep 1
+            print "."
+          rescue Interrupt => e
+            exit 1
+          end
+          res = JSON.parse(status_job(submission_id).body)
+          queued = res['driverState'] == "QUEUED"
+        end
+      end
+      waiting_thread.join
+      puts ""
+    else
+      exit 1
+    end
+  end
+  marathon_http = get_http("http://marathon.#{$cli_args[:host_suffix]}")
+  marathon_response = marathon_http.request(Net::HTTP::Get.new('/v2/info'))
+  unless marathon_response.class.body_permitted?
+    puts 'Unable to fetch Mesos leader url from Marathon'
+    exit 1
+  end
+  res = JSON.parse(marathon_response.body)
+  mesos_http = get_http(res['marathon_config']['mesos_leader_ui_url'])
+  mesos_response = mesos_http.request(Net::HTTP::Get.new('/state.json'))
+  unless mesos_response.class.body_permitted?
+    puts 'Unable to fetch Mesos status'
+    exit 1
+  end
+  res = JSON.parse(mesos_response.body)
+  target_task = get_task(res, submission_id)
+  unless target_task
+    puts "Unable to find submission with id #{submission_id} in Mesos. Maybe the submission is too old?"
+    exit 1
+  end
+  slaves = res['slaves']
+  slave_id = target_task['slave_id']
+  target_slaves = slaves.select do |slave|
+    slave['id'] == slave_id
+  end
+  if target_slaves.empty?
+    puts "Unable to find slave with id #{slave_id}"
+    exit 1
+  end
+  if target_slaves.length != 1
+    puts "Multiple slaves with id #{slave_id}"
+    exit 1
+  end
+  target_slave = target_slaves[0]
+  slave_http = get_http('http://' + target_slave['hostname'] + ':5051')
+  slave_response = slave_http.request(Net::HTTP::Get.new('/state.json'))
+  unless slave_response.class.body_permitted?
+    puts 'Unable to fetch file from slave'
+    exit 1
+  end
+  res = JSON.parse(slave_response.body)
+  target_executor = get_executor(res, submission_id)
+  unless target_executor
+    puts "Unable to find submission with id #{submission_id} in executor. Maybe the submission is too old?"
+    exit 1
+  end
+  directory = target_executor['directory']
+  stdout_file = Tempfile.new('spark' + submission_id)
+  stderr_file = Tempfile.new('spark' + submission_id)
+  threads = []
+  if follow
+    threads.push(Thread.new do
+                   loop do
+                     begin
+                       sleep 1
+                     rescue Interrupt => e
+                       exit 1
+                     end
+                     mesos_download(slave_http, directory + '/stdout', stdout_file)
+                     mesos_download(slave_http, directory + '/stderr', stderr_file)
+                   end
+                 end)
+  else
+    mesos_download(slave_http, directory + '/stdout', stdout_file)
+    mesos_download(slave_http, directory + '/stderr', stderr_file)
+  end
+  if follow
+    threads.push(tail_file(stdout_file))
+    threads.push(tail_file(stderr_file, Proc.new { |line| puts line.chomp.red })) if show_stderr
+    begin
+      threads.each { |thread| thread.join }
+    rescue Interrupt => e
+      exit 1
+    end
+  else
+    if show_stderr
+      stderr_file.rewind
+      puts stderr_file.read.chomp.red
+    end
+    stdout_file.rewind
+    puts stdout_file.read
+  end
+end
+
+# -----------------
+# Program start
+# -----------------
+
+case $action
+when 'submit'
+  warn_missing('--jar') unless $cli_args[:jar]
+  warn_missing('--job') unless $cli_args[:job]
+  response = submit_job($cli_args[:jar], $cli_args[:job])
+  puts response.body
+
+when 'kill'
+  warn_missing('--submission_id') unless $cli_args[:submission_id]
+  response = kill_job($cli_args[:submission_id])
+  puts response.body
+
+when 'status'
+  warn_missing('--submission_id') unless $cli_args[:submission_id]
+  response = status_job($cli_args[:submission_id])
+  puts response.body
+
+when 'log'
+  warn_missing('--submission_id') unless $cli_args[:submission_id]
+  log_job($cli_args[:submission_id], $cli_args[:follow], $cli_args[:show_stderr])
+
+else
+  puts "Unrecognized action: #{$action}"
+  exit 1
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: adstax-spark-job-manager
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- ShiftForward
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: file-tail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Allow submitting, querying the status, outputting the log and killing
+  Spark jobs on an AdStax cluster.
+email:
+- info@shiftforward.eu
+executables:
+- adstax-spark-job-manager
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- adstax-spark-job-manager.gemspec
+- bin/adstax-spark-job-manager
+homepage: 
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Manage Spark jobs running on an AdStax cluster.
+test_files: []