azkaban_scheduler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bb696acb2c67ae0e468b3289302b4066dea786a9
+  data.tar.gz: c9be59214ff6f278d726a8475143634c85fe4734
+SHA512:
+  metadata.gz: a700b32c8abb25fe772212a55f6a568deb46cb1399e7da86c5fc2a52bad57b712da93e249ae411abed96c9e14eed8a4cc0b14f68d7a72ef185470bcb7f541d31
+  data.tar.gz: ade57a7249c77c9ae77d245fdbaa6cad406b60769398ab5accf537c8260d17ea3073086ac681c15d730470547a514e90a66036b207a0717870c1825cb8e99773

data/.gitignore

@@ -0,0 +1,24 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+test/azkaban.yml
+.ruby-version

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Dylan Thacker-Smith
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# AzkabanScheduler
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'azkaban_scheduler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install azkaban_scheduler
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/azkaban_scheduler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+task :default => 'test'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << '.' << 'lib' << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = false
+end

data/azkaban_scheduler.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'azkaban_scheduler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "azkaban_scheduler"
+  spec.version       = AzkabanScheduler::VERSION
+  spec.authors       = ["Dylan Thacker-Smith"]
+  spec.email         = ["Dylan.Smith@shopify.com"]
+  spec.summary       = "Azkaban client that can update the schedule"
+  spec.homepage      = "https://github.com/dylanahsmith/azkaban_scheduler"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency("rubyzip", "~> 1.1")
+  spec.add_dependency("multipart-post", "~> 2.0")
+
+  spec.add_development_dependency("bundler", "~> 1.6")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("minitest", "~> 5.4")
+end

data/lib/azkaban_scheduler.rb

@@ -0,0 +1,6 @@
+require 'azkaban_scheduler/errors'
+require 'azkaban_scheduler/client'
+require 'azkaban_scheduler/job'
+require 'azkaban_scheduler/project'
+require 'azkaban_scheduler/session'
+require 'azkaban_scheduler/version'

data/lib/azkaban_scheduler/client.rb

@@ -0,0 +1,49 @@
+require 'net/http'
+require 'json'
+
+module AzkabanScheduler
+  class Client
+    def initialize(url)
+      uri = URI(url)
+      @http = Net::HTTP.new(uri.host, uri.port)
+      @http.use_ssl = uri.scheme == 'https'
+    end
+
+    def get(path, params=nil, headers=nil)
+      path += "?#{URI.encode_www_form(params)}" if params
+      req = Net::HTTP::Get.new(path)
+      send_request(req, headers)
+    end
+
+    def post(path, params=nil, headers=nil)
+      req = Net::HTTP::Post.new(path)
+      req.set_form_data(params) if params
+      send_request(req, headers)
+    end
+
+    def multipart_post(path, params, headers=nil)
+      req = Net::HTTP::Post::Multipart.new(path, params)
+      send_request(req, headers)
+    end
+
+    private
+
+    def send_request(request, headers)
+      request['Accept'] = 'application/json'
+      headers.each { |name, value| request[name] = value } if headers
+      response = @http.request(request)
+      dump_response(response) if ENV['DUMP_AZKABAN_RESPONSES']
+      response
+    end
+
+    def dump_response(response)
+      puts "HTTP/#{response.http_version} #{response.code} #{response.message}"
+      response.each_header do |name, value|
+        puts "#{name}: #{value}"
+      end
+      puts
+      puts "#{response.body}"
+      puts "-" * 60
+    end
+  end
+end

data/lib/azkaban_scheduler/errors.rb

@@ -0,0 +1,8 @@
+module AzkabanScheduler
+  class AzkabanError < StandardError; end
+  class AuthenticationError < AzkabanError; end
+  class ProjectNotFoundError < AzkabanError; end
+  class ProjectExistsError < AzkabanError; end
+  class InvalidProjectNameError < AzkabanError; end
+  class ProjectDescriptionEmptyError < AzkabanError; end
+end

data/lib/azkaban_scheduler/job.rb

@@ -0,0 +1,13 @@
+module AzkabanScheduler
+  class Job
+    def initialize(params)
+      @params = params
+    end
+
+    def write(io)
+      @params.each do |key, value|
+        io.puts("#{key}=#{value}")
+      end
+    end
+  end
+end

data/lib/azkaban_scheduler/project.rb

@@ -0,0 +1,35 @@
+require 'stringio'
+require 'zip'
+
+module AzkabanScheduler
+  class Project
+    attr_accessor :name, :description, :jobs
+    attr_accessor :id, :version
+
+    def initialize(name, description)
+      @name = name
+      @description = description
+      @jobs = {}
+    end
+
+    def add_job(name, job)
+      @jobs[name] = job
+    end
+
+    def build
+      io = StringIO.new
+      write(io)
+      io.rewind
+      io
+    end
+
+    def write(io)
+      Zip::OutputStream.write_buffer(io) do |out|
+        @jobs.each do |name, job|
+          out.put_next_entry("#{name}.job")
+          job.write(out)
+        end
+      end
+    end
+  end
+end

data/lib/azkaban_scheduler/schedule.rb

@@ -0,0 +1,16 @@
+module AzkabanScheduler
+  class Schedule
+    Stats = Struct.new(:min, :max, :average)
+    attr_accessor :id, :stats
+
+    def initialize(project_name, flow_name, options={})
+      @project_name = project_name
+      @flow_name = flow_name
+      @period = options[:period]
+      @is_recurring = !!options[:period]
+      @time = options[:time] || Time.now.to_i
+      @failure_emails = options[:failure_emails]
+      @success_emails = options[:success_emails]
+    end
+  end
+end

data/lib/azkaban_scheduler/session.rb

@@ -0,0 +1,163 @@
+require 'cgi'
+require 'net/http/post/multipart'
+
+module AzkabanScheduler
+  class Session
+    attr_accessor :id
+
+    def self.start(client, username, password)
+      params = {'action' => 'login', 'username' => username, 'password' => password}
+      response = client.post('/', params)
+      response.error! unless response.kind_of?(Net::HTTPSuccess)
+      result = JSON.parse(response.body)
+      unless result["status"] == "success"
+        error_message = result["error"]
+        if error_message == "Incorrect Login. Username\/Password not found."
+          raise AuthenticationError, error_message
+        end
+        raise AzkabanError, error_message
+      end
+      new(client, result['session.id'])
+    end
+
+    def initialize(client, id)
+      @client = client
+      @id = id
+    end
+
+    def create_project(project)
+      response = @client.post('/manager', {
+        'session.id' => @id,
+        'action' => 'create',
+        'name' => project.name,
+        'description' => project.description,
+      })
+      response.error! unless response.kind_of?(Net::HTTPSuccess)
+      result = JSON.parse(response.body)
+      if result['status'] != 'success'
+        error_message = result['message']
+        if error_message == "Active project with name #{project.name} already exists in db."
+          raise ProjectExistsError, error_message
+        elsif error_message == "Project names must start with a letter, followed by any number of letters, digits, '-' or '_'."
+          raise InvalidProjectNameError, error_message
+        elsif error_message == "Description cannot be empty."
+          raise ProjectDescriptionEmptyError, error_message
+        end
+        raise AzkabanError, error_message
+      end
+      result
+    end
+
+    def upload_project(project)
+      response = @client.multipart_post('/manager', {
+        'session.id' => @id,
+        'ajax' => 'upload',
+        'project' => project.name,
+        'file' => UploadIO.new(project.build, 'application/zip', 'file.zip'),
+      })
+      response.error! unless response.kind_of?(Net::HTTPSuccess)
+      result = JSON.parse(response.body)
+      if error_message = result['error']
+        if error_message == "Installation Failed. Project '#{project.name}' doesn't exist."
+          raise ProjectNotFoundError, error_message
+        end
+        raise AzkabanError, error_message
+      end
+      project.id = result['projectId']
+      project.version = result['version']
+      result
+    end
+
+    def delete_project(project_name)
+      response = @client.get('/manager', {
+        'session.id' => @id,
+        'project' => project_name,
+        'delete' => 'true',
+      })
+      response.error! unless response.kind_of?(Net::HTTPSuccess) || response.kind_of?(Net::HTTPRedirection)
+      cookies = response_cookies(response)
+      unless cookies['azkaban.success.message']
+        error_message = cookies['azkaban.failure.message']
+        if error_message == "Project #{project_name} doesn't exist."
+          return false
+        end
+        raise AzkabanError, error_message
+      end
+      true
+    end
+
+    def list_schedules
+      response = @client.post('/schedule', { 'ajax' => 'loadFlow' }, session_id_cookie)
+      response.error! unless response.kind_of?(Net::HTTPSuccess)
+      result = JSON.parse(response.body)
+      result['items'] || []
+    end
+
+    def remove_schedule(schedule_id)
+      response = @client.post('/schedule', {
+        'action' => 'removeSched',
+        'scheduleId' => schedule_id,
+      }, session_id_cookie)
+      response.error! unless response.kind_of?(Net::HTTPSuccess)
+      result = JSON.parse(response.body)
+      unless result['status'] == 'success'
+        error_message = result['message']
+        return false if error_message == "Schedule with ID #{schedule_id} does not exist"
+        raise AzkabanError, error_message
+      end
+      true
+    end
+
+    def remove_all_schedules(project_name)
+      list_schedules.each do |schedule|
+        next unless schedule['projectname'] == project_name
+        schedule_id = schedule['scheduleid']
+        remove_schedule(schedule_id)
+      end
+    end
+
+    def post_schedule(project_id, project_name, flow, start_time, options={})
+      response = @client.post('/schedule', {
+        'ajax' => 'scheduleFlow',
+        'project' => project_name,
+        'projectName' => project_name,
+        'projectId' => project_id,
+        'flow' => flow,
+        'disabled' => options[:disabled] || '[]',
+        'period' => options[:period] || "1d",
+        'scheduleTime' => start_time.utc.strftime("%I,%M,%p,UTC"),
+        'scheduleDate' => start_time.utc.strftime("%m/%d/%Y"),
+        'is_recurring' => options[:period] ? 'on' : 'off',
+        'concurrentOption' => options[:concurrent_option] || 'skip',
+        'failureEmailsOverride' => (!!options[:failure_emails_override]).to_s,
+        'successEmailsOverride' => (!!options[:success_emails_override]).to_s,
+        'failureAction' => options[:failure_action] || 'finishCurrent',
+        'failureEmails' => Array(options[:failure_emails]).join(', '),
+        'successEmails' => Array(options[:success_emails]).join(', '),
+        'notifyFailureFirst' => (!!options[:notify_failure_first]).to_s,
+        'notifyFailureLast' => (!!options[:notify_failure_last]).to_s,
+      }, session_id_cookie)
+      response.error! unless response.kind_of?(Net::HTTPSuccess)
+      result = JSON.parse(response.body)
+      unless result['status'] == 'success'
+        raise AzkabanError, result['message']
+      end
+      nil
+    end
+
+    private
+
+    def response_cookies(response)
+      response['Set-Cookie'].split(',').each_with_object({}) do |cookie, hash|
+        name, value = cookie.split(';')[0].split('=', 2)
+        value = value.to_s.strip
+        value = value[1...-1] if value[0] == '"'.freeze && value[-1] == '"'.freeze
+        hash[name.strip] = value
+      end
+    end
+
+    def session_id_cookie
+      { 'Cookie' => "azkaban.browser.session.id=#{@id}" }
+    end
+  end
+end

data/lib/azkaban_scheduler/version.rb

@@ -0,0 +1,3 @@
+module AzkabanScheduler
+  VERSION = "0.0.1"
+end

data/test/session_test.rb

@@ -0,0 +1,88 @@
+require 'test_helper'
+
+class SessionTest < Minitest::Test
+  include SessionTestHelper
+
+  def setup
+    @project = AzkabanScheduler::Project.new('Azkaban_Scheduler_Test', 'Used by the test suite')
+    @project.add_job('first', AzkabanScheduler::Job.new(type: 'command', command: 'echo "hello world"'))
+    @@once ||= setup_once || true
+  end
+
+  def setup_once
+    session.remove_all_schedules(@project.name)
+    session.delete_project(@project.name)
+  end
+
+  def test_start
+    refute session.id.empty?
+  end
+
+  def test_start_with_incorrect_password
+    assert_raises(AzkabanScheduler::AuthenticationError) do
+      AzkabanScheduler::Session.start(client, 'foo', 'bar')
+    end
+  end
+
+  def test_create_and_upload_project
+    session.create_project(@project)
+    result = session.upload_project(@project)
+    assert result['projectId']
+    assert result['version']
+    assert_equal result['projectId'], @project.id
+    assert_equal result['version'], @project.version
+    assert_equal true, session.delete_project(@project.name)
+  end
+
+  def test_post_schedule
+    session.create_project(@project)
+    session.upload_project(@project)
+    flow_name = 'first'
+    start_time = Time.now
+    session.post_schedule(@project.id, @project.name, flow_name, start_time,
+                          period: '6h',
+                          failure_emails_override: true,
+                          notifyFailureFirst: true,
+                          failure_emails: ['azkaban-scheduler-test@localhost'])
+    schedule = session.list_schedules.detect { |item| item['projectname'] == @project.name && item['flowname'] == flow_name }
+    assert schedule['scheduleid']
+    assert_equal (start_time.to_i - start_time.sec) * 1000, schedule['time']
+    assert_equal 6 * 60 * 60 * 1000, schedule['period']
+  ensure
+    session.remove_all_schedules(@project.name)
+    session.delete_project(@project.name)
+  end
+
+  def test_create_existing_project
+    session.create_project(@project)
+    assert_raises(AzkabanScheduler::ProjectExistsError) do
+      session.create_project(@project)
+    end
+  ensure
+    session.delete_project(@project.name)
+  end
+
+  def test_create_project_with_invalid_name
+    project = AzkabanScheduler::Project.new('Azkaban Scheduler Test', "description")
+    assert_raises(AzkabanScheduler::InvalidProjectNameError) do
+      session.create_project(project)
+    end
+  end
+
+  def test_create_project_with_invalid_name
+    project = AzkabanScheduler::Project.new('Azkaban_Scheduler_Test', "")
+    assert_raises(AzkabanScheduler::ProjectDescriptionEmptyError) do
+      session.create_project(project)
+    end
+  end
+
+  def test_upload_missing_project
+    assert_raises(AzkabanScheduler::ProjectNotFoundError) do
+      session.upload_project(@project)
+    end
+  end
+
+  def test_delete_missing_project
+    assert_equal false, session.delete_project(@project.name)
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+require 'azkaban_scheduler'
+
+require "minitest/autorun"
+require 'yaml'
+require 'pathname'
+
+
+module SessionTestHelper
+  def config
+    @@config ||= YAML.load(Pathname.new(File.dirname(__FILE__)).join('azkaban.yml').read)
+  end
+
+  def client
+    @@client ||= AzkabanScheduler::Client.new(config['url'])
+  end
+
+  def session
+    @@session ||= AzkabanScheduler::Session.start(client, config['username'], config['password'])
+  end
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: azkaban_scheduler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Dylan Thacker-Smith
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: multipart-post
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: minitest
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.4'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.4'
+  prerelease: false
+  type: :development
+description:
+email:
+- Dylan.Smith@shopify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- azkaban_scheduler.gemspec
+- lib/azkaban_scheduler.rb
+- lib/azkaban_scheduler/client.rb
+- lib/azkaban_scheduler/errors.rb
+- lib/azkaban_scheduler/job.rb
+- lib/azkaban_scheduler/project.rb
+- lib/azkaban_scheduler/schedule.rb
+- lib/azkaban_scheduler/session.rb
+- lib/azkaban_scheduler/version.rb
+- test/session_test.rb
+- test/test_helper.rb
+homepage: https://github.com/dylanahsmith/azkaban_scheduler
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Azkaban client that can update the schedule
+test_files:
+- test/session_test.rb
+- test/test_helper.rb