azkaban-rb 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/Rakefile +1 -0
- data/azkaban-rb.gemspec +22 -0
- data/lib/azkaban-rb.rb +8 -0
- data/lib/azkaban-rb/tasks.rb +129 -0
- data/lib/azkaban-rb/version.rb +5 -0
- metadata +73 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
data/azkaban-rb.gemspec
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "azkaban-rb/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "azkaban-rb"
|
7
|
+
s.version = Azkaban::Rb::VERSION
|
8
|
+
s.authors = ["Matt Hayes"]
|
9
|
+
s.email = ["matthew.terence.hayes@gmail.com"]
|
10
|
+
s.homepage = "https://github.com/matthayes/azkaban-rb"
|
11
|
+
s.summary = %q{Azkaban job generation using Ruby}
|
12
|
+
s.description = %q{azkaban-rb allows Azkaban jobs to be modeled as rake tasks}
|
13
|
+
|
14
|
+
s.rubyforge_project = "azkaban-rb"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
s.add_dependency "httpclient", "~> 2.1.6"
|
22
|
+
end
|
data/lib/azkaban-rb.rb
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
require 'httpclient'
|
2
|
+
|
3
|
+
module Azkaban
|
4
|
+
|
5
|
+
def self.deploy(uri, path, zip_file)
|
6
|
+
client = HTTPClient.new
|
7
|
+
|
8
|
+
client.send_timeout = 1200
|
9
|
+
|
10
|
+
File.open(zip_file) do |file|
|
11
|
+
body = { 'path' => path, 'file' => file }
|
12
|
+
puts "Uploading jobs ZIP file to #{uri}"
|
13
|
+
result = client.post(uri,body)
|
14
|
+
|
15
|
+
# We expect to be redirected after uploading
|
16
|
+
raise "Error while uploading to Azkaban" if result.status != 302
|
17
|
+
|
18
|
+
location = result.header["Location"][0]
|
19
|
+
|
20
|
+
success = /Installation Succeeded/
|
21
|
+
failure = /Installation Failed:\s*(.+)/
|
22
|
+
|
23
|
+
if location =~ success
|
24
|
+
puts "Successfully uploaded to Azkaban"
|
25
|
+
elsif (m = failure.match(location))
|
26
|
+
reason = m[1]
|
27
|
+
raise "Failed to upload to Azkaban: #{reason}"
|
28
|
+
else
|
29
|
+
raise "Failed to upload to Azkaban for unknown reason"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# custom MIME type handler so ZIP is uploaded as 'application/zip' as required by Azkaban UI
|
35
|
+
def self.mime_type_handler(path)
|
36
|
+
case path
|
37
|
+
when /\.txt$/i
|
38
|
+
'text/plain'
|
39
|
+
when /\.zip$/i
|
40
|
+
'application/zip'
|
41
|
+
when /\.(htm|html)$/i
|
42
|
+
'text/html'
|
43
|
+
when /\.doc$/i
|
44
|
+
'application/msword'
|
45
|
+
when /\.png$/i
|
46
|
+
'image/png'
|
47
|
+
when /\.gif$/i
|
48
|
+
'image/gif'
|
49
|
+
when /\.(jpg|jpeg)$/i
|
50
|
+
'image/jpeg'
|
51
|
+
else
|
52
|
+
'application/octet-stream'
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# register our custom MIME handler
|
57
|
+
HTTP::Message.mime_type_handler = Proc.new { |path| Azkaban::mime_type_handler(path) }
|
58
|
+
|
59
|
+
class JobFile
|
60
|
+
|
61
|
+
@output_dir = "conf/"
|
62
|
+
|
63
|
+
def initialize(task, ext)
|
64
|
+
@task = task
|
65
|
+
@ext = ext
|
66
|
+
@args = {}
|
67
|
+
end
|
68
|
+
|
69
|
+
class << self
|
70
|
+
attr_accessor :output_dir
|
71
|
+
end
|
72
|
+
|
73
|
+
def set(params)
|
74
|
+
params.each do |k,v|
|
75
|
+
@args[k] = v
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def write
|
80
|
+
if @args.size > 0
|
81
|
+
file_name = @task.name.gsub(":", "-") + @ext
|
82
|
+
if @task.prerequisites.size > 0
|
83
|
+
scope = @task.scope.map { |s| s.to_s }.join("-")
|
84
|
+
@args["dependencies"] = @task.prerequisites.map{ |p|
|
85
|
+
# look up the prerequisite in the scope of its task
|
86
|
+
prereq_task = Rake.application.lookup(p, @task.scope)
|
87
|
+
prereq_task.name.gsub(":", "-")
|
88
|
+
}.join(",")
|
89
|
+
end
|
90
|
+
create_properties_file(file_name, @args)
|
91
|
+
puts "Created #{file_name}"
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
def create_properties_file(file_name, props)
|
98
|
+
unless File.exists? Azkaban::JobFile.output_dir
|
99
|
+
Dir.mkdir Azkaban::JobFile.output_dir
|
100
|
+
end
|
101
|
+
file = File.new(Azkaban::JobFile.output_dir + file_name, "w+")
|
102
|
+
props.each do |k,v|
|
103
|
+
file.write("#{k}=#{v}\n")
|
104
|
+
end
|
105
|
+
file.close
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
|
111
|
+
def props(*args, &b)
|
112
|
+
task(*args) do |t|
|
113
|
+
unless b.nil?
|
114
|
+
job = Azkaban::JobFile.new(t, ".properties")
|
115
|
+
job.instance_eval(&b)
|
116
|
+
job.write
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def job(*args,&b)
|
122
|
+
task(*args) do |t|
|
123
|
+
unless b.nil?
|
124
|
+
job = Azkaban::JobFile.new(t, ".job")
|
125
|
+
job.instance_eval(&b)
|
126
|
+
job.write
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
metadata
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: azkaban-rb
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.4
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Matt Hayes
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-06-23 00:00:00 -07:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: httpclient
|
18
|
+
prerelease: false
|
19
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
20
|
+
none: false
|
21
|
+
requirements:
|
22
|
+
- - ~>
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 2.1.6
|
25
|
+
type: :runtime
|
26
|
+
version_requirements: *id001
|
27
|
+
description: azkaban-rb allows Azkaban jobs to be modeled as rake tasks
|
28
|
+
email:
|
29
|
+
- matthew.terence.hayes@gmail.com
|
30
|
+
executables: []
|
31
|
+
|
32
|
+
extensions: []
|
33
|
+
|
34
|
+
extra_rdoc_files: []
|
35
|
+
|
36
|
+
files:
|
37
|
+
- .gitignore
|
38
|
+
- Gemfile
|
39
|
+
- Rakefile
|
40
|
+
- azkaban-rb.gemspec
|
41
|
+
- lib/azkaban-rb.rb
|
42
|
+
- lib/azkaban-rb/tasks.rb
|
43
|
+
- lib/azkaban-rb/version.rb
|
44
|
+
has_rdoc: true
|
45
|
+
homepage: https://github.com/matthayes/azkaban-rb
|
46
|
+
licenses: []
|
47
|
+
|
48
|
+
post_install_message:
|
49
|
+
rdoc_options: []
|
50
|
+
|
51
|
+
require_paths:
|
52
|
+
- lib
|
53
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
+
none: false
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: "0"
|
59
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
60
|
+
none: false
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: "0"
|
65
|
+
requirements: []
|
66
|
+
|
67
|
+
rubyforge_project: azkaban-rb
|
68
|
+
rubygems_version: 1.6.2
|
69
|
+
signing_key:
|
70
|
+
specification_version: 3
|
71
|
+
summary: Azkaban job generation using Ruby
|
72
|
+
test_files: []
|
73
|
+
|