jackal 0.1.16 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -1
- data/bin/jackal +16 -1
- data/jackal.gemspec +5 -3
- data/lib/jackal.rb +2 -0
- data/lib/jackal/callback.rb +38 -0
- data/lib/jackal/formatter.rb +56 -0
- data/lib/jackal/loader.rb +53 -42
- data/lib/jackal/utils.rb +3 -0
- data/lib/jackal/utils/config.rb +12 -2
- data/lib/jackal/utils/constants.rb +19 -0
- data/lib/jackal/utils/process.rb +52 -0
- data/lib/jackal/utils/spec/helpers.rb +1 -2
- data/lib/jackal/utils/spec/loader.rb +0 -2
- data/lib/jackal/version.rb +1 -1
- metadata +48 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 914f2d57a1083c0adc67afe2bda1f310201779bd
|
4
|
+
data.tar.gz: 2323ad17173d7dd0d2325cdd36de6399abc42fca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ba7be9cc5ad19e6ddf23086226291f7cda5b58efea577309d52b4bf17ed39408e5a3ace5ac8ed01391fe1eab16b33675fbaa988472d8d4a9340e75d04b559755
|
7
|
+
data.tar.gz: e09419edd04c3f59efa6f41c9e0d4579e61480c77a8d5f98545aaea15e5e4251fa0c0607cca15be0e147d24aeeb79b23800385d314f8fbdc3206df509d76d5d2
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
|
+
# v0.2.0
|
2
|
+
* Add abstract payload formatter
|
3
|
+
* Provide common helper method to shell out
|
4
|
+
* Remove custom CLI and integrate bogo-cli
|
5
|
+
|
1
6
|
# v0.1.16
|
2
7
|
* Load HTTP hook directly prior to service start
|
3
|
-
|
8
|
+
* Fixes some issues with spec loads
|
4
9
|
|
5
10
|
# v0.1.14
|
6
11
|
* Move orphan handlers into source init arg
|
data/bin/jackal
CHANGED
@@ -1,3 +1,18 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
require 'bogo-cli'
|
2
3
|
require 'jackal'
|
3
|
-
|
4
|
+
|
5
|
+
Bogo::Cli::Setup.define do
|
6
|
+
|
7
|
+
on :v, :version, 'Print version' do
|
8
|
+
puts "jackal - Message Pipeline - [Version: #{Jackal::VERSION}]"
|
9
|
+
exit
|
10
|
+
end
|
11
|
+
on :c, :config=, 'Configuration file or directory'
|
12
|
+
on :V, :verbosity=, 'Log verbosity (debug info warn error)'
|
13
|
+
run do |opts, args|
|
14
|
+
Jackal::Loader.run!(opts)
|
15
|
+
exit
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
data/jackal.gemspec
CHANGED
@@ -10,9 +10,11 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.description = 'Message processing helper'
|
11
11
|
s.require_path = 'lib'
|
12
12
|
s.license = 'Apache 2.0'
|
13
|
-
s.add_dependency 'carnivore', '>= 0.
|
14
|
-
s.add_dependency '
|
15
|
-
s.
|
13
|
+
s.add_dependency 'carnivore', '>= 0.3.0', '< 1.0.0'
|
14
|
+
s.add_dependency 'bogo-cli', '~> 0.1'
|
15
|
+
s.add_dependency 'bogo-config', '>= 0.1.6', '< 1.0.0'
|
16
|
+
s.add_dependency 'childprocess'
|
17
|
+
s.files = Dir['{lib,bin}/**/**/*'] + %w(jackal.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
|
16
18
|
s.executables << 'jackal'
|
17
19
|
s.executables << 'jackal-test'
|
18
20
|
end
|
data/lib/jackal.rb
CHANGED
@@ -4,7 +4,9 @@ module Jackal
|
|
4
4
|
autoload :Cli, 'jackal/cli'
|
5
5
|
autoload :Callback, 'jackal/callback'
|
6
6
|
autoload :Error, 'jackal/error'
|
7
|
+
autoload :Formatter, 'jackal/formatter'
|
7
8
|
autoload :Utils, 'jackal/utils'
|
9
|
+
autoload :Loader, 'jackal/loader'
|
8
10
|
end
|
9
11
|
|
10
12
|
require 'jackal/version'
|
data/lib/jackal/callback.rb
CHANGED
@@ -6,8 +6,33 @@ module Jackal
|
|
6
6
|
|
7
7
|
include Utils::Payload
|
8
8
|
include Utils::Config
|
9
|
+
include Utils::Config
|
9
10
|
# @!parse include Jackal::Utils::Payload
|
10
11
|
# @!parse include Jackal::Utils::Config
|
12
|
+
# @!parse include Jackal::Utils::Constants
|
13
|
+
|
14
|
+
# @return [Array<Formatter>] formatters
|
15
|
+
attr_reader :formatters
|
16
|
+
|
17
|
+
# Create new instance
|
18
|
+
#
|
19
|
+
# @return [self]
|
20
|
+
def initialize(*_)
|
21
|
+
super
|
22
|
+
if(service_config[:formatters])
|
23
|
+
@formatters = service_config[:formatters].map do |klass_name|
|
24
|
+
constantize(klass_name).new
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# @return [Utils::Process]
|
30
|
+
def process_manager
|
31
|
+
unless(@_process)
|
32
|
+
@_process = Utils::Process.new
|
33
|
+
end
|
34
|
+
@_process
|
35
|
+
end
|
11
36
|
|
12
37
|
# Validity of message
|
13
38
|
#
|
@@ -86,9 +111,22 @@ module Jackal
|
|
86
111
|
# @param message [Carnivore::Message]
|
87
112
|
def job_completed(name, payload, message)
|
88
113
|
info "Processing of message #{message} has completed within this component #{name}"
|
114
|
+
if(formatters)
|
115
|
+
apply_formatters!(payload)
|
116
|
+
end
|
89
117
|
message.confirm!
|
90
118
|
forward(payload)
|
91
119
|
end
|
92
120
|
|
121
|
+
# Apply configured formatters to payload
|
122
|
+
#
|
123
|
+
# @param payload [Smash]
|
124
|
+
# @return [Smash]
|
125
|
+
def apply_formatters!(payload)
|
126
|
+
formatters.each do |formatter|
|
127
|
+
formatter.format(payload)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
93
131
|
end
|
94
132
|
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'jackal'
|
2
|
+
|
3
|
+
module Jackal
|
4
|
+
# Payload formatter
|
5
|
+
class Formatter
|
6
|
+
|
7
|
+
class << self
|
8
|
+
|
9
|
+
# Register formatter
|
10
|
+
def inherited(klass)
|
11
|
+
Formatter.descendants.push(klass).uniq!
|
12
|
+
end
|
13
|
+
|
14
|
+
# @return [Array<Class>] registered formatters
|
15
|
+
def descendants
|
16
|
+
@_descendants ||= []
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [String, Symbol]
|
22
|
+
SOURCE = nil
|
23
|
+
# @return [String, Symbol]
|
24
|
+
DESTINATION = nil
|
25
|
+
|
26
|
+
# Create a new instance
|
27
|
+
#
|
28
|
+
# @return [self]
|
29
|
+
def initialize
|
30
|
+
[:SOURCE, :DESTINATION].each do |key|
|
31
|
+
unless(self.class.const_get(key))
|
32
|
+
raise NotImplementedError.new("Formatter class must define #{key} constant")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# @return [Symbol]
|
38
|
+
def source
|
39
|
+
self.class.const_get(:SOURCE).to_sym
|
40
|
+
end
|
41
|
+
|
42
|
+
# @return [Symbol]
|
43
|
+
def destination
|
44
|
+
self.class.const_get(:DESTINATION).to_sym
|
45
|
+
end
|
46
|
+
|
47
|
+
# Apply format to payload
|
48
|
+
#
|
49
|
+
# @param payload [Smash]
|
50
|
+
# @return payload [Smash]
|
51
|
+
def format(payload)
|
52
|
+
raise NotImplementedError
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
data/lib/jackal/loader.rb
CHANGED
@@ -1,57 +1,68 @@
|
|
1
1
|
require 'carnivore'
|
2
2
|
require 'jackal'
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
4
|
+
module Jackal
|
5
|
+
class Loader
|
6
|
+
class << self
|
7
|
+
|
8
|
+
# Run the jackal
|
9
|
+
#
|
10
|
+
# @param opts [Hash]
|
11
|
+
def run!(opts)
|
9
12
|
|
10
|
-
|
13
|
+
if(ENV['JACKAL_TESTING_MODE'] && !opts[:config])
|
14
|
+
Carnivore.configure!(:verify)
|
15
|
+
else
|
16
|
+
Carnivore.configure!(opts[:config])
|
17
|
+
Carnivore::Config.immutable!
|
18
|
+
end
|
11
19
|
|
12
|
-
Celluloid.logger.level = Celluloid.logger.class.const_get(
|
13
|
-
|
14
|
-
)
|
20
|
+
Celluloid.logger.level = Celluloid.logger.class.const_get(
|
21
|
+
(opts[:verbosity] || Carnivore::Config[:verbosity] || :debug).to_s.upcase
|
22
|
+
)
|
15
23
|
|
16
|
-
|
17
|
-
|
18
|
-
end
|
24
|
+
Carnivore::Config.fetch(:jackal, :require, []).each do |path|
|
25
|
+
require path
|
26
|
+
end
|
19
27
|
|
20
|
-
begin
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
28
|
+
begin
|
29
|
+
Carnivore::Config.to_smash.each do |namespace, args|
|
30
|
+
next unless args.is_a?(Hash)
|
31
|
+
args.each do |key, opts|
|
32
|
+
next unless opts.is_a?(Hash) && opts[:sources]
|
33
|
+
Carnivore::Utils.debug "Processing: #{opts.inspect}"
|
34
|
+
Carnivore.configure do
|
35
|
+
opts.fetch(:sources, {}).each do |kind, source_args|
|
36
|
+
source = Carnivore::Source.build(
|
37
|
+
:type => source_args[:type].to_sym,
|
38
|
+
:args => source_args.fetch(:args, {}).merge(:name => "#{namespace}_#{key}_#{kind}"),
|
39
|
+
:orphan_callback => lambda{|message|
|
40
|
+
error "No callbacks matched message. Failed to process. Removed from bus. (#{message})"
|
41
|
+
message.confirm!
|
42
|
+
}
|
43
|
+
)
|
44
|
+
Carnivore::Utils.info "Registered new source: #{namespace}_#{key}_#{kind}"
|
45
|
+
if(kind == :input)
|
46
|
+
opts.fetch(:callbacks, []).each do |klass_name|
|
47
|
+
klass = Utils.constantize(klass_name)
|
48
|
+
source.add_callback(klass_name, klass)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
41
52
|
end
|
42
|
-
source.add_callback(klass_name, klass)
|
43
53
|
end
|
44
54
|
end
|
55
|
+
Jackal::Utils.load_http_hook
|
56
|
+
Carnivore.start!
|
57
|
+
rescue => e
|
58
|
+
$stderr.puts "Unexpected failure encountered: #{e.class}: #{e}"
|
59
|
+
if(ENV['DEBUG'])
|
60
|
+
$stderr.puts "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
61
|
+
end
|
62
|
+
exit -1
|
45
63
|
end
|
64
|
+
|
46
65
|
end
|
47
66
|
end
|
48
67
|
end
|
49
|
-
Jackal::Utils.load_http_hook
|
50
|
-
Carnivore.start!
|
51
|
-
rescue => e
|
52
|
-
$stderr.puts "Unexpected failure encountered: #{e.class}: #{e}"
|
53
|
-
if(ENV['DEBUG'])
|
54
|
-
$stderr.puts "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
55
|
-
end
|
56
|
-
exit -1
|
57
68
|
end
|
data/lib/jackal/utils.rb
CHANGED
@@ -7,9 +7,12 @@ module Jackal
|
|
7
7
|
autoload :Spec, 'jackal/utils/spec'
|
8
8
|
autoload :Payload, 'jackal/utils/payload'
|
9
9
|
autoload :Config, 'jackal/utils/config'
|
10
|
+
autoload :Constants, 'jackal/utils/constants'
|
10
11
|
autoload :HttpApi, 'jackal/utils/http_api'
|
12
|
+
autoload :Process, 'jackal/utils/process'
|
11
13
|
|
12
14
|
extend Payload
|
15
|
+
extend Constants
|
13
16
|
|
14
17
|
class << self
|
15
18
|
|
data/lib/jackal/utils/config.rb
CHANGED
@@ -4,6 +4,11 @@ module Jackal
|
|
4
4
|
module Utils
|
5
5
|
module Config
|
6
6
|
|
7
|
+
# @return [Symbol] name of service
|
8
|
+
def service_name(class_name = self.class.name)
|
9
|
+
config_path(class_name).last.to_sym
|
10
|
+
end
|
11
|
+
|
7
12
|
# @return [Array] key path in configuration
|
8
13
|
def config_path(class_name = self.class.name)
|
9
14
|
class_name.split('::')[0,2].map do |string|
|
@@ -16,9 +21,14 @@ module Jackal
|
|
16
21
|
config_path.join('_')
|
17
22
|
end
|
18
23
|
|
19
|
-
# @return [
|
24
|
+
# @return [Smash] service configuration
|
25
|
+
def service_config
|
26
|
+
Carnivore::Config.get(*config_path) || Smash.new
|
27
|
+
end
|
28
|
+
|
29
|
+
# @return [Smash] configuration
|
20
30
|
def config
|
21
|
-
|
31
|
+
service_config[:config] || Smash.new
|
22
32
|
end
|
23
33
|
|
24
34
|
# Generation destination key based on direction
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'jackal'
|
2
|
+
|
3
|
+
module Jackal
|
4
|
+
module Utils
|
5
|
+
module Constants
|
6
|
+
|
7
|
+
# Provide constant defined by string
|
8
|
+
#
|
9
|
+
# @param string [String] constant to locate
|
10
|
+
# @return [Class]
|
11
|
+
def constantize(string)
|
12
|
+
klass = string.to_s.split('::').inject(Object) do |memo, name|
|
13
|
+
memo.const_get(name)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'jackal'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'childprocess'
|
4
|
+
require 'shellwords'
|
5
|
+
require 'tempfile'
|
6
|
+
|
7
|
+
module Jackal
|
8
|
+
module Utils
|
9
|
+
class Process
|
10
|
+
|
11
|
+
# Create new instance
|
12
|
+
#
|
13
|
+
# @return [self]
|
14
|
+
def initialize
|
15
|
+
@storage_directory = Carnivore::Config.fetch(
|
16
|
+
:fission, :utils, :process_manager, :storage,
|
17
|
+
'/tmp/fission/process_manager'
|
18
|
+
)
|
19
|
+
FileUtils.mkdir_p(@storage_directory)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Create new process
|
23
|
+
#
|
24
|
+
# @param identifier [String] command identifier (compat argument)
|
25
|
+
# @param command [String] command in single string or splatted array
|
26
|
+
# @yieldparam [ChildProcess]
|
27
|
+
# @return [TrueClass]
|
28
|
+
def process(identifier, *command)
|
29
|
+
if(command.size == 1)
|
30
|
+
command = Shellwords.shellsplit(command.first)
|
31
|
+
end
|
32
|
+
if(block_given?)
|
33
|
+
yield ChildProcess.build(*command)
|
34
|
+
end
|
35
|
+
true
|
36
|
+
end
|
37
|
+
|
38
|
+
# Temporary IO for logging
|
39
|
+
#
|
40
|
+
# @param args [String] argument list joined for filename
|
41
|
+
# @return [IO]
|
42
|
+
def create_io_tmp(*args)
|
43
|
+
path = File.join(@storage_directory, args.join('-'))
|
44
|
+
FileUtils.mkdir_p(File.dirname(path))
|
45
|
+
t_file = File.open(path, 'w+')
|
46
|
+
t_file.sync
|
47
|
+
t_file
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -55,10 +55,9 @@ end
|
|
55
55
|
# @return [Thread] thread with running source
|
56
56
|
def run_setup(config)
|
57
57
|
path = File.join(Dir.pwd, 'test/specs/config', "#{config}.json")
|
58
|
-
Carnivore::Config.configure(:config_path => path)
|
59
58
|
Thread.abort_on_exception = true
|
60
59
|
runner = Thread.new do
|
61
|
-
|
60
|
+
Jackal::Loader.run!(:config => path)
|
62
61
|
end
|
63
62
|
source_wait(:setup)
|
64
63
|
runner
|
data/lib/jackal/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jackal
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Roberts
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-01-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: carnivore
|
@@ -16,16 +16,56 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.3.0
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 1.0.0
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.3.0
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: bogo-cli
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0.1'
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0.1'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bogo-config
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 0.1.6
|
54
|
+
- - "<"
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 1.0.0
|
20
57
|
type: :runtime
|
21
58
|
prerelease: false
|
22
59
|
version_requirements: !ruby/object:Gem::Requirement
|
23
60
|
requirements:
|
24
61
|
- - ">="
|
25
62
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
63
|
+
version: 0.1.6
|
64
|
+
- - "<"
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 1.0.0
|
27
67
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
68
|
+
name: childprocess
|
29
69
|
requirement: !ruby/object:Gem::Requirement
|
30
70
|
requirements:
|
31
71
|
- - ">="
|
@@ -57,11 +97,14 @@ files:
|
|
57
97
|
- lib/jackal/callback.rb
|
58
98
|
- lib/jackal/cli.rb
|
59
99
|
- lib/jackal/error.rb
|
100
|
+
- lib/jackal/formatter.rb
|
60
101
|
- lib/jackal/loader.rb
|
61
102
|
- lib/jackal/utils.rb
|
62
103
|
- lib/jackal/utils/config.rb
|
104
|
+
- lib/jackal/utils/constants.rb
|
63
105
|
- lib/jackal/utils/http_api.rb
|
64
106
|
- lib/jackal/utils/payload.rb
|
107
|
+
- lib/jackal/utils/process.rb
|
65
108
|
- lib/jackal/utils/spec.rb
|
66
109
|
- lib/jackal/utils/spec/callback_local.rb
|
67
110
|
- lib/jackal/utils/spec/helpers.rb
|