jackal 0.1.16 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 238f64730bc68ab50fe7f24e2de11f5e07a084ec
4
- data.tar.gz: 305ba06c52248d48f704965d33494cdaffe82715
3
+ metadata.gz: 914f2d57a1083c0adc67afe2bda1f310201779bd
4
+ data.tar.gz: 2323ad17173d7dd0d2325cdd36de6399abc42fca
5
5
  SHA512:
6
- metadata.gz: acc58c09217457638e88ff5486bce667630d1d568fde6b79a88bafdf7702235162a5d42386524c445d9f9a5491c781e9276127bd835a6e28239bb2f6933997ad
7
- data.tar.gz: 70c6d38554e1ead5e410ece8896cee42cd34f35aec97297afa7f97705b60c048b7a7bc86a1eb6fd10217e412829a38b89a92cf8056ae420a0a9d03afaa72607f
6
+ metadata.gz: ba7be9cc5ad19e6ddf23086226291f7cda5b58efea577309d52b4bf17ed39408e5a3ace5ac8ed01391fe1eab16b33675fbaa988472d8d4a9340e75d04b559755
7
+ data.tar.gz: e09419edd04c3f59efa6f41c9e0d4579e61480c77a8d5f98545aaea15e5e4251fa0c0607cca15be0e147d24aeeb79b23800385d314f8fbdc3206df509d76d5d2
@@ -1,6 +1,11 @@
1
+ # v0.2.0
2
+ * Add abstract payload formatter
3
+ * Provide common helper method to shell out
4
+ * Remove custom CLI and integrate bogo-cli
5
+
1
6
  # v0.1.16
2
7
  * Load HTTP hook directly prior to service start
3
- * Fixes some issues with spec loads
8
+ * Fixes some issues with spec loads
4
9
 
5
10
  # v0.1.14
6
11
  * Move orphan handlers into source init arg
data/bin/jackal CHANGED
@@ -1,3 +1,18 @@
1
1
  #!/usr/bin/env ruby
2
+ require 'bogo-cli'
2
3
  require 'jackal'
3
- require 'jackal/loader'
4
+
5
+ Bogo::Cli::Setup.define do
6
+
7
+ on :v, :version, 'Print version' do
8
+ puts "jackal - Message Pipeline - [Version: #{Jackal::VERSION}]"
9
+ exit
10
+ end
11
+ on :c, :config=, 'Configuration file or directory'
12
+ on :V, :verbosity=, 'Log verbosity (debug info warn error)'
13
+ run do |opts, args|
14
+ Jackal::Loader.run!(opts)
15
+ exit
16
+ end
17
+
18
+ end
@@ -10,9 +10,11 @@ Gem::Specification.new do |s|
10
10
  s.description = 'Message processing helper'
11
11
  s.require_path = 'lib'
12
12
  s.license = 'Apache 2.0'
13
- s.add_dependency 'carnivore', '>= 0.2.6'
14
- s.add_dependency 'mixlib-cli'
15
- s.files = Dir['lib/**/*'] + %w(jackal.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
13
+ s.add_dependency 'carnivore', '>= 0.3.0', '< 1.0.0'
14
+ s.add_dependency 'bogo-cli', '~> 0.1'
15
+ s.add_dependency 'bogo-config', '>= 0.1.6', '< 1.0.0'
16
+ s.add_dependency 'childprocess'
17
+ s.files = Dir['{lib,bin}/**/**/*'] + %w(jackal.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
16
18
  s.executables << 'jackal'
17
19
  s.executables << 'jackal-test'
18
20
  end
@@ -4,7 +4,9 @@ module Jackal
4
4
  autoload :Cli, 'jackal/cli'
5
5
  autoload :Callback, 'jackal/callback'
6
6
  autoload :Error, 'jackal/error'
7
+ autoload :Formatter, 'jackal/formatter'
7
8
  autoload :Utils, 'jackal/utils'
9
+ autoload :Loader, 'jackal/loader'
8
10
  end
9
11
 
10
12
  require 'jackal/version'
@@ -6,8 +6,33 @@ module Jackal
6
6
 
7
7
  include Utils::Payload
8
8
  include Utils::Config
9
+ include Utils::Config
9
10
  # @!parse include Jackal::Utils::Payload
10
11
  # @!parse include Jackal::Utils::Config
12
+ # @!parse include Jackal::Utils::Constants
13
+
14
+ # @return [Array<Formatter>] formatters
15
+ attr_reader :formatters
16
+
17
+ # Create new instance
18
+ #
19
+ # @return [self]
20
+ def initialize(*_)
21
+ super
22
+ if(service_config[:formatters])
23
+ @formatters = service_config[:formatters].map do |klass_name|
24
+ constantize(klass_name).new
25
+ end
26
+ end
27
+ end
28
+
29
+ # @return [Utils::Process]
30
+ def process_manager
31
+ unless(@_process)
32
+ @_process = Utils::Process.new
33
+ end
34
+ @_process
35
+ end
11
36
 
12
37
  # Validity of message
13
38
  #
@@ -86,9 +111,22 @@ module Jackal
86
111
  # @param message [Carnivore::Message]
87
112
  def job_completed(name, payload, message)
88
113
  info "Processing of message #{message} has completed within this component #{name}"
114
+ if(formatters)
115
+ apply_formatters!(payload)
116
+ end
89
117
  message.confirm!
90
118
  forward(payload)
91
119
  end
92
120
 
121
+ # Apply configured formatters to payload
122
+ #
123
+ # @param payload [Smash]
124
+ # @return [Smash]
125
+ def apply_formatters!(payload)
126
+ formatters.each do |formatter|
127
+ formatter.format(payload)
128
+ end
129
+ end
130
+
93
131
  end
94
132
  end
@@ -0,0 +1,56 @@
1
+ require 'jackal'
2
+
3
+ module Jackal
4
+ # Payload formatter
5
+ class Formatter
6
+
7
+ class << self
8
+
9
+ # Register formatter
10
+ def inherited(klass)
11
+ Formatter.descendants.push(klass).uniq!
12
+ end
13
+
14
+ # @return [Array<Class>] registered formatters
15
+ def descendants
16
+ @_descendants ||= []
17
+ end
18
+
19
+ end
20
+
21
+ # @return [String, Symbol]
22
+ SOURCE = nil
23
+ # @return [String, Symbol]
24
+ DESTINATION = nil
25
+
26
+ # Create a new instance
27
+ #
28
+ # @return [self]
29
+ def initialize
30
+ [:SOURCE, :DESTINATION].each do |key|
31
+ unless(self.class.const_get(key))
32
+ raise NotImplementedError.new("Formatter class must define #{key} constant")
33
+ end
34
+ end
35
+ end
36
+
37
+ # @return [Symbol]
38
+ def source
39
+ self.class.const_get(:SOURCE).to_sym
40
+ end
41
+
42
+ # @return [Symbol]
43
+ def destination
44
+ self.class.const_get(:DESTINATION).to_sym
45
+ end
46
+
47
+ # Apply format to payload
48
+ #
49
+ # @param payload [Smash]
50
+ # @return payload [Smash]
51
+ def format(payload)
52
+ raise NotImplementedError
53
+ end
54
+
55
+ end
56
+ end
@@ -1,57 +1,68 @@
1
1
  require 'carnivore'
2
2
  require 'jackal'
3
3
 
4
- unless(ENV['JACKAL_TESTING_MODE'])
5
- cli = Jackal::Cli.new
6
- cli.parse_options
7
- Carnivore::Config.configure(cli.config)
8
- end
4
+ module Jackal
5
+ class Loader
6
+ class << self
7
+
8
+ # Run the jackal
9
+ #
10
+ # @param opts [Hash]
11
+ def run!(opts)
9
12
 
10
- Carnivore::Config.auto_symbolize(true)
13
+ if(ENV['JACKAL_TESTING_MODE'] && !opts[:config])
14
+ Carnivore.configure!(:verify)
15
+ else
16
+ Carnivore.configure!(opts[:config])
17
+ Carnivore::Config.immutable!
18
+ end
11
19
 
12
- Celluloid.logger.level = Celluloid.logger.class.const_get(
13
- (Carnivore::Config.get(:verbosity) || :debug).to_s.upcase
14
- )
20
+ Celluloid.logger.level = Celluloid.logger.class.const_get(
21
+ (opts[:verbosity] || Carnivore::Config[:verbosity] || :debug).to_s.upcase
22
+ )
15
23
 
16
- (Carnivore::Config.get(:jackal, :require) || []).each do |path|
17
- require path
18
- end
24
+ Carnivore::Config.fetch(:jackal, :require, []).each do |path|
25
+ require path
26
+ end
19
27
 
20
- begin
21
- Carnivore::Utils.symbolize_hash(Carnivore::Config.hash_dup).each do |namespace, args|
22
- next unless args.is_a?(Hash)
23
- args.each do |key, opts|
24
- next unless opts.is_a?(Hash) && opts[:sources]
25
- Carnivore::Utils.debug "Processing: #{opts.inspect}"
26
- Carnivore.configure do
27
- opts.fetch(:sources, {}).each do |kind, source_args|
28
- source = Carnivore::Source.build(
29
- :type => source_args[:type].to_sym,
30
- :args => source_args.fetch(:args, {}).merge(:name => "#{namespace}_#{key}_#{kind}"),
31
- :orphan_callback => lambda{|message|
32
- error "No callbacks matched message. Failed to process. Removed from bus. (#{message})"
33
- message.confirm!
34
- }
35
- )
36
- Carnivore::Utils.info "Registered new source: #{namespace}_#{key}_#{kind}"
37
- if(kind == :input)
38
- opts.fetch(:callbacks, []).each do |klass_name|
39
- klass = klass_name.split('::').inject(Object) do |memo, name|
40
- memo.const_get(name)
28
+ begin
29
+ Carnivore::Config.to_smash.each do |namespace, args|
30
+ next unless args.is_a?(Hash)
31
+ args.each do |key, opts|
32
+ next unless opts.is_a?(Hash) && opts[:sources]
33
+ Carnivore::Utils.debug "Processing: #{opts.inspect}"
34
+ Carnivore.configure do
35
+ opts.fetch(:sources, {}).each do |kind, source_args|
36
+ source = Carnivore::Source.build(
37
+ :type => source_args[:type].to_sym,
38
+ :args => source_args.fetch(:args, {}).merge(:name => "#{namespace}_#{key}_#{kind}"),
39
+ :orphan_callback => lambda{|message|
40
+ error "No callbacks matched message. Failed to process. Removed from bus. (#{message})"
41
+ message.confirm!
42
+ }
43
+ )
44
+ Carnivore::Utils.info "Registered new source: #{namespace}_#{key}_#{kind}"
45
+ if(kind == :input)
46
+ opts.fetch(:callbacks, []).each do |klass_name|
47
+ klass = Utils.constantize(klass_name)
48
+ source.add_callback(klass_name, klass)
49
+ end
50
+ end
51
+ end
41
52
  end
42
- source.add_callback(klass_name, klass)
43
53
  end
44
54
  end
55
+ Jackal::Utils.load_http_hook
56
+ Carnivore.start!
57
+ rescue => e
58
+ $stderr.puts "Unexpected failure encountered: #{e.class}: #{e}"
59
+ if(ENV['DEBUG'])
60
+ $stderr.puts "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
61
+ end
62
+ exit -1
45
63
  end
64
+
46
65
  end
47
66
  end
48
67
  end
49
- Jackal::Utils.load_http_hook
50
- Carnivore.start!
51
- rescue => e
52
- $stderr.puts "Unexpected failure encountered: #{e.class}: #{e}"
53
- if(ENV['DEBUG'])
54
- $stderr.puts "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
55
- end
56
- exit -1
57
68
  end
@@ -7,9 +7,12 @@ module Jackal
7
7
  autoload :Spec, 'jackal/utils/spec'
8
8
  autoload :Payload, 'jackal/utils/payload'
9
9
  autoload :Config, 'jackal/utils/config'
10
+ autoload :Constants, 'jackal/utils/constants'
10
11
  autoload :HttpApi, 'jackal/utils/http_api'
12
+ autoload :Process, 'jackal/utils/process'
11
13
 
12
14
  extend Payload
15
+ extend Constants
13
16
 
14
17
  class << self
15
18
 
@@ -4,6 +4,11 @@ module Jackal
4
4
  module Utils
5
5
  module Config
6
6
 
7
+ # @return [Symbol] name of service
8
+ def service_name(class_name = self.class.name)
9
+ config_path(class_name).last.to_sym
10
+ end
11
+
7
12
  # @return [Array] key path in configuration
8
13
  def config_path(class_name = self.class.name)
9
14
  class_name.split('::')[0,2].map do |string|
@@ -16,9 +21,14 @@ module Jackal
16
21
  config_path.join('_')
17
22
  end
18
23
 
19
- # @return [Hash] configuration
24
+ # @return [Smash] service configuration
25
+ def service_config
26
+ Carnivore::Config.get(*config_path) || Smash.new
27
+ end
28
+
29
+ # @return [Smash] configuration
20
30
  def config
21
- Carnivore::Config.get(*config_path.push(:config)) || Smash.new
31
+ service_config[:config] || Smash.new
22
32
  end
23
33
 
24
34
  # Generation destination key based on direction
@@ -0,0 +1,19 @@
1
+ require 'jackal'
2
+
3
+ module Jackal
4
+ module Utils
5
+ module Constants
6
+
7
+ # Provide constant defined by string
8
+ #
9
+ # @param string [String] constant to locate
10
+ # @return [Class]
11
+ def constantize(string)
12
+ klass = string.to_s.split('::').inject(Object) do |memo, name|
13
+ memo.const_get(name)
14
+ end
15
+ end
16
+
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,52 @@
1
+ require 'jackal'
2
+ require 'fileutils'
3
+ require 'childprocess'
4
+ require 'shellwords'
5
+ require 'tempfile'
6
+
7
+ module Jackal
8
+ module Utils
9
+ class Process
10
+
11
+ # Create new instance
12
+ #
13
+ # @return [self]
14
+ def initialize
15
+ @storage_directory = Carnivore::Config.fetch(
16
+ :fission, :utils, :process_manager, :storage,
17
+ '/tmp/fission/process_manager'
18
+ )
19
+ FileUtils.mkdir_p(@storage_directory)
20
+ end
21
+
22
+ # Create new process
23
+ #
24
+ # @param identifier [String] command identifier (compat argument)
25
+ # @param command [String] command in single string or splatted array
26
+ # @yieldparam [ChildProcess]
27
+ # @return [TrueClass]
28
+ def process(identifier, *command)
29
+ if(command.size == 1)
30
+ command = Shellwords.shellsplit(command.first)
31
+ end
32
+ if(block_given?)
33
+ yield ChildProcess.build(*command)
34
+ end
35
+ true
36
+ end
37
+
38
+ # Temporary IO for logging
39
+ #
40
+ # @param args [String] argument list joined for filename
41
+ # @return [IO]
42
+ def create_io_tmp(*args)
43
+ path = File.join(@storage_directory, args.join('-'))
44
+ FileUtils.mkdir_p(File.dirname(path))
45
+ t_file = File.open(path, 'w+')
46
+ t_file.sync
47
+ t_file
48
+ end
49
+
50
+ end
51
+ end
52
+ end
@@ -55,10 +55,9 @@ end
55
55
  # @return [Thread] thread with running source
56
56
  def run_setup(config)
57
57
  path = File.join(Dir.pwd, 'test/specs/config', "#{config}.json")
58
- Carnivore::Config.configure(:config_path => path)
59
58
  Thread.abort_on_exception = true
60
59
  runner = Thread.new do
61
- require 'jackal/loader'
60
+ Jackal::Loader.run!(:config => path)
62
61
  end
63
62
  source_wait(:setup)
64
63
  runner
@@ -1,5 +1,3 @@
1
- require 'carnivore/config'
2
-
3
1
  ENV['JACKAL_TESTING_MODE'] = 'true'
4
2
 
5
3
  path = File.join(Dir.pwd, 'test')
@@ -1,4 +1,4 @@
1
1
  module Jackal
2
2
  # Current library version
3
- VERSION = Gem::Version.new('0.1.16')
3
+ VERSION = Gem::Version.new('0.2.0')
4
4
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: jackal
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.16
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Roberts
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-10-30 00:00:00.000000000 Z
11
+ date: 2015-01-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: carnivore
@@ -16,16 +16,56 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.2.6
19
+ version: 0.3.0
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: 1.0.0
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 0.3.0
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: 1.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ name: bogo-cli
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '0.1'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '0.1'
47
+ - !ruby/object:Gem::Dependency
48
+ name: bogo-config
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: 0.1.6
54
+ - - "<"
55
+ - !ruby/object:Gem::Version
56
+ version: 1.0.0
20
57
  type: :runtime
21
58
  prerelease: false
22
59
  version_requirements: !ruby/object:Gem::Requirement
23
60
  requirements:
24
61
  - - ">="
25
62
  - !ruby/object:Gem::Version
26
- version: 0.2.6
63
+ version: 0.1.6
64
+ - - "<"
65
+ - !ruby/object:Gem::Version
66
+ version: 1.0.0
27
67
  - !ruby/object:Gem::Dependency
28
- name: mixlib-cli
68
+ name: childprocess
29
69
  requirement: !ruby/object:Gem::Requirement
30
70
  requirements:
31
71
  - - ">="
@@ -57,11 +97,14 @@ files:
57
97
  - lib/jackal/callback.rb
58
98
  - lib/jackal/cli.rb
59
99
  - lib/jackal/error.rb
100
+ - lib/jackal/formatter.rb
60
101
  - lib/jackal/loader.rb
61
102
  - lib/jackal/utils.rb
62
103
  - lib/jackal/utils/config.rb
104
+ - lib/jackal/utils/constants.rb
63
105
  - lib/jackal/utils/http_api.rb
64
106
  - lib/jackal/utils/payload.rb
107
+ - lib/jackal/utils/process.rb
65
108
  - lib/jackal/utils/spec.rb
66
109
  - lib/jackal/utils/spec/callback_local.rb
67
110
  - lib/jackal/utils/spec/helpers.rb