jackal 0.1.0 → 0.1.2

data/CHANGELOG.md

@@ -1,2 +1,7 @@
+# v0.1.2
+* Update configuration convention to require toplevel namespace
+* Clean up testing helper and setup
+* Include basic test for proper setup
+
 # v0.1.0
 * Initial commit

data/README.md

@@ -1,3 +1,89 @@
 # Jackal
 
-Run your carnivores
+Run your carnivores.
+
+## Configuration
+
+```json
+{
+  "jackal": {
+    "require": [
+      "carnivore-http",
+      "fubar-helloer"
+    ]
+  },
+  "fubar": {
+    "helloer": {
+      "sources": {
+        "input": {
+          ...
+        },
+        "output": {
+          ...
+        },
+        "error": {
+          ...
+        }
+      },
+      "callbacks": [
+        "Fubar::Helloer::SayHello"
+      ],
+      "config": {
+        "output_prefix": "Received message: "
+      }
+    }
+  }
+}
+```
+
+* `jackal` provides subsystem configuration
+  * `require` libraries to load at startup
+* `fubar` configuration of components (snake cased top level module)
+  * `helloer` configuration of specific component (snake cased second level module)
+    * `sources` configuration for carnivore sources
+    * `callbacks` callback class names to initialize and attach to input source
+    * `config` configuration hash used by callbacks
+
+## Jackal Callbacks
+
+Jackal callbacks are subclassed Carnivore callbacks adding a bit more structure. The
+general implementation of a Jackal callback:
+
+```ruby
+module Fubar
+  module Helloer
+    class SayHello < Jackal::Callback
+
+      def valid?(message)
+        super do |payload|
+          payload.get(:data, :helloer, :output)
+        end
+      end
+
+      def execute(message)
+        failure_wrap(message) do |payload|
+          info config[:output_prefix] + payload.get(:data, :helloer, :output)
+          job_completed(:helloer, payload, message)
+        end
+      end
+
+    end
+  end
+end
+```
+
+## Testing
+
+Jackal provides test helpers building upon the helpers provided by
+Carnivore.
+
+### jackal-test
+
+This executable will load minitest and auto run all files matched
+by the glob: `test/specs/*.rb`.
+
+## Info
+
+* Repository: https://github.com/carnivore-rb/jackal
+* Carnivore: https://github.com/carnivore-rb/carnivore
+* IRC: Freenode @ #carnivore

data/bin/jackal-test

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+# -*- encoding: utf-8 -*-
+
+require 'jackal/utils/spec/loader'

data/jackal.gemspec

@@ -14,4 +14,5 @@ Gem::Specification.new do |s|
   s.add_dependency 'mixlib-cli'
   s.files = Dir['lib/**/*'] + %w(jackal.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
   s.executables << 'jackal'
+  s.executables << 'jackal-test'
 end

data/lib/jackal.rb

@@ -4,6 +4,7 @@ module Jackal
   autoload :Cli, 'jackal/cli'
   autoload :Callback, 'jackal/callback'
   autoload :Error, 'jackal/error'
+  autoload :Utils, 'jackal/utils'
 end
 
 require 'jackal/version'

data/lib/jackal/callback.rb

@@ -1,25 +1,45 @@
 require 'jackal'
 
 module Jackal
+  # Jackal customized callback
   class Callback < Carnivore::Callback
 
-    def config_key
-      self.class.name.split('::').first.gsub(/(?<![A-Z])([A-Z])/, '_\1').sub(/^_/, '').downcase
+    include Utils::Payload
+
+    # @return [Array] key path in configuration
+    def config_path
+      self.class.name.split('::')[0,2].map do |string|
+        string.gsub(/(?<![A-Z])([A-Z])/, '_\1').sub(/^_/, '').downcase
+      end
+    end
+
+    # @return [String] prefix of source for this callback
+    def source_prefix
+      config_path.join('_')
     end
 
-    def new_payload(payload)
-      Smash.new(
-        :id => Celluloid.uuid,
-        :data => payload
-      )
+    # @return [Hash] configuration
+    def config
+      Carnviore::Config.get(*config_path.push(:config)) || Smash.new
+    end
+
+    # Validity of message
+    #
+    # @param message [Carnivore::Message]
+    # @return [TrueClass, FalseClass]
+    def valid?(message)
+      m = unpack(message)
+      block_given? ? yield(m) : true
     end
 
-    # message:: Original message
     # Executes block and catches unexpected exceptions if encountered
+    #
+    # @param message [Carnivore::Message]
+    # @return [Object]
     def failure_wrap(message)
       abort 'Failure wrap requires block for execution' unless block_given?
       begin
-        payload = message[:message]
+        payload = unpack(message)
         yield payload
       rescue => e
         error "!!! Unexpected failure encountered -> #{e.class}: #{e}"
@@ -28,29 +48,51 @@ module Jackal
       end
     end
 
+    # Send payload to error handler
+    #
+    # @param payload [Hash]
+    # @param message [Carnivore::Message]
+    # @param reason [String]
     def failed(payload, message, reason='No reason provided')
       error "Processing of #{message} failed! Reason: #{reason}"
       message.confirm!
-      destination = "#{config_key}_error"
+      destination = "#{source_prefix}_error"
       source = Carnivore::Supervisor.supervisor[destination]
       error "Sending #{message} to error handler: #{source}"
       source.transmit(payload)
     end
 
+    # Mark payload complete and forward
+    #
+    # @param payload [Hash]
+    # @param message [Carnivore::Message]
     def completed(payload, message)
       message.confirm!
       info "Processing of #{message} complete on this callback"
       forward(payload)
     end
 
+    # Forward payload to output source
+    #
+    # @param payload [Hash]
     def forward(payload)
-      destination = "#{config_key}_output"
+      destination = "#{source_prefix}_output"
       source = Carnivore::Supervisor.supervisor[destination]
-      info "Forwarding payload to output destination... (#{source})"
-      debug "Forwarded payload: #{payload.inspect}"
-      source.transmit(payload)
+      if(source)
+        info "Forwarding payload to output destination... (#{source})"
+        debug "Forwarded payload: #{payload.inspect}"
+        source.transmit(payload)
+      else
+        warn "No destination source found for generated source path: #{destination}"
+        info "Processing of message #{message} has completed. Message now discarded."
+      end
     end
 
+    # Mark job as completed
+    #
+    # @param name [String]
+    # @param payload [Hash]
+    # @param message [Carnivore::Message]
     def job_completed(name, payload, message)
       info "Processing of message #{message} has completed within this component #{name}"
       message.confirm!

data/lib/jackal/cli.rb

@@ -1,6 +1,7 @@
 require 'mixlib/cli'
 
 module Jackal
+  # CLI interface
   class Cli
     include Mixlib::CLI
 

data/lib/jackal/error.rb

@@ -1,6 +1,7 @@
 require 'jackal'
 
 module Jackal
+  # Base error class
   class Error < StandardError
   end
 end

data/lib/jackal/loader.rb

@@ -1,9 +1,12 @@
 require 'carnivore'
 require 'jackal'
 
-cli = Jackal::Cli.new
-cli.parse_options
-Carnivore::Config.configure(cli.config)
+unless(ENV['JACKAL_TESTING_MODE'])
+  cli = Jackal::Cli.new
+  cli.parse_options
+  Carnivore::Config.configure(cli.config)
+end
+
 Carnivore::Config.auto_symbolize(true)
 
 (Carnivore::Config.get(:jackal, :require) || []).each do |path|
@@ -11,26 +14,25 @@ Carnivore::Config.auto_symbolize(true)
 end
 
 begin
-  Carnivore::Utils.symbolize_hash(Carnivore::Config.hash_dup).each do |key, opts|
-    next if key == :jackal || !opts.is_a?(Hash)
-    Carnivore::Utils.info "Processing: #{opts.inspect}"
-    Carnivore.configure do
-      opts.fetch(:sources, {}).each do |kind, source_args|
-        source = Carnivore::Source.build(
-          :type => source_args[:type].to_sym,
-          :args => source_args.fetch(:args, {}).merge(:name => "#{key}_#{kind}")
-        )
-        Carnivore::Utils.info "Initialized new source: #{key}_#{kind}"
-        if(kind == :input)
-          opts.fetch(:callbacks, []).each do |klass_name|
-            klass = klass_name.split('::').inject(
-              Object.const_get(
-                key.to_s.split('_').map(&:capitalize).join
-              )
-            ) do |memo, name|
-              memo.const_get(name)
+  Carnivore::Utils.symbolize_hash(Carnivore::Config.hash_dup).each do |namespace, args|
+    next unless args.is_a?(Hash)
+    args.each do |key, opts|
+      next unless opts.is_a?(Hash) && opts[:sources]
+      Carnivore::Utils.info "Processing: #{opts.inspect}"
+      Carnivore.configure do
+        opts.fetch(:sources, {}).each do |kind, source_args|
+          source = Carnivore::Source.build(
+            :type => source_args[:type].to_sym,
+            :args => source_args.fetch(:args, {}).merge(:name => "#{namespace}_#{key}_#{kind}")
+          )
+          Carnivore::Utils.info "Initialized new source: #{namespace}_#{key}_#{kind}"
+          if(kind == :input)
+            opts.fetch(:callbacks, []).each do |klass_name|
+              klass = klass_name.split('::').inject(Object) do |memo, name|
+                memo.const_get(name)
+              end
+              source.add_callback(klass_name, klass)
             end
-            source.add_callback(klass_name, klass)
           end
         end
       end

data/lib/jackal/utils.rb

@@ -0,0 +1,38 @@
+require 'jackal'
+
+module Jackal
+  # Helper utilities
+  module Utils
+
+    autoload :Spec, 'jackal/utils/spec'
+
+    # Module utilities
+    module Payload
+
+      # Generate a new payload
+      #
+      # @param name [String]
+      # @param payload [Hash]
+      # @return [Smash]
+      def new_payload(name, payload)
+        Smash.new(
+          :name => name,
+          :id => Celluloid.uuid,
+          :data => payload
+        )
+      end
+
+      # Extract payload from message
+      #
+      # @param message [Carnivore::Message]
+      # @return [Smash]
+      def unpack(message)
+        Smash.new(message[:message])
+      end
+
+    end
+
+    extend Payload
+
+  end
+end

data/lib/jackal/utils/spec.rb

@@ -0,0 +1,10 @@
+require 'jackal'
+
+module Jackal
+  module Utils
+    # Testing helpers
+    module Spec
+      autoload :CallbackLocal, 'jackal/utils/spec/callback_local'
+    end
+  end
+end

data/lib/jackal/utils/spec/callback_local.rb

@@ -0,0 +1,24 @@
+require 'jackal'
+
+module Jackal
+  module Utils
+    module Spec
+      # Callback helper module for isolated testing
+      module CallbackLocal
+
+        # @return [Array] forwarded payloads
+        def forwarded
+          @forwarded ||= []
+        end
+
+        # Force payload into local store
+        #
+        # @param payload [Hash]
+        def forward(payload)
+          @forwarded << payload
+        end
+
+      end
+    end
+  end
+end

data/lib/jackal/utils/spec/helpers.rb

@@ -0,0 +1,49 @@
+require 'multi_json'
+require 'carnivore/spec_helper'
+
+Celluloid.logger.level = 0 if ENV['DEBUG']
+
+# Default source setup higher than base carivore default
+unless(ENV['CARNIVORE_SOURCE_SETUP'])
+  ENV['CARNIVORE_SOURCE_SETUP'] = '0.5'
+end
+
+# Pass any jackal specific wait settings down to carnivore
+ENV.each do |key, value|
+  if(key.start_with?('JACKAL_SOURCE_'))
+    carnivore_key = key.sub('JACKAL_SOURCE', 'CARNIVORE_SOURCE')
+    ENV[carnivore_key] = value
+  end
+end
+
+def payload_for(style, args={})
+  file = "#{style}.json"
+  path = [File.join(Dir.pwd, 'test/specs/payloads'), File.join(File.dirname(__FILE__), 'payloads')].map do |dir|
+    if(File.exists?(full_path = File.join(dir, file)))
+      full_path
+    end
+  end.compact.first
+  if(path)
+    if(args[:raw])
+      MultiJson.load(File.read(path))
+    else
+      if(args[:nest])
+        Jackal::Utils.new_payload(:test, args[:nest] => MultiJson.load(File.read(path)))
+      else
+        Jackal::Utils.new_payload(:test, File.read(path))
+      end
+    end
+  else
+    raise "Requested payload path for test does not exist: #{File.expand_path(path)}"
+  end
+end
+
+def run_setup(config)
+  path = File.join(Dir.pwd, 'test/specs/config', "#{config}.json")
+  Carnivore::Config.configure(:config_path => path)
+  runner = Thread.new do
+    require 'jackal/loader'
+  end
+  source_wait(:setup)
+  runner
+end

data/lib/jackal/utils/spec/loader.rb

@@ -0,0 +1,14 @@
+require 'carnivore/config'
+
+ENV['JACKAL_TESTING_MODE'] = 'true'
+
+path = File.join(Dir.pwd, 'test')
+
+if(File.directory?(path))
+  if(File.exists?(spec_file = File.join(path, 'spec.rb')))
+    require spec_file
+  end
+  require 'jackal/utils/spec/runner'
+else
+  raise "No test directory found: #{path}"
+end

data/lib/jackal/utils/spec/runner.rb

@@ -0,0 +1,11 @@
+require 'jackal/utils/spec/helpers'
+
+path = File.join(Dir.pwd, 'test', 'specs')
+
+if(File.directory?(path))
+  Dir.glob(File.join(path, '*.rb')).each do |spec|
+    require spec
+  end
+else
+  raise "No specs directory found: #{path}"
+end

data/lib/jackal/version.rb

@@ -1,5 +1,5 @@
 module Jackal
   class Version < Gem::Version
   end
-  VERSION = Version.new('0.1.0')
+  VERSION = Version.new('0.1.2')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jackal
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-01 00:00:00.000000000 Z
+date: 2014-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: carnivore
@@ -47,14 +47,21 @@ description: Message processing helper
 email: code@chrisroberts.org
 executables:
 - jackal
+- jackal-test
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/jackal/callback.rb
 - lib/jackal/version.rb
+- lib/jackal/utils/spec/callback_local.rb
+- lib/jackal/utils/spec/runner.rb
+- lib/jackal/utils/spec/helpers.rb
+- lib/jackal/utils/spec/loader.rb
+- lib/jackal/utils/spec.rb
 - lib/jackal/cli.rb
 - lib/jackal/loader.rb
 - lib/jackal/error.rb
+- lib/jackal/utils.rb
 - lib/jackal.rb
 - jackal.gemspec
 - README.md
@@ -62,6 +69,7 @@ files:
 - CONTRIBUTING.md
 - LICENSE
 - bin/jackal
+- bin/jackal-test
 homepage: https://github.com/carnivore-rb/jackal
 licenses:
 - Apache 2.0