chasqui 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7dc418405a5b23f20b67dc597c78926ee37f2f97
4
- data.tar.gz: 8dbef72f6a1fe7546c3fa770d461d931ee396312
3
+ metadata.gz: 1bb216d9f86e895b3a2c48b686a9d3d92f991cf8
4
+ data.tar.gz: e736ccd90ee44c1aa827c4d59e09136ea63f9958
5
5
  SHA512:
6
- metadata.gz: 6619038c7980b5bdb191014f3e14a3753c5a0256eeaa844035895fc23422736a3584772c8c77dd6b6eb4437ac5bbad019c79138ce0fdab9e8120f8069f03278d
7
- data.tar.gz: f6b7da7f471ce28f665f44cfcd82edcf1702bb2de0d29dcebfc83baf4662ccfa5cbbbff5aed883b550b554a0d4702026ec42b7f65e713bfc022ecb2cc376b8fa
6
+ metadata.gz: d174c6cb4f929a2135f86fe7bf4c06e32d8f66088e25d18fa4048b872aaaa9430888f7cb5f901665dff430064dd24772164fb1b19aa6b48f3b3e74f7fa9d14a2
7
+ data.tar.gz: e4e64b8ee8453138def77a8f229e3fb002f0e794d2c0e0013db5eaa12bdd6bae46e109d707e95b024a6dae4081dbdcfcf2ff7288ea4ea5c9f8ce4ba55eb19532
data/.travis.yml CHANGED
@@ -1,3 +1,6 @@
1
1
  language: ruby
2
+ services:
3
+ - redis-server
2
4
  rvm:
3
- - 2.1.3
5
+ - 2.2.2
6
+ - 1.9.3
data/Guardfile ADDED
@@ -0,0 +1,14 @@
1
+ guard :rspec, cmd: "bundle exec rspec" do
2
+ require "guard/rspec/dsl"
3
+ dsl = Guard::RSpec::Dsl.new(self)
4
+
5
+ # RSpec files
6
+ rspec = dsl.rspec
7
+ watch(rspec.spec_helper) { rspec.spec_dir }
8
+ watch(rspec.spec_support) { rspec.spec_dir }
9
+ watch(rspec.spec_files)
10
+
11
+ # Ruby files
12
+ ruby = dsl.ruby
13
+ dsl.watch_spec_files_for(ruby.lib_files)
14
+ end
data/README.md CHANGED
@@ -1,3 +1,5 @@
1
+ [![Build Status](https://travis-ci.org/jbgo/chasqui.svg?branch=master)](https://travis-ci.org/jbgo/chasqui)
2
+
1
3
  # Chasqui
2
4
 
3
5
  Chasqui is a simple, lightweight, persistent implementation of the publish-subscribe (pub/sub)
@@ -68,12 +70,12 @@ Publishing events is simple.
68
70
  Chasqui.publish 'user.sign-up', user_id
69
71
  ```
70
72
 
71
- To prevent conflicts with other applications, you can choose a unique namespace for your events.
73
+ To prevent conflicts with other applications, you can choose a unique channel for your events.
72
74
 
73
75
  ```rb
74
76
  # config/initializers/chasqui.rb
75
77
  Chasqui.configure do |config|
76
- config.namespace = 'com.example.myapp'
78
+ config.channel = 'com.example.myapp'
77
79
  end
78
80
  ```
79
81
 
@@ -82,11 +84,9 @@ Now when you call `Chasqui.publish 'event.name', data, ...`, Chasqui will publis
82
84
 
83
85
  ## Subscribing to events
84
86
 
85
- __NOT IMPLMENTED YET__
86
-
87
87
  ```rb
88
88
  # file: otherapp/app/subscribers/user_events.rb
89
- Chasqui.subscribe queue: 'unique_queue_name_for_app', namespace: 'com.example.myapp' do
89
+ Chasqui.subscribe queue: 'unique_queue_name_for_app', channel: 'com.example.myapp' do
90
90
 
91
91
  on 'user.sign-up' do |user_id|
92
92
  user = User.find user_id
@@ -106,7 +106,7 @@ end
106
106
 
107
107
  ```rb
108
108
  Chasqui.configure do |config|
109
- config.namespace = 'com.example.transcoder'
109
+ config.channel = 'com.example.transcoder'
110
110
  config.redis = ENV.fetch('REDIS_URL')
111
111
  config.workers = :sidekiq # or :resque
112
112
  ...
data/Rakefile CHANGED
@@ -1,7 +1,23 @@
1
1
  require "bundler/gem_tasks"
2
2
  require "rspec/core/rake_task"
3
3
 
4
+ require 'resque/tasks'
5
+ task 'resque:setup' => :environment
6
+
4
7
  RSpec::Core::RakeTask.new(:spec)
5
8
 
6
9
  task :default => :spec
7
10
 
11
+ task :environment do
12
+ $LOAD_PATH.unshift './lib'
13
+ require 'bundler/setup'
14
+ require 'chasqui'
15
+
16
+ if ENV['CHASQUI_ENV'] == 'test'
17
+ require './spec/integration/subscribers'
18
+ end
19
+
20
+ require 'resque'
21
+ Resque.redis = ENV['REDIS_URL'] if ENV['REDIS_URL']
22
+ Resque.redis.namespace = 'chasqui'
23
+ end
data/bin/chasqui ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+ require 'chasqui/cli'
5
+
6
+ Chasqui::CLI.new(ARGV).run
data/chasqui.gemspec CHANGED
@@ -17,7 +17,13 @@ Gem::Specification.new do |spec|
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
19
 
20
+ spec.add_dependency "redis"
21
+ spec.add_dependency "redis-namespace"
22
+
20
23
  spec.add_development_dependency "bundler", "~> 1.7"
21
24
  spec.add_development_dependency "rake", "~> 10.0"
22
25
  spec.add_development_dependency "rspec"
26
+ spec.add_development_dependency "guard-rspec"
27
+ spec.add_development_dependency "resque"
28
+ spec.add_development_dependency "sidekiq"
23
29
  end
data/examples/full.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  # file: admin/config/initializers/chasqui.rb
2
2
  Chasqui.configure do |config|
3
- config.namespace = 'com.example.admin'
3
+ config.channel = 'com.example.admin'
4
4
  config.redis = ENV.fetch('REDIS_URL')
5
5
  end
6
6
 
@@ -28,7 +28,7 @@ Chasqui.configure do |config|
28
28
  end
29
29
 
30
30
  # file: transcoder/app/subscribers/video_subscriber.rb
31
- Chasqui.subscribe queue: 'transcoder.video', namespace: 'com.example.admin' do
31
+ Chasqui.subscribe queue: 'transcoder.video', channel: 'com.example.admin' do
32
32
  on 'video.upload' do |video_id|
33
33
  begin
34
34
  Transcorder.transcode video_url(video_id)
@@ -41,14 +41,14 @@ Chasqui.subscribe queue: 'transcoder.video', namespace: 'com.example.admin' do
41
41
  end
42
42
 
43
43
  # file: admin/app/subscribers/video_subscriber.rb
44
- Chasqui.subscribe queue: 'admin.events', namespace: 'com.example.transcoder' do
44
+ Chasqui.subscribe queue: 'admin.events', channel: 'com.example.transcoder' do
45
45
 
46
46
  on 'transcoder.video.complete' do |video_id|
47
47
  video = Video.find video_id
48
48
  VideoMailer.transcode_complete(video).deliver
49
49
  end
50
50
 
51
- on 'transcoder.video.complete' do |video_id, error|
51
+ on 'transcoder.video.error' do |video_id, error|
52
52
  video = Video.find video_id
53
53
  VideoMailer.transcode_error(video, error).deliver
54
54
  end
@@ -0,0 +1,70 @@
1
+ require 'timeout'
2
+
3
+ class Chasqui::Broker
4
+ attr_reader :config
5
+
6
+ extend Forwardable
7
+ def_delegators :@config, :redis, :inbox, :logger
8
+
9
+ ShutdownSignals = %w(INT QUIT ABRT TERM).freeze
10
+
11
+ # To prevent unsuspecting clients from blocking forever, the broker uses
12
+ # it's own private redis connection.
13
+ def initialize
14
+ @shutdown_requested = nil
15
+ @config = Chasqui.config.dup
16
+ @config.redis = Redis.new @config.redis.client.options
17
+ end
18
+
19
+ def start
20
+ install_signal_handlers
21
+
22
+ logger.info "broker started with pid #{Process.pid}"
23
+ logger.info "configured to fetch events from #{inbox} on #{redis.inspect}"
24
+
25
+ until_shutdown_requested { forward_event }
26
+ end
27
+
28
+ def forward_event
29
+ raise NotImplementedError.new "please define #forward_event in a subclass of #{self.class.name}"
30
+ end
31
+
32
+ class << self
33
+ def start
34
+ Chasqui::MultiBroker.new.start
35
+ end
36
+ end
37
+
38
+ private
39
+
40
+ def install_signal_handlers
41
+ ShutdownSignals.each do |signal|
42
+ trap(signal) { @shutdown_requested = signal }
43
+ end
44
+ end
45
+
46
+ def until_shutdown_requested
47
+ catch :shutdown do
48
+ loop do
49
+ with_timeout do
50
+ if @shutdown_requested
51
+ logger.info "broker received signal, #@shutdown_requested. shutting down"
52
+ throw :shutdown
53
+ else
54
+ yield
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+
61
+ def with_timeout
62
+ # This timeout is a failsafe for an improperly configured broker
63
+ Timeout::timeout(config.broker_poll_interval + 1) do
64
+ yield
65
+ end
66
+ rescue TimeoutError
67
+ logger.warn "broker poll interval exceeded for broker, #{self.class.name}"
68
+ end
69
+
70
+ end
@@ -0,0 +1,78 @@
1
+ require 'optparse'
2
+ require 'ostruct'
3
+ require 'chasqui'
4
+
5
+ class Chasqui::CLI
6
+ extend Forwardable
7
+ def_delegators :@options, :logfile, :redis_url, :inbox_queue, :debug, :version
8
+
9
+ def initialize(argv)
10
+ build_options(argv)
11
+ end
12
+
13
+ def configure
14
+ Chasqui.configure do |config|
15
+ config.logger = options.logfile if options.logfile
16
+ config.redis = options.redis_url if options.redis_url
17
+ config.inbox_queue = options.inbox_queue if options.inbox_queue
18
+ config.logger.level = Logger::DEBUG if options.debug
19
+ end
20
+ end
21
+
22
+ def run
23
+ configure
24
+
25
+ if options.version
26
+ puts "chasqui #{Chasqui::VERSION}"
27
+ elsif options.help
28
+ puts @parser.help()
29
+ else
30
+ Chasqui::Broker.start
31
+ end
32
+ rescue => ex
33
+ Chasqui.logger.fatal ex.inspect
34
+ Chasqui.logger.fatal ex.backtrace.join("\n")
35
+ end
36
+
37
+ private
38
+
39
+ def options
40
+ @options ||= build_options
41
+ end
42
+
43
+ def build_options(argv)
44
+ opts = {}
45
+
46
+ @parser = OptionParser.new do |o|
47
+ o.banner = "Usage: #{argv[0]} [options]"
48
+
49
+ o.on('-f', '--logfile PATH', 'log file path') do |arg|
50
+ opts[:logfile] = arg
51
+ end
52
+
53
+ o.on('-r', '--redis URL', 'redis connection URL') do |arg|
54
+ opts[:redis_url] = arg
55
+ end
56
+
57
+ o.on('-q', '--inbox-queue NAME', 'name of the queue from which chasqui broker consumes events') do |arg|
58
+ opts[:inbox_queue] = arg
59
+ end
60
+
61
+ o.on('-d', '--debug', 'enable debug logging') do |arg|
62
+ opts[:debug] = arg
63
+ end
64
+
65
+ o.on('-v', '--version', 'show version and exit') do |arg|
66
+ opts[:version] = arg
67
+ end
68
+
69
+ o.on('-h', '--help', 'show this help pessage') do |arg|
70
+ opts[:help] = arg
71
+ end
72
+ end
73
+
74
+ @parser.parse!(argv)
75
+ @options = OpenStruct.new opts
76
+ end
77
+
78
+ end
@@ -0,0 +1,78 @@
1
+ module Chasqui
2
+
3
+ Defaults = {
4
+ inbox_queue: 'inbox',
5
+ redis_namespace: 'chasqui',
6
+ publish_channel: '__default',
7
+ broker_poll_interval: 3
8
+ }.freeze
9
+
10
+ class ConfigurationError < StandardError
11
+ end
12
+
13
+ CONFIG_SETTINGS = [
14
+ :broker_poll_interval,
15
+ :channel,
16
+ :inbox_queue,
17
+ :logger,
18
+ :redis,
19
+ :worker_backend
20
+ ]
21
+
22
+ class Config < Struct.new(*CONFIG_SETTINGS)
23
+ def channel
24
+ self[:channel] ||= Defaults.fetch(:publish_channel)
25
+ end
26
+
27
+ def inbox_queue
28
+ self[:inbox_queue] ||= Defaults.fetch(:inbox_queue)
29
+ end
30
+ alias inbox inbox_queue
31
+
32
+ def redis
33
+ unless self[:redis]
34
+ self.redis = Redis.new
35
+ end
36
+
37
+ self[:redis]
38
+ end
39
+
40
+ def redis=(redis_config)
41
+ client = case redis_config
42
+ when Redis
43
+ redis_config
44
+ when String
45
+ Redis.new url: redis_config
46
+ else
47
+ Redis.new redis_config
48
+ end
49
+
50
+ self[:redis] = Redis::Namespace.new(Defaults.fetch(:redis_namespace), redis: client)
51
+ end
52
+
53
+ def logger
54
+ unless self[:logger]
55
+ self.logger = STDOUT
56
+ end
57
+
58
+ self[:logger]
59
+ end
60
+
61
+ def logger=(new_logger)
62
+ lg = if new_logger.respond_to? :info
63
+ new_logger
64
+ else
65
+ Logger.new(new_logger).tap do |lg|
66
+ lg.level = Logger::INFO
67
+ end
68
+ end
69
+
70
+ lg.progname = 'chasqui'
71
+ self[:logger] = lg
72
+ end
73
+
74
+ def broker_poll_interval
75
+ self[:broker_poll_interval] ||= Defaults.fetch(:broker_poll_interval)
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,56 @@
1
+ class Chasqui::MultiBroker < Chasqui::Broker
2
+
3
+ def forward_event
4
+ event = receive or return
5
+ queues = subscriber_queues event
6
+
7
+ redis.multi do
8
+ queues.each do |queue|
9
+ dispatch event, queue
10
+ end
11
+ redis.rpop(in_progress_queue)
12
+ end
13
+
14
+ logger.info "processed event: #{event['event']}, on channel: #{event['channel']}"
15
+ end
16
+
17
+ def in_progress_queue
18
+ "#{inbox}:in_progress"
19
+ end
20
+
21
+ private
22
+
23
+ def receive
24
+ event = retry_failed_event || dequeue
25
+ logger.debug "received event: #{event['event']}, on channel: #{event['channel']}"
26
+
27
+ JSON.parse event
28
+ end
29
+
30
+ def retry_failed_event
31
+ redis.lrange(in_progress_queue, -1, -1).first.tap do |event|
32
+ unless event.nil?
33
+ logger.warn "detected failed event delivery, attempting recovery"
34
+ end
35
+ end
36
+ end
37
+
38
+ def dequeue
39
+ redis.brpoplpush(inbox, in_progress_queue, timeout: config.broker_poll_interval).tap do |event|
40
+ if event.nil?
41
+ logger.debug "reached timeout for broker poll interval: #{config.broker_poll_interval} seconds"
42
+ end
43
+ end
44
+ end
45
+
46
+ def dispatch(event, queue)
47
+ job = { class: "Chasqui::Subscriber__#{queue}", args: [event] }.to_json
48
+ logger.debug "dispatching event to queue: #{queue}, with job: #{job}"
49
+ redis.rpush "queue:#{queue}", job
50
+ end
51
+
52
+ def subscriber_queues(event)
53
+ redis.smembers "subscribers:#{event['channel']}"
54
+ end
55
+
56
+ end
@@ -0,0 +1,74 @@
1
+ require 'set'
2
+
3
+ module Chasqui
4
+
5
+ HandlerAlreadyRegistered = Class.new StandardError
6
+
7
+ class Subscriber
8
+ attr_accessor :redis, :current_event
9
+ attr_reader :queue, :channel
10
+
11
+ def initialize(queue, channel)
12
+ @queue = queue
13
+ @channel = channel
14
+ end
15
+
16
+ def on(event_name, &block)
17
+ pattern = pattern_for_event event_name
18
+
19
+ if handler_patterns.include? pattern
20
+ raise HandlerAlreadyRegistered.new "handler already registered for event: #{event_name}"
21
+ else
22
+ handler_patterns << pattern
23
+ define_handler_method pattern, &block
24
+ end
25
+ end
26
+
27
+ def perform(redis_for_worker, event)
28
+ self.redis = redis_for_worker
29
+ self.current_event = event
30
+
31
+ matching_handler_patterns_for(event['event']).each do |pattern|
32
+ call_handler pattern, *event['data']
33
+ end
34
+ end
35
+
36
+ def matching_handler_patterns_for(event_name)
37
+ handler_patterns.select do |pattern|
38
+ pattern =~ event_name
39
+ end
40
+ end
41
+
42
+ def call_handler(pattern, *args)
43
+ send "handler__#{pattern.to_s}", *args
44
+ end
45
+
46
+ def evaluate(&block)
47
+ @self_before_instance_eval = eval "self", block.binding
48
+ instance_eval &block
49
+ end
50
+
51
+ private
52
+
53
+ def handler_patterns
54
+ @handler_patterns ||= Set.new
55
+ end
56
+
57
+ def method_missing(method, *args, &block)
58
+ if @self_before_instance_eval
59
+ @self_before_instance_eval.send method, *args, &block
60
+ else
61
+ super
62
+ end
63
+ end
64
+
65
+ def pattern_for_event(event_name)
66
+ /\A#{event_name.to_s.downcase.gsub('*', '.*')}\z/
67
+ end
68
+
69
+ def define_handler_method(pattern, &block)
70
+ self.class.send :define_method, "handler__#{pattern.to_s}", &block
71
+ end
72
+
73
+ end
74
+ end
@@ -1,3 +1,3 @@
1
1
  module Chasqui
2
- VERSION = "0.0.1"
2
+ VERSION = "0.1.0"
3
3
  end
@@ -0,0 +1,40 @@
1
+ class Chasqui::ResqueWorker
2
+
3
+ class << self
4
+
5
+ # Factory method to create a Resque worker class for a Chasqui::Subscriber instance.
6
+ def create(subscriber)
7
+ find_or_build_worker(subscriber).tap do |worker|
8
+ worker.class_eval do
9
+ @queue = subscriber.queue
10
+ @subscriber = subscriber
11
+
12
+ def self.perform(event)
13
+ @subscriber.perform Resque.redis, event
14
+ end
15
+ end
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ def find_or_build_worker(subscriber)
22
+ class_name = class_name_for subscriber
23
+
24
+ if Chasqui.const_defined? class_name
25
+ Chasqui.const_get class_name
26
+ else
27
+ Class.new(Chasqui::ResqueWorker).tap do |worker|
28
+ Chasqui.const_set class_name, worker
29
+ end
30
+ end
31
+ end
32
+
33
+ def class_name_for(subscriber)
34
+ queue_name_constant = subscriber.queue.gsub(/[^\w]/, '_')
35
+ "Subscriber__#{queue_name_constant}".to_sym
36
+ end
37
+
38
+ end
39
+
40
+ end
@@ -0,0 +1,25 @@
1
+ class Chasqui::SidekiqWorker
2
+
3
+ class << self
4
+
5
+ def create(subscriber)
6
+ Class.new(Chasqui::SidekiqWorker) do
7
+ include Sidekiq::Worker
8
+ sidekiq_options queue: subscriber.queue
9
+ @subscriber = subscriber
10
+
11
+ def perform(event)
12
+ self.class.subscriber.perform "TODO: redis", event
13
+ end
14
+
15
+ private
16
+
17
+ def self.subscriber
18
+ @subscriber
19
+ end
20
+ end
21
+ end
22
+
23
+ end
24
+
25
+ end
data/lib/chasqui.rb CHANGED
@@ -1,39 +1,77 @@
1
- require 'ostruct'
1
+ require 'forwardable'
2
2
  require 'json'
3
+ require 'logger'
4
+ require 'redis'
5
+ require 'redis-namespace'
6
+
3
7
  require "chasqui/version"
8
+ require "chasqui/config"
9
+ require "chasqui/broker"
10
+ require "chasqui/multi_broker"
11
+ require "chasqui/subscriber"
12
+ require "chasqui/workers/resque_worker"
13
+ require "chasqui/workers/sidekiq_worker"
4
14
 
5
15
  module Chasqui
16
+ class << self
17
+ extend Forwardable
18
+ def_delegators :config, :redis, :channel, :inbox, :inbox_queue, :logger
6
19
 
7
- Config = Struct.new :namespace, :redis
20
+ def configure(&block)
21
+ @config ||= Config.new
22
+ yield @config
23
+ end
8
24
 
9
- Defaults = {
10
- publish_queue: 'chasqui.inbox'
11
- }.freeze
25
+ def config
26
+ @config ||= Config.new
27
+ end
12
28
 
13
- module ClassMethods
14
- def namespace
15
- @config.namespace unless @config.nil?
29
+ def publish(event, *args)
30
+ payload = { event: event, channel: channel, data: args }
31
+ redis.lpush inbox_queue, payload.to_json
16
32
  end
17
33
 
18
- def redis
19
- @config.redis unless @config.nil?
34
+ def subscribe(options={}, &block)
35
+ queue = options.fetch :queue
36
+ channel = options.fetch :channel
37
+
38
+ register_subscriber(queue, channel).tap do |sub|
39
+ sub.evaluate(&block) if block_given?
40
+ Chasqui::ResqueWorker.create sub
41
+ redis.sadd "subscribers:#{channel}", queue
42
+ end
20
43
  end
21
44
 
22
- def publish_queue
23
- Defaults[:publish_queue]
45
+ def subscriber(queue)
46
+ subscribers[queue.to_s]
24
47
  end
25
48
 
26
- def configure(&block)
27
- @config ||= Config.new
28
- yield @config
49
+ def create_worker(subscriber)
50
+ case config.worker_backend
51
+ when :resque
52
+ Chasqui::ResqueWorker.create subscriber
53
+ when :sidekiq
54
+ Chasqui::SidekiqWorker.create subscriber
55
+ else
56
+ raise ConfigurationError.new(
57
+ "Please choose a supported worker_backend. Choices: #{supported_worker_backends}")
58
+ end
29
59
  end
30
60
 
31
- def publish(event, *args)
32
- name = namespace ? "#{namespace}.#{event}" : event
33
- redis.rpush publish_queue, { name: name, data: args }.to_json
61
+ private
62
+
63
+ def register_subscriber(queue, channel)
64
+ subscribers[queue.to_s] ||= Subscriber.new queue, channel
34
65
  end
35
- end
36
66
 
37
- end
67
+ def subscribers
68
+ @subscribers ||= {}
69
+ end
70
+
71
+ SUPPORTED_WORKER_BACKENDS = [:resque, :sidekiq].freeze
38
72
 
39
- Chasqui.extend Chasqui::ClassMethods
73
+ def supported_worker_backends
74
+ SUPPORTED_WORKER_BACKENDS.join(', ')
75
+ end
76
+ end
77
+ end