oni 0.0.1 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
data/lib/oni.rb CHANGED
@@ -1,2 +1,8 @@
1
+ require 'thread'
2
+ require 'benchmark'
3
+
1
4
  require_relative 'oni/version'
2
- require_relative 'oni/thread_pool'
5
+ require_relative 'oni/configurable'
6
+ require_relative 'oni/mapper'
7
+ require_relative 'oni/worker'
8
+ require_relative 'oni/daemon'
@@ -0,0 +1,107 @@
1
+ module Oni
2
+ ##
3
+ # Configurable is a basic configuration mixin that can be used to set options
4
+ # on class level and easily access them on instance level, optionally only
5
+ # evaluating the setting when it's accessed.
6
+ #
7
+ # Basic usage:
8
+ #
9
+ # class SomeClass
10
+ # include Oni::Configurable
11
+ #
12
+ # set :threads, 5
13
+ # set :logger, proc { Logger.new(STDOUT) }
14
+ #
15
+ # def some_method
16
+ # option(:threads).times do
17
+ # # ...
18
+ # end
19
+ # end
20
+ # end
21
+ #
22
+ module Configurable
23
+ ##
24
+ # @param [Class|Module] into
25
+ #
26
+ def self.included(into)
27
+ into.extend(ClassMethods)
28
+ end
29
+
30
+ ##
31
+ # Returns the value of the given option. If the value responds to `#call`
32
+ # the method is invoked and the return value of this call is returned.
33
+ #
34
+ # @param [Symbol|String] name
35
+ # @param [Mixed] default The default value to return if no custom one was
36
+ # found.
37
+ # @return [Mixed]
38
+ #
39
+ def option(name, default = nil)
40
+ value = self.class.options[name.to_sym]
41
+
42
+ if default and !value
43
+ value = default
44
+ end
45
+
46
+ return value.respond_to?(:call) ? value.call : value
47
+ end
48
+
49
+ ##
50
+ # Raises an error if the given option isn't set.
51
+ #
52
+ # @param [Symbol|String] option
53
+ # @raise [ArgumentError]
54
+ #
55
+ def require_option!(option)
56
+ unless option(option)
57
+ raise ArgumentError, "The option #{option} is required but isn't set"
58
+ end
59
+ end
60
+
61
+ module ClassMethods
62
+ ##
63
+ # Returns a Hash containing the options of the current class.
64
+ #
65
+ # @return [Hash]
66
+ #
67
+ def options
68
+ return @options ||= {}
69
+ end
70
+
71
+ ##
72
+ # Sets the option to the given value. If a Proc (or any object that
73
+ # responds to `#call`) is given it's not evaluated until it's accessed.
74
+ # This makes it possible to for example set a logger that's not created
75
+ # until an instance of the including class is created.
76
+ #
77
+ # @example Setting a regular option
78
+ # set :number, 10
79
+ #
80
+ # @example Setting an option using a proc
81
+ # # This means the logger won't be shared between different instances of
82
+ # # the including class.
83
+ # set :logger, proc { Logger.new(STDOUT) }
84
+ #
85
+ # @param [Symbol|String] option
86
+ # @param [Mixed] value
87
+ #
88
+ def set(option, value)
89
+ options[option.to_sym] = value
90
+ end
91
+
92
+ ##
93
+ # Sets a number of options based on the given Hash.
94
+ #
95
+ # @example
96
+ # set_multiple(:a => 10, :b => 20)
97
+ #
98
+ # @param [Hash] options
99
+ #
100
+ def set_multiple(options)
101
+ options.each do |option, value|
102
+ set(option, value)
103
+ end
104
+ end
105
+ end # ClassMethods
106
+ end # Configurable
107
+ end # Oni
data/lib/oni/daemon.rb ADDED
@@ -0,0 +1,190 @@
1
+ module Oni
2
+ ##
3
+ # The Daemon class takes care of retrieving work to be processed, scheduling
4
+ # it and dispatching it to a mapper and worker. In essence a Daemon instance
5
+ # can be seen as a controller when compared with typical MVC frameworks.
6
+ #
7
+ # This daemon starts a number of threads (5 by default) that will each
8
+ # perform work on their own using the corresponding mapper and worker class.
9
+ #
10
+ # @!attribute [r] workers
11
+ # @return [Array<Thread>]
12
+ #
13
+ class Daemon
14
+ include Configurable
15
+
16
+ attr_reader :workers
17
+
18
+ ##
19
+ # The default amount of threads to start.
20
+ #
21
+ # @return [Fixnum]
22
+ #
23
+ DEFAULT_THREAD_AMOUNT = 5
24
+
25
+ ##
26
+ # Creates a new instance of the class and calls `#after_initialize` if it
27
+ # is defined.
28
+ #
29
+ def initialize
30
+ @workers = []
31
+
32
+ after_initialize if respond_to?(:after_initialize)
33
+ end
34
+
35
+ ##
36
+ # Starts the daemon and waits for all threads to finish execution. This
37
+ # method is blocking since it will wait for all threads to finish.
38
+ #
39
+ # If the current class has a `before_start` method defined it's called
40
+ # before starting the daemon.
41
+ #
42
+ def start
43
+ before_start if respond_to?(:before_start)
44
+
45
+ # If we don't have any threads run in non threaded mode.
46
+ if threads > 0
47
+ threads.times do
48
+ workers << spawn_thread
49
+ end
50
+
51
+ workers.each(&:join)
52
+ else
53
+ run_thread
54
+ end
55
+ rescue => error
56
+ error(error)
57
+ end
58
+
59
+ ##
60
+ # Terminates all the threads and clears up the list. Note that calling this
61
+ # method acts much like sending a SIGKILL signal to a process: threads will
62
+ # be shut down *immediately*.
63
+ #
64
+ def stop
65
+ workers.each(&:kill)
66
+ workers.clear
67
+ end
68
+
69
+ ##
70
+ # Returns the amount of threads to use.
71
+ #
72
+ # @return [Fixnum]
73
+ #
74
+ def threads
75
+ return option(:threads, DEFAULT_THREAD_AMOUNT)
76
+ end
77
+
78
+ ##
79
+ # Processes the given message. Upon completion the `#complete` method is
80
+ # called and passed the resulting output.
81
+ #
82
+ # @param [Mixed] message
83
+ #
84
+ def process(message)
85
+ output = nil
86
+ timings = Benchmark.measure do
87
+ output = run_worker(message)
88
+ end
89
+
90
+ complete(message, output, timings)
91
+ end
92
+
93
+ ##
94
+ # Maps the input, runs the worker and then maps the output into something
95
+ # that the daemon can understand.
96
+ #
97
+ # @param [Mixed] message
98
+ # @return [Mixed]
99
+ #
100
+ def run_worker(message)
101
+ mapper = create_mapper
102
+ input = mapper.map_input(message)
103
+ worker = option(:worker).new(*input)
104
+ output = worker.process
105
+
106
+ return mapper.map_output(output)
107
+ end
108
+
109
+ ##
110
+ # Receives a message, by default this method raises an error.
111
+ #
112
+ # @raise [NotImplementedError]
113
+ #
114
+ def receive
115
+ raise NotImplementedError, 'You must manually implement #receive'
116
+ end
117
+
118
+ ##
119
+ # Called when a job has been completed, by default this method is a noop.
120
+ # This method is passed 3 arguments:
121
+ #
122
+ # 1. The raw input message.
123
+ # 2. The output of the worker (remapped by the mapper).
124
+ # 3. A Benchmark::Tms instance that contains the timings for processing the
125
+ # message.
126
+ #
127
+ # @param [Mixed] message The raw input message (e.g. an AWS SQS message)
128
+ # @param [Mixed] output The output of the worker.
129
+ # @param [Benchmark::Tms] timings
130
+ #
131
+ def complete(message, output, timings)
132
+ end
133
+
134
+ ##
135
+ # Called whenever an error is raised in the daemon, mapper or worker. By
136
+ # default this method just re-raises the error.
137
+ #
138
+ # Note that this callback method is called from a thread in which the
139
+ # exception occured, not from the main thread.
140
+ #
141
+ # @param [StandardError] error
142
+ #
143
+ def error(error)
144
+ raise error
145
+ end
146
+
147
+ ##
148
+ # Creates a new mapper and passes it a set of arguments as defined in
149
+ # {Oni::Daemon#mapper_arguments}.
150
+ #
151
+ # @return [Oni::Mapper]
152
+ #
153
+ def create_mapper
154
+ unless option(:mapper)
155
+ raise ArgumentError, 'No mapper has been set in the `:mapper` option'
156
+ end
157
+
158
+ return option(:mapper).new
159
+ end
160
+
161
+ ##
162
+ # Spawns a new thread that waits for daemon input.
163
+ #
164
+ # @return [Thread]
165
+ #
166
+ def spawn_thread
167
+ thread = Thread.new { run_thread }
168
+
169
+ thread.abort_on_exception = true
170
+
171
+ return thread
172
+ end
173
+
174
+ ##
175
+ # The main code to execute in individual threads.
176
+ #
177
+ # If an error occurs in the receive method or processing a job the error
178
+ # handler is executed and the process is retried. It's the responsibility
179
+ # of the `error` method to determine if the process should fail only once
180
+ # (and fail hard) or if it should continue running.
181
+ #
182
+ def run_thread
183
+ receive { |message| process(message) }
184
+ rescue => error
185
+ error(error)
186
+
187
+ retry
188
+ end
189
+ end # Daemon
190
+ end # Oni
@@ -0,0 +1,61 @@
1
+ require 'aws-sdk'
2
+
3
+ module Oni
4
+ module Daemons
5
+ ##
6
+ # The SQS daemon is a basic daemon skeleton that can be used to process
7
+ # jobs from an Amazon SQS queue.
8
+ #
9
+ # Basic usage:
10
+ #
11
+ # class MyDaemon < Oni::Daemons::SQS
12
+ # set :queue_name, 'my_queue'
13
+ # end
14
+ #
15
+ # The following options can be set:
16
+ #
17
+ # * `queue_name` (required): the name of the queue to poll as a String.
18
+ # * `poll_options`: a Hash of options to pass to the `poll` method of the
19
+ # AWS SQS queue. See the documentation of `AWS::SQS::Queue#poll` for more
20
+ # information on the available options.
21
+ #
22
+ class SQS < Daemon
23
+ ##
24
+ # Checks if the `queue_name` option is set.
25
+ #
26
+ def after_initialize
27
+ require_option!(:queue_name)
28
+ end
29
+
30
+ ##
31
+ # Polls an SQS queue for a message and processes it.
32
+ #
33
+ def receive
34
+ queue.poll(poll_options) do |message|
35
+ yield message
36
+ end
37
+ end
38
+
39
+ ##
40
+ # Returns a Hash containing the options to use for the `poll` method of
41
+ # the SQS queue.
42
+ #
43
+ # @return [Hash]
44
+ #
45
+ def poll_options
46
+ return option(:poll_options, {})
47
+ end
48
+
49
+ ##
50
+ # Returns the queue to use for the current thread.
51
+ #
52
+ # @return [AWS::SQS::Queue]
53
+ #
54
+ #:nocov:
55
+ def queue
56
+ return AWS::SQS.new.queues.named(option(:queue_name))
57
+ end
58
+ #:nocov:
59
+ end # SQS
60
+ end # Daemons
61
+ end # Oni
data/lib/oni/mapper.rb ADDED
@@ -0,0 +1,30 @@
1
+ module Oni
2
+ ##
3
+ # Abstract mapper class that takes care of some common boilerplate code.
4
+ #
5
+ class Mapper
6
+ include Configurable
7
+
8
+ ##
9
+ # Remaps the input of the daemon into a format that's easy to use for the
10
+ # worker.
11
+ #
12
+ # @param [Mixed] input
13
+ # @return [Mixed]
14
+ #
15
+ def map_input(input)
16
+ return input
17
+ end
18
+
19
+ ##
20
+ # Remaps the output of the working into a format that's easy to use for the
21
+ # daemon layer.
22
+ #
23
+ # @param [Mixed] output
24
+ # @return [Mixed]
25
+ #
26
+ def map_output(output)
27
+ return output
28
+ end
29
+ end # Mapper
30
+ end # Oni
data/lib/oni/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Oni
2
- VERSION = '0.0.1'
2
+ VERSION = '1.0.0'
3
3
  end # Oni
data/lib/oni/worker.rb ADDED
@@ -0,0 +1,18 @@
1
+ module Oni
2
+ ##
3
+ # An abstract worker class that takes care of some common boilerplate code.
4
+ #
5
+ class Worker
6
+ include Configurable
7
+
8
+ ##
9
+ # Runs the worker and returns some kind of output.
10
+ #
11
+ # @return [Mixed]
12
+ # @raise [NotImplementedError]
13
+ #
14
+ def process
15
+ raise NotImplementedError, 'You must implement #process yourself'
16
+ end
17
+ end # Worker
18
+ end # Oni
data/oni.gemspec ADDED
@@ -0,0 +1,30 @@
1
+ require File.expand_path('../lib/oni/version', __FILE__)
2
+
3
+ Gem::Specification.new do |gem|
4
+ gem.name = 'oni'
5
+ gem.version = Oni::VERSION
6
+ gem.authors = [
7
+ 'Yorick Peterse',
8
+ 'Wilco van Duinkerken'
9
+ ]
10
+
11
+ gem.summary = 'Framework for building concurrent daemons in Ruby.'
12
+ gem.description = gem.summary
13
+ gem.has_rdoc = 'yard'
14
+ gem.license = 'MIT'
15
+
16
+ gem.required_ruby_version = '>= 1.9.3'
17
+
18
+ gem.files = `git ls-files`.split("\n").sort
19
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
20
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
21
+
22
+ gem.add_development_dependency 'rake'
23
+ gem.add_development_dependency 'bundler'
24
+ gem.add_development_dependency 'rspec'
25
+ gem.add_development_dependency 'yard'
26
+ gem.add_development_dependency 'simplecov'
27
+ gem.add_development_dependency 'kramdown'
28
+ gem.add_development_dependency 'ci_reporter'
29
+ gem.add_development_dependency 'aws-sdk'
30
+ end