cod 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/Gemfile +1 -1
  2. data/HISTORY.txt +5 -1
  3. data/README +12 -13
  4. data/Rakefile +7 -1
  5. data/examples/{ping.rb → ping_pong/ping.rb} +1 -1
  6. data/examples/{pong.rb → ping_pong/pong.rb} +1 -1
  7. data/examples/{presence_client.rb → presence/client.rb} +0 -0
  8. data/examples/{presence_server.rb → presence/server.rb} +0 -0
  9. data/examples/queue/README +9 -0
  10. data/examples/queue/client.rb +31 -0
  11. data/examples/queue/queue.rb +51 -0
  12. data/examples/queue/send.rb +9 -0
  13. data/examples/service.rb +2 -2
  14. data/examples/tcp.rb +1 -1
  15. data/lib/cod.rb +47 -82
  16. data/lib/cod/beanstalk.rb +7 -0
  17. data/lib/cod/beanstalk/channel.rb +170 -0
  18. data/lib/cod/beanstalk/serializer.rb +80 -0
  19. data/lib/cod/beanstalk/service.rb +53 -0
  20. data/lib/cod/channel.rb +54 -12
  21. data/lib/cod/pipe.rb +188 -0
  22. data/lib/cod/select.rb +47 -0
  23. data/lib/cod/select_group.rb +87 -0
  24. data/lib/cod/service.rb +55 -42
  25. data/lib/cod/simple_serializer.rb +19 -0
  26. data/lib/cod/tcp_client.rb +202 -0
  27. data/lib/cod/tcp_server.rb +124 -0
  28. data/lib/cod/work_queue.rb +129 -0
  29. metadata +31 -45
  30. data/examples/pubsub/README +0 -12
  31. data/examples/pubsub/client.rb +0 -13
  32. data/examples/pubsub/directory.rb +0 -13
  33. data/examples/service_directory.rb +0 -32
  34. data/lib/cod/channel/base.rb +0 -185
  35. data/lib/cod/channel/beanstalk.rb +0 -69
  36. data/lib/cod/channel/pipe.rb +0 -137
  37. data/lib/cod/channel/tcp.rb +0 -16
  38. data/lib/cod/channel/tcpconnection.rb +0 -67
  39. data/lib/cod/channel/tcpserver.rb +0 -84
  40. data/lib/cod/client.rb +0 -81
  41. data/lib/cod/connection/beanstalk.rb +0 -77
  42. data/lib/cod/directory.rb +0 -98
  43. data/lib/cod/directory/countdown.rb +0 -31
  44. data/lib/cod/directory/subscription.rb +0 -59
  45. data/lib/cod/object_io.rb +0 -6
  46. data/lib/cod/objectio/connection.rb +0 -106
  47. data/lib/cod/objectio/reader.rb +0 -98
  48. data/lib/cod/objectio/serializer.rb +0 -26
  49. data/lib/cod/objectio/writer.rb +0 -27
  50. data/lib/cod/topic.rb +0 -95
data/Gemfile CHANGED
@@ -1,6 +1,6 @@
1
1
  source "http://rubygems.org"
2
2
 
3
- gem 'uuid'
3
+ gem 'rake'
4
4
  gem 'beanstalk-client'
5
5
 
6
6
  group :test do
data/HISTORY.txt CHANGED
@@ -1,5 +1,9 @@
1
+ == 0.4.0 / 29Nov2011
1
2
 
2
- == 0.3.1 / ???
3
+ * A complete rewrite, focusing on lean implementation and logical/useful
4
+ error conditions.
5
+
6
+ == 0.3.1 / 25Aug2011
3
7
 
4
8
  + Large improvements in resilience to crashes, reconnects are now attempted
5
9
  but can be improved further.
data/README CHANGED
@@ -1,33 +1,32 @@
1
1
 
2
- cod is a simple ipc abstraction layer. It allows you to focus on interaction
3
- between processes instead of having to think about interaction with the OS.
2
+ IPC for babies and those who want to become. Another thin API layer over what
3
+ Ruby offers for IO.pipe, TCP sockets and FIFOs: Makes sending and receiving
4
+ of Ruby objects a breeze.
5
+
6
+ A good place to start is the documentation for the Cod module.
4
7
 
5
8
  SYNOPSIS
6
9
 
7
10
  # Cod's basic elements are channels, unidirectional communication links.
8
11
  pipe = Cod.pipe
9
- beanstalk = Cod.beanstalk('localhost:11300', 'a_channel')
10
12
 
11
13
  # You can use those either directly:
12
14
  pipe.put :some_ruby_object # Process A
13
15
  pipe.get # => :some_ruby_object # Process B
14
16
 
15
17
  # Or use them as bricks for more:
16
- service = Cod::Service.new(beanstalk)
17
- client = Cod::Client.new(beanstalk, pipe)
18
+ service = beanstalk.service
19
+ client = beanstalk.client(pipe)
18
20
 
19
21
  service.one { |msg| :response } # Process A
20
22
  client.call :ruby_object # => :response # Process B
21
-
22
- # And more: Publish/Subscribe, easy construction of more advanced
23
- # distributed communication.
24
-
23
+
25
24
  STATUS
26
25
 
27
- Becoming more useful by the day. Most things will work nicely already,
28
- although error handling is not production quality. Toy around with it now
29
- and give me feedback!
26
+ Complete rewrite of the code: Did away with some of the complexities that
27
+ stemmed from early design work. The functionality that is there is much like
28
+ that we had before, but some things have been designed more logically.
30
29
 
31
- At version 0.3.1
30
+ At version 0.4.0
32
31
 
33
32
  (c) 2011 Kaspar Schiess
data/Rakefile CHANGED
@@ -1,4 +1,3 @@
1
- require 'psych'
2
1
  require "rubygems"
3
2
  require "rdoc/task"
4
3
  require 'rspec/core/rake_task'
@@ -9,6 +8,13 @@ RSpec::Core::RakeTask.new
9
8
 
10
9
  task :default => :spec
11
10
 
11
+ task :stats do
12
+ %w(lib spec).each do |path|
13
+ printf "%10s:", path
14
+ system %Q(find #{path} -name "*.rb" | xargs wc -l | grep total)
15
+ end
16
+ end
17
+
12
18
  require 'sdoc'
13
19
 
14
20
  # Generate documentation
@@ -1,4 +1,4 @@
1
- $:.unshift File.expand_path(File.dirname(__FILE__) + "/../lib")
1
+ $:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
2
2
  require 'cod'
3
3
 
4
4
  pipe = Cod.beanstalk('localhost:11300', 'pingpong')
@@ -1,4 +1,4 @@
1
- $:.unshift File.expand_path(File.dirname(__FILE__) + "/../lib")
1
+ $:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
2
2
  require 'cod'
3
3
 
4
4
  pipe = Cod.beanstalk('localhost:11300', "pingpong")
File without changes
File without changes
@@ -0,0 +1,9 @@
1
+ A small queue like example.
2
+
3
+ run ruby send.rb N to send N messages through queue.rb to client.rb.
4
+ Client will print statistics (calls per second) every 100 messages.
5
+
6
+ Problems:
7
+ - queue.rb seems to always consume 100% CPU
8
+ - number of messages per second is way low
9
+ - some exceptions are still not handled correctly
@@ -0,0 +1,31 @@
1
+ $:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
2
+ require 'cod'
3
+
4
+ queue = Cod.tcp('localhost:12345')
5
+
6
+ queue.put [:join, queue]
7
+
8
+ class CallCounter
9
+ attr_reader :n
10
+ def initialize
11
+ @n = 0
12
+ @last_lap = [0, Time.now]
13
+ end
14
+ def inc
15
+ @n += 1
16
+ end
17
+ def calls_per_sec
18
+ ln, ll = @last_lap
19
+ @last_lap = [@n, Time.now]
20
+
21
+ (@n - ln) / (Time.now - ll)
22
+ end
23
+ end
24
+
25
+ cc = CallCounter.new
26
+ loop do
27
+ wi = queue.get
28
+
29
+ cc.inc
30
+ puts cc.calls_per_sec if cc.n%100==0
31
+ end
@@ -0,0 +1,51 @@
1
+ $:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
2
+ require 'cod'
3
+
4
+ class Queue
5
+ def initialize(url)
6
+ @url = url
7
+ connect
8
+ end
9
+
10
+ def connect
11
+ @clients = []
12
+ @server = Cod.tcpserver(@url)
13
+ end
14
+
15
+ def run
16
+ loop do
17
+ handle_commands
18
+
19
+ check_connections
20
+ end
21
+ end
22
+
23
+ def handle_commands
24
+ while @server.waiting?
25
+ cmd, *rest = @server.get
26
+
27
+ dispatch_command cmd, rest
28
+ end
29
+ end
30
+
31
+ def dispatch_command(cmd, rest)
32
+ self.send("cmd_#{cmd}", *rest)
33
+ end
34
+
35
+ def cmd_join(connection)
36
+ puts "Join at #{Time.now}."
37
+ @clients << connection
38
+ end
39
+
40
+ def cmd_work_item
41
+ @clients.each do |client|
42
+ client.put :work_item
43
+ end
44
+ end
45
+
46
+ def check_connections
47
+ @clients.keep_if { |client| client.connected? }
48
+ end
49
+ end
50
+
51
+ Queue.new('localhost:12345').run
@@ -0,0 +1,9 @@
1
+ $:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
2
+ require 'cod'
3
+
4
+ queue = Cod.tcp('localhost:12345')
5
+
6
+ n = Integer(ARGV.first)
7
+ n.times do
8
+ queue.put :work_item
9
+ end
data/examples/service.rb CHANGED
@@ -9,7 +9,7 @@ service_channel = Cod.pipe
9
9
  answer_channel = Cod.pipe
10
10
 
11
11
  child_pid = fork do
12
- service = Cod::Service.new(service_channel)
12
+ service = Cod.service(service_channel)
13
13
  service.one { |call|
14
14
  puts "Service got called with #{call.inspect}"
15
15
  time = Time.now
@@ -17,7 +17,7 @@ child_pid = fork do
17
17
  time }
18
18
  end
19
19
 
20
- client = Cod::Client.new(service_channel, answer_channel)
20
+ client = Cod.client(service_channel, answer_channel)
21
21
  puts "Calling service..."
22
22
  answer = client.call('42')
23
23
 
data/examples/tcp.rb CHANGED
@@ -5,7 +5,7 @@ require 'cod'
5
5
  require 'example_scaffold'
6
6
 
7
7
  server {
8
- channel = Cod.tcpserver('127.0.0.1:5454')
8
+ channel = Cod.tcp_server('127.0.0.1:5454')
9
9
 
10
10
  client = channel.get
11
11
  client.put 'heiho from server'
data/lib/cod.rb CHANGED
@@ -1,5 +1,3 @@
1
- require 'uuid'
2
-
3
1
  # The core concept of Cod are 'channels'. (Cod::Channel::Base) You can create
4
2
  # such channels on top of the various transport layers. Once you have such a
5
3
  # channel, you #put messages into it and you #get messages out of it. Messages
@@ -8,10 +6,7 @@ require 'uuid'
8
6
  #
9
7
  # Cod also brings a few abstractions layered on top of channels: You can use
10
8
  # channels to present 'services' (Cod::Service) to the network: A service is a
11
- # simple one or two way RPC call. (one way = asynchronous) You can also use
12
- # channels to run a 'directory' (Cod::Directory) where processes subscribe to
13
- # information using a filter. They then get information that matches their
14
- # filter written to their inbound channel. (also called pub/sub)
9
+ # simple one or two way RPC call. (one way = asynchronous)
15
10
  #
16
11
  # Cod channels are serializable whereever possible. If you want to tell
17
12
  # somebody where to write his answers and/or questions to, send him the
@@ -28,101 +23,71 @@ require 'uuid'
28
23
  # most of the tricky stuff!
29
24
  #
30
25
  module Cod
31
- # This gets raised in #create_reference when the identifier passed in is
32
- # either invalid (has never existed) or when it cannot be turned into an
33
- # object instance. (Because it might have been garbage collected or other
34
- # such reasons)
35
- #
36
- class InvalidIdentifier < StandardError; end
37
-
38
- # Creates a beanstalkd based channel. Messages are written to a tube and
39
- # read from it. This channel is read/write. Multiple readers will obtain
40
- # messages in a round-robin fashion from the beanstalk server.
41
- #
42
- # Returns an instance of the Cod::Channel::Beanstalk class.
26
+ # Creates a pipe connection that is visible to this process and its
27
+ # children. (see Cod::Pipe)
43
28
  #
44
- # Example:
45
- # chan = Cod.beanstalk('localhost:11300', 'my_tube')
46
- #
47
- def beanstalk(url, name)
48
- Cod::Channel::Beanstalk.new(
49
- Connection::Beanstalk.new(url),
50
- name)
51
- end
52
- module_function :beanstalk
53
-
54
- # Creates a IO.pipe based channel. Messages are written to one end of the
55
- # pipe and come out on the other end. This channel can have only one reader,
56
- # but of course multiple writers. Also, once you either write or read from
57
- # such a channel, it will not be available for the other operation anymore.
58
- #
59
- # A common trick is to #dup the channel before using it to either read or
60
- # write, so that the copy can still be used for both operations.
61
- #
62
- # Note that Cod.pipe channels are usable from process childs (#fork) as
63
- # well. As such, they are ideally suited for process control.
64
- #
65
- # Returns an instance of the Cod::Channel::Pipe class.
66
- #
67
- # Example:
68
- # chan = Cod.pipe
69
- #
70
- def pipe(name=nil)
71
- Cod::Channel::Pipe.new(name)
29
+ def pipe
30
+ Cod::Pipe.new
72
31
  end
73
32
  module_function :pipe
74
33
 
75
- # Creates a tcp connection to the destination and returns a channel for it.
34
+ # Creates a tcp connection to the destination and returns a channel for it.
35
+ # (see Cod::TcpClient)
76
36
  #
77
- def tcp(destination)
78
- Cod::Channel::TCPConnection.new(destination)
37
+ def tcp(destination, serializer=nil)
38
+ Cod::TcpClient.new(
39
+ destination,
40
+ serializer || SimpleSerializer.new)
79
41
  end
80
42
  module_function :tcp
81
43
 
82
- # Creates a tcp listener on bind_to and returns a channel for it.
44
+ # Creates a tcp listener on bind_to and returns a channel for it. (see
45
+ # Cod::TcpServer)
83
46
  #
84
- def tcpserver(bind_to)
85
- Cod::Channel::TCPServer.new(bind_to)
47
+ def tcp_server(bind_to)
48
+ Cod::TcpServer.new(bind_to)
86
49
  end
87
- module_function :tcpserver
88
-
89
- # Returns a UUID that should be unique for this machine (based on MAC), this
90
- # Thread and Process. Even after a fork.
50
+ module_function :tcp_server
51
+
52
+ # Creates a channel based on the beanstalkd messaging queue. (see
53
+ # Cod::Beanstalk::Channel)
91
54
  #
92
- # This is used to create identity on the network. Internal method.
55
+ def beanstalk(tube_name, server=nil)
56
+ Cod::Beanstalk::Channel.new(tube_name, server||'localhost:11300')
57
+ end
58
+ module_function :beanstalk
59
+
60
+ # Indicates that the given channel is write only. This gets raised on
61
+ # operations like #put.
62
+ #
63
+ class WriteOnlyChannel < StandardError
64
+ def initialize
65
+ super("This channel is write only, attempted read operation.")
66
+ end
67
+ end
68
+
69
+ # Indicates that the channel is read only. This gets raised on operations
70
+ # like #get.
93
71
  #
94
- def uuid
95
- uuid_generator.generate
96
- end
97
- def uuid_generator
98
- pid, generator = Thread.current[:_cod_uuid_generator]
99
-
100
- if pid && Process.pid == pid
101
- return generator
72
+ class ReadOnlyChannel < StandardError
73
+ def initialize
74
+ super("This channel is read only, attempted write operation.")
102
75
  end
103
-
104
- pid, generator = Thread.current[:_cod_uuid_generator] = [Process.pid, UUID.new]
105
- return generator
106
76
  end
107
- module_function :uuid, :uuid_generator
108
77
  end
109
78
 
110
- module Cod::Connection; end
111
- require 'cod/connection/beanstalk'
112
-
113
- require 'cod/object_io'
79
+ require 'cod/select_group'
80
+ require 'cod/select'
114
81
 
115
82
  require 'cod/channel'
116
- require 'cod/channel/base'
117
- require 'cod/channel/pipe'
118
- require 'cod/channel/beanstalk'
119
- require 'cod/channel/tcpconnection'
120
- require 'cod/channel/tcpserver'
121
83
 
122
- require 'cod/client'
84
+ require 'cod/simple_serializer'
85
+
86
+ require 'cod/pipe'
87
+
88
+ require 'cod/tcp_client'
89
+ require 'cod/tcp_server'
123
90
 
124
91
  require 'cod/service'
92
+ require 'cod/beanstalk'
125
93
 
126
- require 'cod/directory'
127
- require 'cod/directory/subscription'
128
- require 'cod/topic'
@@ -0,0 +1,7 @@
1
+ module Cod::Beanstalk
2
+ end
3
+
4
+ require 'cod/beanstalk/serializer'
5
+ require 'cod/beanstalk/channel'
6
+ require 'cod/beanstalk/service'
7
+
@@ -0,0 +1,170 @@
1
+ module Cod::Beanstalk
2
+
3
+ # NOTE: Beanstalk channels cannot currently be used in Cod.select. This is
4
+ # due to limitations inherent in the beanstalkd protocol. We'll probably
5
+ # try to get a patch into beanstalkd to change this.
6
+ #
7
+ # NOTE: If you embed a beanstalk channel into one of your messages, you will
8
+ # get a channel that connects to the same server and the same tube on the
9
+ # other end. This behaviour is useful for Cod::Service.
10
+ #
11
+ class Channel < Cod::Channel
12
+ JOB_PRIORITY = 0
13
+
14
+ def initialize(tube_name, server_url)
15
+ @tube_name, @server_url = tube_name, server_url
16
+
17
+ @body_serializer = Cod::SimpleSerializer.new
18
+ @transport = connection(server_url, tube_name)
19
+ end
20
+
21
+ def put(msg)
22
+ pri = JOB_PRIORITY
23
+ delay = 0
24
+ ttr = 120
25
+ body = @body_serializer.en(msg)
26
+
27
+ answer, *rest = @transport.interact([:put, pri, delay, ttr, body])
28
+ fail "#put fails, #{answer.inspect}" unless answer == :inserted
29
+ end
30
+
31
+ def get
32
+ id, msg = bs_reserve
33
+
34
+ # We delete the job immediately, since #get should be definitive.
35
+ bs_delete(id)
36
+
37
+ deserialize(msg)
38
+ end
39
+
40
+ def close
41
+ @transport.close
42
+ end
43
+
44
+ def to_read_fds
45
+ fail "Cod.select not supported with beanstalkd channels.\n"+
46
+ "To support this, we will have to extend the beanstalkd protocol."
47
+ end
48
+
49
+ # --------------------------------------------------------- service/client
50
+ def service
51
+ Service.new(self)
52
+ end
53
+ def client(answers_to)
54
+ Service::Client.new(self, answers_to)
55
+ end
56
+
57
+ # -------------------------------------------------------- queue interface
58
+ def try_get
59
+ fail "No block given to #try_get" unless block_given?
60
+
61
+ id, msg = bs_reserve
62
+ control = Control.new(id, self)
63
+
64
+ begin
65
+ retval = yield(deserialize(msg), control)
66
+ rescue Exception
67
+ control.release unless control.command_given?
68
+ raise
69
+ ensure
70
+ control.delete unless control.command_given?
71
+ end
72
+
73
+ return retval
74
+ end
75
+
76
+ # Holds a message id of a reserved message. Allows several commands to be
77
+ # executed on the message. See #try_get.
78
+ class Control # :nodoc:
79
+ attr_reader :msg_id
80
+
81
+ def initialize(msg_id, channel)
82
+ @msg_id = msg_id
83
+ @channel = channel
84
+ @command_given = false
85
+ end
86
+
87
+ def command_given?
88
+ @command_given
89
+ end
90
+
91
+ def delete
92
+ @command_given = true
93
+ @channel.bs_delete(@msg_id)
94
+ end
95
+ def release
96
+ @command_given = true
97
+ @channel.bs_release(@msg_id)
98
+ end
99
+ def release_with_delay(seconds)
100
+ fail ArgumentError, "Only integer number of seconds are allowed." \
101
+ unless seconds.floor == seconds
102
+
103
+ @command_given = true
104
+ @channel.bs_release_with_delay(@msg_id, seconds)
105
+ end
106
+ def bury
107
+ @command_given = true
108
+ @channel.bs_bury(@msg_id)
109
+ end
110
+ end
111
+
112
+ # ---------------------------------------------------------- serialization
113
+ def _dump(level) # :nodoc:
114
+ Marshal.dump(
115
+ [@tube_name, @server_url])
116
+ end
117
+ def self._load(str) # :nodoc:
118
+ tube_name, server_url = Marshal.load(str)
119
+ Cod.beanstalk(tube_name, server_url)
120
+ end
121
+
122
+ # ----------------------------------------------------- beanstalk commands
123
+ def bs_delete(msg_id) # :nodoc:
124
+ bs_command([:delete, msg_id], :deleted)
125
+ end
126
+ def bs_release(msg_id) # :nodoc:
127
+ bs_command([:release, msg_id, JOB_PRIORITY, 0], :released)
128
+ end
129
+ def bs_release_with_delay(msg_id, seconds) # :nodoc:
130
+ bs_command([:release, msg_id, JOB_PRIORITY, seconds], :released)
131
+ end
132
+ def bs_bury(msg_id)
133
+ # NOTE: Why I need to assign a priority when burying I fail to
134
+ # understand. Like a priority for rapture?
135
+ bs_command([:bury, msg_id, JOB_PRIORITY], :buried)
136
+ end
137
+ private
138
+ def bs_reserve
139
+ answer, *rest = bs_command([:reserve], :reserved)
140
+ rest
141
+ end
142
+ def bs_command(cmd, good_answer)
143
+ answer, *rest = @transport.interact(cmd)
144
+ fail "#{cmd.first.inspect} fails, #{answer.inspect}" \
145
+ unless answer == good_answer
146
+ [answer, *rest]
147
+ end
148
+
149
+ def deserialize(msg)
150
+ @body_serializer.de(StringIO.new(msg))
151
+ end
152
+
153
+ def connection(server_url, tube_name)
154
+ conn = Cod.tcp(server_url, Serializer.new)
155
+
156
+ begin
157
+ answer, *rest = conn.interact([:use, tube_name])
158
+ fail "#init_tube fails, #{answer.inspect}" unless answer == :using
159
+
160
+ answer, *rest = conn.interact([:watch, tube_name])
161
+ fail "#init_tube fails, #{answer.inspect}" unless answer == :watching
162
+ rescue
163
+ conn.close
164
+ raise
165
+ end
166
+
167
+ conn
168
+ end
169
+ end
170
+ end