pants 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gemtest +1 -0
- data/.gitignore +23 -0
- data/.rspec +3 -0
- data/.travis.yml +4 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +42 -0
- data/History.rdoc +6 -0
- data/README.rdoc +339 -0
- data/Rakefile +23 -0
- data/bin/pants +59 -0
- data/lib/pants.rb +84 -0
- data/lib/pants/core.rb +216 -0
- data/lib/pants/error.rb +5 -0
- data/lib/pants/logger.rb +8 -0
- data/lib/pants/network_helpers.rb +25 -0
- data/lib/pants/readers/base_reader.rb +302 -0
- data/lib/pants/readers/file_reader.rb +81 -0
- data/lib/pants/readers/udp_reader.rb +80 -0
- data/lib/pants/seam.rb +120 -0
- data/lib/pants/version.rb +3 -0
- data/lib/pants/writers/base_writer.rb +73 -0
- data/lib/pants/writers/file_writer.rb +59 -0
- data/lib/pants/writers/udp_writer.rb +125 -0
- data/pants.gemspec +31 -0
- data/spec/acceptance/file_reads_spec.rb +24 -0
- data/spec/spec_helper.rb +10 -0
- data/spec/support/matchers/file_same_size_as.rb +20 -0
- data/spec/support/pants.wav +0 -0
- data/spec/unit/pants/core_spec.rb +105 -0
- data/spec/unit/pants/readers/base_reader_spec.rb +340 -0
- data/spec/unit/pants/readers/file_reader_spec.rb +94 -0
- data/spec/unit/pants/version_spec.rb +7 -0
- data/spec/unit/pants/writers/base_writer_spec.rb +35 -0
- data/spec/unit/pants/writers/file_writer_spec.rb +71 -0
- data/spec/unit/pants/writers/udp_writer_spec.rb +146 -0
- data/spec/unit/pants_spec.rb +6 -0
- data/tasks/pantsmark.thor +66 -0
- data/test_readers.rb +43 -0
- data/test_seams.rb +107 -0
- metadata +215 -0
data/.gemtest
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
|
data/.gitignore
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# rcov generated
|
2
|
+
coverage
|
3
|
+
|
4
|
+
# rdoc generated
|
5
|
+
rdoc
|
6
|
+
|
7
|
+
# yard generated
|
8
|
+
doc
|
9
|
+
.yardoc
|
10
|
+
|
11
|
+
# bundler
|
12
|
+
.bundle
|
13
|
+
|
14
|
+
# packaging
|
15
|
+
pkg
|
16
|
+
pants*.gem
|
17
|
+
|
18
|
+
# Rubinius
|
19
|
+
.rbx
|
20
|
+
|
21
|
+
# IDE files
|
22
|
+
.idea/
|
23
|
+
atlassian-ide-plugin.xml
|
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
pants (0.1.0)
|
5
|
+
eventmachine (>= 1.0.0)
|
6
|
+
log_switch (>= 0.4.0)
|
7
|
+
thor
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: http://rubygems.org/
|
11
|
+
specs:
|
12
|
+
diff-lcs (1.1.3)
|
13
|
+
eventmachine (1.0.0)
|
14
|
+
eventmachine (1.0.0-java)
|
15
|
+
log_switch (0.4.0)
|
16
|
+
multi_json (1.5.0)
|
17
|
+
rake (10.0.3)
|
18
|
+
rspec (2.12.0)
|
19
|
+
rspec-core (~> 2.12.0)
|
20
|
+
rspec-expectations (~> 2.12.0)
|
21
|
+
rspec-mocks (~> 2.12.0)
|
22
|
+
rspec-core (2.12.2)
|
23
|
+
rspec-expectations (2.12.1)
|
24
|
+
diff-lcs (~> 1.1.3)
|
25
|
+
rspec-mocks (2.12.1)
|
26
|
+
simplecov (0.7.1)
|
27
|
+
multi_json (~> 1.0)
|
28
|
+
simplecov-html (~> 0.7.1)
|
29
|
+
simplecov-html (0.7.1)
|
30
|
+
thor (0.17.0)
|
31
|
+
yard (0.8.3)
|
32
|
+
|
33
|
+
PLATFORMS
|
34
|
+
java
|
35
|
+
ruby
|
36
|
+
|
37
|
+
DEPENDENCIES
|
38
|
+
pants!
|
39
|
+
rake
|
40
|
+
rspec (>= 2.6.0)
|
41
|
+
simplecov
|
42
|
+
yard (>= 0.7.2)
|
data/History.rdoc
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,339 @@
|
|
1
|
+
= pants
|
2
|
+
|
3
|
+
* https://github.com/turboladen/pants
|
4
|
+
|
5
|
+
{<img src="https://travis-ci.org/turboladen/pants.png?branch=master" alt="Build Status" />}[https://travis-ci.org/turboladen/pants]
|
6
|
+
{<img src="https://codeclimate.com/badge.png" />}[https://codeclimate.com/github/turboladen/pants]
|
7
|
+
|
8
|
+
== DESCRIPTION:
|
9
|
+
|
10
|
+
Pants redirects IO using {http://rubyeventmachine.com EventMachine} from one
|
11
|
+
input source to many different destinations. In some senses, pants is like a
|
12
|
+
*nix pipe that (works on Windows and) allows for duplicating data across many
|
13
|
+
pipes (like splice and tee).
|
14
|
+
|
15
|
+
== FEATURES/PROBLEMS:
|
16
|
+
|
17
|
+
Read raw data from a
|
18
|
+
* file
|
19
|
+
* UDP socket (unicast or multicast)
|
20
|
+
|
21
|
+
...and write to any number and combination of
|
22
|
+
* files
|
23
|
+
* UDP sockets (unicast or multicast)
|
24
|
+
* pants seams
|
25
|
+
|
26
|
+
Also:
|
27
|
+
* All readers can write to any other writer
|
28
|
+
* Pluggable: write your own reader or writer
|
29
|
+
|
30
|
+
|
31
|
+
== SYNOPSIS:
|
32
|
+
|
33
|
+
=== Concepts
|
34
|
+
|
35
|
+
==== Readers & Writers
|
36
|
+
|
37
|
+
The core concept of pants is that without using threads and system calls, it's
|
38
|
+
difficult to duplicate data. Pants simplifies duplicating data from a single
|
39
|
+
source by using "readers" to read data from a source, then packetize it in some
|
40
|
+
form that "writers" can consume and use for their individual needs. These
|
41
|
+
readers and writers are similar to read and write ends of a *nix pipe, but with
|
42
|
+
pants there can be more than one write end of the pipe.
|
43
|
+
|
44
|
+
For example, you can use a UDP reader to read data on an IP and port, then
|
45
|
+
simultaneously write that data to a file and forward it on to another IP and
|
46
|
+
port using writers.
|
47
|
+
|
48
|
+
==== Seams
|
49
|
+
|
50
|
+
Next, pants also uses "seams" to act as a middle-man for readers and writers. A
|
51
|
+
seam is just like another reader, but it can read from other readers. Seams are
|
52
|
+
primarily useful for doing something to the data from a reader before passing it
|
53
|
+
on to writers. Think of it like a pipe with one read end and the ability to
|
54
|
+
have many write ends.
|
55
|
+
|
56
|
+
For example, the {rtp}[https://github.com/turboladen/rtp] gem
|
57
|
+
is responsible for (amongst other things) pulling an A/V stream out of an A/V
|
58
|
+
file, making sure those chunks of data (A/V frames) are sized well for sending
|
59
|
+
out via UDP, then adding some headers to each on of those chunks (packets), then
|
60
|
+
actually sending each of those packets to some number of UDP endpoints. To do
|
61
|
+
this with pants, it would:
|
62
|
+
|
63
|
+
* Use a reader (well, a demuxer to be clear) to read the A/V stream in the file
|
64
|
+
* Use a seam to accept the A/V stream data chunks, make sure the chunks are
|
65
|
+
sized right, then add an RTP header to each chunk
|
66
|
+
* Use a writer (or many writers) to read from the seam and send the data out to
|
67
|
+
a UDP IP/port (or many UDP IPs/ports)
|
68
|
+
|
69
|
+
=== Pros and Cons
|
70
|
+
|
71
|
+
==== Pros
|
72
|
+
|
73
|
+
If you plan to do the above for just one UDP client, the benefits of pants might
|
74
|
+
not quite shine through (although it will do this quite quickly, and only in a
|
75
|
+
maybe three lines of code); if you intend to be sending the RTP-encoded data to
|
76
|
+
a number of UDP clients though, pants shines in that you will have only had to
|
77
|
+
spend system resources for reading the file once and encoding the data (which
|
78
|
+
can be an expensive operation) once before sending it out over the network to
|
79
|
+
all of your clients. Quite often, network servers will use multicast to achieve
|
80
|
+
this, but pants can do this with both unicast and multicast--or both at the same
|
81
|
+
time, while writing to a file (<-- maybe for debug purposes, for example?).
|
82
|
+
|
83
|
+
Also, since pants uses eventmachine, you don't have to worry about dealing with
|
84
|
+
threads. If you're writing your own reader or writer, you do, however, need to
|
85
|
+
consider if you should <tt>defer</tt> (eventmachine's means of putting your code
|
86
|
+
into a thread) the code you want to do the reading/writing. This is just a
|
87
|
+
matter of wrapping that code in block though, while eventmachine handles the
|
88
|
+
threading for you.
|
89
|
+
|
90
|
+
==== Cons
|
91
|
+
|
92
|
+
The amount of data you can replicate really depends, of course, on system
|
93
|
+
resources. On an i7 MacBook Pro with 16GB RAM and 2 wired NICs, I've been able
|
94
|
+
to duplicate a single 720p video + audio stream over unicast UDP 200 times (pushing ~1.4
|
95
|
+
Gbps out) with almost no quality loss on client ends. If you plan to duplicate
|
96
|
+
more than 20 streams, you'll need to start tweaking EventMachine thread pool
|
97
|
+
settings; generally you should set EventMachines +threadpool_size+ to the number
|
98
|
+
of output streams so it can process all of the data concurrently. If you're on
|
99
|
+
OSX or *nix, you might benefit from using EventMachine's .epoll and .kqueue
|
100
|
+
feature. More on that {http://eventmachine.rubyforge.org/docs/EPOLL.html here}.
|
101
|
+
|
102
|
+
Just like in any case when dealing with I/O, file reading/writing also depends
|
103
|
+
on many factors, such as the number and size of files, disk capabilities, and
|
104
|
+
general I/O capabilities of your system. If you'd like to benchmark, there's a
|
105
|
+
{Thor}[http://whatisthor.com] task in +tasks/+ that will compare pants to
|
106
|
+
FileUtils.cp and `cp` in copying a file some number of times (default to 100).
|
107
|
+
|
108
|
+
Example:
|
109
|
+
|
110
|
+
$ tasks/pantsmark.thor file_copy some_song.mp3 --times=50
|
111
|
+
|
112
|
+
If you're wanting to write your own readers/writers/seams and you're not
|
113
|
+
familiar with eventmachine, getting plugged in to some of its concepts can bek
|
114
|
+
frustrating, as it's a different paradigm to work off than the standard paradigm
|
115
|
+
for doing I/O. Some may say that this is a con, and some may not.
|
116
|
+
|
117
|
+
=== Examples
|
118
|
+
|
119
|
+
==== As a library
|
120
|
+
|
121
|
+
Read unicast UDP inbound data and write to a number of other unicast UDP clients:
|
122
|
+
|
123
|
+
Pants.read('udp://0.0.0.0:1234') do |reader|
|
124
|
+
reader.write_to 'udp://10.0.0.10:1234'
|
125
|
+
reader.write_to 'udp://10.0.0.11:1234'
|
126
|
+
end
|
127
|
+
|
128
|
+
Read multicast UDP inbound data and write to a number of other unicast UDP clients:
|
129
|
+
|
130
|
+
Pants.read('udp://239.0.0.1:1234') do |reader|
|
131
|
+
reader.write_to 'udp://10.0.0.10:1234'
|
132
|
+
reader.write_to 'udp://10.0.0.11:1234'
|
133
|
+
end
|
134
|
+
|
135
|
+
Read unicast UDP inbound data and write to a UDP client and a file:
|
136
|
+
|
137
|
+
Pants.read('udp://0.0.0.0:1234') do |reader|
|
138
|
+
reader.write_to 'udp://10.0.0.10:1234'
|
139
|
+
reader.write_to 'socket_data.raw'
|
140
|
+
end
|
141
|
+
|
142
|
+
Read a file and send out via UDP:
|
143
|
+
|
144
|
+
Pants.read('socket_data.raw') do |reader|
|
145
|
+
reader.write_to 'udp://10.0.0.10:1234'
|
146
|
+
reader.write_to 'udp://239.0.0.1:1234'
|
147
|
+
end
|
148
|
+
|
149
|
+
Get kray-kray:
|
150
|
+
|
151
|
+
EM.threadpool_size = 110
|
152
|
+
EM.kqueue # This has been problematic for me...
|
153
|
+
EM.epoll
|
154
|
+
|
155
|
+
Pants.read('udp://0.0.0.0:1234') do |reader|
|
156
|
+
100.times do |i|
|
157
|
+
reader.write_to "udp://10.0.0.10:#{1235 + i}"
|
158
|
+
end
|
159
|
+
|
160
|
+
10.times do |i|
|
161
|
+
reader.write_to "socket_data_#{i}.raw"
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
The block form above is really just some syntactic sugar for doing:
|
166
|
+
|
167
|
+
core = Pants::Core.new
|
168
|
+
reader = core.read 'udp://0.0.0.0:1234'
|
169
|
+
reader.write_to 'udp://10.0.0.10:1234'
|
170
|
+
reader.write_to 'udp://10.0.0.11:1234'
|
171
|
+
core.run
|
172
|
+
|
173
|
+
...and that's actually short for for:
|
174
|
+
|
175
|
+
core = Pants::Core.new
|
176
|
+
reader = core.add_reader(Pants::Readers::UDPReader, '0.0.0.0', 1234)
|
177
|
+
reader.add_writer(Pants::Writers::UDPWriter, '10.0.0.10', 1234)
|
178
|
+
reader.add_writer(Pants::Writers::UDPWriter, '10.0.0.11', 1234)
|
179
|
+
core.run
|
180
|
+
|
181
|
+
...which can be made even longer (but potentially more helpful):
|
182
|
+
|
183
|
+
core = Pants::Core.new
|
184
|
+
reader = Pants::Readers::UDPReader.new('0.0.0.0', 1234, core.callback)
|
185
|
+
core.add_reader(reader)
|
186
|
+
|
187
|
+
writer1 = Pants::Writers::UDPWriter.new('10.0.0.10', 1234, reader.write_to_channel)
|
188
|
+
reader.add_writer(writer1)
|
189
|
+
|
190
|
+
writer2 = Pants::Writers::UDPWriter.new('10.0.0.11', 1234, reader.write_to_channel)
|
191
|
+
reader.add_writer(writer2)
|
192
|
+
|
193
|
+
core.run
|
194
|
+
|
195
|
+
Using this unsugared form, does give you some flexibility though to use within
|
196
|
+
your code. If, for example, you're writing a server of some sort that needs to
|
197
|
+
handle a varying number of clients, you can add and remove writers at will:
|
198
|
+
|
199
|
+
require 'sinatra'
|
200
|
+
|
201
|
+
core = Pants::Core.new
|
202
|
+
reader = core.read 'udp://0.0.0.0:1234'
|
203
|
+
|
204
|
+
post '/:client' do
|
205
|
+
reader.write_to "udp://#{params[:client]}"
|
206
|
+
end
|
207
|
+
|
208
|
+
delete '/:client' do
|
209
|
+
reader.remove_writer("udp://#{params[:client]}")
|
210
|
+
end
|
211
|
+
|
212
|
+
core.run
|
213
|
+
|
214
|
+
==== As an executable
|
215
|
+
|
216
|
+
The examples from above can also be run via the command-line app like this:
|
217
|
+
|
218
|
+
$ pants udp://0.0.0.0:1234 --dest=udp://10.0.0.10:1234 udp://10.0.0.11:1234 udp://10.0.0.11:2345
|
219
|
+
$ pants udp://239.0.0.1:1234 --dest=udp://10.0.0.10:1234 udp://10.0.0.11:1234 udp://10.0.0.11:2345
|
220
|
+
$ pants udp://0.0.0.0:1234 --dest=udp://10.0.0.10:1234 file:///home/me/socket_data.raw
|
221
|
+
$ pants /home/me/socket_data.raw --dest=udp://10.0.0.10:1234 udp://239.0.0.1:1234
|
222
|
+
|
223
|
+
==== Use a seam to inspect and alter packets
|
224
|
+
|
225
|
+
require 'pants'
|
226
|
+
|
227
|
+
class PantsInspector < Pants::Seam
|
228
|
+
def initialize(core_callback, reader_channel, host)
|
229
|
+
@host = host
|
230
|
+
|
231
|
+
super(core_callback, reader_channel)
|
232
|
+
end
|
233
|
+
|
234
|
+
# Pants will call this for you when it starts the reader that the seam is
|
235
|
+
# reading from.
|
236
|
+
def start
|
237
|
+
|
238
|
+
# You need to define a callback here so pants can call you back after its
|
239
|
+
# made sure that the seam's writers have been started. This makes sure
|
240
|
+
# the seam doesn't start reading and pushing out data before the writers
|
241
|
+
# are ready to receive it.
|
242
|
+
callback = EM.Callback do
|
243
|
+
puts "Starting #{self.class}..."
|
244
|
+
|
245
|
+
read_items do |data|
|
246
|
+
if data.match(/pants/mi)
|
247
|
+
data << "Pants party at #{@host}!!!!!!"
|
248
|
+
end
|
249
|
+
|
250
|
+
write data
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
super(callback)
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
# Assuming you have data coming in on this IP and UDP port, this reads each
|
259
|
+
# packet and hands it over to its writers--in this case, the file dump file
|
260
|
+
# writer, the UDP writer that sends to 10.0.0.50:9001, and our PacketInspector
|
261
|
+
# seam.
|
262
|
+
Pants.read('udp://127.0.0.1:1234') do |reader|
|
263
|
+
|
264
|
+
# Dump UDP packets to a file
|
265
|
+
reader.write_to 'file_dump_of_udp_packets.udp'
|
266
|
+
|
267
|
+
# Redirect the UDP packets to another socket
|
268
|
+
reader.write_to 'udp://127.0.0.1:9000'
|
269
|
+
|
270
|
+
# Inspect the UDP packets and notify about the party going on
|
271
|
+
pants_inspector = reader.add_seam(PantsInspector, 'localhost')
|
272
|
+
|
273
|
+
# The packet inspector seam behaves like a reader, which lets writers read
|
274
|
+
# from it. Dump those party packets to a file...
|
275
|
+
pants_inspector.write_to 'file_dump_of_party_packets.party'
|
276
|
+
|
277
|
+
# Forward our partying packets on to another host.
|
278
|
+
pants_inspector.write_to 'udp://10.0.0.1:1234'
|
279
|
+
end
|
280
|
+
|
281
|
+
If you actually run this, you'll have to ctrl-c out of it, as pants will read on
|
282
|
+
that UDP socket indefinitely. After doing that, you'll see that you've now got
|
283
|
+
a 'file_dump_of_udp_packets.udp', which contains the raw UDP data that came in
|
284
|
+
on port 1234, and a 'file_dump_of_party_packets.party' file, which contains
|
285
|
+
those UDP packets with our party message at the end of each packet. If you had
|
286
|
+
a client listening at 127.0.0.1:9000 and 10.0.0.1:1234, it would received the
|
287
|
+
exact same data that was written to the respective files.
|
288
|
+
|
289
|
+
== REQUIREMENTS:
|
290
|
+
|
291
|
+
* Rubies (tested)
|
292
|
+
* MRI 1.9.3
|
293
|
+
* JRuby 1.7.2 (failing due to what looks like EventMachine bugs)
|
294
|
+
* Rubinius (failing due to what looks like EventMachine bugs)
|
295
|
+
* Gems
|
296
|
+
* eventmachine (>=1.0.0)
|
297
|
+
* log_switch
|
298
|
+
* thor
|
299
|
+
|
300
|
+
_NOTE:_ Multicasting with JRuby doesn't seem to work; EM fails to allow setting
|
301
|
+
socket options, which is necessary to do multicasting.
|
302
|
+
|
303
|
+
== INSTALL:
|
304
|
+
|
305
|
+
* (sudo) gem install
|
306
|
+
|
307
|
+
== DEVELOPERS:
|
308
|
+
|
309
|
+
After checking out the source, run:
|
310
|
+
|
311
|
+
$ bundle install
|
312
|
+
|
313
|
+
This task will install any missing dependencies, run the tests/specs,
|
314
|
+
and generate the RDoc.
|
315
|
+
|
316
|
+
== LICENSE:
|
317
|
+
|
318
|
+
(The MIT License)
|
319
|
+
|
320
|
+
Copyright (c) 2013 Steve Loveless
|
321
|
+
|
322
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
323
|
+
a copy of this software and associated documentation files (the
|
324
|
+
'Software'), to deal in the Software without restriction, including
|
325
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
326
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
327
|
+
permit persons to whom the Software is furnished to do so, subject to
|
328
|
+
the following conditions:
|
329
|
+
|
330
|
+
The above copyright notice and this permission notice shall be
|
331
|
+
included in all copies or substantial portions of the Software.
|
332
|
+
|
333
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
334
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
335
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
336
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
337
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
338
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
339
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler/gem_tasks'
|
5
|
+
require 'rspec/core/rake_task'
|
6
|
+
require 'yard'
|
7
|
+
|
8
|
+
YARD::Rake::YardocTask.new do |t|
|
9
|
+
t.files = %w(lib/**/*.rb - History.rdoc)
|
10
|
+
t.options = %w(--title pants Documentation (#{Pants::VERSION}))
|
11
|
+
t.options += %w(--main README.rdoc)
|
12
|
+
end
|
13
|
+
|
14
|
+
RSpec::Core::RakeTask.new do |t|
|
15
|
+
t.ruby_opts = %w(-w)
|
16
|
+
end
|
17
|
+
|
18
|
+
# Alias for rubygems-test
|
19
|
+
task :test => :spec
|
20
|
+
|
21
|
+
task :default => :test
|
22
|
+
|
23
|
+
# vim: syntax=ruby
|