iop 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGES.md +3 -0
- data/README.md +90 -0
- data/lib/iop/digest.rb +45 -0
- data/lib/iop/file.rb +209 -0
- data/lib/iop/net/ftp.rb +206 -0
- data/lib/iop/openssl.rb +147 -0
- data/lib/iop/securerandom.rb +49 -0
- data/lib/iop/string.rb +89 -0
- data/lib/iop/zlib.rb +179 -0
- data/lib/iop/zstdlib.rb +104 -0
- data/lib/iop.rb +250 -0
- data/test/digest_test.rb +18 -0
- data/test/io_test.rb +23 -0
- data/test/net_ftp_test.rb +39 -0
- data/test/secureransom_test.rb +16 -0
- data/test/string_test.rb +17 -0
- metadata +70 -0
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'iop'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
|
5
|
+
module IOP
|
6
|
+
|
7
|
+
|
8
|
+
#
|
9
|
+
# Feed class to generate and send a random sequence of bytes of specified size.
|
10
|
+
#
|
11
|
+
# This is the adapter for standard {SecureRandom} generator module.
|
12
|
+
#
|
13
|
+
# ### Use case: generate 1024 bytes of random data and compute MD5 hash sum of it.
|
14
|
+
#
|
15
|
+
# require 'iop/digest'
|
16
|
+
# require 'iop/securerandom'
|
17
|
+
# ( IOP::SecureRandomGenerator.new(1024) | IOP::DigestComputer.new(Digest::MD5.new) ).process!
|
18
|
+
#
|
19
|
+
# @since 0.1
|
20
|
+
#
|
21
|
+
class SecureRandomGenerator
|
22
|
+
|
23
|
+
include Feed
|
24
|
+
|
25
|
+
# Creates class instance.
|
26
|
+
#
|
27
|
+
# @param size [Integer] total random data size
|
28
|
+
#
|
29
|
+
# @param block_size [Integer] size of block the data in split into
|
30
|
+
def initialize(size, block_size: DEFAULT_BLOCK_SIZE)
|
31
|
+
@size = size
|
32
|
+
@block_size = block_size
|
33
|
+
end
|
34
|
+
|
35
|
+
def process!
|
36
|
+
written = 0
|
37
|
+
(0..@size/@block_size - 1).each do
|
38
|
+
process(SecureRandom.bytes(@block_size))
|
39
|
+
written += @block_size
|
40
|
+
end
|
41
|
+
left = @size - written
|
42
|
+
process(SecureRandom.bytes(left)) unless left.zero?
|
43
|
+
process
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
|
48
|
+
|
49
|
+
end
|
data/lib/iop/string.rb
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'iop'
|
2
|
+
|
3
|
+
|
4
|
+
module IOP
|
5
|
+
|
6
|
+
|
7
|
+
#
|
8
|
+
# Feed class to send arbitrary string in blocks of specified size.
|
9
|
+
#
|
10
|
+
# ### Use case: split the string into 3-byte blocks and reconstruct it.
|
11
|
+
#
|
12
|
+
# require 'iop/string'
|
13
|
+
# ( IOP::StringSplitter.new('Hello IOP', 3) | IOP::StringMerger.new ).process!
|
14
|
+
#
|
15
|
+
# @since 0.1
|
16
|
+
#
|
17
|
+
class StringSplitter
|
18
|
+
|
19
|
+
include Feed
|
20
|
+
|
21
|
+
# Creates class instance.
|
22
|
+
#
|
23
|
+
# @param string [String] string to be sent in blocks
|
24
|
+
#
|
25
|
+
# @param block_size [Integer] size of block the string is split into
|
26
|
+
def initialize(string, block_size: DEFAULT_BLOCK_SIZE)
|
27
|
+
@string = string
|
28
|
+
@block_size = block_size
|
29
|
+
end
|
30
|
+
|
31
|
+
def process!
|
32
|
+
offset = 0
|
33
|
+
(0..@string.size / @block_size - 1).each do
|
34
|
+
process(@string[offset, @block_size])
|
35
|
+
offset += @block_size
|
36
|
+
end
|
37
|
+
process(offset.zero? ? @string : @string[offset..-1]) unless offset == @string.size
|
38
|
+
process
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
#
|
45
|
+
# Sink class to receive data blocks and merge them into a single string.
|
46
|
+
#
|
47
|
+
# ### Use case: read current source file into a string.
|
48
|
+
#
|
49
|
+
# require 'iop/file'
|
50
|
+
# require 'iop/string'
|
51
|
+
# ( IOP::FileReader.new($0) | (s = IOP::StringMerger.new) ).process!
|
52
|
+
# puts s.to_s
|
53
|
+
#
|
54
|
+
# The actual string assembly is performed by the {#to_s} method.
|
55
|
+
#
|
56
|
+
# @note instance of this class can be used to collect data from multiple processing runs.
|
57
|
+
#
|
58
|
+
# @since 0.1
|
59
|
+
#
|
60
|
+
class StringMerger
|
61
|
+
|
62
|
+
include Sink
|
63
|
+
|
64
|
+
# Creates class instance.
|
65
|
+
def initialize
|
66
|
+
@size = 0
|
67
|
+
@data = []
|
68
|
+
end
|
69
|
+
|
70
|
+
def process(data = nil)
|
71
|
+
unless data.nil?
|
72
|
+
@data << data.dup # CHECKME is duplication really needed when the upstream continuously resending its internal data buffer with new contents
|
73
|
+
@size += data.size
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# Returns concatenation of all received data blocks into a single string.
|
78
|
+
#
|
79
|
+
# @return [String]
|
80
|
+
def to_s
|
81
|
+
string = IOP.allocate_string(@size)
|
82
|
+
@data.each {|x| string << x}
|
83
|
+
string
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
|
88
|
+
|
89
|
+
end
|
data/lib/iop/zlib.rb
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
require 'iop'
|
2
|
+
require 'zlib'
|
3
|
+
|
4
|
+
|
5
|
+
module IOP
|
6
|
+
|
7
|
+
|
8
|
+
#
|
9
|
+
# Filter class to perform data compression with Zlib algorithm.
|
10
|
+
#
|
11
|
+
# This class is an adapter for the standard Ruby +Zlib::Deflate+ class.
|
12
|
+
#
|
13
|
+
# Note that this class does not produce valid _.gz_ files - use {GzipCompressor} for this purpose.
|
14
|
+
#
|
15
|
+
# ### Use case: compress a string.
|
16
|
+
#
|
17
|
+
# require 'iop/zlib'
|
18
|
+
# require 'iop/string'
|
19
|
+
# ( IOP::StringSplitter.new('Hello IOP') | IOP::ZlibCompressor.new | (s = IOP::StringMerger.new) ).process!
|
20
|
+
# puts s.to_s
|
21
|
+
#
|
22
|
+
# @since 0.1
|
23
|
+
#
|
24
|
+
class ZlibCompressor
|
25
|
+
|
26
|
+
include Feed
|
27
|
+
include Sink
|
28
|
+
|
29
|
+
# Creates class instance.
|
30
|
+
#
|
31
|
+
# @param args [Array] arguments passed to +Zlib::Deflate+ constructor
|
32
|
+
def initialize(*args)
|
33
|
+
@args = args
|
34
|
+
end
|
35
|
+
|
36
|
+
def process(data = nil)
|
37
|
+
if data.nil?
|
38
|
+
super(@deflate.finish)
|
39
|
+
super
|
40
|
+
else
|
41
|
+
super(@deflate.deflate(data))
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def process!
|
46
|
+
@deflate = Zlib::Deflate.new(*@args)
|
47
|
+
begin
|
48
|
+
super
|
49
|
+
ensure
|
50
|
+
@deflate.close
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
#
|
57
|
+
# Filter class to perform data decompression with Zlib algorithm.
|
58
|
+
#
|
59
|
+
# This class is an adapter for the standard Ruby +Zlib::Inflate+ class.
|
60
|
+
#
|
61
|
+
# Note that this class can not decompress _.gz_ files - use {GzipDecompressor} for this purpose.
|
62
|
+
#
|
63
|
+
# ### Use case: decompress a Zlib-compressed part of a file skipping a header and compute MD5 hash sum of the uncompressed data.
|
64
|
+
#
|
65
|
+
# require 'iop/zlib'
|
66
|
+
# require 'iop/file'
|
67
|
+
# require 'iop/digest'
|
68
|
+
# ( IOP::FileReader.new('input.dat', offset: 16) | IOP::ZlibDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
|
69
|
+
# puts d.digest.hexdigest
|
70
|
+
#
|
71
|
+
# @since 0.1
|
72
|
+
#
|
73
|
+
class ZlibDecompressor
|
74
|
+
|
75
|
+
include Feed
|
76
|
+
include Sink
|
77
|
+
|
78
|
+
# Creates class instance.
|
79
|
+
#
|
80
|
+
# @param args [Array] arguments passed to +Zlib::Inflate+ constructor
|
81
|
+
def initialize(*args)
|
82
|
+
@args = args
|
83
|
+
end
|
84
|
+
|
85
|
+
def process(data = nil)
|
86
|
+
if data.nil?
|
87
|
+
super(@inflate.finish)
|
88
|
+
super
|
89
|
+
else
|
90
|
+
super(@inflate.inflate(data))
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def process!
|
95
|
+
@inflate = Zlib::Inflate.new(*@args)
|
96
|
+
begin
|
97
|
+
super
|
98
|
+
ensure
|
99
|
+
@inflate.close
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
|
105
|
+
#
|
106
|
+
# Filter class to perform Gzip data compression.
|
107
|
+
#
|
108
|
+
# This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
|
109
|
+
#
|
110
|
+
# This class produces valid _.gz_ files.
|
111
|
+
#
|
112
|
+
# ### Use case: compress a string and store it to .gz file.
|
113
|
+
#
|
114
|
+
# require 'iop/zlib'
|
115
|
+
# require 'iop/file'
|
116
|
+
# require 'iop/string'
|
117
|
+
# ( IOP::StringSplitter.new('Hello IOP') | IOP::GzipCompressor.new | IOP::FileWriter.new('hello.gz') ).process!
|
118
|
+
#
|
119
|
+
# @since 0.1
|
120
|
+
#
|
121
|
+
class GzipCompressor
|
122
|
+
|
123
|
+
include Feed
|
124
|
+
include Sink
|
125
|
+
|
126
|
+
# Creates class instance.
|
127
|
+
#
|
128
|
+
# @param args [Array] arguments passed to +Zlib::GzipWriter+ constructor
|
129
|
+
def initialize(*args)
|
130
|
+
@args = args
|
131
|
+
end
|
132
|
+
|
133
|
+
def process(data = nil)
|
134
|
+
if data.nil?
|
135
|
+
@compressor.finish
|
136
|
+
super
|
137
|
+
else
|
138
|
+
@compressor.write(data)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def write(data)
|
143
|
+
downstream&.process(data)
|
144
|
+
end
|
145
|
+
|
146
|
+
def process!
|
147
|
+
@compressor = Zlib::GzipWriter.new(self, *@args)
|
148
|
+
super
|
149
|
+
ensure
|
150
|
+
@compressor.close unless @compressor.nil?
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
|
155
|
+
#
|
156
|
+
# Filter class to perform Gzip data compression.
|
157
|
+
#
|
158
|
+
# This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
|
159
|
+
#
|
160
|
+
# This class can decompress _.gz_ files.
|
161
|
+
#
|
162
|
+
# ### Use case: decompress a .gz file and compute MD5 hash sum of uncompressed data.
|
163
|
+
#
|
164
|
+
# require 'iop/zlib'
|
165
|
+
# require 'iop/file'
|
166
|
+
# require 'iop/digest'
|
167
|
+
# ( IOP::FileReader.new('hello.gz') | IOP::GzipDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
|
168
|
+
# puts d.digest.hexdigest
|
169
|
+
#
|
170
|
+
# @since 0.1
|
171
|
+
#
|
172
|
+
class GzipDecompressor < ZlibDecompressor
|
173
|
+
def initialize
|
174
|
+
super(16)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
|
179
|
+
end
|
data/lib/iop/zstdlib.rb
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
require 'iop'
|
2
|
+
require 'zstdlib'
|
3
|
+
|
4
|
+
|
5
|
+
module IOP
|
6
|
+
|
7
|
+
|
8
|
+
#
|
9
|
+
# Filter class to perform data compression with Zstandard algorithm.
|
10
|
+
#
|
11
|
+
# This class produces valid _.zst_ files.
|
12
|
+
#
|
13
|
+
# ### Use case: compress a string and store it to .zst file.
|
14
|
+
#
|
15
|
+
# require 'iop/file'
|
16
|
+
# require 'iop/string'
|
17
|
+
# require 'iop/zstdlib'
|
18
|
+
# ( IOP::StringSplitter.new('Hello IOP') | IOP::ZstdCompressor.new(Zstdlib::BEST_COMPRESSION) | IOP::FileWriter.new('hello.zst') ).process!
|
19
|
+
#
|
20
|
+
# @note this class depends on external +zstdlib+ gem.
|
21
|
+
# @since 0.1
|
22
|
+
#
|
23
|
+
class ZstdCompressor
|
24
|
+
|
25
|
+
include Feed
|
26
|
+
include Sink
|
27
|
+
|
28
|
+
# Creates class instance.
|
29
|
+
#
|
30
|
+
# @param args [Array] arguments passed to +Zstdlib::Deflate+ constructor
|
31
|
+
def initialize(*args)
|
32
|
+
@args = args
|
33
|
+
end
|
34
|
+
|
35
|
+
def process(data = nil)
|
36
|
+
if data.nil?
|
37
|
+
super(@deflate.finish)
|
38
|
+
super
|
39
|
+
else
|
40
|
+
super(@deflate.deflate(data))
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def process!
|
45
|
+
@deflate = Zstdlib::Deflate.new(*@args)
|
46
|
+
begin
|
47
|
+
super
|
48
|
+
ensure
|
49
|
+
@deflate.close
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
#
|
56
|
+
# Filter class to perform Gzip data compression.
|
57
|
+
#
|
58
|
+
# This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
|
59
|
+
#
|
60
|
+
# This class can decompress _.zst_ files.
|
61
|
+
#
|
62
|
+
# ### Use case: decompress a .zst file and compute MD5 hash sum of uncompressed data.
|
63
|
+
#
|
64
|
+
# require 'iop/file'
|
65
|
+
# require 'iop/digest'
|
66
|
+
# require 'iop/zstdlib'
|
67
|
+
# ( IOP::FileReader.new('hello.zst') | IOP::ZstdDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
|
68
|
+
# puts d.digest.hexdigest
|
69
|
+
#
|
70
|
+
# @note this class depends on external +zstdlib+ gem.
|
71
|
+
# @since 0.1
|
72
|
+
#
|
73
|
+
class ZstdDecompressor
|
74
|
+
|
75
|
+
include Feed
|
76
|
+
include Sink
|
77
|
+
|
78
|
+
# Creates class instance.
|
79
|
+
#
|
80
|
+
# @param args [Array] arguments passed to +Zstdlib::Inflate+ constructor
|
81
|
+
def initialize(*args)
|
82
|
+
@args = args
|
83
|
+
end
|
84
|
+
|
85
|
+
def process(data = nil)
|
86
|
+
if data.nil?
|
87
|
+
super(@inflate.finish)
|
88
|
+
super
|
89
|
+
else
|
90
|
+
super(@inflate.inflate(data))
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def process!
|
95
|
+
@inflate = Zstdlib::Inflate.new(*@args)
|
96
|
+
begin
|
97
|
+
super
|
98
|
+
ensure
|
99
|
+
@inflate.close
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
end
|
data/lib/iop.rb
ADDED
@@ -0,0 +1,250 @@
|
|
1
|
+
#
|
2
|
+
# IOP is intended for constructing the data processing pipelines in a manner of UNIX command-line pipes.
|
3
|
+
#
|
4
|
+
# There are three principle types of the pipe nodes which can be composed:
|
5
|
+
#
|
6
|
+
# * Feed node.
|
7
|
+
#
|
8
|
+
# This is the start point of the pipe. It has no upstream node and may have downstream node.
|
9
|
+
# Its purpose its to generate blocks of data and send them downstream in sequence.
|
10
|
+
# A typical feed class is implemented by including the {Feed} module and defining the +#process!+ method
|
11
|
+
# which calls {Feed#process} method to send the data.
|
12
|
+
# An example of the feed node is a file reader ({FileReader}) which reads file and sends its contents in blocks.
|
13
|
+
#
|
14
|
+
# * Sink node.
|
15
|
+
#
|
16
|
+
# This is the end point of the pipe. It has upstream node and no downstream node.
|
17
|
+
# Its purpose is to consume the received data.
|
18
|
+
# A typical sink class is implemented by including the {Sink} module and defining the +#process+ method.
|
19
|
+
# An example of the sink node is a file writer ({FileWriter}) which receives the data in blocks and writes it into file.
|
20
|
+
#
|
21
|
+
# * Filter node.
|
22
|
+
#
|
23
|
+
# A filter is a pass-through node which sits between feed and sink and therefore has both upstream and downstream nodes.
|
24
|
+
# The simplest way to create a filter class is to include both {Feed} and {Sink} which manifest
|
25
|
+
# both mandatory +#process!+ and +#process+ methods. Such filter is a no-op that is it does nothing apart passing
|
26
|
+
# the received data downstream.
|
27
|
+
# An example of the filter node is the digest computer ({DigestComputer}) which computes hash sum of the data it passes through.
|
28
|
+
# In order to perform intended processing of the data a filter class overrides the {Feed#process} method.
|
29
|
+
#
|
30
|
+
# The basic control flow for an {IOP}-aware pipe is as follows:
|
31
|
+
#
|
32
|
+
# 1. The pipe is constructed from one or more {IOP}-aware class instances. The two or more objects are linked together
|
33
|
+
# with the | operator implemented as the {Feed#|} method by default.
|
34
|
+
#
|
35
|
+
# 2. The actual processing is then triggered by the {Sink#process!} method of the very last object in the pipe.
|
36
|
+
# By default, this method calls the same method of the upstream node thus forming the stack of nested calls
|
37
|
+
# for all objects in the pipe.
|
38
|
+
#
|
39
|
+
# 3. Upon reaching the very first object in the pipe (which by definition has no upstream node),
|
40
|
+
# the feed, starts sending blocks of data downstream with the {Feed#process} method. All objects' method implementations
|
41
|
+
# (except for the one of the last object in the pipe) are expected to push either this or transformed data further downstream.
|
42
|
+
#
|
43
|
+
# 4. After all data has been processed the finalizing call +#process(nil)+ signifies the end-of-data after which
|
44
|
+
# no data should be sent.
|
45
|
+
#
|
46
|
+
# In case the {Sink#process!} method is overridden in concrete class it is normally organized as follows:
|
47
|
+
#
|
48
|
+
# def process!
|
49
|
+
# # ...initialization code...
|
50
|
+
# super
|
51
|
+
# ensure
|
52
|
+
# # ...finalization code...
|
53
|
+
# end
|
54
|
+
#
|
55
|
+
# to perform specific setup/cleanup actions, including exception handling and to pass the control flow upstream
|
56
|
+
# with +super+ call.
|
57
|
+
#
|
58
|
+
# Note that when an exception is caught and processed in overridden +#process!+ method it must be re-raised in order
|
59
|
+
# for other upstream objects to have a chance to react to it as well.
|
60
|
+
#
|
61
|
+
# In case the {Feed#process} is overridden in concrete class it is organized as follows:
|
62
|
+
#
|
63
|
+
# def process(data = nil)
|
64
|
+
# # ... do something with data, convert data to new_data...
|
65
|
+
# super(new_data)
|
66
|
+
# end
|
67
|
+
#
|
68
|
+
# The data being sent is expected to be a +String+ of arbitrary size. It is however advisable to detect and omit
|
69
|
+
# zero-sized strings.
|
70
|
+
#
|
71
|
+
# Note that the data passed to this method may be a reusable buffer of some other upstream object therefore a duplication
|
72
|
+
# (or cloning) should be performed if the data is stored between the method invocations.
|
73
|
+
#
|
74
|
+
module IOP
|
75
|
+
|
76
|
+
|
77
|
+
VERSION = '0.1.0'
|
78
|
+
|
79
|
+
|
80
|
+
# Default read block size in bytes for adapters which don't have this parameter externally imposed.
|
81
|
+
DEFAULT_BLOCK_SIZE = 1024**2
|
82
|
+
|
83
|
+
|
84
|
+
if RUBY_VERSION >= '2.4'
|
85
|
+
# @private
|
86
|
+
def self.allocate_string(size)
|
87
|
+
String.new(capacity: size)
|
88
|
+
end
|
89
|
+
else
|
90
|
+
# @private
|
91
|
+
def self.allocate_string(size)
|
92
|
+
String.new
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
# @private
|
98
|
+
INSUFFICIENT_DATA = 'premature end-of-data encountered'.freeze
|
99
|
+
|
100
|
+
|
101
|
+
# @private
|
102
|
+
EXTRA_DATA = 'superfluous data received'.freeze
|
103
|
+
|
104
|
+
|
105
|
+
# @private
|
106
|
+
# Finds minimum of the values
|
107
|
+
def self.min(a, b)
|
108
|
+
a < b ? a : b
|
109
|
+
end
|
110
|
+
|
111
|
+
|
112
|
+
#
|
113
|
+
# Module to be included into classes which generate and send the data downstream.
|
114
|
+
#
|
115
|
+
# @since 0.1
|
116
|
+
#
|
117
|
+
module Feed
|
118
|
+
|
119
|
+
#
|
120
|
+
# Commences the data processing operation.
|
121
|
+
#
|
122
|
+
# @abstract
|
123
|
+
#
|
124
|
+
# @note this method should be implemented in concrete classes including this module.
|
125
|
+
#
|
126
|
+
# Refer to {Sink#process!} for details.
|
127
|
+
#
|
128
|
+
def process!
|
129
|
+
raise
|
130
|
+
end
|
131
|
+
remove_method :process!
|
132
|
+
|
133
|
+
#
|
134
|
+
# Sends the data block downstream.
|
135
|
+
#
|
136
|
+
# @note by convention, the very last call to this method should pass +nil+ to indicate the end-of-data and no data should be sent afterwards.
|
137
|
+
#
|
138
|
+
# This implementation simply passes through the received data block downstream if there exists an attached downstream
|
139
|
+
# object otherwise the data is simply thrown away.
|
140
|
+
#
|
141
|
+
# The overriding method in concrete class which includes {Feed} would normally want to call this one as +super+ after
|
142
|
+
# performing specific actions.
|
143
|
+
#
|
144
|
+
def process(data = nil)
|
145
|
+
downstream&.process(data) # Ruby 2.3+
|
146
|
+
end
|
147
|
+
|
148
|
+
# Returns the downstream object or +nil+ if +self+ is the last object in processing pipe.
|
149
|
+
attr_reader :downstream
|
150
|
+
|
151
|
+
#
|
152
|
+
# Links +self+ and +downstream+ together forming a processing pipe.
|
153
|
+
# The subsequent objects may be linked in turn.
|
154
|
+
# @return downstream object
|
155
|
+
#
|
156
|
+
def |(downstream)
|
157
|
+
downstream.upstream = self
|
158
|
+
@downstream = downstream
|
159
|
+
end
|
160
|
+
|
161
|
+
end
|
162
|
+
|
163
|
+
|
164
|
+
#
|
165
|
+
# Module to be included into classes which receive and process the upstream data.
|
166
|
+
#
|
167
|
+
# @since 0.1
|
168
|
+
#
|
169
|
+
module Sink
|
170
|
+
|
171
|
+
# Commences the data processing operation.
|
172
|
+
#
|
173
|
+
# This implementation calls {#process!} method of the upstream object.
|
174
|
+
def process!
|
175
|
+
upstream.process!
|
176
|
+
end
|
177
|
+
|
178
|
+
# @abstract
|
179
|
+
#
|
180
|
+
# @note this method should be implemented in concrete classes including this module.
|
181
|
+
#
|
182
|
+
# Refer to {Feed#process} for more information.
|
183
|
+
def process(data = nil)
|
184
|
+
raise
|
185
|
+
end
|
186
|
+
remove_method :process
|
187
|
+
|
188
|
+
# Returns the upstream object or +nil+ if +self+ is the first object in processing pipe.
|
189
|
+
attr_accessor :upstream
|
190
|
+
|
191
|
+
end
|
192
|
+
|
193
|
+
|
194
|
+
#
|
195
|
+
# @private
|
196
|
+
#
|
197
|
+
# @note a class including this module must implement the {#next_data} method.
|
198
|
+
#
|
199
|
+
# @since 0.1
|
200
|
+
#
|
201
|
+
module BufferingFeed
|
202
|
+
|
203
|
+
include Feed
|
204
|
+
|
205
|
+
def read!(size)
|
206
|
+
@left = @size = size
|
207
|
+
self
|
208
|
+
end
|
209
|
+
|
210
|
+
def process!
|
211
|
+
unless @buffer.nil?
|
212
|
+
if @buffer.size > @size
|
213
|
+
@left = 0
|
214
|
+
process(@buffer[0, @size])
|
215
|
+
@buffer = @buffer[@size..-1]
|
216
|
+
else
|
217
|
+
@left -= @buffer.size
|
218
|
+
process(@buffer)
|
219
|
+
@buffer = nil
|
220
|
+
end
|
221
|
+
end
|
222
|
+
until @left.zero?
|
223
|
+
raise EOFError, INSUFFICIENT_DATA if (data = next_data).nil?
|
224
|
+
if @left < data.size
|
225
|
+
process(data[0, @left])
|
226
|
+
@buffer = data[@left..-1]
|
227
|
+
@left = 0
|
228
|
+
else
|
229
|
+
process(data)
|
230
|
+
@left -= data.size
|
231
|
+
end
|
232
|
+
end
|
233
|
+
@left = @size = nil
|
234
|
+
process
|
235
|
+
end
|
236
|
+
|
237
|
+
# @abstract
|
238
|
+
#
|
239
|
+
# Returns the data portion of non-zero size or +nil+ on EOF.
|
240
|
+
#
|
241
|
+
# @return [String] data chunk recently read or +nil+
|
242
|
+
def next_data
|
243
|
+
raise
|
244
|
+
end
|
245
|
+
remove_method :next_data
|
246
|
+
|
247
|
+
end
|
248
|
+
|
249
|
+
|
250
|
+
end
|
data/test/digest_test.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
require 'iop/digest'
|
3
|
+
require 'iop/file'
|
4
|
+
require 'openssl'
|
5
|
+
|
6
|
+
class DigestTest < Test::Unit::TestCase
|
7
|
+
|
8
|
+
include IOP
|
9
|
+
|
10
|
+
def test_digest
|
11
|
+
( FileReader.new(__FILE__) | DigestComputer.new(Digest::MD5.new) ).process!
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_openssl_digest
|
15
|
+
( FileReader.new(__FILE__) | DigestComputer.new(OpenSSL::Digest::MD5.new) ).process!
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
data/test/io_test.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
require 'iop/file'
|
3
|
+
require 'iop/string'
|
4
|
+
require 'stringio'
|
5
|
+
|
6
|
+
class IOTest < Test::Unit::TestCase
|
7
|
+
|
8
|
+
include IOP
|
9
|
+
|
10
|
+
def test_iosegmentreader_small
|
11
|
+
s = '0123456789'
|
12
|
+
(1..s.size-1).each do |b|
|
13
|
+
(1..11).each do |i|
|
14
|
+
m = StringMerger.new
|
15
|
+
r = IOSegmentReader.new(StringIO.open(s), block_size: i)
|
16
|
+
(r.read!(b) | m).process!
|
17
|
+
(r.read!(s.size-b) | m).process!
|
18
|
+
assert_equal s, m.to_s
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|