iop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,49 @@
1
+ require 'iop'
2
+ require 'securerandom'
3
+
4
+
5
+ module IOP
6
+
7
+
8
+ #
9
+ # Feed class to generate and send a random sequence of bytes of specified size.
10
+ #
11
+ # This is the adapter for standard {SecureRandom} generator module.
12
+ #
13
+ # ### Use case: generate 1024 bytes of random data and compute MD5 hash sum of it.
14
+ #
15
+ # require 'iop/digest'
16
+ # require 'iop/securerandom'
17
+ # ( IOP::SecureRandomGenerator.new(1024) | IOP::DigestComputer.new(Digest::MD5.new) ).process!
18
+ #
19
+ # @since 0.1
20
+ #
21
+ class SecureRandomGenerator
22
+
23
+ include Feed
24
+
25
+ # Creates class instance.
26
+ #
27
+ # @param size [Integer] total random data size
28
+ #
29
+ # @param block_size [Integer] size of block the data in split into
30
+ def initialize(size, block_size: DEFAULT_BLOCK_SIZE)
31
+ @size = size
32
+ @block_size = block_size
33
+ end
34
+
35
+ def process!
36
+ written = 0
37
+ (0..@size/@block_size - 1).each do
38
+ process(SecureRandom.bytes(@block_size))
39
+ written += @block_size
40
+ end
41
+ left = @size - written
42
+ process(SecureRandom.bytes(left)) unless left.zero?
43
+ process
44
+ end
45
+
46
+ end
47
+
48
+
49
+ end
data/lib/iop/string.rb ADDED
@@ -0,0 +1,89 @@
1
+ require 'iop'
2
+
3
+
4
+ module IOP
5
+
6
+
7
+ #
8
+ # Feed class to send arbitrary string in blocks of specified size.
9
+ #
10
+ # ### Use case: split the string into 3-byte blocks and reconstruct it.
11
+ #
12
+ # require 'iop/string'
13
+ # ( IOP::StringSplitter.new('Hello IOP', 3) | IOP::StringMerger.new ).process!
14
+ #
15
+ # @since 0.1
16
+ #
17
+ class StringSplitter
18
+
19
+ include Feed
20
+
21
+ # Creates class instance.
22
+ #
23
+ # @param string [String] string to be sent in blocks
24
+ #
25
+ # @param block_size [Integer] size of block the string is split into
26
+ def initialize(string, block_size: DEFAULT_BLOCK_SIZE)
27
+ @string = string
28
+ @block_size = block_size
29
+ end
30
+
31
+ def process!
32
+ offset = 0
33
+ (0..@string.size / @block_size - 1).each do
34
+ process(@string[offset, @block_size])
35
+ offset += @block_size
36
+ end
37
+ process(offset.zero? ? @string : @string[offset..-1]) unless offset == @string.size
38
+ process
39
+ end
40
+
41
+ end
42
+
43
+
44
+ #
45
+ # Sink class to receive data blocks and merge them into a single string.
46
+ #
47
+ # ### Use case: read current source file into a string.
48
+ #
49
+ # require 'iop/file'
50
+ # require 'iop/string'
51
+ # ( IOP::FileReader.new($0) | (s = IOP::StringMerger.new) ).process!
52
+ # puts s.to_s
53
+ #
54
+ # The actual string assembly is performed by the {#to_s} method.
55
+ #
56
+ # @note instance of this class can be used to collect data from multiple processing runs.
57
+ #
58
+ # @since 0.1
59
+ #
60
+ class StringMerger
61
+
62
+ include Sink
63
+
64
+ # Creates class instance.
65
+ def initialize
66
+ @size = 0
67
+ @data = []
68
+ end
69
+
70
+ def process(data = nil)
71
+ unless data.nil?
72
+ @data << data.dup # CHECKME is duplication really needed when the upstream continuously resending its internal data buffer with new contents
73
+ @size += data.size
74
+ end
75
+ end
76
+
77
+ # Returns concatenation of all received data blocks into a single string.
78
+ #
79
+ # @return [String]
80
+ def to_s
81
+ string = IOP.allocate_string(@size)
82
+ @data.each {|x| string << x}
83
+ string
84
+ end
85
+
86
+ end
87
+
88
+
89
+ end
data/lib/iop/zlib.rb ADDED
@@ -0,0 +1,179 @@
1
+ require 'iop'
2
+ require 'zlib'
3
+
4
+
5
+ module IOP
6
+
7
+
8
+ #
9
+ # Filter class to perform data compression with Zlib algorithm.
10
+ #
11
+ # This class is an adapter for the standard Ruby +Zlib::Deflate+ class.
12
+ #
13
+ # Note that this class does not produce valid _.gz_ files - use {GzipCompressor} for this purpose.
14
+ #
15
+ # ### Use case: compress a string.
16
+ #
17
+ # require 'iop/zlib'
18
+ # require 'iop/string'
19
+ # ( IOP::StringSplitter.new('Hello IOP') | IOP::ZlibCompressor.new | (s = IOP::StringMerger.new) ).process!
20
+ # puts s.to_s
21
+ #
22
+ # @since 0.1
23
+ #
24
+ class ZlibCompressor
25
+
26
+ include Feed
27
+ include Sink
28
+
29
+ # Creates class instance.
30
+ #
31
+ # @param args [Array] arguments passed to +Zlib::Deflate+ constructor
32
+ def initialize(*args)
33
+ @args = args
34
+ end
35
+
36
+ def process(data = nil)
37
+ if data.nil?
38
+ super(@deflate.finish)
39
+ super
40
+ else
41
+ super(@deflate.deflate(data))
42
+ end
43
+ end
44
+
45
+ def process!
46
+ @deflate = Zlib::Deflate.new(*@args)
47
+ begin
48
+ super
49
+ ensure
50
+ @deflate.close
51
+ end
52
+ end
53
+ end
54
+
55
+
56
+ #
57
+ # Filter class to perform data decompression with Zlib algorithm.
58
+ #
59
+ # This class is an adapter for the standard Ruby +Zlib::Inflate+ class.
60
+ #
61
+ # Note that this class can not decompress _.gz_ files - use {GzipDecompressor} for this purpose.
62
+ #
63
+ # ### Use case: decompress a Zlib-compressed part of a file skipping a header and compute MD5 hash sum of the uncompressed data.
64
+ #
65
+ # require 'iop/zlib'
66
+ # require 'iop/file'
67
+ # require 'iop/digest'
68
+ # ( IOP::FileReader.new('input.dat', offset: 16) | IOP::ZlibDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
69
+ # puts d.digest.hexdigest
70
+ #
71
+ # @since 0.1
72
+ #
73
+ class ZlibDecompressor
74
+
75
+ include Feed
76
+ include Sink
77
+
78
+ # Creates class instance.
79
+ #
80
+ # @param args [Array] arguments passed to +Zlib::Inflate+ constructor
81
+ def initialize(*args)
82
+ @args = args
83
+ end
84
+
85
+ def process(data = nil)
86
+ if data.nil?
87
+ super(@inflate.finish)
88
+ super
89
+ else
90
+ super(@inflate.inflate(data))
91
+ end
92
+ end
93
+
94
+ def process!
95
+ @inflate = Zlib::Inflate.new(*@args)
96
+ begin
97
+ super
98
+ ensure
99
+ @inflate.close
100
+ end
101
+ end
102
+ end
103
+
104
+
105
+ #
106
+ # Filter class to perform Gzip data compression.
107
+ #
108
+ # This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
109
+ #
110
+ # This class produces valid _.gz_ files.
111
+ #
112
+ # ### Use case: compress a string and store it to .gz file.
113
+ #
114
+ # require 'iop/zlib'
115
+ # require 'iop/file'
116
+ # require 'iop/string'
117
+ # ( IOP::StringSplitter.new('Hello IOP') | IOP::GzipCompressor.new | IOP::FileWriter.new('hello.gz') ).process!
118
+ #
119
+ # @since 0.1
120
+ #
121
+ class GzipCompressor
122
+
123
+ include Feed
124
+ include Sink
125
+
126
+ # Creates class instance.
127
+ #
128
+ # @param args [Array] arguments passed to +Zlib::GzipWriter+ constructor
129
+ def initialize(*args)
130
+ @args = args
131
+ end
132
+
133
+ def process(data = nil)
134
+ if data.nil?
135
+ @compressor.finish
136
+ super
137
+ else
138
+ @compressor.write(data)
139
+ end
140
+ end
141
+
142
+ def write(data)
143
+ downstream&.process(data)
144
+ end
145
+
146
+ def process!
147
+ @compressor = Zlib::GzipWriter.new(self, *@args)
148
+ super
149
+ ensure
150
+ @compressor.close unless @compressor.nil?
151
+ end
152
+ end
153
+
154
+
155
+ #
156
+ # Filter class to perform Gzip data compression.
157
+ #
158
+ # This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
159
+ #
160
+ # This class can decompress _.gz_ files.
161
+ #
162
+ # ### Use case: decompress a .gz file and compute MD5 hash sum of uncompressed data.
163
+ #
164
+ # require 'iop/zlib'
165
+ # require 'iop/file'
166
+ # require 'iop/digest'
167
+ # ( IOP::FileReader.new('hello.gz') | IOP::GzipDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
168
+ # puts d.digest.hexdigest
169
+ #
170
+ # @since 0.1
171
+ #
172
+ class GzipDecompressor < ZlibDecompressor
173
+ def initialize
174
+ super(16)
175
+ end
176
+ end
177
+
178
+
179
+ end
@@ -0,0 +1,104 @@
1
+ require 'iop'
2
+ require 'zstdlib'
3
+
4
+
5
+ module IOP
6
+
7
+
8
+ #
9
+ # Filter class to perform data compression with Zstandard algorithm.
10
+ #
11
+ # This class produces valid _.zst_ files.
12
+ #
13
+ # ### Use case: compress a string and store it to .zst file.
14
+ #
15
+ # require 'iop/file'
16
+ # require 'iop/string'
17
+ # require 'iop/zstdlib'
18
+ # ( IOP::StringSplitter.new('Hello IOP') | IOP::ZstdCompressor.new(Zstdlib::BEST_COMPRESSION) | IOP::FileWriter.new('hello.zst') ).process!
19
+ #
20
+ # @note this class depends on external +zstdlib+ gem.
21
+ # @since 0.1
22
+ #
23
+ class ZstdCompressor
24
+
25
+ include Feed
26
+ include Sink
27
+
28
+ # Creates class instance.
29
+ #
30
+ # @param args [Array] arguments passed to +Zstdlib::Deflate+ constructor
31
+ def initialize(*args)
32
+ @args = args
33
+ end
34
+
35
+ def process(data = nil)
36
+ if data.nil?
37
+ super(@deflate.finish)
38
+ super
39
+ else
40
+ super(@deflate.deflate(data))
41
+ end
42
+ end
43
+
44
+ def process!
45
+ @deflate = Zstdlib::Deflate.new(*@args)
46
+ begin
47
+ super
48
+ ensure
49
+ @deflate.close
50
+ end
51
+ end
52
+ end
53
+
54
+
55
+ #
56
+ # Filter class to perform Gzip data compression.
57
+ #
58
+ # This class is an adapter for the standard Ruby +Zlib::GzipWriter+ class.
59
+ #
60
+ # This class can decompress _.zst_ files.
61
+ #
62
+ # ### Use case: decompress a .zst file and compute MD5 hash sum of uncompressed data.
63
+ #
64
+ # require 'iop/file'
65
+ # require 'iop/digest'
66
+ # require 'iop/zstdlib'
67
+ # ( IOP::FileReader.new('hello.zst') | IOP::ZstdDecompressor.new | (d = IOP::DigestComputer.new(Digest::MD5.new)) ).process!
68
+ # puts d.digest.hexdigest
69
+ #
70
+ # @note this class depends on external +zstdlib+ gem.
71
+ # @since 0.1
72
+ #
73
+ class ZstdDecompressor
74
+
75
+ include Feed
76
+ include Sink
77
+
78
+ # Creates class instance.
79
+ #
80
+ # @param args [Array] arguments passed to +Zstdlib::Inflate+ constructor
81
+ def initialize(*args)
82
+ @args = args
83
+ end
84
+
85
+ def process(data = nil)
86
+ if data.nil?
87
+ super(@inflate.finish)
88
+ super
89
+ else
90
+ super(@inflate.inflate(data))
91
+ end
92
+ end
93
+
94
+ def process!
95
+ @inflate = Zstdlib::Inflate.new(*@args)
96
+ begin
97
+ super
98
+ ensure
99
+ @inflate.close
100
+ end
101
+ end
102
+ end
103
+
104
+ end
data/lib/iop.rb ADDED
@@ -0,0 +1,250 @@
1
+ #
2
+ # IOP is intended for constructing the data processing pipelines in a manner of UNIX command-line pipes.
3
+ #
4
+ # There are three principle types of the pipe nodes which can be composed:
5
+ #
6
+ # * Feed node.
7
+ #
8
+ # This is the start point of the pipe. It has no upstream node and may have downstream node.
9
+ # Its purpose its to generate blocks of data and send them downstream in sequence.
10
+ # A typical feed class is implemented by including the {Feed} module and defining the +#process!+ method
11
+ # which calls {Feed#process} method to send the data.
12
+ # An example of the feed node is a file reader ({FileReader}) which reads file and sends its contents in blocks.
13
+ #
14
+ # * Sink node.
15
+ #
16
+ # This is the end point of the pipe. It has upstream node and no downstream node.
17
+ # Its purpose is to consume the received data.
18
+ # A typical sink class is implemented by including the {Sink} module and defining the +#process+ method.
19
+ # An example of the sink node is a file writer ({FileWriter}) which receives the data in blocks and writes it into file.
20
+ #
21
+ # * Filter node.
22
+ #
23
+ # A filter is a pass-through node which sits between feed and sink and therefore has both upstream and downstream nodes.
24
+ # The simplest way to create a filter class is to include both {Feed} and {Sink} which manifest
25
+ # both mandatory +#process!+ and +#process+ methods. Such filter is a no-op that is it does nothing apart passing
26
+ # the received data downstream.
27
+ # An example of the filter node is the digest computer ({DigestComputer}) which computes hash sum of the data it passes through.
28
+ # In order to perform intended processing of the data a filter class overrides the {Feed#process} method.
29
+ #
30
+ # The basic control flow for an {IOP}-aware pipe is as follows:
31
+ #
32
+ # 1. The pipe is constructed from one or more {IOP}-aware class instances. The two or more objects are linked together
33
+ # with the | operator implemented as the {Feed#|} method by default.
34
+ #
35
+ # 2. The actual processing is then triggered by the {Sink#process!} method of the very last object in the pipe.
36
+ # By default, this method calls the same method of the upstream node thus forming the stack of nested calls
37
+ # for all objects in the pipe.
38
+ #
39
+ # 3. Upon reaching the very first object in the pipe (which by definition has no upstream node),
40
+ # the feed, starts sending blocks of data downstream with the {Feed#process} method. All objects' method implementations
41
+ # (except for the one of the last object in the pipe) are expected to push either this or transformed data further downstream.
42
+ #
43
+ # 4. After all data has been processed the finalizing call +#process(nil)+ signifies the end-of-data after which
44
+ # no data should be sent.
45
+ #
46
+ # In case the {Sink#process!} method is overridden in concrete class it is normally organized as follows:
47
+ #
48
+ # def process!
49
+ # # ...initialization code...
50
+ # super
51
+ # ensure
52
+ # # ...finalization code...
53
+ # end
54
+ #
55
+ # to perform specific setup/cleanup actions, including exception handling and to pass the control flow upstream
56
+ # with +super+ call.
57
+ #
58
+ # Note that when an exception is caught and processed in overridden +#process!+ method it must be re-raised in order
59
+ # for other upstream objects to have a chance to react to it as well.
60
+ #
61
+ # In case the {Feed#process} is overridden in concrete class it is organized as follows:
62
+ #
63
+ # def process(data = nil)
64
+ # # ... do something with data, convert data to new_data...
65
+ # super(new_data)
66
+ # end
67
+ #
68
+ # The data being sent is expected to be a +String+ of arbitrary size. It is however advisable to detect and omit
69
+ # zero-sized strings.
70
+ #
71
+ # Note that the data passed to this method may be a reusable buffer of some other upstream object therefore a duplication
72
+ # (or cloning) should be performed if the data is stored between the method invocations.
73
+ #
74
+ module IOP
75
+
76
+
77
+ VERSION = '0.1.0'
78
+
79
+
80
+ # Default read block size in bytes for adapters which don't have this parameter externally imposed.
81
+ DEFAULT_BLOCK_SIZE = 1024**2
82
+
83
+
84
+ if RUBY_VERSION >= '2.4'
85
+ # @private
86
+ def self.allocate_string(size)
87
+ String.new(capacity: size)
88
+ end
89
+ else
90
+ # @private
91
+ def self.allocate_string(size)
92
+ String.new
93
+ end
94
+ end
95
+
96
+
97
+ # @private
98
+ INSUFFICIENT_DATA = 'premature end-of-data encountered'.freeze
99
+
100
+
101
+ # @private
102
+ EXTRA_DATA = 'superfluous data received'.freeze
103
+
104
+
105
+ # @private
106
+ # Finds minimum of the values
107
+ def self.min(a, b)
108
+ a < b ? a : b
109
+ end
110
+
111
+
112
+ #
113
+ # Module to be included into classes which generate and send the data downstream.
114
+ #
115
+ # @since 0.1
116
+ #
117
+ module Feed
118
+
119
+ #
120
+ # Commences the data processing operation.
121
+ #
122
+ # @abstract
123
+ #
124
+ # @note this method should be implemented in concrete classes including this module.
125
+ #
126
+ # Refer to {Sink#process!} for details.
127
+ #
128
+ def process!
129
+ raise
130
+ end
131
+ remove_method :process!
132
+
133
+ #
134
+ # Sends the data block downstream.
135
+ #
136
+ # @note by convention, the very last call to this method should pass +nil+ to indicate the end-of-data and no data should be sent afterwards.
137
+ #
138
+ # This implementation simply passes through the received data block downstream if there exists an attached downstream
139
+ # object otherwise the data is simply thrown away.
140
+ #
141
+ # The overriding method in concrete class which includes {Feed} would normally want to call this one as +super+ after
142
+ # performing specific actions.
143
+ #
144
+ def process(data = nil)
145
+ downstream&.process(data) # Ruby 2.3+
146
+ end
147
+
148
+ # Returns the downstream object or +nil+ if +self+ is the last object in processing pipe.
149
+ attr_reader :downstream
150
+
151
+ #
152
+ # Links +self+ and +downstream+ together forming a processing pipe.
153
+ # The subsequent objects may be linked in turn.
154
+ # @return downstream object
155
+ #
156
+ def |(downstream)
157
+ downstream.upstream = self
158
+ @downstream = downstream
159
+ end
160
+
161
+ end
162
+
163
+
164
+ #
165
+ # Module to be included into classes which receive and process the upstream data.
166
+ #
167
+ # @since 0.1
168
+ #
169
+ module Sink
170
+
171
+ # Commences the data processing operation.
172
+ #
173
+ # This implementation calls {#process!} method of the upstream object.
174
+ def process!
175
+ upstream.process!
176
+ end
177
+
178
+ # @abstract
179
+ #
180
+ # @note this method should be implemented in concrete classes including this module.
181
+ #
182
+ # Refer to {Feed#process} for more information.
183
+ def process(data = nil)
184
+ raise
185
+ end
186
+ remove_method :process
187
+
188
+ # Returns the upstream object or +nil+ if +self+ is the first object in processing pipe.
189
+ attr_accessor :upstream
190
+
191
+ end
192
+
193
+
194
+ #
195
+ # @private
196
+ #
197
+ # @note a class including this module must implement the {#next_data} method.
198
+ #
199
+ # @since 0.1
200
+ #
201
+ module BufferingFeed
202
+
203
+ include Feed
204
+
205
+ def read!(size)
206
+ @left = @size = size
207
+ self
208
+ end
209
+
210
+ def process!
211
+ unless @buffer.nil?
212
+ if @buffer.size > @size
213
+ @left = 0
214
+ process(@buffer[0, @size])
215
+ @buffer = @buffer[@size..-1]
216
+ else
217
+ @left -= @buffer.size
218
+ process(@buffer)
219
+ @buffer = nil
220
+ end
221
+ end
222
+ until @left.zero?
223
+ raise EOFError, INSUFFICIENT_DATA if (data = next_data).nil?
224
+ if @left < data.size
225
+ process(data[0, @left])
226
+ @buffer = data[@left..-1]
227
+ @left = 0
228
+ else
229
+ process(data)
230
+ @left -= data.size
231
+ end
232
+ end
233
+ @left = @size = nil
234
+ process
235
+ end
236
+
237
+ # @abstract
238
+ #
239
+ # Returns the data portion of non-zero size or +nil+ on EOF.
240
+ #
241
+ # @return [String] data chunk recently read or +nil+
242
+ def next_data
243
+ raise
244
+ end
245
+ remove_method :next_data
246
+
247
+ end
248
+
249
+
250
+ end
@@ -0,0 +1,18 @@
1
+ require 'test/unit'
2
+ require 'iop/digest'
3
+ require 'iop/file'
4
+ require 'openssl'
5
+
6
+ class DigestTest < Test::Unit::TestCase
7
+
8
+ include IOP
9
+
10
+ def test_digest
11
+ ( FileReader.new(__FILE__) | DigestComputer.new(Digest::MD5.new) ).process!
12
+ end
13
+
14
+ def test_openssl_digest
15
+ ( FileReader.new(__FILE__) | DigestComputer.new(OpenSSL::Digest::MD5.new) ).process!
16
+ end
17
+
18
+ end
data/test/io_test.rb ADDED
@@ -0,0 +1,23 @@
1
+ require 'test/unit'
2
+ require 'iop/file'
3
+ require 'iop/string'
4
+ require 'stringio'
5
+
6
+ class IOTest < Test::Unit::TestCase
7
+
8
+ include IOP
9
+
10
+ def test_iosegmentreader_small
11
+ s = '0123456789'
12
+ (1..s.size-1).each do |b|
13
+ (1..11).each do |i|
14
+ m = StringMerger.new
15
+ r = IOSegmentReader.new(StringIO.open(s), block_size: i)
16
+ (r.read!(b) | m).process!
17
+ (r.read!(s.size-b) | m).process!
18
+ assert_equal s, m.to_s
19
+ end
20
+ end
21
+ end
22
+
23
+ end