piperator 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +27 -1
- data/lib/piperator.rb +1 -0
- data/lib/piperator/io.rb +119 -0
- data/lib/piperator/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eec6368ee594785c21b367e0575ff8e56b1333c9
|
4
|
+
data.tar.gz: 7f4dc49b0dbab3b79bbd171b0099b7c0d623c157
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a784c23791a28b994d4db90626589478b58dfff806a35f3fc812792ed8c247fef556d5e72b1a2aa7195b97fbc0f73e0deab4d80cf9a03e91e393e1af809bc7e6
|
7
|
+
data.tar.gz: f0e310f29dc9965e0f8a94826e0a66830761d4333aae8884cfb05e64cba35ea359657aa25e8046d37e98e30a1c18029dc670303765cd2f139a75ce9f87faf1fc
|
data/README.md
CHANGED
@@ -23,7 +23,9 @@ Start by requiring the gem
|
|
23
23
|
require 'piperator'
|
24
24
|
```
|
25
25
|
|
26
|
-
|
26
|
+
### Pipelines
|
27
|
+
|
28
|
+
As an appetizer, here's a pipeline that triples all input values and then sums the values.
|
27
29
|
|
28
30
|
```ruby
|
29
31
|
Piperator.
|
@@ -124,6 +126,30 @@ Piperator.pipe(double).pipe(prepend_append).call([1, 2, 3]).to_a
|
|
124
126
|
# => ['start', 2, 4, 6, 'end']
|
125
127
|
```
|
126
128
|
|
129
|
+
### Enumerators as IO objects
|
130
|
+
|
131
|
+
Piperator also provides a helper class that allows `Enumerator`s to be used as
|
132
|
+
IO objects. This is useful to provide integration with libraries that work only
|
133
|
+
with IO objects such as [Nokogiri](http://www.nokogiri.org) or
|
134
|
+
[Oj](https://github.com/ohler55/oj).
|
135
|
+
|
136
|
+
An example pipe that would yield all XML node in a document read in streams:
|
137
|
+
|
138
|
+
```ruby
|
139
|
+
|
140
|
+
require 'nokogiri'
|
141
|
+
streaming_xml = lambda do |enumerable|
|
142
|
+
Enumerator.new do |yielder|
|
143
|
+
io = Piperator::IO.new(enumerable.each)
|
144
|
+
reader = Nokogiri::XML::Reader(io)
|
145
|
+
reader.each { |node| yielder << node }
|
146
|
+
end
|
147
|
+
end
|
148
|
+
```
|
149
|
+
|
150
|
+
In real-world scenarios, the pipe would need to filter the nodes. Passing every
|
151
|
+
single XML node forward is not that useful.
|
152
|
+
|
127
153
|
## Development
|
128
154
|
|
129
155
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/lib/piperator.rb
CHANGED
data/lib/piperator/io.rb
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
require 'English'
|
2
|
+
|
3
|
+
module Piperator
|
4
|
+
# Pseudo I/O on Enumerators
|
5
|
+
class IO
|
6
|
+
FLUSH_THRESHOLD = 128 * 1028 # 128KiB
|
7
|
+
|
8
|
+
attr_reader :eof
|
9
|
+
|
10
|
+
def initialize(enumerator, flush_threshold: FLUSH_THRESHOLD)
|
11
|
+
@enumerator = enumerator
|
12
|
+
@flush_threshold = flush_threshold
|
13
|
+
@io = StringIO.new
|
14
|
+
@buffer_start_pos = 0
|
15
|
+
@io_read_pos = 0
|
16
|
+
@eof = false
|
17
|
+
end
|
18
|
+
|
19
|
+
alias eof? eof
|
20
|
+
|
21
|
+
# Return the first bytes of the buffer without marking the buffer as read.
|
22
|
+
def peek(bytes)
|
23
|
+
while @eof == false && readable_bytes < (bytes || 1)
|
24
|
+
@io.write(@enumerator.next)
|
25
|
+
end
|
26
|
+
peek_buffer(bytes)
|
27
|
+
rescue StopIteration
|
28
|
+
@eof = true
|
29
|
+
peek_buffer(bytes)
|
30
|
+
end
|
31
|
+
|
32
|
+
# Reads the next "line" from the I/O stream; lines are separated by
|
33
|
+
# separator.
|
34
|
+
#
|
35
|
+
# @param separator [String] separator to split input
|
36
|
+
# @param _limit Unused parameter for compatiblity
|
37
|
+
# @return [String]
|
38
|
+
def gets(separator = $INPUT_RECORD_SEPARATOR, _limit = nil)
|
39
|
+
while !@eof && !contains_line?(separator)
|
40
|
+
begin
|
41
|
+
@io.write(@enumerator.next)
|
42
|
+
rescue StopIteration
|
43
|
+
@eof = true
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
end
|
47
|
+
read_with { @io.gets(separator) }
|
48
|
+
end
|
49
|
+
|
50
|
+
# Flush internal buffer until the last unread byte
|
51
|
+
def flush
|
52
|
+
if @io.pos == @io_read_pos
|
53
|
+
initialize_buffer
|
54
|
+
else
|
55
|
+
@io.pos = @io_read_pos
|
56
|
+
initialize_buffer(@io.read)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# Reads length bytes from the I/O stream.
|
61
|
+
#
|
62
|
+
# @param length [Integer] number of bytes to read
|
63
|
+
# @return String
|
64
|
+
def read(length = nil)
|
65
|
+
return @enumerator.next if length.nil? && readable_bytes.zero?
|
66
|
+
@io.write(@enumerator.next) while !@eof && readable_bytes < (length || 1)
|
67
|
+
read_with { @io.read(length) }
|
68
|
+
rescue StopIteration
|
69
|
+
@eof = true
|
70
|
+
read_with { @io.read(length) } if readable_bytes > 0
|
71
|
+
end
|
72
|
+
|
73
|
+
# Current buffer size - including non-freed read content
|
74
|
+
#
|
75
|
+
# @return [Integer] number of bytes stored in buffer
|
76
|
+
def used
|
77
|
+
@io.size
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def readable_bytes
|
83
|
+
@io.pos - @io_read_pos
|
84
|
+
end
|
85
|
+
|
86
|
+
def read_with
|
87
|
+
pos = @io.pos
|
88
|
+
@io.pos = @io_read_pos
|
89
|
+
|
90
|
+
yield.tap do |data|
|
91
|
+
@io_read_pos += data.bytesize if data
|
92
|
+
@io.pos = pos
|
93
|
+
flush if flush?
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def peek_buffer(bytes)
|
98
|
+
@io.string.byteslice(@io_read_pos...@io_read_pos + bytes)
|
99
|
+
end
|
100
|
+
|
101
|
+
def flush?
|
102
|
+
@io.pos == @io_read_pos || @io.pos > @flush_threshold
|
103
|
+
end
|
104
|
+
|
105
|
+
def initialize_buffer(data = nil)
|
106
|
+
@io_read_pos = 0
|
107
|
+
@buffer_start_pos += @io.pos if @io
|
108
|
+
@io = StringIO.new
|
109
|
+
@io.write(data) if data
|
110
|
+
end
|
111
|
+
|
112
|
+
def contains_line?(separator = $INPUT_RECORD_SEPARATOR)
|
113
|
+
return true if @eof
|
114
|
+
@io.string.byteslice(@io_read_pos..-1).include?(separator)
|
115
|
+
rescue ArgumentError # Invalid UTF-8
|
116
|
+
false
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
data/lib/piperator/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: piperator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ville Lautanala
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -70,6 +70,7 @@ files:
|
|
70
70
|
- bin/console
|
71
71
|
- bin/setup
|
72
72
|
- lib/piperator.rb
|
73
|
+
- lib/piperator/io.rb
|
73
74
|
- lib/piperator/pipeline.rb
|
74
75
|
- lib/piperator/version.rb
|
75
76
|
- piperator.gemspec
|