d-stream 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: '0398736030470a1452a92ff161692a046b783399'
4
+ data.tar.gz: 9d19b1c1af96896470b800e8fcfa6ba438716526
5
+ SHA512:
6
+ metadata.gz: 8a1bf1c8937184aa7eeb5c89be8359a7e252d6ec617c47f4e61611576b9740f56cc9736569aa55dc37d44bb6f0efeb94f17dc52fa9a2d2e62bdb791263e27d02
7
+ data.tar.gz: 2182ccd0402d0dcd5522e76a44c09b77e6c58ec1306d08353f644e75b236219bb27b8390d823eae7280b297509d47183ea6c6716a606bb11c9491acf808c5c82
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ gem 'rspec'
6
+ gem 'rubocop'
@@ -0,0 +1,49 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ d-stream (0.0.1)
5
+
6
+ GEM
7
+ remote: https://rubygems.org/
8
+ specs:
9
+ ast (2.3.0)
10
+ diff-lcs (1.3)
11
+ parser (2.4.0.0)
12
+ ast (~> 2.2)
13
+ powerpack (0.1.1)
14
+ rainbow (2.2.2)
15
+ rake
16
+ rake (12.0.0)
17
+ rspec (3.6.0)
18
+ rspec-core (~> 3.6.0)
19
+ rspec-expectations (~> 3.6.0)
20
+ rspec-mocks (~> 3.6.0)
21
+ rspec-core (3.6.0)
22
+ rspec-support (~> 3.6.0)
23
+ rspec-expectations (3.6.0)
24
+ diff-lcs (>= 1.2.0, < 2.0)
25
+ rspec-support (~> 3.6.0)
26
+ rspec-mocks (3.6.0)
27
+ diff-lcs (>= 1.2.0, < 2.0)
28
+ rspec-support (~> 3.6.0)
29
+ rspec-support (3.6.0)
30
+ rubocop (0.48.1)
31
+ parser (>= 2.3.3.1, < 3.0)
32
+ powerpack (~> 0.1)
33
+ rainbow (>= 1.99.1, < 3.0)
34
+ ruby-progressbar (~> 1.7)
35
+ unicode-display_width (~> 1.0, >= 1.0.1)
36
+ ruby-progressbar (1.8.1)
37
+ unicode-display_width (1.2.1)
38
+
39
+ PLATFORMS
40
+ ruby
41
+
42
+ DEPENDENCIES
43
+ bundler (~> 1.14)
44
+ d-stream!
45
+ rspec
46
+ rubocop
47
+
48
+ BUNDLED WITH
49
+ 1.14.6
@@ -0,0 +1,3 @@
1
+ guard 'rake', task: 'default' do
2
+ watch(%r{^(lib|spec)/})
3
+ end
data/LICENSE ADDED
@@ -0,0 +1,19 @@
1
+ Copyright (c) 2017 Denis Defreyne and contributors
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the "Software"), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in all
11
+ copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19
+ SOFTWARE.
data/NEWS.md ADDED
@@ -0,0 +1,5 @@
1
+ # D★Stream news
2
+
3
+ ## 0.0.1 (2017-05-05)
4
+
5
+ Initial release.
@@ -0,0 +1,240 @@
1
+ # D★Stream
2
+
3
+ [![Gem version](http://img.shields.io/gem/v/d-stream.svg)](http://rubygems.org/gems/d-stream)
4
+ [![Build status](http://img.shields.io/travis/ddfreyne/d-stream.svg)](https://travis-ci.org/ddfreyne/d-stream)
5
+ [![Code Climate](http://img.shields.io/codeclimate/github/ddfreyne/d-stream.svg)](https://codeclimate.com/github/ddfreyne/d-stream)
6
+ [![Code Coverage](http://img.shields.io/codecov/c/github/ddfreyne/d-stream.svg)](https://codecov.io/github/ddfreyne/d-stream)
7
+
8
+ _D★Stream_ is a set of extensions for writing stream-processing code in Ruby.
9
+
10
+ **CAUTION:** D★Stream is work in progress, and pre-alpha quality.
11
+
12
+ ## Examples
13
+
14
+ ### Example 1: straightforward
15
+
16
+ The following example takes a sequence of events for a given ticket, and calculates the history for that ticket, using slowly changing dimensions:
17
+
18
+ ```ruby
19
+ events =
20
+ Enumerator.new do |y|
21
+ y << { id: 40562348, at: Time.now - 400, status: 'new' }
22
+ y << { id: 40564682, at: Time.now - 300, assignee_id: 2 }
23
+ y << { id: 40565795, at: Time.now - 250, priority: 'high' }
24
+ y << { id: 40569932, at: Time.now - 100, status: 'solved' }
25
+ end.lazy
26
+
27
+ S = DStream
28
+
29
+ indices = (1..(1.0 / 0.0))
30
+
31
+ history =
32
+ S.apply(
33
+ events,
34
+
35
+ # calculate new state
36
+ S.scan({}, &:merge),
37
+
38
+ # add `version`
39
+ S.zip(indices),
40
+ S.map { |(e, i)| e.merge(version: i) },
41
+
42
+ # remove `id`
43
+ S.map { |e| e.reject { |k, _v| k == :id } },
44
+
45
+ # add `valid_to` and `valid_from`, and remove `at`
46
+ S.with_next,
47
+ S.map { |(a, b)| a.merge(valid_to: b ? b.fetch(:at) : nil) },
48
+ S.map { |e| e.merge(valid_from: e.fetch(:at)) },
49
+ S.map { |e| e.reject { |k, _v| k == :at } },
50
+
51
+ # add `row_is_current`
52
+ S.with_next,
53
+ S.map { |(a, b)| a.merge(row_is_current: b.nil?) },
54
+ )
55
+
56
+ history.each { |e| p e }
57
+ ```
58
+
59
+ The output is as follows:
60
+
61
+ ```
62
+ {
63
+ :status=>"new",
64
+ :valid_from=>2017-05-05 20:18:14 +0200,
65
+ :valid_to=>2017-05-05 20:19:54 +0200,
66
+ :version=>1,
67
+ :row_is_current=>false
68
+ }
69
+ {
70
+ :status=>"new",
71
+ :assignee_id=>2,
72
+ :valid_from=>2017-05-05 20:19:54 +0200,
73
+ :valid_to=>2017-05-05 20:20:44 +0200,
74
+ :version=>2,
75
+ :row_is_current=>false
76
+ }
77
+ {
78
+ :status=>"new",
79
+ :assignee_id=>2,
80
+ :priority=>"high",
81
+ :valid_from=>2017-05-05 20:20:44 +0200,
82
+ :valid_to=>2017-05-05 20:23:14 +0200,
83
+ :version=>3,
84
+ :row_is_current=>false
85
+ }
86
+ {
87
+ :status=>"solved",
88
+ :assignee_id=>2,
89
+ :priority=>"high",
90
+ :valid_from=>2017-05-05 20:23:14 +0200,
91
+ :valid_to=>nil,
92
+ :version=>4,
93
+ :row_is_current=>true
94
+ }
95
+ ```
96
+
97
+ ### Example 2: better factored
98
+
99
+ This example is functionally identical to the one above, but uses `S.compose` in order to make the final process, `history_builder`, easier to understand.
100
+
101
+ ```ruby
102
+ events =
103
+ Enumerator.new do |y|
104
+ y << { id: 40562348, at: Time.now - 400, status: 'new' }
105
+ y << { id: 40564682, at: Time.now - 300, assignee_id: 2 }
106
+ y << { id: 40565795, at: Time.now - 250, priority: 'high' }
107
+ y << { id: 40569932, at: Time.now - 100, status: 'solved' }
108
+ end.lazy
109
+
110
+ S = DStream
111
+
112
+ merge =
113
+ S.scan({}, &:merge),
114
+
115
+ indices = (1..(1.0 / 0.0))
116
+ add_version =
117
+ S.compose(
118
+ S.zip(indices),
119
+ S.map { |(e,i)| e.merge(version: i) },
120
+ )
121
+
122
+ remove_id =
123
+ S.map { |e| e.reject { |k, _v| k == :id } }
124
+
125
+ add_valid_dates =
126
+ S.compose(
127
+ S.with_next,
128
+ S.map { |(a,b)| a.merge(valid_to: b ? b.fetch(:at) : nil) },
129
+ S.map { |e| e.merge(valid_from: e.fetch(:at)) },
130
+ S.map { |e| e.reject { |k, _v| k == :at } },
131
+ )
132
+
133
+ add_row_is_current =
134
+ S.compose(
135
+ S.with_next,
136
+ S.map { |(a,b)| a.merge(row_is_current: b.nil?) },
137
+ )
138
+
139
+ history_builder =
140
+ S.compose(
141
+ merge,
142
+ add_version,
143
+ remove_id,
144
+ add_valid_dates,
145
+ add_row_is_current,
146
+ )
147
+
148
+ history = S.apply(events, history_builder)
149
+
150
+ history.each { |h| p h }
151
+ ```
152
+
153
+ ## API
154
+
155
+ The following functions create individual processors:
156
+
157
+ * `map(&block)` (similar to `Enumerable#map`)
158
+
159
+ ```ruby
160
+ S.apply((1..5), S.map(&:odd?)).to_a
161
+ # => [true, false, true, false, true]
162
+ ```
163
+
164
+ * `select(&block)` (similar to `Enumerable#select`)
165
+
166
+ ```ruby
167
+ S.apply((1..5), S.select(&:odd?)).to_a
168
+ # => [1, 3, 5]
169
+ ```
170
+
171
+ * `reduce(&block)` (similar to `Enumerable#reduce`)
172
+
173
+ ```ruby
174
+ S.apply((1..5), S.reduce(&:+))
175
+ # => 15
176
+ ```
177
+
178
+ * `take(n)` (similar to `Enumerable#take`)
179
+
180
+ ```ruby
181
+ S.apply((1..10), S.take(3)).to_a
182
+ # => [1, 2, 3]
183
+ ```
184
+
185
+ * `zip(other)` (similar to `Enumerable#zip`):
186
+
187
+ ```ruby
188
+ S.apply((1..3), S.zip((10..13))).to_a
189
+ # => [[1, 10], [2, 11], [3, 12]]
190
+ ```
191
+
192
+ * `buffer(size)` yields each stream element, but keeps an internal buffer of not-yet-yielded stream elements. This is useful when reading from a slow and bursty data source, such as a paginated HTTP API.
193
+
194
+ * `with_next` yields an array containing the stream element and the next stream element, or nil when the end of the stream is reached:
195
+
196
+ ```ruby
197
+ S.apply((1..5), S.with_next).to_a
198
+ # => [[1, 2], [2, 3], [3, 4], [4, 5], [5, nil]]
199
+ ```
200
+
201
+ * `scan(init, &block)` is similar to `reduce`, but rather than returning a single aggregated value, returns all intermediate aggregated values:
202
+
203
+ ```ruby
204
+ S.apply((1..5), S.scan(0, &:+)).to_a
205
+ # => [1, 3, 6, 10, 15]
206
+ ```
207
+
208
+ * `flatten2` yields the stream element if it is not an array, otherwise yields the stream element array’s contents:
209
+
210
+ ```ruby
211
+ S.apply((1..5), S.with_next, S.flatten2).to_a
212
+ # => [1, 2, 2, 3, 3, 4, 4, 5, 5, nil]
213
+ ```
214
+
215
+ To apply one or more processors to a stream, use `.apply`:
216
+
217
+ ```ruby
218
+ S = DStream
219
+
220
+ stream = ['hi']
221
+
222
+ S.apply(stream, S.map(&:upcase)).to_a
223
+ # => ["HI"]
224
+ ```
225
+
226
+ To combine one or more processors, use `.compose`:
227
+
228
+ ```ruby
229
+ S = DStream
230
+
231
+ stream = ['hi']
232
+
233
+ processor = S.compose(
234
+ S.map(&:upcase),
235
+ S.map(&:reverse),
236
+ )
237
+
238
+ S.apply(stream, processor).to_a
239
+ # => ["IH"]
240
+ ```
@@ -0,0 +1,14 @@
1
+ require 'rspec/core/rake_task'
2
+ require 'rubocop/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec) do |t|
5
+ t.rspec_opts = '-r ./spec/spec_helper.rb --color'
6
+ t.verbose = false
7
+ end
8
+
9
+ RuboCop::RakeTask.new(:rubocop) do |task|
10
+ task.options = %w[--display-cop-names --format simple]
11
+ task.patterns = ['lib/**/*.rb', 'spec/**/*.rb', 'Gemfile*', '*.gemspec', 'Rakefile']
12
+ end
13
+
14
+ task default: %i[spec rubocop]
@@ -0,0 +1,26 @@
1
+ require_relative 'lib/d-stream/version'
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = 'd-stream'
5
+ s.version = DStream::VERSION
6
+ s.homepage = 'http://rubygems.org/gems/d-stream'
7
+ s.summary = 'lazy streaming utils'
8
+ s.description = 'D★Stream is a set of utilities for dealing with lazy streams.'
9
+
10
+ s.author = 'Denis Defreyne'
11
+ s.email = 'denis.defreyne@stoneship.org'
12
+ s.license = 'MIT'
13
+
14
+ s.files =
15
+ Dir['[A-Z]*'] +
16
+ Dir['{lib,spec}/**/*'] +
17
+ ['d-stream.gemspec']
18
+ s.require_paths = ['lib']
19
+
20
+ s.rdoc_options = ['--main', 'README.md']
21
+ s.extra_rdoc_files = ['LICENSE', 'README.md', 'NEWS.md']
22
+
23
+ s.required_ruby_version = '~> 2.3'
24
+
25
+ s.add_development_dependency('bundler', '~> 1.14')
26
+ end
@@ -0,0 +1,258 @@
1
+ module DStream
2
+ class Abstract
3
+ def inspect
4
+ "<#{self.class.to_s}>"
5
+ end
6
+ end
7
+
8
+ class SimpleB < Abstract
9
+ def initialize(sym, &block)
10
+ @sym = sym
11
+ @block = block
12
+ end
13
+
14
+ def inspect
15
+ "<#{self.class.to_s} #{@sym.inspect}>"
16
+ end
17
+
18
+ def apply(s)
19
+ s.to_enum.lazy.__send__(@sym, &@block)
20
+ end
21
+ end
22
+
23
+ class Simple1 < Abstract
24
+ def initialize(sym, arg)
25
+ @sym = sym
26
+ @arg = arg
27
+ end
28
+
29
+ def apply(s)
30
+ s.to_enum.lazy.__send__(@sym, @arg)
31
+ end
32
+ end
33
+
34
+ class Scan < Abstract
35
+ def initialize(init, &block)
36
+ @init = init
37
+ @block = block
38
+ end
39
+
40
+ def apply(s)
41
+ Enumerator.new do |y|
42
+ acc = @init
43
+
44
+ s.each do |e|
45
+ acc = @block.call(acc, e)
46
+ y << acc
47
+ end
48
+ end.lazy
49
+ end
50
+ end
51
+
52
+ class Trickle < Abstract
53
+ def initialize(rate)
54
+ @rate = rate
55
+ end
56
+
57
+ def apply(s)
58
+ q = SizedQueue.new(1)
59
+ stop = Object.new
60
+
61
+ t =
62
+ Thread.new do
63
+ Thread.current.abort_on_exception = true
64
+ s.each do |e|
65
+ q << e
66
+ sleep(1.0 / @rate)
67
+ end
68
+ q << stop
69
+ end
70
+
71
+ Enumerator.new do |y|
72
+ loop do
73
+ e = q.pop
74
+ break if stop.equal?(e)
75
+ y << e
76
+ end
77
+ t.join
78
+ end.lazy
79
+ end
80
+ end
81
+
82
+ class Burst < Abstract
83
+ def initialize(delay, size)
84
+ @delay = delay
85
+ @size = size
86
+ end
87
+
88
+ def apply(s)
89
+ q = SizedQueue.new(1)
90
+ stop = Object.new
91
+
92
+ t =
93
+ Thread.new do
94
+ Thread.current.abort_on_exception = true
95
+ i = 0
96
+ s.each do |e|
97
+ sleep(@delay) if (i % @size).zero?
98
+ q << e
99
+ i += 1
100
+ end
101
+ q << stop
102
+ end
103
+
104
+ Enumerator.new do |y|
105
+ loop do
106
+ e = q.pop
107
+ break if stop.equal?(e)
108
+ y << e
109
+ end
110
+ t.join
111
+ end.lazy
112
+ end
113
+ end
114
+
115
+ class Buffer < Abstract
116
+ def initialize(size)
117
+ @size = size
118
+ end
119
+
120
+ def apply(s)
121
+ q = SizedQueue.new(@size)
122
+ stop = Object.new
123
+
124
+ t =
125
+ Thread.new do
126
+ Thread.current.abort_on_exception = true
127
+ s.each { |e| q << e }
128
+ q << stop
129
+ end
130
+
131
+ Enumerator.new do |y|
132
+ loop do
133
+ e = q.pop
134
+ break if stop.equal?(e)
135
+ y << e
136
+ end
137
+ t.join
138
+ end.lazy
139
+ end
140
+ end
141
+
142
+ class WithNext < Abstract
143
+ def apply(s)
144
+ Enumerator.new do |y|
145
+ prev = nil
146
+ have_prev = false
147
+
148
+ s.each do |e|
149
+ if have_prev
150
+ y << [prev, e]
151
+ else
152
+ have_prev = true
153
+ end
154
+ prev = e
155
+ end
156
+ y << [prev, nil]
157
+ end.lazy
158
+ end
159
+ end
160
+
161
+ class Flatten2 < Abstract
162
+ def apply(s)
163
+ Enumerator.new do |y|
164
+ s.each do |es|
165
+ es.each { |e| y << e }
166
+ end
167
+ end.lazy
168
+ end
169
+ end
170
+
171
+ class Zip < Abstract
172
+ def initialize(other)
173
+ @other = other
174
+ end
175
+
176
+ def inspect
177
+ "<DStream::Zip #{@other.inspect}>"
178
+ end
179
+
180
+ def apply(s)
181
+ Enumerator.new do |y|
182
+ s.lazy.zip(@other).each do |e, i|
183
+ y << [e, i]
184
+ end
185
+ end.lazy
186
+ end
187
+ end
188
+
189
+ class Compose < Abstract
190
+ def initialize(procs)
191
+ @procs = procs
192
+ end
193
+
194
+ def inspect
195
+ "<DStream::Compose #{@procs.map(&:inspect).join(' -> ')}>"
196
+ end
197
+
198
+ def apply(s)
199
+ @procs.inject(s) { |acc, pr| pr.apply(acc) }
200
+ end
201
+ end
202
+
203
+ def self.map(&block)
204
+ SimpleB.new(:map, &block)
205
+ end
206
+
207
+ def self.trickle(rate)
208
+ Trickle.new(rate)
209
+ end
210
+
211
+ def self.burst(rate, size)
212
+ Burst.new(rate, size)
213
+ end
214
+
215
+ def self.buffer(size)
216
+ Buffer.new(size)
217
+ end
218
+
219
+ def self.with_next
220
+ WithNext.new
221
+ end
222
+
223
+ def self.select(&block)
224
+ SimpleB.new(:select, &block)
225
+ end
226
+
227
+ def self.reduce(&block)
228
+ SimpleB.new(:reduce, &block)
229
+ end
230
+
231
+ def self.scan(init, &block)
232
+ Scan.new(init, &block)
233
+ end
234
+
235
+ def self.flatten2
236
+ Flatten2.new
237
+ end
238
+
239
+ def self.take(n)
240
+ Simple1.new(:take, n)
241
+ end
242
+
243
+ def self.chunk_by
244
+ # …
245
+ end
246
+
247
+ def self.zip(other)
248
+ Zip.new(other)
249
+ end
250
+
251
+ def self.compose(*procs)
252
+ Compose.new(procs)
253
+ end
254
+
255
+ def self.apply(s, *procs)
256
+ compose(*procs).apply(s)
257
+ end
258
+ end
@@ -0,0 +1,3 @@
1
+ module DStream
2
+ VERSION = '0.0.1'.freeze
3
+ end
@@ -0,0 +1,2 @@
1
+ describe DStream do
2
+ end
@@ -0,0 +1,3 @@
1
+ require 'rspec'
2
+
3
+ require 'd-stream'
metadata ADDED
@@ -0,0 +1,74 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: d-stream
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Denis Defreyne
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-05-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.14'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.14'
27
+ description: D★Stream is a set of utilities for dealing with lazy streams.
28
+ email: denis.defreyne@stoneship.org
29
+ executables: []
30
+ extensions: []
31
+ extra_rdoc_files:
32
+ - LICENSE
33
+ - README.md
34
+ - NEWS.md
35
+ files:
36
+ - Gemfile
37
+ - Gemfile.lock
38
+ - Guardfile
39
+ - LICENSE
40
+ - NEWS.md
41
+ - README.md
42
+ - Rakefile
43
+ - d-stream.gemspec
44
+ - lib/d-stream.rb
45
+ - lib/d-stream/version.rb
46
+ - spec/d-stream/main_spec.rb
47
+ - spec/spec_helper.rb
48
+ homepage: http://rubygems.org/gems/d-stream
49
+ licenses:
50
+ - MIT
51
+ metadata: {}
52
+ post_install_message:
53
+ rdoc_options:
54
+ - "--main"
55
+ - README.md
56
+ require_paths:
57
+ - lib
58
+ required_ruby_version: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - "~>"
61
+ - !ruby/object:Gem::Version
62
+ version: '2.3'
63
+ required_rubygems_version: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ requirements: []
69
+ rubyforge_project:
70
+ rubygems_version: 2.6.12
71
+ signing_key:
72
+ specification_version: 4
73
+ summary: lazy streaming utils
74
+ test_files: []