d-stream 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: '0398736030470a1452a92ff161692a046b783399'
+  data.tar.gz: 9d19b1c1af96896470b800e8fcfa6ba438716526
+SHA512:
+  metadata.gz: 8a1bf1c8937184aa7eeb5c89be8359a7e252d6ec617c47f4e61611576b9740f56cc9736569aa55dc37d44bb6f0efeb94f17dc52fa9a2d2e62bdb791263e27d02
+  data.tar.gz: 2182ccd0402d0dcd5522e76a44c09b77e6c58ec1306d08353f644e75b236219bb27b8390d823eae7280b297509d47183ea6c6716a606bb11c9491acf808c5c82

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'rspec'
+gem 'rubocop'

data/Gemfile.lock

@@ -0,0 +1,49 @@
+PATH
+  remote: .
+  specs:
+    d-stream (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.3.0)
+    diff-lcs (1.3)
+    parser (2.4.0.0)
+      ast (~> 2.2)
+    powerpack (0.1.1)
+    rainbow (2.2.2)
+      rake
+    rake (12.0.0)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    rubocop (0.48.1)
+      parser (>= 2.3.3.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.8.1)
+    unicode-display_width (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.14)
+  d-stream!
+  rspec
+  rubocop
+
+BUNDLED WITH
+   1.14.6

data/Guardfile

@@ -0,0 +1,3 @@
+guard 'rake', task: 'default' do
+  watch(%r{^(lib|spec)/})
+end

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 Denis Defreyne and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/NEWS.md

@@ -0,0 +1,5 @@
+# D★Stream news
+
+## 0.0.1 (2017-05-05)
+
+Initial release.

data/README.md

@@ -0,0 +1,240 @@
+# D★Stream
+
+[![Gem version](http://img.shields.io/gem/v/d-stream.svg)](http://rubygems.org/gems/d-stream)
+[![Build status](http://img.shields.io/travis/ddfreyne/d-stream.svg)](https://travis-ci.org/ddfreyne/d-stream)
+[![Code Climate](http://img.shields.io/codeclimate/github/ddfreyne/d-stream.svg)](https://codeclimate.com/github/ddfreyne/d-stream)
+[![Code Coverage](http://img.shields.io/codecov/c/github/ddfreyne/d-stream.svg)](https://codecov.io/github/ddfreyne/d-stream)
+
+_D★Stream_ is a set of extensions for writing stream-processing code in Ruby.
+
+**CAUTION:** D★Stream is work in progress, and pre-alpha quality.
+
+## Examples
+
+### Example 1: straightforward
+
+The following example takes a sequence of events for a given ticket, and calculates the history for that ticket, using slowly changing dimensions:
+
+```ruby
+events =
+  Enumerator.new do |y|
+    y << { id: 40562348, at: Time.now - 400, status: 'new' }
+    y << { id: 40564682, at: Time.now - 300, assignee_id: 2 }
+    y << { id: 40565795, at: Time.now - 250, priority: 'high' }
+    y << { id: 40569932, at: Time.now - 100, status: 'solved' }
+  end.lazy
+
+S = DStream
+
+indices = (1..(1.0 / 0.0))
+
+history =
+  S.apply(
+    events,
+
+    # calculate new state
+    S.scan({}, &:merge),
+
+    # add `version`
+    S.zip(indices),
+    S.map { |(e, i)| e.merge(version: i) },
+
+    # remove `id`
+    S.map { |e| e.reject { |k, _v| k == :id } },
+
+    # add `valid_to` and `valid_from`, and remove `at`
+    S.with_next,
+    S.map { |(a, b)| a.merge(valid_to: b ? b.fetch(:at) : nil) },
+    S.map { |e| e.merge(valid_from: e.fetch(:at)) },
+    S.map { |e| e.reject { |k, _v| k == :at } },
+
+    # add `row_is_current`
+    S.with_next,
+    S.map { |(a, b)| a.merge(row_is_current: b.nil?) },
+  )
+
+history.each { |e| p e }
+```
+
+The output is as follows:
+
+```
+{
+  :status=>"new",
+  :valid_from=>2017-05-05 20:18:14 +0200,
+  :valid_to=>2017-05-05 20:19:54 +0200,
+  :version=>1,
+  :row_is_current=>false
+}
+{
+  :status=>"new",
+  :assignee_id=>2,
+  :valid_from=>2017-05-05 20:19:54 +0200,
+  :valid_to=>2017-05-05 20:20:44 +0200,
+  :version=>2,
+  :row_is_current=>false
+}
+{
+  :status=>"new",
+  :assignee_id=>2,
+  :priority=>"high",
+  :valid_from=>2017-05-05 20:20:44 +0200,
+  :valid_to=>2017-05-05 20:23:14 +0200,
+  :version=>3,
+  :row_is_current=>false
+}
+{
+  :status=>"solved",
+  :assignee_id=>2,
+  :priority=>"high",
+  :valid_from=>2017-05-05 20:23:14 +0200,
+  :valid_to=>nil,
+  :version=>4,
+  :row_is_current=>true
+}
+```
+
+### Example 2: better factored
+
+This example is functionally identical to the one above, but uses `S.compose` in order to make the final process, `history_builder`, easier to understand.
+
+```ruby
+events =
+  Enumerator.new do |y|
+    y << { id: 40562348, at: Time.now - 400, status: 'new' }
+    y << { id: 40564682, at: Time.now - 300, assignee_id: 2 }
+    y << { id: 40565795, at: Time.now - 250, priority: 'high' }
+    y << { id: 40569932, at: Time.now - 100, status: 'solved' }
+  end.lazy
+
+S = DStream
+
+merge =
+  S.scan({}, &:merge),
+
+indices = (1..(1.0 / 0.0))
+add_version =
+  S.compose(
+    S.zip(indices),
+    S.map { |(e,i)| e.merge(version: i) },
+  )
+
+remove_id =
+  S.map { |e| e.reject { |k, _v| k == :id } }
+
+add_valid_dates =
+  S.compose(
+    S.with_next,
+    S.map { |(a,b)| a.merge(valid_to: b ? b.fetch(:at) : nil) },
+    S.map { |e| e.merge(valid_from: e.fetch(:at)) },
+    S.map { |e| e.reject { |k, _v| k == :at } },
+  )
+
+add_row_is_current =
+  S.compose(
+    S.with_next,
+    S.map { |(a,b)| a.merge(row_is_current: b.nil?) },
+  )
+
+history_builder =
+  S.compose(
+    merge,
+    add_version,
+    remove_id,
+    add_valid_dates,
+    add_row_is_current,
+  )
+
+history = S.apply(events, history_builder)
+
+history.each { |h| p h }
+```
+
+## API
+
+The following functions create individual processors:
+
+* `map(&block)` (similar to `Enumerable#map`)
+
+    ```ruby
+    S.apply((1..5), S.map(&:odd?)).to_a
+    # => [true, false, true, false, true]
+    ```
+
+* `select(&block)` (similar to `Enumerable#select`)
+
+    ```ruby
+    S.apply((1..5), S.select(&:odd?)).to_a
+    # => [1, 3, 5]
+    ```
+
+* `reduce(&block)` (similar to `Enumerable#reduce`)
+
+    ```ruby
+    S.apply((1..5), S.reduce(&:+))
+    # => 15
+    ```
+
+* `take(n)` (similar to `Enumerable#take`)
+
+    ```ruby
+    S.apply((1..10), S.take(3)).to_a
+    # => [1, 2, 3]
+    ```
+
+* `zip(other)` (similar to `Enumerable#zip`):
+
+    ```ruby
+    S.apply((1..3), S.zip((10..13))).to_a
+    # => [[1, 10], [2, 11], [3, 12]]
+    ```
+
+* `buffer(size)` yields each stream element, but keeps an internal buffer of not-yet-yielded stream elements. This is useful when reading from a slow and bursty data source, such as a paginated HTTP API.
+
+* `with_next` yields an array containing the stream element and the next stream element, or nil when the end of the stream is reached:
+
+    ```ruby
+    S.apply((1..5), S.with_next).to_a
+    # => [[1, 2], [2, 3], [3, 4], [4, 5], [5, nil]]
+    ```
+
+* `scan(init, &block)` is similar to `reduce`, but rather than returning a single aggregated value, returns all intermediate aggregated values:
+
+    ```ruby
+    S.apply((1..5), S.scan(0, &:+)).to_a
+    # => [1, 3, 6, 10, 15]
+    ```
+
+* `flatten2` yields the stream element if it is not an array, otherwise yields the stream element array’s contents:
+
+    ```ruby
+    S.apply((1..5), S.with_next, S.flatten2).to_a
+    # => [1, 2, 2, 3, 3, 4, 4, 5, 5, nil]
+    ```
+
+To apply one or more processors to a stream, use `.apply`:
+
+```ruby
+S = DStream
+
+stream = ['hi']
+
+S.apply(stream, S.map(&:upcase)).to_a
+# => ["HI"]
+```
+
+To combine one or more processors, use `.compose`:
+
+```ruby
+S = DStream
+
+stream = ['hi']
+
+processor = S.compose(
+  S.map(&:upcase),
+  S.map(&:reverse),
+)
+
+S.apply(stream, processor).to_a
+# => ["IH"]
+```

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '-r ./spec/spec_helper.rb --color'
+  t.verbose = false
+end
+
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.options  = %w[--display-cop-names --format simple]
+  task.patterns = ['lib/**/*.rb', 'spec/**/*.rb', 'Gemfile*', '*.gemspec', 'Rakefile']
+end
+
+task default: %i[spec rubocop]

data/d-stream.gemspec

@@ -0,0 +1,26 @@
+require_relative 'lib/d-stream/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'd-stream'
+  s.version     = DStream::VERSION
+  s.homepage    = 'http://rubygems.org/gems/d-stream'
+  s.summary     = 'lazy streaming utils'
+  s.description = 'D★Stream is a set of utilities for dealing with lazy streams.'
+
+  s.author  = 'Denis Defreyne'
+  s.email   = 'denis.defreyne@stoneship.org'
+  s.license = 'MIT'
+
+  s.files =
+    Dir['[A-Z]*'] +
+    Dir['{lib,spec}/**/*'] +
+    ['d-stream.gemspec']
+  s.require_paths = ['lib']
+
+  s.rdoc_options     = ['--main', 'README.md']
+  s.extra_rdoc_files = ['LICENSE', 'README.md', 'NEWS.md']
+
+  s.required_ruby_version = '~> 2.3'
+
+  s.add_development_dependency('bundler', '~> 1.14')
+end

data/lib/d-stream.rb

@@ -0,0 +1,258 @@
+module DStream
+  class Abstract
+    def inspect
+      "<#{self.class.to_s}>"
+    end
+  end
+
+  class SimpleB < Abstract
+    def initialize(sym, &block)
+      @sym = sym
+      @block = block
+    end
+
+    def inspect
+      "<#{self.class.to_s} #{@sym.inspect}>"
+    end
+
+    def apply(s)
+      s.to_enum.lazy.__send__(@sym, &@block)
+    end
+  end
+
+  class Simple1 < Abstract
+    def initialize(sym, arg)
+      @sym = sym
+      @arg = arg
+    end
+
+    def apply(s)
+      s.to_enum.lazy.__send__(@sym, @arg)
+    end
+  end
+
+  class Scan < Abstract
+    def initialize(init, &block)
+      @init = init
+      @block = block
+    end
+
+    def apply(s)
+      Enumerator.new do |y|
+        acc = @init
+
+        s.each do |e|
+          acc = @block.call(acc, e)
+          y << acc
+        end
+      end.lazy
+    end
+  end
+
+  class Trickle < Abstract
+    def initialize(rate)
+      @rate = rate
+    end
+
+    def apply(s)
+      q = SizedQueue.new(1)
+      stop = Object.new
+
+      t =
+        Thread.new do
+          Thread.current.abort_on_exception = true
+          s.each do |e|
+            q << e
+            sleep(1.0 / @rate)
+          end
+          q << stop
+        end
+
+      Enumerator.new do |y|
+        loop do
+          e = q.pop
+          break if stop.equal?(e)
+          y << e
+        end
+        t.join
+      end.lazy
+    end
+  end
+
+  class Burst < Abstract
+    def initialize(delay, size)
+      @delay = delay
+      @size = size
+    end
+
+    def apply(s)
+      q = SizedQueue.new(1)
+      stop = Object.new
+
+      t =
+        Thread.new do
+          Thread.current.abort_on_exception = true
+          i = 0
+          s.each do |e|
+            sleep(@delay) if (i % @size).zero?
+            q << e
+            i += 1
+          end
+          q << stop
+        end
+
+      Enumerator.new do |y|
+        loop do
+          e = q.pop
+          break if stop.equal?(e)
+          y << e
+        end
+        t.join
+      end.lazy
+    end
+  end
+
+  class Buffer < Abstract
+    def initialize(size)
+      @size = size
+    end
+
+    def apply(s)
+      q = SizedQueue.new(@size)
+      stop = Object.new
+
+      t =
+        Thread.new do
+          Thread.current.abort_on_exception = true
+          s.each { |e| q << e }
+          q << stop
+        end
+
+      Enumerator.new do |y|
+        loop do
+          e = q.pop
+          break if stop.equal?(e)
+          y << e
+        end
+        t.join
+      end.lazy
+    end
+  end
+
+  class WithNext < Abstract
+    def apply(s)
+      Enumerator.new do |y|
+        prev = nil
+        have_prev = false
+
+        s.each do |e|
+          if have_prev
+            y << [prev, e]
+          else
+            have_prev = true
+          end
+          prev = e
+        end
+        y << [prev, nil]
+      end.lazy
+    end
+  end
+
+  class Flatten2 < Abstract
+    def apply(s)
+      Enumerator.new do |y|
+        s.each do |es|
+          es.each { |e| y << e }
+        end
+      end.lazy
+    end
+  end
+
+  class Zip < Abstract
+    def initialize(other)
+      @other = other
+    end
+
+    def inspect
+      "<DStream::Zip #{@other.inspect}>"
+    end
+
+    def apply(s)
+      Enumerator.new do |y|
+        s.lazy.zip(@other).each do |e, i|
+          y << [e, i]
+        end
+      end.lazy
+    end
+  end
+
+  class Compose < Abstract
+    def initialize(procs)
+      @procs = procs
+    end
+
+    def inspect
+      "<DStream::Compose #{@procs.map(&:inspect).join(' -> ')}>"
+    end
+
+    def apply(s)
+      @procs.inject(s) { |acc, pr| pr.apply(acc) }
+    end
+  end
+
+  def self.map(&block)
+    SimpleB.new(:map, &block)
+  end
+
+  def self.trickle(rate)
+    Trickle.new(rate)
+  end
+
+  def self.burst(rate, size)
+    Burst.new(rate, size)
+  end
+
+  def self.buffer(size)
+    Buffer.new(size)
+  end
+
+  def self.with_next
+    WithNext.new
+  end
+
+  def self.select(&block)
+    SimpleB.new(:select, &block)
+  end
+
+  def self.reduce(&block)
+    SimpleB.new(:reduce, &block)
+  end
+
+  def self.scan(init, &block)
+    Scan.new(init, &block)
+  end
+
+  def self.flatten2
+    Flatten2.new
+  end
+
+  def self.take(n)
+    Simple1.new(:take, n)
+  end
+
+  def self.chunk_by
+    # …
+  end
+
+  def self.zip(other)
+    Zip.new(other)
+  end
+
+  def self.compose(*procs)
+    Compose.new(procs)
+  end
+
+  def self.apply(s, *procs)
+    compose(*procs).apply(s)
+  end
+end

data/lib/d-stream/version.rb

@@ -0,0 +1,3 @@
+module DStream
+  VERSION = '0.0.1'.freeze
+end

data/spec/d-stream/main_spec.rb

@@ -0,0 +1,2 @@
+describe DStream do
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'rspec'
+
+require 'd-stream'

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: d-stream
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Denis Defreyne
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+description: D★Stream is a set of utilities for dealing with lazy streams.
+email: denis.defreyne@stoneship.org
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+- NEWS.md
+files:
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- LICENSE
+- NEWS.md
+- README.md
+- Rakefile
+- d-stream.gemspec
+- lib/d-stream.rb
+- lib/d-stream/version.rb
+- spec/d-stream/main_spec.rb
+- spec/spec_helper.rb
+homepage: http://rubygems.org/gems/d-stream
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: lazy streaming utils
+test_files: []