piperator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e41e83c8480b4089277077c8c9e067a973113f2a
+  data.tar.gz: '0780f80b1baf8104a16abd923df4664f909b9756'
+SHA512:
+  metadata.gz: 3b433fd2b72437ac5b02a0905c39fb94d1a4b62a084f4907f427c7103100347d2fdd318588d93ef081bcf1faca37174deca31018e0d720f954950750e14e7018
+  data.tar.gz: 59dcd7ce0c99342b7e02fa14d2705cc05a0abe6c5d8a65511fc632872659e03ca44347589b14e361424b9b50735062d12a4ddc7ebf3cd3eb25e45e7b212c7657

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+before_install: gem install bundler -v 1.14.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in piperator.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Ville Lautanala
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,139 @@
+# Piperator
+
+Pipelines for streaming large collections. The pipeline enables composition of streaming pipelines with lazy enumerables.
+
+The library is heavily inspired by [Elixir pipe operator](https://elixirschool.com/lessons/basics/pipe-operator/) and [Node.js Stream](https://nodejs.org/api/stream.html).
+
+[![Build Status](https://travis-ci.org/lautis/piperator.svg?branch=master)](https://travis-ci.org/lautis/piperator)
+
+
+## Installation
+
+Piperator is distributed as a ruby gem and can be installed with
+
+```
+$ gem install piperator
+```
+
+## Usage
+
+Start by requiring the gem
+
+```ruby
+require 'piperator'
+```
+
+As an appetiser, here's a pipeline that triples all input values and then sums the values.
+
+```ruby
+Piperator.
+  pipe(->(values) { values.lazy.map { |i| i * 3 } }).
+  pipe(->(values) { values.sum }).
+  call([1, 2, 3])
+# => 18
+```
+
+If desired, the input enumerable can also be given as the first pipe.
+
+```ruby
+Piperator.
+  pipe([1, 2, 3]).
+  pipe(->(values) { values.lazy.map { |i| i * 3 } }).
+  pipe(->(values) { values.sum }).
+  call
+# => 18
+```
+
+There is, of course, a much more idiomatic alternative in Ruby:
+
+```ruby
+[1, 2, 3].map { |i| i * 3 }.sum
+```
+
+So why bother?
+
+To run code before the stream processing start and after processing has ended. Let's use the same pattern to calculate the decompressed length of a GZip file fetched over HTTP with streaming.
+
+```ruby
+require 'piperator'
+require 'uri'
+require 'em-http-request'
+require 'net/http'
+
+module HTTPFetch
+  def self.call(url)
+    uri = URI(url)
+    Enumerator.new do |yielder|
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        request = Net::HTTP::Get.new(uri.request_uri)
+        http.request request do |response|
+          response.read_body { |chunk| yielder << chunk }
+        end
+      end
+    end
+  end
+end
+
+module GZipDecoder
+  def self.call(enumerable)
+    Enumerator.new do |yielder|
+      decoder = EventMachine::HttpDecoders::GZip.new do |chunk|
+        yielder << chunk
+      end
+
+      enumerable.each { |chunk| decoder << chunk }
+      yielder << decoder.finalize.to_s
+    end
+  end
+end
+
+length = proc do |enumerable|
+  enumerable.lazy.reduce(0) { |aggregate, chunk| aggregate + chunk.length }
+end
+
+Piperator.
+  pipe(HTTPFetch).
+  pipe(GZipDecoder).
+  pipe(length).
+  call('http://ftp.gnu.org/gnu/gzip/gzip-1.2.4.tar.gz')
+```
+
+At no point is it necessary to keep the full response or decompressed content in memory. This is a huge win when the file sizes grow beyond the 780kB seen in the example.
+
+Pipelines themselves respond to `#call`. This enables using pipelines as pipes in other pipelines.
+
+```ruby
+append_end = proc do |enumerator|
+  Enumerator.new do |yielder|
+    enumerator.each { |item| yielder << item }
+    yielder << 'end'
+  end
+end
+
+prepend_start = proc do |enumerator|
+  Enumerator.new do |yielder|
+    yielder << 'start'
+    enumerator.each { |item| yielder << item }
+  end
+end
+
+double = ->(enumerator) { enumerator.lazy.map { |i| i * 2 } }
+
+prepend_append = Piperator.pipe(prepend_start).pipe(append_end)
+Piperator.pipe(double).pipe(prepend_append).call([1, 2, 3]).to_a
+# => ['start', 2, 4, 6, 'end']
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/lautis/piperator.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'piperator'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/piperator.rb

@@ -0,0 +1,14 @@
+require 'piperator/version'
+require 'piperator/pipeline'
+
+# Top-level shortcuts
+module Piperator
+  # Build a new pipeline from a callable or an enumerable object
+  #
+  # @see Piperator::Pipeline.pipe
+  # @param callable An object responding to call(enumerable)
+  # @return [Pipeline] A pipeline containing only the callable
+  def self.pipe(enumerable)
+    Pipeline.pipe(enumerable)
+  end
+end

data/lib/piperator/pipeline.rb

@@ -0,0 +1,57 @@
+module Piperator
+  # Pipeline is responsible of composition of a lazy enumerable from callables.
+  # It contains a collection of pipes that respond to #call and return a
+  # enumerable.
+  #
+  # For streaming purposes, it usually is desirable to have pipes that takes
+  # a lazy Enumerator as an argument a return a (modified) lazy Enumerator.
+  class Pipeline
+    # Build a new pipeline from a callable or an enumerable object
+    #
+    # @param callable An object responding to call(enumerable)
+    # @return [Pipeline] A pipeline containing only the callable
+    def self.pipe(callable)
+      if callable.respond_to?(:call)
+        Pipeline.new([callable])
+      else
+        Pipeline.new([->(_) { callable }])
+      end
+    end
+
+    # Returns enumerable given as an argument without modifications. Usable when
+    # Pipeline is used as an identity transformation.
+    #
+    # @param enumerable [Enumerable]
+    # @return [Enumerable]
+    def self.call(enumerable = [])
+      enumerable
+    end
+
+    def initialize(pipes = [])
+      @pipes = pipes
+    end
+
+    # Compute the pipeline and return a lazy enumerable with all the pipes.
+    #
+    # @param enumerable Argument passed to the first pipe in the pipeline.
+    # @return [Enumerable] A lazy enumerable containing all the pipes
+    def call(enumerable = [])
+      @pipes.reduce(enumerable) { |pipe, memo| memo.call(pipe) }
+    end
+
+    # Compute the pipeline and strictly evaluate the result
+    #
+    # @return [Array]
+    def to_a(enumerable = [])
+      call(enumerable).to_a
+    end
+
+    # Add a new part to the pipeline
+    #
+    # @param other A pipe to append in pipeline. Responds to #call.
+    # @return [Pipeline] A new pipeline instance
+    def pipe(other)
+      Pipeline.new(@pipes + [other])
+    end
+  end
+end

data/lib/piperator/version.rb

@@ -0,0 +1,3 @@
+module Piperator
+  VERSION = '0.1.0'.freeze
+end

data/piperator.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'piperator/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'piperator'
+  spec.version = Piperator::VERSION
+  spec.authors = ['Ville Lautanala']
+  spec.email = ['lautis@gmail.com']
+
+  spec.summary = 'Composable pipelines for streaming large collections'
+  spec.description = 'Pipelines for streaming large collections with ' \
+                     'composition inspired by Elixir pipes.'
+  spec.homepage = 'https://github.com/lautis/piperator'
+  spec.license = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: piperator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ville Lautanala
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-05-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Pipelines for streaming large collections with composition inspired by
+  Elixir pipes.
+email:
+- lautis@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/piperator.rb
+- lib/piperator/pipeline.rb
+- lib/piperator/version.rb
+- piperator.gemspec
+homepage: https://github.com/lautis/piperator
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Composable pipelines for streaming large collections
+test_files: []