piperator 0.3.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5ce14dd4a45b8ce9bc5453cb30df67c1357c8903
-  data.tar.gz: b3df089403c086b52d8f0a48d2292d9f06495c51
+SHA256:
+  metadata.gz: 1475b2b74289ce3c32f638a67156b943101bff0b548283e2e1ea635d5048a6e5
+  data.tar.gz: 3d06228a633975fde9409387471bb505eb192971c8e50e7ce0df34bbc74e2ae1
 SHA512:
-  metadata.gz: e93f487947ca4c0bacc904b3b04d1e90a245a3bc2efa68bb2b461f3df0427b6e34182cfc01c8cb50be9ae79906de91848ee70102687b55b31d769ed4938f40d9
-  data.tar.gz: 0a2e61d19a4533ac76bb302f6fbc505f7a8e4cf5e97f0652ab9bd6b8f34aff1e79e22d271c12656a7e7a0b0cb460be2f6f3e681bbdf6e31d4c55c06f2296499c
+  metadata.gz: 9a11be79d16a0a54c21cb62518ce6426a184c9c9d6609dc6424533413d6cee875f57d74411d92b39dc57fbf9d25d5e712b112149d7bc16d3f42a6f972e52284a
+  data.tar.gz: e2cec664b6832086b53cdaa200b03852f6dcd62cb4f3c5b18532ea636054b0c599471d84a181857e36919cea400a68f1e6fdd93bdeb03e9f612107acdcd4768f

data/.travis.yml

@@ -1,5 +1,11 @@
 sudo: false
+cache: bundler
 language: ruby
 rvm:
-  - 2.4.0
-before_install: gem install bundler -v 1.14.6
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
+  - 2.7
+  - jruby
+  - truffleruby

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 1.1.0 (6 December 2019)
+
+- add `#pos` to Piperator::IO to get current position
+
+## 1.0.0. (13 October 2019)
+
+- add `Piperator.build` to build pipelines with DSL
+
 ## 0.3.0 (13 July 2017)
 
 - remove implicit wrapping to callable from `Pipeline.pipe`

data/README.md

@@ -6,6 +6,20 @@ The library is heavily inspired by [Elixir pipe operator](https://elixirschool.c
 
 [![Build Status](https://travis-ci.org/lautis/piperator.svg?branch=master)](https://travis-ci.org/lautis/piperator)
 
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+- [Installation](#installation)
+- [Usage](#usage)
+  - [Pipelines](#pipelines)
+  - [Enumerators as IO objects](#enumerators-as-io-objects)
+- [Development](#development)
+- [Related Projects](#related-projects)
+- [Contributing](#contributing)
+- [License](#license)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
 ## Installation
 
@@ -35,7 +49,16 @@ Piperator.
 # => 18
 ```
 
-If desired, the input enumerable can also be given as the first elmenent of the pipeline using `Piperator.wrap`.
+The same could also be achieved using DSL instead of method chaining:
+
+```ruby
+Piperator.build do
+  pipe(->(values) { values.lazy.map { |i| i * 3 } })
+  pipe(->(values) { values.sum })
+end.call([1, 2, 3])
+```
+
+If desired, the input enumerable can also be given as the first element of the pipeline using `Piperator.wrap`.
 
 ```ruby
 Piperator.
@@ -46,6 +69,18 @@ Piperator.
 # => 18
 ```
 
+Have reasons to defer constructing a pipe? Evaluate it lazily:
+
+```ruby
+summing = ->(values) { values.sum }
+Piperator.build
+  pipe(->(values) { values.lazy.map { |i| i * 3 } })
+  lazy do
+    summing
+  end
+end.call([1, 2, 3])
+```
+
 There is, of course, a much more idiomatic alternative in Ruby:
 
 ```ruby
@@ -156,6 +191,12 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Related Projects
+
+* [D★Stream](https://github.com/ddfreyne/d-stream) - Set of extensions for writing stream-processing code.
+* [ddbuffer](https://github.com/ddfreyne/ddbuffer) - Buffer enumerables/enumerators.
+* [Down::ChunkedIO](https://github.com/janko-m/down/blob/master/lib/down/chunked_io.rb) - A similar IO class as Piperator::IO.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/lautis/piperator.

data/lib/piperator/builder.rb

@@ -0,0 +1,54 @@
+module Piperator
+  # Builder is used to provide DSL-based Pipeline building. Using Builder,
+  # Pipelines can be built without pipe chaining, which might be easier if
+  # some steps need to be included only on specific conditions.
+  #
+  # @see Piperator.build
+  class Builder
+    # Expose a chained method in Pipeline in DSL
+    #
+    # @param method_name Name of method in Pipeline
+    # @see Pipeline
+    #
+    # @!macro [attach] dsl_method
+    #   @method $1
+    #   Call Pipeline#$1 given arguments and use the return value as builder state.
+    #
+    #   @see Pipeline.$1
+    def self.dsl_method(method_name)
+      define_method(method_name) do |*arguments, &block|
+        @pipeline = @pipeline.send(method_name, *arguments, &block)
+      end
+    end
+
+    dsl_method :lazy
+    dsl_method :pipe
+    dsl_method :wrap
+
+    def initialize(saved_binding, pipeline = Pipeline.new)
+      @pipeline = pipeline
+      @saved_binding = saved_binding
+    end
+
+    # Return build pipeline
+    #
+    # @return [Pipeline]
+    def to_pipeline
+      @pipeline
+    end
+
+    private
+
+    def method_missing(method_name, *arguments, &block)
+      if @saved_binding.receiver.respond_to?(method_name, true)
+        @saved_binding.receiver.send(method_name, *arguments, &block)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      @saved_binding.receiver.respond_to?(method_name, include_private) || super
+    end
+  end
+end

data/lib/piperator/io.rb

@@ -6,22 +6,24 @@ module Piperator
     FLUSH_THRESHOLD = 128 * 1028 # 128KiB
 
     attr_reader :eof
+    attr_reader :pos
 
     def initialize(enumerator, flush_threshold: FLUSH_THRESHOLD)
       @enumerator = enumerator
       @flush_threshold = flush_threshold
-      @io = StringIO.new
-      @buffer_start_pos = 0
-      @io_read_pos = 0
+      @buffer = StringIO.new
+      @pos = 0
+      @buffer_read_pos = 0
       @eof = false
     end
 
     alias eof? eof
+    alias tell pos
 
     # Return the first bytes of the buffer without marking the buffer as read.
     def peek(bytes)
       while @eof == false && readable_bytes < (bytes || 1)
-        @io.write(@enumerator.next)
+        @buffer.write(@enumerator.next)
       end
       peek_buffer(bytes)
     rescue StopIteration
@@ -38,22 +40,22 @@ module Piperator
     def gets(separator = $INPUT_RECORD_SEPARATOR, _limit = nil)
       while !@eof && !contains_line?(separator)
         begin
-          @io.write(@enumerator.next)
+          @buffer.write(@enumerator.next)
         rescue StopIteration
           @eof = true
           nil
         end
       end
-      read_with { @io.gets(separator) }
+      read_with { @buffer.gets(separator) }
     end
 
     # Flush internal buffer until the last unread byte
     def flush
-      if @io.pos == @io_read_pos
+      if @buffer.pos == @buffer_read_pos
         initialize_buffer
       else
-        @io.pos = @io_read_pos
-        initialize_buffer(@io.read)
+        @buffer.pos = @buffer_read_pos
+        initialize_buffer(@buffer.read)
       end
     end
 
@@ -62,56 +64,56 @@ module Piperator
     # @param length [Integer] number of bytes to read
     # @return String
     def read(length = nil)
-      return @enumerator.next if length.nil? && readable_bytes.zero?
-      @io.write(@enumerator.next) while !@eof && readable_bytes < (length || 1)
-      read_with { @io.read(length) }
+      return @enumerator.next.tap { |e| @pos += e.bytesize } if length.nil? && readable_bytes.zero?
+      @buffer.write(@enumerator.next) while !@eof && readable_bytes < (length || 1)
+      read_with { @buffer.read(length) }
     rescue StopIteration
       @eof = true
-      read_with { @io.read(length) } if readable_bytes > 0
+      read_with { @buffer.read(length) } if readable_bytes > 0
     end
 
     # Current buffer size - including non-freed read content
     #
     # @return [Integer] number of bytes stored in buffer
     def used
-      @io.size
+      @buffer.size
     end
 
     private
 
     def readable_bytes
-      @io.pos - @io_read_pos
+      @buffer.pos - @buffer_read_pos
     end
 
     def read_with
-      pos = @io.pos
-      @io.pos = @io_read_pos
+      pos = @buffer.pos
+      @buffer.pos = @buffer_read_pos
 
       yield.tap do |data|
-        @io_read_pos += data.bytesize if data
-        @io.pos = pos
+        @buffer_read_pos += data.bytesize if data
+        @buffer.pos = pos
         flush if flush?
       end
     end
 
     def peek_buffer(bytes)
-      @io.string.byteslice(@io_read_pos...@io_read_pos + bytes)
+      @buffer.string.byteslice(@buffer_read_pos...@buffer_read_pos + bytes)
     end
 
     def flush?
-      @io.pos == @io_read_pos || @io.pos > @flush_threshold
+      @buffer.pos == @buffer_read_pos || @buffer.pos > @flush_threshold
     end
 
     def initialize_buffer(data = nil)
-      @io_read_pos = 0
-      @buffer_start_pos += @io.pos if @io
-      @io = StringIO.new
-      @io.write(data) if data
+      @pos += @buffer_read_pos
+      @buffer_read_pos = 0
+      @buffer = StringIO.new
+      @buffer.write(data) if data
     end
 
     def contains_line?(separator = $INPUT_RECORD_SEPARATOR)
       return true if @eof
-      @io.string.byteslice(@io_read_pos..-1).include?(separator)
+      @buffer.string.byteslice(@buffer_read_pos..-1).include?(separator)
     rescue ArgumentError # Invalid UTF-8
       false
     end

data/lib/piperator/pipeline.rb

@@ -6,6 +6,20 @@ module Piperator
   # For streaming purposes, it usually is desirable to have pipes that takes
   # a lazy Enumerator as an argument a return a (modified) lazy Enumerator.
   class Pipeline
+    # Build a new pipeline from a lazily evaluated callable or an enumerable
+    # object.
+    #
+    # @param block A block returning a callable(enumerable)
+    # @return [Pipeline] A pipeline containing only the lazily evaluated
+    # callable.
+    def self.lazy(&block)
+      callable = nil
+      Pipeline.new([lambda do |e|
+        callable ||= block.call
+        callable.call(e)
+      end])
+    end
+
     # Build a new pipeline from a callable or an enumerable object
     #
     # @param callable An object responding to call(enumerable)
@@ -59,6 +73,19 @@ module Piperator
       call(enumerable).to_a
     end
 
+    # Add a new lazily evaluated part to the pipeline.
+    #
+    # @param block A block returning a callable(enumerable) to append in
+    # pipeline.
+    # @return [Pipeline] A new pipeline instance
+    def lazy(&block)
+      callable = nil
+      Pipeline.new(@pipes + [lambda do |e|
+        callable ||= block.call
+        callable.call(e)
+      end])
+    end
+
     # Add a new part to the pipeline
     #
     # @param other A pipe to append in pipeline. Responds to #call.

data/lib/piperator/version.rb

@@ -1,3 +1,4 @@
 module Piperator
-  VERSION = '0.3.0'.freeze
+  # Piperator version
+  VERSION = '1.2.0'.freeze
 end

data/lib/piperator.rb

@@ -1,13 +1,43 @@
 require 'piperator/version'
 require 'piperator/pipeline'
 require 'piperator/io'
+require 'piperator/builder'
 
 # Top-level shortcuts
 module Piperator
+  # Build a new pipeline using DSL. This enables easy control of the pipeline
+  # stucture.
+  #
+  #   Piperator.build do
+  #     wrap [1, 2, 3]
+  #     pipe(-> (enumerable) { enumerable.map { |i| i + 1 } })
+  #   end
+  #   # => Pipeline that returns [2, 3, 4] called
+  #
+  #   # Alternatively, the Builder is also given as argument to the block
+  #   Piperator.build do |p|
+  #     p.wrap [1, 2, 3]
+  #     p.pipe(-> (enumerable) { enumerable.map { |i| i + 1 } })
+  #   end
+  #   # This is similar, but allows access to instance variables.
+  #
+  # @return [Pipeline] Pipeline containing defined steps
+  def self.build(&block)
+    return Pipeline.new unless block_given?
+
+    Builder.new(block&.binding).tap do |builder|
+      if block.arity.positive?
+        yield builder
+      else
+        builder.instance_eval(&block)
+      end
+    end.to_pipeline
+  end
+
   # Build a new pipeline from a callable or an enumerable object
   #
   # @see Piperator::Pipeline.pipe
-  # @param callable An object responding to call(enumerable)
+  # @param enumerable An object responding to call(enumerable)
   # @return [Pipeline] A pipeline containing only the callable
   def self.pipe(enumerable)
     Pipeline.pipe(enumerable)

data/piperator.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'bundler', '>= 1.14'
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
 end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: piperator
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Ville Lautanala
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-13 00:00:00.000000000 Z
+date: 2020-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.14'
 - !ruby/object:Gem::Dependency
@@ -71,6 +71,7 @@ files:
 - bin/console
 - bin/setup
 - lib/piperator.rb
+- lib/piperator/builder.rb
 - lib/piperator/io.rb
 - lib/piperator/pipeline.rb
 - lib/piperator/version.rb
@@ -94,8 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Composable pipelines for streaming large collections