piperator 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e41e83c8480b4089277077c8c9e067a973113f2a
-  data.tar.gz: '0780f80b1baf8104a16abd923df4664f909b9756'
+  metadata.gz: eec6368ee594785c21b367e0575ff8e56b1333c9
+  data.tar.gz: 7f4dc49b0dbab3b79bbd171b0099b7c0d623c157
 SHA512:
-  metadata.gz: 3b433fd2b72437ac5b02a0905c39fb94d1a4b62a084f4907f427c7103100347d2fdd318588d93ef081bcf1faca37174deca31018e0d720f954950750e14e7018
-  data.tar.gz: 59dcd7ce0c99342b7e02fa14d2705cc05a0abe6c5d8a65511fc632872659e03ca44347589b14e361424b9b50735062d12a4ddc7ebf3cd3eb25e45e7b212c7657
+  metadata.gz: a784c23791a28b994d4db90626589478b58dfff806a35f3fc812792ed8c247fef556d5e72b1a2aa7195b97fbc0f73e0deab4d80cf9a03e91e393e1af809bc7e6
+  data.tar.gz: f0e310f29dc9965e0f8a94826e0a66830761d4333aae8884cfb05e64cba35ea359657aa25e8046d37e98e30a1c18029dc670303765cd2f139a75ce9f87faf1fc

data/README.md

@@ -23,7 +23,9 @@ Start by requiring the gem
 require 'piperator'
 ```
 
-As an appetiser, here's a pipeline that triples all input values and then sums the values.
+### Pipelines
+
+As an appetizer, here's a pipeline that triples all input values and then sums the values.
 
 ```ruby
 Piperator.
@@ -124,6 +126,30 @@ Piperator.pipe(double).pipe(prepend_append).call([1, 2, 3]).to_a
 # => ['start', 2, 4, 6, 'end']
 ```
 
+### Enumerators as IO objects
+
+Piperator also provides a helper class that allows `Enumerator`s to be used as
+IO objects. This is useful to provide integration with libraries that work only
+with IO objects such as [Nokogiri](http://www.nokogiri.org) or
+[Oj](https://github.com/ohler55/oj).
+
+An example pipe that would yield all XML node in a document read in streams:
+
+```ruby
+
+require 'nokogiri'
+streaming_xml = lambda do |enumerable|
+  Enumerator.new do |yielder|
+    io = Piperator::IO.new(enumerable.each)
+    reader = Nokogiri::XML::Reader(io)
+    reader.each { |node| yielder << node }
+  end
+end
+```
+
+In real-world scenarios, the pipe would need to filter the nodes. Passing every
+single XML node forward is not that useful.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/piperator.rb

@@ -1,5 +1,6 @@
 require 'piperator/version'
 require 'piperator/pipeline'
+require 'piperator/io'
 
 # Top-level shortcuts
 module Piperator

data/lib/piperator/io.rb

@@ -0,0 +1,119 @@
+require 'English'
+
+module Piperator
+  # Pseudo I/O on Enumerators
+  class IO
+    FLUSH_THRESHOLD = 128 * 1028 # 128KiB
+
+    attr_reader :eof
+
+    def initialize(enumerator, flush_threshold: FLUSH_THRESHOLD)
+      @enumerator = enumerator
+      @flush_threshold = flush_threshold
+      @io = StringIO.new
+      @buffer_start_pos = 0
+      @io_read_pos = 0
+      @eof = false
+    end
+
+    alias eof? eof
+
+    # Return the first bytes of the buffer without marking the buffer as read.
+    def peek(bytes)
+      while @eof == false && readable_bytes < (bytes || 1)
+        @io.write(@enumerator.next)
+      end
+      peek_buffer(bytes)
+    rescue StopIteration
+      @eof = true
+      peek_buffer(bytes)
+    end
+
+    # Reads the next "line" from the I/O stream; lines are separated by
+    # separator.
+    #
+    # @param separator [String] separator to split input
+    # @param _limit Unused parameter for compatiblity
+    # @return [String]
+    def gets(separator = $INPUT_RECORD_SEPARATOR, _limit = nil)
+      while !@eof && !contains_line?(separator)
+        begin
+          @io.write(@enumerator.next)
+        rescue StopIteration
+          @eof = true
+          nil
+        end
+      end
+      read_with { @io.gets(separator) }
+    end
+
+    # Flush internal buffer until the last unread byte
+    def flush
+      if @io.pos == @io_read_pos
+        initialize_buffer
+      else
+        @io.pos = @io_read_pos
+        initialize_buffer(@io.read)
+      end
+    end
+
+    # Reads length bytes from the I/O stream.
+    #
+    # @param length [Integer] number of bytes to read
+    # @return String
+    def read(length = nil)
+      return @enumerator.next if length.nil? && readable_bytes.zero?
+      @io.write(@enumerator.next) while !@eof && readable_bytes < (length || 1)
+      read_with { @io.read(length) }
+    rescue StopIteration
+      @eof = true
+      read_with { @io.read(length) } if readable_bytes > 0
+    end
+
+    # Current buffer size - including non-freed read content
+    #
+    # @return [Integer] number of bytes stored in buffer
+    def used
+      @io.size
+    end
+
+    private
+
+    def readable_bytes
+      @io.pos - @io_read_pos
+    end
+
+    def read_with
+      pos = @io.pos
+      @io.pos = @io_read_pos
+
+      yield.tap do |data|
+        @io_read_pos += data.bytesize if data
+        @io.pos = pos
+        flush if flush?
+      end
+    end
+
+    def peek_buffer(bytes)
+      @io.string.byteslice(@io_read_pos...@io_read_pos + bytes)
+    end
+
+    def flush?
+      @io.pos == @io_read_pos || @io.pos > @flush_threshold
+    end
+
+    def initialize_buffer(data = nil)
+      @io_read_pos = 0
+      @buffer_start_pos += @io.pos if @io
+      @io = StringIO.new
+      @io.write(data) if data
+    end
+
+    def contains_line?(separator = $INPUT_RECORD_SEPARATOR)
+      return true if @eof
+      @io.string.byteslice(@io_read_pos..-1).include?(separator)
+    rescue ArgumentError # Invalid UTF-8
+      false
+    end
+  end
+end

data/lib/piperator/version.rb

@@ -1,3 +1,3 @@
 module Piperator
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.2.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: piperator
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Ville Lautanala
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-23 00:00:00.000000000 Z
+date: 2017-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -70,6 +70,7 @@ files:
 - bin/console
 - bin/setup
 - lib/piperator.rb
+- lib/piperator/io.rb
 - lib/piperator/pipeline.rb
 - lib/piperator/version.rb
 - piperator.gemspec