datapipes 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a09f50e26a2e9da8b66f7bd6904a78e451a8a78a
-  data.tar.gz: 047393dea516f11e7de3ccac83b9a936cd84ed50
+  metadata.gz: 8abb0451500188b58038dbb2731ff2fcb8977f4b
+  data.tar.gz: 7f6348febf36db66e7939bca687b820f5a029fe4
 SHA512:
-  metadata.gz: b2e0b92e8f32a6b45e711d9f129255046b02089e27fbd43370b4e4272e85e50da209b5b4596ad9750bb034dfb424680b0fc6c12098724c3ca991489789646c0e
-  data.tar.gz: aac12fc41369a05128f0bfbfb3e07ad3b4cc95dd450f89d396e5aeaa303da53283ccd4aa65e0d4b42b0b0cc5e8a315e4e2f5a4062bd60746881612f50a8f2010
+  metadata.gz: 43ba18b6df1af0b1118ac7c39c719d0f7d63160cd90247e1e2a0b4dd8baf3992a888fede632491d7f276b949b07910f327f761758f719e4ff36604626e3db385
+  data.tar.gz: 413545e4c611a78e48f98ce0cfb791003b7ae9512b16adee0535159e49cd30368c81b6a2671e7bc62179cd1ae52984de6ebd1846a31a2b9f1973aa90faeb4a6f

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.1.4
+- Add documents.
+
 ## 0.1.3
 - Fix bug with composing sources.
 

data/README.md

@@ -46,6 +46,7 @@ are composable individually. So the diagram above will be:
 ```
 
 ## Installation
+__datapipes requires Ruby >= 2.0.__
 
 Add this line to your application's Gemfile:
 
@@ -63,7 +64,7 @@ Or install it yourself as:
 You have to define your own Source, Tube and Sink. See more in `examples`.
 
 ### Composing objects
-TODO...
+See more in `examples/composing.rb`.
 
 ## Contributing
 

data/examples/composing.rb

@@ -1,17 +1,19 @@
 require 'datapipes'
 
-$: << File.expand_path('../lib', __FILE__)
+$:.unshift File.expand_path('../lib', __FILE__)
 require 'list'
 require 'long_task'
 require 'mul'
 require 'triple'
 require 'acc'
+require 'reverse_acc'
 
 acc = Acc.new
+rev_acc = ReverseAcc.new
 
 source = List.new + LongTask.new(21..30)
 tube = Mul.new >> Triple.new
-sink = acc
+sink = acc + rev_acc
 
 datapipe = Datapipes.new(
   source,
@@ -22,3 +24,4 @@ datapipe = Datapipes.new(
 datapipe.run_resource
 
 p acc.stock
+p rev_acc.stock

data/examples/lib/reverse_acc.rb

@@ -0,0 +1,11 @@
+class ReverseAcc < Datapipes::Sink
+  attr_reader :stock
+
+  def initialize
+    @stock = []
+  end
+
+  def run(data)
+    @stock.unshift data
+  end
+end

data/lib/datapipes.rb

@@ -9,6 +9,11 @@ require 'datapipes/version'
 Thread.abort_on_exception = true
 
 class Datapipes
+  # Pass datapipes components instances.
+  # Each component can be composed. See detail in examples.
+  #
+  # tube and pipe are optional.
+  # If not given tube, a default tube which takes no effect is used.
   def initialize(source, sink, tube: Tube.new, pipe: Pipe.new)
     @source = source
     @tube = tube
@@ -16,6 +21,11 @@ class Datapipes
     @pipe = pipe
   end
 
+  # Run sources, data flow via pipe, tubes and sinks work.
+  # Everything work with just call this method.
+  #
+  # When all sources finished producing, and all sinks did their jobs,
+  # this method returns.
   def run_resource
     @source.pipe = @pipe
     runners = @source.run_all

data/lib/datapipes/composable.rb

@@ -2,6 +2,8 @@ class Datapipes
   module Composable
     attr_accessor :accumulated
 
+    # Source and Sink can't composed as function composition.
+    # So accumulates composed objects internaly, then use them later.
     def +(op2)
       op1 = self
       op1_acc = (op1.accumulated || [op1])

data/lib/datapipes/pipe.rb

@@ -1,13 +1,26 @@
 class Datapipes
+  # Pipe is not only data pipeline but handling asynchronous.
+  #
+  # You can make your own pipe with database or something, but
+  # becareful this object used in multi thread.
+  #
+  # If you make your own, you can override _initialize_, because
+  # this is not used in internal.
+  #
+  # Then supply _recieve_ and _pull_. _pull_ must cause thread blocking
+  # when it is empty.
   class Pipe
+    # You can override and don't need to call super in sub class.
     def initialize
       @queue ||= Queue.new
     end
 
+    # Emit data to pipe.
     def recieve(data)
       @queue.enq data
     end
 
+    # _pull_ must cause thread blocking when it is empty.
     def pull
       @queue.deq
     end

data/lib/datapipes/sink.rb

@@ -1,9 +1,49 @@
 class Datapipes
   # Build your own sink logic in `run` method.
+  #
+  # Be careful each sinks are executed concurrently.
+  # Avoid race condition in multi sinks.
+  #
+  # This is bad:
+  #
+  #   $shared = []
+  #
+  #   class A < Datapipes::Sink
+  #     def run(data)
+  #       $shared << data
+  #     end
+  #   end
+  #
+  #   class B < Datapipes::Sink
+  #     def run(data)
+  #       $shared << data
+  #     end
+  #   end
+  #
+  # On the other hand, a sink is called serially. So you can
+  # touch shared object in one sink logic.
+  #
+  # This is good:
+  #
+  #   class A < Datapipes::Source
+  #     def initialize
+  #       @shared = []
+  #     end
+  #
+  #     def run(data)
+  #       @shared << data
+  #     end
+  #   end
+  #
   class Sink
     include Composable
 
-    # Run all sinks concurrently.
+    # Override this in sub class
+    def run(data)
+      data
+    end
+
+    # For internal uses.
     def run_all(data)
       @accumulated ||= [self]
       count = Parallel.processor_count

data/lib/datapipes/source.rb

@@ -1,5 +1,4 @@
 class Datapipes
-  #
   # Build your own source logic in `run` method.
   # Use `produce` method to emitt data to pipe.
   #
@@ -7,17 +6,34 @@ class Datapipes
   #     10.times {|i| produce(i) }
   #   end
   #
+  # You can use infinitie stream like:
+  #
+  #   def run
+  #     twitter_client.userstream do |event|
+  #       produce(event)
+  #     end
+  #   end
+  #
   class Source
     include Composable
 
-    attr_accessor :pipe
+    # Override in sub class.
+    def run
+    end
 
+    # For internal used.
+    #
+    # Run accumulated sources which are set by composition.
+    # Each source works in new thread.
     def run_all
       @accumulated ||= [self]
       set_pipe
       @accumulated.map {|s| Thread.new { s.run } }
     end
 
+    # For internal uses. Do not touch.
+    attr_accessor :pipe
+
     private
 
     def produce(data)

data/lib/datapipes/tube.rb

@@ -3,6 +3,9 @@ class Datapipes
   #
   # Build your own tube logic in `run` method.
   class Tube
+    # _>>_ is used to compose tubes. See usage in examples.
+    #
+    # Tube composition satisfies associative law. See more in spec.
     def >>(op2)
       op1 = self
       Tube.new.tap do |o|
@@ -12,6 +15,24 @@ class Datapipes
       end
     end
 
+    # Override this in sub class.
+    #
+    # _run_ recieves any data, so you have to ignore
+    # unexpected data.
+    #
+    #   def run(data)
+    #     if accept? data
+    #       [data, data, data]
+    #     else
+    #       data
+    #     end
+    #   end
+    #
+    #    def accept?(data)
+    #      data.is_a? Integer and data > 3
+    #    end
+    #
+    # Don't forget to return data in else clause.
     def run(data)
       data
     end

data/lib/datapipes/version.rb

@@ -1,3 +1,3 @@
 class Datapipes
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

data/spec/basics_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-$: << File.expand_path('../../examples/lib', __FILE__)
+$:.unshift File.expand_path('../../examples/lib', __FILE__)
 require 'list'
 require 'triple'
 require 'print'

data/spec/composing_spec.rb

@@ -1,27 +1,33 @@
 require 'spec_helper'
 
-$: << File.expand_path('../../examples/lib', __FILE__)
+$:.unshift File.expand_path('../../examples/lib', __FILE__)
 require 'list'
 require 'long_task'
 require 'mul'
 require 'triple'
 require 'acc'
+require 'reverse_acc'
 
 describe 'composability' do
   let(:acc) { Acc.new }
+  let(:rev_acc) { ReverseAcc.new }
+
   let(:source) { List.new + LongTask.new(21..30) }
   let(:tube) { Mul.new >> Triple.new }
+  let(:sink) { acc + rev_acc }
 
   let(:datapipe) do
     Datapipes.new(
       source,
-      acc,
+      sink,
       tube: tube
     )
   end
 
   it 'runs without errors' do
     datapipe.run_resource
+
     expect(acc.stock).to have(20).items
+    expect(rev_acc.stock).to have(20).items
   end
 end

data/spec/sink_spec.rb

@@ -7,7 +7,7 @@ describe Datapipes::Sink do
 
       it 'composes' do
         subject.run_all(5)
-        expect(list).to eq [8, 20, 6]
+        expect(list).to have(3).items
       end
     end
 
@@ -16,7 +16,7 @@ describe Datapipes::Sink do
 
       it 'composes' do
         subject.run_all(5)
-        expect(list).to eq [8, 20, 6, 50]
+        expect(list).to have(4).items
       end
     end
 
@@ -25,11 +25,18 @@ describe Datapipes::Sink do
       let(:b) { sink_a + (sink_b + sink_c) }
 
       it 'keeps' do
-        expect(a.run_all(3)).to eq b.run_all(3)
+        a.run_all(3)
+        size_a = list.size
+        list.clear
+
+        b.run_all(3)
+        size_b = list.size
+
+        expect(size_a).to eq size_b
       end
     end
 
-    let(:list) { [] }
+    let(:list) { Queue.new }
 
     let(:sink_a) do
       list_ = list

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datapipes
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Taiki ONO
@@ -132,6 +132,7 @@ files:
 - examples/lib/long_task.rb
 - examples/lib/mul.rb
 - examples/lib/print.rb
+- examples/lib/reverse_acc.rb
 - examples/lib/triple.rb
 - lib/datapipes.rb
 - lib/datapipes/composable.rb