linepipe 0.1.0

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.ruby-version
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in linepipe.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Wimdu GmbH
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,124 @@
+# Linepipe
+
+A tool to aid in processing data in a pipeline, making every step easily
+testable and benchmarkable.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'linepipe'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install linepipe
+
+## Usage
+
+Linepipe's DSL consists of 4 different parts:
+
+* `setup`: Optional setup that will be run at the beginning.
+* `data`: The input data.
+* `process`: As many of these as you want will conform the steps of your
+  algorithm.
+* `expect`: In development mode, each of these will be run against your final
+  output data to ensure its conformity with your expectations.
+
+While developing a processing algorithm, `Linepipe.develop` is your friend. Each
+`process` block will be reduced against your data in order, and then each
+`expect` block will be run against the final output to ensure that it works.
+
+```ruby
+linepipe = Linepipe.run do
+  data {
+    %w(foo bar baz)
+  }
+
+  process("Upcasing") { |data|
+    data.map(&:upcase)
+  }
+
+  process("Reversing") { |data|
+    data.reverse
+  }
+
+  expect { |data|
+    data == %w(BAZ BAR FOO)
+  }
+end
+
+linepipe.result # => %W(BAZ BAR FOO)
+```
+
+Once you're comfortable with your algorithm, just change your call to
+`Linepipe.develop` to `Linepipe.run` and no expectations will be run.
+
+## Testing your linepipes
+
+`Linepipe.run`, `Linepipe.benchmark` and `Linepipe.develop` return a `Linepipe::Process` object that
+responds to two important methods: `output` and a hash-like
+interface to access each step. In our case above we would access the second step
+"Reversing" (from a test or wherever) like this:
+
+```ruby
+step = linepipe["Reversing"]
+# => #<Step ...>
+expect(step.apply([1,2,3])).to eq([3,2,1])
+# => [3,2,1]
+```
+
+This way you can test every stage of your linepipe separately against as many
+inputs as you want.
+
+## Benchmarking your linepipes
+
+To switch Linepipe into benchmark mode, just call `Linepipe.benchmark` instead
+of `.develop` or `.run`. This will print a detailed benchmark for every step of
+your algorithm so you can easily identify and fix bottlenecks.
+
+```ruby
+linepipe = Linepipe.benchmark(10_000) do
+  data {
+    %w(foo bar baz)
+  }
+
+  process("Upcasing") { |data|
+    data.map(&:upcase)
+  }
+
+  process("Reversing") { |data|
+    data.reverse
+  }
+
+  expect { |data|
+    data == %w(BAZ BAR FOO)
+  }
+end
+```
+
+Will output to the screen:
+
+    Rehearsal ---------------------------------------------
+    Upcasing    0.020000   0.000000   0.020000 (  0.024458)
+    Reversing   0.000000   0.000000   0.000000 (  0.004000)
+    ------------------------------------ total: 0.020000sec
+
+                    user     system      total        real
+    Upcasing    0.020000   0.000000   0.020000 (  0.022565)
+    Reversing   0.010000   0.000000   0.010000 (  0.007034)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+Copyright (c) 2013 Wimdu GmbH (MIT License). See LICENSE.txt for details.

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "bundler/setup"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = ["--color", '--format doc', '--order rand']
+end
+task :default => :spec

data/examples/benchmark.rb

@@ -0,0 +1,23 @@
+require_relative '../lib/linepipe'
+
+Linepipe.benchmark(100_000) do
+  data {
+    %w(foo bar baz)
+  }
+
+  step("Upcasing") { |data|
+    data.map(&:upcase)
+  }
+
+  step("Reversing") { |data|
+    data.reverse
+  }
+
+  step("Sorting") { |data|
+    data.sort
+  }
+
+  expect { |data|
+    data == %w(BAZ BAR FOO)
+  }
+end

data/examples/develop.rb

@@ -0,0 +1,40 @@
+require_relative '../lib/linepipe'
+require 'minitest/autorun'
+
+class DevelopTest < MiniTest::Unit::TestCase
+  def linepipe
+    @linepipe ||= Linepipe.develop do
+      data {
+        %w(foo bar baz)
+      }
+
+      step("Upcasing") { |data|
+        data.map(&:upcase)
+      }
+
+      step("Reversing") { |data|
+        data.reverse
+      }
+
+      step("Sorting") { |data|
+        data.sort
+      }
+
+      expect { |data|
+        data == %w(BAR BAZ FOO)
+      }
+    end
+  end
+
+  def test_upcasing
+    assert_equal %w(A B), linepipe['Upcasing'].apply(%w(a b))
+  end
+
+  def test_reversing
+    assert_equal %w(b a), linepipe['Reversing'].apply(%w(a b))
+  end
+
+  def test_sorting
+    assert_equal %w(a b c), linepipe['Sorting'].apply(%w(c a b))
+  end
+end

data/lib/linepipe/dsl.rb

@@ -0,0 +1,22 @@
+require_relative "expectation"
+require_relative "step"
+
+module Linepipe
+  module DSL
+    def setup(&block)
+      @setup = block
+    end
+
+    def data(data=nil, &block)
+      @data = data ? -> { data } : block
+    end
+
+    def step(name=nil, &block)
+      @steps << Step.new(name, &block)
+    end
+
+    def expect(msg=nil, &block)
+      @expectations << Expectation.new(msg, io, &block)
+    end
+  end
+end

data/lib/linepipe/expectation.rb

@@ -0,0 +1,21 @@
+module Linepipe
+  class Expectation
+    def initialize(msg="Assertion failed", io=STDOUT, &block)
+      @msg   = msg
+      @io    = io
+      @block = block
+    end
+
+    def successful?(data)
+      if !block.call(data)
+        io.puts "Expectation failed at #{block.source_location.join(':')} (#{msg})"
+        return false
+      end
+      true
+    end
+
+    private
+    attr_reader :block, :msg, :io
+  end
+end
+

data/lib/linepipe/process.rb

@@ -0,0 +1,83 @@
+require_relative "dsl"
+
+module Linepipe
+  class Process
+    include DSL
+    attr_reader :output, :steps
+
+    def initialize(io=STDOUT)
+      @data         = []
+      @steps        = []
+      @setup        = nil
+      @output       = nil
+      @expectations = []
+      @io           = io
+    end
+
+    def [](name)
+      steps.detect { |s| s.name == name }
+    end
+
+    def run
+      run_setup
+      @output = steps.reduce(initial_data) { |d, step| step.apply(d) }
+    end
+
+    def develop
+      run_setup
+      @output = steps.to_enum.with_index.reduce(initial_data) { |d, (step, idx)|
+        io.puts "Stage #{idx} #{step.name}"
+        io.puts "Input: #{d}"
+        step.apply(d).tap do |r|
+          io.puts "Output: #{r}"
+        end
+      }
+
+      if expectations.all? { |exp| exp.successful?(output) }
+        io.puts "SUCCESS!"
+      end
+    end
+
+    def benchmark(iterations)
+      require 'benchmark'
+      require 'stringio'
+
+      run_setup
+
+      label_length = steps.map(&:name).map(&:length).max
+
+      out = $stdout
+      $stdout = stringio = StringIO.new
+
+      Benchmark.bmbm(label_length) do |x|
+        @output = steps.reduce(initial_data) { |d, step|
+          result = step.apply(d)
+          x.report(step.name) { iterations.times { step.apply(d) } }
+          result
+        }
+      end
+
+      io.puts stringio.string
+    ensure
+      $stdout = out
+    end
+
+
+    private
+    attr_reader :expectations, :io
+
+    def run_setup
+      @setup.call if @setup
+    end
+
+    def initial_data
+      if @data.is_a?(Proc)
+        @data.call
+      else
+        puts "[Linepipe] Warn: You need to specify an initial data set"
+      end
+    end
+
+  end
+end
+

data/lib/linepipe/step.rb

@@ -0,0 +1,18 @@
+module Linepipe
+  class Step
+    attr_reader :name
+
+    def initialize(name=nil, &block)
+      @name  = name
+      @block = block
+    end
+
+    def apply(data)
+      block.call(data)
+    end
+
+    private
+    attr_reader :block
+  end
+end
+

data/lib/linepipe/version.rb

@@ -0,0 +1,3 @@
+module Linepipe
+  VERSION = "0.1.0"
+end

data/lib/linepipe.rb

@@ -0,0 +1,29 @@
+require_relative "linepipe/version"
+require_relative "linepipe/process"
+
+module Linepipe
+  class << self
+
+    def develop(&block)
+      build_process(block) { |process| process.develop }
+    end
+
+    def run(&block)
+      build_process(block) { |process| process.run }
+    end
+
+    def benchmark(iterations, &block)
+      build_process(block) { |process| process.benchmark(iterations) }
+    end
+
+
+    private
+
+    def build_process(dsl_block, &block)
+      Process.new.tap do |process|
+        process.instance_eval(&dsl_block)
+        yield process
+      end
+    end
+  end
+end

data/linepipe.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'linepipe/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "linepipe"
+  gem.version       = Linepipe::VERSION
+  gem.authors       = ["Josep M. Bach"]
+  gem.email         = ["josep.bach@wimdu.com"]
+  gem.description   = %q{Process data one step at a time.}
+  gem.summary       = %q{Process data one step at a time.}
+  gem.homepage      = "https://github.com/wimdu/linepipe"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency 'rspec'
+end

data/spec/pipeline/expectation_spec.rb

@@ -0,0 +1,35 @@
+require 'rspec'
+require 'linepipe'
+require 'stringio'
+
+module Linepipe
+  describe Expectation, '#successful?' do
+    let(:io) { StringIO.new }
+
+    describe 'when it fails' do
+      let(:expectation) do
+        Expectation.new('Failure message', io) { false }
+      end
+
+      it 'prints the message to the output' do
+        expectation.successful?(%w(some data))
+        expect(io.string).to match(/expectation_spec/)
+        expect(io.string).to match(/Failure message/)
+      end
+
+      it 'returns false' do
+        expect(expectation.successful?(%w(some data))).to be_false
+      end
+    end
+
+    describe 'when it passes' do
+      let(:expectation) do
+        Expectation.new('Failure message', io) { true }
+      end
+
+      it 'returns true' do
+        expect(expectation.successful?(%w(some data))).to be_true
+      end
+    end
+  end
+end

data/spec/pipeline/process_spec.rb

@@ -0,0 +1,99 @@
+require 'rspec'
+require 'linepipe'
+require 'stringio'
+
+module Linepipe
+  describe Process do
+    let(:io) { StringIO.new }
+
+    let(:process) do
+      Process.new(io).tap do |process|
+        process.setup { process.taint }
+        process.data { %w(foo bar baz) }
+        process.step('Upcasing') { |data| data.map(&:upcase) }
+        process.step('Reversing', &:reverse)
+        process.expect { |data| data.first == 'BAZ' }
+      end
+    end
+
+    describe '#[]' do
+      it 'gets a step' do
+        step = process['Upcasing']
+        expect(step.apply(['blah'])).to eq(['BLAH'])
+      end
+    end
+
+    describe '#run' do
+      before do
+        process.run
+      end
+
+      it 'runs the setup' do
+        expect(process).to be_tainted
+      end
+
+      it 'assigns the output' do
+        expect(process.output).to eq(%w(BAZ BAR FOO))
+      end
+    end
+
+    describe '#develop' do
+      it 'runs the setup' do
+        process.develop
+        expect(process).to be_tainted
+      end
+
+      it 'assigns the output' do
+        process.develop
+        expect(process.output).to eq(%w(BAZ BAR FOO))
+      end
+
+      it 'outputs information to the io stream' do
+        process.develop
+        expect(io.string).to match(/Stage 0 Upcasing/)
+        expect(io.string).to match(/Stage 1 Reversing/)
+      end
+
+      describe 'when the expectations pass' do
+        it 'outputs SUCCESS' do
+          process.develop
+          expect(io.string).to match(/SUCCESS/)
+        end
+      end
+
+      describe 'when the expectations fail' do
+        before do
+          process.expect('is not a number!') { |data| data.kind_of?(Numeric) }
+          process.develop
+        end
+
+        it 'does not output SUCCESS' do
+          expect(io.string).to_not match(/SUCCESS/)
+        end
+
+        it 'outputs errors' do
+          expect(io.string).to match(/is not a number!/)
+        end
+      end
+    end
+
+    describe '#benchmark' do
+      before do
+        process.benchmark(2)
+      end
+
+      it 'runs the setup' do
+        expect(process).to be_tainted
+      end
+
+      it 'assigns the output' do
+        expect(process.output).to eq(%w(BAZ BAR FOO))
+      end
+
+      it 'outputs the benchmark results' do
+        expect(io.string).to match(/real/)
+      end
+    end
+  end
+end
+

data/spec/pipeline/step_spec.rb

@@ -0,0 +1,17 @@
+require 'rspec'
+require 'linepipe'
+
+module Linepipe
+  describe Step, '#apply' do
+    let(:step) { Step.new('upcase', &:upcase) }
+
+    it 'exposes a name' do
+      expect(step.name).to eq('upcase')
+    end
+
+    it 'calls the block with the data' do
+      expect(step.apply('data')).to eq('DATA')
+    end
+  end
+end
+

data/spec/pipeline_spec.rb

@@ -0,0 +1,40 @@
+require 'rspec'
+require 'linepipe'
+
+describe Linepipe do
+  describe '.develop' do
+    it 'creates a new linepipe process in development mode' do
+      process = Linepipe::Process.new
+      Linepipe::Process.stub(:new).and_return process
+
+      process.should_receive(:instance_eval)
+      process.should_receive(:develop)
+
+      expect(Linepipe.develop {}).to eq(process)
+    end
+  end
+
+  describe '.run' do
+    it 'creates a new linepipe process in run mode' do
+      process = Linepipe::Process.new
+      Linepipe::Process.stub(:new).and_return process
+
+      process.should_receive(:instance_eval)
+      process.should_receive(:run)
+
+      expect(Linepipe.run {}).to eq(process)
+    end
+  end
+
+  describe '.benchmark' do
+    it 'creates a new linepipe process in benchmark mode' do
+      process = Linepipe::Process.new
+      Linepipe::Process.stub(:new).and_return process
+
+      process.should_receive(:instance_eval)
+      process.should_receive(:benchmark).with(2)
+
+      expect(Linepipe.benchmark(2) {}).to eq(process)
+    end
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: linepipe
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Josep M. Bach
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Process data one step at a time.
+email:
+- josep.bach@wimdu.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/benchmark.rb
+- examples/develop.rb
+- lib/linepipe.rb
+- lib/linepipe/dsl.rb
+- lib/linepipe/expectation.rb
+- lib/linepipe/process.rb
+- lib/linepipe/step.rb
+- lib/linepipe/version.rb
+- linepipe.gemspec
+- spec/pipeline/expectation_spec.rb
+- spec/pipeline/process_spec.rb
+- spec/pipeline/step_spec.rb
+- spec/pipeline_spec.rb
+homepage: https://github.com/wimdu/linepipe
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Process data one step at a time.
+test_files:
+- spec/pipeline/expectation_spec.rb
+- spec/pipeline/process_spec.rb
+- spec/pipeline/step_spec.rb
+- spec/pipeline_spec.rb