spanbars 0.1.2beta

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7698dcae6b52fb67ba0636e83ed8c818bc517ed0ccce2559c4ae61e9c37bad62
+  data.tar.gz: c7472948c2e2f5809e910548677f016b510eeadd711414bae016c027e04b46a9
+SHA512:
+  metadata.gz: cda1887acff3b220c43e0667202ccf6eaa83c1c9ccfa80b6610056e1eea1a786a83baa409f34b146b165a60a769c1b11caed87037a879725a7b42a9964ff6c0f
+  data.tar.gz: c9a98b5a5bf355ee6cec2c9602309d05da34f9293189c0fdfd9a4d1537de96eda539b116d5ec6438a62a3c72c8df4f987fedcc953e1c760272bd9559996d9a13

data/README.md

@@ -0,0 +1,69 @@
+# SpanBars
+
+SpanBars is a tiny tool to produce span bars from time series data, either directly from a time 
+series or based on OHLC bars. 
+
+## Description
+
+SpanBars reads record-by-record (or line-by-line) and provides the according span bars. 
+The current bar is closed and a new bar is created as soon as the given span is _exceeded_. 
+
+Spanbars are comparable to classic OHLC bars or candle sticks, but have the major
+advantage of a CLOSE value always equalling either HIGH or LOW. All data eliminated
+can be considered as _noise_.
+
+Therefore exists 4 types of bars:
+
+* TOP:     CLOSE == HIGH
+* UP:      CLOSE == HIGH and OPEN == LOW
+* BOTTOM:  CLOSE == LOW 
+* DOWN:    CLOSE == LOW  and OPEN == HIGH
+
+When generating spanbars based on a plain time series, these bars can be sent to output using 
+the parameter _--simple_. Without it will  
+process the resulting data again, aggregating all bars created in the first run to start
+at an absolute HIGH (or LOW) and end at an absolute LOW (or HIGH resp.). Although these
+spans of second type might be much larger than _--span_, they can be considered to
+contain no noise _smaller or equal than_ span.
+
+Creating spanbars from given OHLCs is slightly inaccurate, as even one OHLC might
+contain several spanbars, but that cannot be reflected by the given input data. Also, for
+processing OHLC-input data, note that the entire algorithm (simple spanbars first, strict spanbars seconds) is run 3 times: 
+1. using (span / 2) on input HIGHS
+2. using (span / 2) on input LOWS
+1. using (span)     on [ resulting Highs, resulting Lows ].sorted\_by\_time
+
+The application area this gem is written for is denoising data for trend recognition 
+within monitored timeseries. 
+
+## Basic usage
+    processor = SpanBarProcessor.new(span: 5)
+    File.read("./timeseries.csv").map{|x| { t: x[0], v: x[1] } }.sort_by{|x| x[:t]}.each do |data|
+      processor.add(data)
+    end
+    processor.bars.each {|x| puts x}
+
+## Basic usage via commandline
+
+Using _spanbars_ on the commandline expects data on STDIN as CSV with timestamps on column 1 and 
+values on column 2. With _--ohlc_ enabled, it expects CSV with "timestamps,open,high,low,close".
+
+Provided output will be CSV as well, for
+
+* _simple_: "timestamp\_open,open, timestamp\_high,high, timestamp\_low, low, timestamp\_close, close, direction, path, momentum, direction"
+* _strict_: "timestamp\_open,open, timestamp\_close, close, duration, path, momentum, effective\_span, direction"
+
+I currently plan to implement 2 more parameters: _--human_ to create a human-readable table 
+(particularly concerning the time format), and _--intraday_ (ommiting the date part when using _--human_).
+
+    $ cat timeseries.csv | spanbars 
+    $ spanbars --input ./timeseries.csv  --span 5
+
+## List of parameters
+
+* --span     (defaults to 10)
+* --ticksize (default  to 1.0)
+* --ohlc     (defaults to false)
+* --simple   (defaults to false)
+* --human    (planned, defaults to false)
+* --intraday (planned, defaults to false)

data/bin/spanbars

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+
+THIS_FILE = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+require File.dirname(THIS_FILE) + '/../lib/spanbarprocessor.rb'
+require File.dirname(THIS_FILE) + '/../lib/spanbar.rb'
+
+
+# prepare processing of incoming control commands (e.g. Ctrl-C)
+Signal.trap("TERM") { local_interrupt }
+Signal.trap("INT")  { local_interrupt }
+
+def local_interrupt
+  exit
+end
+
+# prepare command line parser and help text
+
+op = Slop::Options.new
+op.banner = "\n#{"Usage:".light_white} spanbars [options]"
+op.separator "" 
+op.bool    '--simple'   , "Disable processing of strict StanBars", default: false
+op.float   '--ticksize' , "Set ticksize for processing",           default: 1.0
+op.integer '--span'     , "Set span for processing",               default: 10
+op.boolean  '--ohlc'     , "Define OHLC input file instead of timeseries (overrides --simple and --both)", default: false
+op.boolean '--human'    , "Define human output",                   default: false
+op.boolean '--intraday' , "Strip date portion (affects --human only)", default: false
+op.boolean '--help'     , "Print this help", default: false
+op.boolean '--both'     , "Returns both simple and strict bars (overrides simple)",   default: false
+op.separator "" 
+op.separator "#{"Please note:".light_white} spanbars relies on STDIN data, e.g. "
+op.separator "    #{"$".light_white} cat /tmp/timeseries.csv | spanbars --simple --span 5 --ticksize 0.1"
+op.separator ""
+
+optparser = Slop::Parser.new(op)
+
+# print help upon unparsable commandline options
+begin
+  opts = optparser.parse(ARGV)
+rescue StandardError => e
+  puts e.inspect
+  puts op.to_s
+  exit
+end
+
+if opts.help? 
+  puts op.to_s
+  exit
+end
+
+unless STDIN.tty? 
+  unless opts.ohlc?
+    s = SpanBarProcessor.new(opts)
+    while csv = STDIN.gets
+      line   = CSV.parse(csv.chomp).flatten
+      result = s.add(line[0].to_i, line[1].to_f)
+      if result
+        if opts[:human] 
+          result.each {|r| r.set_intraday if opts[:intraday]; puts ([:up, :bottom].include? r.type) ? "#{r.to_human}".green : "#{r.to_human}".red } 
+        else # CSV output
+          result.each {|r| CSV {|out| out << r.to_a  } }
+        end
+      end
+    end
+    exit
+  else # if ohlc is given
+    data = [ ] 
+    data << CSV.parse(csv.chomp).flatten while csv = STDIN.gets
+    opts = opts.to_hash
+    opts.delete(:simple)
+    opts.delete(:both)
+    highOpts = opts.dup
+    highOpts[:span] = (opts[:span] / 2.0).floor.to_i
+    highProc = SpanBarProcessor.new(highOpts)
+    lowProc  = SpanBarProcessor.new(highOpts)
+    finProc  = SpanBarProcessor.new(opts)
+    data.each do |d| 
+      highProc.add(d[0].to_i, d[2].to_f)
+      lowProc. add(d[0].to_i, d[3].to_f)
+    end
+    res = highProc.spanBars.map{|bar| bar.type == :up ? bar.highval : bar.lowval } #.map{|peak| bar.highval }
+    res.flatten!
+    res.sort!{|a,b| a[:t] <=> b[:t]}
+    res.each {|peak| finProc.add peak[:t],peak[:p]}
+    if opts[:human]
+      finProc.spanBars.each {|r| r.set_intraday if opts[:intraday]; puts ([:up, :bottom].include? r.type) ? "#{r.to_human}".green : "#{r.to_human}".red }
+    else # CSV output
+      finProc.spanBars.each {|r| CSV {|out| out << r.to_a  } }
+    end
+  end
+else
+  puts op.to_s
+end
+
+
+
+
+

data/features/01_spanbarprocessor_initialization.feature

@@ -0,0 +1,29 @@
+Feature: Initializiation of SpanBarProcessor
+  Scenario: Testing provided arguments during initialization
+    * a new SpanBarProcessor created with span that is not Integer or <= 1 should raise ArgumentError
+    * a new SpanBarProcessor created with a valid span but non-Numeric ticksize or <= 0 should raise ArgumentError
+
+  Scenario Outline: Testing well initialized instance_variables
+    Given a SpanBarProcessor is initialized with "3", "3"
+    Then <instance_var> should be set to <value>
+    Examples: 
+      | instance_var | value | 
+      | @ticks       |  []   |
+      | @limit       |  9    |
+      | @simpleMax   |  0    |
+      | @simpleMin   |  Float::INFINITY | 
+      | @simpleBar   |  []   | 
+      | @simpleBars  |  []   | 
+      | @intraday    | false |
+      | @both        | false |
+      | @simple      | false |
+
+    #Scenario Outline: Testing methods and attr
+    #Given a SpanBarProcessor is initialized with "4", "3"
+    #Then it should respond to "<method>"
+    #Examples:
+    #  | method |
+    #  | add    |
+    #  | simpleBars | 
+    #  | create_strict_from |
+    #

data/features/02_spanbarprocessor_add.feature

@@ -0,0 +1,40 @@
+Feature: Using #add
+
+  Scenario: Falsey input
+     Given a simple SpanBarProcessor is initialized with "3", "3"
+     Then it should raise ArgumentError when calling #add with less than 2 arguments
+     Then it should raise ArgumentError when calling #add and first argument is neither Integer nor Time
+     Then it should raise ArgumentError when calling #add and second argument is neither Numeric nor Nil
+
+  Scenario: Correct return values
+     Given a simple SpanBarProcessor is initialized with "3", "3"
+     Then calling #add with params 123 and nil should return nil
+     Then calling #add with params 123 and 2.5 should return false
+
+  Scenario: eesting a linear raising series
+     Given a simple SpanBarProcessor is initialized with "3", "1"
+     Then  #add with params 1, 1 should return false
+     Then  #add with params 2, 2 should return false
+     Then  #add with params 3, 3 should return false
+     Then  #add with params 4, 4 should return false
+     Then  #add with params 5, 5 should return an Array
+
+  Scenario Outline: Running with reference data
+    Given a SpanBarProcessor is initialized with "<deviation>", "<ticksize>"
+    And a feeder with data from "<file>" is prepared
+    Then adding all data from file should not raise any error
+
+    Examples: 
+      | file                         | deviation | ticksize   | 
+      | ./features/support/ref1.csv  | 5         | 0.25       |
+      | ./features/support/ref1.csv  | 8         | 0.1        |
+      | ./features/support/ref1.csv  | 12        | 0.66       |
+      | ./features/support/ref2.csv  | 5         | 1          |
+      | ./features/support/ref2.csv  | 8         | 2          |
+      | ./features/support/ref2.csv  | 12        | 5.5        |
+      | ./features/support/ref3.csv  | 5         | 0.0000005  |
+      | ./features/support/ref3.csv  | 8         | 0.000001   |
+      | ./features/support/ref3.csv  | 12        | 0.00000025 |
+
+    
+

data/features/03_spanbar_initialization.feature

@@ -0,0 +1,30 @@
+Feature: Initializiation of SpanBar
+  Scenario: Testing provided arguments during initialization
+    * a new simple SpanBar created without options should raise ArgumentError
+    * a new simple SpanBar created by SpanBarProcessor should not raise 
+
+  Scenario Outline: Testing well initialized instance_variables
+    Given a valid simple SpanBar is created by SpanBarProcessor
+    Then <instance_var> should be set to <value>
+    Examples: 
+      | instance_var | value | 
+      | @strict      | false | 
+      | @type        | :up   |
+      | @open        | 1     |
+      | @momentum    | Float::INFINITY   |
+    
+ 
+
+  Scenario Outline: Testing methods and attr of SpanBar
+     Given a valid simple SpanBar is created by SpanBarProcessor
+     Then it should respond to "<method>"
+     Examples:
+       | method   |
+       | inspect  | 
+       | to_human |
+       | to_a     |
+       | valid?   |
+       | path     |
+       | momentum |
+       | split_for|
+

data/features/04_spanbars_commandline.feature

@@ -0,0 +1,19 @@
+Feature: Using spanbars on commandline
+  Scenario: When using bin/spanbars on the commandline without parameters
+    Given bin/spanbars is run on the commandline and neither parameters nor STDIN is given
+    #Then bin/spanbars should display help
+
+  Scenario Outline: Checking known input to produce known output
+    Given bin/spanbars is run with following parameters it should ouput
+    Then cat <input> | spanbars --both --ticksize <ticksize> --span <span> should produce <output>
+    Examples:
+      | input | ticksize  | span | output                 |
+      | ref1  | 0.25      | 25   | ref1_out_0.25_25       |
+      | ref2  |    1      | 25   | ref2_out_1_25          |
+      | ref3  | 0.0000005 | 10   | ref3_out_0.0000005_10  |
+
+  Scenario: Checking whether false reference is recognized
+    Given bin/spanbars is run with following parameters it should ouput
+    Then 'cat ref3 | bin/spanbars --both --span 10 --ticksize 0.0000005' should not produce ref2_out_1_25
+
+      

data/features/step_definitions/01_spanbar_processor_initialization_steps.rb

@@ -0,0 +1,36 @@
+require './lib/spanbar.rb'
+require './lib/spanbarprocessor.rb'
+
+Given "a new SpanBarProcessor created with span that is not Integer or <= 1 should raise ArgumentError" do
+  expect{ SpanBarProcessor.new(span: "t") }.to raise_error(ArgumentError)
+  expect{ SpanBarProcessor.new(span: 1.6) }.to raise_error(ArgumentError)
+  expect{ SpanBarProcessor.new(span: 1)   }.to raise_error(ArgumentError)
+end
+  
+Given "a new SpanBarProcessor created with a valid span but non-Numeric ticksize or <= 0 should raise ArgumentError" do 
+  expect{ SpanBarProcessor.new(span: 2, ticksize: "foo")}.to raise_error(ArgumentError)
+  expect{ SpanBarProcessor.new(span: 2, ticksize: 0)}.to raise_error(ArgumentError)
+  expect{ SpanBarProcessor.new(span: 2, ticksize: -1.2)}.to raise_error(ArgumentError)
+end
+
+Given /^a SpanBarProcessor is initialized with "([^"]*)", "([^"]*)"$/ do |span, ticksize|
+  @s = SpanBarProcessor.new(span: span.to_i, ticksize: ticksize.to_f)
+end
+
+Given /^a simple SpanBarProcessor is initialized with "([^"]*)", "([^"]*)"$/ do |span, ticksize|
+    @s = SpanBarProcessor.new(span: span.to_i, ticksize: ticksize.to_f, simple: true)
+end
+
+
+Then /^it should respond to "([^"]*)"$/ do |method| 
+  expect(@s).to respond_to(method.to_sym)
+end
+
+Then /^([^\s]*) should be set to ([^\s]*)$/ do |var,value|
+  expect(@s.instance_variable_defined?(var.to_sym)).to be_truthy
+  res = nil
+  puts "#{var}---"
+  puts "#{value}---"
+  eval "res = @s.instance_variable_get(var.to_sym) == #{value}"
+  expect(res).to be_truthy
+end

data/features/step_definitions/02_spanbarprocessor_add_steps.rb

@@ -0,0 +1,42 @@
+Then "it should raise ArgumentError when calling #add with less than 2 arguments" do
+  expect{ @s.add }.to raise_error(ArgumentError)
+  expect{ @s.add(1)}.to raise_error(ArgumentError)
+end
+
+Then "it should raise ArgumentError when calling #add and first argument is neither Integer nor Time" do
+  expect{ @s.add("foo", "bar") }.to raise_error(ArgumentError)
+  expect{ @s.add(124, 1.22) }.not_to raise_error
+  expect{ @s.add(Time.now, 1.22) }.not_to raise_error
+end
+
+Then "it should raise ArgumentError when calling #add and second argument is neither Numeric nor Nil" do
+  expect{ @s.add(123, "foo") }.to raise_error(ArgumentError)
+  expect{ @s.add(123, 11234) }.not_to raise_error
+  expect{ @s.add(123, nil  ) }.not_to raise_error
+end
+
+Then /^calling #add with params (\d+) and nil should return nil$/ do |timestamp|  
+  expect(@s.add(timestamp, nil)).to be_nil
+end
+
+Then /^calling #add with params (\d+) and ([0-9.]+) should return false$/ do |timestamp, value|
+  expect(@s.add(timestamp, value.to_f)).to be(false)
+end
+
+Then /^#add with params (\d+), ([0-9.]+) should return false$/ do |timestamp, value|
+  expect(@s.add(timestamp, value.to_f)).to be(false)
+end
+
+Then /^#add with params (\d+), ([0-9.]+) should return an Array$/ do |timestamp, value|
+  result = @s.add(timestamp, value.to_f)
+  expect(result.class).to be(Array)
+end
+
+Given /^a feeder with data from "([^"]+)" is prepared$/ do |filename| 
+  require 'csv'
+  @data = CSV.read(filename).map{|x| [ x[0].to_i, x[1].to_f] }
+end
+
+Then "adding all data from file should not raise any error" do 
+  expect { @data.each {|d| @s.add(d[0],d[1]) } } .not_to raise_error
+end

data/features/step_definitions/03_spanbar_initialization_steps.rb

@@ -0,0 +1,18 @@
+Then "a new simple SpanBar created without options should raise ArgumentError" do
+  expect { SpanBar.new }.to raise_error(ArgumentError)
+end
+
+Then "a new simple SpanBar created by SpanBarProcessor should not raise" do
+  @p = SpanBarProcessor.new(simple: true)
+  expect { for i in (1..5); @p.add(i,i); end }.not_to raise_error
+end
+
+Given "a valid simple SpanBar is created by SpanBarProcessor" do 
+  @p = SpanBarProcessor.new(simple: true);  @s = true; i = 1; 
+  while (not @s.is_a?(Array)) do @s = @p.add(i,i); i+=1; end
+  @s = @s[0]
+end
+
+
+  
+

data/features/step_definitions/04_spanbars_commandline_steps.rb

@@ -0,0 +1,26 @@
+Given /^bin\/spanbars is run on the commandline and neither parameters nor STDIN is given$/ do
+end
+
+Then /^bin\/spanbars should display help$/ do
+  expect{ system("bin/spanbars --help") }.to output(/usage/).to_stdout_from_any_process
+end
+
+Given /^bin\/spanbars is run with following parameters it should ouput$/ do
+end
+
+Then /^cat ([^ ]*) \| spanbars --both --ticksize ([^ ]*) --span ([^ ]*) should produce ([^ ]*)$/ do |input, ticksize, span, output|
+  inputfile = "features/support/#{input}.csv"
+  outputfile = "features/support/#{output}.csv"
+  result = `cat #{inputfile} | bin/spanbars --both --span #{span} --ticksize #{ticksize}`
+  reference = File.read(outputfile)
+  expect(result).to eq(reference)
+end
+
+Then /^'cat ([^ ]*) \| bin\/spanbars --both --span ([^ ]*) --ticksize ([^' ]*)' should not produce ([^"$ ]*)$/ do |input, span, ticksize, output|
+  inputfile = "features/support/#{input}.csv"
+  outputfile = "features/support/#{output}.csv"
+  result = `cat #{inputfile} | bin/spanbars --both --span #{span} --ticksize #{ticksize}`
+  reference = File.read(outputfile)
+  expect(result).not_to eq(reference)
+end
+

data/features/support/env.rb

@@ -0,0 +1,6 @@
+require 'rspec' 
+RSpec.configure do |config|
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end