chawk 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75c272978a418036f10a0f03297b48af7f40b7f4
-  data.tar.gz: 44b65c108649fb2fd8ab58f337c919ad02e7e959
+  metadata.gz: 3043b4d0a900dcc861435dd5377a7acabca5329c
+  data.tar.gz: f170e087c3f43e1770db9d4071bbc2f798c367f9
 SHA512:
-  metadata.gz: 72f85f23a8d05e8adc1308e389796fb605867d6f9126655744e03c04ae9f9bc298a302418078f1ab3e2bba57c56d3b34e4b747a9c6212912a84fcc10c83fdf90
-  data.tar.gz: 05640ad20ba4b52f3671d7a1ffeb401f4445b9a2edc4c5bafb76794ebdf35241e5b7f36a4508aa840daa43d0279582850fcd551908716e87bd1b80997440db65
+  metadata.gz: 2db59a3c7073ad42f2abdd659ce9c037b107cdf9f7d732559d9a35eaaa6f3fb2f0e495d8f22c83f5e67a6f22e9c1794de09f73138b41bd9b67bb4053b48e7ea6
+  data.tar.gz: 44c77569af5f921eb69eacc838d34083efe9f1c01fe0dc26ccdf75e83daf908a72f6bf07a23e63fff6c4e270e0f50d354efc67121b8a1737ba87d322f2169002

data/README.md

@@ -119,6 +119,14 @@ This will return all data from the Node parent_node in the range from timestamp
 
 This will return all data from the Node parent_node in the range from timestamp 1085 to 8100, resampled to the quarter beat. (2 data points, no matter how many are actually present in the sample)
 
+Chawk::Models::Range also accepts a "strategy" field which indicates how it builds it's data points.  
+
+The default is "recent_point", which constructs timestamps and fills them with the most recent value for that timestamp.  
+
+"cluster" creates clustered group timestamps and fills them with a sum of all values within the timeframe from one timestamp to another.  
+
+"tally" creates timestamps and fills them with a sum of all values from the beginning of the range to the timestamp.  
+
 ## Chawk::Models::NodeAggregator
 The NodeAggregator is a (currently expensive) object for aggregate calculations on a Node.  It's intended ot be use on a Range's data_node property, since doing aggregate math on an entire datase can be prohibitively expensive.
 

data/chawk.gemspec

@@ -25,9 +25,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rack-test', "0.6.2")
   spec.add_development_dependency('json', "1.8.1")
   spec.add_development_dependency('simplecov', "0.8.2")
-  spec.add_development_dependency('pg', "0.17.1")
-  spec.add_development_dependency('sqlite3', "1.3.9")
-  # spec.add_development_dependency('pry')
-  # spec.add_development_dependency('pry-debugger')
-  # spec.add_development_dependency('pry-stack_explorer')
+  spec.add_development_dependency('pg')
+  spec.add_development_dependency('sqlite3')
+  #spec.add_development_dependency('pry')
+  #spec.add_development_dependency('pry-debugger')
+  #spec.add_development_dependency('pry-stack_explorer')
 end

data/lib/aggregator.rb

@@ -0,0 +1,46 @@
+require 'active_record'
+module Chawk
+  module Models
+    # TODO: Most expensive version imaginable.  To be replaced.
+    class Aggregator
+
+      attr_reader :dataset
+
+      def initialize(node)
+        node.check_read_access
+        if node.points.length > 0
+          @dataset = node.points.to_a.reduce([]) {|ary,p| ary << p.value}
+        end
+      end
+
+      def max()
+        @dataset.max
+      end
+
+      def min()
+        @dataset.min
+      end
+
+      def mean
+        sum.to_f / @dataset.length
+      end
+
+      def sum
+        @dataset.reduce(0) {|sum,p| sum+=p}
+      end
+
+      def count
+        @dataset.length
+      end
+
+      def sumsqr
+        @dataset.map {|x| x * x}.reduce(&:+)
+      end
+
+      def stdev
+        m = mean
+        Math.sqrt((sumsqr - count * m * m)/(count-1))
+      end
+    end
+  end
+end

data/lib/chawk.rb

@@ -1,6 +1,9 @@
 require "chawk/version"
 require 'quantizer'
 require 'models'
+require 'node'
+require 'range'
+require 'aggregator'
 
 # Chawk is a gem for storing and retrieving time seris data.
 module Chawk

data/lib/chawk/migration.rb

@@ -10,6 +10,7 @@ class CreateChawkBase < ActiveRecord::Migration
 			t.float  "stop_ts"
 			t.integer  "beats"
 			t.integer  "default"
+			t.string  "strategy"
 			t.timestamp "expires"
 			t.timestamps
 		end

data/lib/chawk/version.rb

@@ -1,4 +1,4 @@
 module Chawk
 	# The current version of Chawk
-	VERSION = "0.3.0"
+	VERSION = "0.4.0"
 end

data/lib/models.rb

@@ -1,6 +1,4 @@
 require 'active_record'
-require 'node'
-require 'range'
 module Chawk
 	# Models used in Chawk.  ActiveRecord classes.
 	module Models

data/lib/node.rb

@@ -23,47 +23,6 @@ module Chawk
       end
     end
 
-    class NodeAggregator
-
-      attr_reader :dataset
-
-      def initialize(node)
-        node.check_read_access
-        if node.points.length > 0
-          @dataset = node.points.to_a.reduce([]) {|ary,p| ary << p.value}
-        end
-      end
-
-      def max()
-        @dataset.max
-      end
-
-      def min()
-        @dataset.min
-      end
-
-      def mean
-        sum.to_f / @dataset.length
-      end
-
-      def sum
-        @dataset.reduce(0) {|sum,p| sum+=p}
-      end
-
-      def count
-        @dataset.length
-      end
-
-      def sumsqr
-        @dataset.map {|x| x * x}.reduce(&:+)
-      end
-
-      def stdev
-        m = mean
-        Math.sqrt((sumsqr - count * m * m)/(count-1))
-      end
-    end
-
     # The Node, where most Chawk:Node information is persisted..
     class Node < ActiveRecord::Base
       attr_accessor :agent

data/lib/range.rb

@@ -44,7 +44,19 @@ module Chawk
         populate!
       end
 
-      def point_from_parent_point(now)
+      def cluster(now, benow)
+        sum = parent_node.points.where("observed_at > ? AND observed_at <= ?",benow,now).sum(:value)
+        data_node.points.create(observed_at:now, recorded_at:Time.now, value:sum)
+      end
+
+      def tally(now, benow, beval)
+        sum = parent_node.points.where("observed_at > ? AND observed_at <= ?",benow,now).sum(:value)
+        #puts "\n\n VAL #{beval} SUM #{sum}"
+        value = beval + sum
+        data_node.points.create(observed_at:now, recorded_at:Time.now, value:value)
+      end
+
+      def recent_point(now, benow)
         point = parent_node.points.where("observed_at <= :dt_to",{dt_to:now}).order(observed_at: :desc, id: :desc).first
         if point
           value = point.value
@@ -60,8 +72,20 @@ module Chawk
         self.data_node.points.destroy_all
         step = 0.25 * self.beats
         now = (self.start_ts*4).round/4.to_f
+        benow = now - step
+        beval = 0
         while now < self.stop_ts
-          point = point_from_parent_point now
+          #binding.pry
+          case strategy
+          when "cluster"
+            point = cluster now, benow
+          when "tally"
+            point = tally now, benow, beval
+          when "recent_point","",nil
+            point = recent_point now, benow
+          end
+          beval = point.value
+          benow = now
           now += step
         end
       end

data/test/lib/range_test.rb

@@ -13,12 +13,12 @@ describe Chawk do
     Chawk::Models::Range.new(start_ts:1000.0,stop_ts:1140.0,beats:1,parent_node:node1).save.must_equal true
   end
 
-  it "calculates range" do 
+  it "calculates recent point" do 
     node1 = Chawk.node(@agent,'a:b')
     node1.clear_points!
 
     range = Chawk::Models::Range.create(start_ts:1085.0,stop_ts:1140.0,beats:1,parent_node:node1)
-    ag = Chawk::Models::NodeAggregator.new(range.data_node)
+    ag = Chawk::Models::Aggregator.new(range.data_node)
     ag.sum.must_equal(0)
     ag.mean.round(2).must_equal(0.0)
 
@@ -36,7 +36,7 @@ describe Chawk do
     range.data_node.points.length.must_equal(220)
     range.data_node.points[25].value.must_equal(92)
     range.data_node.points[140].value.must_equal(94)
-    ag = Chawk::Models::NodeAggregator.new(range.data_node)
+    ag = Chawk::Models::Aggregator.new(range.data_node)
     ag.sum.must_equal(20066)
     ag.mean.round(2).must_equal(91.21)
 
@@ -44,7 +44,7 @@ describe Chawk do
 
     range.reload
     range.data_node.points[200].value.must_equal(1500)
-    ag = Chawk::Models::NodeAggregator.new(range.data_node)
+    ag = Chawk::Models::Aggregator.new(range.data_node)
     ag.sum.must_equal(48306)
     ag.max.must_equal(1500)
     ag.min.must_equal(0)
@@ -52,10 +52,47 @@ describe Chawk do
     range = Chawk::Models::Range.create(start_ts:1088.0,stop_ts:8100.0,beats:14400,parent_node:node1)
     range.data_node.points.length.must_equal(2)
 
-    ag = Chawk::Models::NodeAggregator.new(range.data_node)
+    ag = Chawk::Models::Aggregator.new(range.data_node)
     ag.sum.must_equal(1592)
     ag.mean.must_equal(796)
     ag.stdev.round(2).must_equal(995.61)
 
   end
+
+  it "calculates clusters" do
+    node1 = Chawk.node(@agent,'vals')
+
+    node1._insert_point(1,Time.parse("Nov 29, 2014 9:33:20 ").to_f)
+    node1._insert_point(1,Time.parse("Nov 29, 2014 12:40:20").to_f)
+    node1._insert_point(1,Time.parse("Nov 29, 2014 17:44:20").to_f)
+    node1._insert_point(1,Time.parse("Nov 30, 2014 9:33:20").to_f)
+    node1._insert_point(1,Time.parse("Nov 30, 2014 12:40:20").to_f)
+    node1._insert_point(1,Time.parse("Dec 1, 2014 17:44:20").to_f)
+
+    range = Chawk::Models::Range.create(start_ts:Time.parse("Nov 29, 2014 8:00:00").to_f,stop_ts:Time.parse("Dec 2, 2014 10:00:00"),beats:345600,parent_node:node1,strategy:"cluster")
+    range.data_node.points.length.must_equal(4)
+    range.data_node.points[0].value.must_equal(0)
+    range.data_node.points[1].value.must_equal(3)
+    range.data_node.points[2].value.must_equal(2)
+    range.data_node.points[3].value.must_equal(1)
+  end
+
+  it "calculates tallies" do
+    node1 = Chawk.node(@agent,'vals')
+
+    node1._insert_point(1,Time.parse("Nov 29, 2014 9:33:20 ").to_f)
+    node1._insert_point(1,Time.parse("Nov 29, 2014 12:40:20").to_f)
+    node1._insert_point(1,Time.parse("Nov 29, 2014 17:44:20").to_f)
+    node1._insert_point(1,Time.parse("Nov 30, 2014 9:33:20").to_f)
+    node1._insert_point(1,Time.parse("Nov 30, 2014 12:40:20").to_f)
+    node1._insert_point(1,Time.parse("Dec 1, 2014 17:44:20").to_f)
+
+    range = Chawk::Models::Range.create(start_ts:Time.parse("Nov 29, 2014 8:00:00").to_f,stop_ts:Time.parse("Dec 2, 2014 10:00:00"),beats:345600,parent_node:node1,strategy:"tally")
+    range.data_node.points.length.must_equal(4)
+    range.data_node.points[0].value.must_equal(0)
+    range.data_node.points[1].value.must_equal(3)
+    range.data_node.points[2].value.must_equal(5)
+    range.data_node.points[3].value.must_equal(6)
+  end
+
 end

data/test/schema.rb

@@ -44,6 +44,7 @@ ActiveRecord::Schema.define(version: 0) do
     t.float    "stop_ts"
     t.integer  "beats"
     t.integer  "default"
+    t.string   "strategy"
     t.datetime "expires"
     t.datetime "created_at"
     t.datetime "updated_at"

data/test/test_helper.rb

@@ -27,6 +27,7 @@ ENV["CHAWK_DEBUG"] ? debug_level=ENV["CHAWK_DEBUG"] : debug_level=:info
 #end
 
 require 'active_record'
+require 'yaml'
 
 if ENV["TEST_DATABASE_URLX"]
 	ActiveRecord::Base.establish_connection ENV["TEST_DATABASE_URL"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chawk
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Scott Russell
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-29 00:00:00.000000000 Z
+date: 2014-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -98,30 +98,30 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.9
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.9
+        version: '0'
 description: A storage engine for time-series data.  Eventually to include resampling,
   statistical and aggregate data management.
 email:
@@ -139,6 +139,7 @@ files:
 - README.md
 - Rakefile
 - chawk.gemspec
+- lib/aggregator.rb
 - lib/chawk.rb
 - lib/chawk/Jackdaw.svg
 - lib/chawk/migration.rb