data_store 0.0.1 → 0.0.2

data/lib/data_store/table.rb

@@ -0,0 +1,119 @@
+module DataStore
+
+  class Table
+
+    include Celluloid
+
+    attr_reader :identifier, :table_index, :original_value
+
+    # Initialize the table by passsing an identifier
+    def initialize(identifier, table_index = 0)
+      @identifier  = identifier
+      @table_index = table_index
+    end
+
+    # Return a the corresponding parent class, i.e the settings from the data_stores table
+    def parent
+      @parent ||= DataStore::Base.find(identifier: identifier)
+    end
+
+    # Return a table object enriched with Sequel::Model behaviour
+    def model
+      @model ||= Class.new(Sequel::Model(dataset))
+    end
+
+    # Add a new datapoint to the table
+    # In case of a counter type, store the difference between current and last value
+    # And calculates average values on the fly according to compression schema
+    #
+    # Options (hash):
+    #  * created: timestamp
+    #  * type: gauge or counter
+    #  * table_index: in which compressed table
+    def add(value, options = {})
+      created      = options[:created] || Time.now.utc.to_f
+      type         = options[:type] || parent.type
+      @table_index = options[:table_index] if options[:table_index]
+      push(value, type, created)
+    end
+
+    # Return the most recent datapoint added
+    def last
+      model.order(:created).last
+    end
+
+    # Return the total number of datapoints in the table
+    def count
+      dataset.count
+    end
+
+    # Return the corresponding dataset with the datapoitns
+    def dataset
+      database[table_name]
+    end
+
+    # Fetch the corresponding datapoints
+    #
+    # Options:
+    #  * :from
+    #  * :till
+    #
+    def fetch(options)
+      datapoints = []
+      query = parent.db[timeslot(options)].where{created >= options[:from]}.where{created <= options[:till]}.order(:created)
+      query.all.map{|record| datapoints <<[record[:value], record[:created]]}
+      datapoints
+    end
+
+    # Import original datapoints, mostly to recreate compression tables
+    def import(datapoints)
+      datapoints.each do |data|
+        add!(data[0], table_index: 0, created: data[1])
+      end      
+    end
+
+    private
+
+    def timeslot(options)
+      distance = options[:till] - options[:from]
+      index = 0
+      parent.time_borders.each_with_index do |value, idx|
+        index = idx
+        break if value >= distance
+      end
+      parent.table_names[index]
+    end
+
+    def push(value, type, created)
+      value = difference_with_previous(value) if type.to_s == 'counter'
+      datapoint = { value: value, created: created }
+      datapoint[:original_value] = original_value if original_value
+      dataset << datapoint
+      calculate_average_values
+    end
+
+    def calculate_average_values
+      calculator = AverageCalculator.new(self)
+      calculator.perform
+    end
+
+    def difference_with_previous(value)
+      @original_value = value
+      unless last.nil?
+        value = value - last[:original_value]
+        last.delete if last[:value] == last[:original_value]
+      end
+      value
+    end
+
+    def database
+      @database ||= DataStore::Base.db
+    end
+
+    def table_name
+      parent.table_names[table_index]
+    end
+
+  end
+
+end

data/lib/data_store/version.rb

@@ -1,3 +1,3 @@
 module DataStore
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/lib/data_store.rb

@@ -1,5 +1,79 @@
+# encoding: UTF-8
+
+require 'sequel'
+require 'yaml'
+require 'logger'
+require 'celluloid'
+
+$: << File.expand_path('../', __FILE__)
+$: << File.expand_path('../data_store/', __FILE__)
+
+Sequel.extension :migration
+Sequel::Model.plugin :timestamps, :force=>true, :update_on_create=>true
+
 require 'data_store/version'
+require 'data_store/connector'
+require 'data_store/configuration'
+require 'data_store/definitions'
+require 'data_store/table'
+require 'data_store/average_calculator'
+
+module Kernel
+  def suppress_warnings
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    result = yield
+    $VERBOSE = original_verbosity
+    return result
+  end
+end
 
 module DataStore
-  # Your code goes here...
+
+  # Base class will be redefined during configure
+  # In order to assign Sequel::Model behaviour to it
+  # with the correctly defined (or configured) database connector
+  class Base
+  end
+    
+  class << self
+
+    # Configure DataStore
+    #
+    # Example
+    #   DataStore.configure |config|
+    #     config.prefix   = 'data_store_'
+    #     config.database = :postgres
+    #   end
+    def configure
+      yield(configuration)
+      define_base_class
+    end
+
+    # The configuration object. See {Configuration}
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    private
+
+    def define_base_class
+      connector = DataStore::Connector.new     
+      set_logger(connector.database)
+      connector.create_table!
+      suppress_warnings { self.const_set(:Base, Class.new(Sequel::Model(connector.dataset)))}
+      load 'base.rb'
+      connector.database.disconnect
+    end
+
+    def set_logger(db)
+      if configuration.enable_logging
+        logger = Logger.new(configuration.log_file)
+        logger.level = configuration.log_level
+        db.logger = logger
+      end
+    end
+
+  end
+
 end

data/test/average_calculator_test.rb

@@ -0,0 +1,196 @@
+require File.expand_path '../test_helper', __FILE__
+
+class AverageCalculatorTest < Test::Unit::TestCase
+
+  context 'AverageCalculator for a gauge type' do
+  
+    setup do
+      DataStore::Base.db.tables.each do |table|
+        DataStore::Base.db.drop_table(table)
+      end
+
+      DataStore::Connector.new.reset!
+      @record = DataStore::Base.create(identifier:  1,
+                                       type:        'gauge', 
+                                       name:        'Electra',
+                                       frequency:    10,
+                                       description: 'Actual usage of electra in the home',
+                                       compression_schema: [2,2,2])
+
+      @table = DataStore::Table.new(1)
+      @calculator = DataStore::AverageCalculator.new(@table)
+    end
+
+    should 'be valid' do
+      assert @calculator
+    end
+
+    should 'return the identifier' do
+      assert_equal 1, @calculator.identifier
+    end
+
+    should 'calculate the average value for the first' do
+      @table.model.insert(value: 10, created: 0)
+      @table.model.insert(value: 11, created: 10)
+
+      @calculator.perform
+      assert_equal 10.5, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+    end
+
+    context 'Scenario: adding values according expected frequency' do
+      should 'calculate the average values' do
+        time_now_utc_returns(10)
+       
+        @table.model.insert(value: 10, created: 0)
+        @table.model.insert(value: 11, created: 10)
+
+        @calculator.perform
+
+        assert_equal 10.5, DataStore::Base.db[:ds_1_2].order(:created).first[:value]       
+
+        @table.model.insert(value: 12, created: 20)
+        @table.model.insert(value: 13, created: 30)
+
+        time_now_utc_returns(30)
+
+        @calculator.perform
+
+        assert_equal 12.5, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+        assert_equal 11.5, DataStore::Base.db[:ds_1_4].order(:created).last[:value]
+
+        @table.model.insert(value: 14, created: 40)
+        @table.model.insert(value: 15, created: 50)
+
+        time_now_utc_returns(50)
+
+        @calculator.perform
+
+        assert_equal 14.5, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+
+        @table.model.insert(value: 16, created: 60)
+        @table.model.insert(value: 17, created: 70)
+
+        time_now_utc_returns(70)
+
+        @calculator.perform
+
+        assert_equal 16.5, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+        assert_equal 15.5, DataStore::Base.db[:ds_1_4].order(:created).last[:value]
+        assert_equal 13.5, DataStore::Base.db[:ds_1_8].order(:created).last[:value]
+
+        assert_equal [:data_stores, :ds_1, :ds_1_2, :ds_1_4, :ds_1_8],  DataStore::Base.db.tables.sort
+      end
+    end
+
+    context 'Scenario: adding values with an unexpected failure' do
+      should 'calculate the average values' do
+        time_now_utc_returns(10)
+       
+        @table.model.insert(value: 10, created: 0)
+        @table.model.insert(value: 11, created: 10)
+
+        @calculator.perform
+
+        assert_equal 10.5, DataStore::Base.db[:ds_1_2].order(:created).first[:value]       
+
+        @table.model.insert(value: 12, created: 20)
+
+        #No value at timestamp 30!
+        @table.model.insert(value: 14, created: 40)
+
+        time_now_utc_returns(40)
+
+        @calculator.perform
+
+        assert_equal 13.0, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+        assert_equal 11.75, DataStore::Base.db[:ds_1_4].order(:created).last[:value]
+
+        @table.model.insert(value: 15, created: 50)
+        @table.model.insert(value: 16, created: 60)
+
+        time_now_utc_returns(60)
+
+        @calculator.perform
+
+        assert_equal 15.5, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+
+        @table.model.insert(value: 17, created: 70)
+        @table.model.insert(value: 18, created: 80)
+
+        time_now_utc_returns(80)
+
+        @calculator.perform
+
+        assert_equal 17.5, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+        assert_equal 16.5, DataStore::Base.db[:ds_1_4].order(:created).last[:value]
+        assert_equal 14.125, DataStore::Base.db[:ds_1_8].order(:created).last[:value]
+      end
+    end
+  end
+
+  context 'AverageCalculator for a counter type' do
+  
+    setup do
+      DataStore::Base.db.tables.each do |table|
+        DataStore::Base.db.drop_table(table)
+      end
+
+      DataStore::Connector.new.reset!
+      @record = DataStore::Base.create(identifier:  1,
+                                       type:        'counter', 
+                                       name:        'Electra',
+                                       frequency:    10,
+                                       description: 'Actual usage of gas in the home',
+                                       compression_schema: [2])
+
+      @table = DataStore::Table.new(1)
+      @calculator = DataStore::AverageCalculator.new(@table)
+    end
+
+    should 'be valid' do
+      assert @calculator
+    end
+
+    should 'return the identifier' do
+      assert_equal 1, @calculator.identifier
+    end
+
+    should 'calculate the average value' do
+      @table.model.insert(value: 10, original_value: 1010, created: 10)
+      @table.model.insert(value: 10, original_value: 1020, created: 20)
+      
+      @calculator.perform
+
+      assert_equal 10.0, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+    end
+
+    should 'calculate the average values according to compression_schema' do
+      @table.model.insert(value: 10, original_value: 1010, created: 10)
+      @table.model.insert(value: 10, original_value: 1020, created: 20)
+
+      time_now_utc_returns(20)      
+      @calculator.perform
+
+      @table.model.insert(value: 20, original_value: 1040, created: 30)
+      @table.model.insert(value: 30, original_value: 1070, created: 40)
+
+      time_now_utc_returns(40)
+      @calculator.perform
+     
+      assert_equal 25.0, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+
+      assert_equal [:data_stores, :ds_1, :ds_1_2],  DataStore::Base.db.tables.sort
+    end
+
+    should 'calculate the average value by ignoring the original values' do
+      @table.model.insert(value: 20, original_value: 12345, created: 10)
+      @table.model.insert(value: 30, original_value: 67890, created: 20)
+      
+      @calculator.perform
+
+      assert_equal 25.0, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+    end
+
+  end
+
+end

data/test/configuration_test.rb

@@ -0,0 +1,48 @@
+require File.expand_path '../test_helper', __FILE__
+
+class ConfigurationTest < Test::Unit::TestCase
+
+  context 'Configuration' do
+
+    should "provide default values" do
+      assert_config_default :prefix,               'ds_'
+      assert_config_default :database,             :postgres
+      assert_config_default :compression_schema,   [6,5,3,4,4,3]
+      assert_config_default :frequency,            10
+      assert_config_default :frequency_tolerance,  0.05
+      assert_config_default :maximum_datapoints,   800
+      assert_config_default :data_type,            :double
+      assert_config_default :database_config_file, File.expand_path('../../config/database.yml', __FILE__)
+      assert_config_default :enable_logging,       true
+      assert_config_default :log_file,             $stdout
+      assert_config_default :log_level,            Logger::ERROR
+    end
+
+    should "allow values to be overwritten" do
+      assert_config_overridable :prefix
+      assert_config_overridable :database
+      assert_config_overridable :compression_schema
+      assert_config_overridable :frequency
+      assert_config_overridable :frequency_tolerance
+      assert_config_overridable :maximum_datapoints
+      assert_config_overridable :data_type
+      assert_config_overridable :database_config_file
+      assert_config_overridable :enable_logging
+      assert_config_overridable :log_file
+      assert_config_overridable :log_level
+    end
+
+  end
+
+  def assert_config_default(option, default_value, config = nil)
+    config ||= DataStore::Configuration.new
+    assert_equal default_value, config.send(option)
+  end
+
+  def assert_config_overridable(option, value = 'a value')
+    config = DataStore::Configuration.new
+    config.send(:"#{option}=", value)
+    assert_equal value, config.send(option)
+  end
+
+end

data/test/connector_test.rb

@@ -0,0 +1,32 @@
+require File.expand_path '../test_helper', __FILE__
+
+class ConnectorTest < Test::Unit::TestCase
+
+  context 'DataStore::Connector connection with database' do
+
+    setup do
+      @connector = DataStore::Connector.new
+    end
+
+    should 'trigger the migration to create the database table' do
+      migration = mock
+      DataStore.expects(:create_data_stores).returns(migration)
+      migration.expects(:apply)
+      @connector.create_table!
+    end
+
+    should 'reset by dropping and recreating the database table' do
+      migration = mock
+      @connector.expects(:drop_table!)
+      DataStore.expects(:create_data_stores).returns(migration)
+      migration.expects(:apply)
+      @connector.reset!
+    end
+
+    teardown do
+      @connector.database.disconnect
+    end
+
+  end
+
+end

data/test/data_store_test.rb

@@ -0,0 +1,117 @@
+require File.expand_path '../test_helper', __FILE__
+
+class DataStoreTest < Test::Unit::TestCase
+
+  context 'DataStore configuration' do
+
+    should 'have a configuration object' do
+      assert_equal true, DataStore.configuration.is_a?(DataStore::Configuration)
+    end
+
+    should 'be able to define the configuration' do
+      assert_equal ENV['DB'] || :postgres, DataStore.configuration.database
+    end
+
+  end
+
+  context 'DataStore::Base general' do
+
+    setup do
+      DataStore::Connector.new.reset!
+    end
+
+    context 'with added behaviour through Sequel::Model' do
+
+      setup do
+        @record = DataStore::Base.create(identifier:         1,
+                                         type:               'gauge', 
+                                         name:               'Electra',
+                                         description:        'Actual usage of electra in the home',
+                                         compression_schema: [5,4,3])
+      end
+
+      should 'be valid' do
+        assert @record
+      end
+
+      should 'have added a record to the database' do
+        assert_equal 1, DataStore::Base.count
+      end
+
+      should 'have created the necessary tables' do
+        assert_equal 0, DataStore::Base.db[:ds_1].count
+        assert_equal 0, DataStore::Base.db[:ds_1_5].count
+        assert_equal 0, DataStore::Base.db[:ds_1_20].count
+        assert_equal 0, DataStore::Base.db[:ds_1_60].count
+      end
+
+      should 'return all table_names' do
+        assert_equal [:ds_1, :ds_1_5, :ds_1_20, :ds_1_60], @record.table_names
+      end
+
+      should 'return its time_borders' do
+        assert_equal [8000, 40000, 160000, 480000], @record.time_borders
+      end
+
+      should 'return its attributes' do
+        record = DataStore::Base.order(:created_at).last
+        assert_equal 1, record.identifier
+        assert_equal 'gauge', record.type
+        assert_equal 'Electra', record.name
+        assert_equal 'Actual usage of electra in the home', record.description
+        assert_equal [5,4,3], @record.compression_schema
+      end
+
+      should 'return default values if not set' do
+        assert_equal 10, @record.frequency
+        assert_equal 'double', @record.data_type
+        assert_equal 800, @record.maximum_datapoints
+      end
+
+      should 'have timestamps' do
+        assert @record.created_at
+        assert @record.updated_at
+      end
+
+      should 'create a record with a uniq identifier' do
+        assert_raise 'Sequel::DatabaseError(<SQLite3::ConstraintException: column identifier is not unique>)' do
+          DataStore::Base.create(identifier: 1, type: 'gauge', name: 'Electra')
+        end
+      end
+
+      should 'be able to update a record' do
+        @record.name = 'Gas'
+        @record.save
+        assert_equal 'Gas', DataStore::Base.order(:created_at).last.name
+      end
+
+    end
+
+    should 'create with the correct data type for value' do
+      record = DataStore::Base.create(identifier: 2, type: 'gauge', name: 'Electra', data_type: 'integer')
+      assert_equal :integer,Sequel::Model(DataStore::Base.db[:ds_2]).db_schema[:value][:type]
+      record.destroy
+    end
+
+    context 'handling of database tables for the datapoints' do
+
+      should 'create the necessary datapoint tables on create' do
+        DataStore::Base.any_instance.expects(:drop_tables!)
+        DataStore::Base.any_instance.expects(:create_tables!)
+        DataStore::Base.create(identifier: 1, type: 'gauge', name: 'Electra')
+      end
+
+      should 'destroy the corresponding datapoint tables on destroy' do
+        record = DataStore::Base.create(identifier: 1, type: 'gauge', name: 'Electra')
+        record.destroy        
+        assert_raise { DataStore::Base.db[:ds_1].count }
+        assert_raise { DataStore::Base.db[:ds_5].count }
+        assert_raise { DataStore::Base.db[:ds_20].count }
+        assert_raise { DataStore::Base.db[:ds_60].count }
+      end
+
+    end
+
+  end
+
+end

data/test/integration_test.rb

@@ -0,0 +1,66 @@
+require File.expand_path '../test_helper', __FILE__
+
+class IntegrationTest < Test::Unit::TestCase
+
+  context 'Integration test by adding datapoints through table object' do
+  
+    setup do
+      DataStore::Connector.new.reset!
+      @record = DataStore::Base.create(identifier:  1,
+                                       type:        'counter', 
+                                       name:        'Electra',
+                                       frequency:    10,
+                                       description: 'Actual usage of gas in the home',
+                                       compression_schema: [2,2])
+
+      @table = DataStore::Table.new(1)
+      @calculator = DataStore::AverageCalculator.new(@table)
+    end
+
+    should 'also calculate the average value' do
+      time_now_utc_returns(0)
+      @table.add(1000)
+
+      time_now_utc_returns(10)
+      @table.add(1010)
+
+      time_now_utc_returns(20)
+      @table.add(1020)
+
+      assert_equal 10.0, DataStore::Base.db[:ds_1_2].order(:created).last[:value]
+    end
+
+  end
+
+  context 'Import datapoints (gauge type)' do
+    setup do
+      start_time  = 1349042407.00000
+      values      = [2380.0, 2370.0, 2380.0, 2380.0, 2390.0, 2390.0, 2390.0, 2380.0, 2380.0, 2380.0, 2380.0, 2370.0, 2370.0, 2370.0, 
+                     2380.0, 2380.0, 2380.0, 2380.0, 230.0, 230.0, 230.0, 230.0, 230.0, 230.0]
+      @datapoints = []
+      values.each do |value|
+        @datapoints << [value, start_time]
+        start_time += rand(9.95..10.05)
+      end               
+      DataStore::Connector.new.reset!
+      @record = DataStore::Base.create(identifier:  1,
+                                       type:        'gauge', 
+                                       name:        'Electra',
+                                       description: 'Actual usage of electra in the home',
+                                       compression_schema: [2,3])
+      @table = DataStore::Table.new(1)
+    end
+
+    should 'store the data and calculate all averages' do
+      @table.import(@datapoints)
+      assert_equal 24, @table.model.db[:ds_1].count
+      assert_equal 12, @table.model.db[:ds_1_2].count
+      assert_equal 4, @table.model.db[:ds_1_6].count
+
+      assert_equal 1842, @table.model.db[:ds_1].avg(:value).round
+      assert_equal 1842, @table.model.db[:ds_1_2].avg(:value).round
+      assert_equal 1842, @table.model.db[:ds_1_6].avg(:value).round
+    end
+  end
+
+end