data_store 0.0.1 → 0.0.2

data/.gitignore

@@ -1,8 +1,11 @@
 *.gem
 *.rbc
+*.log
 .bundle
 .config
 .yardoc
+.rbx/
+config/emonweb.yml
 Gemfile.lock
 InstalledFiles
 _yardoc

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.3@data_store --create

data/.travis.yml

@@ -0,0 +1,27 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - rbx-19mode
+  - jruby-19mode
+
+matrix:
+  exclude:
+    - rvm: jruby-19mode
+      env: DB=sqlite
+    - rvm: rbx-19mode
+      env: DB=sqlite
+
+branches:
+  only:
+    - master
+
+script: "bundle exec rake test"
+
+env:
+  - DB=postgres
+  - DB=mysql
+  - DB=sqlite
+
+before_script:
+  - psql -c 'create database data_store_test;' -U postgres
+  - mysql -e 'create database data_store_test'

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+##0.0.2
+
+  * Readme driven development
+  * Configuration of DataStore with Connector class for database connection and dataset definition
+  * Introduction of DataStore::Base enriched behaviour with the use of Sequel::Model
+  * introduction of the DataStore::Table to add datapoints
+
+##0.0.1
+
+  * Initial commit and release of gem

data/Gemfile

@@ -2,3 +2,33 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in data_store.gemspec
 gemspec
+
+gem 'rake'
+gem 'sequel'
+
+platforms :jruby do
+  gem 'jdbc-mysql'
+  gem 'jdbc-sqlite3'
+  gem 'jdbc-postgres', '9.1.901' #9.2 throws an error NameError: missing class or uppercase package name (`org.postgresql.Driver')
+end
+
+platforms :ruby do
+  gem 'mysql2'
+  gem 'sqlite3'
+  gem 'pg'
+end
+
+gem 'celluloid'
+
+group :test, :development do
+  gem 'shoulda'
+  gem 'mocha'
+  gem 'guard'
+  gem 'guard-test'
+  gem 'rb-fsevent'
+  gem 'pry'
+  platforms :ruby do
+    gem 'yard'
+    gem 'redcarpet'
+  end
+end

data/Guardfile

@@ -0,0 +1,10 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :test do
+  watch(%r{^lib/(.+)\.rb$})  { |m| "test/#{m[1]}_test.rb" }
+  watch(%r{^lib/data_store/(.+)\.rb$})  { |m| "test/#{m[1]}_test.rb" }
+  watch(%r{^test/.+_test\.rb$})
+  watch('test/test_helper.rb')  { "test" }
+  watch(%r{^lib/data_store/base.rb$})  { "test" }
+end

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012 Frank Oxener
+Copyright (c) 2012 Agile Dovadi B.V. - Frank Oxener
 
 MIT License
 

data/README.md

@@ -1,6 +1,10 @@
 # DataStore
 
-TODO: Write a gem description
+<a href='http://travis-ci.org/dovadi/data_store'>
+![http://travis-ci.org/dovadi/data_store](https://secure.travis-ci.org/dovadi/data_store.png)
+</a>
+
+DataStore is designed to store real time data and manage the growth of your dataset by deciding the time period of your historical data. DataStore is tested with Ruby 1.9.3, Rubinius and JRuby and works with three database adapters Sqlite, Mysql and Postgresql.
 
 ## Installation
 
@@ -18,7 +22,86 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### Configuration
+
+    DataStore.configure do |config|
+      config.prefix              = 'ds_'
+      config.database            = :mysql
+      config.compression_schema  = [6,5,3]
+      config.data_type           = :double
+      config.frequency           = 10
+      config.maximum_datapoints  = 800
+      config.log_file            = 'data_store.log'
+      config.log_level           = Logger::INFO
+    end
+
+### Creation of a DataStore
+
+    DataStore::Base.create(identifier: 1, type: 'gauge', name: 'Electra', description: 'Actual usage of electra in the home')
+
+This will result in the creation of 4 tables,
+
+    'ds_1'
+    'ds_1_6'
+    'ds_1_30'
+    'ds_1_90'
+
+with the following structure:
+
+    id:      integer
+    value:   double
+    created: double #for unix timestamp
+
+and a record to the main data_stores table with the corresponding field names
+
+    id
+    name
+    description
+    compression_schema
+    frequency
+    maximum_datapoints
+    data_type
+
+### Add a datapoint
+
+    table = DataStore::Table.new(1)
+    table.add(120.34)
+    table.add(123.09)
+    table.add(125.01)
+
+### Fetching datapoints
+
+    DataStore::Table.new(1).fetch(:from => (Time.now.utc - 3600).to_f, :till => Time.now.utc.to_f)
+
+will result in an array of the maximum data points. An data point consists of an unix timestamp (UTC) and a value
+
+    [[1352668356, 120], [1352678356, 123.09], [1352688356, 125.01]]
+
+### Getting meta data of your data set
+
+    DataStore::Table.new(1).parent
+
+will return the corresponding record from the general data_stores table
+
+or more specific count of the number of records
+
+    DataStore::Table.new(1).count #=> 1249336
+
+last record
+
+    DataStore::Table.new(1).last
+
+results
+
+    #< @values={:id=>2, :value=>120.38, :created=>1356621436.67489}>
+
+### Managing the size of your data set
+
+### Export a data store (NOT implemented yet)
+
+    DDataStore::Table.new(1).export
+    
+will result in a csv file with the name data_store_1.csv
 
 ## Contributing
 
@@ -27,3 +110,9 @@ TODO: Write usage instructions here
 3. Commit your changes (`git commit -am 'Added some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+## Copyright
+
+Copyright (c) 2013 Agile Dovadi BV - Frank Oxener.
+
+See LICENSE.txt for further details.

data/REMINDERS

@@ -0,0 +1,8 @@
+Publish gem
+===========
+
+# git push
+# git tag -a 0.0.1 -m "Initial commit"
+# git push --tags
+# gem build data_store.gemspec 
+# gem push data_store-0.0.1.gem 

data/Rakefile

@@ -1,2 +1,14 @@
 #!/usr/bin/env rake
-require 'bundler/gem_tasks'
+# encoding: utf-8
+require 'rubygems'
+require 'bundler/gem_helper'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/config/database.yml

@@ -0,0 +1,16 @@
+postgres:
+  adapter: postgres
+  username: postgres
+  password:
+  database: data_store_test
+  host: localhost
+
+sqlite:
+  adapter: sqlite
+  database: db/data_store.db
+
+mysql:
+  adapter: mysql2
+  username: root
+  database: data_store_test
+  host: localhost

data/data_store.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
   gem.email         = ['frank.oxener@gmail.com']
   gem.description   = %q{DataStore is designed to store real time data but still manage the growth of your dataset and still keeping historical data}
   gem.summary       = %q{DataStore for storing real time data}
-  gem.homepage      = ''
+  gem.homepage      = 'https://github.com/dovadi/data_store'
 
   gem.licenses    = ['MIT']
   gem.files         = `git ls-files`.split($\)
@@ -15,7 +15,4 @@ Gem::Specification.new do |gem|
   gem.name          = 'data_store'
   gem.require_paths = ['lib']
   gem.version       = DataStore::VERSION
-
-  gem.add_development_dependency 'shoulda'
-  gem.add_development_dependency 'rspec'
 end

data/lib/data_store/average_calculator.rb

@@ -0,0 +1,93 @@
+module DataStore
+
+  class AverageCalculator
+
+    TIMESTAMP_CORRECTION_FACTOR = 0.0001
+
+    attr_reader :identifier, :base, :table_index, :table
+
+    def initialize(table)
+      @table       = table
+      @identifier  = table.identifier
+      @table_index = table.table_index
+      @base        = Base.find(identifier: identifier)
+    end
+
+    # Calculate average value if needed
+    # Average value is store dthrough an add call by a Table object
+    # So the average calculator is called again recursively
+    def perform
+      if calculation_needed?
+        average = previous_average_record ? calculate! : dataset.avg(:value)
+        table.add(average, table_index: table_index + 1, created: last[:created], type: :gauge)
+      end
+    end
+
+    private
+
+    def calculate!
+      last_time = previous_average_record[:created] + TIMESTAMP_CORRECTION_FACTOR
+      dataset.where{created > last_time}.avg(:value)
+    end
+
+    def calculation_needed?
+      return false if compression_finished
+      if previous_average_record
+        tolerance = DataStore.configuration.frequency_tolerance
+        time_difference_since_last_calculation >= (time_resolution - (time_resolution * tolerance)) 
+      else
+        dataset.count == base.compression_schema[table_index]
+      end
+    end
+
+    def time_difference_since_last_calculation
+      last[:created].round - previous_average_record[:created].round
+    end
+
+    def time_resolution
+      table.parent.frequency * compression_factors[table_index]
+    end
+
+    def compression_factors
+      array, factor = [], 1
+      base.compression_schema.each do |compression|
+        factor = (factor * compression)
+        array << factor
+      end
+      array
+    end
+
+    def previous_average_record
+      base.db[next_table].order(:created).last
+    end
+
+    def compression_finished
+      table_index == base.compression_schema.size
+    end
+
+    def last
+      dataset.order(:created).last
+    end
+
+    def dataset
+      base.db[table_name]
+    end
+
+    def table_name
+      if table_index == 0
+        prefix.chop.to_sym
+      else
+        (prefix + compression_factors[table_index - 1].to_s).to_sym
+      end
+    end
+
+    def next_table
+      (prefix + compression_factors[table_index].to_s).to_sym
+    end
+
+    def prefix
+      DataStore.configuration.prefix + identifier.to_s + '_'
+    end
+  end
+
+end

data/lib/data_store/base.rb

@@ -0,0 +1,108 @@
+module DataStore
+
+  class Base
+
+    # Set the default values with the globally defined values
+    #  * :compression_schema
+    #  * :frequency
+    #  * :maximum_datapoints
+    #  * :data_type
+    # See {Configuration}
+    def before_save
+      set_default_values  
+    end
+
+    def after_create
+      drop_tables!
+      create_tables!
+    end
+
+    def before_destroy
+      drop_tables!
+    end
+
+    # Convert serialized compression schema as a string back into the array object itself.
+    # For example: "[5,4,3]" => [5,4,3]
+    def compression_schema
+      value = self.values[:compression_schema]
+      value.gsub(/\[|\]/,'').split(',').map(&:to_i) unless value.nil?
+    end
+
+    def table_names
+      names  = [table_name]
+      factor = 1
+      compression_schema.each do |compression|
+        factor = (factor * compression)
+        names << (table_name.to_s + '_' + factor.to_s).to_sym
+      end
+      names
+    end
+
+    def time_borders
+      width = time_width
+      borders = [width]
+      compression_schema.each do |compression|
+        width =  width * compression
+        borders << width
+      end
+      borders
+    end
+
+    def time_width
+      DataStore.configuration.frequency * DataStore.configuration.maximum_datapoints
+    end
+
+    private
+
+    def default_values
+      ['compression_schema', 'frequency', 'maximum_datapoints', 'data_type']
+    end
+
+    def set_default_values
+      default_values.each do |variable|
+        value = DataStore.configuration.send(variable)
+        self.send(variable+ '=', value) if self.send(variable).nil?
+      end
+    end
+
+    # Create the database tables which are used for storing the datapoints
+    def create_tables!
+      migrate(:up)
+    end
+
+    # Drop the database tables which are used for storing the datapoints
+    def drop_tables!
+      migrate(:down)
+    end
+
+    def migrate(direction = :up)
+      # Establish new connection to prevent mix up with associated db connection of the Base object
+      # Unless connected to a sqlite db, otherwise it is too time consuming
+      database = sqlite_db? ? db : DataStore::Connector.new.database
+      table_names.each do |name|
+        begin
+          settings = {name: name, data_type: data_type}
+          settings[:original_value] = type == 'counter'
+          DataStore.create_table(settings).apply(database, direction)
+        rescue Sequel::DatabaseError => e
+          raise e if e.message.include?('FATAL')
+        end
+      end
+      database.disconnect unless sqlite_db?
+    end
+
+    def table_name
+      (prefix + identifier.to_s).to_sym
+    end
+
+    def prefix
+      DataStore.configuration.prefix
+    end
+
+    def sqlite_db?
+      DataStore.configuration.database.to_s == 'sqlite'
+    end
+
+  end
+
+end

data/lib/data_store/configuration.rb

@@ -0,0 +1,66 @@
+module DataStore
+
+  # Used to set up and modify settings for the data store.
+  class Configuration
+
+    # The prefix is used as a prefix for the database table name.
+    attr_accessor :prefix
+
+    # The database used for storing the data.
+    attr_accessor :database
+
+    # The schema is the way avarages of the datapoints is calculated/
+    #  Default: [6,5,3,4,4,3]
+    attr_accessor :compression_schema
+
+    # The frequency tells how often data entry is done.
+    # A frequency of 10 means a data entry once every 10 seconds.
+    #  Default: 10 sec
+    attr_accessor :frequency
+
+    # Tolerance of the frequency in which datapoints are added
+    #  Default: 0.05 
+    # This means a 5% margin. So with a frequency of 10s, 
+    # the next datapoint within 9.95 - 10.5 is considered the next datapoint
+    attr_accessor :frequency_tolerance
+
+    # The maximum datapoints is the maximum number of datapoint within a given timeframe
+    #  Default: 800
+    attr_accessor :maximum_datapoints
+
+    # The data type in which the value is stored
+    #  Default: double
+    attr_accessor :data_type
+
+    #The location of the database.yml file
+    attr_accessor :database_config_file
+
+    #Enable logging. 
+    #  Default: true
+    attr_accessor :enable_logging
+
+    #The location of the log file
+    #  Default $stdout
+    attr_accessor :log_file
+
+    #The level of logging
+    #  Default: Logger::ERROR
+    attr_accessor :log_level
+
+    def initialize
+      @prefix                      = 'ds_'
+      @database                    = :postgres
+      @compression_schema          = [6,5,3,4,4,3]
+      @frequency                   = 10
+      @maximum_datapoints          = 800
+      @data_type                   = :double
+      @database_config_file        = File.expand_path('../../../config/database.yml', __FILE__)
+      @log_file                    = $stdout
+      @log_level                   = Logger::ERROR
+      @enable_logging              = true
+      @frequency_tolerance         = 0.05
+    end
+
+  end
+
+end

data/lib/data_store/connector.rb

@@ -0,0 +1,85 @@
+module DataStore
+
+  class Connector
+
+    # Create the data_stores table with the following attributes
+    #
+    # * primary_key :id
+    # * Integer     :identifier, unique: true, null: false
+    # * String      :name, null: false
+    # * String      :type, null: false
+    # * String      :description
+    # * DateTime    :created_at
+    # * DateTime    :updated_at
+    #
+    def create_table!
+      DataStore.create_data_stores.apply(database, :up)
+    rescue Sequel::DatabaseError => e
+      raise e if e.message.include?('FATAL')
+    end
+
+    # Drop data_stores table and recreate it
+    def reset!
+      drop_table!
+      create_table!
+      disconnect
+    end
+
+    # Return the dataset associated with data_stores
+    def dataset
+      @dataset ||= begin
+        create_table!
+        database[:data_stores]
+      end
+    end
+
+    # Return the database object to which its connected.
+    def database
+      @database ||= begin
+        if RUBY_PLATFORM == 'java'
+          Sequel.connect(jdbc_settings)
+        else
+          Sequel.connect(database_settings)
+        end
+      end
+    end
+
+    private
+
+    def jdbc_settings
+      settings = database_settings
+      db =  case settings['adapter']
+            when 'postgres'
+              'postgresql'
+            when 'mysql2'
+              'mysql'
+            else
+              settings['adapter']
+            end
+      if db == 'sqlite'
+        uri = "jdbc:#{db}:#{settings['database']}"
+      else
+        uri = "jdbc:#{db}://#{settings['host']}/#{settings['database']}?user=#{settings['username']}"
+      end
+      settings['password'] ? uri + "&password=#{settings['password']}" : uri
+    end
+
+    def disconnect
+      database.disconnect
+      @database = nil
+    end
+ 
+    def drop_table!
+      DataStore.create_data_stores.apply(database, :down)
+    rescue Sequel::DatabaseError => e
+      raise e if e.message.include?('FATAL')
+    end
+
+    def database_settings
+      config_file = DataStore.configuration.database_config_file
+      YAML.load(File.open(config_file))[DataStore.configuration.database.to_s]
+    end
+
+  end
+
+end

data/lib/data_store/definitions.rb

@@ -0,0 +1,42 @@
+module DataStore
+
+  # Definition of the data_stores table
+  def self.create_data_stores
+    Sequel.migration do
+      change do
+        create_table(:data_stores) do
+          primary_key :id
+          Integer     :identifier, unique: true, null: false
+          String      :name, null: false
+          String      :type, null: false
+          String      :description
+          String      :data_type
+          String      :compression_schema
+          Integer     :frequency
+          Integer     :maximum_datapoints
+          DateTime    :created_at
+          DateTime    :updated_at
+          index       :identifier
+        end
+      end
+    end
+  end
+
+  def self.create_table(settings)
+    name           = settings[:name]
+    original_value = settings[:original_value] || false
+    data_type      = settings[:data_type]
+
+    Sequel.migration do
+      change do
+        create_table(name) do
+          primary_key :id
+          column :value, data_type
+          column :original_value, data_type if original_value
+          column :created, :double
+        end
+      end
+    end
+  end
+
+end