simple_stats_store 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 03dc5993bc4a5e7ec010d5601add13295a9d3cfc
+  data.tar.gz: cfe61488fb2db2b0a2562515ee6efc2fcde61501
+SHA512:
+  metadata.gz: 4bb37b8ff0d17d9d5bac6258bc6af92a840ba68dda73e3f50d45a247a10d37e5b27e13d885910e3fe51379c3b51a00975113cdf07401ae8776ae90ba0063f382
+  data.tar.gz: fd35eec5a1d2904de47e6da818ab395b16b887b605506d05e4d27a10d48a7e9cb4868a007dbca4bc9c1427300446363b3a77308472c2cdae4c1d64354b9b1a32

data/README.md

@@ -0,0 +1,89 @@
+# Simple Stats Store
+
+## Introduction
+
+The purpose of Simple Stats Store is to provide a simple and lightweight method
+for multiple processes to dump data into an SQLite database without contention.
+
+It is appropriate to be used when:
+
+* There are multiple threads or processes submitting statistics
+* Data concurrency is less important than avoiding waiting on database locks
+
+This is achieved by the threads dropping uniquely named files into a common
+directory containing the statistics which are then picked up by a single thread
+that is the sole accessor of the database.
+
+## Usage
+
+### General
+
+Create the repository for temporary data files:
+
+```ruby
+require 'simple_stats_store/file_dump'
+
+dir = '/path/to/temporary/data/directory'
+Dir.mkdir(dir)
+data_dump = SimpleStatsStore::FileDump.new(dir)
+```
+
+### Server
+
+Set up the database:
+
+```ruby
+require 'simple_stats_store/server'
+
+db_file = '/path/to/database.sql'
+ActiveRecord::Base.establish_connection(
+  adapter: :sqlite3,
+  database: db_file
+)
+ActiveRecord::Schema.define do
+  create_table :table do |table|
+    table.column :timestamp, :string
+    table.column :key_1, :integer
+    table.column :key_2, :float
+    # etc.
+  end
+end
+class Table < ActiveRecord::Base
+end
+
+ssss = SimpleStatsStore::Server.new(
+  data_dump: data_dump,
+  models: { table_ref: Table }
+)
+
+t_next = Time.new + 300
+server_pid = ssss.run do
+  if Time.new >= t_next
+    # Code to be executed every 5 minutes
+    # ...
+    t_next += 300
+  end
+end
+```
+
+### Client
+
+Write data
+
+```ruby
+data_dump.write(
+  table_ref,
+  {
+    timestamp: Time.new.to_s,
+    key_1: value_1,
+    key_2: value_2,
+    # etc.
+  }
+)
+```
+
+## License
+
+Simple Stats Store is available to everyone under the terms of the MIT open source licence. Take a look at the LICENSE file in the code.
+
+Copyright (c) 2015 BBC

data/lib/simple_stats_store.rb

@@ -0,0 +1,5 @@
+require 'simple_stats_store/server'
+require 'simple_stats_store/file_dump'
+
+module SimpleStatsStore
+end

data/lib/simple_stats_store/file_dump.rb

@@ -0,0 +1,40 @@
+require 'tempfile'
+
+module SimpleStatsStore
+  class FileDump
+    def initialize(dir)
+      @dir = dir
+    end
+
+    def files_contents
+      contents = []
+      Dir["#{@dir}/*"].each do |f|
+        data = File.open(f, 'r').read
+        if /\n---\n$/.match(data)
+          contents << data
+          File.delete(f)
+        end
+      end
+      contents
+    end
+
+    def each(&block)
+      files_contents.each &block
+    end
+
+    def write(model, data)
+      i = 0
+      while File.exists?(File.expand_path("sss-#{$$}-#{Time.new.to_i}-#{i}.stats", @dir))
+        i += 1
+      end
+      File.open(File.expand_path("sss-#{$$}-#{Time.new.to_i}-#{i}.stats", @dir), 'w') do |f|
+        f.puts "---"
+        f.puts model
+        data.each do |key, value|
+          f.puts "#{key.to_s}: #{value}"
+        end
+        f.puts "---"
+      end
+    end
+  end
+end

data/lib/simple_stats_store/nil_dump.rb

@@ -0,0 +1,7 @@
+module SimpleStatsStore
+  class NilDump
+    def each(&block)
+      [].each(&block)
+    end
+  end
+end

data/lib/simple_stats_store/server.rb

@@ -0,0 +1,37 @@
+module SimpleStatsStore
+  class Server
+    def initialize(options)
+      @data_dump = options[:data_dump]
+      @models = options[:models]
+    end
+
+    def scan
+      @data_dump.each do |stats|
+        lines = stats.split("\n")
+        if lines.shift != '---' or lines.pop != '---'
+          puts "Corrupt statistics"
+          return false
+        end
+
+        model = lines.shift.strip
+        data = {}
+        lines.each do |l|
+          k, v = l.split(/:/, 2)
+          data[k.strip.to_sym] = v.strip
+        end
+
+        @models[model.to_sym].create(data)
+      end
+    end
+
+    def run(&block)
+      Process.fork do
+        loop do
+          self.scan
+          yield if block_given?
+          sleep 0.1
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: simple_stats_store
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Joe Haig
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-15 00:00:00.000000000 Z
+dependencies: []
+description: Using SQLite3 to store statistics from a multithreaded application
+email: joe.haig@bbc.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/simple_stats_store.rb
+- lib/simple_stats_store/file_dump.rb
+- lib/simple_stats_store/nil_dump.rb
+- lib/simple_stats_store/server.rb
+homepage: https://github.com/bbc/simple_stats_store
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Simple Stats Store
+test_files: []