frame_tree 0.0.1.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8eef4bf9e7de2f6ff90c1998e0d9a9c8b5c1e70bbfa4beef5b7d79aaf70f1cfb
+  data.tar.gz: 6d0187ce44f22ed81c33d35c856211ce029dab995ae22adf3ec8c69a2508d8a1
+SHA512:
+  metadata.gz: e67540bd4fc3a030ad06d5b51fe88624e85f71a514902cb8050d762c3eb3446c7356d4b29f31ffb80fa00425f44a5d86a399ca4e3ad4c275aae806a1300e2abc
+  data.tar.gz: e468c128b8e0574583adfc9ccc534712dcb5344edb8f69bde8bd2eda30e14477683aa85ed8d0bf7bbe3ed4627d64db30cf85031d5e7646c182ec36f94be34164

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 creadone
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,87 @@
+# FrameTree
+
+It's a prototype of a fast in-memory tree-like structure to store and retrieve values from the counters through a rolling time-frame/window. FrameTree was created to reduce unnecessary database queries and to provide data to a sparkline from ui-component dataGrid in one of my projects. At the moment some features have been hardcoded, in particular the Frame, but in the future this is planned to rewrite more flexible.
+
+## Features:
+
+* FT is fast. Batch insert of 50 000 ordered items by 100 in batch took 425 ms and mixed 390 ms., for batch get 390 and 170 ms. accordingly. Update by previously inserted keys (no need to create new objects) took 150 ms. for both.
+* FT is simple. Only built-in Ruby objects, limited set of base operations: get/set and get_batch/set_batch, data can be dumped to well-known format MessagePack.
+
+## Structure:
+
+The FrameTree consists of four levels:
+* Tree — starting point and set of management methods. The Tree keeps a list of existing branches:
+```
++-----------+------+
+| Branch ID | Size |
++-----------+------+
+|         0 | 12   |
+|         1 | 10   |
+|         2 | 10   |
+|         3 | 10   |
+|         4 | 10   |
+|         5 | 10   |
+|         6 | 10   |
+|         7 | 10   |
+|         8 | 10   |
+|         9 | 10   |
+|        10 | 10   |
++-----------+------+
+```
+
+* Branch — section like partition in DB to store number of frames. The main role of the branchs is to increase the lookup speed. Testing has shown that it is most efficient to set the maximum number of branches to 1/5 of the total amount of data. Each branch keeps a list of existing frames:
+
+```
++----------+---+---+---+---+---+---+---+---+
+| Frame ID | T | 0 | 1 | 2 | 3 | 4 | 5 | 6 |
++----------+---+---+---+---+---+---+---+---+
+|    40000 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 |
+|        0 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 |
+|    30000 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 |
+|    10000 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 |
+|    50000 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 |
+|    20000 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 |
++----------+---+---+---+---+---+---+---+---+
+```
+
+* Frame — group of counters like a table
+* Counter — simple class with increment, decrement and replace operations.
+
+## Install
+```
+gem install frame_tree --pre
+```
+
+## Usage
+
+```Ruby
+require 'frame_tree'
+
+FrameTree.setup do |s|
+  s.max_branches = 2_000    # Max possible branches, better 1/5 of the total
+  s.frame_length = 6        # Number of days for which values are stored
+  s.data_path    = '/tmp'   # Path to directory to drop dumps
+end
+
+# Init
+storage = FrameTree::Tree.new :db_name
+
+# Insert only key (will be considered an increment)
+storage.set 13441946 # => 1
+
+# Output values will be in the format [total, sum of 7 day, sum of 6 day, sum of 5 day, ..., sum of current day]
+# total — is not a sum of 7 days, it's just value that will be incrementing.
+storage.get 13441946 # => [1, 0, 0, 0, 0, 0, 0, 1]
+
+# Will return ordered arrays of counters value
+storage.get_batch [key1, key2, key3, ...]
+
+# will return array of total of each key
+storage.set_batch [key1, key2, key3, ...]
+
+# Dump data to file
+storage.dump('path_to_file')
+
+# Load data from file
+storage.restore('path_to_file')
+```

data/lib/frame_tree.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'msgpack'
+require_relative 'frame_tree/config'
+require_relative 'frame_tree/helper'
+require_relative 'frame_tree/counter'
+require_relative 'frame_tree/branch'
+require_relative 'frame_tree/error'
+require_relative 'frame_tree/frame'
+require_relative 'frame_tree/tree'
+
+module FrameTree
+end

data/lib/frame_tree/branch.rb

@@ -0,0 +1,74 @@
+module FrameTree
+  class Branch
+    attr_reader :size
+    attr_accessor :frames
+
+    class << self
+      def from_hash id, data
+        branch = Branch.new id
+        frames = data.each_with_object({}) do |(idx, val), acc|
+          acc[idx] = Frame.from_hash idx, val
+        end
+        branch.frames = frames
+        branch
+      end
+    end
+
+    def initialize id
+      @id     = id
+      @size   = 0
+      @frames = {}
+    end
+
+    def get(key)
+      if @frames.has_key?(key)
+        @frames[key].get
+      end
+    end
+
+    def set(key, val = nil)
+      if @frames.key? key
+        @frames[key].set val
+      else
+        @frames[key] = Frame.new @frames.size + 1
+        @frames[key].set val
+      end
+      @size += 1
+    end
+
+    # def [](key)
+    #   get key
+    # end
+
+    def frame(id)
+      @frames[id]
+    end
+
+    def to_hash
+      @frames.each_with_object({}) do |(k,v), acc|
+        acc[k] = v.to_h
+      end
+    end
+
+    def to_s
+      "<FrameTree::Branch @id=#{@id},
+        @size=#{@size},
+        @frames=[#{@frames.keys.first(3).join(', ')}, ...]>"
+    end
+
+    def to_table
+      Helper.render do |rows, headings|
+        headings << ['Frame ID', 'T', 0, 1, 2, 3, 4, 5, 6]
+        @frames.each do |(id, frame)|
+          rows << [id, frame.values].flatten
+        end
+      end
+    end
+
+    def each
+      @frames.each do |frame|
+        yield frame
+      end
+    end
+  end
+end

data/lib/frame_tree/config.rb

@@ -0,0 +1,27 @@
+module FrameTree
+  class << self
+    attr_accessor :config
+  end
+
+  class Config
+    attr_accessor :frame_length
+    attr_accessor :max_branches
+    attr_accessor :data_path
+    attr_accessor :tree
+
+    def initialize
+      @frame_length = 6
+      @max_branches = 1
+      @data_path    = '/tmp'
+      @tree = nil
+    end
+  end
+
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.setup
+    yield(config)
+  end
+end

data/lib/frame_tree/counter.rb

@@ -0,0 +1,37 @@
+module FrameTree
+  class Counter
+    def initialize id
+      @id      = id
+      @storage = 0
+    end
+
+    def get
+      @storage
+    end
+
+    def set(val)
+      @storage = if val.nil?
+        @storage += 1
+      else
+        @storage += val
+      end
+    end
+
+    def incr
+      @storage = @storage + 1
+    end
+
+    def decr
+      @storage = @storage - 1
+    end
+
+    def clear
+      @storage = 0
+    end
+
+    def to_s
+      "<FrameTree::Counter @id=#{@id}, @value=#{@storage}>"
+    end
+
+  end
+end

data/lib/frame_tree/error.rb

@@ -0,0 +1,20 @@
+module FrameTree
+  class WrongInputType < ArgumentError
+    def initialize(msg = 'Key or Value must be Integer')
+      super(msg)
+    end
+  end
+
+  class WrongDataLength < ArgumentError
+    def initialize(msg = 'Data must be the same length as Frame')
+      super(msg)
+    end
+  end
+
+  class MissingOption < ArgumentError
+    def initialize(option)
+      @msg = "Missing config option #{option}"
+      super(@msg)
+    end
+  end
+end

data/lib/frame_tree/frame.rb

@@ -0,0 +1,120 @@
+module FrameTree
+  class Frame
+    attr_reader   :size
+    attr_accessor :total
+    attr_accessor :config
+    attr_accessor :counters
+
+    class << self
+      def from_hash id, data
+        frame = Frame.new id
+        frame.total = [data.shift].to_h['total']
+        frame.counters = data.each_with_object({}) do |(idx, val), acc|
+          acc[idx] = Counter.new idx
+          acc[idx].set val
+        end
+        frame
+      end
+    end
+
+    def initialize(id, config: FrameTree.config)
+      @config       = config
+      @id           = id
+      @wday         = Time.now.wday
+      @size         = 0
+      @total        = Counter.new :total
+      @count_num    = @config.frame_length
+      @counters     = setup_counters
+      @last_access  = nil
+    end
+
+    def set val = nil
+      update { val }
+    end
+
+    def get
+      [@total.get] + @counters.values.map(&:get)
+    end
+
+    def [](idx)
+      @counters[idx]
+    end
+
+    def values
+      get
+    end
+
+    def to_h
+      (['total'] + @counters.keys).zip(values).to_h
+    end
+
+    def to_table
+      Helper.render do |rows, headers|
+        headers << ['ID', 'Value']
+        rows << ['T', @total.get]
+        @counters.each do |(idx, value)|
+          rows << [idx, value]
+        end
+      end
+    end
+
+    def to_s
+      "<FrameTree::Frame @id=#{@id}, @size=#{@size}, @counters=[#{values.join(', ')}]>"
+    end
+
+    private
+
+    def update
+      time_now = Time.now
+      current_wday = time_now.wday
+      rotate_counters if @wday != current_wday
+      data = yield
+      if data.is_a?(Hash) || data.is_a?(Array)
+        update_counters data
+      else
+        active_counter.incr
+        @total.incr
+      end
+      @wday = current_wday
+      @last_access = time_now.to_i
+      values
+    end
+
+    def update_counters data
+      data = data.values if data.is_a? Hash
+      raise WrongDataLength.new unless data.size == @count_num
+      raise WrongInputType.new  unless data.all?{ |val| val.is_a?(Integer) }
+
+      @total.set data.shift
+      data.each_with_index do |val, idx|
+        @counters[idx].set val
+      end
+    end
+
+    def rotate_counters
+      @counters.shift
+      tmp_hash = @counters.each_with_object({}) do |(pos, val), acc|
+        acc[pos - 1] = val
+      end
+      tmp_hash[tmp_hash.size] = Counter.new tmp_hash.size
+      @counters = tmp_hash
+    end
+
+    def setup_counters
+      unless @count_num.is_a?(Integer)
+        raise MissingOption.new('frame_length')
+      end
+      tmp_hash = {}
+      (0..@count_num).each do |idx|
+        tmp_hash[idx] = Counter.new idx
+        @size = idx
+      end
+      tmp_hash
+    end
+
+    def active_counter
+      counters[@count_num]
+    end
+
+  end
+end

data/lib/frame_tree/helper.rb

@@ -0,0 +1,21 @@
+require 'terminal-table'
+
+module FrameTree
+  module Helper
+    class << self
+      def fetch_obj(oid)
+        ObjectSpace._id2ref oid
+      end
+
+      def render
+        rows = []
+        headings = []
+        yield rows, headings
+        table = Terminal::Table.new rows: rows
+        table.align_column(0, :right)
+        table.headings = headings
+        table
+      end
+    end
+  end
+end

data/lib/frame_tree/tree.rb

@@ -0,0 +1,117 @@
+module FrameTree
+  class Tree
+    attr_reader :size
+    attr_reader :name
+    attr_reader :statistics
+    attr_accessor :branches
+    attr_accessor :config
+
+    def initialize(name, config = FrameTree.config)
+      @config       = config
+      @config.tree  = self
+      @name         = name
+      @size         = 0
+      @today        = Time.now.wday
+      @branches     = {}
+      @max_branches = @config.max_branches
+      @statistics   = {
+        get: 0, set: 0, lookup: 0,
+      }
+    end
+
+    # Select and insert data
+    def set(key, val = nil)
+      @statistics[:set] += 1
+      with_branch_for(key) do |branch|
+        branch.set key, val
+      end
+    end
+
+    def get(key)
+      @statistics[:get] += 1
+      with_branch_for(key) do |branch|
+        branch.get key
+      end
+    end
+
+    def set_batch(items)
+      items.map do |(key, val)|
+        set key, val
+      end
+    end
+
+    def get_batch(keys)
+      keys.map do |key|
+        get key
+      end
+    end
+
+    def branch(id)
+      @branches[id]
+    end
+
+    # Explore structure
+    def to_table
+      Helper.render do |rows, headings|
+        headings << ['Branch ID', 'Size']
+        @branches.each do |(id, branch)|
+          rows << [id, branch.size]
+        end
+      end
+    end
+
+    def each
+      @branches.each do |branch|
+        yield branch
+      end
+    end
+
+    def to_hash
+      @branches.each_with_object({}) do |(k,v), acc|
+        acc[k] = v.to_hash
+      end
+    end
+
+    def from_hash data
+      @branches = data.each_with_object({}) do |(idx, values), acc|
+        acc[idx] = Branch.from_hash idx, values
+      end
+    end
+
+    # Dump and restore data
+    def dump(file_path = nil)
+      file_path ||= File.join(@config.data_path, "#{Time.now.to_i.to_s}.bin")
+      File.open(file_path, 'wb') do |io|
+        io << MessagePack.pack(to_hash)
+      end
+      file_path
+    end
+
+    def restore(file_path = nil)
+      data = nil
+      file_path ||= Dir.glob(File.join(@config.data_path, '*.bin')).sort.last
+      File.open(file_path, 'rb') do |io|
+        data = MessagePack.unpack(io.read)
+      end
+      from_hash data
+    end
+
+    private
+
+    def hash(key)
+      @statistics[:lookup] += 1
+      key % @max_branches
+    end
+
+    def with_branch_for(key)
+      index = hash key
+      if @branches.has_key? index
+        @branches[index]
+      else
+        @size += 1
+        @branches[index] = Branch.new index
+      end
+      yield @branches[index]
+    end
+  end
+end

data/lib/frame_tree/version.rb

@@ -0,0 +1,3 @@
+module FrameTree
+  VERSION = '0.0.1.pre'
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: frame_tree
+version: !ruby/object:Gem::Version
+  version: 0.0.1.pre
+platform: ruby
+authors:
+- Sergey Fedorov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-02-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- creadone@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- lib/frame_tree.rb
+- lib/frame_tree/branch.rb
+- lib/frame_tree/config.rb
+- lib/frame_tree/counter.rb
+- lib/frame_tree/error.rb
+- lib/frame_tree/frame.rb
+- lib/frame_tree/helper.rb
+- lib/frame_tree/tree.rb
+- lib/frame_tree/version.rb
+homepage: https://github.com/creadone/frame_tree
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: In-memory Tree-like structure to store counters with rolling frame/window.
+test_files: []