trifle-stats 0.4.1 → 1.1.2

data/.devops/docker/environment/.zshrc

@@ -0,0 +1,13 @@
+# Enable Powerlevel10k instant prompt. Should stay close to the top of ~/.zshrc.
+# Initialization code that may require console input (password prompts, [y/n]
+# confirmations, etc.) must go above this block; everything else may go below.
+if [[ -r "${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh" ]]; then
+  source "${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh"
+fi
+
+source /root/powerlevel10k/powerlevel10k.zsh-theme
+
+# To customize prompt, run `p10k configure` or edit ~/.p10k.zsh.
+[[ ! -f /root/.p10k.zsh ]] || source /root/.p10k.zsh
+
+POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD=true

data/.devops/docker/environment/Dockerfile

@@ -0,0 +1,54 @@
+FROM debian:latest
+
+ENV TZ=Europe/Bratislava
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# install ubuntu packages
+RUN apt-get update -q \
+ && apt-get install -y \
+    build-essential \
+    apt-transport-https \
+    zsh \
+    libpq-dev \
+    git \
+    curl \
+    wget \
+    unzip \
+    gpg \
+    gnupg2 \
+    locales \
+    autoconf \
+    libssl-dev \
+    libreadline-dev \
+    zlib1g-dev \
+    tmux \
+    htop \
+    vim \
+ && apt-get clean
+
+#set the locale
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh || true
+RUN git clone --depth=1 https://github.com/romkatv/powerlevel10k.git /root/powerlevel10k
+COPY .zshrc /root/.zshrc
+COPY .p10k.zsh /root/p10k.zsh
+
+#install asdf
+ENV ASDF_ROOT /root/.asdf
+ENV PATH "${ASDF_ROOT}/bin:${ASDF_ROOT}/shims:$PATH"
+RUN git clone https://github.com/asdf-vm/asdf.git ${ASDF_ROOT} --branch v0.10.0
+
+RUN asdf plugin-add ruby https://github.com/asdf-vm/asdf-ruby.git
+
+# install ruby
+ENV RUBY_VERSION 3.0.0
+RUN ASDF_RUBY_BUILD_VERSION=v20201225 asdf install ruby ${RUBY_VERSION} \
+  && asdf global ruby ${RUBY_VERSION}
+
+RUN gem install bundler
+
+CMD ["zsh"]

data/.gitpod.yml

@@ -1,5 +1,5 @@
 image:
-  file: .gitpod/Dockerfile
+  file: .devops/docker/gitpod/Dockerfile
 tasks:
   - init: bundle install
     command: ./bin/console

data/Gemfile.lock

@@ -1,10 +1,7 @@
 PATH
   remote: .
   specs:
-    trifle-stats (0.4.1)
-      mongo (>= 2.14.0)
-      pg (>= 1.2)
-      redis (>= 4.2)
+    trifle-stats (1.1.2)
       tzinfo (~> 2.0)
 
 GEM
@@ -13,7 +10,7 @@ GEM
     ast (2.4.2)
     bson (4.12.1)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     diff-lcs (1.4.4)
     dotenv (2.7.6)
     mongo (2.14.0)
@@ -52,7 +49,7 @@ GEM
     rubocop-ast (1.4.1)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
-    tzinfo (2.0.4)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
 
@@ -64,7 +61,10 @@ DEPENDENCIES
   bundler (~> 2.1)
   byebug
   dotenv
+  mongo (>= 2.14.0)
+  pg (>= 1.2)
   rake (~> 12.0)
+  redis (>= 4.2)
   rspec (~> 3.0)
   rubocop (= 1.0.0)
   trifle-stats!

data/README.md

@@ -8,7 +8,7 @@ Simple analytics backed by Redis, Postgres, MongoDB, Google Analytics, Segment,
 
 `Trifle::Stats` is a _way too_ simple timeline analytics that helps you track custom metrics. Automatically increments counters for each enabled range. It supports timezones and different week beginning.
 
-[^1]: TBH only Redis for now 💔.
+[^1]: TBH only Redis, Postgres and MongoDB for now 💔.
 
 ## Documentation
 
@@ -24,11 +24,22 @@ gem 'trifle-stats'
 
 And then execute:
 
-    $ bundle install
+```sh
+$ bundle install
+```
 
 Or install it yourself as:
 
-    $ gem install trifle-stats
+```sh
+$ gem install trifle-stats
+```
+
+Depending on driver you would like to use, make sure you add required gems into your `Gemfile`.
+```ruby
+gem 'mongo', '>= 2.14.0'
+gem 'pg', '>= 1.2'
+gem 'redis', '>= 4.2'
+```
 
 ## Usage
 
@@ -45,25 +56,72 @@ end
 
 ### Track values
 
-Available ranges are `:minute`, `:hour`, `:day`, `:week`, `:month`, `:quarter`, `:year`.
+Track your first metrics
 
-Now track your first metrics
 ```ruby
 Trifle::Stats.track(key: 'event::logs', at: Time.now, values: {count: 1, duration: 2, lines: 241})
 => [{2021-01-25 16:00:00 +0100=>{:count=>1, :duration=>2, :lines=>241}}, {2021-01-25 00:00:00 +0100=>{:count=>1, :duration=>2, :lines=>241}}]
-# or do it few more times
+```
+
+Then do it few more times
+
+```ruby
 Trifle::Stats.track(key: 'event::logs', at: Time.now, values: {count: 1, duration: 1, lines: 56})
 => [{2021-01-25 16:00:00 +0100=>{:count=>1, :duration=>1, :lines=>56}}, {2021-01-25 00:00:00 +0100=>{:count=>1, :duration=>1, :lines=>56}}]
 Trifle::Stats.track(key: 'event::logs', at: Time.now, values: {count: 1, duration: 5, lines: 361})
 => [{2021-01-25 16:00:00 +0100=>{:count=>1, :duration=>5, :lines=>361}}, {2021-01-25 00:00:00 +0100=>{:count=>1, :duration=>5, :lines=>361}}]
 ```
 
-### Get values
+You can also store nested counters like
+
+```ruby
+Trifle::Stats.track(key: 'event::logs', at: Time.now, values: {
+  count: 1,
+  duration: {
+    parsing: 21,
+    compression: 8,
+    upload: 1
+  },
+  lines: 25432754
+})
+```
+
+#### Get values
+
+Retrieve your values for specific `range`. Adding increments above will return sum of all the values you've tracked.
+
+```ruby
+Trifle::Stats.values(key: 'event::logs', from: Time.now, to: Time.now, range: :day)
+=> {:at=>[2021-01-25 00:00:00 +0200], :values=>[{"count"=>3, "duration"=>8, "lines"=>658}]}
+```
+
+### Assert values
+
+Asserting values works same way like incrementing, but instead of increment, it sets the value. Duh.
+
+Set your first metrics
+
+```ruby
+Trifle::Stats.assert(key: 'event::logs', at: Time.now, values: {count: 1, duration: 2, lines: 241})
+=> [{2021-01-25 16:00:00 +0100=>{:count=>1, :duration=>2, :lines=>241}}, {2021-01-25 00:00:00 +0100=>{:count=>1, :duration=>2, :lines=>241}}]
+```
+
+Then do it few more times
+
+```ruby
+Trifle::Stats.assert(key: 'event::logs', at: Time.now, values: {count: 1, duration: 1, lines: 56})
+=> [{2021-01-25 16:00:00 +0100=>{:count=>1, :duration=>1, :lines=>56}}, {2021-01-25 00:00:00 +0100=>{:count=>1, :duration=>1, :lines=>56}}]
+Trifle::Stats.assert(key: 'event::logs', at: Time.now, values: {count: 1, duration: 5, lines: 361})
+=> [{2021-01-25 16:00:00 +0100=>{:count=>1, :duration=>5, :lines=>361}}, {2021-01-25 00:00:00 +0100=>{:count=>1, :duration=>5, :lines=>361}}]
+```
+
+#### Get values
+
+Retrieve your values for specific `range`. As you just used `assert` above, it will return latest value you've asserted.
 
-Retrieve your values for specific `range`.
 ```ruby
 Trifle::Stats.values(key: 'event::logs', from: Time.now, to: Time.now, range: :day)
-=> [{2021-01-25 00:00:00 +0100=>{"count"=>3, "duration"=>8, "lines"=>658}}]
+=> {:at=>[2021-01-25 00:00:00 +0200], :values=>[{"count"=>1, "duration"=>5, "lines"=>361}]}
 ```
 
 ## Contributing

data/lib/trifle/stats/configuration.rb

@@ -6,12 +6,13 @@ module Trifle
   module Stats
     class Configuration
       attr_writer :driver
-      attr_accessor :track_ranges, :time_zone, :beginning_of_week
+      attr_accessor :track_ranges, :time_zone, :beginning_of_week, :designator
 
       def initialize
         @ranges = %i[minute hour day week month quarter year]
         @beginning_of_week = :monday
         @time_zone = 'GMT'
+        @designator = nil
       end
 
       def tz

data/lib/trifle/stats/designator/custom.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Designator
+      class Custom
+        attr_reader :buckets
+
+        def initialize(buckets:)
+          @buckets = buckets.sort
+        end
+
+        def designate(value:)
+          return buckets.first.to_s if value <= buckets.first
+          return "#{buckets.last}+" if value > buckets.last
+
+          (buckets.find { |b| value.ceil < b }).to_s
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/designator/geometric.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Designator
+      class Geometric
+        attr_reader :min, :max
+
+        def initialize(min:, max:)
+          @min = min.negative? ? 0 : min
+          @max = max
+        end
+
+        def designate(value:) # rubocop:disable Metrics/AbcSize
+          return min.to_f.to_s if value <= min
+          return "#{max.to_f}+" if value > max
+          return (10**value.floor.to_s.length).to_f.to_s if value > 1
+          return 1.0.to_s if value > 0.1 # ugh?
+
+          (1.0 / 10**value.to_s.gsub('0.', '').split(/[1-9]/).first.length).to_s
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/designator/linear.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Designator
+      class Linear
+        attr_reader :min, :max, :step
+
+        def initialize(min:, max:, step:)
+          @min = min
+          @max = max
+          @step = step.to_i
+        end
+
+        def designate(value:) # rubocop:disable Metrics/AbcSize
+          return min.to_s if value <= min
+          return "#{max}+" if value > max
+
+          (value.ceil / step * step + ((value.ceil % step).zero? ? 0 : step)).to_s
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/driver/README.md

@@ -2,9 +2,9 @@
 
 Driver is a wrapper class that persists and retrieves values from backend. It needs to implement:
 
-- `inc(key:, **values)` method increment values
-- `set(key:, **values)` method set values
-- `get(key:)` method to retrieve values
+- `inc(keys:, **values)` method increment values
+- `set(keys:, **values)` method set values
+- `get(keys:)` method to retrieve values
 
 ## Documentation
 

data/lib/trifle/stats/driver/mongo.rb

@@ -16,24 +16,27 @@ module Trifle
           @separator = '::'
         end
 
-        def inc(key:, **values)
-          pkey = key.join(separator)
+        def inc(keys:, **values)
+          data = self.class.pack(hash: { data: values })
 
           collection.bulk_write(
-            [upsert_operation('$inc', pkey: pkey, values: values)]
+            keys.map do |key|
+              upsert_operation('$inc', pkey: key.join(separator), data: data)
+            end
           )
         end
 
-        def set(key:, **values)
-          pkey = key.join(separator)
+        def set(keys:, **values)
+          data = self.class.pack(hash: { data: values })
 
           collection.bulk_write(
-            [upsert_operation('$set', pkey: pkey, values: values)]
+            keys.map do |key|
+              upsert_operation('$set', pkey: key.join(separator), data: data)
+            end
           )
         end
 
-        def upsert_operation(operation, pkey:, values:)
-          data = self.class.pack(hash: { data: values })
+        def upsert_operation(operation, pkey:, data:)
           {
             update_many: {
               filter: { key: pkey },
@@ -43,13 +46,12 @@ module Trifle
           }
         end
 
-        def get(key:)
-          pkey = key.join(separator)
-
-          data = collection.find(key: pkey).limit(1).first
-          return {} if data.nil? || data['data'].nil?
+        def get(keys:)
+          pkeys = keys.map { |key| key.join(separator) }
+          data = collection.find(key: { '$in' => pkeys })
+          map = data.inject({}) { |o, d| o.merge(d['key'] => d['data']) }
 
-          data['data']
+          pkeys.map { |pkey| map[pkey] || {} }
         end
 
         private

data/lib/trifle/stats/driver/postgres.rb

@@ -16,11 +16,13 @@ module Trifle
           @separator = '::'
         end
 
-        def inc(key:, **values)
-          pkey = key.join(separator)
+        def inc(keys:, **values)
+          keys.map do |key|
+            pkey = key.join(separator)
 
-          self.class.pack(hash: values).each do |k, c|
-            _inc_one(key: pkey, name: k, value: c)
+            self.class.pack(hash: values).each do |k, c|
+              _inc_one(key: pkey, name: k, value: c)
+            end
           end
         end
 
@@ -31,10 +33,12 @@ module Trifle
           client.exec(query)
         end
 
-        def set(key:, **values)
-          pkey = key.join(separator)
+        def set(keys:, **values)
+          keys.map do |key|
+            pkey = key.join(separator)
 
-          _set_all(key: pkey, **values)
+            _set_all(key: pkey, **values)
+          end
         end
 
         def _set_all(key:, **values)
@@ -44,13 +48,15 @@ module Trifle
           client.exec(query)
         end
 
-        def get(key:)
-          pkey = key.join(separator)
+        def get(keys:)
+          keys.map do |key|
+            pkey = key.join(separator)
 
-          data = _get(key: pkey)
-          return {} if data.nil?
+            data = _get(key: pkey)
+            return {} if data.nil?
 
-          self.class.unpack(hash: data)
+            self.class.unpack(hash: data)
+          end
         end
 
         def _get(key:)
@@ -59,7 +65,7 @@ module Trifle
           ).to_a.first
           return nil if result.nil?
 
-          JSON.parse(result.try(:fetch, 'data'))
+          JSON.parse(result['data'])
         rescue JSON::ParserError
           nil
         end

data/lib/trifle/stats/driver/process.rb

@@ -12,26 +12,32 @@ module Trifle
           @separator = '::'
         end
 
-        def inc(key:, **values)
-          self.class.pack(hash: values).each do |k, c|
-            d = @data.fetch(key.join(@separator), {})
-            d[k] = d[k].to_i + c
-            @data[key.join(@separator)] = d
+        def inc(keys:, **values)
+          keys.map do |key|
+            self.class.pack(hash: values).each do |k, c|
+              d = @data.fetch(key.join(@separator), {})
+              d[k] = d[k].to_i + c
+              @data[key.join(@separator)] = d
+            end
           end
         end
 
-        def set(key:, **values)
-          self.class.pack(hash: values).each do |k, c|
-            d = @data.fetch(key.join(@separator), {})
-            d[k] = c
-            @data[key.join(@separator)] = d
+        def set(keys:, **values)
+          keys.map do |key|
+            self.class.pack(hash: values).each do |k, c|
+              d = @data.fetch(key.join(@separator), {})
+              d[k] = c
+              @data[key.join(@separator)] = d
+            end
           end
         end
 
-        def get(key:)
-          self.class.unpack(
-            hash: @data.fetch(key.join(@separator), {})
-          )
+        def get(keys:)
+          keys.map do |key|
+            self.class.unpack(
+              hash: @data.fetch(key.join(@separator), {})
+            )
+          end
         end
       end
     end

data/lib/trifle/stats/driver/redis.rb

@@ -16,26 +16,32 @@ module Trifle
           @separator = '::'
         end
 
-        def inc(key:, **values)
-          pkey = ([prefix] + key).join(separator)
+        def inc(keys:, **values)
+          keys.map do |key|
+            pkey = ([prefix] + key).join(separator)
 
-          self.class.pack(hash: values).each do |k, c|
-            client.hincrby(pkey, k, c)
+            self.class.pack(hash: values).each do |k, c|
+              client.hincrby(pkey, k, c)
+            end
           end
         end
 
-        def set(key:, **values)
-          pkey = ([prefix] + key).join(separator)
+        def set(keys:, **values)
+          keys.map do |key|
+            pkey = ([prefix] + key).join(separator)
 
-          client.hmset(pkey, *self.class.pack(hash: values))
+            client.hmset(pkey, *self.class.pack(hash: values))
+          end
         end
 
-        def get(key:)
-          pkey = ([prefix] + key).join(separator)
+        def get(keys:)
+          keys.map do |key|
+            pkey = ([prefix] + key).join(separator)
 
-          self.class.unpack(
-            hash: client.hgetall(pkey)
-          )
+            self.class.unpack(
+              hash: client.hgetall(pkey)
+            )
+          end
         end
       end
     end

data/lib/trifle/stats/operations/timeseries/classify.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    module Operations
+      module Timeseries
+        class Classify
+          attr_reader :key, :values
+
+          def initialize(**keywords)
+            @key = keywords.fetch(:key)
+            @at = keywords.fetch(:at)
+            @values = keywords.fetch(:values)
+            @config = keywords[:config]
+          end
+
+          def config
+            @config || Trifle::Stats.default
+          end
+
+          def deep_classify(hash)
+            hash.transform_values do |value|
+              next deep_classify(value) if value.is_a?(Hash)
+
+              { classify(value) => 1 }
+            end
+          end
+
+          def classify(value)
+            config.designator.designate(value: value).to_s.gsub('.', '_')
+          end
+
+          def key_for(range:)
+            at = Nocturnal.new(@at, config: config).send(range)
+            [key, range, at.to_i]
+          end
+
+          def perform
+            config.driver.inc(
+              keys: config.ranges.map { |range| key_for(range: range) },
+              **deep_classify(values)
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/operations/timeseries/increment.rb

@@ -18,14 +18,16 @@ module Trifle
             @config || Trifle::Stats.default
           end
 
+          def key_for(range:)
+            at = Nocturnal.new(@at, config: config).send(range)
+            [key, range, at.to_i]
+          end
+
           def perform
-            config.ranges.map do |range|
-              at = Nocturnal.new(@at, config: config).send(range)
-              config.driver.inc(
-                key: [key, range, at.to_i],
-                **values
-              )
-            end
+            config.driver.inc(
+              keys: config.ranges.map { |range| key_for(range: range) },
+              **values
+            )
           end
         end
       end

data/lib/trifle/stats/operations/timeseries/set.rb

@@ -18,14 +18,16 @@ module Trifle
             @config || Trifle::Stats.default
           end
 
+          def key_for(range:)
+            at = Nocturnal.new(@at, config: config).send(range)
+            [key, range, at.to_i]
+          end
+
           def perform
-            config.ranges.map do |range|
-              at = Nocturnal.new(@at, config: config).send(range)
-              config.driver.set(
-                key: [key, range, at.to_i],
-                **values
-              )
-            end
+            config.driver.set(
+              keys: config.ranges.map { |range| key_for(range: range) },
+              **values
+            )
           end
         end
       end

data/lib/trifle/stats/operations/timeseries/values.rb

@@ -24,13 +24,14 @@ module Trifle
           end
 
           def perform
-            timeline.map do |at|
-              {
-                at => config.driver.get(
-                  key: [key, range, at.to_i]
-                )
-              }
-            end
+            {
+              at: timeline,
+              values: config.driver.get(
+                keys: timeline.map do |at|
+                  [key, range, at.to_i]
+                end
+              )
+            }
           end
         end
       end