trifle-stats 1.3.0 → 1.6.0

data/.devops/docker/gitpod/base/.zshrc

@@ -0,0 +1,13 @@
+# Enable Powerlevel10k instant prompt. Should stay close to the top of ~/.zshrc.
+# Initialization code that may require console input (password prompts, [y/n]
+# confirmations, etc.) must go above this block; everything else may go below.
+if [[ -r "${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh" ]]; then
+  source "${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh"
+fi
+
+source /root/powerlevel10k/powerlevel10k.zsh-theme
+
+# To customize prompt, run `p10k configure` or edit ~/.p10k.zsh.
+[[ ! -f /root/.p10k.zsh ]] || source /root/.p10k.zsh
+
+POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD=true

data/.devops/docker/gitpod/base/Dockerfile

@@ -1,41 +1,27 @@
-FROM ubuntu:20.04
+FROM debian:latest
 
 ENV TZ=Europe/Bratislava
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-RUN useradd gitpod -u 33333
-RUN mkdir -p /home/gitpod && chown gitpod:gitpod /home/gitpod
-
 # install ubuntu packages
 RUN apt-get update -q \
  && apt-get install -y \
     build-essential \
     apt-transport-https \
-    bash \
-    xterm \
-    xvfb \
-    x11vnc \
+    zsh \
     libpq-dev \
     git \
     curl \
     wget \
     unzip \
-    dirmngr \
     gpg \
     gnupg2 \
     locales \
     autoconf \
-    libncurses5-dev \
-    libgl1-mesa-dev \
-    libglu1-mesa-dev \
-    libpng-dev \
-    unixodbc-dev \
     libssl-dev \
     libreadline-dev \
     zlib1g-dev \
-    ffmpeg \
     tmux \
-    runit-systemd \
     htop \
     vim \
  && apt-get clean
@@ -46,23 +32,15 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
-RUN curl -fsSL https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -
-RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list
-
-RUN apt-get update -q \
- && apt-get install -y \
-    postgresql postgresql-contrib \
-    # mongodb-org \
-    redis-server \
-    mariadb-server \
- && apt-get clean
-
-USER gitpod
+RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh || true
+RUN git clone --depth=1 https://github.com/romkatv/powerlevel10k.git /root/powerlevel10k
+COPY .zshrc /root/.zshrc
+COPY .p10k.zsh /root/p10k.zsh
 
 #install asdf
-ENV ASDF_ROOT /home/gitpod/.asdf
+ENV ASDF_ROOT /root/.asdf
 ENV PATH "${ASDF_ROOT}/bin:${ASDF_ROOT}/shims:$PATH"
-RUN git clone https://github.com/asdf-vm/asdf.git ${ASDF_ROOT} --branch v0.8.0
+RUN git clone https://github.com/asdf-vm/asdf.git ${ASDF_ROOT} --branch v0.10.0
 
 RUN asdf plugin-add ruby https://github.com/asdf-vm/asdf-ruby.git
 
@@ -71,7 +49,6 @@ ENV RUBY_VERSION 3.0.0
 RUN ASDF_RUBY_BUILD_VERSION=v20201225 asdf install ruby ${RUBY_VERSION} \
   && asdf global ruby ${RUBY_VERSION}
 
-# throw errors if Gemfile has been modified since Gemfile.lock
 RUN gem install bundler
 
-CMD ["bash"]
+CMD ["zsh"]

data/.devops/docker/local/Dockerfile

@@ -0,0 +1 @@
+FROM trifle/environment:ruby_3.1.0_1

data/.devops/docker/local/docker-compose.yml

@@ -0,0 +1,30 @@
+version: "3.7"
+name: "stats"
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_PASSWORD: password
+  redis:
+    image: redis:latest
+  mongo:
+    image: mongo:latest
+  app:
+    command: /bin/sh -c "while sleep 1000; do :; done"
+    build:
+      context: ../../..
+      dockerfile: .devops/docker/local/Dockerfile
+    depends_on:
+      - postgres
+      - redis
+      - mongo
+    environment:
+      PGHOST: postgres
+      PGUSER: postgres
+      REDIS_HOST: redis
+      REDIS_URL: redis://redis:6379
+    expose:
+      - 4000
+    volumes:
+      - ../../..:/workspaces/stats
+    working_dir: /workspaces/stats

data/.github/workflows/ruby.yml

@@ -16,21 +16,100 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_DB: test_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+      mongodb:
+        image: mongo:6.0
+        env:
+          MONGO_INITDB_ROOT_USERNAME: root
+          MONGO_INITDB_ROOT_PASSWORD: password
+        options: >-
+          --health-cmd "mongosh --eval 'db.adminCommand(\"ping\")'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 27017:27017
+
     strategy:
       matrix:
-        ruby-version: ["3.0"]
+        ruby-version: ["3.1"]
+
+    env:
+      # Database configuration
+      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
+      REDIS_URL: redis://localhost:6379/0
+      MONGODB_URL: mongodb://root:password@localhost:27017/test_db?authSource=admin
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
       - name: Set up Ruby
-        # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
-        # change this to (see https://github.com/ruby/setup-ruby#versioning):
-        # uses: ruby/setup-ruby@v1
-        uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+        uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version }}
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Wait for services to be ready
+        run: |
+          # Wait for PostgreSQL
+          until pg_isready -h localhost -p 5432 -U postgres; do
+            echo "Waiting for PostgreSQL..."
+            sleep 2
+          done
+
+          # Wait for Redis
+          until timeout 1 bash -c "</dev/tcp/localhost/6379"; do
+            echo "Waiting for Redis..."
+            sleep 2
+          done
+
+          # Wait for MongoDB
+          until timeout 1 bash -c "</dev/tcp/localhost/27017"; do
+            echo "Waiting for MongoDB..."
+            sleep 2
+          done
+
+      - name: Setup Database
+        run: |
+          # Create database if needed (adjust based on your setup)
+          # bundle exec rails db:create db:migrate
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
+
       - name: Rspec
         run: bundle exec rspec
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
+          REDIS_URL: redis://localhost:6379/0
+          MONGODB_URL: mongodb://root:password@localhost:27017/test_db?authSource=admin
+
       - name: Rubocop
         run: bundle exec rubocop
+
+      # - name: Performance
+      #   run: cd spec/performance && bundle install && ruby run.rb 1000 '{"a":1,"b":2,"c":1,"d":2,"e":1,"f":2,"g":1,"h":2,"i":1,"j":2,"k":1,"l":2,"m":1,"n":2,"o":1,"p":2,"q":1,"r":2,"s":1,"t":2,"u":1,"v":2,"w":1}'

data/.gitignore

@@ -10,4 +10,6 @@
 # rspec failure tracking
 .rspec_status
 /.byebug_history
-stats.db
+stats.db
+.DS_Store
+spec/performance/.byebug_history

data/.ruby-version

@@ -1 +1 @@
-ruby-3.0.0
+ruby-3.1.0

data/.tool-versions

@@ -1 +1 @@
-ruby 3.0.0
+ruby 3.1.0

data/Gemfile

@@ -5,3 +5,7 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+gem "mongo", require: false
+gem "pg", require: false
+gem "redis", require: false
+gem "sqlite3", require: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    trifle-stats (1.3.0)
+    trifle-stats (1.6.0)
       tzinfo (~> 2.0)
 
 GEM
@@ -10,9 +10,10 @@ GEM
     ast (2.4.2)
     bson (4.12.1)
     byebug (11.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.3.5)
     diff-lcs (1.4.4)
     dotenv (2.7.6)
+    mini_portile2 (2.8.9)
     mongo (2.14.0)
       bson (>= 4.8.2, < 5.0.0)
     parallel (1.20.1)
@@ -49,8 +50,10 @@ GEM
     rubocop-ast (1.4.1)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
-    sqlite3 (1.4.4)
-    tzinfo (2.0.5)
+    sqlite3 (2.7.3)
+      mini_portile2 (~> 2.8.0)
+    sqlite3 (2.7.3-x86_64-darwin)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
 
@@ -62,13 +65,13 @@ DEPENDENCIES
   bundler (~> 2.1)
   byebug
   dotenv
-  mongo (>= 2.14.0)
-  pg (>= 1.2)
+  mongo
+  pg
   rake (~> 12.0)
-  redis (>= 4.2)
+  redis
   rspec (~> 3.0)
   rubocop (= 1.0.0)
-  sqlite3 (>= 1.4.4)
+  sqlite3
   trifle-stats!
 
 BUNDLED WITH

data/README.md

@@ -124,6 +124,37 @@ Trifle::Stats.values(key: 'event::logs', from: Time.now, to: Time.now, range: :d
 => {:at=>[2021-01-25 00:00:00 +0200], :values=>[{"count"=>1, "duration"=>5, "lines"=>361}]}
 ```
 
+## Testing
+
+### Testing Principles
+
+Tests are structured to be simple, isolated, and mirror the class structure. Each test is independent and self-contained.
+
+#### Key Rules:
+
+1. **Keep tests simple and isolated** - Each test should focus on a single class/method
+2. **Independent tests** - Tests should not depend on each other and can be run in any order
+3. **Self-contained setup** - Every test configures its own variables and dependencies
+4. **Single layer testing** - Test only the specific class, not multiple layers of functionality
+5. **Use appropriate stubbing** - When testing operations, stub driver methods. Let driver tests verify driver behavior
+6. **Repeat yourself** - It's okay to repeat setup code for clarity and independence
+
+#### Driver Testing:
+
+- Driver tests use **real database connections** (Redis, PostgreSQL, MongoDB, SQLite)
+- Clean data between tests to ensure isolation
+- Use appropriate test databases (e.g., Redis database 15, test-specific DB names)
+- The **Process driver** is ideal for testing environments as it uses in-memory storage
+
+#### Test Structure:
+
+Tests follow the same structure as the classes they test:
+- `spec/stats/driver/` - Driver class tests
+- `spec/stats/operations/` - Operation class tests  
+- `spec/stats/mixins/` - Mixin tests
+
+This approach makes it easier to see initial configuration and expected results for each test.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/trifle-io/trifle-stats.

data/lib/trifle/stats/aggregator/avg.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Avg
+        Trifle::Stats::Series.register_aggregator(:avg, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            sum = slice.compact.sum
+            count = slice.compact.count
+
+            count.zero? ? 0 : sum / count
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/aggregator/max.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Max
+        Trifle::Stats::Series.register_aggregator(:max, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            slice.compact.max
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/aggregator/min.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Min
+        Trifle::Stats::Series.register_aggregator(:min, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            slice.compact.min
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/aggregator/sum.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Sum
+        Trifle::Stats::Series.register_aggregator(:sum, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            slice.compact.sum
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/driver/mongo.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'mongo'
 require_relative '../mixins/packer'
 
 module Trifle
@@ -8,55 +7,109 @@ module Trifle
     module Driver
       class Mongo
         include Mixins::Packer
-        attr_accessor :client, :collection_name, :separator
+        attr_accessor :client, :collection_name
 
-        def initialize(client, collection_name: 'trifle_stats')
+        def initialize(client, collection_name: 'trifle_stats', joined_identifier: true, expire_after: nil)
           @client = client
           @collection_name = collection_name
+          @joined_identifier = joined_identifier
+          @expire_after = expire_after
           @separator = '::'
         end
 
-        def self.setup!(client, collection_name: 'trifle_stats')
-          client[collection_name].create
-          client[collection_name].indexes.create_one({ key: 1 }, unique: true)
+        def self.setup!(client, collection_name: 'trifle_stats', joined_identifier: true, expire_after: nil)
+          collection = client[collection_name]
+          collection.create
+          if joined_identifier
+            collection.indexes.create_one({ key: 1 }, unique: true)
+          else
+            collection.indexes.create_one({ key: 1, range: 1, at: -1 }, unique: true)
+          end
+          collection.indexes.create_one({ expire_at: 1 }, expire_after_seconds: 0) if expire_after
+        end
+
+        def description
+          "#{self.class.name}(#{@joined_identifier ? 'J' : 'S'})"
+        end
+
+        def separator
+          @joined_identifier ? @separator : nil
         end
 
         def inc(keys:, **values)
           data = self.class.pack(hash: { data: values })
 
-          collection.bulk_write(
-            keys.map do |key|
-              upsert_operation('$inc', pkey: key.join(separator), data: data)
-            end
-          )
+          operations = keys.map do |key|
+            filter = key.identifier(separator)
+            expire_at = @expire_after ? key.at + @expire_after : nil
+
+            upsert_operation('$inc', filter: filter, data: data, expire_at: expire_at)
+          end
+
+          collection.bulk_write(operations)
         end
 
         def set(keys:, **values)
           data = self.class.pack(hash: { data: values })
 
-          collection.bulk_write(
-            keys.map do |key|
-              upsert_operation('$set', pkey: key.join(separator), data: data)
-            end
-          )
+          operations = keys.map do |key|
+            filter = key.identifier(separator)
+            expire_at = @expire_after ? key.at + @expire_after : nil
+
+            upsert_operation('$set', filter: filter, data: data, expire_at: expire_at)
+          end
+
+          collection.bulk_write(operations)
+        end
+
+        def ping(key:, **values)
+          data = self.class.pack(hash: { data: values, at: key.at })
+          identifier = key.identifier(separator)
+          expire_at = @expire_after ? key.at + @expire_after : nil
+
+          operations = [
+            upsert_operation('$set', filter: identifier.slice(:key), data: data, expire_at: expire_at)
+          ]
+
+          collection.bulk_write(operations)
         end
 
-        def upsert_operation(operation, pkey:, data:)
-          {
-            update_many: {
-              filter: { key: pkey },
-              update: { operation => data },
-              upsert: true
-            }
+        def upsert_operation(operation, filter:, data:, expire_at: nil)
+          # Merge if $set and $set
+          update_data = operation == '$set' && expire_at ? data.merge(expire_at: expire_at) : data
+
+          # Add if $inc and $set
+          update = {
+            operation => update_data,
+            **(operation != '$set' && expire_at ? { '$set' => { expire_at: expire_at } } : {})
           }
+
+          { update_many: { filter: filter, update: update, upsert: true } }
         end
 
-        def get(keys:)
-          pkeys = keys.map { |key| key.join(separator) }
-          data = collection.find(key: { '$in' => pkeys })
-          map = data.inject({}) { |o, d| o.merge(d['key'] => d['data']) }
+        def get(keys:) # rubocop:disable Metrics/AbcSize
+          combinations = keys.map { |key| key.identifier(separator) }
+          data = collection.find('$or' => combinations)
+          map = data.inject({}) do |o, d|
+            o.merge(
+              Nocturnal::Key.new(
+                key: d['key'], range: d['range'], at: d['at']
+              ).identifier(separator) => d['data']
+            )
+          end
+
+          combinations.map { |combination| map[combination] || {} }
+        end
+
+        def scan(key:)
+          return [] if @joined_identifier
+
+          data = collection.find(
+            **key.identifier(separator)
+          ).sort(at: -1).first # rubocop:disable Style/RedundantSort
+          return [] if data.nil?
 
-          pkeys.map { |pkey| map[pkey] || {} }
+          [data['at'], data['data']]
         end
 
         private

data/lib/trifle/stats/driver/postgres.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'pg'
 require_relative '../mixins/packer'
 
 module Trifle
@@ -20,6 +19,10 @@ module Trifle
           client.exec("CREATE TABLE #{table_name} (key VARCHAR(255) PRIMARY KEY, data JSONB NOT NULL DEFAULT '{}'::jsonb)") # rubocop:disable Layout/LineLength
         end
 
+        def description
+          "#{self.class.name}(J)"
+        end
+
         def inc(keys:, **values)
           data = self.class.pack(hash: values)
           client.transaction do |c|
@@ -34,7 +37,7 @@ module Trifle
           <<-SQL
             INSERT INTO #{table_name} (key, data) VALUES ('#{key}', '#{data.to_json}')
             ON CONFLICT (key) DO UPDATE SET data =
-            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (COALESCE(trifle_stats.data->>'#{k}', '0')::int + #{v})::text::jsonb)" }};
+            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (COALESCE(#{table_name}.data->>'#{k}', '0')::int + #{v})::text::jsonb)" }};
           SQL
         end
 
@@ -52,7 +55,7 @@ module Trifle
           <<-SQL
             INSERT INTO #{table_name} (key, data) VALUES ('#{key}', '#{data.to_json}')
             ON CONFLICT (key) DO UPDATE SET data =
-            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (#{v})::text::jsonb)" }}
+            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', '#{v.to_json}'::jsonb)" }}
           SQL
         end
 
@@ -79,6 +82,14 @@ module Trifle
             SELECT * FROM #{table_name} WHERE key IN ('#{keys.join("', '")}');
           SQL
         end
+
+        def ping(*)
+          []
+        end
+
+        def scan(*)
+          []
+        end
       end
     end
   end