trifle-stats 1.3.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e169bb4a0d962b9d16ff5ba4b437ffc1948223a8d3e3ef599baa7c510b57c6f
-  data.tar.gz: 01637b0e31df2f49107cce66cd02addb4463a96fd96d66d6f41ca1a44b0f94e7
+  metadata.gz: eece6621a74a0dfc38e7e1932897ad0481e3b85744ec85d016320cc6c7e5b395
+  data.tar.gz: a90a4b7bf5e885fdc77af3638b2fcb0fdef0f48e17001b2da925161164fd31f4
 SHA512:
-  metadata.gz: 773d0d82001e990090a7fed8ac1e129d2235b62d02d258f5da28c89f6578166ab63aa69f8795861a2870646e236a0e408ac9053b00c705486105cee3e0160dd1
-  data.tar.gz: abf4dd13853d4911bd0fead246b609ac3f28470018af6c5647588f0ccae1228083391c96b6d7dd443b60f8346ff65dcd23fcc5d2f2b90e205cf057581970a177
+  metadata.gz: 7bb55a42c115a0e353277b7445f33db93bc16a6011202de515adb63126d0d154615bbc2b13bfad45bd6ca990aedec32de4911c5843553b2c5148676c01d21b99
+  data.tar.gz: 475e37cab204a8b07ea90ead1c209cb784844ba3683884362f284d41c0f2e8170f9f7ec858125dece018312561d934efd555a41ac6457ec16af2d31e36be0695

data/.devcontainer.json

@@ -1,6 +1,6 @@
 {
   "dockerComposeFile": ".devops/docker/codespaces/docker-compose.yml",
   "service": "app",
-  "workspaceFolder": "/",
+  "workspaceFolder": "/workspaces/trifle-stats",
   "shutdownAction": "stopCompose"
 }

data/.devops/docker/codespaces/Dockerfile

@@ -1 +1 @@
-FROM trifle/environment:1.0.4
+FROM trifle/environment:ruby_3.1.0_1

data/.devops/docker/environment/Dockerfile

@@ -21,6 +21,7 @@ RUN apt-get update -q \
     libssl-dev \
     libreadline-dev \
     zlib1g-dev \
+    libsqlite3-dev \
     tmux \
     htop \
     vim \
@@ -45,8 +46,8 @@ RUN git clone https://github.com/asdf-vm/asdf.git ${ASDF_ROOT} --branch v0.10.0
 RUN asdf plugin-add ruby https://github.com/asdf-vm/asdf-ruby.git
 
 # install ruby
-ENV RUBY_VERSION 3.0.0
-RUN ASDF_RUBY_BUILD_VERSION=v20201225 asdf install ruby ${RUBY_VERSION} \
+ENV RUBY_VERSION 3.1.0
+RUN asdf install ruby ${RUBY_VERSION} \
   && asdf global ruby ${RUBY_VERSION}
 
 RUN gem install bundler

data/.devops/docker/local/Dockerfile

@@ -0,0 +1 @@
+FROM trifle/environment:ruby_3.1.0_1

data/.devops/docker/local/docker-compose.yml

@@ -0,0 +1,30 @@
+version: "3.7"
+name: "stats"
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_PASSWORD: password
+  redis:
+    image: redis:latest
+  mongo:
+    image: mongo:latest
+  app:
+    command: /bin/sh -c "while sleep 1000; do :; done"
+    build:
+      context: ../../..
+      dockerfile: .devops/docker/local/Dockerfile
+    depends_on:
+      - postgres
+      - redis
+      - mongo
+    environment:
+      PGHOST: postgres
+      PGUSER: postgres
+      REDIS_HOST: redis
+      REDIS_URL: redis://redis:6379
+    expose:
+      - 4000
+    volumes:
+      - ../../..:/workspaces/stats
+    working_dir: /workspaces/stats

data/.github/workflows/ruby.yml

@@ -16,21 +16,100 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_DB: test_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+      mongodb:
+        image: mongo:6.0
+        env:
+          MONGO_INITDB_ROOT_USERNAME: root
+          MONGO_INITDB_ROOT_PASSWORD: password
+        options: >-
+          --health-cmd "mongosh --eval 'db.adminCommand(\"ping\")'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 27017:27017
+
     strategy:
       matrix:
-        ruby-version: ["3.0"]
+        ruby-version: ["3.1"]
+
+    env:
+      # Database configuration
+      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
+      REDIS_URL: redis://localhost:6379/0
+      MONGODB_URL: mongodb://root:password@localhost:27017/test_db?authSource=admin
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
       - name: Set up Ruby
-        # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
-        # change this to (see https://github.com/ruby/setup-ruby#versioning):
-        # uses: ruby/setup-ruby@v1
-        uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+        uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version }}
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Wait for services to be ready
+        run: |
+          # Wait for PostgreSQL
+          until pg_isready -h localhost -p 5432 -U postgres; do
+            echo "Waiting for PostgreSQL..."
+            sleep 2
+          done
+
+          # Wait for Redis
+          until timeout 1 bash -c "</dev/tcp/localhost/6379"; do
+            echo "Waiting for Redis..."
+            sleep 2
+          done
+
+          # Wait for MongoDB
+          until timeout 1 bash -c "</dev/tcp/localhost/27017"; do
+            echo "Waiting for MongoDB..."
+            sleep 2
+          done
+
+      - name: Setup Database
+        run: |
+          # Create database if needed (adjust based on your setup)
+          # bundle exec rails db:create db:migrate
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
+
       - name: Rspec
         run: bundle exec rspec
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
+          REDIS_URL: redis://localhost:6379/0
+          MONGODB_URL: mongodb://root:password@localhost:27017/test_db?authSource=admin
+
       - name: Rubocop
         run: bundle exec rubocop
+
+      # - name: Performance
+      #   run: cd spec/performance && bundle install && ruby run.rb 1000 '{"a":1,"b":2,"c":1,"d":2,"e":1,"f":2,"g":1,"h":2,"i":1,"j":2,"k":1,"l":2,"m":1,"n":2,"o":1,"p":2,"q":1,"r":2,"s":1,"t":2,"u":1,"v":2,"w":1}'

data/.gitignore

@@ -12,3 +12,4 @@
 /.byebug_history
 stats.db
 .DS_Store
+spec/performance/.byebug_history

data/.ruby-version

@@ -1 +1 @@
-ruby-3.0.0
+ruby-3.1.0

data/.tool-versions

@@ -1 +1 @@
-ruby 3.0.0
+ruby 3.1.0

data/Gemfile

@@ -8,4 +8,4 @@ gem "rspec", "~> 3.0"
 gem "mongo", require: false
 gem "pg", require: false
 gem "redis", require: false
-gem "sqlite", require: false
+gem "sqlite3", require: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    trifle-stats (1.3.1)
+    trifle-stats (1.6.0)
       tzinfo (~> 2.0)
 
 GEM
@@ -10,9 +10,10 @@ GEM
     ast (2.4.2)
     bson (4.12.1)
     byebug (11.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.3.5)
     diff-lcs (1.4.4)
     dotenv (2.7.6)
+    mini_portile2 (2.8.9)
     mongo (2.14.0)
       bson (>= 4.8.2, < 5.0.0)
     parallel (1.20.1)
@@ -49,9 +50,10 @@ GEM
     rubocop-ast (1.4.1)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
-    sqlite (1.0.2)
-    sqlite3 (1.4.4)
-    tzinfo (2.0.5)
+    sqlite3 (2.7.3)
+      mini_portile2 (~> 2.8.0)
+    sqlite3 (2.7.3-x86_64-darwin)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
 
@@ -69,8 +71,7 @@ DEPENDENCIES
   redis
   rspec (~> 3.0)
   rubocop (= 1.0.0)
-  sqlite
-  sqlite3 (>= 1.4.4)
+  sqlite3
   trifle-stats!
 
 BUNDLED WITH

data/README.md

@@ -124,6 +124,37 @@ Trifle::Stats.values(key: 'event::logs', from: Time.now, to: Time.now, range: :d
 => {:at=>[2021-01-25 00:00:00 +0200], :values=>[{"count"=>1, "duration"=>5, "lines"=>361}]}
 ```
 
+## Testing
+
+### Testing Principles
+
+Tests are structured to be simple, isolated, and mirror the class structure. Each test is independent and self-contained.
+
+#### Key Rules:
+
+1. **Keep tests simple and isolated** - Each test should focus on a single class/method
+2. **Independent tests** - Tests should not depend on each other and can be run in any order
+3. **Self-contained setup** - Every test configures its own variables and dependencies
+4. **Single layer testing** - Test only the specific class, not multiple layers of functionality
+5. **Use appropriate stubbing** - When testing operations, stub driver methods. Let driver tests verify driver behavior
+6. **Repeat yourself** - It's okay to repeat setup code for clarity and independence
+
+#### Driver Testing:
+
+- Driver tests use **real database connections** (Redis, PostgreSQL, MongoDB, SQLite)
+- Clean data between tests to ensure isolation
+- Use appropriate test databases (e.g., Redis database 15, test-specific DB names)
+- The **Process driver** is ideal for testing environments as it uses in-memory storage
+
+#### Test Structure:
+
+Tests follow the same structure as the classes they test:
+- `spec/stats/driver/` - Driver class tests
+- `spec/stats/operations/` - Operation class tests  
+- `spec/stats/mixins/` - Mixin tests
+
+This approach makes it easier to see initial configuration and expected results for each test.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/trifle-io/trifle-stats.

data/lib/trifle/stats/aggregator/avg.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Avg
+        Trifle::Stats::Series.register_aggregator(:avg, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            sum = slice.compact.sum
+            count = slice.compact.count
+
+            count.zero? ? 0 : sum / count
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/aggregator/max.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Max
+        Trifle::Stats::Series.register_aggregator(:max, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            slice.compact.max
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/aggregator/min.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Min
+        Trifle::Stats::Series.register_aggregator(:min, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            slice.compact.min
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/aggregator/sum.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Stats
+    class Aggregator
+      class Sum
+        Trifle::Stats::Series.register_aggregator(:sum, self)
+
+        def aggregate(series:, path:, slices: 1)
+          return [] if series[:at].empty?
+
+          keys = path.split('.')
+          result = series[:values].map do |data|
+            data.dig(*keys)
+          end
+          sliced(result: result, slices: slices)
+        end
+
+        private
+
+        def sliced(result:, slices:)
+          result[(result.count - (result.count / slices * slices))..].each_slice(result.count / slices).map do |slice|
+            slice.compact.sum
+          end
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/driver/mongo.rb

@@ -7,55 +7,109 @@ module Trifle
     module Driver
       class Mongo
         include Mixins::Packer
-        attr_accessor :client, :collection_name, :separator
+        attr_accessor :client, :collection_name
 
-        def initialize(client, collection_name: 'trifle_stats')
+        def initialize(client, collection_name: 'trifle_stats', joined_identifier: true, expire_after: nil)
           @client = client
           @collection_name = collection_name
+          @joined_identifier = joined_identifier
+          @expire_after = expire_after
           @separator = '::'
         end
 
-        def self.setup!(client, collection_name: 'trifle_stats')
-          client[collection_name].create
-          client[collection_name].indexes.create_one({ key: 1 }, unique: true)
+        def self.setup!(client, collection_name: 'trifle_stats', joined_identifier: true, expire_after: nil)
+          collection = client[collection_name]
+          collection.create
+          if joined_identifier
+            collection.indexes.create_one({ key: 1 }, unique: true)
+          else
+            collection.indexes.create_one({ key: 1, range: 1, at: -1 }, unique: true)
+          end
+          collection.indexes.create_one({ expire_at: 1 }, expire_after_seconds: 0) if expire_after
+        end
+
+        def description
+          "#{self.class.name}(#{@joined_identifier ? 'J' : 'S'})"
+        end
+
+        def separator
+          @joined_identifier ? @separator : nil
         end
 
         def inc(keys:, **values)
           data = self.class.pack(hash: { data: values })
 
-          collection.bulk_write(
-            keys.map do |key|
-              upsert_operation('$inc', pkey: key.join(separator), data: data)
-            end
-          )
+          operations = keys.map do |key|
+            filter = key.identifier(separator)
+            expire_at = @expire_after ? key.at + @expire_after : nil
+
+            upsert_operation('$inc', filter: filter, data: data, expire_at: expire_at)
+          end
+
+          collection.bulk_write(operations)
         end
 
         def set(keys:, **values)
           data = self.class.pack(hash: { data: values })
 
-          collection.bulk_write(
-            keys.map do |key|
-              upsert_operation('$set', pkey: key.join(separator), data: data)
-            end
-          )
+          operations = keys.map do |key|
+            filter = key.identifier(separator)
+            expire_at = @expire_after ? key.at + @expire_after : nil
+
+            upsert_operation('$set', filter: filter, data: data, expire_at: expire_at)
+          end
+
+          collection.bulk_write(operations)
+        end
+
+        def ping(key:, **values)
+          data = self.class.pack(hash: { data: values, at: key.at })
+          identifier = key.identifier(separator)
+          expire_at = @expire_after ? key.at + @expire_after : nil
+
+          operations = [
+            upsert_operation('$set', filter: identifier.slice(:key), data: data, expire_at: expire_at)
+          ]
+
+          collection.bulk_write(operations)
         end
 
-        def upsert_operation(operation, pkey:, data:)
-          {
-            update_many: {
-              filter: { key: pkey },
-              update: { operation => data },
-              upsert: true
-            }
+        def upsert_operation(operation, filter:, data:, expire_at: nil)
+          # Merge if $set and $set
+          update_data = operation == '$set' && expire_at ? data.merge(expire_at: expire_at) : data
+
+          # Add if $inc and $set
+          update = {
+            operation => update_data,
+            **(operation != '$set' && expire_at ? { '$set' => { expire_at: expire_at } } : {})
           }
+
+          { update_many: { filter: filter, update: update, upsert: true } }
         end
 
-        def get(keys:)
-          pkeys = keys.map { |key| key.join(separator) }
-          data = collection.find(key: { '$in' => pkeys })
-          map = data.inject({}) { |o, d| o.merge(d['key'] => d['data']) }
+        def get(keys:) # rubocop:disable Metrics/AbcSize
+          combinations = keys.map { |key| key.identifier(separator) }
+          data = collection.find('$or' => combinations)
+          map = data.inject({}) do |o, d|
+            o.merge(
+              Nocturnal::Key.new(
+                key: d['key'], range: d['range'], at: d['at']
+              ).identifier(separator) => d['data']
+            )
+          end
+
+          combinations.map { |combination| map[combination] || {} }
+        end
+
+        def scan(key:)
+          return [] if @joined_identifier
+
+          data = collection.find(
+            **key.identifier(separator)
+          ).sort(at: -1).first # rubocop:disable Style/RedundantSort
+          return [] if data.nil?
 
-          pkeys.map { |pkey| map[pkey] || {} }
+          [data['at'], data['data']]
         end
 
         private

data/lib/trifle/stats/driver/postgres.rb

@@ -19,6 +19,10 @@ module Trifle
           client.exec("CREATE TABLE #{table_name} (key VARCHAR(255) PRIMARY KEY, data JSONB NOT NULL DEFAULT '{}'::jsonb)") # rubocop:disable Layout/LineLength
         end
 
+        def description
+          "#{self.class.name}(J)"
+        end
+
         def inc(keys:, **values)
           data = self.class.pack(hash: values)
           client.transaction do |c|
@@ -33,7 +37,7 @@ module Trifle
           <<-SQL
             INSERT INTO #{table_name} (key, data) VALUES ('#{key}', '#{data.to_json}')
             ON CONFLICT (key) DO UPDATE SET data =
-            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (COALESCE(trifle_stats.data->>'#{k}', '0')::int + #{v})::text::jsonb)" }};
+            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (COALESCE(#{table_name}.data->>'#{k}', '0')::int + #{v})::text::jsonb)" }};
           SQL
         end
 
@@ -51,7 +55,7 @@ module Trifle
           <<-SQL
             INSERT INTO #{table_name} (key, data) VALUES ('#{key}', '#{data.to_json}')
             ON CONFLICT (key) DO UPDATE SET data =
-            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (#{v})::text::jsonb)" }}
+            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', '#{v.to_json}'::jsonb)" }}
           SQL
         end
 
@@ -78,6 +82,14 @@ module Trifle
             SELECT * FROM #{table_name} WHERE key IN ('#{keys.join("', '")}');
           SQL
         end
+
+        def ping(*)
+          []
+        end
+
+        def scan(*)
+          []
+        end
       end
     end
   end

data/lib/trifle/stats/driver/process.rb

@@ -12,6 +12,10 @@ module Trifle
           @separator = '::'
         end
 
+        def description
+          "#{self.class.name}(J)"
+        end
+
         def inc(keys:, **values)
           keys.map do |key|
             self.class.pack(hash: values).each do |k, c|
@@ -39,6 +43,14 @@ module Trifle
             )
           end
         end
+
+        def ping(*)
+          []
+        end
+
+        def scan(*)
+          []
+        end
       end
     end
   end

data/lib/trifle/stats/driver/redis.rb

@@ -9,15 +9,20 @@ module Trifle
         include Mixins::Packer
         attr_accessor :client, :prefix, :separator
 
-        def initialize(client = ::Redis.current, prefix: 'trfl')
+        def initialize(client, prefix: 'trfl')
           @client = client
           @prefix = prefix
           @separator = '::'
         end
 
+        def description
+          "#{self.class.name}(J)"
+        end
+
         def inc(keys:, **values)
           keys.map do |key|
-            pkey = ([prefix] + key).join(separator)
+            key.prefix = prefix
+            pkey = key.join(separator)
 
             self.class.pack(hash: values).each do |k, c|
               client.hincrby(pkey, k, c)
@@ -27,7 +32,8 @@ module Trifle
 
         def set(keys:, **values)
           keys.map do |key|
-            pkey = ([prefix] + key).join(separator)
+            key.prefix = prefix
+            pkey = key.join(separator)
 
             client.hmset(pkey, *self.class.pack(hash: values))
           end
@@ -35,13 +41,22 @@ module Trifle
 
         def get(keys:)
           keys.map do |key|
-            pkey = ([prefix] + key).join(separator)
+            key.prefix = prefix
+            pkey = key.join(separator)
 
             self.class.unpack(
               hash: client.hgetall(pkey)
             )
           end
         end
+
+        def ping(*)
+          []
+        end
+
+        def scan(*)
+          []
+        end
       end
     end
   end