trifle-stats 2.4.1 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 574331154d00b65ea5a994f7c7fbc754a1225c39617294acbeed3886c88d5b98
-  data.tar.gz: d4cca3dd912c0642594805395f727fcce350cee4cd6b388ddf09fb3032895317
+  metadata.gz: 053aa86693f158ba5a395ab80396f0b3274bff69481ec80dd4335e782d000bfe
+  data.tar.gz: 86be7c68e870d4fe38eac8dcae73634d8237a5334c8071c610932b22b57c735d
 SHA512:
-  metadata.gz: 423c591af0e2f61178be6553afef6ec4a52ab96268f30b82720e8bf591d079ece3b5b4753766f1ebeecffe442b92162ea89619a4aa7fa58298aee62b226f1cba
-  data.tar.gz: a1fdba88136d21091e95eb59b1b87fab7e917626e40860d5e7128f7771c0428547e2070ba98ae8a9cf5437ee5d61c7671745336226fc25c46492fd6aee1756f1
+  metadata.gz: cfc8e1a9b317808a37acc6799d87a99d856fe4e4f4b792ac223d96f4ac500b2e6c77e25a5e99e1861047524984cf7ce2bd691a6d930404384b973a173ed49e99
+  data.tar.gz: 383c36c748fa4266b2d741054f1f22a35baff5806ccb5ab169d3a0cf49c9fabad2fae97e64acb86fe2fbd534600acad0fe5ee6edac7a01faea6340c5b1eb1c13

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    trifle-stats (2.4.1)
+    trifle-stats (2.6.0)
       tzinfo (~> 2.0)
 
 GEM
@@ -11,7 +11,7 @@ GEM
     bigdecimal (4.0.1)
     bson (4.12.1)
     byebug (11.1.3)
-    concurrent-ruby (1.3.6)
+    concurrent-ruby (1.3.8)
     diff-lcs (1.4.4)
     dotenv (2.7.6)
     mini_portile2 (2.8.9)

data/README.md

@@ -57,23 +57,42 @@ Trifle::Stats.values(
 #=> { at: [Mon, Tue, Wed, ...], values: [{ "count" => 12, "revenue" => 598_80, ... }, ...] }
 ```
 
+### 5. Process with Series
+
+```ruby
+series = Trifle::Stats.series(
+  key: 'orders',
+  from: 1.week.ago,
+  to: Time.now,
+  granularity: :day
+)
+
+series.transpond.expression(
+  paths: ['revenue', 'count'],
+  expression: 'a / b',
+  response: 'avg_order'
+)
+
+series.aggregate.sum(path: 'count')
+```
+
 ## Drivers
 
 | Driver | Backend | Best for |
 |--------|---------|----------|
+| **API** | Trifle Cloud Projects | Hosted metrics without your own database |
 | **Postgres** | JSONB upsert | Most production apps |
 | **Redis** | Hash increment | High-throughput counters |
 | **MongoDB** | Document upsert | Document-oriented stacks |
 | **MySQL** | JSON column | MySQL shops |
 | **SQLite** | JSON1 extension | Single-server apps, dev/test |
 | **Process** | In-memory | Testing |
-| **Dummy** | No-op | Disabled analytics |
 
 ## Features
 
 - **Dynamic time granularities.** Use any interval like `1m`, `10m`, `1h`, `6h`, `1d`, `1w`, `1mo`, `1q`, `1y`.
 - **Nested value hierarchies.** Track dimensional breakdowns in a single call.
-- **Series operations.** Aggregators (sum, avg, min, max), transponders, formatters.
+- **Series operations.** Aggregators, the expression transponder, and formatters.
 - **Buffered writes.** Queue metrics in-memory before flushing to reduce write load.
 - **Driver flexibility.** Switch backends without changing application code.
 

data/lib/trifle/stats/configuration.rb

@@ -42,6 +42,7 @@ module Trifle
 
       def storage
         return driver unless buffer_enabled
+        return driver if driver.respond_to?(:bypass_buffer?) && driver.bypass_buffer?
 
         @storage ||= Trifle::Stats::Buffer.new(
           driver: driver,

data/lib/trifle/stats/driver/api.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'stringio'
+require 'time'
+require 'uri'
+require 'zlib'
+require_relative '../version'
+
+module Trifle
+  module Stats
+    module Driver
+      class Api
+        ENDPOINT = URI('https://app.trifle.io/api/v1/metrics')
+        TIMEOUT = 10
+        ERROR_BODY_LIMIT = 1024
+
+        class Error < Trifle::Stats::Error
+          attr_reader :status, :response_body, :retry_after, :delivery_unknown
+
+          def initialize(message, status: nil, response_body: nil, retry_after: nil, delivery_unknown: false)
+            super(message)
+            @status = status
+            @response_body = response_body
+            @retry_after = retry_after
+            @delivery_unknown = delivery_unknown
+          end
+        end
+
+        class NetHttpTransport
+          def call(uri:, request:, timeout:)
+            Net::HTTP.start(
+              uri.hostname,
+              uri.port,
+              use_ssl: uri.scheme == 'https',
+              open_timeout: timeout,
+              read_timeout: timeout
+            ) { |http| http.request(request) }
+          end
+        end
+
+        attr_reader :token, :project_id
+
+        def initialize(token:, project_id:, transport: NetHttpTransport.new)
+          raise ArgumentError, 'token must not be empty' if token.to_s.strip.empty?
+          raise ArgumentError, 'project_id must not be empty' if project_id.to_s.strip.empty?
+
+          @token = token.to_s
+          @project_id = project_id.to_s
+          @transport = transport
+        end
+
+        def description
+          self.class.name
+        end
+
+        def bypass_buffer?
+          true
+        end
+
+        def direct_write(operation:, key:, at:, values:, untracked: false)
+          body = gzip(JSON.generate(
+                        operation: operation.to_s,
+                        key: key,
+                        at: at.to_time.iso8601(6),
+                        values: values,
+                        untracked: untracked
+                      ))
+          request = Net::HTTP::Post.new(ENDPOINT)
+          request['Authorization'] = "Bearer #{token}"
+          request['X-Trifle-Source-Id'] = project_id
+          request['Content-Type'] = 'application/json'
+          request['Accept'] = 'application/json'
+          request['Content-Encoding'] = 'gzip'
+          request['User-Agent'] = "trifle-stats-ruby/#{Trifle::Stats::VERSION}"
+          request.body = body
+
+          response = @transport.call(uri: ENDPOINT, request: request, timeout: TIMEOUT)
+          return true if response.code.to_i.between?(200, 299)
+
+          response_body = response.body.to_s.byteslice(0, ERROR_BODY_LIMIT)
+          raise Error.new(
+            "Trifle API returned HTTP #{response.code}",
+            status: response.code.to_i,
+            response_body: response_body,
+            retry_after: response['Retry-After']
+          )
+        rescue Error
+          raise
+        rescue StandardError => e
+          raise Error.new("Trifle API request failed: #{e.message}", delivery_unknown: true), cause: e
+        end
+
+        def inc(*)
+          unsupported!(:track)
+        end
+
+        def set(*)
+          unsupported!(:assert)
+        end
+
+        def get(*)
+          unsupported!(:values)
+        end
+
+        def ping(*)
+          unsupported!(:beam)
+        end
+
+        def scan(*)
+          unsupported!(:scan)
+        end
+
+        private
+
+        def gzip(value)
+          output = StringIO.new
+          Zlib::GzipWriter.wrap(output) { |writer| writer.write(value) }
+          output.string
+        end
+
+        def unsupported!(operation)
+          message = "#{operation} must be called through Trifle::Stats; " \
+                    'API driver does not support direct storage operations'
+          raise Error, message
+        end
+      end
+    end
+  end
+end

data/lib/trifle/stats/driver/mysql.rb

@@ -95,10 +95,9 @@ module Trifle
         end
 
         def get(keys:)
-          keys.map do |key|
-            identifier = identifier_for(key)
-            self.class.unpack(hash: fetch_packed_data(identifier))
-          end
+          identifiers = keys.map { |key| identifier_for(key) }
+          data = get_all(identifiers: identifiers)
+          identifiers.map { |identifier| self.class.unpack(hash: data.fetch(lookup_key_for(identifier), {})) }
         end
 
         def ping(key:, values:)
@@ -157,23 +156,42 @@ module Trifle
           execute_prepared(connection, query, args)
         end
 
-        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        def fetch_packed_data(identifier)
-          conditions = identifier.keys.map { |column| "#{self.class.quote_identifier(column)} = ?" }.join(' AND ')
+        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        def get_all(identifiers:)
+          columns = identifiers.first.keys
+          # Select `at` as a plain string to avoid client time zone casting
+          columns_sql = columns.map do |column|
+            if column == :at
+              "DATE_FORMAT(`at`, '%Y-%m-%d %H:%i:%s.%f') AS at"
+            else
+              self.class.quote_identifier(column)
+            end
+          end.join(', ')
+          conditions = identifiers.map do |identifier|
+            "(#{identifier.keys.map { |column| "#{self.class.quote_identifier(column)} = ?" }.join(' AND ')})"
+          end.join(' OR ')
+
           query = <<~SQL
-            SELECT CAST(`data` AS CHAR) AS data
+            SELECT #{columns_sql}, CAST(`data` AS CHAR) AS data
             FROM #{self.class.quote_identifier(table_name)}
             WHERE #{conditions}
-            LIMIT 1
           SQL
-          packed_data = execute_prepared(client, query, query_values(identifier)).first&.fetch('data', nil)
-          return {} if packed_data.nil? || packed_data.empty?
+          args = identifiers.flat_map { |identifier| query_values(identifier) }
 
-          JSON.parse(packed_data)
-        rescue JSON::ParserError
-          {}
+          execute_prepared(client, query, args).each_with_object({}) do |row, o|
+            identifier = columns.to_h { |column| [column, row[column.to_s]] }
+            o[lookup_key_for(identifier)] = JSON.parse(row['data'])
+          rescue JSON::ParserError
+            nil
+          end
+        end
+        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+        def lookup_key_for(identifier)
+          identifier.map do |_column, value|
+            value.is_a?(Time) || value.is_a?(DateTime) ? format_time_value(value) : value.to_s
+          end.join('|')
         end
-        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
         def inc_query(identifier:, data:)
           upsert_query(

data/lib/trifle/stats/driver/postgres.rb

@@ -61,22 +61,26 @@ module Trifle
           client.transaction do |c|
             keys.map do |key|
               identifier = identifier_for(key)
-              c.exec(inc_query(identifier: identifier, data: data))
+              query, params = inc_query(identifier: identifier, data: data)
+              c.exec_params(query, params)
               track_system_data(c, key, count, tracking_key)
             end
           end
         end
 
-        def inc_query(identifier:, data:)
+        def inc_query(identifier:, data:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
           columns = identifier.keys.join(', ')
-          values = identifier.values.map { |v| format_value(v) }.join(', ')
-          conflict_columns = identifier.keys.join(', ')
+          placeholders = identifier.each_with_index.map { |_, i| "$#{i + 1}" }.join(', ')
+          expression = data.inject("to_jsonb(#{table_name}.data)") do |o, (k, v)|
+            path = escape_string(k.to_s)
+            "jsonb_set(#{o}, '{#{path}}', (COALESCE(#{table_name}.data->>'#{path}', '0')::numeric + #{numeric_value(k, v)})::text::jsonb)" # rubocop:disable Layout/LineLength
+          end
 
-          <<-SQL
-            INSERT INTO #{table_name} (#{columns}, data) VALUES (#{values}, '#{data.to_json}')
-            ON CONFLICT (#{conflict_columns}) DO UPDATE SET data =
-            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', (COALESCE(#{table_name}.data->>'#{k}', '0')::int + #{v})::text::jsonb)" }};
+          query = <<-SQL
+            INSERT INTO #{table_name} (#{columns}, data) VALUES (#{placeholders}, $#{identifier.size + 1})
+            ON CONFLICT (#{columns}) DO UPDATE SET data = #{expression};
           SQL
+          [query, query_params(identifier) + [data.to_json]]
         end
 
         def set(keys:, values:, count: 1, tracking_key: nil)
@@ -84,7 +88,8 @@ module Trifle
           client.transaction do |c|
             keys.map do |key|
               identifier = identifier_for(key)
-              c.exec(set_query(identifier: identifier, data: data))
+              query, params = set_query(identifier: identifier, data: data)
+              c.exec_params(query, params)
               track_system_data(c, key, count, tracking_key)
             end
           end
@@ -94,21 +99,24 @@ module Trifle
           return unless @system_tracking
 
           system_data = system_data_for(key: key, count: count, tracking_key: tracking_key)
-          connection.exec(
-            inc_query(identifier: system_identifier_for(key: key), data: system_data)
-          )
+          query, params = inc_query(identifier: system_identifier_for(key: key), data: system_data)
+          connection.exec_params(query, params)
         end
 
-        def set_query(identifier:, data:)
+        def set_query(identifier:, data:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
           columns = identifier.keys.join(', ')
-          values = identifier.values.map { |v| format_value(v) }.join(', ')
-          conflict_columns = identifier.keys.join(', ')
+          placeholders = identifier.each_with_index.map { |_, i| "$#{i + 1}" }.join(', ')
+          params = query_params(identifier) + [data.to_json]
+          expression = data.inject("to_jsonb(#{table_name}.data)") do |o, (k, v)|
+            params << v.to_json
+            "jsonb_set(#{o}, '{#{escape_string(k.to_s)}}', $#{params.size}::jsonb)"
+          end
 
-          <<-SQL
-            INSERT INTO #{table_name} (#{columns}, data) VALUES (#{values}, '#{data.to_json}')
-            ON CONFLICT (#{conflict_columns}) DO UPDATE SET data =
-            #{data.inject("to_jsonb(#{table_name}.data)") { |o, (k, v)| "jsonb_set(#{o}, '{#{k}}', '#{v.to_json}'::jsonb)" }}
+          query = <<-SQL
+            INSERT INTO #{table_name} (#{columns}, data) VALUES (#{placeholders}, $#{identifier.size + 1})
+            ON CONFLICT (#{columns}) DO UPDATE SET data = #{expression}
           SQL
+          [query, params]
         end
 
         def get(keys:)
@@ -118,7 +126,8 @@ module Trifle
         end
 
         def get_all(identifiers:) # rubocop:disable Metrics/AbcSize
-          results = client.exec_params(get_query(identifiers: identifiers)).to_a
+          query, params = get_query(identifiers: identifiers)
+          results = client.exec_params(query, params).to_a
           sample = identifiers.first
 
           results.each_with_object(Hash.new({})) do |r, o|
@@ -130,37 +139,44 @@ module Trifle
           end
         end
 
-        def get_query(identifiers:)
+        def get_query(identifiers:) # rubocop:disable Metrics/MethodLength
+          params = []
           conditions = identifiers.map do |identifier|
-            identifier.map { |k, v| "#{k} = #{format_value(v)}" }.join(' AND ')
+            identifier.map do |k, v|
+              params << query_param(v)
+              "#{k} = $#{params.size}"
+            end.join(' AND ')
           end.join(' OR ')
 
-          <<-SQL
+          query = <<-SQL
             SELECT * FROM #{table_name} WHERE #{conditions};
           SQL
+          [query, params]
         end
 
         def ping(key:, values:)
           return [] if @joined_identifier
 
           data = self.class.pack(hash: { data: values, at: key.at })
-          operation = ping_query(key: key.key, at: key.at, data: data)
+          query, params = ping_query(key: key.key, at: key.at, data: data)
           client.transaction do |c|
-            c.exec(operation)
+            c.exec_params(query, params)
           end
         end
 
         def ping_query(key:, at:, data:)
-          <<-SQL
-            INSERT INTO #{ping_table_name} (key, at, data) VALUES ('#{key}', '#{at.iso8601}', '#{data.to_json}')
-            ON CONFLICT (key) DO UPDATE SET at = '#{at.iso8601}', data = '#{data.to_json}'::jsonb;
+          query = <<-SQL
+            INSERT INTO #{ping_table_name} (key, at, data) VALUES ($1, $2, $3)
+            ON CONFLICT (key) DO UPDATE SET at = $2, data = $3::jsonb;
           SQL
+          [query, [key.to_s, at.iso8601, data.to_json]]
         end
 
         def scan(key:)
           return [] if @joined_identifier
 
-          result = client.exec_params(scan_query(key: key.key)).to_a.first
+          query, params = scan_query(key: key.key)
+          result = client.exec_params(query, params).to_a.first
           return [] if result.nil?
 
           [Time.parse(result['at']), self.class.unpack(hash: JSON.parse(result['data']))]
@@ -169,9 +185,10 @@ module Trifle
         end
 
         def scan_query(key:)
-          <<-SQL
-            SELECT at, data FROM #{ping_table_name} WHERE key = '#{key}' ORDER BY at DESC LIMIT 1;
+          query = <<-SQL
+            SELECT at, data FROM #{ping_table_name} WHERE key = $1 ORDER BY at DESC LIMIT 1;
           SQL
+          [query, [key.to_s]]
         end
 
         def self.normalize_joined_identifier(value)
@@ -185,19 +202,31 @@ module Trifle
 
         private
 
-        def format_value(value)
+        def query_params(identifier)
+          identifier.values.map { |value| query_param(value) }
+        end
+
+        def query_param(value)
           case value
-          when String
-            "'#{value}'"
           when Time
-            "'#{value.utc.strftime('%Y-%m-%d %H:%M:%S+00')}'"
+            value.utc.strftime('%Y-%m-%d %H:%M:%S+00')
           when Integer, Float
-            value.to_s
+            value
           else
-            "'#{value}'"
+            value.to_s
           end
         end
 
+        def numeric_value(key, value)
+          return value.to_s if value.is_a?(Numeric)
+
+          raise ArgumentError, "increment requires numeric value for key #{key.inspect}"
+        end
+
+        def escape_string(value)
+          value.gsub("'", "''")
+        end
+
         def build_map_key(data)
           return data[:key] if @joined_identifier == :full
           return "#{data[:key]}::#{data[:at]}" if @joined_identifier == :partial

data/lib/trifle/stats/driver/redis.rb

@@ -31,47 +31,43 @@ module Trifle
           self.class.pack(hash: { count: count, keys: { tracking_key => count } })
         end
 
-        def inc(keys:, values:, count: 1, tracking_key: nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-          keys.map do |key|
-            key.prefix = prefix
-            pkey = key.join(separator)
-
-            self.class.pack(hash: values).each do |k, c|
-              client.hincrby(pkey, k, c)
-            end
-            next unless @system_tracking
-
-            skey = system_join_for(key: key)
-            system_data_for(key: key, count: count, tracking_key: tracking_key).each do |k, c|
-              client.hincrby(skey, k, c)
+        def inc(keys:, values:, count: 1, tracking_key: nil) # rubocop:disable Metrics/MethodLength
+          packed = self.class.pack(hash: values)
+          client.pipelined do |pipeline|
+            keys.each do |key|
+              key.prefix = prefix
+              pkey = key.join(separator)
+
+              packed.each do |k, c|
+                pipeline.hincrby(pkey, k, c)
+              end
+              track_system_data(pipeline, key, count, tracking_key)
             end
           end
         end
 
         def set(keys:, values:, count: 1, tracking_key: nil)
-          keys.map do |key|
-            key.prefix = prefix
-            pkey = key.join(separator)
-
-            client.hmset(pkey, *self.class.pack(hash: values))
-            next unless @system_tracking
-
-            skey = system_join_for(key: key)
-            system_data_for(key: key, count: count, tracking_key: tracking_key).each do |k, c|
-              client.hincrby(skey, k, c)
+          packed = self.class.pack(hash: values)
+          client.pipelined do |pipeline|
+            keys.each do |key|
+              key.prefix = prefix
+              pkey = key.join(separator)
+
+              pipeline.hmset(pkey, *packed)
+              track_system_data(pipeline, key, count, tracking_key)
             end
           end
         end
 
         def get(keys:)
-          keys.map do |key|
-            key.prefix = prefix
-            pkey = key.join(separator)
-
-            self.class.unpack(
-              hash: client.hgetall(pkey)
-            )
+          results = client.pipelined do |pipeline|
+            keys.each do |key|
+              key.prefix = prefix
+              pipeline.hgetall(key.join(separator))
+            end
           end
+
+          results.map { |hash| self.class.unpack(hash: hash) }
         end
 
         def ping(*)
@@ -81,6 +77,17 @@ module Trifle
         def scan(*)
           []
         end
+
+        private
+
+        def track_system_data(pipeline, key, count, tracking_key)
+          return unless @system_tracking
+
+          skey = system_join_for(key: key)
+          system_data_for(key: key, count: count, tracking_key: tracking_key).each do |k, c|
+            pipeline.hincrby(skey, k, c)
+          end
+        end
       end
     end
   end