billdogeng 1.0.2.pre.beta.3

data/lib/billdogeng/analytics.rb

@@ -0,0 +1,530 @@
+# frozen_string_literal: true
+
+require_relative "quota_limited"
+
+module BilldogEng
+  # Analytics client: capture/identify/group_identify/alias + batching.
+  #
+  # Events accumulate in an in-memory queue and are flushed as a single batched
+  # POST to `/ingest-events` when:
+  #   - the queue reaches `flush_at`,
+  #   - the background timer fires every `flush_interval`, or
+  #   - `flush` / `shutdown` is called.
+  #
+  # A failed flush re-queues its batch (front of the queue) so events are never
+  # lost on a single transient failure; the Transport also retries with backoff
+  # before a flush is considered failed.
+  #
+  # A 200 carrying `quota_limited` is NOT a failure and NOT a success: the batch was
+  # refused, not stored (see QuotaLimited). It SUSPENDS the flush loop — capture becomes
+  # a no-op, because data that cannot be stored is a memory leak rather than a rescue —
+  # and one probe batch is retried every QUOTA_SUSPEND_COOLDOWN_MS until the allowance
+  # returns.
+  class Analytics
+    # How long a quota suspension holds the flush loop closed before ONE probe batch is
+    # let through. A server process runs for months: the allowance resets monthly and the
+    # account can be upgraded at any moment, so — unlike the mobile SDKs, which suspend
+    # for the session and recover on next launch — this one has to heal itself.
+    QUOTA_SUSPEND_COOLDOWN_MS = 15 * 60 * 1000
+
+    # @param transport [Transport]
+    # @param api_key   [String]
+    # @param options   [Hash] resolved options
+    def initialize(transport, api_key:, options:)
+      @transport = transport
+      @api_key = api_key
+      @options = options
+
+      @queue = []
+      @mutex = Mutex.new       # guards @queue, @async_flush_running and the quota state
+      @flush_mutex = Mutex.new # serializes flushes so re-queued batches don't interleave
+      @timer = nil
+      @running = true
+      @async_flush_running = false # at-most-one background threshold flush in flight
+      # Resolves a user's active flags for capture(..., send_feature_flags: true). Late-bound by the
+      # client to avoid a construction-order cycle (analytics is built before flags).
+      @flag_resolver = nil
+      # Quota suspension (see QUOTA_SUSPEND_COOLDOWN_MS). Monotonic ms, so a wall-clock
+      # step on a long-lived host cannot skew — or cancel — the cooldown.
+      @quota_suspended = false
+      @quota_suspended_at = 0
+
+      # Disabled SDK: stay fully inert — no background timer, no network.
+      start_timer unless @options[:disabled]
+    end
+
+    # Late-bind the flag resolver used by `send_feature_flags` (avoids a construction-order cycle).
+    # @param callable [#call] `->(distinct_id, groups) { { key => value, ... } }`
+    def set_flag_resolver(callable)
+      @flag_resolver = callable
+    end
+
+    # Capture an arbitrary event for a user. Batched; flushed per policy.
+    #
+    # @param timestamp [Integer, Time, nil] override the event timestamp — epoch ms or a native Time
+    #   (backfill). A Time is normalized to epoch ms; nil → now.
+    # @param send_feature_flags [Boolean] attach the user's active flags as `$feature/<key>` properties
+    #   (plus `$active_feature_flags`). Best-effort, LOCAL-only, SYNC via the late-bound resolver — never
+    #   blocks capture on a fetch; server-only/cold flags are skipped.
+    def capture(distinct_id, event, properties = {}, groups = {}, timestamp: nil, send_feature_flags: false)
+      return if @options[:disabled]
+
+      props = with_groups(properties, groups)
+      if send_feature_flags && @flag_resolver
+        active = @flag_resolver.call(distinct_id, groups) || {}
+        enabled = []
+        active.each do |key, value|
+          props["$feature/#{key}"] = value
+          enabled << key unless value == false
+        end
+        props["$active_feature_flags"] = enabled
+      end
+      enqueue(distinct_id, event, props, resolve_timestamp(timestamp))
+    end
+
+    # Capture an exception as a canonical `$exception` event (PostHog error-tracking parity). Parses the
+    # error's class / message / backtrace into the `$exception_list` shape so downstream error-tracking can
+    # group and symbolicate it — do NOT hand-build `$exception` via #capture. A server SDK requires the
+    # distinct_id (there is no ambient user). Disabled => no-op. Parses defensively: a malformed/absent
+    # backtrace never raises out of here.
+    #
+    # @param error       [Exception, Object] a raised exception (preferred) or any value (its string form
+    #   becomes the message with an empty stacktrace).
+    # @param distinct_id [String]
+    # @param properties  [Hash] extra properties merged into the event.
+    def capture_exception(error, distinct_id, properties = {})
+      return if @options[:disabled]
+
+      exc = build_exception_entry(error)
+      capture(distinct_id, "$exception", stringify_keys(properties).merge(
+                                           "$exception_list" => [exc],
+                                           "$exception_type" => exc["type"],
+                                           "$exception_message" => exc["value"],
+                                           "$exception_level" => "error",
+                                         ))
+    end
+
+    # Normalize a caller-supplied timestamp (Time or epoch ms) to epoch ms; nil → now.
+    def resolve_timestamp(ts)
+      return (ts.to_f * 1000).to_i if ts.is_a?(Time)
+      return ts if ts.is_a?(Integer)
+
+      now_ms
+    end
+
+    # Set person properties. Emits the `$identify` event (identity/analytics) and, when `set_once`
+    # is supplied, ALSO persists `$set_once` properties to the profile store via
+    # #set_person_properties ("only if absent" — e.g. signup_date). Default behaviour (no set_once)
+    # is unchanged — it does NOT hit /user-properties.
+    def identify(distinct_id, properties = {}, set_once: nil)
+      enqueue(distinct_id, "$identify", { "$set" => properties }.merge(stringify_keys(properties)))
+      set_person_properties(distinct_id, set_once: set_once) if set_once && !set_once.empty?
+    end
+
+    # Apply person-property operations to the profile store (POST `/user-properties`), the
+    # PostHog-parity property-op vocabulary the `$identify` event alone cannot express:
+    # `$set` (overwrite), `$set_once` (write only if absent), `$add` (numeric increment),
+    # `$append` (push to a list), `$unset` (delete). Operations are built in the order
+    # set -> set_once -> add -> append -> unset. Empty operations => no request.
+    #
+    # Fire-and-forget on a background thread (same discipline as the threshold flush): the
+    # Transport retries transient failures; a final error is swallowed so this never blocks
+    # or raises out of the caller (best-effort telemetry contract).
+    def set_person_properties(distinct_id, set: nil, set_once: nil, add: nil, append: nil, unset: nil)
+      return if @options[:disabled]
+
+      operations = []
+      stringify_keys(set).each { |k, v| operations << { "operation" => "$set", "key" => k, "value" => v } }
+      stringify_keys(set_once).each { |k, v| operations << { "operation" => "$set_once", "key" => k, "value" => v } }
+      stringify_keys(add).each { |k, v| operations << { "operation" => "$add", "key" => k, "value" => v } }
+      stringify_keys(append).each { |k, v| operations << { "operation" => "$append", "key" => k, "value" => v } }
+      (unset || []).each { |k| operations << { "operation" => "$unset", "key" => k.to_s } }
+      return if operations.empty?
+
+      body = { "api_key" => @api_key, "distinct_id" => distinct_id, "operations" => operations }
+
+      Thread.new do
+        begin
+          @transport.request(
+            path: "/user-properties",
+            body: body,
+            headers: { "X-BillDog-API-Key" => @api_key },
+            gzip: false,
+          )
+        rescue BilldogEng::Error => e
+          log("set_person_properties failed: #{e.message}")
+        end
+      end
+    end
+
+    # Set properties on a group. Emits `$groupidentify`.
+    def group_identify(group_type, group_key, properties = {})
+      enqueue(group_key, "$groupidentify", {
+                "$group_type" => group_type,
+                "$group_key" => group_key,
+                "$group_set" => properties,
+              })
+    end
+
+    # Alias one distinct id to another. Emits `$create_alias`.
+    def alias(distinct_id, alias_id)
+      enqueue(distinct_id, "$create_alias", { "alias" => alias_id, "distinct_id" => distinct_id })
+    end
+
+    # Force-flush the queue immediately. Raises on failure (after re-queue). A quota
+    # refusal is neither: it suspends quietly, so this never raises or hangs once the
+    # allowance is gone.
+    def flush
+      @flush_mutex.synchronize { do_flush }
+    end
+
+    # Stop the background timer. Use #shutdown to also drain the queue.
+    def stop_timer
+      @running = false
+      timer = nil
+      @mutex.synchronize do
+        timer = @timer
+        @timer = nil
+      end
+      if timer
+        timer.kill
+        timer.join
+      end
+    end
+
+    # Flush remaining events and stop the background timer.
+    def shutdown
+      stop_timer
+      flush
+    end
+
+    # Current number of queued (not-yet-flushed) events.
+    def queue_length
+      @mutex.synchronize { @queue.length }
+    end
+
+    private
+
+    def log(*args)
+      warn("[BilldogEng:analytics] #{args.join(" ")}") if @options[:enable_logging]
+    end
+
+    def start_timer
+      interval_s = @options[:flush_interval] / 1000.0
+      @timer = Thread.new do
+        while @running
+          sleep(interval_s)
+          break unless @running
+
+          begin
+            flush
+          rescue BilldogEng::Error => e
+            log("background flush failed: #{e.message}")
+          end
+        end
+      end
+      @timer.abort_on_exception = false
+    end
+
+    def with_groups(properties, groups)
+      props = stringify_keys(properties)
+      return props if groups.nil? || groups.empty?
+
+      # `$groups` ({type => key}) is the only group wire shape.
+      props.merge("$groups" => stringify_keys(groups))
+    end
+
+    # PostHog-parity person-profile control (choke point). Resolves the
+    # `person_profiles` option into the reserved `$process_person_profile` wire
+    # flag the ingest pipeline honors:
+    #
+    #   "never"  => false (event stays anonymous even after identify)
+    #   "always" => true  (ingest promotes even a pre-identify anon id to a
+    #                      person — makes every device a billable MAU)
+    #   "identified_only" (default) / unset => attach nothing
+    #
+    # A caller-supplied "$process_person_profile" in properties always wins; the
+    # flag is only injected when absent.
+    def apply_person_profile_flag(properties)
+      return if properties.key?("$process_person_profile")
+
+      case @options[:person_profiles]
+      when "never"
+        properties["$process_person_profile"] = false
+      when "always"
+        properties["$process_person_profile"] = true
+      end
+    end
+
+    def enqueue(distinct_id, event_name, properties, timestamp = nil)
+      return if @options[:disabled]
+
+      # Quota-suspended: the allowance is gone, so this event CANNOT be stored however
+      # long we hold it — queueing it would only grow a long-lived process's memory.
+      return if quota_suspended?
+
+      apply_person_profile_flag(properties)
+      wire_event = {
+        "event_name" => event_name,
+        "event_timestamp" => timestamp || now_ms,
+        "properties" => properties,
+        "user_id" => distinct_id,
+      }
+
+      # before_send hook: mutate or DROP (nil) the event before it is queued. Runs at this single choke
+      # point, so it sees capture/identify/group/alias AND the `$feature_flag_called` exposure.
+      hook = @options[:before_send]
+      if hook
+        wire_event = hook.call(wire_event)
+        return if wire_event.nil?
+      end
+
+      should_flush = false
+      @mutex.synchronize do
+        @queue.push(wire_event)
+
+        max = @options[:max_queue_size]
+        if @queue.length > max
+          dropped = @queue.length - max
+          @queue.shift(dropped)
+          log("max queue size exceeded, dropped #{dropped} oldest event(s)")
+        end
+
+        should_flush = @queue.length >= @options[:flush_at]
+      end
+
+      trigger_async_flush if should_flush
+    end
+
+    # Fire-and-forget the threshold flush on a background thread so capture never
+    # blocks the caller through the transport's retry backoff and never lets a flush
+    # error propagate out of capture(). Node fire-and-forgets `void this.flush()`;
+    # the synchronous port used to block AND raise into host request code. Bounded to
+    # at most one in-flight async flush (further threshold crossings coalesce onto the
+    # next one), so this can't spawn a thread per event.
+    def trigger_async_flush
+      start = false
+      @mutex.synchronize do
+        unless @async_flush_running
+          @async_flush_running = true
+          start = true
+        end
+      end
+      return unless start
+
+      Thread.new do
+        begin
+          flush
+        rescue BilldogEng::Error => e
+          log("async flush failed: #{e.message}")
+        ensure
+          @mutex.synchronize { @async_flush_running = false }
+        end
+      end
+    end
+
+    def do_flush
+      # A quota suspension stops the flush loop dead until the cooldown expires, at which
+      # point exactly one probe batch is let through to test whether the allowance is back.
+      return if quota_flush_blocked?
+
+      batch = nil
+      @mutex.synchronize do
+        return if @queue.empty?
+
+        batch = @queue
+        @queue = []
+      end
+
+      body = { "api_key" => @api_key, "events" => batch }
+
+      # GeoIP control (PostHog `disable_geoip` parity): a server SDK's request IP is the customer's
+      # backend, so tell ingest how precisely to resolve it — or not to (`"none"`). Only sent when
+      # configured; otherwise the project default applies. Header rides ONLY the /ingest-events flush.
+      headers = { "X-BillDog-API-Key" => @api_key }
+      level = @options[:geoip_resolution_level]
+      headers["x-geoip-resolution-level"] = level if level
+
+      begin
+        result = @transport.request(
+          path: "/ingest-events",
+          body: body,
+          headers: headers,
+        )
+        # A 2xx is no longer proof of storage: `quota_limited` means this batch was
+        # refused and dropped. Keep it as the probe batch and go quiet — never report
+        # success for data the server just discarded.
+        if QuotaLimited.limited?(result)
+          @mutex.synchronize { @queue = batch.concat(@queue) }
+          enter_quota_suspension(result)
+          return
+        end
+
+        leave_quota_suspension
+        log("flushed #{batch.length} event(s)")
+      rescue BilldogEng::Error => e
+        # Re-queue the batch at the front so it is retried on the next flush;
+        # never lose events on a single failure.
+        @mutex.synchronize { @queue = batch.concat(@queue) }
+        log("flush failed, re-queued #{batch.length} event(s): #{e.message}")
+        raise e
+      end
+    end
+
+    # True while a quota suspension holds. Stays true across the whole cooldown so the
+    # queue cannot grow while the pipeline is dark.
+    def quota_suspended?
+      @mutex.synchronize { @quota_suspended }
+    end
+
+    # True while suspended AND still inside the cooldown, i.e. no probe flush is due yet.
+    def quota_flush_blocked?
+      @mutex.synchronize do
+        @quota_suspended && (monotonic_ms - @quota_suspended_at) < QUOTA_SUSPEND_COOLDOWN_MS
+      end
+    end
+
+    # Enter — or re-arm — the quota suspension. The reason is logged EXACTLY ONCE, on
+    # entry: a probe that finds the allowance still gone only resets the cooldown, so a
+    # dark pipeline does not also fill the host's log every QUOTA_SUSPEND_COOLDOWN_MS.
+    def enter_quota_suspension(result)
+      first = false
+      @mutex.synchronize do
+        first = !@quota_suspended
+        @quota_suspended = true
+        @quota_suspended_at = monotonic_ms
+      end
+      quota_warn(QuotaLimited.reason(result)) if first
+    end
+
+    # Leave the suspension after a probe batch was accepted, logging the recovery once so
+    # the silence has a visible end as well as a visible start.
+    def leave_quota_suspension
+      return unless @quota_suspended
+
+      resumed = false
+      @mutex.synchronize do
+        resumed = @quota_suspended
+        @quota_suspended = false
+        @quota_suspended_at = 0
+      end
+      quota_warn("allowance restored — resuming event flushes") if resumed
+    end
+
+    # Unconditional stderr warning, NOT gated on :enable_logging like #log: a quota
+    # suspension is the only signal a developer gets that their pipeline went dark.
+    def quota_warn(message)
+      warn("[BilldogEng:analytics] #{message}")
+    end
+
+    # Monotonic clock for the cooldown only; #now_ms stays wall-clock because event
+    # timestamps have to mean something to the server.
+    def monotonic_ms
+      (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i
+    end
+
+    def now_ms
+      (Time.now.to_f * 1000).to_i
+    end
+
+    # Build one `$exception_list` entry (PostHog canonical shape) from a raised exception or any value.
+    # A real Exception contributes its class name / message / backtrace frames; any other value yields
+    # type "Error", its string form as the message, and no frames. Never raises.
+    def build_exception_entry(error)
+      is_exception = error.is_a?(Exception)
+      type  = is_exception ? (error.class.name || "Error") : "Error"
+      value = is_exception ? error.message.to_s : error.to_s
+      frames = is_exception ? parse_backtrace(error) : []
+      {
+        "type" => type,
+        "value" => value,
+        "mechanism" => { "handled" => true, "synthetic" => false },
+        "stacktrace" => { "type" => "raw", "frames" => frames },
+      }
+    rescue StandardError
+      # Defensive: a pathological error object (e.g. #message raises) must not break capture_exception.
+      { "type" => "Error", "value" => "", "mechanism" => { "handled" => true, "synthetic" => false },
+        "stacktrace" => { "type" => "raw", "frames" => [] } }
+    end
+
+    # Ruby / bundled-gem path markers → in_app = false (stdlib + vendored/dependency frames).
+    IN_APP_EXCLUDE = %r{
+      /gems/ | /ruby/ | /rubygems/ | /vendor/ | /bundle/ |
+      /\.rbenv/ | /\.rvm/ | /\.gem/ | /ruby[0-9]
+    }x
+    private_constant :IN_APP_EXCLUDE
+
+    # Parse an exception's backtrace into frames, OLDEST-FIRST. Prefers structured
+    # `backtrace_locations` (Thread::Backtrace::Location: #path/#label/#lineno); falls back to parsing the
+    # `backtrace` strings ("path:lineno:in 'method'"). A nil/absent backtrace yields []. Never raises.
+    def parse_backtrace(error)
+      locations = safe_backtrace_locations(error)
+      frames =
+        if locations && !locations.empty?
+          locations.map { |loc| frame_from_location(loc) }.compact
+        else
+          (safe_backtrace(error) || []).map { |line| frame_from_line(line) }.compact
+        end
+      # Ruby backtraces are newest-first (call site first); PostHog orders oldest-first.
+      frames.reverse
+    rescue StandardError
+      []
+    end
+
+    def safe_backtrace_locations(error)
+      error.backtrace_locations
+    rescue StandardError
+      nil
+    end
+
+    def safe_backtrace(error)
+      error.backtrace
+    rescue StandardError
+      nil
+    end
+
+    def frame_from_location(loc)
+      filename = (loc.path || "").to_s
+      {
+        "filename" => filename,
+        "function" => (loc.label || "<unknown>").to_s,
+        "lineno" => loc.lineno.to_i,
+        "in_app" => in_app?(filename),
+      }
+    rescue StandardError
+      nil
+    end
+
+    # Parse a backtrace string: "path/to/file.rb:123:in 'method'" (Ruby 3.4+ single-quote) or the older
+    # backtick form "...:123:in `method'". Trailing method chunk is optional.
+    def frame_from_line(line)
+      s = line.to_s
+      m = /\A(.+):(\d+)(?::in [`']?(.*?)'?)?\z/.match(s)
+      return nil unless m
+
+      filename = m[1]
+      {
+        "filename" => filename,
+        "function" => (m[3] && !m[3].empty? ? m[3] : "<unknown>"),
+        "lineno" => m[2].to_i,
+        "in_app" => in_app?(filename),
+      }
+    rescue StandardError
+      nil
+    end
+
+    def in_app?(filename)
+      return false if filename.nil? || filename.empty?
+
+      !IN_APP_EXCLUDE.match?(filename)
+    end
+
+    def stringify_keys(hash)
+      return {} if hash.nil?
+
+      hash.each_with_object({}) do |(k, v), acc|
+        acc[k.is_a?(Symbol) ? k.to_s : k] = v
+      end
+    end
+  end
+end