@noego/telemetry 0.0.1 → 0.0.3

package/dist/cli/telemetry_bin.cjs

@@ -244,6 +244,7 @@ var TelemetryTableMap = class _TelemetryTableMap {
       rawEventsOccurredIndex: this.indexName(mapping.rawEvents, "occurred"),
       rawEventsConversationIndex: this.indexName(mapping.rawEvents, "conversation"),
       rawEventsLabelIndex: this.indexName(mapping.rawEvents, "label"),
+      rawEventsLevelIndex: this.indexName(mapping.rawEvents, "level"),
       metricSamplesTimeIndex: this.indexName(mapping.metricSamples, "time"),
       metricSamplesEventIndex: this.indexName(mapping.metricSamples, "event"),
       metricRollupsTimeIndex: this.indexName(mapping.metricRollups, "time")
@@ -418,6 +419,7 @@ var SqliteTargetSchema = import_zod.z.object({
     raw: DurationSchema,
     lateArrival: DurationSchema.optional(),
     samples: DurationSchema.optional(),
+    levels: import_zod.z.record(import_zod.z.string().min(1), DurationSchema).optional(),
     rollups: import_zod.z.array(
       import_zod.z.object({
         resolution: DurationSchema,
@@ -560,6 +562,7 @@ var NodeTelemetryConfigurationParser = class {
       true
     );
     const samplesMs = target.retention.samples === void 0 ? rawMs + lateArrivalMs : this.durations.parse(target.retention.samples, `${field}.retention.samples`);
+    const levelMs = this.resolveLevelRetention(target.retention.levels, field);
     const rollups = target.retention.rollups.map(
       (tier, tierIndex) => Object.freeze({
         resolutionMs: this.durations.parse(
@@ -594,6 +597,7 @@ var NodeTelemetryConfigurationParser = class {
         rawMs,
         lateArrivalMs,
         samplesMs,
+        levelMs,
         rollups: Object.freeze(rollups)
       }),
       maintenance: Object.freeze({
@@ -624,6 +628,26 @@ var NodeTelemetryConfigurationParser = class {
     }
     return resolved;
   }
+  resolveLevelRetention(levels, field) {
+    if (levels === void 0) {
+      return Object.freeze({});
+    }
+    const resolved = {};
+    for (const [level, duration] of Object.entries(levels)) {
+      const normalized = level.trim().toLowerCase();
+      if (normalized.length === 0) {
+        throw new Error(`${field}.retention.levels contains an empty level name`);
+      }
+      if (resolved[normalized] !== void 0) {
+        throw new Error(`${field}.retention.levels declares "${normalized}" more than once`);
+      }
+      resolved[normalized] = this.durations.parse(
+        duration,
+        `${field}.retention.levels.${normalized}`
+      );
+    }
+    return Object.freeze(resolved);
+  }
   validateRetention(target, field) {
     const retention = target.retention;
     const requiredSamplesMs = retention.rawMs + retention.lateArrivalMs;
@@ -3377,73 +3401,49 @@ var TelemetryRepository = class {
       lastSequence: sample.sequence
     };
   }
+  /**
+   * Raw retention is purely time-based, matching the native collector: every
+   * expired row goes in one set-based statement per level band. No page size,
+   * no row cap, and no rollup-completion gate — metric samples are extracted at
+   * ingest and retained on their own clock, so dropping a raw row costs no
+   * metric fidelity, and no row can outlive its window waiting on a rollup.
+   */
   createRawEventRetentionStep(nowMs) {
-    const cutoffMs = nowMs - this.options.retention.rawMs;
     const table = this.options.tableMap.quoted("rawEvents");
-    const readFirstPage = this.options.connection.prepare(
-      `SELECT producer_id, sequence, event_id, occurred_at_ms, monotonic_ns,
-              source, process_kind, level, label, conversation_id,
-              conversation_id_kind, correlation_id, metadata_json, schema_version
-       FROM ${table}
-       WHERE target_id = ? AND occurred_at_ms < ?
-       ORDER BY occurred_at_ms ASC, producer_id ASC, sequence ASC
-       LIMIT ${RETENTION_PAGE_SIZE}`
-    );
-    const readNextPage = this.options.connection.prepare(
-      `SELECT producer_id, sequence, event_id, occurred_at_ms, monotonic_ns,
-              source, process_kind, level, label, conversation_id,
-              conversation_id_kind, correlation_id, metadata_json, schema_version
-       FROM ${table}
-       WHERE target_id = ? AND occurred_at_ms < ?
-         AND (occurred_at_ms, producer_id, sequence) > (?, ?, ?)
-       ORDER BY occurred_at_ms ASC, producer_id ASC, sequence ASC
-       LIMIT ${RETENTION_PAGE_SIZE}`
+    const levelMs = this.options.retention.levelMs ?? {};
+    const levels = Object.keys(levelMs);
+    const removeByLevel = this.options.connection.prepare(
+      `DELETE FROM ${table}
+       WHERE target_id = ? AND lower(coalesce(level, '')) = ? AND occurred_at_ms < ?`
     );
-    const remove = this.options.connection.prepare(
-      `DELETE FROM ${table} WHERE target_id = ? AND producer_id = ? AND sequence = ?`
+    const fallbackPlaceholders = levels.map(() => "?").join(", ");
+    const removeFallback = this.options.connection.prepare(
+      `DELETE FROM ${table}
+       WHERE target_id = ? AND occurred_at_ms < ?` + (levels.length === 0 ? "" : ` AND lower(coalesce(level, '')) NOT IN (${fallbackPlaceholders})`)
     );
-    let cursor = null;
+    let done = false;
     return () => {
-      const rows = cursor === null ? readFirstPage.all(this.options.targetId, cutoffMs) : readNextPage.all(
-        this.options.targetId,
-        cutoffMs,
-        cursor.occurredAtMs,
-        cursor.producerId,
-        cursor.sequence
-      );
-      if (rows.length === 0) {
+      if (done) {
         return { deleted: 0, done: true };
       }
-      const scanned = rows.map((row) => {
-        const parsed = this.rowParser.parseRawEnvelope(row);
-        if (!parsed.ok) {
-          throw new Error(parsed.error);
-        }
-        return parsed.value;
-      });
-      const eligible = scanned.filter((envelope) => this.rawEventHasRequiredRollups(envelope));
+      done = true;
       const transaction = this.options.connection.transaction(() => {
-        let deleted2 = 0;
-        for (const envelope of eligible) {
-          deleted2 += remove.run(
-            this.options.targetId,
-            envelope.producerId,
-            envelope.sequence
-          ).changes;
+        let deleted = 0;
+        for (const level of levels) {
+          const retainMs = levelMs[level];
+          if (retainMs === void 0) {
+            continue;
+          }
+          deleted += removeByLevel.run(this.options.targetId, level, nowMs - retainMs).changes;
         }
-        return deleted2;
+        deleted += removeFallback.run(
+          this.options.targetId,
+          nowMs - this.options.retention.rawMs,
+          ...levels
+        ).changes;
+        return deleted;
       });
-      const deleted = transaction();
-      const last = scanned[scanned.length - 1];
-      if (last === void 0 || rows.length < RETENTION_PAGE_SIZE) {
-        return { deleted, done: true };
-      }
-      cursor = {
-        occurredAtMs: last.occurredAtMs,
-        producerId: last.producerId,
-        sequence: last.sequence
-      };
-      return { deleted, done: false };
+      return { deleted: transaction(), done: true };
     };
   }
   rawEventHasRequiredRollups(envelope) {