@jeffjassky/telemetry 0.5.0 → 0.6.0

package/dist/index.js

@@ -551,10 +551,11 @@ var truncate = (d, b) => {
   if (b === "day") return day;
   return new Date(day.getTime() - (day.getUTCDay() + 6) % 7 * 864e5);
 };
-async function recordRollup(RollupModel, doc, name, spec, counters) {
+async function recordRollup(RollupModel, doc, name, spec, counters, opts) {
+  const dry = opts?.dryRun === true;
   if (spec.actors && doc.actor) {
     const actorType = String(doc.actor).split(":")[0];
-    if (!spec.actors.includes(actorType)) return;
+    if (!spec.actors.includes(actorType)) return 0;
   }
   const as = spec.as ?? name;
   const at = doc.occurredAt;
@@ -566,9 +567,11 @@ async function recordRollup(RollupModel, doc, name, spec, counters) {
     let v = resolveDim(src, doc);
     if (v == null || v === "") {
       if (spec.dimDefault === void 0) {
-        counters.rollupSkipped++;
-        bumpCounterMap(counters.rollupSkippedBy, `${as}|${label(src)}`);
-        return;
+        if (!dry) {
+          counters.rollupSkipped++;
+          bumpCounterMap(counters.rollupSkippedBy, `${as}|${label(src)}`);
+        }
+        return 0;
       }
       v = spec.dimDefault;
     }
@@ -578,48 +581,48 @@ async function recordRollup(RollupModel, doc, name, spec, counters) {
   const refs = fansOut ? (doc.subjectKeys ?? []).filter(
     (r) => !spec.subjects || spec.subjects.includes(r.split(":")[0])
   ) : [null];
-  if (!refs.length) return;
+  if (!refs.length) return 0;
   const firstCapture = Object.fromEntries(
     (spec.capture ?? []).map((src) => [label(src), resolveDim(src, doc)]).filter(([, v]) => v != null).map(([k, v]) => [k, String(v)])
   );
   const expiresAt = spec.retentionDays != null ? new Date(at.getTime() + spec.retentionDays * 864e5) : void 0;
-  await RollupModel.bulkWrite(
-    refs.map((ref) => {
-      const dims = spec.by.map((src) => src === "subject" ? ref : fixed.get(src));
-      const isNewFirst = {
-        $or: [{ $eq: [{ $type: "$firstAt" }, "missing"] }, { $lt: [at, "$firstAt"] }]
-      };
-      const sums = Object.fromEntries(
-        (spec.sum ?? []).map((k) => [k, doc.metrics?.get(k)]).filter(([, v]) => typeof v === "number").map(([k, v]) => [`sums.${k}`, { $add: [{ $ifNull: [`$sums.${k}`, 0] }, v] }])
-      );
-      return {
-        updateOne: {
-          filter: { _id: `${doc.tenantId}|${as}|${dims.join("|")}|${bucketKey}` },
-          update: [
-            {
-              $set: {
-                tenantId: doc.tenantId,
-                as,
-                dims,
-                ...ref ? { subjectType: ref.split(":")[0] } : {},
-                ...bucketAt ? { bucketAt } : {},
-                ...expiresAt ? { expiresAt } : {},
-                // aggregation $min/$max ignore missing, so correct on insert too
-                firstAt: { $min: ["$firstAt", at] },
-                lastAt: { $max: ["$lastAt", at] },
-                count: { $add: [{ $ifNull: ["$count", 0] }, 1] },
-                ...sums,
-                firstTraceId: { $cond: [isNewFirst, doc.traceId ?? null, "$firstTraceId"] },
-                firstCapture: { $cond: [isNewFirst, { $literal: firstCapture }, "$firstCapture"] }
-              }
+  const ops = refs.map((ref) => {
+    const dims = spec.by.map((src) => src === "subject" ? ref : fixed.get(src));
+    const isNewFirst = {
+      $or: [{ $eq: [{ $type: "$firstAt" }, "missing"] }, { $lt: [at, "$firstAt"] }]
+    };
+    const sums = Object.fromEntries(
+      (spec.sum ?? []).map((k) => [k, doc.metrics?.get(k)]).filter(([, v]) => typeof v === "number").map(([k, v]) => [`sums.${k}`, { $add: [{ $ifNull: [`$sums.${k}`, 0] }, v] }])
+    );
+    return {
+      updateOne: {
+        filter: { _id: `${doc.tenantId}|${as}|${dims.join("|")}|${bucketKey}` },
+        update: [
+          {
+            $set: {
+              tenantId: doc.tenantId,
+              as,
+              dims,
+              ...ref ? { subjectType: ref.split(":")[0] } : {},
+              ...bucketAt ? { bucketAt } : {},
+              ...expiresAt ? { expiresAt } : {},
+              // aggregation $min/$max ignore missing, so correct on insert too
+              firstAt: { $min: ["$firstAt", at] },
+              lastAt: { $max: ["$lastAt", at] },
+              count: { $add: [{ $ifNull: ["$count", 0] }, 1] },
+              ...sums,
+              firstTraceId: { $cond: [isNewFirst, doc.traceId ?? null, "$firstTraceId"] },
+              firstCapture: { $cond: [isNewFirst, { $literal: firstCapture }, "$firstCapture"] }
             }
-          ],
-          upsert: true
-        }
-      };
-    }),
-    { ordered: false }
-  );
+          }
+        ],
+        upsert: true
+      }
+    };
+  });
+  if (dry) return ops.length;
+  await RollupModel.bulkWrite(ops, { ordered: false });
+  return ops.length;
 }
 function buildCheckpointModel(connection, modelName, collection) {
   const existing = connection.models?.[modelName];
@@ -983,6 +986,109 @@ function createForget(ctx) {
   };
 }
 
+// src/server/relink.ts
+var RELINK_BATCH_SIZE = 500;
+var refKey = (s) => `${s.type}:${s.id}`;
+function createRelink(ctx) {
+  const { registry, TelemetryModel, RollupModel, counters, logger, linkSubjects } = ctx;
+  return async function relink(opts = {}) {
+    const dryRun = opts.dryRun !== false;
+    const batchSize = opts.batchSize ?? RELINK_BATCH_SIZE;
+    const result = {
+      examined: 0,
+      linked: 0,
+      subjects: 0,
+      rollups: 0,
+      misses: 0,
+      errors: 0,
+      skipped: 0
+    };
+    if (opts.names) {
+      const unknown = opts.names.filter((n) => !registry[n]);
+      if (unknown.length) {
+        throw new Error(
+          `telemetry: relink() was given names this registry does not declare: ${unknown.join(", ")}. A name with no spec has no rollup families to replay, so relinking it could only desynchronize its rows from its aggregates. Omit \`names\` to sweep everything.`
+        );
+      }
+    }
+    if (!linkSubjects) {
+      logger.warn(
+        "[telemetry] relink() has no subjectLinker to ask \u2014 nothing was read and nothing was written. Configure createTelemetry({ subjectLinker }) first; relink() backfills what that hook would have done, it does not replace it."
+      );
+      result.skipped = 1;
+      return result;
+    }
+    if (opts.limit != null && opts.limit <= 0) return result;
+    const filter = {};
+    if (opts.names) filter.name = { $in: opts.names };
+    if (opts.since) filter.occurredAt = { $gte: opts.since };
+    const query = TelemetryModel.find(filter).hint({ _id: 1 }).batchSize(batchSize);
+    if (opts.limit != null) query.limit(opts.limit);
+    const progress = () => {
+      if (!opts.onProgress) return;
+      try {
+        opts.onProgress({ ...result });
+      } catch (e) {
+        logger.warn(`[telemetry] relink() onProgress threw \u2014 ignored, the backfill continues: ${e}`);
+      }
+    };
+    const cursor = query.cursor();
+    let sinceProgress = 0;
+    try {
+      for await (const doc of cursor) {
+        result.examined++;
+        sinceProgress++;
+        if (sinceProgress >= batchSize) {
+          sinceProgress = 0;
+          progress();
+        }
+        const spec = registry[doc.name];
+        if (!spec) {
+          result.skipped++;
+          continue;
+        }
+        const declared = (doc.subjects ?? []).map(
+          (s) => s.role ? { type: s.type, id: s.id, role: s.role } : { type: s.type, id: s.id }
+        );
+        const m0 = counters.subjectLinkMisses;
+        const e0 = counters.subjectLinkErrors + counters.subjectLinkTimeouts;
+        const merged = await linkSubjects(doc.name, spec, doc.tenantId, declared);
+        result.misses += counters.subjectLinkMisses - m0;
+        if (counters.subjectLinkErrors + counters.subjectLinkTimeouts > e0) result.errors++;
+        if (!merged) continue;
+        const had = new Set(declared.map(refKey));
+        const mergedKeys = [];
+        const newKeys = [];
+        for (const s of merged) {
+          const key = refKey(s);
+          if (mergedKeys.includes(key)) continue;
+          mergedKeys.push(key);
+          if (!had.has(key)) newKeys.push(key);
+        }
+        if (!newKeys.length) continue;
+        result.linked++;
+        result.subjects += newKeys.length;
+        if (!dryRun) {
+          await TelemetryModel.updateOne(
+            { _id: doc._id },
+            { $set: { subjects: merged, subjectKeys: mergedKeys } }
+          );
+        }
+        doc.subjectKeys = newKeys;
+        for (const r of spec.rollups ?? []) {
+          if (!r.by.includes("subject")) continue;
+          result.rollups += await recordRollup(RollupModel, doc, doc.name, r, counters, { dryRun });
+        }
+      }
+    } finally {
+      await cursor.close().catch(() => {
+      });
+    }
+    if (sinceProgress) progress();
+    return result;
+  };
+}
+
 // src/server/indexes.ts
 var INDEX_BUDGET = 24;
 function createSyncIndexes(ctx) {
@@ -3774,6 +3880,14 @@ function createTelemetry(config) {
     },
     globalSubjectRefs: () => config.globalSubjectRefs === true
   });
+  const relink = createRelink({
+    registry,
+    TelemetryModel,
+    RollupModel,
+    counters,
+    logger,
+    linkSubjects
+  });
   const syncIndexes = createSyncIndexes({
     registry,
     TelemetryModel,
@@ -3785,6 +3899,21 @@ function createTelemetry(config) {
     emit,
     /** erasure: delete sole-party rows, redact shared ones, rekey rollups, drop aliases */
     forget,
+    /**
+     * Backfill: re-ask the `subjectLinker` about records ALREADY on disk, and
+     * replay the rollups the new subjects reach.
+     *
+     * Linking happens at write time, so configuring it fixes the future and
+     * nothing else — a lifetime `by:['subject']` family is keyed on the subject
+     * the record was written with, and a read-time join cannot reach back into
+     * it. This is how a host catches up the backlog it adopted the hook with.
+     *
+     * DRY RUN BY DEFAULT: it rewrites historical aggregates, so the short call
+     * reports and the writing call says `{ dryRun: false }`. Idempotent by
+     * construction — a row that already carries the linked subject yields
+     * nothing new, so a second run is a no-op.
+     */
+    relink,
     /**
      * Tenant scope is not optional — force every read through here. The five
      * dashboard query primitives (records/series/distribution/rollups/journey)
@@ -3846,6 +3975,6 @@ function createTelemetry(config) {
   };
 }
 
-export { BODY_MAX_CHARS, COUNTER_MAP_MAX, COUNTER_OVERFLOW_KEY, DEFAULT_LIMITS, Env, INDEX_BUDGET, KeyKind, LogLevel, MAX_SUGGESTIONS, Origin, PLATFORM_SCOPE, RETENTION_DAYS, SAMPLE_RATE, SCHEMA_VERSION, SUBJECT_LINK_TIMEOUT_MS, SUBJECT_MAX, TelemetryKind, TenantMode, boundedMeta, createDashboard, createIngest, createKey, createQueries, createTelemetry, createValues, defaultSpaDir, defineRegistry, deriveCatalog, deriveSuggestions, deriveViews, executeReport, findFamily, foldRollups, hashSecret, intervalForRange, isPlatformScope, median, newId, normalizeQuery, parseKeyString, parseReportQuery, plain, projectRegistry, rangeOf, reportToQuery, requireMilestoneFamily, resolveDim, resolveReport, summarizeStages, traceKeep, truncate, validateRegistry, verifySecret };
+export { BODY_MAX_CHARS, COUNTER_MAP_MAX, COUNTER_OVERFLOW_KEY, DEFAULT_LIMITS, Env, INDEX_BUDGET, KeyKind, LogLevel, MAX_SUGGESTIONS, Origin, PLATFORM_SCOPE, RELINK_BATCH_SIZE, RETENTION_DAYS, SAMPLE_RATE, SCHEMA_VERSION, SUBJECT_LINK_TIMEOUT_MS, SUBJECT_MAX, TelemetryKind, TenantMode, boundedMeta, createDashboard, createIngest, createKey, createQueries, createTelemetry, createValues, defaultSpaDir, defineRegistry, deriveCatalog, deriveSuggestions, deriveViews, executeReport, findFamily, foldRollups, hashSecret, intervalForRange, isPlatformScope, median, newId, normalizeQuery, parseKeyString, parseReportQuery, plain, projectRegistry, rangeOf, reportToQuery, requireMilestoneFamily, resolveDim, resolveReport, summarizeStages, traceKeep, truncate, validateRegistry, verifySecret };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map