@jeffjassky/telemetry 0.6.1 → 0.8.0

package/dist/index.js

@@ -2,6 +2,7 @@ import { uuidv7 } from 'uuidv7';
 import { z } from 'zod';
 import { Schema } from 'mongoose';
 import { randomBytes, scryptSync, timingSafeEqual, createHash } from 'crypto';
+import { originalPositionFor, sourceContentFor, TraceMap } from '@jridgewell/trace-mapping';
 import express2 from 'express';
 import fs from 'fs';
 import path2 from 'path';
@@ -62,6 +63,8 @@ var newCounters = () => ({
   truncated: 0,
   rollupSkippedBy: {},
   undeclaredAttrs: {},
+  attrsDropped: {},
+  metricsDropped: {},
   subjectsLinked: 0,
   subjectLinkMisses: 0,
   subjectLinkErrors: 0,
@@ -271,6 +274,7 @@ var UsageDetailSchema = new Schema(
   { _id: false }
 );
 function buildBaseSchema(collection, registry, counters, opts) {
+  const validation = opts.validation;
   const schema = new Schema(
     {
       /** UUIDv7 — sortable, insertion-local, replaces the ObjectId */
@@ -412,13 +416,39 @@ function buildBaseSchema(collection, registry, counters, opts) {
     }
     const check = (label3, m, zschema) => {
       const obj = Object.fromEntries(m ?? []);
+      const dropped = label3 === "attrs" ? counters.attrsDropped : counters.metricsDropped;
+      const note = (key) => bumpCounterMap(dropped, `${this.name}|${key}`);
       if (!zschema) {
-        if (Object.keys(obj).length) throw new Error(`telemetry: "${this.name}" declares no ${label3}`);
+        if (!Object.keys(obj).length) return;
+        if (validation === "strict") throw new Error(`telemetry: "${this.name}" declares no ${label3}`);
+        for (const key of Object.keys(obj)) {
+          note(key);
+          m?.delete(key);
+        }
         return;
       }
       const s = zschema.strict?.() ?? zschema;
       const r = s.safeParse(obj);
-      if (!r.success) throw new Error(`telemetry: ${label3} invalid for "${this.name}": ${r.error.message}`);
+      if (r.success) return;
+      if (validation === "strict") {
+        throw new Error(`telemetry: ${label3} invalid for "${this.name}": ${r.error.message}`);
+      }
+      const bad = /* @__PURE__ */ new Set();
+      for (const issue of r.error.issues) {
+        if (issue.code === "unrecognized_keys" && Array.isArray(issue.keys)) {
+          for (const k of issue.keys) bad.add(String(k));
+        } else if (Array.isArray(issue.path) && issue.path.length) {
+          bad.add(String(issue.path[0]));
+        }
+      }
+      for (const key of bad) {
+        if (m?.has(key)) {
+          note(key);
+          m.delete(key);
+        } else {
+          note("(missing)");
+        }
+      }
     };
     check("attrs", this.attrs, spec.attrs);
     check("metrics", this.metrics, spec.metrics);
@@ -451,7 +481,11 @@ function buildTelemetryModels(opts) {
       )
     };
   }
-  const base = buildBaseSchema(collection, registry, counters, { platforms, bodyMax });
+  const base = buildBaseSchema(collection, registry, counters, {
+    platforms,
+    bodyMax,
+    validation: opts.validation ?? "lenient"
+  });
   const TelemetryModel = connection.model(modelName, base);
   const disc = (kind, build) => TelemetryModel.discriminator(`${modelName}_${kind}`, build(), kind);
   const byKind = {
@@ -1128,6 +1162,145 @@ function createSyncIndexes(ctx) {
     await ctx.rejects().createIndex({ at: 1 }, { expireAfterSeconds: REJECT_TTL_DAYS * 86400 });
   };
 }
+var SOURCEMAP_MAX_BYTES = 15 * 1024 * 1024;
+var SOURCEMAP_RETENTION_DAYS = 90;
+var CACHE_SIZE = 8;
+var CONTEXT_MAX = 200;
+var bundleFile = (filename) => String(filename ?? "").split(/[?#]/)[0].split(/[\\/]/).pop() ?? "";
+var cleanSource = (source) => (
+  // webpack:///src/x.ts, webpack://app/./src/x.ts → src/x.ts
+  source.replace(/^webpack:\/\/[^/]*\//, "").replace(/^\.\//, "")
+);
+function createSourcemaps(opts) {
+  const coll = () => opts.connection.db.collection(opts.collection);
+  const cache = /* @__PURE__ */ new Map();
+  const remember = (key, value) => {
+    cache.delete(key);
+    cache.set(key, value);
+    while (cache.size > CACHE_SIZE) cache.delete(cache.keys().next().value);
+  };
+  const keyOf = (tenantId, service, release, file) => `${tenantId}\0${service}\0${release}\0${file}`;
+  async function load(tenantId, service, release, file) {
+    const key = keyOf(tenantId, service, release, file);
+    if (cache.has(key)) {
+      const hit = cache.get(key);
+      remember(key, hit);
+      return hit;
+    }
+    let parsed = null;
+    try {
+      const doc = await coll().findOne({ tenantId, service, release, file }, { projection: { map: 1 } });
+      if (doc?.map) parsed = new TraceMap(String(doc.map));
+    } catch (e) {
+      opts.logger.warn("[telemetry] sourcemap load failed", file, e?.message);
+    }
+    remember(key, parsed);
+    return parsed;
+  }
+  return {
+    async ensureIndexes() {
+      await coll().createIndex(
+        { tenantId: 1, service: 1, release: 1, file: 1 },
+        { unique: true, name: "sourcemap_identity" }
+      );
+      await coll().createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0, name: "sourcemap_ttl" });
+    },
+    /**
+     * Store maps for one release. Idempotent: registering the same release
+     * again replaces its maps and refreshes their retention, so a host can
+     * simply call this on every boot.
+     */
+    async register(input) {
+      const { tenantId, service, release } = input;
+      if (!tenantId || !service || !release) {
+        throw new Error("telemetry: sourcemaps.register needs tenantId, service and release");
+      }
+      if (release === "unknown") {
+        throw new Error('telemetry: refusing to register sourcemaps for release "unknown" \u2014 set a real release on the client');
+      }
+      const now = /* @__PURE__ */ new Date();
+      const expiresAt = new Date(now.getTime() + SOURCEMAP_RETENTION_DAYS * 864e5);
+      const skipped = [];
+      let stored = 0;
+      for (const f of input.files) {
+        const file = bundleFile(f.file);
+        const map = typeof f.map === "string" ? f.map : JSON.stringify(f.map);
+        if (!file || !map) {
+          skipped.push(f.file);
+          continue;
+        }
+        if (Buffer.byteLength(map) > SOURCEMAP_MAX_BYTES) {
+          opts.logger.warn(`[telemetry] sourcemap ${file} exceeds ${SOURCEMAP_MAX_BYTES} bytes \u2014 skipped`);
+          skipped.push(f.file);
+          continue;
+        }
+        try {
+          JSON.parse(map);
+        } catch {
+          skipped.push(f.file);
+          continue;
+        }
+        await coll().updateOne(
+          { tenantId, service, release, file },
+          { $set: { map, bytes: Buffer.byteLength(map), registeredAt: now, expiresAt } },
+          { upsert: true }
+        );
+        cache.delete(keyOf(tenantId, service, release, file));
+        stored++;
+      }
+      return { stored, skipped };
+    },
+    /** translate one position; null when no map is registered or it has no mapping there */
+    async resolve(where, lineno, colno) {
+      const file = bundleFile(where.filename);
+      if (!file || !where.release || where.release === "unknown") return null;
+      if (!Number.isFinite(lineno) || !Number.isFinite(colno)) return null;
+      const map = await load(where.tenantId, where.service, where.release, file);
+      if (!map) return null;
+      const pos = originalPositionFor(map, { line: lineno, column: Math.max(0, colno - 1) });
+      if (!pos.source || pos.line == null) return null;
+      const out = {
+        source: cleanSource(pos.source),
+        line: pos.line,
+        column: (pos.column ?? 0) + 1
+      };
+      if (pos.name) out.name = pos.name;
+      const content = sourceContentFor(map, pos.source);
+      const text = content?.split("\n")[pos.line - 1];
+      if (text != null) out.context = text.trim().slice(0, CONTEXT_MAX);
+      return out;
+    },
+    /**
+     * A copy of an error record with `original` set on every frame a map
+     * could translate. Anything else comes back unchanged — never throws.
+     */
+    async symbolicate(record) {
+      const frames = record?.error?.frames;
+      if (record?.kind !== "error" || !Array.isArray(frames) || !frames.length) return record;
+      try {
+        const translated = await Promise.all(
+          frames.map(async (f) => {
+            const original = await this.resolve(
+              {
+                tenantId: record.tenantId,
+                service: record.service,
+                release: record.release,
+                filename: f?.filename
+              },
+              Number(f?.lineno),
+              Number(f?.colno)
+            );
+            return original ? { ...f, original } : f;
+          })
+        );
+        return { ...record, error: { ...record.error, frames: translated } };
+      } catch (e) {
+        opts.logger.warn("[telemetry] symbolicate failed", e?.message);
+        return record;
+      }
+    }
+  };
+}
 var KeyKind = { Publishable: "publishable", Secret: "secret" };
 var TenantMode = { Fixed: "fixed", Session: "session", Claimed: "claimed" };
 var KEY_RE = /^(pk|sk)_([a-z0-9]+)_(tk_[a-f0-9]{24})(?:_([a-f0-9]{48}))?$/;
@@ -3587,12 +3760,14 @@ function createDashboard(opts) {
     scope: req.viewer.tenantId,
     platform: isPlatformScope(req.viewer.tenantId)
   })));
-  api.get("/records", h(
-    async (req) => q.records(req.viewer.tenantId, parseRange(req.query), parseFilter(req.query), {
+  api.get("/records", h(async (req) => {
+    const page = await q.records(req.viewer.tenantId, parseRange(req.query), parseFilter(req.query), {
       limit: req.query.limit ? Number(req.query.limit) : void 0,
       cursor: typeof req.query.cursor === "string" ? req.query.cursor : void 0
-    })
-  ));
+    });
+    if (!t.sourcemaps || !page?.items?.some((r) => r?.kind === "error")) return page;
+    return { ...page, items: await Promise.all(page.items.map((r) => t.sourcemaps.symbolicate(r))) };
+  }));
   api.get("/series", h(
     async (req) => q.series(req.viewer.tenantId, parseRange(req.query), parseFilter(req.query), {
       measure: typeof req.query.measure === "string" ? req.query.measure : void 0,
@@ -3834,7 +4009,8 @@ function createTelemetry(config) {
     modelName,
     collection,
     platforms: config.platforms,
-    bodyMax: config.bodyMax
+    bodyMax: config.bodyMax,
+    validation: config.validation
   });
   const RollupModel = buildRollupModel(conn, `${modelName}Rollup`, `${collection}_rollups`);
   const CheckpointModel = buildCheckpointModel(conn, `${modelName}Checkpoint`, `${collection}_checkpoints`);
@@ -3888,12 +4064,18 @@ function createTelemetry(config) {
     logger,
     linkSubjects
   });
-  const syncIndexes = createSyncIndexes({
+  const syncModelIndexes = createSyncIndexes({
     registry,
     TelemetryModel,
     models: [TelemetryModel, ...Object.values(byKind), RollupModel, CheckpointModel, KeyModel],
     rejects
   });
+  const sourcemaps = createSourcemaps({ connection: conn, collection: `${collection}_sourcemaps`, logger });
+  const syncIndexes = async (...args) => {
+    const result = await syncModelIndexes(...args);
+    await sourcemaps.ensureIndexes();
+    return result;
+  };
   return {
     /** write — the only write */
     emit,
@@ -3960,6 +4142,13 @@ function createTelemetry(config) {
      */
     linkSubjects,
     logger,
+    /**
+     * Sourcemaps for minified clients. `register()` stores a release's maps
+     * (server-side only — there is no HTTP route); the dashboard translates
+     * error frames against them at read time, so errors recorded before the
+     * maps were registered are translated too. See sourcemaps.ts.
+     */
+    sourcemaps,
     /** mint an ingest key; the full key string is returned once, never again */
     createKey: (input) => createKey(KeyModel, input),
     /** the models, exposed for hosts and the router factories */