@anuma/sdk 6.19.2 → 6.21.0

package/dist/server/index.mjs

@@ -141,7 +141,7 @@ import {
   updateVaultMemoryOp,
   upsertMessageOp,
   vaultMemoryToStored
-} from "./chunk-X5EZQFGR.mjs";
+} from "./chunk-4DSDATYG.mjs";
 import {
   createStreamingDeAnonymizer,
   notifyConsolidationFallback,
@@ -5496,6 +5496,7 @@ async function retain(content, ctx, options = {}) {
   const threshold = options.autoMergeThreshold ?? DEFAULT_AUTO_MERGE_THRESHOLD;
   const resolvedScope = options.scope ?? DEFAULT_SCOPE;
   let supersedeTargetIds = [];
+  let consolidationDecidedCreate = false;
   let supersedeContent;
   let prepared;
   if (enableAutoMerge) {
@@ -5523,8 +5524,12 @@ async function retain(content, ctx, options = {}) {
       const outcome = await tryConsolidate(trimmed, ctx, options, prepared);
       if (outcome) {
         if ("done" in outcome) return outcome.done;
-        supersedeTargetIds = outcome.supersede;
-        supersedeContent = outcome.content;
+        if ("create" in outcome) {
+          consolidationDecidedCreate = true;
+        } else {
+          supersedeTargetIds = outcome.supersede;
+          supersedeContent = outcome.content;
+        }
       }
     }
     if (supersedeTargetIds.length === 0) {
@@ -5574,7 +5579,14 @@ async function retain(content, ctx, options = {}) {
               action: "merge",
               memoryId: targetId,
               targetId,
-              proofCount: updated.proofCount ?? (existing.proofCount ?? 1) + 1
+              proofCount: updated.proofCount ?? (existing.proofCount ?? 1) + 1,
+              similarity: matches[0].similarity,
+              // Stage 1 may have run and explicitly said `create`, and Stage 2
+              // then found a strict-cosine match anyway. Reporting the decision
+              // here is what makes that DISAGREEMENT visible — without it the
+              // model's create silently vanished from the distribution whenever
+              // the cosine stage won.
+              ...consolidationDecidedCreate && { consolidation: "create" }
             };
           }
           await assertMergeTargetGoneOrThrow(ctx, targetId);
@@ -5587,12 +5599,18 @@ async function retain(content, ctx, options = {}) {
   const embedding = reusableQueryEmbedding ?? await generateEmbedding(contentToWrite, ctx.embeddingOptions);
   const embeddingModel = ctx.embeddingOptions.model ?? DEFAULT_API_EMBEDDING_MODEL;
   if (options.respectTombstones) {
-    const tombstoneId = await findTombstoneMatch(embedding, embeddingModel, resolvedScope, ctx, {
+    const tombstone = await findTombstoneMatch(embedding, embeddingModel, resolvedScope, ctx, {
       threshold,
       folderId: options.folderId
     });
-    if (tombstoneId) {
-      return { action: "suppressed", memoryId: tombstoneId, tombstoneId, proofCount: 0 };
+    if (tombstone) {
+      return {
+        action: "suppressed",
+        memoryId: tombstone.id,
+        tombstoneId: tombstone.id,
+        proofCount: 0,
+        similarity: tombstone.similarity
+      };
     }
   }
   const createOpts = {
@@ -5659,7 +5677,8 @@ async function retain(content, ctx, options = {}) {
         action: "supersede",
         memoryId: created2.uniqueId,
         targetId: primaryTargetId,
-        proofCount: 1
+        proofCount: 1,
+        consolidation: "supersede"
       };
     }
     notifyConsolidationFallback(
@@ -5673,7 +5692,8 @@ async function retain(content, ctx, options = {}) {
   return {
     action: "create",
     memoryId: created.uniqueId,
-    proofCount: 1
+    proofCount: 1,
+    ...consolidationDecidedCreate && { consolidation: "create" }
   };
 }
 function unionStrings(a, b) {
@@ -5703,7 +5723,7 @@ async function findTombstoneMatch(embedding, embeddingModel, scope, ctx, opts) {
       bestId = row.uniqueId;
     }
   }
-  return bestId;
+  return bestId === null ? null : { id: bestId, similarity: bestSim };
 }
 function pickEventTimeUpdate(existing, incoming) {
   if (incoming === void 0 || incoming === null) return void 0;
@@ -5745,7 +5765,7 @@ async function tryConsolidate(trimmed, ctx, options, prepared) {
   }));
   const { consolidateMemory: doConsolidate } = await import("./consolidate-54P6QZKU.mjs");
   const decision = await doConsolidate(trimmed, candidates, consolidateOptions);
-  if (decision.action === "create") return null;
+  if (decision.action === "create") return decision.fallbackReason ? null : { create: true };
   if (decision.action === "supersede" && decision.content) {
     const requestedIds = decision.targetIds?.length ? decision.targetIds : decision.targetId ? [decision.targetId] : [];
     if (requestedIds.length === 0) return null;
@@ -5804,7 +5824,8 @@ async function tryConsolidate(trimmed, ctx, options, prepared) {
         action: "merge",
         memoryId: decision.targetId,
         targetId: decision.targetId,
-        proofCount: updated.proofCount ?? (existing.proofCount ?? 1) + 1
+        proofCount: updated.proofCount ?? (existing.proofCount ?? 1) + 1,
+        consolidation: "noop"
       }
     };
   }
@@ -5857,7 +5878,8 @@ async function tryConsolidate(trimmed, ctx, options, prepared) {
         action: "update",
         memoryId: decision.targetId,
         targetId: decision.targetId,
-        proofCount: updated.proofCount ?? (existing.proofCount ?? 1) + 1
+        proofCount: updated.proofCount ?? (existing.proofCount ?? 1) + 1,
+        consolidation: "update"
       }
     };
   }
@@ -5999,6 +6021,11 @@ Extract durable user facts.`,
     fallbackSourceId,
     options.userIdentity ?? []
   );
+  const rawCandidates = parsed.candidates;
+  options.onCandidatesParsed?.({
+    rawCount: Array.isArray(rawCandidates) ? rawCandidates.length : 0,
+    validCount: candidates.length
+  });
   if (!redactor) return candidates;
   const restored = restoreCandidates(candidates, redactor, options.userIdentity ?? []);
   if (candidates.length > 0 && restored.length === 0) {
@@ -6035,8 +6062,12 @@ async function extractAndRetain(messages, retainCtx, options) {
   let exhaustedEmpty = false;
   let exhaustedFailure;
   let droppedAfterRedaction = false;
+  let rawCandidateCount = 0;
+  let validCandidateCount = 0;
   const callerOnExhaustedEmpty = options.extract.onExhaustedEmpty;
   const callerOnCandidatesDropped = options.extract.onCandidatesDropped;
+  const callerOnCandidatesParsed = options.extract.onCandidatesParsed;
+  const tExtract = Date.now();
   const candidates = await extractFacts(messages, {
     ...options.extract,
     onExhaustedEmpty: (failure) => {
@@ -6047,8 +6078,14 @@ async function extractAndRetain(messages, retainCtx, options) {
     onCandidatesDropped: () => {
       droppedAfterRedaction = true;
       callerOnCandidatesDropped?.();
+    },
+    onCandidatesParsed: (stats) => {
+      rawCandidateCount = stats.rawCount;
+      validCandidateCount = stats.validCount;
+      callerOnCandidatesParsed?.(stats);
     }
   });
+  const extractMs = Date.now() - tExtract;
   const filtered = candidates.filter((c) => c.confidence >= minConfidence);
   const log = getLogger();
   const screened = screenCandidatesForInjection(filtered);
@@ -6096,6 +6133,7 @@ async function extractAndRetain(messages, retainCtx, options) {
   const results = [];
   const quarantinedInfo = [];
   let failedWrites = 0;
+  const tRetain = Date.now();
   for (const { candidate, isQuarantined, reason, signature } of toRetain) {
     try {
       const result = await retain(candidate.content, retainCtx, {
@@ -6162,6 +6200,21 @@ async function extractAndRetain(messages, retainCtx, options) {
     failedCount: failedWrites,
     outcome,
     quarantined: quarantinedInfo,
+    funnel: {
+      rawCandidateCount,
+      validCandidateCount,
+      afterRedactionCount: candidates.length,
+      aboveConfidenceCount: filtered.length,
+      // The PERSISTED quarantines (`quarantinedInfo`), not the screened list: a
+      // screened candidate whose retain() throws is counted in `failedCount`,
+      // and counting it here too would double it and break the identity below.
+      // This also keeps the count equal to `quarantined.length` on the result.
+      quarantinedCount: quarantinedInfo.length,
+      retainedCount: results.length,
+      failedCount: failedWrites
+    },
+    timings: { extractMs, retainMs: Date.now() - tRetain },
+    model: options.extract.model ?? DEFAULT_EXTRACTION_MODEL,
     // Only meaningful alongside `outcome: "empty-after-retry"`. Returned as well
     // as pushed through the callback so a consumer that only inspects the result
     // (rather than wiring a hook) can still report WHY.
@@ -6415,27 +6468,23 @@ function createAutoExtractor(options) {
     const t0 = Date.now();
     void (async () => {
       try {
-        const { candidates, results, failedCount, outcome, failure } = await extractAndRetain(
-          window2,
-          retainCtx,
-          {
-            extract,
-            ...options.minConfidence !== void 0 && { minConfidence: options.minConfidence },
-            ...options.entityCtx !== void 0 && { entityCtx: options.entityCtx },
-            ...options.scope !== void 0 && { scope: options.scope },
-            ...options.folderId !== void 0 && { folderId: options.folderId },
-            ...consolidateOptions !== void 0 && { consolidateOptions },
-            ...injectionClassifierOptions !== void 0 && {
-              injectionClassifier: injectionClassifierOptions
-            },
-            ...options.onCandidateFailed && {
-              onCandidateFailed: (candidate, error) => options.onCandidateFailed?.({ candidate, error, conversationId })
-            },
-            ...options.onMemoryQuarantined && {
-              onQuarantined: (info) => options.onMemoryQuarantined?.({ ...info, conversationId })
-            }
+        const { candidates, results, failedCount, outcome, failure, funnel, timings, model } = await extractAndRetain(window2, retainCtx, {
+          extract,
+          ...options.minConfidence !== void 0 && { minConfidence: options.minConfidence },
+          ...options.entityCtx !== void 0 && { entityCtx: options.entityCtx },
+          ...options.scope !== void 0 && { scope: options.scope },
+          ...options.folderId !== void 0 && { folderId: options.folderId },
+          ...consolidateOptions !== void 0 && { consolidateOptions },
+          ...injectionClassifierOptions !== void 0 && {
+            injectionClassifier: injectionClassifierOptions
+          },
+          ...options.onCandidateFailed && {
+            onCandidateFailed: (candidate, error) => options.onCandidateFailed?.({ candidate, error, conversationId })
+          },
+          ...options.onMemoryQuarantined && {
+            onQuarantined: (info) => options.onMemoryQuarantined?.({ ...info, conversationId })
           }
-        );
+        });
         if (outcome !== "empty-after-retry") {
           const advancedTo = window2[window2.length - 1].id;
           stateFor(conversationId).watermark = advancedTo;
@@ -6456,6 +6505,9 @@ function createAutoExtractor(options) {
           durationMs: Date.now() - t0,
           conversationId,
           outcome,
+          funnel,
+          timings,
+          model,
           // Only set alongside `outcome: "empty-after-retry"`. Spread so the key
           // is absent rather than explicitly undefined on a healthy turn —
           // analytics backends store an explicit undefined as a real value.

package/dist/server/{recall-M3KBKU74.mjs → recall-HA4UX4BB.mjs}

@@ -1,6 +1,6 @@
 import {
   recall
-} from "./chunk-X5EZQFGR.mjs";
+} from "./chunk-4DSDATYG.mjs";
 import "./chunk-YXL5VKCO.mjs";
 import "./chunk-3RPGXXPZ.mjs";
 export {

package/dist/telemetry/index.cjs

@@ -213,6 +213,23 @@ function createRecallDiagnosticsHandler(sink) {
       usedBudget: diagnostics.usedBudget,
       reranked: diagnostics.reranked,
       candidateCount: diagnostics.candidateCount,
+      // What the caller actually GOT. `candidateCount` is what was considered,
+      // and the two are routinely far apart — the fact lane pulls `limit * 2`
+      // when fusing — so a consumer reading candidateCount as "memories this
+      // turn received" overcounts.
+      admittedCount: diagnostics.admittedCount,
+      // Both ends of the admitted range, and the lane counts, ride the EVENT as
+      // well as the metrics: a track-only sink is a supported configuration
+      // (the module's own PostHog example is exactly that shape), and it could
+      // otherwise see neither the bottom of the score range nor which side lane
+      // contributed.
+      topScore: diagnostics.topScore,
+      lowestAdmittedScore: diagnostics.lowestAdmittedScore,
+      minScoreApplied: diagnostics.minScoreApplied,
+      truncated: diagnostics.truncated,
+      emptyReason: diagnostics.emptyReason,
+      graphLaneCount: diagnostics.graphLaneCount,
+      temporalLaneCount: diagnostics.temporalLaneCount,
       factCount: diagnostics.factCount,
       chunkCount: diagnostics.chunkCount,
       totalMs: diagnostics.timings.total,
@@ -226,8 +243,11 @@ function createRecallDiagnosticsHandler(sink) {
       metric("recall.duration", ms, { lane });
     }
     metric("recall.candidates", diagnostics.candidateCount, {});
+    metric("recall.admitted", diagnostics.admittedCount, {});
     metric("recall.facts", diagnostics.factCount, {});
     metric("recall.chunks", diagnostics.chunkCount, {});
+    metric("recall.lane.graph", diagnostics.graphLaneCount, {});
+    metric("recall.lane.temporal", diagnostics.temporalLaneCount, {});
     if (diagnostics.vaultSize !== void 0) {
       metric("recall.vault.size", diagnostics.vaultSize, {});
     }

package/dist/telemetry/index.d.mts

@@ -422,7 +422,25 @@ type RecallDegradation =
  *  failing back to the single-query path, or a row batch failing while other
  *  rows keep usable vectors) — those are logged, not reported, so this stays a
  *  reliable outage signal rather than a general embedding-error counter. */
- | "embeddings-unavailable";
+ | "embeddings-unavailable"
+/** The W5 graph (entity) side lane threw and was dropped. Auxiliary, so recall
+ *  still returned — but its RRF signal is missing from the ranking, and until
+ *  this existed the drop was a log line and nothing else. */
+ | "graph-lane-failed"
+/** The W6 temporal side lane threw and was dropped. Same posture as the graph
+ *  lane: recall still returned, ranked without the temporal signal. */
+ | "temporal-lane-failed";
+/**
+ * Why a recall returned nothing. `""` when it returned something, so the field
+ * is always present and groupable rather than being absent on the healthy path.
+ *
+ * The four are different problems: an empty query is a caller bug, no-lanes is a
+ * context wiring bug (the requested kinds have no store), vault-empty is a new
+ * user, and no-candidates is the only one that is about retrieval quality. They
+ * were previously indistinguishable from outside — every one reported
+ * `candidateCount: 0`.
+ */
+type RecallEmptyReason = "" | "empty-query" | "no-lanes" | "vault-empty" | "no-candidates";
 /**
  * Per-call recall observability payload (see {@link RecallOptions.onDiagnostics}).
  * All timings are wall-clock milliseconds. Lane counts are post-dedupe,
@@ -473,6 +491,50 @@ interface RecallDiagnostics {
     factCount: number;
     /** Chunks the chunk lane returned (post-dedupe, pre-fusion). */
     chunkCount: number;
+    /**
+     * Memories actually RETURNED — `memories.length` after fusion, cross-lane
+     * dedup and the `limit` slice.
+     *
+     * `candidateCount` is what was considered; this is what the caller got, and
+     * the two are routinely far apart (the fact lane pulls `limit * 2` when fusing).
+     * Every consumer that wanted "how many memories did this turn actually get"
+     * was reading `candidateCount` and overcounting.
+     */
+    admittedCount: number;
+    /** Highest score among the returned memories; -1 when none were returned. */
+    topScore: number;
+    /** Lowest score among the returned memories; -1 when none were returned. */
+    lowestAdmittedScore: number;
+    /**
+     * The similarity floor the lane that produced these scores actually applied:
+     * the fact lane's when it RETURNED results, otherwise the chunk lane's, and
+     * **-1 when neither ran** (empty query, unwired context). Gated on results
+     * rather than on the lane running, because a fact lane that ran and came
+     * back empty filtered none of the scores in the payload — reporting its
+     * 0.1 default against chunks that cleared 0.5 corrupted the telemetry.
+     * When NOTHING was admitted, the floor a lane did apply is still reported:
+     * "searched at this floor, found nothing" is the useful reading.
+     *
+     * Per-lane rather than one constant because the two defaults differ (0.1 fact
+     * / 0.5 chunk), so a single seeded value reported a floor that a chunk-only
+     * recall never applied. Reported next to the scores because the scores alone
+     * cannot say what they cleared.
+     */
+    minScoreApplied: number;
+    /**
+     * Whether the `limit` cut an ELIGIBLE result — recorded at the cut, not
+     * derived from `candidateCount > limit`. In the fused path `candidateCount`
+     * counts before provenance suppression, so a recall whose suppressed chunks
+     * brought it under the limit would otherwise report a truncation that never
+     * happened.
+     */
+    truncated: boolean;
+    /** Memory ids the W5 graph (entity) side lane contributed to the fusion. */
+    graphLaneCount: number;
+    /** Memory ids the W6 temporal side lane contributed to the fusion. */
+    temporalLaneCount: number;
+    /** Why nothing came back — see {@link RecallEmptyReason}. `""` when something did. */
+    emptyReason: RecallEmptyReason;
     /** Wall-clock phase timings (ms). */
     timings: {
         /** Whole `recall()` call. */

package/dist/telemetry/index.d.ts

@@ -422,7 +422,25 @@ type RecallDegradation =
  *  failing back to the single-query path, or a row batch failing while other
  *  rows keep usable vectors) — those are logged, not reported, so this stays a
  *  reliable outage signal rather than a general embedding-error counter. */
- | "embeddings-unavailable";
+ | "embeddings-unavailable"
+/** The W5 graph (entity) side lane threw and was dropped. Auxiliary, so recall
+ *  still returned — but its RRF signal is missing from the ranking, and until
+ *  this existed the drop was a log line and nothing else. */
+ | "graph-lane-failed"
+/** The W6 temporal side lane threw and was dropped. Same posture as the graph
+ *  lane: recall still returned, ranked without the temporal signal. */
+ | "temporal-lane-failed";
+/**
+ * Why a recall returned nothing. `""` when it returned something, so the field
+ * is always present and groupable rather than being absent on the healthy path.
+ *
+ * The four are different problems: an empty query is a caller bug, no-lanes is a
+ * context wiring bug (the requested kinds have no store), vault-empty is a new
+ * user, and no-candidates is the only one that is about retrieval quality. They
+ * were previously indistinguishable from outside — every one reported
+ * `candidateCount: 0`.
+ */
+type RecallEmptyReason = "" | "empty-query" | "no-lanes" | "vault-empty" | "no-candidates";
 /**
  * Per-call recall observability payload (see {@link RecallOptions.onDiagnostics}).
  * All timings are wall-clock milliseconds. Lane counts are post-dedupe,
@@ -473,6 +491,50 @@ interface RecallDiagnostics {
     factCount: number;
     /** Chunks the chunk lane returned (post-dedupe, pre-fusion). */
     chunkCount: number;
+    /**
+     * Memories actually RETURNED — `memories.length` after fusion, cross-lane
+     * dedup and the `limit` slice.
+     *
+     * `candidateCount` is what was considered; this is what the caller got, and
+     * the two are routinely far apart (the fact lane pulls `limit * 2` when fusing).
+     * Every consumer that wanted "how many memories did this turn actually get"
+     * was reading `candidateCount` and overcounting.
+     */
+    admittedCount: number;
+    /** Highest score among the returned memories; -1 when none were returned. */
+    topScore: number;
+    /** Lowest score among the returned memories; -1 when none were returned. */
+    lowestAdmittedScore: number;
+    /**
+     * The similarity floor the lane that produced these scores actually applied:
+     * the fact lane's when it RETURNED results, otherwise the chunk lane's, and
+     * **-1 when neither ran** (empty query, unwired context). Gated on results
+     * rather than on the lane running, because a fact lane that ran and came
+     * back empty filtered none of the scores in the payload — reporting its
+     * 0.1 default against chunks that cleared 0.5 corrupted the telemetry.
+     * When NOTHING was admitted, the floor a lane did apply is still reported:
+     * "searched at this floor, found nothing" is the useful reading.
+     *
+     * Per-lane rather than one constant because the two defaults differ (0.1 fact
+     * / 0.5 chunk), so a single seeded value reported a floor that a chunk-only
+     * recall never applied. Reported next to the scores because the scores alone
+     * cannot say what they cleared.
+     */
+    minScoreApplied: number;
+    /**
+     * Whether the `limit` cut an ELIGIBLE result — recorded at the cut, not
+     * derived from `candidateCount > limit`. In the fused path `candidateCount`
+     * counts before provenance suppression, so a recall whose suppressed chunks
+     * brought it under the limit would otherwise report a truncation that never
+     * happened.
+     */
+    truncated: boolean;
+    /** Memory ids the W5 graph (entity) side lane contributed to the fusion. */
+    graphLaneCount: number;
+    /** Memory ids the W6 temporal side lane contributed to the fusion. */
+    temporalLaneCount: number;
+    /** Why nothing came back — see {@link RecallEmptyReason}. `""` when something did. */
+    emptyReason: RecallEmptyReason;
     /** Wall-clock phase timings (ms). */
     timings: {
         /** Whole `recall()` call. */

package/dist/telemetry/index.mjs

@@ -185,6 +185,23 @@ function createRecallDiagnosticsHandler(sink) {
       usedBudget: diagnostics.usedBudget,
       reranked: diagnostics.reranked,
       candidateCount: diagnostics.candidateCount,
+      // What the caller actually GOT. `candidateCount` is what was considered,
+      // and the two are routinely far apart — the fact lane pulls `limit * 2`
+      // when fusing — so a consumer reading candidateCount as "memories this
+      // turn received" overcounts.
+      admittedCount: diagnostics.admittedCount,
+      // Both ends of the admitted range, and the lane counts, ride the EVENT as
+      // well as the metrics: a track-only sink is a supported configuration
+      // (the module's own PostHog example is exactly that shape), and it could
+      // otherwise see neither the bottom of the score range nor which side lane
+      // contributed.
+      topScore: diagnostics.topScore,
+      lowestAdmittedScore: diagnostics.lowestAdmittedScore,
+      minScoreApplied: diagnostics.minScoreApplied,
+      truncated: diagnostics.truncated,
+      emptyReason: diagnostics.emptyReason,
+      graphLaneCount: diagnostics.graphLaneCount,
+      temporalLaneCount: diagnostics.temporalLaneCount,
       factCount: diagnostics.factCount,
       chunkCount: diagnostics.chunkCount,
       totalMs: diagnostics.timings.total,
@@ -198,8 +215,11 @@ function createRecallDiagnosticsHandler(sink) {
       metric("recall.duration", ms, { lane });
     }
     metric("recall.candidates", diagnostics.candidateCount, {});
+    metric("recall.admitted", diagnostics.admittedCount, {});
     metric("recall.facts", diagnostics.factCount, {});
     metric("recall.chunks", diagnostics.chunkCount, {});
+    metric("recall.lane.graph", diagnostics.graphLaneCount, {});
+    metric("recall.lane.temporal", diagnostics.temporalLaneCount, {});
     if (diagnostics.vaultSize !== void 0) {
       metric("recall.vault.size", diagnostics.vaultSize, {});
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anuma/sdk",
-  "version": "6.19.2",
+  "version": "6.21.0",
   "description": "",
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",