hippo-memory 1.71.0 → 1.72.0

package/dist/connectors/github/ingest.d.ts

@@ -1,10 +1,6 @@
-import { type Context } from '../../api/index.js';
+import type { Context } from '../../api/index.js';
+import { type IngestResult } from '../ingest.js';
 import type { GitHubIssueEvent, GitHubIssueCommentEvent, GitHubPullRequestEvent, GitHubPullRequestReviewCommentEvent } from './types.js';
-export type IngestStatus = 'ingested' | 'duplicate' | 'skipped' | 'skipped_duplicate';
-export interface IngestResult {
-    status: IngestStatus;
-    memoryId: string | null;
-}
 /** Discriminated union of the four event shapes V1 ingests; eventName MUST equal the X-GitHub-Event header,
  *  since computeIdempotencyKey folds it into the dedupe key. */
 export type IngestEvent = {

package/dist/connectors/github/ingest.js

@@ -1,9 +1,8 @@
 // GitHub event ingest, once per idempotency key. The key comes from the event's own content, never from the unsigned
 // X-GitHub-Delivery header, so a replay cannot get past it by rotating the delivery id. The pre-check is only the fast path:
 // the store logs the key in the memory's own transaction, which is what holds when two workers race.
-import { remember } from '../../api/index.js';
 import { requireGroup, storeFor } from '../../store/index.js';
-import { RejectedValueError } from '../../store/rejection.js';
+import { rememberWithEventLog } from '../ingest.js';
 import { computeIdempotencyKey } from './signature.js';
 import { issueEventToRememberOpts, issueCommentEventToRememberOpts, pullRequestEventToRememberOpts, prReviewCommentEventToRememberOpts, } from './transform.js';
 function transformEvent(event) {
@@ -54,31 +53,7 @@ export async function ingestEvent(ctx, input) {
     const seen = await events.eventRecord(event);
     if (seen.seen)
         return { status: 'duplicate', memoryId: seen.memoryId };
-    const opts = transformEvent(input.event);
-    if (!opts) {
-        // Empty body: no memory to write, but mark seen so a retry returns 'duplicate' instead of re-running the transform.
-        await events.markEventSeen(event);
-        return { status: 'skipped', memoryId: null };
-    }
-    try {
-        return await rememberWithEventLog(ctx, event, opts);
-    }
-    catch (e) {
-        if (e instanceof RejectedValueError) {
-            // A tombstone hit is a PERMANENT skip, never DLQ-retried: mark the key seen like the empty-body
-            // branch above so a GitHub retry of the same delivery acks as done, not error.
-            await events.markEventSeen(event);
-            return { status: 'skipped', memoryId: null };
-        }
-        throw e;
-    }
-}
-async function rememberWithEventLog(ctx, event, opts) {
-    // No `|| 'connector:github'` fallback: the caller always builds ctx with the connector subject (same as slack/ingest.ts).
-    const result = await remember(ctx, { ...opts, untrusted: true, event });
-    // Another worker logged this key between the pre-check and the write: its memory stands and ours was not stored.
-    if (result.duplicate)
-        return { status: 'skipped_duplicate', memoryId: result.duplicate.memoryId };
-    return { status: 'ingested', memoryId: result.id };
+    // An empty body is logged too, so a retry returns 'duplicate' instead of re-running the transform.
+    return rememberWithEventLog(ctx, event, transformEvent(input.event));
 }
 //# sourceMappingURL=ingest.js.map

package/dist/connectors/github/webhook.d.ts

@@ -1,6 +1,7 @@
 import type { HippoStore } from '../../store/index.js';
 import { type WebhookRequest } from '../../util/http-util.js';
 /** GitHub webhook receiver: HMAC SHA-256 over the raw body, event type from the X-GitHub-Event header, tenant by installation.id then repository.full_name.
- *  ACKs 200 on signed envelopes (DLQ too); 401 only on a bad signature; 404 when GITHUB_WEBHOOK_SECRET is unset. No Bearer: server.ts checks isPublicRoute. */
+ *  ACKs 200 on signed envelopes (DLQ too); 401 only on a bad signature; 409 on a delivery id seen in the replay window;
+ *  404 when GITHUB_WEBHOOK_SECRET is unset. No Bearer: its route row declares webhook-signature. */
 export declare function handleGitHubEventsWebhook(request: WebhookRequest, store?: HippoStore): Promise<void>;
 //# sourceMappingURL=webhook.d.ts.map

package/dist/connectors/github/webhook.js

@@ -10,7 +10,8 @@ import { computeDeletionKey as computeGitHubDeletionKey } from './signature.js';
 import { resolveTenantId } from '../../store/tenant.js';
 import { HttpError, JSON_HEADERS, isHeaderString, closeIfBodyUnread, readWebhookBody, sendJson, } from '../../util/http-util.js';
 /** GitHub webhook receiver: HMAC SHA-256 over the raw body, event type from the X-GitHub-Event header, tenant by installation.id then repository.full_name.
- *  ACKs 200 on signed envelopes (DLQ too); 401 only on a bad signature; 404 when GITHUB_WEBHOOK_SECRET is unset. No Bearer: server.ts checks isPublicRoute. */
+ *  ACKs 200 on signed envelopes (DLQ too); 401 only on a bad signature; 409 on a delivery id seen in the replay window;
+ *  404 when GITHUB_WEBHOOK_SECRET is unset. No Bearer: its route row declares webhook-signature. */
 export async function handleGitHubEventsWebhook(request, store) {
     const { req, res, opts } = request;
     // Secret and signature header before the body, so a caller with neither cannot make the server buffer one.
@@ -43,7 +44,37 @@ export async function handleGitHubEventsWebhook(request, store) {
     }
     // Signature OK from here on. Everything else is ACK-200; bad envelopes go
     // to the DLQ and a human can replay later.
-    await routeSignedDelivery({ hippoRoot: opts.hippoRoot, store, res, rawBody, eventName, deliveryId, signature: sigStr });
+    const replayKey = deliveryId === null ? null : `${opts.hippoRoot}\n${deliveryId}`;
+    if (replayKey !== null && !claimDelivery(replayKey, Date.now()))
+        throw new HttpError(409, 'duplicate X-GitHub-Delivery');
+    try {
+        await routeSignedDelivery({ hippoRoot: opts.hippoRoot, store, res, rawBody, eventName, deliveryId, signature: sigStr });
+    }
+    catch (err) {
+        // A delivery that failed left nothing stored, so GitHub's redelivery of the same id must get through.
+        if (replayKey !== null)
+            seenDeliveries.delete(replayKey);
+        throw err;
+    }
+}
+// GitHub signs no timestamp, so a captured delivery verifies forever; this window is what refuses its replay.
+const DELIVERY_REPLAY_TTL_MS = 24 * 60 * 60 * 1000;
+const DELIVERY_REPLAY_MAX_IDS = 10_000;
+// SHORTCUT: seen ids live in this process and reset on restart or eviction; move them to the store if serve ever runs as several processes.
+const seenDeliveries = new Map();
+/** Claims a delivery id for the window; false when it was claimed within the window already. The oldest claim goes first at the cap. */
+function claimDelivery(key, now) {
+    const seenAt = seenDeliveries.get(key);
+    if (seenAt !== undefined && now - seenAt < DELIVERY_REPLAY_TTL_MS)
+        return false;
+    seenDeliveries.delete(key);
+    if (seenDeliveries.size >= DELIVERY_REPLAY_MAX_IDS) {
+        const oldest = seenDeliveries.keys().next();
+        if (!oldest.done)
+            seenDeliveries.delete(oldest.value);
+    }
+    seenDeliveries.set(key, now);
+    return true;
 }
 const ALLOWED_EVENTS = new Set([
     'issues',

package/dist/connectors/ingest.d.ts

@@ -0,0 +1,10 @@
+import { type Context, type RememberOpts } from '../api/index.js';
+import type { ConnectorEvent } from '../store/port.js';
+export type IngestStatus = 'ingested' | 'duplicate' | 'skipped' | 'skipped_duplicate';
+export interface IngestResult {
+    status: IngestStatus;
+    memoryId: string | null;
+}
+/** Stores `opts` once per event key; null `opts` (an empty body) and a tombstoned value are logged with no memory, so a redelivery acks instead of retrying. */
+export declare function rememberWithEventLog(ctx: Context, event: ConnectorEvent, opts: RememberOpts | null): Promise<IngestResult>;
+//# sourceMappingURL=ingest.d.ts.map

package/dist/connectors/ingest.js

@@ -0,0 +1,28 @@
+// The write half every connector's ingest shares, after its own pre-check of the event log.
+import { remember } from '../api/index.js';
+import { requireGroup, storeFor } from '../store/index.js';
+import { RejectedValueError } from '../core/api-errors.js';
+/** Stores `opts` once per event key; null `opts` (an empty body) and a tombstoned value are logged with no memory, so a redelivery acks instead of retrying. */
+export async function rememberWithEventLog(ctx, event, opts) {
+    if (!opts)
+        return markSkipped(ctx, event);
+    try {
+        // No actor fallback: every caller builds ctx with its connector subject.
+        const result = await remember(ctx, { ...opts, untrusted: true, event });
+        // Another worker logged this key between the pre-check and the write: its memory stands and ours was not stored.
+        if (result.duplicate)
+            return { status: 'skipped_duplicate', memoryId: result.duplicate.memoryId };
+        return { status: 'ingested', memoryId: result.id };
+    }
+    catch (e) {
+        // A tombstone refusal is permanent, so a DLQ retry would hit it forever.
+        if (e instanceof RejectedValueError)
+            return markSkipped(ctx, event);
+        throw e;
+    }
+}
+async function markSkipped(ctx, event) {
+    await requireGroup(storeFor(ctx), 'connectorEvents').markEventSeen(event);
+    return { status: 'skipped', memoryId: null };
+}
+//# sourceMappingURL=ingest.js.map

package/dist/connectors/slack/dlq.js

@@ -1,96 +1,75 @@
 import { adminActor } from '../../api/index.js';
-import { bumpSlackDlqRetryCount, listSlackDlq, markSlackDlqRetried, slackDlqEntry, } from '../../store/connectors/slack.js';
-import { replayFailed } from '../dlq.js';
+import { listSlackDlq, markSlackDlqRetried, slackDlqEntry, } from '../../store/connectors/slack.js';
+import { replayParked } from '../dlq.js';
 import { ingestMessage } from './ingest.js';
 import { resolveTenantForSlackTeam } from './tenant-routing.js';
 import { verifySlackSignature } from './signature.js';
 import { isSlackEventEnvelope, isSlackMessageEvent } from './types.js';
 import { handleMessageDeleted } from './deletion.js';
-import { errorMessage } from '../../util/log.js';
+import { YEAR_S } from '../../util/time.js';
 /** Slack's dead-letter table; `teamId` and `slackTimestamp` are the columns only it has. */
 export const slackDlq = {
     letter: (row) => ({ connector: 'slack', ...row }),
     list: listSlackDlq,
+    entry: slackDlqEntry,
+    bump: markSlackDlqRetried,
 };
 /** Replay a DLQ row via the normal ingest path (`hippo slack dlq replay <id> [--force]`), re-verifying the signature with the CURRENT secret unless forced.
  *  Rows with a NULL signature need --force; replays use today's routing, because the DLQ exists to be drained after the operator fixed routing. */
 export async function replayDlqEntry(ctx, id, opts = {}) {
-    const row = slackDlqEntry(ctx.hippoRoot, id);
-    if (!row)
-        return replayFailed('not_found', 0, `dlq id ${id} not found`);
-    // Signature verification (current secret, not previous).
-    if (!opts.force) {
-        const sigFailure = checkReplaySignature(row, opts);
-        if (sigFailure)
-            return sigFailure;
-    }
-    // Parse + dispatch.
-    let parsed;
-    try {
-        parsed = JSON.parse(row.rawPayload);
-    }
-    catch (e) {
-        return failAndBump(ctx.hippoRoot, row, 'parse_error', `still unparseable: ${errorMessage(e)}`);
-    }
-    if (!isSlackEventEnvelope(parsed)) {
-        return failAndBump(ctx.hippoRoot, row, 'unhandled', 'not an event_callback envelope');
-    }
-    // Resolve tenant against current state. If still unroutable, bail.
-    const tenant = await resolveTenantForSlackTeam(ctx.hippoRoot, parsed.team_id);
-    if (!tenant) {
-        return failAndBump(ctx.hippoRoot, row, 'unroutable', `team_id ${parsed.team_id} still unroutable`);
-    }
-    const replayCtx = {
-        hippoRoot: ctx.hippoRoot,
-        tenantId: tenant,
-        actor: adminActor('connector:slack:replay'),
-    };
-    return dispatchReplay(replayCtx, row, parsed);
+    return replayParked({
+        dlq: slackDlq,
+        refuseSignature: (row) => refuseSignature(row, opts),
+        isEnvelope: isSlackEventEnvelope,
+        notEnvelope: 'not an event_callback envelope',
+        reingest: (_row, envelope) => reingest(ctx.hippoRoot, envelope),
+    }, ctx.hippoRoot, id, opts.force === true);
 }
-/** The failure result when the row cannot pass the signature gate, else null; never bumps the count. */
-function checkReplaySignature(row, opts) {
+/** A missing signature or timestamp refuses even without a secret; a mismatch needs the current secret to show. */
+function refuseSignature(row, opts) {
     if (!row.signature || !row.slackTimestamp) {
-        return replayFailed('sig_missing', row.retryCount, 'row has no signature/timestamp (legacy, or redacted before storing); pass --force to replay');
+        return {
+            status: 'sig_missing',
+            reason: 'row has no signature/timestamp (legacy, or redacted before storing); pass --force to replay',
+        };
     }
-    if (opts.signingSecret) {
-        const ok = verifySlackSignature({
-            rawBody: row.rawPayload,
-            signature: row.signature,
-            timestamp: row.slackTimestamp,
-            signingSecret: opts.signingSecret,
-            now: opts.now,
-            // Replays happen long after the fact — give them a wider skew unless overridden.
-            skewSeconds: opts.skewSeconds ?? 60 * 60 * 24 * 365,
-        });
-        if (!ok) {
-            return replayFailed('sig_fail', row.retryCount, 'signature did not verify against current SLACK_SIGNING_SECRET; pass --force to replay anyway');
-        }
-    }
-    return null;
-}
-function failAndBump(hippoRoot, row, status, reason) {
-    bumpSlackDlqRetryCount(hippoRoot, row.id);
-    return replayFailed(status, row.retryCount + 1, reason);
+    if (!opts.signingSecret)
+        return null;
+    const ok = verifySlackSignature({
+        rawBody: row.rawPayload,
+        signature: row.signature,
+        timestamp: row.slackTimestamp,
+        signingSecret: opts.signingSecret,
+        now: opts.now,
+        // Replays happen long after the fact, so give them a wider skew unless overridden.
+        skewSeconds: opts.skewSeconds ?? YEAR_S,
+    });
+    if (ok)
+        return null;
+    return {
+        status: 'sig_fail',
+        reason: 'signature did not verify against current SLACK_SIGNING_SECRET; pass --force to replay anyway',
+    };
 }
-async function dispatchReplay(replayCtx, row, parsed) {
-    const { hippoRoot } = replayCtx;
-    const id = row.id;
-    const inner = parsed.event;
-    if (!isSlackMessageEvent(inner)) {
-        return failAndBump(hippoRoot, row, 'unhandled', `unhandled inner event type`);
-    }
+async function reingest(hippoRoot, envelope) {
+    const tenantId = await resolveTenantForSlackTeam(hippoRoot, envelope.team_id);
+    if (!tenantId)
+        return { ok: false, status: 'unroutable', reason: `team_id ${envelope.team_id} still unroutable` };
+    const inner = envelope.event;
+    if (!isSlackMessageEvent(inner))
+        return { ok: false, status: 'unhandled', reason: 'unhandled inner event type' };
+    const replayCtx = { hippoRoot, tenantId, actor: adminActor('connector:slack:replay') };
     if (inner.subtype === 'message_deleted' && inner.deleted_ts) {
         const r = await handleMessageDeleted(replayCtx, {
-            teamId: parsed.team_id,
+            teamId: envelope.team_id,
             channelId: inner.channel,
             deletedTs: inner.deleted_ts,
-            eventId: parsed.event_id,
+            eventId: envelope.event_id,
         });
-        markSlackDlqRetried(hippoRoot, id);
-        return { ok: true, status: r.status, memoryId: r.memoryId, retryCount: row.retryCount + 1 };
+        return { ok: true, status: r.status, memoryId: r.memoryId };
     }
     const result = await ingestMessage(replayCtx, {
-        teamId: parsed.team_id,
+        teamId: envelope.team_id,
         channel: {
             id: inner.channel,
             is_private: inner.channel_type !== 'channel',
@@ -98,9 +77,8 @@ async function dispatchReplay(replayCtx, row, parsed) {
             is_mpim: inner.channel_type === 'mpim',
         },
         message: inner,
-        eventId: parsed.event_id,
+        eventId: envelope.event_id,
     });
-    markSlackDlqRetried(hippoRoot, id);
-    return { ok: true, status: result.status, memoryId: result.memoryId, retryCount: row.retryCount + 1 };
+    return { ok: true, status: result.status, memoryId: result.memoryId };
 }
 //# sourceMappingURL=dlq.js.map

package/dist/connectors/slack/ingest.d.ts

@@ -1,4 +1,5 @@
-import { type Context } from '../../api/index.js';
+import type { Context } from '../../api/index.js';
+import { type IngestResult } from '../ingest.js';
 import type { ChannelMeta } from './scope.js';
 import type { SlackMessageEvent } from './types.js';
 export interface IngestInput {
@@ -8,11 +9,6 @@ export interface IngestInput {
     /** Slack event_id for the envelope (or for backfill, a synthesized stable id). */
     eventId: string;
 }
-export type IngestStatus = 'ingested' | 'duplicate' | 'skipped' | 'skipped_duplicate';
-export interface IngestResult {
-    status: IngestStatus;
-    memoryId: string | null;
-}
 /** Ingests a Slack message as a kind='raw' memory once per event id (Slack redelivers within a minute); the pre-check is only the fast path,
  *  the store logs the event in the memory's own transaction, which holds when two workers race. An empty body is logged with no memory. */
 export declare function ingestMessage(ctx: Context, input: IngestInput): Promise<IngestResult>;

package/dist/connectors/slack/ingest.js

@@ -1,6 +1,5 @@
-import { remember } from '../../api/index.js';
 import { requireGroup, storeFor } from '../../store/index.js';
-import { RejectedValueError } from '../../store/rejection.js';
+import { rememberWithEventLog } from '../ingest.js';
 import { messageToRememberOpts } from './transform.js';
 /** Ingests a Slack message as a kind='raw' memory once per event id (Slack redelivers within a minute); the pre-check is only the fast path,
  *  the store logs the event in the memory's own transaction, which holds when two workers race. An empty body is logged with no memory. */
@@ -15,30 +14,6 @@ export async function ingestMessage(ctx, input) {
         // they first returned; a non-NULL memory_id means a memory was ingested, so 'duplicate'.
         return { status: seen.memoryId === null ? 'skipped' : 'duplicate', memoryId: seen.memoryId };
     }
-    const opts = messageToRememberOpts(input);
-    if (!opts) {
-        await events.markEventSeen(event);
-        return { status: 'skipped', memoryId: null };
-    }
-    try {
-        return await rememberWithEventLog(ctx, event, opts);
-    }
-    catch (e) {
-        if (!(e instanceof RejectedValueError))
-            throw e;
-        // A tombstone hit is a PERMANENT skip: a DLQ retry would hit the same refusal forever, so mark the
-        // event seen like the empty-body branch above and let a Slack retry ack as done, not error.
-        await events.markEventSeen(event);
-        return { status: 'skipped', memoryId: null };
-    }
-}
-async function rememberWithEventLog(ctx, event, opts) {
-    // No `|| 'connector:slack'` fallback: the caller always builds ctx with the connector subject, and with
-    // an object-shaped Context.actor an OR-fallback would never fire anyway.
-    const result = await remember(ctx, { ...opts, untrusted: true, event });
-    // Another worker logged this event between the pre-check and the write: its memory stands and ours was not stored.
-    if (result.duplicate)
-        return { status: 'skipped_duplicate', memoryId: result.duplicate.memoryId };
-    return { status: 'ingested', memoryId: result.id };
+    return rememberWithEventLog(ctx, event, messageToRememberOpts(input));
 }
 //# sourceMappingURL=ingest.js.map

package/dist/connectors/slack/webhook.d.ts

@@ -1,6 +1,7 @@
 import type { HippoStore } from '../../store/index.js';
 import { type WebhookRequest } from '../../util/http-util.js';
-/** Slack Events API webhook: HMAC over the raw body with SLACK_SIGNING_SECRET, no Bearer (hence PUBLIC_ROUTES); malformed payloads go to slack_dlq, ACK 200.
+/** Slack Events API webhook: HMAC over the raw body with SLACK_SIGNING_SECRET, no Bearer (its route row declares webhook-signature);
+ *  malformed payloads go to slack_dlq, ACK 200.
  *  404 (not 503) when the secret is unset, so a probe cannot tell a gated-off route from a missing one. */
 export declare function handleSlackEventsWebhook(request: WebhookRequest, store?: HippoStore): Promise<void>;
 //# sourceMappingURL=webhook.d.ts.map

package/dist/connectors/slack/webhook.js

@@ -9,7 +9,8 @@ import { resolveTenantForSlackTeam } from './tenant-routing.js';
 import { resolveTenantId } from '../../store/tenant.js';
 import { HttpError, JSON_HEADERS, isHeaderString, closeIfBodyUnread, readWebhookBody, sendJson } from '../../util/http-util.js';
 import { isJsonObject } from '../../util/json.js';
-/** Slack Events API webhook: HMAC over the raw body with SLACK_SIGNING_SECRET, no Bearer (hence PUBLIC_ROUTES); malformed payloads go to slack_dlq, ACK 200.
+/** Slack Events API webhook: HMAC over the raw body with SLACK_SIGNING_SECRET, no Bearer (its route row declares webhook-signature);
+ *  malformed payloads go to slack_dlq, ACK 200.
  *  404 (not 503) when the secret is unset, so a probe cannot tell a gated-off route from a missing one. */
 export async function handleSlackEventsWebhook(request, store) {
     const { req, res, opts } = request;

package/dist/consolidate/conflicts.js

@@ -1,7 +1,7 @@
 import { Layer, calculateStrength } from '../core/memory.js';
 import { jaccardMinShared, overlapPartners } from './overlap-index.js';
 import { isQuarantineScope } from '../trust/quarantine.js';
-import { isPersonalScope } from '../store/recall-scope.js';
+import { isPersonalScope } from '../core/recall-scope.js';
 import { DECAY_THRESHOLD } from './decay.js';
 // Gate contradictions on content overlap, not shared tags (`feedback`/`policy` are too coarse). The Jaccard threshold on stopword-filtered tokens
 // only applies after a polarity signal, so it filters drive-by topic similarity, not semantic drift.

package/dist/consolidate/dag.js

@@ -1,18 +1,24 @@
 import { createMemory, Layer } from '../core/memory.js';
 import { writeEntry, writeEntriesTogether } from '../store/entry-writes.js';
 import { loadAllDirtySummaries, loadChildrenOfSummary, applyRebuildResult, clearSummaryDirtyAfterBuild, } from '../store/summaries.js';
-import { RejectedValueError } from '../store/rejection.js';
+import { RejectedValueError } from '../core/api-errors.js';
 import { generateDagSummary } from './dag-summary.js';
-import { derivationScope, derivationPartitionKey } from '../store/recall-scope.js';
+import { overlapPartners } from './overlap-index.js';
+import { derivationScope, derivationPartitionKey } from '../core/recall-scope.js';
 import { loadConfig } from '../core/config.js';
 import { neverAutoShareTags } from '../sharing/share.js';
 import { errorFields, errorMessage, log } from '../util/log.js';
 const SUMMARY_FALLBACK_CHARS = 40;
 const DEFAULT_REBUILD_CAP = 20;
+// Facts join a cluster when their tag sets overlap at least this much.
+const CLUSTER_JACCARD_MIN = 0.5;
 export function clusterFacts(facts) {
     if (facts.length === 0)
         return [];
     const entityTags = facts.map((f) => f.tags.filter((t) => t.startsWith('speaker:') || t.startsWith('topic:')));
+    const tagSets = entityTags.map((tags) => new Set(tags));
+    // Every tag indexed, not a prefix: a repeated tag counts twice in `shared`, so the prefix-filter bound would not hold.
+    const partnersOf = overlapPartners(tagSets, () => 1);
     const assigned = new Set();
     const clusters = [];
     for (let i = 0; i < facts.length; i++) {
@@ -20,19 +26,18 @@ export function clusterFacts(facts) {
             continue;
         const cluster = [i];
         assigned.add(i);
-        for (let j = i + 1; j < facts.length; j++) {
+        for (const j of partnersOf(i)) {
             if (assigned.has(j))
                 continue;
-            const shared = entityTags[i].filter((t) => entityTags[j].includes(t));
-            const union = new Set([...entityTags[i], ...entityTags[j]]);
-            const jaccard = union.size > 0 ? shared.length / union.size : 0;
-            if (jaccard >= 0.5) {
+            const shared = entityTags[i].filter((t) => tagSets[j].has(t)).length;
+            const inBoth = [...tagSets[i]].filter((t) => tagSets[j].has(t)).length;
+            if (shared / (tagSets[i].size + tagSets[j].size - inBoth) >= CLUSTER_JACCARD_MIN) {
                 cluster.push(j);
                 assigned.add(j);
             }
         }
         const members = cluster.map((idx) => facts[idx]);
-        const sharedTags = entityTags[cluster[0]].filter((t) => cluster.every((idx) => entityTags[idx].includes(t)));
+        const sharedTags = entityTags[cluster[0]].filter((t) => cluster.every((idx) => tagSets[idx].has(t)));
         const label = sharedTags
             .map((t) => t.split(':')[1])
             .join(': ') || members[0].content.slice(0, SUMMARY_FALLBACK_CHARS);
@@ -179,7 +184,8 @@ function summaryLabel(summary) {
  * A concurrent sleep's second writer gets changed=false (UPDATE requires summary_dirty=1) and is skipped silently; a tombstone hit counts as `refused`. */
 export async function rebuildDirtySummaries(hippoRoot, opts) {
     const cap = opts.cap ?? DEFAULT_REBUILD_CAP;
-    const dirty = loadAllDirtySummaries(hippoRoot);
+    // One past the cap tells whether more were waiting without reading the rest.
+    const dirty = loadAllDirtySummaries(hippoRoot, cap + 1);
     const capped = dirty.length > cap;
     const queue = dirty.slice(0, cap);
     const result = {

package/dist/consolidate/dedupe.d.ts

@@ -1,5 +1,6 @@
 /** Store-level dedup: same text apart from spacing, keeping the stronger copy; split out so `api.sleep` can dedupe without cli -> api imports.
  * Survivor order is total and tenant-scoped: strength bucket, retrieval_count, then compareEntryIdentity; raw and superseded rows are not candidates. */
+import { type MemoryEntry } from '../core/memory.js';
 export interface DedupPair {
     kept: string;
     keptContent: string;
@@ -19,11 +20,17 @@ export interface DedupResult {
 /** Quantize strength to an integer bucket so ties are a true equivalence relation (equal iff same multiple of `STRENGTH_TIE_EPSILON`).
  * Non-finite input maps to 0: a NaN bucket would make the comparator return NaN and break the total order. */
 export declare function strengthBucket(strength: number | null | undefined): number;
+/** Every row of the store, read once by a caller that also needs them, with the ids of rows backing an object. */
+export interface LoadedStore {
+    readonly entries: readonly MemoryEntry[];
+    readonly backing: ReadonlySet<string>;
+}
 /** Remove the weaker copy of same-text (apart from spacing) memories within one tenant; cross-tenant pairs are never duplicates (isolation boundary).
  * Higher strength wins, then more retrievals; `threshold` is accepted for old callers and ignored. */
 export declare function deduplicateStore(hippoRoot: string, options?: {
     threshold?: number;
     dryRun?: boolean;
     actor?: string;
+    loaded?: LoadedStore;
 }): DedupResult;
 //# sourceMappingURL=dedupe.d.ts.map

package/dist/consolidate/dedupe.js

@@ -5,7 +5,7 @@ import { loadCurrentDistilledEntries } from '../store/entry-reads.js';
 import { deleteEntriesOneByOne, memoriesBackingObjects } from '../store/delete-and-batch.js';
 import { compareEntryIdentity } from '../core/compare.js';
 import { canAutoDelete } from '../core/memory.js';
-import { derivationPartitionKey } from '../store/recall-scope.js';
+import { derivationPartitionKey } from '../core/recall-scope.js';
 import { duplicateKey } from '../util/same-text.js';
 /** Quantization step for strength-tie comparisons: the 0.01 epsilon applied by rounding instead of an abs-diff threshold, so the tiebreak is transitive. */
 const STRENGTH_TIE_EPSILON = 0.01;
@@ -73,18 +73,20 @@ function partitionPairs(tenantEntries, removed, backing) {
     }
     return [...groups.values()].flatMap((group) => group.pairs);
 }
+/** The rows loadCurrentDistilledEntries selects, picked from a whole-store read. */
+const isCurrentDistilled = (e) => (e.kind ?? 'distilled') === 'distilled' && !e.superseded_by;
 /** Remove the weaker copy of same-text (apart from spacing) memories within one tenant; cross-tenant pairs are never duplicates (isolation boundary).
  * Higher strength wins, then more retrievals; `threshold` is accepted for old callers and ignored. */
 export function deduplicateStore(hippoRoot, options = {}) {
     const dryRun = options.dryRun ?? false;
     // Only current distilled rows compete: raw rows are append-only (the delete
     // trigger would abort sleep mid-loop) and superseded rows are history.
-    const entries = loadCurrentDistilledEntries(hippoRoot);
+    const entries = options.loaded?.entries.filter(isCurrentDistilled) ?? loadCurrentDistilledEntries(hippoRoot);
     const entriesByTenant = entriesByPartition(entries);
     // Shared across tenant groups: ids are globally unique UUIDs, so `removed` cannot collide across tenants and deleteEntry deletes by primary key alone.
     const removed = new Set();
     const pairs = [];
-    const backing = memoriesBackingObjects(hippoRoot);
+    const backing = options.loaded?.backing ?? memoriesBackingObjects(hippoRoot);
     for (const tenantEntries of entriesByTenant.values()) {
         // The survivor total order (see the file-level docstring), scoped per
         // tenant group: the total order holds within a tenant only, matching the partition above.

package/dist/consolidate/llm-passes.js

@@ -1,8 +1,11 @@
 import { envAnthropicApiKey, envDagRebuildCap } from '../util/env.js';
 import { Layer } from '../core/memory.js';
-import { log } from '../util/log.js';
+import { errorFields, errorMessage, log } from '../util/log.js';
 import { keptAsWritten } from './run.js';
 import { isReusable } from '../core/memory-quality.js';
+import { isPermanentFailure } from '../util/anthropic-messages.js';
+// Bounds the LLM calls one sleep spends on extraction.
+const EXTRACTION_BATCH_LIMIT = 20;
 /** The key, model options and once-per-line error reporter every LLM phase shares. */
 function sleepLlm(run, fetcher) {
     const { config, result } = run;
@@ -28,21 +31,10 @@ export async function llmPasses(run, fetcher) {
     run.result.extractionCandidates = extractionCandidates.length;
     const llm = sleepLlm(run, fetcher);
     if (llm.apiKey && extractionCandidates.length > 0 && !run.dryRun) {
-        const { extractFacts, storeExtractedFacts } = await import('../learn/extract.js');
-        const batchLimit = 20;
-        let extractedCount = 0;
-        for (const candidate of extractionCandidates.slice(0, batchLimit)) {
-            try {
-                const facts = await extractFacts(candidate.content, { ...llm.llmOpts, onError: llm.llmError('extraction') });
-                if (facts.length > 0) {
-                    extractedCount += storeExtractedFacts(run.hippoRoot, candidate, facts).length;
-                }
-            }
-            catch (err) {
-                llm.llmError('extraction')(String(err));
-            }
-        }
-        run.result.extracted = extractedCount;
+        const stopped = await extractionPass(run, llm, extractionCandidates.slice(0, EXTRACTION_BATCH_LIMIT));
+        // The DAG passes call the same API with the same key and model, so they would fail the same way.
+        if (stopped)
+            return;
     }
     await dagBuildPass(run, llm);
     if (llm.apiKey && !run.dryRun) {
@@ -50,6 +42,37 @@ export async function llmPasses(run, fetcher) {
         await entityProfilePass(run, llm);
     }
 }
+/** Extracts facts from each candidate; true when it stopped on a failure every later call would repeat (bad key, no credit, unknown model). */
+async function extractionPass(run, llm, batch) {
+    const { extractFacts, storeExtractedFacts } = await import('../learn/extract.js');
+    const report = llm.llmError('extraction');
+    let extractedCount = 0;
+    let stopped = false;
+    const onError = (msg, failure) => {
+        report(msg);
+        if (failure && isPermanentFailure(failure))
+            stopped = true;
+    };
+    for (const [index, candidate] of batch.entries()) {
+        try {
+            const facts = await extractFacts(candidate.content, { ...llm.llmOpts, onError });
+            if (facts.length > 0) {
+                extractedCount += storeExtractedFacts(run.hippoRoot, candidate, facts).length;
+            }
+        }
+        catch (err) {
+            report(errorMessage(err));
+            log.warn('consolidate extraction threw', errorFields(err));
+        }
+        if (stopped) {
+            const skipped = batch.length - index - 1;
+            report(`stopped after a failure that will not clear on retry; ${skipped} of ${batch.length} candidates skipped, and the DAG passes with them`);
+            break;
+        }
+    }
+    run.result.extracted = extractedCount;
+    return stopped;
+}
 // 1.7. DAG summarization — cluster extracted facts and generate summaries
 async function dagBuildPass(run, { apiKey, llmError, llmOpts }) {
     const extractedFacts = run.survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1 && !e.superseded_by && isReusable(e));
@@ -65,7 +88,8 @@ async function dagBuildPass(run, { apiKey, llmError, llmOpts }) {
         }
     }
     catch (err) {
-        llmError('dag')(String(err));
+        llmError('dag')(errorMessage(err));
+        log.warn('consolidate dag threw', errorFields(err));
     }
 }
 // 1.8. DAG summary rebuild: drain the summary_dirty queue set by child-write hooks; same apiKey/dryRun gate as buildDag.
@@ -99,7 +123,8 @@ async function dagRebuildPass(run, { llmError, llmOpts }) {
         }
     }
     catch (err) {
-        llmError('dag rebuild')(String(err));
+        llmError('dag rebuild')(errorMessage(err));
+        log.warn('consolidate dag rebuild threw', errorFields(err));
     }
 }
 // 1.9. DAG entity profiles: cluster L2 summaries into per-entity L3 profiles; runs even when 1.7 was skipped (re-clusters existing L2s every sleep).
@@ -118,7 +143,8 @@ async function entityProfilePass(run, { llmError, llmOpts }) {
         }
     }
     catch (err) {
-        llmError('dag profiles')(String(err));
+        llmError('dag profiles')(errorMessage(err));
+        log.warn('consolidate dag profiles threw', errorFields(err));
     }
 }
 //# sourceMappingURL=llm-passes.js.map