@atomicmemory/core 1.0.0

package/dist/services/first-mention-service.js

@@ -0,0 +1,186 @@
+/**
+ * First-mention extraction service — produces a chronological list of
+ * topic-introduction events from a conversation transcript.
+ *
+ * One LLM call scans the full transcript and outputs a JSON array of
+ * `{topic, turn_id, session_id, anchor_date}` records. The service
+ * maps those onto core's stricter `FirstMentionEvent` shape (joining
+ * turn_id to memory_id via a caller-supplied map) and persists via
+ * `FirstMentionRepository.store`.
+ *
+ * Best-effort: extraction failures are logged to stderr and produce an
+ * empty array. Storage errors are propagated (no silent swallow).
+ *
+ * Prompts and salvage parser were ported verbatim from the BEAM harness
+ * (`atomicmemory-benchmarks/data/exp-stage7-beam-dryrun/lib.ts`) so the
+ * core implementation matches the validated extraction behaviour.
+ */
+const FIRST_MENTIONS_MAX_TOKENS = 8000;
+const FIRST_MENTIONS_SYSTEM = 'You scan a conversation chronologically and identify FIRST-MENTION events: ' +
+    'the moment when a NEW major topic is introduced for the first time. ' +
+    'A "topic" is a specific aspect, feature, tech decision, problem, milestone, ' +
+    'or planning item — NOT a sub-aspect of a previously-introduced topic and ' +
+    'NOT a generic concept. Use SPECIFIC TECHNICAL PHRASES from the conversation ' +
+    'verbatim (version numbers, library names, config flags, named operations). ' +
+    'Order strictly by turn appearance. Output ONLY a JSON array.';
+function buildFirstMentionsUser(turnsText) {
+    return (`Conversation turns (in chronological order, with turn_id and session_id markers):\n\n` +
+        turnsText +
+        `\n\nWalk through the turns sequentially. For each turn that introduces ` +
+        `a NEW major topic (not discussed in any earlier turn), record one event:\n` +
+        `  {"topic": "<COMPACT specific topic phrase, 5-12 words MAX>", "turn_id": <int>, ` +
+        `"session_id": <int>, "anchor_date": "<time_anchor or null>"}\n\n` +
+        `Output ONLY a valid JSON array (no markdown fences, no preamble). ` +
+        `Aim for 15-30 first-mentions across the whole conversation, covering ` +
+        `distinct major aspects spanning all sessions. ` +
+        `Each topic phrase MUST be SHORT (5-12 words). Use verbatim tech tokens ` +
+        `(version numbers, library names) inside the short phrase. ` +
+        `Skip generic chatter, sub-aspects of already-listed topics, and assistant ` +
+        `explanatory content. Close the JSON array with ] before any other output.`);
+}
+/**
+ * Salvage parser: if the model truncated mid-array (no closing ]),
+ * find the last complete object and synthesize a closing bracket.
+ */
+function salvageJsonArray(text) {
+    const start = text.indexOf('[');
+    if (start < 0)
+        return null;
+    const lastBrace = text.lastIndexOf('}');
+    if (lastBrace < start)
+        return null;
+    return text.slice(start, lastBrace + 1) + ']';
+}
+function isRawFirstMention(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const v = value;
+    return typeof v.topic === 'string' && typeof v.turn_id === 'number';
+}
+function parseAnchorDate(raw) {
+    if (typeof raw !== 'string' || raw.length === 0)
+        return null;
+    const ms = Date.parse(raw);
+    if (Number.isNaN(ms))
+        return null;
+    return new Date(ms);
+}
+/** Extract a JSON array from raw LLM text, salvaging truncated output. */
+function extractJsonArray(text) {
+    const trimmed = text.trim();
+    const start = trimmed.indexOf('[');
+    if (start < 0) {
+        console.error(`[first-mention-llm-failed] no opening [ in response (text len=${trimmed.length})`);
+        return null;
+    }
+    const end = trimmed.lastIndexOf(']');
+    let jsonSlice;
+    if (end <= start) {
+        const salvaged = salvageJsonArray(trimmed);
+        if (!salvaged)
+            return null;
+        console.warn(`[first-mention-llm-salvaged] response truncated; salvaged ${salvaged.length} chars`);
+        jsonSlice = salvaged;
+    }
+    else {
+        jsonSlice = trimmed.slice(start, end + 1);
+    }
+    const parsed = JSON.parse(jsonSlice);
+    if (!Array.isArray(parsed)) {
+        console.error(`[first-mention-llm-failed] parsed JSON is not an array (type=${typeof parsed})`);
+        return null;
+    }
+    return parsed;
+}
+export class FirstMentionService {
+    repo;
+    chatFn;
+    constructor(repo, chatFn) {
+        this.repo = repo;
+        this.chatFn = chatFn;
+    }
+    /**
+     * Extract first-mention events from a conversation transcript and
+     * persist them. Returns the parsed events (post-mapping). Best-effort:
+     * if the LLM call fails or returns unparseable output the method
+     * logs to stderr and returns `[]` without throwing.
+     *
+     * `memoryIdsByTurnId` provides the mapping the LLM cannot produce —
+     * any event whose `turn_id` is not present in the map is dropped.
+     */
+    async extractAndStore(userId, conversationText, sourceSite, memoryIdsByTurnId) {
+        const raw = await this.invokeLlm(conversationText);
+        if (raw === null)
+            return [];
+        const events = this.mapToEvents(raw, memoryIdsByTurnId);
+        if (events.length > 0) {
+            await this.repo.store(userId, sourceSite, events);
+        }
+        return events;
+    }
+    /**
+     * Run the extraction LLM call; return parsed array or null on failure.
+     *
+     * Fail-open by design: a flaky upstream LLM call should not crash the
+     * caller's `extractAndStore` invocation; the EO read path treats a
+     * missing first-mention list as "no signal" rather than a hard
+     * failure. The deliberate fallback is a structured
+     * `[first-mention-llm-failed]` log line plus null return so the
+     * failure is observable.
+     */
+    async invokeLlm(conversationText) {
+        let rawText = '';
+        try {
+            const res = await this.chatFn(FIRST_MENTIONS_SYSTEM, buildFirstMentionsUser(conversationText), FIRST_MENTIONS_MAX_TOKENS);
+            rawText = res.text;
+            return extractJsonArray(rawText);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message.slice(0, 200) : String(err);
+            console.error(`[first-mention-llm-failed] ${msg}`);
+            if (rawText) {
+                console.error(`[first-mention-llm-failed] raw response (first 500 chars): ${rawText.slice(0, 500).replace(/\n/g, ' ')}`);
+            }
+            return null;
+        }
+    }
+    /**
+     * Map LLM output records onto core's `FirstMentionEvent` shape.
+     *
+     * `positionInConversation` is the 0-based index in the FINAL turn-id-
+     * sorted output, NOT `turn_id` itself. This makes the mapping stable
+     * across re-runs even when the LLM emits a slightly different
+     * `turn_id` for the same logical topic on a second pass — the
+     * `(user_id, memory_id)` UNIQUE constraint in `first_mention_events`
+     * skips the duplicate insert, but if `position_in_conversation`
+     * tracked `turn_id` directly, an unrelated drift in the second run's
+     * turn assignment would mean callers reading the row see whichever
+     * turn_id was first written. Encoding position by post-sort index
+     * eliminates that variability — read-side ordering is now a stable
+     * sequence (0, 1, 2, ...) regardless of which LLM run produced it.
+     */
+    mapToEvents(raw, memoryIdsByTurnId) {
+        const filtered = raw.filter(isRawFirstMention);
+        const candidates = [];
+        for (const m of filtered) {
+            const memoryId = memoryIdsByTurnId.get(m.turn_id);
+            if (!memoryId)
+                continue;
+            candidates.push({
+                topic: m.topic,
+                turnId: m.turn_id,
+                memoryId,
+                anchorDate: parseAnchorDate(m.anchor_date),
+            });
+        }
+        candidates.sort((a, b) => a.turnId - b.turnId);
+        const events = candidates.map((c, index) => ({
+            ...c,
+            positionInConversation: index,
+        }));
+        if (events.length === 0 && filtered.length > 0) {
+            console.warn(`[first-mention-mapping] ${filtered.length} parsed entries had no matching memory_id`);
+        }
+        return events;
+    }
+}

package/dist/services/hierarchical-retrieval.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Hierarchical retrieval — the 5th RRF arm (T2.3 scaffold).
+ *
+ * Three-stage pipeline for BEAM-10M scale:
+ *   stage 1: top-K conv_summaries by query-embedding cosine
+ *   stage 2: top-K session_summaries within those conversations
+ *   stage 3: expand to atomic-fact ids (memories.id) for the matched sessions
+ *
+ * Output ids are handed to the existing search-pipeline's RRF fusion as a 5th
+ * arm. Full pipeline integration lives in T2.4 (memory-search.ts wiring); this
+ * module exposes the arm as a stand-alone callable so it can be tested
+ * independently and unit-mocked.
+ *
+ * Gated by `hierarchicalRetrievalEnabled`. With the flag off, the function
+ * short-circuits and returns an empty result with zero work done — preserves
+ * byte-for-byte behavior on existing deployments.
+ */
+import type { SummariesRepository } from '../db/summaries-repository.js';
+/** Result of one hierarchical retrieval pass. */
+export interface HierarchicalArmResult {
+    /** Atomic memory ids surfaced by the arm. Empty when flag off / no data. */
+    memoryIds: string[];
+    /** Conversation ids matched at stage 1 (for trace observability). */
+    matchedConvs: string[];
+    /** Session ids matched at stage 2 (for trace observability). */
+    matchedSessions: string[];
+    /** LLM cost incurred by this arm. Always 0 today — only embedding ops. */
+    cost: number;
+}
+export interface HierarchicalArmDeps {
+    config: {
+        hierarchicalRetrievalEnabled: boolean;
+    };
+    summariesRepo: SummariesRepository;
+}
+export interface HierarchicalArmOptions {
+    /** Stage 1: how many conversations to surface. Default 3. */
+    topConvs?: number;
+    /** Stage 2: how many sessions to surface across selected convs. Default 10. */
+    topSessions?: number;
+    /** Stage 3: how many atomic facts to return. Default 50. */
+    factLimit?: number;
+}
+/**
+ * Run the hierarchical arm. Caller is responsible for embedding the query
+ * (we keep this function pure — it doesn't call an embedding provider so
+ * tests don't need to mock one).
+ */
+export declare function runHierarchicalArm(deps: HierarchicalArmDeps, userId: string, queryEmbedding: number[], opts?: HierarchicalArmOptions): Promise<HierarchicalArmResult>;

package/dist/services/hierarchical-retrieval.js

@@ -0,0 +1,50 @@
+/**
+ * Hierarchical retrieval — the 5th RRF arm (T2.3 scaffold).
+ *
+ * Three-stage pipeline for BEAM-10M scale:
+ *   stage 1: top-K conv_summaries by query-embedding cosine
+ *   stage 2: top-K session_summaries within those conversations
+ *   stage 3: expand to atomic-fact ids (memories.id) for the matched sessions
+ *
+ * Output ids are handed to the existing search-pipeline's RRF fusion as a 5th
+ * arm. Full pipeline integration lives in T2.4 (memory-search.ts wiring); this
+ * module exposes the arm as a stand-alone callable so it can be tested
+ * independently and unit-mocked.
+ *
+ * Gated by `hierarchicalRetrievalEnabled`. With the flag off, the function
+ * short-circuits and returns an empty result with zero work done — preserves
+ * byte-for-byte behavior on existing deployments.
+ */
+const EMPTY = {
+    memoryIds: [],
+    matchedConvs: [],
+    matchedSessions: [],
+    cost: 0,
+};
+/**
+ * Run the hierarchical arm. Caller is responsible for embedding the query
+ * (we keep this function pure — it doesn't call an embedding provider so
+ * tests don't need to mock one).
+ */
+export async function runHierarchicalArm(deps, userId, queryEmbedding, opts = {}) {
+    if (!deps.config.hierarchicalRetrievalEnabled) {
+        return EMPTY;
+    }
+    const topConvs = opts.topConvs ?? 3;
+    const topSessions = opts.topSessions ?? 10;
+    const factLimit = opts.factLimit ?? 50;
+    // Stage 1
+    const convHits = await deps.summariesRepo.searchTopConvSummaries(userId, queryEmbedding, topConvs);
+    if (convHits.length === 0)
+        return EMPTY;
+    // Stage 2
+    const matchedConvs = convHits.map((h) => h.conversationId);
+    const sessionHits = await deps.summariesRepo.searchTopSessionSummaries(userId, matchedConvs, queryEmbedding, topSessions);
+    if (sessionHits.length === 0) {
+        return { memoryIds: [], matchedConvs, matchedSessions: [], cost: 0 };
+    }
+    // Stage 3
+    const matchedSessions = sessionHits.map((h) => h.sessionId);
+    const memoryIds = await deps.summariesRepo.getMemoryIdsForSessions(userId, matchedSessions, factLimit);
+    return { memoryIds, matchedConvs, matchedSessions, cost: 0 };
+}

package/dist/services/ingest-fact-pipeline.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Per-fact ingest pipeline: embed, gate, find candidates, decide, store.
+ *
+ * Unifies the three per-fact paths (full, quick, workspace) behind a single
+ * parameterized function. Each path is a different combination of options
+ * rather than a separate code path.
+ */
+import type { WorkspaceContext } from '../db/repository-types.js';
+import type { EntropyContext, FactInput, FactResult, IngestFactTrace, MemoryServiceDeps } from './memory-service-types.js';
+/** Controls which stages of the per-fact pipeline are active. */
+export interface FactPipelineOptions {
+    /** When set, scopes candidate finding and storage to this workspace. */
+    workspace?: WorkspaceContext;
+    /** Run the entropy gate before candidate search (off for quick-ingest). */
+    entropyGate: boolean;
+    /** Run the full AUDN path (fast + deferred + LLM). When false, uses quick duplicate threshold only. */
+    fullAudn: boolean;
+    /** Mutable set of superseded target IDs, shared across a batch. */
+    supersededTargets: Set<string>;
+    /** Mutable entropy context, shared across a batch. */
+    entropyCtx: EntropyContext;
+    /** Optional logical timestamp for backdating. */
+    logicalTimestamp?: Date;
+    /** Timing label prefix for timed() wrappers. */
+    timingPrefix: string;
+    /** Optional per-request trace collector. */
+    traceCollector?: {
+        record(trace: IngestFactTrace): void;
+    };
+}
+/** Process a single extracted fact through the ingest pipeline. */
+export declare function processFactThroughPipeline(deps: MemoryServiceDeps, userId: string, fact: FactInput, sourceSite: string, sourceUrl: string, episodeId: string, options: FactPipelineOptions): Promise<FactResult>;

package/dist/services/ingest-fact-pipeline.js

@@ -0,0 +1,212 @@
+/**
+ * Per-fact ingest pipeline: embed, gate, find candidates, decide, store.
+ *
+ * Unifies the three per-fact paths (full, quick, workspace) behind a single
+ * parameterized function. Each path is a different combination of options
+ * rather than a separate code path.
+ */
+import { embedText } from './embedding.js';
+import { mergeCandidates } from './conflict-policy.js';
+import { computeEntropyScore } from './entropy-gate.js';
+import { assessWriteSecurity, recordRejectedWrite } from './write-security.js';
+import { previewContent } from './ingest-trace.js';
+import { timed } from './timing.js';
+import { storeCanonicalFact, resolveDeterministicClaimSlot, findSlotConflictCandidates } from './memory-storage.js';
+import { findFilteredCandidates, resolveAndExecuteAudn } from './memory-audn.js';
+// ---------------------------------------------------------------------------
+// Main pipeline function
+// ---------------------------------------------------------------------------
+/** Process a single extracted fact through the ingest pipeline. */
+export async function processFactThroughPipeline(deps, userId, fact, sourceSite, sourceUrl, episodeId, options) {
+    if (options.workspace) {
+        return processWorkspaceFact(deps, userId, fact, sourceSite, sourceUrl, episodeId, options);
+    }
+    if (options.fullAudn) {
+        return processFullAudnFact(deps, userId, fact, sourceSite, sourceUrl, episodeId, options);
+    }
+    return processQuickFact(deps, userId, fact, sourceSite, sourceUrl, episodeId, options);
+}
+// ---------------------------------------------------------------------------
+// Full AUDN path (performIngest)
+// ---------------------------------------------------------------------------
+async function processFullAudnFact(deps, userId, fact, sourceSite, sourceUrl, episodeId, options) {
+    const embedding = await timed(`${options.timingPrefix}.fact.embed`, () => embedText(fact.fact));
+    const writeSecurity = assessWriteSecurity(fact.fact, sourceSite, deps.config);
+    if (!writeSecurity.allowed) {
+        await recordRejectedWrite(userId, fact.fact, sourceSite, writeSecurity, deps.config, deps.stores.lesson);
+        return blockedResult(options, fact, writeSecurity, undefined, blockedReasonCode(writeSecurity.blockedBy));
+    }
+    const entropyResult = options.entropyGate
+        ? assessEntropyGate(fact, embedding, options.entropyCtx, deps.config)
+        : null;
+    if (entropyResult && !entropyResult.accepted) {
+        return blockedResult(options, fact, writeSecurity, entropyResult, 'entropy-gate');
+    }
+    const claimSlot = await resolveDeterministicClaimSlot(deps, userId, fact);
+    const filteredCandidates = await findFilteredCandidates(deps, userId, fact, embedding, claimSlot, options.supersededTargets);
+    const candidates = toTraceCandidates(filteredCandidates);
+    const ctx = {
+        userId, fact, embedding, sourceSite, sourceUrl, episodeId,
+        trustScore: writeSecurity.trust.score, claimSlot, logicalTimestamp: options.logicalTimestamp,
+    };
+    if (filteredCandidates.length === 0) {
+        const result = await storeCanonicalFact(deps, ctx);
+        return storedDirectResult(options, result, embedding, fact, writeSecurity, candidates, entropyResult, 'direct-store-no-candidates');
+    }
+    return tracedResult(options, await resolveAndExecuteAudn(deps, userId, fact, embedding, sourceSite, sourceUrl, episodeId, writeSecurity.trust.score, claimSlot, options.logicalTimestamp, filteredCandidates, options.supersededTargets, undefined, {
+        fact,
+        logicalTimestamp: options.logicalTimestamp,
+        writeSecurity,
+        entropyResult,
+        candidates,
+    }));
+}
+// ---------------------------------------------------------------------------
+// Quick path (performQuickIngest)
+// ---------------------------------------------------------------------------
+async function processQuickFact(deps, userId, fact, sourceSite, sourceUrl, episodeId, options) {
+    const embedding = await timed(`${options.timingPrefix}.fact.embed`, () => embedText(fact.fact));
+    const writeSecurity = assessWriteSecurity(fact.fact, sourceSite, deps.config);
+    if (!writeSecurity.allowed) {
+        return blockedResult(options, fact, writeSecurity, undefined, blockedReasonCode(writeSecurity.blockedBy));
+    }
+    const claimSlot = await resolveDeterministicClaimSlot(deps, userId, fact);
+    const [vectorCandidates, slotCandidates] = await timed(`${options.timingPrefix}.fact.find-dupes`, async () => Promise.all([
+        deps.stores.search.findNearDuplicates(userId, embedding, deps.config.audnCandidateThreshold),
+        findSlotConflictCandidates(deps, userId, claimSlot),
+    ]));
+    const candidates = mergeCandidates(vectorCandidates, slotCandidates);
+    const traceCandidates = toTraceCandidates(candidates);
+    if (candidates.length > 0) {
+        const topCandidate = candidates.reduce((a, b) => a.similarity > b.similarity ? a : b);
+        if (topCandidate.similarity >= deps.config.fastAudnDuplicateThreshold) {
+            return tracedResult(options, {
+                outcome: 'skipped',
+                memoryId: topCandidate.id,
+                trace: buildFactTrace(fact, options.logicalTimestamp, {
+                    writeSecurity,
+                    candidates: traceCandidates,
+                    decision: makeDecision('quick-dedup', 'NOOP', 'quick-duplicate-noop', topCandidate.id, ['raw-near-duplicate']),
+                }, 'skipped', topCandidate.id),
+            });
+        }
+    }
+    const ctx = { userId, fact, embedding, sourceSite, sourceUrl, episodeId, trustScore: writeSecurity.trust.score, claimSlot, logicalTimestamp: options.logicalTimestamp };
+    const result = await storeCanonicalFact(deps, ctx);
+    return storedDirectResult(options, result, embedding, fact, writeSecurity, traceCandidates, undefined, 'direct-store-no-candidates');
+}
+// ---------------------------------------------------------------------------
+// Workspace path (performWorkspaceIngest)
+// ---------------------------------------------------------------------------
+async function processWorkspaceFact(deps, userId, fact, sourceSite, sourceUrl, episodeId, options) {
+    const embedding = await timed(`${options.timingPrefix}.fact.embed`, () => embedText(fact.fact));
+    const writeSecurity = assessWriteSecurity(fact.fact, sourceSite, deps.config);
+    if (!writeSecurity.allowed) {
+        await recordRejectedWrite(userId, fact.fact, sourceSite, writeSecurity, deps.config);
+        return blockedResult(options, fact, writeSecurity, undefined, blockedReasonCode(writeSecurity.blockedBy));
+    }
+    const candidates = await deps.stores.search.findNearDuplicatesInWorkspace(options.workspace.workspaceId, embedding, deps.config.audnCandidateThreshold, 10, 'all', options.workspace.agentId);
+    const traceCandidates = toTraceCandidates(candidates);
+    const ctx = {
+        userId, fact, embedding, sourceSite, sourceUrl, episodeId,
+        trustScore: writeSecurity.trust.score, workspace: options.workspace,
+    };
+    if (candidates.length === 0) {
+        const result = await storeCanonicalFact(deps, ctx);
+        return storedDirectResult(options, result, embedding, fact, writeSecurity, traceCandidates, undefined, 'workspace-direct-store');
+    }
+    return tracedResult(options, await resolveAndExecuteAudn(deps, userId, fact, embedding, sourceSite, sourceUrl, episodeId, writeSecurity.trust.score, null, undefined, candidates.map((c) => ({ ...c, content: c.content ?? '' })), options.supersededTargets, options.workspace, {
+        fact,
+        logicalTimestamp: options.logicalTimestamp,
+        writeSecurity,
+        candidates: traceCandidates,
+    }));
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Check entropy gate; returns false if the fact should be skipped. */
+function assessEntropyGate(fact, embedding, entropyCtx, runtimeConfig) {
+    if (!runtimeConfig.entropyGateEnabled)
+        return null;
+    const entropyResult = computeEntropyScore({
+        windowEntities: fact.keywords,
+        existingEntities: entropyCtx.seenEntities,
+        windowEmbedding: embedding,
+        previousEmbedding: entropyCtx.previousEmbedding,
+    }, { threshold: runtimeConfig.entropyGateThreshold, alpha: runtimeConfig.entropyGateAlpha });
+    entropyCtx.previousEmbedding = embedding;
+    for (const kw of fact.keywords)
+        entropyCtx.seenEntities.add(kw);
+    return entropyResult;
+}
+function buildFactTrace(fact, logicalTimestamp, details, outcome, memoryId) {
+    return {
+        factText: fact.fact,
+        headline: fact.headline,
+        factType: fact.type,
+        importance: fact.importance,
+        ...(logicalTimestamp ? { logicalTimestamp: logicalTimestamp.toISOString() } : {}),
+        ...(details.writeSecurity ? {
+            writeSecurity: {
+                allowed: details.writeSecurity.allowed,
+                blockedBy: details.writeSecurity.blockedBy,
+                trustScore: details.writeSecurity.trust.score,
+            },
+        } : {}),
+        ...(details.entropyGate ? { entropyGate: details.entropyGate } : {}),
+        ...(details.candidates ? { candidates: details.candidates } : {}),
+        decision: details.decision,
+        outcome,
+        memoryId,
+    };
+}
+function makeDecision(source, action, reasonCode, targetMemoryId, rawAction) {
+    return {
+        source,
+        action,
+        reasonCode,
+        targetMemoryId,
+        ...(rawAction?.[0] ? { rawAction: rawAction[0] } : {}),
+    };
+}
+function toTraceCandidates(candidates) {
+    return candidates.map((candidate) => ({
+        id: candidate.id,
+        similarity: Math.round(candidate.similarity * 10000) / 10000,
+        contentPreview: previewContent(candidate.content),
+    }));
+}
+function blockedReasonCode(blockedBy) {
+    return blockedBy === 'sanitization'
+        ? 'write-security-sanitization'
+        : 'write-security-trust';
+}
+function tracedResult(options, result) {
+    if (result.trace)
+        options.traceCollector?.record(result.trace);
+    return result;
+}
+function blockedResult(options, fact, writeSecurity, entropyResult, reasonCode) {
+    return tracedResult(options, {
+        outcome: 'skipped',
+        memoryId: null,
+        trace: buildFactTrace(fact, options.logicalTimestamp, {
+            writeSecurity,
+            entropyGate: entropyResult,
+            decision: makeDecision(reasonCode === 'entropy-gate' ? 'entropy-gate' : 'write-security', 'SKIP', reasonCode, null),
+        }, 'skipped', null),
+    });
+}
+function storedDirectResult(options, result, embedding, fact, writeSecurity, candidates, entropyResult, reasonCode) {
+    return tracedResult(options, {
+        ...result,
+        embedding,
+        trace: buildFactTrace(fact, options.logicalTimestamp, {
+            writeSecurity,
+            entropyGate: entropyResult,
+            candidates,
+            decision: makeDecision('direct-store', 'ADD', reasonCode, null),
+        }, result.outcome, result.memoryId),
+    });
+}

package/dist/services/ingest-post-write.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Post-write processors for the ingest pipeline.
+ *
+ * Runs after the per-fact loop completes: backdates memories to a session
+ * timestamp, generates inter-memory links, and clusters related facts
+ * into composite memories. Each processor is independently skippable via
+ * the batch context.
+ *
+ * Composite generation is full-ingest-only. The caller controls this via
+ * the `compositesEnabled` field — only `performIngest` sets it to true.
+ */
+import type { FactInput, MemoryServiceDeps } from './memory-service-types.js';
+/** Everything the post-write processors need from the completed fact loop. */
+export interface PostWriteBatchContext {
+    episodeId: string;
+    sourceSite: string;
+    sourceUrl: string;
+    /** Facts that were actually stored (with their memoryIds). Only populated by performIngest. */
+    storedFacts: Array<{
+        memoryId: string;
+        fact: FactInput;
+    }>;
+    /** All memory IDs produced by the fact loop (stored + updated). */
+    memoryIds: string[];
+    /** Embedding cache keyed by memoryId, populated during the fact loop. */
+    embeddingCache: Map<string, number[]>;
+    /** When set, memories are backdated to this timestamp. */
+    sessionTimestamp?: Date;
+    /** Caller controls this. Only performIngest sets true. */
+    compositesEnabled: boolean;
+    /** Timing label prefix for timed() wrappers. */
+    timingPrefix: string;
+    /**
+     * Original conversation chunk text. Required when topicAbstractionEnabled
+     * is true; the topic-abstraction processor extracts a conceptual topic from
+     * the full chunk (one LLM call per chunk, not per fact) and tags every
+     * stored memory from this chunk with the same topic + topic_embedding.
+     */
+    chunkText?: string;
+}
+export interface PostWriteResult {
+    linksCreated: number;
+    compositesCreated: number;
+    topicAbstractionApplied: boolean;
+}
+/**
+ * Run all post-write processors for a completed ingest batch.
+ * Order: backdate → links → composites (if caller-enabled).
+ */
+export declare function runPostWriteProcessors(deps: MemoryServiceDeps, userId: string, ctx: PostWriteBatchContext): Promise<PostWriteResult>;