@atomicmemory/core 1.0.0

package/dist/services/deferred-audn.js

@@ -0,0 +1,162 @@
+/**
+ * Deferred AUDN Reconciliation Service.
+ *
+ * When DEFERRED_AUDN_ENABLED=true, facts with conflict candidates
+ * (0.7 ≤ similarity < 0.95) are stored immediately as ADD and flagged
+ * for background reconciliation. This eliminates the 500–2000ms LLM
+ * AUDN call from the synchronous ingest path.
+ *
+ * The reconciliation pass processes flagged memories in batches,
+ * running the full LLM AUDN pipeline and applying decisions
+ * (NOOP → delete, SUPERSEDE → soft-delete original, UPDATE → merge).
+ *
+ * Expected latency improvement: 60–80% reduction for ingest batches
+ * with moderate conflict rates (30–70% of facts hitting candidates).
+ */
+import { config } from '../config.js';
+import { findDeferredMemories, findAllDeferredMemories, clearDeferredFlag, countDeferredMemories, markMemoryDeferred, } from '../db/repository-deferred-audn.js';
+import { cachedResolveAUDN } from './extraction-cache.js';
+import { applyClarificationOverrides } from './conflict-policy.js';
+import { embedText } from './embedding.js';
+import { emitAuditEvent } from './audit-events.js';
+/**
+ * Check whether a set of candidates should be deferred rather than
+ * resolved via LLM AUDN synchronously.
+ */
+export function shouldDeferAudn(fastDecisionResolved, candidateCount) {
+    return config.deferredAudnEnabled && !fastDecisionResolved && candidateCount > 0;
+}
+/**
+ * Mark a newly stored memory for deferred reconciliation.
+ */
+export async function deferMemoryForReconciliation(pool, memoryId, candidates) {
+    const serialized = candidates.map((c) => ({
+        id: c.id,
+        content: c.content,
+        similarity: c.similarity,
+    }));
+    await markMemoryDeferred(pool, memoryId, serialized);
+}
+/** Run a reconciliation pass for a single user. */
+export async function reconcileUser(pool, repo, userId, batchSize = config.deferredAudnBatchSize) {
+    const start = Date.now();
+    const deferred = await findDeferredMemories(pool, userId, batchSize);
+    return processReconciliationBatch(pool, repo, deferred, start);
+}
+/** Run a reconciliation pass across all users. */
+export async function reconcileAll(pool, repo, batchSize = config.deferredAudnBatchSize) {
+    const start = Date.now();
+    const deferred = await findAllDeferredMemories(pool, batchSize);
+    return processReconciliationBatch(pool, repo, deferred, start);
+}
+/** Get reconciliation status for a user. */
+export async function getReconciliationStatus(pool, userId) {
+    const pending = await countDeferredMemories(pool, userId);
+    return { pending, enabled: config.deferredAudnEnabled };
+}
+async function processReconciliationBatch(pool, repo, deferred, startMs) {
+    const result = {
+        processed: 0, resolved: 0, noops: 0, updates: 0,
+        supersedes: 0, deletes: 0, adds: 0, errors: 0, durationMs: 0,
+    };
+    for (const memory of deferred) {
+        result.processed++;
+        try {
+            const action = await reconcileSingleMemory(pool, repo, memory);
+            result.resolved++;
+            switch (action) {
+                case 'NOOP':
+                    result.noops++;
+                    break;
+                case 'UPDATE':
+                    result.updates++;
+                    break;
+                case 'SUPERSEDE':
+                    result.supersedes++;
+                    break;
+                case 'DELETE':
+                    result.deletes++;
+                    break;
+                case 'ADD':
+                    result.adds++;
+                    break;
+            }
+        }
+        catch (err) {
+            result.errors++;
+            console.error(`[deferred-audn] Error reconciling memory ${memory.id}:`, err);
+        }
+    }
+    result.durationMs = Date.now() - startMs;
+    return result;
+}
+async function reconcileSingleMemory(pool, repo, memory) {
+    const candidates = await refreshCandidates(repo, memory.userId, memory.candidates);
+    if (candidates.length === 0) {
+        await clearDeferredFlag(pool, memory.id);
+        return 'ADD';
+    }
+    const decision = applyClarificationOverrides(await cachedResolveAUDN(memory.content, candidates), memory.content, candidates, [], 'knowledge');
+    await applyDeferredDecision(pool, repo, memory, decision);
+    await clearDeferredFlag(pool, memory.id);
+    if (config.auditLoggingEnabled) {
+        emitAuditEvent('deferred-audn:reconcile', memory.userId, {
+            memoryId: memory.id,
+            action: decision.action,
+            targetMemoryId: decision.targetMemoryId,
+        });
+    }
+    return decision.action;
+}
+/**
+ * Refresh candidate data — candidates stored at ingest time may have
+ * been modified or deleted since. Re-fetch from DB to ensure accuracy.
+ */
+async function refreshCandidates(repo, userId, storedCandidates) {
+    const refreshed = [];
+    for (const candidate of storedCandidates) {
+        const memory = await repo.getMemory(candidate.id, userId);
+        if (memory && !memory.deleted_at) {
+            refreshed.push({
+                id: memory.id,
+                content: memory.content,
+                similarity: candidate.similarity,
+                importance: memory.importance,
+            });
+        }
+    }
+    return refreshed;
+}
+async function applyDeferredDecision(pool, repo, memory, decision) {
+    switch (decision.action) {
+        case 'NOOP':
+            await repo.softDeleteMemory(memory.userId, memory.id);
+            console.log(`[deferred-audn] NOOP: deleted duplicate ${memory.id}`);
+            break;
+        case 'UPDATE':
+            if (decision.targetMemoryId && decision.updatedContent) {
+                const target = await repo.getMemory(decision.targetMemoryId, memory.userId);
+                const newEmbedding = await embedText(decision.updatedContent);
+                await repo.updateMemoryContent(memory.userId, decision.targetMemoryId, decision.updatedContent, newEmbedding, target?.importance ?? 0.5);
+                await repo.softDeleteMemory(memory.userId, memory.id);
+                console.log(`[deferred-audn] UPDATE: merged ${memory.id} into ${decision.targetMemoryId}`);
+            }
+            break;
+        case 'SUPERSEDE':
+            if (decision.targetMemoryId) {
+                await repo.softDeleteMemory(memory.userId, decision.targetMemoryId);
+                console.log(`[deferred-audn] SUPERSEDE: ${memory.id} replaces ${decision.targetMemoryId}`);
+            }
+            break;
+        case 'DELETE':
+            await repo.softDeleteMemory(memory.userId, memory.id);
+            console.log(`[deferred-audn] DELETE: removed ${memory.id}`);
+            break;
+        case 'ADD':
+            console.log(`[deferred-audn] ADD: confirmed ${memory.id} is distinct`);
+            break;
+        default:
+            console.log(`[deferred-audn] ${decision.action}: no action for ${memory.id}`);
+            break;
+    }
+}

package/dist/services/document-chunker.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Deterministic text chunker for the document pipeline (Phase 2).
+ *
+ * Pure function: same (text, options) → same chunks. No timing, no
+ * randomness, no provider state. Each chunk carries an absolute
+ * character offset range, a content_hash, a stable index, and a token
+ * estimate; the index field is what the active-unique partial index on
+ * `document_chunks` keys on alongside `chunker_version`.
+ *
+ * Phase 2 ships a single chunker_version (`PHASE2_CHUNKER_VERSION`).
+ * If the algorithm changes meaningfully, bump the constant — that
+ * triggers a fresh insert generation rather than colliding with the
+ * old run on the unique index.
+ *
+ * See `Atomicmemory-research/docs/core-repo/design/large-file-ingestion-and-raw-storage-plan-2026-05-08.md`
+ * Phase 2.
+ */
+/**
+ * Pinned chunker identifier. Bump when the algorithm output changes for
+ * the same input text — that lets the active-unique index treat the new
+ * generation as fresh inserts rather than colliding with prior runs.
+ */
+export declare const PHASE2_CHUNKER_VERSION = "phase2-fixed-v1";
+/** Pinned parser identifier. Phase 2 accepts text input only. */
+export declare const PHASE2_PARSER_VERSION = "phase2-text-v1";
+export interface ChunkOptions {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+}
+export interface ChunkResult {
+    chunkIndex: number;
+    content: string;
+    contentHash: string;
+    charStart: number;
+    charEnd: number;
+    tokenCount: number;
+}
+/**
+ * Chunk `text` deterministically. Returns `[]` for empty or
+ * whitespace-only input.
+ */
+export declare function chunkText(text: string, options?: ChunkOptions): ChunkResult[];
+/** Fingerprint a chunk's content; stable across runs for byte-identical input. */
+export declare function hashChunkContent(content: string): string;
+/**
+ * Whole-text fingerprint, used by the indexer's idempotency check.
+ * Distinct helper so tests can pin both invariants independently.
+ */
+export declare function hashIndexedText(text: string): string;

package/dist/services/document-chunker.js

@@ -0,0 +1,153 @@
+/**
+ * Deterministic text chunker for the document pipeline (Phase 2).
+ *
+ * Pure function: same (text, options) → same chunks. No timing, no
+ * randomness, no provider state. Each chunk carries an absolute
+ * character offset range, a content_hash, a stable index, and a token
+ * estimate; the index field is what the active-unique partial index on
+ * `document_chunks` keys on alongside `chunker_version`.
+ *
+ * Phase 2 ships a single chunker_version (`PHASE2_CHUNKER_VERSION`).
+ * If the algorithm changes meaningfully, bump the constant — that
+ * triggers a fresh insert generation rather than colliding with the
+ * old run on the unique index.
+ *
+ * See `Atomicmemory-research/docs/core-repo/design/large-file-ingestion-and-raw-storage-plan-2026-05-08.md`
+ * Phase 2.
+ */
+import { createHash } from 'node:crypto';
+/**
+ * Character size of one chunk before word-boundary trimming. ~250 tokens
+ * for `text-embedding-3-small`'s typical English-text ratio (4 chars/token);
+ * well under the 8192-token model limit.
+ */
+const DEFAULT_CHUNK_SIZE = 1500;
+/**
+ * Overlap between adjacent chunks. ~10% of chunk size keeps adjacent
+ * sentences findable via either chunk without exploding the chunk count.
+ */
+const DEFAULT_CHUNK_OVERLAP = 150;
+/** Reject chunks that fall below this size after trimming. */
+const DEFAULT_MIN_CHUNK_SIZE = 100;
+/**
+ * Pinned chunker identifier. Bump when the algorithm output changes for
+ * the same input text — that lets the active-unique index treat the new
+ * generation as fresh inserts rather than colliding with prior runs.
+ */
+export const PHASE2_CHUNKER_VERSION = 'phase2-fixed-v1';
+/** Pinned parser identifier. Phase 2 accepts text input only. */
+export const PHASE2_PARSER_VERSION = 'phase2-text-v1';
+/**
+ * Chunk `text` deterministically. Returns `[]` for empty or
+ * whitespace-only input.
+ */
+export function chunkText(text, options = {}) {
+    if (!text || text.trim().length === 0)
+        return [];
+    const opts = resolveOptions(options);
+    if (text.length <= opts.chunkSize) {
+        const leading = text.length - text.trimStart().length;
+        const trailing = text.length - text.trimEnd().length;
+        const trimmed = text.slice(leading, text.length - trailing);
+        return trimmed.length === 0
+            ? []
+            : [makeChunk(0, trimmed, leading, text.length - trailing)];
+    }
+    return slidingWindowChunks(text, opts);
+}
+/** Fingerprint a chunk's content; stable across runs for byte-identical input. */
+export function hashChunkContent(content) {
+    return createHash('sha256').update(content, 'utf8').digest('hex');
+}
+/**
+ * Whole-text fingerprint, used by the indexer's idempotency check.
+ * Distinct helper so tests can pin both invariants independently.
+ */
+export function hashIndexedText(text) {
+    return createHash('sha256').update(text, 'utf8').digest('hex');
+}
+function resolveOptions(input) {
+    const chunkSize = Math.max(1, input.chunkSize ?? DEFAULT_CHUNK_SIZE);
+    const chunkOverlap = clampOverlap(input.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP, chunkSize);
+    const minChunkSize = Math.max(1, Math.min(input.minChunkSize ?? DEFAULT_MIN_CHUNK_SIZE, chunkSize));
+    return { chunkSize, chunkOverlap, minChunkSize };
+}
+function clampOverlap(requested, chunkSize) {
+    if (requested < 0)
+        return 0;
+    // Overlap must leave room to advance past it on each step; cap at half.
+    return Math.min(requested, Math.floor(chunkSize / 2));
+}
+function slidingWindowChunks(text, opts) {
+    const chunks = [];
+    let cursor = 0;
+    let chunkIndex = 0;
+    while (cursor < text.length) {
+        const window = openWindow(text, cursor, opts);
+        if (window.content.length >= opts.minChunkSize) {
+            chunks.push(makeChunk(chunkIndex, window.content, window.charStart, window.charEnd));
+            chunkIndex++;
+        }
+        if (window.advanceTo >= text.length)
+            break;
+        cursor = advanceCursor(window.advanceTo, opts.chunkOverlap, cursor);
+    }
+    return chunks;
+}
+function openWindow(text, cursor, opts) {
+    const rawEnd = Math.min(cursor + opts.chunkSize, text.length);
+    const wordEnd = preserveWordBoundary(text, cursor, rawEnd, opts.minChunkSize);
+    // Recompute the offsets so they exactly bound the trimmed content; this
+    // is the invariant downstream relies on (text.slice(charStart, charEnd)
+    // === content), and downstream callers — provenance audit, future raw
+    // re-fetch — would otherwise see ranges that include leading/trailing
+    // whitespace not present in the chunk's stored content.
+    const slice = text.slice(cursor, wordEnd);
+    const leading = slice.length - slice.trimStart().length;
+    const trailing = slice.length - slice.trimEnd().length;
+    const charStart = cursor + leading;
+    const charEnd = wordEnd - trailing;
+    const content = slice.slice(leading, slice.length - trailing);
+    return { content, charStart, charEnd, advanceTo: wordEnd };
+}
+/**
+ * Walk back from `rawEnd` to the previous whitespace so the chunk doesn't
+ * end mid-word. Bails out (returning rawEnd unchanged) if no whitespace
+ * is found within the [cursor + minChunkSize, rawEnd) window — that
+ * keeps the slider from collapsing on inputs with no spaces.
+ */
+function preserveWordBoundary(text, cursor, rawEnd, minChunkSize) {
+    if (rawEnd >= text.length)
+        return rawEnd;
+    const lower = cursor + minChunkSize;
+    for (let i = rawEnd; i > lower; i--) {
+        if (/\s/.test(text[i - 1]))
+            return i;
+    }
+    return rawEnd;
+}
+function advanceCursor(charEnd, overlap, prevCursor) {
+    const next = charEnd - overlap;
+    // Guard against the case where overlap >= last window length, which
+    // would otherwise make the cursor stand still and loop.
+    return next > prevCursor ? next : charEnd;
+}
+function makeChunk(chunkIndex, content, charStart, charEnd) {
+    return {
+        chunkIndex,
+        content,
+        contentHash: hashChunkContent(content),
+        charStart,
+        charEnd,
+        tokenCount: estimateTokens(content),
+    };
+}
+/**
+ * Cheap token estimate: ~4 chars / token for English ASCII. We intentionally
+ * don't use a real tokenizer here — that would couple the chunker to the
+ * embedding model and add a heavy dependency. The number is metadata for
+ * downstream cost reporting, not a control.
+ */
+function estimateTokens(content) {
+    return Math.max(1, Math.ceil(content.length / 4));
+}

package/dist/services/document-failure-markers.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Phase C constrained failure-marker transitions.
+ *
+ * `POST /v1/documents/:id/extraction-failure` and
+ * `POST /v1/documents/:id/index-failure` need to be **constrained**
+ * client-side surfaces, not arbitrary status writes - clients can
+ * declare *that* extraction or indexing failed and *what category*,
+ * but cannot put a document into arbitrary status combinations or
+ * smuggle log content into `last_error`. This module owns the
+ * load-then-transition logic for both endpoints:
+ *
+ *   1. Open a transaction and take the per-document
+ *      `pg_advisory_xact_lock` so concurrent markers serialize.
+ *   2. Load the row's raw / extraction / semantic-index status.
+ *   3. Match the current state against the allowed source set and
+ *      apply the corresponding write (or throw an
+ *      `*InvalidStateError` -> 409).
+ *   4. Read the row back and COMMIT so the caller can see the
+ *      durable post-transition shape.
+ *
+ * The audit fix the Phase B plan calls out (rev 18 Phase C section):
+ * the marker MUST sit in front of `markExtractionStatus` /
+ * `markSemanticIndexStatus` so the row's status pair stays internally
+ * consistent (e.g. `extraction='failed'` + `semantic_index='not_required'`
+ * always travel together).
+ */
+import pg from 'pg';
+import type { ExtractionStatus, RawDocumentRow, RawStorageStatus, SemanticIndexStatus } from '../db/raw-document-types.js';
+import type { ExtractionErrorCode, IndexErrorCode } from '../schemas/documents.js';
+/** State snapshot echoed in `*InvalidStateError` for 409 bodies. */
+export interface DocumentLayerStateSnapshot {
+    raw_storage_status: RawStorageStatus;
+    extraction_status: ExtractionStatus;
+    semantic_index_status: SemanticIndexStatus;
+}
+/** Document not found / not owned by user. Routes map to 404. */
+export declare class FailureMarkerDocumentNotFoundError extends Error {
+    readonly documentId: string;
+    constructor(documentId: string);
+}
+/**
+ * Phase C - the row's current state does not allow the requested
+ * extraction-layer transition. Routes map to 409 and echo `current`
+ * in the response so the caller can decide whether to retry.
+ */
+export declare class ExtractionFailureInvalidStateError extends Error {
+    readonly documentId: string;
+    readonly current: DocumentLayerStateSnapshot;
+    constructor(documentId: string, current: DocumentLayerStateSnapshot);
+}
+/**
+ * Phase C - the row's current state does not allow the requested
+ * index-layer transition. Routes map to 409 and echo `current` in
+ * the response.
+ */
+export declare class IndexFailureInvalidStateError extends Error {
+    readonly documentId: string;
+    readonly current: DocumentLayerStateSnapshot;
+    constructor(documentId: string, current: DocumentLayerStateSnapshot);
+}
+export interface MarkerInput<C> {
+    userId: string;
+    documentId: string;
+    errorCode: C;
+    errorMessage: string;
+}
+export interface MarkerResult {
+    document: RawDocumentRow;
+    /**
+     * `true` when the row was already in the target failed state and
+     * the call only refreshed `last_error` (or was a complete no-op
+     * for same-code retries). `false` for a first-time transition.
+     */
+    idempotent: boolean;
+}
+/**
+ * Phase C constrained transition for the extraction layer.
+ *
+ * Allowed source states:
+ *   * `extraction_status='failed'` + same `errorCode` -> idempotent
+ *     no-op; caller sees the existing row with `idempotent: true`.
+ *   * `extraction_status='failed'` + different `errorCode` -> refresh
+ *     `last_error` only; status stays `'failed'`. `idempotent: true`.
+ *   * `extraction_status='pending'` + raw stored ->
+ *     `extraction_status='failed'` + `semantic_index_status='not_required'`
+ *     + new `last_error.layer='extraction'`. `idempotent: false`.
+ *
+ * Any other state throws `ExtractionFailureInvalidStateError` (-> 409).
+ */
+export declare function markExtractionFailure(pool: pg.Pool, input: MarkerInput<ExtractionErrorCode>): Promise<MarkerResult>;
+export declare function markIndexFailure(pool: pg.Pool, input: MarkerInput<IndexErrorCode>): Promise<MarkerResult>;