@atomicmemory/core 1.0.0

package/dist/services/document-failure-markers.js

@@ -0,0 +1,305 @@
+/**
+ * Phase C constrained failure-marker transitions.
+ *
+ * `POST /v1/documents/:id/extraction-failure` and
+ * `POST /v1/documents/:id/index-failure` need to be **constrained**
+ * client-side surfaces, not arbitrary status writes - clients can
+ * declare *that* extraction or indexing failed and *what category*,
+ * but cannot put a document into arbitrary status combinations or
+ * smuggle log content into `last_error`. This module owns the
+ * load-then-transition logic for both endpoints:
+ *
+ *   1. Open a transaction and take the per-document
+ *      `pg_advisory_xact_lock` so concurrent markers serialize.
+ *   2. Load the row's raw / extraction / semantic-index status.
+ *   3. Match the current state against the allowed source set and
+ *      apply the corresponding write (or throw an
+ *      `*InvalidStateError` -> 409).
+ *   4. Read the row back and COMMIT so the caller can see the
+ *      durable post-transition shape.
+ *
+ * The audit fix the Phase B plan calls out (rev 18 Phase C section):
+ * the marker MUST sit in front of `markExtractionStatus` /
+ * `markSemanticIndexStatus` so the row's status pair stays internally
+ * consistent (e.g. `extraction='failed'` + `semantic_index='not_required'`
+ * always travel together).
+ */
+import { getRawDocumentById } from '../db/raw-document-repository.js';
+import { buildLastError, markExtractionStatus, markSemanticIndexStatus, } from '../db/raw-document-status-repository.js';
+/** Document not found / not owned by user. Routes map to 404. */
+export class FailureMarkerDocumentNotFoundError extends Error {
+    documentId;
+    constructor(documentId) {
+        super(`document ${documentId} not found`);
+        this.documentId = documentId;
+        this.name = 'FailureMarkerDocumentNotFoundError';
+    }
+}
+/**
+ * Phase C - the row's current state does not allow the requested
+ * extraction-layer transition. Routes map to 409 and echo `current`
+ * in the response so the caller can decide whether to retry.
+ */
+export class ExtractionFailureInvalidStateError extends Error {
+    documentId;
+    current;
+    constructor(documentId, current) {
+        super(`document ${documentId} cannot transition to extraction_status='failed' ` +
+            `from current state ${JSON.stringify(current)}`);
+        this.documentId = documentId;
+        this.current = current;
+        this.name = 'ExtractionFailureInvalidStateError';
+    }
+}
+/**
+ * Phase C - the row's current state does not allow the requested
+ * index-layer transition. Routes map to 409 and echo `current` in
+ * the response.
+ */
+export class IndexFailureInvalidStateError extends Error {
+    documentId;
+    current;
+    constructor(documentId, current) {
+        super(`document ${documentId} cannot transition to semantic_index_status='failed' ` +
+            `from current state ${JSON.stringify(current)}`);
+        this.documentId = documentId;
+        this.current = current;
+        this.name = 'IndexFailureInvalidStateError';
+    }
+}
+/** Raw-storage states that mean "raw bytes are recoverable for retry / forensics". */
+const RAW_OK = new Set([
+    'blob_stored',
+    'inline_text_stored',
+    'pointer_recorded',
+]);
+function snapshot(row) {
+    return {
+        raw_storage_status: row.rawStorageStatus,
+        extraction_status: row.extractionStatus,
+        semantic_index_status: row.semanticIndexStatus,
+    };
+}
+async function withDocumentLock(pool, userId, documentId, body) {
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        await client.query('SELECT pg_advisory_xact_lock(hashtext($1))', [documentId]);
+        const row = await getRawDocumentById(client, userId, documentId);
+        if (!row) {
+            await client.query('ROLLBACK').catch(() => undefined);
+            throw new FailureMarkerDocumentNotFoundError(documentId);
+        }
+        let result;
+        try {
+            result = await body(client, row);
+        }
+        catch (err) {
+            await client.query('ROLLBACK').catch(() => undefined);
+            throw err;
+        }
+        await client.query('COMMIT');
+        return result;
+    }
+    finally {
+        client.release();
+    }
+}
+async function reload(client, userId, documentId) {
+    const row = await getRawDocumentById(client, userId, documentId);
+    if (!row) {
+        // Cannot happen inside the advisory lock unless the row was hard-
+        // deleted by something outside this code path; treat as not-found
+        // rather than corrupting the caller's flow.
+        throw new FailureMarkerDocumentNotFoundError(documentId);
+    }
+    return row;
+}
+/**
+ * Apply the indexer-layer `'failed'` write + reload-and-return the
+ * fresh row. Used by both branches of `markIndexFailure` (idempotent
+ * `last_error` refresh + first-time failure from pending) so the SQL
+ * shape is owned in one place. The caller decides the `idempotent`
+ * flag.
+ */
+async function applySemanticIndexFailedAndReload(args) {
+    await markSemanticIndexStatus({
+        q: args.client,
+        userId: args.userId,
+        documentId: args.documentId,
+        status: 'failed',
+        lastError: args.lastError,
+    });
+    const reloaded = await reload(args.client, args.userId, args.documentId);
+    return { document: reloaded, idempotent: args.idempotent };
+}
+/**
+ * Phase C constrained transition for the extraction layer.
+ *
+ * Allowed source states:
+ *   * `extraction_status='failed'` + same `errorCode` -> idempotent
+ *     no-op; caller sees the existing row with `idempotent: true`.
+ *   * `extraction_status='failed'` + different `errorCode` -> refresh
+ *     `last_error` only; status stays `'failed'`. `idempotent: true`.
+ *   * `extraction_status='pending'` + raw stored ->
+ *     `extraction_status='failed'` + `semantic_index_status='not_required'`
+ *     + new `last_error.layer='extraction'`. `idempotent: false`.
+ *
+ * Any other state throws `ExtractionFailureInvalidStateError` (-> 409).
+ */
+export async function markExtractionFailure(pool, input) {
+    return withDocumentLock(pool, input.userId, input.documentId, async (client, row) => {
+        const lastError = buildLastError('extraction', input.errorCode, input.errorMessage);
+        // Branch 1: idempotent retry on an already-failed row.
+        if (row.extractionStatus === 'failed') {
+            const sameCode = row.lastError?.layer === 'extraction'
+                && row.lastError.code === input.errorCode;
+            if (sameCode) {
+                // Same code: do not touch the row - preserves the original
+                // `occurred_at` so per-incident timestamps stay stable across
+                // retries from the same caller.
+                return { document: row, idempotent: true };
+            }
+            // Different code (or stale envelope): refresh `last_error`.
+            // `markExtractionStatus(failed, lastError)` writes the new
+            // envelope unconditionally on the failed-status branch.
+            await markExtractionStatus({
+                q: client,
+                userId: input.userId,
+                documentId: input.documentId,
+                status: 'failed',
+                lastError,
+            });
+            const reloaded = await reload(client, input.userId, input.documentId);
+            return { document: reloaded, idempotent: true };
+        }
+        // Branch 2: first-time failure from `'pending'` + raw stored.
+        if (row.extractionStatus === 'pending' && RAW_OK.has(row.rawStorageStatus)) {
+            await markExtractionStatus({
+                q: client,
+                userId: input.userId,
+                documentId: input.documentId,
+                status: 'failed',
+                lastError,
+            });
+            // Failed extraction implies nothing to index. Service-owned
+            // semantic-index status flip; the helper's success-branch
+            // `last_error` clear ignores envelopes scoped to other layers,
+            // so the extraction `last_error` we just wrote is preserved.
+            await markSemanticIndexStatus({
+                q: client,
+                userId: input.userId,
+                documentId: input.documentId,
+                status: 'not_required',
+            });
+            const reloaded = await reload(client, input.userId, input.documentId);
+            return { document: reloaded, idempotent: false };
+        }
+        // Anything else - 409.
+        throw new ExtractionFailureInvalidStateError(input.documentId, snapshot(row));
+    });
+}
+/**
+ * Phase C constrained transition for the semantic-index layer.
+ *
+ * Allowed source states:
+ *   * `semantic_index_status='failed'` + same `errorCode` -> idempotent
+ *     no-op (`idempotent: true`).
+ *   * `semantic_index_status='failed'` + different `errorCode` ->
+ *     refresh `last_error` only; status stays `'failed'`.
+ *     `idempotent: true`.
+ *   * `extraction_status='complete'` +
+ *     `semantic_index_status='pending'` + raw stored ->
+ *     `semantic_index_status='failed'` +
+ *     `last_error.layer='semantic_index'`. `idempotent: false`.
+ *   * `extraction_status='pending'` +
+ *     `semantic_index_status='pending'` AND
+ *     `errorCode='index_text_too_large'` + raw stored -> atomically
+ *     `extraction_status='complete'` +
+ *     `semantic_index_status='failed'` + `last_error`. The
+ *     atomic-extraction-complete shortcut models the upload-pipeline
+ *     case where the webapp has the extracted text in hand but it
+ *     exceeded the index byte cap before reaching `POST /:id/index`.
+ *
+ * Any other state -> `IndexFailureInvalidStateError` (-> 409).
+ */
+/** True when the row matches the "extraction completed, indexing pending" branch. */
+function isCompleteThenPendingIndex(row) {
+    return (row.extractionStatus === 'complete'
+        && row.semanticIndexStatus === 'pending'
+        && RAW_OK.has(row.rawStorageStatus));
+}
+/** True when the row matches the upload-pipeline atomic shortcut branch. */
+function isPendingPendingIndexTooLarge(row, errorCode) {
+    return (row.extractionStatus === 'pending'
+        && row.semanticIndexStatus === 'pending'
+        && RAW_OK.has(row.rawStorageStatus)
+        && errorCode === 'index_text_too_large');
+}
+/**
+ * Idempotent-retry handler for a row already in
+ * `semantic_index_status='failed'`. Same code -> no-op; different
+ * code -> refresh `last_error`. Always reports `idempotent: true`.
+ */
+async function indexFailureIdempotentRetry(args) {
+    const sameCode = args.row.lastError?.layer === 'semantic_index'
+        && args.row.lastError.code === args.input.errorCode;
+    if (sameCode) {
+        return { document: args.row, idempotent: true };
+    }
+    return applySemanticIndexFailedAndReload({
+        client: args.client,
+        userId: args.input.userId,
+        documentId: args.input.documentId,
+        lastError: args.lastError,
+        idempotent: true,
+    });
+}
+/**
+ * Atomic extraction-complete shortcut for the upload-pipeline
+ * `index_text_too_large` case. Order matters within the transaction:
+ *   1. `markExtractionStatus(complete)` clears any prior
+ *      extraction-scoped `last_error` (none expected, but the
+ *      helper guards regardless) and flips extraction.
+ *   2. `markSemanticIndexStatus(failed, lastError)` writes the
+ *      durable failure envelope. The two writes COMMIT together
+ *      so external readers never observe the intermediate
+ *      `extraction='complete'` + `semantic_index='pending'` state
+ *      (the lock plus the single COMMIT serializes them).
+ */
+async function indexFailurePendingShortcut(args) {
+    await markExtractionStatus({
+        q: args.client,
+        userId: args.input.userId,
+        documentId: args.input.documentId,
+        status: 'complete',
+    });
+    return applySemanticIndexFailedAndReload({
+        client: args.client,
+        userId: args.input.userId,
+        documentId: args.input.documentId,
+        lastError: args.lastError,
+        idempotent: false,
+    });
+}
+export async function markIndexFailure(pool, input) {
+    return withDocumentLock(pool, input.userId, input.documentId, async (client, row) => {
+        const lastError = buildLastError('semantic_index', input.errorCode, input.errorMessage);
+        if (row.semanticIndexStatus === 'failed') {
+            return indexFailureIdempotentRetry({ client, row, input, lastError });
+        }
+        if (isCompleteThenPendingIndex(row)) {
+            return applySemanticIndexFailedAndReload({
+                client,
+                userId: input.userId,
+                documentId: input.documentId,
+                lastError,
+                idempotent: false,
+            });
+        }
+        if (isPendingPendingIndexTooLarge(row, input.errorCode)) {
+            return indexFailurePendingShortcut({ client, input, lastError });
+        }
+        throw new IndexFailureInvalidStateError(input.documentId, snapshot(row));
+    });
+}

package/dist/services/document-indexer.d.ts

@@ -0,0 +1,122 @@
+/**
+ * Document indexer (Phase 2 + Phase B hardening).
+ *
+ * Takes a registered document + a body of text, deterministically
+ * chunks it, embeds every chunk in one batch via the existing core
+ * `embedTexts` helper, persists chunks to `document_chunks`, and writes
+ * one row per chunk into `memories` with `raw_document_id` +
+ * `document_chunk_id` provenance set so the existing
+ * `/v1/memories/search` retrieval pipeline finds them.
+ *
+ * Phase B (rev-18 plan, Phase B section "document-indexer.ts") rewrote
+ * the flow so it cannot leave a row in a stuck pending/running state:
+ *
+ *   1. Schema validation — type check only, no row touched (`IndexInputError` → 400).
+ *   2. Open transaction, take per-document advisory lock, load row.
+ *   3. Idempotent / re-index short-circuits BEFORE marking running:
+ *        - `'complete'` + same `indexed_content_hash` → COMMIT, return idempotent skip.
+ *        - `'complete'` + different hash → fall through; the conditional
+ *          UPDATE below moves the row through 'running' and the existing
+ *          `clearPriorGeneration` path replaces chunks atomically.
+ *   4. Atomic conditional UPDATE → `'running'` (CAS). Only fires when
+ *      `semantic_index_status IN ('pending','failed','stale','complete')`;
+ *      `'running'` (concurrent writer) and `'not_required'` (registered
+ *      not-to-index) yield rowCount=0 → ROLLBACK + 409.
+ *   5. Known-document semantic validation: text-too-large / empty bodies
+ *      ROLLBACK the running write, then write durable `'failed'` from
+ *      a fresh statement so direct SDK callers cannot leave a stuck
+ *      pending row.
+ *   6. Prepare chunks (chunkText + embedTexts), persist inside the same
+ *      transaction, mark `'complete'` + clear `last_error.semantic_index`,
+ *      COMMIT.
+ *
+ * Visibility note: the `'running'` state is written *inside* the
+ * BEGIN..COMMIT transaction, so under READ COMMITTED isolation other
+ * connections never observe it — the row reads as the prior committed
+ * state until COMMIT, at which point the row reads as `'complete'`
+ * (success) or — after the catch-path fresh-tx write — `'failed'`. The
+ * `'running'` value exists for the conditional-UPDATE concurrency
+ * guard, not as a UI state. UI rendering of "indexing in progress"
+ * requires a future async-worker design that commits `'running'`
+ * before doing the work, with a lease/heartbeat (out of scope; see
+ * the rev-18 "Out of scope" section).
+ *
+ * Idempotency contract (preserved): a re-index with byte-identical text
+ * under the current `chunker_version` is a no-op (no fresh chunks, no
+ * fresh memories, `indexed_content_hash` unchanged). A re-index with
+ * new text soft-deletes the prior chunk + memory generation in
+ * user-scope before inserting the fresh one. Retry from
+ * `semantic_index_status='failed'` proceeds normally (no skip), clears
+ * `last_error`, and lands `'complete'`.
+ */
+import pg from 'pg';
+import { type ChunkOptions } from './document-chunker.js';
+import type { SemanticIndexStatus } from '../db/raw-document-types.js';
+export interface IndexDocumentInput {
+    userId: string;
+    documentId: string;
+    text: string;
+    chunkOptions?: ChunkOptions;
+}
+export interface IndexDocumentResult {
+    documentId: string;
+    /**
+     * SHA-256 of the indexed text under the current `chunker_version`.
+     * Distinct from `RawDocumentRow.contentHash` (which is the
+     * upstream/provider raw-content fingerprint) — both can co-exist on
+     * the same document.
+     */
+    indexedContentHash: string;
+    chunksCreated: number;
+    memoriesCreated: number;
+    /** True when the input text matched the prior indexed text and no work was done. */
+    idempotentSkip: boolean;
+    chunkerVersion: string;
+    parserVersion: string;
+}
+/** Document not found / not owned by user. Routes map to 404. */
+export declare class DocumentNotFoundError extends Error {
+    readonly documentId: string;
+    constructor(documentId: string);
+}
+/**
+ * Schema-level (pre-document) input failure. Routes map to 400; no
+ * row update fires because the document hasn't been loaded yet (and
+ * may not even exist).
+ */
+export declare class IndexInputError extends Error {
+    constructor(message: string);
+}
+/**
+ * Phase B — the conditional UPDATE that moves
+ * `semantic_index_status` to `'running'` returned rowCount=0. The
+ * row is in a state that does not allow indexing (`'running'`
+ * concurrent writer, `'not_required'`, or vanished between load
+ * and the CAS write). Routes map to 409.
+ */
+export declare class IndexInvalidStateError extends Error {
+    readonly documentId: string;
+    readonly currentStatus: SemanticIndexStatus;
+    constructor(documentId: string, currentStatus: SemanticIndexStatus);
+}
+/**
+ * Phase B — known-document semantic validation failure. Thrown after
+ * the row has been loaded and the running CAS has fired, so the
+ * indexer's catch path writes durable `semantic_index_status='failed'`
+ * + `last_error` BEFORE rethrowing. Routes map to 413/400 with the
+ * `documentId` echoed in the body so callers can navigate to the
+ * failed row's detail view.
+ */
+export type IndexSemanticValidationCode = 'index_text_too_large' | 'extraction_empty';
+export declare class IndexSemanticValidationError extends Error {
+    readonly documentId: string;
+    readonly code: IndexSemanticValidationCode;
+    constructor(documentId: string, code: IndexSemanticValidationCode, message: string);
+}
+/**
+ * Run the full index path. See file docstring for the six-phase
+ * structure. The `pool` is used to check out one client for the
+ * mutation transaction; failure markers ride a fresh statement on the
+ * same pool so they survive the rollback.
+ */
+export declare function indexDocumentText(pool: pg.Pool, input: IndexDocumentInput): Promise<IndexDocumentResult>;