@atomicmemory/core 1.0.0

package/dist/db/raw-document-repository.js

@@ -0,0 +1,410 @@
+/**
+ * Postgres queries for the document pipeline (Phase 1 — pointer-only).
+ *
+ * Function-style module mirroring `repository-write.ts`. The split
+ * `*WithClient` variants exist so `deleteBySource` can include
+ * document soft-deletion inside its existing transaction.
+ *
+ * Phase 1 contract: callers only ever pass `storageMode = 'pointer_only'`.
+ * The CHECK constraint on `raw_documents.storage_mode` accepts
+ * `managed_blob` and `inline_small_text` as well, but the service layer
+ * rejects them until those phases land. No raw content is ever written
+ * by this module — `storage_uri` and `storage_provider` are NEVER set
+ * in Phase 1.
+ */
+import { createStorageArtifact } from './storage-artifact-repository.js';
+import { EXTERNAL_POINTER_PROVIDER } from './storage-artifact-providers.js';
+const RAW_SOURCE_COLUMNS = 'id, user_id, source_site, provider, account_id, storage_mode, retention_policy, consent_policy, created_at, updated_at';
+/**
+ * Full column set on `raw_documents`. Exported so the Phase D list /
+ * recovery / passport-feed repositories (which moved to focused
+ * modules to keep this file under the 400 LOC rule) can build their
+ * SELECT lists from the same canonical source.
+ */
+export const RAW_DOCUMENT_COLUMNS = 'id, user_id, raw_source_id, external_id, external_uri, display_name, mime_type, size_bytes, content_hash, provider_version, source_modified_at, storage_mode, storage_uri, storage_provider, registration_status, raw_storage_status, raw_storage_metadata, metadata, created_at, updated_at, deleted_at, indexed_content_hash, indexed_at, extraction_status, semantic_index_status, last_error, raw_storage_claim_id, raw_storage_claimed_at, raw_storage_last_checked_at, raw_storage_next_check_at, raw_storage_reconcile_attempts, raw_storage_pending_since, storage_artifact_id';
+// ---------------------------------------------------------------------------
+// raw_sources
+// ---------------------------------------------------------------------------
+/**
+ * Idempotently insert (or update) a `raw_sources` row keyed on
+ * `(user_id, source_site, provider, COALESCE(account_id, ''))`. Returns
+ * the persisted row.
+ *
+ * On conflict the storage_mode + retention/consent policies are
+ * **overwritten** with the input values — those describe the caller's
+ * current intent, and silently keeping a stale policy on the existing
+ * row would mask configuration drift.
+ */
+export async function upsertRawSource(pool, input) {
+    const accountId = input.accountId ?? null;
+    const storageMode = input.storageMode ?? 'pointer_only';
+    const retentionPolicy = input.retentionPolicy ?? {};
+    const consentPolicy = input.consentPolicy ?? {};
+    const result = await pool.query(`INSERT INTO raw_sources (user_id, source_site, provider, account_id, storage_mode, retention_policy, consent_policy)
+     VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7::jsonb)
+     ON CONFLICT (user_id, source_site, provider, COALESCE(account_id, ''))
+     DO UPDATE SET
+       storage_mode = EXCLUDED.storage_mode,
+       retention_policy = EXCLUDED.retention_policy,
+       consent_policy = EXCLUDED.consent_policy,
+       updated_at = NOW()
+     RETURNING ${RAW_SOURCE_COLUMNS}`, [
+        input.userId,
+        input.sourceSite,
+        input.provider,
+        accountId,
+        storageMode,
+        JSON.stringify(retentionPolicy),
+        JSON.stringify(consentPolicy),
+    ]);
+    return rowToRawSource(result.rows[0]);
+}
+// ---------------------------------------------------------------------------
+// raw_documents
+// ---------------------------------------------------------------------------
+/**
+ * Idempotently register a `raw_documents` row. Returns
+ * `{ document, created: true }` when a fresh row was inserted, or
+ * `{ document, created: false }` when an active row already existed
+ * (matched on `idx_raw_documents_active_unique`).
+ *
+ * Implemented as a single atomic `INSERT ... ON CONFLICT DO NOTHING
+ * RETURNING ...` followed by a fallback SELECT when no row was
+ * inserted. Two concurrent calls for the same namespace see exactly
+ * one INSERT win and one fallback SELECT — neither caller observes
+ * the unique-violation error that the previous find-then-insert
+ * pattern could surface.
+ *
+ * Soft-deleted rows are excluded by the partial unique index, so a
+ * re-registration after `softDeleteRawDocument` always inserts a new id.
+ */
+export async function registerRawDocument(pool, input) {
+    // Step 7 of the storage-sibling plan: when the caller supplies an
+    // `external_uri`, this insert ALSO writes a paired pointer-mode
+    // `storage_artifacts` row and links `raw_documents.storage_artifact_id`
+    // in the same transaction. Documents registered without an
+    // `external_uri` remain pure registration stubs (NULL link). A
+    // later managed upload (Phase β2 in `document-upload.ts`) is the
+    // first place the row gets a managed artifact.
+    if (input.externalUri)
+        return registerWithPointerArtifact(pool, input);
+    return registerWithoutArtifact(pool, input);
+}
+async function registerWithoutArtifact(pool, input) {
+    const inserted = await tryInsertRawDocument(pool, input);
+    if (inserted)
+        return { document: inserted, created: true };
+    const existing = await findActiveDocumentRow(pool, input);
+    if (existing)
+        return { document: existing, created: false };
+    throw new Error('raw_documents: concurrent soft-delete observed during registration');
+}
+/**
+ * Registration that pairs the new document with a pointer-mode
+ * `storage_artifacts` row in one transaction. On idempotent re-
+ * registration (the partial-unique index already has an active row),
+ * the caller's external_uri is ignored and the existing artifact
+ * link is preserved.
+ */
+async function registerWithPointerArtifact(pool, input) {
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const inserted = await tryInsertRawDocumentWithClient(client, input);
+        if (inserted === null) {
+            await client.query('COMMIT');
+            const existing = await findActiveDocumentRow(pool, input);
+            if (existing)
+                return { document: existing, created: false };
+            throw new Error('raw_documents: concurrent soft-delete observed during registration');
+        }
+        const artifact = await createStorageArtifact(client, {
+            userId: input.userId,
+            provider: EXTERNAL_POINTER_PROVIDER,
+            mode: 'pointer',
+            uri: input.externalUri,
+            status: 'stored',
+            sizeBytes: input.sizeBytes ?? null,
+            contentType: input.mimeType ?? null,
+            contentEncoding: 'identity',
+            discloseContentHash: false,
+            identifiers: input.contentHash ? { contentHash: input.contentHash } : {},
+            metadata: {},
+        });
+        const linked = await client.query(`UPDATE raw_documents SET storage_artifact_id = $1, updated_at = NOW()
+         WHERE id = $2 RETURNING ${RAW_DOCUMENT_COLUMNS}`, [artifact.id, inserted.id]);
+        await client.query('COMMIT');
+        return { document: rowToRawDocument(linked.rows[0]), created: true };
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => undefined);
+        throw err;
+    }
+    finally {
+        client.release();
+    }
+}
+const INSERT_RAW_DOCUMENT_COLUMNS = 'user_id, raw_source_id, external_id, external_uri, display_name, mime_type, size_bytes, content_hash, provider_version, source_modified_at, storage_mode, metadata, extraction_status, semantic_index_status';
+const INSERT_RAW_DOCUMENT_PLACEHOLDERS = '$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13, $14';
+/**
+ * Treat `undefined` as `null` at the wire-to-SQL boundary. Tiny helper so
+ * the param builder doesn't fan out one `?? null` branch per nullable
+ * column — keeping the builder's cyclomatic complexity at one path.
+ */
+function nullify(value) {
+    return value ?? null;
+}
+function buildInsertRawDocumentParams(input) {
+    return [
+        input.userId,
+        input.rawSourceId,
+        input.externalId,
+        nullify(input.externalUri),
+        nullify(input.displayName),
+        nullify(input.mimeType),
+        nullify(input.sizeBytes),
+        nullify(input.contentHash),
+        nullify(input.providerVersion),
+        nullify(input.sourceModifiedAt),
+        input.storageMode ?? 'pointer_only',
+        JSON.stringify(input.metadata ?? {}),
+        // Phase B — restricted-initial-state writes. Default
+        // `'not_required'` matches the column default; clients that opt
+        // into the document pipeline pass `'pending'` (see
+        // `RegisterDocumentBodySchema` for the wire-side guard).
+        input.extractionStatus ?? 'not_required',
+        input.semanticIndexStatus ?? 'not_required',
+    ];
+}
+/**
+ * Atomic INSERT on the active-unique partial index. Returns the
+ * persisted row when this call won the insert; null when an active
+ * row already exists (a concurrent caller, or this was a re-register
+ * with the same namespace).
+ */
+async function tryInsertRawDocument(pool, input) {
+    return tryInsertRawDocumentWithClient(pool, input);
+}
+async function tryInsertRawDocumentWithClient(q, input) {
+    const result = await q.query(`INSERT INTO raw_documents (${INSERT_RAW_DOCUMENT_COLUMNS})
+     VALUES (${INSERT_RAW_DOCUMENT_PLACEHOLDERS})
+     ON CONFLICT (user_id, raw_source_id, external_id, COALESCE(provider_version, ''))
+     WHERE deleted_at IS NULL DO NOTHING
+     RETURNING ${RAW_DOCUMENT_COLUMNS}`, buildInsertRawDocumentParams(input));
+    if (result.rows.length === 0)
+        return null;
+    return rowToRawDocument(result.rows[0]);
+}
+async function findActiveDocumentRow(pool, input) {
+    const result = await pool.query(`SELECT ${RAW_DOCUMENT_COLUMNS}
+       FROM raw_documents
+      WHERE user_id = $1
+        AND raw_source_id = $2
+        AND external_id = $3
+        AND COALESCE(provider_version, '') = COALESCE($4, '')
+        AND deleted_at IS NULL`, [input.userId, input.rawSourceId, input.externalId, input.providerVersion ?? null]);
+    if (result.rows.length === 0)
+        return null;
+    return rowToRawDocument(result.rows[0]);
+}
+/**
+ * Fetch one active document by id, scoped to a user. Returns null when
+ * the row is missing, deleted, or owned by a different user.
+ */
+export async function getRawDocumentById(q, userId, id) {
+    const result = await q.query(`SELECT ${RAW_DOCUMENT_COLUMNS}
+       FROM raw_documents
+      WHERE id = $1 AND user_id = $2 AND deleted_at IS NULL`, [id, userId]);
+    if (result.rows.length === 0)
+        return null;
+    return rowToRawDocument(result.rows[0]);
+}
+/**
+ * List active documents for a user, optionally filtered by source_site.
+ * Limit is clamped to [1, 100]; offset is non-negative.
+ */
+export async function listRawDocuments(pool, input) {
+    const limit = Math.max(1, Math.min(100, Math.floor(input.limit ?? 20)));
+    const offset = Math.max(0, Math.floor(input.offset ?? 0));
+    const params = [input.userId, limit, offset];
+    let where = `d.user_id = $1 AND d.deleted_at IS NULL`;
+    if (input.sourceSite) {
+        params.push(input.sourceSite);
+        where += ` AND s.source_site = $${params.length}`;
+    }
+    const result = await pool.query(`SELECT ${RAW_DOCUMENT_COLUMNS.split(', ').map(c => `d.${c}`).join(', ')}
+       FROM raw_documents d
+       JOIN raw_sources s ON s.id = d.raw_source_id
+      WHERE ${where}
+      ORDER BY d.created_at DESC
+      LIMIT $2 OFFSET $3`, params);
+    return result.rows.map(rowToRawDocument);
+}
+/**
+ * Soft-delete one document by id, scoped to a user. Returns true when
+ * a row transitioned from active to deleted; false when the row was
+ * missing, already deleted, or owned by a different user.
+ */
+export async function softDeleteRawDocument(q, userId, id) {
+    const result = await q.query(`UPDATE raw_documents
+        SET deleted_at = NOW(), updated_at = NOW()
+      WHERE id = $1 AND user_id = $2 AND deleted_at IS NULL`, [id, userId]);
+    return (result.rowCount ?? 0) > 0;
+}
+/**
+ * Fetch one active document together with its parent source's
+ * `source_site`. The Phase 2 indexer uses `source_site` to populate the
+ * `memories.source_site` column on derived memories without needing a
+ * second round-trip.
+ */
+export async function getDocumentWithSourceSite(q, userId, id) {
+    const result = await q.query(`SELECT ${RAW_DOCUMENT_COLUMNS.split(', ').map((c) => `d.${c}`).join(', ')}, s.source_site
+       FROM raw_documents d
+       JOIN raw_sources s ON s.id = d.raw_source_id
+      WHERE d.id = $1 AND d.user_id = $2 AND d.deleted_at IS NULL`, [id, userId]);
+    if (result.rows.length === 0)
+        return null;
+    const row = result.rows[0];
+    return {
+        document: rowToRawDocument(row),
+        sourceSite: row.source_site,
+    };
+}
+/**
+ * Update the indexed-text fingerprint on an active document. Called by
+ * the Phase 2 indexer on a successful re-chunk pass so subsequent index
+ * calls can short-circuit on byte-identical input.
+ *
+ * Writes only `indexed_content_hash` + `indexed_at` (and `updated_at`).
+ * `content_hash` is reserved for the upstream/provider raw-content
+ * fingerprint and must not be touched by the indexer.
+ */
+export async function setRawDocumentIndexedHashWithClient(client, userId, id, indexedContentHash) {
+    await client.query(`UPDATE raw_documents
+        SET indexed_content_hash = $1,
+            indexed_at = NOW(),
+            updated_at = NOW()
+      WHERE id = $2 AND user_id = $3 AND deleted_at IS NULL`, [indexedContentHash, id, userId]);
+}
+/**
+ * Soft-delete every active document whose source_site matches.
+ * Joins through `raw_sources` so callers don't need to know
+ * `raw_source_id`. Used by `deleteBySource` to keep document deletion
+ * inside the same transactional reset.
+ */
+export async function deleteDocumentsBySourceWithClient(client, userId, sourceSite) {
+    const result = await client.query(`UPDATE raw_documents d
+        SET deleted_at = NOW(), updated_at = NOW()
+       FROM raw_sources s
+      WHERE d.raw_source_id = s.id
+        AND d.user_id = $1
+        AND s.source_site = $2
+        AND d.deleted_at IS NULL`, [userId, sourceSite]);
+    return result.rowCount ?? 0;
+}
+// ---------------------------------------------------------------------------
+// Row mappers — accept the snake_case shape Postgres returns and emit
+// the camelCase TypeScript types the rest of the codebase consumes.
+// ---------------------------------------------------------------------------
+function rowToRawSource(row) {
+    return {
+        id: row.id,
+        userId: row.user_id,
+        sourceSite: row.source_site,
+        provider: row.provider,
+        accountId: row.account_id ?? null,
+        storageMode: row.storage_mode,
+        retentionPolicy: row.retention_policy ?? {},
+        consentPolicy: row.consent_policy ?? {},
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}
+function nullableString(value) {
+    return value ?? null;
+}
+function nullableDate(value) {
+    return value ?? null;
+}
+function nullableBigInt(value) {
+    return value === null || value === undefined ? null : Number(value);
+}
+/** Identifying fields shared by every active raw_documents row. */
+function rowToRawDocumentIdentity(row) {
+    return {
+        id: row.id,
+        userId: row.user_id,
+        rawSourceId: row.raw_source_id,
+        externalId: row.external_id,
+    };
+}
+/** Provider/source-side metadata fields. */
+function rowToRawDocumentSourceMeta(row) {
+    return {
+        externalUri: nullableString(row.external_uri),
+        displayName: nullableString(row.display_name),
+        mimeType: nullableString(row.mime_type),
+        sizeBytes: nullableBigInt(row.size_bytes),
+        contentHash: nullableString(row.content_hash),
+        providerVersion: nullableString(row.provider_version),
+        sourceModifiedAt: nullableDate(row.source_modified_at),
+    };
+}
+/** Storage + lifecycle status fields. */
+function rowToRawDocumentStatus(row) {
+    return {
+        storageMode: row.storage_mode,
+        storageUri: nullableString(row.storage_uri),
+        storageProvider: nullableString(row.storage_provider),
+        registrationStatus: row.registration_status,
+        rawStorageStatus: row.raw_storage_status,
+        rawStorageMetadata: row.raw_storage_metadata ?? {},
+    };
+}
+/**
+ * Phase 5 / Phase 6 private worker columns. Read into RawDocumentRow
+ * so the upload service and the reconciler can drive their state
+ * machines off typed fields; never projected to the wire.
+ */
+function rowToRawDocumentWorkerState(row) {
+    return {
+        rawStorageClaimId: nullableString(row.raw_storage_claim_id),
+        rawStorageClaimedAt: nullableDate(row.raw_storage_claimed_at),
+        rawStorageLastCheckedAt: nullableDate(row.raw_storage_last_checked_at),
+        rawStorageNextCheckAt: nullableDate(row.raw_storage_next_check_at),
+        rawStorageReconcileAttempts: typeof row.raw_storage_reconcile_attempts === 'number'
+            ? row.raw_storage_reconcile_attempts
+            : 0,
+        rawStoragePendingSince: nullableDate(row.raw_storage_pending_since),
+    };
+}
+/** Phase B — per-layer status fields + last_error envelope. */
+function rowToRawDocumentLayerStatus(row) {
+    return {
+        extractionStatus: row.extraction_status,
+        semanticIndexStatus: row.semantic_index_status,
+        lastError: row.last_error ?? null,
+    };
+}
+/**
+ * Map a flat Postgres row into the typed `RawDocumentRow` shape.
+ * Exported so the Phase D list / recovery / passport-feed
+ * repositories can reuse it without re-implementing the column-by-
+ * column projection.
+ */
+export function rowToRawDocument(row) {
+    return {
+        ...rowToRawDocumentIdentity(row),
+        ...rowToRawDocumentSourceMeta(row),
+        ...rowToRawDocumentStatus(row),
+        ...rowToRawDocumentLayerStatus(row),
+        ...rowToRawDocumentWorkerState(row),
+        metadata: row.metadata ?? {},
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+        deletedAt: nullableDate(row.deleted_at),
+        indexedContentHash: nullableString(row.indexed_content_hash),
+        indexedAt: nullableDate(row.indexed_at),
+        storageArtifactId: nullableString(row.storage_artifact_id),
+    };
+}

package/dist/db/raw-document-status-repository.d.ts

@@ -0,0 +1,122 @@
+/**
+ * Per-layer status repository helpers (Phase B).
+ *
+ * The audit at
+ * `Atomicmemory-research/docs/core-repo/design/document-ingest-audit.md`
+ * and the rev-18 hardening plan call for **service-owned** transitions
+ * of `extraction_status`, `semantic_index_status`, `raw_storage_status`
+ * and the `last_error` JSONB envelope. This module exposes the SQL
+ * primitives that the indexer (`document-indexer.ts`), the upload
+ * service (`document-upload.ts`), and any future async worker call to
+ * record those transitions.
+ *
+ * Design notes (cross-referenced from the plan):
+ *
+ *   * Each helper accepts `pg.Pool | pg.PoolClient` so it works inside
+ *     an in-flight transaction (e.g. the indexer's running-tx) and
+ *     from a **fresh** statement after a parent rolled back. The catch
+ *     paths in `document-upload.ts` and `document-indexer.ts` use the
+ *     fresh-statement form deliberately so the durable failure write
+ *     survives the rollback that drops the in-tx work.
+ *
+ *   * Mark helpers are scoped by `user_id` + `id` and skip soft-deleted
+ *     rows. They are **idempotent** at the SQL level - re-applying the
+ *     same status (e.g. `'failed'` -> `'failed'` with the same code)
+ *     returns the row unchanged.
+ *
+ *   * `last_error` write rule: a failure transition writes the supplied
+ *     envelope; a success transition clears `last_error` **only when
+ *     the existing envelope was scoped to the same layer**. A
+ *     successful semantic-index pass should not silently erase a
+ *     previous extraction failure that someone else still needs to
+ *     reconcile.
+ */
+import pg from 'pg';
+import type { ExtractionStatus, LastError, LastErrorLayer, SemanticIndexStatus } from './raw-document-types.js';
+/** Pool or in-flight client - matches the union the rest of the repo uses. */
+type Querier = pg.Pool | pg.PoolClient;
+interface MarkArgs {
+    q: Querier;
+    userId: string;
+    documentId: string;
+}
+/**
+ * Update `extraction_status` on an active row. Pass `lastError` for
+ * failure / unsupported transitions; success transitions clear the
+ * column when the prior `last_error` was scoped to the extraction
+ * layer.
+ */
+export declare function markExtractionStatus(args: MarkArgs & {
+    status: ExtractionStatus;
+    lastError?: LastError | null;
+}): Promise<void>;
+/**
+ * Update `semantic_index_status` on an active row. Same `last_error`
+ * scoping rule as {@link markExtractionStatus} - a successful
+ * indexing pass clears the column only when it was last set by the
+ * semantic-index layer.
+ */
+export declare function markSemanticIndexStatus(args: MarkArgs & {
+    status: SemanticIndexStatus;
+    lastError?: LastError | null;
+}): Promise<void>;
+/**
+ * Mark a row as `raw_storage_status='raw_storage_failed'` from a
+ * **document id** (not a storage URI). The audit fix the upload
+ * service relies on: when `store.put` throws before returning a URI
+ * the URI-keyed marker can't fire, so this doc-id form lives next to
+ * it and runs in the catch path of `uploadRawDocument` in a fresh
+ * statement after the parent transaction rolled back.
+ */
+export declare function markRawStorageFailedByDocumentId(args: MarkArgs & {
+    lastError: LastError;
+}): Promise<void>;
+/**
+ * Clear `last_error` on a row when the existing envelope is scoped to
+ * `layer`. Optional helper for retry paths that want to reset only
+ * one layer's failure marker without touching status fields.
+ */
+export declare function clearLastError(args: MarkArgs & {
+    layer: LastErrorLayer;
+}): Promise<void>;
+/**
+ * Hard cap on the persisted `last_error.message` length (UTF-16 code
+ * units, the JS string-length unit). Producer-facing failure messages
+ * vary widely - DB-driver stack traces, embedding-provider error
+ * blobs, file-extractor exceptions - and persisting them verbatim
+ * risks (a) blowing JSONB row-size budgets on operational noise and
+ * (b) leaking sensitive operational strings (URIs, keys baked into
+ * exception messages). This cap is the central knob; producers funnel
+ * through {@link buildLastError} so the cap applies uniformly to every
+ * `last_error` write.
+ *
+ * The number is sized to fit a useful human-readable detail line
+ * (e.g. one stack-frame's worth of context) without becoming a
+ * dumping ground for raw exception text.
+ */
+export declare const MAX_LAST_ERROR_MESSAGE_CHARS = 1000;
+/**
+ * Normalize a producer-supplied error message before persisting it on
+ * `raw_documents.last_error`:
+ *   * control chars (including TAB, CR, LF, and NUL) are replaced
+ *     with a single space - keeps multi-line stack traces from
+ *     breaking the UI's row layout and prevents NUL bytes from
+ *     reaching clients that mishandle them;
+ *   * runs of whitespace collapse to a single space - the message
+ *     reads as a single line of operator detail;
+ *   * the result is truncated to {@link MAX_LAST_ERROR_MESSAGE_CHARS}
+ *     code units. No truncation marker is appended; readers compare
+ *     against the cap if they want to flag truncation client-side.
+ *
+ * Exported so producer-side code paths (route handlers, services
+ * that need to build envelopes outside `buildLastError`) can apply
+ * the same sanitization rule without re-implementing it.
+ */
+export declare function sanitizeLastErrorMessage(raw: string): string;
+/**
+ * Build a `LastError` envelope. Producers should funnel through this
+ * helper so the wire / JSONB shape stays uniform AND the message is
+ * sanitized + capped (see {@link sanitizeLastErrorMessage}).
+ */
+export declare function buildLastError(layer: LastErrorLayer, code: string, message: string, occurredAt?: Date): LastError;
+export {};