@atomicmemory/core 1.0.0

package/dist/db/raw-document-status-repository.js

@@ -0,0 +1,183 @@
+/**
+ * Per-layer status repository helpers (Phase B).
+ *
+ * The audit at
+ * `Atomicmemory-research/docs/core-repo/design/document-ingest-audit.md`
+ * and the rev-18 hardening plan call for **service-owned** transitions
+ * of `extraction_status`, `semantic_index_status`, `raw_storage_status`
+ * and the `last_error` JSONB envelope. This module exposes the SQL
+ * primitives that the indexer (`document-indexer.ts`), the upload
+ * service (`document-upload.ts`), and any future async worker call to
+ * record those transitions.
+ *
+ * Design notes (cross-referenced from the plan):
+ *
+ *   * Each helper accepts `pg.Pool | pg.PoolClient` so it works inside
+ *     an in-flight transaction (e.g. the indexer's running-tx) and
+ *     from a **fresh** statement after a parent rolled back. The catch
+ *     paths in `document-upload.ts` and `document-indexer.ts` use the
+ *     fresh-statement form deliberately so the durable failure write
+ *     survives the rollback that drops the in-tx work.
+ *
+ *   * Mark helpers are scoped by `user_id` + `id` and skip soft-deleted
+ *     rows. They are **idempotent** at the SQL level - re-applying the
+ *     same status (e.g. `'failed'` -> `'failed'` with the same code)
+ *     returns the row unchanged.
+ *
+ *   * `last_error` write rule: a failure transition writes the supplied
+ *     envelope; a success transition clears `last_error` **only when
+ *     the existing envelope was scoped to the same layer**. A
+ *     successful semantic-index pass should not silently erase a
+ *     previous extraction failure that someone else still needs to
+ *     reconcile.
+ */
+/**
+ * Update `extraction_status` on an active row. Pass `lastError` for
+ * failure / unsupported transitions; success transitions clear the
+ * column when the prior `last_error` was scoped to the extraction
+ * layer.
+ */
+export async function markExtractionStatus(args) {
+    const { q, userId, documentId, status, lastError } = args;
+    const isFailure = status === 'failed' || status === 'unsupported';
+    if (isFailure && !lastError) {
+        throw new Error(`markExtractionStatus: lastError is required for status '${status}'`);
+    }
+    if (isFailure) {
+        await q.query(`UPDATE raw_documents
+          SET extraction_status = $1,
+              last_error = $2::jsonb,
+              updated_at = NOW()
+        WHERE id = $3 AND user_id = $4 AND deleted_at IS NULL`, [status, JSON.stringify(lastError), documentId, userId]);
+        return;
+    }
+    // Success / interim - selectively clear last_error.
+    await q.query(`UPDATE raw_documents
+        SET extraction_status = $1,
+            last_error = CASE
+              WHEN last_error IS NOT NULL AND last_error->>'layer' = 'extraction'
+              THEN NULL
+              ELSE last_error
+            END,
+            updated_at = NOW()
+      WHERE id = $2 AND user_id = $3 AND deleted_at IS NULL`, [status, documentId, userId]);
+}
+/**
+ * Update `semantic_index_status` on an active row. Same `last_error`
+ * scoping rule as {@link markExtractionStatus} - a successful
+ * indexing pass clears the column only when it was last set by the
+ * semantic-index layer.
+ */
+export async function markSemanticIndexStatus(args) {
+    const { q, userId, documentId, status, lastError } = args;
+    const isFailure = status === 'failed';
+    if (isFailure && !lastError) {
+        throw new Error(`markSemanticIndexStatus: lastError is required for status '${status}'`);
+    }
+    if (isFailure) {
+        await q.query(`UPDATE raw_documents
+          SET semantic_index_status = $1,
+              last_error = $2::jsonb,
+              updated_at = NOW()
+        WHERE id = $3 AND user_id = $4 AND deleted_at IS NULL`, [status, JSON.stringify(lastError), documentId, userId]);
+        return;
+    }
+    await q.query(`UPDATE raw_documents
+        SET semantic_index_status = $1,
+            last_error = CASE
+              WHEN last_error IS NOT NULL AND last_error->>'layer' = 'semantic_index'
+              THEN NULL
+              ELSE last_error
+            END,
+            updated_at = NOW()
+      WHERE id = $2 AND user_id = $3 AND deleted_at IS NULL`, [status, documentId, userId]);
+}
+/**
+ * Mark a row as `raw_storage_status='raw_storage_failed'` from a
+ * **document id** (not a storage URI). The audit fix the upload
+ * service relies on: when `store.put` throws before returning a URI
+ * the URI-keyed marker can't fire, so this doc-id form lives next to
+ * it and runs in the catch path of `uploadRawDocument` in a fresh
+ * statement after the parent transaction rolled back.
+ */
+export async function markRawStorageFailedByDocumentId(args) {
+    const { q, userId, documentId, lastError } = args;
+    await q.query(`UPDATE raw_documents
+        SET raw_storage_status = 'raw_storage_failed',
+            last_error = $1::jsonb,
+            updated_at = NOW()
+      WHERE id = $2 AND user_id = $3 AND deleted_at IS NULL`, [JSON.stringify(lastError), documentId, userId]);
+}
+/**
+ * Clear `last_error` on a row when the existing envelope is scoped to
+ * `layer`. Optional helper for retry paths that want to reset only
+ * one layer's failure marker without touching status fields.
+ */
+export async function clearLastError(args) {
+    const { q, userId, documentId, layer } = args;
+    await q.query(`UPDATE raw_documents
+        SET last_error = NULL,
+            updated_at = NOW()
+      WHERE id = $1
+        AND user_id = $2
+        AND deleted_at IS NULL
+        AND last_error IS NOT NULL
+        AND last_error->>'layer' = $3`, [documentId, userId, layer]);
+}
+/**
+ * Hard cap on the persisted `last_error.message` length (UTF-16 code
+ * units, the JS string-length unit). Producer-facing failure messages
+ * vary widely - DB-driver stack traces, embedding-provider error
+ * blobs, file-extractor exceptions - and persisting them verbatim
+ * risks (a) blowing JSONB row-size budgets on operational noise and
+ * (b) leaking sensitive operational strings (URIs, keys baked into
+ * exception messages). This cap is the central knob; producers funnel
+ * through {@link buildLastError} so the cap applies uniformly to every
+ * `last_error` write.
+ *
+ * The number is sized to fit a useful human-readable detail line
+ * (e.g. one stack-frame's worth of context) without becoming a
+ * dumping ground for raw exception text.
+ */
+export const MAX_LAST_ERROR_MESSAGE_CHARS = 1000;
+/**
+ * Normalize a producer-supplied error message before persisting it on
+ * `raw_documents.last_error`:
+ *   * control chars (including TAB, CR, LF, and NUL) are replaced
+ *     with a single space - keeps multi-line stack traces from
+ *     breaking the UI's row layout and prevents NUL bytes from
+ *     reaching clients that mishandle them;
+ *   * runs of whitespace collapse to a single space - the message
+ *     reads as a single line of operator detail;
+ *   * the result is truncated to {@link MAX_LAST_ERROR_MESSAGE_CHARS}
+ *     code units. No truncation marker is appended; readers compare
+ *     against the cap if they want to flag truncation client-side.
+ *
+ * Exported so producer-side code paths (route handlers, services
+ * that need to build envelopes outside `buildLastError`) can apply
+ * the same sanitization rule without re-implementing it.
+ */
+export function sanitizeLastErrorMessage(raw) {
+    // Replace any character at code point < 0x20 OR DEL (0x7f) with a
+    // single space. Covers ASCII control chars including \r, \n, \t,
+    // \0; preserves all printable / non-control Unicode glyphs.
+    // eslint-disable-next-line no-control-regex
+    const stripped = raw.replace(/[\x00-\x1f\x7f]/g, ' ');
+    const collapsed = stripped.replace(/ {2,}/g, ' ').trim();
+    if (collapsed.length <= MAX_LAST_ERROR_MESSAGE_CHARS)
+        return collapsed;
+    return collapsed.slice(0, MAX_LAST_ERROR_MESSAGE_CHARS);
+}
+/**
+ * Build a `LastError` envelope. Producers should funnel through this
+ * helper so the wire / JSONB shape stays uniform AND the message is
+ * sanitized + capped (see {@link sanitizeLastErrorMessage}).
+ */
+export function buildLastError(layer, code, message, occurredAt = new Date()) {
+    return {
+        layer,
+        code,
+        message: sanitizeLastErrorMessage(message),
+        occurred_at: occurredAt.toISOString(),
+    };
+}

package/dist/db/raw-document-types.d.ts

@@ -0,0 +1,236 @@
+/**
+ * Row + input types for the document pipeline (Phase 1).
+ *
+ * Mirrors the columns defined in `schema.sql` for `raw_sources` and
+ * `raw_documents`. The CHECK constraints in SQL accept the full enum
+ * range so later phases (managed_blob, inline_small_text) can populate
+ * those values without a schema change; Phase 1 service-layer code
+ * still only writes `storage_mode = 'pointer_only'`.
+ */
+export type RawStorageMode = 'pointer_only' | 'managed_blob' | 'inline_small_text';
+export type RegistrationStatus = 'registered' | 'registration_failed';
+export type RawStorageStatus = 'pointer_recorded' | 'blob_stored' | 'inline_text_stored' | 'raw_storage_failed'
+/**
+ * Terminal state for a tombstoned managed-blob row whose bytes were
+ * successfully deleted from the configured `RawContentStore`. Set by
+ * the Phase-3 cleanup path *after* the soft-delete and *after* the
+ * adapter's delete call returned; lets retries of `DELETE
+ * /v1/documents/:id` distinguish "blob is gone" from
+ * "blob still needs cleanup" (raw_storage_failed).
+ */
+ | 'blob_deleted'
+/**
+ * Filecoin lifecycle refactor (Slice 2): the provider accepted the
+ * upload but storage / retrievability is not yet confirmed. The
+ * upload service writes this when the adapter's `put()` returned
+ * `status: 'pending'` (an eventual provider). The future Phase 3
+ * reconciliation worker polls `head()` and promotes the row to
+ * `blob_available` (success) or `blob_archival_failed` (permanent
+ * failure).
+ */
+ | 'blob_pending'
+/**
+ * Reserved for the Phase 3 reconciliation worker. No Phase-1
+ * writer; the schema and the union just have to accept it so the
+ * Phase 3 worker can flip a row from `blob_pending` once `head()`
+ * confirms retrievability.
+ */
+ | 'blob_available'
+/**
+ * Reserved for the Phase 3 reconciler's permanent-failure path. No
+ * Phase-1 writer.
+ */
+ | 'blob_archival_failed'
+/**
+ * Reserved for Phase 2 Filecoin deletes against providers that
+ * support unpin-only semantics. No Phase-1 writer.
+ */
+ | 'blob_tombstoned'
+/**
+ * Filecoin lifecycle refactor (Phase 5) — transient state during
+ * the upload pipeline's α/β/β2/γ split. Phase α writes this with a
+ * `raw_storage_claim_id` after seizing the slot; Phase γ flips it
+ * to the final terminal state (`blob_stored` / `blob_pending` /
+ * `blob_available`) once the adapter returns. A row that stays in
+ * `blob_uploading` past a process restart is recoverable via
+ * same-bytes idempotent retry of `uploadRaw`; the Phase 6
+ * reconciler does NOT process `blob_uploading` rows.
+ */
+ | 'blob_uploading';
+/**
+ * Phase B (document-ingest hardening) — text-extraction layer status.
+ * Mirrors the values in `raw_documents.extraction_status` (CHECK
+ * constraint in `schema.sql`).
+ */
+export type ExtractionStatus = 'not_required' | 'pending' | 'running' | 'complete' | 'unsupported' | 'failed';
+/**
+ * Phase B — semantic indexing layer status (chunk + embed + store).
+ * Mirrors `raw_documents.semantic_index_status`. Note that `'running'`
+ * is internally written inside the indexer's BEGIN..COMMIT
+ * transaction; under READ COMMITTED isolation other connections do
+ * NOT observe it. The state lives on the row only as a CAS marker;
+ * UI rendering of "indexing in progress" requires a future
+ * async-worker design that commits `'running'` before doing the
+ * work, with a lease/heartbeat for crash recovery.
+ */
+export type SemanticIndexStatus = 'not_required' | 'pending' | 'running' | 'complete' | 'failed' | 'stale';
+/** Layer that produced the most-recent failure recorded on a row. */
+export type LastErrorLayer = 'raw_storage' | 'extraction' | 'semantic_index';
+/**
+ * Phase B — most-recent failure envelope persisted on `raw_documents.last_error`.
+ *
+ * Single most-recent failure per row; cleared on the next successful
+ * transition for that layer. Snake_case at the storage layer and on
+ * the wire so JSONB columns and HTTP responses share one shape.
+ */
+export interface LastError {
+    layer: LastErrorLayer;
+    /** Bounded enum chosen by the producing layer (e.g. `'managed_storage_disabled'`, `'index_text_too_large'`). */
+    code: string;
+    /** Human-readable detail. Producers should truncate to a sensible cap. */
+    message: string;
+    /** ISO 8601 UTC timestamp. */
+    occurred_at: string;
+}
+/**
+ * Persisted shape of a `raw_sources` row. `account_id` is nullable for
+ * sources that do not have a per-account scoping concept (e.g. manual
+ * webapp uploads).
+ */
+export interface RawSourceRow {
+    id: string;
+    userId: string;
+    sourceSite: string;
+    provider: string;
+    accountId: string | null;
+    storageMode: RawStorageMode;
+    retentionPolicy: Record<string, unknown>;
+    consentPolicy: Record<string, unknown>;
+    createdAt: Date;
+    updatedAt: Date;
+}
+export interface UpsertRawSourceInput {
+    userId: string;
+    sourceSite: string;
+    provider: string;
+    accountId?: string | null;
+    storageMode?: RawStorageMode;
+    retentionPolicy?: Record<string, unknown>;
+    consentPolicy?: Record<string, unknown>;
+}
+/**
+ * Persisted shape of a `raw_documents` row. Phase 1 callers see
+ * `storageUri = null`, `storageProvider = null`, and
+ * `rawStorageStatus = 'pointer_recorded'` for every active row.
+ */
+export interface RawDocumentRow {
+    id: string;
+    userId: string;
+    rawSourceId: string;
+    externalId: string;
+    externalUri: string | null;
+    displayName: string | null;
+    mimeType: string | null;
+    sizeBytes: number | null;
+    contentHash: string | null;
+    providerVersion: string | null;
+    sourceModifiedAt: Date | null;
+    storageMode: RawStorageMode;
+    storageUri: string | null;
+    storageProvider: string | null;
+    registrationStatus: RegistrationStatus;
+    rawStorageStatus: RawStorageStatus;
+    /**
+     * Provider-side identifiers for the managed blob — CID, piece CID,
+     * deal id, onramp request id, gateway URL, etc. Set by the upload
+     * service from `StoredRawContent.providerMetadata`. Opaque to the
+     * upload pipeline; surfaced verbatim on the wire as
+     * `raw_storage_metadata`. Defaults to `{}` for pointer-only rows
+     * and for immediate providers (local_fs / s3) that don't yet
+     * populate the field.
+     */
+    rawStorageMetadata: Record<string, unknown>;
+    metadata: Record<string, unknown>;
+    createdAt: Date;
+    updatedAt: Date;
+    deletedAt: Date | null;
+    /**
+     * Hash of the text last fed through `POST /v1/documents/:id/index`.
+     * Distinct from `contentHash` (the upstream/provider raw-content
+     * fingerprint). NULL until first indexed. Used by the Phase 2
+     * indexer's idempotency check.
+     */
+    indexedContentHash: string | null;
+    indexedAt: Date | null;
+    /**
+     * Phase B per-layer status fields. Default `'not_required'` for
+     * legacy rows that predate the columns; new registrations
+     * targeting the document pipeline set safe initial states
+     * (`'pending'`) at registration time.
+     */
+    extractionStatus: ExtractionStatus;
+    semanticIndexStatus: SemanticIndexStatus;
+    /** Most-recent failure across any layer; null when no layer is currently failed. */
+    lastError: LastError | null;
+    /**
+     * Phase 5 / Phase 6 private worker state. Holds the upload-pipeline
+     * claim during α/β/β2/γ and the reconciler claim during the
+     * blob_pending → blob_available promotion. These columns are
+     * INTERNAL — `formatRawDocument` and `formatPublicRawStorageMetadata`
+     * do NOT project them onto the wire; the upload service and the
+     * reconciler are the only readers/writers.
+     */
+    rawStorageClaimId: string | null;
+    rawStorageClaimedAt: Date | null;
+    rawStorageLastCheckedAt: Date | null;
+    rawStorageNextCheckAt: Date | null;
+    rawStorageReconcileAttempts: number;
+    /**
+     * Durable "row entered blob_pending at" timestamp; the observability
+     * layer reads this for the `pending_age_seconds` metric. Set by
+     * Phase γ when writing `blob_pending`; cleared on terminal
+     * transitions out of pending (blob_available, blob_archival_failed)
+     * by the reconciler.
+     */
+    rawStoragePendingSince: Date | null;
+    /**
+     * Step 7 of the storage-sibling plan — FK to the row's paired
+     * `storage_artifacts` entry. NULL for rows that pre-date Step 7
+     * and for registration stubs (no `external_uri`, no managed
+     * upload yet). The composite FK on
+     * `(storage_artifact_id, user_id)` makes cross-user links
+     * impossible at the persistence layer.
+     */
+    storageArtifactId: string | null;
+}
+export interface RegisterRawDocumentInput {
+    userId: string;
+    rawSourceId: string;
+    externalId: string;
+    externalUri?: string | null;
+    displayName?: string | null;
+    mimeType?: string | null;
+    sizeBytes?: number | null;
+    contentHash?: string | null;
+    providerVersion?: string | null;
+    sourceModifiedAt?: Date | null;
+    storageMode?: RawStorageMode;
+    metadata?: Record<string, unknown>;
+    /**
+     * Phase B — restricted-initial-state status fields. Clients may
+     * declare `'pending'` (document pipeline expected to extract +
+     * index this row), `'not_required'` (default; pointer-only flow),
+     * or `'unsupported'` (extraction layer only — for known
+     * non-extractable file types). Service-owned transitions handle
+     * `'complete'` / `'failed'` / `'running'`; clients that supply
+     * those values are rejected at the schema layer.
+     */
+    extractionStatus?: 'pending' | 'not_required' | 'unsupported';
+    semanticIndexStatus?: 'pending' | 'not_required';
+}
+export interface ListRawDocumentsInput {
+    userId: string;
+    sourceSite?: string;
+    limit?: number;
+    offset?: number;
+}

package/dist/db/raw-document-types.js

@@ -0,0 +1,10 @@
+/**
+ * Row + input types for the document pipeline (Phase 1).
+ *
+ * Mirrors the columns defined in `schema.sql` for `raw_sources` and
+ * `raw_documents`. The CHECK constraints in SQL accept the full enum
+ * range so later phases (managed_blob, inline_small_text) can populate
+ * those values without a schema change; Phase 1 service-layer code
+ * still only writes `storage_mode = 'pointer_only'`.
+ */
+export {};

package/dist/db/raw-storage-reconciliation-repository.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Raw-storage reconciler DB layer. The reconciler promotes
+ * `raw_storage_status='blob_pending'` rows on eventual providers once the
+ * adapter's `head()` confirms gateway retrievability (→
+ * `blob_available`) or, after exhausted retries, marks them
+ * `blob_archival_failed`.
+ *
+ * Concurrency model (rev-2 §3 + rev-6 §1): claim-then-release. Phase
+ * A grabs a batch in a short transaction with `FOR UPDATE SKIP
+ * LOCKED` so multiple reconciler instances partition the work
+ * deterministically without coordinator overhead. Slow
+ * network calls run with NO DB locks held. failure-transition's per-row UPDATE
+ * is guarded on `raw_storage_claim_id` so a stale claim cannot
+ * clobber a row another worker has since reclaimed.
+ *
+ * Eligibility predicate gates on TYPED columns only — no JSONB casts
+ * (rev-6 §2): a malformed JSONB shape can't masquerade as a fresh
+ * claim. The predicate also explicitly excludes `blob_uploading` rows
+ * — those recover through `uploadRaw`'s same-bytes idempotent retry,
+ * not the reconciler.
+ */
+import pg from 'pg';
+/** One row's worth of state the reconciler needs to probe + update. */
+export interface ReconcilerClaimedRow {
+    id: string;
+    userId: string;
+    storageUri: string;
+    storageProvider: string;
+    contentHash: string | null;
+    rawStorageStatus: 'blob_pending';
+    rawStorageMetadata: Record<string, unknown>;
+    rawStorageReconcileAttempts: number;
+    /**
+     * `raw_storage_pending_since` carried through so the reconciler
+     * can emit the `pending_age_seconds` observability gauge against
+     * the durable lifecycle timestamp (rev-8 §7). NULL only on rows
+     * seeded by a path that bypassed the finalization step — production rows always
+     * carry it.
+     */
+    rawStoragePendingSince: Date | null;
+    /**
+     * True when the claim batch reclaimed a row whose previous
+     * `raw_storage_claim_id` was non-NULL (an earlier worker died /
+     * crashed). The `filecoin.reconcile.stale_claim_recovered`
+     * event uses this to differentiate fresh claims from recoveries.
+     */
+    recoveredStaleClaim: boolean;
+}
+export interface ClaimBatchArgs {
+    claimId: string;
+    batchSize: number;
+    staleAfterMs: number;
+    provider: string;
+}
+/**
+ * claim step: claim a batch of eligible eventual-provider `blob_pending` rows by
+ * writing `raw_storage_claim_id` + `raw_storage_claimed_at`. Skips
+ * rows another worker holds a fresh claim on. Returns the typed row
+ * state the reconciler will probe; the network call runs after this
+ * function returns + the caller commits the transaction.
+ *
+ * Eligibility predicate (rev-7 §3 — two-axis: claim ownership AND
+ * scheduled backoff):
+ *   - `raw_storage_status = 'blob_pending'`
+ *   - `storage_provider = args.provider`
+ *   - `storage_uri IS NOT NULL` (rev-7 §1: the claim step-only rows with no
+ *     URI recover via uploadRaw idempotency, NOT the reconciler)
+ *   - `deleted_at IS NULL`
+ *   - backoff elapsed: `next_check_at IS NULL OR next_check_at <= NOW()`
+ *   - no live owner: claim_id NULL OR claimed_at older than staleAfterMs
+ */
+export declare function claimReconcileBatch(pool: pg.Pool, args: ClaimBatchArgs): Promise<ReadonlyArray<ReconcilerClaimedRow>>;
+/**
+ * failure-transition — success: promote to `blob_available`. Guarded on claim_id
+ * + status='blob_pending' so a stale claim cannot promote a row
+ * another worker has since failed. Deep-merges the new
+ * provider keys into the existing sibling (rev-4 §3) so a status-
+ * only probe doesn't drop `cid`/`piece_cid`/`deals` the original
+ * upload wrote. Layer-scoped `last_error` clear (rev-4 §2): drops
+ * the envelope only when it was scoped to `raw_storage`.
+ */
+export declare function promoteToAvailableWithClient(pool: pg.Pool | pg.PoolClient, args: {
+    rowId: string;
+    claimId: string;
+    provider: string;
+    providerFields: Record<string, unknown>;
+}): Promise<number>;
+/**
+ * failure-transition — still-pending probe: clear the claim (release ownership)
+ * + increment attempts + advance `next_check_at` per the
+ * caller-computed backoff. Status STAYS `blob_pending` and
+ * `pending_since` is preserved.
+ */
+export declare function markStillPendingWithClient(pool: pg.Pool, args: {
+    rowId: string;
+    claimId: string;
+    nextCheckAtMs: number;
+    provider: string;
+}): Promise<number>;
+/**
+ * failure-transition — terminal failure: status → `blob_archival_failed`. Sets
+ * a fresh raw-storage `last_error` envelope, clears claim/pending
+ * state, resets attempts. Guarded on claim_id.
+ */
+export declare function markArchivalFailedWithClient(pool: pg.Pool | pg.PoolClient, args: {
+    rowId: string;
+    claimId: string;
+    lastError: Record<string, unknown>;
+    provider: string;
+}): Promise<number>;