@atomicmemory/core 1.0.0

package/dist/db/raw-doc-artifact-sync.js

@@ -0,0 +1,259 @@
+/**
+ * @file Sync hook keeping `storage_artifacts.status` in lockstep
+ * with `raw_documents.raw_storage_status`.
+ *
+ * Step 7 of the storage-sibling plan. Every existing raw-storage
+ * transition (upload finalize, reconciler success/failure, cleanup
+ * success/failure) calls this hook after mutating `raw_documents`
+ * so the paired `storage_artifacts` row tracks the same lifecycle.
+ *
+ * v1 keeps `raw_documents` as the source of truth for document-
+ * backed uploads — the artifact row is a synchronized projection.
+ * The follow-up Webapp Storage UI PR (source plan Phase 8) makes
+ * `storage_artifacts` the source of truth and switches the document
+ * formatter to a join-projection. Until then, the helper guarantees
+ * both rows agree.
+ *
+ * Status mapping (plan rev 6 §Step 7):
+ *
+ *   pointer_recorded (with external_uri / pointer artifact) → 'stored'
+ *   pointer_recorded (no artifact linked)                   → no-op
+ *   blob_stored                                              → 'stored'
+ *   blob_pending                                             → 'pending'
+ *   blob_uploading                                           → 'pending' (transient)
+ *   blob_available                                           → 'available'
+ *   blob_archival_failed                                     → 'failed'
+ *   raw_storage_failed                                       → 'failed'
+ *   blob_tombstoned                                          → 'deleted'
+ *   blob_deleted                                             → 'deleted'
+ *   inline_text_stored                                       → no-op (inline; no artifact)
+ */
+import { markArchivalFailedWithClient, promoteToAvailableWithClient, } from './raw-storage-reconciliation-repository.js';
+import { buildLastError } from './raw-document-status-repository.js';
+/**
+ * Canonical raw_storage cleanup-failure envelope. Persisted to both
+ * `raw_documents.last_error` (via `markCleanupFailedAndSyncArtifact`)
+ * and the linked `storage_artifacts.last_error`. The provider's raw
+ * exception message is funneled through `buildLastError` so the
+ * central `sanitizeLastErrorMessage` cap (1 KiB) and control-char
+ * collapse apply uniformly — multi-line stack traces, sensitive
+ * URLs in error blobs, and oversized provider envelopes never reach
+ * the JSONB column unfiltered. The `storage_provider` field is
+ * appended after sanitization so the envelope shape stays
+ * additive on top of the canonical `LastError`.
+ */
+export function buildRawStorageCleanupFailureEnvelope(message, storageProvider) {
+    return {
+        ...buildLastError('raw_storage', 'managed_blob_cleanup_failed', message),
+        storage_provider: storageProvider,
+    };
+}
+const STATUS_MAP = {
+    pointer_recorded: 'stored',
+    blob_stored: 'stored',
+    blob_pending: 'pending',
+    blob_uploading: 'pending',
+    blob_available: 'available',
+    blob_archival_failed: 'failed',
+    raw_storage_failed: 'failed',
+    blob_tombstoned: 'deleted',
+    blob_deleted: 'deleted',
+};
+/**
+ * Map a single `raw_documents` row's new status onto its linked
+ * `storage_artifacts` row. Resolves the link inside the same query
+ * pair (no separate read) so the hook is safe to call inside the
+ * existing transactions that wrap the raw-storage transitions.
+ *
+ * No-op (silently) when:
+ *   - the document is not linked to an artifact (`storage_artifact_id IS NULL`),
+ *   - the new raw status maps to no artifact state (e.g. `inline_text_stored`).
+ *
+ * The function deliberately does NOT scope by `user_id` — the
+ * caller has already vouched for the row by holding the row's
+ * advisory lock or having authenticated the request that triggered
+ * the transition.
+ */
+export async function syncArtifactStatusFromRawDocument(client, args) {
+    const mapped = STATUS_MAP[args.newRawStatus];
+    if (mapped === undefined)
+        return;
+    const lookup = await client.query(`SELECT storage_artifact_id FROM raw_documents WHERE id = $1`, [args.rawDocumentId]);
+    if (lookup.rowCount === 0)
+        return;
+    const artifactId = lookup.rows[0].storage_artifact_id;
+    if (artifactId === null)
+        return;
+    await applyArtifactTransition(client, artifactId, mapped, args.lastError);
+}
+/**
+ * Reconciler paired promote: flip the document to `blob_available`
+ * AND sync the artifact in one transaction. Two-row drift is
+ * impossible — either both rows commit or both roll back.
+ */
+export async function promoteAndSyncArtifact(pool, args) {
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const rowCount = await promoteToAvailableWithClient(client, args);
+        if (rowCount === 0) {
+            await client.query('ROLLBACK');
+            return 0;
+        }
+        await syncArtifactStatusFromRawDocument(client, {
+            rawDocumentId: args.rowId,
+            newRawStatus: 'blob_available',
+        });
+        await client.query('COMMIT');
+        return rowCount;
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => undefined);
+        throw err;
+    }
+    finally {
+        client.release();
+    }
+}
+/**
+ * Reconciler paired archival-fail: flip the document to
+ * `blob_archival_failed` AND sync the artifact (with the
+ * `last_error` envelope) in one transaction.
+ */
+export async function markArchivalFailedAndSyncArtifact(pool, args) {
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const rowCount = await markArchivalFailedWithClient(client, args);
+        if (rowCount === 0) {
+            await client.query('ROLLBACK');
+            return 0;
+        }
+        await syncArtifactStatusFromRawDocument(client, {
+            rawDocumentId: args.rowId,
+            newRawStatus: 'blob_archival_failed',
+            lastError: args.lastError,
+        });
+        await client.query('COMMIT');
+        return rowCount;
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => undefined);
+        throw err;
+    }
+    finally {
+        client.release();
+    }
+}
+/**
+ * Document-keyed paired marker for the cleanup path. Each blob
+ * cleanup carries its `rawDocumentId` (see
+ * `ManagedBlobRefRow.rawDocumentId`) so the sync never needs to
+ * disambiguate by URI. The raw_documents UPDATE is owner-scoped;
+ * if zero rows match (cross-user mismatch or already-terminal),
+ * the artifact sync is skipped — preventing a mismatched caller
+ * from flipping another user's artifact.
+ */
+export async function markCleanupSuccessAndSyncArtifact(pool, args) {
+    const newRawStatus = args.semantics === 'deleted' ? 'blob_deleted' : 'blob_tombstoned';
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const result = await client.query(`UPDATE raw_documents
+          SET raw_storage_status = $1, updated_at = NOW()
+        WHERE user_id = $2 AND id = $3`, [newRawStatus, args.userId, args.rawDocumentId]);
+        if ((result.rowCount ?? 0) === 0) {
+            // Cross-user mismatch or row missing — leave the artifact
+            // untouched. The link belongs to another user (or no row);
+            // mirroring through id-only would clobber theirs.
+            await client.query('ROLLBACK');
+            return;
+        }
+        await syncArtifactStatusFromRawDocument(client, {
+            rawDocumentId: args.rawDocumentId,
+            newRawStatus,
+        });
+        await client.query('COMMIT');
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => undefined);
+        throw err;
+    }
+    finally {
+        client.release();
+    }
+}
+/**
+ * Document-keyed paired failure marker. Mirrors
+ * `markCleanupSuccessAndSyncArtifact` but for the failure branch
+ * (`raw_storage_failed`). Same owner-scope gating.
+ *
+ * Retry-friendly: the UPDATE matches on `(user_id, id)` only — a row
+ * that is ALREADY `raw_storage_failed` is re-marked so a fresh
+ * provider error replaces the prior envelope on both
+ * `raw_documents.last_error` AND the linked
+ * `storage_artifacts.last_error`. Mirrors the failure-marker pattern
+ * elsewhere in the codebase, where retries land the newest, most-
+ * useful error rather than discarding it. Cross-user mismatches
+ * still roll back via the owner-scoped WHERE.
+ */
+export async function markCleanupFailedAndSyncArtifact(pool, args) {
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const hasError = args.lastError !== undefined;
+        const result = await client.query(`UPDATE raw_documents
+          SET raw_storage_status = 'raw_storage_failed',
+              last_error = CASE WHEN $3::boolean THEN $4::jsonb ELSE last_error END,
+              updated_at = NOW()
+        WHERE user_id = $1 AND id = $2`, [
+            args.userId,
+            args.rawDocumentId,
+            hasError,
+            hasError ? JSON.stringify(args.lastError) : null,
+        ]);
+        if ((result.rowCount ?? 0) === 0) {
+            await client.query('ROLLBACK');
+            return;
+        }
+        await syncArtifactStatusFromRawDocument(client, {
+            rawDocumentId: args.rawDocumentId,
+            newRawStatus: 'raw_storage_failed',
+            lastError: args.lastError,
+        });
+        await client.query('COMMIT');
+    }
+    catch (err) {
+        await client.query('ROLLBACK').catch(() => undefined);
+        throw err;
+    }
+    finally {
+        client.release();
+    }
+}
+async function applyArtifactTransition(client, artifactId, mapped, lastError) {
+    if (mapped === 'deleted') {
+        await client.query(`UPDATE storage_artifacts
+         SET status = 'deleted',
+             deleted_at = COALESCE(deleted_at, NOW()),
+             delete_attempt_id = NULL,
+             updated_at = NOW()
+         WHERE id = $1`, [artifactId]);
+        return;
+    }
+    if (mapped === 'failed') {
+        await client.query(`UPDATE storage_artifacts
+         SET status = 'failed',
+             last_error = COALESCE($2::jsonb, last_error),
+             updated_at = NOW()
+         WHERE id = $1`, [artifactId, lastError === undefined ? null : JSON.stringify(lastError)]);
+        return;
+    }
+    // Success transitions clear last_error so a retry path doesn't
+    // surface a stale envelope alongside `status='available'`.
+    await client.query(`UPDATE storage_artifacts
+       SET status = $2,
+           last_error = NULL,
+           updated_at = NOW()
+       WHERE id = $1`, [artifactId, mapped]);
+}

package/dist/db/raw-document-blob-repository.d.ts

@@ -0,0 +1,148 @@
+/**
+ * Phase-3 managed-blob helpers for the document pipeline.
+ *
+ * Split out of `raw-document-repository.ts` to keep that module under
+ * the workspace 400-LOC cap as the managed-blob surface grew. This
+ * module owns every read/update that touches the
+ * `storage_mode='managed_blob'` slice of `raw_documents`:
+ *
+ *   - Lookups for blob URIs that need cleanup
+ *     (`listManagedBlobs*`, `listOrphanedManagedBlobs*`).
+ *   - The post-upload row-promotion helper
+ *     (`updateRawDocumentBlobStorageWithClient`).
+ *
+ *   Cleanup state-machine markers used to live here as URI-keyed
+ *   updates; the paired artifact-sync implementation replaced them with the id-keyed paired helpers
+ *   in `db/raw-doc-artifact-sync.ts`.
+ *
+ * The base registry CRUD (insert / list / get / soft-delete /
+ * source-site lookups) and the indexer hash setter stay in
+ * `raw-document-repository.ts` so this module can stay tightly
+ * scoped to the managed-blob lifecycle.
+ *
+ * No row-mapping helpers are needed here — every function returns
+ * `(storage_provider, storage_uri)` tuples or executes UPDATEs, so
+ * the snake_case → camelCase mapping the registry CRUD needs lives
+ * with the registry helpers, not here.
+ */
+import pg from 'pg';
+/** Shape returned by every managed-blob lookup in this module.
+ * `rawDocumentId` lets the cleanup loop sync the linked
+ * `storage_artifacts` row by id — URIs are not globally unique
+ * across documents, so id-keyed sync removes the ambiguity.
+ * `rawStorageMetadata` carries the provider-specific sidecar so
+ * `cleanupManagedBlobs` can pass it as opaque hints to
+ * `RawContentStore.delete` (e.g. the Filecoin adapter uses
+ * `filecoin.copies[].piece_id` + `data_set_id` to delete a
+ * freshly-uploaded piece without relying on the SDK's
+ * CID→active-piece lookup). Non-Filecoin adapters ignore it. */
+export interface ManagedBlobRefRow {
+    rawDocumentId: string;
+    storageProvider: string;
+    storageUri: string;
+    rawStorageMetadata: Record<string, unknown>;
+}
+/**
+ * `(storage_provider, storage_uri)` tuples for the *active* managed-blob
+ * row of a single document. Caller hands the result to
+ * `cleanupManagedBlobs()` *after* the DB transaction commits. Active-only
+ * (deleted_at IS NULL) — soft-deleted rows are skipped to avoid
+ * double-deleting blobs after a prior cascade.
+ */
+export declare function listManagedBlobsForDocumentWithClient(client: pg.Pool | pg.PoolClient, userId: string, documentId: string): Promise<ManagedBlobRefRow[]>;
+/**
+ * Source-scoped sibling of `listManagedBlobsForDocumentWithClient`.
+ * Used by `deleteBySource` so source-reset can hand a complete blob
+ * list to the post-commit cleanup pass.
+ */
+export declare function listManagedBlobsBySourceWithClient(client: pg.PoolClient, userId: string, sourceSite: string): Promise<ManagedBlobRefRow[]>;
+/**
+ * Active managed-blob URIs for a user (`userId` set) or globally
+ * (`userId` undefined). Used by `deleteAll`'s pre-hard-delete cleanup
+ * hook so the wipe can flush the blob store before dropping the rows.
+ */
+export declare function listManagedBlobsForUser(pool: pg.Pool, userId?: string): Promise<ManagedBlobRefRow[]>;
+/**
+ * Retry-safe lookup: managed-blob URIs that need cleanup *for a
+ * soft-deleted document* whose prior cleanup pass failed. Consulted by
+ * a retry of `DELETE /v1/documents/:id` so it can re-attempt
+ * `store.delete()` on the orphan instead of returning
+ * `already_deleted=true` while the blob is still present.
+ *
+ * Filters: `deleted_at IS NOT NULL` (only soft-deleted rows), and
+ * `raw_storage_status = 'raw_storage_failed'` (only the rows whose
+ * cleanup failed — `blob_deleted` rows are already clean).
+ */
+export declare function listOrphanedManagedBlobsForDocument(pool: pg.Pool, userId: string, documentId: string): Promise<ManagedBlobRefRow[]>;
+/**
+ * Source-scoped sibling of `listOrphanedManagedBlobsForDocument`. Used
+ * by the retry path of `POST /v1/memories/reset-source` so a second
+ * reset-by-source after a partially-failed cleanup can still find the
+ * orphan blobs to retry on.
+ */
+export declare function listOrphanedManagedBlobsBySource(pool: pg.Pool, userId: string, sourceSite: string): Promise<ManagedBlobRefRow[]>;
+/**
+ * the claim step: short-tx slot claim. Writes `raw_storage_status =
+ * 'blob_uploading'` + claim_id + claimed_at + plaintext content_hash +
+ * plaintext size_bytes.
+ *
+ * `clearDurableUri` (rev-fix HIGH 2) selects between the two
+ * recovery paths:
+ *   - `true` (reclaimAndUpload): the row may carry a stale URI /
+ *     provider / metadata from a prior failed attempt — clear them
+ *     so a crash before the durable URI-write step cannot leave a finalize-recovery
+ *     window pointing at the old bytes. `raw_storage_pending_since`
+ *     is cleared too (the row is leaving the pending state).
+ *   - `false` (finalize-recovery): preserve `storage_uri`,
+ *     `storage_provider`, and `raw_storage_metadata`. the finalization step alone
+ *     promotes the row; β + β2 are skipped because the bytes are
+ *     already durable on the provider.
+ */
+export declare function claimUploadSlotWithClient(client: pg.PoolClient, args: {
+    userId: string;
+    documentId: string;
+    claimId: string;
+    contentHash: string;
+    sizeBytes: number;
+    clearDurableUri: boolean;
+}): Promise<number>;
+/**
+ * the durable URI-write step: short-tx durable URI record. Writes storage_mode,
+ * storage_uri, storage_provider, raw_storage_metadata. Status STAYS
+ * `blob_uploading` — the finalization step flips it. Guarded by claim_id so a
+ * stale claim's β2 cannot clobber a row another worker has since
+ * reclaimed. Returns rowCount; caller compensates with a delete on 0.
+ */
+export declare function recordUploadResultWithClient(client: pg.PoolClient | pg.Pool, args: {
+    userId: string;
+    documentId: string;
+    claimId: string;
+    storageUri: string;
+    storageProvider: string;
+    rawStorageMetadata: Record<string, unknown>;
+}): Promise<number>;
+/**
+ * the finalization step: compare-and-set status flip. Writes the final
+ * raw_storage_status, sets pending_since when entering blob_pending,
+ * clears the claim, and layer-scopes the last_error clear (drops
+ * raw_storage envelopes only). Guarded on claim_id + storage_uri NOT
+ * NULL. Returns rowCount.
+ */
+export declare function finalizeUploadStatusWithClient(client: pg.PoolClient | pg.Pool, args: {
+    userId: string;
+    documentId: string;
+    claimId: string;
+    finalStatus: 'blob_stored' | 'blob_pending' | 'blob_available';
+}): Promise<number>;
+/**
+ * the provider-write step failure-path: clear an active claim and flip the row to
+ * raw_storage_failed with a sanitized last_error envelope. Guarded
+ * by claim_id so a stale claim's failure marker cannot overwrite a
+ * subsequent successful retry. Returns rowCount.
+ */
+export declare function failClaimedUploadWithClient(client: pg.PoolClient | pg.Pool, args: {
+    userId: string;
+    documentId: string;
+    claimId: string;
+    lastError: Record<string, unknown>;
+}): Promise<number>;