@atomicmemory/core 1.0.0

package/dist/services/storage-key-prefix.d.ts

@@ -0,0 +1,42 @@
+/**
+ * @file PII-safe storage-key prefix derivation.
+ *
+ * The storage-sibling plan removed plaintext `users/${userId}/...`
+ * path segments from every provider key/URI we hand to a backend.
+ * Replacement: an HMAC-SHA256 prefix that is:
+ *
+ *   - deterministic (same `(secret, userId)` → same 32-hex prefix),
+ *     so retries / same-bytes re-uploads still collide on the same
+ *     key and the idempotency contract holds end-to-end;
+ *   - non-reversible (HMAC of a server-side secret), so an
+ *     operator scanning backend listings cannot recover the
+ *     `user_id` from a key — addresses the PII leak that motivated
+ *     this commit;
+ *   - per-user (different `userId` derives a different prefix), so
+ *     ownership is still visually separable in the backend listing
+ *     (one prefix = one user).
+ *
+ * Output shape: the leading 32 hex chars (16 bytes) of the HMAC.
+ * 16 bytes is enough collision resistance — birthday at 2^64 over
+ * the user-space — while keeping keys short. The `s/<hex32>/`
+ * route is the only consumer; do NOT slice differently elsewhere.
+ *
+ * The secret is required at startup via
+ * `RuntimeConfig.storageKeyHmacSecret`; callers thread it through
+ * the service constructors. NEVER read `process.env` directly here.
+ */
+/**
+ * Length of the hex slice we keep as the per-user prefix.
+ * Workspace constant — do not change without a key-migration plan
+ * (existing keys would no longer collide with their re-upload).
+ */
+export declare const STORAGE_KEY_PREFIX_LENGTH = 32;
+/**
+ * Derive the per-user HMAC prefix used at the head of every
+ * managed-storage key. Stable across retries; non-reversible.
+ *
+ * `secret` is the deployment-wide `STORAGE_KEY_HMAC_SECRET`. The
+ * caller MUST source it from `RuntimeConfig.storageKeyHmacSecret`
+ * (validated at startup as >=64 hex chars); never read env here.
+ */
+export declare function deriveStorageKeyPrefix(secret: string, userId: string): string;

package/dist/services/storage-key-prefix.js

@@ -0,0 +1,45 @@
+/**
+ * @file PII-safe storage-key prefix derivation.
+ *
+ * The storage-sibling plan removed plaintext `users/${userId}/...`
+ * path segments from every provider key/URI we hand to a backend.
+ * Replacement: an HMAC-SHA256 prefix that is:
+ *
+ *   - deterministic (same `(secret, userId)` → same 32-hex prefix),
+ *     so retries / same-bytes re-uploads still collide on the same
+ *     key and the idempotency contract holds end-to-end;
+ *   - non-reversible (HMAC of a server-side secret), so an
+ *     operator scanning backend listings cannot recover the
+ *     `user_id` from a key — addresses the PII leak that motivated
+ *     this commit;
+ *   - per-user (different `userId` derives a different prefix), so
+ *     ownership is still visually separable in the backend listing
+ *     (one prefix = one user).
+ *
+ * Output shape: the leading 32 hex chars (16 bytes) of the HMAC.
+ * 16 bytes is enough collision resistance — birthday at 2^64 over
+ * the user-space — while keeping keys short. The `s/<hex32>/`
+ * route is the only consumer; do NOT slice differently elsewhere.
+ *
+ * The secret is required at startup via
+ * `RuntimeConfig.storageKeyHmacSecret`; callers thread it through
+ * the service constructors. NEVER read `process.env` directly here.
+ */
+import { createHmac } from 'node:crypto';
+/**
+ * Length of the hex slice we keep as the per-user prefix.
+ * Workspace constant — do not change without a key-migration plan
+ * (existing keys would no longer collide with their re-upload).
+ */
+export const STORAGE_KEY_PREFIX_LENGTH = 32;
+/**
+ * Derive the per-user HMAC prefix used at the head of every
+ * managed-storage key. Stable across retries; non-reversible.
+ *
+ * `secret` is the deployment-wide `STORAGE_KEY_HMAC_SECRET`. The
+ * caller MUST source it from `RuntimeConfig.storageKeyHmacSecret`
+ * (validated at startup as >=64 hex chars); never read env here.
+ */
+export function deriveStorageKeyPrefix(secret, userId) {
+    return createHmac('sha256', secret).update(userId).digest('hex').slice(0, STORAGE_KEY_PREFIX_LENGTH);
+}

package/dist/services/storage-put-recovery.d.ts

@@ -0,0 +1,71 @@
+/**
+ * @file Pending-row-first put recovery helpers for `StorageService`.
+ *
+ * `storage-service.ts` orchestrates managed-mode puts: claim a
+ * pending row, call `backend.put`, finalize via a CAS to `'stored'`.
+ * This module owns every branch the orchestrator hands off to when
+ * something on either side fails, plus the closed-set event names
+ * the helpers emit:
+ *
+ *   - `persistUploadedOrRecover` — happy success + retry, then
+ *     reconciliation (commit-after-throw), then recovery.
+ *   - `recordBackendPutFailure` — `backend.put` threw before
+ *     finalization; CAS-flip to `'failed'`.
+ *   - the post-put recovery branches (cleanup OK / cleanup failed)
+ *     and the durable orphan-URI marker that the reconciler reads.
+ *
+ * Extracted from `storage-service.ts` to keep that file under the
+ * workspace 400-LOC cap. No behavior change vs the inline form.
+ */
+import type pg from 'pg';
+import { type StorageArtifactRow } from '../db/storage-artifact-repository.js';
+import type { PutBackendResult, StorageBackend } from '../storage/storage-backend.js';
+/** Re-export of the backend `put` result shape used by the recovery helpers. */
+export type PutResult = PutBackendResult;
+/**
+ * Closed enum of structured-log event names emitted by the put
+ * recovery pipeline. Adding a name here is the explicit permission
+ * to surface it in the `[STORAGE]` log stream; the TS checker
+ * blocks accidental typos.
+ */
+export type StorageEventName = 'storage.put.post_put_unrecoverable' | 'storage.put.post_put_failed_cleaned_up' | 'storage.put.post_put_unrecoverable_mark_error' | 'storage.put.post_put_failed_cleaned_up_mark_error' | 'storage.put.backend_put_failed_mark_skipped' | 'storage.put.backend_put_failed_mark_error';
+/**
+ * Normalize an arbitrary thrown value into a single-line string.
+ * Exported because `StorageService.executeBackendDelete` also
+ * needs to format `backend.delete` errors into `last_error`
+ * envelopes; the function shape is identical to the one the
+ * recovery helpers use so they share a single implementation.
+ */
+export declare function errorMessage(err: unknown): string;
+export interface PutClaim {
+    row: StorageArtifactRow;
+    claimId: string;
+}
+/**
+ * Success-path orchestrator: CAS-flip the pending row to `stored`,
+ * with one retry. On every failure mode (throw, CAS miss, or the
+ * commit-after-throw race) call `reconcileAlreadyFinalized` BEFORE
+ * recovery so an already-finalized row is not erroneously cleaned
+ * up.
+ */
+export declare function persistUploadedOrRecover(args: {
+    pool: pg.Pool;
+    userId: string;
+    backend: StorageBackend;
+    claim: PutClaim;
+    putResult: PutResult;
+}): Promise<StorageArtifactRow>;
+/**
+ * Failure branch invoked from `putManaged` when `backend.put`
+ * itself threw. CAS-flip the pending row to `'failed'`. If the CAS
+ * marker misses or its UPDATE throws, emit a typed event so the
+ * failure stays observable; the caller re-throws the original put
+ * error regardless.
+ */
+export declare function recordBackendPutFailure(args: {
+    pool: pg.Pool;
+    userId: string;
+    claim: PutClaim;
+    provider: string;
+    putError: unknown;
+}): Promise<void>;

package/dist/services/storage-put-recovery.js

@@ -0,0 +1,269 @@
+/**
+ * @file Pending-row-first put recovery helpers for `StorageService`.
+ *
+ * `storage-service.ts` orchestrates managed-mode puts: claim a
+ * pending row, call `backend.put`, finalize via a CAS to `'stored'`.
+ * This module owns every branch the orchestrator hands off to when
+ * something on either side fails, plus the closed-set event names
+ * the helpers emit:
+ *
+ *   - `persistUploadedOrRecover` — happy success + retry, then
+ *     reconciliation (commit-after-throw), then recovery.
+ *   - `recordBackendPutFailure` — `backend.put` threw before
+ *     finalization; CAS-flip to `'failed'`.
+ *   - the post-put recovery branches (cleanup OK / cleanup failed)
+ *     and the durable orphan-URI marker that the reconciler reads.
+ *
+ * Extracted from `storage-service.ts` to keep that file under the
+ * workspace 400-LOC cap. No behavior change vs the inline form.
+ */
+import { getStorageArtifactByIdIncludingDeleted, markPutFailed, recordUploadedArtifact, } from '../db/storage-artifact-repository.js';
+import { projectArtifactProviderFields } from '../storage/provider-metadata-projection.js';
+import { PutPostPersistError } from './storage-service-errors.js';
+/** Emit a structured `[STORAGE]` event to stderr (one JSON line). */
+function emitStorageEvent(event, detail) {
+    const payload = { event, timestamp: new Date().toISOString(), detail };
+    process.stderr.write(`[STORAGE] ${JSON.stringify(payload)}\n`);
+}
+/**
+ * Normalize an arbitrary thrown value into a single-line string.
+ * Exported because `StorageService.executeBackendDelete` also
+ * needs to format `backend.delete` errors into `last_error`
+ * envelopes; the function shape is identical to the one the
+ * recovery helpers use so they share a single implementation.
+ */
+export function errorMessage(err) {
+    if (err instanceof Error)
+        return err.message;
+    return String(err ?? 'unknown error');
+}
+/**
+ * Success-path orchestrator: CAS-flip the pending row to `stored`,
+ * with one retry. On every failure mode (throw, CAS miss, or the
+ * commit-after-throw race) call `reconcileAlreadyFinalized` BEFORE
+ * recovery so an already-finalized row is not erroneously cleaned
+ * up.
+ */
+export async function persistUploadedOrRecover(args) {
+    const projected = projectArtifactProviderFields(args.backend.provider, args.putResult.providerMetadata);
+    const recordInput = {
+        userId: args.userId,
+        artifactId: args.claim.row.id,
+        putAttemptId: args.claim.claimId,
+        uri: args.putResult.uri,
+        sizeBytes: args.putResult.sizeBytes,
+        plaintextHash: args.putResult.plaintextHash,
+        storedHash: args.putResult.storedHash,
+        identifiers: projected.identifiers,
+        providerDetails: projected.providerDetails,
+    };
+    let firstError;
+    try {
+        const recorded = await recordUploadedArtifact(args.pool, recordInput);
+        if (recorded !== null)
+            return recorded;
+    }
+    catch (err) {
+        firstError = err;
+    }
+    const reconciled = await reconcileAlreadyFinalized(args.pool, args.userId, args.claim.row.id, args.putResult);
+    if (reconciled !== null)
+        return reconciled;
+    let retryError;
+    try {
+        const retried = await recordUploadedArtifact(args.pool, recordInput);
+        if (retried !== null)
+            return retried;
+    }
+    catch (err) {
+        retryError = err;
+    }
+    const reconciledAfterRetry = await reconcileAlreadyFinalized(args.pool, args.userId, args.claim.row.id, args.putResult);
+    if (reconciledAfterRetry !== null)
+        return reconciledAfterRetry;
+    // Synthesize a typed error so recovery has a meaningful message
+    // when both calls returned null without throwing. URI stays
+    // server-side; only the artifact id is in the message.
+    const persistError = firstError ?? retryError ?? new Error(`recordUploadedArtifact CAS missed twice for artifact '${args.claim.row.id}' ` +
+        'and the row was not reconcilable; treating as post-put DB failure');
+    return recoverPostPutFailure({
+        pool: args.pool,
+        userId: args.userId,
+        claim: args.claim,
+        backend: args.backend,
+        putResult: args.putResult,
+        persistError,
+    });
+}
+/**
+ * Commit-after-throw reconciliation. Reads the row by id (within
+ * the user scope) and returns it if `status='stored'` AND its
+ * `(uri, plaintextHash, storedHash)` match the bytes we just
+ * uploaded. Returns null otherwise.
+ */
+async function reconcileAlreadyFinalized(pool, userId, artifactId, putResult) {
+    const row = await getStorageArtifactByIdIncludingDeleted(pool, userId, artifactId);
+    if (row === null)
+        return null;
+    if (row.status !== 'stored')
+        return null;
+    if (row.uri !== putResult.uri)
+        return null;
+    if (row.plaintextHash !== putResult.plaintextHash)
+        return null;
+    if (row.storedHash !== putResult.storedHash)
+        return null;
+    return row;
+}
+/**
+ * Cleanup-then-mark recovery. Attempts `backend.delete(uri)` to
+ * roll back the just-uploaded bytes; depending on the outcome
+ * either marks the row `failed` with `put_post_persist_failed_cleaned_up`
+ * and re-throws the original DB error, OR persists a durable
+ * `put_post_persist_unrecoverable` marker (with the orphan URI on
+ * the internal `last_error` envelope) and throws `PutPostPersistError`.
+ */
+async function recoverPostPutFailure(args) {
+    const persistMessage = errorMessage(args.persistError);
+    try {
+        await args.backend.delete(args.putResult.uri);
+    }
+    catch (cleanupError) {
+        await markPostPutUnrecoverableBestEffort({
+            pool: args.pool,
+            userId: args.userId,
+            artifactId: args.claim.row.id,
+            putAttemptId: args.claim.claimId,
+            provider: args.backend.provider,
+            uri: args.putResult.uri,
+            persistMessage,
+            cleanupMessage: errorMessage(cleanupError),
+        });
+        throw new PutPostPersistError(args.claim.row.id, args.backend.provider, args.putResult.uri, persistMessage);
+    }
+    await markCleanedUpBestEffort({
+        pool: args.pool,
+        userId: args.userId,
+        artifactId: args.claim.row.id,
+        putAttemptId: args.claim.claimId,
+        provider: args.backend.provider,
+        persistMessage,
+    });
+    throw args.persistError;
+}
+async function markCleanedUpBestEffort(args) {
+    const envelope = {
+        layer: 'raw_storage',
+        code: 'put_post_persist_failed_cleaned_up',
+        message: args.persistMessage,
+        storage_provider: args.provider,
+        occurred_at: new Date().toISOString(),
+    };
+    const matched = await runScopedFailureMarker({
+        pool: args.pool,
+        userId: args.userId,
+        artifactId: args.artifactId,
+        putAttemptId: args.putAttemptId,
+        envelope,
+        dbErrorEvent: 'storage.put.post_put_failed_cleaned_up_mark_error',
+    });
+    if (!matched) {
+        emitStorageEvent('storage.put.post_put_failed_cleaned_up', {
+            artifact_id: args.artifactId,
+            provider: args.provider,
+            persist_error: args.persistMessage,
+            reason: 'cas_miss_or_db_error',
+        });
+    }
+}
+async function markPostPutUnrecoverableBestEffort(args) {
+    const envelope = {
+        layer: 'raw_storage',
+        code: 'put_post_persist_unrecoverable',
+        message: args.persistMessage,
+        cleanup_error: args.cleanupMessage,
+        orphan_uri: args.uri,
+        storage_provider: args.provider,
+        occurred_at: new Date().toISOString(),
+    };
+    const matched = await runScopedFailureMarker({
+        pool: args.pool,
+        userId: args.userId,
+        artifactId: args.artifactId,
+        putAttemptId: args.putAttemptId,
+        envelope,
+        dbErrorEvent: 'storage.put.post_put_unrecoverable_mark_error',
+    });
+    emitStorageEvent('storage.put.post_put_unrecoverable', {
+        artifact_id: args.artifactId,
+        provider: args.provider,
+        uri: args.uri,
+        persist_error: args.persistMessage,
+        cleanup_error: args.cleanupMessage,
+        marker_persisted: matched,
+    });
+}
+/** Shared CAS-scoped failure marker — CAS on
+ *  `(id, user_id, put_attempt_id, status='pending')`. Emits the
+ *  supplied typed event on DB error; never swallows. */
+async function runScopedFailureMarker(args) {
+    try {
+        const result = await args.pool.query(`UPDATE storage_artifacts
+         SET status = 'failed',
+             last_error = $4::jsonb,
+             put_attempt_id = NULL,
+             updated_at = NOW()
+         WHERE id = $1
+           AND user_id = $2
+           AND put_attempt_id = $3
+           AND status = 'pending'`, [args.artifactId, args.userId, args.putAttemptId, JSON.stringify(args.envelope)]);
+        return (result.rowCount ?? 0) > 0;
+    }
+    catch (markError) {
+        emitStorageEvent(args.dbErrorEvent, {
+            artifact_id: args.artifactId,
+            mark_error: errorMessage(markError),
+        });
+        return false;
+    }
+}
+/**
+ * Failure branch invoked from `putManaged` when `backend.put`
+ * itself threw. CAS-flip the pending row to `'failed'`. If the CAS
+ * marker misses or its UPDATE throws, emit a typed event so the
+ * failure stays observable; the caller re-throws the original put
+ * error regardless.
+ */
+export async function recordBackendPutFailure(args) {
+    const envelope = {
+        layer: 'raw_storage',
+        code: 'backend_put_failed',
+        message: errorMessage(args.putError),
+        storage_provider: args.provider,
+        occurred_at: new Date().toISOString(),
+    };
+    try {
+        const marked = await markPutFailed(args.pool, {
+            userId: args.userId,
+            artifactId: args.claim.row.id,
+            putAttemptId: args.claim.claimId,
+            lastError: envelope,
+        });
+        if (marked !== null)
+            return;
+        emitStorageEvent('storage.put.backend_put_failed_mark_skipped', {
+            artifact_id: args.claim.row.id,
+            provider: args.provider,
+            reason: 'cas_miss',
+            put_error: envelope.message,
+        });
+    }
+    catch (markError) {
+        emitStorageEvent('storage.put.backend_put_failed_mark_error', {
+            artifact_id: args.claim.row.id,
+            provider: args.provider,
+            put_error: envelope.message,
+            mark_error: errorMessage(markError),
+        });
+    }
+}

package/dist/services/storage-service-errors.d.ts

@@ -0,0 +1,124 @@
+/**
+ * @file Typed errors thrown by `StorageService`.
+ *
+ * The route layer pattern-matches these by `instanceof` to emit the
+ * correct HTTP status + error envelope. Service code never embeds
+ * status codes itself; the route is the single place where service
+ * outcomes map to HTTP.
+ */
+export declare class StorageArtifactNotFoundError extends Error {
+    readonly artifactId: string;
+    constructor(artifactId: string);
+}
+export declare class PointerContentNotManagedError extends Error {
+    readonly artifactId: string;
+    readonly uri: string;
+    constructor(artifactId: string, uri: string);
+}
+export declare class FilecoinDirectStorageNotSupportedError extends Error {
+    readonly provider: string;
+    constructor(provider?: string);
+}
+export declare class UnsupportedPointerSchemeError extends Error {
+    readonly uri: string;
+    readonly allowedSchemes: ReadonlyArray<string>;
+    constructor(uri: string, allowedSchemes: ReadonlyArray<string>);
+}
+export declare class ArtifactInUseError extends Error {
+    readonly artifactId: string;
+    readonly referencedByDocumentCount: number;
+    constructor(artifactId: string, count: number);
+}
+export declare class ManagedStorageDisabledError extends Error {
+    constructor();
+}
+export declare class InvalidArtifactMetadataError extends Error {
+    readonly reason: string;
+    constructor(reason: string);
+}
+/**
+ * Thrown when a managed-mode artifact row's `provider` is not
+ * registered with the active `StorageBackendRegistry`. This is an
+ * operational unavailability (the deployment dropped a backend that
+ * still has live data), NOT a 5xx crash — the route layer maps it
+ * to HTTP 503 `storage_backend_unavailable`. The envelope carries
+ * only `provider_id`; internal stack/error details stay server-side.
+ */
+export declare class BackendNotRegisteredError extends Error {
+    readonly providerId: string;
+    readonly artifactId: string;
+    constructor(providerId: string, artifactId: string);
+}
+/**
+ * Thrown by `StorageService.putManaged` when the put would write
+ * through a backend whose provider isn't part of the registry's
+ * known set (registered backends + the pointer-only marker). The
+ * route maps this to HTTP 503 `storage_backend_unavailable`. Unlike
+ * `BackendNotRegisteredError` (which fires on read against an
+ * already-persisted row), this fires BEFORE any DB write — the
+ * upload is rejected at the entry point.
+ */
+export declare class UnregisteredProviderError extends Error {
+    readonly providerId: string;
+    constructor(providerId: string);
+}
+/**
+ * Thrown by `StorageService.putManaged` when `backend.put` succeeded
+ * but the post-put DB update and the subsequent recovery (DB retry +
+ * backend cleanup of the just-uploaded bytes) ALL failed. At that
+ * point the bytes survive at the backend AND the DB does not know
+ * the URI. The route maps this to HTTP 503 `put_post_persist_failed`
+ * with the artifact id and provider so ops can investigate. The
+ * orphan URI is captured server-side (in the row's `last_error`
+ * envelope + a structured `storage.put.post_put_unrecoverable`
+ * event) but NEVER on the wire.
+ */
+export declare class PutPostPersistError extends Error {
+    readonly artifactId: string;
+    readonly providerId: string;
+    /** Orphan URI — internal only; the route never surfaces this on the wire. */
+    readonly uri: string;
+    constructor(artifactId: string, providerId: string, uri: string, cause: string);
+}
+/**
+ * Thrown when a public API caller requests bytes or verifies an
+ * artifact whose managed upload is still in flight (`status='pending'`).
+ * Route maps this to HTTP 409 `artifact_not_ready`. Distinct from
+ * `ArtifactUnavailableError` (terminal `failed`) because the row
+ * MIGHT still reach `stored` if the upload finishes.
+ */
+export declare class ArtifactNotReadyError extends Error {
+    readonly artifactId: string;
+    constructor(artifactId: string);
+}
+/**
+ * Thrown when a public API caller requests bytes or verifies an
+ * artifact whose managed upload terminally failed (`status='failed'`
+ * with no URI). The row exists for ops visibility — the caller's
+ * action against it would never succeed because the backend never
+ * persisted the bytes. Route maps this to HTTP 410 `artifact_unavailable`.
+ */
+export declare class ArtifactUnavailableError extends Error {
+    readonly artifactId: string;
+    readonly reason: string;
+    constructor(artifactId: string, reason: string);
+}
+/**
+ * Thrown by `deleteArtifact` when `claimDeleteAttempt` returns null
+ * AND the row is currently in `status='deleting'` — another caller
+ * holds an active claim and is mid-cascade / mid-backend-delete.
+ *
+ * Distinct from the idempotent terminal `status='deleted'` path
+ * (which still returns the prior terminal envelope as success) and
+ * from `StorageArtifactNotFoundError` (missing / cross-user). Route
+ * maps this to HTTP 409 `delete_in_flight` with `retryable=true` so
+ * the caller knows the action isn't fatal — just contended — and a
+ * later retry will either find the row `deleted` (idempotent
+ * success) or re-claim it (if the other caller's attempt failed
+ * and the row transitioned to `delete_failed`).
+ */
+export declare class ArtifactDeleteInFlightError extends Error {
+    readonly artifactId: string;
+    readonly currentStatus: string;
+    constructor(artifactId: string, currentStatus: string);
+}