@atomicmemory/core 1.0.0

package/dist/routes/document-response-formatters.js

@@ -0,0 +1,243 @@
+/**
+ * Wire-format formatters for /v1/documents/* responses.
+ *
+ * Mirrors the snake_case envelope the rest of the API uses: handlers
+ * emit camelCase shapes from the service layer, this module flattens
+ * them to the on-the-wire JSON.
+ */
+import { formatPublicRawStorageMetadata } from './public-raw-storage-metadata.js';
+import { getDeleteSemantics, } from '../storage/store-registry.js';
+export function formatRawDocument(row, registry) {
+    return {
+        id: row.id,
+        user_id: row.userId,
+        raw_source_id: row.rawSourceId,
+        external_id: row.externalId,
+        external_uri: row.externalUri,
+        display_name: row.displayName,
+        mime_type: row.mimeType,
+        size_bytes: row.sizeBytes,
+        content_hash: row.contentHash,
+        provider_version: row.providerVersion,
+        source_modified_at: row.sourceModifiedAt instanceof Date ? row.sourceModifiedAt.toISOString() : null,
+        storage_mode: row.storageMode,
+        storage_uri: row.storageUri,
+        storage_provider: row.storageProvider,
+        registration_status: row.registrationStatus,
+        raw_storage_status: row.rawStorageStatus,
+        // Public allowlist: `upload_result` (internal upload sidecar)
+        // + AES-GCM internals (`nonce`/`tag`/`key_id`/`encoded_*`) +
+        // unknown filecoin fields are stripped. The internal Filecoin
+        // sidecar `{ipfs_cid?, piece_cid, copies:[{provider_id, status}], ...}`
+        // is projected through the shared
+        // `projectFilecoinPublicMetadata` helper to the Synapse
+        // public shape `{ipfs_cid?, piece_cid, copy_count,
+        // provider_ids, copy_statuses}` (Phase 4 renamed the legacy
+        // `cid` slot to `ipfs_cid`).
+        raw_storage_metadata: formatPublicRawStorageMetadata(row.rawStorageMetadata),
+        // Per-row capability advertisement for the UI delete-confirm
+        // dialog. Resolved from the row's own
+        // `storage_provider` via the registry — supports mixed-provider
+        // deployments where legacy local_fs rows coexist with new
+        // filecoin rows. NULL for pointer-only rows or
+        // providers the deployment doesn't have registered.
+        delete_semantics: getDeleteSemantics(registry, row.storageProvider),
+        metadata: row.metadata,
+        created_at: row.createdAt instanceof Date ? row.createdAt.toISOString() : String(row.createdAt),
+        updated_at: row.updatedAt instanceof Date ? row.updatedAt.toISOString() : String(row.updatedAt),
+        indexed_content_hash: row.indexedContentHash,
+        indexed_at: row.indexedAt instanceof Date ? row.indexedAt.toISOString() : null,
+        // Per-layer status + last_error envelope. Snake_case
+        // matches the wire and the JSONB column shape; the UI layer
+        // converts to camelCase at the React Query boundary.
+        extraction_status: row.extractionStatus,
+        semantic_index_status: row.semanticIndexStatus,
+        last_error: sanitizeLastErrorForWire(row.lastError),
+        // Additive link to the row's paired `storage_artifacts` entry.
+        // NULL for pointer-only registration stubs and older rows.
+        // The existing `raw_storage_status`/`storage_provider`/
+        // `raw_storage_metadata`/`delete_semantics` fields stay
+        // populated unchanged; artifact-native data can be read from the
+        // storage API.
+        storage_artifact_id: row.storageArtifactId,
+    };
+}
+export function formatRegisterDocumentResponse(result, registry) {
+    return {
+        document: formatRawDocument(result.document, registry),
+        created: result.created,
+    };
+}
+export function formatListDocumentsResponse(documents, registry) {
+    return {
+        documents: documents.map((d) => formatRawDocument(d, registry)),
+        count: documents.length,
+    };
+}
+/**
+ * Cursor-paginated document list. Mirrors
+ * `DocumentListRootResponseSchema` in `schemas/responses.ts`.
+ */
+export function formatDocumentListRootResponse(result, registry) {
+    return {
+        documents: result.documents.map((d) => formatRawDocument(d, registry)),
+        next_cursor: result.nextCursor,
+    };
+}
+/**
+ * Public passport-feed row formatter. Strips internal
+ * fields the repository never returns; renames camelCase to
+ * snake_case for the wire; ISO-encodes timestamp instances. The
+ * discriminated union on `kind` is preserved verbatim so consumers
+ * branch on it.
+ */
+function formatPassportFeedRow(row, registry) {
+    if (row.kind === 'document_grouped') {
+        return formatPassportFeedGroupedRow(row, registry);
+    }
+    return formatPassportFeedStandaloneRow(row);
+}
+function formatPassportFeedGroupedRow(row, registry) {
+    return {
+        kind: 'document_grouped',
+        document_id: row.documentId,
+        sort_at: isoOf(row.sortAt),
+        sort_id: row.sortId,
+        representative: {
+            id: row.representative.id,
+            content: row.representative.content,
+            created_at: isoOf(row.representative.createdAt),
+            source_site: row.representative.sourceSite,
+        },
+        chunk_count: row.chunkCount,
+        raw_storage_status: row.rawStorageStatus,
+        extraction_status: row.extractionStatus,
+        semantic_index_status: row.semanticIndexStatus,
+        last_error: sanitizeLastErrorForWire(row.lastError),
+        display_name: row.displayName,
+        mime_type: row.mimeType,
+        // Grouped rows carry the row-specific `storage_provider` +
+        // public-allowlisted `raw_storage_metadata` + per-row
+        // `delete_semantics`. Standalone-memory rows stay status-only
+        // because they have no backing document.
+        storage_provider: row.storageProvider,
+        raw_storage_metadata: formatPublicRawStorageMetadata(row.rawStorageMetadata),
+        delete_semantics: getDeleteSemantics(registry, row.storageProvider),
+    };
+}
+function formatPassportFeedStandaloneRow(row) {
+    return {
+        kind: 'standalone_memory',
+        sort_at: isoOf(row.sortAt),
+        sort_id: row.sortId,
+        memory: {
+            id: row.memory.id,
+            content: row.memory.content,
+            created_at: isoOf(row.memory.createdAt),
+            source_site: row.memory.sourceSite,
+        },
+    };
+}
+/**
+ * Passport-feed envelope formatter. Maps the discriminated
+ * repository result to the public snake_case wire shape.
+ */
+export function formatPassportFeedResponse(result, registry) {
+    return {
+        rows: result.rows.map((r) => formatPassportFeedRow(r, registry)),
+        next_cursor: result.nextCursor,
+    };
+}
+function isoOf(value) {
+    return value instanceof Date ? value.toISOString() : String(value);
+}
+/**
+ * Strip server-only keys (currently `internal_recovery_hint`) from
+ * the `last_error` envelope before it leaves the trust boundary.
+ *
+ * The `runPhaseBeta2` orphan-recovery path embeds the failed
+ * `store.delete` URI + provider under `internal_recovery_hint` so
+ * a reconciler / ops can find abandoned bytes — but those values
+ * (especially the storage URI) reveal internal storage layout
+ * and must not surface on the public API. Add new
+ * `internal_*` keys to the deny-list here as they appear.
+ */
+function sanitizeLastErrorForWire(value) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+        return value;
+    }
+    const out = {};
+    for (const [key, val] of Object.entries(value)) {
+        if (key.startsWith('internal_'))
+            continue;
+        out[key] = val;
+    }
+    return out;
+}
+export function formatDeleteDocumentResponse(result) {
+    return {
+        success: true,
+        already_deleted: result.alreadyDeleted,
+    };
+}
+export function formatUploadRawDocumentResponse(result, registry) {
+    return {
+        document_id: result.documentId,
+        storage_provider: result.storageProvider,
+        storage_uri: result.storageUri,
+        content_hash: result.contentHash,
+        size_bytes: result.sizeBytes,
+        raw_storage_status: result.rawStorageStatus,
+        storage_mode: result.storageMode,
+        raw_storage_metadata: formatPublicRawStorageMetadata(result.rawStorageMetadata),
+        // Per-row capability advertisement (same as `formatRawDocument`).
+        // Resolved from the upload's
+        // `storageProvider` so a Filecoin upload emits
+        // `delete_semantics: 'tombstone'` end-to-end even when the
+        // deployment also has legacy local_fs adapters registered.
+        delete_semantics: getDeleteSemantics(registry, result.storageProvider),
+        idempotent_skip: result.idempotentSkip,
+    };
+}
+export function formatIndexDocumentResponse(result) {
+    return {
+        document_id: result.documentId,
+        indexed_content_hash: result.indexedContentHash,
+        chunks_created: result.chunksCreated,
+        memories_created: result.memoriesCreated,
+        idempotent_skip: result.idempotentSkip,
+        chunker_version: result.chunkerVersion,
+        parser_version: result.parserVersion,
+    };
+}
+export function formatDocumentFailureMarkerResponse(result, registry) {
+    return {
+        document: formatRawDocument(result.document, registry),
+        idempotent: result.idempotent,
+    };
+}
+export function formatDocumentLimitsResponse(snapshot) {
+    const rawStorage = {
+        enabled: snapshot.rawStorage.enabled,
+        mode: snapshot.rawStorage.mode,
+    };
+    if (snapshot.rawStorage.reason !== undefined) {
+        rawStorage.reason = snapshot.rawStorage.reason;
+    }
+    if (snapshot.rawStorage.provider !== undefined) {
+        rawStorage.provider = snapshot.rawStorage.provider;
+    }
+    if (snapshot.rawStorage.capabilities !== undefined) {
+        const c = snapshot.rawStorage.capabilities;
+        rawStorage.addressing = c.addressing;
+        rawStorage.retrieval_consistency = c.retrievalConsistency;
+        rawStorage.delete_semantics = c.deleteSemantics;
+        rawStorage.supports_head = c.supportsHead;
+        rawStorage.supports_get = c.supportsGet;
+    }
+    return {
+        raw_upload_max_bytes: snapshot.rawUploadMaxBytes,
+        index_max_text_bytes: snapshot.indexMaxTextBytes,
+        raw_storage: rawStorage,
+    };
+}

package/dist/routes/documents.d.ts

@@ -0,0 +1,74 @@
+/**
+ * /v1/documents — multi-phase document pipeline router.
+ *
+ * Routes the document registry + lifecycle surface, layered up through
+ * the rev-18 hardening plan:
+ *   - Phase 1: pointer-only registration, get-by-id, legacy
+ *     `/list` (offset/limit + source_site), soft-delete.
+ *   - Phase 2: `POST /:id/index` chunks + embeds supplied text.
+ *   - Phase 3: `PUT /:id/raw` managed-blob upload (when
+ *     `RAW_STORAGE_MODE=managed_blob` is configured); the post-upload
+ *     row promotion sets `storage_mode='managed_blob'` and
+ *     `raw_storage_status='blob_stored'`.
+ *   - Phase B/C: per-layer status + `last_error` envelope + the
+ *     constrained-transition routes
+ *     `POST /:id/extraction-failure` and `POST /:id/index-failure`.
+ *   - Phase D: cursor-paginated `GET /` (root list),
+ *     `GET /without-memories` (recovery-status filter), and
+ *     `GET /passport-feed` (data-layer grouped feed) on top of
+ *     `GET /limits` for the runtime preflight surface.
+ *
+ * Validation + response-shape conventions match `/v1/memories`
+ * (`createMemoryRouter`, `validateBody`, `validateResponse`).
+ *
+ * `POST /v1/documents` (register) still only accepts
+ * `storage_mode='pointer_only'` on the wire — the `managed_blob` /
+ * `inline_small_text` modes are populated by `PUT /:id/raw` (and a
+ * future inline-text path) rather than declared at register time.
+ *
+ * See `Atomicmemory-research/docs/core-repo/design/large-file-ingestion-and-raw-storage-plan-2026-05-08.md`
+ * and the rev-18 hardening plan.
+ */
+import { Router } from 'express';
+import { type DocumentLimitsSnapshot } from './document-response-formatters.js';
+import type { DocumentService } from '../services/document-service.js';
+/** Composition-time options for {@link createDocumentRouter}. */
+export interface DocumentRouterOptions {
+    /**
+     * Body-size cap (bytes) for `PUT /v1/documents/:id/raw`. Required so
+     * the limit is wired explicitly at composition time — this module
+     * intentionally does not read `config.rawUploadMaxBytes`, keeping the
+     * route layer free of singleton imports.
+     */
+    rawUploadMaxBytes: number;
+    /**
+     * Public preflight snapshot for `GET /v1/documents/limits`. Surfaces
+     * byte caps and raw-storage capability so clients can size requests
+     * and decide whether to attempt a managed-blob upload before
+     * touching the database. Sourced from the runtime config at
+     * composition time; the route layer does not import config singletons.
+     */
+    limits: DocumentLimitsSnapshot;
+}
+/**
+ * Build the documents router with composition-owned limits.
+ *
+ * Body-parser ordering inside this router (deliberate — see
+ * `Atomicmemory-research/docs/core-repo/design/document-ingest-audit.md`
+ * and the rev-18 hardening plan):
+ *   1. validate-response (response-shape contract).
+ *   2. Query-only GETs first (`/limits`, `/list`) so they can never be
+ *      shadowed by `/:id` and never need a body parser.
+ *   3. Per-route large parsers (`POST /:id/index` → 25 MiB JSON,
+ *      `PUT /:id/raw` → raw bytes), registered BEFORE the router-level
+ *      JSON middleware so the larger cap wins for that single path.
+ *   4. Router-level `express.json({ limit: '1mb' })` fallthrough for the
+ *      remaining JSON-body routes.
+ *   5. Remaining JSON-body routes (`POST /`, `DELETE /:id`).
+ *   6. `GET /:id` LAST so it cannot shadow `/limits`, `/list`, etc.
+ *
+ * The router takes ownership of body parsing — `create-app.ts` mounts
+ * the documents router WITHOUT an upstream `express.json` so the
+ * larger `/index` parser is not silently overridden.
+ */
+export declare function createDocumentRouter(service: DocumentService, options: DocumentRouterOptions): Router;