@nacre.work/api 0.2.0 → 0.4.0

package/dist/server.js

@@ -3,7 +3,7 @@ import { createHash, randomUUID, timingSafeEqual } from 'node:crypto';
 // Imported rather than taken from the global scope: `lib` is ES2023 with no
 // DOM, so the global URL is not typed here.
 import { URL } from 'node:url';
-import { logger, MetadataError, queryAudit, parseFilters, parseMetadata, PROTECTED_RESOURCE_PATH, TooBusy, } from '@nacre.work/core';
+import { logger, MetadataError, MultipartError, multipartBoundary, parseMultipart, queryAudit, parseFilters, parseMetadata, PROTECTED_RESOURCE_PATH, JWKS_PATH, ADMIN_PREFIX, adminRoutes, withAuditSinks, TooBusy, } from '@nacre.work/core';
 import { authenticate, rejectTenantOverride } from './auth.js';
 import { badRequest, internal, notFound, Problem } from './errors.js';
 import { isConflict, isReplay } from './idempotency.js';
@@ -114,7 +114,7 @@ function parseGrant(body) {
         permission: permission,
     };
 }
-async function readBody(req, limit = MAX_BODY_BYTES) {
+async function readRaw(req, limit) {
     const chunks = [];
     let size = 0;
     for await (const chunk of req) {
@@ -123,9 +123,55 @@ async function readBody(req, limit = MAX_BODY_BYTES) {
             throw new BodyTooLarge('body too large');
         chunks.push(chunk);
     }
-    if (chunks.length === 0)
+    return Buffer.concat(chunks);
+}
+async function readBody(req, limit = MAX_BODY_BYTES) {
+    const raw = await readRaw(req, limit);
+    if (raw.length === 0)
         return undefined;
-    return JSON.parse(Buffer.concat(chunks).toString('utf8'));
+    return JSON.parse(raw.toString('utf8'));
+}
+/**
+ * A multipart body, reduced to the same shape a JSON one has.
+ *
+ * Two things depend on this being a plain object of fields rather than a
+ * special case threaded through the handler.
+ *
+ * The first is T2. `rejectTenantOverride` scans the body for an organization
+ * named at any depth, before routing and before validation, and it runs on
+ * whatever `body` is. A multipart request whose fields never became `body`
+ * would be a second door into the ingest endpoint with that check on the other
+ * side of it — which is exactly the shape of hole the rate limiter and the
+ * metrics each had when MCP was a second surface.
+ *
+ * The second is that everything downstream stays one code path: the same
+ * required-field checks, the same metadata parsing, the same audit event.
+ *
+ * The file is kept out of it. Its bytes are not a field, and putting a
+ * document body into an object that gets scanned, logged and error-messaged is
+ * how content ends up somewhere it should not be.
+ */
+function multipartBody(parts) {
+    const fields = {};
+    let file;
+    for (const part of parts) {
+        // The file is the part with a filename, or the one called `file` — which
+        // is what openapi.yaml names it and what every form sends.
+        if (part.filename !== undefined || part.name === 'file') {
+            if (file !== undefined) {
+                throw new MultipartError('more than one file part; a document is one file');
+            }
+            file = part;
+            continue;
+        }
+        if (part.name in fields) {
+            // Refused rather than last-wins. A repeated field is a caller who
+            // believes something different from what would be stored.
+            throw new MultipartError(`the field ${part.name} appears more than once`);
+        }
+        fields[part.name] = Buffer.from(part.bytes).toString('utf8');
+    }
+    return { fields, ...(file === undefined ? {} : { file }) };
 }
 /** The wire shape of a reindex, snake case like every other response here. */
 function reindexJson(status) {
@@ -146,8 +192,84 @@ function reindexJson(status) {
         failed: status.failed,
         progress: status.progress,
         error: status.error,
+        // `null` and not omitted. Absent would read as "this deployment does not do
+        // recall checks"; null says "this migration has not been scored", which for
+        // a layer with no reference set is the permanent and correct answer.
+        check: status.check === null
+            ? null
+            : {
+                recall: status.check.recall,
+                floor: status.check.floor,
+                passed: status.check.passed,
+                queries: status.check.queries,
+                scores: status.check.scores.map((s) => ({ query_id: s.queryId, recall: s.recall })),
+                ...(status.check.unresolved === undefined
+                    ? {}
+                    : { unresolved: [...status.check.unresolved] }),
+            },
     };
 }
+/** One reference query on the wire, snake case like every other response here. */
+function referenceQueryJson(q) {
+    return { id: q.id, query: q.query, expected: [...q.expected] };
+}
+/** At most this many queries in a set, and this many expected documents in one. */
+const MAX_REFERENCE_QUERIES = 50;
+const MAX_EXPECTED_PER_QUERY = 10;
+/**
+ * The reference set from a request body.
+ *
+ * Every bound here is a refusal rather than a truncation, for the reason the
+ * multipart parser gives: a truncation is a silent disagreement between what
+ * was sent and what got stored, and this one would be measured later as a
+ * recall number the operator cannot reconcile with what they wrote.
+ *
+ * `MAX_EXPECTED_PER_QUERY` is not a size limit, it is `RECALL_K`. A query
+ * naming more expected documents than the check retrieves could never score
+ * 1.0, so its floor would be unreachable and would read as a regression in the
+ * model rather than as a mistake in the set.
+ *
+ * An empty list is accepted and means "no gate on this layer", which is how a
+ * set is removed. Refusing it would leave no way back from having written one.
+ */
+function parseReferenceQueries(body) {
+    if (typeof body !== 'object' || body === null || Array.isArray(body)) {
+        return { error: 'The body must be an object with a "queries" array.' };
+    }
+    const raw = body['queries'];
+    if (!Array.isArray(raw))
+        return { error: "'queries' must be an array." };
+    if (raw.length > MAX_REFERENCE_QUERIES) {
+        return { error: `A reference set may hold at most ${MAX_REFERENCE_QUERIES} queries.` };
+    }
+    const queries = [];
+    for (const [i, entry] of raw.entries()) {
+        if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) {
+            return { error: `queries[${i}] must be an object.` };
+        }
+        const { query, expected } = entry;
+        if (typeof query !== 'string' || query.trim() === '' || query.length > 1024) {
+            return { error: `queries[${i}].query must be a string of 1 to 1024 characters.` };
+        }
+        if (!Array.isArray(expected) || expected.length === 0) {
+            return { error: `queries[${i}].expected must be a non-empty array of external ids.` };
+        }
+        if (expected.length > MAX_EXPECTED_PER_QUERY) {
+            return {
+                error: `queries[${i}].expected may name at most ${MAX_EXPECTED_PER_QUERY} documents, ` +
+                    'which is how many the check retrieves. A longer list could never score 1.0.',
+            };
+        }
+        if (!expected.every((e) => typeof e === 'string' && e !== '')) {
+            return { error: `queries[${i}].expected must hold non-empty external ids.` };
+        }
+        // Deduplicated rather than refused: a repeated id is a typo with an obvious
+        // reading, and leaving it in would divide the score by a denominator the
+        // caller did not mean.
+        queries.push({ query, expected: [...new Set(expected)] });
+    }
+    return { queries };
+}
 function send(res, status, body, requestId, extra = {}) {
     // 204 means no content, and it meant it while this function wrote the four
     // bytes `null` into the body. Some clients tolerate that and some treat a
@@ -433,8 +555,22 @@ function tokenJson(tokens) {
     };
 }
 export function createApi(options) {
+    // Applied once, here, so every event this surface records reaches a module's
+    // sinks whatever adapter is behind the port. It used to live inside the
+    // Postgres adapter, which made forwarding a property of how an event was
+    // stored rather than of it having been stored.
+    const withSinks = {
+        ...options,
+        audit: withAuditSinks(options.audit, (sink, event, error) => {
+            logger.warn('audit sink failed; the event is still in the table', {
+                sink,
+                action: event.action,
+                error: String(error).slice(0, 200),
+            });
+        }),
+    };
     return createServer((req, res) => {
-        void handle(req, res, options).catch(() => {
+        void handle(req, res, withSinks).catch(() => {
             // handle() converts everything it can into a Problem. Reaching here means
             // the failure was in the error path itself; say nothing about it.
             if (!res.headersSent)
@@ -502,6 +638,25 @@ async function handle(req, res, options) {
         send(res, 200, options.resourceMetadata, requestId);
         return;
     }
+    if (req.method === 'GET' && instance === JWKS_PATH) {
+        // The public half of the signing key, so anything outside this process can
+        // verify a token without a secret — a gateway, a sidecar, a second service
+        // in the same deployment.
+        //
+        // Unauthenticated, like every other `/.well-known` document, and that is
+        // not a concession: a public key is public. What would be a leak is the
+        // other mode, which is exactly why this answers `404` for a deployment
+        // signing with `NACRE_JWT_SECRET`. A shared secret has no publishable half,
+        // and an endpoint that "helpfully" served one would be serving the key that
+        // mints tokens.
+        if (options.jwks === undefined) {
+            const problem = notFound(instance, requestId);
+            send(res, problem.status, problem.toJSON(), requestId);
+            return;
+        }
+        send(res, 200, { keys: options.jwks }, requestId);
+        return;
+    }
     if (req.method === 'GET' && instance === '/v1/health') {
         // Liveness touches no dependency. A health check that calls Postgres turns
         // one slow database into a cascading restart loop.
@@ -570,8 +725,21 @@ async function handle(req, res, options) {
         return;
     }
     let body;
+    // Held aside from `body` on purpose — see multipartBody.
+    let uploaded;
+    let wasMultipart = false;
     try {
-        body = await readBody(req, options.maxBodyBytes ?? MAX_BODY_BYTES);
+        const limit = options.maxBodyBytes ?? MAX_BODY_BYTES;
+        const boundary = multipartBoundary(req.headers['content-type']);
+        if (boundary === undefined) {
+            body = await readBody(req, limit);
+        }
+        else {
+            const reduced = multipartBody(parseMultipart(await readRaw(req, limit), boundary));
+            body = reduced.fields;
+            uploaded = reduced.file;
+            wasMultipart = true;
+        }
     }
     catch (error) {
         // 413 for size and 400 for anything else. They were one answer, and the
@@ -586,7 +754,12 @@ async function handle(req, res, options) {
                 instance,
                 requestId,
             })
-            : badRequest(instance, requestId, 'The request body could not be read.');
+            : error instanceof MultipartError
+                ? // Named, because every one of them is a caller mistake with a fix,
+                    // and "the request body could not be read" sends them looking at
+                    // their bytes rather than at their boundary.
+                    badRequest(instance, requestId, `${error.message}.`)
+                : badRequest(instance, requestId, 'The request body could not be read.');
         send(res, problem.status, problem.toJSON(), requestId);
         return;
     }
@@ -779,15 +952,97 @@ async function handle(req, res, options) {
         if (req.method === 'POST' && instance === '/v1/documents') {
             const body_ = (body ?? {});
             const layer = body_.layer;
-            const externalId = body_.external_id;
-            const content = body_.content;
+            // A file part is content. The external id defaults to its filename,
+            // because a form that uploads `q3-plan.md` has already said what the
+            // document is called and asking for the same string twice is how a
+            // client ends up with two names for one document.
+            //
+            // The filename is used for that and for nothing else. It never reaches a
+            // path, and never an object key — `documentKey` hashes the external id,
+            // so a caller cannot choose the shape of anything in the bucket.
+            const externalId = typeof body_.external_id === 'string'
+                ? body_.external_id
+                : (uploaded?.filename ?? undefined);
+            let content = body_.content;
+            let binary;
+            if (uploaded !== undefined) {
+                if (typeof content === 'string' || typeof body_.url === 'string') {
+                    const problem = badRequest(instance, requestId, "A multipart upload carries the document; 'content' and 'url' are for the JSON body.");
+                    send(res, problem.status, problem.toJSON(), requestId);
+                    return;
+                }
+                // PDF first, and both signals must agree: the part must declare
+                // `application/pdf` AND the bytes must begin with `%PDF-`. Either
+                // alone is a refusal that names the other — a declared type the bytes
+                // contradict is exactly the disagreement the multipart parser's
+                // strictness doctrine exists to refuse, and sniffing alone would turn
+                // the declared type into decoration. Other formats extend this table;
+                // nothing falls through to a guess.
+                const declared = (uploaded.contentType ?? '').split(';')[0]?.trim().toLowerCase() ?? '';
+                const MAGIC = [0x25, 0x50, 0x44, 0x46, 0x2d]; // %PDF-
+                const magic = uploaded.bytes.length >= MAGIC.length && MAGIC.every((b, i) => uploaded.bytes[i] === b);
+                if (declared === 'application/pdf' && !magic) {
+                    const problem = badRequest(instance, requestId, "The file part declares 'application/pdf' but the bytes do not begin with the %PDF- magic. " +
+                        'Both must agree; a declared type the bytes contradict is refused rather than trusted.');
+                    send(res, problem.status, problem.toJSON(), requestId);
+                    return;
+                }
+                if (declared !== 'application/pdf' && magic) {
+                    const problem = badRequest(instance, requestId, "The bytes begin with the %PDF- magic but the file part does not declare 'application/pdf'. " +
+                        'Both must agree; declare the type rather than relying on sniffing.');
+                    send(res, problem.status, problem.toJSON(), requestId);
+                    return;
+                }
+                if (declared === 'application/pdf' && magic) {
+                    // Binary requires object storage, at the edge. The bytes' only home
+                    // is the bucket — `documents.source_ref` is text and stays text — so
+                    // a deployment without one learns on the request, naming the
+                    // variables, not from a `failed` row minutes later.
+                    if (options.objectStorage !== true) {
+                        const problem = badRequest(instance, requestId, 'A PDF upload needs object storage, and this deployment has none configured. ' +
+                            'Set NACRE_S3_* (endpoint, bucket, access key, secret key) to enable binary ingest.');
+                        send(res, problem.status, problem.toJSON(), requestId);
+                        return;
+                    }
+                    binary = { bytes: uploaded.bytes, contentType: 'application/pdf' };
+                }
+                else {
+                    // Decoded here, and refused here, rather than queued and failed
+                    // later.
+                    //
+                    // The parser extracts exactly the formats in the table above — it
+                    // took its first dependency for PDF and nothing else. Until this
+                    // check existed the sidecar decoded with `errors="replace"`, so a
+                    // binary file became a string of replacement characters that was
+                    // chunked, embedded, stored as the document body and reported as
+                    // indexed.
+                    //
+                    // At the edge the caller learns immediately and nothing is queued.
+                    // Deep in the worker they would have learned from a `failed` row
+                    // minutes later, if they looked.
+                    const decoder = new TextDecoder('utf-8', { fatal: true });
+                    try {
+                        content = decoder.decode(uploaded.bytes);
+                    }
+                    catch {
+                        const problem = badRequest(instance, requestId, 'The uploaded file is not UTF-8 text. This installation extracts PDF and nothing else — ' +
+                            'a Word file or an image needs an extractor the parser deliberately does not carry, ' +
+                            'and a PDF must declare application/pdf on the file part.');
+                        send(res, problem.status, problem.toJSON(), requestId);
+                        return;
+                    }
+                }
+            }
             const url_ = body_.url;
             if (typeof layer !== 'string' || typeof externalId !== 'string') {
                 const problem = badRequest(instance, requestId, "'layer' and 'external_id' are required.");
                 send(res, problem.status, problem.toJSON(), requestId);
                 return;
             }
-            if ((typeof content === 'string') === (typeof url_ === 'string')) {
+            // A PDF file part is the third source and already excludes the other
+            // two: `content` and `url` beside a file were refused above, before the
+            // bytes were even looked at.
+            if (binary === undefined && (typeof content === 'string') === (typeof url_ === 'string')) {
                 // Accepting both would mean choosing silently, and the choice would
                 // differ from whatever the caller assumed.
                 const problem = badRequest(instance, requestId, "Exactly one of 'content' or 'url' is required.");
@@ -799,9 +1054,28 @@ async function handle(req, res, options) {
             // a document, got 202, and the tag existed nowhere. Refused rather than
             // trimmed when it is malformed — a dropped key is a document the caller
             // believes is tagged and a filter that will never match it.
+            // Every multipart field is a string, so `metadata` arrives as JSON text
+            // where the JSON body carries an object. Parsed here rather than taught
+            // to parseMetadata, which is shared with `PATCH` and with MCP and should
+            // keep meaning one thing.
+            //
+            // Found by running it: the first version of this branch answered 400 for
+            // a perfectly good `metadata` field, because the string never became an
+            // object.
+            let rawMetadata = body_.metadata;
+            if (wasMultipart && typeof rawMetadata === 'string') {
+                try {
+                    rawMetadata = JSON.parse(rawMetadata);
+                }
+                catch {
+                    const problem = badRequest(instance, requestId, "The 'metadata' field is not JSON. In a multipart upload it carries a JSON object as text.");
+                    send(res, problem.status, problem.toJSON(), requestId);
+                    return;
+                }
+            }
             let metadata;
             try {
-                metadata = parseMetadata(body_.metadata);
+                metadata = parseMetadata(rawMetadata);
             }
             catch (error) {
                 const problem = badRequest(instance, requestId, error instanceof MetadataError ? error.message : "'metadata' is not usable.");
@@ -815,6 +1089,7 @@ async function handle(req, res, options) {
                 ...(typeof body_.title === 'string' ? { title: body_.title } : {}),
                 ...(typeof content === 'string' ? { content } : {}),
                 ...(typeof url_ === 'string' ? { url: url_ } : {}),
+                ...(binary === undefined ? {} : { bytes: binary.bytes, contentType: binary.contentType }),
                 metadata,
             });
             // The accept stage only — parse, chunk and embed happen in the worker,
@@ -841,6 +1116,32 @@ async function handle(req, res, options) {
                 send(res, problem.status, problem.toJSON(), requestId);
                 return;
             }
+            if ('refused' in outcome) {
+                // A module's ingest gate declined a document the caller may write — a
+                // quota, a suspension. Not a 404: the layer is not being hidden, the
+                // caller was allowed to write against it. Recorded as a denial so an
+                // operator can see quota-refused ingests; the gate's reason is the
+                // detail, and the gate chose the status.
+                await options.audit.write({
+                    orgId: auth.orgId,
+                    actor: `${auth.principal.type}:${auth.principal.id}`,
+                    action: 'ingest',
+                    result: 'deny',
+                    target: { layer },
+                    detail: { layer, reason: outcome.reason },
+                    requestId,
+                });
+                const problem = new Problem({
+                    type: 'https://nacre.work/errors/ingest-refused',
+                    title: outcome.status === 429 ? 'Too many requests' : 'Forbidden',
+                    status: outcome.status,
+                    detail: outcome.reason,
+                    instance,
+                    requestId,
+                });
+                send(res, outcome.status, problem.toJSON(), requestId);
+                return;
+            }
             await options.audit.write({
                 orgId: auth.orgId,
                 actor: `${auth.principal.type}:${auth.principal.id}`,
@@ -1248,6 +1549,107 @@ async function handle(req, res, options) {
             send(res, 204, null, requestId);
             return;
         }
+        // `/v1/admin/...` — routes a commercial module mounted.
+        //
+        // After authentication and after `rejectTenantOverride`, deliberately. A
+        // module gets an already-authenticated principal and a body that has
+        // already been scanned for a tenant override, so invariants 1 and 2 hold
+        // for its routes without it having to know they exist. It cannot opt out of
+        // either, because it never sees the request before this point.
+        //
+        // `platform_admin` and `org_admin` only. There is no module-supplied role
+        // check to get wrong: an administrative surface is administrative, and a
+        // member reaching one would be a widening decided in the closed half.
+        if (instance.startsWith(ADMIN_PREFIX)) {
+            // Role before route lookup. Both answer the same 404, so this is not
+            // about what a caller can tell apart — it is that a member must not
+            // reach module code at all, and "the module happened to have no matching
+            // route" is not a reason to be safe.
+            if (auth.role !== 'platform_admin' && auth.role !== 'org_admin') {
+                const problem = notFound(instance, requestId);
+                send(res, problem.status, problem.toJSON(), requestId);
+                return;
+            }
+            const route = adminRoutes().find((r) => r.method === req.method && r.pattern.test(instance));
+            if (route === undefined) {
+                // 404 whether nothing is mounted or nothing matched — the two are the
+                // same answer, and a deployment without the module must not be
+                // distinguishable from one where the path is simply wrong.
+                const problem = notFound(instance, requestId);
+                send(res, problem.status, problem.toJSON(), requestId);
+                return;
+            }
+            const url = new URL(req.url ?? '/', 'http://internal');
+            const matched = route.pattern.exec(instance);
+            const answer = await route.handle({
+                method: req.method ?? 'GET',
+                path: instance,
+                params: (matched ?? []).slice(1),
+                query: url.searchParams,
+                body,
+                auth,
+            });
+            await options.audit.write({
+                orgId: auth.orgId,
+                actor: `${auth.principal.type}:${auth.principal.id}`,
+                action: `admin.${req.method?.toLowerCase() ?? 'get'}`,
+                surface: 'admin',
+                result: answer.status < 400 ? 'allow' : 'deny',
+                target: { path: instance },
+                detail: { status: answer.status },
+                requestId,
+            });
+            send(res, answer.status, answer.body ?? null, requestId);
+            return;
+        }
+        // `/v1/layers/{id}/reference-queries`
+        const referencePath = /^\/v1\/layers\/([0-9a-f-]{36})\/reference-queries$/i.exec(instance);
+        if (referencePath !== null) {
+            const layerId = referencePath[1];
+            if (options.referenceQueries === undefined || (req.method !== 'GET' && req.method !== 'PUT')) {
+                const problem = notFound(instance, requestId);
+                send(res, problem.status, problem.toJSON(), requestId);
+                return;
+            }
+            if (req.method === 'GET') {
+                const found = await options.referenceQueries.list(auth, layerId);
+                if (found === undefined) {
+                    // No such layer and no permission to administer it, one answer.
+                    const problem = notFound(instance, requestId);
+                    send(res, problem.status, problem.toJSON(), requestId);
+                    return;
+                }
+                send(res, 200, { items: found.map(referenceQueryJson) }, requestId);
+                return;
+            }
+            const parsed = parseReferenceQueries(body);
+            if ('error' in parsed) {
+                const problem = badRequest(instance, requestId, parsed.error);
+                send(res, problem.status, problem.toJSON(), requestId);
+                return;
+            }
+            const replaced = await options.referenceQueries.replace(auth, layerId, parsed.queries);
+            await options.audit.write({
+                orgId: auth.orgId,
+                actor: `${auth.principal.type}:${auth.principal.id}`,
+                action: 'reference_queries.replace',
+                result: replaced === undefined ? 'deny' : 'allow',
+                target: { layer_id: layerId },
+                // The count and never the queries. They are the operator's own text
+                // rather than a caller's search, so this is not the rule about query
+                // text — but the journal is read by more people than the endpoint is,
+                // and a document's title has already reached it once by this route.
+                detail: { layer_id: layerId, queries: parsed.queries.length },
+                requestId,
+            });
+            if (replaced === undefined) {
+                const problem = notFound(instance, requestId);
+                send(res, problem.status, problem.toJSON(), requestId);
+                return;
+            }
+            send(res, 200, { items: replaced.map(referenceQueryJson) }, requestId);
+            return;
+        }
         // `/v1/layers/{id}/reindex`
         const reindexPath = /^\/v1\/layers\/([0-9a-f-]{36})\/reindex$/i.exec(instance);
         if (reindexPath !== null) {