@opendatalabs/personal-server-ts-core 1.33.7 → 1.34.0

package/dist/api/index.js

@@ -1,9 +1,12 @@
+import { isSystemScope } from "../policy/reserved-scopes.js";
 import { DataDeletedError, DeleteTombstoneFailedError, GrantRequiredError, InvalidCascadeError, LineageCascadeUnavailableError, LineageGatewayError, LineageUnavailableError, ProtocolError, ServerNotConfiguredError, } from "../errors/catalog.js";
 import { computeDataPointId } from "../sync/data-point-id.js";
 import { isEntryCoveredByTombstone, } from "../sync/scope-deletions.js";
+import { isPermanentRejection, } from "../sync/pdpp-import.js";
 import { supersedeLegacyVersions, } from "../storage/legacy-projection.js";
 import { deleteScope as deleteScopeLocally, } from "../sync/workers/delete.js";
-import { ingestDataContract, ingestBinaryDataContract, listAccessLogsContract, listDataScopesContract, listDataVersionsContract, parseDataScopeContract, parseJsonObjectBody, readDataContract, syncFileContract, triggerSyncContract, getSyncStatusContract, configReadErrorContract, configWriteErrorContract, createGrantContract, listGrantsContract, oauthTokenContract, revokeGrantContract, validateServerConfigContract, verifyGrantContract, decodeBinaryEnvelope, isBinaryEnvelope, parseMetadataHeader, stringifyMetadataHeader, IngestPersistedError, } from "../contracts/index.js";
+import { ingestDataContract, ingestBinaryDataContract, listAccessLogsContract, listDataScopesContract, listDataVersionsContract, parseDataScopeContract, parseJsonObjectBody, readDataContract, syncFileContract, triggerSyncContract, getSyncStatusContract, configReadErrorContract, configWriteErrorContract, createGrantContract, listGrantsContract, oauthTokenContract, revokeGrantContract, validateServerConfigContract, verifyGrantContract, decodeBinaryEnvelope, isBinaryEnvelope, parseMetadataHeader, stringifyMetadataHeader, IngestPersistedError, sha256Hex, } from "../contracts/index.js";
+import { canonicalJsonBytes } from "../derivatives/e2ee/jcs.js";
 // Imported directly: the contracts barrel does not re-export this addition.
 import { summarizeDataAdditionsContract } from "../contracts/data.js";
 import { scopeCoveredByGrant } from "@opendatalabs/vana-sdk/browser";
@@ -86,17 +89,80 @@ function methodNotAllowed() {
 function notFound() {
     return errorResponse(404, "NOT_FOUND", "Not found");
 }
-async function withApiErrors(handler) {
+async function withApiErrors(handler, context) {
     try {
-        return await handler();
+        const response = await handler();
+        // A handled 4xx that means "the index and the disk disagree" is an
+        // operator-facing fault even though it is not a 500 — log it once, here,
+        // rather than at the contract that cannot see the route.
+        if (response.status === 404 && context) {
+            await logMissingDataFile(response, context);
+        }
+        return response;
     }
     catch (err) {
         if (err instanceof ProtocolError) {
             return protocolErrorResponse(err);
         }
+        if (context) {
+            context.logger?.error?.({
+                route: context.route,
+                requestId: context.requestId ?? crypto.randomUUID(),
+                errorCode: "INTERNAL_ERROR",
+                ...stripUndefined(context.detail),
+                err: describeError(err),
+            }, "Unhandled error in personal server API");
+        }
         return errorResponse(500, "INTERNAL_ERROR", "Internal server error");
     }
 }
+/**
+ * The `DATA_FILE_MISSING` 404 the read contract produces, logged at the one
+ * layer that knows which route served it. Peeks at a clone so the response
+ * body stays readable by the caller.
+ */
+async function logMissingDataFile(response, context) {
+    if (!context.logger?.error)
+        return;
+    try {
+        const body = await response.clone().json();
+        if (typeof body !== "object" ||
+            body === null ||
+            body.error !== "DATA_FILE_MISSING") {
+            return;
+        }
+        context.logger.error({
+            route: context.route,
+            requestId: context.requestId ?? crypto.randomUUID(),
+            errorCode: "DATA_FILE_MISSING",
+            ...stripUndefined(context.detail),
+        }, "Indexed data file is missing on disk");
+    }
+    catch {
+        // A diagnostic log must not change the response sent to the caller.
+    }
+}
+/**
+ * An error rendered for a log line — name, message and stack only. Never the
+ * whole object: a thrown error can carry a request, a config or a token on an
+ * arbitrary property, and a structured logger would serialize all of it.
+ */
+function describeError(err) {
+    if (err instanceof Error) {
+        return {
+            name: err.name,
+            message: err.message,
+            ...("code" in err && { code: err.code }),
+            stack: err.stack,
+        };
+    }
+    return { message: String(err) };
+}
+function stripUndefined(detail) {
+    if (!detail)
+        return {};
+    return Object.fromEntries(Object.entries(detail).filter((pair) => pair[1] !== undefined));
+}
 function normalizeLimit(value, fallback) {
     if (value === null)
         return fallback;
@@ -129,7 +195,8 @@ export function selectedGrantId(request, url) {
         undefined);
 }
 function listAuthCovers(listAuth, scope) {
-    return scopeCoveredByGrant(scope, [...listAuth.grantedScopes]);
+    return (!isSystemScope(scope) &&
+        scopeCoveredByGrant(scope, [...listAuth.grantedScopes]));
 }
 /**
  * Run the X402 dispatch for one read:
@@ -293,33 +360,102 @@ export async function handleX402Cycle(input) {
     }
     return { kind: "gateway-error", status: gatewayRes.status, body: errorBody };
 }
-const MAX_INGEST_STAMP_ADVANCE_SEC = 60;
 const pendingIngestStamps = new WeakMap();
-/** Reserve before awaiting ingestion; keep the existing second-precision paths. */
-function reserveIngestStamp(storage, scope, now) {
-    let pending = pendingIngestStamps.get(storage);
-    if (!pending) {
-        pending = new Set();
-        pendingIngestStamps.set(storage, pending);
-    }
-    const base = Math.floor(now().getTime() / 1000) * 1000;
-    for (let bump = 0; bump < MAX_INGEST_STAMP_ADVANCE_SEC; bump += 1) {
-        const value = new Date(base + bump * 1000)
-            .toISOString()
-            .replace(/\.\d{3}Z$/, "Z");
-        const key = `${scope}:${value}`;
-        if (pending.has(key) ||
-            storage.findEntry({ scope, at: value })?.collectedAt === value)
+function reserveIngestStamp(now, storage, scope) {
+    // Bound allocation to 60 millisecond slots per scope, anchored to wall time.
+    // Failed requests release their slot; neither failures nor other scopes
+    // advance this scope's clock. The durable head protects restart/clock rollback.
+    const wall = now().getTime();
+    const latest = Date.parse(storage.findEntry({ scope })?.collectedAt ?? "");
+    let scopes = pendingIngestStamps.get(storage);
+    const pending = scopes?.get(scope) ?? new Set();
+    for (let stamp = wall; stamp < wall + 60; stamp += 1) {
+        if ((Number.isFinite(latest) && stamp <= latest) || pending.has(stamp)) {
             continue;
-        pending.add(key);
+        }
+        if (!scopes) {
+            scopes = new Map();
+            pendingIngestStamps.set(storage, scopes);
+        }
+        scopes.set(scope, pending);
+        pending.add(stamp);
+        const reservations = scopes;
         return {
-            value,
-            release: () => {
-                pending.delete(key);
+            collectedAt: new Date(stamp).toISOString(),
+            release() {
+                pending.delete(stamp);
+                if (pending.size === 0)
+                    reservations.delete(scope);
             },
         };
     }
-    throw new ProtocolError(429, "INGEST_CAPACITY", "Too many writes to this scope; retry after a minute");
+    return undefined;
+}
+function parseLegacyPrecondition(request) {
+    const none = request.headers.get("if-none-match");
+    const match = request.headers.get("if-match");
+    if (none !== null && match !== null) {
+        return errorResponse(400, "INVALID_PRECONDITION", "Supply only one precondition");
+    }
+    if (none !== null) {
+        return none === "*"
+            ? { kind: "none" }
+            : errorResponse(400, "INVALID_PRECONDITION", "If-None-Match must be *");
+    }
+    if (match !== null) {
+        const parsed = /^"([1-9]\d*)"$/.exec(match);
+        const version = parsed ? Number(parsed[1]) : NaN;
+        return Number.isSafeInteger(version)
+            ? { kind: "match", version }
+            : errorResponse(400, "INVALID_PRECONDITION", "If-Match must be a quoted positive version");
+    }
+    return undefined;
+}
+function parseProducerHeader(request) {
+    const producer = request.headers.get("vana-producer");
+    const encodedProvenance = request.headers.get("vana-producer-provenance") ?? undefined;
+    if (producer === null && !encodedProvenance)
+        return {};
+    if (producer !== "pdpp-projector" && producer !== "pdpp-import-projection") {
+        return errorResponse(400, "INVALID_PRODUCER", "Vana-Producer is unrecognized or missing");
+    }
+    if (encodedProvenance &&
+        (encodedProvenance.length > 8192 ||
+            !/^[A-Za-z0-9_-]+$/.test(encodedProvenance))) {
+        return errorResponse(400, "INVALID_PROVENANCE", "Vana-Producer-Provenance is malformed");
+    }
+    return { producer, encodedProvenance };
+}
+async function parseProducerProvenance(encoded, body) {
+    if (!encoded)
+        return undefined;
+    try {
+        const value = JSON.parse(new TextDecoder().decode(Uint8Array.from(atob(encoded.replace(/-/g, "+").replace(/_/g, "/")), (char) => char.charCodeAt(0))));
+        if (!value ||
+            typeof value !== "object" ||
+            typeof value.projector_version !== "string" ||
+            !value.projector_version ||
+            typeof value.declaration_digest !== "string" ||
+            !value.declaration_digest ||
+            !Array.isArray(value.inputs) ||
+            !value.inputs.every((input) => input &&
+                typeof input.stream === "string" &&
+                input.stream &&
+                typeof input.changes_since_token === "string" &&
+                input.changes_since_token) ||
+            typeof value.payload_sha256 !== "string" ||
+            !/^(?:0x)?[0-9a-fA-F]{64}$/.test(value.payload_sha256)) {
+            return errorResponse(400, "INVALID_PROVENANCE", "Producer provenance has invalid fields");
+        }
+        const actual = (await sha256Hex(canonicalJsonBytes(body))).slice(2);
+        if (actual !== value.payload_sha256.replace(/^0x/, "").toLowerCase()) {
+            return errorResponse(400, "PAYLOAD_HASH_MISMATCH", "Provenance payload_sha256 does not match the JSON body");
+        }
+        return value;
+    }
+    catch {
+        return errorResponse(400, "INVALID_PROVENANCE", "Vana-Producer-Provenance is not base64url JSON");
+    }
 }
 /**
  * The deletion that applies to a read of `scope`, or null when the read may
@@ -335,15 +471,25 @@ function reserveIngestStamp(storage, scope, now) {
  * Gateway unreachability never fails the read: the tracker answers from its
  * last known state, which is the documented offline behaviour.
  */
-export async function resolveReadDeletion(deps, scope, entry) {
+export async function resolveReadDeletion(deps, scope, entry, body = "file") {
     if (!deps.scopeDeletions)
         return null;
     const verdict = await deps.scopeDeletions.resolve(scope, {
         consultGateway: entry ? "if-stale" : "always",
     });
-    if (!verdict.deleted)
+    // A live DPv2 file cannot establish that a different canonical body was
+    // re-added. Keep this refusal until a genuine canonical re-add seam exists.
+    const tombstone = body === "canonical"
+        ? (deps.scopeDeletions.deletionHistory?.get(scope) ??
+            (verdict.deleted ? verdict : null))
+        : verdict.deleted
+            ? verdict
+            : null;
+    if (!tombstone)
         return null;
-    if (entry && !isEntryCoveredByTombstone(entry, verdict)) {
+    if (body === "file" &&
+        entry &&
+        !isEntryCoveredByTombstone(entry, tombstone)) {
         return null;
     }
     return {
@@ -351,7 +497,7 @@ export async function resolveReadDeletion(deps, scope, entry) {
         dataPointId: deps.serverOwner
             ? computeDataPointId(deps.serverOwner, scope)
             : null,
-        deletedAt: verdict.deletedAt,
+        deletedAt: tombstone.deletedAt,
     };
 }
 /**
@@ -359,8 +505,8 @@ export async function resolveReadDeletion(deps, scope, entry) {
  * served. Call it before any payment step: a builder must never be charged
  * for a scope the owner deleted.
  */
-export async function assertScopeNotDeleted(deps, scope, entry) {
-    const deletion = await resolveReadDeletion(deps, scope, entry);
+export async function assertScopeNotDeleted(deps, scope, entry, body = "file") {
+    const deletion = await resolveReadDeletion(deps, scope, entry, body);
     if (deletion)
         throw new DataDeletedError(deletion);
 }
@@ -374,6 +520,82 @@ function discoveryVisibility(deps) {
         return undefined;
     return async (scope, entry) => (await resolveReadDeletion(deps, scope, entry)) === null;
 }
+/**
+ * Discovery filter for the scope listing: the deletion filter, and a
+ * canonical scope is listed only when the view could serve it.
+ */
+function listingVisibility(deps) {
+    const deletion = discoveryVisibility(deps);
+    const view = deps.legacyScopeView;
+    if (!view)
+        return deletion;
+    return async (scope, entry) => {
+        const listing = view.listingFor(scope);
+        if (listing.kind === "refused")
+            return false;
+        const canonicalOwned = view.ownsScope(scope);
+        // A file's re-add marker cannot revive the canonical body this row
+        // describes. Canonical re-adds have no registered version yet.
+        return ((await resolveReadDeletion(deps, scope, canonicalOwned ? undefined : entry, canonicalOwned ? "canonical" : "file")) === null);
+    };
+}
+/**
+ * Bound scopes served only from canonical records, for the listing: no
+ * legacy write is needed to discover them. A gateway deletion hides them
+ * the way it hides a stored scope.
+ */
+function canonicalOnlyScopes(deps) {
+    const view = deps.legacyScopeView;
+    if (!view)
+        return undefined;
+    return async (scopePrefix, stored) => {
+        const listed = [];
+        for (const { scope, collectedAt } of view.listCanonicalScopes(scopePrefix)) {
+            if (stored.has(scope))
+                continue;
+            if ((await resolveReadDeletion(deps, scope, undefined, "canonical")) !==
+                null) {
+                continue;
+            }
+            // No legacy version exists to count or to ask `/versions` for.
+            listed.push({ scope, latestCollectedAt: collectedAt, versionCount: 0 });
+        }
+        return listed;
+    };
+}
+/**
+ * A canonical scope's listing row carries the canonical freshness. The
+ * legacy head's size and block readiness describe a file the latest read
+ * does not serve, so they are dropped.
+ */
+function withCanonicalListing(deps, summary) {
+    const listing = deps.legacyScopeView?.listingFor(summary.scope);
+    if (listing?.kind !== "canonical")
+        return summary;
+    return {
+        scope: summary.scope,
+        latestCollectedAt: listing.collectedAt,
+        versionCount: summary.versionCount,
+    };
+}
+/**
+ * 409 when the owner must choose between connections, 503 otherwise. The
+ * body names the reason and stream names only: no instance id, no row data,
+ * no legacy content.
+ */
+function legacyViewRefusalResponse(scope, refusal) {
+    const status = refusal.reason === "ambiguous_connection" ? 409 : 503;
+    return jsonResponse({
+        error: "CANONICAL_VIEW_UNAVAILABLE",
+        message: `The latest data for scope "${scope}" cannot be served from the owner's connector records (${refusal.reason}).`,
+        status: refusal.status,
+        reason: refusal.reason,
+        ...("detail" in refusal ? { detail: refusal.detail } : {}),
+        ...("streams" in refusal ? { streams: refusal.streams } : {}),
+        ...("stream" in refusal ? { stream: refusal.stream } : {}),
+        ...("candidates" in refusal ? { candidates: refusal.candidates } : {}),
+    }, { status });
+}
 /**
  * The causal marker a fresh ingest carries: the tombstone version this
  * replica currently knows for the scope, if any, so the new entry is a
@@ -393,12 +615,11 @@ async function ingestTombstoneMarker(deps, scope) {
 }
 // The delete worker wants a full Logger; the API logger is all-optional.
 function apiLoggerAsLogger(logger) {
-    const noop = () => undefined;
     return {
-        debug: (payload, message) => (logger?.debug ?? noop)(payload, message ?? ""),
-        info: (payload, message) => (logger?.info ?? noop)(payload, message ?? ""),
-        warn: (payload, message) => (logger?.warn ?? noop)(payload, message ?? ""),
-        error: (payload, message) => (logger?.error ?? noop)(payload, message ?? ""),
+        debug: (payload, message) => logger?.debug?.(payload, message ?? ""),
+        info: (payload, message) => logger?.info?.(payload, message ?? ""),
+        warn: (payload, message) => logger?.warn?.(payload, message ?? ""),
+        error: (payload, message) => logger?.error?.(payload, message ?? ""),
     };
 }
 function notifyDataWritten(deps, event) {
@@ -414,6 +635,50 @@ function notifyDataWritten(deps, event) {
         }, "onDataWritten hook failed; record already stored");
     }
 }
+/**
+ * Offer a just-ingested envelope to the PDPP importer.
+ *
+ * The local-ingest twin of the download worker's `importIntoPdpp`, and
+ * deliberately the same shape: same `ImportableEnvelope`, same total handling
+ * of every outcome, same permanent-vs-transient logging split. Keeping the
+ * two symmetric is the point — a `$pdpp` envelope must mean the same thing
+ * and reach the same store whether the gateway handed it over or a local
+ * producer POSTed it, otherwise "imported" would depend on which door the
+ * bytes came through.
+ *
+ * `skipped` is the ordinary legacy case (no `$pdpp` block) and stays silent.
+ * A rejection is a WARNING, not an error: it means the producer sent
+ * something this deployment could not verify — actionable, but not a failure
+ * of the write, which has already committed.
+ */
+function importIngestedIntoPdpp(deps, envelope) {
+    if (!deps.pdppImporter)
+        return;
+    try {
+        const outcome = deps.pdppImporter.importEnvelope(envelope);
+        if (outcome.status === "rejected") {
+            const permanent = isPermanentRejection(outcome.rejection);
+            deps.logger?.warn?.({
+                scope: envelope.scope,
+                collectedAt: envelope.collectedAt,
+                code: outcome.rejection.code,
+                message: outcome.rejection.message,
+                permanent,
+            }, permanent
+                ? "Ingested data point was refused by the PDPP importer and will not be retried"
+                : "PDPP import unavailable for this ingested data point");
+        }
+    }
+    catch (err) {
+        // A throwing importer is infrastructure failing, not the envelope being
+        // wrong. The record is stored and indexed already, so this must never
+        // turn a committed write into a 500.
+        deps.logger?.warn?.({
+            scope: envelope.scope,
+            error: err instanceof Error ? err.message : String(err),
+        }, "PDPP import threw on ingest; record already stored and indexed");
+    }
+}
 function notifyDataRead(deps, event) {
     if (!deps.onDataRead)
         return;
@@ -571,6 +836,15 @@ export function redactEnvelopeForGrantee(envelope) {
 }
 export { redactJsonEnvelopeBytesForGrantee } from "./raw-envelope.js";
 export async function handlePersonalServerDataRequest(request, deps, options = {}) {
+    // Filled in as the dispatch narrows, so a 500 thrown deep in a read is
+    // logged against the route that served it rather than a generic label. It
+    // is mutable for exactly that reason: the route is not known until the path
+    // has been parsed, and by then we are already inside the try.
+    const logContext = {
+        logger: deps.logger,
+        route: `${request.method} ${options.basePath ?? "/v1/data"}`,
+        requestId: crypto.randomUUID(),
+    };
     return withApiErrors(async () => {
         const url = new URL(request.url);
         const pathname = stripBasePath(url.pathname, options.basePath);
@@ -585,12 +859,15 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 scopePrefix: url.searchParams.get("scopePrefix") ?? undefined,
                 limit: normalizeLimit(url.searchParams.get("limit"), 20),
                 offset: normalizeLimit(url.searchParams.get("offset"), 0),
-                isVisible: discoveryVisibility(deps),
-                isListable: listAuth
-                    ? (scope) => listAuthCovers(listAuth, scope)
-                    : undefined,
+                isVisible: listingVisibility(deps),
+                isListable: (scope) => !isSystemScope(scope) &&
+                    (!listAuth || listAuthCovers(listAuth, scope)),
+                extraScopes: canonicalOnlyScopes(deps),
+            });
+            return jsonResponse({
+                ...result.response,
+                scopes: result.response.scopes.map((summary) => withCanonicalListing(deps, summary)),
             });
-            return jsonResponse(result.response);
         }
         // "additions" has no dot, so it can never be a real scope name (scopes are
         // "source.dataset"); handle it before the single-segment scope path.
@@ -601,6 +878,7 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
             await deps.auth.authorizeOwner(request);
             const daysRaw = url.searchParams.get("days");
             const result = await summarizeDataAdditionsContract({
+                additions: deps.additions,
                 storage: deps.storage,
                 isVisible: discoveryVisibility(deps),
                 timezone: url.searchParams.get("tz") ?? "UTC",
@@ -617,15 +895,80 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
             return jsonResponse(result);
         }
         const parts = pathname.split("/").filter(Boolean);
+        if (parts.length === 3 && parts[1] === "imports") {
+            if (request.method !== "PUT")
+                return methodNotAllowed();
+            await deps.auth.authorizeOwner(request);
+            const scopeResult = parseDataScopeContract(decodePathPart(parts[0]));
+            if (!scopeResult.ok)
+                return contractErrorResponse(scopeResult);
+            const operation = decodePathPart(parts[2]);
+            if (!/^[a-f0-9]{64}$/.test(operation)) {
+                return errorResponse(400, "INVALID_IMPORT_OPERATION", "operation must be 64 lowercase hexadecimal characters");
+            }
+            if (!deps.storage.commitVersion) {
+                throw new ProtocolError(501, "IMPORT_UNSUPPORTED", "This storage backend does not support durable owner imports");
+            }
+            if (!deps.serverOwner)
+                throw new ServerNotConfiguredError({
+                    reason: "serverOwner is required for durable owner imports",
+                });
+            if (!isJsonContentType(request) || url.search.length > 0) {
+                return errorResponse(400, "INVALID_IMPORT", "Owner imports accept JSON root writes without query parameters");
+            }
+            const parsed = await parseJsonObjectBody(request, "Request body must be valid JSON");
+            if (!parsed.ok)
+                return contractResponse(parsed.result);
+            if ("lineage" in parsed.body || "$binary" in parsed.body) {
+                return errorResponse(400, "INVALID_IMPORT", "Owner imports do not accept binary or lineage writes");
+            }
+            const bodySha256 = await sha256Hex(canonicalJsonBytes(parsed.body));
+            const authority = deps.gatewayConfig
+                ? `${deps.gatewayConfig.chainId}:${deps.gatewayConfig.contracts.dataRegistry.toLowerCase()}`
+                : "local";
+            const result = await ingestDataContract({
+                storage: deps.storage,
+                scopeParam: scopeResult.scope,
+                body: parsed.body,
+                collectedAt: (deps.now ?? (() => new Date()))().toISOString(),
+                status: deps.syncManager ? "syncing" : "stored",
+                afterTombstoneVersion: await ingestTombstoneMarker(deps, scopeResult.scope),
+                import: {
+                    operation,
+                    bodySha256,
+                    owner: deps.serverOwner.toLowerCase(),
+                    authority,
+                },
+            });
+            if (!result.ok)
+                return contractErrorResponse(result);
+            notifyNewData(deps.syncManager);
+            if (!result.replay)
+                notifyDataWritten(deps, {
+                    scope: result.scope,
+                    collectedAt: result.collectedAt,
+                });
+            return jsonResponse(result.response, {
+                status: result.replay ? 200 : 201,
+            });
+        }
         if (parts.length === 2 && parts[1] === "versions") {
             if (request.method !== "GET")
                 return methodNotAllowed();
-            const listAuth = await deps.auth.authorizeBuilderList(request, {
-                grantId: selectedGrantId(request, url),
-            });
+            const versionsScopeParam = decodePathPart(parts[0]);
+            const versionsGrantId = selectedGrantId(request, url);
+            const listAuth = deps.auth.authorizeScopeVersions
+                ? await deps.auth.authorizeScopeVersions({
+                    request,
+                    scope: versionsScopeParam,
+                    grantId: versionsGrantId,
+                })
+                : await deps.auth.authorizeBuilderList(request, {
+                    grantId: versionsGrantId,
+                });
             // A scope's version history is as much its data as its content is, so
             // a non-owner needs a grant covering it, exactly like a read.
-            const scopeResult = parseDataScopeContract(decodePathPart(parts[0]));
+            const scopeResult = parseDataScopeContract(versionsScopeParam);
             if (scopeResult.ok &&
                 listAuth &&
                 !listAuthCovers(listAuth, scopeResult.scope)) {
@@ -738,10 +1081,16 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
         if (parts.length !== 1)
             return notFound();
         const scopeParam = decodePathPart(parts[0]);
+        logContext.route = `${request.method} ${options.basePath ?? "/v1/data"}/:scope`;
         if (request.method === "GET") {
             const scopeResult = parseDataScopeContract(scopeParam);
             if (!scopeResult.ok)
                 return contractErrorResponse(scopeResult);
+            logContext.detail = { scope: scopeResult.scope };
+            // An empty `fileId=` or `at=` selects no version: it is the latest
+            // read, as storage already treats it.
+            const fileIdParam = url.searchParams.get("fileId") || undefined;
+            const atParam = url.searchParams.get("at") || undefined;
             // `?view=stored` returns the body as stored, without the read-time
             // legacy projection. Owner only; checked after auth below.
             const view = url.searchParams.get("view");
@@ -750,8 +1099,8 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
             }
             const selectedEntry = deps.storage.findEntry({
                 scope: scopeResult.scope,
-                fileId: url.searchParams.get("fileId") ?? undefined,
-                at: url.searchParams.get("at") ?? undefined,
+                fileId: fileIdParam,
+                at: atParam,
             });
             // Holds the base64-encoded gateway payForOperation response body when
             // X402 succeeds — emitted on the final read response as X-PAYMENT-RESPONSE.
@@ -761,13 +1110,22 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 request,
                 scope: scopeResult.scope,
                 grantId,
-                fileId: url.searchParams.get("fileId") ?? selectedEntry?.fileId ?? undefined,
-                at: url.searchParams.get("at") ?? undefined,
+                fileId: fileIdParam ?? selectedEntry?.fileId ?? undefined,
+                at: atParam,
             });
+            // Select the body after auth and before deletion or payment. A file's
+            // version and re-add marker authorize that file, not canonical rows.
+            // `?view=stored` asks for the stored file's body, so it never reads the
+            // canonical view either.
+            const latest = fileIdParam === undefined && atParam === undefined && view === null
+                ? await deps.legacyScopeView?.readLatest(scopeResult.scope)
+                : undefined;
+            const canonicalOwned = latest !== undefined &&
+                deps.legacyScopeView?.ownsScope(scopeResult.scope);
             // Deletion gate, after auth (so unauthenticated callers learn nothing)
             // and before payment (so nobody is charged for deleted data). Covers
             // both a stale local copy of a tombstoned scope and a local miss.
-            await assertScopeNotDeleted(deps, scopeResult.scope, selectedEntry);
+            await assertScopeNotDeleted(deps, scopeResult.scope, canonicalOwned ? undefined : selectedEntry, canonicalOwned ? "canonical" : "file");
             if (view === "stored" && !isOwnerView(authResult)) {
                 return errorResponse(403, "FORBIDDEN", "Only the owner can read the stored view");
             }
@@ -780,6 +1138,13 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
             // answer unreachable through this route. Nothing here blocks: a
             // previous version is served now, the fresh one lands in a later read.
             notifyDataRead(deps, { scope: scopeResult.scope });
+            // A canonical scope's latest body is the canonical view, and only it:
+            // a refusal is answered, never replaced by the legacy file. Explicit
+            // versions stay legacy history. Read before payment, so a builder is
+            // never challenged or charged for a read that is then refused.
+            if (latest?.kind === "refused") {
+                return legacyViewRefusalResponse(scopeResult.scope, latest.refusal);
+            }
             // X402 payment dance for builder reads. Owner-exempt reads (the
             // grantId sentinels "owner" / "policy-bypass") skip payment entirely
             // since there's no payable op to attach the payment to.
@@ -833,6 +1198,12 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                     willCharge,
                 });
             }
+            if (willCharge && latest?.kind === "canonical") {
+                return legacyViewRefusalResponse(scopeResult.scope, {
+                    status: "unavailable",
+                    reason: "paid_read_unsupported",
+                });
+            }
             // Keep the full condition inline (not `if (willCharge)`) so TS narrows
             // deps.gateway/gatewayConfig/gatewayUrl to non-undefined in the block.
             if (deps.paymentEnabled &&
@@ -847,8 +1218,8 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                     deps,
                     request,
                     scope: scopeResult.scope,
-                    fileIdParam: url.searchParams.get("fileId") ?? undefined,
-                    atParam: url.searchParams.get("at") ?? undefined,
+                    fileIdParam,
+                    atParam,
                     grantId: resolvedGrantId,
                     builder: builder,
                     gateway: deps.gateway,
@@ -869,25 +1240,38 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 // breakdown / paidAt / paymentNonce without a second gateway round-trip.
                 paymentResponseHeader = encodePaymentResponseHeader(x402Result.payResponse);
             }
-            const result = await readDataContract({
-                storage: deps.storage,
-                scopeParam: scopeResult.scope,
-                fileId: url.searchParams.get("fileId") ?? undefined,
-                at: url.searchParams.get("at") ?? undefined,
-                ...(view === "stored" ? { view } : {}),
-            });
-            if (!result.ok) {
-                if (result.status === 404) {
-                    // The entry selected above can vanish before the read (a sync
-                    // reconcile applying a tombstone concurrently); answer 410, not
-                    // 404, when that is why.
-                    await assertScopeNotDeleted(deps, scopeResult.scope, undefined);
+            const headers = {};
+            headers["X-Vana-Data-Source"] =
+                latest?.kind === "canonical" ? "canonical" : "legacy";
+            let envelope;
+            let servedFileId;
+            if (latest?.kind === "canonical") {
+                envelope = latest.envelope;
+                headers.ETag = `"${latest.revision}"`;
+            }
+            else {
+                const result = await readDataContract({
+                    storage: deps.storage,
+                    scopeParam: scopeResult.scope,
+                    fileId: fileIdParam,
+                    at: atParam,
+                    ...(view === "stored" ? { view } : {}),
+                });
+                if (!result.ok) {
+                    if (result.status === 404) {
+                        // The entry selected above can vanish before the read (a sync
+                        // reconcile applying a tombstone concurrently); answer 410, not
+                        // 404, when that is why.
+                        await assertScopeNotDeleted(deps, scopeResult.scope, undefined);
+                    }
+                    return contractErrorResponse(result);
                 }
-                return contractErrorResponse(result);
+                envelope = result.envelope;
+                servedFileId = fileIdParam ?? selectedEntry?.fileId ?? undefined;
             }
             const servedEnvelope = isOwnerView(authResult)
-                ? result.envelope
-                : redactEnvelopeForGrantee(result.envelope);
+                ? envelope
+                : redactEnvelopeForGrantee(envelope);
             const logId = deps.createLogId?.() ?? crypto.randomUUID();
             const timestamp = (deps.now ?? (() => new Date()))().toISOString();
             const ipAddress = request.headers.get("x-forwarded-for") ??
@@ -911,9 +1295,7 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                     return;
                 reportPersonalServerReadFulfillment(deps, {
                     builder: authResult.builder,
-                    fileId: url.searchParams.get("fileId") ??
-                        selectedEntry?.fileId ??
-                        undefined,
+                    fileId: servedFileId,
                     grantId: authResult.grantId,
                     ipAddress,
                     logId,
@@ -924,7 +1306,6 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                     userAgent,
                 });
             };
-            const headers = {};
             if (paymentResponseHeader) {
                 headers["X-PAYMENT-RESPONSE"] = paymentResponseHeader;
             }
@@ -989,7 +1370,7 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
             // so the builder's retry with the same still-valid proof is accepted;
             // after commit the proof stays consumed (a retry would be a duplicate).
             let committed = false;
-            let releaseStamp;
+            let ingestStamp;
             const failWrite = async (response) => {
                 await writeAuth?.releaseProof?.();
                 return response;
@@ -998,9 +1379,20 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 const scopeResult = parseDataScopeContract(scopeParam);
                 if (!scopeResult.ok)
                     return failWrite(contractErrorResponse(scopeResult));
-                const stamp = reserveIngestStamp(deps.storage, scopeResult.scope, deps.now ?? (() => new Date()));
-                releaseStamp = stamp.release;
-                const collectedAtValue = stamp.value;
+                const parsedPrecondition = parseLegacyPrecondition(request);
+                if (parsedPrecondition instanceof Response)
+                    return failWrite(parsedPrecondition);
+                const producerHeader = parseProducerHeader(request);
+                if (producerHeader instanceof Response)
+                    return failWrite(producerHeader);
+                if (writeAuth && producerHeader.producer) {
+                    return failWrite(errorResponse(403, "NOT_OWNER", "Only the owner may assert a producer"));
+                }
+                ingestStamp = reserveIngestStamp(deps.now ?? (() => new Date()), deps.storage, scopeResult.scope);
+                if (!ingestStamp) {
+                    return failWrite(errorResponse(429, "INGEST_CAPACITY", "Too many writes for this scope; retry when the clock advances"));
+                }
+                const collectedAtValue = ingestStamp.collectedAt;
                 const status = deps.syncManager ? "syncing" : "stored";
                 const afterTombstoneVersion = await ingestTombstoneMarker(deps, scopeResult.scope);
                 // Builder writes land in the same access log as builder reads, so the
@@ -1040,6 +1432,9 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 // data needs no schema at all — we ingest it schemaless. (Structured JSON
                 // below still resolves a schema for validation/metadata.)
                 if (!isJsonContentType(request)) {
+                    if (producerHeader.producer) {
+                        return failWrite(errorResponse(400, "INVALID_PRODUCER", "Producer attribution requires a JSON body"));
+                    }
                     if (supersede !== null) {
                         return failWrite(errorResponse(400, "INVALID_SUPERSEDE", "supersede applies to JSON writes only"));
                     }
@@ -1060,6 +1455,7 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                         attribution: writeAuth?.attribution,
                         lineage,
                         afterTombstoneVersion,
+                        precondition: parsedPrecondition,
                     });
                     if (!result.ok)
                         return failWrite(contractErrorResponse(result));
@@ -1084,6 +1480,9 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 const parsed = await parseJsonObjectBody(request, "Request body must be valid JSON");
                 if (!parsed.ok)
                     return failWrite(contractResponse(parsed.result));
+                const provenance = await parseProducerProvenance(producerHeader.encodedProvenance, parsed.body);
+                if (provenance instanceof Response)
+                    return failWrite(provenance);
                 // Derivative writes: validate the caller's `lineage` before anything
                 // is stored (a failure here throws, releasing the builder's proof).
                 const lineage = await prepareWriteLineage(deps, scopeResult.scope, extractLineageField(parsed.body));
@@ -1098,9 +1497,14 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                     attribution: writeAuth?.attribution,
                     lineage,
                     afterTombstoneVersion,
+                    precondition: parsedPrecondition,
+                    producer: producerHeader.producer,
+                    producerProvenance: provenance,
                 });
                 if (!result.ok)
                     return failWrite(contractErrorResponse(result));
+                if (result.replay)
+                    throw new Error("Ordinary POST cannot replay an import");
                 committed = true;
                 deps.logger?.info?.({
                     scope: scopeResult.scope,
@@ -1113,6 +1517,14 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                     ? await supersedeAfterCommit(deps, scopeResult.scope, collectedAtValue)
                     : undefined;
                 notifyNewData(deps.syncManager);
+                // The parsed body is the envelope's `data`, which is where a producer
+                // stamps `$pdpp`. Offered here rather than inside the ingest contract
+                // so the import stays strictly after the record is durable.
+                importIngestedIntoPdpp(deps, {
+                    scope: scopeResult.scope,
+                    collectedAt: collectedAtValue,
+                    data: parsed.body,
+                });
                 notifyDataWritten(deps, {
                     scope: scopeResult.scope,
                     collectedAt: collectedAtValue,
@@ -1139,7 +1551,7 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                         error: err.cause instanceof Error
                             ? err.cause.message
                             : String(err.cause),
-                    }, "Envelope written but indexing failed; record persisted unindexed");
+                    }, "Envelope write incomplete after durable commit; boot recovery may finish it");
                 }
                 else if (!committed) {
                     await writeAuth?.releaseProof?.();
@@ -1147,7 +1559,7 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 throw err;
             }
             finally {
-                releaseStamp?.();
+                ingestStamp?.release();
             }
         }
         if (request.method === "DELETE") {
@@ -1170,6 +1582,18 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
                 // first tombstone). Until that lands the only honest answer is 501.
                 throw new LineageCascadeUnavailableError({ scope: parsed.scope });
             }
+            // A canonical-owned scope's latest body is its connection's rows. No
+            // store seam deletes one legacy scope's rows (streams can be shared,
+            // and the account and its generation are the unit of erasure), so a
+            // 200 here would leave the data readable. Refuse before any gateway or
+            // file change; the connection's own delete erases its rows.
+            if (deps.legacyScopeView?.ownsScope(parsed.scope)) {
+                return jsonResponse({
+                    error: "CANONICAL_DELETE_UNSUPPORTED",
+                    message: `Scope "${parsed.scope}" is served from the owner's connector records, which this route cannot delete. Nothing was deleted; delete the connection to erase its records.`,
+                    scope: parsed.scope,
+                }, { status: 409 });
+            }
             // Durable order: gateway tombstone -> storage blobs -> local copy. The
             // sync manager owns the remote ports; without one (sync disabled or
             // no owner key material) only the local copy goes and the result says
@@ -1225,7 +1649,7 @@ export async function handlePersonalServerDataRequest(request, deps, options = {
             return jsonResponse(result, { status: 200 });
         }
         return methodNotAllowed();
-    });
+    }, logContext);
 }
 export async function handlePersonalServerAccessLogsRequest(request, deps, options = {}) {
     return withApiErrors(async () => {
@@ -1271,6 +1695,9 @@ export async function handlePersonalServerSyncRequest(request, deps, options = {
             }));
         }
         return notFound();
+    }, {
+        logger: deps.logger,
+        route: `${request.method} ${options.basePath ?? "/v1/sync"}`,
     });
 }
 export async function handlePersonalServerGrantsRequest(request, deps, options = {}) {