@north-light/api 0.3.579 → 0.3.587

package/dist/core/import/bundle/layout.d.ts

@@ -0,0 +1,90 @@
+/** `bodies` (revision text, design-fleet D106) is copied like any table but dumped apart: its 667 MB do not fit one row file (`encodeRows` builds one string), so its rows
+ *  travel as the part files named in `source.bodyParts` (see `rowKeys`), each written and read a line at a time (`RowFileWriter`, `rowLines`).
+ *  Every row is canonical: `hash` is the sha256 of `text`, which holds no NUL. The exporter dumps only such rows and the lane refuses a bundle with any other (`stage`). */
+export declare const BODIES_TABLE = "bodies";
+/** Tables whose rows are dumped, staged and (after rewrites) copied, in no particular order: the lane computes the copy
+ *  order from the target's foreign-key catalog. Each row file holds the table's rows in primary-key order (identity-ordered
+ *  tables in source `seq` order). */
+export declare const COPIED_TABLES: readonly ["objects", "documents", "revisions", "revision_parents", "identities", "jobs", "custom_objects", "context_sessions", "environments", "object_kinds", "edges", "watches", "watch_subscriptions", "exposures", "rules", "unresolved_links", "nodes", "sessions", "worktrees", "node_faults", "waits", "structured_requests", "memory_reads", "deleted_isolated_nodes", "node_outcome_deliveries", "run_events", "messages", "message_causes", "deliveries", "crons", "cron_runs", "daemon_actions", "human_requests", "human_action_deliveries", "review_comments", "review_comment_events", "audit_rows", "app_users", "idempotency_keys", "uploads", "upload_uses", "shares", "provider_approvals", "relations", "charges", "limits", "conversation_parts", "conversation_entries", "profiles", "bodies"];
+/** Dumped and staged so rewrite 2 can map them, never inserted into the target (`grants`: the directory writes them). */
+export declare const STAGED_ONLY_TABLES: readonly ["grants"];
+/** Every table the exporter dumps. */
+export declare const DUMPED_TABLES: readonly string[];
+/** Tables with an identity primary key, tracked in `import_key_map` (source key → target key). */
+export declare const KEY_MAPPED_TABLES: {
+    readonly audit_rows: "id";
+    readonly review_comment_events: "seq";
+    readonly app_users: "uid";
+};
+/** Tables the target numbers by `seq`/identity in source order (the staged `seq` is not carried over). */
+export declare const SEQ_TABLES: readonly ["objects", "revisions", "watches", "watch_subscriptions", "sessions", "waits", "structured_requests", "deliveries", "run_events"];
+export declare const bundleKeys: {
+    readonly prefix: (run: string) => string;
+    readonly rows: (run: string, table: string) => string;
+    readonly source: (run: string) => string;
+    readonly manifest: (run: string) => string;
+    readonly nodeFiles: (run: string, node: string) => string;
+    readonly bodyRows: (run: string, part: string) => string;
+    readonly request: (requestId: string, relative: string) => string;
+};
+/** The blob keys of a table's row files: one file, or for `bodies` its parts. */
+export declare function rowKeys(run: string, table: string, source: Pick<SourceJson, 'bodyParts'>): string[];
+/** `imports/<run>/source.json`. */
+export interface SourceJson {
+    installId: string;
+    kind: 'machine' | 'hosted';
+    /** The source's person (`user:<id>`, or the local person on a runtime that never joined). */
+    person: string;
+    release: string;
+    schemaVersion: number;
+    /** Rows per dumped table. The lane refuses a row file whose line count differs. */
+    counts: Record<string, number>;
+    /** The source's machine environment object id, when it has one (`environments.object`). */
+    environmentObject?: string;
+    /** Hosted sources: every directory the exporter archived for a running node (`nodeFiles` holds its entries as `<from minus leading slash>/<relative path>`).
+     *  The lane picks the sandbox directory each one lands in (`sandboxPathMap`); the exporter does not. */
+    pathPrefixes?: PathPrefix[];
+    /** The part files of the `bodies` rows (`imports/<run>/rows/bodies/<part>.jsonl.zst`): the pre-pause dump's parts, then the pause dump's. `counts.bodies` is their total. */
+    bodyParts?: string[];
+    /** Shipped documents (`documents.source_path` set): document object id → hash of the body text the exporter read from its file; that text is a `bodies` row in the bundle.
+     *  The lane uses it for a shipped document the target has no copy of, which becomes an ordinary document (I23). */
+    shippedBodies?: Record<string, string>;
+}
+/** One archived directory of a running node: `app` is its grantee (`app:<slug>`), `role` says which directory it is. */
+export interface PathPrefix {
+    node: string;
+    app: string;
+    role: 'context' | 'worktree' | 'cwd';
+    from: string;
+}
+/** Rewrite 4: source path prefix → target path prefix, applied to the rows of running nodes of a hosted source. */
+export interface PathMapEntry {
+    from: string;
+    to: string;
+}
+export type PathMap = readonly PathMapEntry[];
+/** Rewrite one path through the map: the longest matching prefix wins; a path under no prefix is returned unchanged. */
+export declare function mapPath(map: PathMap, path: string): string;
+export type RowObject = Record<string, unknown>;
+/** Row-file codec: one JSON object per line, newline-terminated, zstd-compressed. */
+export declare function encodeRows(rows: Iterable<RowObject>): Buffer;
+export declare function decodeRows(data: Buffer | Uint8Array): RowObject[];
+/** One JSON object per line, newline-terminated: the line `RowFileWriter.write` takes. */
+export declare function rowLine(row: RowObject): string;
+/** Writes a row file as a zstd stream, a line at a time, so no string larger than one line exists (`encodeRows` builds the whole file as one string, past V8's
+ *  ~512 MiB string limit for a large file). `bytes` is the size of the JSON written so far, before compression. */
+export declare class RowFileWriter {
+    private readonly zst;
+    private readonly finished;
+    private written;
+    constructor(path: string);
+    get bytes(): number;
+    /** Write one line (`rowLine`), waiting for the compressor when it is full. */
+    write(line: string, size?: number): Promise<void>;
+    /** Finish the file; resolves when it is on disk. */
+    close(): Promise<void>;
+    destroy(): void;
+}
+/** The lines of a zstd row file, decompressed as a stream: each yielded string is one line (without its newline), so memory is one line plus a chunk. A file whose last
+ *  line is not newline-terminated, or that is not valid zstd, throws when the reader reaches it. */
+export declare function rowLines(data: Buffer | Uint8Array): AsyncGenerator<string>;

package/dist/core/import/bundle/layout.js

@@ -0,0 +1,117 @@
+// The import bundle: what the exporter (I-1) writes to R2 and the import lane (I-2) reads. One definition for both.
+// Design: design-import, "Components" (Exporter, Import lane), "Data model" (blob keys) and "Table by table".
+import { once } from 'node:events';
+import { createWriteStream } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { createZstdCompress, createZstdDecompress, zstdCompressSync, zstdDecompressSync } from 'node:zlib';
+/** `bodies` (revision text, design-fleet D106) is copied like any table but dumped apart: its 667 MB do not fit one row file (`encodeRows` builds one string), so its rows
+ *  travel as the part files named in `source.bodyParts` (see `rowKeys`), each written and read a line at a time (`RowFileWriter`, `rowLines`).
+ *  Every row is canonical: `hash` is the sha256 of `text`, which holds no NUL. The exporter dumps only such rows and the lane refuses a bundle with any other (`stage`). */
+export const BODIES_TABLE = 'bodies';
+/** Tables whose rows are dumped, staged and (after rewrites) copied, in no particular order: the lane computes the copy
+ *  order from the target's foreign-key catalog. Each row file holds the table's rows in primary-key order (identity-ordered
+ *  tables in source `seq` order). */
+export const COPIED_TABLES = [
+    'objects', 'documents', 'revisions', 'revision_parents', 'identities', 'jobs', 'custom_objects', 'context_sessions', 'environments',
+    'object_kinds', 'edges', 'watches', 'watch_subscriptions', 'exposures', 'rules', 'unresolved_links',
+    'nodes', 'sessions', 'worktrees', 'node_faults', 'waits', 'structured_requests', 'memory_reads', 'deleted_isolated_nodes', 'node_outcome_deliveries',
+    'run_events', 'messages', 'message_causes', 'deliveries', 'crons', 'cron_runs', 'daemon_actions',
+    'human_requests', 'human_action_deliveries', 'review_comments', 'review_comment_events', 'audit_rows', 'app_users',
+    'idempotency_keys', 'uploads', 'upload_uses', 'shares', 'provider_approvals', 'relations', 'charges', 'limits',
+    'conversation_parts', 'conversation_entries', 'profiles', 'bodies',
+];
+/** Dumped and staged so rewrite 2 can map them, never inserted into the target (`grants`: the directory writes them). */
+export const STAGED_ONLY_TABLES = ['grants'];
+/** Every table the exporter dumps. */
+export const DUMPED_TABLES = [...COPIED_TABLES, ...STAGED_ONLY_TABLES];
+/** Tables with an identity primary key, tracked in `import_key_map` (source key → target key). */
+export const KEY_MAPPED_TABLES = { audit_rows: 'id', review_comment_events: 'seq', app_users: 'uid' };
+/** Tables the target numbers by `seq`/identity in source order (the staged `seq` is not carried over). */
+export const SEQ_TABLES = [
+    'objects', 'revisions', 'watches', 'watch_subscriptions', 'sessions', 'waits', 'structured_requests', 'deliveries', 'run_events',
+];
+export const bundleKeys = {
+    prefix: (run) => `imports/${run}/`,
+    rows: (run, table) => `imports/${run}/rows/${table}.jsonl.zst`,
+    source: (run) => `imports/${run}/source.json`,
+    manifest: (run) => `imports/${run}/manifest.jsonl.zst`,
+    nodeFiles: (run, node) => `imports/${run}/node-files/${node}.tar.zst`,
+    bodyRows: (run, part) => `imports/${run}/rows/bodies/${part}.jsonl.zst`,
+    request: (requestId, relative) => `requests/${requestId}/${relative}`,
+};
+/** The blob keys of a table's row files: one file, or for `bodies` its parts. */
+export function rowKeys(run, table, source) {
+    return table === BODIES_TABLE ? (source.bodyParts ?? []).map((part) => bundleKeys.bodyRows(run, part)) : [bundleKeys.rows(run, table)];
+}
+/** Rewrite one path through the map: the longest matching prefix wins; a path under no prefix is returned unchanged. */
+export function mapPath(map, path) {
+    let best;
+    for (const entry of map) {
+        if ((path === entry.from || path.startsWith(entry.from.endsWith('/') ? entry.from : `${entry.from}/`)) && (best === undefined || entry.from.length > best.from.length))
+            best = entry;
+    }
+    return best === undefined ? path : best.to + path.slice(best.from.length);
+}
+/** Row-file codec: one JSON object per line, newline-terminated, zstd-compressed. */
+export function encodeRows(rows) {
+    let text = '';
+    for (const row of rows)
+        text += `${JSON.stringify(row)}\n`;
+    return zstdCompressSync(Buffer.from(text, 'utf8'));
+}
+export function decodeRows(data) {
+    const text = zstdDecompressSync(data).toString('utf8');
+    if (text === '')
+        return [];
+    if (!text.endsWith('\n'))
+        throw new Error('row file is truncated: the last line is not newline-terminated');
+    return text.slice(0, -1).split('\n').map((line) => JSON.parse(line));
+}
+/** One JSON object per line, newline-terminated: the line `RowFileWriter.write` takes. */
+export function rowLine(row) {
+    return `${JSON.stringify(row)}\n`;
+}
+/** Writes a row file as a zstd stream, a line at a time, so no string larger than one line exists (`encodeRows` builds the whole file as one string, past V8's
+ *  ~512 MiB string limit for a large file). `bytes` is the size of the JSON written so far, before compression. */
+export class RowFileWriter {
+    zst = createZstdCompress();
+    finished;
+    written = 0;
+    constructor(path) {
+        this.finished = pipeline(this.zst, createWriteStream(path));
+        this.finished.catch(() => undefined);
+    }
+    get bytes() { return this.written; }
+    /** Write one line (`rowLine`), waiting for the compressor when it is full. */
+    async write(line, size = Buffer.byteLength(line, 'utf8')) {
+        this.written += size;
+        if (!this.zst.write(line, 'utf8'))
+            await Promise.race([once(this.zst, 'drain'), this.finished]);
+    }
+    /** Finish the file; resolves when it is on disk. */
+    async close() {
+        this.zst.end();
+        await this.finished;
+    }
+    destroy() { this.zst.destroy(); }
+}
+/** The lines of a zstd row file, decompressed as a stream: each yielded string is one line (without its newline), so memory is one line plus a chunk. A file whose last
+ *  line is not newline-terminated, or that is not valid zstd, throws when the reader reaches it. */
+export async function* rowLines(data) {
+    const zst = createZstdDecompress({ chunkSize: 1 << 20 });
+    zst.end(data);
+    let pending = [];
+    for await (const chunk of zst) {
+        let start = 0;
+        for (let nl = chunk.indexOf(0x0a, start); nl !== -1; nl = chunk.indexOf(0x0a, start)) {
+            pending.push(chunk.subarray(start, nl));
+            yield (pending.length === 1 ? pending[0] : Buffer.concat(pending)).toString('utf8');
+            pending = [];
+            start = nl + 1;
+        }
+        if (start < chunk.length)
+            pending.push(chunk.subarray(start));
+    }
+    if (pending.length > 0)
+        throw new Error('row file is truncated: the last line is not newline-terminated');
+}

package/dist/core/import/export/archive.d.ts

@@ -0,0 +1,11 @@
+import type { PathPrefix } from '../bundle/layout.js';
+/** One node's archive and the directories in it: context dir, worktree, and a cwd outside both. The lane picks each directory's sandbox target. */
+export declare function nodeArchive(node: {
+    node: string;
+    app: string;
+    cwd: string;
+    worktree: string | null;
+}, contextDir?: string): Promise<{
+    data: Buffer;
+    prefixes: PathPrefix[];
+}>;

package/dist/core/import/export/archive.js

@@ -0,0 +1,22 @@
+// A hosted node's files for the sandbox (design-import "node files", I-4's contract): one zstd tar per node whose entries are named
+// `<prefix from without its leading slash>/<path relative to it>`, regular files and directories only, no symlinks.
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { zstdCompressSync } from 'node:zlib';
+import * as tar from 'tar';
+import { nodeAgentDir } from '../../canvas/paths.js';
+/** One node's archive and the directories in it: context dir, worktree, and a cwd outside both. The lane picks each directory's sandbox target. */
+export async function nodeArchive(node, contextDir = join(nodeAgentDir(node.node, node.app), 'context')) {
+    const base = { node: node.node, app: node.app };
+    const prefixes = [{ ...base, role: 'context', from: contextDir }];
+    if (node.worktree !== null)
+        prefixes.push({ ...base, role: 'worktree', from: node.worktree });
+    const covered = prefixes.some((p) => node.cwd === p.from || node.cwd.startsWith(`${p.from}/`));
+    if (node.cwd !== '' && !covered)
+        prefixes.push({ ...base, role: 'cwd', from: node.cwd });
+    const chunks = [];
+    const stream = tar.create({ cwd: '/', portable: true, filter: (_path, entry) => !('isSymbolicLink' in entry && entry.isSymbolicLink()) }, prefixes.filter((p) => existsSync(p.from)).map((p) => p.from.replace(/^\//, '')));
+    for await (const chunk of stream)
+        chunks.push(chunk);
+    return { data: zstdCompressSync(Buffer.concat(chunks)), prefixes };
+}

package/dist/core/import/export/cloud.d.ts

@@ -0,0 +1,23 @@
+import type { ImportRunDTO, ImportStartRequest, ImportStartedDTO, ImportStoreNameDTO, ImportUploadUrlsDTO } from '../../../api/dto/imports.js';
+/** The dump directory's file holding the opened file stores, and the order they are uploaded in. */
+export declare const STORE_FILE = "stores.json";
+export declare const STORE_ORDER: readonly ImportStoreNameDTO[];
+export declare class ImportCloud {
+    private readonly apiUrl;
+    private readonly token;
+    /** `token` is a fixed import token, or a getter for a person bearer (5-minute tokens, so a long import asks per call). */
+    constructor(apiUrl: string, token: string | (() => Promise<string>));
+    private call;
+    start(request: ImportStartRequest): Promise<ImportStartedDTO>;
+    uploadUrls(run: string, keys: string[]): Promise<ImportUploadUrlsDTO>;
+    putStore(run: string, name: ImportStoreNameDTO, body: unknown): Promise<void>;
+    load(run: string): Promise<ImportRunDTO>;
+    get(run: string): Promise<ImportRunDTO>;
+    /** `rotate` replaces a credential already issued for this run's machine (without it a second call answers 409 `credential_exists`). */
+    machineCredential(run: string, rotate?: boolean): Promise<{
+        environment_id: string;
+        credential: string;
+    }>;
+    /** PUT bytes to a signed URL. */
+    put(url: string, data: Buffer): Promise<void>;
+}

package/dist/core/import/export/cloud.js

@@ -0,0 +1,47 @@
+// The exporter's HTTP side: `/v1/imports` on the control plane with the person's (or the operator's) bearer token, and signed R2 PUTs.
+// Imports Node built-ins only, so a CLI leaf can use it.
+import { general } from '../../errors.js';
+/** The dump directory's file holding the opened file stores, and the order they are uploaded in. */
+export const STORE_FILE = 'stores.json';
+export const STORE_ORDER = ['auth', 'credentials', 'env', 'settings', 'plugins'];
+export class ImportCloud {
+    apiUrl;
+    token;
+    /** `token` is a fixed import token, or a getter for a person bearer (5-minute tokens, so a long import asks per call). */
+    constructor(apiUrl, token) {
+        this.apiUrl = apiUrl;
+        this.token = token;
+    }
+    async call(method, path, body) {
+        const response = await fetch(new URL(path, this.apiUrl), {
+            method,
+            headers: { authorization: `Bearer ${typeof this.token === 'string' ? this.token : await this.token()}`, ...(body === undefined ? {} : { 'content-type': 'application/json' }) },
+            ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+        });
+        const text = await response.text();
+        if (!response.ok) {
+            let code;
+            try {
+                code = JSON.parse(text).error?.code;
+            }
+            catch {
+                code = undefined;
+            }
+            throw general(`${method} ${path} answered ${response.status}: ${text.slice(0, 500)}`, { status: response.status, ...(typeof code === 'string' ? { code } : {}) });
+        }
+        return (text === '' ? undefined : JSON.parse(text));
+    }
+    start(request) { return this.call('POST', '/v1/imports', request); }
+    uploadUrls(run, keys) { return this.call('POST', `/v1/imports/${run}/upload-urls`, { keys }); }
+    putStore(run, name, body) { return this.call('PUT', `/v1/imports/${run}/stores/${name}`, body); }
+    load(run) { return this.call('POST', `/v1/imports/${run}/load`, {}); }
+    get(run) { return this.call('GET', `/v1/imports/${run}`); }
+    /** `rotate` replaces a credential already issued for this run's machine (without it a second call answers 409 `credential_exists`). */
+    machineCredential(run, rotate = false) { return this.call('POST', `/v1/imports/${run}/machine-credential${rotate ? '?rotate=1' : ''}`, {}); }
+    /** PUT bytes to a signed URL. */
+    async put(url, data) {
+        const response = await fetch(url, { method: 'PUT', body: new Uint8Array(data) });
+        if (!response.ok)
+            throw general(`upload answered ${response.status}: ${(await response.text()).slice(0, 300)}`);
+    }
+}

package/dist/core/import/export/dump.d.ts

@@ -0,0 +1,34 @@
+import { type SourceJson } from '../bundle/layout.js';
+export interface DumpRequestRef {
+    id: string;
+    dir: string;
+    file: string | null;
+}
+export interface DumpRunningNode {
+    node: string;
+    app: string;
+    cwd: string;
+    worktree: string | null;
+}
+export interface DumpResult {
+    source: SourceJson;
+    /** Human requests (page and review) with the folder and reviewed file their rows name. */
+    requests: DumpRequestRef[];
+    /** Nodes with status active or idle, for the hosted node-file archives. */
+    runningNodes: DumpRunningNode[];
+    /** `body_ref` files that could not be read and stayed as references. */
+    bodiesMissing: number;
+    bodiesInlined: number;
+}
+/** A `bodies` part file holds about this much JSON (the encoded lines, before compression); a row that alone exceeds it is a part of its own. */
+export declare const PART_JSON_BYTES: number;
+/** A body longer than this many UTF-8 bytes fails the export. The largest body on Silas's Mac is 5.2 MB. JSON escapes one character into at most six (`\u0001`), so a body at
+ *  the ceiling is a line of at most 192 MiB, well inside V8's ~512 MiB string limit. */
+export declare const BODY_MAX_BYTES: number;
+/** The pre-pause dump of `bodies` (design-import cutover step 2): every row, while nodes still run. Rows are never updated or deleted, so the pause dump
+ *  needs only the rows whose hash is not in the list this leaves in `<out>/bodies-pre.json`. */
+export declare function dumpBodiesPre(out: string): Promise<{
+    parts: string[];
+    count: number;
+}>;
+export declare function dumpRows(out: string, kind: SourceJson['kind']): Promise<DumpResult>;

package/dist/core/import/export/dump.js

@@ -0,0 +1,235 @@
+// The row dump, run in the daemon generation's entry (`crtrd-cli import-dump`) because a CLI process may not load canvas database
+// code (scripts/check-cli-no-canvas.mjs). One REPEATABLE READ transaction reads every dumped table; each becomes
+// `<out>/rows/<table>.jsonl.zst` (primary-key order, `seq` order for identity-ordered tables). The result is one JSON line.
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { openDb, withCanvasWrite } from '../../canvas/db.js';
+import { hashOf, storable } from '../../graph/bodies.js';
+import { personIdAsync } from '../../canvas/person.js';
+import { CANVAS_SCHEMA_VERSION } from '../../canvas/schema-version.js';
+import { crtrVersion } from '../../../shared/crtr-version.js';
+import { BODIES_TABLE, DUMPED_TABLES, RowFileWriter, SEQ_TABLES, bundleKeys, encodeRows, rowLine } from '../bundle/layout.js';
+import { parseFrontmatterGeneric } from '../../frontmatter.js';
+import { inlineMessageBody } from './inline.js';
+/** A `bodies` part file holds about this much JSON (the encoded lines, before compression); a row that alone exceeds it is a part of its own. */
+export const PART_JSON_BYTES = 16 * 1024 * 1024;
+/** `bodies` rows are read from the database in pages of about this much text. */
+const PAGE_TEXT_BYTES = 8 * 1024 * 1024;
+/** A body longer than this many UTF-8 bytes fails the export. The largest body on Silas's Mac is 5.2 MB. JSON escapes one character into at most six (`\u0001`), so a body at
+ *  the ceiling is a line of at most 192 MiB, well inside V8's ~512 MiB string limit. */
+export const BODY_MAX_BYTES = 32 * 1024 * 1024;
+/** Hashes whose sizes one query reads. */
+const SIZE_BATCH = 5000;
+const PRE_FILE = 'bodies-pre.json';
+const q = (name) => `"${name.replace(/"/g, '""')}"`;
+function readText(path) {
+    try {
+        const text = readFileSync(path, 'utf8');
+        return text.trim() === '' ? null : text;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return null;
+        throw error;
+    }
+}
+function readFileOrNull(path) {
+    try {
+        return readFileSync(path, 'utf8');
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return null;
+        throw error;
+    }
+}
+function tooLarge(hash, bytes) {
+    return new Error(`revision text ${hash} is ${bytes} bytes; the import carries at most ${BODY_MAX_BYTES} bytes of text per body`);
+}
+/** A bundle carries only canonical bodies: the key is the sha256 of the text (the boot repair rekeys a row whose key is not). A row that is not fails the export. */
+function assertCanonical(row) {
+    const actual = hashOf(row.text);
+    if (actual !== row.hash) {
+        throw new Error(`body ${row.hash} does not hash to its key (its text hashes to ${actual}); upgrade and restart the daemon so its boot repair rekeys the row, then export again`);
+    }
+}
+/** Read the `bodies` rows of `hashes` (in that order) and write them as part files `<phase>-<n>`; returns the part names. Rows are read in pages of about `PAGE_TEXT_BYTES`
+ *  of text (sizes first, by `octet_length`), each row is written as one line, and a part is closed when its encoded JSON reaches `PART_JSON_BYTES`. */
+async function writeBodyParts(db, out, phase, hashes, extra) {
+    const dir = join(out, 'rows', BODIES_TABLE);
+    mkdirSync(dir, { recursive: true });
+    const parts = [];
+    let file = null;
+    const close = async () => { const done = file; file = null; if (done !== null)
+        await done.close(); };
+    const add = async (row) => {
+        assertCanonical(row);
+        const line = rowLine(row);
+        const size = Buffer.byteLength(line, 'utf8');
+        if (file !== null && file.bytes > 0 && file.bytes + size > PART_JSON_BYTES)
+            await close();
+        if (file === null) {
+            const part = `${phase}-${parts.length}`;
+            file = new RowFileWriter(join(dir, `${part}.jsonl.zst`));
+            parts.push(part);
+        }
+        await file.write(line, size);
+    };
+    try {
+        for (let i = 0; i < hashes.length;) {
+            // A page: the next hashes whose text adds up to about PAGE_TEXT_BYTES (a row of its own when larger). Sizes come from the database, not from the text.
+            const sizes = await db.prepare('SELECT hash, octet_length(text) AS n FROM bodies WHERE hash = ANY(?::text[]) ORDER BY hash').all(hashes.slice(i, i + SIZE_BATCH));
+            let page = [];
+            let pageBytes = 0;
+            const fetchPage = async () => {
+                if (page.length === 0)
+                    return;
+                const rows = await db.prepare('SELECT hash, text FROM bodies WHERE hash = ANY(?::text[]) ORDER BY hash').all(page);
+                for (const row of rows)
+                    await add(row);
+                page = [];
+                pageBytes = 0;
+            };
+            for (const { hash, n } of sizes) {
+                const bytes = Number(n);
+                if (bytes > BODY_MAX_BYTES)
+                    throw tooLarge(hash, bytes);
+                if (page.length > 0 && pageBytes + bytes > PAGE_TEXT_BYTES)
+                    await fetchPage();
+                page.push(hash);
+                pageBytes += bytes;
+            }
+            await fetchPage();
+            i += SIZE_BATCH;
+        }
+        for (const row of extra) {
+            const bytes = Buffer.byteLength(row.text, 'utf8');
+            if (bytes > BODY_MAX_BYTES)
+                throw tooLarge(row.hash, bytes);
+            await add(row);
+        }
+        await close();
+    }
+    catch (error) {
+        file?.destroy();
+        throw error;
+    }
+    return parts;
+}
+const allHashes = async (db) => (await db.prepare('SELECT hash FROM bodies ORDER BY hash').all()).map((r) => r.hash);
+/** The pre-pause dump of `bodies` (design-import cutover step 2): every row, while nodes still run. Rows are never updated or deleted, so the pause dump
+ *  needs only the rows whose hash is not in the list this leaves in `<out>/bodies-pre.json`. */
+export async function dumpBodiesPre(out) {
+    const db = await openDb();
+    const hashes = await allHashes(db);
+    const parts = await writeBodyParts(db, out, 'pre', hashes, []);
+    const pre = { hashes, parts, count: hashes.length, canonical: true };
+    writeFileSync(join(out, PRE_FILE), JSON.stringify(pre));
+    return { parts, count: hashes.length };
+}
+function readPre(out) {
+    try {
+        return JSON.parse(readFileSync(join(out, PRE_FILE), 'utf8'));
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return { hashes: [], parts: [], count: 0 };
+        throw error;
+    }
+}
+/** The pause dump reuses the pre-pause parts as they are, so it refuses a pre-pause dump that may hold a row the database no longer has under that key: one written without
+ *  the canonical check, or one whose rows the boot repair has rekeyed since (the repair removes the old-key row). The run must be restarted from a new pre-pause dump. */
+function assertPreCurrent(pre, current) {
+    if (pre.count === 0 && pre.parts.length === 0)
+        return;
+    if (pre.canonical !== true)
+        throw new Error(`${PRE_FILE} was written before bodies were checked against their keys; discard the export directory and run the pre-pause dump again`);
+    const gone = pre.hashes.filter((hash) => !current.has(hash));
+    if (gone.length > 0) {
+        throw new Error(`the pre-pause dump holds ${gone.length} body row(s) the database no longer has (first: ${gone[0]}); the boot repair ran after it. Discard the export directory and run the pre-pause dump again`);
+    }
+}
+export async function dumpRows(out, kind) {
+    const person = await personIdAsync();
+    mkdirSync(join(out, 'rows'), { recursive: true });
+    return withCanvasWrite(async (db) => {
+        await db.exec('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ');
+        const present = new Set((await db.prepareUncached(`SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema()`).all()).map((r) => r.table_name));
+        const missingTables = DUMPED_TABLES.filter((table) => !present.has(table));
+        if (missingTables.length > 0)
+            throw new Error(`the canvas database lacks tables the bundle needs: ${missingTables.join(', ')}`);
+        const keys = await db.prepareUncached(`SELECT c.conrelid::regclass::text AS tbl, (SELECT array_agg(a.attname::text ORDER BY k.ord) FROM unnest(c.conkey) WITH ORDINALITY k(attnum, ord)
+        JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum) AS cols FROM pg_constraint c WHERE c.contype = 'p' AND c.connamespace = current_schema()::regnamespace`).all();
+        const primary = new Map(keys.map((k) => [k.tbl.replace(/"/g, '').replace(/^.*\./, ''), k.cols]));
+        // A table with no primary key (`memory_reads`) is ordered by every column, in column order, each compared as text (a json column has no ordering).
+        const columns = await db.prepareUncached(`SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = current_schema() ORDER BY table_name, ordinal_position`).all();
+        const identityOrdered = new Set(SEQ_TABLES);
+        const counts = {};
+        let inlined = 0;
+        let missing = 0;
+        // `bodies` is dumped apart (below): its part files are not one row file per table.
+        for (const table of DUMPED_TABLES.filter((t) => t !== BODIES_TABLE)) {
+            const keyed = identityOrdered.has(table) ? ['seq'] : primary.get(table);
+            const order = keyed !== undefined && keyed.length > 0 ? keyed.map((c) => `t.${q(c)}`) : columns.filter((c) => c.table_name === table).map((c) => `t.${q(c.column_name)}::text`);
+            if (order.length === 0)
+                throw new Error(`table ${table} has no columns to order its dump by`);
+            const found = await db.prepareUncached(`SELECT row_to_json(t)::text AS j FROM ${q(table)} t ORDER BY ${order.join(', ')}`).all();
+            let rows = found.map((r) => JSON.parse(r.j));
+            if (table === 'messages') {
+                rows = rows.map((row) => {
+                    const ref = row['body_ref'];
+                    const next = inlineMessageBody(row, readText);
+                    if (typeof ref === 'string' && ref.startsWith('/')) {
+                        if (next === row)
+                            missing += 1;
+                        else
+                            inlined += 1;
+                    }
+                    return next;
+                });
+            }
+            counts[table] = rows.length;
+            writeFileSync(join(out, bundleKeys.rows('', table).replace(/^imports\/\//, '')), encodeRows(rows));
+        }
+        const installId = (await db.prepare(`SELECT value FROM canvas_meta WHERE key = 'install_id'`).get())?.value;
+        if (installId === undefined)
+            throw new Error('this canvas database has no install id');
+        const environment = await db.prepare(`SELECT object FROM environments WHERE kind = 'machine' ORDER BY object LIMIT 1`).get();
+        const source = { installId, kind, person, release: crtrVersion(), schemaVersion: CANVAS_SCHEMA_VERSION, counts, ...(environment === undefined ? {} : { environmentObject: environment.object }) };
+        const requests = await db.prepare('SELECT request_id, request_dir, file FROM human_requests ORDER BY request_id').all();
+        // A shipped document's body is its file's body after the frontmatter (`bodyOf`); a file the source no longer has is skipped. The text travels as a `bodies` row.
+        const shipped = await db.prepare('SELECT object, source_path FROM documents WHERE source_path IS NOT NULL ORDER BY object').all();
+        const shippedBodies = {};
+        const shippedRows = new Map();
+        for (const doc of shipped) {
+            const file = readFileOrNull(doc.source_path);
+            if (file === null)
+                continue;
+            const text = storable(parseFrontmatterGeneric(file).body);
+            const hash = hashOf(text);
+            shippedBodies[doc.object] = hash;
+            shippedRows.set(hash, { hash, text });
+        }
+        // `bodies` rows added since the pre-pause dump (all of them when there was none), and the shipped texts the pre-pause dump did not hold.
+        const pre = readPre(out);
+        const had = new Set(pre.hashes);
+        const now = await allHashes(db);
+        assertPreCurrent(pre, new Set(now));
+        const fresh = now.filter((hash) => !had.has(hash));
+        const added = new Set(fresh);
+        const pauseParts = await writeBodyParts(db, out, 'pause', fresh, [...shippedRows.values()].filter((row) => !had.has(row.hash) && !added.has(row.hash)));
+        const bodyCount = pre.count + fresh.length + [...shippedRows.values()].filter((row) => !had.has(row.hash) && !added.has(row.hash)).length;
+        counts[BODIES_TABLE] = bodyCount;
+        source.bodyParts = [...pre.parts, ...pauseParts];
+        const running = await db.prepare(`SELECT n.node_id, r.grantee, n.cwd, w.path AS worktree FROM nodes n JOIN node_rows r ON r.node_id = n.node_id LEFT JOIN worktrees w ON w.node_id = n.node_id AND w.state = 'open'
+      WHERE n.status IN ('active', 'idle') ORDER BY n.node_id`).all();
+        source.shippedBodies = shippedBodies;
+        return {
+            source,
+            requests: requests.map((r) => ({ id: r.request_id, dir: r.request_dir, file: r.file })),
+            runningNodes: running.map((r) => ({ node: r.node_id, app: r.grantee, cwd: r.cwd, worktree: r.worktree })),
+            bodiesInlined: inlined,
+            bodiesMissing: missing,
+        };
+    });
+}

package/dist/core/import/export/files.d.ts

@@ -0,0 +1,22 @@
+import type { ImportCloud } from './cloud.js';
+export interface LocalObject {
+    key: string;
+    path: string;
+}
+/** Every file the runtime holds that the cloud serves from a blob key. */
+export declare function localObjects(): LocalObject[];
+/** The reviewed file of a review that lies outside the review's own directory, at the key the lane rewrites `file` to. */
+export declare function reviewSourceObject(requestId: string, requestDir: string, file: string | null): LocalObject | null;
+export declare class Uploader {
+    private readonly cloud;
+    private readonly run;
+    private readonly progress;
+    private readonly sent;
+    uploaded: number;
+    missing: string[];
+    constructor(cloud: ImportCloud, run: string, progress: (line: string) => void);
+    /** Upload source blobs, each to `imports/<run>/blobs/<its key>`: the upload route signs only under the run's own prefix, and the lane moves each blob to its key. */
+    upload(objects: readonly LocalObject[]): Promise<void>;
+    /** Upload bytes held in memory (row files, source.json, node archives). */
+    uploadBytes(key: string, data: Buffer): Promise<void>;
+}