@north-light/api 0.3.589 → 0.3.591

package/dist/core/graph/package-docs.js

@@ -5,102 +5,44 @@
  *  A plugin is an app: its documents belong to the app `<plugin>` (identity `app:<plugin>`,
  *  the one `ownerForGrantee` gives), and a document with a `source_path` is shipped. */
 import { randomUUID } from 'node:crypto';
-import { CrtrError } from '../errors.js';
-import { emitEvent } from '../events/emit.js';
-import { quarantineBrokenPlugins } from '../plugin-quarantine.js';
-import { listInstalledPlugins } from '../resolver.js';
-import { builtinMemoryRoot, pluginMemoryDir } from '../scope.js';
 import { documentFields } from '../../migrations/004-canvas-documents/fields.js';
-import { put as putBody } from './bodies.js';
+import { get as getBody, put as putBody } from './bodies.js';
 import { applyDocumentFields, deleteDocument, documentRecord, heads, importDocument, linksIn, moveDocument } from './documents.js';
 import { deleteShippedReader, writeShippedReader } from '../permissions/facts.js';
 import { setLinks } from './edges.js';
 import { publish } from './events.js';
-import { resolveName } from './names.js';
-import { packageFiles } from './package-files.js';
-import { ensureOwner, getById, setObjectKind } from './objects.js';
-import { checkoutsForCwd } from './repo-sync/repos.js';
+import { getById, setObjectKind } from './objects.js';
 import { rulesOf } from './rules.js';
-/** The builtin corpus (app `crtr`), then every enabled user-scope plugin's `memory/` folder. */
-export function packageSources() {
-    return [
-        { plugin: 'crtr', dir: builtinMemoryRoot() },
-        ...listInstalledPlugins('user').filter((p) => p.enabled).map((p) => ({ plugin: p.name, dir: pluginMemoryDir(p) })),
-    ];
-}
-/** Converge every package's documents onto its files. Run inside the caller's transaction.
- *  Idempotent: a second run over unchanged files changes no document, revision or rule (it re-derives
- *  each document's link edges, which land the same). A changed body replaces the
- *  document's single revision and publishes `edited`; a vanished file (or plugin) marks its
- *  document deleted and publishes `deleted`. */
-export async function reconcilePackageDocuments(db, explicitSources) {
-    const report = { created: 0, updated: 0, deleted: 0, conflicts: [], invalid: [], quarantined: [] };
-    // One damaged plugin directory (a 0-byte manifest from an interrupted write) is set aside, never fatal.
-    // Only the real installed set is swept; a caller passing its own sources has no plugins directory here.
-    if (explicitSources === undefined) {
-        report.quarantined = quarantineBrokenPlugins();
-        for (const q of report.quarantined) {
-            emitEvent({ level: 'error', event: 'daemon.plugins.quarantined', fields: { plugin: q.name, file: q.file, reason: q.reason, ...(q.movedTo !== undefined ? { moved_to: q.movedTo } : {}) } });
-        }
-    }
-    const sources = explicitSources ?? packageSources();
-    const reposFn = async (cwd) => await checkoutsForCwd(db, cwd);
-    const reconciled = new Set();
-    const converged = [];
-    const names = new Set();
-    for (const source of sources) {
-        let owner;
-        try {
-            if (names.has(source.plugin))
-                throw new CrtrError('conflict', `plugin "${source.plugin}" is already provided by another package`);
-            names.add(source.plugin);
-            owner = await ensureOwner(db, 'app', 'app', `app:${source.plugin}`, source.plugin);
-        }
-        catch (error) {
-            if (!(error instanceof CrtrError) || error.code !== 'conflict')
-                throw error;
-            report.conflicts.push({ plugin: source.plugin, message: error.message });
-            continue;
-        }
-        reconciled.add(owner.id);
-        const { files, invalid } = packageFiles(source.dir);
-        report.invalid.push(...invalid);
-        const seen = new Set();
-        for (const file of files) {
-            const id = await convergeFile(db, owner, file, report, reposFn);
-            seen.add(id);
-            converged.push({ owner, id, body: file.body });
-        }
-        await deleteMissing(db, owner, seen, report, reposFn);
+/** The live shipped document `name` (`<name>`, `<app>/<name>` or `app:<app>/<name>`) names for a package published by `owner`: the package's own documents
+ *  first, then every other app's, `crtr` last. It looks across all published packages, whoever installed them: whether a reader may follow the edge is the
+ *  reader's permission check when the link is rendered, not a property of the publisher. */
+async function shippedNamed(db, owner, name) {
+    const typed = /^(?:app:)?([^/:]+)\/(.+)$/.exec(name);
+    const candidates = [{ app: null, doc: name }];
+    if (typed !== null)
+        candidates.push({ app: typed[1], doc: typed[2] });
+    for (const { app, doc } of candidates) {
+        const row = await db.prepare(`SELECT o.id FROM objects o JOIN documents d ON d.object = o.id AND d.source_path IS NOT NULL JOIN objects a ON a.id = o.owner AND a.type = 'app'
+      WHERE o.type = 'document' AND o.deleted_at IS NULL AND o.home_machine IS NULL AND o.name = ? AND (?::text IS NULL OR a.name = ?)
+      ORDER BY (o.owner = ?) DESC, a.name = 'crtr', a.name LIMIT 1`).get(doc, app, app, owner.id);
+        if (row !== undefined)
+            return await getById(db, row.id) ?? undefined;
     }
-    // A plugin no longer installed (or disabled) keeps its app and the app's other objects;
-    // only its shipped documents are deleted.
-    const stale = await db.prepare(`SELECT DISTINCT o.owner AS id FROM objects o JOIN documents d ON d.object = o.id
-    WHERE o.type = 'document' AND o.deleted_at IS NULL AND d.source_path IS NOT NULL`).all();
-    for (const { id } of stale) {
-        if (!reconciled.has(id))
-            await deleteMissing(db, (await getById(db, id)), new Set(), report, reposFn);
-    }
-    // Links resolve only after every package's documents are in place, so a link to a document
-    // another package ships, or one reconciled later in this pass, still attaches.
-    for (const { owner, id, body } of converged)
-        await deriveLinks(db, owner, (await getById(db, id)), body);
-    return report;
+    return undefined;
 }
 /** Record a package document's `links-to` edges the way a runtime document gets them, so delivery
- *  attaches the link previews. Each `[[name]]` resolves in its own app's view and links only when it
- *  names another shipped document (builtin or plugin): a package never links to a user's or a
+ *  attaches the link previews. Each `[[name]]` links only when it names a shipped document of any
+ *  published package (builtin, provider or plugin): a package never links to a user's or a
  *  runtime document that happens to share a bare name. A name that resolves to nothing stays text.
  *  Edges are rewritten only when they differ, so an unchanged pass writes nothing. */
 export async function deriveLinks(db, owner, doc, body) {
     if (doc.deleted_at !== null)
         return;
-    const reader = { class: 'app', app: owner, scopes: [] };
     const links = [];
     for (const name of linksIn(body)) {
-        const hit = await resolveName(db, reader, name);
-        if ('object' in hit && hit.object.id !== doc.id && await shippedDocument(db, hit.object.id))
-            links.push({ written_as: name, dst: hit.object });
+        const hit = await shippedNamed(db, owner, name);
+        if (hit !== undefined && hit.id !== doc.id)
+            links.push({ written_as: name, dst: hit });
     }
     const key = (rows) => JSON.stringify(rows.map((r) => [r.dst, r.written_as]).sort());
     const current = await db.prepare(`SELECT dst, written_as FROM edges WHERE type = 'links-to' AND src = ?`)
@@ -109,9 +51,17 @@ export async function deriveLinks(db, owner, doc, body) {
         return;
     await setLinks(db, doc, links);
 }
-/** Whether the object is a live shipped document (its `documents.source_path` is set). */
-async function shippedDocument(db, id) {
-    return await db.prepare('SELECT 1 FROM documents WHERE object = ? AND source_path IS NOT NULL').get(id) !== undefined;
+/** Re-derive the links of every live shipped document of every app. A package published before the one its documents name has no edge to it yet;
+ *  publishing the missing one calls this, so the edges do not depend on which package arrived first. */
+export async function relinkAllShipped(db) {
+    const docs = await db.prepare(`SELECT o.id, o.owner FROM objects o JOIN documents d ON d.object = o.id
+    WHERE o.type = 'document' AND o.deleted_at IS NULL AND d.source_path IS NOT NULL ORDER BY o.id`).all();
+    for (const { id, owner } of docs) {
+        const doc = (await getById(db, id));
+        const [head] = await heads(db, doc);
+        if (head !== undefined)
+            await deriveLinks(db, (await getById(db, owner)), doc, await getBody(db, head.body_hash));
+    }
 }
 export async function convergeFile(db, owner, file, report, reposFn) {
     const f = documentFields(file.frontmatter, file.body, { missingPrivacy: 'public' });

package/dist/core/graph/package-publish.d.ts

@@ -0,0 +1,60 @@
+import type { KindConfig, ObjectKindDeclaration } from '../../types.js';
+import { type PackageReconcileReport } from './package-docs.js';
+import { type PackageFile } from './package-files.js';
+import type { CanvasDb } from '../canvas/db.js';
+/** What one app's package folder holds right now, read before the publishing transaction opens. */
+export interface PackageSnapshot {
+    app: string;
+    origin: 'builtin' | 'provider' | 'plugin';
+    /** crtr release (builtin), bundle generation (provider), manifest version (plugin) */
+    version: string;
+    /** the directory's revision for a directory package; null for builtin and for a Mac's local package */
+    revision: number | null;
+    /** normalized marketplace URL of a plugin package; null otherwise */
+    source: string | null;
+    /** the manifest's launch `kinds` (persona kinds); null when it declares none */
+    launchKinds: Record<string, Partial<KindConfig>> | null;
+    /** the manifest's custom-object kinds, keyed by kind name; null when it declares none */
+    objectKinds: Record<string, ObjectKindDeclaration> | null;
+    digest: string;
+    /** files with `path` already package-relative, prefixed by the app: `<app>/<relative path>` */
+    files: PackageFile[];
+    invalid: Array<{
+        path: string;
+        error: string;
+    }>;
+}
+export interface PublishReport extends PackageReconcileReport {
+    /** the row's release is newer than this snapshot: nothing was written */
+    superseded: boolean;
+    /** what declaring the snapshot's object kinds did; empty when the bytes were already published */
+    kinds: {
+        applied: Array<{
+            kind: string;
+            outcome: 'inserted' | 'updated' | 'unchanged';
+        }>;
+        refused: Array<{
+            kind: string;
+            message: string;
+        }>;
+    };
+}
+/** The package folder `dir` as a snapshot of app `app`. Throws when the folder or any directory in it cannot be read. The digest covers the
+ *  path and raw bytes of every markdown file, invalid ones included, so removing or changing an invalid file changes it. */
+export declare function readPackageSnapshot(dir: string, meta: Pick<PackageSnapshot, 'app' | 'origin' | 'version'> & Partial<Pick<PackageSnapshot, 'revision' | 'source' | 'launchKinds' | 'objectKinds'>>): PackageSnapshot;
+/** JSON with every object's keys sorted at every depth. */
+export declare function canonicalJson(value: unknown): string;
+/** App `crtr` as this process's release ships it. */
+export declare function readBuiltinSnapshot(): PackageSnapshot;
+/** Publish one snapshot inside the caller's write transaction. Serialized per app on the publication row. A `capture` is called after the lock is
+ *  taken, so a Mac reads its disk with no older read able to commit after it. Returns what changed. */
+export declare function publish(db: CanvasDb, input: PackageSnapshot | {
+    app: string;
+    origin: PackageSnapshot['origin'];
+    capture: () => PackageSnapshot;
+}, publishedBy: string): Promise<PublishReport>;
+/** A package folder that does not exist ships nothing: a plugin without a `memory/` folder. */
+export declare function emptyPackageSnapshot(meta: Pick<PackageSnapshot, 'app' | 'origin' | 'version'> & Partial<Pick<PackageSnapshot, 'launchKinds' | 'objectKinds'>>): PackageSnapshot;
+/** Take a package down: soft-delete the app's shipped documents, undeclare its manifest kinds, mark the row withdrawn. Nothing else of the
+ *  app (its ordinary documents, its kind rows) is touched. The caller decides the package is no longer wanted. */
+export declare function withdraw(db: CanvasDb, app: string): Promise<PublishReport>;

package/dist/core/graph/package-publish.js

@@ -0,0 +1,158 @@
+/** The package publisher, for the one package a release owns: app `crtr`, from `builtinMemoryRoot()`. A cloud role publishes it at every
+ *  daemon start and touches nothing else: no other app's documents, no manifest kind, no `crtr_app_` token, no plugin directory.
+ *  One `package_publications` row per published app is the lock a publisher takes and the release it compares against, so two pods
+ *  of different releases (a rolling deploy) never leave older text over newer text. The Mac pass (`reconcilePackages`) publishes through the same function. */
+import { createHash } from 'node:crypto';
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, relative, sep } from 'node:path';
+import { compareReleaseVersions } from '../../api/runtime-version.js';
+import { upsertKind } from './custom.js';
+import { crtrVersion } from '../../shared/crtr-version.js';
+import { emitEvent } from '../events/emit.js';
+import { builtinMemoryRoot } from '../scope.js';
+import { deleteMissing, convergeFile, relinkAllShipped } from './package-docs.js';
+import { packageFiles } from './package-files.js';
+import { ensureOwner } from './objects.js';
+import { checkoutsForCwd } from './repo-sync/repos.js';
+/** Every markdown file the package reader considers, as sorted `[package-relative path, raw bytes]`. Unlike `walkFiles`, a directory that
+ *  cannot be read throws: a partial or empty read must never look like a package that dropped its documents. */
+function strictPackageBytes(dir) {
+    if (!statSync(dir, { throwIfNoEntry: false })?.isDirectory())
+        throw new Error(`package folder is missing or not a directory: ${dir}`);
+    const out = [];
+    const stack = [dir];
+    while (stack.length > 0) {
+        const current = stack.pop();
+        for (const entry of readdirSync(current, { withFileTypes: true })) {
+            const full = join(current, entry.name);
+            if (entry.isDirectory()) {
+                if (!entry.name.startsWith('.'))
+                    stack.push(full);
+            }
+            else if (entry.isFile() && entry.name.endsWith('.md') && entry.name !== 'SKILL.md')
+                out.push([relative(dir, full).split(sep).join('/'), readFileSync(full)]);
+        }
+    }
+    return out.sort((a, b) => a[0] < b[0] ? -1 : 1);
+}
+/** The package folder `dir` as a snapshot of app `app`. Throws when the folder or any directory in it cannot be read. The digest covers the
+ *  path and raw bytes of every markdown file, invalid ones included, so removing or changing an invalid file changes it. */
+export function readPackageSnapshot(dir, meta) {
+    const bytes = strictPackageBytes(dir);
+    const { files, invalid } = packageFiles(dir);
+    const shipped = files.map((f) => ({ ...f, path: `${meta.app}/${relative(dir, f.path).split(sep).join('/')}` }));
+    const hash = createHash('sha256');
+    for (const [rel, data] of bytes)
+        hash.update(rel).update('\0').update(data).update('\0');
+    const shippedInvalid = invalid.map((i) => ({ ...i, path: `${meta.app}/${relative(dir, i.path).split(sep).join('/')}` }));
+    return snapshotOf({ revision: null, source: null, launchKinds: null, objectKinds: null, ...meta }, shipped, shippedInvalid, hash);
+}
+/** JSON with every object's keys sorted at every depth. */
+export function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(',')}]`;
+    if (value !== null && typeof value === 'object') {
+        const record = value;
+        return `{${Object.keys(record).sort().filter((k) => record[k] !== undefined).map((k) => `${JSON.stringify(k)}:${canonicalJson(record[k])}`).join(',')}}`;
+    }
+    return JSON.stringify(value) ?? 'null';
+}
+/** Close a digest over the package's kinds and build the snapshot. */
+function snapshotOf(meta, files, invalid, hash) {
+    hash.update(canonicalJson({ launchKinds: meta.launchKinds, objectKinds: meta.objectKinds, source: meta.source }));
+    return { ...meta, digest: hash.digest('hex'), files, invalid };
+}
+/** App `crtr` as this process's release ships it. */
+export function readBuiltinSnapshot() {
+    return readPackageSnapshot(builtinMemoryRoot(), { app: 'crtr', origin: 'builtin', version: crtrVersion() });
+}
+/** Whether `snapshot` is older than the published row: a builtin by release, a directory package by its revision. A Mac's local package
+ *  has no order of its own; it is captured under the row lock, so the last commit carries the newest bytes. */
+function olderThanPublished(snapshot, row) {
+    if (snapshot.revision !== null)
+        return row.revision !== null && snapshot.revision < Number(row.revision);
+    if (snapshot.origin === 'builtin')
+        return compareReleaseVersions(snapshot.version, row.version) < 0;
+    return false;
+}
+/** Publish one snapshot inside the caller's write transaction. Serialized per app on the publication row. A `capture` is called after the lock is
+ *  taken, so a Mac reads its disk with no older read able to commit after it. Returns what changed. */
+export async function publish(db, input, publishedBy) {
+    const { app, origin } = input;
+    const owner = await ensureOwner(db, 'app', 'app', `app:${app}`, app);
+    const now = new Date().toISOString();
+    await db.prepare(`INSERT INTO package_publications (app, origin, version, digest, published_at, published_by)
+    VALUES (?, ?, '', '', ?, ?) ON CONFLICT (app) DO NOTHING`).run(owner.id, origin, now, publishedBy);
+    const row = await db.prepare('SELECT version, digest, revision, withdrawn_at FROM package_publications WHERE app = ? FOR UPDATE').get(owner.id);
+    const snapshot = 'capture' in input ? input.capture() : input;
+    const report = { created: 0, updated: 0, deleted: 0, conflicts: [], invalid: [...snapshot.invalid], quarantined: [], superseded: false, kinds: { applied: [], refused: [] } };
+    if (olderThanPublished(snapshot, row)) {
+        report.superseded = true;
+        emitEvent({ level: 'warn', event: 'daemon.packages.superseded', fields: { app: snapshot.app, snapshot_version: snapshot.version, published_version: row.version, snapshot_revision: snapshot.revision ?? 'none', published_revision: row.revision === null ? 'none' : Number(row.revision) } });
+        return report;
+    }
+    const sameBytes = row.digest === snapshot.digest && row.withdrawn_at === null;
+    if (!sameBytes) {
+        const reposFn = async (cwd) => await checkoutsForCwd(db, cwd);
+        const seen = new Set();
+        for (const file of snapshot.files)
+            seen.add(await convergeFile(db, owner, file, report, reposFn));
+        // An invalid file keeps its prior document: its name is not known, so a snapshot with any invalid file removes nothing.
+        if (snapshot.invalid.length === 0)
+            await deleteMissing(db, owner, seen, report, reposFn);
+        report.kinds = await convergeKinds(db, owner, snapshot.objectKinds);
+        // A package published before the one its documents name has no edge to it yet, so any change revisits every shipped document.
+        if (report.created + report.updated + report.deleted > 0)
+            await relinkAllShipped(db);
+    }
+    else if (row.version === snapshot.version && String(row.revision ?? '') === String(snapshot.revision ?? '')) {
+        return report;
+    }
+    await db.prepare(`UPDATE package_publications SET origin = ?, version = ?, revision = ?, source = ?, launch_kinds_json = ?, digest = ?, published_at = ?, published_by = ?, withdrawn_at = NULL
+    WHERE app = ?`).run(snapshot.origin, snapshot.version, snapshot.revision, snapshot.source, snapshot.launchKinds === null ? null : canonicalJson(snapshot.launchKinds), snapshot.digest, now, publishedBy, owner.id);
+    return report;
+}
+/** A package folder that does not exist ships nothing: a plugin without a `memory/` folder. */
+export function emptyPackageSnapshot(meta) {
+    return snapshotOf({ revision: null, source: null, launchKinds: null, objectKinds: null, ...meta }, [], [], createHash('sha256'));
+}
+/** Declare the snapshot's object kinds under the app and undeclare the app's manifest kinds it no longer lists. A kind another owner holds is
+ *  left alone (its document text still publishes). A kind row is never deleted. */
+async function convergeKinds(db, owner, kinds) {
+    const report = { applied: [], refused: [] };
+    const declared = new Set();
+    for (const [name, decl] of Object.entries(kinds ?? {})) {
+        try {
+            const result = await upsertKind(db, { name, owner: owner, description: decl.description, events: decl.events,
+                read: decl.read === undefined ? null : { command: decl.read.command }, source: 'manifest' });
+            if (result.outcome !== 'skipped') {
+                declared.add(name);
+                report.applied.push({ kind: name, outcome: result.outcome });
+            }
+        }
+        catch (error) {
+            if (error.code !== 'kind_owned')
+                throw error;
+            report.refused.push({ kind: name, message: error.message });
+            emitEvent({ level: 'warn', event: 'daemon.packages.kind_owned', fields: { app: owner.id, kind: name } });
+        }
+    }
+    const rows = await db.prepare("SELECT name FROM object_kinds WHERE owner = ? AND source = 'manifest' AND declared = 1").all(owner.id);
+    for (const { name } of rows)
+        if (!declared.has(name))
+            await db.prepare('UPDATE object_kinds SET declared = 0 WHERE name = ?').run(name);
+    return report;
+}
+/** Take a package down: soft-delete the app's shipped documents, undeclare its manifest kinds, mark the row withdrawn. Nothing else of the
+ *  app (its ordinary documents, its kind rows) is touched. The caller decides the package is no longer wanted. */
+export async function withdraw(db, app) {
+    const report = { created: 0, updated: 0, deleted: 0, conflicts: [], invalid: [], quarantined: [], superseded: false, kinds: { applied: [], refused: [] } };
+    const owner = await ensureOwner(db, 'app', 'app', `app:${app}`, app);
+    const row = await db.prepare('SELECT withdrawn_at FROM package_publications WHERE app = ? FOR UPDATE').get(owner.id);
+    if (row === undefined || row.withdrawn_at !== null)
+        return report;
+    await deleteMissing(db, owner, new Set(), report, async (cwd) => await checkoutsForCwd(db, cwd));
+    await convergeKinds(db, owner, null);
+    await db.prepare("UPDATE package_publications SET withdrawn_at = ?, digest = '' WHERE app = ?").run(new Date().toISOString(), owner.id);
+    return report;
+}

package/dist/core/graph/package-reconcile.d.ts

@@ -1,4 +1,3 @@
-import { type KindsReport } from './package-kinds.js';
 import { type PackageReconcileReport } from './package-docs.js';
 import type { CanvasDb } from '../canvas/db.js';
 export interface TokensReport {
@@ -10,6 +9,26 @@ export interface TokensReport {
     /** plugins whose `crtr_app_` token was removed because they no longer qualify */
     revoked: string[];
 }
+export interface KindsReport {
+    /** every kind a valid plugin declares that now has its row, and what upserting it did */
+    applied: Array<{
+        plugin: string;
+        kind: string;
+        outcome: 'inserted' | 'updated' | 'unchanged';
+    }>;
+    /** kinds another owner holds; the plugin's other kinds still applied */
+    refused: Array<{
+        plugin: string;
+        kind: string;
+        code: 'kind_owned';
+        message: string;
+    }>;
+    /** enabled plugins whose manifest or commands are invalid (no kind update, no token); a provider bundle's kinds reconcile without a token, and plugins whose app handle another owner holds */
+    invalid: Array<{
+        plugin: string;
+        error: string;
+    }>;
+}
 export interface PackagesReport extends PackageReconcileReport {
     kinds: KindsReport;
     tokens: TokensReport;

package/dist/core/graph/package-reconcile.js

@@ -1,31 +1,99 @@
-/** The one package reconcile pass: documents, then kinds, then plugin app tokens. Runs in the caller's write
- *  transaction. The token writes go to the secrets store and are idempotent, so a rolled-back pass re-converges
- *  on the next one. */
-import { appPlugins } from '../plugin-app.js';
+/** The Mac's package pass: the builtin corpus and every enabled local plugin or provider bundle are published through the publisher (each package
+ *  captured from disk under its publication row's lock), a package whose app is no longer enabled is withdrawn, then plugin app tokens are
+ *  minted and revoked. Runs in the caller's write transaction. The token writes go to the secrets store and are idempotent, so a rolled-back
+ *  pass re-converges on the next one. A cloud role never runs this: it sees no person's plugins (it publishes `crtr` and the directory's packages). */
+import { statSync } from 'node:fs';
+import { CrtrError } from '../errors.js';
+import { emitEvent } from '../events/emit.js';
+import { invalidAppPlugins, appPlugins, kindPlugins } from '../plugin-app.js';
+import { quarantineBrokenPlugins } from '../plugin-quarantine.js';
+import { listInstalledPlugins } from '../resolver.js';
+import { pluginMemoryDir } from '../scope.js';
 import { ensurePluginAppToken, listScopedApiTokens, revokePluginAppTokens } from '../secrets.js';
-import { reconcilePackageKinds } from './package-kinds.js';
-import { reconcilePackageDocuments } from './package-docs.js';
+import { deleteMissing } from './package-docs.js';
+import { emptyPackageSnapshot, publish, readBuiltinSnapshot, readPackageSnapshot, withdraw } from './package-publish.js';
+import { getById } from './objects.js';
+import { checkoutsForCwd } from './repo-sync/repos.js';
+const PUBLISHED_BY = 'mac';
+function isDirectory(path) {
+    return statSync(path, { throwIfNoEntry: false })?.isDirectory() === true;
+}
 export async function reconcilePackages(db) {
-    const documents = await reconcilePackageDocuments(db);
-    const kinds = await reconcilePackageKinds(db);
-    const tokens = { granted: [], revoked: [] };
+    const report = {
+        created: 0, updated: 0, deleted: 0, conflicts: [], invalid: [], quarantined: quarantineBrokenPlugins(),
+        kinds: { applied: [], refused: [], invalid: invalidAppPlugins().map((i) => ({ plugin: i.plugin.name, error: i.error })) },
+        tokens: { granted: [], revoked: [] },
+    };
+    for (const q of report.quarantined) {
+        emitEvent({ level: 'error', event: 'daemon.plugins.quarantined', fields: { plugin: q.name, file: q.file, reason: q.reason, ...(q.movedTo !== undefined ? { moved_to: q.movedTo } : {}) } });
+    }
+    const validKinds = new Set(kindPlugins().map((p) => p.name));
+    const published = new Set(['crtr']);
+    const add = (r, plugin) => {
+        report.created += r.created;
+        report.updated += r.updated;
+        report.deleted += r.deleted;
+        report.invalid.push(...r.invalid);
+        for (const k of r.kinds.applied)
+            report.kinds.applied.push({ plugin, kind: k.kind, outcome: k.outcome });
+        for (const k of r.kinds.refused)
+            report.kinds.refused.push({ plugin, kind: k.kind, code: 'kind_owned', message: k.message });
+    };
+    add(await publish(db, { app: 'crtr', origin: 'builtin', capture: () => readBuiltinSnapshot() }, PUBLISHED_BY), 'crtr');
+    for (const plugin of listInstalledPlugins('user').filter((p) => p.enabled)) {
+        const origin = plugin.manifest.provider === undefined ? 'plugin' : 'provider';
+        const dir = pluginMemoryDir(plugin);
+        const kinds = plugin.manifest.kinds !== undefined && Object.keys(plugin.manifest.kinds).length > 0 ? plugin.manifest.kinds : null;
+        const meta = { app: plugin.name, origin, version: plugin.manifest.version ?? '', launchKinds: kinds, objectKinds: validKinds.has(plugin.name) ? plugin.manifest.object_kinds ?? null : null };
+        try {
+            if (published.has(plugin.name))
+                throw new CrtrError('conflict', `plugin "${plugin.name}" is already provided by another package`);
+            published.add(plugin.name);
+            add(await publish(db, { app: plugin.name, origin, capture: () => isDirectory(dir) ? readPackageSnapshot(dir, meta) : emptyPackageSnapshot(meta) }, PUBLISHED_BY), plugin.name);
+        }
+        catch (error) {
+            if (!(error instanceof CrtrError) || error.code !== 'conflict')
+                throw error;
+            report.conflicts.push({ plugin: plugin.name, message: error.message });
+        }
+    }
+    // A package no longer installed (or disabled) keeps its app and the app's other objects; only its shipped documents go and its row is withdrawn.
+    const gone = await db.prepare(`SELECT i.external_id FROM package_publications p JOIN identities i ON i.object = p.app AND i.scheme = 'app'
+    WHERE p.withdrawn_at IS NULL ORDER BY i.external_id`).all();
+    for (const { external_id } of gone) {
+        const app = external_id.replace(/^app:/, '');
+        if (published.has(app))
+            continue;
+        const w = await withdraw(db, app);
+        report.deleted += w.deleted;
+    }
+    // Shipped documents of an app that never had a publication row (written by a release before the publisher) are removed the same way.
+    const stale = await db.prepare(`SELECT DISTINCT o.owner AS id FROM objects o JOIN documents d ON d.object = o.id
+    WHERE o.type = 'document' AND o.deleted_at IS NULL AND d.source_path IS NOT NULL`).all();
+    for (const { id } of stale) {
+        const owner = (await getById(db, id));
+        const ext = await db.prepare(`SELECT external_id FROM identities WHERE object = ? AND scheme = 'app'`).get(id);
+        if (ext !== undefined && published.has(ext.external_id.replace(/^app:/, '')))
+            continue;
+        await deleteMissing(db, owner, new Set(), report, async (cwd) => await checkoutsForCwd(db, cwd));
+    }
     const holders = new Set();
     for (const plugin of appPlugins()) {
         const scopes = plugin.manifest.scopes ?? [];
         ensurePluginAppToken(plugin.name, scopes);
         holders.add(plugin.name);
-        tokens.granted.push({ plugin: plugin.name, scopes: [...scopes] });
+        report.tokens.granted.push({ plugin: plugin.name, scopes: [...scopes] });
     }
-    const stale = new Set();
+    const staleTokens = new Set();
     for (const entry of listScopedApiTokens('user')) {
         if (!entry.token.startsWith('crtr_app_') || !entry.grantee.startsWith('app:'))
             continue;
         const plugin = entry.grantee.slice('app:'.length);
         if (!holders.has(plugin))
-            stale.add(plugin);
+            staleTokens.add(plugin);
     }
-    for (const plugin of stale)
+    for (const plugin of staleTokens)
         if (revokePluginAppTokens(plugin))
-            tokens.revoked.push(plugin);
-    return { ...documents, kinds, tokens };
+            report.tokens.revoked.push(plugin);
+    return report;
 }

package/dist/core/import/export/dump.js

@@ -4,12 +4,11 @@
 import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { openDb, withCanvasWrite } from '../../canvas/db.js';
-import { hashOf, storable } from '../../graph/bodies.js';
+import { hashOf } from '../../graph/bodies.js';
 import { personIdAsync } from '../../canvas/person.js';
 import { CANVAS_SCHEMA_VERSION } from '../../canvas/schema-version.js';
 import { crtrVersion } from '../../../shared/crtr-version.js';
 import { BODIES_TABLE, DUMPED_TABLES, RowFileWriter, SEQ_TABLES, bundleKeys, encodeRows, rowLine } from '../bundle/layout.js';
-import { parseFrontmatterGeneric } from '../../frontmatter.js';
 import { inlineMessageBody } from './inline.js';
 /** A `bodies` part file holds about this much JSON (the encoded lines, before compression); a row that alone exceeds it is a part of its own. */
 export const PART_JSON_BYTES = 16 * 1024 * 1024;
@@ -33,16 +32,6 @@ function readText(path) {
         throw error;
     }
 }
-function readFileOrNull(path) {
-    try {
-        return readFileSync(path, 'utf8');
-    }
-    catch (error) {
-        if (error.code === 'ENOENT')
-            return null;
-        throw error;
-    }
-}
 function tooLarge(hash, bytes) {
     return new Error(`revision text ${hash} is ${bytes} bytes; the import carries at most ${BODY_MAX_BYTES} bytes of text per body`);
 }
@@ -55,7 +44,7 @@ function assertCanonical(row) {
 }
 /** Read the `bodies` rows of `hashes` (in that order) and write them as part files `<phase>-<n>`; returns the part names. Rows are read in pages of about `PAGE_TEXT_BYTES`
  *  of text (sizes first, by `octet_length`), each row is written as one line, and a part is closed when its encoded JSON reaches `PART_JSON_BYTES`. */
-async function writeBodyParts(db, out, phase, hashes, extra) {
+async function writeBodyParts(db, out, phase, hashes) {
     const dir = join(out, 'rows', BODIES_TABLE);
     mkdirSync(dir, { recursive: true });
     const parts = [];
@@ -102,12 +91,6 @@ async function writeBodyParts(db, out, phase, hashes, extra) {
             await fetchPage();
             i += SIZE_BATCH;
         }
-        for (const row of extra) {
-            const bytes = Buffer.byteLength(row.text, 'utf8');
-            if (bytes > BODY_MAX_BYTES)
-                throw tooLarge(row.hash, bytes);
-            await add(row);
-        }
         await close();
     }
     catch (error) {
@@ -122,7 +105,7 @@ const allHashes = async (db) => (await db.prepare('SELECT hash FROM bodies ORDER
 export async function dumpBodiesPre(out) {
     const db = await openDb();
     const hashes = await allHashes(db);
-    const parts = await writeBodyParts(db, out, 'pre', hashes, []);
+    const parts = await writeBodyParts(db, out, 'pre', hashes);
     const pre = { hashes, parts, count: hashes.length, canonical: true };
     writeFileSync(join(out, PRE_FILE), JSON.stringify(pre));
     return { parts, count: hashes.length };
@@ -197,28 +180,21 @@ export async function dumpRows(out, kind) {
         const environment = await db.prepare(`SELECT object FROM environments WHERE kind = 'machine' ORDER BY object LIMIT 1`).get();
         const source = { installId, kind, person, release: crtrVersion(), schemaVersion: CANVAS_SCHEMA_VERSION, counts, ...(environment === undefined ? {} : { environmentObject: environment.object }) };
         const requests = await db.prepare('SELECT request_id, request_dir, file FROM human_requests ORDER BY request_id').all();
-        // A shipped document's body is its file's body after the frontmatter (`bodyOf`); a file the source no longer has is skipped. The text travels as a `bodies` row.
-        const shipped = await db.prepare('SELECT object, source_path FROM documents WHERE source_path IS NOT NULL ORDER BY object').all();
+        // A shipped document's text is its head revision's `bodies` row, the same row every reader gets; `source_path` is provenance only and is never opened.
+        // The row travels with the table, so the bundle needs only the hash.
+        const shipped = await db.prepare(`SELECT d.object, r.body_hash FROM documents d JOIN revisions r ON r.document = d.object
+      WHERE d.source_path IS NOT NULL AND NOT EXISTS (SELECT 1 FROM revision_parents p WHERE p.parent = r.rev_id) ORDER BY d.object, r.number DESC, r.rev_id`).all();
         const shippedBodies = {};
-        const shippedRows = new Map();
-        for (const doc of shipped) {
-            const file = readFileOrNull(doc.source_path);
-            if (file === null)
-                continue;
-            const text = storable(parseFrontmatterGeneric(file).body);
-            const hash = hashOf(text);
-            shippedBodies[doc.object] = hash;
-            shippedRows.set(hash, { hash, text });
-        }
-        // `bodies` rows added since the pre-pause dump (all of them when there was none), and the shipped texts the pre-pause dump did not hold.
+        for (const doc of shipped)
+            shippedBodies[doc.object] ??= doc.body_hash;
+        // `bodies` rows added since the pre-pause dump (all of them when there was none); the shipped texts are among them.
         const pre = readPre(out);
         const had = new Set(pre.hashes);
         const now = await allHashes(db);
         assertPreCurrent(pre, new Set(now));
         const fresh = now.filter((hash) => !had.has(hash));
-        const added = new Set(fresh);
-        const pauseParts = await writeBodyParts(db, out, 'pause', fresh, [...shippedRows.values()].filter((row) => !had.has(row.hash) && !added.has(row.hash)));
-        const bodyCount = pre.count + fresh.length + [...shippedRows.values()].filter((row) => !had.has(row.hash) && !added.has(row.hash)).length;
+        const pauseParts = await writeBodyParts(db, out, 'pause', fresh);
+        const bodyCount = pre.count + fresh.length;
         counts[BODIES_TABLE] = bodyCount;
         source.bodyParts = [...pre.parts, ...pauseParts];
         const running = await db.prepare(`SELECT n.node_id, r.grantee, n.cwd, w.path AS worktree FROM nodes n JOIN node_rows r ON r.node_id = n.node_id LEFT JOIN worktrees w ON w.node_id = n.node_id AND w.state = 'open'