@capybara-hq/kernel 0.1.0-rc.1 → 0.1.0-rc.10

package/dist/catalog/reserved-names.d.ts

@@ -0,0 +1,8 @@
+import type { KernelSchemaDb } from "../db/kernel-schema.js";
+/**
+ * 409 `plugin.name_reserved` when an org upload names a plugin (and so its extension: one name
+ * space) of the bundled bootstrap manifest or of an entry of any marketplace the deployment has
+ * read, official or custom (ruling P3-28): an org cannot take a name a later bootstrap pin or a
+ * platform import of a public plugin needs, since the name would then belong to the org (P3-14).
+ */
+export declare function assertNameNotReserved(db: KernelSchemaDb, name: string): Promise<void>;

package/dist/catalog/reserved-names.js

@@ -0,0 +1,42 @@
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { parseBootstrapManifest } from "@capybara-hq/plugin-sdk";
+import { ApiError } from "../errors.js";
+/**
+ * The bootstrap manifest shipped with the kernel package (the same file as
+ * `extensions/bootstrap.ts`'s `BOOTSTRAP_MANIFEST_FILE`, which this module does not import: that
+ * module depends on the plugin import).
+ */
+const BUNDLED_BOOTSTRAP_MANIFEST = fileURLToPath(new URL("../../bootstrap.json", import.meta.url));
+/** The plugin names of the bundled bootstrap manifest (an unreadable or invalid one: none). */
+async function bootstrapNames() {
+    let raw;
+    try {
+        raw = JSON.parse(await readFile(BUNDLED_BOOTSTRAP_MANIFEST, "utf8"));
+    }
+    catch {
+        return [];
+    }
+    const parsed = parseBootstrapManifest(raw);
+    return "errors" in parsed ? [] : parsed.plugins.map((p) => p.name);
+}
+/**
+ * 409 `plugin.name_reserved` when an org upload names a plugin (and so its extension: one name
+ * space) of the bundled bootstrap manifest or of an entry of any marketplace the deployment has
+ * read, official or custom (ruling P3-28): an org cannot take a name a later bootstrap pin or a
+ * platform import of a public plugin needs, since the name would then belong to the org (P3-14).
+ */
+export async function assertNameNotReserved(db, name) {
+    const where = (await bootstrapNames()).includes(name)
+        ? "a bootstrap plugin of this deployment"
+        : (await db
+            .selectFrom("marketplace_entries")
+            .select("name")
+            .where("name", "=", name)
+            .executeTakeFirst()) !== undefined
+            ? "a plugin listed by a marketplace of this deployment"
+            : undefined;
+    if (where === undefined)
+        return;
+    throw new ApiError("plugin.name_reserved", `Plugin name ${name} is reserved for ${where}; rename the plugin (plugin.json name) and upload it again`);
+}

package/dist/catalog/scope-orgs.d.ts

@@ -0,0 +1,21 @@
+import type { KernelSchemaDb } from "../db/kernel-schema.js";
+/**
+ * Adds one org to a `named` item's list in the caller's transaction (the caller holds the item row
+ * `FOR UPDATE`): bumps `scope_version` and audits `catalog.scope_changed` the way a scope change
+ * does (`from` / `to` with the sorted lists; `requestId` when a request asked for it). Nothing is
+ * written when the item is not `named` (or builtin) or the org is listed already. Returns whether
+ * the list changed; the caller enqueues the item's scope sync after the commit. Adding an org other
+ * than its owner to an org-owned item that is not shareable -> 409 `catalog.not_shareable` (the
+ * share gate, ruling P3-16, read on the caller's locked row).
+ */
+export declare function addScopeOrg(db: KernelSchemaDb, actor: string | null, item: {
+    id: string;
+    name: string;
+    scope: "all" | "optional" | "named";
+    builtin: boolean;
+    owner_org_id: string | null;
+    shareable: boolean;
+}, orgId: string, opts?: {
+    actorTokenId?: string;
+    requestId?: string;
+}): Promise<boolean>;

package/dist/catalog/scope-orgs.js

@@ -0,0 +1,67 @@
+import { sql } from "kysely";
+import { audit } from "../identity/audit.js";
+import { notifyItemShared } from "../inbox/org-notices.js";
+import { assertShareable, shareGate } from "./ownership.js";
+/**
+ * Adds one org to a `named` item's list in the caller's transaction (the caller holds the item row
+ * `FOR UPDATE`): bumps `scope_version` and audits `catalog.scope_changed` the way a scope change
+ * does (`from` / `to` with the sorted lists; `requestId` when a request asked for it). Nothing is
+ * written when the item is not `named` (or builtin) or the org is listed already. Returns whether
+ * the list changed; the caller enqueues the item's scope sync after the commit. Adding an org other
+ * than its owner to an org-owned item that is not shareable -> 409 `catalog.not_shareable` (the
+ * share gate, ruling P3-16, read on the caller's locked row).
+ */
+export async function addScopeOrg(db, actor, item, orgId, opts = {}) {
+    if (item.builtin || item.scope !== "named")
+        return false;
+    if (!shareGate(item) && orgId !== item.owner_org_id) {
+        const listed = await db
+            .selectFrom("catalog_item_scope_orgs")
+            .select("org_id")
+            .where("item_id", "=", item.id)
+            .where("org_id", "=", orgId)
+            .executeTakeFirst();
+        if (listed !== undefined)
+            return false;
+        assertShareable(item);
+    }
+    const added = await db
+        .insertInto("catalog_item_scope_orgs")
+        .values({ item_id: item.id, org_id: orgId })
+        .onConflict((oc) => oc.doNothing())
+        .returning("org_id")
+        .executeTakeFirst();
+    if (added === undefined)
+        return false;
+    await db
+        .updateTable("catalog_items")
+        .set({
+        scope_version: sql `scope_version + 1`,
+        updated_at: sql `now()`,
+    })
+        .where("id", "=", item.id)
+        .execute();
+    const rows = await db
+        .selectFrom("catalog_item_scope_orgs")
+        .select("org_id")
+        .where("item_id", "=", item.id)
+        .orderBy("org_id")
+        .execute();
+    const to = rows.map((r) => r.org_id);
+    await audit(db, {
+        ...(actor !== null && { actorAccountId: actor }),
+        ...(opts.actorTokenId !== undefined && { actorTokenId: opts.actorTokenId }),
+        action: "catalog.scope_changed",
+        target: item.id,
+        detail: {
+            name: item.name,
+            from: { scope: "named", orgs: to.filter((o) => o !== orgId) },
+            to: { scope: "named", orgs: to },
+            ...(opts.requestId !== undefined && { requestId: opts.requestId }),
+        },
+    });
+    await notifyItemShared(db, item.owner_org_id, { kind: "plugin", id: item.id }, item.name, [
+        orgId,
+    ]);
+    return true;
+}

package/dist/catalog/scope-state.d.ts

@@ -0,0 +1,35 @@
+import type { KernelSchemaDb } from "../db/kernel-schema.js";
+/**
+ * Whether the item's own scope wants it on in the org (`wantedOn` over the item's scope, builtin
+ * and disabled flags, the org's list membership and its switch), read through `db` (the caller's
+ * transaction under the org row lock, or the pool). A deleted or missing item is not wanted.
+ */
+export declare function wantedByScope(db: KernelSchemaDb, orgId: string, itemId: string): Promise<boolean>;
+/**
+ * A scope sync install (`via: "scope"`) found, under the org row lock, that the item is no longer
+ * wanted in the org (a person switched it off, or the scope changed, after the sync's decision):
+ * nothing was written; the sync treats it as converged.
+ */
+export declare class ScopeNoLongerWanted extends Error {
+    readonly orgId: string;
+    readonly itemId: string;
+    constructor(orgId: string, itemId: string);
+}
+/**
+ * A scope sync uninstall (`via: "scope"`) found, under the org row lock, that the item is wanted
+ * in the org again (a person switched it on, or the scope changed, after the sync's decision), or
+ * that another live install of the org depends on it: nothing was written; the sync treats it as
+ * converged.
+ */
+export declare class ScopeStillWanted extends Error {
+    readonly orgId: string;
+    readonly itemId: string;
+    constructor(orgId: string, itemId: string);
+}
+/**
+ * The items among `itemIds` (live ones) the org can see in its catalog: offered to it (`all`,
+ * `optional`, builtin, or `named` with the org listed) or requestable (`named`, shown to other
+ * orgs, through the share gate). A hidden `named` item outside its list is left out, so nothing
+ * reveals it to the org.
+ */
+export declare function itemsSeenBy(db: KernelSchemaDb, orgId: string, itemIds: readonly string[]): Promise<Set<string>>;

package/dist/catalog/scope-state.js

@@ -0,0 +1,87 @@
+import { requestable, visibleTo, wantedOn } from "./scope.js";
+/**
+ * Whether the item's own scope wants it on in the org (`wantedOn` over the item's scope, builtin
+ * and disabled flags, the org's list membership and its switch), read through `db` (the caller's
+ * transaction under the org row lock, or the pool). A deleted or missing item is not wanted.
+ */
+export async function wantedByScope(db, orgId, itemId) {
+    const item = await db
+        .selectFrom("catalog_items")
+        .select(["scope", "builtin", "disabled"])
+        .where("id", "=", itemId)
+        .where("deleted_at", "is", null)
+        .executeTakeFirst();
+    if (!item)
+        return false;
+    // one after the other: `db` may be a transaction's single connection
+    const listed = await db
+        .selectFrom("catalog_item_scope_orgs")
+        .select("item_id")
+        .where("item_id", "=", itemId)
+        .where("org_id", "=", orgId)
+        .executeTakeFirst();
+    const sw = await db
+        .selectFrom("catalog_item_org_switches")
+        .select("enabled")
+        .where("item_id", "=", itemId)
+        .where("org_id", "=", orgId)
+        .executeTakeFirst();
+    return wantedOn(item, listed !== undefined, sw?.enabled ?? null);
+}
+/**
+ * A scope sync install (`via: "scope"`) found, under the org row lock, that the item is no longer
+ * wanted in the org (a person switched it off, or the scope changed, after the sync's decision):
+ * nothing was written; the sync treats it as converged.
+ */
+export class ScopeNoLongerWanted extends Error {
+    orgId;
+    itemId;
+    constructor(orgId, itemId) {
+        super(`Plugin ${itemId} is no longer wanted in org ${orgId} by its scope`);
+        this.orgId = orgId;
+        this.itemId = itemId;
+        this.name = "ScopeNoLongerWanted";
+    }
+}
+/**
+ * A scope sync uninstall (`via: "scope"`) found, under the org row lock, that the item is wanted
+ * in the org again (a person switched it on, or the scope changed, after the sync's decision), or
+ * that another live install of the org depends on it: nothing was written; the sync treats it as
+ * converged.
+ */
+export class ScopeStillWanted extends Error {
+    orgId;
+    itemId;
+    constructor(orgId, itemId) {
+        super(`Plugin ${itemId} is still wanted in org ${orgId}`);
+        this.orgId = orgId;
+        this.itemId = itemId;
+        this.name = "ScopeStillWanted";
+    }
+}
+/**
+ * The items among `itemIds` (live ones) the org can see in its catalog: offered to it (`all`,
+ * `optional`, builtin, or `named` with the org listed) or requestable (`named`, shown to other
+ * orgs, through the share gate). A hidden `named` item outside its list is left out, so nothing
+ * reveals it to the org.
+ */
+export async function itemsSeenBy(db, orgId, itemIds) {
+    if (itemIds.length === 0)
+        return new Set();
+    const items = await db
+        .selectFrom("catalog_items")
+        .select(["id", "scope", "builtin", "shown_to_others", "owner_org_id", "shareable"])
+        .where("id", "in", [...itemIds])
+        .where("deleted_at", "is", null)
+        .execute();
+    const listed = new Set((await db
+        .selectFrom("catalog_item_scope_orgs")
+        .select("item_id")
+        .where("org_id", "=", orgId)
+        .where("item_id", "in", [...itemIds])
+        .execute()).map((r) => r.item_id));
+    return new Set(items
+        .filter((i) => visibleTo({ scope: i.builtin ? "all" : i.scope }, listed.has(i.id)) ||
+        requestable(i, listed.has(i.id)))
+        .map((i) => i.id));
+}

package/dist/catalog/scope-sync-workers.d.ts

@@ -0,0 +1,7 @@
+/**
+ * The plugin scope queue's consumer: {@link PluginScopeSyncProcessor}. Imported only by the worker
+ * role (`WorkerModule`), so other roles get no processor and no BullMQ worker. It never starts by
+ * itself: the worker role (`WorkerRole`) starts it after the extensions loaded.
+ */
+export declare class PluginScopeWorkersModule {
+}

package/dist/catalog/scope-sync-workers.js

@@ -0,0 +1,29 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { Module } from "@nestjs/common";
+import { configuredProcessor } from "../infra/queue/configured-processor.js";
+import { CatalogModule } from "./catalog.module.js";
+import { PLUGIN_SCOPE_CONCURRENCY, PLUGIN_SCOPE_QUEUE, PluginScopeSyncProcessor, } from "./scope-sync.js";
+/**
+ * The plugin scope queue's consumer: {@link PluginScopeSyncProcessor}. Imported only by the worker
+ * role (`WorkerModule`), so other roles get no processor and no BullMQ worker. It never starts by
+ * itself: the worker role (`WorkerRole`) starts it after the extensions loaded.
+ */
+let PluginScopeWorkersModule = class PluginScopeWorkersModule {
+};
+PluginScopeWorkersModule = __decorate([
+    Module({
+        imports: [CatalogModule],
+        providers: [
+            configuredProcessor(PluginScopeSyncProcessor, PLUGIN_SCOPE_QUEUE, {
+                concurrency: PLUGIN_SCOPE_CONCURRENCY,
+            }),
+        ],
+        exports: [PluginScopeSyncProcessor],
+    })
+], PluginScopeWorkersModule);
+export { PluginScopeWorkersModule };

package/dist/catalog/scope-sync.d.ts

@@ -0,0 +1,176 @@
+import { WorkerHost } from "@nestjs/bullmq";
+import type { Job, Queue, Worker } from "bullmq";
+import { ExtensionLifecycle } from "../extensions/lifecycle.js";
+import { ExtensionRegistry } from "../extensions/registry.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import type { OneJob } from "../infra/queue/add-job-once.js";
+import type { KernelLogger } from "../log.js";
+import { PluginInstaller } from "./plugin-install.js";
+/**
+ * The plugin scope sync (plugin and component management spec §5, phase 1): converges every org
+ * to each platform plugin's scope. In phase 1 "on in an org" is the org's plugin install, so for
+ * each (item, org) the sync:
+ * - wanted on (`wantedOn`, or a live install in the org depends on it: rule 9) and no live install
+ *   -> installs it as the kernel itself (actor null, audits `via: "scope"`), unless the install
+ *   needs the org's input (connector templates, or a required org config of its extension not set):
+ *   that is `needs_config`, derived by the org catalog, never attempted here;
+ * - wanted off and a live install -> uninstalls it (`via: "scope"`; the person who changed the
+ *   scope or the switch confirmed the preview); the connectors the install created stay as the
+ *   org's own (spec §5 rule 8); a dependency install it took along that its own scope still wants
+ *   on stays as a standalone install (the installer's rule), the others are synced again.
+ * The decision is made outside the org row lock; the installer re-checks the scope under it for a
+ * `via: "scope"` install or uninstall, so a person's switch committed meanwhile wins.
+ * The sync writes no switch row: those record people's choices, and `wantedOn` honours them for
+ * every scope (an org that switched an `all` item off stays off, also across a scope change away
+ * and back, layering spec §4). Builtin items are left alone (on everywhere through their required
+ * extension). A failed install, uninstall or dependency install is logged, audited
+ * `plugin.scope_sync_failed`, kept as the pair's open failure (`plugin_scope_sync_failures`, the
+ * super admin's inbox, until a later sync of the pair does not fail) and not retried until the next
+ * enqueue. An error outside those steps (a read, the failure write itself) ends the attempt without
+ * touching the pair's row, so an older failure stays open.
+ * Every worker start enqueues every live item once ({@link PluginScopeSync.enqueueAll}), which
+ * also catches a lost enqueue.
+ *
+ * Jobs (queue {@link PLUGIN_SCOPE_QUEUE}): one per item and scope version
+ * (`scope-item-<itemId>-<scope_version>`, every org), one per org (`scope-org-<orgId>`, every item,
+ * enqueued after the org is created). Both are removed when done, so the next enqueue with the same
+ * id is not swallowed. Pairs of one org are serialized across replicas by a Redis lock per org; the
+ * installer's org row lock and its one-live-install rule are the backstop.
+ */
+/** The scope sync queue (no `:` in the name). */
+export declare const PLUGIN_SCOPE_QUEUE = "cap-kernel-plugin-scope";
+/** The job converging one item in every org. */
+export declare const SCOPE_ITEM_JOB = "scope-item";
+/** The job converging every item in one org. */
+export declare const SCOPE_ORG_JOB = "scope-org";
+/** One item job per scope version (no `:`: item ids are UUIDs). */
+export declare const scopeItemJobId: (itemId: string, version: number) => string;
+/** One org job at a time per org. */
+export declare const scopeOrgJobId: (orgId: string) => string;
+/**
+ * The delayed retry `n` of an item job whose extension this replica had not loaded yet (ruling
+ * P3-38); deterministic, so replicas deferring the same item add it once.
+ */
+export declare const scopeRetryJobId: (itemId: string, version: number, n: number) => string;
+/** How many delayed retries an item job gets while its extension is not loaded here (P3-38). */
+export declare const SCOPE_LOAD_RETRIES = 6;
+/** The delay of each such retry. */
+export declare const SCOPE_LOAD_RETRY_DELAY_MS = 5000;
+/** How long a pair waits for this replica's reconcile before deferring (P3-38). */
+export declare const SCOPE_RECONCILE_WAIT_MS = 10000;
+/**
+ * The job's data. `retry`: the item job's delayed retry number (P3-38); at
+ * {@link SCOPE_LOAD_RETRIES} an extension still not loaded is a failure as before.
+ */
+export type ScopeJob = {
+    kind: "item";
+    itemId: string;
+    retry?: number;
+} | {
+    kind: "org";
+    orgId: string;
+};
+/**
+ * Awaits this replica's extension reconcile for at most `ms` (a slower one carries on in the
+ * background); a failure only warns.
+ */
+export declare function reconcileWithin(lifecycle: Pick<ExtensionLifecycle, "reconcile">, reason: string, ms: number, log: Pick<KernelLogger, "warn">): Promise<void>;
+/** The item job of a scope version (what {@link PluginScopeSync.enqueueItem} adds). */
+export declare function scopeItemJob(itemId: string, version: number): OneJob<ScopeJob>;
+/**
+ * What a sync of one (item, org) did. `deferred`: the plugin's extension is enabled on the
+ * deployment but not loaded on this replica yet (P3-38); the item job retries later.
+ */
+export type ScopeSyncResult = "installed" | "removed" | "needs_config" | "unchanged" | "deferred";
+/** The worker's concurrency. */
+export declare const PLUGIN_SCOPE_CONCURRENCY = 2;
+export declare class PluginScopeSync {
+    private readonly kdb;
+    private readonly queue;
+    private readonly installer;
+    private readonly registry;
+    /** This replica's reconcile, awaited (bounded) before deferring an install (P3-38). */
+    private readonly lifecycle;
+    private readonly locks;
+    private readonly log;
+    readonly txHost: TransactionHost;
+    constructor(kdb: KernelDb, queue: Queue<ScopeJob>, installer: PluginInstaller, registry: ExtensionRegistry, 
+    /** This replica's reconcile, awaited (bounded) before deferring an install (P3-38). */
+    lifecycle: ExtensionLifecycle, locks: LockService, log: KernelLogger, txHost: TransactionHost);
+    private get db();
+    /** Enqueues the sync of an item in every org, once per scope version. Call after the commit. */
+    enqueueItem(itemId: string): Promise<void>;
+    /**
+     * Enqueues the sync of every live, not builtin item (a worker start): retries what failed or was
+     * never enqueued (a lost `afterCommit` enqueue, a migrated scope). The job ids are the items'
+     * scope versions, so replicas starting together add each one once.
+     */
+    enqueueAll(): Promise<void>;
+    /** Enqueues the sync of every item in an org (a new org). Call after the commit. */
+    enqueueOrg(orgId: string): Promise<void>;
+    /** One job: every org of an item, or every item of an org; `signal` stops between pairs. */
+    process(data: ScopeJob, signal?: AbortSignal): Promise<void>;
+    /** The delayed retry `n` of an item job (deterministic id, P3-38). */
+    private enqueueRetry;
+    /**
+     * Whether the install of the item's `version` may go ahead on this replica (P3-38): its
+     * extension is loaded here, or it is not enabled on the deployment (the install reports
+     * `catalog.not_deployed` as before). Enabled but not loaded here: this replica's reconcile is
+     * awaited (bounded); still not loaded -> false, unless this is the last try.
+     */
+    private extensionReady;
+    /** Converges one (item, org), under the org's sync lock; `signal` ends the wait for it. */
+    syncPair(itemId: string, orgId: string, signal?: AbortSignal, opts?: {
+        finalTry?: boolean;
+    }): Promise<ScopeSyncResult>;
+    /**
+     * {@link syncLocked} plus the pair's open failure (`plugin_scope_sync_failures`, the super
+     * admin's inbox): an attempt that failed records it, any other attempt clears it. An attempt
+     * that throws (outside the recorded steps) leaves the row as it was.
+     */
+    private syncRecorded;
+    private syncLocked;
+    /** Whether a live install of the org depends on this install (spec §5 rule 9). */
+    dependedOn(orgId: string, installId: string): Promise<boolean>;
+    /** The items of the dependency installs this install created. */
+    private createdDependencies;
+    /**
+     * Whether installing the item (with the dependencies the org lacks) needs the org's input: a
+     * connector template, or an extension whose org config is required and not set in the org. The
+     * same rule as the org catalog's `needs_config`, applied to the whole tree.
+     */
+    needsInput(orgId: string, itemId: string, version: string): Promise<boolean>;
+    /**
+     * Whether installing plugin `itemId` (manifest `m`) needs the org's input: a connector template
+     * on in the org (ruling P2-6: templates switched off there, outside their own scope or disabled
+     * make no connector), or a required org config of its extension not set yet.
+     */
+    private manifestNeedsInput;
+    /**
+     * Logs and audits a failed install or uninstall (`plugin.scope_sync_failed`, actor null) and
+     * records it as the pair's open failure (the super admin's inbox).
+     */
+    private failed;
+    private recordFailure;
+}
+/** The consumer of {@link PLUGIN_SCOPE_QUEUE}: {@link PluginScopeSync}. */
+export declare class PluginScopeSyncProcessor extends WorkerHost<Worker<ScopeJob>> {
+    private readonly sync;
+    private readonly log;
+    private readonly stopping;
+    constructor(sync: PluginScopeSync, log: Pick<KernelLogger, "error">);
+    process(job: Job<ScopeJob>): Promise<void>;
+    /** Starts consuming (worker role, after the extensions loaded; built with `autorun: false`). */
+    start(): void;
+    /**
+     * Enqueues every live item's sync once (the worker role, after the start gate opened), in the
+     * background; a failure is only logged (the next worker start, or the next change, enqueues).
+     */
+    resync(): void;
+    /** Stops between pairs and closes the worker. */
+    close(): Promise<void>;
+    failed(job: Job<ScopeJob> | undefined, err: Error): void;
+    error(err: Error): void;
+}