@vellumai/assistant 0.8.0 → 0.8.2

package/src/plugins/external-api.ts

@@ -0,0 +1,104 @@
+/**
+ * External plugin runtime — `globalThis.__vellumPluginRuntime` bridge.
+ *
+ * Workspace-local plugins (`<workspaceDir>/plugins/*`) get dynamic-imported by
+ * {@link loadUserPlugins}. They need to call `registerPlugin()` from their
+ * module body and they typically also want to read secrets, subscribe to
+ * runtime events, etc.
+ *
+ * Importing those symbols by absolute path
+ * (`/abs/path/to/assistant/src/plugins/registry.js`) works when the daemon is
+ * running from source — both the daemon and the dynamic-imported plugin
+ * resolve the same on-disk file and share module identity. It DOES NOT work
+ * when the daemon is a `bun --compile` binary: the daemon's modules are baked
+ * into the executable, and any absolute-path import re-loads a fresh disk
+ * copy. `registerPlugin()` then writes into a disjoint registry instance and
+ * the daemon never sees the plugin.
+ *
+ * The fix is to expose a single, stable handle on `globalThis` that plugins
+ * read at module-load time. The daemon's bundled modules attach themselves
+ * here once at startup; plugins consume the same instance regardless of how
+ * the daemon was built.
+ *
+ * Plugins use the bridge like this:
+ *
+ *   const runtime = (globalThis as { __vellumPluginRuntime?: ... })
+ *     .__vellumPluginRuntime;
+ *   if (!runtime || runtime.version !== 1) throw new Error("...");
+ *   const { registerPlugin, assistantEventHub, getSecureKeyAsync } = runtime;
+ *
+ * Type-only imports (`import type { Plugin } from "..."`) remain free to use
+ * absolute paths or workspace-local copies — the TypeScript compiler erases
+ * them and they have no module-identity effect at runtime.
+ *
+ * See `assistant/docs/plugins.md` for the full authoring contract and
+ * `assistant/examples/plugins/echo/register.ts` for a worked example.
+ */
+
+import { assistantEventHub } from "../runtime/assistant-event-hub.js";
+import { getSecureKeyAsync } from "../security/secure-keys.js";
+import { registerPlugin } from "./registry.js";
+
+/**
+ * The handle plugins read from `globalThis.__vellumPluginRuntime`.
+ *
+ * `version` is the contract version. Plugins should assert against the
+ * version they were authored against and refuse to register if they see a
+ * different one — bumping it is a breaking change to the plugin surface.
+ *
+ * The field set is intentionally small: the most-commonly-needed symbols
+ * across both static (module-load-time) and dynamic (init-time) plugin
+ * lifecycle. Plugins that need richer runtime state can still receive it
+ * through {@link PluginInitContext} during `init()`.
+ */
+export interface VellumPluginRuntime {
+  readonly version: 1;
+  readonly registerPlugin: typeof registerPlugin;
+  readonly assistantEventHub: typeof assistantEventHub;
+  readonly getSecureKeyAsync: typeof getSecureKeyAsync;
+}
+
+/** Stable globalThis key. Don't rename — plugins reference it by string. */
+const RUNTIME_GLOBAL_KEY = "__vellumPluginRuntime" as const;
+
+interface GlobalWithRuntime {
+  [RUNTIME_GLOBAL_KEY]?: VellumPluginRuntime;
+}
+
+/**
+ * Install the plugin runtime bridge on `globalThis`. Idempotent — repeat
+ * calls are no-ops, so it's safe to invoke from tests that also touch the
+ * lifecycle path.
+ *
+ * Must be called BEFORE {@link loadUserPlugins} runs, otherwise plugins that
+ * touch `globalThis.__vellumPluginRuntime` in their module body will throw.
+ */
+export function installPluginRuntime(): void {
+  const g = globalThis as GlobalWithRuntime;
+  if (g[RUNTIME_GLOBAL_KEY]) return;
+  g[RUNTIME_GLOBAL_KEY] = {
+    version: 1,
+    registerPlugin,
+    assistantEventHub,
+    getSecureKeyAsync,
+  };
+}
+
+/**
+ * Read the installed runtime. Returns `undefined` if {@link installPluginRuntime}
+ * hasn't been called yet — plugins should treat this as a fatal error.
+ *
+ * Exposed mainly for tests and for the rare in-process consumer that wants
+ * to read the bridge from the same module graph that installed it.
+ */
+export function getPluginRuntime(): VellumPluginRuntime | undefined {
+  return (globalThis as GlobalWithRuntime)[RUNTIME_GLOBAL_KEY];
+}
+
+/**
+ * Tear down the runtime handle. Test-only — production code never uninstalls
+ * because the runtime lifetime is the daemon lifetime.
+ */
+export function uninstallPluginRuntimeForTests(): void {
+  delete (globalThis as GlobalWithRuntime)[RUNTIME_GLOBAL_KEY];
+}

package/src/plugins/external-plugin-loader.ts

@@ -0,0 +1,367 @@
+/**
+ * External plugin loader — builds a {@link Plugin} from a directory and
+ * registers it with the runtime.
+ *
+ * The convention this loader walks is currently **experimental**: surface
+ * set, manifest fields, and discovery shape may all change before the
+ * framework stabilizes. We keep this module's identifiers harness-neutral
+ * ("external") so the stable call path through the harness —
+ * `loadUserPlugins → loadExternalPlugin → registerPlugin` — does not
+ * need to be renamed when the convention shifts.
+ *
+ *     <pluginDir>/
+ *       package.json              ← manifest.name comes from `name`
+ *                                   (npm scope stripped);
+ *                                   peerDependencies["@vellumai/plugin-api"]
+ *                                   semver range is checked against the
+ *                                   running assistant version and rejects
+ *                                   the plugin if unsatisfied
+ *       hooks/
+ *         <name>.ts               ← default export → plugin.hooks[<name>]
+ *                                   (today the runtime invokes "init" at
+ *                                    bootstrap and "shutdown" at teardown;
+ *                                    other filenames sit in the map for
+ *                                    forward compatibility)
+ *       tools/
+ *         *.ts                    ← each file's default export → plugin.tools[]
+ *       src/                      ← internal helpers, ignored by the loader
+ *
+ * Per-surface, `.js` is preferred over `.ts` (compiled-binary semantics).
+ * Missing surface files are silently omitted (the harness treats absent
+ * fields as "this plugin contributes nothing here"). Surface files
+ * present without a usable default export are a hard failure: the
+ * loader logs with attribution and skips the plugin.
+ *
+ * This function owns the per-plugin isolation contract: it never throws,
+ * times out after `importTimeoutMs`, catches any error from the build or
+ * `registerPlugin` call, and logs an attributed entry. Callers can
+ * `await loadExternalPlugin(...)` in a loop with no per-iteration guard.
+ */
+
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import semver from "semver";
+import { z } from "zod";
+
+import assistantPkg from "../../package.json" with { type: "json" };
+import { getLogger } from "../util/logger.js";
+import { registerPlugin } from "./registry.js";
+import type {
+  Plugin,
+  PluginHookFn,
+  PluginHooks,
+  PluginManifest,
+  PluginToolRegistration,
+} from "./types.js";
+
+const PLUGIN_API_PEER_DEP = "@vellumai/plugin-api";
+
+const log = getLogger("external-plugin-loader");
+
+/** Default upper bound on how long a single plugin load may take. */
+const DEFAULT_IMPORT_TIMEOUT_MS = 10_000;
+
+/**
+ * Zod schema for the subset of `package.json` the external loader reads.
+ *
+ * - `name` is the only required field; everything else is best-effort.
+ * - `peerDependencies["@vellumai/plugin-api"]` is the canonical host-compat
+ *   declaration. If present, the loader checks `semver.satisfies(host, range)`
+ *   against the running assistant version and rejects the plugin on
+ *   mismatch. If absent, the plugin loads without a host-compat claim
+ *   (with a warning).
+ * - Unknown fields pass through (`passthrough`) so the loader does not
+ *   destructively reshape the file when the rest of the npm ecosystem
+ *   writes to it.
+ */
+const PluginPackageJsonSchema = z
+  .object({
+    name: z.string().min(1, "package.json `name` must be a non-empty string"),
+    version: z.string().optional(),
+    peerDependencies: z.record(z.string(), z.string()).optional(),
+  })
+  .passthrough();
+
+type PluginPackageJson = z.infer<typeof PluginPackageJsonSchema>;
+
+export interface LoadExternalPluginOptions {
+  /**
+   * Maximum time to spend building the `Plugin` from disk before bailing.
+   * The build runs to completion in the background if it eventually
+   * resolves, but the loader has already moved on. Defaults to
+   * {@link DEFAULT_IMPORT_TIMEOUT_MS}.
+   */
+  readonly importTimeoutMs?: number;
+}
+
+/**
+ * Strip the npm scope from a package name. `@vellumai/simple-memory` →
+ * `simple-memory`; an unscoped name passes through unchanged.
+ */
+function stripScope(name: string): string {
+  const match = /^@[^/]+\/(.+)$/.exec(name);
+  return match ? match[1]! : name;
+}
+
+/**
+ * Dynamic-import `absolutePath` and return its default export. Throws when
+ * the module has no default export — callers attribute the error.
+ */
+async function importDefault<T>(absolutePath: string): Promise<T> {
+  const url = pathToFileURL(absolutePath).href;
+  const mod = (await import(url)) as { default?: T };
+  if (mod.default === undefined) {
+    throw new Error(
+      `module ${absolutePath} has no default export — external plugins must default-export their interface surfaces`,
+    );
+  }
+  return mod.default;
+}
+
+interface SurfaceFile {
+  /** Basename without `.js`/`.ts` extension. */
+  readonly name: string;
+  /** Absolute path on disk. */
+  readonly path: string;
+}
+
+/**
+ * List every `.js`/`.ts` file directly under `dir`, deduplicating `.js`
+ * over `.ts` when both are present for the same basename. Returns entries
+ * sorted by basename so plugin authors get a deterministic per-plugin
+ * registration sequence; cross-plugin order remains the registry's job.
+ *
+ * Used to walk both `hooks/` and `tools/` — neither surface needs
+ * subdirectory recursion today, so this stays flat on purpose.
+ */
+function listSurfaceDir(dir: string): SurfaceFile[] {
+  if (!existsSync(dir) || !statSync(dir).isDirectory()) return [];
+  const entries = readdirSync(dir);
+  const byBase = new Map<string, string>();
+  for (const entry of entries) {
+    // `.d.ts` declaration files are TypeScript type-only artifacts shipped
+    // alongside compiled `.js`. They have no default-exported runtime
+    // function and would crash `importDefault`, so the walker filters
+    // them out before the `.js`/`.ts` extension check.
+    if (entry.endsWith(".d.ts")) continue;
+    const base =
+      entry.endsWith(".js") || entry.endsWith(".ts")
+        ? entry.slice(0, -3)
+        : null;
+    if (base === null) continue;
+    const existing = byBase.get(base);
+    if (
+      existing === undefined ||
+      (existing.endsWith(".ts") && entry.endsWith(".js"))
+    ) {
+      byBase.set(base, entry);
+    }
+  }
+  return [...byBase.keys()]
+    .sort()
+    .map((name) => ({ name, path: join(dir, byBase.get(name)!) }));
+}
+
+/**
+ * Walk every file under `<pluginDir>/hooks/` and import each as a
+ * lifecycle hook keyed by filename basename. The runtime today invokes
+ * `init` at bootstrap and `shutdown` at teardown; other filenames are
+ * loaded into the map for forward compatibility with future lifecycle
+ * events but stay inert.
+ */
+async function loadHooks(
+  pluginDir: string,
+  pluginName: string,
+): Promise<PluginHooks | undefined> {
+  const files = listSurfaceDir(join(pluginDir, "hooks"));
+  if (files.length === 0) return undefined;
+  const hooks: PluginHooks = {};
+  for (const { name, path } of files) {
+    const fn = await importDefault<PluginHookFn>(path);
+    if (typeof fn !== "function") {
+      throw new Error(
+        `external plugin ${pluginName}: hooks/${name} default export must be a function (got ${typeof fn})`,
+      );
+    }
+    hooks[name] = fn;
+  }
+  return hooks;
+}
+
+/**
+ * Build a `Plugin` object from the directory layout. Internal — the
+ * public entry point ({@link loadExternalPlugin}) wraps this in the
+ * timeout/try-catch/register triple.
+ */
+async function buildPluginFromDir(pluginDir: string): Promise<Plugin> {
+  const pkgPath = join(pluginDir, "package.json");
+  let rawPkg: unknown;
+  try {
+    rawPkg = JSON.parse(await readFile(pkgPath, "utf8"));
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `package.json at ${pluginDir} could not be read or parsed: ${reason}`,
+    );
+  }
+  const parsed = PluginPackageJsonSchema.safeParse(rawPkg);
+  if (!parsed.success) {
+    throw new Error(
+      `package.json at ${pluginDir} failed schema validation: ${parsed.error.message}`,
+    );
+  }
+  const pkg: PluginPackageJson = parsed.data;
+  const name = stripScope(pkg.name);
+  const version = pkg.version && pkg.version.length > 0 ? pkg.version : "0.0.0";
+
+  // Host-compat negotiation: plugins declare their plugin-api version
+  // range via standard `peerDependencies["@vellumai/plugin-api"]`. We
+  // inspect the range and report unparseable / unsatisfied cases via
+  // `log.error` but still load the plugin — the plugin-installation
+  // flow is in flux and a strict gate here would block experimentation
+  // for the customers driving the install UX. Once the install path
+  // settles, the two `log.error` branches below should harden into
+  // throws so a stale plugin can't silently run against a mismatched
+  // host.
+  //
+  // If the peerDep is absent, the plugin loads without a host-compat
+  // claim; we log a warning so the omission is visible at boot.
+  const range = pkg.peerDependencies?.[PLUGIN_API_PEER_DEP];
+  if (range !== undefined) {
+    if (!semver.validRange(range)) {
+      log.error(
+        { pluginDir, plugin: name, peerDep: PLUGIN_API_PEER_DEP, range },
+        `external plugin ${name}: peerDependencies["${PLUGIN_API_PEER_DEP}"] is not a valid semver range — loading anyway`,
+      );
+    } else if (
+      !semver.satisfies(assistantPkg.version, range, {
+        includePrerelease: true,
+      })
+    ) {
+      log.error(
+        {
+          pluginDir,
+          plugin: name,
+          peerDep: PLUGIN_API_PEER_DEP,
+          range,
+          assistantVersion: assistantPkg.version,
+        },
+        `external plugin ${name}: peerDependencies["${PLUGIN_API_PEER_DEP}"] requires "${range}" but assistant is ${assistantPkg.version} — loading anyway`,
+      );
+    }
+  } else {
+    log.warn(
+      { pluginDir, plugin: name, peerDep: PLUGIN_API_PEER_DEP },
+      "external plugin missing plugin-api peerDependency — loading without host-compat claim",
+    );
+  }
+
+  const manifest: PluginManifest = { name, version };
+  const plugin: Plugin = { manifest };
+
+  const hooks = await loadHooks(pluginDir, name);
+  if (hooks !== undefined) plugin.hooks = hooks;
+
+  const tools: PluginToolRegistration[] = [];
+  for (const { path: toolPath } of listSurfaceDir(join(pluginDir, "tools"))) {
+    const tool = await importDefault<PluginToolRegistration>(toolPath);
+    if (
+      tool === null ||
+      typeof tool !== "object" ||
+      typeof (tool as { name?: unknown }).name !== "string"
+    ) {
+      throw new Error(
+        `external plugin ${name}: ${toolPath} default export must be a Tool object with a string "name"`,
+      );
+    }
+    tools.push(tool);
+  }
+  if (tools.length > 0) plugin.tools = tools;
+
+  return plugin;
+}
+
+/**
+ * Build a {@link Plugin} from `pluginDir` with the same timeout +
+ * per-plugin isolation contract as {@link loadExternalPlugin}, but
+ * without registering it. The plugin-source watcher consumes this so it
+ * can decide between first-time registration (init once, then publish) and
+ * hot-reload (replace + skip init) based on what's already in the registry.
+ *
+ * Returns `undefined` on timeout, build failure, or abandoned surface
+ * import. Never throws — failures are logged with directory attribution.
+ */
+export async function buildExternalPlugin(
+  pluginDir: string,
+  opts: LoadExternalPluginOptions = {},
+): Promise<Plugin | undefined> {
+  const timeoutMs = opts.importTimeoutMs ?? DEFAULT_IMPORT_TIMEOUT_MS;
+  let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
+  try {
+    const timeoutSentinel = Symbol("external-plugin-build-timeout");
+    const buildPromise = buildPluginFromDir(pluginDir);
+    const timeoutPromise = new Promise<typeof timeoutSentinel>((resolve) => {
+      timeoutHandle = setTimeout(() => resolve(timeoutSentinel), timeoutMs);
+    });
+    const result = await Promise.race([buildPromise, timeoutPromise]);
+    if (result === timeoutSentinel) {
+      // Abandoned build — surface imports may still be running. Attach a
+      // terminal `.catch` so a late rejection does not surface as an
+      // unhandled-rejection crash. Callers who feed the returned plugin
+      // into `registerPlugin` rely on the closed-registration latch
+      // (registry.ts) to reject any stale late-arriving registration.
+      buildPromise.catch(() => {
+        /* swallow — see comment above */
+      });
+      log.warn(
+        { pluginDir, timeoutMs },
+        `Timed out building external plugin ${pluginDir} after ${timeoutMs}ms — skipping`,
+      );
+      return undefined;
+    }
+    return result;
+  } catch (err) {
+    // Per-plugin isolation: one bad external plugin must not crash the
+    // daemon. Surface the failure with attribution and move on.
+    const message = err instanceof Error ? err.message : String(err);
+    log.error(
+      { err, pluginDir },
+      `Failed to build external plugin ${pluginDir}: ${message}`,
+    );
+    return undefined;
+  } finally {
+    if (timeoutHandle !== undefined) clearTimeout(timeoutHandle);
+  }
+}
+
+/**
+ * Load the external plugin at `pluginDir` and register it. Thin wrapper
+ * over {@link buildExternalPlugin} that calls `registerPlugin` on the
+ * built plugin, preserving the existing `loadUserPlugins` call shape.
+ */
+export async function loadExternalPlugin(
+  pluginDir: string,
+  opts: LoadExternalPluginOptions = {},
+): Promise<void> {
+  const plugin = await buildExternalPlugin(pluginDir, opts);
+  if (plugin === undefined) {
+    // buildExternalPlugin already logged the failure with attribution.
+    return;
+  }
+  try {
+    registerPlugin(plugin);
+    log.info(
+      { pluginDir, name: plugin.manifest.name },
+      "loaded external plugin",
+    );
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    log.error(
+      { err, pluginDir, plugin: plugin.manifest.name },
+      `Failed to register external plugin ${pluginDir}: ${message}`,
+    );
+  }
+}

package/src/plugins/feature-gate.ts

@@ -0,0 +1,22 @@
+/**
+ * External-plugin feature gate.
+ *
+ * Single source of truth for whether the experimental external-plugin
+ * surface is enabled. The flag gates both the `assistant plugins` CLI
+ * command tree and (in future) the declarative external-plugin loader
+ * pathway in the daemon.
+ *
+ * The flag key uses the simple kebab-case format and is declared in
+ * `meta/feature-flags/feature-flag-registry.json`.
+ */
+
+import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
+import type { AssistantConfig } from "../config/schema.js";
+
+/** Gate key for the external-plugin surface. */
+const EXTERNAL_PLUGINS_FLAG_KEY = "external-plugins" as const;
+
+/** Whether the external-plugin surface is enabled. */
+export function isExternalPluginsEnabled(config: AssistantConfig): boolean {
+  return isAssistantFeatureFlagEnabled(EXTERNAL_PLUGINS_FLAG_KEY, config);
+}

package/src/plugins/pipeline.ts

@@ -26,7 +26,9 @@
 
 import type { Logger } from "pino";
 
+import type { HookName } from "../plugin-api/constants.js";
 import { getLogger } from "../util/logger.js";
+import { getHooksFor } from "./registry.js";
 import {
   type Middleware,
   type PipelineName,
@@ -314,3 +316,38 @@ export async function runPipeline<A, R>(
     logger.info(record, "plugin.pipeline");
   }
 }
+
+// ─── Hook runner ────────────────────────────────────────────────────────────
+
+/**
+ * Execute a hook chain: walk every registered plugin's hook for `name` in
+ * registration order, threading `initialCtx` through each. Hooks may either
+ * mutate the context in place (returning `void`) or return a new context
+ * that replaces the threaded value for subsequent hooks. The final context
+ * after the chain settles is returned.
+ *
+ * `runHook` is the hook-side counterpart to {@link runPipeline}:
+ * `runPipeline` composes middleware around a terminal handler for stateful
+ * request/response pipelines (memory retrieval, history repair, etc.);
+ * `runHook` walks ordered hook functions for declarative chain-style
+ * context transformations (`user-prompt-submit` today).
+ *
+ * @param name        The hook identifier — pick one from {@link HOOKS}.
+ * @param initialCtx  Context the first hook receives.
+ * @returns The final context after the chain settles. Same reference as
+ *          `initialCtx` when no plugin registers `name`, and when every
+ *          chained hook returns `void` (mutation-in-place style).
+ */
+export async function runHook<TCtx>(
+  name: HookName,
+  initialCtx: TCtx,
+): Promise<TCtx> {
+  let active = initialCtx;
+  for (const hook of getHooksFor<TCtx>(name)) {
+    const result = await hook(active);
+    if (result !== undefined) {
+      active = result;
+    }
+  }
+  return active;
+}