@nyte-ai/host 0.0.0-stage → 0.0.19

package/src/open.ts

@@ -0,0 +1,449 @@
+import { watch } from "node:fs";
+import { mkdir, readFile } from "node:fs/promises";
+import { homedir, hostname } from "node:os";
+import { dirname, join } from "node:path";
+import process from "node:process";
+import type { Models, Provider } from "@nyte-ai/ai";
+import { defaultProviderAuthContext } from "@nyte-ai/ai/auth/context";
+import { FileModelsStore, nyteProviders } from "@nyte-ai/ai";
+import { createModels } from "@nyte-ai/ai/models";
+import { isJson } from "@nyte-ai/schema";
+import { serial } from "@nyte-ai/schema/serial";
+import type {
+  HostDescription,
+  HostView,
+  Json,
+  PluginReplacementReport,
+  ValidationIssue,
+  WorkspaceTarget,
+} from "@nyte-ai/schema";
+import { dev, processEnvironment } from "@nyte-ai/core";
+import { composeHost } from "@nyte-ai/core/server";
+import { readModelSelection } from "@nyte-ai/core/store";
+import type { Host, ListenOptions, Listener, OpenOptions, Plugin, Store } from "@nyte-ai/core";
+import approvals from "@nyte-ai/plugin/core/approvals";
+import { acpAgents } from "@nyte-ai/plugin/core/acp";
+import bashDescription from "@nyte-ai/plugin/core/bash-description";
+import browserTools from "@nyte-ai/plugin/core/browser-tools";
+import canvas from "@nyte-ai/plugin/core/canvas";
+import { claudeCodeSdk } from "@nyte-ai/plugin/core/claude-code-sdk";
+import codemode from "@nyte-ai/plugin/core/codemode";
+import openaiAstraContext from "@nyte-ai/plugin/core/openai-astra-context";
+import { openaiCompaction } from "@nyte-ai/plugin/core/openai-compaction";
+import question from "@nyte-ai/plugin/core/question";
+import rename from "@nyte-ai/plugin/core/rename";
+import telemetry from "@nyte-ai/plugin/core/telemetry";
+import webSearch from "@nyte-ai/plugin/core/web-search";
+import manifest from "../package.json" with { type: "json" };
+import { credentialStore } from "./credentials.ts";
+import { identitySigner } from "./identity-key.ts";
+import { readErrorCode } from "./local/tree.ts";
+import { localGitHubAccount } from "./local/github.ts";
+import { localProvider } from "./local/provider.ts";
+import { importLegacyTrust } from "./local/legacy-trust.ts";
+import { migrateLocalWorkspaceIdentities } from "./local/workspace-migration.ts";
+import { listen } from "./listen.ts";
+import { nodeImages } from "./images/node.ts";
+import { createMcpHub } from "./mcp/hub.ts";
+import { loadPluginDirectory, pluginLoader } from "./plugins/loader.ts";
+import { defaultSecrets } from "./secrets.ts";
+import type { SecretsSource } from "./secrets.ts";
+import { sqliteStore, storePath } from "./sqlite.ts";
+
+const RELOAD_DEBOUNCE_MS = 250;
+
+const STORE_POLL_MS = 500;
+
+export type OpenHostOptions = Omit<OpenOptions, "store" | "workspace" | "models" | "plugins"> & {
+  readonly store?: Store;
+  readonly workspace?: WorkspaceTarget;
+  readonly models?: Models;
+  readonly plugins?: readonly Plugin<"core">[];
+  readonly home?: string;
+  readonly shell?: () => string | null;
+  readonly label?: string;
+  readonly onRevoked?: () => void;
+};
+
+export async function openHost(options: OpenHostOptions): Promise<Host> {
+  const home = options.home ?? homedir();
+  const workspace = options.workspace ?? { kind: "local", path: process.cwd() };
+
+  if (options.store !== undefined)
+    return composeOpen(options, { home, workspace, store: options.store, owned: null });
+  const owned = await defaultStore(home);
+
+  try {
+    return await composeOpen(options, { home, workspace, store: owned, owned });
+  } catch (cause) {
+    await owned.close().catch(() => undefined);
+    throw cause;
+  }
+}
+
+async function defaultStore(home: string): Promise<Store> {
+  const path = storePath(home);
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+
+  return sqliteStore({ path });
+}
+
+async function composeOpen(
+  options: OpenHostOptions,
+  input: {
+    readonly home: string;
+    readonly workspace: WorkspaceTarget;
+    readonly store: Store;
+    readonly owned: Store | null;
+  },
+): Promise<Host> {
+  const { home, workspace, store, owned } = input;
+  const secrets = options.secrets ?? defaultSecrets(home);
+  const description = hostDescription();
+  const given = options.plugins ?? [];
+  const usesLocalProvider = !given.some((plugin) => plugin.environment?.kind === "local");
+
+  const claudeCode = claudeCodeSdk();
+  const acp = acpAgents();
+
+  const [models, files, settings] = await Promise.all([
+    options.models ??
+      nyteModels({ store, secrets, home, providers: [claudeCode.provider, acp.provider] }),
+    filePlugins(home),
+    settingsSeed(home),
+    usesLocalProvider
+      ? store.open().then(async (connection) => {
+          await migrateLocalWorkspaceIdentities(connection);
+          await importLegacyTrust(connection, home);
+        })
+      : undefined,
+  ]);
+
+  const builtins = [
+    approvals,
+    question,
+    rename,
+    bashDescription,
+    webSearch,
+    codemode,
+    openaiCompaction(models),
+    openaiAstraContext,
+    telemetry,
+    canvas,
+    claudeCode.plugin,
+    acp.plugin,
+  ];
+
+  const plugins = [
+    ...(usesLocalProvider
+      ? [{ id: "local", environment: localProvider({ home, shell: options.shell }) }]
+      : []),
+    ...builtins.filter((builtin) => !given.some((plugin) => plugin.id === builtin.id)),
+    ...given,
+  ];
+
+  let projectDirs: () => Promise<readonly string[]> = async () => [];
+
+  const host = await composeHost({
+    ...options,
+    store,
+    workspace,
+    loadPlugins: async (paths) => pluginLoader([...files.dirs, ...(await projectDirs())])(paths),
+    models,
+    mcp: options.mcp ?? createMcpHub({ home }),
+    github: options.github ?? (usesLocalProvider ? localGitHubAccount({ home }) : undefined),
+    images: options.images ?? nodeImages(),
+    plugins: [...plugins, ...files.plugins],
+    pluginSources: files.sources,
+    secrets,
+    home,
+    description,
+    ...(settings.seed === null ? {} : { seed: settings.seed }),
+    ...(owned === null ? {} : { poll: STORE_POLL_MS }),
+    member: {
+      label: options.label ?? "nyte",
+      pid: process.pid,
+      capabilities: given.some((plugin) => plugin.id === browserTools.id) ? [browserTools.id] : [],
+    },
+    revoked: options.onRevoked,
+  });
+
+  projectDirs = async () =>
+    (await host.workspaces.list()).flatMap((entry) =>
+      entry.trusted === true && "path" in entry.target
+        ? [join(entry.target.path, ".nyte", "plugins")]
+        : [],
+    );
+
+  for (const row of files.failed) host.server.report(row);
+
+  if (settings.issue !== null)
+    host.server.report({
+      id: "settings",
+      source: "user",
+      path: settingsPath(home),
+      issues: [settings.issue],
+    });
+
+  dev("open", "host", {
+    home,
+    seedIds: Object.keys(settings.seed?.settings ?? {}).length,
+    pluginFiles: files.plugins.length + files.failed.length,
+    failedFiles: files.failed.length,
+  });
+
+  const reported = new Map(
+    host.server
+      .pluginRows()
+      .filter((row) => row.path !== null && row.id === row.path)
+      .map((row) => [row.id, row]),
+  );
+
+  const reload = async (): Promise<void> => {
+    const next = await filePlugins(home);
+
+    for (const [plugin, source] of next.sources) files.sources.set(plugin, source);
+    const report = await host.plugins.replace([...plugins, ...next.plugins]);
+    const failed = [...next.failed, ...refused(next, report)];
+
+    for (const [id, row] of reported)
+      if (!failed.some((row) => row.id === id)) host.server.report({ ...row, issues: [] });
+
+    for (const row of failed) host.server.report(row);
+    reported.clear();
+
+    for (const row of failed) reported.set(row.id, row);
+
+    for (const plugin of files.sources.keys())
+      if (!host.server.loaded(plugin)) files.sources.delete(plugin);
+    dev("open", "plugins reloaded", {
+      generation: report.generation,
+      pluginFiles: next.plugins.length,
+      failedFiles: failed.length,
+    });
+  };
+
+  const unwatch = watchPlugins(files.dirs, reload);
+  const listeners = new Set<Listener>();
+  let identity: ReturnType<typeof identitySigner> | null = null;
+
+  const attach = async (listenOptions: ListenOptions): Promise<Listener> => {
+    const listener = await listen(host, {
+      describe: () => description,
+      ...listenOptions,
+      identity: listenOptions.identity ?? (await (identity ??= identitySigner(host.hostId, home))),
+    });
+
+    listeners.add(listener);
+    dev("open", "listen", { address: listener.address });
+
+    return listener;
+  };
+
+  let closing: Promise<void> | null = null;
+
+  const close = (): Promise<void> => {
+    closing ??= (async () => {
+      await unwatch();
+
+      for (const listener of listeners) await listener.close();
+      await host.close();
+      await owned?.close();
+    })();
+
+    return closing;
+  };
+
+  return { ...host, listen: attach, close, [Symbol.asyncDispose]: close };
+}
+
+async function nyteModels(input: {
+  readonly store: Store;
+  readonly secrets: SecretsSource;
+  readonly home: string;
+  readonly providers: readonly Provider[];
+}): Promise<Models> {
+  const credentials = credentialStore(input.store, input.secrets);
+  const auth = defaultProviderAuthContext();
+
+  const models = createModels({
+    credentials,
+    modelsStore: new FileModelsStore(join(input.home, ".nyte", "models-store.json")),
+    authContext: {
+      ...auth,
+      env: async (name) => {
+        await processEnvironment();
+
+        return auth.env(name);
+      },
+    },
+    catalog: { url: process.env.NYTE_MODELS_URL },
+  });
+
+  for (const provider of [...nyteProviders(), ...input.providers]) models.setProvider(provider);
+
+  const [stored, selected] = await Promise.all([
+    credentials.list(),
+    input.store.open().then(readModelSelection),
+  ]);
+
+  const restored = await models.refresh({
+    providers: [
+      ...new Set([
+        ...stored.map((row) => row.providerId),
+        ...input.providers.map((provider) => provider.id),
+        ...(selected === null ? [] : [selected.provider]),
+      ]),
+    ],
+    allowNetwork: false,
+  });
+
+  dev("open", "models restored", {
+    providers: stored.length,
+    failed: [...restored.errors.keys()],
+  });
+
+  return models;
+}
+
+async function filePlugins(home: string): Promise<{
+  readonly plugins: readonly Plugin<"core">[];
+  readonly sources: Map<
+    Plugin<"core">,
+    { readonly source: "user" | "project"; readonly path: string }
+  >;
+  readonly dirs: readonly string[];
+  readonly failed: readonly HostView["plugins"][number][];
+}> {
+  const scopes = [{ source: "user" as const, dir: join(home, ".nyte", "plugins") }];
+
+  const plugins: Plugin<"core">[] = [];
+  const sources = new Map<Plugin<"core">, { source: "user" | "project"; path: string }>();
+  const failed: HostView["plugins"][number][] = [];
+
+  for (const { source, dir } of scopes) {
+    const found = await loadPluginDirectory(dir);
+
+    for (const { plugin, path } of found.loaded) {
+      plugins.push(plugin);
+      sources.set(plugin, { source, path });
+    }
+
+    for (const { path, issue } of found.failed)
+      failed.push({ id: path, source, path, issues: [issue] });
+  }
+
+  return { plugins, sources, dirs: scopes.map((scope) => scope.dir), failed };
+}
+
+function refused(
+  files: Awaited<ReturnType<typeof filePlugins>>,
+  report: PluginReplacementReport,
+): readonly HostView["plugins"][number][] {
+  return report.plugins.flatMap((outcome) => {
+    const plugin = files.plugins.find((candidate) => candidate.id === outcome.id);
+    const file = plugin === undefined ? undefined : files.sources.get(plugin);
+
+    return outcome.kind === "activated" || file === undefined
+      ? []
+      : [{ id: file.path, source: file.source, path: file.path, issues: outcome.issues }];
+  });
+}
+
+function watchPlugins(dirs: readonly string[], reload: () => Promise<void>): () => Promise<void> {
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  const reloads = serial();
+
+  const run = (): void => {
+    timer = null;
+    void reloads.run(reload).catch((error: unknown) =>
+      dev("open", "plugin reload failed", {
+        message: error instanceof Error ? error.message : String(error),
+      }),
+    );
+  };
+
+  const changed = (): void => {
+    if (timer !== null) clearTimeout(timer);
+    timer = setTimeout(run, RELOAD_DEBOUNCE_MS);
+    timer.unref();
+  };
+
+  const watchers = dirs.flatMap((dir) => {
+    try {
+      const watcher = watch(dir, { recursive: true, persistent: false }, changed);
+
+      watcher.on("error", (error) => {
+        dev("open", "plugin watch failed", { dir, message: error.message });
+        watcher.close();
+      });
+      dev("open", "plugins watched", { dir });
+
+      return [watcher];
+    } catch (error) {
+      dev("open", "plugins unwatched", { dir, code: readErrorCode(error) });
+
+      return [];
+    }
+  });
+
+  return async () => {
+    for (const watcher of watchers) watcher.close();
+
+    if (timer !== null) clearTimeout(timer);
+    await reloads.settled();
+  };
+}
+
+async function settingsSeed(home: string): Promise<{
+  readonly seed: {
+    readonly settings: Readonly<Record<string, Json>>;
+    readonly path: string;
+  } | null;
+  readonly issue: ValidationIssue | null;
+}> {
+  const path = settingsPath(home);
+
+  const text = await readFile(path, "utf8").catch((error: unknown) =>
+    readErrorCode(error) === "ENOENT"
+      ? null
+      : error instanceof Error
+        ? error
+        : new Error(String(error)),
+  );
+
+  if (text === null) return { seed: null, issue: null };
+
+  if (text instanceof Error)
+    return {
+      seed: null,
+      issue: { severity: "warning", path: [], message: `${path} cannot be read: ${text.message}` },
+    };
+  let parsed: unknown;
+
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    parsed = undefined;
+  }
+
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
+    return {
+      seed: null,
+      issue: { severity: "warning", path: [], message: `${path} is not a JSON object.` },
+    };
+
+  const settings: Record<string, Json> = {};
+
+  for (const [id, value] of Object.entries(parsed)) if (isJson(value)) settings[id] = value;
+
+  return { seed: { settings, path }, issue: null };
+}
+
+function hostDescription(): HostDescription {
+  return { name: hostname(), platform: process.platform, version: manifest.version };
+}
+
+function settingsPath(home: string): string {
+  return join(home, ".nyte", "settings.json");
+}

package/src/plugins/loader.ts

@@ -0,0 +1,186 @@
+import { readdir, realpath, stat } from "node:fs/promises";
+import { isAbsolute, join, relative } from "node:path";
+import { pathToFileURL } from "node:url";
+import { SdkError } from "@nyte-ai/schema";
+import type { ValidationIssue } from "@nyte-ai/schema";
+import { dev } from "@nyte-ai/core";
+import type { Plugin } from "@nyte-ai/core";
+import type { ComposeOptions } from "@nyte-ai/core/server";
+import { readErrorCode } from "../local/tree.ts";
+
+const EXTENSIONS = [".ts", ".mts", ".js", ".mjs"];
+
+/** Returns the core plugins in a plugin directory (`~/.nyte/plugins`, a workspace's `.nyte/plugins`), by file name: each `.ts`, `.mts`, `.js` or `.mjs` file directly in it and each subdirectory's `index` of those, loaded with a dynamic `import()` of its file URL. That `import()` is one of the two dynamic loads the rewrite allows (§41a), host-only, because a plugin file is code the owner placed there and its path is known only at run time; Node strips `.ts` types natively (erasable syntax only). The URL carries the file's modification time and size as a query, so a replaced file loads fresh instead of from the module cache. The default export must be a core plugin (an object with a string `id`, and, where present, `contribute` and `hooks` objects, an `activate` function and an `environment` with a string `kind` and `inspect` and `attach` functions); every file that does not load or export one is reported, together, as `SdkError("invalid")` with an issue per file naming it. A missing directory has none. */
+export async function loadPluginFiles(
+  dir: string,
+): Promise<readonly { readonly plugin: Plugin<"core">; readonly path: string }[]> {
+  return loadAll(await pluginFilesIn(dir));
+}
+
+/** Returns a plugin directory's plugins as `loadPluginFiles` finds and imports them, but file by file: every file that loads, with its path, and every file that does not, with its path and one `error` issue (its `path` the file, `source.file` the file, the message `loadPluginFiles` would have put in its `invalid`), never rejecting for a bad file; a missing directory has neither. What `openHost` loads `~/.nyte/plugins` and a local workspace's `.nyte/plugins` through, so one bad file is reported on `HostView.plugins` instead of failing the open (§41j); `loadPluginFiles` becomes this, throwing its one `invalid` when anything failed. */
+export async function loadPluginDirectory(dir: string): Promise<{
+  readonly loaded: readonly { readonly plugin: Plugin<"core">; readonly path: string }[];
+  readonly failed: readonly { readonly path: string; readonly issue: ValidationIssue }[];
+}> {
+  return importEach(await pluginFilesIn(dir));
+}
+
+/** Returns the host's `loadPlugins` (`composeHost({ loadPlugins })`), which `plugins.replace` and `plugins.check` load their files through (§40b, §47m): each path, resolved to its real path, must lie inside one of `dirs` (else `SdkError("denied")`) and is imported as `loadPluginFiles` imports a file; the plugins come back in path order. */
+export function pluginLoader(dirs: readonly string[]): NonNullable<ComposeOptions["loadPlugins"]> {
+  return async (paths) => {
+    const roots = (await Promise.all(dirs.map((dir) => realpath(dir).catch(() => null)))).filter(
+      (root) => root !== null,
+    );
+
+    const files = await Promise.all(
+      paths.map(async (path) => {
+        const real = await realpath(path).catch(() => null);
+
+        if (real === null) throw new SdkError("invalid", `No plugin file is at ${path}.`);
+
+        if (!roots.some((root) => inside(root, real)))
+          throw new SdkError("denied", `${path} is not in a plugin directory this host loads.`);
+
+        return real;
+      }),
+    );
+
+    return (await loadAll(files)).map((loaded) => loaded.plugin);
+  };
+}
+
+async function loadAll(
+  files: readonly string[],
+): Promise<readonly { readonly plugin: Plugin<"core">; readonly path: string }[]> {
+  const { loaded, failed } = await importEach(files);
+  const issues = failed.map((entry) => entry.issue);
+
+  if (issues.length > 0)
+    throw new SdkError(
+      "invalid",
+      `${issues.length === 1 ? "A plugin file" : `${issues.length} plugin files`} did not load: ${failed.map((entry) => `${entry.path}: ${entry.issue.message}`).join("; ")}`,
+      { issues },
+    );
+
+  return loaded;
+}
+
+async function importEach(files: readonly string[]): Promise<{
+  readonly loaded: readonly { readonly plugin: Plugin<"core">; readonly path: string }[];
+  readonly failed: readonly { readonly path: string; readonly issue: ValidationIssue }[];
+}> {
+  const loaded: { readonly plugin: Plugin<"core">; readonly path: string }[] = [];
+  const failed: { readonly path: string; readonly issue: ValidationIssue }[] = [];
+
+  for (const path of files) {
+    const outcome = await importPlugin(path);
+    dev("plugins", "file", { path, outcome: "plugin" in outcome ? "loaded" : "failed" });
+
+    if ("plugin" in outcome) loaded.push({ plugin: outcome.plugin, path });
+    else
+      failed.push({
+        path,
+        issue: {
+          severity: "error",
+          path: [path],
+          message: outcome.reason,
+          source: { plugin: path, file: path },
+        },
+      });
+  }
+
+  return { loaded, failed };
+}
+
+async function pluginFilesIn(dir: string): Promise<readonly string[]> {
+  const entries = await readdir(dir, { withFileTypes: true }).catch((error: unknown) => {
+    if (readErrorCode(error) === "ENOENT" || readErrorCode(error) === "ENOTDIR") return [];
+
+    throw error;
+  });
+
+  return (
+    await Promise.all(
+      entries
+        .sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0))
+        .map(async (entry) => {
+          const path = join(dir, entry.name);
+
+          if (entry.isFile())
+            return EXTENSIONS.some((extension) => entry.name.endsWith(extension)) ? [path] : [];
+
+          if (!entry.isDirectory()) return [];
+
+          for (const extension of EXTENSIONS) {
+            const index = join(path, `index${extension}`);
+
+            if ((await stat(index).catch(() => null))?.isFile() === true) return [index];
+          }
+
+          return [];
+        }),
+    )
+  ).flat();
+}
+
+async function importPlugin(
+  path: string,
+): Promise<{ readonly plugin: Plugin<"core"> } | { readonly reason: string }> {
+  try {
+    const stats = await stat(path);
+    const url = pathToFileURL(path);
+    url.searchParams.set("nyte", `${stats.mtimeMs}-${stats.size}`);
+    const imported: unknown = await import(url.href);
+
+    const exported =
+      typeof imported === "object" && imported !== null && "default" in imported
+        ? imported.default
+        : undefined;
+
+    return fitsCorePlugin(exported)
+      ? { plugin: exported }
+      : { reason: "Its default export is not a core plugin (an object with a string id)." };
+  } catch (error) {
+    return { reason: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+function fitsCorePlugin(value: unknown): value is Plugin<"core"> {
+  if (
+    typeof value !== "object" ||
+    value === null ||
+    !("id" in value) ||
+    typeof value.id !== "string"
+  )
+    return false;
+
+  const object = (field: unknown): boolean =>
+    field === undefined || (typeof field === "object" && field !== null);
+
+  if ("contribute" in value && !object(value.contribute)) return false;
+
+  if ("hooks" in value && !object(value.hooks)) return false;
+
+  if ("activate" in value && value.activate !== undefined && typeof value.activate !== "function")
+    return false;
+
+  if (!("environment" in value) || value.environment === undefined) return true;
+  const environment = value.environment;
+
+  return (
+    typeof environment === "object" &&
+    environment !== null &&
+    "kind" in environment &&
+    typeof environment.kind === "string" &&
+    "inspect" in environment &&
+    typeof environment.inspect === "function" &&
+    "attach" in environment &&
+    typeof environment.attach === "function"
+  );
+}
+
+function inside(root: string, path: string): boolean {
+  const within = relative(root, path);
+
+  return within !== "" && !within.startsWith("..") && !isAbsolute(within);
+}