@plasm_lang/vercel-agent 0.3.63

package/src/engine/napi-binding.ts

@@ -0,0 +1,265 @@
+/**
+ * Plasm engine binding — in-process NAPI (`@plasm_lang/engine`) with stub fallback.
+ */
+
+import { createRequire } from "node:module";
+import type { LoadedCatalog } from "../catalog/loader.js";
+import { toNapiHostTransport } from "./host-transport-bridge.js";
+
+const require = createRequire(import.meta.url);
+
+export interface TeachingExposureResult {
+  tsv: string;
+  deltaRefs: string[];
+  executeSessionRef?: string;
+}
+
+export interface DryRunResult {
+  planCommitRef: string;
+  summary: string;
+  compJson?: unknown;
+}
+
+export interface ResolvedPlanPayload {
+  /** application/vnd.plasm.resolved-plan+json */
+  planJson: unknown;
+  catalogCgsHash: string;
+}
+
+/** Host-injected outbound HTTP transport (Connect bearer, etc.). */
+export type HostTransportRequest = {
+  method: string;
+  url: string;
+  headers?: Record<string, string>;
+  body?: string;
+  entryId?: string;
+};
+
+export type HostTransportResponse = {
+  status: number;
+  body: string;
+  nextUrl?: string;
+};
+
+export type HostTransportFn = (request: HostTransportRequest) => Promise<HostTransportResponse>;
+
+export interface PlasmEngine {
+  loadCatalog(catalog: LoadedCatalog): Promise<void>;
+  synthesizeTeaching(
+    intent: string,
+    seeds: Array<{ api: string; entity: string }>,
+  ): Promise<TeachingExposureResult>;
+  dryRun(program: string, executeSessionRef?: string): Promise<DryRunResult>;
+  discover(intent: string): Promise<{ markdown: string }>;
+  runPlan(planCommitRef: string): Promise<{ ok: boolean; message: string; rowsJson?: string; metaJson?: string }>;
+  runPlanLive?(
+    planCommitRef: string,
+    transport: HostTransportFn,
+  ): Promise<{ ok: boolean; message: string; rowsJson?: string; metaJson?: string }>;
+  run(resolved: ResolvedPlanPayload, transport: HostTransportFn): Promise<unknown>;
+  introspectCatalog(entryId: string): Promise<string>;
+}
+
+type NativePlasmEngine = {
+  loadCatalog(catalogDir: string): {
+    entryId: string;
+    catalogCgsHash: string;
+  };
+  exposeSeeds(
+    intent: string,
+    seeds: Array<{ api: string; entity: string }>,
+  ): {
+    tsv: string;
+    deltaRefs: string[];
+  };
+  dryRun(program: string): {
+    planCommitRef: string;
+    summary: string;
+    compJson: string;
+  };
+  discover(intent: string): {
+    markdown: string;
+  };
+  runPlan(planCommitRef: string): {
+    ok: boolean;
+    message: string;
+    rowsJson?: string;
+    metaJson?: string;
+  };
+  runPlanLive(
+    planCommitRef: string,
+    transport: (request: HostTransportRequest) => Promise<HostTransportResponse>,
+  ): Promise<{
+    ok: boolean;
+    message: string;
+    rowsJson?: string;
+    metaJson?: string;
+  }>;
+  introspectCatalog(entryId: string): string;
+};
+
+type NativeConstructor = new () => NativePlasmEngine;
+
+function loadNativeConstructor(): NativeConstructor | null {
+  try {
+    const mod = require("@plasm_lang/engine") as { PlasmEngine: NativeConstructor };
+    return mod.PlasmEngine ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/** NAPI-backed engine (`@plasm_lang/engine`). Build with `npm run build` in `packages/plasm-engine`. */
+export class NapiPlasmEngine implements PlasmEngine {
+  private readonly native: NativePlasmEngine;
+  private loaded: LoadedCatalog[] = [];
+
+  constructor(NativeEngine: NativeConstructor = loadNativeConstructor()!) {
+    if (!NativeEngine) {
+      throw new Error(
+        "@plasm_lang/engine native binding not found — run `npm run build` in packages/plasm-engine",
+      );
+    }
+    this.native = new NativeEngine();
+  }
+
+  async loadCatalog(catalog: LoadedCatalog): Promise<void> {
+    const info = this.native.loadCatalog(catalog.rootDir);
+    this.loaded.push({
+      ...catalog,
+      manifest: {
+        ...catalog.manifest,
+        entryId: info.entryId,
+        cgsHash: info.catalogCgsHash,
+      },
+    });
+  }
+
+  async synthesizeTeaching(
+    intent: string,
+    seeds: Array<{ api: string; entity: string }>,
+  ): Promise<TeachingExposureResult> {
+    void this.loaded;
+    const result = this.native.exposeSeeds(intent, seeds);
+    return {
+      tsv: result.tsv,
+      deltaRefs: result.deltaRefs,
+    };
+  }
+
+  async dryRun(program: string, executeSessionRef?: string): Promise<DryRunResult> {
+    void executeSessionRef;
+    const result = this.native.dryRun(program);
+    let compJson: unknown;
+    try {
+      compJson = JSON.parse(result.compJson) as unknown;
+    } catch {
+      compJson = result.compJson;
+    }
+    return {
+      planCommitRef: result.planCommitRef,
+      summary: result.summary,
+      compJson,
+    };
+  }
+
+  async discover(intent: string): Promise<{ markdown: string }> {
+    return this.native.discover(intent);
+  }
+
+  async runPlan(planCommitRef: string): Promise<{ ok: boolean; message: string; rowsJson?: string }> {
+    return this.native.runPlan(planCommitRef);
+  }
+
+  async runPlanLive(
+    planCommitRef: string,
+    transport: HostTransportFn,
+  ): Promise<{ ok: boolean; message: string; rowsJson?: string; metaJson?: string }> {
+    return this.native.runPlanLive(planCommitRef, toNapiHostTransport(transport));
+  }
+
+  async run(_resolved: ResolvedPlanPayload, transport: HostTransportFn): Promise<unknown> {
+    void _resolved;
+    throw new Error(
+      "NapiPlasmEngine.run(resolvedPlan): pass planCommitRef via runPlanLive(pcN, transport) instead",
+    );
+  }
+
+  async introspectCatalog(entryId: string): Promise<string> {
+    return this.native.introspectCatalog(entryId);
+  }
+}
+
+/** Stub until `@plasm_lang/engine` is built or on unsupported platforms. */
+export class StubPlasmEngine implements PlasmEngine {
+  private loaded: LoadedCatalog[] = [];
+
+  async loadCatalog(catalog: LoadedCatalog): Promise<void> {
+    this.loaded.push(catalog);
+  }
+
+  async synthesizeTeaching(
+    intent: string,
+    seeds: Array<{ api: string; entity: string }>,
+  ): Promise<TeachingExposureResult> {
+    void intent;
+    return {
+      tsv: `# teaching TSV placeholder\n# seeds: ${JSON.stringify(seeds)}`,
+      deltaRefs: [],
+    };
+  }
+
+  async dryRun(program: string, executeSessionRef?: string): Promise<DryRunResult> {
+    void executeSessionRef;
+    return {
+      planCommitRef: "pc0",
+      summary: `[stub dry-run] ${program.slice(0, 80)}`,
+    };
+  }
+
+  async discover(intent: string): Promise<{ markdown: string }> {
+    return {
+      markdown: `# Discovery stub\n\nIntent: ${intent}\n\nLoad @plasm_lang/engine native binding for real discovery TSV.`,
+    };
+  }
+
+  async runPlan(planCommitRef: string): Promise<{ ok: boolean; message: string; rowsJson?: string }> {
+    return {
+      ok: false,
+      message: `[stub] plan ${planCommitRef} — wire NAPI binding`,
+    };
+  }
+
+  async runPlanLive(
+    planCommitRef: string,
+    _transport: HostTransportFn,
+  ): Promise<{ ok: boolean; message: string; rowsJson?: string }> {
+    return {
+      ok: false,
+      message: `[stub] live run ${planCommitRef} — build @plasm_lang/engine`,
+    };
+  }
+
+  async run(_resolved: ResolvedPlanPayload, _transport: HostTransportFn): Promise<unknown> {
+    throw new Error("StubPlasmEngine.run: wire NAPI binding or plasm-server sidecar");
+  }
+
+  async introspectCatalog(_entryId: string): Promise<string> {
+    throw new Error("StubPlasmEngine.introspectCatalog: build @plasm_lang/engine");
+  }
+}
+
+export function createEngine(mode: "napi" | "stub" = "napi"): PlasmEngine {
+  if (mode === "stub") {
+    return new StubPlasmEngine();
+  }
+  const Native = loadNativeConstructor();
+  if (Native) {
+    return new NapiPlasmEngine(Native);
+  }
+  return new StubPlasmEngine();
+}
+
+export function isNativeEngineAvailable(): boolean {
+  return loadNativeConstructor() !== null;
+}

package/src/evals/define-eval.ts

@@ -0,0 +1,56 @@
+export const PLASM_EVAL_KIND = "eval" as const;
+
+export type EvalToolName =
+  | "discover_capabilities"
+  | "plasm_context"
+  | "plasm"
+  | "plasm_run"
+  | "read_skill"
+  | "delegate_subagent";
+
+export interface EvalAssert {
+  /** Tools that must appear in at least one step (live agent loop). */
+  toolsUsedAny?: EvalToolName[];
+  /** Final assistant text must match. */
+  responseIncludes?: string | RegExp;
+  /** Minimum tool-calling steps observed. */
+  minSteps?: number;
+}
+
+export interface EvalDefinition {
+  readonly __plasmSlotKind: typeof PLASM_EVAL_KIND;
+  name: string;
+  /** Natural-language goal sent to the live agent loop. */
+  goal: string;
+  assert: EvalAssert;
+}
+
+export interface DefineEvalInput {
+  name: string;
+  goal: string;
+  assert: EvalAssert;
+}
+
+/** Live LLM eval — requires AI Gateway; fixture-only chains are not supported. */
+export function defineEval(input: DefineEvalInput): EvalDefinition {
+  if (!input.name?.trim()) {
+    throw new Error("defineEval: name is required");
+  }
+  if (!input.goal?.trim()) {
+    throw new Error("defineEval: goal is required");
+  }
+  return Object.freeze({
+    __plasmSlotKind: PLASM_EVAL_KIND,
+    name: input.name.trim(),
+    goal: input.goal.trim(),
+    assert: input.assert,
+  });
+}
+
+export function isEvalDefinition(value: unknown): value is EvalDefinition {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    (value as EvalDefinition).__plasmSlotKind === PLASM_EVAL_KIND
+  );
+}

package/src/evals/run-eval.ts

@@ -0,0 +1,136 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+import type { PlasmAgent } from "../runtime/plasm-agent.js";
+import type { EvalDefinition, EvalToolName } from "./define-eval.js";
+import { isEvalDefinition } from "./define-eval.js";
+
+export interface EvalRunResult {
+  name: string;
+  ok: boolean;
+  recordedTools: EvalToolName[];
+  responseText?: string;
+  stepCount: number;
+  error?: string;
+}
+
+function collectToolNames(steps: unknown[]): EvalToolName[] {
+  const names = new Set<EvalToolName>();
+  for (const step of steps) {
+    if (!step || typeof step !== "object") continue;
+    const toolCalls = (step as { toolCalls?: Array<{ toolName?: string }> }).toolCalls ?? [];
+    for (const call of toolCalls) {
+      const name = call.toolName;
+      if (name) names.add(name as EvalToolName);
+    }
+  }
+  return [...names];
+}
+
+async function importEvalModule(filePath: string): Promise<EvalDefinition> {
+  const mod = await import(pathToFileURL(filePath).href);
+  const exported = mod.default ?? mod;
+  if (!isEvalDefinition(exported)) {
+    throw new Error(`default export must be defineEval() result`);
+  }
+  return exported;
+}
+
+export async function discoverEvalFiles(evalsDir: string): Promise<string[]> {
+  const entries = await readdir(evalsDir, { withFileTypes: true });
+  return entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".eval.ts"))
+    .map((entry) => path.join(evalsDir, entry.name))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+/** Run one live LLM eval via agent.generate(). */
+export async function runEval(
+  agent: PlasmAgent,
+  definition: EvalDefinition,
+): Promise<EvalRunResult> {
+  let recordedTools: EvalToolName[] = [];
+  let responseText: string | undefined;
+  let stepCount = 0;
+
+  try {
+    const result = await agent.generate(definition.goal, { resetConversation: true });
+    responseText = result.text;
+    stepCount = result.steps.length;
+    recordedTools = collectToolNames(result.steps);
+
+    if (definition.assert.minSteps !== undefined && stepCount < definition.assert.minSteps) {
+      throw new Error(`expected at least ${definition.assert.minSteps} steps, got ${stepCount}`);
+    }
+
+    if (definition.assert.toolsUsedAny?.length) {
+      for (const tool of definition.assert.toolsUsedAny) {
+        if (!recordedTools.includes(tool)) {
+          throw new Error(`expected tool ${tool} in live run, got [${recordedTools.join(", ")}]`);
+        }
+      }
+    }
+
+    if (definition.assert.responseIncludes) {
+      const needle = definition.assert.responseIncludes;
+      const haystack = responseText ?? "";
+      const matches =
+        typeof needle === "string" ? haystack.includes(needle) : needle.test(haystack);
+      if (!matches) {
+        throw new Error(`response missing expected fragment`);
+      }
+    }
+
+    return {
+      name: definition.name,
+      ok: true,
+      recordedTools,
+      responseText,
+      stepCount,
+    };
+  } catch (err) {
+    return {
+      name: definition.name,
+      ok: false,
+      recordedTools,
+      responseText,
+      stepCount,
+      error: String(err),
+    };
+  }
+}
+
+export async function runEvalFile(
+  agent: PlasmAgent,
+  evalPath: string,
+): Promise<EvalRunResult> {
+  const definition = await importEvalModule(evalPath);
+  return runEval(agent, definition);
+}
+
+export function requireLiveEvalGateway(): void {
+  const key =
+    process.env.AI_GATEWAY_API_KEY?.trim() ||
+    process.env.AI_API_GATEWAY_KEY?.trim() ||
+    process.env.AI_GATEWAY_KEY?.trim();
+  if (!key) {
+    throw new Error(
+      "Live evals require AI_GATEWAY_API_KEY (Vercel AI Gateway). Fixture-only eval chains are not supported.",
+    );
+  }
+  process.env.AI_GATEWAY_API_KEY ??= key;
+  process.env.PLASM_AGENT_MOCK_HTTP = "1";
+}
+
+export async function runAllEvals(
+  agent: PlasmAgent,
+  evalsDir: string,
+): Promise<EvalRunResult[]> {
+  const files = await discoverEvalFiles(evalsDir);
+  const results: EvalRunResult[] = [];
+  for (const file of files) {
+    results.push(await runEvalFile(agent, file));
+  }
+  return results;
+}

package/src/gateway-model.ts

@@ -0,0 +1,43 @@
+import { gateway, type LanguageModel } from "ai";
+
+import type { AgentModelOptions } from "./define-agent.js";
+
+function ensureGatewayApiKey(): void {
+  if (process.env.AI_GATEWAY_API_KEY?.trim()) return;
+  const alias =
+    process.env.AI_API_GATEWAY_KEY?.trim() ?? process.env.AI_GATEWAY_KEY?.trim();
+  if (alias) {
+    process.env.AI_GATEWAY_API_KEY = alias;
+  }
+}
+
+/** Whether `resolveGatewayModel` can run (API key or mapped alias present). */
+export function isGatewayConfigured(): boolean {
+  ensureGatewayApiKey();
+  return Boolean(process.env.AI_GATEWAY_API_KEY?.trim());
+}
+
+/** Resolve a Gateway model slug (`provider/model`) to an AI SDK `LanguageModel`. */
+export function resolveGatewayModel(
+  model: string | LanguageModel,
+  _options?: AgentModelOptions,
+): LanguageModel {
+  if (typeof model !== "string") return model;
+  const slug = model.trim();
+  if (!slug) {
+    throw new Error("agent `model` must be a non-empty AI Gateway slug");
+  }
+
+  ensureGatewayApiKey();
+  if (!process.env.AI_GATEWAY_API_KEY?.trim()) {
+    throw new Error(
+      [
+        "Vercel AI Gateway requires AI_GATEWAY_API_KEY.",
+        "Create one in the Vercel dashboard (AI → Gateway → API Keys) and add it to plasm-oss/.env.",
+        "Local dev: `vercel env pull` from a linked project also works.",
+      ].join(" "),
+    );
+  }
+
+  return gateway(slug);
+}