@intentius/chant-lexicon-e2b 0.0.0-stage → 0.124.32

package/src/op/sandbox.ts

@@ -0,0 +1,138 @@
+/**
+ * The seam between the e2b runtime and an E2B-compatible API.
+ *
+ * {@link StewardSandboxes} is all the runtime asks of a sandbox provider: find
+ * the steward's sandbox by its metadata, make one, wake one, run a command in
+ * it, list and kill its processes, keep it awake and pause it. The production
+ * implementation below is a thin wrapper over the `e2b` SDK, so the runtime
+ * works against hosted E2B and anything that serves E2B's API (loam, for one);
+ * a test hands the runtime a fake and drives the whole path with no network.
+ */
+
+/** The metadata key a steward's sandbox is found by. */
+export const STEWARD_METADATA_KEY = "chant/steward";
+
+/** The environment variables that mark a process in the sandbox as a chant run. */
+export const RUN_OP_ENV = "CHANT_E2B_OP";
+export const RUN_ID_ENV = "CHANT_E2B_RUN";
+export const RUN_STARTED_ENV = "CHANT_E2B_STARTED";
+
+export interface CommandResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+export interface RunCommandOptions {
+  envs?: Record<string, string>;
+  onStdout?: (chunk: string) => void;
+  onStderr?: (chunk: string) => void;
+  /** Called with the process id once the command has started. */
+  onStart?: (pid: number) => void;
+}
+
+export interface SandboxProcess {
+  pid: number;
+  envs: Record<string, string>;
+}
+
+/** One steward's sandbox, awake. */
+export interface StewardSandbox {
+  readonly id: string;
+  /** Run `bash -c <script>` to completion, however long it takes. A non-zero exit is a result, not a throw. */
+  run(script: string, opts?: RunCommandOptions): Promise<CommandResult>;
+  processes(): Promise<SandboxProcess[]>;
+  kill(pid: number): Promise<void>;
+  /** Hold the sandbox awake for `ms` from now. When that runs out it pauses rather than dying. */
+  keepAwake(ms: number): Promise<void>;
+  pause(): Promise<void>;
+}
+
+export interface StewardSandboxes {
+  /** The steward's sandbox, running or paused, or `undefined` when it has none. */
+  find(steward: string): Promise<{ id: string; state: string } | undefined>;
+  /** Make the steward's sandbox. It pauses, rather than dies, when it is left alone. */
+  create(steward: string, opts: { template: string; awakeMs: number }): Promise<StewardSandbox>;
+  /** Wake (or reach) a sandbox by id. */
+  connect(id: string, opts: { awakeMs: number }): Promise<StewardSandbox>;
+}
+
+export interface E2bConnection {
+  apiKey: string;
+  /** Omitted: the SDK's own default. */
+  domain?: string;
+}
+
+/** The production seam, over the `e2b` SDK. The SDK is imported on first use, so loading the plugin costs nothing. */
+export function e2bSandboxes(conn: E2bConnection): StewardSandboxes {
+  const base = { apiKey: conn.apiKey, ...(conn.domain ? { domain: conn.domain } : {}) };
+  const sdk = () => import("e2b");
+
+  const wrap = (sb: import("e2b").Sandbox): StewardSandbox => ({
+    id: sb.sandboxId,
+    async run(script, opts = {}) {
+      const { CommandExitError } = await sdk();
+      // `timeoutMs: 0` lifts the SDK's one-minute default: an op runs as long as it runs.
+      const handle = await sb.commands.run(script, {
+        background: true,
+        timeoutMs: 0,
+        ...(opts.envs ? { envs: opts.envs } : {}),
+        ...(opts.onStdout ? { onStdout: opts.onStdout } : {}),
+        ...(opts.onStderr ? { onStderr: opts.onStderr } : {}),
+      });
+      opts.onStart?.(handle.pid);
+      try {
+        const done = await handle.wait();
+        return { exitCode: done.exitCode, stdout: done.stdout, stderr: done.stderr };
+      } catch (err) {
+        if (err instanceof CommandExitError) return { exitCode: err.exitCode, stdout: err.stdout, stderr: err.stderr };
+        throw err;
+      }
+    },
+    async processes() {
+      return (await sb.commands.list()).map((p) => ({ pid: p.pid, envs: p.envs ?? {} }));
+    },
+    async kill(pid) {
+      await sb.commands.kill(pid);
+    },
+    async keepAwake(ms) {
+      await sb.setTimeout(ms);
+    },
+    async pause() {
+      await sb.pause();
+    },
+  });
+
+  return {
+    async find(steward) {
+      const { Sandbox } = await sdk();
+      const page = await Sandbox.list({
+        ...base,
+        query: { metadata: { [STEWARD_METADATA_KEY]: steward }, state: ["running", "paused"] },
+        limit: 2,
+      }).nextItems();
+      if (page.length > 1) {
+        throw new Error(
+          `e2b runtime: ${page.length} sandboxes carry ${STEWARD_METADATA_KEY}=${steward}. ` +
+            `A steward is one machine; kill all but one (${page.map((s) => s.sandboxId).join(", ")}).`,
+        );
+      }
+      return page[0] ? { id: page[0].sandboxId, state: page[0].state } : undefined;
+    },
+    async create(steward, opts) {
+      const { Sandbox } = await sdk();
+      const sb = await Sandbox.create(opts.template, {
+        ...base,
+        timeoutMs: opts.awakeMs,
+        metadata: { [STEWARD_METADATA_KEY]: steward },
+        // Run out of time paused, not killed: the disk, and the memory, are the steward.
+        lifecycle: { onTimeout: "pause" },
+      });
+      return wrap(sb);
+    },
+    async connect(id, opts) {
+      const { Sandbox } = await sdk();
+      return wrap(await Sandbox.connect(id, { ...base, timeoutMs: opts.awakeMs }));
+    },
+  };
+}

package/src/package-cli.ts

@@ -0,0 +1,16 @@
+#!/usr/bin/env tsx
+/**
+ * `npm run bundle` for the e2b lexicon: writes dist/manifest.json, dist/meta.json,
+ * the rules and the skills.
+ */
+import { packageLexicon } from "./codegen/package";
+import { writeBundleSpec } from "@intentius/chant/codegen/package";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+
+const distDir = join(dirname(fileURLToPath(import.meta.url)), "..", "dist");
+const verbose = process.argv.includes("--verbose");
+
+const { spec, stats } = await packageLexicon({ verbose });
+writeBundleSpec(spec, distDir);
+console.error(`Packaged ${stats.resources} resources, ${stats.ruleCount} rules, ${stats.skillCount} skills into ${distDir}`);

package/src/plugin.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "vitest";
+import { isLexiconPlugin } from "@intentius/chant/lexicon";
+import { e2bPlugin } from "./plugin";
+
+describe("e2b plugin", () => {
+  it("is a valid LexiconPlugin", () => {
+    expect(isLexiconPlugin(e2bPlugin)).toBe(true);
+  });
+
+  it("has the e2b name and serializer", () => {
+    expect(e2bPlugin.name).toBe("e2b");
+    expect(e2bPlugin.serializer.name).toBe("e2b");
+    expect(e2bPlugin.serializer.rulePrefix).toBe("E2B");
+  });
+
+  it("ships lint rules and post-synth checks under the E2B prefix, every check catalogued", () => {
+    expect((e2bPlugin.lintRules?.() ?? []).map((r) => r.id)).toEqual(["E2B001", "E2B002"]);
+    const checks = (e2bPlugin.postSynthChecks?.() ?? []).map((c) => c.id);
+    expect(checks).toEqual(["E2B011", "E2B012"]);
+    for (const id of checks) expect(e2bPlugin.auditCatalog?.()[id]).toBeDefined();
+  });
+
+  it("keeps its op runtime and config namespace", () => {
+    expect(e2bPlugin.opRuntime).toBeDefined();
+    expect(e2bPlugin.configSchema?.safeParse({ profiles: { prod: { apiKey: { env: "E2B_API_KEY" }, template: "chant-steward" } }, defaultProfile: "prod" }).success).toBe(true);
+  });
+
+  it("registers completion, hover and docs", () => {
+    expect(typeof e2bPlugin.completionProvider).toBe("function");
+    expect(typeof e2bPlugin.hoverProvider).toBe("function");
+    expect(typeof e2bPlugin.docs).toBe("function");
+  });
+
+  it("loads its skill with content", () => {
+    const skills = e2bPlugin.skills?.() ?? [];
+    expect(skills.map((s) => s.name)).toEqual(["chant-e2b-ops"]);
+    expect(skills[0].content.length).toBeGreaterThan(500);
+  });
+});

package/src/plugin.ts

@@ -0,0 +1,79 @@
+import type { LexiconPlugin } from "@intentius/chant/lexicon";
+import type { CompletionContext, HoverContext } from "@intentius/chant/lsp/types";
+import { createSkillsLoader } from "@intentius/chant/lexicon-plugin-helpers";
+import { e2bSerializer } from "./serializer";
+import { e2bConfigSchema } from "./config";
+import { createE2bOpRuntime } from "./op/runtime";
+import { rules } from "./lint/rules";
+import { postSynthChecks } from "./lint/post-synth";
+import { e2bAuditCatalog } from "./lint/audit-catalog";
+import { completions } from "./lsp/completions";
+import { hover } from "./lsp/hover";
+
+/**
+ * e2b lexicon plugin: `chant run <op> --on e2b`, and the `Template` a
+ * steward's sandbox is made from.
+ *
+ * The op runtime is most of the lexicon. Its one declarable kind is written
+ * by hand, so `generate` only writes the post-synth barrel, there is no
+ * upstream spec to validate against or measure coverage of, and `package`
+ * bundles the hand-written catalog.
+ */
+export const e2bPlugin: LexiconPlugin = {
+  name: "e2b",
+  serializer: e2bSerializer,
+  configSchema: e2bConfigSchema,
+  auditCatalog: () => e2bAuditCatalog,
+
+  // Construction is pure: the key, the steward and the repository are
+  // resolved on the first call, so loading the plugin costs nothing.
+  opRuntime: createE2bOpRuntime(),
+
+  skills: createSkillsLoader(import.meta.url, [
+    {
+      file: "chant-e2b-ops.md",
+      name: "chant-e2b-ops",
+      description: "Run chant ops on a steward sandbox with `--on e2b`, on hosted E2B or any API that speaks E2B's",
+      triggers: [{ type: "context" as const, value: "e2b" }],
+    },
+  ]),
+
+  async generate(): Promise<void> {
+    const { generate } = await import("./codegen/generate");
+    generate();
+  },
+
+  // Nothing upstream to check the hand-written kind against.
+  async validate(): Promise<void> {},
+  async coverage(): Promise<void> {},
+
+  async package(options?: { verbose?: boolean; force?: boolean }): Promise<void> {
+    const { packageLexicon } = await import("./codegen/package");
+    const { writeBundleSpec } = await import("@intentius/chant/codegen/package");
+    const { join, dirname } = await import("path");
+    const { fileURLToPath } = await import("url");
+    const { spec } = await packageLexicon(options);
+    writeBundleSpec(spec, join(dirname(dirname(fileURLToPath(import.meta.url))), "dist"));
+  },
+
+  lintRules() {
+    return rules;
+  },
+
+  postSynthChecks() {
+    return postSynthChecks;
+  },
+
+  completionProvider(ctx: CompletionContext) {
+    return completions(ctx);
+  },
+
+  hoverProvider(ctx: HoverContext) {
+    return hover(ctx);
+  },
+
+  async docs(options?) {
+    const { generateDocs } = await import("./codegen/docs");
+    return generateDocs(options);
+  },
+};

package/src/resources.ts

@@ -0,0 +1,50 @@
+/**
+ * The e2b lexicon's one declarable kind: a sandbox template.
+ *
+ * A steward's sandbox is made from a template (`template` in the profile,
+ * default `base`). The runtime needs git and Node in it, and a project usually
+ * wants its own tools there too, installed once rather than on every run. A
+ * `Template` declares that image; `chant build` writes it out as a Dockerfile
+ * and the `e2b template create` command line that builds it.
+ */
+
+import type { Declarable } from "@intentius/chant/declarable";
+import { createResource } from "@intentius/chant/runtime";
+import { TEMPLATE_TYPE } from "./catalog";
+
+export interface TemplateProps {
+  /** The template's name on E2B: lowercase letters, digits, dashes and underscores. A profile's `template` names it. */
+  name: string;
+  /** The image the template starts from, such as `node:22-bookworm`. Pin a tag. */
+  fromImage: string;
+  /** Shell commands run in order while the template builds, one `RUN` line each. */
+  run?: string[];
+  /** Environment variables set in every sandbox made from the template. */
+  env?: Record<string, string>;
+  /** The working directory for the `run` steps and the start command. */
+  workdir?: string;
+  /** The start command (`--cmd`): E2B runs it when the template is snapshotted, so a sandbox starts with it running. */
+  startCmd?: string;
+  /** The ready command (`--ready-cmd`): E2B polls it until it exits 0 before it snapshots the template. */
+  readyCmd?: string;
+  /** vCPUs per sandbox. A positive integer; E2B's default is 2. */
+  cpuCount?: number;
+  /** Memory per sandbox in MiB. A positive even integer; E2B's default is 1024. */
+  memoryMB?: number;
+}
+
+export interface Template extends Declarable {
+  readonly props: TemplateProps;
+}
+
+/** A sandbox template, serialized to `e2b.<name>.Dockerfile` and an `e2b template create` command. */
+export const Template = /* @__PURE__ */ createResource(TEMPLATE_TYPE, "e2b", {}) as unknown as new (
+  props: TemplateProps,
+) => Template;
+
+/** The props of a declared `Template`, or `undefined` for anything else. */
+export function templateProps(entity: unknown): TemplateProps | undefined {
+  if (!entity || typeof entity !== "object") return undefined;
+  if ((entity as { entityType?: unknown }).entityType !== TEMPLATE_TYPE) return undefined;
+  return ((entity as { props?: unknown }).props ?? {}) as TemplateProps;
+}

package/src/serializer.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from "vitest";
+import type { Declarable } from "@intentius/chant/declarable";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { e2bSerializer } from "./serializer";
+import { Template } from "./resources";
+
+function serialize(entries: Array<[string, unknown]>): SerializerResult {
+  return e2bSerializer.serialize(new Map(entries as Array<[string, Declarable]>)) as SerializerResult;
+}
+
+describe("e2b serializer", () => {
+  it("writes a Dockerfile per template, named after it, and the command that builds it", () => {
+    const out = serialize([
+      [
+        "steward",
+        new Template({
+          name: "chant-steward",
+          fromImage: "node:22-bookworm",
+          env: { CI: "1", GREETING: 'say "hi"' },
+          workdir: "/home/user",
+          run: ["apt-get update", "apt-get install -y jq"],
+          startCmd: "node server.js",
+          readyCmd: "curl -sf localhost:3000",
+          cpuCount: 2,
+          memoryMB: 2048,
+        }),
+      ],
+    ]);
+
+    expect(Object.keys(out.files ?? {})).toEqual(["e2b.chant-steward.Dockerfile"]);
+    expect(out.verbatimFiles).toEqual(["e2b.chant-steward.Dockerfile"]);
+    expect(out.files?.["e2b.chant-steward.Dockerfile"]).toBe(
+      [
+        "FROM node:22-bookworm",
+        'ENV CI="1"',
+        'ENV GREETING="say \\"hi\\""',
+        "WORKDIR /home/user",
+        "RUN apt-get update",
+        "RUN apt-get install -y jq",
+        "",
+      ].join("\n"),
+    );
+    expect(JSON.parse(out.primary)).toEqual({
+      templates: {
+        steward: {
+          name: "chant-steward",
+          dockerfile: "e2b.chant-steward.Dockerfile",
+          build:
+            "e2b template create chant-steward --dockerfile e2b.chant-steward.Dockerfile " +
+            "--cmd 'node server.js' --ready-cmd 'curl -sf localhost:3000' --cpu-count 2 --memory-mb 2048",
+          cpuCount: 2,
+          memoryMB: 2048,
+        },
+      },
+    });
+  });
+
+  it("leaves out what a template does not set", () => {
+    const out = serialize([["base", new Template({ name: "slim", fromImage: "debian:12" })]]);
+    expect(out.files?.["e2b.slim.Dockerfile"]).toBe("FROM debian:12\n");
+    expect(JSON.parse(out.primary).templates.base).toEqual({
+      name: "slim",
+      dockerfile: "e2b.slim.Dockerfile",
+      build: "e2b template create slim --dockerfile e2b.slim.Dockerfile",
+    });
+  });
+
+  it("quotes a start command with a single quote in it as one shell word", () => {
+    const out = serialize([["s", new Template({ name: "s", fromImage: "node:22-bookworm", startCmd: "echo 'hi'" })]]);
+    expect(JSON.parse(out.primary).templates.s.build).toContain(`--cmd 'echo '\\''hi'\\'''`);
+  });
+
+  it("orders templates by export name and ignores anything that is not one", () => {
+    const out = serialize([
+      ["zeta", new Template({ name: "zeta", fromImage: "node:22-bookworm" })],
+      ["alpha", new Template({ name: "alpha", fromImage: "node:22-bookworm" })],
+      ["other", { entityType: "Other::Thing", props: {} }],
+    ]);
+    expect(Object.keys(JSON.parse(out.primary).templates)).toEqual(["alpha", "zeta"]);
+    expect(Object.keys(out.files ?? {})).toHaveLength(2);
+  });
+
+  it("gives a build with no templates an empty index", () => {
+    const out = serialize([]);
+    expect(JSON.parse(out.primary)).toEqual({ templates: {} });
+    expect(out.files).toEqual({});
+  });
+});

package/src/serializer.ts

@@ -0,0 +1,73 @@
+import type { Declarable } from "@intentius/chant/declarable";
+import type { Serializer, SerializerResult } from "@intentius/chant/serializer";
+import { templateProps, type TemplateProps } from "./resources";
+
+/**
+ * The e2b serializer: each declared `Template` becomes `e2b.<name>.Dockerfile`
+ * (`FROM`, then `ENV`, `WORKDIR` and one `RUN` per step), named after the
+ * template so several can sit in one directory.
+ *
+ * The primary output is a JSON index of the templates, keyed by the name each
+ * was exported as. Each entry carries the Dockerfile's name and `build`, the
+ * `e2b template create` command line that builds the template from it, run in
+ * the output directory. A build with no templates (a project that only runs
+ * Ops here) gets an empty index.
+ */
+export const e2bSerializer: Serializer = {
+  name: "e2b",
+  rulePrefix: "E2B",
+  serialize(entities: Map<string, Declarable>): SerializerResult {
+    const index: Record<string, unknown> = {};
+    const files: Record<string, string> = {};
+
+    const templates = [...entities]
+      .map(([exportName, entity]) => [exportName, templateProps(entity)] as const)
+      .filter((e): e is readonly [string, TemplateProps] => e[1] !== undefined)
+      .sort(([a], [b]) => a.localeCompare(b));
+
+    for (const [exportName, props] of templates) {
+      const name = props.name || exportName;
+      const dockerfile = `e2b.${name}.Dockerfile`;
+      files[dockerfile] = renderDockerfile(props);
+      index[exportName] = {
+        name,
+        dockerfile,
+        build: createCommand(name, dockerfile, props),
+        ...(props.cpuCount !== undefined ? { cpuCount: props.cpuCount } : {}),
+        ...(props.memoryMB !== undefined ? { memoryMB: props.memoryMB } : {}),
+      };
+    }
+
+    return {
+      primary: JSON.stringify({ templates: index }, null, 2) + "\n",
+      files,
+      verbatimFiles: Object.keys(files),
+    };
+  },
+};
+
+/** A Dockerfile string literal: Docker reads double-quoted values with JSON's escapes. */
+const quote = (value: string): string => JSON.stringify(value);
+
+/** A word as one shell argument. */
+function shellWord(value: string): string {
+  return /^[A-Za-z0-9._:@+=/,-]+$/.test(value) ? value : `'${value.replace(/'/g, `'\\''`)}'`;
+}
+
+export function renderDockerfile(props: TemplateProps): string {
+  const lines = [`FROM ${props.fromImage}`];
+  for (const [key, value] of Object.entries(props.env ?? {})) lines.push(`ENV ${key}=${quote(value)}`);
+  if (props.workdir) lines.push(`WORKDIR ${props.workdir}`);
+  for (const step of props.run ?? []) lines.push(`RUN ${step}`);
+  return lines.join("\n") + "\n";
+}
+
+/** The `e2b template create` command line that builds the template, run where the Dockerfile is. */
+export function createCommand(name: string, dockerfile: string, props: TemplateProps): string {
+  const args = ["e2b", "template", "create", name, "--dockerfile", dockerfile];
+  if (props.startCmd) args.push("--cmd", props.startCmd);
+  if (props.readyCmd) args.push("--ready-cmd", props.readyCmd);
+  if (props.cpuCount !== undefined) args.push("--cpu-count", String(props.cpuCount));
+  if (props.memoryMB !== undefined) args.push("--memory-mb", String(props.memoryMB));
+  return args.map(shellWord).join(" ");
+}

package/src/skills/chant-e2b-ops.md

@@ -0,0 +1,69 @@
+---
+skill: chant-e2b-ops
+description: Run chant ops on a steward sandbox with `--on e2b`, on hosted E2B or any API that speaks E2B's
+user-invocable: true
+---
+
+# Running chant ops on an e2b steward
+
+`chant run <op> --on e2b` runs the op on a steward: one sandbox that outlives
+every run. It keeps a clone of the project and its installed tools on its disk,
+runs one op at a time, and sleeps paused between runs. Any API that speaks
+E2B's hosts one: hosted E2B, or a platform that serves the same API.
+
+## Set it up
+
+```ts
+// chant.config.ts
+import type { ChantConfig } from "@intentius/chant/config";
+import "@intentius/chant-lexicon-e2b";
+
+export default {
+  lexicons: ["e2b"],
+  e2b: {
+    profiles: {
+      prod: {
+        apiKey: { env: "E2B_API_KEY" },
+        domain: "e2b.example.com",        // omit for hosted E2B
+        steward: "prod-steward",           // default: the repository's name
+        gitToken: { env: "GITHUB_TOKEN" }, // fetch a private repo, push the ledger
+      },
+    },
+    defaultProfile: "prod",
+  },
+} satisfies ChantConfig;
+```
+
+The project must depend on `@intentius/chant`, because the steward runs the
+chant the project installs. The template (default `base`) needs git and Node.
+
+## Run, read, cancel
+
+```bash
+chant run prod-apply --on e2b              # streams the steps, then pauses the steward
+chant run status prod-apply --on e2b       # a run in progress, or the last one recorded
+chant run log prod-apply --on e2b
+chant run cancel prod-apply --on e2b --force
+```
+
+The steward runs the commit this checkout is at, fetched from the repository,
+so **push first**. Uncommitted changes are not sent; the run warns about them.
+
+One run at a time: a run while the steward is busy is refused, naming the run
+in progress. A client that dies mid-run (Ctrl-C excepted) leaves the run going;
+`status` reports it, and the steward pauses itself when it is done.
+
+## Gates
+
+A gate works as it does locally. The run reaches it in the sandbox, pushes the
+pending fact to the repository's `chant/lifecycle` branch and ends `gated`
+(exit 3). `chant approve <op> <gate>` from any checkout of the repository
+records the answer, and the next `--on e2b` run walks through. The steward also
+pushes its run records, so `chant run log` on any checkout sees its runs.
+
+## Running on a schedule
+
+Nothing in E2B fires on a clock. Run `chant run <op> --on e2b` from whatever
+already does: a CI `schedule:` trigger (the github lexicon renders an op's
+`schedule` as one), a CronJob, or `chant operator` on a machine that stays up.
+The job is short, because the steward does the work.