esoul-sdk 0.3.0

package/README.md

@@ -0,0 +1,61 @@
+# esoul-sdk
+
+Build **ExternalSoul apps**: native apps for an event-sourced workspace where people and AI
+agents share one canvas. An app you write with this SDK is indistinguishable from the platform's
+own once it ships — the same events, the same timeline, the same durability, the same tools that
+chat, voice, agents and MCP call.
+
+```bash
+npm install --save-dev esoul-sdk
+```
+
+Inside ExternalSoul the package name resolves to the platform's real implementations. Outside it
+(your editor, your tests) the package gives you the types, the manifest validator, and the test
+helpers; the server functions throw "host only" if called, because they run in the platform.
+
+## Where to start
+
+- **Build it in the Forge, not on your laptop.** Open a Forge board in your workspace and ask it
+  to open a workbench for your app. You get a cloud machine with the platform on it, a live
+  preview in your frame, your app's tools callable before it is installed, checks, and a submit
+  button. Read [docs/01-getting-started.md](docs/01-getting-started.md).
+- **The contract, one page per part:**
+  1. [Getting started](docs/01-getting-started.md) — the loop, the package layout
+  2. [The manifest](docs/02-manifest.md) — `plugin.json`, every field
+  3. [Events and state](docs/03-events-and-state.md) — the heart: dataCreator, processor, replay
+  4. [Tools](docs/04-tools.md) — what agents call, on every surface
+  5. [The UI](docs/05-ui.md) — React, hooks, theme, responsive rules
+  6. [The server half](docs/06-server.md) — ops, webhooks, reading state, calling other apps
+  7. [Background tasks](docs/07-background-tasks.md) — durable work, polling, the replay model
+  8. [Connections and OAuth](docs/08-connections.md) — tokens the platform holds for you
+  9. [Files](docs/09-files.md) — workspace files, Drive, your own provider
+  10. [Testing](docs/10-testing.md) — the fold contract as tests
+  11. [Shipping](docs/11-shipping.md) — submit, review, release, install; the import wall
+  12. [Rules and failures](docs/12-rules.md) — every rule with the failure that earned it
+- **For a coding model:** `llms.txt` (short) and `llms-full.txt` (the whole contract in one file).
+
+## The one rule that explains the others
+
+**Events are the truth.** Your app's state is the fold of its events, re-run on every replay,
+scrub and sync. So a reducer must be pure and idempotent, ids and timestamps are minted in the
+`dataCreator` (never in a reducer), whole-replace events carry a collapse key, and the agent-facing
+state description never claims something it could not read. Everything in the docs follows from
+that.
+
+## What is in the package
+
+| Entry | What it gives you |
+|---|---|
+| `esoul-sdk` | `ApplicationSchema`, `EventDefinition`, `EventTypes`, `ApplicationIdentifier`, `incompleteStateNotice`, `deterministicReducerId`, `stableStringify`, `timingSafeEqual`, `nanoid`, `callPluginOp`, the manifest schema |
+| `esoul-sdk/react` | `usePluginEventDispatch`, `useAppCanEdit`, `usePluginCurrentChatId`, `useWorkspaceTools`, file hooks |
+| `esoul-sdk/server` | `PluginServerModule`, `readAppState`, `callWorkspaceTool`, `emitPluginAppEvent`, `getPluginConnectionCredentials`, file provider types |
+| `esoul-sdk/testing` | a mock OAuth server for connection tests |
+| `esoul-app validate <dir>` | validates a package folder against the manifest schema |
+
+## Versions
+
+- **0.3.0** — renamed from `@externalsoul/plugin-sdk` (still resolved as an alias inside the
+  platform). `readAppState`, `callWorkspaceTool`, `nanoid` on the index. The import wall: an app
+  reaches the platform only through this package. Docs rewritten for the Forge workbench loop.
+- 0.2.0 — file sources and providers.
+- 0.1.0 — the contract: manifest, schema, events, tools, tasks, webhooks, ops, connections.

package/bin/esoul-plugin.mjs

@@ -0,0 +1,110 @@
+#!/usr/bin/env node
+/**
+ * esoul-plugin — repo-free plugin package checks.
+ *
+ *   esoul-app validate <dir>   exit 0 = the folder is a well-formed
+ *                                 package; exit 1 = every problem listed.
+ *
+ * This is the AUTHOR-SIDE half of validation (manifest shape, entry
+ * existence, forbidden imports, secret-shaped keys). The host's
+ * plugins:sync re-checks everything plus host-only facts (type collisions
+ * with installed apps) at install time — passing here is necessary, not
+ * sufficient, and the CLI says so.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import url from "node:url";
+
+const here = path.dirname(url.fileURLToPath(import.meta.url));
+let PluginManifestSchema;
+try {
+  ({ PluginManifestSchema } = await import(
+    url.pathToFileURL(path.join(here, "..", "dist", "manifest.js")).href
+  ));
+} catch {
+  console.error("esoul-plugin: package not built (dist/ missing) — reinstall esoul-sdk");
+  process.exit(2);
+}
+
+const [cmd, dirArg] = process.argv.slice(2);
+if (cmd !== "validate" || !dirArg) {
+  console.error("usage: esoul-app validate <pluginDir>");
+  process.exit(2);
+}
+const dir = path.resolve(dirArg);
+const problems = [];
+const problem = (m) => problems.push(m);
+
+if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
+  console.error(`not a directory: ${dir}`);
+  process.exit(1);
+}
+
+/* manifest */
+let manifest = null;
+const manifestPath = path.join(dir, "plugin.json");
+if (!fs.existsSync(manifestPath)) {
+  problem("plugin.json missing — not a plugin package");
+} else {
+  try {
+    const raw = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+    const parsed = PluginManifestSchema.safeParse(raw);
+    if (!parsed.success) {
+      for (const issue of parsed.error.issues) {
+        problem(`plugin.json ${issue.path.join(".") || "(root)"}: ${issue.message}`);
+      }
+    } else {
+      manifest = parsed.data;
+    }
+  } catch (e) {
+    problem(`plugin.json is not valid JSON: ${e.message}`);
+  }
+}
+
+/* entry + server module */
+if (manifest) {
+  const entryOk = ["tsx", "ts"].some((ext) =>
+    fs.existsSync(path.join(dir, `${manifest.entry}.${ext}`)),
+  );
+  if (!entryOk) problem(`entry module "${manifest.entry}.(tsx|ts)" not found`);
+  const needsServer = manifest.webhooks.length > 0 || manifest.ops.length > 0;
+  const hasServer = ["ts", "tsx"].some((ext) =>
+    fs.existsSync(path.join(dir, `server.${ext}`)),
+  );
+  if (needsServer && !hasServer) {
+    problem("declares webhooks/ops but has no server.ts exporting pluginServer");
+  }
+}
+
+/* source lints (the init-cycle traps + secret smells) */
+const walk = (d) =>
+  fs.readdirSync(d, { withFileTypes: true }).flatMap((e) => {
+    const p = path.join(d, e.name);
+    if (e.isDirectory()) {
+      return ["node_modules", ".git", "dist"].includes(e.name) ? [] : walk(p);
+    }
+    return /\.(ts|tsx)$/.test(e.name) ? [p] : [];
+  });
+for (const f of walk(dir)) {
+  const src = fs.readFileSync(f, "utf8");
+  const rel = path.relative(dir, f);
+  if (/(from\s+|import\s*\(\s*|require\s*\(\s*)["'][^"']*client-registry["']/.test(src)) {
+    problem(`${rel}: imports client-registry — the init-cycle trap; plugins never import the registry`);
+  }
+  if (/from\s+["']@\/store\/venus-slice["']/.test(src)) {
+    problem(`${rel}: statically imports venus-slice — use the hooks from esoul-sdk/react`);
+  }
+  if (/(sk-[A-Za-z0-9]{20,}|-----BEGIN [A-Z ]*PRIVATE KEY-----)/.test(src)) {
+    problem(`${rel}: looks like a hard-coded secret — credentials belong in connections, never in code`);
+  }
+}
+
+if (problems.length) {
+  console.error(`✗ ${dir}`);
+  for (const p of problems) console.error(`  - ${p}`);
+  process.exit(1);
+}
+console.log(`✓ ${manifest.name} (${manifest.id} / ${manifest.applicationType}) is a well-formed package.`);
+console.log(
+  "  Host-side checks (type collisions, gate tests) still run at install — this pass is necessary, not sufficient.",
+);

package/dist/files.d.ts

@@ -0,0 +1,59 @@
+/**
+ * File sources — the client-safe half of the files API
+ * (plugin-file-sources.md). PURE mirror of the host's
+ * src/lib/file-sources/types.ts; the sdk-drift gate keeps them in lockstep.
+ *
+ * A FileSource is a MOUNT: one place files come from (the workspace, the
+ * workspace's Google Drive, a local folder on the desktop build, or a mount
+ * contributed by a plugin's own connection). Identity = `sourceId` +
+ * provider-opaque `ref`; labels are display only. A source that cannot
+ * answer reports `source_unavailable` — never an empty listing.
+ */
+export type FileSourceId = string;
+export interface FileSource {
+    sourceId: FileSourceId;
+    providerKey: string;
+    label: string;
+    detail?: string;
+    caps: {
+        write: boolean;
+        watch: boolean;
+        durable: boolean;
+    };
+    online: boolean;
+}
+export interface FileRef {
+    sourceId: FileSourceId;
+    /** Provider-opaque id. Workspace source: the WorkspaceFile.id uuid. */
+    ref: string;
+}
+export interface FileEntry {
+    ref: FileRef;
+    name: string;
+    kind: "file" | "folder";
+    sizeBytes?: number;
+    mimeType?: string;
+    /** Epoch ms. */
+    modifiedAt?: number;
+    path?: string;
+}
+export type FileSourceErrorKind = "not_declared" | "source_unavailable" | "not_found" | "too_large" | "read_only" | "disabled" | "bad_ref";
+export declare class FileSourceError extends Error {
+    readonly kind: FileSourceErrorKind;
+    constructor(kind: FileSourceErrorKind, message: string);
+}
+export declare const FILE_READ_CAP_BYTES: number;
+export declare const FILE_READ_HARD_CAP_BYTES: number;
+export type ParsedSourceId = {
+    kind: "workspace";
+} | {
+    kind: "google-drive";
+} | {
+    kind: "local";
+    mountId: string;
+} | {
+    kind: "plugin";
+    providerKey: string;
+    connectionId: string;
+};
+export declare function parseSourceId(sourceId: string): ParsedSourceId;

package/dist/files.js

@@ -0,0 +1,39 @@
+/**
+ * File sources — the client-safe half of the files API
+ * (plugin-file-sources.md). PURE mirror of the host's
+ * src/lib/file-sources/types.ts; the sdk-drift gate keeps them in lockstep.
+ *
+ * A FileSource is a MOUNT: one place files come from (the workspace, the
+ * workspace's Google Drive, a local folder on the desktop build, or a mount
+ * contributed by a plugin's own connection). Identity = `sourceId` +
+ * provider-opaque `ref`; labels are display only. A source that cannot
+ * answer reports `source_unavailable` — never an empty listing.
+ */
+export class FileSourceError extends Error {
+    kind;
+    constructor(kind, message) {
+        super(message);
+        this.name = "FileSourceError";
+        this.kind = kind;
+    }
+}
+export const FILE_READ_CAP_BYTES = 25 * 1024 * 1024;
+export const FILE_READ_HARD_CAP_BYTES = 100 * 1024 * 1024;
+export function parseSourceId(sourceId) {
+    if (sourceId === "workspace")
+        return { kind: "workspace" };
+    if (sourceId === "google-drive")
+        return { kind: "google-drive" };
+    const colon = sourceId.indexOf(":");
+    if (colon <= 0 || colon === sourceId.length - 1) {
+        throw new FileSourceError("bad_ref", `unknown file source "${sourceId}"`);
+    }
+    const head = sourceId.slice(0, colon);
+    const tail = sourceId.slice(colon + 1);
+    if (head === "local")
+        return { kind: "local", mountId: tail };
+    if (head === "workspace" || head === "google-drive") {
+        throw new FileSourceError("bad_ref", `unknown file source "${sourceId}"`);
+    }
+    return { kind: "plugin", providerKey: head, connectionId: tail };
+}

package/dist/helpers.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Pure helpers — byte-for-byte vendored from the platform (a drift gate in
+ * the host asserts behavioural equality on every build).
+ */
+export declare function stableStringify(v: unknown): string;
+/** Refold-stable id from event payload — the ONLY way a processor may mint
+ * an id when the dataCreator didn't (never nanoid/Date.now in a reducer). */
+export declare function deterministicReducerId(prefix: string, seed: unknown): string;
+/** Fields whose ABSENCE means "did not load" (they are seeded at genesis). */
+export interface RequiredStateShape {
+    /** Keys that must be arrays. */
+    lists?: Record<string, unknown>;
+    /** Keys that must be non-null objects. */
+    maps?: Record<string, unknown>;
+}
+export declare function missingStateKeys(shape: RequiredStateShape): string[];
+/**
+ * `null` when the state is whole — the describer proceeds normally.
+ * Otherwise the block to return INSTEAD of a description. A missing
+ * collection is "state did not load", NEVER "empty" — flooring to [] tells
+ * the model something false and invites it to rebuild over real data.
+ */
+export declare function incompleteStateNotice(args: {
+    title: string;
+    instanceName?: unknown;
+    shape: RequiredStateShape;
+}): string | null;
+/**
+ * Call one of your plugin's server ops from client-graph code (tools,
+ * tasks, UI). Server side authenticates with the internal secret; browser
+ * side rides the session. Throws the op's honest error — relay it, never
+ * swallow it.
+ */
+export declare function callPluginOp<T = unknown>(pluginId: string, op: string, nodeId: string, args?: unknown): Promise<T>;
+/**
+ * Constant-time string compare for webhook secrets/signatures. A plain
+ * `===` leaks length/prefix timing; use THIS in every webhook handler.
+ * Pure JS (no node:crypto) so it is safe in any runtime the SDK reaches.
+ */
+export declare function timingSafeEqual(a: string, b: string): boolean;

package/dist/helpers.js

@@ -0,0 +1,93 @@
+/**
+ * Pure helpers — byte-for-byte vendored from the platform (a drift gate in
+ * the host asserts behavioural equality on every build).
+ */
+/* ── deterministic ids (from event-spec.ts) ─────────────────────────── */
+export function stableStringify(v) {
+    if (v === null || typeof v !== "object")
+        return JSON.stringify(v) ?? "null";
+    if (Array.isArray(v))
+        return `[${v.map(stableStringify).join(",")}]`;
+    const keys = Object.keys(v).sort();
+    return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(v[k])}`).join(",")}}`;
+}
+/** Refold-stable id from event payload — the ONLY way a processor may mint
+ * an id when the dataCreator didn't (never nanoid/Date.now in a reducer). */
+export function deterministicReducerId(prefix, seed) {
+    const s = stableStringify(seed);
+    let h = 5381;
+    for (let i = 0; i < s.length; i++)
+        h = ((h * 33) ^ s.charCodeAt(i)) >>> 0;
+    return `${prefix}_${h.toString(36)}${(s.length & 0xffff).toString(36)}`;
+}
+export function missingStateKeys(shape) {
+    const missing = [];
+    for (const [key, value] of Object.entries(shape.lists ?? {})) {
+        if (!Array.isArray(value))
+            missing.push(key);
+    }
+    for (const [key, value] of Object.entries(shape.maps ?? {})) {
+        if (value === null || value === undefined || typeof value !== "object") {
+            missing.push(key);
+        }
+    }
+    return missing;
+}
+/**
+ * `null` when the state is whole — the describer proceeds normally.
+ * Otherwise the block to return INSTEAD of a description. A missing
+ * collection is "state did not load", NEVER "empty" — flooring to [] tells
+ * the model something false and invites it to rebuild over real data.
+ */
+export function incompleteStateNotice(args) {
+    const missing = missingStateKeys(args.shape);
+    if (missing.length === 0)
+        return null;
+    const name = typeof args.instanceName === "string" && args.instanceName
+        ? args.instanceName
+        : "(unnamed)";
+    return (`## ${args.title} — "${name}"\n` +
+        `- ⚠ STATE UNAVAILABLE: ${missing.join(", ")} did not load, so this app's contents are UNKNOWN.\n` +
+        `- This is NOT an empty app. Do not add, overwrite, re-import or re-run anything here on the assumption that it is empty — read it with this app's own read/list tools first, or tell the user it could not be read.\n`);
+}
+/* ── ops caller (from lib/plugins/call-op.ts) ───────────────────────── */
+/**
+ * Call one of your plugin's server ops from client-graph code (tools,
+ * tasks, UI). Server side authenticates with the internal secret; browser
+ * side rides the session. Throws the op's honest error — relay it, never
+ * swallow it.
+ */
+export async function callPluginOp(pluginId, op, nodeId, args) {
+    const serverBase = typeof window === "undefined"
+        ? process.env.NEXT_PUBLIC_APP_URL ?? "http://localhost:3000"
+        : "";
+    const headers = { "Content-Type": "application/json" };
+    if (typeof window === "undefined" && process.env.INTERNAL_TOOL_SECRET) {
+        headers["x-esoul-internal-secret"] = process.env.INTERNAL_TOOL_SECRET;
+    }
+    const res = await fetch(`${serverBase}/api/plugins/${pluginId}/op/${op}`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({ nodeId, args }),
+    });
+    const body = (await res.json().catch(() => null));
+    if (!res.ok || !body?.ok) {
+        throw new Error(body?.error ?? `plugin op ${pluginId}/${op} failed (HTTP ${res.status})`);
+    }
+    return body.result;
+}
+/* ── webhook secret comparison ──────────────────────────────────────── */
+/**
+ * Constant-time string compare for webhook secrets/signatures. A plain
+ * `===` leaks length/prefix timing; use THIS in every webhook handler.
+ * Pure JS (no node:crypto) so it is safe in any runtime the SDK reaches.
+ */
+export function timingSafeEqual(a, b) {
+    const len = Math.max(a.length, b.length, 1);
+    let diff = a.length === b.length ? 0 : 1;
+    for (let i = 0; i < len; i++) {
+        diff |= (a.charCodeAt(i % Math.max(a.length, 1)) || 0) ^
+            (b.charCodeAt(i % Math.max(b.length, 1)) || 0);
+    }
+    return diff === 0;
+}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export * from "./types.js";
+export * from "./manifest.js";
+export * from "./helpers.js";
+export * from "./files.js";
+/** Id minting for dataCreators; an app depends on the SDK, not on nanoid. */
+export { nanoid } from "nanoid";

package/dist/index.js

@@ -0,0 +1,6 @@
+export * from "./types.js";
+export * from "./manifest.js";
+export * from "./helpers.js";
+export * from "./files.js";
+/** Id minting for dataCreators; an app depends on the SDK, not on nanoid. */
+export { nanoid } from "nanoid";