@junejs/server 0.1.3 → 0.2.0

package/src/agent-compile.ts

@@ -0,0 +1,271 @@
+// agent-compile.ts — compile an agent/ directory into a static module
+// (_agent.gen.ts) so the SAME directory convention mounts where filesystem
+// discovery cannot run: workerd has no fs, so discoverAgent (dynamic import +
+// readFile) is native-only. The generated module is plain erasable TypeScript —
+// static imports of tools/channels/connections, markdown inlined as string
+// literals — bundleable by wrangler or Rolldown, importable under Node type
+// stripping and bun test alike. No [[rules]] Text hack, no md loader plugins,
+// no hand-maintained tool registry.
+//
+// Mirrors the content freeze (content.ts): a pure emitter (emitAgentModule)
+// over a plain scan (scanAgentDir), with a thin fs wrapper
+// (generateAgentModule) — the emitter is unit-testable without touching disk.
+// Assembly logic is NOT duplicated here: the generated module exports the raw
+// AgentModule shape, and both runtimes assemble it through
+// @junejs/core/agent-config (assembleAgent natively, assembleDurable in a DO
+// shell). Skill files are inlined raw and parsed at module init via core's
+// parseSkill, so a parser improvement never requires regeneration.
+
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+
+import { stripJsonc } from "./tsconfig-jsx";
+
+export const AGENT_MODULE_FILE = "_agent.gen.ts";
+
+// The raw scan of an agent directory: file paths for code (imported statically
+// by the emitted module), file contents for prose (inlined as literals).
+export type AgentDirScan = {
+  dir: string;
+  // basename fallback when agent.ts is absent (mirrors discoverAgent).
+  name: string;
+  hasConfig: boolean;
+  instructions: string | null;
+  // instructions.<source>.md at the agent root — per-surface variants (#149).
+  surfaceVariants: { source: string; raw: string }[];
+  tools: string[]; // relative specifiers, e.g. "./tools/create_order.ts"
+  skills: { name: string; raw: string }[];
+  channels: string[]; // "./channels/slack.ts"
+  channelOverlays: { source: string; raw: string }[]; // channels/<source>.md
+  connections: string[]; // "./connections/teachify.ts"
+};
+
+// `_`-prefixed files are private by convention — most importantly the generated
+// module itself, which must never be scanned back in. Sorted for determinism.
+function list(dir: string, ext: string): string[] {
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir, { withFileTypes: true })
+    .filter((e) => e.isFile() && e.name.endsWith(ext) && !e.name.startsWith("_"))
+    .map((e) => e.name)
+    .sort();
+}
+
+// Scan an agent directory (fs only — no dynamic imports; the point is to defer
+// module loading to the bundler). Returns null when the directory holds nothing
+// agent-shaped, so callers can probe candidate paths cheaply.
+export function scanAgentDir(dir: string): AgentDirScan | null {
+  if (!existsSync(dir)) return null;
+  const instructionsFile = join(dir, "instructions.md");
+  const scan: AgentDirScan = {
+    dir,
+    name: basename(dir),
+    hasConfig: existsSync(join(dir, "agent.ts")),
+    instructions: existsSync(instructionsFile) ? readFileSync(instructionsFile, "utf8") : null,
+    surfaceVariants: readdirSync(dir, { withFileTypes: true })
+      .filter((e) => e.isFile())
+      .map((e) => e.name.match(/^instructions\.([a-z0-9_-]+)\.md$/))
+      .filter((m): m is RegExpMatchArray => m !== null)
+      .sort((a, b) => a[1]!.localeCompare(b[1]!))
+      .map((m) => ({ source: m[1]!, raw: readFileSync(join(dir, m[0]!), "utf8") })),
+    tools: list(join(dir, "tools"), ".ts").map((f) => `./tools/${f}`),
+    skills: list(join(dir, "skills"), ".md").map((f) => ({
+      name: basename(f, ".md"),
+      raw: readFileSync(join(dir, "skills", f), "utf8"),
+    })),
+    channels: list(join(dir, "channels"), ".ts").map((f) => `./channels/${f}`),
+    channelOverlays: list(join(dir, "channels"), ".md").map((f) => ({
+      source: basename(f, ".md"),
+      raw: readFileSync(join(dir, "channels", f), "utf8"),
+    })),
+    connections: list(join(dir, "connections"), ".ts").map((f) => `./connections/${f}`),
+  };
+  const empty =
+    !scan.hasConfig &&
+    scan.instructions === null &&
+    !scan.tools.length &&
+    !scan.skills.length &&
+    !scan.channels.length &&
+    !scan.surfaceVariants.length &&
+    !scan.channelOverlays.length &&
+    !scan.connections.length;
+  return empty ? null : scan;
+}
+
+// A stable, readable import identifier from a file specifier: "./tools/create_order.ts"
+// → "tool_create_order". Collisions (foo-bar vs foo_bar) get a numeric suffix.
+function identFor(prefix: string, spec: string, taken: Set<string>): string {
+  const stem = basename(spec).replace(/\.ts$/, "").replace(/[^A-Za-z0-9]/g, "_");
+  let id = `${prefix}_${stem}`;
+  for (let i = 2; taken.has(id); i++) id = `${prefix}_${stem}_${i}`;
+  taken.add(id);
+  return id;
+}
+
+export type EmitOptions = {
+  // Keep ".ts" on relative import specifiers. Required by consumers that run
+  // the module under Node's native type stripping (Node never resolves
+  // extensionless imports); rejected (TS5097) by tsconfigs without
+  // allowImportingTsExtensions. generateAgentModule sniffs the consumer's
+  // tsconfig; pass explicitly to override.
+  tsExtensions?: boolean;
+};
+
+// Emit the module source for a scan. Pure — the golden test pins this output.
+export function emitAgentModule(scan: AgentDirScan, opts?: EmitOptions): string {
+  const spec = (s: string) => JSON.stringify(opts?.tsExtensions ? s : s.replace(/\.ts$/, ""));
+  const taken = new Set<string>();
+  const imports: string[] = [];
+  const toolIds = scan.tools.map((s) => {
+    const id = identFor("tool", s, taken);
+    imports.push(`import ${id} from ${spec(s)};`);
+    return id;
+  });
+  const channelIds = scan.channels.map((s) => {
+    const id = identFor("channel", s, taken);
+    imports.push(`import ${id} from ${spec(s)};`);
+    return { key: basename(s, ".ts"), id };
+  });
+  const connectionIds = scan.connections.map((s) => {
+    const id = identFor("connection", s, taken);
+    imports.push(`import ${id} from ${spec(s)};`);
+    return id;
+  });
+  if (scan.hasConfig) imports.push(`import config from ${spec("./agent.ts")};`);
+
+  const lines: string[] = [
+    "// AUTO-GENERATED by June — do not edit. Compiled from this agent/ directory so",
+    "// the definition mounts without filesystem discovery (workerd has no fs).",
+    "// Regenerate with `june gen` after adding, removing, or editing agent files.",
+    ...imports,
+    scan.skills.length
+      ? `import { parseSkill, type AgentModule } from "@junejs/core/agent-config";`
+      : `import type { AgentModule } from "@junejs/core/agent-config";`,
+    "",
+  ];
+  if (!scan.hasConfig) lines.push(`const config = { name: ${JSON.stringify(scan.name)} };`, "");
+
+  lines.push(
+    "export const agentModule: AgentModule = {",
+    "  config,",
+    `  instructions: ${JSON.stringify(scan.instructions ?? "")},`,
+    scan.surfaceVariants.length
+      ? `  surfaceInstructions: {\n${scan.surfaceVariants.map((v) => `    ${JSON.stringify(v.source)}: ${JSON.stringify(v.raw)},`).join("\n")}\n  },`
+      : "  surfaceInstructions: {},",
+    `  tools: [${toolIds.join(", ")}],`,
+    scan.skills.length
+      ? `  skills: [\n${scan.skills.map((s) => `    parseSkill(${JSON.stringify(s.name)}, ${JSON.stringify(s.raw)}),`).join("\n")}\n  ],`
+      : "  skills: [],",
+    channelIds.length
+      ? `  channels: { ${channelIds.map((c) => `${JSON.stringify(c.key)}: ${c.id}`).join(", ")} },`
+      : "  channels: {},",
+    scan.channelOverlays.length
+      ? `  channelInstructions: {\n${scan.channelOverlays.map((o) => `    ${JSON.stringify(o.source)}: ${JSON.stringify(o.raw)},`).join("\n")}\n  },`
+      : "  channelInstructions: {},",
+    `  connections: [${connectionIds.join(", ")}],`,
+    "};",
+    "export default agentModule;",
+    "",
+  );
+  return lines.join("\n");
+}
+
+// Locate the agent directory for a project root: a June app keeps it at
+// app/<dir> (the app.ts convention); a wrangler-first worker keeps it at
+// <root>/<dir>. `dirName` follows june.config's agent.runtime.dir (default
+// "agent").
+export function findAgentDir(root: string, dirName = "agent"): string | null {
+  for (const candidate of [join(root, "app", dirName), join(root, dirName)]) {
+    if (existsSync(candidate)) return candidate;
+  }
+  return null;
+}
+
+// Whether the consumer's tsconfig keeps ".ts" on import specifiers
+// (allowImportingTsExtensions): the nearest tsconfig.json walking up from the
+// agent dir, parsed as JSONC (same stripJsonc as tsconfig-jsx.ts — a commented-
+// out option must not count) and resolved through `extends` (relative, bare
+// package specifiers, and the TS 5 array form; own options win, later array
+// entries beat earlier ones). Explicit opts.tsExtensions overrides the sniff.
+type TsconfigLite = { extends?: string | string[]; compilerOptions?: { allowImportingTsExtensions?: boolean } };
+
+function readTsconfig(path: string): TsconfigLite | undefined {
+  try {
+    return JSON.parse(stripJsonc(readFileSync(path, "utf8"))) as TsconfigLite;
+  } catch {
+    return undefined; // absent or unparsable — treat as "not declared"
+  }
+}
+
+// Resolve an `extends` specifier from the extending config's directory: relative
+// paths directly, bare specifiers through the nearest node_modules walking up.
+// tsc appends ".json" when the target has no extension — mirror that.
+function resolveExtends(fromDir: string, spec: string): string | undefined {
+  const withJson = (p: string) => (existsSync(p) ? p : !p.endsWith(".json") && existsSync(`${p}.json`) ? `${p}.json` : undefined);
+  if (spec.startsWith(".")) return withJson(resolve(fromDir, spec));
+  for (let d = fromDir; ; ) {
+    const candidate = withJson(join(d, "node_modules", spec));
+    if (candidate) return candidate;
+    const parent = dirname(d);
+    if (parent === d) return undefined;
+    d = parent;
+  }
+}
+
+function resolveAllowTsExtensions(path: string, depth: number): boolean | undefined {
+  if (depth > 8) return undefined; // extends cycles/degenerate chains — give up
+  const tc = readTsconfig(path);
+  if (!tc) return undefined;
+  const own = tc.compilerOptions?.allowImportingTsExtensions;
+  if (own !== undefined) return own;
+  // TS extends-array semantics: later entries override earlier ones.
+  const bases = typeof tc.extends === "string" ? [tc.extends] : [...(tc.extends ?? [])].reverse();
+  for (const spec of bases) {
+    const base = resolveExtends(dirname(path), spec);
+    const inherited = base ? resolveAllowTsExtensions(base, depth + 1) : undefined;
+    if (inherited !== undefined) return inherited;
+  }
+  return undefined;
+}
+
+function sniffTsExtensions(dir: string): boolean {
+  for (let d = dir; ; ) {
+    const candidate = join(d, "tsconfig.json");
+    if (existsSync(candidate)) return resolveAllowTsExtensions(candidate, 0) ?? false;
+    const parent = dirname(d);
+    if (parent === d) return false;
+    d = parent;
+  }
+}
+
+// Generate <dir>/_agent.gen.ts. With `check: true` nothing is written — `stale`
+// reports whether the file on disk differs from what would be generated (the CI
+// gate; `june gen && git diff --exit-code` works too). Returns null when the
+// directory holds nothing agent-shaped.
+export function generateAgentModule(
+  dir: string,
+  opts?: { check?: boolean } & EmitOptions,
+): { file: string; code: string; stale: boolean; written: boolean } | null {
+  const scan = scanAgentDir(dir);
+  if (!scan) return null;
+  for (const o of scan.channelOverlays) {
+    console.warn(
+      `[june] deprecated: channels/${o.source}.md — move it to instructions.${o.source}.md at the agent root (per-surface behavior is agent-level; see junebuild/june#149).`,
+    );
+  }
+  // A variant-shaped file the grammar doesn't parse yet must never be silently
+  // inert (the same failure class this whole convention exists to kill).
+  for (const e of readdirSync(dir, { withFileTypes: true })) {
+    if (e.isFile() && /^instructions\..+\.md$/.test(e.name) && !/^instructions\.[a-z0-9_-]+\.md$/.test(e.name)) {
+      console.warn(
+        `[june] ${e.name}: unrecognized instructions variant — only single-segment sources (instructions.<source>.md) are parsed today; locale variants are a planned follow-up (junebuild/june#149). This file is ignored.`,
+      );
+    }
+  }
+  const code = emitAgentModule(scan, { tsExtensions: opts?.tsExtensions ?? sniffTsExtensions(dir) });
+  const file = join(dir, AGENT_MODULE_FILE);
+  const existing = existsSync(file) ? readFileSync(file, "utf8") : null;
+  const stale = existing !== code;
+  if (opts?.check) return { file, code, stale, written: false };
+  if (stale) writeFileSync(file, code);
+  return { file, code, stale, written: stale };
+}

package/src/agent-discover.ts

@@ -3,75 +3,82 @@
 // An agent is a DIRECTORY, not a config object. Location determines function:
 //
 //   agent/
-//     agent.ts         → default-exports a plain config { name, model?, description? }
-//     instructions.md  → system prompt
-//     tools/*.ts       → each default-exports a defineAction (a tool)
-//     skills/*.md      → each a procedure, loaded on demand (progressive disclosure)
-//     channels/*.ts    → each default-exports a Channel (an inbound edge)
-//     connections/*.ts → each default-exports a Connection (an outbound tool source)
+//     agent.ts                  → default-exports a plain config { name, model?, surfaces? }
+//     instructions.md           → base system prompt
+//     instructions.<source>.md  → per-surface instruction variant (#149)
+//     tools/*.ts                → each default-exports a defineAction (a tool),
+//                                 OR an array of defineActions (an integration
+//                                 shipping several tools, e.g. googleDriveTools())
+//     skills/*.md               → each a procedure, loaded on demand (progressive disclosure)
+//     channels/*.ts             → each default-exports a Channel (an inbound edge — pure transport)
+//     channels/*.md             → DEPRECATED overlay location (warned; use instructions.<source>.md)
+//     connections/*.ts          → each default-exports a Connection (an outbound tool source)
 //
 // There is no central registry to keep in sync — the directory IS the manifest.
 // Because tools are `defineAction`s, the SAME directory is also an MCP server and
 // a `.agent` manifest (surfaces land in a later step). fs lives here in the host
-// package; the pure assembly is @junejs/core/agent-config's defineAgent().
+// package; the pure assembly is @junejs/core/agent-config's assembleAgent() — the
+// SAME entry point a compiled agent module (agent-compile.ts, the edge path) goes
+// through, so native discovery and the generated module cannot drift.
 
 import { existsSync } from "node:fs";
 import { readdir, readFile } from "node:fs/promises";
 import { basename, extname, join } from "node:path";
 import { pathToFileURL } from "node:url";
 
-import type { AnyAction } from "@junejs/core/agent";
-import { defineAgent, resolveChannel, type AgentConfigFile, type AgentDefinition, type Channel, type ChannelFactory, type Skill } from "@junejs/core/agent-config";
-import { connectAll, type Connection } from "@junejs/core/connections";
+import { assembleAgent, parseSkill, type AgentConfigFile, type AgentDefinition, type AgentModule, type Channel, type ChannelFactory, type Skill, type ToolEntry } from "@junejs/core/agent-config";
+import type { Connection } from "@junejs/core/connections";
 
+// `_`-prefixed files are private by convention (mirrors the app/ router) — most
+// importantly the generated `_agent.gen.ts` itself, which must never be scanned
+// back in as a tool/channel.
 async function scan(dir: string, ext: string): Promise<string[]> {
   if (!existsSync(dir)) return [];
   const entries = await readdir(dir, { withFileTypes: true });
   return entries
-    .filter((e) => e.isFile() && extname(e.name) === ext)
+    .filter((e) => e.isFile() && extname(e.name) === ext && !e.name.startsWith("_"))
     .map((e) => join(dir, e.name))
     .sort();
 }
 
-// Optional YAML-ish frontmatter: --- \n key: value \n --- \n body. Falls back to
-// the first non-empty line (minus a leading "# ") as the description.
-function parseSkill(name: string, text: string): Skill {
-  if (text.startsWith("---")) {
-    const end = text.indexOf("\n---", 3);
-    if (end !== -1) {
-      const front = text.slice(3, end);
-      const body = text.slice(end + 4).replace(/^\s*\n/, "");
-      const meta: Record<string, string> = {};
-      for (const line of front.split("\n")) {
-        const m = line.match(/^\s*(\w+)\s*:\s*(.+?)\s*$/);
-        if (m) meta[m[1]!] = m[2]!;
-      }
-      return { name: meta.name ?? name, description: meta.description ?? "", body };
-    }
-  }
-  const firstLine = text.split("\n").find((l) => l.trim()) ?? "";
-  return { name, description: firstLine.replace(/^#\s*/, ""), body: text };
-}
-
-// Discover an agent from its directory, returning the assembled AgentDefinition
-// (tools already adapted from defineActions; read_skill added if any skills
-// exist). Mount it on a runtime with createNativeRuntime({ [agent.name]: { model,
-// tools: agent.tools } }).
-export async function discoverAgent(dir: string): Promise<AgentDefinition> {
+// Read an agent directory into its raw, unassembled shape (fs + dynamic import —
+// native only). The pure sibling for the edge is agent-compile.ts, which emits
+// the same AgentModule as static code.
+export async function discoverAgentModule(dir: string): Promise<AgentModule> {
   const configFile = join(dir, "agent.ts");
   const config: AgentConfigFile = existsSync(configFile)
     ? ((await import(pathToFileURL(configFile).href)).default as AgentConfigFile)
     : { name: basename(dir) };
 
   const instructionsFile = join(dir, "instructions.md");
-  const instructions = existsSync(instructionsFile)
-    ? await readFile(instructionsFile, "utf8")
-    : (config.instructions ?? "");
+  const instructions = existsSync(instructionsFile) ? await readFile(instructionsFile, "utf8") : "";
 
-  const tools: AnyAction[] = [];
+  // instructions.<source>.md — the agent's per-surface instruction variants
+  // (#149): applied to turns arriving through that source, composed per
+  // config.surfaces (append by default, "replace" to stand alone).
+  const surfaceInstructions: Record<string, string> = {};
+  if (existsSync(dir)) {
+    for (const e of await readdir(dir, { withFileTypes: true })) {
+      if (!e.isFile() || !/^instructions\..+\.md$/.test(e.name)) continue;
+      const m = e.name.match(/^instructions\.([a-z0-9_-]+)\.md$/);
+      if (m) {
+        surfaceInstructions[m[1]!] = await readFile(join(dir, e.name), "utf8");
+      } else {
+        // A variant-shaped file the grammar doesn't parse yet (e.g. a locale
+        // suffix, instructions.slack.zh-TW.md) must never be SILENTLY inert.
+        console.warn(
+          `[june] ${e.name}: unrecognized instructions variant — only single-segment sources (instructions.<source>.md) are parsed today; locale variants are a planned follow-up (junebuild/june#149). This file is ignored.`,
+        );
+      }
+    }
+  }
+
+  // A tool file default-exports one tool OR an array of tools (an integration
+  // like googleDriveTools()); assembly flattens arrays.
+  const tools: ToolEntry[] = [];
   for (const f of await scan(join(dir, "tools"), ".ts")) {
     const mod = await import(pathToFileURL(f).href);
-    if (mod.default) tools.push(mod.default as AnyAction);
+    if (mod.default) tools.push(mod.default as ToolEntry);
   }
 
   const skills: Skill[] = [];
@@ -80,34 +87,41 @@ export async function discoverAgent(dir: string): Promise<AgentDefinition> {
   }
 
   // A channel module default-exports a Channel OR a `(env) => Channel` factory (the
-  // form workerd needs — secrets live in env, not at module scope). On native, env is
-  // process.env (present at load), so resolve here; the plain form stays supported.
-  const channels: Channel[] = [];
+  // form workerd needs — secrets live in env, not at module scope). Resolution is
+  // assembleAgent's job (native: process.env; a DO: its own env) — keep raw here.
+  const channels: Record<string, Channel | ChannelFactory> = {};
   for (const f of await scan(join(dir, "channels"), ".ts")) {
     const mod = await import(pathToFileURL(f).href);
-    if (mod.default) channels.push(resolveChannel(mod.default as Channel | ChannelFactory, process.env));
+    if (mod.default) channels[basename(f, ".ts")] = mod.default as Channel | ChannelFactory;
   }
 
-  // connections/*.ts — outbound: wire external MCP/OpenAPI servers, turning their
-  // operations into `<connection>__<tool>` defineActions that join `tools`.
-  const connectionDefs: Connection[] = [];
+  // channels/<source>.md — the DEPRECATED overlay convention (#149): behavior
+  // prose belongs at the agent level (instructions.<source>.md), not beside
+  // transport code. Still honored for one dev-series, loudly.
+  const channelInstructions: Record<string, string> = {};
+  for (const f of await scan(join(dir, "channels"), ".md")) {
+    const source = basename(f, ".md");
+    console.warn(
+      `[june] deprecated: channels/${source}.md — move it to instructions.${source}.md at the agent root (per-surface behavior is agent-level; see junebuild/june#149).`,
+    );
+    channelInstructions[source] = await readFile(f, "utf8");
+  }
+
+  // connections/*.ts — outbound: definitions of external MCP/OpenAPI servers;
+  // assembleAgent wires them (connectAll) where the agent actually runs.
+  const connections: Connection[] = [];
   for (const f of await scan(join(dir, "connections"), ".ts")) {
     const mod = await import(pathToFileURL(f).href);
-    if (mod.default) connectionDefs.push(mod.default as Connection);
+    if (mod.default) connections.push(mod.default as Connection);
   }
-  const { actions: connectionActions, report: connections } = connectionDefs.length
-    ? await connectAll(connectionDefs)
-    : { actions: [], report: [] };
-  tools.push(...connectionActions);
 
-  return defineAgent({
-    name: config.name,
-    model: config.model,
-    description: config.description,
-    instructions,
-    tools,
-    skills,
-    channels,
-    connections,
-  });
+  return { config, instructions, surfaceInstructions, tools, skills, channels, channelInstructions, connections };
+}
+
+// Discover an agent from its directory, returning the assembled AgentDefinition
+// (tools already adapted from defineActions; read_skill added if any skills
+// exist). Mount it on a runtime with createNativeRuntime({ [agent.name]:
+// toAgentDef(agent, model) }).
+export async function discoverAgent(dir: string): Promise<AgentDefinition> {
+  return assembleAgent(await discoverAgentModule(dir), process.env);
 }