@vauban-org/agent-sdk 0.17.4 → 1.2.0

package/src/skills/poc-md-loader/e2e-parity.test.ts

@@ -0,0 +1,237 @@
+/**
+ * E2E parity test — POC-B SKILL.md vs original web-search.ts
+ *
+ * Runs BOTH the original TS skill and the MD-loaded variant against
+ * the same mocked fetch, then compares output shape + content.
+ * Measures cold start of the MD loader path.
+ *
+ * No network calls — fetch is mocked via globalThis.fetch override.
+ *
+ * @poc-jetable
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ── Original skill (TS) ───────────────────────────────────────────────────────
+import { webSearch } from "../web-search.js";
+import type { WebSearchOutput } from "../web-search.js";
+
+// ── POC loader ────────────────────────────────────────────────────────────────
+import { loadSkillMd } from "./markdown-loader.js";
+import { runSkillMd } from "./runner.js";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SKILL_DIR = join(__dirname, "web-search");
+
+// ── Mock fetch helpers ────────────────────────────────────────────────────────
+
+const BRAVE_MOCK_RESPONSE = {
+  web: {
+    results: [
+      {
+        title: "Mock Result 1",
+        url: "https://example.com/1",
+        description: "Snippet one",
+      },
+      {
+        title: "Mock Result 2",
+        url: "https://example.com/2",
+        description: "Snippet two",
+      },
+    ],
+  },
+};
+
+function mockFetch(response: unknown, status = 200) {
+  return vi.fn().mockResolvedValue({
+    ok: status >= 200 && status < 300,
+    status,
+    json: async () => response,
+  });
+}
+
+// ── Replay context for original skill ────────────────────────────────────────
+
+const REPLAY_CTX = {
+  isReplay: true,
+  dryRunMocks: {
+    web_search: (_input: unknown) => ({
+      results: [
+        {
+          title: "Mock Result 1",
+          url: "https://example.com/1",
+          snippet: "Snippet one",
+        },
+        {
+          title: "Mock Result 2",
+          url: "https://example.com/2",
+          snippet: "Snippet two",
+        },
+      ],
+      provider: "replay" as const,
+    }),
+  },
+  agentId: "poc-test",
+  runId: "poc-run-1",
+};
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("POC-B: SKILL.md parity + cold start", () => {
+  let originalFetch: typeof globalThis.fetch;
+
+  beforeEach(() => {
+    originalFetch = globalThis.fetch;
+  });
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+    delete process.env.BRAVE_SEARCH_KEY;
+    delete process.env.TAVILY_API_KEY;
+  });
+
+  // ── 1. Manifest loading ──────────────────────────────────────────────────
+
+  it("loads SKILL.md and returns valid manifest", async () => {
+    const { manifest } = await loadSkillMd(join(SKILL_DIR, "SKILL.md"));
+
+    expect(manifest.name).toBe("web-search");
+    expect(manifest.description).toBeTruthy();
+    expect(manifest.description.length).toBeLessThanOrEqual(1024);
+    expect(manifest["allowed-tools"]).toBe("Bash(curl:*)");
+    expect(manifest.metadata?.version).toBe("1.0.0");
+    expect(manifest.metadata?.category).toBe("search");
+    expect(manifest.metadata?.["env-mode"]).toBe("at-least-one");
+  });
+
+  // ── 2. Output shape parity (replay vs MD) ───────────────────────────────
+
+  it("MD output shape matches original skill output shape (replay)", async () => {
+    // Original skill in replay mode
+    const originalOutput = await webSearch.execute(
+      { query: "test", limit: 2 },
+      REPLAY_CTX as never
+    );
+
+    // MD script in replay mode (mock fetch, BRAVE_SEARCH_KEY set → path taken but mocked)
+    process.env.BRAVE_SEARCH_KEY = "test-key";
+    globalThis.fetch = mockFetch(BRAVE_MOCK_RESPONSE) as never;
+
+    const { output: mdOutput } = await runSkillMd<WebSearchOutput>(SKILL_DIR, {
+      query: "test",
+      limit: 2,
+    });
+
+    // Shape parity: both have results[] and provider
+    expect(Object.keys(mdOutput)).toEqual(
+      expect.arrayContaining(["results", "provider"])
+    );
+    expect(Array.isArray(mdOutput.results)).toBe(true);
+
+    // Original replay returns empty results (no mock for live), MD returns from brave mock
+    // Key parity: result items have title/url/snippet
+    if (mdOutput.results.length > 0) {
+      const r = mdOutput.results[0];
+      expect(r).toHaveProperty("title");
+      expect(r).toHaveProperty("url");
+      expect(r).toHaveProperty("snippet");
+    }
+
+    // Provider field exists and is valid
+    expect(["brave", "tavily", "replay"]).toContain(mdOutput.provider);
+  });
+
+  // ── 3. Live path parity (both use mocked fetch, same brave response) ────
+
+  it("MD script returns same data as original skill on Brave success path", async () => {
+    process.env.BRAVE_SEARCH_KEY = "test-brave-key";
+    globalThis.fetch = mockFetch(BRAVE_MOCK_RESPONSE) as never;
+
+    // Original skill — live path with mocked fetch
+    const originalOutput = await webSearch.execute(
+      { query: "starknet", limit: 2 },
+      { isReplay: false, dryRunMocks: {}, agentId: "poc", runId: "r1" } as never
+    );
+
+    // Reset mock (fresh call counter)
+    globalThis.fetch = mockFetch(BRAVE_MOCK_RESPONSE) as never;
+
+    const { output: mdOutput } = await runSkillMd<WebSearchOutput>(SKILL_DIR, {
+      query: "starknet",
+      limit: 2,
+    });
+
+    // Full output equality
+    expect(mdOutput.provider).toBe(originalOutput.provider);
+    expect(mdOutput.results).toEqual(originalOutput.results);
+  });
+
+  // ── 4. Tavily fallback parity ────────────────────────────────────────────
+
+  it("MD script Tavily fallback matches original skill", async () => {
+    process.env.TAVILY_API_KEY = "test-tavily-key";
+    // No BRAVE key → goes straight to Tavily
+
+    const tavilyMock = {
+      results: [
+        {
+          title: "Tavily R1",
+          url: "https://tavily.com/1",
+          content: "Tavily snippet 1",
+        },
+      ],
+    };
+
+    globalThis.fetch = mockFetch(tavilyMock) as never;
+    const originalOutput = await webSearch.execute(
+      { query: "cairo", limit: 1 },
+      { isReplay: false, dryRunMocks: {}, agentId: "poc", runId: "r2" } as never
+    );
+
+    globalThis.fetch = mockFetch(tavilyMock) as never;
+    const { output: mdOutput } = await runSkillMd<WebSearchOutput>(SKILL_DIR, {
+      query: "cairo",
+      limit: 1,
+    });
+
+    expect(mdOutput.provider).toBe("tavily");
+    expect(mdOutput.provider).toBe(originalOutput.provider);
+    expect(mdOutput.results).toEqual(originalOutput.results);
+  });
+
+  // ── 5. Error parity: missing env ────────────────────────────────────────
+
+  it("MD runner throws when no API key configured (env-mode: at-least-one)", async () => {
+    // No BRAVE_SEARCH_KEY, no TAVILY_API_KEY
+    await expect(runSkillMd(SKILL_DIR, { query: "test" })).rejects.toThrow(
+      /at least one of/i
+    );
+  });
+
+  // ── 6. Cold start measurement ────────────────────────────────────────────
+
+  it("cold start (load+parse+execute) is under 500ms", async () => {
+    process.env.BRAVE_SEARCH_KEY = "perf-test-key";
+    globalThis.fetch = mockFetch(BRAVE_MOCK_RESPONSE) as never;
+
+    const t0 = performance.now();
+    const { coldStartMs } = await runSkillMd(SKILL_DIR, {
+      query: "perf",
+      limit: 1,
+    });
+    const wallMs = performance.now() - t0;
+
+    console.log(
+      `[POC-B] cold start: ${coldStartMs.toFixed(2)}ms (wall: ${wallMs.toFixed(
+        2
+      )}ms)`
+    );
+
+    // Primary assertion
+    expect(coldStartMs).toBeLessThan(500);
+    // Sanity: internal timer matches wall clock within 50ms
+    expect(Math.abs(coldStartMs - wallMs)).toBeLessThan(50);
+  });
+});

package/src/skills/poc-md-loader/markdown-loader.ts

@@ -0,0 +1,161 @@
+/**
+ * markdown-loader — prototype SKILL.md parser.
+ *
+ * Parses frontmatter (YAML-subset, no external dep) + markdown body.
+ * Returns a typed SkillManifest validated against a Zod schema.
+ *
+ * Constraint: no new npm deps — uses only node:fs/promises + zod (already in pkg).
+ *
+ * @poc-jetable
+ */
+
+import { readFile } from "node:fs/promises";
+import { z } from "zod";
+
+// ── Zod schema for SKILL.md frontmatter ──────────────────────────────────────
+
+const MetadataSchema = z
+  .object({
+    version: z.string().default("1.0.0"),
+    category: z.string().optional(),
+    tags: z.string().optional(),
+    "model-hint": z.string().optional(),
+    "env-required": z.string().optional(),
+    "env-mode": z.enum(["all", "at-least-one"]).default("all"),
+  })
+  .strict();
+
+export const SkillManifestSchema = z
+  .object({
+    /** kebab-case, 1-64 chars */
+    name: z
+      .string()
+      .min(1)
+      .max(64)
+      .regex(/^[a-z0-9-]+$/, "name must be kebab-case"),
+    /** 1-1024 chars */
+    description: z.string().min(1).max(1024),
+    /** Space-separated tool grant list, e.g. "Bash(curl:*) Read" */
+    "allowed-tools": z.string().optional(),
+    metadata: MetadataSchema.optional(),
+  })
+  .strict();
+
+export type SkillManifest = z.infer<typeof SkillManifestSchema>;
+
+// ── Minimal YAML-subset parser (no external dep) ─────────────────────────────
+// Supports: string scalars, nested objects (2-space indent), multi-line > strings.
+
+function parseYamlSubset(raw: string): Record<string, unknown> {
+  const lines = raw.split("\n");
+  const result: Record<string, unknown> = {};
+  let i = 0;
+
+  while (i < lines.length) {
+    const line = lines[i];
+    if (!line.trim() || line.trim().startsWith("#")) {
+      i++;
+      continue;
+    }
+
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) {
+      i++;
+      continue;
+    }
+
+    const indent = line.match(/^(\s*)/)?.[1].length ?? 0;
+    if (indent > 0) {
+      // nested key — handled by parent
+      i++;
+      continue;
+    }
+
+    const key = line.slice(0, colonIdx).trim();
+    const rest = line.slice(colonIdx + 1).trim();
+
+    // Multi-line folded scalar (>) — collect continuation lines
+    if (rest === ">") {
+      const parts: string[] = [];
+      i++;
+      while (
+        i < lines.length &&
+        (lines[i].startsWith("  ") || lines[i].trim() === "")
+      ) {
+        parts.push(lines[i].trim());
+        i++;
+      }
+      result[key] = parts.filter(Boolean).join(" ");
+      continue;
+    }
+
+    // Nested object — collect child lines with 2-space indent
+    if (rest === "") {
+      const childLines: string[] = [];
+      i++;
+      while (i < lines.length && lines[i].startsWith("  ")) {
+        childLines.push(lines[i].slice(2)); // strip 2-space indent
+        i++;
+      }
+      if (childLines.length > 0) {
+        result[key] = parseYamlSubset(childLines.join("\n"));
+      }
+      continue;
+    }
+
+    // Quoted string
+    if (
+      (rest.startsWith('"') && rest.endsWith('"')) ||
+      (rest.startsWith("'") && rest.endsWith("'"))
+    ) {
+      result[key] = rest.slice(1, -1);
+      i++;
+      continue;
+    }
+
+    // Plain scalar
+    result[key] = rest;
+    i++;
+  }
+
+  return result;
+}
+
+// ── SKILL.md parser ───────────────────────────────────────────────────────────
+
+export interface ParsedSkillFile {
+  manifest: SkillManifest;
+  body: string;
+}
+
+/**
+ * Parse a SKILL.md file at `filePath`.
+ * Extracts frontmatter (between --- delimiters) and validates with Zod.
+ * Returns typed manifest + raw markdown body.
+ */
+export async function loadSkillMd(filePath: string): Promise<ParsedSkillFile> {
+  const raw = await readFile(filePath, "utf-8");
+
+  // Extract frontmatter block
+  const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
+  if (!fmMatch) {
+    throw new Error(
+      `SKILL.md at ${filePath}: missing or malformed frontmatter (expected --- delimiters)`
+    );
+  }
+
+  const [, frontmatter, body] = fmMatch;
+  const rawParsed = parseYamlSubset(frontmatter);
+
+  // Validate with Zod
+  const result = SkillManifestSchema.safeParse(rawParsed);
+  if (!result.success) {
+    throw new Error(
+      `SKILL.md at ${filePath}: invalid frontmatter — ${result.error.issues
+        .map((i) => `${i.path.join(".")}: ${i.message}`)
+        .join("; ")}`
+    );
+  }
+
+  return { manifest: result.data, body: body.trim() };
+}

package/src/skills/poc-md-loader/runner.ts

@@ -0,0 +1,82 @@
+/**
+ * runner — prototype SKILL.md execution harness.
+ *
+ * Loads a SKILL.md manifest, validates env requirements,
+ * dynamically imports the companion script.ts, and executes it.
+ *
+ * No integration with SkillRegistry — standalone POC harness only.
+ *
+ * @poc-jetable
+ */
+
+import { join } from "node:path";
+import { loadSkillMd, type SkillManifest } from "./markdown-loader.js";
+
+export interface RunnerResult<T = unknown> {
+  output: T;
+  manifest: SkillManifest;
+  coldStartMs: number;
+}
+
+/**
+ * Run a SKILL.md skill by directory path.
+ *
+ * @param skillDir  Absolute path to directory containing SKILL.md + script.ts
+ * @param input     Raw input object passed to script.execute()
+ */
+export async function runSkillMd<T = unknown>(
+  skillDir: string,
+  input: unknown
+): Promise<RunnerResult<T>> {
+  const t0 = performance.now();
+
+  // Step 1: Load + validate SKILL.md
+  const skillMdPath = join(skillDir, "SKILL.md");
+  const { manifest } = await loadSkillMd(skillMdPath);
+
+  // Step 2: Check env requirements
+  const envRequired = manifest.metadata?.["env-required"];
+  const envMode = manifest.metadata?.["env-mode"] ?? "all";
+  if (envRequired) {
+    const keys = envRequired.split(" ").filter(Boolean);
+    const missing = keys.filter((k) => !process.env[k]);
+    if (envMode === "all" && missing.length > 0) {
+      throw new Error(
+        `Skill '${manifest.name}': missing required env vars: ${missing.join(
+          ", "
+        )}`
+      );
+    }
+    if (envMode === "at-least-one" && missing.length === keys.length) {
+      throw new Error(
+        `Skill '${manifest.name}': at least one of ${keys.join(
+          ", "
+        )} must be set`
+      );
+    }
+  }
+
+  // Step 3: Dynamic import of companion script
+  const scriptPath = join(skillDir, "script.js");
+  // In test context (tsx/ts-node), import .ts directly; in compiled, import .js
+  let scriptModule: { execute: (input: unknown) => Promise<T> };
+  try {
+    scriptModule = (await import(scriptPath)) as typeof scriptModule;
+  } catch {
+    // Fallback: try .ts extension (tsx / ts-node context)
+    const scriptTsPath = join(skillDir, "script.ts");
+    scriptModule = (await import(scriptTsPath)) as typeof scriptModule;
+  }
+
+  if (typeof scriptModule.execute !== "function") {
+    throw new Error(
+      `Skill '${manifest.name}': script must export an execute() function`
+    );
+  }
+
+  // Step 4: Execute
+  const output = await scriptModule.execute(input);
+  const coldStartMs = performance.now() - t0;
+
+  return { output, manifest, coldStartMs };
+}

package/src/skills/poc-md-loader/vitest.poc.config.ts

@@ -0,0 +1,13 @@
+/**
+ * Vitest config for POC-B tests only — not part of production test suite.
+ * @poc-jetable
+ */
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    environment: "node",
+    include: ["src/skills/poc-md-loader/**/*.test.ts"],
+    globals: false,
+  },
+});

package/src/skills/poc-md-loader/web-search/SKILL.md

@@ -0,0 +1,42 @@
+---
+name: web-search
+description: >
+  Search the web using Brave Search API (primary) or Tavily (fallback).
+  Returns ranked results with title, URL, and snippet. Requires at least
+  one of BRAVE_SEARCH_KEY or TAVILY_API_KEY environment variables.
+allowed-tools: "Bash(curl:*)"
+metadata:
+  version: "1.0.0"
+  category: "search"
+  tags: "brave tavily web-search"
+  model-hint: "sonnet"
+  env-required: "BRAVE_SEARCH_KEY TAVILY_API_KEY"
+  env-mode: "at-least-one"
+---
+
+# web-search
+
+Searches the web and returns structured results.
+
+## Input
+
+| Field   | Type    | Required | Default | Description              |
+|---------|---------|----------|---------|--------------------------|
+| `query` | string  | yes      | —       | Search query (1-512 chars) |
+| `limit` | integer | no       | 5       | Max results (1-20)       |
+
+## Output
+
+```json
+{
+  "results": [
+    { "title": "...", "url": "...", "snippet": "..." }
+  ],
+  "provider": "brave | tavily | replay"
+}
+```
+
+## Execution
+
+See `script.ts` for the execution logic. The script implements the same
+Brave → Tavily fallback chain as the original `web-search.ts` TS skill.

package/src/skills/poc-md-loader/web-search/script.ts

@@ -0,0 +1,109 @@
+/**
+ * web-search execution script — POC MD loader variant.
+ *
+ * Identical logic to web-search.ts but as a standalone function
+ * with no SDK imports (no Skill<> interface, no withSkillSpan, no errors.ts).
+ * This measures the "pure script" execution path of the SKILL.md pattern.
+ *
+ * @poc-jetable
+ */
+
+export interface WebSearchInput {
+  query: string;
+  limit?: number;
+}
+
+export interface WebSearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
+export interface WebSearchOutput {
+  results: WebSearchResult[];
+  provider: "brave" | "tavily" | "replay";
+}
+
+export class SkillNotConfiguredError extends Error {
+  constructor(missing: string[]) {
+    super(`web-search not configured: missing env ${missing.join(", ")}`);
+    this.name = "SkillNotConfiguredError";
+  }
+}
+
+export class SkillExecutionError extends Error {
+  constructor(message: string) {
+    super(`web-search failed: ${message}`);
+    this.name = "SkillExecutionError";
+  }
+}
+
+/**
+ * Execute web search. Same logic as web-search.ts#webSearch.execute().
+ */
+export async function execute(input: WebSearchInput): Promise<WebSearchOutput> {
+  const query = input.query;
+  const limit = input.limit ?? 5;
+
+  const braveKey = process.env.BRAVE_SEARCH_KEY;
+  const tavilyKey = process.env.TAVILY_API_KEY;
+
+  if (!braveKey && !tavilyKey) {
+    throw new SkillNotConfiguredError(["BRAVE_SEARCH_KEY", "TAVILY_API_KEY"]);
+  }
+
+  if (braveKey) {
+    const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(
+      query
+    )}&count=${limit}`;
+    const res = await fetch(url, {
+      headers: {
+        Accept: "application/json",
+        "X-Subscription-Token": braveKey,
+      },
+    });
+    if (res.ok) {
+      const data = (await res.json()) as {
+        web?: {
+          results?: Array<{
+            title?: string;
+            url?: string;
+            description?: string;
+          }>;
+        };
+      };
+      const results: WebSearchResult[] = (data.web?.results ?? []).map((r) => ({
+        title: r.title ?? "",
+        url: r.url ?? "",
+        snippet: r.description ?? "",
+      }));
+      return { results, provider: "brave" };
+    }
+    if (!tavilyKey) {
+      throw new SkillExecutionError(`brave ${res.status}`);
+    }
+  }
+
+  // Tavily fallback
+  const res = await fetch("https://api.tavily.com/search", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      api_key: tavilyKey,
+      query,
+      max_results: limit,
+    }),
+  });
+  if (!res.ok) {
+    throw new SkillExecutionError(`tavily ${res.status}`);
+  }
+  const data = (await res.json()) as {
+    results?: Array<{ title?: string; url?: string; content?: string }>;
+  };
+  const results: WebSearchResult[] = (data.results ?? []).map((r) => ({
+    title: r.title ?? "",
+    url: r.url ?? "",
+    snippet: r.content ?? "",
+  }));
+  return { results, provider: "tavily" };
+}

package/src/skills/send-email.ts

@@ -29,6 +29,18 @@ export interface SendEmailOutput {
   message_id: string | null;
 }
 
+/**
+ * Prefer ctx.secrets (audit-trail-bearing) over process.env. When no secrets
+ * accessor is configured, falls back to env so legacy callers see no behavior
+ * change. When the secret is absent in both, returns undefined.
+ */
+function readSecret(ctx: SkillContext, name: string): string | undefined {
+  if (ctx.secrets) {
+    if (ctx.secrets.has(name)) return ctx.secrets.get(name);
+  }
+  return process.env[name];
+}
+
 export const sendEmail: Skill<SendEmailInput, SendEmailOutput> = {
   name: "send_email",
   inputSchema,
@@ -39,9 +51,9 @@ export const sendEmail: Skill<SendEmailInput, SendEmailOutput> = {
       return { delivered: false, provider: "replay", message_id: null };
     }
     return withSkillSpan("send_email", async () => {
-      const resendKey = process.env.RESEND_API_KEY;
-      const smtpUrl = process.env.SMTP_URL;
-      const defaultFrom = process.env.EMAIL_FROM ?? input.from;
+      const resendKey = readSecret(ctx, "RESEND_API_KEY");
+      const smtpUrl = readSecret(ctx, "SMTP_URL");
+      const defaultFrom = readSecret(ctx, "EMAIL_FROM") ?? input.from;
       if (!resendKey && !smtpUrl) {
         throw new SkillNotConfiguredError("send_email", [
           "RESEND_API_KEY",