okengine 0.1.6 → 0.2.2

package/src/mcp/docs-index.ts

@@ -0,0 +1,252 @@
+/**
+ * Read-only docs index for the docs MCP surface.
+ *
+ * Loads generated pages under `site/content/docs` (same tree as the docs site).
+ * Bodies are YAML-frontmatter-stripped so `oke.docs.get` matches Copy Markdown.
+ */
+
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+
+/** One indexed documentation page. */
+export interface DocsPage {
+  /** URL slug under `/docs` (`""` for the index page). */
+  readonly slug: string;
+  /** Content-relative path (e.g. `get-started/introduction.mdx`). */
+  readonly path: string;
+  /** Frontmatter title. */
+  readonly title: string;
+  /** Frontmatter description when present. */
+  readonly description: string;
+  /** Docs site URL (`/docs/...`). */
+  readonly url: string;
+  /** Markdown body with YAML frontmatter removed. */
+  readonly body: string;
+}
+
+/** Search hit returned by {@link DocsIndex.search}. */
+export interface DocsSearchHit {
+  readonly slug: string;
+  readonly title: string;
+  readonly url: string;
+  readonly excerpt: string;
+}
+
+/** In-memory docs catalogue. */
+export interface DocsIndex {
+  /** All loaded pages. */
+  readonly pages: readonly DocsPage[];
+  /**
+   * Look up a page by slug or content path.
+   *
+   * @param id - Slug (`get-started/introduction`) or path (`…/introduction.mdx`)
+   */
+  readonly get: (id: string) => DocsPage | null;
+  /**
+   * Case-insensitive search over title, description, and body.
+   *
+   * @param query - Search string
+   * @param limit - Max hits (default 20)
+   */
+  readonly search: (query: string, limit?: number) => readonly DocsSearchHit[];
+}
+
+/**
+ * Strip a leading YAML frontmatter block (`---` … `---`).
+ *
+ * @param raw - Full file contents
+ */
+export function stripYamlFrontmatter(raw: string): string {
+  const normalized = raw.replace(/\r\n/g, "\n");
+  if (!normalized.startsWith("---\n")) {
+    return normalized;
+  }
+  const end = normalized.indexOf("\n---\n", 4);
+  if (end < 0) {
+    return normalized;
+  }
+  return normalized.slice(end + "\n---\n".length).replace(/^\n/, "");
+}
+
+/**
+ * Parse `title` / `description` from a YAML frontmatter block.
+ *
+ * @param raw - Full file contents
+ */
+export function parseDocsFrontmatter(raw: string): {
+  readonly title: string;
+  readonly description: string;
+} {
+  const normalized = raw.replace(/\r\n/g, "\n");
+  if (!normalized.startsWith("---\n")) {
+    return { title: "", description: "" };
+  }
+  const end = normalized.indexOf("\n---\n", 4);
+  if (end < 0) {
+    return { title: "", description: "" };
+  }
+  const block = normalized.slice(4, end);
+  let title = "";
+  let description = "";
+  for (const line of block.split("\n")) {
+    const titleMatch = /^title:\s*(.*)$/.exec(line);
+    if (titleMatch) {
+      title = unquoteYaml(titleMatch[1] ?? "");
+      continue;
+    }
+    const descMatch = /^description:\s*(.*)$/.exec(line);
+    if (descMatch) {
+      description = unquoteYaml(descMatch[1] ?? "");
+    }
+  }
+  return { title, description };
+}
+
+/**
+ * @param value - YAML scalar (optionally quoted)
+ */
+function unquoteYaml(value: string): string {
+  const trimmed = value.trim();
+  if (
+    (trimmed.startsWith('"') && trimmed.endsWith('"')) ||
+    (trimmed.startsWith("'") && trimmed.endsWith("'"))
+  ) {
+    try {
+      return JSON.parse(trimmed.startsWith("'")
+        ? `"${trimmed.slice(1, -1).replace(/"/g, '\\"')}"`
+        : trimmed) as string;
+    } catch {
+      return trimmed.slice(1, -1);
+    }
+  }
+  return trimmed;
+}
+
+/**
+ * Default absolute path to `site/content/docs` from the monorepo root.
+ */
+export function defaultDocsContentDir(): string {
+  return join(import.meta.dir, "..", "..", "site", "content", "docs");
+}
+
+/**
+ * Load every `.md` / `.mdx` file under a docs content directory into an index.
+ *
+ * @param contentDir - Absolute `content/docs` directory
+ */
+export async function loadDocsIndex(
+  contentDir: string = defaultDocsContentDir(),
+): Promise<DocsIndex> {
+  const files = await listDocsSourceFiles(contentDir);
+  const pages: DocsPage[] = [];
+
+  for (const abs of files) {
+    const relative = abs.slice(contentDir.length + 1).replaceAll("\\", "/");
+    const raw = await Bun.file(abs).text();
+    const { title, description } = parseDocsFrontmatter(raw);
+    const body = stripYamlFrontmatter(raw);
+    const slug = pathToSlug(relative);
+    pages.push({
+      slug,
+      path: relative,
+      title: title || slug || "Documentation",
+      description,
+      url: slug.length === 0 ? "/docs" : `/docs/${slug}`,
+      body,
+    });
+  }
+
+  pages.sort((a, b) => a.slug.localeCompare(b.slug));
+
+  const bySlug = new Map(pages.map((p) => [p.slug, p]));
+  const byPath = new Map(pages.map((p) => [p.path, p]));
+
+  return {
+    pages,
+    get(id) {
+      const normalized = id
+        .replace(/^\/docs\/?/, "")
+        .replace(/^\//, "")
+        .replace(/\/$/, "");
+      if (bySlug.has(normalized)) return bySlug.get(normalized) ?? null;
+      if (byPath.has(normalized)) return byPath.get(normalized) ?? null;
+      if (byPath.has(`${normalized}.mdx`)) {
+        return byPath.get(`${normalized}.mdx`) ?? null;
+      }
+      if (byPath.has(`${normalized}.md`)) {
+        return byPath.get(`${normalized}.md`) ?? null;
+      }
+      if (normalized === "docs" || normalized === "") {
+        return bySlug.get("") ?? null;
+      }
+      return null;
+    },
+    search(query, limit = 20) {
+      const q = query.trim().toLowerCase();
+      if (q.length === 0) return [];
+      const hits: DocsSearchHit[] = [];
+      for (const page of pages) {
+        const hay = `${page.title}\n${page.description}\n${page.body}`.toLowerCase();
+        const idx = hay.indexOf(q);
+        if (idx < 0) continue;
+        hits.push({
+          slug: page.slug,
+          title: page.title,
+          url: page.url,
+          excerpt: excerptAround(page.body, q, idx),
+        });
+        if (hits.length >= limit) break;
+      }
+      return hits;
+    },
+  };
+}
+
+/**
+ * @param relativePath - e.g. `get-started/introduction.mdx`
+ */
+function pathToSlug(relativePath: string): string {
+  const noExt = relativePath.replace(/\.mdx?$/i, "");
+  if (noExt === "index") return "";
+  if (noExt.endsWith("/index")) {
+    return noExt.slice(0, -"/index".length);
+  }
+  return noExt;
+}
+
+/**
+ * @param dir - Absolute directory
+ */
+async function listDocsSourceFiles(dir: string): Promise<string[]> {
+  const out: string[] = [];
+  const entries = await readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const abs = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      out.push(...(await listDocsSourceFiles(abs)));
+    } else if (
+      entry.isFile() &&
+      (entry.name.endsWith(".md") || entry.name.endsWith(".mdx"))
+    ) {
+      out.push(abs);
+    }
+  }
+  return out;
+}
+
+/**
+ * @param body - Page body
+ * @param query - Lowercase query
+ * @param idx - Match index in lowercased haystack (approx)
+ */
+function excerptAround(body: string, query: string, _idx: number): string {
+  const lower = body.toLowerCase();
+  const at = lower.indexOf(query);
+  if (at < 0) {
+    return body.slice(0, 160).trim();
+  }
+  const start = Math.max(0, at - 60);
+  const end = Math.min(body.length, at + query.length + 100);
+  const slice = body.slice(start, end).replace(/\s+/g, " ").trim();
+  return `${start > 0 ? "…" : ""}${slice}${end < body.length ? "…" : ""}`;
+}

package/src/mcp/docs-mcp.test.ts

@@ -0,0 +1,176 @@
+/**
+ * Docs MCP acceptance:
+ * - Host/Origin validation (same class as app MCP)
+ * - tools/list exposes only search + get
+ * - oke.docs.get matches on-disk source (frontmatter stripped)
+ * - search finds a known page
+ */
+
+import { describe, expect, test } from "bun:test";
+import { join } from "node:path";
+import { isDataEnvelope } from "./data.ts";
+import {
+  defaultDocsContentDir,
+  loadDocsIndex,
+  stripYamlFrontmatter,
+} from "./docs-index.ts";
+import { createDocsMcpServer } from "./docs-server.ts";
+import { createMcpServer } from "./server.ts";
+import { createSessionStore } from "../auth/sessions.ts";
+
+const CONTENT = defaultDocsContentDir();
+
+function mcpPost(
+  fetch: (request: Request) => Promise<Response>,
+  body: unknown,
+  headers: Record<string, string> = {},
+): Promise<Response> {
+  return fetch(
+    new Request("http://127.0.0.1:6536/mcp", {
+      method: "POST",
+      headers: {
+        host: "127.0.0.1:6536",
+        "content-type": "application/json",
+        ...headers,
+      },
+      body: JSON.stringify(body),
+    }),
+  );
+}
+
+describe("docs MCP Host/Origin", () => {
+  test("rejects attacker Host with 403", async () => {
+    const mcp = await createDocsMcpServer({ contentDir: CONTENT });
+    const res = await mcp.fetch(
+      new Request("http://127.0.0.1:6536/health", {
+        headers: { host: "attacker.example" },
+      }),
+    );
+    expect(res.status).toBe(403);
+    expect(await res.text()).toContain("Host");
+  });
+
+  test("rejects disallowed Origin with 403", async () => {
+    const mcp = await createDocsMcpServer({ contentDir: CONTENT });
+    const res = await mcp.fetch(
+      new Request("http://127.0.0.1:6536/health", {
+        headers: {
+          host: "127.0.0.1:6536",
+          origin: "https://evil.example",
+        },
+      }),
+    );
+    expect(res.status).toBe(403);
+    expect(await res.text()).toContain("Origin");
+  });
+
+  test("allows loopback Host on /health", async () => {
+    const mcp = await createDocsMcpServer({ contentDir: CONTENT });
+    const res = await mcp.fetch(
+      new Request("http://127.0.0.1:6536/health", {
+        headers: { host: "127.0.0.1:6536" },
+      }),
+    );
+    expect(res.status).toBe(200);
+    const json = (await res.json()) as { surface: string };
+    expect(json.surface).toBe("docs-mcp");
+  });
+});
+
+describe("app MCP Host/Origin (behavioral)", () => {
+  test("rejects attacker Host with 403 before auth", async () => {
+    const store = createSessionStore();
+    const mcp = createMcpServer({
+      sessions: store,
+      secret: "test-secret",
+      context: {
+        getManifest: () => null,
+        listRuns: async () => [],
+      },
+      hostname: "127.0.0.1",
+    });
+    const res = await mcp.fetch(
+      new Request("http://127.0.0.1:6535/health", {
+        headers: { host: "attacker.example" },
+      }),
+    );
+    expect(res.status).toBe(403);
+  });
+});
+
+describe("docs MCP tools", () => {
+  test("tools/list exposes only oke.docs.search and oke.docs.get", async () => {
+    const mcp = await createDocsMcpServer({ contentDir: CONTENT });
+    const res = await mcpPost(mcp.fetch, {
+      jsonrpc: "2.0",
+      id: 1,
+      method: "tools/list",
+    });
+    expect(res.status).toBe(200);
+    const json = (await res.json()) as {
+      result: { tools: Array<{ name: string }> };
+    };
+    const names = json.result.tools.map((t) => t.name).sort();
+    expect(names).toEqual(["oke.docs.get", "oke.docs.search"]);
+  });
+
+  test("oke.docs.get returns body byte-identical to on-disk source", async () => {
+    const mcp = await createDocsMcpServer({ contentDir: CONTENT });
+    const slug = "get-started/introduction";
+    const res = await mcpPost(mcp.fetch, {
+      jsonrpc: "2.0",
+      id: 2,
+      method: "tools/call",
+      params: {
+        name: "oke.docs.get",
+        arguments: { slug },
+      },
+    });
+    expect(res.status).toBe(200);
+    const json = (await res.json()) as {
+      result: { structuredContent: unknown };
+    };
+    const envelope = json.result.structuredContent;
+    expect(isDataEnvelope(envelope)).toBe(true);
+    if (!isDataEnvelope(envelope)) return;
+    expect(envelope.provenance).toBe("docs");
+    const content = envelope.content as { body: string; slug: string };
+    expect(content.slug).toBe(slug);
+
+    const raw = await Bun.file(
+      join(CONTENT, "get-started", "introduction.mdx"),
+    ).text();
+    expect(content.body).toBe(stripYamlFrontmatter(raw));
+  });
+
+  test("oke.docs.search finds a known page", async () => {
+    const index = await loadDocsIndex(CONTENT);
+    const mcp = await createDocsMcpServer({ index });
+    const res = await mcpPost(mcp.fetch, {
+      jsonrpc: "2.0",
+      id: 3,
+      method: "tools/call",
+      params: {
+        name: "oke.docs.search",
+        arguments: { query: "eight elements" },
+      },
+    });
+    expect(res.status).toBe(200);
+    const json = (await res.json()) as {
+      result: { structuredContent: unknown };
+    };
+    const envelope = json.result.structuredContent;
+    expect(isDataEnvelope(envelope)).toBe(true);
+    if (!isDataEnvelope(envelope)) return;
+    const content = envelope.content as {
+      hits: Array<{ slug: string; title: string }>;
+    };
+    expect(content.hits.length).toBeGreaterThan(0);
+    expect(
+      content.hits.some(
+        (h) =>
+          h.slug.includes("introduction") || h.slug.includes("elements"),
+      ),
+    ).toBe(true);
+  });
+});

package/src/mcp/docs-server.ts

@@ -0,0 +1,233 @@
+/**
+ * Read-only docs MCP HTTP server on port 6536.
+ *
+ * Host/Origin validation uses the shared {@link checkRequestSecurity}
+ * validator (same rules as the app MCP on :6535). No Bearer auth — this
+ * surface serves public documentation, not a live Manifest.
+ */
+
+import {
+  checkRequestSecurity,
+  forbiddenResponse,
+  resolveAllowedHosts,
+} from "../runtime/security.ts";
+import { DOCS_MCP_PORT, type ServerHandle } from "../runtime/types.ts";
+import { asData } from "./data.ts";
+import {
+  defaultDocsContentDir,
+  loadDocsIndex,
+  type DocsIndex,
+} from "./docs-index.ts";
+import { createDocsToolRuntime } from "./docs-tools.ts";
+import {
+  MCP_PROTOCOL_VERSION,
+  parseJsonRpcRequest,
+  parseToolsCallParams,
+  rpcError,
+  rpcSuccess,
+  RpcErrorCode,
+  type JsonRpcId,
+  type McpInitializeResult,
+} from "./protocol.ts";
+import { newMcpTransportSessionId } from "./session.ts";
+
+/** Options for {@link createDocsMcpServer} / {@link serveDocsMcp}. */
+export interface CreateDocsMcpServerOptions {
+  /** Preloaded index (tests). When omitted, load from {@link contentDir}. */
+  readonly index?: DocsIndex;
+  /** Docs content directory. Defaults to `site/content/docs`. */
+  readonly contentDir?: string;
+  readonly version?: string;
+  readonly allowedHosts?: readonly string[];
+  readonly hostname?: string;
+}
+
+/** In-process docs MCP handler (no listen). */
+export interface DocsMcpServer {
+  /** Handle one HTTP request. */
+  readonly fetch: (request: Request) => Promise<Response>;
+  /** Bound docs index. */
+  readonly index: DocsIndex;
+  /** Tool runtime (tests). */
+  readonly tools: ReturnType<typeof createDocsToolRuntime>;
+}
+
+/**
+ * Create a docs MCP request handler without listening.
+ *
+ * @param options - Index / security options
+ */
+export async function createDocsMcpServer(
+  options: CreateDocsMcpServerOptions = {},
+): Promise<DocsMcpServer> {
+  const index =
+    options.index ??
+    (await loadDocsIndex(options.contentDir ?? defaultDocsContentDir()));
+  const tools = createDocsToolRuntime(index);
+  const hostname = options.hostname ?? "127.0.0.1";
+  const allowed = resolveAllowedHosts(hostname, options.allowedHosts);
+  const version = options.version ?? "0.0.19";
+  const transportSessions = new Set<string>();
+
+  const fetch = async (request: Request): Promise<Response> => {
+    const security = checkRequestSecurity(request, allowed);
+    if (!security.ok) return forbiddenResponse(security.reason);
+
+    const url = new URL(request.url);
+    if (request.method === "GET" && url.pathname === "/health") {
+      return Response.json({ ok: true, surface: "docs-mcp" });
+    }
+
+    if (request.method !== "POST" || url.pathname !== "/mcp") {
+      return new Response("Not Found", { status: 404 });
+    }
+
+    let body: unknown;
+    try {
+      body = await request.json();
+    } catch {
+      return jsonRpcHttp(
+        rpcError(null, RpcErrorCode.parse, "invalid JSON body"),
+        400,
+      );
+    }
+
+    const parsed = parseJsonRpcRequest(body);
+    if (!parsed.ok) {
+      return jsonRpcHttp(
+        rpcError(null, RpcErrorCode.invalidRequest, parsed.message),
+        400,
+      );
+    }
+
+    const { request: rpc } = parsed;
+    const id: JsonRpcId = rpc.id;
+
+    switch (rpc.method) {
+      case "initialize": {
+        const sessionId = newMcpTransportSessionId();
+        transportSessions.add(sessionId);
+        const result: McpInitializeResult = {
+          protocolVersion: MCP_PROTOCOL_VERSION,
+          capabilities: { tools: { listChanged: false } },
+          serverInfo: { name: "okengine-docs-mcp", version },
+          sessionId,
+        };
+        return jsonRpcHttp(rpcSuccess(id, result));
+      }
+      case "ping":
+        return jsonRpcHttp(rpcSuccess(id, { ok: true }));
+      case "tools/list": {
+        const listed = tools.listTools().map((t) => ({
+          name: t.name,
+          description: t.description,
+          inputSchema: t.inputSchema,
+          annotations: {
+            readOnlyHint: true,
+            destructiveHint: false,
+          },
+        }));
+        return jsonRpcHttp(
+          rpcSuccess(id, {
+            tools: listed,
+            _oke: asData({ count: listed.length }, "catalog"),
+          }),
+        );
+      }
+      case "tools/call": {
+        const call = parseToolsCallParams(rpc.params);
+        if (!call.ok) {
+          return jsonRpcHttp(
+            rpcError(id, RpcErrorCode.invalidParams, call.message),
+            400,
+          );
+        }
+        const result = tools.callTool(call.name, call.arguments);
+        if (!result.ok) {
+          const code =
+            result.code === "not-found"
+              ? RpcErrorCode.methodNotFound
+              : RpcErrorCode.invalidParams;
+          return jsonRpcHttp(
+            rpcError(id, code, result.message, result.data),
+            result.code === "not-found" ? 404 : 400,
+          );
+        }
+        return jsonRpcHttp(
+          rpcSuccess(id, {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify(result.data),
+              },
+            ],
+            structuredContent: result.data,
+            isError: false,
+          }),
+        );
+      }
+      default:
+        return jsonRpcHttp(
+          rpcError(
+            id,
+            RpcErrorCode.methodNotFound,
+            `method not found: ${rpc.method}`,
+          ),
+          404,
+        );
+    }
+  };
+
+  return { fetch, index, tools };
+}
+
+/** Options for {@link serveDocsMcp}. */
+export interface ServeDocsMcpOptions extends CreateDocsMcpServerOptions {
+  readonly port?: number;
+}
+
+/** Running docs MCP server handle. */
+export interface DocsMcpServerHandle extends ServerHandle {
+  readonly mcp: DocsMcpServer;
+}
+
+/**
+ * Listen on port 6536 (or override) with Host/Origin validation.
+ *
+ * @param options - Server options
+ */
+export async function serveDocsMcp(
+  options: ServeDocsMcpOptions = {},
+): Promise<DocsMcpServerHandle> {
+  const hostname = options.hostname ?? "127.0.0.1";
+  const port = options.port ?? DOCS_MCP_PORT;
+  const mcp = await createDocsMcpServer({ ...options, hostname });
+  const server = Bun.serve({
+    hostname,
+    port,
+    fetch: mcp.fetch,
+  });
+  const boundPort = server.port ?? port;
+  const boundHost = hostname;
+  const url = new URL(`http://${boundHost}:${boundPort}/`);
+  return {
+    mcp,
+    url,
+    port: boundPort,
+    hostname: boundHost,
+    fetch: mcp.fetch,
+    stop(closeActive = false) {
+      server.stop(closeActive);
+    },
+  };
+}
+
+function jsonRpcHttp(body: unknown, status = 200): Response {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: {
+      "content-type": "application/json; charset=utf-8",
+      "cache-control": "no-store",
+    },
+  });
+}