@nkmc/gateway 0.1.0

package/src/onboard/types.ts

@@ -0,0 +1,72 @@
+import type { HttpAuth } from "@nkmc/agent-fs";
+
+export interface RpcManifestDef {
+  url: string;
+  convention?: "crud" | "evm" | "raw";
+  methods: Array<{
+    rpcMethod: string;
+    description: string;
+    resource?: string;
+    fsOp?: "list" | "read" | "write" | "create" | "remove" | "search";
+  }>;
+}
+
+/** A single service to onboard */
+export interface ManifestEntry {
+  domain: string;
+  /** OpenAPI spec URL — triggers compilation */
+  specUrl?: string;
+  /** skill.md URL — fetched and registered directly */
+  skillMdUrl?: string;
+  /** Inline skill.md content */
+  skillMd?: string;
+  /** JSON-RPC definition — triggers RPC compilation */
+  rpcDef?: RpcManifestDef;
+  /** Pool credential — values can be "${ENV_VAR}" references */
+  auth?: ManifestAuth;
+  /** Tags for categorization */
+  tags?: string[];
+  /** Skip this entry (default false) */
+  disabled?: boolean;
+}
+
+export interface ManifestAuth {
+  type: "bearer" | "api-key" | "basic" | "oauth2";
+  token?: string;
+  prefix?: string;
+  header?: string;
+  key?: string;
+  username?: string;
+  password?: string;
+  tokenUrl?: string;
+  clientId?: string;
+  clientSecret?: string;
+  scope?: string;
+}
+
+/** Result of onboarding one service */
+export interface OnboardResult {
+  domain: string;
+  status: "ok" | "failed" | "skipped";
+  error?: string;
+  source: "openapi" | "skillmd" | "wellknown" | "jsonrpc" | "none";
+  endpoints: number;
+  resources: number;
+  hasCredentials: boolean;
+  smokeTest?: {
+    ls: boolean;
+    cat: boolean;
+    catEndpoint?: string;
+  };
+  durationMs: number;
+}
+
+/** Summary of a batch onboard run */
+export interface OnboardReport {
+  total: number;
+  ok: number;
+  failed: number;
+  skipped: number;
+  results: OnboardResult[];
+  durationMs: number;
+}

package/src/proxy/__tests__/tool-registry.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect } from "vitest";
+import type { HttpAuth } from "@nkmc/agent-fs";
+import {
+  ToolRegistry,
+  createDefaultToolRegistry,
+} from "../tool-registry.js";
+
+describe("ToolRegistry", () => {
+  it("resolves a known tool", () => {
+    const registry = createDefaultToolRegistry();
+    const gh = registry.get("gh");
+
+    expect(gh).not.toBeNull();
+    expect(gh!.name).toBe("gh");
+    expect(gh!.credentialDomain).toBe("github.com");
+  });
+
+  it("returns null for an unknown tool", () => {
+    const registry = createDefaultToolRegistry();
+    expect(registry.get("nonexistent")).toBeNull();
+  });
+
+  it("lists all registered tools", () => {
+    const registry = createDefaultToolRegistry();
+    const tools = registry.list();
+
+    expect(tools.length).toBe(5);
+
+    const names = tools.map((t) => t.name).sort();
+    expect(names).toEqual(["anthropic", "aws", "gh", "openai", "stripe"]);
+  });
+
+  it("builds env vars from bearer credential (gh → GH_TOKEN)", () => {
+    const registry = createDefaultToolRegistry();
+    const gh = registry.get("gh")!;
+
+    const auth: HttpAuth = { type: "bearer", token: "ghp_abc123" };
+    const env = registry.buildEnv(gh, auth);
+
+    expect(env).toEqual({ GH_TOKEN: "ghp_abc123" });
+  });
+
+  it("builds env vars for api-key auth type (stripe → STRIPE_API_KEY)", () => {
+    const registry = createDefaultToolRegistry();
+    const stripe = registry.get("stripe")!;
+
+    const auth: HttpAuth = {
+      type: "api-key",
+      header: "Authorization",
+      key: "sk_test_xyz",
+    };
+    const env = registry.buildEnv(stripe, auth);
+
+    expect(env).toEqual({ STRIPE_API_KEY: "sk_test_xyz" });
+  });
+
+  it("builds env vars for basic auth (aws → ACCESS_KEY_ID + SECRET)", () => {
+    const registry = createDefaultToolRegistry();
+    const aws = registry.get("aws")!;
+
+    const auth: HttpAuth = {
+      type: "basic",
+      username: "AKIAIOSFODNN7EXAMPLE",
+      password: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    };
+    const env = registry.buildEnv(aws, auth);
+
+    expect(env).toEqual({
+      AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE",
+      AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    });
+  });
+
+  it("omits env vars when auth type does not match requested field", () => {
+    const registry = new ToolRegistry();
+    registry.register({
+      name: "test",
+      credentialDomain: "example.com",
+      envMapping: { MY_TOKEN: "token" },
+    });
+    const tool = registry.get("test")!;
+
+    // Pass basic auth but tool wants "token" → should get empty env
+    const auth: HttpAuth = {
+      type: "basic",
+      username: "user",
+      password: "pass",
+    };
+    const env = registry.buildEnv(tool, auth);
+
+    expect(env).toEqual({});
+  });
+});

package/src/proxy/tool-registry.ts

@@ -0,0 +1,122 @@
+import type { HttpAuth } from "@nkmc/agent-fs";
+
+/** Field names that can be extracted from an HttpAuth credential. */
+export type AuthField = "token" | "key" | "username" | "password";
+
+/**
+ * Defines how a CLI tool maps to a credential domain and which
+ * environment variables should be injected at runtime.
+ */
+export interface ToolDefinition {
+  /** CLI tool name, e.g. "gh", "stripe" */
+  name: string;
+  /** Domain used to look up credentials in the vault */
+  credentialDomain: string;
+  /** Maps env var name → field to extract from HttpAuth */
+  envMapping: Record<string, AuthField>;
+}
+
+/**
+ * Registry of CLI tools that can be proxied through the gateway.
+ * Each tool declares the credential domain it needs and how to
+ * translate stored credentials into environment variables.
+ */
+export class ToolRegistry {
+  private tools = new Map<string, ToolDefinition>();
+
+  /** Register a tool definition. Overwrites any existing entry with the same name. */
+  register(tool: ToolDefinition): void {
+    this.tools.set(tool.name, tool);
+  }
+
+  /** Look up a tool by name. Returns null if not found. */
+  get(name: string): ToolDefinition | null {
+    return this.tools.get(name) ?? null;
+  }
+
+  /** Return all registered tool definitions. */
+  list(): ToolDefinition[] {
+    return [...this.tools.values()];
+  }
+
+  /**
+   * Build a record of environment variables for the given tool by
+   * extracting the requested fields from the HttpAuth credential.
+   *
+   * If the auth type does not contain the requested field (e.g.
+   * requesting "token" from a basic-auth credential), that env var
+   * is silently omitted.
+   */
+  buildEnv(tool: ToolDefinition, auth: HttpAuth): Record<string, string> {
+    const env: Record<string, string> = {};
+
+    for (const [envVar, field] of Object.entries(tool.envMapping)) {
+      const value = extractField(auth, field);
+      if (value !== undefined) {
+        env[envVar] = value;
+      }
+    }
+
+    return env;
+  }
+}
+
+/** Extract a named field from an HttpAuth credential. */
+function extractField(auth: HttpAuth, field: AuthField): string | undefined {
+  switch (field) {
+    case "token":
+      return auth.type === "bearer" ? auth.token : undefined;
+    case "key":
+      return auth.type === "api-key" ? auth.key : undefined;
+    case "username":
+      return auth.type === "basic" ? auth.username : undefined;
+    case "password":
+      return auth.type === "basic" ? auth.password : undefined;
+    default:
+      return undefined;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Default tools
+// ---------------------------------------------------------------------------
+
+/** Create a ToolRegistry pre-populated with common CLI tools. */
+export function createDefaultToolRegistry(): ToolRegistry {
+  const registry = new ToolRegistry();
+
+  registry.register({
+    name: "gh",
+    credentialDomain: "github.com",
+    envMapping: { GH_TOKEN: "token" },
+  });
+
+  registry.register({
+    name: "stripe",
+    credentialDomain: "api.stripe.com",
+    envMapping: { STRIPE_API_KEY: "key" },
+  });
+
+  registry.register({
+    name: "openai",
+    credentialDomain: "api.openai.com",
+    envMapping: { OPENAI_API_KEY: "key" },
+  });
+
+  registry.register({
+    name: "anthropic",
+    credentialDomain: "api.anthropic.com",
+    envMapping: { ANTHROPIC_API_KEY: "key" },
+  });
+
+  registry.register({
+    name: "aws",
+    credentialDomain: "aws.amazon.com",
+    envMapping: {
+      AWS_ACCESS_KEY_ID: "username",
+      AWS_SECRET_ACCESS_KEY: "password",
+    },
+  });
+
+  return registry;
+}

package/src/proxy.ts

@@ -0,0 +1,12 @@
+export {
+  ToolRegistry,
+  createDefaultToolRegistry,
+  type ToolDefinition,
+  type AuthField,
+} from "./proxy/tool-registry.js";
+
+export {
+  proxyRoutes,
+  type ProxyRouteOptions,
+  type ExecResult,
+} from "./http/routes/proxy.js";

package/src/registry/context7-backend.ts

@@ -0,0 +1,93 @@
+import type { FsBackend } from "@nkmc/agent-fs";
+import { Context7Client, type Context7Options, type LibrarySearchResult } from "./context7.js";
+
+export interface Context7BackendOptions extends Context7Options {}
+
+/**
+ * FsBackend that maps filesystem operations to Context7 documentation queries.
+ *
+ * Filesystem mapping:
+ *   grep "react" /         → searchLibraries("react")  — search for libraries
+ *   cat /{owner}/{repo}    → queryDocs("/{owner}/{repo}", repo) — library overview
+ *   grep "hooks" /{o}/{r}  → queryDocs("/{o}/{r}", "hooks")   — query specific docs
+ *   ls /                   → usage instructions
+ */
+export class Context7Backend implements FsBackend {
+  private client: Context7Client;
+
+  constructor(options?: Context7BackendOptions) {
+    this.client = new Context7Client(options);
+  }
+
+  async list(path: string): Promise<string[]> {
+    const cleaned = path.replace(/^\/+/, "").replace(/\/+$/, "");
+
+    if (!cleaned) {
+      return [
+        'grep "<关键词>" /context7/     — 搜索库',
+        'grep "<问题>" /context7/{id}   — 查询文档',
+        'cat /context7/{owner}/{repo}   — 库概览',
+      ];
+    }
+
+    // ls /{owner}/{repo}/ — not much to list, return hint
+    return ['grep "<问题>" /context7/' + cleaned + " — 查询此库文档"];
+  }
+
+  async read(path: string): Promise<unknown> {
+    const libraryId = parseLibraryId(path);
+    if (!libraryId) {
+      return { usage: 'grep "<关键词>" /context7/ — 搜索库' };
+    }
+
+    // cat /{owner}/{repo} → query overview
+    const name = libraryId.split("/").pop() ?? libraryId;
+    const docs = await this.client.queryDocs(libraryId, `${name} overview getting started`);
+    return { libraryId, docs };
+  }
+
+  async write(_path: string, _data: unknown): Promise<{ id: string }> {
+    throw new Error("context7 is read-only");
+  }
+
+  async remove(_path: string): Promise<void> {
+    throw new Error("context7 is read-only");
+  }
+
+  async search(path: string, pattern: string): Promise<unknown[]> {
+    const cleaned = path.replace(/^\/+/, "").replace(/\/+$/, "");
+
+    if (!cleaned) {
+      // grep at root → search libraries
+      const results = await this.client.searchLibraries(pattern);
+      return results.map(formatSearchResult);
+    }
+
+    // grep at /{owner}/{repo} → query docs
+    const libraryId = parseLibraryId(path);
+    if (!libraryId) return [];
+
+    const docs = await this.client.queryDocs(libraryId, pattern);
+    if (!docs) return [];
+    return [{ libraryId, query: pattern, docs }];
+  }
+}
+
+function parseLibraryId(path: string): string | null {
+  const cleaned = path.replace(/^\/+/, "").replace(/\/+$/, "");
+  if (!cleaned) return null;
+
+  // Expect owner/repo format
+  const parts = cleaned.split("/");
+  if (parts.length < 2) return null;
+  return "/" + parts.slice(0, 2).join("/");
+}
+
+function formatSearchResult(r: LibrarySearchResult): Record<string, unknown> {
+  return {
+    id: r.id,
+    name: r.name,
+    description: r.description ?? "",
+    snippets: r.totalSnippets ?? 0,
+  };
+}

package/src/registry/context7.ts

@@ -0,0 +1,54 @@
+export interface Context7Options {
+  apiKey?: string;
+  baseUrl?: string;
+  fetchFn?: typeof globalThis.fetch;
+}
+
+export interface LibrarySearchResult {
+  id: string;
+  name: string;
+  description?: string;
+  totalSnippets?: number;
+  trustScore?: number;
+}
+
+export class Context7Client {
+  private apiKey?: string;
+  private baseUrl: string;
+  private fetchFn: typeof globalThis.fetch;
+
+  constructor(options?: Context7Options) {
+    this.apiKey = options?.apiKey;
+    this.baseUrl = options?.baseUrl ?? "https://context7.com/api/v2";
+    this.fetchFn = options?.fetchFn ?? globalThis.fetch.bind(globalThis);
+  }
+
+  /** Search for a library by name. Returns matching library entries. */
+  async searchLibraries(libraryName: string, query?: string): Promise<LibrarySearchResult[]> {
+    const params = new URLSearchParams({ libraryName });
+    if (query) params.set("query", query);
+
+    const resp = await this.fetchFn(`${this.baseUrl}/libs/search?${params}`, {
+      headers: this.headers(),
+    });
+    if (!resp.ok) throw new Error(`Context7 search failed: ${resp.status}`);
+    return resp.json() as Promise<LibrarySearchResult[]>;
+  }
+
+  /** Query documentation for a specific library. Returns documentation text. */
+  async queryDocs(libraryId: string, query: string): Promise<string> {
+    const params = new URLSearchParams({ libraryId, query, type: "txt" });
+
+    const resp = await this.fetchFn(`${this.baseUrl}/context?${params}`, {
+      headers: this.headers(),
+    });
+    if (!resp.ok) throw new Error(`Context7 query failed: ${resp.status}`);
+    return resp.text();
+  }
+
+  private headers(): Record<string, string> {
+    const h: Record<string, string> = {};
+    if (this.apiKey) h["Authorization"] = `Bearer ${this.apiKey}`;
+    return h;
+  }
+}

package/src/registry/d1-store.ts

@@ -0,0 +1,242 @@
+import type { D1Database } from "../d1/types.js";
+import type {
+  RegistryStore,
+  RegistryStats,
+  SearchResult,
+  ServiceRecord,
+  ServiceSummary,
+  VersionSummary,
+} from "./types.js";
+
+const CREATE_SERVICES = `
+CREATE TABLE IF NOT EXISTS services (
+  domain TEXT NOT NULL,
+  version TEXT NOT NULL,
+  name TEXT NOT NULL,
+  description TEXT,
+  roles TEXT,
+  skill_md TEXT NOT NULL,
+  endpoints TEXT,
+  is_first_party INTEGER DEFAULT 0,
+  status TEXT DEFAULT 'active',
+  is_default INTEGER DEFAULT 1,
+  source TEXT,
+  sunset_date INTEGER,
+  auth_mode TEXT,
+  created_at INTEGER NOT NULL,
+  updated_at INTEGER NOT NULL,
+  PRIMARY KEY (domain, version)
+)`;
+
+const CREATE_SERVICES_DEFAULT_INDEX = `
+CREATE INDEX IF NOT EXISTS idx_services_default ON services(domain, is_default)`;
+
+const CREATE_DOMAIN_CHALLENGES = `
+CREATE TABLE IF NOT EXISTS domain_challenges (
+  domain TEXT PRIMARY KEY,
+  challenge_code TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending',
+  created_at INTEGER NOT NULL,
+  verified_at INTEGER,
+  expires_at INTEGER NOT NULL
+)`;
+
+interface ServiceRow {
+  domain: string;
+  version: string;
+  name: string;
+  description: string;
+  roles: string;
+  skill_md: string;
+  endpoints: string;
+  is_first_party: number;
+  status: string;
+  is_default: number;
+  source: string | null;
+  sunset_date: number | null;
+  auth_mode: string | null;
+  created_at: number;
+  updated_at: number;
+}
+
+function rowToRecord(row: ServiceRow): ServiceRecord {
+  return {
+    domain: row.domain,
+    name: row.name,
+    description: row.description,
+    version: row.version,
+    roles: JSON.parse(row.roles),
+    skillMd: row.skill_md,
+    endpoints: JSON.parse(row.endpoints),
+    isFirstParty: row.is_first_party === 1,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+    status: row.status as ServiceRecord["status"],
+    isDefault: row.is_default === 1,
+    ...(row.source ? { source: JSON.parse(row.source) } : {}),
+    ...(row.sunset_date ? { sunsetDate: row.sunset_date } : {}),
+    ...(row.auth_mode ? { authMode: row.auth_mode as ServiceRecord["authMode"] } : {}),
+  };
+}
+
+function toSummary(row: ServiceRow): ServiceSummary {
+  return {
+    domain: row.domain,
+    name: row.name,
+    description: row.description,
+    isFirstParty: row.is_first_party === 1,
+  };
+}
+
+export class D1RegistryStore implements RegistryStore {
+  constructor(private db: D1Database) {}
+
+  async initSchema(): Promise<void> {
+    await this.db.exec(CREATE_SERVICES);
+    await this.db.exec(CREATE_SERVICES_DEFAULT_INDEX);
+    await this.db.exec(CREATE_DOMAIN_CHALLENGES);
+  }
+
+  async get(domain: string): Promise<ServiceRecord | null> {
+    const row = await this.db
+      .prepare("SELECT * FROM services WHERE domain = ? AND is_default = 1")
+      .bind(domain)
+      .first<ServiceRow>();
+
+    return row ? rowToRecord(row) : null;
+  }
+
+  async getVersion(domain: string, version: string): Promise<ServiceRecord | null> {
+    const row = await this.db
+      .prepare("SELECT * FROM services WHERE domain = ? AND version = ?")
+      .bind(domain, version)
+      .first<ServiceRow>();
+
+    return row ? rowToRecord(row) : null;
+  }
+
+  async listVersions(domain: string): Promise<VersionSummary[]> {
+    const { results } = await this.db
+      .prepare(
+        "SELECT version, status, is_default, created_at, updated_at FROM services WHERE domain = ? ORDER BY created_at DESC",
+      )
+      .bind(domain)
+      .all<ServiceRow>();
+
+    return results.map((row) => ({
+      version: row.version,
+      status: row.status as ServiceRecord["status"],
+      isDefault: row.is_default === 1,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+    }));
+  }
+
+  /** Max endpoints JSON size before stripping verbose fields (parameters, requestBody, responses). */
+  static readonly ENDPOINTS_SIZE_LIMIT = 800_000; // ~800 KB, well under D1's ~1 MB per-value limit
+
+  async put(domain: string, record: ServiceRecord): Promise<void> {
+    let endpointsJson = JSON.stringify(record.endpoints);
+
+    // Only strip verbose fields when the full JSON exceeds the size limit.
+    // Small APIs keep parameters/requestBody/responses for the explore detail page.
+    if (endpointsJson.length > D1RegistryStore.ENDPOINTS_SIZE_LIMIT) {
+      const slim = record.endpoints.map(({ method, path, description, price, pricing }) => ({
+        method,
+        path,
+        description,
+        ...(price ? { price } : {}),
+        ...(pricing ? { pricing } : {}),
+      }));
+      endpointsJson = JSON.stringify(slim);
+    }
+
+    await this.db
+      .prepare(
+        `INSERT OR REPLACE INTO services
+         (domain, version, name, description, roles, skill_md, endpoints, is_first_party, status, is_default, source, sunset_date, auth_mode, created_at, updated_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      )
+      .bind(
+        domain,
+        record.version,
+        record.name,
+        record.description,
+        JSON.stringify(record.roles),
+        record.skillMd,
+        endpointsJson,
+        record.isFirstParty ? 1 : 0,
+        record.status,
+        record.isDefault ? 1 : 0,
+        record.source ? JSON.stringify(record.source) : null,
+        record.sunsetDate ?? null,
+        record.authMode ?? null,
+        record.createdAt,
+        record.updatedAt,
+      )
+      .run();
+  }
+
+  async delete(domain: string): Promise<void> {
+    await this.db
+      .prepare("DELETE FROM services WHERE domain = ?")
+      .bind(domain)
+      .run();
+  }
+
+  async list(): Promise<ServiceSummary[]> {
+    const { results } = await this.db
+      .prepare("SELECT domain, name, description, is_first_party FROM services WHERE is_default = 1")
+      .all<ServiceRow>();
+
+    return results.map(toSummary);
+  }
+
+  async search(query: string): Promise<SearchResult[]> {
+    const pattern = `%${query}%`;
+    const { results: rows } = await this.db
+      .prepare(
+        `SELECT * FROM services
+         WHERE is_default = 1 AND (name LIKE ? OR description LIKE ? OR endpoints LIKE ?)`,
+      )
+      .bind(pattern, pattern, pattern)
+      .all<ServiceRow>();
+
+    const q = query.toLowerCase();
+    return rows.map((row) => {
+      const endpoints = JSON.parse(row.endpoints) as Array<{
+        method: string;
+        path: string;
+        description: string;
+      }>;
+      const matched = endpoints.filter(
+        (e) =>
+          e.description.toLowerCase().includes(q) ||
+          e.method.toLowerCase().includes(q) ||
+          e.path.toLowerCase().includes(q),
+      );
+      return {
+        ...toSummary(row),
+        matchedEndpoints: matched.map((e) => ({
+          method: e.method,
+          path: e.path,
+          description: e.description,
+        })),
+      };
+    });
+  }
+
+  async stats(): Promise<RegistryStats> {
+    const row = await this.db
+      .prepare(
+        `SELECT COUNT(*) as service_count, COALESCE(SUM(json_array_length(endpoints)), 0) as endpoint_count
+         FROM services WHERE is_default = 1`,
+      )
+      .first<{ service_count: number; endpoint_count: number }>();
+
+    return {
+      serviceCount: row?.service_count ?? 0,
+      endpointCount: row?.endpoint_count ?? 0,
+    };
+  }
+}