@corsenai/corsen-context 1.2.0 → 2.0.0

package/README.md

@@ -1,27 +1,88 @@
-# @corsenai/corsen-context
-
-The core engine of **[Corsen Context](https://github.com/CorsenAI/corsen-context)** — the Universal AI Context Layer that makes any website AI-native with an MCP server + enhanced `llms.txt`.
-
-```bash
-npm install @corsenai/corsen-context
-```
-
-```typescript
-import { CorsenContext, createInMemoryProvider } from '@corsenai/corsen-context';
-
-const cc = new CorsenContext(
-  { siteUrl: 'https://example.com' },
-  createInMemoryProvider([
-    { url: 'https://example.com/', title: 'Home', description: 'Welcome', markdown: '# Home', metadata: {}, type: 'page' },
-  ]),
-);
-
-const server = cc.createMCPServer();
-const result = await server.handleRequest(requestBody, clientIp);
-```
-
-This package provides the framework-agnostic core: the MCP JSON-RPC 2.0 server, `llms.txt` generators, HTML→Markdown converter, SSRF-safe fetching, rate limiting, caching, and the content-access policy.
-
-- **Full docs:** https://github.com/CorsenAI/corsen-context#readme
-- **Security model:** [SECURITY.md](https://github.com/CorsenAI/corsen-context/blob/main/SECURITY.md)
-- **License:** MIT
+# @corsenai/corsen-context
+
+The core engine of **[Corsen Context](https://github.com/CorsenAI/corsen-context)** — owner-controlled public content through MCP, WebMCP, and bounded `llms.txt` output.
+
+```bash
+npm install @corsenai/corsen-context
+```
+
+```typescript
+import {
+  CorsenContext,
+  createInMemoryProvider,
+  generateWebMCPScript,
+  toWebMCPTools,
+} from '@corsenai/corsen-context';
+
+const cc = new CorsenContext(
+  { siteUrl: 'https://example.com' },
+  createInMemoryProvider([
+    {
+      url: 'https://example.com/',
+      title: 'Home',
+      description: 'Welcome',
+      markdown: '# Home',
+      metadata: {},
+      type: 'page',
+    },
+  ]),
+);
+
+const server = cc.createMCPServer();
+const result = await server.handleRequest(requestBody, clientIp);
+
+// Serve this string as GET /webmcp.js on the same origin as POST /v1/mcp.
+const webmcpScript = generateWebMCPScript(toWebMCPTools(server.getToolDefinitions()), {
+  mcpEndpoint: '/v1/mcp',
+});
+```
+
+Load `/webmcp.js` from the site's pages with
+`<script src="/webmcp.js" defer></script>`. The bridge registers the four
+tools through `document.modelContext` and forwards calls to the same-origin MCP
+endpoint. It intentionally sends no API key or visitor credentials: use a
+public, read-only, rate-limited endpoint for WebMCP, or omit the browser bridge
+when MCP requires server-side authentication.
+
+This package provides the framework-agnostic core: the MCP JSON-RPC 2.0 server,
+WebMCP bridge generation, `llms.txt` generators, HTML-to-Markdown converter,
+SSRF-safe fetching, rate limiting, caching, and the content-access policy. A
+working site integration must also implement a `ContentProvider` backed only
+by the public content it intends to expose.
+
+The resolved configuration enforces these owner controls and bounds:
+
+```typescript
+const cc = new CorsenContext(
+  {
+    siteUrl: 'https://example.com',
+    mcp: { enabled: true },
+    content: { maxPages: 500 }, // 1–5000
+    static: {
+      generateLlmsTxt: true,
+      includeFullContent: false,
+      maxOutputBytes: 5_242_880, // 64 KiB–10 MiB
+    },
+  },
+  provider,
+);
+```
+
+With `mcp.enabled: false`, direct MCP dispatch is rejected before the provider
+is invoked; adapters also use this value to return `404` from their MCP and
+WebMCP routes. With `static.generateLlmsTxt: false`, both `CorsenContext`
+generation methods refuse generation. Full-content generation additionally
+requires
+`static.includeFullContent: true`, which is off by default. Both static outputs
+are capped at `maxOutputBytes` without splitting a UTF-8 code point and append
+a truncation notice when the complete output would exceed the limit.
+Full-content iteration returns as soon as the next block would exceed the budget.
+
+Static headings, labels, descriptions, dates, and Markdown destinations are
+normalized and escaped. The provider's page-body `markdown` is passed through
+unchanged and remains untrusted site-authored content; this package does not
+claim to neutralize it.
+
+- **Full docs:** https://github.com/CorsenAI/corsen-context#readme
+- **Security model:** [SECURITY.md](https://github.com/CorsenAI/corsen-context/blob/main/SECURITY.md)
+- **License:** MIT

package/dist/index.d.mts

@@ -41,7 +41,7 @@ interface JSONRPCRequest {
     jsonrpc: '2.0';
     method: string;
     params?: Record<string, unknown>;
-    id?: string | number | null;
+    id?: string | number;
 }
 interface JSONRPCResponse {
     jsonrpc: '2.0';
@@ -61,6 +61,7 @@ interface MCPToolDefinition {
         type: 'object';
         properties: Record<string, unknown>;
         required?: string[];
+        additionalProperties?: boolean;
     };
 }
 interface MCPResourceDefinition {
@@ -95,22 +96,50 @@ interface RateLimitStore {
     addTimestamp(key: string, timestamp: number): Promise<void>;
     cleanup(): Promise<void>;
     /**
-     * Optional atomic prune + record + count. When present, the RateLimiter uses
-     * this instead of the getTimestamps()/addTimestamp() pair to avoid the
-     * check-then-add race. Returns counts that INCLUDE the current request.
+     * Optional combined record + count path. When present, the RateLimiter uses
+     * it instead of getTimestamps()/addTimestamp(). Implementations document
+     * whether their backing store makes the whole operation atomic. Returned
+     * counts INCLUDE the current request.
      */
     hit?(key: string, windowStart: number, burstWindowStart: number, now: number): Promise<{
         windowCount: number;
         burstCount: number;
     }>;
 }
-interface RedisClient {
-    get(key: string): Promise<string | null>;
-    set(key: string, value: string, options?: {
-        ex?: number;
-    } | 'EX', ttlSeconds?: number): Promise<unknown>;
+/** Commands required by RedisCache. SET with EX makes value + TTL one atomic write. */
+interface RedisCacheClient {
+    get(key: string): Promise<unknown | null>;
+    set(key: string, value: string, options: {
+        ex: number;
+    }): Promise<unknown>;
+    del(...keys: string[]): Promise<number>;
+}
+/** Numeric sorted-set bound accepted by Redis and the Upstash TypeScript SDK. */
+type RedisScoreBoundary = number | '-inf' | '+inf' | `(${number}`;
+/**
+ * Upstash-compatible sorted-set shape used by RedisRateLimitStore. ioredis
+ * callers can use adaptIORedisClient() to normalize its variadic commands.
+ */
+interface RedisRateLimitClient {
+    expire(key: string, seconds: number): Promise<number | boolean>;
+    zadd(key: string, entry: {
+        score: number;
+        member: string;
+    }): Promise<number | null>;
+    zremrangebyscore(key: string, min: RedisScoreBoundary, max: RedisScoreBoundary): Promise<number>;
+    zcard(key: string): Promise<number>;
+    zrange(key: string, min: RedisScoreBoundary, max: RedisScoreBoundary, options: {
+        byScore: true;
+    }): Promise<string[]>;
+}
+/** Direct @upstash/redis-compatible client contract. */
+interface RedisClient extends RedisCacheClient, RedisRateLimitClient {
+}
+/** Minimal ioredis surface normalized by adaptIORedisClient(). */
+interface IORedisClient {
+    get(key: string): Promise<unknown | null>;
+    set(key: string, value: string, mode: 'EX', ttlSeconds: number): Promise<unknown>;
     del(...keys: string[]): Promise<number>;
-    incr(key: string): Promise<number>;
     expire(key: string, seconds: number): Promise<number | boolean>;
     zadd(key: string, score: number, member: string): Promise<number>;
     zremrangebyscore(key: string, min: number | string, max: number | string): Promise<number>;
@@ -201,12 +230,15 @@ declare const corsenContextConfigSchema: z.ZodObject<{
     static: z.ZodDefault<z.ZodObject<{
         generateLlmsTxt: z.ZodDefault<z.ZodBoolean>;
         includeFullContent: z.ZodDefault<z.ZodBoolean>;
+        maxOutputBytes: z.ZodDefault<z.ZodNumber>;
     }, "strip", z.ZodTypeAny, {
         generateLlmsTxt: boolean;
         includeFullContent: boolean;
+        maxOutputBytes: number;
     }, {
         generateLlmsTxt?: boolean | undefined;
         includeFullContent?: boolean | undefined;
+        maxOutputBytes?: number | undefined;
     }>>;
     security: z.ZodDefault<z.ZodObject<{
         rateLimit: z.ZodDefault<z.ZodNumber>;
@@ -259,6 +291,7 @@ declare const corsenContextConfigSchema: z.ZodObject<{
     static: {
         generateLlmsTxt: boolean;
         includeFullContent: boolean;
+        maxOutputBytes: number;
     };
     security: {
         rateLimit: number;
@@ -293,6 +326,7 @@ declare const corsenContextConfigSchema: z.ZodObject<{
     static?: {
         generateLlmsTxt?: boolean | undefined;
         includeFullContent?: boolean | undefined;
+        maxOutputBytes?: number | undefined;
     } | undefined;
     security?: {
         rateLimit?: number | undefined;
@@ -326,6 +360,7 @@ declare class MCPServer {
     private provider;
     private rateLimiter;
     private cache;
+    private cacheNamespace;
     private log;
     constructor(config: ResolvedConfig, provider: ContentProvider, options?: {
         cache?: CacheDriver;
@@ -334,6 +369,15 @@ declare class MCPServer {
     });
     getSecurityHeaders(): Record<string, string>;
     getCorsHeaders(origin?: string): Record<string, string>;
+    /**
+     * Validate a browser Origin for the Streamable HTTP endpoint.
+     *
+     * Non-browser clients commonly omit Origin and remain accepted. When an
+     * Origin is present, MCP requires validation to prevent DNS rebinding. The
+     * canonical site origin is always allowed; operators can add explicit
+     * browser origins through security.allowedOrigins.
+     */
+    validateRequestOrigin(origin?: string): boolean;
     checkRateLimit(clientIp: string, apiKey?: string): Promise<{
         allowed: boolean;
         headers: Record<string, string>;
@@ -356,36 +400,47 @@ declare class MCPServer {
     private cacheSet;
     /**
      * Drop the cached body for a single page URL. Call this from your CMS's
-     * publish/update/delete hooks so edits and unpublishes propagate before the
-     * TTL expires (otherwise stale content can be served for up to cache.ttl).
+     * publish/update/delete hooks. Aggregate surfaces are intentionally read
+     * through so an unpublished URL is not retained behind an unenumerable key.
      */
     invalidatePage(url: string): Promise<void>;
     /**
-     * Clear all cached MCP responses (search, page, list, sitemap). Call after
-     * bulk content changes. No-op for cache drivers without prefix enumeration
-     * (see RedisCache.clear notes).
+     * Clear all cached page bodies. Cache drivers that cannot prove a complete
+     * purge reject instead of reporting success.
      */
     clearCache(): Promise<void>;
-    searchSite(query: string, limit?: number): Promise<{} | undefined>;
+    searchSite(query: string, limit?: number): Promise<SearchResult[]>;
     getPageContent(uri: string): Promise<{} | null | undefined>;
-    listContent(type: string, page?: number, limit?: number): Promise<{} | undefined>;
-    getSitemap(): Promise<{} | undefined>;
+    listContent(type: string, page?: number, limit?: number): Promise<{
+        items: ContentListItem[];
+        total: number;
+        page: number;
+        limit: number;
+        hasMore: boolean;
+    }>;
+    getSitemap(): Promise<{
+        url: string;
+        title: string;
+        type: string;
+        lastModified: string | undefined;
+    }[]>;
     getToolDefinitions(): MCPToolDefinition[];
     getCapabilities(): MCPCapabilities;
     private successResponse;
+    private toolErrorResponse;
     private errorResponse;
 }
 
 /**
  * Single source of truth for version strings.
  *
- * Keep CORSEN_CONTEXT_VERSION in sync with each package.json `version` on release
- * (the release tooling does this). serverInfo and the CLI both read from here so
- * the version can never silently drift between what is published and what is
- * reported over MCP.
+ * CORSEN_CONTEXT_VERSION is bumped by hand on every release, in the same
+ * commit that updates the package.json versions. serverInfo and the CLI both
+ * read from here so the version can never silently drift between what is
+ * published and what is reported over MCP.
  */
 /** Corsen Context release version. */
-declare const CORSEN_CONTEXT_VERSION = "1.2.0";
+declare const CORSEN_CONTEXT_VERSION = "2.0.0";
 /** MCP protocol version implemented by this server. */
 declare const MCP_PROTOCOL_VERSION = "2025-11-25";
 
@@ -461,6 +516,65 @@ declare function mcpLinkTag(config: DiscoveryConfig): string;
 declare function generateLlmsTxt(config: ResolvedConfig, provider: ContentProvider): Promise<string>;
 declare function generateLlmsFullTxt(config: ResolvedConfig, provider: ContentProvider): Promise<string>;
 
+/**
+ * WebMCP exposes the same tools to an agent running inside the page, through
+ * `document.modelContext`. The browser never reimplements a tool: it receives
+ * the definitions from the server and every `execute()` calls back into the
+ * existing MCP endpoint, so there is one implementation per runtime and one
+ * contract for every transport.
+ *
+ * Spec: https://webmachinelearning.github.io/webmcp/
+ */
+/** Tool annotations defined by the WebMCP `ToolAnnotations` dictionary. */
+interface WebMCPToolAnnotations {
+    /** Tool only reads state. Lets an agent decide when a call is safe. */
+    readOnlyHint: boolean;
+    /** Tool output is untrusted data, from the perspective of this site. */
+    untrustedContentHint: boolean;
+}
+interface WebMCPTool extends MCPToolDefinition {
+    annotations: WebMCPToolAnnotations;
+}
+interface WebMCPScriptConfig {
+    /**
+     * MCP endpoint the browser bridge calls. Defaults to `/v1/mcp`.
+     *
+     * The bridge is deliberately keyless: any credential embedded in a public
+     * page's script is disclosed to every visitor, so a key-protected endpoint
+     * should not enable the WebMCP bridge at all.
+     */
+    mcpEndpoint?: string;
+}
+/**
+ * Every tool Corsen Context exposes reads published site content, so all of
+ * them are read-only and all of them return untrusted data: page bodies come
+ * from authors, comments and imports, and an agent must treat that output as
+ * data rather than as instructions.
+ */
+declare const WEBMCP_TOOL_ANNOTATIONS: Readonly<Record<string, WebMCPToolAnnotations>>;
+/** Annotations for a tool. Unknown tools fall back to the safest pair. */
+declare function webMCPAnnotationsFor(name: string): WebMCPToolAnnotations;
+/** Attach WebMCP annotations to MCP tool definitions. */
+declare function toWebMCPTools(tools: MCPToolDefinition[]): WebMCPTool[];
+/**
+ * Build the inline script that registers the tools with the in-page agent.
+ *
+ * Deliberate constraints:
+ * - `exposedTo` is never set, so tools stay same-origin by default.
+ * - Registration is refused inside a frame: the Permissions Policy `tools`
+ *   feature already defaults to `['self']`, and this keeps a same-origin
+ *   frame from registering the set a second time.
+ * - The bridge only forwards calls to this site's own MCP endpoint; the page
+ *   cannot introduce a tool the server does not already serve.
+ * - Every forwarded call carries the MCP-Protocol-Version header, which the
+ *   endpoint requires on every request after initialize.
+ * - Chrome 153+ passes an AbortSignal as execute's second argument. Each
+ *   caller can stop waiting for the shared handshake, and its own tool fetch
+ *   receives that signal. The handshake itself uses an independent timeout so
+ *   cancelling one concurrent execution cannot fail every other caller.
+ */
+declare function generateWebMCPScript(tools: WebMCPTool[], config?: WebMCPScriptConfig): string;
+
 declare function parseSitemap(sitemapUrl: string, maxPages?: number): Promise<SitemapEntry[]>;
 declare function discoverSitemap(siteUrl: string): Promise<string | null>;
 
@@ -500,18 +614,20 @@ declare class MemoryCache implements CacheDriver {
 
 /**
  * Redis-backed cache driver for distributed / multi-instance deployments.
- * Compatible with ioredis, @upstash/redis, or any client implementing RedisClient.
+ * Accepts @upstash/redis directly. Wrap ioredis with adaptIORedisClient().
+ * Custom RedisCacheClient implementations must expose atomic SET with EX.
  *
  * Usage:
  *   import Redis from 'ioredis';
- *   const redis = new Redis();
+ *   import { adaptIORedisClient } from '@corsenai/corsen-context';
+ *   const redis = adaptIORedisClient(new Redis());
  *   const cache = new RedisCache(redis);
  *   const ctx = new CorsenContext(config, provider, cache);
  */
 declare class RedisCache implements CacheDriver {
     private redis;
     private prefix;
-    constructor(redis: RedisClient, options?: {
+    constructor(redis: RedisCacheClient, options?: {
         prefix?: string;
     });
     get<T>(key: string): Promise<T | null>;
@@ -532,13 +648,13 @@ declare class RedisCache implements CacheDriver {
  * still occupies a slot), which keeps the overshoot small and bounded. For a
  * strict guarantee, back this with a client exposing an atomic script (Lua).
  *
- * Compatible with ioredis, @upstash/redis, or any client implementing RedisClient.
+ * Accepts @upstash/redis directly. Wrap ioredis with adaptIORedisClient().
  */
 declare class RedisRateLimitStore implements RateLimitStore {
     private redis;
     private prefix;
     private windowMs;
-    constructor(redis: RedisClient, options?: {
+    constructor(redis: RedisRateLimitClient, options?: {
         prefix?: string;
         windowMs?: number;
     });
@@ -557,6 +673,12 @@ declare class RedisRateLimitStore implements RateLimitStore {
     cleanup(): Promise<void>;
 }
 
+/**
+ * Normalize ioredis's variadic sorted-set API to the Upstash-compatible
+ * RedisClient contract used by the core stores.
+ */
+declare function adaptIORedisClient(redis: IORedisClient): RedisClient;
+
 /**
  * Check if an IP address is private/internal.
  * Sync function — works on resolved IPs, not hostnames.
@@ -648,9 +770,9 @@ declare function extractClientIp(headers: Record<string, string | string[] | und
  */
 declare function buildRateLimitKey(clientIp: string, apiKey?: string): string;
 declare const searchParamsSchema: z.ZodObject<{
-    query: z.ZodString;
+    query: z.ZodEffects<z.ZodString, string, string>;
     limit: z.ZodDefault<z.ZodNumber>;
-}, "strip", z.ZodTypeAny, {
+}, "strict", z.ZodTypeAny, {
     query: string;
     limit: number;
 }, {
@@ -658,17 +780,17 @@ declare const searchParamsSchema: z.ZodObject<{
     limit?: number | undefined;
 }>;
 declare const getPageParamsSchema: z.ZodObject<{
-    uri: z.ZodString;
-}, "strip", z.ZodTypeAny, {
+    uri: z.ZodEffects<z.ZodString, string, string>;
+}, "strict", z.ZodTypeAny, {
     uri: string;
 }, {
     uri: string;
 }>;
 declare const listContentParamsSchema: z.ZodObject<{
-    type: z.ZodDefault<z.ZodString>;
+    type: z.ZodDefault<z.ZodEffects<z.ZodString, string, string>>;
     page: z.ZodDefault<z.ZodNumber>;
     limit: z.ZodDefault<z.ZodNumber>;
-}, "strip", z.ZodTypeAny, {
+}, "strict", z.ZodTypeAny, {
     type: string;
     page: number;
     limit: number;
@@ -753,7 +875,7 @@ declare class CorsenContext {
     }): MCPServer;
     /** Drop the cached body for a single page URL (wire to CMS update/delete hooks). */
     invalidatePage(url: string): Promise<void>;
-    /** Clear all cached MCP responses. Call after bulk content changes. */
+    /** Clear all cached page bodies. Call after bulk content changes. */
     clearCache(): Promise<void>;
     discoverSitemap(url?: string): Promise<string | null>;
     parseSitemap(sitemapUrl: string): Promise<SitemapEntry[]>;
@@ -774,6 +896,7 @@ declare class CorsenContext {
         static: {
             generateLlmsTxt: boolean;
             includeFullContent: boolean;
+            maxOutputBytes: number;
         };
         security: {
             rateLimit: number;
@@ -794,4 +917,4 @@ declare class CorsenContext {
     };
 }
 
-export { API_VERSION, ApiKeyManager, type ApiKeyRecord, CORSEN_CONTEXT_VERSION, CREDIT_LINE, type CacheDriver, type ContentList, type ContentListItem, type ContentProvider, CorsenContext, type CorsenContextConfig, type DiscoveryConfig, type InMemoryPage, type JSONRPCRequest, type JSONRPCResponse, JSONRPC_ERRORS, type LogLevel, type Logger, type LoggerOptions, MAX_BODY_SIZE, MAX_JSON_DEPTH, type MCPCapabilities, type MCPResourceDefinition, MCPServer, type MCPToolDefinition, MCP_PROTOCOL_VERSION, MemoryCache, MemoryRateLimitStore, type PageContent, REQUEST_TIMEOUT_MS, type RateLimitResult, type RateLimitStore, RateLimiter, RedisCache, type RedisClient, RedisRateLimitStore, type ResolvedConfig, SECURITY_HEADERS, type SearchResult, type SitemapEntry, buildRateLimitKey, corsenContextConfigSchema, createInMemoryProvider, createLogger, createSitemapProvider, discoverSitemap, extractClientIp, extractMetadata, filterPublicPages, filterPublicSearchResults, generateLlmsFullTxt, generateLlmsTxt, generateRobotsTxt, generateWellKnownMcp, getLogger, getPageParamsSchema, hashApiKey, htmlToMarkdown, isPrivateIp, isPrivateUrl, isPublicListItem, isPublicPageContent, listContentParamsSchema, mcpLinkTag, mcpLogger, parseSitemap, resolveConfig, resolvePublicPageUrl, safeFetch, searchParamsSchema, securityLogger, setLogger, validateApiKey, validateHost, validateOrigin };
+export { API_VERSION, ApiKeyManager, type ApiKeyRecord, CORSEN_CONTEXT_VERSION, CREDIT_LINE, type CacheDriver, type ContentList, type ContentListItem, type ContentProvider, CorsenContext, type CorsenContextConfig, type DiscoveryConfig, type IORedisClient, type InMemoryPage, type JSONRPCRequest, type JSONRPCResponse, JSONRPC_ERRORS, type LogLevel, type Logger, type LoggerOptions, MAX_BODY_SIZE, MAX_JSON_DEPTH, type MCPCapabilities, type MCPResourceDefinition, MCPServer, type MCPToolDefinition, MCP_PROTOCOL_VERSION, MemoryCache, MemoryRateLimitStore, type PageContent, REQUEST_TIMEOUT_MS, type RateLimitResult, type RateLimitStore, RateLimiter, RedisCache, type RedisCacheClient, type RedisClient, type RedisRateLimitClient, RedisRateLimitStore, type ResolvedConfig, SECURITY_HEADERS, type SearchResult, type SitemapEntry, WEBMCP_TOOL_ANNOTATIONS, type WebMCPScriptConfig, type WebMCPTool, type WebMCPToolAnnotations, adaptIORedisClient, buildRateLimitKey, corsenContextConfigSchema, createInMemoryProvider, createLogger, createSitemapProvider, discoverSitemap, extractClientIp, extractMetadata, filterPublicPages, filterPublicSearchResults, generateLlmsFullTxt, generateLlmsTxt, generateRobotsTxt, generateWebMCPScript, generateWellKnownMcp, getLogger, getPageParamsSchema, hashApiKey, htmlToMarkdown, isPrivateIp, isPrivateUrl, isPublicListItem, isPublicPageContent, listContentParamsSchema, mcpLinkTag, mcpLogger, parseSitemap, resolveConfig, resolvePublicPageUrl, safeFetch, searchParamsSchema, securityLogger, setLogger, toWebMCPTools, validateApiKey, validateHost, validateOrigin, webMCPAnnotationsFor };