crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/schemas/toolOutputSchemas.js

@@ -0,0 +1,269 @@
+/**
+ * toolOutputSchemas — MCP `outputSchema` shapes for structured tool results (Phase 6).
+ *
+ * Each entry is a plain zod RAW SHAPE (an object of validators), not a
+ * z.object(shape) — callers wrap it themselves (see src/server/registerTool.js).
+ *
+ * These are documentation/discovery schemas for MCP clients, not runtime
+ * enforcement: the SDK validates a tool's `structuredContent` against its
+ * outputSchema on every successful call, so every field here must be
+ * `.optional()` (every top-level key, and every nested object uses
+ * `.passthrough()`) to guarantee a legitimate result can never fail
+ * validation. Shapes are derived directly from the object each tool actually
+ * returns (see server.js registrations + the tool source files under
+ * src/tools/).
+ */
+
+import { z } from 'zod';
+
+// `_cost` is injected by withAuth into the legacy JSON-text copy of the
+// result (never into structuredContent directly) — included here anyway so
+// clients that copy `content[0].text` into structuredContent-shaped code
+// don't fail validation.
+const costShape = z.object({
+  projected: z.number().optional().describe('Credits projected for this call before execution'),
+  actual: z.number().optional().describe('Credits actually charged (0 in creator mode, half-rate on error)'),
+  remaining_credits: z.number().nullable().optional().describe('Credits remaining on the account after this call, if known'),
+  projection_note: z.string().optional().describe('Human-readable note about how the cost was projected')
+}).passthrough().optional().describe('Cost-transparency metadata (D3.5), present when injected into the text copy of the result');
+
+// ── scrape ──────────────────────────────────────────────────────────────────
+
+const scrapeLinkShape = z.object({
+  href: z.string().optional(),
+  text: z.string().optional(),
+  is_external: z.boolean().optional(),
+  original_href: z.string().optional()
+}).passthrough();
+
+const scrapeLinksShape = z.object({
+  links: z.array(scrapeLinkShape).optional(),
+  total_count: z.number().optional(),
+  internal_count: z.number().optional(),
+  external_count: z.number().optional()
+}).passthrough();
+
+const scrapeMetadataShape = z.object({
+  title: z.string().optional(),
+  description: z.string().optional(),
+  keywords: z.array(z.string()).optional(),
+  canonical_url: z.string().optional(),
+  author: z.string().optional(),
+  robots: z.string().optional(),
+  viewport: z.string().optional(),
+  og_tags: z.record(z.unknown()).optional(),
+  twitter_tags: z.record(z.unknown()).optional(),
+  json_ld: z.array(z.unknown()).optional(),
+  microdata: z.array(z.unknown()).optional(),
+  url: z.string().optional()
+}).passthrough();
+
+const scrapeShape = {
+  success: z.boolean().optional().describe('Whether the scrape completed'),
+  url: z.string().optional().describe('Final URL after redirects'),
+  content: z.object({
+    markdown: z.string().optional(),
+    html: z.string().optional(),
+    rawHtml: z.string().optional(),
+    text: z.string().optional(),
+    links: scrapeLinksShape.optional(),
+    metadata: scrapeMetadataShape.optional(),
+    branding: z.record(z.unknown()).optional().describe('Static design tokens: colors, fonts, logo'),
+    screenshots: z.array(z.object({}).passthrough()).optional().describe('Present for the "screenshot" format; each item carries a resourceUri once published'),
+    json: z.unknown().optional().describe('Result of the {type:"json"} format (LLM-structured extraction)')
+  }).passthrough().optional().describe('One key per requested format'),
+  warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
+  _cost: costShape
+};
+
+// ── map_site ────────────────────────────────────────────────────────────────
+
+const mapSiteShape = {
+  base_url: z.string().optional(),
+  total_urls: z.number().optional(),
+  urls: z.union([
+    z.array(z.string()),
+    z.record(z.array(z.string()))
+  ]).optional().describe('Flat array of URLs, or grouped-by-path object when group_by_path=true (default)'),
+  metadata: z.record(z.unknown()).optional().describe('Per-URL metadata when include_metadata=true'),
+  site_map: z.object({
+    root: z.array(z.string()).optional(),
+    sections: z.record(z.unknown()).optional(),
+    depth_levels: z.record(z.unknown()).optional()
+  }).passthrough().optional(),
+  statistics: z.object({
+    total_urls: z.number().optional(),
+    unique_paths: z.number().optional(),
+    file_extensions: z.record(z.number()).optional(),
+    query_parameters: z.number().optional(),
+    secure_urls: z.number().optional(),
+    max_depth: z.number().optional(),
+    average_depth: z.number().optional(),
+    url_lengths: z.object({
+      min: z.number().nullable().optional(),
+      max: z.number().optional(),
+      average: z.number().optional()
+    }).passthrough().optional()
+  }).passthrough().optional(),
+  domain_filter_config: z.unknown().nullable().optional(),
+  filter_stats: z.unknown().nullable().optional(),
+  ranked_urls: z.array(z.object({
+    url: z.string().optional(),
+    score: z.number().optional()
+  }).passthrough()).optional().describe('Present only when the `search` param was set'),
+  _cost: costShape
+};
+
+// ── serp_rank ───────────────────────────────────────────────────────────────
+
+const serpRankResultShape = z.object({
+  position: z.number().nullable().optional(),
+  rankAbsolute: z.number().nullable().optional(),
+  domain: z.string().optional(),
+  url: z.string().nullable().optional(),
+  title: z.string().nullable().optional(),
+  snippet: z.string().optional()
+}).passthrough();
+
+const serpRankShape = {
+  configured: z.boolean().optional().describe('False when DATAFORSEO_LOGIN/PASSWORD are unset — no rank was fabricated'),
+  keyword: z.string().optional(),
+  target: z.string().optional().describe('Bare target domain, normalized'),
+  note: z.string().optional().describe('Present when configured=false, explains how to enable'),
+  found: z.boolean().optional().describe('Whether the target appeared anywhere in the scanned SERP'),
+  position: z.number().nullable().optional().describe('Best (lowest) organic rank; null = not within top `depth`'),
+  rankAbsolute: z.number().nullable().optional(),
+  url: z.string().nullable().optional().describe('URL of the target\'s best-ranking result'),
+  title: z.string().nullable().optional(),
+  allPositions: z.array(serpRankResultShape).optional().describe('Every position the target holds on this SERP'),
+  results: z.array(serpRankResultShape).optional().describe('Top organic competitors as Google actually ranks them (capped)'),
+  location: z.unknown().optional(),
+  device: z.string().optional(),
+  depthScanned: z.number().optional(),
+  organicResults: z.number().optional(),
+  seResultsCount: z.number().optional(),
+  checkUrl: z.string().optional().describe('Link to view the real SERP on DataForSEO'),
+  cost: z.number().optional().describe('USD charged by DataForSEO for this lookup (separate from CrawlForge credits)'),
+  checkedAt: z.string().optional(),
+  _cost: costShape
+};
+
+// ── search_web ──────────────────────────────────────────────────────────────
+
+const searchWebResultShape = z.object({
+  title: z.string().optional(),
+  link: z.string().optional(),
+  snippet: z.string().optional(),
+  displayLink: z.string().optional(),
+  formattedUrl: z.string().optional(),
+  htmlSnippet: z.string().optional(),
+  pagemap: z.record(z.unknown()).optional(),
+  metadata: z.record(z.unknown()).optional()
+}).passthrough();
+
+const searchWebShape = {
+  query: z.string().optional(),
+  effective_query: z.string().optional().describe('Present when query expansion changed the query actually used'),
+  expanded_queries: z.array(z.string()).optional(),
+  results: z.array(searchWebResultShape).optional(),
+  total_results: z.union([z.string(), z.number()]).optional(),
+  search_time: z.number().optional(),
+  offset: z.number().optional(),
+  limit: z.number().optional(),
+  cached: z.boolean().optional(),
+  provider: z.object({
+    name: z.string().optional(),
+    backend: z.string().optional(),
+    note: z.string().optional(),
+    instanceUrl: z.string().nullable().optional(),
+    capabilities: z.record(z.unknown()).optional()
+  }).passthrough().optional(),
+  localization: z.object({
+    applied: z.boolean().optional(),
+    countryCode: z.string().optional(),
+    language: z.string().optional(),
+    searchDomain: z.string().optional(),
+    geoTargeting: z.boolean().optional()
+  }).passthrough().nullable().optional(),
+  processing: z.object({
+    ranking: z.record(z.unknown()).nullable().optional(),
+    deduplication: z.record(z.unknown()).nullable().optional(),
+    query_expansion: z.record(z.unknown()).nullable().optional(),
+    localization_applied: z.boolean().optional()
+  }).passthrough().optional(),
+  _cost: costShape
+};
+
+// ── extract_structured ───────────────────────────────────────────────────────
+
+const extractStructuredShape = {
+  url: z.string().optional(),
+  data: z.record(z.unknown()).optional().describe('Extracted fields matching the requested schema'),
+  extraction_method: z.string().optional().describe('"llm" | "css_fallback" | "none"'),
+  confidence: z.number().optional(),
+  schema_used: z.record(z.unknown()).optional(),
+  processingTime: z.number().optional(),
+  error: z.string().optional(),
+  validation: z.object({
+    valid: z.boolean().optional(),
+    errors: z.array(z.string()).optional()
+  }).passthrough().optional(),
+  extractionNotes: z.array(z.string()).optional(),
+  _cost: costShape
+};
+
+// ── crawl_deep ────────────────────────────────────────────────────────────────
+
+const crawlDeepPageShape = z.object({
+  url: z.string().optional(),
+  depth: z.number().optional(),
+  title: z.string().optional(),
+  links_count: z.number().optional(),
+  content_length: z.number().optional(),
+  timestamp: z.union([z.string(), z.number()]).optional(),
+  content: z.string().optional(),
+  truncated: z.boolean().optional(),
+  metadata: z.unknown().optional()
+}).passthrough();
+
+const crawlDeepShape = {
+  success: z.boolean().optional().describe('False only when the crawl was cancelled via elicitation decline'),
+  error: z.string().optional(),
+  url: z.string().optional(),
+  crawl_depth: z.number().optional(),
+  pages_crawled: z.number().optional(),
+  pages_found: z.number().optional(),
+  error_count: z.number().optional(),
+  duration_ms: z.number().optional(),
+  pages_per_second: z.number().optional(),
+  results: z.array(crawlDeepPageShape).optional(),
+  errors: z.array(z.unknown()).optional(),
+  stats: z.unknown().optional(),
+  site_structure: z.object({
+    total_pages: z.number().optional(),
+    depth_distribution: z.record(z.number()).optional(),
+    path_patterns: z.record(z.number()).optional(),
+    file_types: z.record(z.number()).optional(),
+    subdomains: z.array(z.string()).optional()
+  }).passthrough().optional(),
+  domain_filter_config: z.unknown().nullable().optional(),
+  link_analysis: z.unknown().nullable().optional(),
+  session: z.object({
+    enabled: z.boolean().optional(),
+    cookies_captured: z.number().optional()
+  }).passthrough().optional(),
+  _cost: costShape
+};
+
+// ── Export ────────────────────────────────────────────────────────────────────
+
+export const OUTPUT_SCHEMAS = {
+  scrape: scrapeShape,
+  map_site: mapSiteShape,
+  serp_rank: serpRankShape,
+  search_web: searchWebShape,
+  extract_structured: extractStructuredShape,
+  crawl_deep: crawlDeepShape
+};
+
+export default OUTPUT_SCHEMAS;

package/src/server/auth/oauth.js

@@ -188,6 +188,22 @@ async function handleAuthorize(req, res, store, apiKey) {
     return sendJson(res, 400, { error: 'invalid_request', error_description: 'only S256 PKCE is supported' });
   }
 
+  // Proof-of-possession required: the caller must present the operator's
+  // CrawlForge API key — via `Authorization: Bearer <key>` header or an
+  // `api_key` query parameter — before a code is issued. Dynamic client
+  // registration (/oauth/register) is open and unauthenticated by design,
+  // so client_id/PKCE alone must never be sufficient to mint a code; the
+  // key proof is the actual authorization check.
+  //
+  // For a multi-tenant deployment, replace this with a real consent page that
+  // resolves to a CrawlForge user → API key mapping.
+  if (!verifyApiKeyProof(req, params, apiKey)) {
+    return sendJson(res, 401, {
+      error: 'invalid_client',
+      error_description: 'proof of API key possession required (Authorization: Bearer <key> header or api_key query parameter)'
+    });
+  }
+
   const client = await store.getClient(params.client_id);
   if (!client) {
     return sendJson(res, 400, { error: 'invalid_client' });
@@ -196,13 +212,6 @@ async function handleAuthorize(req, res, store, apiKey) {
     return sendJson(res, 400, { error: 'invalid_redirect_uri' });
   }
 
-  // Auto-approve: this server is a personal MCP endpoint backed by a single
-  // CrawlForge API key the operator has already authenticated. No consent UI
-  // is needed — possession of the operator's `apiKey` IS the authorization.
-  // (Same trust model as the static-key transport.)
-  //
-  // For a multi-tenant deployment, replace this with a real consent page that
-  // resolves to a CrawlForge user → API key mapping.
   const code = randomBytes(24).toString('base64url');
   await store.setCode(code, {
     clientId: params.client_id,
@@ -313,6 +322,27 @@ async function issueTokens(store, { clientId, mappedApiKey, scopes, accessTtlMs,
   };
 }
 
+/**
+ * Verify the caller has proven possession of the operator's CrawlForge API key.
+ * Accepted forms: `Authorization: Bearer <key>` header, or `api_key` query param.
+ * Comparison is constant-time over fixed-length SHA-256 digests (never compares
+ * the raw secrets directly, and never leaks timing based on input length).
+ */
+function verifyApiKeyProof(req, params, apiKey) {
+  const authHeader = req.headers?.authorization || req.headers?.Authorization;
+  let provided = null;
+  if (typeof authHeader === 'string' && /^Bearer\s+/i.test(authHeader)) {
+    provided = authHeader.replace(/^Bearer\s+/i, '').trim();
+  } else if (typeof params.api_key === 'string' && params.api_key.length > 0) {
+    provided = params.api_key;
+  }
+  if (!provided) return false;
+
+  const providedHash = createHash('sha256').update(provided).digest();
+  const expectedHash = createHash('sha256').update(apiKey).digest();
+  return timingSafeEqual(providedHash, expectedHash);
+}
+
 function verifyPkce(verifier, expectedChallenge) {
   try {
     const challenge = createHash('sha256').update(verifier).digest('base64url');

package/src/server/specHygiene.js

@@ -0,0 +1,192 @@
+/**
+ * specHygiene — Phase 6 protocol-hygiene wrapper.
+ *
+ * Applies three MCP wire-level upgrades to an already-registered McpServer
+ * without changing how tools/prompts are declared anywhere else:
+ *
+ *   1. JSON Schema 2020-12 dialect stamping on every tool's inputSchema /
+ *      outputSchema (the SDK's zod-to-json-schema conversion still emits
+ *      draft-07-style `definitions` / `#/definitions/...` refs).
+ *   2. Deterministic (alphabetical) tool ordering in tools/list, so clients
+ *      that prompt-cache tools/list get stable hits across restarts.
+ *   3. SEP-973 icons metadata on every tool/prompt lacking one.
+ *   4. SEP-2549-style cache hints on tools/call results for a fixed
+ *      allowlist of read-only, idempotent tools (see note below).
+ *
+ * Wiring: SDK 1.30 has no plugin hook for this, so applySpecHygiene() must be
+ * called once, after all server.registerTool()/registerPrompt() calls and
+ * before transport.connect(). It reaches into `server.server` (the
+ * underlying Protocol), captures the ListTools/CallTool/ListPrompts handlers
+ * McpServer already installed in `_requestHandlers`, and re-registers wrapped
+ * versions via `setRequestHandler` — which silently replaces an existing
+ * handler for the same method (confirmed from the SDK source: only
+ * `assertCanSetRequestHandler`, called by McpServer's own registration path,
+ * throws on a pre-existing handler; `setRequestHandler` itself does not).
+ *
+ * SEP-2549 note: as specified (2026-07-28 RC), `ttlMs` / `cacheScope`
+ * ("public"|"private") are defined on tools/list, prompts/list,
+ * resources/list, resources/read and resources/templates/list results —
+ * there is no CacheableResult for tools/call. Since this deliverable asks
+ * for tools/call cache hints, the same two field names are carried into a
+ * namespaced `_meta` key following the SDK's own
+ * `io.modelcontextprotocol/<name>` convention (see RELATED_TASK_META_KEY in
+ * @modelcontextprotocol/sdk/types.js):
+ *   result._meta["io.modelcontextprotocol/cacheable"] = { ttlMs, cacheScope }
+ * This is a documented adaptation, not a key confirmed by the spec text for
+ * tools/call — revisit if/when a SEP defines call-result caching explicitly.
+ */
+
+import {
+  ListToolsRequestSchema,
+  CallToolRequestSchema,
+  ListPromptsRequestSchema
+} from '@modelcontextprotocol/sdk/types.js';
+
+const APPLIED = Symbol('crawlforge.specHygiene.applied');
+
+const JSON_SCHEMA_2020_12 = 'https://json-schema.org/draft/2020-12/schema';
+const CACHE_META_KEY = 'io.modelcontextprotocol/cacheable';
+
+const DEFAULT_ICON = Object.freeze({
+  src: 'https://www.crawlforge.dev/icon.png',
+  mimeType: 'image/png',
+  sizes: ['any']
+});
+
+// Read-only, idempotent tools whose tools/call results get a SEP-2549-style
+// cache hint (see note above). ttlMs mirrors CACHE_TTL's 5-minute default;
+// cacheScope 'private' because scraped content may reflect the caller's own
+// request context (headers, auth) and should not be shared across clients.
+const DEFAULT_CACHEABLE_TOOLS = Object.freeze({
+  fetch_url: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  extract_text: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  extract_links: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  extract_metadata: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  scrape_structured: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  map_site: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  search_web: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  serp_rank: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  extract_content: Object.freeze({ ttlMs: 300000, cacheScope: 'private' }),
+  scrape: Object.freeze({ ttlMs: 300000, cacheScope: 'private' })
+});
+
+/**
+ * Deep-copies a JSON-schema-like tree, renaming `definitions` -> `$defs` and
+ * rewriting `"#/definitions/..."` refs to `"#/$defs/..."` anywhere in the
+ * tree. Never mutates the input (some schemas, e.g. the SDK's shared
+ * "no params" constant, are frozen and reused across tools).
+ */
+function rewriteDefs(node) {
+  if (Array.isArray(node)) return node.map(rewriteDefs);
+  if (node && typeof node === 'object') {
+    const out = {};
+    for (const [key, value] of Object.entries(node)) {
+      if (key === 'definitions') {
+        out.$defs = rewriteDefs(value);
+      } else if (key === '$ref' && typeof value === 'string' && value.startsWith('#/definitions/')) {
+        out.$ref = `#/$defs/${value.slice('#/definitions/'.length)}`;
+      } else {
+        out[key] = rewriteDefs(value);
+      }
+    }
+    return out;
+  }
+  return node;
+}
+
+function stamp2020_12(schema) {
+  if (!schema || typeof schema !== 'object') return schema;
+  return { ...rewriteDefs(schema), $schema: JSON_SCHEMA_2020_12 };
+}
+
+function withDefaultIcon(entry, icon) {
+  if (Array.isArray(entry.icons) && entry.icons.length > 0) return entry;
+  return { ...entry, icons: [icon] };
+}
+
+function wrapToolsList(innerHandler, icon) {
+  return async (request, extra) => {
+    const result = await innerHandler(request, extra);
+    if (!result || !Array.isArray(result.tools)) return result;
+
+    const tools = result.tools
+      .map((tool) => {
+        let next = { ...tool };
+        if (next.inputSchema) next.inputSchema = stamp2020_12(next.inputSchema);
+        if (next.outputSchema) next.outputSchema = stamp2020_12(next.outputSchema);
+        next = withDefaultIcon(next, icon);
+        return next;
+      })
+      .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+
+    return { ...result, tools };
+  };
+}
+
+function wrapPromptsList(innerHandler, icon) {
+  return async (request, extra) => {
+    const result = await innerHandler(request, extra);
+    if (!result || !Array.isArray(result.prompts)) return result;
+
+    const prompts = result.prompts.map((prompt) => withDefaultIcon(prompt, icon));
+    return { ...result, prompts };
+  };
+}
+
+function wrapToolsCall(innerHandler, cacheableTools) {
+  return async (request, extra) => {
+    const result = await innerHandler(request, extra);
+    if (!result || typeof result !== 'object') return result;
+    if (result.isError) return result;
+    if (!Array.isArray(result.content)) return result; // task-augmented / content-less results pass through untouched
+
+    const hint = cacheableTools[request?.params?.name];
+    if (!hint) return result;
+
+    return {
+      ...result,
+      _meta: {
+        ...(result._meta ?? {}),
+        [CACHE_META_KEY]: { ttlMs: hint.ttlMs, cacheScope: hint.cacheScope }
+      }
+    };
+  };
+}
+
+/**
+ * Applies the protocol-hygiene wrappers to `server`. Call once, after all
+ * tools/prompts are registered and before transport.connect(). Idempotent:
+ * a second call on the same server is a no-op.
+ *
+ * @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} server
+ * @param {object} [overrides]
+ * @param {{src:string,mimeType?:string,sizes?:string[]}} [overrides.icon] — default icon injected into tools/prompts lacking one
+ * @param {Record<string,{ttlMs:number,cacheScope:'public'|'private'}>} [overrides.cacheableTools] — tool-name -> cache hint map for tools/call
+ */
+export function applySpecHygiene(server, overrides = {}) {
+  const protocol = server?.server;
+  if (!protocol || typeof protocol.setRequestHandler !== 'function' || !(protocol._requestHandlers instanceof Map)) {
+    throw new TypeError('applySpecHygiene requires a connected McpServer instance');
+  }
+
+  if (protocol[APPLIED]) return;
+  protocol[APPLIED] = true;
+
+  const icon = overrides.icon ?? DEFAULT_ICON;
+  const cacheableTools = overrides.cacheableTools ?? DEFAULT_CACHEABLE_TOOLS;
+
+  const innerToolsList = protocol._requestHandlers.get('tools/list');
+  if (innerToolsList) {
+    protocol.setRequestHandler(ListToolsRequestSchema, wrapToolsList(innerToolsList, icon));
+  }
+
+  const innerToolsCall = protocol._requestHandlers.get('tools/call');
+  if (innerToolsCall) {
+    protocol.setRequestHandler(CallToolRequestSchema, wrapToolsCall(innerToolsCall, cacheableTools));
+  }
+
+  const innerPromptsList = protocol._requestHandlers.get('prompts/list');
+  if (innerPromptsList) {
+    protocol.setRequestHandler(ListPromptsRequestSchema, wrapPromptsList(innerPromptsList, icon));
+  }
+}