mcp-researchpowerpack 3.9.4 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -208
- package/dist/index.js +280 -337
- package/dist/index.js.map +7 -1
- package/dist/mcp-use.json +7 -0
- package/dist/src/clients/reddit.js +278 -0
- package/dist/src/clients/reddit.js.map +7 -0
- package/dist/src/clients/scraper.js +326 -0
- package/dist/src/clients/scraper.js.map +7 -0
- package/dist/src/clients/search.js +217 -0
- package/dist/src/clients/search.js.map +7 -0
- package/dist/src/config/index.js +138 -0
- package/dist/src/config/index.js.map +7 -0
- package/dist/src/prompts/deep-research.js +41 -0
- package/dist/src/prompts/deep-research.js.map +7 -0
- package/dist/src/prompts/reddit-sentiment.js +47 -0
- package/dist/src/prompts/reddit-sentiment.js.map +7 -0
- package/dist/src/schemas/reddit.js +21 -0
- package/dist/src/schemas/reddit.js.map +7 -0
- package/dist/src/schemas/scrape-links.js +26 -0
- package/dist/src/schemas/scrape-links.js.map +7 -0
- package/dist/src/schemas/start-research.js +17 -0
- package/dist/src/schemas/start-research.js.map +7 -0
- package/dist/src/schemas/web-search.js +53 -0
- package/dist/src/schemas/web-search.js.map +7 -0
- package/dist/src/services/llm-processor.js +675 -0
- package/dist/src/services/llm-processor.js.map +7 -0
- package/dist/src/services/markdown-cleaner.js +62 -0
- package/dist/src/services/markdown-cleaner.js.map +7 -0
- package/dist/src/services/workflow-state.js +116 -0
- package/dist/src/services/workflow-state.js.map +7 -0
- package/dist/src/tools/mcp-helpers.js +50 -0
- package/dist/src/tools/mcp-helpers.js.map +7 -0
- package/dist/src/tools/reddit.js +277 -0
- package/dist/src/tools/reddit.js.map +7 -0
- package/dist/src/tools/registry.js +18 -0
- package/dist/src/tools/registry.js.map +7 -0
- package/dist/src/tools/scrape.js +334 -0
- package/dist/src/tools/scrape.js.map +7 -0
- package/dist/src/tools/search.js +423 -0
- package/dist/src/tools/search.js.map +7 -0
- package/dist/src/tools/start-research.js +199 -0
- package/dist/src/tools/start-research.js.map +7 -0
- package/dist/src/tools/utils.js +29 -0
- package/dist/src/tools/utils.js.map +7 -0
- package/dist/src/utils/bootstrap-guard.js +27 -0
- package/dist/src/utils/bootstrap-guard.js.map +7 -0
- package/dist/src/utils/concurrency.js +62 -0
- package/dist/src/utils/concurrency.js.map +7 -0
- package/dist/src/utils/content-extractor.js +61 -0
- package/dist/src/utils/content-extractor.js.map +7 -0
- package/dist/src/utils/errors.js +211 -0
- package/dist/src/utils/errors.js.map +7 -0
- package/dist/src/utils/logger.js +25 -0
- package/dist/src/utils/logger.js.map +7 -0
- package/dist/src/utils/markdown-formatter.js +15 -0
- package/dist/src/utils/markdown-formatter.js.map +7 -0
- package/dist/src/utils/reddit-keyword-guard.js +29 -0
- package/dist/src/utils/reddit-keyword-guard.js.map +7 -0
- package/dist/src/utils/response.js +81 -0
- package/dist/src/utils/response.js.map +7 -0
- package/dist/src/utils/retry.js +13 -0
- package/dist/src/utils/retry.js.map +7 -0
- package/dist/src/utils/sanitize.js +10 -0
- package/dist/src/utils/sanitize.js.map +7 -0
- package/dist/src/utils/source-type.js +41 -0
- package/dist/src/utils/source-type.js.map +7 -0
- package/dist/src/utils/url-aggregator.js +227 -0
- package/dist/src/utils/url-aggregator.js.map +7 -0
- package/dist/src/utils/workflow-key.js +14 -0
- package/dist/src/utils/workflow-key.js.map +7 -0
- package/dist/src/version.js +32 -0
- package/dist/src/version.js.map +7 -0
- package/package.json +33 -28
- package/dist/clients/reddit.d.ts +0 -69
- package/dist/clients/reddit.d.ts.map +0 -1
- package/dist/clients/reddit.js +0 -369
- package/dist/clients/reddit.js.map +0 -1
- package/dist/clients/research.d.ts +0 -67
- package/dist/clients/research.d.ts.map +0 -1
- package/dist/clients/research.js +0 -290
- package/dist/clients/research.js.map +0 -1
- package/dist/clients/scraper.d.ts +0 -72
- package/dist/clients/scraper.d.ts.map +0 -1
- package/dist/clients/scraper.js +0 -338
- package/dist/clients/scraper.js.map +0 -1
- package/dist/clients/search.d.ts +0 -57
- package/dist/clients/search.d.ts.map +0 -1
- package/dist/clients/search.js +0 -223
- package/dist/clients/search.js.map +0 -1
- package/dist/config/index.d.ts +0 -78
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -201
- package/dist/config/index.js.map +0 -1
- package/dist/config/loader.d.ts +0 -40
- package/dist/config/loader.d.ts.map +0 -1
- package/dist/config/loader.js +0 -322
- package/dist/config/loader.js.map +0 -1
- package/dist/config/types.d.ts +0 -81
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -6
- package/dist/config/types.js.map +0 -1
- package/dist/config/yaml/tools.yaml +0 -146
- package/dist/index.d.ts +0 -7
- package/dist/index.d.ts.map +0 -1
- package/dist/schemas/deep-research.d.ts +0 -64
- package/dist/schemas/deep-research.d.ts.map +0 -1
- package/dist/schemas/deep-research.js +0 -224
- package/dist/schemas/deep-research.js.map +0 -1
- package/dist/schemas/scrape-links.d.ts +0 -32
- package/dist/schemas/scrape-links.d.ts.map +0 -1
- package/dist/schemas/scrape-links.js +0 -34
- package/dist/schemas/scrape-links.js.map +0 -1
- package/dist/schemas/web-search.d.ts +0 -22
- package/dist/schemas/web-search.d.ts.map +0 -1
- package/dist/schemas/web-search.js +0 -21
- package/dist/schemas/web-search.js.map +0 -1
- package/dist/services/file-attachment.d.ts +0 -30
- package/dist/services/file-attachment.d.ts.map +0 -1
- package/dist/services/file-attachment.js +0 -207
- package/dist/services/file-attachment.js.map +0 -1
- package/dist/services/llm-processor.d.ts +0 -29
- package/dist/services/llm-processor.d.ts.map +0 -1
- package/dist/services/llm-processor.js +0 -244
- package/dist/services/llm-processor.js.map +0 -1
- package/dist/services/markdown-cleaner.d.ts +0 -8
- package/dist/services/markdown-cleaner.d.ts.map +0 -1
- package/dist/services/markdown-cleaner.js +0 -74
- package/dist/services/markdown-cleaner.js.map +0 -1
- package/dist/tools/definitions.d.ts +0 -16
- package/dist/tools/definitions.d.ts.map +0 -1
- package/dist/tools/definitions.js +0 -17
- package/dist/tools/definitions.js.map +0 -1
- package/dist/tools/reddit.d.ts +0 -14
- package/dist/tools/reddit.d.ts.map +0 -1
- package/dist/tools/reddit.js +0 -265
- package/dist/tools/reddit.js.map +0 -1
- package/dist/tools/registry.d.ts +0 -71
- package/dist/tools/registry.d.ts.map +0 -1
- package/dist/tools/registry.js +0 -252
- package/dist/tools/registry.js.map +0 -1
- package/dist/tools/research.d.ts +0 -14
- package/dist/tools/research.d.ts.map +0 -1
- package/dist/tools/research.js +0 -196
- package/dist/tools/research.js.map +0 -1
- package/dist/tools/scrape.d.ts +0 -14
- package/dist/tools/scrape.d.ts.map +0 -1
- package/dist/tools/scrape.js +0 -234
- package/dist/tools/scrape.js.map +0 -1
- package/dist/tools/search.d.ts +0 -10
- package/dist/tools/search.d.ts.map +0 -1
- package/dist/tools/search.js +0 -158
- package/dist/tools/search.js.map +0 -1
- package/dist/tools/utils.d.ts +0 -105
- package/dist/tools/utils.d.ts.map +0 -1
- package/dist/tools/utils.js +0 -159
- package/dist/tools/utils.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -28
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -92
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/errors.d.ts +0 -95
- package/dist/utils/errors.d.ts.map +0 -1
- package/dist/utils/errors.js +0 -390
- package/dist/utils/errors.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -39
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -57
- package/dist/utils/logger.js.map +0 -1
- package/dist/utils/markdown-formatter.d.ts +0 -5
- package/dist/utils/markdown-formatter.d.ts.map +0 -1
- package/dist/utils/markdown-formatter.js +0 -15
- package/dist/utils/markdown-formatter.js.map +0 -1
- package/dist/utils/response.d.ts +0 -93
- package/dist/utils/response.d.ts.map +0 -1
- package/dist/utils/response.js +0 -170
- package/dist/utils/response.js.map +0 -1
- package/dist/utils/retry.d.ts +0 -43
- package/dist/utils/retry.d.ts.map +0 -1
- package/dist/utils/retry.js +0 -57
- package/dist/utils/retry.js.map +0 -1
- package/dist/utils/url-aggregator.d.ts +0 -90
- package/dist/utils/url-aggregator.d.ts.map +0 -1
- package/dist/utils/url-aggregator.js +0 -538
- package/dist/utils/url-aggregator.js.map +0 -1
- package/dist/version.d.ts +0 -29
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js +0 -55
- package/dist/version.js.map +0 -1
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/services/llm-processor.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * LLM Processor for content extraction\n * Uses OpenRouter via OPENROUTER_API_KEY for AI-powered content filtering\n * Implements robust retry logic and NEVER throws\n */\n\nimport OpenAI from 'openai';\nimport { LLM_EXTRACTION, getCapabilities } from '../config/index.js';\nimport {\n classifyError,\n sleep,\n ErrorCode,\n withStallProtection,\n type StructuredError,\n} from '../utils/errors.js';\nimport { mcpLog } from '../utils/logger.js';\n\n/** Maximum input characters for LLM processing (~25k tokens) */\nconst MAX_LLM_INPUT_CHARS = 100_000 as const;\n\n/** LLM client timeout in milliseconds */\nconst LLM_CLIENT_TIMEOUT_MS = 120_000 as const;\n\n/** Jitter factor for exponential backoff */\nconst BACKOFF_JITTER_FACTOR = 0.3 as const;\n\n/** Stall detection timeout \u2014 abort if no response in this time */\nconst LLM_STALL_TIMEOUT_MS = 15_000 as const;\n\n/** Hard request deadline for LLM calls */\nconst LLM_REQUEST_DEADLINE_MS = 30_000 as const;\n\n// ============================================================================\n// LLM health tracking \u2014 surfaced via health://status so capability-aware\n// clients can branch on degraded mode without parsing per-call footers.\n// ============================================================================\n\ntype LLMHealthKind = 'planner' | 'extractor';\n\ninterface LLMHealthSnapshot {\n readonly lastPlannerOk: boolean;\n readonly lastExtractorOk: boolean;\n readonly lastPlannerCheckedAt: string | null;\n readonly lastExtractorCheckedAt: string | null;\n readonly lastPlannerError: string | null;\n readonly lastExtractorError: string | null;\n readonly plannerConfigured: boolean;\n readonly extractorConfigured: boolean;\n}\n\nconst llmHealth = {\n lastPlannerOk: false,\n lastExtractorOk: false,\n lastPlannerCheckedAt: null as string | null,\n lastExtractorCheckedAt: null as string | null,\n lastPlannerError: null as string | null,\n lastExtractorError: null as string | null,\n};\n\nexport function markLLMSuccess(kind: LLMHealthKind): void {\n const ts = new Date().toISOString();\n if (kind === 'planner') {\n llmHealth.lastPlannerOk = true;\n llmHealth.lastPlannerCheckedAt = ts;\n llmHealth.lastPlannerError = null;\n } else {\n llmHealth.lastExtractorOk = true;\n llmHealth.lastExtractorCheckedAt = ts;\n llmHealth.lastExtractorError = null;\n }\n}\n\nexport function markLLMFailure(kind: LLMHealthKind, err: unknown): void {\n const ts = new Date().toISOString();\n const message = err instanceof Error ? err.message : String(err ?? 'unknown error');\n if (kind === 'planner') {\n llmHealth.lastPlannerOk = false;\n llmHealth.lastPlannerCheckedAt = ts;\n llmHealth.lastPlannerError = message;\n } else {\n llmHealth.lastExtractorOk = false;\n llmHealth.lastExtractorCheckedAt = ts;\n llmHealth.lastExtractorError = message;\n }\n}\n\nexport function getLLMHealth(): LLMHealthSnapshot {\n const cap = getCapabilities();\n return {\n lastPlannerOk: llmHealth.lastPlannerOk,\n lastExtractorOk: llmHealth.lastExtractorOk,\n lastPlannerCheckedAt: llmHealth.lastPlannerCheckedAt,\n lastExtractorCheckedAt: llmHealth.lastExtractorCheckedAt,\n lastPlannerError: llmHealth.lastPlannerError,\n lastExtractorError: llmHealth.lastExtractorError,\n // Static capability \u2014 based on env presence at boot. Runtime health (above)\n // tells whether the last attempt actually succeeded.\n plannerConfigured: cap.llmExtraction,\n extractorConfigured: cap.llmExtraction,\n };\n}\n\n/** Test-only \u2014 reset state between tests. Not exported from index. */\nexport function _resetLLMHealthForTests(): void {\n llmHealth.lastPlannerOk = false;\n llmHealth.lastExtractorOk = false;\n llmHealth.lastPlannerCheckedAt = null;\n llmHealth.lastExtractorCheckedAt = null;\n llmHealth.lastPlannerError = null;\n llmHealth.lastExtractorError = null;\n}\n\ninterface ProcessingConfig {\n readonly enabled: boolean;\n readonly extract: string | undefined;\n readonly max_tokens?: number;\n readonly url?: string;\n}\n\ninterface LLMResult {\n readonly content: string;\n readonly processed: boolean;\n readonly error?: string;\n readonly errorDetails?: StructuredError;\n}\n\n// LLM-specific retry configuration\nconst LLM_RETRY_CONFIG = {\n maxRetries: 2,\n baseDelayMs: 1000,\n maxDelayMs: 5000,\n} as const;\n\n// OpenRouter/OpenAI specific retryable error codes (using Set for type-safe lookup)\nconst RETRYABLE_LLM_ERROR_CODES = new Set([\n 'rate_limit_exceeded',\n 'server_error',\n 'timeout',\n 'service_unavailable',\n]);\n\n/** Type guard for errors with an HTTP status code */\nfunction hasStatus(error: unknown): error is { status: number } {\n return (\n typeof error === 'object' &&\n error !== null &&\n 'status' in error &&\n typeof (error as Record<string, unknown>).status === 'number'\n );\n}\n\nlet llmClient: OpenAI | null = null;\n\ntype OpenAITextGenerator = Pick<OpenAI, 'chat'>;\n\nexport function createLLMProcessor(): OpenAI | null {\n if (!getCapabilities().llmExtraction) return null;\n\n if (!llmClient) {\n llmClient = new OpenAI({\n baseURL: LLM_EXTRACTION.BASE_URL,\n apiKey: LLM_EXTRACTION.API_KEY,\n timeout: LLM_CLIENT_TIMEOUT_MS,\n maxRetries: 0,\n defaultHeaders: { 'X-Title': 'mcp-research-powerpack' },\n });\n mcpLog('info', `LLM extraction configured (model: ${LLM_EXTRACTION.MODEL}, baseURL: ${LLM_EXTRACTION.BASE_URL})`, 'llm');\n }\n return llmClient;\n}\n\nfunction buildChatRequestBody(model: string, prompt: string, maxTokens: number): Record<string, unknown> {\n const requestBody: Record<string, unknown> = {\n model,\n messages: [{ role: 'user', content: prompt }],\n max_tokens: maxTokens,\n };\n\n if (LLM_EXTRACTION.REASONING_EFFORT !== 'none') {\n requestBody.reasoning_effort = LLM_EXTRACTION.REASONING_EFFORT;\n }\n\n return requestBody;\n}\n\nexport async function requestTextWithFallback(\n processor: OpenAITextGenerator,\n prompt: string,\n maxTokens: number,\n operationLabel: string,\n signal?: AbortSignal,\n): Promise<{ content: string | null; model: string; error?: string }> {\n const models = [...new Set([\n LLM_EXTRACTION.MODEL,\n LLM_EXTRACTION.FALLBACK_MODEL,\n ].filter(Boolean))];\n\n let lastError = 'Unknown LLM error';\n\n for (const model of models) {\n try {\n const response = await withStallProtection(\n (stallSignal) => processor.chat.completions.create(\n buildChatRequestBody(model, prompt, maxTokens) as unknown as OpenAI.ChatCompletionCreateParamsNonStreaming,\n {\n signal: signal ? AbortSignal.any([stallSignal, signal]) : stallSignal,\n timeout: LLM_REQUEST_DEADLINE_MS,\n },\n ),\n LLM_STALL_TIMEOUT_MS,\n 3,\n `${operationLabel} (${model})`,\n );\n\n const content = response.choices?.[0]?.message?.content?.trim();\n if (content) {\n if (model !== LLM_EXTRACTION.MODEL) {\n mcpLog('warning', `${operationLabel} succeeded with fallback model ${model}`, 'llm');\n }\n return { content, model };\n }\n\n lastError = `Empty response from model ${model}`;\n mcpLog('warning', `${operationLabel} returned empty content for model ${model}`, 'llm');\n } catch (err: unknown) {\n lastError = err instanceof Error ? err.message : String(err);\n mcpLog('warning', `${operationLabel} failed for model ${model}: ${lastError}`, 'llm');\n }\n }\n\n return { content: null, model: LLM_EXTRACTION.FALLBACK_MODEL, error: lastError };\n}\n\n/**\n * Check if an LLM error is retryable\n */\nfunction isRetryableLLMError(error: unknown): boolean {\n if (!error || typeof error !== 'object') return false;\n\n // Stall/timeout protection errors - always retry these\n const stallCode = (error as { code?: string })?.code;\n if (stallCode === 'ESTALLED' || stallCode === 'ETIMEDOUT') {\n return true;\n }\n\n // Check HTTP status codes\n if (hasStatus(error)) {\n if (error.status === 429 || error.status === 500 || error.status === 502 || error.status === 503 || error.status === 504) {\n return true;\n }\n }\n\n // Check error codes from OpenAI/OpenRouter\n const record = error as Record<string, unknown>;\n const code = typeof record.code === 'string' ? record.code : undefined;\n const nested =\n typeof record.error === 'object' && record.error !== null\n ? (record.error as Record<string, unknown>)\n : null;\n const errorCode =\n code ??\n (nested && typeof nested.code === 'string' ? nested.code : undefined) ??\n (nested && typeof nested.type === 'string' ? nested.type : undefined);\n if (errorCode && RETRYABLE_LLM_ERROR_CODES.has(errorCode)) {\n return true;\n }\n\n // Check message for common patterns\n const message = typeof record.message === 'string' ? record.message.toLowerCase() : '';\n if (\n message.includes('rate limit') ||\n message.includes('timeout') ||\n message.includes('timed out') ||\n message.includes('service unavailable') ||\n message.includes('server error') ||\n message.includes('connection') ||\n message.includes('econnreset')\n ) {\n return true;\n }\n\n return false;\n}\n\n/**\n * Calculate backoff delay with jitter for LLM retries\n */\nfunction calculateLLMBackoff(attempt: number): number {\n const exponentialDelay = LLM_RETRY_CONFIG.baseDelayMs * Math.pow(2, attempt);\n const jitter = Math.random() * BACKOFF_JITTER_FACTOR * exponentialDelay;\n return Math.min(exponentialDelay + jitter, LLM_RETRY_CONFIG.maxDelayMs);\n}\n\n/**\n * Process content with LLM extraction\n * NEVER throws - always returns a valid LLMResult\n * Implements retry logic with exponential backoff for transient failures\n */\nexport async function processContentWithLLM(\n content: string,\n config: ProcessingConfig,\n processor?: OpenAI | null,\n signal?: AbortSignal\n): Promise<LLMResult> {\n // Early returns for invalid/skip conditions\n if (!config.enabled) {\n return { content, processed: false };\n }\n\n if (!processor) {\n return {\n content,\n processed: false,\n error: 'LLM processor not available (LLM_EXTRACTION_API_KEY or OPENROUTER_API_KEY not set)',\n errorDetails: {\n code: ErrorCode.AUTH_ERROR,\n message: 'LLM processor not available',\n retryable: false,\n },\n };\n }\n\n if (!content?.trim()) {\n return { content: content || '', processed: false, error: 'Empty content provided' };\n }\n\n // Truncate extremely long content to avoid token limits\n const truncatedContent = content.length > MAX_LLM_INPUT_CHARS\n ? content.substring(0, MAX_LLM_INPUT_CHARS) + '\\n\\n[Content truncated due to length]'\n : content;\n\n // Sanitize URL before sending to LLM: drop query string and fragment\n // so signed URLs, session tokens, auth params, or tracking hashes never\n // land in a third-party LLM prompt. Keep origin + path for page-type classification.\n const safeUrl = (() => {\n if (!config.url) return undefined;\n try {\n const u = new URL(config.url);\n return `${u.origin}${u.pathname}`;\n } catch {\n return undefined;\n }\n })();\n const urlLine = safeUrl ? `PAGE URL: ${safeUrl}\\n\\n` : '';\n\n const prompt = config.extract\n ? `You are a factual extractor for a research agent. Extract ONLY the information that matches the instruction below. Do not summarize, interpret, or editorialize.\n\n${urlLine}EXTRACTION INSTRUCTION: ${config.extract}\n\nSTEP 1 \u2014 Classify this page. Look at the URL if present, plus structural cues (code blocks, table patterns, comment threads, marketing copy). Pick ONE:\n\\`docs | changelog | github-readme | github-thread | reddit | hackernews | forum | blog | marketing | announcement | qa | cve | paper | release-notes | other\\`\n\nSTEP 2 \u2014 Adjust emphasis by page type:\n- docs / changelog / github-readme / release-notes \u2192 API signatures, version numbers, flags, exact config keys, code blocks. Copy verbatim. Preserve tables as tables.\n- github-thread \u2192 weight MAINTAINER comments (label \"[maintainer]\") over drive-by commenters. Preserve stacktraces verbatim. Capture chronological resolution \u2014 what was decided and when. Link the accepted-fix commit/PR if referenced.\n- reddit / hackernews / forum \u2192 lived experience. Quote verbatim with attribution (\"u/foo wrote: \u2026\" or \"user <name>\"). Prioritize replies with stack details, specific failure stories, or replies that contradict the OP. Record overall sentiment distribution as one bullet if clear skew (\"~70% agree / ~20% dissent / rest off-topic\"). Drop context-free opinions (\"this sucks\") from Matches.\n- blog \u2192 prioritize concrete reproductions, code, measurements. If the author makes a claim without evidence, mark \"[unsourced claim]\".\n- marketing / announcement \u2192 pricing tiers, feature matrices verbatim, free-tier quotas, enterprise contact. Preserve tables as tables. Treat roadmap/future-tense claims skeptically \u2014 note them as \"[announced, not shipped]\" when framing is future-tense.\n- qa (stackoverflow) \u2192 accepted answer's code + high-voted disagreements. Always note the answer date \u2014 SO rots.\n- cve \u2192 CVSS vector verbatim, CWE, CPE ranges, affected versions, fix version, references. Each with its label.\n- paper \u2192 claim, method, dataset, benchmark numbers, comparison baseline. Preserve numeric deltas verbatim.\n\nSTEP 3 \u2014 Emit markdown with these sections, in order:\n\n## Source\n- URL: <verbatim if visible, else \"unknown\">\n- Page type: <the type you picked>\n- Page date: <verbatim if visible, else \"not visible\">\n- Author / maintainer (if identifiable): <verbatim>\n\n## Matches\nOne bullet per distinct piece of matching info:\n- **<short label>** \u2014 the information. Quote VERBATIM for: numbers, versions, dates, API names, prices, error messages, stacktraces, CVSS vectors, benchmark scores, command flags, proper nouns, and people's words. Backticks for code/identifiers. Preserve tables.\n\n## Not found\nEvery part of the extraction instruction this page did NOT answer. Be explicit. Example: \"Enterprise pricing contact \u2014 not present on this page.\"\n\n## Follow-up signals\nShort bullets \u2014 NEW angles this page surfaced that the agent should investigate. Include: new terms, unexpected vendor names, contradicting claims, referenced-but-unscraped URLs. Copy URLs VERBATIM from the source; if only anchor text is visible, write \"anchor: <text> (URL not in scraped content)\". Skip this section if nothing new surfaced. Do NOT invent.\n\n## Contradictions\n(Include this section only if the page contains internally contradictory claims.) Bullet each contradiction with both sides quoted verbatim.\n\n## Truncation\n(Include only if content appears cut mid-element.) \"Content cut mid-<table row / code block / comment / paragraph>; extraction may be incomplete for <section>.\"\n\nRULES:\n- Never paraphrase numbers, versions, code, or quoted text.\n- If an instruction item is not answered, it goes in \"Not found\" \u2014 do NOT invent an answer to please the caller.\n- Preserve code blocks, command examples, tables exactly.\n- Do NOT add commentary or recommendations outside \"Follow-up signals\".\n- Page language \u2260 English: quote verbatim in the original language AND provide a parenthetical gloss in English.\n- Content clearly failed to load: return ONLY a single line, choosing from:\n \\`## Matches\\\\n_Page did not load: 404_\\`\n \\`## Matches\\\\n_Page did not load: login-wall_\\`\n \\`## Matches\\\\n_Page did not load: paywall_\\`\n \\`## Matches\\\\n_Page did not load: JS-render-empty_\\`\n \\`## Matches\\\\n_Page did not load: non-text-asset_\\`\n \\`## Matches\\\\n_Page did not load: truncated-before-relevant-section_\\`\n\nContent:\n${truncatedContent}`\n : `Clean the following page content: drop navigation, ads, cookie banners, footers, author bios, related-article lists. Preserve headings, paragraphs, code blocks, tables, and inline links as \\`[text](url)\\`. Do NOT summarize \u2014 preserve the full body.\n\n${urlLine}Content:\n${truncatedContent}`;\n\n let lastError: StructuredError | undefined;\n\n // Retry loop\n for (let attempt = 0; attempt <= LLM_RETRY_CONFIG.maxRetries; attempt++) {\n try {\n if (attempt === 0) {\n mcpLog('info', `Starting extraction with ${LLM_EXTRACTION.MODEL}`, 'llm');\n } else {\n mcpLog('warning', `Retry attempt ${attempt}/${LLM_RETRY_CONFIG.maxRetries}`, 'llm');\n }\n\n const response = await requestTextWithFallback(\n processor,\n prompt,\n config.max_tokens || LLM_EXTRACTION.MAX_TOKENS,\n 'LLM extraction',\n signal,\n );\n\n if (response.content) {\n mcpLog('info', `Successfully extracted ${response.content.length} characters`, 'llm');\n markLLMSuccess('extractor');\n return { content: response.content, processed: true };\n }\n\n // Empty response - not retryable\n mcpLog('warning', 'Received empty response from LLM', 'llm');\n markLLMFailure('extractor', 'LLM returned empty response');\n return {\n content,\n processed: false,\n error: 'LLM returned empty response',\n errorDetails: {\n code: ErrorCode.INTERNAL_ERROR,\n message: 'LLM returned empty response',\n retryable: false,\n },\n };\n\n } catch (err: unknown) {\n lastError = classifyError(err);\n\n // Log the error\n const status = hasStatus(err) ? err.status : undefined;\n const code = typeof err === 'object' && err !== null && 'code' in err\n ? String((err as Record<string, unknown>).code)\n : undefined;\n mcpLog('error', `Error (attempt ${attempt + 1}): ${lastError.message} [status=${status}, code=${code}, retryable=${isRetryableLLMError(err)}]`, 'llm');\n\n // Check if we should retry\n if (isRetryableLLMError(err) && attempt < LLM_RETRY_CONFIG.maxRetries) {\n const delayMs = calculateLLMBackoff(attempt);\n mcpLog('warning', `Retrying in ${delayMs}ms...`, 'llm');\n try { await sleep(delayMs, signal); } catch { break; }\n continue;\n }\n\n // Non-retryable or max retries reached\n break;\n }\n }\n\n // All attempts failed - return original content with error info\n const errorMessage = lastError?.message || 'Unknown LLM error';\n mcpLog('error', `All attempts failed: ${errorMessage}. Returning original content.`, 'llm');\n markLLMFailure('extractor', errorMessage);\n\n return {\n content, // Return original content as fallback\n processed: false,\n error: `LLM extraction failed: ${errorMessage}`,\n errorDetails: lastError || {\n code: ErrorCode.UNKNOWN_ERROR,\n message: errorMessage,\n retryable: false,\n },\n };\n}\n\n// ============================================================================\n// Web-Search Result Classification\n// ============================================================================\n\n/** Maximum URLs to send to the LLM for classification */\nconst MAX_CLASSIFICATION_URLS = 50 as const;\n\n/** Classification tiers */\ntype ClassificationTier = 'HIGHLY_RELEVANT' | 'MAYBE_RELEVANT' | 'OTHER';\n\nexport interface ClassificationEntry {\n readonly rank: number;\n readonly tier: ClassificationTier;\n readonly source_type?: string;\n readonly reason?: string;\n}\n\nexport interface ClassificationGap {\n readonly id: number;\n readonly description: string;\n}\n\nexport interface ClassificationResult {\n readonly title: string;\n readonly synthesis: string;\n readonly results: ClassificationEntry[];\n readonly refine_queries?: Array<{\n readonly query: string;\n readonly rationale: string;\n readonly gap_id?: number;\n }>;\n readonly confidence?: 'high' | 'medium' | 'low';\n readonly confidence_reason?: string;\n readonly gaps?: ClassificationGap[];\n}\n\nexport interface RefineQuerySuggestion {\n readonly query: string;\n readonly rationale: string;\n readonly gap_id?: number;\n readonly gap_description?: string;\n}\n\n/**\n * Classify web-search results by relevance to an objective using the LLM.\n * Sends only titles, snippets, and domain names \u2014 does NOT fetch URLs.\n * Returns null on failure (caller should fall back to raw output).\n */\nexport async function classifySearchResults(\n rankedUrls: ReadonlyArray<{\n readonly rank: number;\n readonly url: string;\n readonly title: string;\n readonly snippet: string;\n readonly frequency: number;\n readonly queries: string[];\n }>,\n objective: string,\n totalQueries: number,\n processor: OpenAI,\n previousQueries: readonly string[] = [],\n): Promise<{ result: ClassificationResult | null; error?: string }> {\n const urlsToClassify = rankedUrls.slice(0, MAX_CLASSIFICATION_URLS);\n\n // Build compressed result list \u2014 title + domain + snippet (truncated)\n const lines: string[] = [];\n for (const url of urlsToClassify) {\n let domain: string;\n try {\n domain = new URL(url.url).hostname.replace(/^www\\./, '');\n } catch {\n domain = url.url;\n }\n const snippet = url.snippet.length > 120\n ? url.snippet.slice(0, 117) + '...'\n : url.snippet;\n lines.push(`[${url.rank}] ${url.title} \u2014 ${domain} \u2014 ${snippet}`);\n }\n\n const prevQueriesBlock = previousQueries.length > 0\n ? previousQueries.map((q) => `- ${q}`).join('\\n')\n : '- (none provided)';\n const today = new Date().toISOString().slice(0, 10);\n\n const prompt = `You are the relevance filter for a research agent. Classify each search result below against the objective and produce a structured analysis.\n\nOBJECTIVE: ${objective}\nTODAY: ${today}\n\nPREVIOUS QUERIES (already run \u2014 do NOT paraphrase in refine_queries):\n${prevQueriesBlock}\n\nReturn ONLY a JSON object (no markdown, no code fences):\n\n{\n \"title\": \"2\u20138 word label for this RESULT CLUSTER (not the objective)\",\n \"synthesis\": \"3\u20135 sentences grounded in the results. Every non-trivial claim cites a rank in [brackets], e.g. '[3] documents the flag; [7][12] report it is broken on macOS.' A synthesis with zero citations is invalid.\",\n \"confidence\": \"high | medium | low\",\n \"confidence_reason\": \"one sentence \u2014 why\",\n \"gaps\": [\n { \"id\": 0, \"description\": \"specific, actionable thing the current results do NOT answer \u2014 not 'more info needed'\" }\n ],\n \"refine_queries\": [\n { \"query\": \"concrete next search\", \"gap_id\": 0, \"rationale\": \"\u226412 words\" }\n ],\n \"results\": [\n {\n \"rank\": 1,\n \"tier\": \"HIGHLY_RELEVANT | MAYBE_RELEVANT | OTHER\",\n \"source_type\": \"vendor_doc | github | reddit | hackernews | blog | news | marketing | stackoverflow | cve | paper | release_notes | aggregator | other\",\n \"reason\": \"\u226412 words citing the snippet cue that drove the tier\"\n }\n ]\n}\n\nSOURCE-OF-TRUTH RUBRIC (the \"primary source\" is goal-dependent \u2014 infer goal type from the objective):\n- spec / API / config questions \u2192 vendor_doc, github (README, RFC), release_notes are primary\n- bug / failure-mode questions \u2192 github (issue/PR), stackoverflow are primary\n- migration / sentiment / lived-experience \u2192 reddit, hackernews, blog are primary; docs are secondary\n- pricing / commercial \u2192 marketing (the vendor's own pricing page IS the primary source, but treat feature lists skeptically)\n- security / CVE \u2192 cve databases, distro security trackers (nvd.nist.gov, security-tracker.debian.org, ubuntu.com/security) are primary\n- synthesis / open-ended \u2192 blend; no single type is primary\n- product launch \u2192 vendor_doc + news + marketing for the launch itself; blogs + reddit for independent verification\n\nFRESHNESS: proportional to topic velocity. For a week-old release, demote anything older than 30 days. For general tech questions, demote older than 18 months. For stable protocols (HTTP, TCP, POSIX), don't demote by age.\n\nCONFIDENCE:\n- high = \u22653 HIGHLY_RELEVANT results from INDEPENDENT domains agree on the core answer\n- medium = \u22652 HIGHLY_RELEVANT exist but disagree or share a domain; OR a single authoritative primary source answers it\n- low = otherwise; snippet-only judgments cap at medium\n\nREFINE QUERIES \u2014 each MUST differ from every previousQuery by:\n- a new operator (site:, quotes, verbatim version number), OR\n- a domain-specific noun ABSENT from every prior query\nAdding a year alone does NOT count as differentiation.\nEach refine_query MUST reference a specific gap_id from the gaps array above.\nProduce 4\u20138 refine_queries total. Cover: (a) a primary-source probe, (b) a temporal sharpener, (c) a failure-mode or comparison probe, (d) at least one new-term probe seeded by a specific result's snippet.\n\nRULES:\n- Classify ALL ${urlsToClassify.length} results. Do not skip or collapse any.\n- Use only the three tier values.\n- Judge from title + domain + snippet only. Do NOT invent facts not present in the snippet.\n- If ALL results are OTHER: synthesis = \"\", confidence = \"low\", and \\`gaps\\` must explicitly state why the current queries missed the target.\n- Casing: tier = UPPERCASE_WITH_UNDERSCORES, confidence = lowercase.\n\nSEARCH RESULTS (${urlsToClassify.length} URLs from ${totalQueries} queries):\n${lines.join('\\n')}`;\n\n try {\n mcpLog('info', `Classifying ${urlsToClassify.length} URLs against objective`, 'llm');\n\n // Output budget needs room for: 50 results \u00D7 (rank + tier + source_type + reason)\n // plus gaps[] with descriptions, refine_queries[] with rationales, synthesis with citations,\n // title, confidence, and confidence_reason. 4000 truncates JSON mid-structure for full outputs;\n // 8000 leaves headroom for the richest cases.\n const response = await requestTextWithFallback(\n processor,\n prompt,\n 8000,\n 'Search classification',\n );\n\n if (!response.content) {\n const errMsg = response.error ?? 'LLM returned empty classification response';\n markLLMFailure('planner', errMsg);\n return { result: null, error: errMsg };\n }\n\n // Strip markdown code fences if present\n const cleaned = response.content.replace(/^```(?:json)?\\s*\\n?/m, '').replace(/\\n?```\\s*$/m, '').trim();\n const parsed = JSON.parse(cleaned) as ClassificationResult;\n\n // Validate the response shape.\n // Note: synthesis is typed not truthy \u2014 the prompt explicitly instructs an empty string\n // for the all-OTHER case, and we must not reject that.\n if (!parsed.title || typeof parsed.synthesis !== 'string' || !Array.isArray(parsed.results)) {\n const errMsg = 'LLM response missing required fields (title, synthesis, results)';\n markLLMFailure('planner', errMsg);\n return { result: null, error: errMsg };\n }\n\n mcpLog('info', `Classification complete: ${parsed.results.filter(r => r.tier === 'HIGHLY_RELEVANT').length} highly relevant`, 'llm');\n markLLMSuccess('planner');\n return { result: parsed };\n } catch (err: unknown) {\n const message = err instanceof Error ? err.message : String(err);\n mcpLog('error', `Classification failed: ${message}`, 'llm');\n markLLMFailure('planner', message);\n return { result: null, error: `Classification failed: ${message}` };\n }\n}\n\nexport async function suggestRefineQueriesForRawMode(\n rankedUrls: ReadonlyArray<{\n readonly rank: number;\n readonly url: string;\n readonly title: string;\n }>,\n objective: string,\n originalQueries: readonly string[],\n processor: OpenAI,\n): Promise<{ result: RefineQuerySuggestion[]; error?: string }> {\n const urlsToSummarize = rankedUrls.slice(0, 12);\n const lines = urlsToSummarize.map((url) => {\n let domain: string;\n try {\n domain = new URL(url.url).hostname.replace(/^www\\./, '');\n } catch {\n domain = url.url;\n }\n return `[${url.rank}] ${url.title} \u2014 ${domain}`;\n });\n\n const prompt = `You are generating follow-up search queries for an agent using raw web-search results.\n\nReturn ONLY a JSON object (no markdown, no code fences):\n{\n \"refine_queries\": [\n { \"query\": \"next search query\", \"gap_description\": \"what gap this closes\", \"rationale\": \"\u226412 words on why\" }\n ]\n}\n\nOBJECTIVE: ${objective}\n\nPREVIOUS QUERIES (already run \u2014 do NOT paraphrase):\n${originalQueries.map((query) => `- ${query}`).join('\\n')}\n\nTOP RESULT TITLES (to seed new-term probes):\n${lines.join('\\n')}\n\nRULES:\n- Produce 4\u20136 diverse follow-ups. Cover: (a) a primary-source probe (site:, RFC, vendor docs); (b) a temporal sharpener (changelog, version number); (c) a failure-mode or comparison probe; (d) at least one new-term probe seeded by a specific result title.\n- Each query MUST differ from every previousQuery by either a new operator (site:, quotes, a verbatim version number) OR a domain-specific noun absent from every prior query. Adding a year alone does NOT count.\n- Each refine_query MUST include a \\`gap_description\\` naming what the current results don't answer.\n- Do not include URLs.\n- Keep rationales \u226412 words.`;\n\n try {\n const response = await requestTextWithFallback(\n processor,\n prompt,\n 800,\n 'Raw-mode refine query generation',\n );\n\n if (!response.content) {\n const errMsg = response.error ?? 'LLM returned empty raw-mode refine query response';\n markLLMFailure('planner', errMsg);\n return { result: [], error: errMsg };\n }\n\n const cleaned = response.content.replace(/^```(?:json)?\\s*\\n?/m, '').replace(/\\n?```\\s*$/m, '').trim();\n const parsed = JSON.parse(cleaned) as { refine_queries?: RefineQuerySuggestion[] };\n\n markLLMSuccess('planner');\n return { result: Array.isArray(parsed.refine_queries) ? parsed.refine_queries : [] };\n } catch (err: unknown) {\n const message = err instanceof Error ? err.message : String(err);\n mcpLog('error', `Raw-mode refine query generation failed: ${message}`, 'llm');\n markLLMFailure('planner', message);\n return { result: [], error: message };\n }\n}\n\n// ============================================================================\n// Research Brief \u2014 goal-aware orientation (called by start-research)\n// ============================================================================\n\nexport interface ResearchBriefConceptGroup {\n readonly facet: string;\n readonly queries: readonly string[];\n}\n\nexport interface ResearchBrief {\n readonly goal_class: string;\n readonly goal_class_reason: string;\n readonly source_priority: readonly string[];\n readonly sources_to_deprioritize: readonly string[];\n readonly fire_reddit_branch: boolean;\n readonly fire_reddit_reason: string;\n readonly freshness_window: string;\n readonly concept_groups: readonly ResearchBriefConceptGroup[];\n readonly anticipated_gaps: readonly string[];\n readonly first_scrape_targets: readonly string[];\n readonly success_criteria: readonly string[];\n}\n\nconst VALID_GOAL_CLASSES = new Set([\n 'spec', 'bug', 'migration', 'sentiment', 'pricing', 'security',\n 'synthesis', 'product_launch', 'other',\n]);\n\nconst VALID_FRESHNESS = new Set(['days', 'weeks', 'months', 'years']);\n\nfunction isStringArray(value: unknown): value is string[] {\n return Array.isArray(value) && value.every((v) => typeof v === 'string');\n}\n\nfunction isConceptGroupArray(value: unknown): value is ResearchBriefConceptGroup[] {\n return Array.isArray(value) && value.every((g) => {\n if (typeof g !== 'object' || g === null) return false;\n const facet = (g as Record<string, unknown>).facet;\n const queries = (g as Record<string, unknown>).queries;\n return typeof facet === 'string'\n && facet.trim().length > 0\n && isStringArray(queries)\n && queries.length > 0\n && queries.every((q) => q.trim().length > 0);\n });\n}\n\nexport function parseResearchBrief(raw: string): ResearchBrief | null {\n try {\n const cleaned = raw.replace(/^```(?:json)?\\s*\\n?/m, '').replace(/\\n?```\\s*$/m, '').trim();\n const parsed = JSON.parse(cleaned) as Record<string, unknown>;\n\n const goal_class = typeof parsed.goal_class === 'string' ? parsed.goal_class : null;\n if (!goal_class || !VALID_GOAL_CLASSES.has(goal_class)) return null;\n\n const freshness_window = typeof parsed.freshness_window === 'string' ? parsed.freshness_window : null;\n if (!freshness_window || !VALID_FRESHNESS.has(freshness_window)) return null;\n\n if (typeof parsed.fire_reddit_branch !== 'boolean') return null;\n if (!isConceptGroupArray(parsed.concept_groups) || parsed.concept_groups.length === 0) return null;\n\n return {\n goal_class,\n goal_class_reason: typeof parsed.goal_class_reason === 'string' ? parsed.goal_class_reason : '',\n source_priority: isStringArray(parsed.source_priority) ? parsed.source_priority : [],\n sources_to_deprioritize: isStringArray(parsed.sources_to_deprioritize) ? parsed.sources_to_deprioritize : [],\n fire_reddit_branch: parsed.fire_reddit_branch,\n fire_reddit_reason: typeof parsed.fire_reddit_reason === 'string' ? parsed.fire_reddit_reason : '',\n freshness_window,\n concept_groups: parsed.concept_groups,\n anticipated_gaps: isStringArray(parsed.anticipated_gaps) ? parsed.anticipated_gaps : [],\n first_scrape_targets: isStringArray(parsed.first_scrape_targets) ? parsed.first_scrape_targets : [],\n success_criteria: isStringArray(parsed.success_criteria) ? parsed.success_criteria : [],\n };\n } catch {\n return null;\n }\n}\n\nexport async function generateResearchBrief(\n goal: string,\n processor: OpenAI,\n signal?: AbortSignal,\n): Promise<ResearchBrief | null> {\n const today = new Date().toISOString().slice(0, 10);\n\n const prompt = `You are a research planner. An agent is about to run a multi-pass research loop on the goal below. Produce a tailored research brief in JSON.\n\nGOAL: ${goal}\nTODAY: ${today}\n\nReturn ONLY a JSON object (no markdown, no code fences):\n\n{\n \"goal_class\": \"spec | bug | migration | sentiment | pricing | security | synthesis | product_launch | other\",\n \"goal_class_reason\": \"one sentence \u2014 why this class\",\n \"source_priority\": [\"ordered list from: vendor_docs, changelog, github_code, github_issues, reddit, hackernews, blogs, marketing_pages, stackoverflow, cve_databases, arxiv, news, release_notes\"],\n \"sources_to_deprioritize\": [\"same vocabulary \u2014 sources that add noise for this goal\"],\n \"fire_reddit_branch\": true,\n \"fire_reddit_reason\": \"one sentence \u2014 why yes or why no\",\n \"freshness_window\": \"days | weeks | months | years\",\n \"concept_groups\": [\n {\n \"facet\": \"2-4 word facet name\",\n \"queries\": [\"5-10 concrete Google queries using operators where helpful: site:, quotes, version numbers\"]\n }\n ],\n \"anticipated_gaps\": [\"2-5 things likely missing after pass 1 that the agent should watch for\"],\n \"first_scrape_targets\": [\"domain names or URL patterns most likely to contain the answer\"],\n \"success_criteria\": [\"2-4 concrete facts the agent must verify before declaring done\"]\n}\n\nRules:\n- Concept groups must probe DIFFERENT facets. Same noun-phrase cannot repeat across groups.\n- Queries within a group vary by operator/phrasing but probe the same facet.\n- Total queries across all groups: 25\u201350. Narrow bugs fewer; open synthesis more.\n- If the goal mentions a recent release / date / version, freshness_window = days or weeks.\n- Do NOT invent vendor names you are uncertain exist. Leave shaky queries out.\n- source_priority MUST reflect the goal type \u2014 docs for spec, github_issues for bugs, reddit/hackernews/blogs for migration/sentiment, cve_databases for security.\n- fire_reddit_branch should be false for CVE / pricing / API spec / primary-source lookups.`;\n\n try {\n const response = await requestTextWithFallback(\n processor,\n prompt,\n 2500,\n 'Research brief generation',\n signal,\n );\n\n if (!response.content) {\n mcpLog('warning', `Research brief generation returned no content: ${response.error ?? 'unknown'}`, 'llm');\n markLLMFailure('planner', response.error ?? 'empty response');\n return null;\n }\n\n const brief = parseResearchBrief(response.content);\n if (!brief) {\n mcpLog('warning', 'Research brief JSON parse or shape validation failed', 'llm');\n markLLMFailure('planner', 'brief parse/validation failed');\n return null;\n }\n\n markLLMSuccess('planner');\n return brief;\n } catch (err: unknown) {\n const message = err instanceof Error ? err.message : String(err);\n mcpLog('warning', `Research brief generation failed: ${message}`, 'llm');\n markLLMFailure('planner', message);\n return null;\n }\n}\n\nexport function renderResearchBrief(brief: ResearchBrief): string {\n const lines: string[] = [];\n\n lines.push('## Your research brief (goal-tailored)');\n lines.push('');\n lines.push(`**Goal class**: \\`${brief.goal_class}\\` \u2014 ${brief.goal_class_reason}`);\n lines.push(`**Freshness target**: \\`${brief.freshness_window}\\``);\n lines.push(`**Reddit branch**: ${brief.fire_reddit_branch ? '**fire**' : 'skip'} \u2014 ${brief.fire_reddit_reason}`);\n lines.push('');\n\n if (brief.source_priority.length > 0) {\n lines.push('**Source priority** (highest \u2192 lowest):');\n brief.source_priority.forEach((src, i) => lines.push(`${i + 1}. \\`${src}\\``));\n lines.push('');\n }\n\n if (brief.sources_to_deprioritize.length > 0) {\n lines.push(`**Deprioritize**: ${brief.sources_to_deprioritize.map((s) => `\\`${s}\\``).join(', ')}`);\n lines.push('');\n }\n\n lines.push('### Pass 1 concept groups');\n lines.push('');\n lines.push('Issue every query below in ONE `web-search` call (flat array).');\n lines.push('');\n\n for (const group of brief.concept_groups) {\n lines.push(`#### ${group.facet}`);\n for (const query of group.queries) {\n lines.push(`- ${query}`);\n }\n lines.push('');\n }\n\n if (brief.anticipated_gaps.length > 0) {\n lines.push('### Anticipated gaps (watch the classifier\\'s `gaps[]` output for these)');\n brief.anticipated_gaps.forEach((g) => lines.push(`- ${g}`));\n lines.push('');\n }\n\n if (brief.first_scrape_targets.length > 0) {\n lines.push('### First-pass scrape targets (prioritize these in `scrape-links`)');\n brief.first_scrape_targets.forEach((t) => lines.push(`- ${t}`));\n lines.push('');\n }\n\n if (brief.success_criteria.length > 0) {\n lines.push('### Success criteria (do not declare done until all are verified)');\n brief.success_criteria.forEach((c) => lines.push(`- ${c}`));\n lines.push('');\n }\n\n lines.push('---');\n lines.push('');\n lines.push('Run all concept-group queries above in ONE `web-search` call, then loop per the discipline above.');\n\n return lines.join('\\n');\n}\n"],
|
|
5
|
+
"mappings": "AAMA,OAAO,YAAY;AACnB,SAAS,gBAAgB,uBAAuB;AAChD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP,SAAS,cAAc;AAGvB,MAAM,sBAAsB;AAG5B,MAAM,wBAAwB;AAG9B,MAAM,wBAAwB;AAG9B,MAAM,uBAAuB;AAG7B,MAAM,0BAA0B;AAoBhC,MAAM,YAAY;AAAA,EAChB,eAAe;AAAA,EACf,iBAAiB;AAAA,EACjB,sBAAsB;AAAA,EACtB,wBAAwB;AAAA,EACxB,kBAAkB;AAAA,EAClB,oBAAoB;AACtB;AAEO,SAAS,eAAe,MAA2B;AACxD,QAAM,MAAK,oBAAI,KAAK,GAAE,YAAY;AAClC,MAAI,SAAS,WAAW;AACtB,cAAU,gBAAgB;AAC1B,cAAU,uBAAuB;AACjC,cAAU,mBAAmB;AAAA,EAC/B,OAAO;AACL,cAAU,kBAAkB;AAC5B,cAAU,yBAAyB;AACnC,cAAU,qBAAqB;AAAA,EACjC;AACF;AAEO,SAAS,eAAe,MAAqB,KAAoB;AACtE,QAAM,MAAK,oBAAI,KAAK,GAAE,YAAY;AAClC,QAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,OAAO,eAAe;AAClF,MAAI,SAAS,WAAW;AACtB,cAAU,gBAAgB;AAC1B,cAAU,uBAAuB;AACjC,cAAU,mBAAmB;AAAA,EAC/B,OAAO;AACL,cAAU,kBAAkB;AAC5B,cAAU,yBAAyB;AACnC,cAAU,qBAAqB;AAAA,EACjC;AACF;AAEO,SAAS,eAAkC;AAChD,QAAM,MAAM,gBAAgB;AAC5B,SAAO;AAAA,IACL,eAAe,UAAU;AAAA,IACzB,iBAAiB,UAAU;AAAA,IAC3B,sBAAsB,UAAU;AAAA,IAChC,wBAAwB,UAAU;AAAA,IAClC,kBAAkB,UAAU;AAAA,IAC5B,oBAAoB,UAAU;AAAA;AAAA;AAAA,IAG9B,mBAAmB,IAAI;AAAA,IACvB,qBAAqB,IAAI;AAAA,EAC3B;AACF;AAGO,SAAS,0BAAgC;AAC9C,YAAU,gBAAgB;AAC1B,YAAU,kBAAkB;AAC5B,YAAU,uBAAuB;AACjC,YAAU,yBAAyB;AACnC,YAAU,mBAAmB;AAC7B,YAAU,qBAAqB;AACjC;AAiBA,MAAM,mBAAmB;AAAA,EACvB,YAAY;AAAA,EACZ,aAAa;AAAA,EACb,YAAY;AACd;AAGA,MAAM,4BAA4B,oBAAI,IAAI;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAGD,SAAS,UAAU,OAA6C;AAC9D,SACE,OAAO,UAAU,YACjB,UAAU,QACV,YAAY,SACZ,OAAQ,MAAkC,WAAW;AAEzD;AAEA,IAAI,YAA2B;AAIxB,SAAS,qBAAoC;AAClD,MAAI,CAAC,gBAAgB,EAAE,cAAe,QAAO;AAE7C,MAAI,CAAC,WAAW;AACd,gBAAY,IAAI,OAAO;AAAA,MACrB,SAAS,eAAe;AAAA,MACxB,QAAQ,eAAe;AAAA,MACvB,SAAS;AAAA,MACT,YAAY;AAAA,MACZ,gBAAgB,EAAE,WAAW,yBAAyB;AAAA,IACxD,CAAC;AACD,WAAO,QAAQ,qCAAqC,eAAe,KAAK,cAAc,eAAe,QAAQ,KAAK,KAAK;AAAA,EACzH;AACA,SAAO;AACT;AAEA,SAAS,qBAAqB,OAAe,QAAgB,WAA4C;AACvG,QAAM,cAAuC;AAAA,IAC3C;AAAA,IACA,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC5C,YAAY;AAAA,EACd;AAEA,MAAI,eAAe,qBAAqB,QAAQ;AAC9C,gBAAY,mBAAmB,eAAe;AAAA,EAChD;AAEA,SAAO;AACT;AAEA,eAAsB,wBACpB,WACA,QACA,WACA,gBACA,QACoE;AACpE,QAAM,SAAS,CAAC,GAAG,IAAI,IAAI;AAAA,IACzB,eAAe;AAAA,IACf,eAAe;AAAA,EACjB,EAAE,OAAO,OAAO,CAAC,CAAC;AAElB,MAAI,YAAY;AAEhB,aAAW,SAAS,QAAQ;AAC1B,QAAI;AACF,YAAM,WAAW,MAAM;AAAA,QACrB,CAAC,gBAAgB,UAAU,KAAK,YAAY;AAAA,UAC1C,qBAAqB,OAAO,QAAQ,SAAS;AAAA,UAC7C;AAAA,YACE,QAAQ,SAAS,YAAY,IAAI,CAAC,aAAa,MAAM,CAAC,IAAI;AAAA,YAC1D,SAAS;AAAA,UACX;AAAA,QACF;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,cAAc,KAAK,KAAK;AAAA,MAC7B;AAEA,YAAM,UAAU,SAAS,UAAU,CAAC,GAAG,SAAS,SAAS,KAAK;AAC9D,UAAI,SAAS;AACX,YAAI,UAAU,eAAe,OAAO;AAClC,iBAAO,WAAW,GAAG,cAAc,kCAAkC,KAAK,IAAI,KAAK;AAAA,QACrF;AACA,eAAO,EAAE,SAAS,MAAM;AAAA,MAC1B;AAEA,kBAAY,6BAA6B,KAAK;AAC9C,aAAO,WAAW,GAAG,cAAc,qCAAqC,KAAK,IAAI,KAAK;AAAA,IACxF,SAAS,KAAc;AACrB,kBAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,aAAO,WAAW,GAAG,cAAc,qBAAqB,KAAK,KAAK,SAAS,IAAI,KAAK;AAAA,IACtF;AAAA,EACF;AAEA,SAAO,EAAE,SAAS,MAAM,OAAO,eAAe,gBAAgB,OAAO,UAAU;AACjF;AAKA,SAAS,oBAAoB,OAAyB;AACpD,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAGhD,QAAM,YAAa,OAA6B;AAChD,MAAI,cAAc,cAAc,cAAc,aAAa;AACzD,WAAO;AAAA,EACT;AAGA,MAAI,UAAU,KAAK,GAAG;AACpB,QAAI,MAAM,WAAW,OAAO,MAAM,WAAW,OAAO,MAAM,WAAW,OAAO,MAAM,WAAW,OAAO,MAAM,WAAW,KAAK;AACxH,aAAO;AAAA,IACT;AAAA,EACF;AAGA,QAAM,SAAS;AACf,QAAM,OAAO,OAAO,OAAO,SAAS,WAAW,OAAO,OAAO;AAC7D,QAAM,SACJ,OAAO,OAAO,UAAU,YAAY,OAAO,UAAU,OAChD,OAAO,QACR;AACN,QAAM,YACJ,SACC,UAAU,OAAO,OAAO,SAAS,WAAW,OAAO,OAAO,YAC1D,UAAU,OAAO,OAAO,SAAS,WAAW,OAAO,OAAO;AAC7D,MAAI,aAAa,0BAA0B,IAAI,SAAS,GAAG;AACzD,WAAO;AAAA,EACT;AAGA,QAAM,UAAU,OAAO,OAAO,YAAY,WAAW,OAAO,QAAQ,YAAY,IAAI;AACpF,MACE,QAAQ,SAAS,YAAY,KAC7B,QAAQ,SAAS,SAAS,KAC1B,QAAQ,SAAS,WAAW,KAC5B,QAAQ,SAAS,qBAAqB,KACtC,QAAQ,SAAS,cAAc,KAC/B,QAAQ,SAAS,YAAY,KAC7B,QAAQ,SAAS,YAAY,GAC7B;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAKA,SAAS,oBAAoB,SAAyB;AACpD,QAAM,mBAAmB,iBAAiB,cAAc,KAAK,IAAI,GAAG,OAAO;AAC3E,QAAM,SAAS,KAAK,OAAO,IAAI,wBAAwB;AACvD,SAAO,KAAK,IAAI,mBAAmB,QAAQ,iBAAiB,UAAU;AACxE;AAOA,eAAsB,sBACpB,SACA,QACA,WACA,QACoB;AAEpB,MAAI,CAAC,OAAO,SAAS;AACnB,WAAO,EAAE,SAAS,WAAW,MAAM;AAAA,EACrC;AAEA,MAAI,CAAC,WAAW;AACd,WAAO;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,OAAO;AAAA,MACP,cAAc;AAAA,QACZ,MAAM,UAAU;AAAA,QAChB,SAAS;AAAA,QACT,WAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AAEA,MAAI,CAAC,SAAS,KAAK,GAAG;AACpB,WAAO,EAAE,SAAS,WAAW,IAAI,WAAW,OAAO,OAAO,yBAAyB;AAAA,EACrF;AAGA,QAAM,mBAAmB,QAAQ,SAAS,sBACtC,QAAQ,UAAU,GAAG,mBAAmB,IAAI,0CAC5C;AAKJ,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,OAAO,IAAK,QAAO;AACxB,QAAI;AACF,YAAM,IAAI,IAAI,IAAI,OAAO,GAAG;AAC5B,aAAO,GAAG,EAAE,MAAM,GAAG,EAAE,QAAQ;AAAA,IACjC,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF,GAAG;AACH,QAAM,UAAU,UAAU,aAAa,OAAO;AAAA;AAAA,IAAS;AAEvD,QAAM,SAAS,OAAO,UAClB;AAAA;AAAA,EAEJ,OAAO,2BAA2B,OAAO,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAsDhD,gBAAgB,KACZ;AAAA;AAAA,EAEJ,OAAO;AAAA,EACP,gBAAgB;AAEhB,MAAI;AAGJ,WAAS,UAAU,GAAG,WAAW,iBAAiB,YAAY,WAAW;AACvE,QAAI;AACF,UAAI,YAAY,GAAG;AACjB,eAAO,QAAQ,4BAA4B,eAAe,KAAK,IAAI,KAAK;AAAA,MAC1E,OAAO;AACL,eAAO,WAAW,iBAAiB,OAAO,IAAI,iBAAiB,UAAU,IAAI,KAAK;AAAA,MACpF;AAEA,YAAM,WAAW,MAAM;AAAA,QACrB;AAAA,QACA;AAAA,QACA,OAAO,cAAc,eAAe;AAAA,QACpC;AAAA,QACA;AAAA,MACF;AAEA,UAAI,SAAS,SAAS;AACpB,eAAO,QAAQ,0BAA0B,SAAS,QAAQ,MAAM,eAAe,KAAK;AACpF,uBAAe,WAAW;AAC1B,eAAO,EAAE,SAAS,SAAS,SAAS,WAAW,KAAK;AAAA,MACtD;AAGA,aAAO,WAAW,oCAAoC,KAAK;AAC3D,qBAAe,aAAa,6BAA6B;AACzD,aAAO;AAAA,QACL;AAAA,QACA,WAAW;AAAA,QACX,OAAO;AAAA,QACP,cAAc;AAAA,UACZ,MAAM,UAAU;AAAA,UAChB,SAAS;AAAA,UACT,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IAEF,SAAS,KAAc;AACrB,kBAAY,cAAc,GAAG;AAG7B,YAAM,SAAS,UAAU,GAAG,IAAI,IAAI,SAAS;AAC7C,YAAM,OAAO,OAAO,QAAQ,YAAY,QAAQ,QAAQ,UAAU,MAC9D,OAAQ,IAAgC,IAAI,IAC5C;AACJ,aAAO,SAAS,kBAAkB,UAAU,CAAC,MAAM,UAAU,OAAO,YAAY,MAAM,UAAU,IAAI,eAAe,oBAAoB,GAAG,CAAC,KAAK,KAAK;AAGrJ,UAAI,oBAAoB,GAAG,KAAK,UAAU,iBAAiB,YAAY;AACrE,cAAM,UAAU,oBAAoB,OAAO;AAC3C,eAAO,WAAW,eAAe,OAAO,SAAS,KAAK;AACtD,YAAI;AAAE,gBAAM,MAAM,SAAS,MAAM;AAAA,QAAG,QAAQ;AAAE;AAAA,QAAO;AACrD;AAAA,MACF;AAGA;AAAA,IACF;AAAA,EACF;AAGA,QAAM,eAAe,WAAW,WAAW;AAC3C,SAAO,SAAS,wBAAwB,YAAY,iCAAiC,KAAK;AAC1F,iBAAe,aAAa,YAAY;AAExC,SAAO;AAAA,IACL;AAAA;AAAA,IACA,WAAW;AAAA,IACX,OAAO,0BAA0B,YAAY;AAAA,IAC7C,cAAc,aAAa;AAAA,MACzB,MAAM,UAAU;AAAA,MAChB,SAAS;AAAA,MACT,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAOA,MAAM,0BAA0B;AA2ChC,eAAsB,sBACpB,YAQA,WACA,cACA,WACA,kBAAqC,CAAC,GAC4B;AAClE,QAAM,iBAAiB,WAAW,MAAM,GAAG,uBAAuB;AAGlE,QAAM,QAAkB,CAAC;AACzB,aAAW,OAAO,gBAAgB;AAChC,QAAI;AACJ,QAAI;AACF,eAAS,IAAI,IAAI,IAAI,GAAG,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IACzD,QAAQ;AACN,eAAS,IAAI;AAAA,IACf;AACA,UAAM,UAAU,IAAI,QAAQ,SAAS,MACjC,IAAI,QAAQ,MAAM,GAAG,GAAG,IAAI,QAC5B,IAAI;AACR,UAAM,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,KAAK,WAAM,MAAM,WAAM,OAAO,EAAE;AAAA,EAClE;AAEA,QAAM,mBAAmB,gBAAgB,SAAS,IAC9C,gBAAgB,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,EAAE,KAAK,IAAI,IAC9C;AACJ,QAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY,EAAE,MAAM,GAAG,EAAE;AAElD,QAAM,SAAS;AAAA;AAAA,aAEJ,SAAS;AAAA,SACb,KAAK;AAAA;AAAA;AAAA,EAGZ,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAiDD,eAAe,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,kBAMpB,eAAe,MAAM,cAAc,YAAY;AAAA,EAC/D,MAAM,KAAK,IAAI,CAAC;AAEhB,MAAI;AACF,WAAO,QAAQ,eAAe,eAAe,MAAM,2BAA2B,KAAK;AAMnF,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,SAAS;AACrB,YAAM,SAAS,SAAS,SAAS;AACjC,qBAAe,WAAW,MAAM;AAChC,aAAO,EAAE,QAAQ,MAAM,OAAO,OAAO;AAAA,IACvC;AAGA,UAAM,UAAU,SAAS,QAAQ,QAAQ,wBAAwB,EAAE,EAAE,QAAQ,eAAe,EAAE,EAAE,KAAK;AACrG,UAAM,SAAS,KAAK,MAAM,OAAO;AAKjC,QAAI,CAAC,OAAO,SAAS,OAAO,OAAO,cAAc,YAAY,CAAC,MAAM,QAAQ,OAAO,OAAO,GAAG;AAC3F,YAAM,SAAS;AACf,qBAAe,WAAW,MAAM;AAChC,aAAO,EAAE,QAAQ,MAAM,OAAO,OAAO;AAAA,IACvC;AAEA,WAAO,QAAQ,4BAA4B,OAAO,QAAQ,OAAO,OAAK,EAAE,SAAS,iBAAiB,EAAE,MAAM,oBAAoB,KAAK;AACnI,mBAAe,SAAS;AACxB,WAAO,EAAE,QAAQ,OAAO;AAAA,EAC1B,SAAS,KAAc;AACrB,UAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,WAAO,SAAS,0BAA0B,OAAO,IAAI,KAAK;AAC1D,mBAAe,WAAW,OAAO;AACjC,WAAO,EAAE,QAAQ,MAAM,OAAO,0BAA0B,OAAO,GAAG;AAAA,EACpE;AACF;AAEA,eAAsB,+BACpB,YAKA,WACA,iBACA,WAC8D;AAC9D,QAAM,kBAAkB,WAAW,MAAM,GAAG,EAAE;AAC9C,QAAM,QAAQ,gBAAgB,IAAI,CAAC,QAAQ;AACzC,QAAI;AACJ,QAAI;AACF,eAAS,IAAI,IAAI,IAAI,GAAG,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IACzD,QAAQ;AACN,eAAS,IAAI;AAAA,IACf;AACA,WAAO,IAAI,IAAI,IAAI,KAAK,IAAI,KAAK,WAAM,MAAM;AAAA,EAC/C,CAAC;AAED,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aASJ,SAAS;AAAA;AAAA;AAAA,EAGpB,gBAAgB,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA,EAGvD,MAAM,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAShB,MAAI;AACF,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,SAAS;AACrB,YAAM,SAAS,SAAS,SAAS;AACjC,qBAAe,WAAW,MAAM;AAChC,aAAO,EAAE,QAAQ,CAAC,GAAG,OAAO,OAAO;AAAA,IACrC;AAEA,UAAM,UAAU,SAAS,QAAQ,QAAQ,wBAAwB,EAAE,EAAE,QAAQ,eAAe,EAAE,EAAE,KAAK;AACrG,UAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,mBAAe,SAAS;AACxB,WAAO,EAAE,QAAQ,MAAM,QAAQ,OAAO,cAAc,IAAI,OAAO,iBAAiB,CAAC,EAAE;AAAA,EACrF,SAAS,KAAc;AACrB,UAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,WAAO,SAAS,4CAA4C,OAAO,IAAI,KAAK;AAC5E,mBAAe,WAAW,OAAO;AACjC,WAAO,EAAE,QAAQ,CAAC,GAAG,OAAO,QAAQ;AAAA,EACtC;AACF;AAyBA,MAAM,qBAAqB,oBAAI,IAAI;AAAA,EACjC;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAa;AAAA,EAAa;AAAA,EAAW;AAAA,EACpD;AAAA,EAAa;AAAA,EAAkB;AACjC,CAAC;AAED,MAAM,kBAAkB,oBAAI,IAAI,CAAC,QAAQ,SAAS,UAAU,OAAO,CAAC;AAEpE,SAAS,cAAc,OAAmC;AACxD,SAAO,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAM,CAAC,MAAM,OAAO,MAAM,QAAQ;AACzE;AAEA,SAAS,oBAAoB,OAAsD;AACjF,SAAO,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAM,CAAC,MAAM;AAChD,QAAI,OAAO,MAAM,YAAY,MAAM,KAAM,QAAO;AAChD,UAAM,QAAS,EAA8B;AAC7C,UAAM,UAAW,EAA8B;AAC/C,WAAO,OAAO,UAAU,YACnB,MAAM,KAAK,EAAE,SAAS,KACtB,cAAc,OAAO,KACrB,QAAQ,SAAS,KACjB,QAAQ,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC;AAAA,EAC/C,CAAC;AACH;AAEO,SAAS,mBAAmB,KAAmC;AACpE,MAAI;AACF,UAAM,UAAU,IAAI,QAAQ,wBAAwB,EAAE,EAAE,QAAQ,eAAe,EAAE,EAAE,KAAK;AACxF,UAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAM,aAAa,OAAO,OAAO,eAAe,WAAW,OAAO,aAAa;AAC/E,QAAI,CAAC,cAAc,CAAC,mBAAmB,IAAI,UAAU,EAAG,QAAO;AAE/D,UAAM,mBAAmB,OAAO,OAAO,qBAAqB,WAAW,OAAO,mBAAmB;AACjG,QAAI,CAAC,oBAAoB,CAAC,gBAAgB,IAAI,gBAAgB,EAAG,QAAO;AAExE,QAAI,OAAO,OAAO,uBAAuB,UAAW,QAAO;AAC3D,QAAI,CAAC,oBAAoB,OAAO,cAAc,KAAK,OAAO,eAAe,WAAW,EAAG,QAAO;AAE9F,WAAO;AAAA,MACL;AAAA,MACA,mBAAmB,OAAO,OAAO,sBAAsB,WAAW,OAAO,oBAAoB;AAAA,MAC7F,iBAAiB,cAAc,OAAO,eAAe,IAAI,OAAO,kBAAkB,CAAC;AAAA,MACnF,yBAAyB,cAAc,OAAO,uBAAuB,IAAI,OAAO,0BAA0B,CAAC;AAAA,MAC3G,oBAAoB,OAAO;AAAA,MAC3B,oBAAoB,OAAO,OAAO,uBAAuB,WAAW,OAAO,qBAAqB;AAAA,MAChG;AAAA,MACA,gBAAgB,OAAO;AAAA,MACvB,kBAAkB,cAAc,OAAO,gBAAgB,IAAI,OAAO,mBAAmB,CAAC;AAAA,MACtF,sBAAsB,cAAc,OAAO,oBAAoB,IAAI,OAAO,uBAAuB,CAAC;AAAA,MAClG,kBAAkB,cAAc,OAAO,gBAAgB,IAAI,OAAO,mBAAmB,CAAC;AAAA,IACxF;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,sBACpB,MACA,WACA,QAC+B;AAC/B,QAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY,EAAE,MAAM,GAAG,EAAE;AAElD,QAAM,SAAS;AAAA;AAAA,QAET,IAAI;AAAA,SACH,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAgCZ,MAAI;AACF,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI,CAAC,SAAS,SAAS;AACrB,aAAO,WAAW,kDAAkD,SAAS,SAAS,SAAS,IAAI,KAAK;AACxG,qBAAe,WAAW,SAAS,SAAS,gBAAgB;AAC5D,aAAO;AAAA,IACT;AAEA,UAAM,QAAQ,mBAAmB,SAAS,OAAO;AACjD,QAAI,CAAC,OAAO;AACV,aAAO,WAAW,wDAAwD,KAAK;AAC/E,qBAAe,WAAW,+BAA+B;AACzD,aAAO;AAAA,IACT;AAEA,mBAAe,SAAS;AACxB,WAAO;AAAA,EACT,SAAS,KAAc;AACrB,UAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,WAAO,WAAW,qCAAqC,OAAO,IAAI,KAAK;AACvE,mBAAe,WAAW,OAAO;AACjC,WAAO;AAAA,EACT;AACF;AAEO,SAAS,oBAAoB,OAA8B;AAChE,QAAM,QAAkB,CAAC;AAEzB,QAAM,KAAK,wCAAwC;AACnD,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,qBAAqB,MAAM,UAAU,aAAQ,MAAM,iBAAiB,EAAE;AACjF,QAAM,KAAK,2BAA2B,MAAM,gBAAgB,IAAI;AAChE,QAAM,KAAK,sBAAsB,MAAM,qBAAqB,aAAa,MAAM,WAAM,MAAM,kBAAkB,EAAE;AAC/G,QAAM,KAAK,EAAE;AAEb,MAAI,MAAM,gBAAgB,SAAS,GAAG;AACpC,UAAM,KAAK,8CAAyC;AACpD,UAAM,gBAAgB,QAAQ,CAAC,KAAK,MAAM,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;AAC5E,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,MAAM,wBAAwB,SAAS,GAAG;AAC5C,UAAM,KAAK,qBAAqB,MAAM,wBAAwB,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,EAAE;AACjG,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,KAAK,2BAA2B;AACtC,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,gEAAgE;AAC3E,QAAM,KAAK,EAAE;AAEb,aAAW,SAAS,MAAM,gBAAgB;AACxC,UAAM,KAAK,QAAQ,MAAM,KAAK,EAAE;AAChC,eAAW,SAAS,MAAM,SAAS;AACjC,YAAM,KAAK,KAAK,KAAK,EAAE;AAAA,IACzB;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,MAAM,iBAAiB,SAAS,GAAG;AACrC,UAAM,KAAK,yEAA0E;AACrF,UAAM,iBAAiB,QAAQ,CAAC,MAAM,MAAM,KAAK,KAAK,CAAC,EAAE,CAAC;AAC1D,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,MAAM,qBAAqB,SAAS,GAAG;AACzC,UAAM,KAAK,oEAAoE;AAC/E,UAAM,qBAAqB,QAAQ,CAAC,MAAM,MAAM,KAAK,KAAK,CAAC,EAAE,CAAC;AAC9D,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,MAAM,iBAAiB,SAAS,GAAG;AACrC,UAAM,KAAK,mEAAmE;AAC9E,UAAM,iBAAiB,QAAQ,CAAC,MAAM,MAAM,KAAK,KAAK,CAAC,EAAE,CAAC;AAC1D,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,KAAK,KAAK;AAChB,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,mGAAmG;AAE9G,SAAO,MAAM,KAAK,IAAI;AACxB;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Logger } from "mcp-use";
|
|
2
|
+
import TurndownService from "turndown";
|
|
3
|
+
const logger = Logger.get("markdown-cleaner");
|
|
4
|
+
const turndown = new TurndownService({
|
|
5
|
+
headingStyle: "atx",
|
|
6
|
+
codeBlockStyle: "fenced",
|
|
7
|
+
bulletListMarker: "-"
|
|
8
|
+
});
|
|
9
|
+
turndown.remove(["script", "style", "nav", "footer", "aside", "noscript"]);
|
|
10
|
+
const MAX_CONTENT_LENGTH = 524288;
|
|
11
|
+
function removeHtmlComments(html) {
|
|
12
|
+
const parts = [];
|
|
13
|
+
let pos = 0;
|
|
14
|
+
while (pos < html.length) {
|
|
15
|
+
const start = html.indexOf("<!--", pos);
|
|
16
|
+
if (start === -1) {
|
|
17
|
+
parts.push(html.substring(pos));
|
|
18
|
+
break;
|
|
19
|
+
}
|
|
20
|
+
if (start > pos) parts.push(html.substring(pos, start));
|
|
21
|
+
const end = html.indexOf("-->", start + 4);
|
|
22
|
+
if (end === -1) {
|
|
23
|
+
parts.push(html.substring(start));
|
|
24
|
+
break;
|
|
25
|
+
}
|
|
26
|
+
pos = end + 3;
|
|
27
|
+
}
|
|
28
|
+
return parts.join("");
|
|
29
|
+
}
|
|
30
|
+
class MarkdownCleaner {
|
|
31
|
+
/**
|
|
32
|
+
* Process HTML content and convert to clean Markdown
|
|
33
|
+
* NEVER throws - returns original content on any error for graceful degradation
|
|
34
|
+
*/
|
|
35
|
+
processContent(htmlContent) {
|
|
36
|
+
try {
|
|
37
|
+
if (!htmlContent || typeof htmlContent !== "string") {
|
|
38
|
+
return htmlContent || "";
|
|
39
|
+
}
|
|
40
|
+
if (!htmlContent.includes("<")) {
|
|
41
|
+
return htmlContent.trim();
|
|
42
|
+
}
|
|
43
|
+
if (htmlContent.length > MAX_CONTENT_LENGTH) {
|
|
44
|
+
htmlContent = htmlContent.substring(0, MAX_CONTENT_LENGTH);
|
|
45
|
+
}
|
|
46
|
+
let content = removeHtmlComments(htmlContent);
|
|
47
|
+
content = turndown.turndown(content);
|
|
48
|
+
content = content.replace(/\n{3,}/g, "\n\n");
|
|
49
|
+
content = content.trim();
|
|
50
|
+
return content;
|
|
51
|
+
} catch (error) {
|
|
52
|
+
logger.warn(
|
|
53
|
+
`processContent failed: ${error instanceof Error ? error.message : String(error)} | Content length: ${htmlContent?.length ?? 0}`
|
|
54
|
+
);
|
|
55
|
+
return htmlContent || "";
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
export {
|
|
60
|
+
MarkdownCleaner
|
|
61
|
+
};
|
|
62
|
+
//# sourceMappingURL=markdown-cleaner.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/services/markdown-cleaner.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Markdown cleaner service using Turndown for HTML to Markdown conversion\n */\nimport { Logger } from 'mcp-use';\nimport TurndownService from 'turndown';\n\nconst logger = Logger.get('markdown-cleaner');\n\nconst turndown = new TurndownService({\n headingStyle: 'atx',\n codeBlockStyle: 'fenced',\n bulletListMarker: '-',\n});\n\n// Remove script, style, nav, footer, aside elements\nturndown.remove(['script', 'style', 'nav', 'footer', 'aside', 'noscript']);\n\n/** Maximum HTML content length in characters \u2014 prevents event loop blocking on huge pages */\nconst MAX_CONTENT_LENGTH = 524_288 as const; // 512 * 1024\n\n/**\n * Remove HTML comments using linear-time indexOf loop.\n * Avoids catastrophic backtracking from /<!--[\\s\\S]*?-->/g on malformed HTML.\n */\nfunction removeHtmlComments(html: string): string {\n const parts: string[] = [];\n let pos = 0;\n while (pos < html.length) {\n const start = html.indexOf('<!--', pos);\n if (start === -1) { parts.push(html.substring(pos)); break; }\n if (start > pos) parts.push(html.substring(pos, start));\n const end = html.indexOf('-->', start + 4);\n if (end === -1) {\n parts.push(html.substring(start)); // preserve unclosed comment + rest\n break;\n }\n pos = end + 3;\n }\n return parts.join('');\n}\n\nexport class MarkdownCleaner {\n /**\n * Process HTML content and convert to clean Markdown\n * NEVER throws - returns original content on any error for graceful degradation\n */\n processContent(htmlContent: string): string {\n try {\n // Handle null/undefined/non-string inputs gracefully\n if (!htmlContent || typeof htmlContent !== 'string') {\n return htmlContent || '';\n }\n\n // If already markdown (no HTML tags), return as-is\n if (!htmlContent.includes('<')) {\n return htmlContent.trim();\n }\n\n // Truncate oversized HTML to prevent blocking the event loop\n if (htmlContent.length > MAX_CONTENT_LENGTH) {\n htmlContent = htmlContent.substring(0, MAX_CONTENT_LENGTH);\n }\n\n // Remove HTML comments before conversion (linear-time)\n let content = removeHtmlComments(htmlContent);\n\n // Convert HTML to Markdown using Turndown\n content = turndown.turndown(content);\n\n // Clean up whitespace\n content = content.replace(/\\n{3,}/g, '\\n\\n');\n content = content.trim();\n\n return content;\n } catch (error) {\n // Log error but don't crash - return original content for graceful degradation\n logger.warn(\n `processContent failed: ${error instanceof Error ? error.message : String(error)} | Content length: ${htmlContent?.length ?? 0}`,\n );\n // Return original content if conversion fails\n return htmlContent || '';\n }\n }\n}\n"],
|
|
5
|
+
"mappings": "AAGA,SAAS,cAAc;AACvB,OAAO,qBAAqB;AAE5B,MAAM,SAAS,OAAO,IAAI,kBAAkB;AAE5C,MAAM,WAAW,IAAI,gBAAgB;AAAA,EACnC,cAAc;AAAA,EACd,gBAAgB;AAAA,EAChB,kBAAkB;AACpB,CAAC;AAGD,SAAS,OAAO,CAAC,UAAU,SAAS,OAAO,UAAU,SAAS,UAAU,CAAC;AAGzE,MAAM,qBAAqB;AAM3B,SAAS,mBAAmB,MAAsB;AAChD,QAAM,QAAkB,CAAC;AACzB,MAAI,MAAM;AACV,SAAO,MAAM,KAAK,QAAQ;AACxB,UAAM,QAAQ,KAAK,QAAQ,QAAQ,GAAG;AACtC,QAAI,UAAU,IAAI;AAAE,YAAM,KAAK,KAAK,UAAU,GAAG,CAAC;AAAG;AAAA,IAAO;AAC5D,QAAI,QAAQ,IAAK,OAAM,KAAK,KAAK,UAAU,KAAK,KAAK,CAAC;AACtD,UAAM,MAAM,KAAK,QAAQ,OAAO,QAAQ,CAAC;AACzC,QAAI,QAAQ,IAAI;AACd,YAAM,KAAK,KAAK,UAAU,KAAK,CAAC;AAChC;AAAA,IACF;AACA,UAAM,MAAM;AAAA,EACd;AACA,SAAO,MAAM,KAAK,EAAE;AACtB;AAEO,MAAM,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA,EAK3B,eAAe,aAA6B;AAC1C,QAAI;AAEF,UAAI,CAAC,eAAe,OAAO,gBAAgB,UAAU;AACnD,eAAO,eAAe;AAAA,MACxB;AAGA,UAAI,CAAC,YAAY,SAAS,GAAG,GAAG;AAC9B,eAAO,YAAY,KAAK;AAAA,MAC1B;AAGA,UAAI,YAAY,SAAS,oBAAoB;AAC3C,sBAAc,YAAY,UAAU,GAAG,kBAAkB;AAAA,MAC3D;AAGA,UAAI,UAAU,mBAAmB,WAAW;AAG5C,gBAAU,SAAS,SAAS,OAAO;AAGnC,gBAAU,QAAQ,QAAQ,WAAW,MAAM;AAC3C,gBAAU,QAAQ,KAAK;AAEvB,aAAO;AAAA,IACT,SAAS,OAAO;AAEd,aAAO;AAAA,QACL,0BAA0B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC,sBAAsB,aAAa,UAAU,CAAC;AAAA,MAChI;AAEA,aAAO,eAAe;AAAA,IACxB;AAAA,EACF;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { createClient } from "redis";
|
|
2
|
+
const WORKFLOW_STATE_TTL_SECONDS = 60 * 60 * 24;
|
|
3
|
+
const WORKFLOW_STATE_PREFIX = "research-pack:workflow:";
|
|
4
|
+
const SWEEP_INTERVAL_MS = 6e4;
|
|
5
|
+
function emptyWorkflowState() {
|
|
6
|
+
return {
|
|
7
|
+
bootstrapped: false,
|
|
8
|
+
redditWarningShown: false,
|
|
9
|
+
orientationVersion: 1
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
function createMemoryWorkflowStateStore(opts = {}) {
|
|
13
|
+
const now = opts.now ?? (() => Date.now());
|
|
14
|
+
const sweepIntervalMs = opts.sweepIntervalMs ?? SWEEP_INTERVAL_MS;
|
|
15
|
+
const ttlMs = (opts.ttlSeconds ?? WORKFLOW_STATE_TTL_SECONDS) * 1e3;
|
|
16
|
+
const states = /* @__PURE__ */ new Map();
|
|
17
|
+
let lastSweepAt = 0;
|
|
18
|
+
function sweep(currentTime) {
|
|
19
|
+
if (currentTime - lastSweepAt < sweepIntervalMs) return;
|
|
20
|
+
lastSweepAt = currentTime;
|
|
21
|
+
const cutoff = currentTime - ttlMs;
|
|
22
|
+
for (const [key, entry] of states) {
|
|
23
|
+
if (entry.updatedAt < cutoff) states.delete(key);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return {
|
|
27
|
+
async get(key) {
|
|
28
|
+
const entry = states.get(key);
|
|
29
|
+
if (!entry) return emptyWorkflowState();
|
|
30
|
+
if (now() - entry.updatedAt > ttlMs) {
|
|
31
|
+
states.delete(key);
|
|
32
|
+
return emptyWorkflowState();
|
|
33
|
+
}
|
|
34
|
+
return entry.value;
|
|
35
|
+
},
|
|
36
|
+
async patch(key, patch) {
|
|
37
|
+
const t = now();
|
|
38
|
+
sweep(t);
|
|
39
|
+
const prev = states.get(key);
|
|
40
|
+
const baseValue = prev && t - prev.updatedAt <= ttlMs ? prev.value : emptyWorkflowState();
|
|
41
|
+
const next = {
|
|
42
|
+
...baseValue,
|
|
43
|
+
...patch,
|
|
44
|
+
orientationVersion: 1
|
|
45
|
+
};
|
|
46
|
+
states.set(key, { value: next, updatedAt: t });
|
|
47
|
+
return next;
|
|
48
|
+
},
|
|
49
|
+
size() {
|
|
50
|
+
return states.size;
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
class RedisWorkflowStateStore {
|
|
55
|
+
constructor(client, prefix = WORKFLOW_STATE_PREFIX) {
|
|
56
|
+
this.client = client;
|
|
57
|
+
this.prefix = prefix;
|
|
58
|
+
}
|
|
59
|
+
async get(key) {
|
|
60
|
+
const raw = await this.client.get(this.prefix + key);
|
|
61
|
+
if (!raw) {
|
|
62
|
+
return emptyWorkflowState();
|
|
63
|
+
}
|
|
64
|
+
const parsed = JSON.parse(raw);
|
|
65
|
+
return {
|
|
66
|
+
...emptyWorkflowState(),
|
|
67
|
+
...parsed,
|
|
68
|
+
orientationVersion: 1
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
async patch(key, patch) {
|
|
72
|
+
const next = {
|
|
73
|
+
...await this.get(key),
|
|
74
|
+
...patch,
|
|
75
|
+
orientationVersion: 1
|
|
76
|
+
};
|
|
77
|
+
await this.client.set(this.prefix + key, JSON.stringify(next), {
|
|
78
|
+
EX: WORKFLOW_STATE_TTL_SECONDS
|
|
79
|
+
});
|
|
80
|
+
return next;
|
|
81
|
+
}
|
|
82
|
+
async close() {
|
|
83
|
+
await this.client.quit();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
let workflowStateStore = null;
|
|
87
|
+
async function configureWorkflowStateStore(redisUrl) {
|
|
88
|
+
if (workflowStateStore) {
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (!redisUrl) {
|
|
92
|
+
workflowStateStore = createMemoryWorkflowStateStore();
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
const client = createClient({ url: redisUrl });
|
|
96
|
+
await client.connect();
|
|
97
|
+
workflowStateStore = new RedisWorkflowStateStore(client);
|
|
98
|
+
}
|
|
99
|
+
function getWorkflowStateStore() {
|
|
100
|
+
if (!workflowStateStore) {
|
|
101
|
+
throw new Error("WorkflowStateStore not configured");
|
|
102
|
+
}
|
|
103
|
+
return workflowStateStore;
|
|
104
|
+
}
|
|
105
|
+
async function closeWorkflowStateStore() {
|
|
106
|
+
await workflowStateStore?.close?.();
|
|
107
|
+
workflowStateStore = null;
|
|
108
|
+
}
|
|
109
|
+
export {
|
|
110
|
+
closeWorkflowStateStore,
|
|
111
|
+
configureWorkflowStateStore,
|
|
112
|
+
createMemoryWorkflowStateStore,
|
|
113
|
+
emptyWorkflowState,
|
|
114
|
+
getWorkflowStateStore
|
|
115
|
+
};
|
|
116
|
+
//# sourceMappingURL=workflow-state.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/services/workflow-state.ts"],
|
|
4
|
+
"sourcesContent": ["import { createClient } from 'redis';\n\nexport interface WorkflowState {\n bootstrapped: boolean;\n bootstrappedAt?: string;\n redditWarningShown: boolean;\n orientationVersion: 1;\n}\n\nexport interface WorkflowStateStore {\n get(key: string): Promise<WorkflowState>;\n patch(key: string, patch: Partial<WorkflowState>): Promise<WorkflowState>;\n size?(): number;\n close?(): Promise<void>;\n}\n\nconst WORKFLOW_STATE_TTL_SECONDS = 60 * 60 * 24;\nconst WORKFLOW_STATE_PREFIX = 'research-pack:workflow:';\nconst SWEEP_INTERVAL_MS = 60_000;\n\nexport function emptyWorkflowState(): WorkflowState {\n return {\n bootstrapped: false,\n redditWarningShown: false,\n orientationVersion: 1,\n };\n}\n\ninterface MemoryWorkflowStateOptions {\n /** Override the wall-clock source \u2014 for tests. Defaults to Date.now. */\n readonly now?: () => number;\n /** Minimum interval between sweeps. Defaults to 60s. */\n readonly sweepIntervalMs?: number;\n /** TTL in seconds. Defaults to 24h to match the Redis store. */\n readonly ttlSeconds?: number;\n}\n\ninterface MemoryEntry {\n readonly value: WorkflowState;\n updatedAt: number;\n}\n\n/**\n * In-memory workflow state with TTL eviction. Mirrors the 24h TTL the\n * Redis store gets via `EX`. Without this, every anonymous client minted\n * a key the process could never reclaim \u2014 confirmed in production where\n * health://status reported 521 sessions at 31h uptime.\n *\n * See: docs/code-review/context/04-session-and-workflow-state.md (E7).\n */\nexport function createMemoryWorkflowStateStore(\n opts: MemoryWorkflowStateOptions = {},\n): WorkflowStateStore {\n const now = opts.now ?? (() => Date.now());\n const sweepIntervalMs = opts.sweepIntervalMs ?? SWEEP_INTERVAL_MS;\n const ttlMs = (opts.ttlSeconds ?? WORKFLOW_STATE_TTL_SECONDS) * 1000;\n const states = new Map<string, MemoryEntry>();\n let lastSweepAt = 0;\n\n function sweep(currentTime: number): void {\n if (currentTime - lastSweepAt < sweepIntervalMs) return;\n lastSweepAt = currentTime;\n const cutoff = currentTime - ttlMs;\n for (const [key, entry] of states) {\n if (entry.updatedAt < cutoff) states.delete(key);\n }\n }\n\n return {\n async get(key) {\n const entry = states.get(key);\n if (!entry) return emptyWorkflowState();\n // Per-entry expiry check \u2014 sweep is throttled, so a stale entry\n // could otherwise be returned in the gap between sweeps.\n if (now() - entry.updatedAt > ttlMs) {\n states.delete(key);\n return emptyWorkflowState();\n }\n return entry.value;\n },\n async patch(key, patch) {\n const t = now();\n sweep(t);\n const prev = states.get(key);\n const baseValue = prev && t - prev.updatedAt <= ttlMs\n ? prev.value\n : emptyWorkflowState();\n const next: WorkflowState = {\n ...baseValue,\n ...patch,\n orientationVersion: 1,\n };\n states.set(key, { value: next, updatedAt: t });\n return next;\n },\n size(): number {\n return states.size;\n },\n };\n}\n\nclass RedisWorkflowStateStore implements WorkflowStateStore {\n constructor(\n private readonly client: {\n get(key: string): Promise<string | null>;\n set(\n key: string,\n value: string,\n options: { EX: number },\n ): Promise<unknown>;\n quit(): Promise<unknown>;\n },\n private readonly prefix = WORKFLOW_STATE_PREFIX,\n ) {}\n\n async get(key: string): Promise<WorkflowState> {\n const raw = await this.client.get(this.prefix + key);\n if (!raw) {\n return emptyWorkflowState();\n }\n\n const parsed = JSON.parse(raw) as Partial<WorkflowState>;\n return {\n ...emptyWorkflowState(),\n ...parsed,\n orientationVersion: 1,\n };\n }\n\n async patch(key: string, patch: Partial<WorkflowState>): Promise<WorkflowState> {\n const next: WorkflowState = {\n ...(await this.get(key)),\n ...patch,\n orientationVersion: 1,\n };\n\n await this.client.set(this.prefix + key, JSON.stringify(next), {\n EX: WORKFLOW_STATE_TTL_SECONDS,\n });\n\n return next;\n }\n\n async close(): Promise<void> {\n await this.client.quit();\n }\n}\n\nlet workflowStateStore: WorkflowStateStore | null = null;\n\nexport async function configureWorkflowStateStore(redisUrl: string | undefined): Promise<void> {\n if (workflowStateStore) {\n return;\n }\n\n if (!redisUrl) {\n workflowStateStore = createMemoryWorkflowStateStore();\n return;\n }\n\n const client = createClient({ url: redisUrl });\n await client.connect();\n workflowStateStore = new RedisWorkflowStateStore(client);\n}\n\nexport function getWorkflowStateStore(): WorkflowStateStore {\n if (!workflowStateStore) {\n throw new Error('WorkflowStateStore not configured');\n }\n\n return workflowStateStore;\n}\n\nexport async function closeWorkflowStateStore(): Promise<void> {\n await workflowStateStore?.close?.();\n workflowStateStore = null;\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,oBAAoB;AAgB7B,MAAM,6BAA6B,KAAK,KAAK;AAC7C,MAAM,wBAAwB;AAC9B,MAAM,oBAAoB;AAEnB,SAAS,qBAAoC;AAClD,SAAO;AAAA,IACL,cAAc;AAAA,IACd,oBAAoB;AAAA,IACpB,oBAAoB;AAAA,EACtB;AACF;AAwBO,SAAS,+BACd,OAAmC,CAAC,GAChB;AACpB,QAAM,MAAM,KAAK,QAAQ,MAAM,KAAK,IAAI;AACxC,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,QAAM,SAAS,KAAK,cAAc,8BAA8B;AAChE,QAAM,SAAS,oBAAI,IAAyB;AAC5C,MAAI,cAAc;AAElB,WAAS,MAAM,aAA2B;AACxC,QAAI,cAAc,cAAc,gBAAiB;AACjD,kBAAc;AACd,UAAM,SAAS,cAAc;AAC7B,eAAW,CAAC,KAAK,KAAK,KAAK,QAAQ;AACjC,UAAI,MAAM,YAAY,OAAQ,QAAO,OAAO,GAAG;AAAA,IACjD;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM,IAAI,KAAK;AACb,YAAM,QAAQ,OAAO,IAAI,GAAG;AAC5B,UAAI,CAAC,MAAO,QAAO,mBAAmB;AAGtC,UAAI,IAAI,IAAI,MAAM,YAAY,OAAO;AACnC,eAAO,OAAO,GAAG;AACjB,eAAO,mBAAmB;AAAA,MAC5B;AACA,aAAO,MAAM;AAAA,IACf;AAAA,IACA,MAAM,MAAM,KAAK,OAAO;AACtB,YAAM,IAAI,IAAI;AACd,YAAM,CAAC;AACP,YAAM,OAAO,OAAO,IAAI,GAAG;AAC3B,YAAM,YAAY,QAAQ,IAAI,KAAK,aAAa,QAC5C,KAAK,QACL,mBAAmB;AACvB,YAAM,OAAsB;AAAA,QAC1B,GAAG;AAAA,QACH,GAAG;AAAA,QACH,oBAAoB;AAAA,MACtB;AACA,aAAO,IAAI,KAAK,EAAE,OAAO,MAAM,WAAW,EAAE,CAAC;AAC7C,aAAO;AAAA,IACT;AAAA,IACA,OAAe;AACb,aAAO,OAAO;AAAA,IAChB;AAAA,EACF;AACF;AAEA,MAAM,wBAAsD;AAAA,EAC1D,YACmB,QASA,SAAS,uBAC1B;AAViB;AASA;AAAA,EAChB;AAAA,EAEH,MAAM,IAAI,KAAqC;AAC7C,UAAM,MAAM,MAAM,KAAK,OAAO,IAAI,KAAK,SAAS,GAAG;AACnD,QAAI,CAAC,KAAK;AACR,aAAO,mBAAmB;AAAA,IAC5B;AAEA,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,WAAO;AAAA,MACL,GAAG,mBAAmB;AAAA,MACtB,GAAG;AAAA,MACH,oBAAoB;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,KAAa,OAAuD;AAC9E,UAAM,OAAsB;AAAA,MAC1B,GAAI,MAAM,KAAK,IAAI,GAAG;AAAA,MACtB,GAAG;AAAA,MACH,oBAAoB;AAAA,IACtB;AAEA,UAAM,KAAK,OAAO,IAAI,KAAK,SAAS,KAAK,KAAK,UAAU,IAAI,GAAG;AAAA,MAC7D,IAAI;AAAA,IACN,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,OAAO,KAAK;AAAA,EACzB;AACF;AAEA,IAAI,qBAAgD;AAEpD,eAAsB,4BAA4B,UAA6C;AAC7F,MAAI,oBAAoB;AACtB;AAAA,EACF;AAEA,MAAI,CAAC,UAAU;AACb,yBAAqB,+BAA+B;AACpD;AAAA,EACF;AAEA,QAAM,SAAS,aAAa,EAAE,KAAK,SAAS,CAAC;AAC7C,QAAM,OAAO,QAAQ;AACrB,uBAAqB,IAAI,wBAAwB,MAAM;AACzD;AAEO,SAAS,wBAA4C;AAC1D,MAAI,CAAC,oBAAoB;AACvB,UAAM,IAAI,MAAM,mCAAmC;AAAA,EACrD;AAEA,SAAO;AACT;AAEA,eAAsB,0BAAyC;AAC7D,QAAM,oBAAoB,QAAQ;AAClC,uBAAqB;AACvB;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { error, markdown } from "mcp-use/server";
|
|
2
|
+
const NOOP_REPORTER = {
|
|
3
|
+
async log() {
|
|
4
|
+
},
|
|
5
|
+
async progress() {
|
|
6
|
+
}
|
|
7
|
+
};
|
|
8
|
+
function toolSuccess(content, structuredContent) {
|
|
9
|
+
return {
|
|
10
|
+
isError: false,
|
|
11
|
+
content,
|
|
12
|
+
structuredContent
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
function toolFailure(content) {
|
|
16
|
+
return {
|
|
17
|
+
isError: true,
|
|
18
|
+
content
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function createToolReporter(ctx, loggerName) {
|
|
22
|
+
return {
|
|
23
|
+
log(level, message) {
|
|
24
|
+
return ctx.log(level, message, loggerName);
|
|
25
|
+
},
|
|
26
|
+
progress(loaded, total, message) {
|
|
27
|
+
return ctx.reportProgress?.(loaded, total, message) ?? Promise.resolve();
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
function toToolResponse(result) {
|
|
32
|
+
if (result.isError) {
|
|
33
|
+
return error(result.content);
|
|
34
|
+
}
|
|
35
|
+
if (result.structuredContent) {
|
|
36
|
+
return {
|
|
37
|
+
...markdown(result.content),
|
|
38
|
+
structuredContent: result.structuredContent
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
return markdown(result.content);
|
|
42
|
+
}
|
|
43
|
+
export {
|
|
44
|
+
NOOP_REPORTER,
|
|
45
|
+
createToolReporter,
|
|
46
|
+
toToolResponse,
|
|
47
|
+
toolFailure,
|
|
48
|
+
toolSuccess
|
|
49
|
+
};
|
|
50
|
+
//# sourceMappingURL=mcp-helpers.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/tools/mcp-helpers.ts"],
|
|
4
|
+
"sourcesContent": ["import { error, markdown, type TypedCallToolResult } from 'mcp-use/server';\n\ntype ClientLogLevel =\n | 'debug'\n | 'info'\n | 'notice'\n | 'warning'\n | 'error'\n | 'critical'\n | 'alert'\n | 'emergency';\n\ninterface ReporterContext {\n log(level: ClientLogLevel, message: string, loggerName?: string): Promise<void>;\n reportProgress?: (loaded: number, total?: number, message?: string) => Promise<void>;\n}\n\nexport interface ToolExecutionSuccess<T extends Record<string, unknown>> {\n readonly isError: false;\n readonly content: string;\n readonly structuredContent?: T;\n}\n\nexport interface ToolExecutionFailure {\n readonly isError: true;\n readonly content: string;\n}\n\nexport type ToolExecutionResult<T extends Record<string, unknown>> =\n | ToolExecutionSuccess<T>\n | ToolExecutionFailure;\n\nexport interface ToolReporter {\n log(level: ClientLogLevel, message: string): Promise<void>;\n progress(loaded: number, total?: number, message?: string): Promise<void>;\n}\n\nexport const NOOP_REPORTER: ToolReporter = {\n async log() {},\n async progress() {},\n};\n\nexport function toolSuccess<T extends Record<string, unknown>>(\n content: string,\n structuredContent?: T,\n): ToolExecutionSuccess<T> {\n return {\n isError: false,\n content,\n structuredContent,\n };\n}\n\nexport function toolFailure(content: string): ToolExecutionFailure {\n return {\n isError: true,\n content,\n };\n}\n\nexport function createToolReporter(\n ctx: ReporterContext,\n loggerName: string,\n): ToolReporter {\n return {\n log(level, message) {\n return ctx.log(level, message, loggerName);\n },\n progress(loaded, total, message) {\n return ctx.reportProgress?.(loaded, total, message) ?? Promise.resolve();\n },\n };\n}\n\nexport function toToolResponse<T extends Record<string, unknown>>(\n result: ToolExecutionResult<T>,\n): TypedCallToolResult<T> | TypedCallToolResult<never> {\n if (result.isError) {\n return error(result.content);\n }\n\n if (result.structuredContent) {\n return {\n ...markdown(result.content),\n structuredContent: result.structuredContent,\n } as TypedCallToolResult<T>;\n }\n\n return markdown(result.content) as TypedCallToolResult<never>;\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,OAAO,gBAA0C;AAqCnD,MAAM,gBAA8B;AAAA,EACzC,MAAM,MAAM;AAAA,EAAC;AAAA,EACb,MAAM,WAAW;AAAA,EAAC;AACpB;AAEO,SAAS,YACd,SACA,mBACyB;AACzB,SAAO;AAAA,IACL,SAAS;AAAA,IACT;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,YAAY,SAAuC;AACjE,SAAO;AAAA,IACL,SAAS;AAAA,IACT;AAAA,EACF;AACF;AAEO,SAAS,mBACd,KACA,YACc;AACd,SAAO;AAAA,IACL,IAAI,OAAO,SAAS;AAClB,aAAO,IAAI,IAAI,OAAO,SAAS,UAAU;AAAA,IAC3C;AAAA,IACA,SAAS,QAAQ,OAAO,SAAS;AAC/B,aAAO,IAAI,iBAAiB,QAAQ,OAAO,OAAO,KAAK,QAAQ,QAAQ;AAAA,IACzE;AAAA,EACF;AACF;AAEO,SAAS,eACd,QACqD;AACrD,MAAI,OAAO,SAAS;AAClB,WAAO,MAAM,OAAO,OAAO;AAAA,EAC7B;AAEA,MAAI,OAAO,mBAAmB;AAC5B,WAAO;AAAA,MACL,GAAG,SAAS,OAAO,OAAO;AAAA,MAC1B,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAEA,SAAO,SAAS,OAAO,OAAO;AAChC;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|