@librechat/agents 3.2.64 → 3.2.65

package/dist/esm/llm/anthropic/utils/stream_events.mjs

@@ -0,0 +1,337 @@
+import { getAnthropicUsageMetadata } from "./message_outputs.mjs";
+//#region src/llm/anthropic/utils/stream_events.ts
+/**
+* Convert an async iterable of raw Anthropic stream events into
+* LangChain `ChatModelStreamEvent`s with typed deltas.
+*/
+async function* convertAnthropicStream(source, options = {}) {
+	const shouldStreamUsage = options.streamUsage ?? true;
+	const blockAccumulators = /* @__PURE__ */ new Map();
+	let usageSnapshot;
+	let eventUsage;
+	let stopReason = null;
+	for await (const data of source) switch (data.type) {
+		case "message_start": {
+			const { usage, id, model } = data.message;
+			if (shouldStreamUsage) {
+				eventUsage = {
+					inputTokens: usage.input_tokens,
+					cacheCreationInputTokens: usage.cache_creation_input_tokens ?? 0,
+					cacheReadInputTokens: usage.cache_read_input_tokens ?? 0,
+					outputTokens: usage.output_tokens
+				};
+				usageSnapshot = buildUsageSnapshot(eventUsage);
+			}
+			yield {
+				event: "message-start",
+				id,
+				...usageSnapshot ? { usage: usageSnapshot } : {}
+			};
+			yield {
+				event: "provider",
+				provider: "anthropic",
+				name: "message_start",
+				payload: {
+					model,
+					id
+				}
+			};
+			break;
+		}
+		case "message_delta":
+			stopReason = data.delta.stop_reason;
+			if (shouldStreamUsage) {
+				eventUsage = updateEventUsage(eventUsage, data.usage);
+				usageSnapshot = buildUsageSnapshot(eventUsage);
+				yield {
+					event: "usage",
+					usage: usageSnapshot
+				};
+			}
+			if ("context_management" in data.delta && data.delta.context_management != null) yield {
+				event: "provider",
+				provider: "anthropic",
+				name: "context_management",
+				payload: data.delta.context_management
+			};
+			break;
+		case "message_stop":
+			yield {
+				event: "message-finish",
+				reason: mapStopReason(stopReason),
+				...usageSnapshot ? { usage: usageSnapshot } : {},
+				responseMetadata: { model_provider: "anthropic" }
+			};
+			break;
+		case "error": {
+			const streamError = new Error(data.error.message);
+			streamError.name = data.error.type;
+			throw streamError;
+		}
+		case "content_block_start": {
+			const { index, content_block } = data;
+			const mapped = mapBlockToContentBlock(content_block, index);
+			blockAccumulators.set(index, { ...mapped });
+			yield {
+				event: "content-block-start",
+				index,
+				content: mapped
+			};
+			break;
+		}
+		case "content_block_delta": {
+			const { index, delta } = data;
+			const acc = blockAccumulators.get(index);
+			if (!acc) break;
+			const { contentDelta, accumulated } = applyAnthropicDelta(acc, delta);
+			blockAccumulators.set(index, accumulated);
+			yield {
+				event: "content-block-delta",
+				index,
+				delta: contentDelta
+			};
+			break;
+		}
+		case "content_block_stop": {
+			const { index } = data;
+			const acc = blockAccumulators.get(index);
+			if (!acc) break;
+			yield {
+				event: "content-block-finish",
+				index,
+				content: finalizeBlock(acc)
+			};
+			blockAccumulators.delete(index);
+			break;
+		}
+		default: {
+			const providerData = data;
+			yield {
+				event: "provider",
+				provider: "anthropic",
+				name: providerData.type,
+				payload: providerData
+			};
+			break;
+		}
+	}
+}
+function mapStopReason(stopReason) {
+	switch (stopReason) {
+		case "end_turn":
+		case "stop_sequence": return "stop";
+		case "tool_use": return "tool_use";
+		case "max_tokens":
+		case "model_context_window_exceeded": return "length";
+		case "refusal": return "content_filter";
+		default: return "stop";
+	}
+}
+function updateEventUsage(previous, current) {
+	return {
+		inputTokens: getCumulativeUsageValue(current.input_tokens, previous?.inputTokens),
+		cacheCreationInputTokens: getCumulativeUsageValue(current.cache_creation_input_tokens, previous?.cacheCreationInputTokens),
+		cacheReadInputTokens: getCumulativeUsageValue(current.cache_read_input_tokens, previous?.cacheReadInputTokens),
+		outputTokens: getCumulativeUsageValue(current.output_tokens, previous?.outputTokens)
+	};
+}
+function getCumulativeUsageValue(current, previous) {
+	return Math.max(previous ?? 0, current ?? 0);
+}
+function buildUsageSnapshot(usage) {
+	const metadata = getAnthropicUsageMetadata({
+		input_tokens: usage.inputTokens,
+		output_tokens: usage.outputTokens,
+		cache_creation_input_tokens: usage.cacheCreationInputTokens,
+		cache_read_input_tokens: usage.cacheReadInputTokens
+	});
+	if (metadata == null) throw new Error("Anthropic usage metadata was not created");
+	return metadata;
+}
+function getStringField(record, key) {
+	const value = record[key];
+	return typeof value === "string" ? value : "";
+}
+function getArrayField(record, key) {
+	const value = record[key];
+	return Array.isArray(value) ? value : [];
+}
+function mapBlockToContentBlock(block, index) {
+	switch (block.type) {
+		case "text": return {
+			type: "text",
+			text: block.text,
+			...block.citations != null ? { citations: block.citations } : {},
+			index
+		};
+		case "thinking": return {
+			type: "reasoning",
+			reasoning: block.thinking,
+			index
+		};
+		case "redacted_thinking": return {
+			...block,
+			index
+		};
+		case "tool_use": return {
+			type: "tool_call_chunk",
+			id: block.id,
+			name: block.name,
+			args: "",
+			index
+		};
+		case "server_tool_use": return {
+			type: "server_tool_call_chunk",
+			id: block.id,
+			name: block.name,
+			args: "",
+			index
+		};
+		case "web_search_tool_result": return {
+			...block,
+			index
+		};
+		case "compaction": return {
+			...block,
+			index
+		};
+		default: return {
+			type: "non_standard",
+			value: { ...block },
+			index
+		};
+	}
+}
+/**
+* Map an Anthropic content_block_delta to a content block delta
+* and update the accumulated state.
+*/
+function applyAnthropicDelta(accumulated, delta) {
+	const rawDelta = delta;
+	switch (delta.type) {
+		case "text_delta": return {
+			contentDelta: {
+				type: "text-delta",
+				text: delta.text
+			},
+			accumulated: {
+				...accumulated,
+				text: getStringField(accumulated, "text") + delta.text
+			}
+		};
+		case "thinking_delta": return {
+			contentDelta: {
+				type: "reasoning-delta",
+				reasoning: delta.thinking
+			},
+			accumulated: {
+				...accumulated,
+				reasoning: getStringField(accumulated, "reasoning") + delta.thinking
+			}
+		};
+		case "input_json_delta": {
+			const newArgs = getStringField(accumulated, "args") + delta.partial_json;
+			return {
+				contentDelta: {
+					type: "block-delta",
+					fields: {
+						type: getStringField(accumulated, "type"),
+						args: newArgs
+					}
+				},
+				accumulated: {
+					...accumulated,
+					args: newArgs
+				}
+			};
+		}
+		case "citations_delta": {
+			const citations = [...getArrayField(accumulated, "citations"), delta.citation];
+			return {
+				contentDelta: {
+					type: "block-delta",
+					fields: {
+						type: getStringField(accumulated, "type"),
+						citations
+					}
+				},
+				accumulated: {
+					...accumulated,
+					citations
+				}
+			};
+		}
+		case "signature_delta": return {
+			contentDelta: {
+				type: "block-delta",
+				fields: {
+					type: getStringField(accumulated, "type"),
+					signature: delta.signature
+				}
+			},
+			accumulated: {
+				...accumulated,
+				signature: delta.signature
+			}
+		};
+		case "compaction_delta": {
+			const previousContent = accumulated.content;
+			const content = delta.content == null ? previousContent ?? null : getStringField(accumulated, "content") + delta.content;
+			return {
+				contentDelta: {
+					type: "block-delta",
+					fields: {
+						type: "compaction",
+						content,
+						encrypted_content: delta.encrypted_content
+					}
+				},
+				accumulated: {
+					...accumulated,
+					content,
+					encrypted_content: delta.encrypted_content
+				}
+			};
+		}
+		default: return {
+			contentDelta: {
+				type: "block-delta",
+				fields: {
+					type: getStringField(accumulated, "type"),
+					...rawDelta
+				}
+			},
+			accumulated
+		};
+	}
+}
+function finalizeBlock(accumulated) {
+	if (accumulated.type === "tool_call_chunk" || accumulated.type === "server_tool_call_chunk") {
+		const finalType = accumulated.type === "tool_call_chunk" ? "tool_call" : "server_tool_call";
+		const args = getStringField(accumulated, "args");
+		let parsedArgs;
+		try {
+			parsedArgs = JSON.parse(args || "{}");
+		} catch {
+			return {
+				type: "invalid_tool_call",
+				id: getStringField(accumulated, "id"),
+				name: getStringField(accumulated, "name"),
+				args,
+				error: "Failed to parse tool call arguments as JSON"
+			};
+		}
+		return {
+			type: finalType,
+			id: getStringField(accumulated, "id"),
+			name: getStringField(accumulated, "name"),
+			args: parsedArgs
+		};
+	}
+	const { index: _index, ...rest } = accumulated;
+	return rest;
+}
+//#endregion
+export { convertAnthropicStream };
+
+//# sourceMappingURL=stream_events.mjs.map

package/dist/esm/llm/anthropic/utils/stream_events.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream_events.mjs","names":[],"sources":["../../../../../src/llm/anthropic/utils/stream_events.ts"],"sourcesContent":["/**\n * Converts a raw Anthropic SSE event stream into LangChain ChatModelStreamEvents.\n *\n * @module\n */\n\nimport type {\n  ChatModelStreamEvent,\n  ContentBlockDelta,\n  FinishReason,\n} from '@langchain/core/language_models/event';\nimport type { ContentBlock, UsageMetadata } from '@langchain/core/messages';\nimport type {\n  AnthropicCompactionBlock,\n  AnthropicCompactionContentBlockDelta,\n  AnthropicMessageStreamEvent,\n} from '../types';\nimport type { AnthropicUsageData } from './message_outputs';\nimport { getAnthropicUsageMetadata } from './message_outputs';\n\ntype AnthropicContentBlock =\n  | Extract<\n      AnthropicMessageStreamEvent,\n      { type: 'content_block_start' }\n    >['content_block']\n  | AnthropicCompactionBlock;\n\ntype AnthropicContentDelta =\n  | Extract<\n      AnthropicMessageStreamEvent,\n      { type: 'content_block_delta' }\n    >['delta']\n  | AnthropicCompactionContentBlockDelta;\n\ninterface AnthropicStreamErrorEvent {\n  type: 'error';\n  error: {\n    type: string;\n    message: string;\n  };\n  request_id?: string | null;\n}\n\ntype AnthropicStreamInputEvent =\n  | AnthropicMessageStreamEvent\n  | AnthropicStreamErrorEvent;\n\ntype BlockAccumulator = Record<string, unknown>;\n\ninterface AnthropicEventUsage {\n  inputTokens: number;\n  cacheCreationInputTokens: number;\n  cacheReadInputTokens: number;\n  outputTokens: number;\n}\n\n// ─── Public API ─────────────────────────────────────────────────\n\nexport interface ConvertAnthropicStreamOptions {\n  streamUsage?: boolean;\n}\n\n/**\n * Convert an async iterable of raw Anthropic stream events into\n * LangChain `ChatModelStreamEvent`s with typed deltas.\n */\nexport async function* convertAnthropicStream(\n  source: AsyncIterable<AnthropicStreamInputEvent>,\n  options: ConvertAnthropicStreamOptions = {}\n): AsyncGenerator<ChatModelStreamEvent> {\n  const shouldStreamUsage = options.streamUsage ?? true;\n\n  // Track accumulated state per content block (for finalization)\n  const blockAccumulators = new Map<number, BlockAccumulator>();\n  let usageSnapshot: UsageMetadata | undefined;\n  let eventUsage: AnthropicEventUsage | undefined;\n  let stopReason: string | null = null;\n\n  for await (const data of source) {\n    switch (data.type) {\n    // ── Message lifecycle ──────────────────────────────────\n    case 'message_start': {\n      const { usage, id, model } = data.message;\n      if (shouldStreamUsage) {\n        eventUsage = {\n          inputTokens: usage.input_tokens,\n          cacheCreationInputTokens: usage.cache_creation_input_tokens ?? 0,\n          cacheReadInputTokens: usage.cache_read_input_tokens ?? 0,\n          outputTokens: usage.output_tokens,\n        };\n        usageSnapshot = buildUsageSnapshot(eventUsage);\n      }\n      yield {\n        event: 'message-start' as const,\n        id,\n        ...(usageSnapshot ? { usage: usageSnapshot } : {}),\n      };\n      yield {\n        event: 'provider' as const,\n        provider: 'anthropic',\n        name: 'message_start',\n        payload: { model, id },\n      };\n      break;\n    }\n\n    case 'message_delta': {\n      stopReason = data.delta.stop_reason;\n      if (shouldStreamUsage) {\n        eventUsage = updateEventUsage(eventUsage, data.usage);\n        usageSnapshot = buildUsageSnapshot(eventUsage);\n        yield { event: 'usage' as const, usage: usageSnapshot };\n      }\n      if (\n        'context_management' in data.delta &&\n          data.delta.context_management != null\n      ) {\n        yield {\n          event: 'provider' as const,\n          provider: 'anthropic',\n          name: 'context_management',\n          payload: data.delta.context_management,\n        };\n      }\n      break;\n    }\n\n    case 'message_stop': {\n      const finishEvent = {\n        event: 'message-finish' as const,\n        reason: mapStopReason(stopReason),\n        ...(usageSnapshot ? { usage: usageSnapshot } : {}),\n        responseMetadata: { model_provider: 'anthropic' },\n      };\n      yield finishEvent;\n      break;\n    }\n\n    case 'error': {\n      const streamError = new Error(data.error.message);\n      streamError.name = data.error.type;\n      throw streamError;\n    }\n\n    // ── Content block lifecycle ───────────────────────────\n    case 'content_block_start': {\n      const { index, content_block } = data;\n      const mapped = mapBlockToContentBlock(content_block, index);\n      blockAccumulators.set(index, { ...mapped });\n      yield {\n        event: 'content-block-start' as const,\n        index,\n        content: mapped as unknown as ContentBlock,\n      };\n      break;\n    }\n\n    case 'content_block_delta': {\n      const { index, delta } = data;\n      const acc = blockAccumulators.get(index);\n      if (!acc) break;\n\n      const { contentDelta, accumulated } = applyAnthropicDelta(acc, delta);\n      blockAccumulators.set(index, accumulated);\n\n      yield {\n        event: 'content-block-delta' as const,\n        index,\n        delta: contentDelta,\n      };\n      break;\n    }\n\n    case 'content_block_stop': {\n      const { index } = data;\n      const acc = blockAccumulators.get(index);\n      if (!acc) break;\n\n      const finalized = finalizeBlock(acc);\n      yield {\n        event: 'content-block-finish' as const,\n        index,\n        content: finalized,\n      };\n      blockAccumulators.delete(index);\n      break;\n    }\n\n    // ── Unhandled → provider passthrough ───────────────────\n    default: {\n      const providerData = data as AnthropicMessageStreamEvent;\n      yield {\n        event: 'provider' as const,\n        provider: 'anthropic',\n        name: providerData.type,\n        payload: providerData,\n      };\n      break;\n    }\n    }\n  }\n}\n\n// ─── Internal helpers ───────────────────────────────────────────\n\nfunction mapStopReason(stopReason: string | null | undefined): FinishReason {\n  switch (stopReason) {\n  case 'end_turn':\n  case 'stop_sequence':\n    return 'stop';\n  case 'tool_use':\n    return 'tool_use';\n  case 'max_tokens':\n  case 'model_context_window_exceeded':\n    return 'length';\n  case 'refusal':\n    return 'content_filter';\n  default:\n    return 'stop';\n  }\n}\n\nfunction updateEventUsage(\n  previous: AnthropicEventUsage | undefined,\n  current: AnthropicUsageData\n): AnthropicEventUsage {\n  return {\n    inputTokens: getCumulativeUsageValue(\n      current.input_tokens,\n      previous?.inputTokens\n    ),\n    cacheCreationInputTokens: getCumulativeUsageValue(\n      current.cache_creation_input_tokens,\n      previous?.cacheCreationInputTokens\n    ),\n    cacheReadInputTokens: getCumulativeUsageValue(\n      current.cache_read_input_tokens,\n      previous?.cacheReadInputTokens\n    ),\n    outputTokens: getCumulativeUsageValue(\n      current.output_tokens,\n      previous?.outputTokens\n    ),\n  };\n}\n\nfunction getCumulativeUsageValue(\n  current: number | null | undefined,\n  previous: number | undefined\n): number {\n  return Math.max(previous ?? 0, current ?? 0);\n}\n\nfunction buildUsageSnapshot(usage: AnthropicEventUsage): UsageMetadata {\n  const metadata = getAnthropicUsageMetadata({\n    input_tokens: usage.inputTokens,\n    output_tokens: usage.outputTokens,\n    cache_creation_input_tokens: usage.cacheCreationInputTokens,\n    cache_read_input_tokens: usage.cacheReadInputTokens,\n  });\n  if (metadata == null) {\n    throw new Error('Anthropic usage metadata was not created');\n  }\n  return metadata;\n}\n\nfunction getStringField(record: BlockAccumulator, key: string): string {\n  const value = record[key];\n  return typeof value === 'string' ? value : '';\n}\n\nfunction getArrayField(record: BlockAccumulator, key: string): unknown[] {\n  const value = record[key];\n  return Array.isArray(value) ? value : [];\n}\n\nfunction mapBlockToContentBlock(\n  block: AnthropicContentBlock,\n  index: number\n): BlockAccumulator {\n  switch (block.type) {\n  case 'text':\n    return {\n      type: 'text' as const,\n      text: block.text,\n      ...(block.citations != null ? { citations: block.citations } : {}),\n      index,\n    };\n  case 'thinking':\n    return {\n      type: 'reasoning' as const,\n      reasoning: block.thinking,\n      index,\n    };\n  case 'redacted_thinking':\n    return { ...block, index };\n  case 'tool_use':\n    return {\n      type: 'tool_call_chunk' as const,\n      id: block.id,\n      name: block.name,\n      args: '',\n      index,\n    };\n  case 'server_tool_use':\n    return {\n      type: 'server_tool_call_chunk' as const,\n      id: block.id,\n      name: block.name,\n      args: '',\n      index,\n    };\n  case 'web_search_tool_result':\n    return { ...block, index };\n  case 'compaction':\n    return { ...block, index };\n  default:\n    return { type: 'non_standard' as const, value: { ...block }, index };\n  }\n}\n\n/**\n * Map an Anthropic content_block_delta to a content block delta\n * and update the accumulated state.\n */\nfunction applyAnthropicDelta(\n  accumulated: BlockAccumulator,\n  delta: AnthropicContentDelta\n): {\n  contentDelta: ContentBlockDelta;\n  accumulated: BlockAccumulator;\n} {\n  const rawDelta = delta as unknown as Record<string, unknown>;\n  switch (delta.type) {\n  case 'text_delta':\n    return {\n      contentDelta: { type: 'text-delta' as const, text: delta.text },\n      accumulated: {\n        ...accumulated,\n        text: getStringField(accumulated, 'text') + delta.text,\n      },\n    };\n\n  case 'thinking_delta':\n    return {\n      contentDelta: {\n        type: 'reasoning-delta' as const,\n        reasoning: delta.thinking,\n      },\n      accumulated: {\n        ...accumulated,\n        reasoning: getStringField(accumulated, 'reasoning') + delta.thinking,\n      },\n    };\n\n  case 'input_json_delta': {\n    const newArgs = getStringField(accumulated, 'args') + delta.partial_json;\n    return {\n      contentDelta: {\n        type: 'block-delta' as const,\n        fields: {\n          type: getStringField(accumulated, 'type'),\n          args: newArgs,\n        },\n      },\n      accumulated: { ...accumulated, args: newArgs },\n    };\n  }\n\n  case 'citations_delta': {\n    const citations = [\n      ...getArrayField(accumulated, 'citations'),\n      delta.citation,\n    ];\n    return {\n      contentDelta: {\n        type: 'block-delta' as const,\n        fields: {\n          type: getStringField(accumulated, 'type'),\n          citations,\n        },\n      },\n      accumulated: {\n        ...accumulated,\n        citations,\n      },\n    };\n  }\n\n  case 'signature_delta':\n    return {\n      contentDelta: {\n        type: 'block-delta' as const,\n        fields: {\n          type: getStringField(accumulated, 'type'),\n          signature: delta.signature,\n        },\n      },\n      accumulated: { ...accumulated, signature: delta.signature },\n    };\n\n  case 'compaction_delta': {\n    const previousContent = accumulated.content;\n    const content =\n        delta.content == null\n          ? (previousContent ?? null)\n          : getStringField(accumulated, 'content') + delta.content;\n    return {\n      contentDelta: {\n        type: 'block-delta' as const,\n        fields: {\n          type: 'compaction',\n          content,\n          encrypted_content: delta.encrypted_content,\n        },\n      },\n      accumulated: {\n        ...accumulated,\n        content,\n        encrypted_content: delta.encrypted_content,\n      },\n    };\n  }\n\n  default:\n    return {\n      contentDelta: {\n        type: 'block-delta' as const,\n        fields: {\n          type: getStringField(accumulated, 'type'),\n          ...rawDelta,\n        },\n      },\n      accumulated,\n    };\n  }\n}\n\nfunction finalizeBlock(accumulated: BlockAccumulator): ContentBlock {\n  if (\n    accumulated.type === 'tool_call_chunk' ||\n    accumulated.type === 'server_tool_call_chunk'\n  ) {\n    const finalType =\n      accumulated.type === 'tool_call_chunk'\n        ? ('tool_call' as const)\n        : ('server_tool_call' as const);\n    const args = getStringField(accumulated, 'args');\n    let parsedArgs: unknown;\n    try {\n      parsedArgs = JSON.parse(args || '{}');\n    } catch {\n      return {\n        type: 'invalid_tool_call' as const,\n        id: getStringField(accumulated, 'id'),\n        name: getStringField(accumulated, 'name'),\n        args,\n        error: 'Failed to parse tool call arguments as JSON',\n      } as ContentBlock.Tools.InvalidToolCall;\n    }\n    return {\n      type: finalType,\n      id: getStringField(accumulated, 'id'),\n      name: getStringField(accumulated, 'name'),\n      args: parsedArgs,\n    } as ContentBlock;\n  }\n\n  const { index: _index, ...rest } = accumulated;\n  return rest as ContentBlock;\n}\n"],"mappings":";;;;;;AAkEA,gBAAuB,uBACrB,QACA,UAAyC,CAAC,GACJ;CACtC,MAAM,oBAAoB,QAAQ,eAAe;CAGjD,MAAM,oCAAoB,IAAI,IAA8B;CAC5D,IAAI;CACJ,IAAI;CACJ,IAAI,aAA4B;CAEhC,WAAW,MAAM,QAAQ,QACvB,QAAQ,KAAK,MAAb;EAEA,KAAK,iBAAiB;GACpB,MAAM,EAAE,OAAO,IAAI,UAAU,KAAK;GAClC,IAAI,mBAAmB;IACrB,aAAa;KACX,aAAa,MAAM;KACnB,0BAA0B,MAAM,+BAA+B;KAC/D,sBAAsB,MAAM,2BAA2B;KACvD,cAAc,MAAM;IACtB;IACA,gBAAgB,mBAAmB,UAAU;GAC/C;GACA,MAAM;IACJ,OAAO;IACP;IACA,GAAI,gBAAgB,EAAE,OAAO,cAAc,IAAI,CAAC;GAClD;GACA,MAAM;IACJ,OAAO;IACP,UAAU;IACV,MAAM;IACN,SAAS;KAAE;KAAO;IAAG;GACvB;GACA;EACF;EAEA,KAAK;GACH,aAAa,KAAK,MAAM;GACxB,IAAI,mBAAmB;IACrB,aAAa,iBAAiB,YAAY,KAAK,KAAK;IACpD,gBAAgB,mBAAmB,UAAU;IAC7C,MAAM;KAAE,OAAO;KAAkB,OAAO;IAAc;GACxD;GACA,IACE,wBAAwB,KAAK,SAC3B,KAAK,MAAM,sBAAsB,MAEnC,MAAM;IACJ,OAAO;IACP,UAAU;IACV,MAAM;IACN,SAAS,KAAK,MAAM;GACtB;GAEF;EAGF,KAAK;GAOH,MAAM;IALJ,OAAO;IACP,QAAQ,cAAc,UAAU;IAChC,GAAI,gBAAgB,EAAE,OAAO,cAAc,IAAI,CAAC;IAChD,kBAAkB,EAAE,gBAAgB,YAAY;GAElC;GAChB;EAGF,KAAK,SAAS;GACZ,MAAM,cAAc,IAAI,MAAM,KAAK,MAAM,OAAO;GAChD,YAAY,OAAO,KAAK,MAAM;GAC9B,MAAM;EACR;EAGA,KAAK,uBAAuB;GAC1B,MAAM,EAAE,OAAO,kBAAkB;GACjC,MAAM,SAAS,uBAAuB,eAAe,KAAK;GAC1D,kBAAkB,IAAI,OAAO,EAAE,GAAG,OAAO,CAAC;GAC1C,MAAM;IACJ,OAAO;IACP;IACA,SAAS;GACX;GACA;EACF;EAEA,KAAK,uBAAuB;GAC1B,MAAM,EAAE,OAAO,UAAU;GACzB,MAAM,MAAM,kBAAkB,IAAI,KAAK;GACvC,IAAI,CAAC,KAAK;GAEV,MAAM,EAAE,cAAc,gBAAgB,oBAAoB,KAAK,KAAK;GACpE,kBAAkB,IAAI,OAAO,WAAW;GAExC,MAAM;IACJ,OAAO;IACP;IACA,OAAO;GACT;GACA;EACF;EAEA,KAAK,sBAAsB;GACzB,MAAM,EAAE,UAAU;GAClB,MAAM,MAAM,kBAAkB,IAAI,KAAK;GACvC,IAAI,CAAC,KAAK;GAGV,MAAM;IACJ,OAAO;IACP;IACA,SAJgB,cAAc,GAIb;GACnB;GACA,kBAAkB,OAAO,KAAK;GAC9B;EACF;EAGA,SAAS;GACP,MAAM,eAAe;GACrB,MAAM;IACJ,OAAO;IACP,UAAU;IACV,MAAM,aAAa;IACnB,SAAS;GACX;GACA;EACF;CACA;AAEJ;AAIA,SAAS,cAAc,YAAqD;CAC1E,QAAQ,YAAR;EACA,KAAK;EACL,KAAK,iBACH,OAAO;EACT,KAAK,YACH,OAAO;EACT,KAAK;EACL,KAAK,iCACH,OAAO;EACT,KAAK,WACH,OAAO;EACT,SACE,OAAO;CACT;AACF;AAEA,SAAS,iBACP,UACA,SACqB;CACrB,OAAO;EACL,aAAa,wBACX,QAAQ,cACR,UAAU,WACZ;EACA,0BAA0B,wBACxB,QAAQ,6BACR,UAAU,wBACZ;EACA,sBAAsB,wBACpB,QAAQ,yBACR,UAAU,oBACZ;EACA,cAAc,wBACZ,QAAQ,eACR,UAAU,YACZ;CACF;AACF;AAEA,SAAS,wBACP,SACA,UACQ;CACR,OAAO,KAAK,IAAI,YAAY,GAAG,WAAW,CAAC;AAC7C;AAEA,SAAS,mBAAmB,OAA2C;CACrE,MAAM,WAAW,0BAA0B;EACzC,cAAc,MAAM;EACpB,eAAe,MAAM;EACrB,6BAA6B,MAAM;EACnC,yBAAyB,MAAM;CACjC,CAAC;CACD,IAAI,YAAY,MACd,MAAM,IAAI,MAAM,0CAA0C;CAE5D,OAAO;AACT;AAEA,SAAS,eAAe,QAA0B,KAAqB;CACrE,MAAM,QAAQ,OAAO;CACrB,OAAO,OAAO,UAAU,WAAW,QAAQ;AAC7C;AAEA,SAAS,cAAc,QAA0B,KAAwB;CACvE,MAAM,QAAQ,OAAO;CACrB,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC;AACzC;AAEA,SAAS,uBACP,OACA,OACkB;CAClB,QAAQ,MAAM,MAAd;EACA,KAAK,QACH,OAAO;GACL,MAAM;GACN,MAAM,MAAM;GACZ,GAAI,MAAM,aAAa,OAAO,EAAE,WAAW,MAAM,UAAU,IAAI,CAAC;GAChE;EACF;EACF,KAAK,YACH,OAAO;GACL,MAAM;GACN,WAAW,MAAM;GACjB;EACF;EACF,KAAK,qBACH,OAAO;GAAE,GAAG;GAAO;EAAM;EAC3B,KAAK,YACH,OAAO;GACL,MAAM;GACN,IAAI,MAAM;GACV,MAAM,MAAM;GACZ,MAAM;GACN;EACF;EACF,KAAK,mBACH,OAAO;GACL,MAAM;GACN,IAAI,MAAM;GACV,MAAM,MAAM;GACZ,MAAM;GACN;EACF;EACF,KAAK,0BACH,OAAO;GAAE,GAAG;GAAO;EAAM;EAC3B,KAAK,cACH,OAAO;GAAE,GAAG;GAAO;EAAM;EAC3B,SACE,OAAO;GAAE,MAAM;GAAyB,OAAO,EAAE,GAAG,MAAM;GAAG;EAAM;CACrE;AACF;;;;;AAMA,SAAS,oBACP,aACA,OAIA;CACA,MAAM,WAAW;CACjB,QAAQ,MAAM,MAAd;EACA,KAAK,cACH,OAAO;GACL,cAAc;IAAE,MAAM;IAAuB,MAAM,MAAM;GAAK;GAC9D,aAAa;IACX,GAAG;IACH,MAAM,eAAe,aAAa,MAAM,IAAI,MAAM;GACpD;EACF;EAEF,KAAK,kBACH,OAAO;GACL,cAAc;IACZ,MAAM;IACN,WAAW,MAAM;GACnB;GACA,aAAa;IACX,GAAG;IACH,WAAW,eAAe,aAAa,WAAW,IAAI,MAAM;GAC9D;EACF;EAEF,KAAK,oBAAoB;GACvB,MAAM,UAAU,eAAe,aAAa,MAAM,IAAI,MAAM;GAC5D,OAAO;IACL,cAAc;KACZ,MAAM;KACN,QAAQ;MACN,MAAM,eAAe,aAAa,MAAM;MACxC,MAAM;KACR;IACF;IACA,aAAa;KAAE,GAAG;KAAa,MAAM;IAAQ;GAC/C;EACF;EAEA,KAAK,mBAAmB;GACtB,MAAM,YAAY,CAChB,GAAG,cAAc,aAAa,WAAW,GACzC,MAAM,QACR;GACA,OAAO;IACL,cAAc;KACZ,MAAM;KACN,QAAQ;MACN,MAAM,eAAe,aAAa,MAAM;MACxC;KACF;IACF;IACA,aAAa;KACX,GAAG;KACH;IACF;GACF;EACF;EAEA,KAAK,mBACH,OAAO;GACL,cAAc;IACZ,MAAM;IACN,QAAQ;KACN,MAAM,eAAe,aAAa,MAAM;KACxC,WAAW,MAAM;IACnB;GACF;GACA,aAAa;IAAE,GAAG;IAAa,WAAW,MAAM;GAAU;EAC5D;EAEF,KAAK,oBAAoB;GACvB,MAAM,kBAAkB,YAAY;GACpC,MAAM,UACF,MAAM,WAAW,OACZ,mBAAmB,OACpB,eAAe,aAAa,SAAS,IAAI,MAAM;GACvD,OAAO;IACL,cAAc;KACZ,MAAM;KACN,QAAQ;MACN,MAAM;MACN;MACA,mBAAmB,MAAM;KAC3B;IACF;IACA,aAAa;KACX,GAAG;KACH;KACA,mBAAmB,MAAM;IAC3B;GACF;EACF;EAEA,SACE,OAAO;GACL,cAAc;IACZ,MAAM;IACN,QAAQ;KACN,MAAM,eAAe,aAAa,MAAM;KACxC,GAAG;IACL;GACF;GACA;EACF;CACF;AACF;AAEA,SAAS,cAAc,aAA6C;CAClE,IACE,YAAY,SAAS,qBACrB,YAAY,SAAS,0BACrB;EACA,MAAM,YACJ,YAAY,SAAS,oBAChB,cACA;EACP,MAAM,OAAO,eAAe,aAAa,MAAM;EAC/C,IAAI;EACJ,IAAI;GACF,aAAa,KAAK,MAAM,QAAQ,IAAI;EACtC,QAAQ;GACN,OAAO;IACL,MAAM;IACN,IAAI,eAAe,aAAa,IAAI;IACpC,MAAM,eAAe,aAAa,MAAM;IACxC;IACA,OAAO;GACT;EACF;EACA,OAAO;GACL,MAAM;GACN,IAAI,eAAe,aAAa,IAAI;GACpC,MAAM,eAAe,aAAa,MAAM;GACxC,MAAM;EACR;CACF;CAEA,MAAM,EAAE,OAAO,QAAQ,GAAG,SAAS;CACnC,OAAO;AACT"}

package/dist/esm/tools/search/crw-scraper.mjs

@@ -0,0 +1,163 @@
+import { createDefaultLogger } from "./utils.mjs";
+import { processContent } from "./content.mjs";
+import axios from "axios";
+//#region src/tools/search/crw-scraper.ts
+/** HTTP headroom over the payload render budget: fastCRW's queue/verification
+* overhead is not counted against `timeout`, so the client must wait longer. */
+const CRW_TIMEOUT_BUFFER = 5e3;
+/**
+* fastCRW scraper. Firecrawl-compatible web scraper; single binary;
+* self-host or cloud. Posts to {base}/v1/scrape.
+*/
+var CrwScraper = class {
+	apiKey;
+	apiUrl;
+	defaultFormats;
+	timeout;
+	logger;
+	onlyMainContent;
+	includeTags;
+	excludeTags;
+	waitFor;
+	headers;
+	renderJs;
+	cssSelector;
+	xpath;
+	proxy;
+	stealth;
+	constructor(config = {}) {
+		this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? "";
+		const baseUrl = config.apiUrl ?? process.env.CRW_API_URL ?? "https://api.fastcrw.com";
+		this.apiUrl = `${baseUrl.replace(/\/+$/, "")}/v1/scrape`;
+		this.defaultFormats = config.formats ?? ["markdown", "html"];
+		this.timeout = config.timeout ?? 7500;
+		this.logger = config.logger || createDefaultLogger();
+		this.onlyMainContent = config.onlyMainContent;
+		this.includeTags = config.includeTags;
+		this.excludeTags = config.excludeTags;
+		this.waitFor = config.waitFor;
+		this.headers = config.headers;
+		this.renderJs = config.renderJs;
+		this.cssSelector = config.cssSelector;
+		this.xpath = config.xpath;
+		this.proxy = config.proxy;
+		this.stealth = config.stealth;
+		if (!this.apiKey) this.logger.warn("CRW_API_KEY is not set. Scraping will not work.");
+		this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);
+	}
+	async scrapeUrl(url, options = {}) {
+		try {
+			const payloadTimeout = options.timeout ?? this.timeout;
+			const payload = omitUndefined({
+				url,
+				formats: options.formats ?? this.defaultFormats,
+				onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+				includeTags: options.includeTags ?? this.includeTags,
+				excludeTags: options.excludeTags ?? this.excludeTags,
+				waitFor: options.waitFor ?? this.waitFor,
+				headers: options.headers ?? this.headers,
+				renderJs: options.renderJs ?? this.renderJs,
+				cssSelector: options.cssSelector ?? this.cssSelector,
+				xpath: options.xpath ?? this.xpath,
+				proxy: options.proxy ?? this.proxy,
+				stealth: options.stealth ?? this.stealth,
+				timeout: payloadTimeout,
+				deadlineMs: Math.max(1, Math.min(payloadTimeout, 6e4))
+			});
+			const headers = { "Content-Type": "application/json" };
+			if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`;
+			return [url, normalizeCrwResponse((await axios.post(this.apiUrl, payload, {
+				headers,
+				timeout: payloadTimeout + CRW_TIMEOUT_BUFFER
+			})).data)];
+		} catch (error) {
+			return [url, {
+				success: false,
+				error: `fastCRW API request failed: ${error instanceof Error ? error.message : String(error)}`
+			}];
+		}
+	}
+	/**
+	* Extract content from scrape response. Mirrors FirecrawlScraper — reads
+	* response.data.*, which normalizeCrwResponse guarantees, preserving the
+	* processContent ref-markers used by the reranker. Parameter is typed as the
+	* NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:
+	* TS class-method bivariance accepts this against BaseScraper's
+	* AnyScraperResponse, AND it lets us read response.data.plainText (a
+	* CrwScrapeResponse-only field) with no cast.
+	*/
+	extractContent(response) {
+		if (!response.success || !response.data) return ["", void 0];
+		const htmlSource = response.data.html ?? response.data.rawHtml;
+		if (response.data.markdown != null && htmlSource != null) try {
+			const { markdown, ...rest } = processContent(htmlSource, response.data.markdown);
+			return [markdown, rest];
+		} catch (error) {
+			this.logger.error("Error processing content:", error);
+			return [response.data.markdown, void 0];
+		}
+		else if (response.data.markdown != null) return [response.data.markdown, void 0];
+		if (response.data.html != null) return [response.data.html, void 0];
+		if (response.data.rawHtml != null) return [response.data.rawHtml, void 0];
+		if (response.data.plainText != null) return [response.data.plainText, void 0];
+		return ["", void 0];
+	}
+	extractMetadata(response) {
+		if (!response.success || !response.data || !response.data.metadata) return {};
+		return response.data.metadata;
+	}
+};
+/**
+* Create a fastCRW scraper instance
+* @param config Scraper configuration
+* @returns fastCRW scraper instance
+*/
+const createCrwScraper = (config = {}) => new CrwScraper(config);
+/**
+* fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,
+* ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested
+* container and fall back to top-level fields for self-host/legacy responses.
+*/
+function normalizeCrwResponse(raw) {
+	if (raw == null) return {
+		success: false,
+		error: "Empty fastCRW response"
+	};
+	if (raw.success === false) return {
+		success: false,
+		error: raw.error_code != null ? `[${raw.error_code}] ${raw.error ?? "Unknown error"}` : raw.error ?? "fastCRW scrape failed",
+		error_code: raw.error_code
+	};
+	const data = raw.data ?? raw;
+	return {
+		success: true,
+		data: {
+			markdown: stripBase64DataUris(data.markdown),
+			html: stripBase64DataUris(data.html),
+			rawHtml: stripBase64DataUris(data.rawHtml),
+			plainText: stripBase64DataUris(data.plainText),
+			screenshot: data.screenshot,
+			links: data.links,
+			metadata: data.metadata
+		}
+	};
+}
+/**
+* Replace inline base64 data-URI payloads (typically images) in scraped text
+* with a short placeholder. Such payloads can be hundreds of KB; the content
+* processor builds a per-link RegExp from each URL, and one that large overflows
+* the engine's pattern-size limit ("Invalid regular expression"). Firecrawl
+* drops them via removeBase64Images — mirror that so image-heavy pages stay
+* processable (and don't bloat the LLM context with base64 noise).
+*/
+function stripBase64DataUris(text) {
+	if (text == null) return text;
+	return text.replace(/data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]+/g, "data:base64-content-removed");
+}
+function omitUndefined(obj) {
+	return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== void 0));
+}
+//#endregion
+export { createCrwScraper };
+
+//# sourceMappingURL=crw-scraper.mjs.map

package/dist/esm/tools/search/crw-scraper.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"crw-scraper.mjs","names":[],"sources":["../../../../src/tools/search/crw-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/** HTTP headroom over the payload render budget: fastCRW's queue/verification\n * overhead is not counted against `timeout`, so the client must wait longer. */\nconst CRW_TIMEOUT_BUFFER = 5000;\n\n/**\n * fastCRW scraper. Firecrawl-compatible web scraper; single binary;\n * self-host or cloud. Posts to {base}/v1/scrape.\n */\nexport class CrwScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private onlyMainContent?: boolean;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private headers?: Record<string, string>;\n  private renderJs?: boolean | null;\n  private cssSelector?: string;\n  private xpath?: string;\n  private proxy?: string;\n  private stealth?: boolean;\n\n  constructor(config: t.CrwScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ?? process.env.CRW_API_URL ?? 'https://api.fastcrw.com';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 7500;\n    this.logger = config.logger || createDefaultLogger();\n\n    this.onlyMainContent = config.onlyMainContent;\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.headers = config.headers;\n    this.renderJs = config.renderJs;\n    this.cssSelector = config.cssSelector;\n    this.xpath = config.xpath;\n    this.proxy = config.proxy;\n    this.stealth = config.stealth;\n\n    // Self-host fastCRW may run without auth, so a missing key is only a\n    // warning — unlike Firecrawl/Tavily, scrapeUrl does NOT early-return on it.\n    if (!this.apiKey) {\n      this.logger.warn('CRW_API_KEY is not set. Scraping will not work.');\n    }\n    this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  async scrapeUrl(\n    url: string,\n    options: t.CrwScrapeOptions = {}\n  ): Promise<[string, t.CrwScrapeResponse]> {\n    try {\n      const payloadTimeout = options.timeout ?? this.timeout;\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        waitFor: options.waitFor ?? this.waitFor,\n        headers: options.headers ?? this.headers,\n        renderJs: options.renderJs ?? this.renderJs,\n        cssSelector: options.cssSelector ?? this.cssSelector,\n        xpath: options.xpath ?? this.xpath,\n        proxy: options.proxy ?? this.proxy,\n        stealth: options.stealth ?? this.stealth,\n        // Cloud honors `timeout` (live-verified); the published OpenAPI\n        // documents `deadlineMs` (1..60000) instead. Send both.\n        timeout: payloadTimeout,\n        deadlineMs: Math.max(1, Math.min(payloadTimeout, 60000)),\n      });\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n      if (this.apiKey) {\n        headers.Authorization = `Bearer ${this.apiKey}`;\n      }\n\n      const response = await axios.post<t.CrwRawScrapeResponse>(\n        this.apiUrl,\n        payload,\n        {\n          headers,\n          timeout: payloadTimeout + CRW_TIMEOUT_BUFFER,\n        }\n      );\n\n      return [url, normalizeCrwResponse(response.data)];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `fastCRW API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response. Mirrors FirecrawlScraper — reads\n   * response.data.*, which normalizeCrwResponse guarantees, preserving the\n   * processContent ref-markers used by the reranker. Parameter is typed as the\n   * NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:\n   * TS class-method bivariance accepts this against BaseScraper's\n   * AnyScraperResponse, AND it lets us read response.data.plainText (a\n   * CrwScrapeResponse-only field) with no cast.\n   */\n  extractContent(\n    response: t.CrwScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    const htmlSource = response.data.html ?? response.data.rawHtml;\n    if (response.data.markdown != null && htmlSource != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          htmlSource,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    // CRW-only fallback (no Firecrawl equivalent): plain-text body.\n    if (response.data.plainText != null) {\n      return [response.data.plainText, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  extractMetadata(response: t.CrwScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a fastCRW scraper instance\n * @param config Scraper configuration\n * @returns fastCRW scraper instance\n */\nexport const createCrwScraper = (config: t.CrwScraperConfig = {}): CrwScraper =>\n  new CrwScraper(config);\n\n/**\n * fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,\n * ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested\n * container and fall back to top-level fields for self-host/legacy responses.\n */\nfunction normalizeCrwResponse(\n  raw: t.CrwRawScrapeResponse | null | undefined\n): t.CrwScrapeResponse {\n  if (raw == null) {\n    return { success: false, error: 'Empty fastCRW response' };\n  }\n  if (raw.success === false) {\n    return {\n      success: false,\n      error:\n        raw.error_code != null\n          ? `[${raw.error_code}] ${raw.error ?? 'Unknown error'}`\n          : (raw.error ?? 'fastCRW scrape failed'),\n      error_code: raw.error_code,\n    };\n  }\n  const data = raw.data ?? raw;\n  return {\n    success: true,\n    data: {\n      // Strip inline base64 image payloads from text content (not `screenshot`,\n      // which is base64 by design and never reaches the content processor).\n      markdown: stripBase64DataUris(data.markdown),\n      html: stripBase64DataUris(data.html),\n      rawHtml: stripBase64DataUris(data.rawHtml),\n      plainText: stripBase64DataUris(data.plainText),\n      screenshot: data.screenshot,\n      links: data.links,\n      metadata: data.metadata,\n    },\n  };\n}\n\n/**\n * Replace inline base64 data-URI payloads (typically images) in scraped text\n * with a short placeholder. Such payloads can be hundreds of KB; the content\n * processor builds a per-link RegExp from each URL, and one that large overflows\n * the engine's pattern-size limit (\"Invalid regular expression\"). Firecrawl\n * drops them via removeBase64Images — mirror that so image-heavy pages stay\n * processable (and don't bloat the LLM context with base64 noise).\n */\nfunction stripBase64DataUris(text: string | undefined): string | undefined {\n  if (text == null) {\n    return text;\n  }\n  return text.replace(\n    /data:[\\w.+-]+\\/[\\w.+-]+;base64,[A-Za-z0-9+/=]+/g,\n    'data:base64-content-removed'\n  );\n}\n\n// Helper function to clean up payload for fastCRW\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"mappings":";;;;;;AAOA,MAAM,qBAAqB;;;;;AAM3B,IAAa,aAAb,MAAiD;CAC/C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA6B,CAAC,GAAG;EAC3C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,eAAe;EAE1D,MAAM,UACJ,OAAO,UAAU,QAAQ,IAAI,eAAe;EAC9C,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE;EAE7C,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,MAAM;EAC3D,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,kBAAkB,OAAO;EAC9B,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,cAAc,OAAO;EAC1B,KAAK,QAAQ,OAAO;EACpB,KAAK,QAAQ,OAAO;EACpB,KAAK,UAAU,OAAO;EAItB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,iDAAiD;EAEpE,KAAK,OAAO,MAAM,yCAAyC,KAAK,QAAQ;CAC1E;CAEA,MAAM,UACJ,KACA,UAA8B,CAAC,GACS;EACxC,IAAI;GACF,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,UAAU,QAAQ,YAAY,KAAK;IACnC,aAAa,QAAQ,eAAe,KAAK;IACzC,OAAO,QAAQ,SAAS,KAAK;IAC7B,OAAO,QAAQ,SAAS,KAAK;IAC7B,SAAS,QAAQ,WAAW,KAAK;IAGjC,SAAS;IACT,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,GAAK,CAAC;GACzD,CAAC;GAED,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,KAAK,QACP,QAAQ,gBAAgB,UAAU,KAAK;GAYzC,OAAO,CAAC,KAAK,sBAAqB,MATX,MAAM,KAC3B,KAAK,QACL,SACA;IACE;IACA,SAAS,iBAAiB;GAC5B,CACF,EAAA,CAE2C,IAAI,CAAC;EAClD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,+BALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;;;;;CAWA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,aAAa,SAAS,KAAK,QAAQ,SAAS,KAAK;EACvD,IAAI,SAAS,KAAK,YAAY,QAAQ,cAAc,MAClD,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,YACA,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAI1C,IAAI,SAAS,KAAK,aAAa,MAC7B,OAAO,CAAC,SAAS,KAAK,WAAW,KAAA,CAAS;EAG5C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;CAEA,gBAAgB,UAAiD;EAC/D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,oBAAoB,SAA6B,CAAC,MAC7D,IAAI,WAAW,MAAM;;;;;;AAOvB,SAAS,qBACP,KACqB;CACrB,IAAI,OAAO,MACT,OAAO;EAAE,SAAS;EAAO,OAAO;CAAyB;CAE3D,IAAI,IAAI,YAAY,OAClB,OAAO;EACL,SAAS;EACT,OACE,IAAI,cAAc,OACd,IAAI,IAAI,WAAW,IAAI,IAAI,SAAS,oBACnC,IAAI,SAAS;EACpB,YAAY,IAAI;CAClB;CAEF,MAAM,OAAO,IAAI,QAAQ;CACzB,OAAO;EACL,SAAS;EACT,MAAM;GAGJ,UAAU,oBAAoB,KAAK,QAAQ;GAC3C,MAAM,oBAAoB,KAAK,IAAI;GACnC,SAAS,oBAAoB,KAAK,OAAO;GACzC,WAAW,oBAAoB,KAAK,SAAS;GAC7C,YAAY,KAAK;GACjB,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;CACF;AACF;;;;;;;;;AAUA,SAAS,oBAAoB,MAA8C;CACzE,IAAI,QAAQ,MACV,OAAO;CAET,OAAO,KAAK,QACV,mDACA,6BACF;AACF;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}

package/dist/esm/tools/search/crw-search.mjs

@@ -0,0 +1,103 @@
+import axios from "axios";
+//#region src/tools/search/crw-search.ts
+const DEFAULT_CRW_TIMEOUT = 15e3;
+const getHostname = (link) => {
+	try {
+		return new URL(link).hostname;
+	} catch {
+		return link;
+	}
+};
+/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat
+* SearchResult[] with a `category` field; route rows into source groups. */
+const splitByCategory = (rows) => {
+	const groups = {
+		web: [],
+		images: [],
+		news: []
+	};
+	for (const row of rows) if (row.category === "news") groups.news.push(row);
+	else if (row.category === "images") groups.images.push({
+		...row,
+		imageUrl: row.url
+	});
+	else groups.web.push(row);
+	return groups;
+};
+const createCrwAPI = (apiKey, apiUrl, options) => {
+	const base = (apiUrl ?? process.env.CRW_API_URL ?? "https://api.fastcrw.com").replace(/\/+$/, "");
+	const config = {
+		apiKey: apiKey ?? process.env.CRW_API_KEY,
+		apiUrl: `${base}/v1/search`,
+		timeout: options?.timeout ?? DEFAULT_CRW_TIMEOUT
+	};
+	const getSources = async ({ query, date, numResults = 8, type }) => {
+		if (!query.trim()) return {
+			success: false,
+			error: "Query cannot be empty"
+		};
+		try {
+			const limit = Math.min(Math.max(1, options?.maxResults ?? numResults), 20);
+			let sources;
+			if (type === "images") sources = ["images"];
+			else if (type === "news") sources = ["news"];
+			else sources = options?.includeImages === true ? ["web", "images"] : ["web"];
+			const payload = {
+				query,
+				limit,
+				sources
+			};
+			if (date != null) payload.tbs = `qdr:${date}`;
+			const headers = { "Content-Type": "application/json" };
+			if (config.apiKey != null && config.apiKey !== "") headers.Authorization = `Bearer ${config.apiKey}`;
+			const body = (await axios.post(config.apiUrl, payload, {
+				headers,
+				timeout: config.timeout
+			})).data;
+			if (body.success === false) return {
+				success: false,
+				error: `fastCRW search failed: ${body.error_code != null ? `[${body.error_code}] ` : ""}${body.error ?? "Unknown error"}`
+			};
+			const container = body.data ?? {};
+			const data = Array.isArray(container) ? splitByCategory(container) : container.results ?? container;
+			return {
+				success: true,
+				data: {
+					organic: (data.web ?? []).filter((r) => r.url != null && r.url !== "").map((r) => ({
+						title: r.title ?? "",
+						link: r.url,
+						snippet: r.description ?? r.snippet ?? "",
+						position: r.position
+					})),
+					images: (data.images ?? []).filter((r) => r.imageUrl != null && r.imageUrl !== "").map((r) => ({
+						title: r.title,
+						imageUrl: r.imageUrl,
+						link: r.url,
+						position: r.position
+					})),
+					topStories: [],
+					videos: [],
+					news: (data.news ?? []).filter((r) => r.url != null && r.url !== "").map((r) => ({
+						title: r.title ?? "",
+						link: r.url,
+						snippet: r.description ?? r.snippet ?? "",
+						date: r.publishedDate,
+						source: getHostname(r.url),
+						position: r.position
+					})),
+					relatedSearches: []
+				}
+			};
+		} catch (error) {
+			return {
+				success: false,
+				error: `fastCRW search request failed: ${error instanceof Error ? error.message : String(error)}`
+			};
+		}
+	};
+	return { getSources };
+};
+//#endregion
+export { createCrwAPI };
+
+//# sourceMappingURL=crw-search.mjs.map