@effect-uai/firecrawl 0.10.0 → 0.12.0

package/dist/FirecrawlRead.d.mts

@@ -6,7 +6,7 @@ import { CommonReadRequest, ReadResponse, WebRead } from "@effect-uai/core/WebRe
 
 //#region src/FirecrawlRead.d.ts
 declare namespace FirecrawlRead_d_exports {
-  export { Config, FirecrawlRead, FirecrawlReadRequest, FirecrawlReadService, layer, make };
+  export { Config, FirecrawlMapRequest, FirecrawlRead, FirecrawlReadRequest, FirecrawlReadService, MapLink, MapResponse, layer, make };
 }
 /**
  * Firecrawl-typed read request. Widens {@link CommonReadRequest} with
@@ -21,8 +21,37 @@ type FirecrawlReadRequest = CommonReadRequest & {
   readonly mobile?: boolean; /** Proxy tier - see {@link FirecrawlProxy}. */
   readonly proxy?: FirecrawlProxy;
 };
+/**
+ * Firecrawl-typed map (sitemap / URL discovery) request. Given a base `url`,
+ * Firecrawl returns the site's URLs without scraping each one - handy for
+ * locating specific pages (pricing, docs) before a targeted read. All fields
+ * but `url` are optional.
+ */
+type FirecrawlMapRequest = {
+  readonly url: string; /** Order results by relevance to this query. */
+  readonly search?: string; /** Cap the number of links returned. */
+  readonly limit?: number; /** How to use the site's sitemap: `only` (sitemap alone), `include`, or `skip`. */
+  readonly sitemap?: "only" | "include" | "skip"; /** Include links on subdomains of the base URL. */
+  readonly includeSubdomains?: boolean; /** Abort the map if it runs past this duration. */
+  readonly timeout?: Duration.Duration;
+};
+/** One discovered URL, with page metadata where Firecrawl has it. */
+type MapLink = {
+  readonly url: string;
+  readonly title?: string;
+  readonly description?: string;
+};
+type MapResponse = {
+  readonly links: ReadonlyArray<MapLink>;
+  readonly raw: unknown;
+};
 type FirecrawlReadService = {
   readonly read: (request: FirecrawlReadRequest) => Effect.Effect<ReadResponse, AiError.AiError>;
+  /**
+   * Discover a site's URLs (sitemap-style) without scraping. Firecrawl-specific;
+   * not on the generic `WebRead` tag, whose contract is a single-URL read.
+   */
+  readonly map: (request: FirecrawlMapRequest) => Effect.Effect<MapResponse, AiError.AiError>;
 };
 declare const FirecrawlRead_base: Context.ServiceClass<FirecrawlRead, "@betalyra/effect-uai/providers/firecrawl/FirecrawlRead", FirecrawlReadService>;
 /**
@@ -48,5 +77,5 @@ declare const make: (cfg: Config) => Effect.Effect<FirecrawlReadService, never,
  */
 declare const layer: (cfg: Config) => Layer.Layer<FirecrawlRead | WebRead, never, HttpClient.HttpClient>;
 //#endregion
-export { Config, FirecrawlRead, FirecrawlReadRequest, FirecrawlReadService, layer, make, FirecrawlRead_d_exports as t };
+export { Config, FirecrawlMapRequest, FirecrawlRead, FirecrawlReadRequest, FirecrawlReadService, MapLink, MapResponse, layer, make, FirecrawlRead_d_exports as t };
 //# sourceMappingURL=FirecrawlRead.d.mts.map

package/dist/FirecrawlRead.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"FirecrawlRead.d.mts","names":[],"sources":["../src/FirecrawlRead.ts"],"mappings":";;;;;;;;;;;;;;;KAoBY,oBAAA,GAAuB,iBAAA;iFAExB,eAAA;WAEA,WAAA,GAAc,aAAA;WAEd,WAAA,GAAc,aAAA,UANO;EAAA,SAQrB,OAAA,GAAU,QAAA,CAAS,QAAA,EARK;EAAA,SAUxB,MAAA,YAJc;EAAA,SAMd,KAAA,GAAQ,cAAA;AAAA;AAAA,KAGP,oBAAA;EAAA,SACD,IAAA,GAAO,OAAA,EAAS,oBAAA,KAAyB,MAAA,CAAO,MAAA,CAAO,YAAA,EAAc,OAAA,CAAQ,OAAA;AAAA;AAAA,cACvF,kBAAA;;;;;;cAOY,aAAA,SAAsB,kBAAA;AAAA,KAIvB,MAAA;EAAA,SACD,MAAA,EAAQ,QAAA,CAAS,QAAA;EAAA,SACjB,OAAA;AAAA;;AAfX;;;cAyJa,IAAA,GACX,GAAA,EAAK,MAAA,KACJ,MAAA,CAAO,MAAA,CAAO,oBAAA,SAA6B,UAAA,CAAW,UAAA;;;;;;;cAY5C,KAAA,GACX,GAAA,EAAK,MAAA,KACJ,KAAA,CAAM,KAAA,CAAM,aAAA,GAAgB,OAAA,SAAgB,UAAA,CAAW,UAAA"}
+{"version":3,"file":"FirecrawlRead.d.mts","names":[],"sources":["../src/FirecrawlRead.ts"],"mappings":";;;;;;;;;;;;;;;KAoBY,oBAAA,GAAuB,iBAAA;iFAExB,eAAA;WAEA,WAAA,GAAc,aAAA;WAEd,WAAA,GAAc,aAAA;WAEd,OAAA,GAAU,QAAA,CAAS,QAAA,EARlB;EAAA,SAUD,MAAA;WAEA,KAAA,GAAQ,cAAA;AAAA;;;;;;;KASP,mBAAA;EAAA,SACD,GAAA,UAlBc;EAAA,SAoBd,MAAA,WAlBc;EAAA,SAoBd,KAAA,WAlBU;EAAA,SAoBV,OAAA,gCAlBA;EAAA,SAoBA,iBAAA,YAlBQ;EAAA,SAoBR,OAAA,GAAU,QAAA,CAAS,QAAA;AAAA;AAX9B;AAAA,KAeY,OAAA;EAAA,SACD,GAAA;EAAA,SACA,KAAA;EAAA,SACA,WAAA;AAAA;AAAA,KAGC,WAAA;EAAA,SACD,KAAA,EAAO,aAAA,CAAc,OAAA;EAAA,SACrB,GAAA;AAAA;AAAA,KAGC,oBAAA;EAAA,SACD,IAAA,GAAO,OAAA,EAAS,oBAAA,KAAyB,MAAA,CAAO,MAAA,CAAO,YAAA,EAAc,OAAA,CAAQ,OAAA;EAhBlD;;AAItC;;EAJsC,SAqB3B,GAAA,GAAM,OAAA,EAAS,mBAAA,KAAwB,MAAA,CAAO,MAAA,CAAO,WAAA,EAAa,OAAA,CAAQ,OAAA;AAAA;AAAA,cACpF,kBAAA;;;;;AAZD;cAmBa,aAAA,SAAsB,kBAAA;AAAA,KAIvB,MAAA;EAAA,SACD,MAAA,EAAQ,QAAA,CAAS,QAAA;EAAA,SACjB,OAAA;AAAA;;;;AApBX;cA2Qa,IAAA,GACX,GAAA,EAAK,MAAA,KACJ,MAAA,CAAO,MAAA,CAAO,oBAAA,SAA6B,UAAA,CAAW,UAAA;;;;;;;cAc5C,KAAA,GACX,GAAA,EAAK,MAAA,KACJ,KAAA,CAAM,KAAA,CAAM,aAAA,GAAgB,OAAA,SAAgB,UAAA,CAAW,UAAA"}

package/dist/FirecrawlRead.mjs

@@ -29,6 +29,14 @@ const buildBody = (request) => {
 		...request.proxy !== void 0 && { proxy: request.proxy }
 	};
 };
+const buildMapBody = (request) => ({
+	url: request.url,
+	...request.search !== void 0 && { search: request.search },
+	...request.limit !== void 0 && { limit: request.limit },
+	...request.sitemap !== void 0 && { sitemap: request.sitemap },
+	...request.includeSubdomains !== void 0 && { includeSubdomains: request.includeSubdomains },
+	...request.timeout !== void 0 && { timeout: Duration.toMillis(request.timeout) }
+});
 const WireMetadata = Schema.Struct({
 	title: Schema.optional(Schema.NullOr(Schema.Union([Schema.String, Schema.Array(Schema.String)]))),
 	sourceURL: Schema.optional(Schema.NullOr(Schema.String)),
@@ -41,13 +49,20 @@ const WireData = Schema.Struct({
 	links: Schema.optional(Schema.NullOr(Schema.Array(Schema.String))),
 	metadata: Schema.optional(WireMetadata)
 });
+/**
+* `data` is optional because a failed scrape is not an HTTP failure. Firecrawl
+* answers 200 with `{ success: false, error }` when a page is blocked, times
+* out or renders only under JS — so requiring `data` turned every one of those
+* into a decode error, which was then reported as a transport fault and
+* retried on the backoff schedule.
+*/
 const WireResponse = Schema.Struct({
 	success: Schema.optional(Schema.Boolean),
-	data: WireData
+	data: Schema.optional(WireData),
+	error: Schema.optional(Schema.NullOr(Schema.String))
 });
 const pickTitle = (title) => typeof title === "string" ? title : Array.isArray(title) ? title[0] : void 0;
-const toResponse = (wire, request) => {
-	const { data } = wire;
+const toResponse = (wire, data, request) => {
 	const content = ((request.format ?? "markdown") === "html" ? data.html : data.markdown) ?? "";
 	const title = pickTitle(data.metadata?.title);
 	const links = data.links ?? void 0;
@@ -59,10 +74,43 @@ const toResponse = (wire, request) => {
 		raw: wire
 	};
 };
+const WireMapLink = Schema.Struct({
+	url: Schema.String,
+	title: Schema.optional(Schema.NullOr(Schema.String)),
+	description: Schema.optional(Schema.NullOr(Schema.String))
+});
+const WireMapResponse = Schema.Struct({
+	success: Schema.optional(Schema.Boolean),
+	links: Schema.Array(Schema.Union([WireMapLink, Schema.String]))
+});
+const toMapLink = (link) => typeof link === "string" ? { url: link } : {
+	url: link.url,
+	...link.title != null && { title: link.title },
+	...link.description != null && { description: link.description }
+};
+const toMapResponse = (wire) => ({
+	links: wire.links.map(toMapLink),
+	raw: wire
+});
 const transportFailure = (cause) => new AiError.Unavailable({
 	provider: "firecrawl",
 	raw: cause
 });
+/**
+* The call arrived and the answer is unusable — a scrape Firecrawl itself
+* reports as failed, or a body that does not match what this provider knows how
+* to read.
+*
+* Deliberately not `Unavailable`: that tag is retryable, and neither of these
+* gets better on a second attempt. A blocked page stays blocked while the
+* backoff spends the scrape budget on it, and a changed API shape is a code
+* problem, not a weather problem.
+*/
+const unusableResponse = (message, raw) => new AiError.GenerationFailed({
+	provider: "firecrawl",
+	message,
+	raw
+});
 const httpStatusError = (status, body) => {
 	const provider = "firecrawl";
 	const raw = body;
@@ -109,14 +157,29 @@ const postScrape = (cfg, body) => Effect.gen(function* () {
 		return yield* httpStatusError(response.status, text);
 	}
 	const json = yield* response.json.pipe(Effect.mapError(transportFailure));
-	return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(Effect.mapError(transportFailure));
+	return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(Effect.mapError((issue) => unusableResponse("firecrawl returned a body this provider cannot read", issue)));
+});
+const readImpl = (cfg) => (request) => Effect.flatMap(postScrape(cfg, buildBody(request)), (wire) => wire.data === void 0 || wire.success === false ? Effect.fail(unusableResponse(wire.error ?? `firecrawl could not scrape ${request.url}`, wire)) : Effect.succeed(toResponse(wire, wire.data, request)));
+const postMap = (cfg, body) => Effect.gen(function* () {
+	const client = yield* HttpClient.HttpClient;
+	const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/map`).pipe(HttpClientRequest.setHeader("Authorization", `Bearer ${Redacted.value(cfg.apiKey)}`), HttpClientRequest.bodyJsonUnsafe(body));
+	const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure));
+	if (response.status >= 400) {
+		const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""));
+		return yield* httpStatusError(response.status, text);
+	}
+	const json = yield* response.json.pipe(Effect.mapError(transportFailure));
+	return yield* Schema.decodeUnknownEffect(WireMapResponse)(json).pipe(Effect.mapError(transportFailure));
 });
-const readImpl = (cfg) => (request) => Effect.map(postScrape(cfg, buildBody(request)), (wire) => toResponse(wire, request));
+const mapImpl = (cfg) => (request) => Effect.map(postMap(cfg, buildMapBody(request)), toMapResponse);
 /**
 * Build a `FirecrawlReadService` value. For Layer-based setup, prefer
 * {@link layer}.
 */
-const make = (cfg) => Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({ read: (request) => readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)) }));
+const make = (cfg) => Effect.map(HttpClient.HttpClient, (client) => ({
+	read: (request) => readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
+	map: (request) => mapImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client))
+}));
 /**
 * Layer registering both the provider-typed `FirecrawlRead` tag and the
 * generic `WebRead` tag over one implementation. A `CommonReadRequest` is

package/dist/FirecrawlRead.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"FirecrawlRead.mjs","names":[],"sources":["../src/FirecrawlRead.ts"],"sourcesContent":["import { Context, Duration, Effect, Layer, Redacted, Schema } from \"effect\"\nimport { HttpClient, HttpClientRequest } from \"effect/unstable/http\"\nimport * as AiError from \"@effect-uai/core/AiError\"\nimport {\n  type CommonReadRequest,\n  type ReadResponse,\n  WebRead,\n  type WebReadService,\n} from \"@effect-uai/core/WebRead\"\nimport type { FirecrawlProxy } from \"./models.js\"\n\n// ---------------------------------------------------------------------------\n// Public types\n// ---------------------------------------------------------------------------\n\n/**\n * Firecrawl-typed read request. Widens {@link CommonReadRequest} with\n * Firecrawl's own scrape knobs; the common fields map onto Firecrawl's wire\n * body (see the codec below).\n */\nexport type FirecrawlReadRequest = CommonReadRequest & {\n  /** Keep only the main content, dropping nav/header/footer/sidebar. */\n  readonly onlyMainContent?: boolean\n  /** Restrict extraction to these HTML tags / selectors. */\n  readonly includeTags?: ReadonlyArray<string>\n  /** Drop these HTML tags / selectors before extraction. */\n  readonly excludeTags?: ReadonlyArray<string>\n  /** Wait this long for the page to settle before scraping (JS-heavy sites). */\n  readonly waitFor?: Duration.Duration\n  /** Emulate a mobile device. */\n  readonly mobile?: boolean\n  /** Proxy tier - see {@link FirecrawlProxy}. */\n  readonly proxy?: FirecrawlProxy\n}\n\nexport type FirecrawlReadService = {\n  readonly read: (request: FirecrawlReadRequest) => Effect.Effect<ReadResponse, AiError.AiError>\n}\n\n/**\n * Provider-typed service tag. Yield this for Firecrawl-typed reads (its own\n * scrape knobs); yield the generic `WebRead` tag for provider-portable code.\n * Both are registered by {@link layer}.\n */\nexport class FirecrawlRead extends Context.Service<FirecrawlRead, FirecrawlReadService>()(\n  \"@betalyra/effect-uai/providers/firecrawl/FirecrawlRead\",\n) {}\n\nexport type Config = {\n  readonly apiKey: Redacted.Redacted\n  readonly baseUrl?: string\n}\n\n// ---------------------------------------------------------------------------\n// Codec - request body\n// ---------------------------------------------------------------------------\n\ntype WireBody = {\n  readonly url: string\n  readonly formats: ReadonlyArray<string>\n  readonly onlyMainContent?: boolean\n  readonly includeTags?: ReadonlyArray<string>\n  readonly excludeTags?: ReadonlyArray<string>\n  readonly waitFor?: number\n  readonly timeout?: number\n  readonly mobile?: boolean\n  readonly proxy?: string\n}\n\n// `markdown` / `html` map one-to-one onto Firecrawl's format names. `links` is\n// always requested so {@link ReadResponse.links} can be populated; it does not\n// change the per-page billing.\nconst buildBody = (request: FirecrawlReadRequest): WireBody => {\n  const contentFormat = request.format ?? \"markdown\"\n  return {\n    url: request.url,\n    formats: [contentFormat, \"links\"],\n    ...(request.onlyMainContent !== undefined && { onlyMainContent: request.onlyMainContent }),\n    ...(request.includeTags !== undefined && { includeTags: request.includeTags }),\n    ...(request.excludeTags !== undefined && { excludeTags: request.excludeTags }),\n    ...(request.waitFor !== undefined && { waitFor: Duration.toMillis(request.waitFor) }),\n    ...(request.timeout !== undefined && { timeout: Duration.toMillis(request.timeout) }),\n    ...(request.mobile !== undefined && { mobile: request.mobile }),\n    ...(request.proxy !== undefined && { proxy: request.proxy }),\n  }\n}\n\n// ---------------------------------------------------------------------------\n// Codec - response\n// ---------------------------------------------------------------------------\n\nconst WireMetadata = Schema.Struct({\n  // Firecrawl returns `title` as a string, an array (repeated meta tags), or\n  // null; normalize to a single string in `toResponse`.\n  title: Schema.optional(Schema.NullOr(Schema.Union([Schema.String, Schema.Array(Schema.String)]))),\n  sourceURL: Schema.optional(Schema.NullOr(Schema.String)),\n  statusCode: Schema.optional(Schema.Number),\n})\n\nconst WireData = Schema.Struct({\n  markdown: Schema.optional(Schema.NullOr(Schema.String)),\n  html: Schema.optional(Schema.NullOr(Schema.String)),\n  rawHtml: Schema.optional(Schema.NullOr(Schema.String)),\n  links: Schema.optional(Schema.NullOr(Schema.Array(Schema.String))),\n  metadata: Schema.optional(WireMetadata),\n})\n\nconst WireResponse = Schema.Struct({\n  success: Schema.optional(Schema.Boolean),\n  data: WireData,\n})\ntype WireResponse = typeof WireResponse.Type\n\nconst pickTitle = (title: string | ReadonlyArray<string> | null | undefined): string | undefined =>\n  typeof title === \"string\" ? title : Array.isArray(title) ? title[0] : undefined\n\nconst toResponse = (wire: WireResponse, request: FirecrawlReadRequest): ReadResponse => {\n  const { data } = wire\n  const format = request.format ?? \"markdown\"\n  const content = (format === \"html\" ? data.html : data.markdown) ?? \"\"\n  const title = pickTitle(data.metadata?.title)\n  const links = data.links ?? undefined\n  return {\n    url: data.metadata?.sourceURL ?? request.url,\n    content,\n    ...(title !== undefined && { title }),\n    ...(links !== undefined && links.length > 0 && { links }),\n    raw: wire,\n  }\n}\n\n// ---------------------------------------------------------------------------\n// HTTP\n// ---------------------------------------------------------------------------\n\nconst transportFailure = (cause: unknown): AiError.AiError =>\n  new AiError.Unavailable({ provider: \"firecrawl\", raw: cause })\n\nconst httpStatusError = (status: number, body: string): AiError.AiError => {\n  const provider = \"firecrawl\"\n  const raw = body\n  if (status === 429) return new AiError.RateLimited({ provider, raw })\n  if (status === 408 || status === 504) return new AiError.Timeout({ provider, raw })\n  if (status === 401) return new AiError.AuthFailed({ provider, subtype: \"auth\", raw })\n  if (status === 403) return new AiError.AuthFailed({ provider, subtype: \"permission\", raw })\n  if (status === 402) return new AiError.AuthFailed({ provider, subtype: \"billing\", raw })\n  if (status >= 500) return new AiError.Unavailable({ provider, status, raw })\n  return new AiError.InvalidRequest({ provider, raw })\n}\n\nconst baseUrl = (cfg: Config): string => cfg.baseUrl ?? \"https://api.firecrawl.dev\"\n\nconst postScrape = (\n  cfg: Config,\n  body: WireBody,\n): Effect.Effect<WireResponse, AiError.AiError, HttpClient.HttpClient> =>\n  Effect.gen(function* () {\n    const client = yield* HttpClient.HttpClient\n    const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/scrape`).pipe(\n      HttpClientRequest.setHeader(\"Authorization\", `Bearer ${Redacted.value(cfg.apiKey)}`),\n      HttpClientRequest.bodyJsonUnsafe(body),\n    )\n    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))\n    if (response.status >= 400) {\n      const text = yield* response.text.pipe(Effect.orElseSucceed(() => \"\"))\n      return yield* httpStatusError(response.status, text)\n    }\n    const json = yield* response.json.pipe(Effect.mapError(transportFailure))\n    return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(\n      Effect.mapError(transportFailure),\n    )\n  })\n\nconst readImpl =\n  (cfg: Config) =>\n  (\n    request: FirecrawlReadRequest,\n  ): Effect.Effect<ReadResponse, AiError.AiError, HttpClient.HttpClient> =>\n    Effect.map(postScrape(cfg, buildBody(request)), (wire) => toResponse(wire, request))\n\n// ---------------------------------------------------------------------------\n// Constructors\n// ---------------------------------------------------------------------------\n\n/**\n * Build a `FirecrawlReadService` value. For Layer-based setup, prefer\n * {@link layer}.\n */\nexport const make = (\n  cfg: Config,\n): Effect.Effect<FirecrawlReadService, never, HttpClient.HttpClient> =>\n  Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({\n    read: (request) =>\n      readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),\n  }))\n\n/**\n * Layer registering both the provider-typed `FirecrawlRead` tag and the\n * generic `WebRead` tag over one implementation. A `CommonReadRequest` is\n * structurally a `FirecrawlReadRequest` with the extra knobs unset, so the\n * generic registration forwards directly.\n */\nexport const layer = (\n  cfg: Config,\n): Layer.Layer<FirecrawlRead | WebRead, never, HttpClient.HttpClient> => {\n  const typed = Layer.effect(FirecrawlRead, make(cfg))\n  const generic = Layer.effect(\n    WebRead,\n    Effect.map(make(cfg), (s): WebReadService => ({ read: (request) => s.read(request) })),\n  )\n  return Layer.merge(typed, generic)\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA4CA,IAAa,gBAAb,cAAmC,QAAQ,SAA8C,CACvF,yDACD,CAAC;AA0BF,MAAM,aAAa,YAA4C;CAC7D,MAAM,gBAAgB,QAAQ,UAAU;AACxC,QAAO;EACL,KAAK,QAAQ;EACb,SAAS,CAAC,eAAe,QAAQ;EACjC,GAAI,QAAQ,oBAAoB,KAAA,KAAa,EAAE,iBAAiB,QAAQ,iBAAiB;EACzF,GAAI,QAAQ,gBAAgB,KAAA,KAAa,EAAE,aAAa,QAAQ,aAAa;EAC7E,GAAI,QAAQ,gBAAgB,KAAA,KAAa,EAAE,aAAa,QAAQ,aAAa;EAC7E,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,SAAS,SAAS,QAAQ,QAAQ,EAAE;EACpF,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,SAAS,SAAS,QAAQ,QAAQ,EAAE;EACpF,GAAI,QAAQ,WAAW,KAAA,KAAa,EAAE,QAAQ,QAAQ,QAAQ;EAC9D,GAAI,QAAQ,UAAU,KAAA,KAAa,EAAE,OAAO,QAAQ,OAAO;EAC5D;;AAOH,MAAM,eAAe,OAAO,OAAO;CAGjC,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC,OAAO,QAAQ,OAAO,MAAM,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;CACjG,WAAW,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACxD,YAAY,OAAO,SAAS,OAAO,OAAO;CAC3C,CAAC;AAEF,MAAM,WAAW,OAAO,OAAO;CAC7B,UAAU,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACvD,MAAM,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACnD,SAAS,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACtD,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,OAAO,OAAO,CAAC,CAAC;CAClE,UAAU,OAAO,SAAS,aAAa;CACxC,CAAC;AAEF,MAAM,eAAe,OAAO,OAAO;CACjC,SAAS,OAAO,SAAS,OAAO,QAAQ;CACxC,MAAM;CACP,CAAC;AAGF,MAAM,aAAa,UACjB,OAAO,UAAU,WAAW,QAAQ,MAAM,QAAQ,MAAM,GAAG,MAAM,KAAK,KAAA;AAExE,MAAM,cAAc,MAAoB,YAAgD;CACtF,MAAM,EAAE,SAAS;CAEjB,MAAM,YADS,QAAQ,UAAU,gBACL,SAAS,KAAK,OAAO,KAAK,aAAa;CACnE,MAAM,QAAQ,UAAU,KAAK,UAAU,MAAM;CAC7C,MAAM,QAAQ,KAAK,SAAS,KAAA;AAC5B,QAAO;EACL,KAAK,KAAK,UAAU,aAAa,QAAQ;EACzC;EACA,GAAI,UAAU,KAAA,KAAa,EAAE,OAAO;EACpC,GAAI,UAAU,KAAA,KAAa,MAAM,SAAS,KAAK,EAAE,OAAO;EACxD,KAAK;EACN;;AAOH,MAAM,oBAAoB,UACxB,IAAI,QAAQ,YAAY;CAAE,UAAU;CAAa,KAAK;CAAO,CAAC;AAEhE,MAAM,mBAAmB,QAAgB,SAAkC;CACzE,MAAM,WAAW;CACjB,MAAM,MAAM;AACZ,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,YAAY;EAAE;EAAU;EAAK,CAAC;AACrE,KAAI,WAAW,OAAO,WAAW,IAAK,QAAO,IAAI,QAAQ,QAAQ;EAAE;EAAU;EAAK,CAAC;AACnF,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAQ;EAAK,CAAC;AACrF,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAc;EAAK,CAAC;AAC3F,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAW;EAAK,CAAC;AACxF,KAAI,UAAU,IAAK,QAAO,IAAI,QAAQ,YAAY;EAAE;EAAU;EAAQ;EAAK,CAAC;AAC5E,QAAO,IAAI,QAAQ,eAAe;EAAE;EAAU;EAAK,CAAC;;AAGtD,MAAM,WAAW,QAAwB,IAAI,WAAW;AAExD,MAAM,cACJ,KACA,SAEA,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,OAAO,WAAW;CACjC,MAAM,cAAc,kBAAkB,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,CAAC,KACtE,kBAAkB,UAAU,iBAAiB,UAAU,SAAS,MAAM,IAAI,OAAO,GAAG,EACpF,kBAAkB,eAAe,KAAK,CACvC;CACD,MAAM,WAAW,OAAO,OAAO,QAAQ,YAAY,CAAC,KAAK,OAAO,SAAS,iBAAiB,CAAC;AAC3F,KAAI,SAAS,UAAU,KAAK;EAC1B,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,oBAAoB,GAAG,CAAC;AACtE,SAAO,OAAO,gBAAgB,SAAS,QAAQ,KAAK;;CAEtD,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,iBAAiB,CAAC;AACzE,QAAO,OAAO,OAAO,oBAAoB,aAAa,CAAC,KAAK,CAAC,KAC3D,OAAO,SAAS,iBAAiB,CAClC;EACD;AAEJ,MAAM,YACH,SAEC,YAEA,OAAO,IAAI,WAAW,KAAK,UAAU,QAAQ,CAAC,GAAG,SAAS,WAAW,MAAM,QAAQ,CAAC;;;;;AAUxF,MAAa,QACX,QAEA,OAAO,IAAI,WAAW,WAAW,UAAU,GAAG,YAAY,EACxD,OAAO,YACL,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,OAAO,eAAe,WAAW,YAAY,OAAO,CAAC,EACpF,EAAE;;;;;;;AAQL,MAAa,SACX,QACuE;CACvE,MAAM,QAAQ,MAAM,OAAO,eAAe,KAAK,IAAI,CAAC;CACpD,MAAM,UAAU,MAAM,OACpB,SACA,OAAO,IAAI,KAAK,IAAI,GAAG,OAAuB,EAAE,OAAO,YAAY,EAAE,KAAK,QAAQ,EAAE,EAAE,CACvF;AACD,QAAO,MAAM,MAAM,OAAO,QAAQ"}
+{"version":3,"file":"FirecrawlRead.mjs","names":[],"sources":["../src/FirecrawlRead.ts"],"sourcesContent":["import { Context, Duration, Effect, Layer, Redacted, Schema } from \"effect\"\nimport { HttpClient, HttpClientRequest } from \"effect/unstable/http\"\nimport * as AiError from \"@effect-uai/core/AiError\"\nimport {\n  type CommonReadRequest,\n  type ReadResponse,\n  WebRead,\n  type WebReadService,\n} from \"@effect-uai/core/WebRead\"\nimport type { FirecrawlProxy } from \"./models.js\"\n\n// ---------------------------------------------------------------------------\n// Public types\n// ---------------------------------------------------------------------------\n\n/**\n * Firecrawl-typed read request. Widens {@link CommonReadRequest} with\n * Firecrawl's own scrape knobs; the common fields map onto Firecrawl's wire\n * body (see the codec below).\n */\nexport type FirecrawlReadRequest = CommonReadRequest & {\n  /** Keep only the main content, dropping nav/header/footer/sidebar. */\n  readonly onlyMainContent?: boolean\n  /** Restrict extraction to these HTML tags / selectors. */\n  readonly includeTags?: ReadonlyArray<string>\n  /** Drop these HTML tags / selectors before extraction. */\n  readonly excludeTags?: ReadonlyArray<string>\n  /** Wait this long for the page to settle before scraping (JS-heavy sites). */\n  readonly waitFor?: Duration.Duration\n  /** Emulate a mobile device. */\n  readonly mobile?: boolean\n  /** Proxy tier - see {@link FirecrawlProxy}. */\n  readonly proxy?: FirecrawlProxy\n}\n\n/**\n * Firecrawl-typed map (sitemap / URL discovery) request. Given a base `url`,\n * Firecrawl returns the site's URLs without scraping each one - handy for\n * locating specific pages (pricing, docs) before a targeted read. All fields\n * but `url` are optional.\n */\nexport type FirecrawlMapRequest = {\n  readonly url: string\n  /** Order results by relevance to this query. */\n  readonly search?: string\n  /** Cap the number of links returned. */\n  readonly limit?: number\n  /** How to use the site's sitemap: `only` (sitemap alone), `include`, or `skip`. */\n  readonly sitemap?: \"only\" | \"include\" | \"skip\"\n  /** Include links on subdomains of the base URL. */\n  readonly includeSubdomains?: boolean\n  /** Abort the map if it runs past this duration. */\n  readonly timeout?: Duration.Duration\n}\n\n/** One discovered URL, with page metadata where Firecrawl has it. */\nexport type MapLink = {\n  readonly url: string\n  readonly title?: string\n  readonly description?: string\n}\n\nexport type MapResponse = {\n  readonly links: ReadonlyArray<MapLink>\n  readonly raw: unknown\n}\n\nexport type FirecrawlReadService = {\n  readonly read: (request: FirecrawlReadRequest) => Effect.Effect<ReadResponse, AiError.AiError>\n  /**\n   * Discover a site's URLs (sitemap-style) without scraping. Firecrawl-specific;\n   * not on the generic `WebRead` tag, whose contract is a single-URL read.\n   */\n  readonly map: (request: FirecrawlMapRequest) => Effect.Effect<MapResponse, AiError.AiError>\n}\n\n/**\n * Provider-typed service tag. Yield this for Firecrawl-typed reads (its own\n * scrape knobs); yield the generic `WebRead` tag for provider-portable code.\n * Both are registered by {@link layer}.\n */\nexport class FirecrawlRead extends Context.Service<FirecrawlRead, FirecrawlReadService>()(\n  \"@betalyra/effect-uai/providers/firecrawl/FirecrawlRead\",\n) {}\n\nexport type Config = {\n  readonly apiKey: Redacted.Redacted\n  readonly baseUrl?: string\n}\n\n// ---------------------------------------------------------------------------\n// Codec - request body\n// ---------------------------------------------------------------------------\n\ntype WireBody = {\n  readonly url: string\n  readonly formats: ReadonlyArray<string>\n  readonly onlyMainContent?: boolean\n  readonly includeTags?: ReadonlyArray<string>\n  readonly excludeTags?: ReadonlyArray<string>\n  readonly waitFor?: number\n  readonly timeout?: number\n  readonly mobile?: boolean\n  readonly proxy?: string\n}\n\n// `markdown` / `html` map one-to-one onto Firecrawl's format names. `links` is\n// always requested so {@link ReadResponse.links} can be populated; it does not\n// change the per-page billing.\nconst buildBody = (request: FirecrawlReadRequest): WireBody => {\n  const contentFormat = request.format ?? \"markdown\"\n  return {\n    url: request.url,\n    formats: [contentFormat, \"links\"],\n    ...(request.onlyMainContent !== undefined && { onlyMainContent: request.onlyMainContent }),\n    ...(request.includeTags !== undefined && { includeTags: request.includeTags }),\n    ...(request.excludeTags !== undefined && { excludeTags: request.excludeTags }),\n    ...(request.waitFor !== undefined && { waitFor: Duration.toMillis(request.waitFor) }),\n    ...(request.timeout !== undefined && { timeout: Duration.toMillis(request.timeout) }),\n    ...(request.mobile !== undefined && { mobile: request.mobile }),\n    ...(request.proxy !== undefined && { proxy: request.proxy }),\n  }\n}\n\ntype WireMapBody = {\n  readonly url: string\n  readonly search?: string\n  readonly limit?: number\n  readonly sitemap?: string\n  readonly includeSubdomains?: boolean\n  readonly timeout?: number\n}\n\nconst buildMapBody = (request: FirecrawlMapRequest): WireMapBody => ({\n  url: request.url,\n  ...(request.search !== undefined && { search: request.search }),\n  ...(request.limit !== undefined && { limit: request.limit }),\n  ...(request.sitemap !== undefined && { sitemap: request.sitemap }),\n  ...(request.includeSubdomains !== undefined && { includeSubdomains: request.includeSubdomains }),\n  ...(request.timeout !== undefined && { timeout: Duration.toMillis(request.timeout) }),\n})\n\n// ---------------------------------------------------------------------------\n// Codec - response\n// ---------------------------------------------------------------------------\n\nconst WireMetadata = Schema.Struct({\n  // Firecrawl returns `title` as a string, an array (repeated meta tags), or\n  // null; normalize to a single string in `toResponse`.\n  title: Schema.optional(Schema.NullOr(Schema.Union([Schema.String, Schema.Array(Schema.String)]))),\n  sourceURL: Schema.optional(Schema.NullOr(Schema.String)),\n  statusCode: Schema.optional(Schema.Number),\n})\n\nconst WireData = Schema.Struct({\n  markdown: Schema.optional(Schema.NullOr(Schema.String)),\n  html: Schema.optional(Schema.NullOr(Schema.String)),\n  rawHtml: Schema.optional(Schema.NullOr(Schema.String)),\n  links: Schema.optional(Schema.NullOr(Schema.Array(Schema.String))),\n  metadata: Schema.optional(WireMetadata),\n})\n\n/**\n * `data` is optional because a failed scrape is not an HTTP failure. Firecrawl\n * answers 200 with `{ success: false, error }` when a page is blocked, times\n * out or renders only under JS — so requiring `data` turned every one of those\n * into a decode error, which was then reported as a transport fault and\n * retried on the backoff schedule.\n */\nconst WireResponse = Schema.Struct({\n  success: Schema.optional(Schema.Boolean),\n  data: Schema.optional(WireData),\n  error: Schema.optional(Schema.NullOr(Schema.String)),\n})\ntype WireResponse = typeof WireResponse.Type\n\nconst pickTitle = (title: string | ReadonlyArray<string> | null | undefined): string | undefined =>\n  typeof title === \"string\" ? title : Array.isArray(title) ? title[0] : undefined\n\nconst toResponse = (\n  wire: WireResponse,\n  data: NonNullable<WireResponse[\"data\"]>,\n  request: FirecrawlReadRequest,\n): ReadResponse => {\n  const format = request.format ?? \"markdown\"\n  const content = (format === \"html\" ? data.html : data.markdown) ?? \"\"\n  const title = pickTitle(data.metadata?.title)\n  const links = data.links ?? undefined\n  return {\n    url: data.metadata?.sourceURL ?? request.url,\n    content,\n    ...(title !== undefined && { title }),\n    ...(links !== undefined && links.length > 0 && { links }),\n    raw: wire,\n  }\n}\n\n// v2 returns link objects; older responses returned bare URL strings. Accept\n// both and normalize a string to `{ url }` in `toMapResponse`.\nconst WireMapLink = Schema.Struct({\n  url: Schema.String,\n  title: Schema.optional(Schema.NullOr(Schema.String)),\n  description: Schema.optional(Schema.NullOr(Schema.String)),\n})\n\nconst WireMapResponse = Schema.Struct({\n  success: Schema.optional(Schema.Boolean),\n  links: Schema.Array(Schema.Union([WireMapLink, Schema.String])),\n})\ntype WireMapResponse = typeof WireMapResponse.Type\n\nconst toMapLink = (link: typeof WireMapLink.Type | string): MapLink =>\n  typeof link === \"string\"\n    ? { url: link }\n    : {\n        url: link.url,\n        ...(link.title != null && { title: link.title }),\n        ...(link.description != null && { description: link.description }),\n      }\n\nconst toMapResponse = (wire: WireMapResponse): MapResponse => ({\n  links: wire.links.map(toMapLink),\n  raw: wire,\n})\n\n// ---------------------------------------------------------------------------\n// HTTP\n// ---------------------------------------------------------------------------\n\nconst transportFailure = (cause: unknown): AiError.AiError =>\n  new AiError.Unavailable({ provider: \"firecrawl\", raw: cause })\n\n/**\n * The call arrived and the answer is unusable — a scrape Firecrawl itself\n * reports as failed, or a body that does not match what this provider knows how\n * to read.\n *\n * Deliberately not `Unavailable`: that tag is retryable, and neither of these\n * gets better on a second attempt. A blocked page stays blocked while the\n * backoff spends the scrape budget on it, and a changed API shape is a code\n * problem, not a weather problem.\n */\nconst unusableResponse = (message: string, raw: unknown): AiError.AiError =>\n  new AiError.GenerationFailed({ provider: \"firecrawl\", message, raw })\n\nconst httpStatusError = (status: number, body: string): AiError.AiError => {\n  const provider = \"firecrawl\"\n  const raw = body\n  if (status === 429) return new AiError.RateLimited({ provider, raw })\n  if (status === 408 || status === 504) return new AiError.Timeout({ provider, raw })\n  if (status === 401) return new AiError.AuthFailed({ provider, subtype: \"auth\", raw })\n  if (status === 403) return new AiError.AuthFailed({ provider, subtype: \"permission\", raw })\n  if (status === 402) return new AiError.AuthFailed({ provider, subtype: \"billing\", raw })\n  if (status >= 500) return new AiError.Unavailable({ provider, status, raw })\n  return new AiError.InvalidRequest({ provider, raw })\n}\n\nconst baseUrl = (cfg: Config): string => cfg.baseUrl ?? \"https://api.firecrawl.dev\"\n\nconst postScrape = (\n  cfg: Config,\n  body: WireBody,\n): Effect.Effect<WireResponse, AiError.AiError, HttpClient.HttpClient> =>\n  Effect.gen(function* () {\n    const client = yield* HttpClient.HttpClient\n    const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/scrape`).pipe(\n      HttpClientRequest.setHeader(\"Authorization\", `Bearer ${Redacted.value(cfg.apiKey)}`),\n      HttpClientRequest.bodyJsonUnsafe(body),\n    )\n    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))\n    if (response.status >= 400) {\n      const text = yield* response.text.pipe(Effect.orElseSucceed(() => \"\"))\n      return yield* httpStatusError(response.status, text)\n    }\n    const json = yield* response.json.pipe(Effect.mapError(transportFailure))\n    return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(\n      Effect.mapError((issue) =>\n        unusableResponse(\"firecrawl returned a body this provider cannot read\", issue),\n      ),\n    )\n  })\n\nconst readImpl =\n  (cfg: Config) =>\n  (\n    request: FirecrawlReadRequest,\n  ): Effect.Effect<ReadResponse, AiError.AiError, HttpClient.HttpClient> =>\n    Effect.flatMap(postScrape(cfg, buildBody(request)), (wire) =>\n      // A 200 carrying `success: false` is Firecrawl reporting the page could\n      // not be scraped — blocked, JS-only, or timed out. Its own message is the\n      // useful part, so it becomes the error rather than a decode failure.\n      wire.data === undefined || wire.success === false\n        ? Effect.fail(\n            unusableResponse(wire.error ?? `firecrawl could not scrape ${request.url}`, wire),\n          )\n        : Effect.succeed(toResponse(wire, wire.data, request)),\n    )\n\nconst postMap = (\n  cfg: Config,\n  body: WireMapBody,\n): Effect.Effect<WireMapResponse, AiError.AiError, HttpClient.HttpClient> =>\n  Effect.gen(function* () {\n    const client = yield* HttpClient.HttpClient\n    const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/map`).pipe(\n      HttpClientRequest.setHeader(\"Authorization\", `Bearer ${Redacted.value(cfg.apiKey)}`),\n      HttpClientRequest.bodyJsonUnsafe(body),\n    )\n    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))\n    if (response.status >= 400) {\n      const text = yield* response.text.pipe(Effect.orElseSucceed(() => \"\"))\n      return yield* httpStatusError(response.status, text)\n    }\n    const json = yield* response.json.pipe(Effect.mapError(transportFailure))\n    return yield* Schema.decodeUnknownEffect(WireMapResponse)(json).pipe(\n      Effect.mapError(transportFailure),\n    )\n  })\n\nconst mapImpl =\n  (cfg: Config) =>\n  (\n    request: FirecrawlMapRequest,\n  ): Effect.Effect<MapResponse, AiError.AiError, HttpClient.HttpClient> =>\n    Effect.map(postMap(cfg, buildMapBody(request)), toMapResponse)\n\n// ---------------------------------------------------------------------------\n// Constructors\n// ---------------------------------------------------------------------------\n\n/**\n * Build a `FirecrawlReadService` value. For Layer-based setup, prefer\n * {@link layer}.\n */\nexport const make = (\n  cfg: Config,\n): Effect.Effect<FirecrawlReadService, never, HttpClient.HttpClient> =>\n  Effect.map(HttpClient.HttpClient, (client) => ({\n    read: (request) =>\n      readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),\n    map: (request) =>\n      mapImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),\n  }))\n\n/**\n * Layer registering both the provider-typed `FirecrawlRead` tag and the\n * generic `WebRead` tag over one implementation. A `CommonReadRequest` is\n * structurally a `FirecrawlReadRequest` with the extra knobs unset, so the\n * generic registration forwards directly.\n */\nexport const layer = (\n  cfg: Config,\n): Layer.Layer<FirecrawlRead | WebRead, never, HttpClient.HttpClient> => {\n  const typed = Layer.effect(FirecrawlRead, make(cfg))\n  const generic = Layer.effect(\n    WebRead,\n    Effect.map(make(cfg), (s): WebReadService => ({ read: (request) => s.read(request) })),\n  )\n  return Layer.merge(typed, generic)\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiFA,IAAa,gBAAb,cAAmC,QAAQ,SAA8C,CACvF,yDACD,CAAC;AA0BF,MAAM,aAAa,YAA4C;CAC7D,MAAM,gBAAgB,QAAQ,UAAU;AACxC,QAAO;EACL,KAAK,QAAQ;EACb,SAAS,CAAC,eAAe,QAAQ;EACjC,GAAI,QAAQ,oBAAoB,KAAA,KAAa,EAAE,iBAAiB,QAAQ,iBAAiB;EACzF,GAAI,QAAQ,gBAAgB,KAAA,KAAa,EAAE,aAAa,QAAQ,aAAa;EAC7E,GAAI,QAAQ,gBAAgB,KAAA,KAAa,EAAE,aAAa,QAAQ,aAAa;EAC7E,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,SAAS,SAAS,QAAQ,QAAQ,EAAE;EACpF,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,SAAS,SAAS,QAAQ,QAAQ,EAAE;EACpF,GAAI,QAAQ,WAAW,KAAA,KAAa,EAAE,QAAQ,QAAQ,QAAQ;EAC9D,GAAI,QAAQ,UAAU,KAAA,KAAa,EAAE,OAAO,QAAQ,OAAO;EAC5D;;AAYH,MAAM,gBAAgB,aAA+C;CACnE,KAAK,QAAQ;CACb,GAAI,QAAQ,WAAW,KAAA,KAAa,EAAE,QAAQ,QAAQ,QAAQ;CAC9D,GAAI,QAAQ,UAAU,KAAA,KAAa,EAAE,OAAO,QAAQ,OAAO;CAC3D,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,QAAQ,SAAS;CACjE,GAAI,QAAQ,sBAAsB,KAAA,KAAa,EAAE,mBAAmB,QAAQ,mBAAmB;CAC/F,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,SAAS,SAAS,QAAQ,QAAQ,EAAE;CACrF;AAMD,MAAM,eAAe,OAAO,OAAO;CAGjC,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC,OAAO,QAAQ,OAAO,MAAM,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;CACjG,WAAW,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACxD,YAAY,OAAO,SAAS,OAAO,OAAO;CAC3C,CAAC;AAEF,MAAM,WAAW,OAAO,OAAO;CAC7B,UAAU,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACvD,MAAM,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACnD,SAAS,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACtD,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,OAAO,OAAO,CAAC,CAAC;CAClE,UAAU,OAAO,SAAS,aAAa;CACxC,CAAC;;;;;;;;AASF,MAAM,eAAe,OAAO,OAAO;CACjC,SAAS,OAAO,SAAS,OAAO,QAAQ;CACxC,MAAM,OAAO,SAAS,SAAS;CAC/B,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACrD,CAAC;AAGF,MAAM,aAAa,UACjB,OAAO,UAAU,WAAW,QAAQ,MAAM,QAAQ,MAAM,GAAG,MAAM,KAAK,KAAA;AAExE,MAAM,cACJ,MACA,MACA,YACiB;CAEjB,MAAM,YADS,QAAQ,UAAU,gBACL,SAAS,KAAK,OAAO,KAAK,aAAa;CACnE,MAAM,QAAQ,UAAU,KAAK,UAAU,MAAM;CAC7C,MAAM,QAAQ,KAAK,SAAS,KAAA;AAC5B,QAAO;EACL,KAAK,KAAK,UAAU,aAAa,QAAQ;EACzC;EACA,GAAI,UAAU,KAAA,KAAa,EAAE,OAAO;EACpC,GAAI,UAAU,KAAA,KAAa,MAAM,SAAS,KAAK,EAAE,OAAO;EACxD,KAAK;EACN;;AAKH,MAAM,cAAc,OAAO,OAAO;CAChC,KAAK,OAAO;CACZ,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACpD,aAAa,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CAC3D,CAAC;AAEF,MAAM,kBAAkB,OAAO,OAAO;CACpC,SAAS,OAAO,SAAS,OAAO,QAAQ;CACxC,OAAO,OAAO,MAAM,OAAO,MAAM,CAAC,aAAa,OAAO,OAAO,CAAC,CAAC;CAChE,CAAC;AAGF,MAAM,aAAa,SACjB,OAAO,SAAS,WACZ,EAAE,KAAK,MAAM,GACb;CACE,KAAK,KAAK;CACV,GAAI,KAAK,SAAS,QAAQ,EAAE,OAAO,KAAK,OAAO;CAC/C,GAAI,KAAK,eAAe,QAAQ,EAAE,aAAa,KAAK,aAAa;CAClE;AAEP,MAAM,iBAAiB,UAAwC;CAC7D,OAAO,KAAK,MAAM,IAAI,UAAU;CAChC,KAAK;CACN;AAMD,MAAM,oBAAoB,UACxB,IAAI,QAAQ,YAAY;CAAE,UAAU;CAAa,KAAK;CAAO,CAAC;;;;;;;;;;;AAYhE,MAAM,oBAAoB,SAAiB,QACzC,IAAI,QAAQ,iBAAiB;CAAE,UAAU;CAAa;CAAS;CAAK,CAAC;AAEvE,MAAM,mBAAmB,QAAgB,SAAkC;CACzE,MAAM,WAAW;CACjB,MAAM,MAAM;AACZ,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,YAAY;EAAE;EAAU;EAAK,CAAC;AACrE,KAAI,WAAW,OAAO,WAAW,IAAK,QAAO,IAAI,QAAQ,QAAQ;EAAE;EAAU;EAAK,CAAC;AACnF,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAQ;EAAK,CAAC;AACrF,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAc;EAAK,CAAC;AAC3F,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAW;EAAK,CAAC;AACxF,KAAI,UAAU,IAAK,QAAO,IAAI,QAAQ,YAAY;EAAE;EAAU;EAAQ;EAAK,CAAC;AAC5E,QAAO,IAAI,QAAQ,eAAe;EAAE;EAAU;EAAK,CAAC;;AAGtD,MAAM,WAAW,QAAwB,IAAI,WAAW;AAExD,MAAM,cACJ,KACA,SAEA,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,OAAO,WAAW;CACjC,MAAM,cAAc,kBAAkB,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,CAAC,KACtE,kBAAkB,UAAU,iBAAiB,UAAU,SAAS,MAAM,IAAI,OAAO,GAAG,EACpF,kBAAkB,eAAe,KAAK,CACvC;CACD,MAAM,WAAW,OAAO,OAAO,QAAQ,YAAY,CAAC,KAAK,OAAO,SAAS,iBAAiB,CAAC;AAC3F,KAAI,SAAS,UAAU,KAAK;EAC1B,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,oBAAoB,GAAG,CAAC;AACtE,SAAO,OAAO,gBAAgB,SAAS,QAAQ,KAAK;;CAEtD,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,iBAAiB,CAAC;AACzE,QAAO,OAAO,OAAO,oBAAoB,aAAa,CAAC,KAAK,CAAC,KAC3D,OAAO,UAAU,UACf,iBAAiB,uDAAuD,MAAM,CAC/E,CACF;EACD;AAEJ,MAAM,YACH,SAEC,YAEA,OAAO,QAAQ,WAAW,KAAK,UAAU,QAAQ,CAAC,GAAG,SAInD,KAAK,SAAS,KAAA,KAAa,KAAK,YAAY,QACxC,OAAO,KACL,iBAAiB,KAAK,SAAS,8BAA8B,QAAQ,OAAO,KAAK,CAClF,GACD,OAAO,QAAQ,WAAW,MAAM,KAAK,MAAM,QAAQ,CAAC,CACzD;AAEL,MAAM,WACJ,KACA,SAEA,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,OAAO,WAAW;CACjC,MAAM,cAAc,kBAAkB,KAAK,GAAG,QAAQ,IAAI,CAAC,SAAS,CAAC,KACnE,kBAAkB,UAAU,iBAAiB,UAAU,SAAS,MAAM,IAAI,OAAO,GAAG,EACpF,kBAAkB,eAAe,KAAK,CACvC;CACD,MAAM,WAAW,OAAO,OAAO,QAAQ,YAAY,CAAC,KAAK,OAAO,SAAS,iBAAiB,CAAC;AAC3F,KAAI,SAAS,UAAU,KAAK;EAC1B,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,oBAAoB,GAAG,CAAC;AACtE,SAAO,OAAO,gBAAgB,SAAS,QAAQ,KAAK;;CAEtD,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,iBAAiB,CAAC;AACzE,QAAO,OAAO,OAAO,oBAAoB,gBAAgB,CAAC,KAAK,CAAC,KAC9D,OAAO,SAAS,iBAAiB,CAClC;EACD;AAEJ,MAAM,WACH,SAEC,YAEA,OAAO,IAAI,QAAQ,KAAK,aAAa,QAAQ,CAAC,EAAE,cAAc;;;;;AAUlE,MAAa,QACX,QAEA,OAAO,IAAI,WAAW,aAAa,YAAY;CAC7C,OAAO,YACL,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,OAAO,eAAe,WAAW,YAAY,OAAO,CAAC;CACnF,MAAM,YACJ,QAAQ,IAAI,CAAC,QAAQ,CAAC,KAAK,OAAO,eAAe,WAAW,YAAY,OAAO,CAAC;CACnF,EAAE;;;;;;;AAQL,MAAa,SACX,QACuE;CACvE,MAAM,QAAQ,MAAM,OAAO,eAAe,KAAK,IAAI,CAAC;CACpD,MAAM,UAAU,MAAM,OACpB,SACA,OAAO,IAAI,KAAK,IAAI,GAAG,OAAuB,EAAE,OAAO,YAAY,EAAE,KAAK,QAAQ,EAAE,EAAE,CACvF;AACD,QAAO,MAAM,MAAM,OAAO,QAAQ"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effect-uai/firecrawl",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "Firecrawl provider for @effect-uai/core (web read).",
   "keywords": [
     "ai",
@@ -43,13 +43,13 @@
     "access": "public"
   },
   "devDependencies": {
-    "effect": "4.0.0-beta.57",
+    "effect": "4.0.0-beta.94",
     "typescript": "^6.0.3",
-    "@effect-uai/core": "0.10.0"
+    "@effect-uai/core": "0.12.0"
   },
   "peerDependencies": {
     "@effect-uai/core": ">=0.2.0 <1",
-    "effect": "4.0.0-beta.57"
+    "effect": ">=4.0.0-beta.94 <5.0.0"
   },
   "scripts": {
     "build": "tsdown",

package/src/FirecrawlRead.ts

@@ -33,8 +33,45 @@ export type FirecrawlReadRequest = CommonReadRequest & {
   readonly proxy?: FirecrawlProxy
 }
 
+/**
+ * Firecrawl-typed map (sitemap / URL discovery) request. Given a base `url`,
+ * Firecrawl returns the site's URLs without scraping each one - handy for
+ * locating specific pages (pricing, docs) before a targeted read. All fields
+ * but `url` are optional.
+ */
+export type FirecrawlMapRequest = {
+  readonly url: string
+  /** Order results by relevance to this query. */
+  readonly search?: string
+  /** Cap the number of links returned. */
+  readonly limit?: number
+  /** How to use the site's sitemap: `only` (sitemap alone), `include`, or `skip`. */
+  readonly sitemap?: "only" | "include" | "skip"
+  /** Include links on subdomains of the base URL. */
+  readonly includeSubdomains?: boolean
+  /** Abort the map if it runs past this duration. */
+  readonly timeout?: Duration.Duration
+}
+
+/** One discovered URL, with page metadata where Firecrawl has it. */
+export type MapLink = {
+  readonly url: string
+  readonly title?: string
+  readonly description?: string
+}
+
+export type MapResponse = {
+  readonly links: ReadonlyArray<MapLink>
+  readonly raw: unknown
+}
+
 export type FirecrawlReadService = {
   readonly read: (request: FirecrawlReadRequest) => Effect.Effect<ReadResponse, AiError.AiError>
+  /**
+   * Discover a site's URLs (sitemap-style) without scraping. Firecrawl-specific;
+   * not on the generic `WebRead` tag, whose contract is a single-URL read.
+   */
+  readonly map: (request: FirecrawlMapRequest) => Effect.Effect<MapResponse, AiError.AiError>
 }
 
 /**
@@ -85,6 +122,24 @@ const buildBody = (request: FirecrawlReadRequest): WireBody => {
   }
 }
 
+type WireMapBody = {
+  readonly url: string
+  readonly search?: string
+  readonly limit?: number
+  readonly sitemap?: string
+  readonly includeSubdomains?: boolean
+  readonly timeout?: number
+}
+
+const buildMapBody = (request: FirecrawlMapRequest): WireMapBody => ({
+  url: request.url,
+  ...(request.search !== undefined && { search: request.search }),
+  ...(request.limit !== undefined && { limit: request.limit }),
+  ...(request.sitemap !== undefined && { sitemap: request.sitemap }),
+  ...(request.includeSubdomains !== undefined && { includeSubdomains: request.includeSubdomains }),
+  ...(request.timeout !== undefined && { timeout: Duration.toMillis(request.timeout) }),
+})
+
 // ---------------------------------------------------------------------------
 // Codec - response
 // ---------------------------------------------------------------------------
@@ -105,17 +160,28 @@ const WireData = Schema.Struct({
   metadata: Schema.optional(WireMetadata),
 })
 
+/**
+ * `data` is optional because a failed scrape is not an HTTP failure. Firecrawl
+ * answers 200 with `{ success: false, error }` when a page is blocked, times
+ * out or renders only under JS — so requiring `data` turned every one of those
+ * into a decode error, which was then reported as a transport fault and
+ * retried on the backoff schedule.
+ */
 const WireResponse = Schema.Struct({
   success: Schema.optional(Schema.Boolean),
-  data: WireData,
+  data: Schema.optional(WireData),
+  error: Schema.optional(Schema.NullOr(Schema.String)),
 })
 type WireResponse = typeof WireResponse.Type
 
 const pickTitle = (title: string | ReadonlyArray<string> | null | undefined): string | undefined =>
   typeof title === "string" ? title : Array.isArray(title) ? title[0] : undefined
 
-const toResponse = (wire: WireResponse, request: FirecrawlReadRequest): ReadResponse => {
-  const { data } = wire
+const toResponse = (
+  wire: WireResponse,
+  data: NonNullable<WireResponse["data"]>,
+  request: FirecrawlReadRequest,
+): ReadResponse => {
   const format = request.format ?? "markdown"
   const content = (format === "html" ? data.html : data.markdown) ?? ""
   const title = pickTitle(data.metadata?.title)
@@ -129,6 +195,34 @@ const toResponse = (wire: WireResponse, request: FirecrawlReadRequest): ReadResp
   }
 }
 
+// v2 returns link objects; older responses returned bare URL strings. Accept
+// both and normalize a string to `{ url }` in `toMapResponse`.
+const WireMapLink = Schema.Struct({
+  url: Schema.String,
+  title: Schema.optional(Schema.NullOr(Schema.String)),
+  description: Schema.optional(Schema.NullOr(Schema.String)),
+})
+
+const WireMapResponse = Schema.Struct({
+  success: Schema.optional(Schema.Boolean),
+  links: Schema.Array(Schema.Union([WireMapLink, Schema.String])),
+})
+type WireMapResponse = typeof WireMapResponse.Type
+
+const toMapLink = (link: typeof WireMapLink.Type | string): MapLink =>
+  typeof link === "string"
+    ? { url: link }
+    : {
+        url: link.url,
+        ...(link.title != null && { title: link.title }),
+        ...(link.description != null && { description: link.description }),
+      }
+
+const toMapResponse = (wire: WireMapResponse): MapResponse => ({
+  links: wire.links.map(toMapLink),
+  raw: wire,
+})
+
 // ---------------------------------------------------------------------------
 // HTTP
 // ---------------------------------------------------------------------------
@@ -136,6 +230,19 @@ const toResponse = (wire: WireResponse, request: FirecrawlReadRequest): ReadResp
 const transportFailure = (cause: unknown): AiError.AiError =>
   new AiError.Unavailable({ provider: "firecrawl", raw: cause })
 
+/**
+ * The call arrived and the answer is unusable — a scrape Firecrawl itself
+ * reports as failed, or a body that does not match what this provider knows how
+ * to read.
+ *
+ * Deliberately not `Unavailable`: that tag is retryable, and neither of these
+ * gets better on a second attempt. A blocked page stays blocked while the
+ * backoff spends the scrape budget on it, and a changed API shape is a code
+ * problem, not a weather problem.
+ */
+const unusableResponse = (message: string, raw: unknown): AiError.AiError =>
+  new AiError.GenerationFailed({ provider: "firecrawl", message, raw })
+
 const httpStatusError = (status: number, body: string): AiError.AiError => {
   const provider = "firecrawl"
   const raw = body
@@ -167,7 +274,9 @@ const postScrape = (
     }
     const json = yield* response.json.pipe(Effect.mapError(transportFailure))
     return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(
-      Effect.mapError(transportFailure),
+      Effect.mapError((issue) =>
+        unusableResponse("firecrawl returned a body this provider cannot read", issue),
+      ),
     )
   })
 
@@ -176,7 +285,44 @@ const readImpl =
   (
     request: FirecrawlReadRequest,
   ): Effect.Effect<ReadResponse, AiError.AiError, HttpClient.HttpClient> =>
-    Effect.map(postScrape(cfg, buildBody(request)), (wire) => toResponse(wire, request))
+    Effect.flatMap(postScrape(cfg, buildBody(request)), (wire) =>
+      // A 200 carrying `success: false` is Firecrawl reporting the page could
+      // not be scraped — blocked, JS-only, or timed out. Its own message is the
+      // useful part, so it becomes the error rather than a decode failure.
+      wire.data === undefined || wire.success === false
+        ? Effect.fail(
+            unusableResponse(wire.error ?? `firecrawl could not scrape ${request.url}`, wire),
+          )
+        : Effect.succeed(toResponse(wire, wire.data, request)),
+    )
+
+const postMap = (
+  cfg: Config,
+  body: WireMapBody,
+): Effect.Effect<WireMapResponse, AiError.AiError, HttpClient.HttpClient> =>
+  Effect.gen(function* () {
+    const client = yield* HttpClient.HttpClient
+    const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/map`).pipe(
+      HttpClientRequest.setHeader("Authorization", `Bearer ${Redacted.value(cfg.apiKey)}`),
+      HttpClientRequest.bodyJsonUnsafe(body),
+    )
+    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
+    if (response.status >= 400) {
+      const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
+      return yield* httpStatusError(response.status, text)
+    }
+    const json = yield* response.json.pipe(Effect.mapError(transportFailure))
+    return yield* Schema.decodeUnknownEffect(WireMapResponse)(json).pipe(
+      Effect.mapError(transportFailure),
+    )
+  })
+
+const mapImpl =
+  (cfg: Config) =>
+  (
+    request: FirecrawlMapRequest,
+  ): Effect.Effect<MapResponse, AiError.AiError, HttpClient.HttpClient> =>
+    Effect.map(postMap(cfg, buildMapBody(request)), toMapResponse)
 
 // ---------------------------------------------------------------------------
 // Constructors
@@ -189,9 +335,11 @@ const readImpl =
 export const make = (
   cfg: Config,
 ): Effect.Effect<FirecrawlReadService, never, HttpClient.HttpClient> =>
-  Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
+  Effect.map(HttpClient.HttpClient, (client) => ({
     read: (request) =>
       readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
+    map: (request) =>
+      mapImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
   }))
 
 /**