@effect-uai/firecrawl 0.9.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Betalyra Sociedade Unipessoal Lda.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/dist/FirecrawlRead.d.mts

@@ -0,0 +1,52 @@
+import { FirecrawlProxy } from "./models.mjs";
+import { Context, Duration, Effect, Layer, Redacted } from "effect";
+import { HttpClient } from "effect/unstable/http";
+import * as AiError from "@effect-uai/core/AiError";
+import { CommonReadRequest, ReadResponse, WebRead } from "@effect-uai/core/WebRead";
+
+//#region src/FirecrawlRead.d.ts
+declare namespace FirecrawlRead_d_exports {
+  export { Config, FirecrawlRead, FirecrawlReadRequest, FirecrawlReadService, layer, make };
+}
+/**
+ * Firecrawl-typed read request. Widens {@link CommonReadRequest} with
+ * Firecrawl's own scrape knobs; the common fields map onto Firecrawl's wire
+ * body (see the codec below).
+ */
+type FirecrawlReadRequest = CommonReadRequest & {
+  /** Keep only the main content, dropping nav/header/footer/sidebar. */readonly onlyMainContent?: boolean; /** Restrict extraction to these HTML tags / selectors. */
+  readonly includeTags?: ReadonlyArray<string>; /** Drop these HTML tags / selectors before extraction. */
+  readonly excludeTags?: ReadonlyArray<string>; /** Wait this long for the page to settle before scraping (JS-heavy sites). */
+  readonly waitFor?: Duration.Duration; /** Emulate a mobile device. */
+  readonly mobile?: boolean; /** Proxy tier - see {@link FirecrawlProxy}. */
+  readonly proxy?: FirecrawlProxy;
+};
+type FirecrawlReadService = {
+  readonly read: (request: FirecrawlReadRequest) => Effect.Effect<ReadResponse, AiError.AiError>;
+};
+declare const FirecrawlRead_base: Context.ServiceClass<FirecrawlRead, "@betalyra/effect-uai/providers/firecrawl/FirecrawlRead", FirecrawlReadService>;
+/**
+ * Provider-typed service tag. Yield this for Firecrawl-typed reads (its own
+ * scrape knobs); yield the generic `WebRead` tag for provider-portable code.
+ * Both are registered by {@link layer}.
+ */
+declare class FirecrawlRead extends FirecrawlRead_base {}
+type Config = {
+  readonly apiKey: Redacted.Redacted;
+  readonly baseUrl?: string;
+};
+/**
+ * Build a `FirecrawlReadService` value. For Layer-based setup, prefer
+ * {@link layer}.
+ */
+declare const make: (cfg: Config) => Effect.Effect<FirecrawlReadService, never, HttpClient.HttpClient>;
+/**
+ * Layer registering both the provider-typed `FirecrawlRead` tag and the
+ * generic `WebRead` tag over one implementation. A `CommonReadRequest` is
+ * structurally a `FirecrawlReadRequest` with the extra knobs unset, so the
+ * generic registration forwards directly.
+ */
+declare const layer: (cfg: Config) => Layer.Layer<FirecrawlRead | WebRead, never, HttpClient.HttpClient>;
+//#endregion
+export { Config, FirecrawlRead, FirecrawlReadRequest, FirecrawlReadService, layer, make, FirecrawlRead_d_exports as t };
+//# sourceMappingURL=FirecrawlRead.d.mts.map

package/dist/FirecrawlRead.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FirecrawlRead.d.mts","names":[],"sources":["../src/FirecrawlRead.ts"],"mappings":";;;;;;;;;;;;;;;KAoBY,oBAAA,GAAuB,iBAAA;iFAExB,eAAA;WAEA,WAAA,GAAc,aAAA;WAEd,WAAA,GAAc,aAAA,UANO;EAAA,SAQrB,OAAA,GAAU,QAAA,CAAS,QAAA,EARK;EAAA,SAUxB,MAAA,YAJc;EAAA,SAMd,KAAA,GAAQ,cAAA;AAAA;AAAA,KAGP,oBAAA;EAAA,SACD,IAAA,GAAO,OAAA,EAAS,oBAAA,KAAyB,MAAA,CAAO,MAAA,CAAO,YAAA,EAAc,OAAA,CAAQ,OAAA;AAAA;AAAA,cACvF,kBAAA;;;;;;cAOY,aAAA,SAAsB,kBAAA;AAAA,KAIvB,MAAA;EAAA,SACD,MAAA,EAAQ,QAAA,CAAS,QAAA;EAAA,SACjB,OAAA;AAAA;;AAfX;;;cAyJa,IAAA,GACX,GAAA,EAAK,MAAA,KACJ,MAAA,CAAO,MAAA,CAAO,oBAAA,SAA6B,UAAA,CAAW,UAAA;;;;;;;cAY5C,KAAA,GACX,GAAA,EAAK,MAAA,KACJ,KAAA,CAAM,KAAA,CAAM,aAAA,GAAgB,OAAA,SAAgB,UAAA,CAAW,UAAA"}

package/dist/FirecrawlRead.mjs

@@ -0,0 +1,134 @@
+import { t as __exportAll } from "./chunk-CfYAbeIz.mjs";
+import { Context, Duration, Effect, Layer, Redacted, Schema } from "effect";
+import { HttpClient, HttpClientRequest } from "effect/unstable/http";
+import * as AiError from "@effect-uai/core/AiError";
+import { WebRead } from "@effect-uai/core/WebRead";
+//#region src/FirecrawlRead.ts
+var FirecrawlRead_exports = /* @__PURE__ */ __exportAll({
+	FirecrawlRead: () => FirecrawlRead,
+	layer: () => layer,
+	make: () => make
+});
+/**
+* Provider-typed service tag. Yield this for Firecrawl-typed reads (its own
+* scrape knobs); yield the generic `WebRead` tag for provider-portable code.
+* Both are registered by {@link layer}.
+*/
+var FirecrawlRead = class extends Context.Service()("@betalyra/effect-uai/providers/firecrawl/FirecrawlRead") {};
+const buildBody = (request) => {
+	const contentFormat = request.format ?? "markdown";
+	return {
+		url: request.url,
+		formats: [contentFormat, "links"],
+		...request.onlyMainContent !== void 0 && { onlyMainContent: request.onlyMainContent },
+		...request.includeTags !== void 0 && { includeTags: request.includeTags },
+		...request.excludeTags !== void 0 && { excludeTags: request.excludeTags },
+		...request.waitFor !== void 0 && { waitFor: Duration.toMillis(request.waitFor) },
+		...request.timeout !== void 0 && { timeout: Duration.toMillis(request.timeout) },
+		...request.mobile !== void 0 && { mobile: request.mobile },
+		...request.proxy !== void 0 && { proxy: request.proxy }
+	};
+};
+const WireMetadata = Schema.Struct({
+	title: Schema.optional(Schema.NullOr(Schema.Union([Schema.String, Schema.Array(Schema.String)]))),
+	sourceURL: Schema.optional(Schema.NullOr(Schema.String)),
+	statusCode: Schema.optional(Schema.Number)
+});
+const WireData = Schema.Struct({
+	markdown: Schema.optional(Schema.NullOr(Schema.String)),
+	html: Schema.optional(Schema.NullOr(Schema.String)),
+	rawHtml: Schema.optional(Schema.NullOr(Schema.String)),
+	links: Schema.optional(Schema.NullOr(Schema.Array(Schema.String))),
+	metadata: Schema.optional(WireMetadata)
+});
+const WireResponse = Schema.Struct({
+	success: Schema.optional(Schema.Boolean),
+	data: WireData
+});
+const pickTitle = (title) => typeof title === "string" ? title : Array.isArray(title) ? title[0] : void 0;
+const toResponse = (wire, request) => {
+	const { data } = wire;
+	const content = ((request.format ?? "markdown") === "html" ? data.html : data.markdown) ?? "";
+	const title = pickTitle(data.metadata?.title);
+	const links = data.links ?? void 0;
+	return {
+		url: data.metadata?.sourceURL ?? request.url,
+		content,
+		...title !== void 0 && { title },
+		...links !== void 0 && links.length > 0 && { links },
+		raw: wire
+	};
+};
+const transportFailure = (cause) => new AiError.Unavailable({
+	provider: "firecrawl",
+	raw: cause
+});
+const httpStatusError = (status, body) => {
+	const provider = "firecrawl";
+	const raw = body;
+	if (status === 429) return new AiError.RateLimited({
+		provider,
+		raw
+	});
+	if (status === 408 || status === 504) return new AiError.Timeout({
+		provider,
+		raw
+	});
+	if (status === 401) return new AiError.AuthFailed({
+		provider,
+		subtype: "auth",
+		raw
+	});
+	if (status === 403) return new AiError.AuthFailed({
+		provider,
+		subtype: "permission",
+		raw
+	});
+	if (status === 402) return new AiError.AuthFailed({
+		provider,
+		subtype: "billing",
+		raw
+	});
+	if (status >= 500) return new AiError.Unavailable({
+		provider,
+		status,
+		raw
+	});
+	return new AiError.InvalidRequest({
+		provider,
+		raw
+	});
+};
+const baseUrl = (cfg) => cfg.baseUrl ?? "https://api.firecrawl.dev";
+const postScrape = (cfg, body) => Effect.gen(function* () {
+	const client = yield* HttpClient.HttpClient;
+	const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/scrape`).pipe(HttpClientRequest.setHeader("Authorization", `Bearer ${Redacted.value(cfg.apiKey)}`), HttpClientRequest.bodyJsonUnsafe(body));
+	const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure));
+	if (response.status >= 400) {
+		const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""));
+		return yield* httpStatusError(response.status, text);
+	}
+	const json = yield* response.json.pipe(Effect.mapError(transportFailure));
+	return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(Effect.mapError(transportFailure));
+});
+const readImpl = (cfg) => (request) => Effect.map(postScrape(cfg, buildBody(request)), (wire) => toResponse(wire, request));
+/**
+* Build a `FirecrawlReadService` value. For Layer-based setup, prefer
+* {@link layer}.
+*/
+const make = (cfg) => Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({ read: (request) => readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)) }));
+/**
+* Layer registering both the provider-typed `FirecrawlRead` tag and the
+* generic `WebRead` tag over one implementation. A `CommonReadRequest` is
+* structurally a `FirecrawlReadRequest` with the extra knobs unset, so the
+* generic registration forwards directly.
+*/
+const layer = (cfg) => {
+	const typed = Layer.effect(FirecrawlRead, make(cfg));
+	const generic = Layer.effect(WebRead, Effect.map(make(cfg), (s) => ({ read: (request) => s.read(request) })));
+	return Layer.merge(typed, generic);
+};
+//#endregion
+export { FirecrawlRead, layer, make, FirecrawlRead_exports as t };
+
+//# sourceMappingURL=FirecrawlRead.mjs.map

package/dist/FirecrawlRead.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"FirecrawlRead.mjs","names":[],"sources":["../src/FirecrawlRead.ts"],"sourcesContent":["import { Context, Duration, Effect, Layer, Redacted, Schema } from \"effect\"\nimport { HttpClient, HttpClientRequest } from \"effect/unstable/http\"\nimport * as AiError from \"@effect-uai/core/AiError\"\nimport {\n  type CommonReadRequest,\n  type ReadResponse,\n  WebRead,\n  type WebReadService,\n} from \"@effect-uai/core/WebRead\"\nimport type { FirecrawlProxy } from \"./models.js\"\n\n// ---------------------------------------------------------------------------\n// Public types\n// ---------------------------------------------------------------------------\n\n/**\n * Firecrawl-typed read request. Widens {@link CommonReadRequest} with\n * Firecrawl's own scrape knobs; the common fields map onto Firecrawl's wire\n * body (see the codec below).\n */\nexport type FirecrawlReadRequest = CommonReadRequest & {\n  /** Keep only the main content, dropping nav/header/footer/sidebar. */\n  readonly onlyMainContent?: boolean\n  /** Restrict extraction to these HTML tags / selectors. */\n  readonly includeTags?: ReadonlyArray<string>\n  /** Drop these HTML tags / selectors before extraction. */\n  readonly excludeTags?: ReadonlyArray<string>\n  /** Wait this long for the page to settle before scraping (JS-heavy sites). */\n  readonly waitFor?: Duration.Duration\n  /** Emulate a mobile device. */\n  readonly mobile?: boolean\n  /** Proxy tier - see {@link FirecrawlProxy}. */\n  readonly proxy?: FirecrawlProxy\n}\n\nexport type FirecrawlReadService = {\n  readonly read: (request: FirecrawlReadRequest) => Effect.Effect<ReadResponse, AiError.AiError>\n}\n\n/**\n * Provider-typed service tag. Yield this for Firecrawl-typed reads (its own\n * scrape knobs); yield the generic `WebRead` tag for provider-portable code.\n * Both are registered by {@link layer}.\n */\nexport class FirecrawlRead extends Context.Service<FirecrawlRead, FirecrawlReadService>()(\n  \"@betalyra/effect-uai/providers/firecrawl/FirecrawlRead\",\n) {}\n\nexport type Config = {\n  readonly apiKey: Redacted.Redacted\n  readonly baseUrl?: string\n}\n\n// ---------------------------------------------------------------------------\n// Codec - request body\n// ---------------------------------------------------------------------------\n\ntype WireBody = {\n  readonly url: string\n  readonly formats: ReadonlyArray<string>\n  readonly onlyMainContent?: boolean\n  readonly includeTags?: ReadonlyArray<string>\n  readonly excludeTags?: ReadonlyArray<string>\n  readonly waitFor?: number\n  readonly timeout?: number\n  readonly mobile?: boolean\n  readonly proxy?: string\n}\n\n// `markdown` / `html` map one-to-one onto Firecrawl's format names. `links` is\n// always requested so {@link ReadResponse.links} can be populated; it does not\n// change the per-page billing.\nconst buildBody = (request: FirecrawlReadRequest): WireBody => {\n  const contentFormat = request.format ?? \"markdown\"\n  return {\n    url: request.url,\n    formats: [contentFormat, \"links\"],\n    ...(request.onlyMainContent !== undefined && { onlyMainContent: request.onlyMainContent }),\n    ...(request.includeTags !== undefined && { includeTags: request.includeTags }),\n    ...(request.excludeTags !== undefined && { excludeTags: request.excludeTags }),\n    ...(request.waitFor !== undefined && { waitFor: Duration.toMillis(request.waitFor) }),\n    ...(request.timeout !== undefined && { timeout: Duration.toMillis(request.timeout) }),\n    ...(request.mobile !== undefined && { mobile: request.mobile }),\n    ...(request.proxy !== undefined && { proxy: request.proxy }),\n  }\n}\n\n// ---------------------------------------------------------------------------\n// Codec - response\n// ---------------------------------------------------------------------------\n\nconst WireMetadata = Schema.Struct({\n  // Firecrawl returns `title` as a string, an array (repeated meta tags), or\n  // null; normalize to a single string in `toResponse`.\n  title: Schema.optional(Schema.NullOr(Schema.Union([Schema.String, Schema.Array(Schema.String)]))),\n  sourceURL: Schema.optional(Schema.NullOr(Schema.String)),\n  statusCode: Schema.optional(Schema.Number),\n})\n\nconst WireData = Schema.Struct({\n  markdown: Schema.optional(Schema.NullOr(Schema.String)),\n  html: Schema.optional(Schema.NullOr(Schema.String)),\n  rawHtml: Schema.optional(Schema.NullOr(Schema.String)),\n  links: Schema.optional(Schema.NullOr(Schema.Array(Schema.String))),\n  metadata: Schema.optional(WireMetadata),\n})\n\nconst WireResponse = Schema.Struct({\n  success: Schema.optional(Schema.Boolean),\n  data: WireData,\n})\ntype WireResponse = typeof WireResponse.Type\n\nconst pickTitle = (title: string | ReadonlyArray<string> | null | undefined): string | undefined =>\n  typeof title === \"string\" ? title : Array.isArray(title) ? title[0] : undefined\n\nconst toResponse = (wire: WireResponse, request: FirecrawlReadRequest): ReadResponse => {\n  const { data } = wire\n  const format = request.format ?? \"markdown\"\n  const content = (format === \"html\" ? data.html : data.markdown) ?? \"\"\n  const title = pickTitle(data.metadata?.title)\n  const links = data.links ?? undefined\n  return {\n    url: data.metadata?.sourceURL ?? request.url,\n    content,\n    ...(title !== undefined && { title }),\n    ...(links !== undefined && links.length > 0 && { links }),\n    raw: wire,\n  }\n}\n\n// ---------------------------------------------------------------------------\n// HTTP\n// ---------------------------------------------------------------------------\n\nconst transportFailure = (cause: unknown): AiError.AiError =>\n  new AiError.Unavailable({ provider: \"firecrawl\", raw: cause })\n\nconst httpStatusError = (status: number, body: string): AiError.AiError => {\n  const provider = \"firecrawl\"\n  const raw = body\n  if (status === 429) return new AiError.RateLimited({ provider, raw })\n  if (status === 408 || status === 504) return new AiError.Timeout({ provider, raw })\n  if (status === 401) return new AiError.AuthFailed({ provider, subtype: \"auth\", raw })\n  if (status === 403) return new AiError.AuthFailed({ provider, subtype: \"permission\", raw })\n  if (status === 402) return new AiError.AuthFailed({ provider, subtype: \"billing\", raw })\n  if (status >= 500) return new AiError.Unavailable({ provider, status, raw })\n  return new AiError.InvalidRequest({ provider, raw })\n}\n\nconst baseUrl = (cfg: Config): string => cfg.baseUrl ?? \"https://api.firecrawl.dev\"\n\nconst postScrape = (\n  cfg: Config,\n  body: WireBody,\n): Effect.Effect<WireResponse, AiError.AiError, HttpClient.HttpClient> =>\n  Effect.gen(function* () {\n    const client = yield* HttpClient.HttpClient\n    const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/scrape`).pipe(\n      HttpClientRequest.setHeader(\"Authorization\", `Bearer ${Redacted.value(cfg.apiKey)}`),\n      HttpClientRequest.bodyJsonUnsafe(body),\n    )\n    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))\n    if (response.status >= 400) {\n      const text = yield* response.text.pipe(Effect.orElseSucceed(() => \"\"))\n      return yield* httpStatusError(response.status, text)\n    }\n    const json = yield* response.json.pipe(Effect.mapError(transportFailure))\n    return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(\n      Effect.mapError(transportFailure),\n    )\n  })\n\nconst readImpl =\n  (cfg: Config) =>\n  (\n    request: FirecrawlReadRequest,\n  ): Effect.Effect<ReadResponse, AiError.AiError, HttpClient.HttpClient> =>\n    Effect.map(postScrape(cfg, buildBody(request)), (wire) => toResponse(wire, request))\n\n// ---------------------------------------------------------------------------\n// Constructors\n// ---------------------------------------------------------------------------\n\n/**\n * Build a `FirecrawlReadService` value. For Layer-based setup, prefer\n * {@link layer}.\n */\nexport const make = (\n  cfg: Config,\n): Effect.Effect<FirecrawlReadService, never, HttpClient.HttpClient> =>\n  Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({\n    read: (request) =>\n      readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),\n  }))\n\n/**\n * Layer registering both the provider-typed `FirecrawlRead` tag and the\n * generic `WebRead` tag over one implementation. A `CommonReadRequest` is\n * structurally a `FirecrawlReadRequest` with the extra knobs unset, so the\n * generic registration forwards directly.\n */\nexport const layer = (\n  cfg: Config,\n): Layer.Layer<FirecrawlRead | WebRead, never, HttpClient.HttpClient> => {\n  const typed = Layer.effect(FirecrawlRead, make(cfg))\n  const generic = Layer.effect(\n    WebRead,\n    Effect.map(make(cfg), (s): WebReadService => ({ read: (request) => s.read(request) })),\n  )\n  return Layer.merge(typed, generic)\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA4CA,IAAa,gBAAb,cAAmC,QAAQ,SAA8C,CACvF,yDACD,CAAC;AA0BF,MAAM,aAAa,YAA4C;CAC7D,MAAM,gBAAgB,QAAQ,UAAU;AACxC,QAAO;EACL,KAAK,QAAQ;EACb,SAAS,CAAC,eAAe,QAAQ;EACjC,GAAI,QAAQ,oBAAoB,KAAA,KAAa,EAAE,iBAAiB,QAAQ,iBAAiB;EACzF,GAAI,QAAQ,gBAAgB,KAAA,KAAa,EAAE,aAAa,QAAQ,aAAa;EAC7E,GAAI,QAAQ,gBAAgB,KAAA,KAAa,EAAE,aAAa,QAAQ,aAAa;EAC7E,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,SAAS,SAAS,QAAQ,QAAQ,EAAE;EACpF,GAAI,QAAQ,YAAY,KAAA,KAAa,EAAE,SAAS,SAAS,SAAS,QAAQ,QAAQ,EAAE;EACpF,GAAI,QAAQ,WAAW,KAAA,KAAa,EAAE,QAAQ,QAAQ,QAAQ;EAC9D,GAAI,QAAQ,UAAU,KAAA,KAAa,EAAE,OAAO,QAAQ,OAAO;EAC5D;;AAOH,MAAM,eAAe,OAAO,OAAO;CAGjC,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC,OAAO,QAAQ,OAAO,MAAM,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;CACjG,WAAW,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACxD,YAAY,OAAO,SAAS,OAAO,OAAO;CAC3C,CAAC;AAEF,MAAM,WAAW,OAAO,OAAO;CAC7B,UAAU,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACvD,MAAM,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACnD,SAAS,OAAO,SAAS,OAAO,OAAO,OAAO,OAAO,CAAC;CACtD,OAAO,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,OAAO,OAAO,CAAC,CAAC;CAClE,UAAU,OAAO,SAAS,aAAa;CACxC,CAAC;AAEF,MAAM,eAAe,OAAO,OAAO;CACjC,SAAS,OAAO,SAAS,OAAO,QAAQ;CACxC,MAAM;CACP,CAAC;AAGF,MAAM,aAAa,UACjB,OAAO,UAAU,WAAW,QAAQ,MAAM,QAAQ,MAAM,GAAG,MAAM,KAAK,KAAA;AAExE,MAAM,cAAc,MAAoB,YAAgD;CACtF,MAAM,EAAE,SAAS;CAEjB,MAAM,YADS,QAAQ,UAAU,gBACL,SAAS,KAAK,OAAO,KAAK,aAAa;CACnE,MAAM,QAAQ,UAAU,KAAK,UAAU,MAAM;CAC7C,MAAM,QAAQ,KAAK,SAAS,KAAA;AAC5B,QAAO;EACL,KAAK,KAAK,UAAU,aAAa,QAAQ;EACzC;EACA,GAAI,UAAU,KAAA,KAAa,EAAE,OAAO;EACpC,GAAI,UAAU,KAAA,KAAa,MAAM,SAAS,KAAK,EAAE,OAAO;EACxD,KAAK;EACN;;AAOH,MAAM,oBAAoB,UACxB,IAAI,QAAQ,YAAY;CAAE,UAAU;CAAa,KAAK;CAAO,CAAC;AAEhE,MAAM,mBAAmB,QAAgB,SAAkC;CACzE,MAAM,WAAW;CACjB,MAAM,MAAM;AACZ,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,YAAY;EAAE;EAAU;EAAK,CAAC;AACrE,KAAI,WAAW,OAAO,WAAW,IAAK,QAAO,IAAI,QAAQ,QAAQ;EAAE;EAAU;EAAK,CAAC;AACnF,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAQ;EAAK,CAAC;AACrF,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAc;EAAK,CAAC;AAC3F,KAAI,WAAW,IAAK,QAAO,IAAI,QAAQ,WAAW;EAAE;EAAU,SAAS;EAAW;EAAK,CAAC;AACxF,KAAI,UAAU,IAAK,QAAO,IAAI,QAAQ,YAAY;EAAE;EAAU;EAAQ;EAAK,CAAC;AAC5E,QAAO,IAAI,QAAQ,eAAe;EAAE;EAAU;EAAK,CAAC;;AAGtD,MAAM,WAAW,QAAwB,IAAI,WAAW;AAExD,MAAM,cACJ,KACA,SAEA,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,OAAO,WAAW;CACjC,MAAM,cAAc,kBAAkB,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,CAAC,KACtE,kBAAkB,UAAU,iBAAiB,UAAU,SAAS,MAAM,IAAI,OAAO,GAAG,EACpF,kBAAkB,eAAe,KAAK,CACvC;CACD,MAAM,WAAW,OAAO,OAAO,QAAQ,YAAY,CAAC,KAAK,OAAO,SAAS,iBAAiB,CAAC;AAC3F,KAAI,SAAS,UAAU,KAAK;EAC1B,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,oBAAoB,GAAG,CAAC;AACtE,SAAO,OAAO,gBAAgB,SAAS,QAAQ,KAAK;;CAEtD,MAAM,OAAO,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,iBAAiB,CAAC;AACzE,QAAO,OAAO,OAAO,oBAAoB,aAAa,CAAC,KAAK,CAAC,KAC3D,OAAO,SAAS,iBAAiB,CAClC;EACD;AAEJ,MAAM,YACH,SAEC,YAEA,OAAO,IAAI,WAAW,KAAK,UAAU,QAAQ,CAAC,GAAG,SAAS,WAAW,MAAM,QAAQ,CAAC;;;;;AAUxF,MAAa,QACX,QAEA,OAAO,IAAI,WAAW,WAAW,UAAU,GAAG,YAAY,EACxD,OAAO,YACL,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,OAAO,eAAe,WAAW,YAAY,OAAO,CAAC,EACpF,EAAE;;;;;;;AAQL,MAAa,SACX,QACuE;CACvE,MAAM,QAAQ,MAAM,OAAO,eAAe,KAAK,IAAI,CAAC;CACpD,MAAM,UAAU,MAAM,OACpB,SACA,OAAO,IAAI,KAAK,IAAI,GAAG,OAAuB,EAAE,OAAO,YAAY,EAAE,KAAK,QAAQ,EAAE,EAAE,CACvF;AACD,QAAO,MAAM,MAAM,OAAO,QAAQ"}

package/dist/chunk-CfYAbeIz.mjs

@@ -0,0 +1,13 @@
+//#region \0rolldown/runtime.js
+var __defProp = Object.defineProperty;
+var __exportAll = (all, no_symbols) => {
+	let target = {};
+	for (var name in all) __defProp(target, name, {
+		get: all[name],
+		enumerable: true
+	});
+	if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
+	return target;
+};
+//#endregion
+export { __exportAll as t };

package/dist/index.d.mts

@@ -0,0 +1,3 @@
+import { FirecrawlProxy } from "./models.mjs";
+import { t as FirecrawlRead_d_exports } from "./FirecrawlRead.mjs";
+export { FirecrawlProxy, FirecrawlRead_d_exports as FirecrawlRead };

package/dist/index.mjs

@@ -0,0 +1,3 @@
+import { t as FirecrawlRead_exports } from "./FirecrawlRead.mjs";
+import "./models.mjs";
+export { FirecrawlRead_exports as FirecrawlRead };

package/dist/models.d.mts

@@ -0,0 +1,10 @@
+//#region src/models.d.ts
+/**
+ * Firecrawl's proxy tier for a scrape. Higher tiers survive more aggressive
+ * anti-bot protection at higher cost. Provider-specific, so it lives on
+ * `FirecrawlReadRequest`, not the cross-provider `CommonReadRequest`.
+ */
+type FirecrawlProxy = "basic" | "stealth" | "auto" | (string & {});
+//#endregion
+export { FirecrawlProxy };
+//# sourceMappingURL=models.d.mts.map

package/dist/models.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.d.mts","names":[],"sources":["../src/models.ts"],"mappings":";;AAKA;;;;KAAY,cAAA"}

package/dist/models.mjs

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@effect-uai/firecrawl",
+  "version": "0.9.0",
+  "description": "Firecrawl provider for @effect-uai/core (web read).",
+  "keywords": [
+    "ai",
+    "effect",
+    "firecrawl",
+    "scrape",
+    "web-read"
+  ],
+  "homepage": "https://github.com/betalyra/effect-uai/tree/main/packages/providers/firecrawl",
+  "bugs": {
+    "url": "https://github.com/betalyra/effect-uai/issues"
+  },
+  "license": "MIT",
+  "author": "Betalyra",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/betalyra/effect-uai",
+    "directory": "packages/providers/firecrawl"
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    },
+    "./FirecrawlRead": {
+      "types": "./dist/FirecrawlRead.d.mts",
+      "import": "./dist/FirecrawlRead.mjs"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "effect": "4.0.0-beta.57",
+    "typescript": "^6.0.3",
+    "@effect-uai/core": "0.9.0"
+  },
+  "peerDependencies": {
+    "@effect-uai/core": ">=0.2.0 <1",
+    "effect": "4.0.0-beta.57"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "typecheck": "tsc --noEmit"
+  }
+}

package/src/FirecrawlRead.ts

@@ -0,0 +1,212 @@
+import { Context, Duration, Effect, Layer, Redacted, Schema } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import * as AiError from "@effect-uai/core/AiError"
+import {
+  type CommonReadRequest,
+  type ReadResponse,
+  WebRead,
+  type WebReadService,
+} from "@effect-uai/core/WebRead"
+import type { FirecrawlProxy } from "./models.js"
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+/**
+ * Firecrawl-typed read request. Widens {@link CommonReadRequest} with
+ * Firecrawl's own scrape knobs; the common fields map onto Firecrawl's wire
+ * body (see the codec below).
+ */
+export type FirecrawlReadRequest = CommonReadRequest & {
+  /** Keep only the main content, dropping nav/header/footer/sidebar. */
+  readonly onlyMainContent?: boolean
+  /** Restrict extraction to these HTML tags / selectors. */
+  readonly includeTags?: ReadonlyArray<string>
+  /** Drop these HTML tags / selectors before extraction. */
+  readonly excludeTags?: ReadonlyArray<string>
+  /** Wait this long for the page to settle before scraping (JS-heavy sites). */
+  readonly waitFor?: Duration.Duration
+  /** Emulate a mobile device. */
+  readonly mobile?: boolean
+  /** Proxy tier - see {@link FirecrawlProxy}. */
+  readonly proxy?: FirecrawlProxy
+}
+
+export type FirecrawlReadService = {
+  readonly read: (request: FirecrawlReadRequest) => Effect.Effect<ReadResponse, AiError.AiError>
+}
+
+/**
+ * Provider-typed service tag. Yield this for Firecrawl-typed reads (its own
+ * scrape knobs); yield the generic `WebRead` tag for provider-portable code.
+ * Both are registered by {@link layer}.
+ */
+export class FirecrawlRead extends Context.Service<FirecrawlRead, FirecrawlReadService>()(
+  "@betalyra/effect-uai/providers/firecrawl/FirecrawlRead",
+) {}
+
+export type Config = {
+  readonly apiKey: Redacted.Redacted
+  readonly baseUrl?: string
+}
+
+// ---------------------------------------------------------------------------
+// Codec - request body
+// ---------------------------------------------------------------------------
+
+type WireBody = {
+  readonly url: string
+  readonly formats: ReadonlyArray<string>
+  readonly onlyMainContent?: boolean
+  readonly includeTags?: ReadonlyArray<string>
+  readonly excludeTags?: ReadonlyArray<string>
+  readonly waitFor?: number
+  readonly timeout?: number
+  readonly mobile?: boolean
+  readonly proxy?: string
+}
+
+// `markdown` / `html` map one-to-one onto Firecrawl's format names. `links` is
+// always requested so {@link ReadResponse.links} can be populated; it does not
+// change the per-page billing.
+const buildBody = (request: FirecrawlReadRequest): WireBody => {
+  const contentFormat = request.format ?? "markdown"
+  return {
+    url: request.url,
+    formats: [contentFormat, "links"],
+    ...(request.onlyMainContent !== undefined && { onlyMainContent: request.onlyMainContent }),
+    ...(request.includeTags !== undefined && { includeTags: request.includeTags }),
+    ...(request.excludeTags !== undefined && { excludeTags: request.excludeTags }),
+    ...(request.waitFor !== undefined && { waitFor: Duration.toMillis(request.waitFor) }),
+    ...(request.timeout !== undefined && { timeout: Duration.toMillis(request.timeout) }),
+    ...(request.mobile !== undefined && { mobile: request.mobile }),
+    ...(request.proxy !== undefined && { proxy: request.proxy }),
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Codec - response
+// ---------------------------------------------------------------------------
+
+const WireMetadata = Schema.Struct({
+  // Firecrawl returns `title` as a string, an array (repeated meta tags), or
+  // null; normalize to a single string in `toResponse`.
+  title: Schema.optional(Schema.NullOr(Schema.Union([Schema.String, Schema.Array(Schema.String)]))),
+  sourceURL: Schema.optional(Schema.NullOr(Schema.String)),
+  statusCode: Schema.optional(Schema.Number),
+})
+
+const WireData = Schema.Struct({
+  markdown: Schema.optional(Schema.NullOr(Schema.String)),
+  html: Schema.optional(Schema.NullOr(Schema.String)),
+  rawHtml: Schema.optional(Schema.NullOr(Schema.String)),
+  links: Schema.optional(Schema.NullOr(Schema.Array(Schema.String))),
+  metadata: Schema.optional(WireMetadata),
+})
+
+const WireResponse = Schema.Struct({
+  success: Schema.optional(Schema.Boolean),
+  data: WireData,
+})
+type WireResponse = typeof WireResponse.Type
+
+const pickTitle = (title: string | ReadonlyArray<string> | null | undefined): string | undefined =>
+  typeof title === "string" ? title : Array.isArray(title) ? title[0] : undefined
+
+const toResponse = (wire: WireResponse, request: FirecrawlReadRequest): ReadResponse => {
+  const { data } = wire
+  const format = request.format ?? "markdown"
+  const content = (format === "html" ? data.html : data.markdown) ?? ""
+  const title = pickTitle(data.metadata?.title)
+  const links = data.links ?? undefined
+  return {
+    url: data.metadata?.sourceURL ?? request.url,
+    content,
+    ...(title !== undefined && { title }),
+    ...(links !== undefined && links.length > 0 && { links }),
+    raw: wire,
+  }
+}
+
+// ---------------------------------------------------------------------------
+// HTTP
+// ---------------------------------------------------------------------------
+
+const transportFailure = (cause: unknown): AiError.AiError =>
+  new AiError.Unavailable({ provider: "firecrawl", raw: cause })
+
+const httpStatusError = (status: number, body: string): AiError.AiError => {
+  const provider = "firecrawl"
+  const raw = body
+  if (status === 429) return new AiError.RateLimited({ provider, raw })
+  if (status === 408 || status === 504) return new AiError.Timeout({ provider, raw })
+  if (status === 401) return new AiError.AuthFailed({ provider, subtype: "auth", raw })
+  if (status === 403) return new AiError.AuthFailed({ provider, subtype: "permission", raw })
+  if (status === 402) return new AiError.AuthFailed({ provider, subtype: "billing", raw })
+  if (status >= 500) return new AiError.Unavailable({ provider, status, raw })
+  return new AiError.InvalidRequest({ provider, raw })
+}
+
+const baseUrl = (cfg: Config): string => cfg.baseUrl ?? "https://api.firecrawl.dev"
+
+const postScrape = (
+  cfg: Config,
+  body: WireBody,
+): Effect.Effect<WireResponse, AiError.AiError, HttpClient.HttpClient> =>
+  Effect.gen(function* () {
+    const client = yield* HttpClient.HttpClient
+    const httpRequest = HttpClientRequest.post(`${baseUrl(cfg)}/v2/scrape`).pipe(
+      HttpClientRequest.setHeader("Authorization", `Bearer ${Redacted.value(cfg.apiKey)}`),
+      HttpClientRequest.bodyJsonUnsafe(body),
+    )
+    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
+    if (response.status >= 400) {
+      const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
+      return yield* httpStatusError(response.status, text)
+    }
+    const json = yield* response.json.pipe(Effect.mapError(transportFailure))
+    return yield* Schema.decodeUnknownEffect(WireResponse)(json).pipe(
+      Effect.mapError(transportFailure),
+    )
+  })
+
+const readImpl =
+  (cfg: Config) =>
+  (
+    request: FirecrawlReadRequest,
+  ): Effect.Effect<ReadResponse, AiError.AiError, HttpClient.HttpClient> =>
+    Effect.map(postScrape(cfg, buildBody(request)), (wire) => toResponse(wire, request))
+
+// ---------------------------------------------------------------------------
+// Constructors
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a `FirecrawlReadService` value. For Layer-based setup, prefer
+ * {@link layer}.
+ */
+export const make = (
+  cfg: Config,
+): Effect.Effect<FirecrawlReadService, never, HttpClient.HttpClient> =>
+  Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
+    read: (request) =>
+      readImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
+  }))
+
+/**
+ * Layer registering both the provider-typed `FirecrawlRead` tag and the
+ * generic `WebRead` tag over one implementation. A `CommonReadRequest` is
+ * structurally a `FirecrawlReadRequest` with the extra knobs unset, so the
+ * generic registration forwards directly.
+ */
+export const layer = (
+  cfg: Config,
+): Layer.Layer<FirecrawlRead | WebRead, never, HttpClient.HttpClient> => {
+  const typed = Layer.effect(FirecrawlRead, make(cfg))
+  const generic = Layer.effect(
+    WebRead,
+    Effect.map(make(cfg), (s): WebReadService => ({ read: (request) => s.read(request) })),
+  )
+  return Layer.merge(typed, generic)
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export * as FirecrawlRead from "./FirecrawlRead.js"
+export * from "./models.js"

package/src/models.ts

@@ -0,0 +1,11 @@
+/**
+ * Firecrawl's proxy tier for a scrape. Higher tiers survive more aggressive
+ * anti-bot protection at higher cost. Provider-specific, so it lives on
+ * `FirecrawlReadRequest`, not the cross-provider `CommonReadRequest`.
+ */
+export type FirecrawlProxy =
+  | "basic"
+  | "stealth"
+  | "auto"
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  | (string & {})