@effect-uai/retrieval 0.12.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Betalyra Sociedade Unipessoal Lda.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,64 @@
+# @effect-uai/retrieval
+
+Retrieval-pipeline utilities for [`@effect-uai/core`](https://www.npmjs.com/package/@effect-uai/core):
+chunking, rank fusion, and a tokenizer layer.
+
+The pieces of a retrieval pipeline that are plain functions rather than
+providers. The capability tags they implement (`Chunker`, `Tokenizer`) live in
+core, so your ingest code never names a strategy.
+
+This package exposes:
+
+- **`Chunking`** - four chunkers (`recursive`, `sentences`, `markdown`,
+  `fixed`), each reporting the offsets every passage came from, so a search hit
+  can be traced back to its source. `layer` serves one through the core
+  `Chunker` tag; `withTokenizer` sizes chunks by real tokens.
+- **`Rank`** - `rrf`, reciprocal rank fusion, for merging a keyword leg and a
+  vector leg whose scores mean nothing to each other.
+- **`HuggingFaceTokenizer`** - the core `Tokenizer` tag over any Hugging Face
+  repo with a `tokenizer.json`. Downloading and building are separate, so you
+  cache the vocabulary instead of refetching on every boot.
+
+## Install
+
+```sh
+pnpm add @effect-uai/retrieval @effect-uai/core effect
+```
+
+The tokenizer needs one optional peer:
+
+```sh
+pnpm add @huggingface/tokenizers
+```
+
+ESM-only. Requires `effect@4.x` as a peer.
+
+## Usage
+
+```ts
+import { chunk } from "@effect-uai/core/Chunker"
+import * as Chunking from "@effect-uai/retrieval/Chunking"
+import * as Rank from "@effect-uai/retrieval/Rank"
+
+const ingest = Effect.gen(function* () {
+  const passages = yield* chunk(document)
+  // embed and store
+})
+
+const chunker = Chunking.layer(Chunking.recursive, { targetSize: 512 })
+
+const fused = Rank.rrf([keywordIds, vectorIds], { weights: [1, 2] })
+```
+
+## Docs
+
+Full docs: <https://effect-uai.betalyra.com/retrieval/>
+
+See [Chunking](https://effect-uai.betalyra.com/retrieval/chunking/),
+[Tokenizers](https://effect-uai.betalyra.com/language-models/tokenizers/), and
+[Agentic search](https://effect-uai.betalyra.com/recipes/agentic-search/) for a
+pipeline using all of it.
+
+## License
+
+MIT

package/dist/Chunking.d.mts

@@ -0,0 +1,48 @@
+import { Effect, Layer } from "effect";
+import { Chunk, Chunk as Chunk$1, Chunker } from "@effect-uai/core/Chunker";
+import { Tokenizer } from "@effect-uai/core/Tokenizer";
+declare namespace Chunking_d_exports {
+  export { Chunk, Measure, Options, Splitter, fixed, layer, markdown, recursive, sentences, withTokenizer };
+}
+/** How chunk size is counted. Chunkers call this on candidate chunks. */
+type Measure = (text: string) => number;
+type Options = {
+  /** In units of `measure`, approximately tokens with the default. Default 512. */
+  readonly targetSize?: number;
+  /** Units of `measure` repeated from the end of the previous chunk. Default 0. */
+  readonly overlap?: number;
+  /** Default: characters / 4, a rough token estimate for prose. */
+  readonly measure?: Measure;
+};
+type Splitter = (text: string, options?: Options) => ReadonlyArray<Chunk$1>;
+/**
+ * Blind character windows, sized by calibrating `measure` over the input. Cuts
+ * mid-word and mid-sentence, so reach for it when the input has no structure
+ * worth respecting or a hard size ceiling matters more than readable passages.
+ */
+declare const fixed: Splitter;
+/**
+ * Split on the coarsest separator that makes the pieces fit (blank line,
+ * newline, sentence end, word), then refill up to `targetSize`. The default:
+ * it keeps related prose together without knowing anything about the format.
+ */
+declare const recursive: Splitter;
+/** Pack whole sentences. Only a sentence longer than `targetSize` is cut. */
+declare const sentences: Splitter;
+/**
+ * One chunk per `#`..`######` section, so a heading stays with the prose under
+ * it. A section over `targetSize` is split with `recursive` inside its own
+ * bounds; a chunk never spans two sections. Headings inside fenced code are
+ * ignored; setext (`===` underline) headings are not recognized.
+ */
+declare const markdown: Splitter;
+/** Serve a chunker through the core `Chunker` tag. */
+declare const layer: (chunker: Splitter, options?: Options) => Layer.Layer<Chunker>;
+/**
+ * Run a chunker with a real tokenizer as its measure, so `targetSize` and
+ * `overlap` count tokens rather than estimate them.
+ */
+declare const withTokenizer: (chunker: Splitter) => (text: string, options?: Omit<Options, "measure">) => Effect.Effect<ReadonlyArray<Chunk$1>, never, Tokenizer>;
+//#endregion
+export { type Chunk, Measure, Options, Splitter, fixed, layer, markdown, recursive, sentences, Chunking_d_exports as t, withTokenizer };
+//# sourceMappingURL=Chunking.d.mts.map

package/dist/Chunking.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Chunking.d.mts","names":[],"sources":["../src/Chunking.ts"],"mappings":";;;;;;;KAmBY,WAAW;KAEX;;WAED;;WAEA;;WAEA,UAAU;;KAGT,YAAY,cAAc,UAAU,YAAY,cAAc;;;;;;cAkK7D,OAAO;;;;;;cAkBP,WAAW;;cAMX,WAAW;;;;;;;cAgDX,UAAU;;cAYV,QAAK,SAAa,UAAQ,UAAY,YAAU,MAAM,MAAM;;;;;cAO5D,gBAAa,SACd,cAAQ,cAEJ,UACF,KAAK,wBACd,OAAO,OAAO,cAAc,iBAAe"}

package/dist/Chunking.mjs

@@ -0,0 +1,206 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { Array, Effect, Layer, Option, Result, pipe } from "effect";
+import { Chunker } from "@effect-uai/core/Chunker";
+import { Tokenizer } from "@effect-uai/core/Tokenizer";
+//#region src/Chunking.ts
+/**
+* Splitting documents into retrievable passages. The chunkers are pure and
+* synchronous; `layer` lifts one into the core `Chunker` tag so a hosted
+* chunking service can take its place without touching callers.
+*
+* Every chunk carries the offsets it came from, so
+* `input.slice(chunk.start, chunk.end) === chunk.text` always holds. Overlap
+* makes chunks share regions, never lose them.
+*
+* `recursive` is the one to reach for by default. `fixed` ignores structure,
+* `sentences` never cuts mid-sentence, `markdown` follows headers.
+*/
+var Chunking_exports = /* @__PURE__ */ __exportAll({
+	fixed: () => fixed,
+	layer: () => layer,
+	markdown: () => markdown,
+	recursive: () => recursive,
+	sentences: () => sentences,
+	withTokenizer: () => withTokenizer
+});
+const approximate = (text) => Math.ceil(text.length / 4);
+const settings = (options) => ({
+	target: options?.targetSize ?? 512,
+	overlap: options?.overlap ?? 0,
+	measure: options?.measure ?? approximate
+});
+const sizeOf = (text, span, s) => s.measure(text.slice(span.start, span.end));
+/** Narrow a span to its non-whitespace core; a blank span drops out. */
+const trim = (text) => (span) => {
+	const region = text.slice(span.start, span.end);
+	const start = span.start + (region.length - region.trimStart().length);
+	const end = span.end - (region.length - region.trimEnd().length);
+	return start < end ? Result.succeed({
+		start,
+		end
+	}) : Result.failVoid;
+};
+/** Split a span on every separator match, keeping the trimmed parts between. */
+const splitSpans = (text, span, separator) => {
+	const cuts = Array.map([...text.slice(span.start, span.end).matchAll(separator)], (m) => ({
+		start: span.start + m.index,
+		end: span.start + m.index + m[0].length
+	}));
+	return pipe(Array.zip([span.start, ...Array.map(cuts, (c) => c.end)], [...Array.map(cuts, (c) => c.start), span.end]), Array.filterMap(([start, end]) => trim(text)({
+		start,
+		end
+	})));
+};
+const PARAGRAPH = /\n[ \t]*(?:\n[ \t]*)+/gu;
+const LINE = /\n/gu;
+const SENTENCE = /(?<=[.!?…][)"'”’]*)\s+/gu;
+const SEPARATORS = [
+	PARAGRAPH,
+	LINE,
+	SENTENCE,
+	/\s+/gu
+];
+/** Sentences are split by the time we get here, so oversized ones go to words. */
+const WORD_LEVEL = 3;
+/**
+* `measure` is a black box, so size a character window by calibrating it once
+* over the region instead of probing per chunk.
+*/
+const charBudget = (text, span, s) => {
+	const length = span.end - span.start;
+	const units = sizeOf(text, span, s);
+	return Math.max(1, Math.round(units === 0 ? s.target : s.target * length / units));
+};
+const windows = (span, budget, stride) => Array.unfold(span.start, (start) => start >= span.end ? Option.none() : Option.some([{
+	start,
+	end: Math.min(start + budget, span.end)
+}, start + budget >= span.end ? span.end : start + stride]));
+/** Break a span down until every part fits: coarsest separator first, then blind. */
+const explode = (text, span, level, s) => {
+	if (sizeOf(text, span, s) <= s.target) return [span];
+	const separator = SEPARATORS[level];
+	if (separator === void 0) {
+		const budget = charBudget(text, span, s);
+		return windows(span, budget, budget);
+	}
+	const parts = splitSpans(text, span, separator);
+	return parts.length <= 1 ? explode(text, span, level + 1, s) : Array.flatMap(parts, (part) => explode(text, part, level + 1, s));
+};
+/**
+* The longest suffix of a chunk worth at most `overlap`. Never the whole
+* chunk, or packing would not advance. Relies on `measure` growing with the
+* text it is given, which every sane measure does.
+*/
+const carry = (text, spans, s) => {
+	const tail = Array.drop(spans, 1);
+	return s.overlap <= 0 ? [] : pipe(Array.last(tail), Option.map((end) => Array.dropWhile(tail, (span) => sizeOf(text, {
+		start: span.start,
+		end: end.end
+	}, s) > s.overlap)), Option.getOrElse(() => []));
+};
+/** One chunk covering a group of spans verbatim. An empty group yields none. */
+const toChunk = (text, spans) => pipe(Option.all([Array.head(spans), Array.last(spans)]), Option.map(([first, last]) => ({
+	text: text.slice(first.start, last.end),
+	start: first.start,
+	end: last.end
+})), Option.match({
+	onNone: () => [],
+	onSome: (chunk) => [chunk]
+}));
+/** Greedily fill chunks with consecutive spans, carrying overlap forward. */
+const pack = (text, spans, s) => {
+	const packed = Array.reduce(spans, {
+		chunks: [],
+		current: []
+	}, (acc, span) => {
+		const first = acc.current[0];
+		return first !== void 0 && sizeOf(text, {
+			start: first.start,
+			end: span.end
+		}, s) > s.target ? {
+			chunks: [...acc.chunks, ...toChunk(text, acc.current)],
+			current: [...carry(text, acc.current, s), span]
+		} : {
+			chunks: acc.chunks,
+			current: [...acc.current, span]
+		};
+	});
+	return [...packed.chunks, ...toChunk(text, packed.current)];
+};
+/** The input's content as a span, or nothing if it is blank. */
+const content = (text) => Array.filterMap([{
+	start: 0,
+	end: text.length
+}], trim(text));
+/**
+* Blind character windows, sized by calibrating `measure` over the input. Cuts
+* mid-word and mid-sentence, so reach for it when the input has no structure
+* worth respecting or a hard size ceiling matters more than readable passages.
+*/
+const fixed = (text, options) => {
+	const s = settings(options);
+	return Array.flatMap(content(text), (span) => {
+		const budget = charBudget(text, span, s);
+		const stride = budget - Math.min(budget - 1, Math.round(s.overlap * budget / s.target));
+		return Array.map(windows(span, budget, stride), (w) => ({
+			text: text.slice(w.start, w.end),
+			start: w.start,
+			end: w.end
+		}));
+	});
+};
+/**
+* Split on the coarsest separator that makes the pieces fit (blank line,
+* newline, sentence end, word), then refill up to `targetSize`. The default:
+* it keeps related prose together without knowing anything about the format.
+*/
+const recursive = (text, options) => {
+	const s = settings(options);
+	return Array.flatMap(content(text), (span) => pack(text, explode(text, span, 0, s), s));
+};
+/** Pack whole sentences. Only a sentence longer than `targetSize` is cut. */
+const sentences = (text, options) => {
+	const s = settings(options);
+	return Array.flatMap(content(text), (span) => pack(text, Array.flatMap(splitSpans(text, span, PARAGRAPH), (paragraph) => Array.flatMap(splitSpans(text, paragraph, SENTENCE), (sentence) => explode(text, sentence, WORD_LEVEL, s))), s));
+};
+const HEADER = /^#{1,6}[ \t]+\S.*$/gmu;
+const FENCE = /^[ \t]{0,3}(?:```|~~~).*$/gmu;
+/** Fenced code regions, where a `#` line is content rather than a heading. */
+const fenced = (region) => pipe(Array.map([...region.matchAll(FENCE)], (m) => m.index), Array.chunksOf(2), Array.map(([open, close]) => ({
+	start: open,
+	end: close ?? region.length
+})));
+/** Spans running from each header to the next, plus any preamble. */
+const sections = (text, span) => {
+	const region = text.slice(span.start, span.end);
+	const code = fenced(region);
+	const starts = Array.dedupe([span.start, ...pipe([...region.matchAll(HEADER)], Array.filter((m) => !Array.some(code, (f) => m.index > f.start && m.index < f.end)), Array.map((m) => span.start + m.index))]);
+	return Array.filterMap(starts, (start, i) => trim(text)({
+		start,
+		end: starts[i + 1] ?? span.end
+	}));
+};
+/**
+* One chunk per `#`..`######` section, so a heading stays with the prose under
+* it. A section over `targetSize` is split with `recursive` inside its own
+* bounds; a chunk never spans two sections. Headings inside fenced code are
+* ignored; setext (`===` underline) headings are not recognized.
+*/
+const markdown = (text, options) => {
+	const s = settings(options);
+	return Array.flatMap(content(text), (span) => Array.flatMap(sections(text, span), (section) => pack(text, explode(text, section, 0, s), s)));
+};
+/** Serve a chunker through the core `Chunker` tag. */
+const layer = (chunker, options) => Layer.succeed(Chunker, { chunk: (text) => Effect.succeed(chunker(text, options)) });
+/**
+* Run a chunker with a real tokenizer as its measure, so `targetSize` and
+* `overlap` count tokens rather than estimate them.
+*/
+const withTokenizer = (chunker) => (text, options) => Effect.map(Tokenizer, (tokenizer) => chunker(text, {
+	...options,
+	measure: (part) => tokenizer.encode(part).length
+}));
+//#endregion
+export { fixed, layer, markdown, recursive, sentences, Chunking_exports as t, withTokenizer };
+
+//# sourceMappingURL=Chunking.mjs.map

package/dist/Chunking.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"Chunking.mjs","names":["Arr"],"sources":["../src/Chunking.ts"],"sourcesContent":["/**\n * Splitting documents into retrievable passages. The chunkers are pure and\n * synchronous; `layer` lifts one into the core `Chunker` tag so a hosted\n * chunking service can take its place without touching callers.\n *\n * Every chunk carries the offsets it came from, so\n * `input.slice(chunk.start, chunk.end) === chunk.text` always holds. Overlap\n * makes chunks share regions, never lose them.\n *\n * `recursive` is the one to reach for by default. `fixed` ignores structure,\n * `sentences` never cuts mid-sentence, `markdown` follows headers.\n */\nimport { Array as Arr, Effect, Layer, Option, pipe, Result } from \"effect\"\nimport { type Chunk, Chunker } from \"@effect-uai/core/Chunker\"\nimport { Tokenizer } from \"@effect-uai/core/Tokenizer\"\n\nexport type { Chunk } from \"@effect-uai/core/Chunker\"\n\n/** How chunk size is counted. Chunkers call this on candidate chunks. */\nexport type Measure = (text: string) => number\n\nexport type Options = {\n  /** In units of `measure`, approximately tokens with the default. Default 512. */\n  readonly targetSize?: number\n  /** Units of `measure` repeated from the end of the previous chunk. Default 0. */\n  readonly overlap?: number\n  /** Default: characters / 4, a rough token estimate for prose. */\n  readonly measure?: Measure\n}\n\nexport type Splitter = (text: string, options?: Options) => ReadonlyArray<Chunk>\n\nconst approximate: Measure = (text) => Math.ceil(text.length / 4)\n\ntype Span = { readonly start: number; readonly end: number }\n\ntype Settings = {\n  readonly target: number\n  readonly overlap: number\n  readonly measure: Measure\n}\n\nconst settings = (options: Options | undefined): Settings => ({\n  target: options?.targetSize ?? 512,\n  overlap: options?.overlap ?? 0,\n  measure: options?.measure ?? approximate,\n})\n\n// ---------------------------------------------------------------------------\n// Spans\n// ---------------------------------------------------------------------------\n\nconst sizeOf = (text: string, span: Span, s: Settings): number =>\n  s.measure(text.slice(span.start, span.end))\n\n/** Narrow a span to its non-whitespace core; a blank span drops out. */\nconst trim =\n  (text: string) =>\n  (span: Span): Result.Result<Span, void> => {\n    const region = text.slice(span.start, span.end)\n    const start = span.start + (region.length - region.trimStart().length)\n    const end = span.end - (region.length - region.trimEnd().length)\n    return start < end ? Result.succeed({ start, end }) : Result.failVoid\n  }\n\n/** Split a span on every separator match, keeping the trimmed parts between. */\nconst splitSpans = (text: string, span: Span, separator: RegExp): ReadonlyArray<Span> => {\n  const cuts = Arr.map([...text.slice(span.start, span.end).matchAll(separator)], (m) => ({\n    start: span.start + m.index,\n    end: span.start + m.index + m[0].length,\n  }))\n  return pipe(\n    Arr.zip(\n      [span.start, ...Arr.map(cuts, (c) => c.end)],\n      [...Arr.map(cuts, (c) => c.start), span.end],\n    ),\n    Arr.filterMap(([start, end]) => trim(text)({ start, end })),\n  )\n}\n\n// Coarse to fine. Closing quotes may trail the sentence punctuation.\nconst PARAGRAPH = /\\n[ \\t]*(?:\\n[ \\t]*)+/gu\nconst LINE = /\\n/gu\nconst SENTENCE = /(?<=[.!?…][)\"'”’]*)\\s+/gu\nconst WORD = /\\s+/gu\nconst SEPARATORS: ReadonlyArray<RegExp> = [PARAGRAPH, LINE, SENTENCE, WORD]\n\n/** Sentences are split by the time we get here, so oversized ones go to words. */\nconst WORD_LEVEL = 3\n\n/**\n * `measure` is a black box, so size a character window by calibrating it once\n * over the region instead of probing per chunk.\n */\nconst charBudget = (text: string, span: Span, s: Settings): number => {\n  const length = span.end - span.start\n  const units = sizeOf(text, span, s)\n  return Math.max(1, Math.round(units === 0 ? s.target : (s.target * length) / units))\n}\n\nconst windows = (span: Span, budget: number, stride: number): ReadonlyArray<Span> =>\n  Arr.unfold(span.start, (start) =>\n    start >= span.end\n      ? Option.none()\n      : Option.some([\n          { start, end: Math.min(start + budget, span.end) },\n          // Once a window reaches the end, stop: with overlap the strides that\n          // follow would only produce windows contained in this one.\n          start + budget >= span.end ? span.end : start + stride,\n        ] as const),\n  )\n\n/** Break a span down until every part fits: coarsest separator first, then blind. */\nconst explode = (text: string, span: Span, level: number, s: Settings): ReadonlyArray<Span> => {\n  if (sizeOf(text, span, s) <= s.target) return [span]\n  const separator = SEPARATORS[level]\n  if (separator === undefined) {\n    const budget = charBudget(text, span, s)\n    return windows(span, budget, budget)\n  }\n  const parts = splitSpans(text, span, separator)\n  // A separator that does not divide the region is no progress; try the next.\n  return parts.length <= 1\n    ? explode(text, span, level + 1, s)\n    : Arr.flatMap(parts, (part) => explode(text, part, level + 1, s))\n}\n\n/**\n * The longest suffix of a chunk worth at most `overlap`. Never the whole\n * chunk, or packing would not advance. Relies on `measure` growing with the\n * text it is given, which every sane measure does.\n */\nconst carry = (text: string, spans: ReadonlyArray<Span>, s: Settings): ReadonlyArray<Span> => {\n  const tail = Arr.drop(spans, 1)\n  return s.overlap <= 0\n    ? []\n    : pipe(\n        Arr.last(tail),\n        Option.map((end) =>\n          Arr.dropWhile(\n            tail,\n            (span) => sizeOf(text, { start: span.start, end: end.end }, s) > s.overlap,\n          ),\n        ),\n        Option.getOrElse((): ReadonlyArray<Span> => []),\n      )\n}\n\n/** One chunk covering a group of spans verbatim. An empty group yields none. */\nconst toChunk = (text: string, spans: ReadonlyArray<Span>): ReadonlyArray<Chunk> =>\n  pipe(\n    Option.all([Arr.head(spans), Arr.last(spans)]),\n    Option.map(([first, last]): Chunk => ({\n      text: text.slice(first.start, last.end),\n      start: first.start,\n      end: last.end,\n    })),\n    Option.match({ onNone: (): ReadonlyArray<Chunk> => [], onSome: (chunk) => [chunk] }),\n  )\n\ntype Packing = {\n  readonly chunks: ReadonlyArray<Chunk>\n  readonly current: ReadonlyArray<Span>\n}\n\n/** Greedily fill chunks with consecutive spans, carrying overlap forward. */\nconst pack = (text: string, spans: ReadonlyArray<Span>, s: Settings): ReadonlyArray<Chunk> => {\n  const packed = Arr.reduce(spans, { chunks: [], current: [] } as Packing, (acc, span) => {\n    const first = acc.current[0]\n    return first !== undefined && sizeOf(text, { start: first.start, end: span.end }, s) > s.target\n      ? {\n          chunks: [...acc.chunks, ...toChunk(text, acc.current)],\n          current: [...carry(text, acc.current, s), span],\n        }\n      : { chunks: acc.chunks, current: [...acc.current, span] }\n  })\n  return [...packed.chunks, ...toChunk(text, packed.current)]\n}\n\n/** The input's content as a span, or nothing if it is blank. */\nconst content = (text: string): ReadonlyArray<Span> =>\n  Arr.filterMap([{ start: 0, end: text.length }], trim(text))\n\n// ---------------------------------------------------------------------------\n// Chunkers\n// ---------------------------------------------------------------------------\n\n/**\n * Blind character windows, sized by calibrating `measure` over the input. Cuts\n * mid-word and mid-sentence, so reach for it when the input has no structure\n * worth respecting or a hard size ceiling matters more than readable passages.\n */\nexport const fixed: Splitter = (text, options) => {\n  const s = settings(options)\n  return Arr.flatMap(content(text), (span) => {\n    const budget = charBudget(text, span, s)\n    const stride = budget - Math.min(budget - 1, Math.round((s.overlap * budget) / s.target))\n    return Arr.map(windows(span, budget, stride), (w) => ({\n      text: text.slice(w.start, w.end),\n      start: w.start,\n      end: w.end,\n    }))\n  })\n}\n\n/**\n * Split on the coarsest separator that makes the pieces fit (blank line,\n * newline, sentence end, word), then refill up to `targetSize`. The default:\n * it keeps related prose together without knowing anything about the format.\n */\nexport const recursive: Splitter = (text, options) => {\n  const s = settings(options)\n  return Arr.flatMap(content(text), (span) => pack(text, explode(text, span, 0, s), s))\n}\n\n/** Pack whole sentences. Only a sentence longer than `targetSize` is cut. */\nexport const sentences: Splitter = (text, options) => {\n  const s = settings(options)\n  return Arr.flatMap(content(text), (span) =>\n    pack(\n      text,\n      Arr.flatMap(splitSpans(text, span, PARAGRAPH), (paragraph) =>\n        Arr.flatMap(splitSpans(text, paragraph, SENTENCE), (sentence) =>\n          explode(text, sentence, WORD_LEVEL, s),\n        ),\n      ),\n      s,\n    ),\n  )\n}\n\nconst HEADER = /^#{1,6}[ \\t]+\\S.*$/gmu\nconst FENCE = /^[ \\t]{0,3}(?:```|~~~).*$/gmu\n\n/** Fenced code regions, where a `#` line is content rather than a heading. */\nconst fenced = (region: string): ReadonlyArray<Span> =>\n  pipe(\n    Arr.map([...region.matchAll(FENCE)], (m) => m.index),\n    Arr.chunksOf(2),\n    // An unclosed fence swallows the rest of the document, as a renderer does.\n    Arr.map(([open, close]) => ({ start: open, end: close ?? region.length })),\n  )\n\n/** Spans running from each header to the next, plus any preamble. */\nconst sections = (text: string, span: Span): ReadonlyArray<Span> => {\n  const region = text.slice(span.start, span.end)\n  const code = fenced(region)\n  const starts = Arr.dedupe([\n    span.start,\n    ...pipe(\n      [...region.matchAll(HEADER)],\n      Arr.filter((m) => !Arr.some(code, (f) => m.index > f.start && m.index < f.end)),\n      Arr.map((m) => span.start + m.index),\n    ),\n  ])\n  return Arr.filterMap(starts, (start, i) => trim(text)({ start, end: starts[i + 1] ?? span.end }))\n}\n\n/**\n * One chunk per `#`..`######` section, so a heading stays with the prose under\n * it. A section over `targetSize` is split with `recursive` inside its own\n * bounds; a chunk never spans two sections. Headings inside fenced code are\n * ignored; setext (`===` underline) headings are not recognized.\n */\nexport const markdown: Splitter = (text, options) => {\n  const s = settings(options)\n  return Arr.flatMap(content(text), (span) =>\n    Arr.flatMap(sections(text, span), (section) => pack(text, explode(text, section, 0, s), s)),\n  )\n}\n\n// ---------------------------------------------------------------------------\n// Wiring\n// ---------------------------------------------------------------------------\n\n/** Serve a chunker through the core `Chunker` tag. */\nexport const layer = (chunker: Splitter, options?: Options): Layer.Layer<Chunker> =>\n  Layer.succeed(Chunker, { chunk: (text: string) => Effect.succeed(chunker(text, options)) })\n\n/**\n * Run a chunker with a real tokenizer as its measure, so `targetSize` and\n * `overlap` count tokens rather than estimate them.\n */\nexport const withTokenizer =\n  (chunker: Splitter) =>\n  (\n    text: string,\n    options?: Omit<Options, \"measure\">,\n  ): Effect.Effect<ReadonlyArray<Chunk>, never, Tokenizer> =>\n    Effect.map(Tokenizer, (tokenizer) =>\n      chunker(text, { ...options, measure: (part) => tokenizer.encode(part).length }),\n    )\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgCA,MAAM,eAAwB,SAAS,KAAK,KAAK,KAAK,SAAS,CAAC;AAUhE,MAAM,YAAY,aAA4C;CAC5D,QAAQ,SAAS,cAAc;CAC/B,SAAS,SAAS,WAAW;CAC7B,SAAS,SAAS,WAAW;AAC/B;AAMA,MAAM,UAAU,MAAc,MAAY,MACxC,EAAE,QAAQ,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG,CAAC;;AAG5C,MAAM,QACH,UACA,SAA0C;CACzC,MAAM,SAAS,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG;CAC9C,MAAM,QAAQ,KAAK,SAAS,OAAO,SAAS,OAAO,UAAU,CAAC,CAAC;CAC/D,MAAM,MAAM,KAAK,OAAO,OAAO,SAAS,OAAO,QAAQ,CAAC,CAAC;CACzD,OAAO,QAAQ,MAAM,OAAO,QAAQ;EAAE;EAAO;CAAI,CAAC,IAAI,OAAO;AAC/D;;AAGF,MAAM,cAAc,MAAc,MAAY,cAA2C;CACvF,MAAM,OAAOA,MAAI,IAAI,CAAC,GAAG,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC,SAAS,SAAS,CAAC,IAAI,OAAO;EACtF,OAAO,KAAK,QAAQ,EAAE;EACtB,KAAK,KAAK,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;CACnC,EAAE;CACF,OAAO,KACLA,MAAI,IACF,CAAC,KAAK,OAAO,GAAGA,MAAI,IAAI,OAAO,MAAM,EAAE,GAAG,CAAC,GAC3C,CAAC,GAAGA,MAAI,IAAI,OAAO,MAAM,EAAE,KAAK,GAAG,KAAK,GAAG,CAC7C,GACAA,MAAI,WAAW,CAAC,OAAO,SAAS,KAAK,IAAI,CAAC,CAAC;EAAE;EAAO;CAAI,CAAC,CAAC,CAC5D;AACF;AAGA,MAAM,YAAY;AAClB,MAAM,OAAO;AACb,MAAM,WAAW;AAEjB,MAAM,aAAoC;CAAC;CAAW;CAAM;CAAU;AAAI;;AAG1E,MAAM,aAAa;;;;;AAMnB,MAAM,cAAc,MAAc,MAAY,MAAwB;CACpE,MAAM,SAAS,KAAK,MAAM,KAAK;CAC/B,MAAM,QAAQ,OAAO,MAAM,MAAM,CAAC;CAClC,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,UAAU,IAAI,EAAE,SAAU,EAAE,SAAS,SAAU,KAAK,CAAC;AACrF;AAEA,MAAM,WAAW,MAAY,QAAgB,WAC3CA,MAAI,OAAO,KAAK,QAAQ,UACtB,SAAS,KAAK,MACV,OAAO,KAAK,IACZ,OAAO,KAAK,CACV;CAAE;CAAO,KAAK,KAAK,IAAI,QAAQ,QAAQ,KAAK,GAAG;AAAE,GAGjD,QAAQ,UAAU,KAAK,MAAM,KAAK,MAAM,QAAQ,MAClD,CAAU,CAChB;;AAGF,MAAM,WAAW,MAAc,MAAY,OAAe,MAAqC;CAC7F,IAAI,OAAO,MAAM,MAAM,CAAC,KAAK,EAAE,QAAQ,OAAO,CAAC,IAAI;CACnD,MAAM,YAAY,WAAW;CAC7B,IAAI,cAAc,KAAA,GAAW;EAC3B,MAAM,SAAS,WAAW,MAAM,MAAM,CAAC;EACvC,OAAO,QAAQ,MAAM,QAAQ,MAAM;CACrC;CACA,MAAM,QAAQ,WAAW,MAAM,MAAM,SAAS;CAE9C,OAAO,MAAM,UAAU,IACnB,QAAQ,MAAM,MAAM,QAAQ,GAAG,CAAC,IAChCA,MAAI,QAAQ,QAAQ,SAAS,QAAQ,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC;AACpE;;;;;;AAOA,MAAM,SAAS,MAAc,OAA4B,MAAqC;CAC5F,MAAM,OAAOA,MAAI,KAAK,OAAO,CAAC;CAC9B,OAAO,EAAE,WAAW,IAChB,CAAC,IACD,KACEA,MAAI,KAAK,IAAI,GACb,OAAO,KAAK,QACVA,MAAI,UACF,OACC,SAAS,OAAO,MAAM;EAAE,OAAO,KAAK;EAAO,KAAK,IAAI;CAAI,GAAG,CAAC,IAAI,EAAE,OACrE,CACF,GACA,OAAO,gBAAqC,CAAC,CAAC,CAChD;AACN;;AAGA,MAAM,WAAW,MAAc,UAC7B,KACE,OAAO,IAAI,CAACA,MAAI,KAAK,KAAK,GAAGA,MAAI,KAAK,KAAK,CAAC,CAAC,GAC7C,OAAO,KAAK,CAAC,OAAO,WAAkB;CACpC,MAAM,KAAK,MAAM,MAAM,OAAO,KAAK,GAAG;CACtC,OAAO,MAAM;CACb,KAAK,KAAK;AACZ,EAAE,GACF,OAAO,MAAM;CAAE,cAAoC,CAAC;CAAG,SAAS,UAAU,CAAC,KAAK;AAAE,CAAC,CACrF;;AAQF,MAAM,QAAQ,MAAc,OAA4B,MAAsC;CAC5F,MAAM,SAASA,MAAI,OAAO,OAAO;EAAE,QAAQ,CAAC;EAAG,SAAS,CAAC;CAAE,IAAe,KAAK,SAAS;EACtF,MAAM,QAAQ,IAAI,QAAQ;EAC1B,OAAO,UAAU,KAAA,KAAa,OAAO,MAAM;GAAE,OAAO,MAAM;GAAO,KAAK,KAAK;EAAI,GAAG,CAAC,IAAI,EAAE,SACrF;GACE,QAAQ,CAAC,GAAG,IAAI,QAAQ,GAAG,QAAQ,MAAM,IAAI,OAAO,CAAC;GACrD,SAAS,CAAC,GAAG,MAAM,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI;EAChD,IACA;GAAE,QAAQ,IAAI;GAAQ,SAAS,CAAC,GAAG,IAAI,SAAS,IAAI;EAAE;CAC5D,CAAC;CACD,OAAO,CAAC,GAAG,OAAO,QAAQ,GAAG,QAAQ,MAAM,OAAO,OAAO,CAAC;AAC5D;;AAGA,MAAM,WAAW,SACfA,MAAI,UAAU,CAAC;CAAE,OAAO;CAAG,KAAK,KAAK;AAAO,CAAC,GAAG,KAAK,IAAI,CAAC;;;;;;AAW5D,MAAa,SAAmB,MAAM,YAAY;CAChD,MAAM,IAAI,SAAS,OAAO;CAC1B,OAAOA,MAAI,QAAQ,QAAQ,IAAI,IAAI,SAAS;EAC1C,MAAM,SAAS,WAAW,MAAM,MAAM,CAAC;EACvC,MAAM,SAAS,SAAS,KAAK,IAAI,SAAS,GAAG,KAAK,MAAO,EAAE,UAAU,SAAU,EAAE,MAAM,CAAC;EACxF,OAAOA,MAAI,IAAI,QAAQ,MAAM,QAAQ,MAAM,IAAI,OAAO;GACpD,MAAM,KAAK,MAAM,EAAE,OAAO,EAAE,GAAG;GAC/B,OAAO,EAAE;GACT,KAAK,EAAE;EACT,EAAE;CACJ,CAAC;AACH;;;;;;AAOA,MAAa,aAAuB,MAAM,YAAY;CACpD,MAAM,IAAI,SAAS,OAAO;CAC1B,OAAOA,MAAI,QAAQ,QAAQ,IAAI,IAAI,SAAS,KAAK,MAAM,QAAQ,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;AACtF;;AAGA,MAAa,aAAuB,MAAM,YAAY;CACpD,MAAM,IAAI,SAAS,OAAO;CAC1B,OAAOA,MAAI,QAAQ,QAAQ,IAAI,IAAI,SACjC,KACE,MACAA,MAAI,QAAQ,WAAW,MAAM,MAAM,SAAS,IAAI,cAC9CA,MAAI,QAAQ,WAAW,MAAM,WAAW,QAAQ,IAAI,aAClD,QAAQ,MAAM,UAAU,YAAY,CAAC,CACvC,CACF,GACA,CACF,CACF;AACF;AAEA,MAAM,SAAS;AACf,MAAM,QAAQ;;AAGd,MAAM,UAAU,WACd,KACEA,MAAI,IAAI,CAAC,GAAG,OAAO,SAAS,KAAK,CAAC,IAAI,MAAM,EAAE,KAAK,GACnDA,MAAI,SAAS,CAAC,GAEdA,MAAI,KAAK,CAAC,MAAM,YAAY;CAAE,OAAO;CAAM,KAAK,SAAS,OAAO;AAAO,EAAE,CAC3E;;AAGF,MAAM,YAAY,MAAc,SAAoC;CAClE,MAAM,SAAS,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG;CAC9C,MAAM,OAAO,OAAO,MAAM;CAC1B,MAAM,SAASA,MAAI,OAAO,CACxB,KAAK,OACL,GAAG,KACD,CAAC,GAAG,OAAO,SAAS,MAAM,CAAC,GAC3BA,MAAI,QAAQ,MAAM,CAACA,MAAI,KAAK,OAAO,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,GAC9EA,MAAI,KAAK,MAAM,KAAK,QAAQ,EAAE,KAAK,CACrC,CACF,CAAC;CACD,OAAOA,MAAI,UAAU,SAAS,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC;EAAE;EAAO,KAAK,OAAO,IAAI,MAAM,KAAK;CAAI,CAAC,CAAC;AAClG;;;;;;;AAQA,MAAa,YAAsB,MAAM,YAAY;CACnD,MAAM,IAAI,SAAS,OAAO;CAC1B,OAAOA,MAAI,QAAQ,QAAQ,IAAI,IAAI,SACjCA,MAAI,QAAQ,SAAS,MAAM,IAAI,IAAI,YAAY,KAAK,MAAM,QAAQ,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC,CAC5F;AACF;;AAOA,MAAa,SAAS,SAAmB,YACvC,MAAM,QAAQ,SAAS,EAAE,QAAQ,SAAiB,OAAO,QAAQ,QAAQ,MAAM,OAAO,CAAC,EAAE,CAAC;;;;;AAM5F,MAAa,iBACV,aAEC,MACA,YAEA,OAAO,IAAI,YAAY,cACrB,QAAQ,MAAM;CAAE,GAAG;CAAS,UAAU,SAAS,UAAU,OAAO,IAAI,CAAC,CAAC;AAAO,CAAC,CAChF"}

package/dist/HuggingFaceTokenizer.d.mts

@@ -0,0 +1,46 @@
+import { Effect, Layer, Redacted, Schema } from "effect";
+import { Tokenizer } from "@effect-uai/core/Tokenizer";
+import { HttpClient } from "effect/unstable/http";
+//#region src/HuggingFaceTokenizer.d.ts
+declare const TokenizerLoadError_base: Schema.Class<TokenizerLoadError, Schema.TaggedStruct<"TokenizerLoadError", {
+  readonly model: Schema.optional<Schema.String>;
+  readonly reason: Schema.String;
+  readonly cause: Schema.optional<Schema.Unknown>;
+}>, import("effect/Cause").YieldableError>;
+/** The vocabulary could not be fetched, parsed, or loaded. */
+declare class TokenizerLoadError extends TokenizerLoadError_base {}
+/**
+ * Everything that defines a tokenizer's behaviour, as published. Plain JSON:
+ * write it to disk, a row, or your bundle, and decode it back with this schema.
+ */
+declare const Definition: Schema.Struct<{
+  /** Contents of `tokenizer.json`. */
+  readonly tokenizer: Schema.Unknown;
+  /** Contents of `tokenizer_config.json`, absent for repos that publish none. */
+  readonly config: Schema.optional<Schema.Unknown>;
+}>;
+type Definition = typeof Definition.Type;
+type Options = {
+  /** A Hugging Face repo id, e.g. `"jinaai/jina-embeddings-v3"` or `"Xenova/gpt-4o"`. */
+  readonly model: string;
+  /** Branch, tag, or commit. Default `"main"`. */
+  readonly revision?: string;
+  /**
+   * A Hugging Face access token. Required for gated repos, where the Hub
+   * serves the files only to accounts that have accepted the model's terms
+   * (Google's tokenizers among them), and for private ones.
+   */
+  readonly token?: Redacted.Redacted;
+};
+/**
+ * Fetch a model's definition from the Hugging Face Hub. Cache the result
+ * yourself; a repo without a `tokenizer_config.json` yields `config: undefined`.
+ */
+declare const download: (options: Options) => Effect.Effect<Definition, TokenizerLoadError, HttpClient.HttpClient>;
+/** Build the tokenizer from a definition you already hold. No IO. */
+declare const fromDefinition: (definition: Definition) => Layer.Layer<Tokenizer, TokenizerLoadError>;
+/** Download and build in one step. For scripts; production should cache. */
+declare const layer: (options: Options) => Layer.Layer<Tokenizer, TokenizerLoadError, HttpClient.HttpClient>;
+//#endregion
+export { Definition, Options, TokenizerLoadError, download, fromDefinition, layer };
+//# sourceMappingURL=HuggingFaceTokenizer.d.mts.map

package/dist/HuggingFaceTokenizer.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HuggingFaceTokenizer.d.mts","names":[],"sources":["../src/HuggingFaceTokenizer.ts"],"mappings":";;;;;;;;;;cAiBa,2BAA2B;;;;;cAY3B,YAAU,OAAA;;;;;;KAOX,oBAAoB,WAAW;KAE/B;;WAED;;WAEA;;;;;;WAMA,QAAQ,SAAS;;;;;;cAkCf,WAAQ,SACV,YACR,OAAO,OAAO,YAAY,oBAAoB,WAAW;;cAO/C,iBAAc,YACb,eACX,MAAM,MAAM,WAAW;;cAsBb,QAAK,SACP,YACR,MAAM,MAAM,WAAW,oBAAoB,WAAW"}

package/dist/HuggingFaceTokenizer.mjs

@@ -0,0 +1,75 @@
+import { Array, Effect, Layer, Redacted, Schema } from "effect";
+import { Tokenizer } from "@effect-uai/core/Tokenizer";
+import { HttpClient, HttpClientRequest } from "effect/unstable/http";
+import { Tokenizer as Tokenizer$1 } from "@huggingface/tokenizers";
+//#region src/HuggingFaceTokenizer.ts
+/**
+* The core `Tokenizer` tag over `@huggingface/tokenizers`, which reads a
+* model's `tokenizer.json` and needs no native build. The package is an
+* optional peer dependency: `pnpm add @huggingface/tokenizers`.
+*
+* Fetching and construction are separate on purpose. `download` gives you
+* plain JSON to keep wherever your application keeps things; `fromDefinition`
+* builds the tokenizer from what you kept. Nothing here touches the filesystem
+* or caches behind your back. `layer` composes the two for scripts happy to
+* download on every run.
+*/
+/** The vocabulary could not be fetched, parsed, or loaded. */
+var TokenizerLoadError = class extends Schema.TaggedError("@effect-uai/retrieval/TokenizerLoadError")("TokenizerLoadError", {
+	model: Schema.optional(Schema.String),
+	reason: Schema.String,
+	cause: Schema.optional(Schema.Unknown)
+}) {};
+/**
+* Everything that defines a tokenizer's behaviour, as published. Plain JSON:
+* write it to disk, a row, or your bundle, and decode it back with this schema.
+*/
+const Definition = Schema.Struct({
+	/** Contents of `tokenizer.json`. */
+	tokenizer: Schema.Unknown,
+	/** Contents of `tokenizer_config.json`, absent for repos that publish none. */
+	config: Schema.optional(Schema.Unknown)
+});
+const fetchJson = (options, file) => Effect.gen(function* () {
+	const client = yield* HttpClient.HttpClient;
+	const url = `https://huggingface.co/${options.model}/resolve/${options.revision ?? "main"}/${file}`;
+	const failed = (reason) => (cause) => new TokenizerLoadError({
+		model: options.model,
+		reason,
+		cause
+	});
+	const request = HttpClientRequest.get(url).pipe(options.token === void 0 ? (self) => self : HttpClientRequest.bearerToken(Redacted.value(options.token)));
+	const response = yield* client.execute(request).pipe(Effect.mapError(failed(`could not fetch ${file}`)));
+	return response.status >= 400 ? yield* new TokenizerLoadError({
+		model: options.model,
+		reason: `${url} returned ${response.status}`
+	}) : yield* response.json.pipe(Effect.mapError(failed(`${file} is not valid JSON`)));
+});
+/**
+* Fetch a model's definition from the Hugging Face Hub. Cache the result
+* yourself; a repo without a `tokenizer_config.json` yields `config: undefined`.
+*/
+const download = (options) => Effect.all({
+	tokenizer: fetchJson(options, "tokenizer.json"),
+	config: fetchJson(options, "tokenizer_config.json").pipe(Effect.orElseSucceed(() => void 0))
+});
+/** Build the tokenizer from a definition you already hold. No IO. */
+const fromDefinition = (definition) => Layer.effect(Tokenizer, Effect.try({
+	try: () => {
+		const hf = new Tokenizer$1(definition.tokenizer, definition.config ?? {});
+		return {
+			encode: (text) => hf.encode(text).ids,
+			decode: (tokens) => tokens.length === 0 ? "" : hf.decode(Array.copy(tokens))
+		};
+	},
+	catch: (cause) => new TokenizerLoadError({
+		reason: "could not load the tokenizer",
+		cause
+	})
+}));
+/** Download and build in one step. For scripts; production should cache. */
+const layer = (options) => Layer.unwrap(Effect.map(download(options), fromDefinition));
+//#endregion
+export { Definition, TokenizerLoadError, download, fromDefinition, layer };
+
+//# sourceMappingURL=HuggingFaceTokenizer.mjs.map

package/dist/HuggingFaceTokenizer.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"HuggingFaceTokenizer.mjs","names":["HfTokenizer","Arr"],"sources":["../src/HuggingFaceTokenizer.ts"],"sourcesContent":["/**\n * The core `Tokenizer` tag over `@huggingface/tokenizers`, which reads a\n * model's `tokenizer.json` and needs no native build. The package is an\n * optional peer dependency: `pnpm add @huggingface/tokenizers`.\n *\n * Fetching and construction are separate on purpose. `download` gives you\n * plain JSON to keep wherever your application keeps things; `fromDefinition`\n * builds the tokenizer from what you kept. Nothing here touches the filesystem\n * or caches behind your back. `layer` composes the two for scripts happy to\n * download on every run.\n */\nimport { Array as Arr, Effect, Layer, Redacted, Schema } from \"effect\"\nimport { HttpClient, HttpClientRequest } from \"effect/unstable/http\"\nimport { Tokenizer, type TokenizerService } from \"@effect-uai/core/Tokenizer\"\nimport { Tokenizer as HfTokenizer } from \"@huggingface/tokenizers\"\n\n/** The vocabulary could not be fetched, parsed, or loaded. */\nexport class TokenizerLoadError extends Schema.TaggedError<TokenizerLoadError>(\n  \"@effect-uai/retrieval/TokenizerLoadError\",\n)(\"TokenizerLoadError\", {\n  model: Schema.optional(Schema.String),\n  reason: Schema.String,\n  cause: Schema.optional(Schema.Unknown),\n}) {}\n\n/**\n * Everything that defines a tokenizer's behaviour, as published. Plain JSON:\n * write it to disk, a row, or your bundle, and decode it back with this schema.\n */\nexport const Definition = Schema.Struct({\n  /** Contents of `tokenizer.json`. */\n  tokenizer: Schema.Unknown,\n  /** Contents of `tokenizer_config.json`, absent for repos that publish none. */\n  config: Schema.optional(Schema.Unknown),\n})\n\nexport type Definition = typeof Definition.Type\n\nexport type Options = {\n  /** A Hugging Face repo id, e.g. `\"jinaai/jina-embeddings-v3\"` or `\"Xenova/gpt-4o\"`. */\n  readonly model: string\n  /** Branch, tag, or commit. Default `\"main\"`. */\n  readonly revision?: string\n  /**\n   * A Hugging Face access token. Required for gated repos, where the Hub\n   * serves the files only to accounts that have accepted the model's terms\n   * (Google's tokenizers among them), and for private ones.\n   */\n  readonly token?: Redacted.Redacted\n}\n\nconst fetchJson = (\n  options: Options,\n  file: string,\n): Effect.Effect<unknown, TokenizerLoadError, HttpClient.HttpClient> =>\n  Effect.gen(function* () {\n    const client = yield* HttpClient.HttpClient\n    const url = `https://huggingface.co/${options.model}/resolve/${options.revision ?? \"main\"}/${file}`\n    const failed = (reason: string) => (cause: unknown) =>\n      new TokenizerLoadError({ model: options.model, reason, cause })\n\n    const request = HttpClientRequest.get(url).pipe(\n      options.token === undefined\n        ? (self) => self\n        : HttpClientRequest.bearerToken(Redacted.value(options.token)),\n    )\n    const response = yield* client\n      .execute(request)\n      .pipe(Effect.mapError(failed(`could not fetch ${file}`)))\n    // 401/403 on a gated repo means no token, or terms not yet accepted.\n    return response.status >= 400\n      ? yield* new TokenizerLoadError({\n          model: options.model,\n          reason: `${url} returned ${response.status}`,\n        })\n      : yield* response.json.pipe(Effect.mapError(failed(`${file} is not valid JSON`)))\n  })\n\n/**\n * Fetch a model's definition from the Hugging Face Hub. Cache the result\n * yourself; a repo without a `tokenizer_config.json` yields `config: undefined`.\n */\nexport const download = (\n  options: Options,\n): Effect.Effect<Definition, TokenizerLoadError, HttpClient.HttpClient> =>\n  Effect.all({\n    tokenizer: fetchJson(options, \"tokenizer.json\"),\n    config: fetchJson(options, \"tokenizer_config.json\").pipe(Effect.orElseSucceed(() => undefined)),\n  })\n\n/** Build the tokenizer from a definition you already hold. No IO. */\nexport const fromDefinition = (\n  definition: Definition,\n): Layer.Layer<Tokenizer, TokenizerLoadError> =>\n  Layer.effect(\n    Tokenizer,\n    Effect.try({\n      try: (): TokenizerService => {\n        const hf = new HfTokenizer(\n          definition.tokenizer as object,\n          (definition.config ?? {}) as object,\n        )\n        return {\n          encode: (text) => hf.encode(text).ids,\n          // The library rejects an empty array, but `encode(\"\")` returns one and\n          // `decode` inverts `encode`. Its signature also wants a mutable array,\n          // though it mutates nothing.\n          decode: (tokens) => (tokens.length === 0 ? \"\" : hf.decode(Arr.copy(tokens))),\n        }\n      },\n      catch: (cause) => new TokenizerLoadError({ reason: \"could not load the tokenizer\", cause }),\n    }),\n  )\n\n/** Download and build in one step. For scripts; production should cache. */\nexport const layer = (\n  options: Options,\n): Layer.Layer<Tokenizer, TokenizerLoadError, HttpClient.HttpClient> =>\n  Layer.unwrap(Effect.map(download(options), fromDefinition))\n"],"mappings":";;;;;;;;;;;;;;;;;AAiBA,IAAa,qBAAb,cAAwC,OAAO,YAC7C,0CACF,CAAC,CAAC,sBAAsB;CACtB,OAAO,OAAO,SAAS,OAAO,MAAM;CACpC,QAAQ,OAAO;CACf,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CAAC,CAAC,CAAC,CAAC;;;;;AAMJ,MAAa,aAAa,OAAO,OAAO;;CAEtC,WAAW,OAAO;;CAElB,QAAQ,OAAO,SAAS,OAAO,OAAO;AACxC,CAAC;AAiBD,MAAM,aACJ,SACA,SAEA,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,OAAO,WAAW;CACjC,MAAM,MAAM,0BAA0B,QAAQ,MAAM,WAAW,QAAQ,YAAY,OAAO,GAAG;CAC7F,MAAM,UAAU,YAAoB,UAClC,IAAI,mBAAmB;EAAE,OAAO,QAAQ;EAAO;EAAQ;CAAM,CAAC;CAEhE,MAAM,UAAU,kBAAkB,IAAI,GAAG,CAAC,CAAC,KACzC,QAAQ,UAAU,KAAA,KACb,SAAS,OACV,kBAAkB,YAAY,SAAS,MAAM,QAAQ,KAAK,CAAC,CACjE;CACA,MAAM,WAAW,OAAO,OACrB,QAAQ,OAAO,CAAC,CAChB,KAAK,OAAO,SAAS,OAAO,mBAAmB,MAAM,CAAC,CAAC;CAE1D,OAAO,SAAS,UAAU,MACtB,OAAO,IAAI,mBAAmB;EAC5B,OAAO,QAAQ;EACf,QAAQ,GAAG,IAAI,YAAY,SAAS;CACtC,CAAC,IACD,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,OAAO,GAAG,KAAK,mBAAmB,CAAC,CAAC;AACpF,CAAC;;;;;AAMH,MAAa,YACX,YAEA,OAAO,IAAI;CACT,WAAW,UAAU,SAAS,gBAAgB;CAC9C,QAAQ,UAAU,SAAS,uBAAuB,CAAC,CAAC,KAAK,OAAO,oBAAoB,KAAA,CAAS,CAAC;AAChG,CAAC;;AAGH,MAAa,kBACX,eAEA,MAAM,OACJ,WACA,OAAO,IAAI;CACT,WAA6B;EAC3B,MAAM,KAAK,IAAIA,YACb,WAAW,WACV,WAAW,UAAU,CAAC,CACzB;EACA,OAAO;GACL,SAAS,SAAS,GAAG,OAAO,IAAI,CAAC,CAAC;GAIlC,SAAS,WAAY,OAAO,WAAW,IAAI,KAAK,GAAG,OAAOC,MAAI,KAAK,MAAM,CAAC;EAC5E;CACF;CACA,QAAQ,UAAU,IAAI,mBAAmB;EAAE,QAAQ;EAAgC;CAAM,CAAC;AAC5F,CAAC,CACH;;AAGF,MAAa,SACX,YAEA,MAAM,OAAO,OAAO,IAAI,SAAS,OAAO,GAAG,cAAc,CAAC"}

package/dist/Rank.d.mts

@@ -0,0 +1,22 @@
+declare namespace Rank_d_exports {
+  export { Fused, rrf };
+}
+type Fused<A> = {
+  readonly value: A;
+  readonly score: number;
+};
+/**
+ * Reciprocal rank fusion: `score(v) = Σ weightᵢ / (k + rankᵢ(v))` over
+ * 1-based ranks, where a list omitting `v` contributes nothing. `k`
+ * defaults to 60, `weights` to 1 per ranking.
+ *
+ * Items are keyed by `Map` semantics, so fuse ids rather than fresh
+ * objects. Sorted descending; ties keep first-seen order.
+ */
+declare const rrf: <A>(rankings: ReadonlyArray<ReadonlyArray<A>>, options?: {
+  readonly k?: number;
+  readonly weights?: ReadonlyArray<number>;
+}) => Array<Fused<A>>;
+//#endregion
+export { Fused, rrf, Rank_d_exports as t };
+//# sourceMappingURL=Rank.d.mts.map

package/dist/Rank.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Rank.d.mts","names":[],"sources":["../src/Rank.ts"],"mappings":";;;KAOY,MAAM;WACP,OAAO;WACP;;;;;;;;;;cAaE,MAAO,GAAC,UACT,cAAc,cAAc,KAAG;WACpB;WAAqB,UAAU;MACnD,MAAM,MAAM"}

package/dist/Rank.mjs

@@ -0,0 +1,29 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { Array, Order, pipe } from "effect";
+//#region src/Rank.ts
+/**
+* Rank fusion: merge several ranked lists of the same items into one
+* ranking by position, so retrievers on incomparable score scales
+* (BM25, cosine) can be combined without normalizing them.
+*/
+var Rank_exports = /* @__PURE__ */ __exportAll({ rrf: () => rrf });
+const byScore = Order.mapInput(Order.flip(Order.Number), (f) => f.score);
+/**
+* Reciprocal rank fusion: `score(v) = Σ weightᵢ / (k + rankᵢ(v))` over
+* 1-based ranks, where a list omitting `v` contributes nothing. `k`
+* defaults to 60, `weights` to 1 per ranking.
+*
+* Items are keyed by `Map` semantics, so fuse ids rather than fresh
+* objects. Sorted descending; ties keep first-seen order.
+*/
+const rrf = (rankings, options) => {
+	const k = options?.k ?? 60;
+	return pipe(Array.reduce(rankings, /* @__PURE__ */ new Map(), (scores, ranking, i) => Array.reduce(ranking, scores, (acc, value, rank) => acc.set(value, (acc.get(value) ?? 0) + (options?.weights?.[i] ?? 1) / (k + rank + 1)))), Array.fromIterable, Array.map(([value, score]) => ({
+		value,
+		score
+	})), Array.sort(byScore));
+};
+//#endregion
+export { rrf, Rank_exports as t };
+
+//# sourceMappingURL=Rank.mjs.map

package/dist/Rank.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"Rank.mjs","names":["Arr"],"sources":["../src/Rank.ts"],"sourcesContent":["/**\n * Rank fusion: merge several ranked lists of the same items into one\n * ranking by position, so retrievers on incomparable score scales\n * (BM25, cosine) can be combined without normalizing them.\n */\nimport { Array as Arr, Order, pipe } from \"effect\"\n\nexport type Fused<A> = {\n  readonly value: A\n  readonly score: number\n}\n\nconst byScore = Order.mapInput(Order.flip(Order.Number), (f: { readonly score: number }) => f.score)\n\n/**\n * Reciprocal rank fusion: `score(v) = Σ weightᵢ / (k + rankᵢ(v))` over\n * 1-based ranks, where a list omitting `v` contributes nothing. `k`\n * defaults to 60, `weights` to 1 per ranking.\n *\n * Items are keyed by `Map` semantics, so fuse ids rather than fresh\n * objects. Sorted descending; ties keep first-seen order.\n */\nexport const rrf = <A>(\n  rankings: ReadonlyArray<ReadonlyArray<A>>,\n  options?: { readonly k?: number; readonly weights?: ReadonlyArray<number> },\n): Array<Fused<A>> => {\n  const k = options?.k ?? 60\n  return pipe(\n    Arr.reduce(rankings, new Map<A, number>(), (scores, ranking, i) =>\n      Arr.reduce(ranking, scores, (acc, value, rank) =>\n        acc.set(value, (acc.get(value) ?? 0) + (options?.weights?.[i] ?? 1) / (k + rank + 1)),\n      ),\n    ),\n    Arr.fromIterable,\n    Arr.map(([value, score]): Fused<A> => ({ value, score })),\n    Arr.sort(byScore),\n  )\n}\n"],"mappings":";;;;;;;;;AAYA,MAAM,UAAU,MAAM,SAAS,MAAM,KAAK,MAAM,MAAM,IAAI,MAAkC,EAAE,KAAK;;;;;;;;;AAUnG,MAAa,OACX,UACA,YACoB;CACpB,MAAM,IAAI,SAAS,KAAK;CACxB,OAAO,KACLA,MAAI,OAAO,0BAAU,IAAI,IAAe,IAAI,QAAQ,SAAS,MAC3DA,MAAI,OAAO,SAAS,SAAS,KAAK,OAAO,SACvC,IAAI,IAAI,QAAQ,IAAI,IAAI,KAAK,KAAK,MAAM,SAAS,UAAU,MAAM,MAAM,IAAI,OAAO,EAAE,CACtF,CACF,GACAA,MAAI,cACJA,MAAI,KAAK,CAAC,OAAO,YAAsB;EAAE;EAAO;CAAM,EAAE,GACxDA,MAAI,KAAK,OAAO,CAClB;AACF"}

package/dist/index.d.mts

@@ -0,0 +1,3 @@
+import { t as Chunking_d_exports } from "./Chunking.mjs";
+import { t as Rank_d_exports } from "./Rank.mjs";
+export { Chunking_d_exports as Chunking, Rank_d_exports as Rank };

package/dist/index.mjs

@@ -0,0 +1,3 @@
+import { t as Chunking_exports } from "./Chunking.mjs";
+import { t as Rank_exports } from "./Rank.mjs";
+export { Chunking_exports as Chunking, Rank_exports as Rank };

package/dist/rolldown-runtime-D7D4PA-g.mjs

@@ -0,0 +1,13 @@
+//#region \0rolldown/runtime.js
+var __defProp = Object.defineProperty;
+var __exportAll = (all, no_symbols) => {
+	let target = {};
+	for (var name in all) __defProp(target, name, {
+		get: all[name],
+		enumerable: true
+	});
+	if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
+	return target;
+};
+//#endregion
+export { __exportAll as t };