@effect-uai/retrieval 0.17.0 → 0.19.0-dev-20261109183959
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -31,7 +31,7 @@ The tokenizer needs one optional peer:
|
|
|
31
31
|
pnpm add @huggingface/tokenizers
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
-
ESM-only. Requires `effect
|
|
34
|
+
ESM-only. Requires `effect@>=4.0.1` (Effect 4.0 stable) as a peer.
|
|
35
35
|
|
|
36
36
|
## Usage
|
|
37
37
|
|
|
@@ -52,11 +52,11 @@ const fused = Rank.rrf([keywordIds, vectorIds], { weights: [1, 2] })
|
|
|
52
52
|
|
|
53
53
|
## Docs
|
|
54
54
|
|
|
55
|
-
Full docs: <https://effect-uai.
|
|
55
|
+
Full docs: <https://effect-uai.com/retrieval/>
|
|
56
56
|
|
|
57
|
-
See [Chunking](https://effect-uai.
|
|
58
|
-
[Tokenizers](https://effect-uai.
|
|
59
|
-
[Agentic search](https://effect-uai.
|
|
57
|
+
See [Chunking](https://effect-uai.com/retrieval/chunking/),
|
|
58
|
+
[Tokenizers](https://effect-uai.com/language-models/tokenizers/), and
|
|
59
|
+
[Agentic search](https://effect-uai.com/recipes/agentic-search/) for a
|
|
60
60
|
pipeline using all of it.
|
|
61
61
|
|
|
62
62
|
## License
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Effect, Layer, Redacted, Schema } from "effect";
|
|
2
2
|
import { Tokenizer } from "@effect-uai/core/Tokenizer";
|
|
3
|
-
import { HttpClient } from "effect/
|
|
3
|
+
import { HttpClient } from "effect/http";
|
|
4
4
|
//#region src/HuggingFaceTokenizer.d.ts
|
|
5
5
|
declare const TokenizerLoadError_base: Schema.Class<TokenizerLoadError, Schema.TaggedStruct<"TokenizerLoadError", {
|
|
6
6
|
readonly model: Schema.optional<Schema.String>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Array, Effect, Layer, Redacted, Schema } from "effect";
|
|
2
2
|
import { Tokenizer } from "@effect-uai/core/Tokenizer";
|
|
3
|
-
import { HttpClient, HttpClientRequest } from "effect/
|
|
3
|
+
import { HttpClient, HttpClientRequest } from "effect/http";
|
|
4
4
|
import { Tokenizer as Tokenizer$1 } from "@huggingface/tokenizers";
|
|
5
5
|
//#region src/HuggingFaceTokenizer.ts
|
|
6
6
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HuggingFaceTokenizer.mjs","names":["HfTokenizer","Arr"],"sources":["../src/HuggingFaceTokenizer.ts"],"sourcesContent":["/**\n * The core `Tokenizer` tag over `@huggingface/tokenizers`, which reads a\n * model's `tokenizer.json` and needs no native build. The package is an\n * optional peer dependency: `pnpm add @huggingface/tokenizers`.\n *\n * Fetching and construction are separate on purpose. `download` gives you\n * plain JSON to keep wherever your application keeps things; `fromDefinition`\n * builds the tokenizer from what you kept. Nothing here touches the filesystem\n * or caches behind your back. `layer` composes the two for scripts happy to\n * download on every run.\n */\nimport { Array as Arr, Effect, Layer, Redacted, Schema } from \"effect\"\nimport { HttpClient, HttpClientRequest } from \"effect/
|
|
1
|
+
{"version":3,"file":"HuggingFaceTokenizer.mjs","names":["HfTokenizer","Arr"],"sources":["../src/HuggingFaceTokenizer.ts"],"sourcesContent":["/**\n * The core `Tokenizer` tag over `@huggingface/tokenizers`, which reads a\n * model's `tokenizer.json` and needs no native build. The package is an\n * optional peer dependency: `pnpm add @huggingface/tokenizers`.\n *\n * Fetching and construction are separate on purpose. `download` gives you\n * plain JSON to keep wherever your application keeps things; `fromDefinition`\n * builds the tokenizer from what you kept. Nothing here touches the filesystem\n * or caches behind your back. `layer` composes the two for scripts happy to\n * download on every run.\n */\nimport { Array as Arr, Effect, Layer, Redacted, Schema } from \"effect\"\nimport { HttpClient, HttpClientRequest } from \"effect/http\"\nimport { Tokenizer, type TokenizerService } from \"@effect-uai/core/Tokenizer\"\nimport { Tokenizer as HfTokenizer } from \"@huggingface/tokenizers\"\n\n/** The vocabulary could not be fetched, parsed, or loaded. */\nexport class TokenizerLoadError extends Schema.TaggedError<TokenizerLoadError>(\n \"@effect-uai/retrieval/TokenizerLoadError\",\n)(\"TokenizerLoadError\", {\n model: Schema.optional(Schema.String),\n reason: Schema.String,\n cause: Schema.optional(Schema.Unknown),\n}) {}\n\n/**\n * Everything that defines a tokenizer's behaviour, as published. Plain JSON:\n * write it to disk, a row, or your bundle, and decode it back with this schema.\n */\nexport const Definition = Schema.Struct({\n /** Contents of `tokenizer.json`. */\n tokenizer: Schema.Unknown,\n /** Contents of `tokenizer_config.json`, absent for repos that publish none. */\n config: Schema.optional(Schema.Unknown),\n})\n\nexport type Definition = typeof Definition.Type\n\nexport type Options = {\n /** A Hugging Face repo id, e.g. `\"jinaai/jina-embeddings-v3\"` or `\"Xenova/gpt-4o\"`. */\n readonly model: string\n /** Branch, tag, or commit. Default `\"main\"`. */\n readonly revision?: string\n /**\n * A Hugging Face access token. Required for gated repos, where the Hub\n * serves the files only to accounts that have accepted the model's terms\n * (Google's tokenizers among them), and for private ones.\n */\n readonly token?: Redacted.Redacted\n}\n\nconst fetchJson = (\n options: Options,\n file: string,\n): Effect.Effect<unknown, TokenizerLoadError, HttpClient.HttpClient> =>\n Effect.gen(function* () {\n const client = yield* HttpClient.HttpClient\n const url = `https://huggingface.co/${options.model}/resolve/${options.revision ?? \"main\"}/${file}`\n const failed = (reason: string) => (cause: unknown) =>\n new TokenizerLoadError({ model: options.model, reason, cause })\n\n const request = HttpClientRequest.get(url).pipe(\n options.token === undefined\n ? (self) => self\n : HttpClientRequest.bearerToken(Redacted.value(options.token)),\n )\n const response = yield* client\n .execute(request)\n .pipe(Effect.mapError(failed(`could not fetch ${file}`)))\n // 401/403 on a gated repo means no token, or terms not yet accepted.\n return response.status >= 400\n ? yield* new TokenizerLoadError({\n model: options.model,\n reason: `${url} returned ${response.status}`,\n })\n : yield* response.json.pipe(Effect.mapError(failed(`${file} is not valid JSON`)))\n })\n\n/**\n * Fetch a model's definition from the Hugging Face Hub. Cache the result\n * yourself; a repo without a `tokenizer_config.json` yields `config: undefined`.\n */\nexport const download = (\n options: Options,\n): Effect.Effect<Definition, TokenizerLoadError, HttpClient.HttpClient> =>\n Effect.all({\n tokenizer: fetchJson(options, \"tokenizer.json\"),\n config: fetchJson(options, \"tokenizer_config.json\").pipe(Effect.orElseSucceed(() => undefined)),\n })\n\n/** Build the tokenizer from a definition you already hold. No IO. */\nexport const fromDefinition = (\n definition: Definition,\n): Layer.Layer<Tokenizer, TokenizerLoadError> =>\n Layer.effect(\n Tokenizer,\n Effect.try({\n try: (): TokenizerService => {\n const hf = new HfTokenizer(\n definition.tokenizer as object,\n (definition.config ?? {}) as object,\n )\n return {\n encode: (text) => hf.encode(text).ids,\n // The library rejects an empty array, but `encode(\"\")` returns one and\n // `decode` inverts `encode`. Its signature also wants a mutable array,\n // though it mutates nothing.\n decode: (tokens) => (tokens.length === 0 ? \"\" : hf.decode(Arr.copy(tokens))),\n }\n },\n catch: (cause) => new TokenizerLoadError({ reason: \"could not load the tokenizer\", cause }),\n }),\n )\n\n/** Download and build in one step. For scripts; production should cache. */\nexport const layer = (\n options: Options,\n): Layer.Layer<Tokenizer, TokenizerLoadError, HttpClient.HttpClient> =>\n Layer.unwrap(Effect.map(download(options), fromDefinition))\n"],"mappings":";;;;;;;;;;;;;;;;;AAiBA,IAAa,qBAAb,cAAwC,OAAO,YAC7C,0CACF,CAAC,CAAC,sBAAsB;CACtB,OAAO,OAAO,SAAS,OAAO,MAAM;CACpC,QAAQ,OAAO;CACf,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CAAC,CAAC,CAAC,CAAC;;;;;AAMJ,MAAa,aAAa,OAAO,OAAO;;CAEtC,WAAW,OAAO;;CAElB,QAAQ,OAAO,SAAS,OAAO,OAAO;AACxC,CAAC;AAiBD,MAAM,aACJ,SACA,SAEA,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,OAAO,WAAW;CACjC,MAAM,MAAM,0BAA0B,QAAQ,MAAM,WAAW,QAAQ,YAAY,OAAO,GAAG;CAC7F,MAAM,UAAU,YAAoB,UAClC,IAAI,mBAAmB;EAAE,OAAO,QAAQ;EAAO;EAAQ;CAAM,CAAC;CAEhE,MAAM,UAAU,kBAAkB,IAAI,GAAG,CAAC,CAAC,KACzC,QAAQ,UAAU,KAAA,KACb,SAAS,OACV,kBAAkB,YAAY,SAAS,MAAM,QAAQ,KAAK,CAAC,CACjE;CACA,MAAM,WAAW,OAAO,OACrB,QAAQ,OAAO,CAAC,CAChB,KAAK,OAAO,SAAS,OAAO,mBAAmB,MAAM,CAAC,CAAC;CAE1D,OAAO,SAAS,UAAU,MACtB,OAAO,IAAI,mBAAmB;EAC5B,OAAO,QAAQ;EACf,QAAQ,GAAG,IAAI,YAAY,SAAS;CACtC,CAAC,IACD,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,OAAO,GAAG,KAAK,mBAAmB,CAAC,CAAC;AACpF,CAAC;;;;;AAMH,MAAa,YACX,YAEA,OAAO,IAAI;CACT,WAAW,UAAU,SAAS,gBAAgB;CAC9C,QAAQ,UAAU,SAAS,uBAAuB,CAAC,CAAC,KAAK,OAAO,oBAAoB,KAAA,CAAS,CAAC;AAChG,CAAC;;AAGH,MAAa,kBACX,eAEA,MAAM,OACJ,WACA,OAAO,IAAI;CACT,WAA6B;EAC3B,MAAM,KAAK,IAAIA,YACb,WAAW,WACV,WAAW,UAAU,CAAC,CACzB;EACA,OAAO;GACL,SAAS,SAAS,GAAG,OAAO,IAAI,CAAC,CAAC;GAIlC,SAAS,WAAY,OAAO,WAAW,IAAI,KAAK,GAAG,OAAOC,MAAI,KAAK,MAAM,CAAC;EAC5E;CACF;CACA,QAAQ,UAAU,IAAI,mBAAmB;EAAE,QAAQ;EAAgC;CAAM,CAAC;AAC5F,CAAC,CACH;;AAGF,MAAa,SACX,YAEA,MAAM,OAAO,OAAO,IAAI,SAAS,OAAO,GAAG,cAAc,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@effect-uai/retrieval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0-dev-20261109183959",
|
|
4
4
|
"description": "Retrieval-pipeline utilities for @effect-uai/core: text chunking, rank fusion, and a Hugging Face tokenizer layer.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai",
|
|
@@ -52,16 +52,16 @@
|
|
|
52
52
|
"access": "public"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
55
|
-
"@effect/vitest": "4.
|
|
55
|
+
"@effect/vitest": "4.1.2",
|
|
56
56
|
"@huggingface/tokenizers": "^0.1.3",
|
|
57
|
-
"effect": "4.
|
|
57
|
+
"effect": "4.1.2",
|
|
58
58
|
"typescript": "^7.0.2",
|
|
59
|
-
"@effect-uai/core": "0.
|
|
59
|
+
"@effect-uai/core": "0.19.0-dev-20261109183959"
|
|
60
60
|
},
|
|
61
61
|
"peerDependencies": {
|
|
62
|
-
"@effect-uai/core": "
|
|
62
|
+
"@effect-uai/core": "0.19.0-dev-20261109183959",
|
|
63
63
|
"@huggingface/tokenizers": "^0.1.0",
|
|
64
|
-
"effect": ">=4.0.
|
|
64
|
+
"effect": ">=4.0.1 <5.0.0"
|
|
65
65
|
},
|
|
66
66
|
"peerDependenciesMeta": {
|
|
67
67
|
"@huggingface/tokenizers": {
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* download on every run.
|
|
11
11
|
*/
|
|
12
12
|
import { Array as Arr, Effect, Layer, Redacted, Schema } from "effect"
|
|
13
|
-
import { HttpClient, HttpClientRequest } from "effect/
|
|
13
|
+
import { HttpClient, HttpClientRequest } from "effect/http"
|
|
14
14
|
import { Tokenizer, type TokenizerService } from "@effect-uai/core/Tokenizer"
|
|
15
15
|
import { Tokenizer as HfTokenizer } from "@huggingface/tokenizers"
|
|
16
16
|
|