@effect-uai/core 0.12.1 → 0.13.0

package/dist/chunker/Chunker.d.mts

@@ -0,0 +1,32 @@
+import { t as AiError } from "../AiError-eLV9t1lU.mjs";
+import { Context, Effect } from "effect";
+declare namespace Chunker_d_exports {
+  export { Chunk, Chunker, ChunkerService, chunk };
+}
+/**
+ * A passage of a document, with the character offsets it came from.
+ *
+ * Implementor contract: `text` is the input's `[start, end)` slice verbatim,
+ * so a retrieved passage can be traced back to its source and enriched with
+ * what surrounds it. Chunks may overlap.
+ */
+type Chunk = {
+  readonly text: string;
+  readonly start: number;
+  readonly end: number;
+};
+/**
+ * Splitting a document into retrievable passages. Strategy and sizing are
+ * fixed when the layer is built, so callers stay portable across a local
+ * splitter and a hosted chunking service.
+ */
+type ChunkerService = {
+  readonly chunk: (text: string) => Effect.Effect<ReadonlyArray<Chunk>, AiError>;
+};
+declare const Chunker_base: Context.ServiceClass<Chunker, "@betalyra/effect-uai/Chunker", ChunkerService>;
+declare class Chunker extends Chunker_base {}
+/** Split a document into passages. */
+declare const chunk: (text: string) => Effect.Effect<ReadonlyArray<Chunk>, AiError, Chunker>;
+//#endregion
+export { Chunk, Chunker, ChunkerService, chunk, Chunker_d_exports as t };
+//# sourceMappingURL=Chunker.d.mts.map

package/dist/chunker/Chunker.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Chunker.d.mts","names":[],"sources":["../../src/chunker/Chunker.ts"],"mappings":";;;;;;;;;;;;KAUY;WACD;WACA;WACA;;;;;;;KAQC;WACD,QAAQ,iBAAiB,OAAO,OAAO,cAAc,QAAQ;;;cAG3D,gBAAgB;;cAKhB,QAAK,iBAEf,OAAO,OAAO,cAAc,QAAQ,SAAiB"}

package/dist/chunker/Chunker.mjs

@@ -0,0 +1,14 @@
+import { n as __exportAll } from "../rolldown-runtime-EAsCxrDo.mjs";
+import { Context, Effect } from "effect";
+//#region src/chunker/Chunker.ts
+var Chunker_exports = /* @__PURE__ */ __exportAll({
+	Chunker: () => Chunker,
+	chunk: () => chunk
+});
+var Chunker = class extends Context.Service()("@betalyra/effect-uai/Chunker") {};
+/** Split a document into passages. */
+const chunk = (text) => Effect.flatMap(Chunker, (chunker) => chunker.chunk(text));
+//#endregion
+export { Chunker, chunk, Chunker_exports as t };
+
+//# sourceMappingURL=Chunker.mjs.map

package/dist/chunker/Chunker.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"Chunker.mjs","names":[],"sources":["../../src/chunker/Chunker.ts"],"sourcesContent":["import { Context, Effect } from \"effect\"\nimport type * as AiError from \"../domain/AiError.js\"\n\n/**\n * A passage of a document, with the character offsets it came from.\n *\n * Implementor contract: `text` is the input's `[start, end)` slice verbatim,\n * so a retrieved passage can be traced back to its source and enriched with\n * what surrounds it. Chunks may overlap.\n */\nexport type Chunk = {\n  readonly text: string\n  readonly start: number\n  readonly end: number\n}\n\n/**\n * Splitting a document into retrievable passages. Strategy and sizing are\n * fixed when the layer is built, so callers stay portable across a local\n * splitter and a hosted chunking service.\n */\nexport type ChunkerService = {\n  readonly chunk: (text: string) => Effect.Effect<ReadonlyArray<Chunk>, AiError.AiError>\n}\n\nexport class Chunker extends Context.Service<Chunker, ChunkerService>()(\n  \"@betalyra/effect-uai/Chunker\",\n) {}\n\n/** Split a document into passages. */\nexport const chunk = (\n  text: string,\n): Effect.Effect<ReadonlyArray<Chunk>, AiError.AiError, Chunker> =>\n  Effect.flatMap(Chunker, (chunker) => chunker.chunk(text))\n"],"mappings":";;;;;;;AAyBA,IAAa,UAAb,cAA6B,QAAQ,QAAiC,CAAC,CACrE,8BACF,CAAC,CAAC,CAAC;;AAGH,MAAa,SACX,SAEA,OAAO,QAAQ,UAAU,YAAY,QAAQ,MAAM,IAAI,CAAC"}

package/dist/index.d.mts

@@ -7,6 +7,7 @@ import { _ as Tool_d_exports } from "./Tool-DE-rnvkw.mjs";
 import { t as BrowserTool_d_exports } from "./browser/BrowserTool.mjs";
 import { t as Capabilities_d_exports } from "./capabilities/Capabilities.mjs";
 import { n as AiError_d_exports } from "./AiError-eLV9t1lU.mjs";
+import { t as Chunker_d_exports } from "./chunker/Chunker.mjs";
 import { o as Audio_d_exports } from "./Audio-60lKKHxS.mjs";
 import { t as Citation_d_exports } from "./domain/Citation.mjs";
 import { t as Music_d_exports } from "./domain/Music.mjs";
@@ -24,6 +25,7 @@ import { n as ToolEvent_d_exports } from "./ToolEvent-D99MGQYH.mjs";
 import { l as Toolkit_d_exports } from "./Toolkit-BEZrNPTI.mjs";
 import { t as LanguageModel_d_exports } from "./language-model/LanguageModel.mjs";
 import { t as MusicGenerator_d_exports } from "./music-generator/MusicGenerator.mjs";
+import { t as Reranker_d_exports } from "./reranker/Reranker.mjs";
 import { t as Memory_d_exports } from "./sandbox/Memory.mjs";
 import { o as SandboxError_d_exports } from "./SandboxError-BN8b28Bh.mjs";
 import { t as Sandbox_d_exports } from "./sandbox/Sandbox.mjs";
@@ -36,6 +38,7 @@ import { t as WebReadTool_d_exports } from "./web-read/WebReadTool.mjs";
 import { t as WebSearch_d_exports } from "./web-search/WebSearch.mjs";
 import { t as WebSearchTool_d_exports } from "./web-search/WebSearchTool.mjs";
 import { t as Vector_d_exports } from "./math/Vector.mjs";
+import { t as Tokenizer_d_exports } from "./tokenizer/Tokenizer.mjs";
 import { t as Retry_d_exports } from "./retry/Retry.mjs";
 import { t as Approval_d_exports } from "./tool/Approval.mjs";
 import { t as HistoryCheck_d_exports } from "./tool/HistoryCheck.mjs";
@@ -44,4 +47,4 @@ import { t as Lines_d_exports } from "./streaming/Lines.mjs";
 import { t as SSE_d_exports } from "./streaming/SSE.mjs";
 import { t as Metrics_d_exports } from "./observability/Metrics.mjs";
 import { t as Telemetry_d_exports } from "./observability/Telemetry.mjs";
-export { AiError_d_exports as AiError, Approval_d_exports as Approval, Audio_d_exports as Audio, Browser_d_exports as Browser, BrowserError_d_exports as BrowserError, BrowserTool_d_exports as BrowserTool, Capabilities_d_exports as Capabilities, Citation_d_exports as Citation, DeepResearch_d_exports as DeepResearch, Embedding_d_exports as Embedding, EmbeddingModel_d_exports as EmbeddingModel, HistoryCheck_d_exports as HistoryCheck, Image_d_exports as Image, Items_d_exports as Items, JSONL_d_exports as JSONL, Job_d_exports as Job, LanguageModel_d_exports as LanguageModel, Lines_d_exports as Lines, Loop_d_exports as Loop, Media_d_exports as Media, Memory_d_exports as Memory, Metrics_d_exports as Metrics, Music_d_exports as Music, MusicGenerator_d_exports as MusicGenerator, Research_d_exports as Research, Retry_d_exports as Retry, SSE_d_exports as SSE, Sandbox_d_exports as Sandbox, SandboxError_d_exports as SandboxError, SandboxImage_d_exports as SandboxImage, SandboxNetwork_d_exports as SandboxNetwork, SpeechSynthesizer_d_exports as SpeechSynthesizer, StructuredFormat_d_exports as StructuredFormat, Telemetry_d_exports as Telemetry, Tool_d_exports as Tool, ToolEvent_d_exports as ToolEvent, ToolResult_d_exports as ToolResult, Toolkit_d_exports as Toolkit, Transcriber_d_exports as Transcriber, Transcript_d_exports as Transcript, Turn_d_exports as Turn, Vector_d_exports as Vector, WebRead_d_exports as WebRead, WebReadTool_d_exports as WebReadTool, WebSearch_d_exports as WebSearch, WebSearchTool_d_exports as WebSearchTool };
+export { AiError_d_exports as AiError, Approval_d_exports as Approval, Audio_d_exports as Audio, Browser_d_exports as Browser, BrowserError_d_exports as BrowserError, BrowserTool_d_exports as BrowserTool, Capabilities_d_exports as Capabilities, Chunker_d_exports as Chunker, Citation_d_exports as Citation, DeepResearch_d_exports as DeepResearch, Embedding_d_exports as Embedding, EmbeddingModel_d_exports as EmbeddingModel, HistoryCheck_d_exports as HistoryCheck, Image_d_exports as Image, Items_d_exports as Items, JSONL_d_exports as JSONL, Job_d_exports as Job, LanguageModel_d_exports as LanguageModel, Lines_d_exports as Lines, Loop_d_exports as Loop, Media_d_exports as Media, Memory_d_exports as Memory, Metrics_d_exports as Metrics, Music_d_exports as Music, MusicGenerator_d_exports as MusicGenerator, Reranker_d_exports as Reranker, Research_d_exports as Research, Retry_d_exports as Retry, SSE_d_exports as SSE, Sandbox_d_exports as Sandbox, SandboxError_d_exports as SandboxError, SandboxImage_d_exports as SandboxImage, SandboxNetwork_d_exports as SandboxNetwork, SpeechSynthesizer_d_exports as SpeechSynthesizer, StructuredFormat_d_exports as StructuredFormat, Telemetry_d_exports as Telemetry, Tokenizer_d_exports as Tokenizer, Tool_d_exports as Tool, ToolEvent_d_exports as ToolEvent, ToolResult_d_exports as ToolResult, Toolkit_d_exports as Toolkit, Transcriber_d_exports as Transcriber, Transcript_d_exports as Transcript, Turn_d_exports as Turn, Vector_d_exports as Vector, WebRead_d_exports as WebRead, WebReadTool_d_exports as WebReadTool, WebSearch_d_exports as WebSearch, WebSearchTool_d_exports as WebSearchTool };

package/dist/index.mjs

@@ -13,6 +13,7 @@ import { t as ToolEvent_exports } from "./tool/ToolEvent.mjs";
 import { t as Toolkit_exports } from "./tool/Toolkit.mjs";
 import { t as BrowserTool_exports } from "./browser/BrowserTool.mjs";
 import { t as Capabilities_exports } from "./capabilities/Capabilities.mjs";
+import { t as Chunker_exports } from "./chunker/Chunker.mjs";
 import { t as Citation_exports } from "./domain/Citation.mjs";
 import { t as Job_exports } from "./job/Job.mjs";
 import { t as DeepResearch_exports } from "./research/DeepResearch.mjs";
@@ -24,6 +25,7 @@ import { t as Embedding_exports } from "./embedding-model/Embedding.mjs";
 import { t as EmbeddingModel_exports } from "./embedding-model/EmbeddingModel.mjs";
 import { t as LanguageModel_exports } from "./language-model/LanguageModel.mjs";
 import { t as MusicGenerator_exports } from "./music-generator/MusicGenerator.mjs";
+import { t as Reranker_exports } from "./reranker/Reranker.mjs";
 import { t as Memory_exports } from "./sandbox/Memory.mjs";
 import { t as Sandbox_exports } from "./sandbox/Sandbox.mjs";
 import { t as SandboxError_exports } from "./sandbox/SandboxError.mjs";
@@ -36,6 +38,7 @@ import { t as WebReadTool_exports } from "./web-read/WebReadTool.mjs";
 import { t as WebSearch_exports } from "./web-search/WebSearch.mjs";
 import { t as WebSearchTool_exports } from "./web-search/WebSearchTool.mjs";
 import { t as Vector_exports } from "./math/Vector.mjs";
+import { t as Tokenizer_exports } from "./tokenizer/Tokenizer.mjs";
 import { t as Retry_exports } from "./retry/Retry.mjs";
 import { t as Approval_exports } from "./tool/Approval.mjs";
 import { t as HistoryCheck_exports } from "./tool/HistoryCheck.mjs";
@@ -44,4 +47,4 @@ import { t as Lines_exports } from "./streaming/Lines.mjs";
 import { t as SSE_exports } from "./streaming/SSE.mjs";
 import { t as Metrics_exports } from "./observability/Metrics.mjs";
 import { t as Telemetry_exports } from "./observability/Telemetry.mjs";
-export { AiError_exports as AiError, Approval_exports as Approval, Audio_exports as Audio, Browser_exports as Browser, BrowserError_exports as BrowserError, BrowserTool_exports as BrowserTool, Capabilities_exports as Capabilities, Citation_exports as Citation, DeepResearch_exports as DeepResearch, Embedding_exports as Embedding, EmbeddingModel_exports as EmbeddingModel, HistoryCheck_exports as HistoryCheck, Image_exports as Image, Items_exports as Items, JSONL_exports as JSONL, Job_exports as Job, LanguageModel_exports as LanguageModel, Lines_exports as Lines, Loop_exports as Loop, Media_exports as Media, Memory_exports as Memory, Metrics_exports as Metrics, Music_exports as Music, MusicGenerator_exports as MusicGenerator, Research_exports as Research, Retry_exports as Retry, SSE_exports as SSE, Sandbox_exports as Sandbox, SandboxError_exports as SandboxError, SandboxImage_exports as SandboxImage, SandboxNetwork_exports as SandboxNetwork, SpeechSynthesizer_exports as SpeechSynthesizer, StructuredFormat_exports as StructuredFormat, Telemetry_exports as Telemetry, Tool_exports as Tool, ToolEvent_exports as ToolEvent, ToolResult_exports as ToolResult, Toolkit_exports as Toolkit, Transcriber_exports as Transcriber, Transcript_exports as Transcript, Turn_exports as Turn, Vector_exports as Vector, WebRead_exports as WebRead, WebReadTool_exports as WebReadTool, WebSearch_exports as WebSearch, WebSearchTool_exports as WebSearchTool };
+export { AiError_exports as AiError, Approval_exports as Approval, Audio_exports as Audio, Browser_exports as Browser, BrowserError_exports as BrowserError, BrowserTool_exports as BrowserTool, Capabilities_exports as Capabilities, Chunker_exports as Chunker, Citation_exports as Citation, DeepResearch_exports as DeepResearch, Embedding_exports as Embedding, EmbeddingModel_exports as EmbeddingModel, HistoryCheck_exports as HistoryCheck, Image_exports as Image, Items_exports as Items, JSONL_exports as JSONL, Job_exports as Job, LanguageModel_exports as LanguageModel, Lines_exports as Lines, Loop_exports as Loop, Media_exports as Media, Memory_exports as Memory, Metrics_exports as Metrics, Music_exports as Music, MusicGenerator_exports as MusicGenerator, Reranker_exports as Reranker, Research_exports as Research, Retry_exports as Retry, SSE_exports as SSE, Sandbox_exports as Sandbox, SandboxError_exports as SandboxError, SandboxImage_exports as SandboxImage, SandboxNetwork_exports as SandboxNetwork, SpeechSynthesizer_exports as SpeechSynthesizer, StructuredFormat_exports as StructuredFormat, Telemetry_exports as Telemetry, Tokenizer_exports as Tokenizer, Tool_exports as Tool, ToolEvent_exports as ToolEvent, ToolResult_exports as ToolResult, Toolkit_exports as Toolkit, Transcriber_exports as Transcriber, Transcript_exports as Transcript, Turn_exports as Turn, Vector_exports as Vector, WebRead_exports as WebRead, WebReadTool_exports as WebReadTool, WebSearch_exports as WebSearch, WebSearchTool_exports as WebSearchTool };

package/dist/reranker/Reranker.d.mts

@@ -0,0 +1,46 @@
+import { t as AiError } from "../AiError-eLV9t1lU.mjs";
+import { Context, Effect } from "effect";
+declare namespace Reranker_d_exports {
+  export { CommonRerankRequest, RerankResponse, RerankResult, RerankUsage, Reranker, RerankerService, rerank };
+}
+/**
+ * Cross-provider rerank request. Vendor knobs (multimodal documents,
+ * instruction fields, truncation) live on the provider's typed request.
+ */
+type CommonRerankRequest = {
+  readonly query: string;
+  readonly documents: ReadonlyArray<string>;
+  /** Each provider narrows this to its typed literal union. */
+  readonly model: string;
+  /** Keep only the top N results. Default: all documents. */
+  readonly topN?: number;
+};
+type RerankResult = {
+  /** Position in the request's `documents`. */
+  readonly index: number;
+  readonly score: number;
+};
+/** Optional throughout: some providers bill per search unit, not per token. */
+type RerankUsage = {
+  readonly totalTokens?: number;
+};
+/**
+ * Score contract: `results` is sorted descending, higher is better. Scores
+ * order candidates within one call; no range or calibration is promised and
+ * they are not comparable across requests. Implementors must sort descending
+ * if the wire does not.
+ */
+type RerankResponse = {
+  readonly results: ReadonlyArray<RerankResult>;
+  readonly usage: RerankUsage;
+};
+type RerankerService = {
+  readonly rerank: (request: CommonRerankRequest) => Effect.Effect<RerankResponse, AiError>;
+};
+declare const Reranker_base: Context.ServiceClass<Reranker, "@betalyra/effect-uai/Reranker", RerankerService>;
+declare class Reranker extends Reranker_base {}
+/** Score a candidate set against a query, best first. */
+declare const rerank: (request: CommonRerankRequest) => Effect.Effect<RerankResponse, AiError, Reranker>;
+//#endregion
+export { CommonRerankRequest, RerankResponse, RerankResult, RerankUsage, Reranker, RerankerService, rerank, Reranker_d_exports as t };
+//# sourceMappingURL=Reranker.d.mts.map

package/dist/reranker/Reranker.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Reranker.d.mts","names":[],"sources":["../../src/reranker/Reranker.ts"],"mappings":";;;;;;;;;KAOY;WACD;WACA,WAAW;;WAEX;;WAEA;;KAGC;;WAED;WACA;;;KAIC;WACD;;;;;;;;KASC;WACD,SAAS,cAAc;WACvB,OAAO;;KAGN;WACD,SAAS,SAAS,wBAAwB,OAAO,OAAO,gBAAgB;;;cAGtE,iBAAiB;;cAKjB,SAAM,SACR,wBACR,OAAO,OAAO,gBAAgB,SAAiB"}

package/dist/reranker/Reranker.mjs

@@ -0,0 +1,14 @@
+import { n as __exportAll } from "../rolldown-runtime-EAsCxrDo.mjs";
+import { Context, Effect } from "effect";
+//#region src/reranker/Reranker.ts
+var Reranker_exports = /* @__PURE__ */ __exportAll({
+	Reranker: () => Reranker,
+	rerank: () => rerank
+});
+var Reranker = class extends Context.Service()("@betalyra/effect-uai/Reranker") {};
+/** Score a candidate set against a query, best first. */
+const rerank = (request) => Effect.flatMap(Reranker, (r) => r.rerank(request));
+//#endregion
+export { Reranker, rerank, Reranker_exports as t };
+
+//# sourceMappingURL=Reranker.mjs.map

package/dist/reranker/Reranker.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"Reranker.mjs","names":[],"sources":["../../src/reranker/Reranker.ts"],"sourcesContent":["import { Context, Effect } from \"effect\"\nimport type * as AiError from \"../domain/AiError.js\"\n\n/**\n * Cross-provider rerank request. Vendor knobs (multimodal documents,\n * instruction fields, truncation) live on the provider's typed request.\n */\nexport type CommonRerankRequest = {\n  readonly query: string\n  readonly documents: ReadonlyArray<string>\n  /** Each provider narrows this to its typed literal union. */\n  readonly model: string\n  /** Keep only the top N results. Default: all documents. */\n  readonly topN?: number\n}\n\nexport type RerankResult = {\n  /** Position in the request's `documents`. */\n  readonly index: number\n  readonly score: number\n}\n\n/** Optional throughout: some providers bill per search unit, not per token. */\nexport type RerankUsage = {\n  readonly totalTokens?: number\n}\n\n/**\n * Score contract: `results` is sorted descending, higher is better. Scores\n * order candidates within one call; no range or calibration is promised and\n * they are not comparable across requests. Implementors must sort descending\n * if the wire does not.\n */\nexport type RerankResponse = {\n  readonly results: ReadonlyArray<RerankResult>\n  readonly usage: RerankUsage\n}\n\nexport type RerankerService = {\n  readonly rerank: (request: CommonRerankRequest) => Effect.Effect<RerankResponse, AiError.AiError>\n}\n\nexport class Reranker extends Context.Service<Reranker, RerankerService>()(\n  \"@betalyra/effect-uai/Reranker\",\n) {}\n\n/** Score a candidate set against a query, best first. */\nexport const rerank = (\n  request: CommonRerankRequest,\n): Effect.Effect<RerankResponse, AiError.AiError, Reranker> =>\n  Effect.flatMap(Reranker, (r) => r.rerank(request))\n"],"mappings":";;;;;;;AA0CA,IAAa,WAAb,cAA8B,QAAQ,QAAmC,CAAC,CACxE,+BACF,CAAC,CAAC,CAAC;;AAGH,MAAa,UACX,YAEA,OAAO,QAAQ,WAAW,MAAM,EAAE,OAAO,OAAO,CAAC"}

package/dist/tokenizer/Tokenizer.d.mts

@@ -0,0 +1,22 @@
+import { Context } from "effect";
+declare namespace Tokenizer_d_exports {
+  export { Tokenizer, TokenizerService };
+}
+/**
+ * Turning text into tokens, for sizing chunks, budgeting context, and
+ * estimating usage a provider does not report. Count with `encode(text).length`.
+ *
+ * Implementor contract: both methods are synchronous, so counting inside a loop
+ * never awaits; loading a vocabulary is the layer's job. `decode` inverts
+ * `encode`. Implement this only over a real vocabulary, never a heuristic:
+ * a character-count estimate is a plain function, not a service.
+ */
+type TokenizerService = {
+  readonly encode: (text: string) => ReadonlyArray<number>;
+  readonly decode: (tokens: ReadonlyArray<number>) => string;
+};
+declare const Tokenizer_base: Context.ServiceClass<Tokenizer, "@betalyra/effect-uai/Tokenizer", TokenizerService>;
+declare class Tokenizer extends Tokenizer_base {}
+//#endregion
+export { Tokenizer, TokenizerService, Tokenizer_d_exports as t };
+//# sourceMappingURL=Tokenizer.d.mts.map

package/dist/tokenizer/Tokenizer.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Tokenizer.d.mts","names":[],"sources":["../../src/tokenizer/Tokenizer.ts"],"mappings":";;;;;;;;;;;;;KAWY;WACD,SAAS,iBAAiB;WAC1B,SAAS,QAAQ;;;cAGf,kBAAkB"}

package/dist/tokenizer/Tokenizer.mjs

@@ -0,0 +1,9 @@
+import { n as __exportAll } from "../rolldown-runtime-EAsCxrDo.mjs";
+import { Context } from "effect";
+//#region src/tokenizer/Tokenizer.ts
+var Tokenizer_exports = /* @__PURE__ */ __exportAll({ Tokenizer: () => Tokenizer });
+var Tokenizer = class extends Context.Service()("@betalyra/effect-uai/Tokenizer") {};
+//#endregion
+export { Tokenizer, Tokenizer_exports as t };
+
+//# sourceMappingURL=Tokenizer.mjs.map

package/dist/tokenizer/Tokenizer.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"Tokenizer.mjs","names":[],"sources":["../../src/tokenizer/Tokenizer.ts"],"sourcesContent":["import { Context } from \"effect\"\n\n/**\n * Turning text into tokens, for sizing chunks, budgeting context, and\n * estimating usage a provider does not report. Count with `encode(text).length`.\n *\n * Implementor contract: both methods are synchronous, so counting inside a loop\n * never awaits; loading a vocabulary is the layer's job. `decode` inverts\n * `encode`. Implement this only over a real vocabulary, never a heuristic:\n * a character-count estimate is a plain function, not a service.\n */\nexport type TokenizerService = {\n  readonly encode: (text: string) => ReadonlyArray<number>\n  readonly decode: (tokens: ReadonlyArray<number>) => string\n}\n\nexport class Tokenizer extends Context.Service<Tokenizer, TokenizerService>()(\n  \"@betalyra/effect-uai/Tokenizer\",\n) {}\n"],"mappings":";;;;AAgBA,IAAa,YAAb,cAA+B,QAAQ,QAAqC,CAAC,CAC3E,gCACF,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effect-uai/core",
-  "version": "0.12.1",
+  "version": "0.13.0",
   "description": "Low-level primitives (loop, conversation, items, tools, streaming codecs) for building AI agents with Effect.",
   "keywords": [
     "agents",
@@ -42,6 +42,10 @@
       "types": "./dist/domain/Audio.d.mts",
       "import": "./dist/domain/Audio.mjs"
     },
+    "./Chunker": {
+      "types": "./dist/chunker/Chunker.d.mts",
+      "import": "./dist/chunker/Chunker.mjs"
+    },
     "./Citation": {
       "types": "./dist/domain/Citation.d.mts",
       "import": "./dist/domain/Citation.mjs"
@@ -110,6 +114,10 @@
       "types": "./dist/music-generator/MusicGenerator.d.mts",
       "import": "./dist/music-generator/MusicGenerator.mjs"
     },
+    "./Reranker": {
+      "types": "./dist/reranker/Reranker.d.mts",
+      "import": "./dist/reranker/Reranker.mjs"
+    },
     "./Memory": {
       "types": "./dist/sandbox/Memory.d.mts",
       "import": "./dist/sandbox/Memory.mjs"
@@ -166,6 +174,10 @@
       "types": "./dist/math/Vector.d.mts",
       "import": "./dist/math/Vector.mjs"
     },
+    "./Tokenizer": {
+      "types": "./dist/tokenizer/Tokenizer.d.mts",
+      "import": "./dist/tokenizer/Tokenizer.mjs"
+    },
     "./Tool": {
       "types": "./dist/tool/Tool.d.mts",
       "import": "./dist/tool/Tool.mjs"

package/src/chunker/Chunker.ts

@@ -0,0 +1,34 @@
+import { Context, Effect } from "effect"
+import type * as AiError from "../domain/AiError.js"
+
+/**
+ * A passage of a document, with the character offsets it came from.
+ *
+ * Implementor contract: `text` is the input's `[start, end)` slice verbatim,
+ * so a retrieved passage can be traced back to its source and enriched with
+ * what surrounds it. Chunks may overlap.
+ */
+export type Chunk = {
+  readonly text: string
+  readonly start: number
+  readonly end: number
+}
+
+/**
+ * Splitting a document into retrievable passages. Strategy and sizing are
+ * fixed when the layer is built, so callers stay portable across a local
+ * splitter and a hosted chunking service.
+ */
+export type ChunkerService = {
+  readonly chunk: (text: string) => Effect.Effect<ReadonlyArray<Chunk>, AiError.AiError>
+}
+
+export class Chunker extends Context.Service<Chunker, ChunkerService>()(
+  "@betalyra/effect-uai/Chunker",
+) {}
+
+/** Split a document into passages. */
+export const chunk = (
+  text: string,
+): Effect.Effect<ReadonlyArray<Chunk>, AiError.AiError, Chunker> =>
+  Effect.flatMap(Chunker, (chunker) => chunker.chunk(text))

package/src/index.ts

@@ -4,6 +4,7 @@ export * as Browser from "./browser/Browser.js"
 export * as BrowserError from "./browser/BrowserError.js"
 export * as BrowserTool from "./browser/BrowserTool.js"
 export * as Capabilities from "./capabilities/Capabilities.js"
+export * as Chunker from "./chunker/Chunker.js"
 export * as Citation from "./domain/Citation.js"
 export * as DeepResearch from "./research/DeepResearch.js"
 export * as Image from "./domain/Image.js"
@@ -18,6 +19,7 @@ export * as Embedding from "./embedding-model/Embedding.js"
 export * as EmbeddingModel from "./embedding-model/EmbeddingModel.js"
 export * as LanguageModel from "./language-model/LanguageModel.js"
 export * as MusicGenerator from "./music-generator/MusicGenerator.js"
+export * as Reranker from "./reranker/Reranker.js"
 export * as Memory from "./sandbox/Memory.js"
 export * as Sandbox from "./sandbox/Sandbox.js"
 export * as SandboxError from "./sandbox/SandboxError.js"
@@ -30,6 +32,7 @@ export * as WebReadTool from "./web-read/WebReadTool.js"
 export * as WebSearch from "./web-search/WebSearch.js"
 export * as WebSearchTool from "./web-search/WebSearchTool.js"
 export * as Vector from "./math/Vector.js"
+export * as Tokenizer from "./tokenizer/Tokenizer.js"
 export * as Loop from "./loop/Loop.js"
 export * as Retry from "./retry/Retry.js"
 export * as Tool from "./tool/Tool.js"

package/src/reranker/Reranker.ts

@@ -0,0 +1,51 @@
+import { Context, Effect } from "effect"
+import type * as AiError from "../domain/AiError.js"
+
+/**
+ * Cross-provider rerank request. Vendor knobs (multimodal documents,
+ * instruction fields, truncation) live on the provider's typed request.
+ */
+export type CommonRerankRequest = {
+  readonly query: string
+  readonly documents: ReadonlyArray<string>
+  /** Each provider narrows this to its typed literal union. */
+  readonly model: string
+  /** Keep only the top N results. Default: all documents. */
+  readonly topN?: number
+}
+
+export type RerankResult = {
+  /** Position in the request's `documents`. */
+  readonly index: number
+  readonly score: number
+}
+
+/** Optional throughout: some providers bill per search unit, not per token. */
+export type RerankUsage = {
+  readonly totalTokens?: number
+}
+
+/**
+ * Score contract: `results` is sorted descending, higher is better. Scores
+ * order candidates within one call; no range or calibration is promised and
+ * they are not comparable across requests. Implementors must sort descending
+ * if the wire does not.
+ */
+export type RerankResponse = {
+  readonly results: ReadonlyArray<RerankResult>
+  readonly usage: RerankUsage
+}
+
+export type RerankerService = {
+  readonly rerank: (request: CommonRerankRequest) => Effect.Effect<RerankResponse, AiError.AiError>
+}
+
+export class Reranker extends Context.Service<Reranker, RerankerService>()(
+  "@betalyra/effect-uai/Reranker",
+) {}
+
+/** Score a candidate set against a query, best first. */
+export const rerank = (
+  request: CommonRerankRequest,
+): Effect.Effect<RerankResponse, AiError.AiError, Reranker> =>
+  Effect.flatMap(Reranker, (r) => r.rerank(request))

package/src/tokenizer/Tokenizer.ts

@@ -0,0 +1,19 @@
+import { Context } from "effect"
+
+/**
+ * Turning text into tokens, for sizing chunks, budgeting context, and
+ * estimating usage a provider does not report. Count with `encode(text).length`.
+ *
+ * Implementor contract: both methods are synchronous, so counting inside a loop
+ * never awaits; loading a vocabulary is the layer's job. `decode` inverts
+ * `encode`. Implement this only over a real vocabulary, never a heuristic:
+ * a character-count estimate is a plain function, not a service.
+ */
+export type TokenizerService = {
+  readonly encode: (text: string) => ReadonlyArray<number>
+  readonly decode: (tokens: ReadonlyArray<number>) => string
+}
+
+export class Tokenizer extends Context.Service<Tokenizer, TokenizerService>()(
+  "@betalyra/effect-uai/Tokenizer",
+) {}