@effect/ai 0.26.0 → 0.27.0

package/dist/dts/Tokenizer.d.ts

@@ -1,36 +1,154 @@
 /**
+ * The `Tokenizer` module provides tokenization and text truncation capabilities
+ * for large language model text processing workflows.
+ *
+ * This module offers services for converting text into tokens and truncating
+ * prompts based on token limits, essential for managing context length
+ * constraints in large language models.
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer, Prompt } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * const tokenizeText = Effect.gen(function* () {
+ *   const tokenizer = yield* Tokenizer.Tokenizer
+ *   const tokens = yield* tokenizer.tokenize("Hello, world!")
+ *   console.log(`Token count: ${tokens.length}`)
+ *   return tokens
+ * })
+ * ```
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer, Prompt } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * // Truncate a prompt to fit within token limits
+ * const truncatePrompt = Effect.gen(function* () {
+ *   const tokenizer = yield* Tokenizer.Tokenizer
+ *   const longPrompt = "This is a very long prompt..."
+ *   const truncated = yield* tokenizer.truncate(longPrompt, 100)
+ *   return truncated
+ * })
+ * ```
+ *
  * @since 1.0.0
  */
 import * as Context from "effect/Context";
 import * as Effect from "effect/Effect";
-import type { AiError } from "./AiError.js";
-import * as AiInput from "./AiInput.js";
-declare const Tokenizer_base: Context.TagClass<Tokenizer, "@effect/ai/Tokenizer", Tokenizer.Service>;
+import type * as AiError from "./AiError.js";
+import * as Prompt from "./Prompt.js";
+declare const Tokenizer_base: Context.TagClass<Tokenizer, "@effect/ai/Tokenizer", Service>;
 /**
+ * The `Tokenizer` service tag for dependency injection.
+ *
+ * This tag provides access to tokenization functionality throughout your
+ * application, enabling token counting and prompt truncation capabilities.
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * const useTokenizer = Effect.gen(function* () {
+ *   const tokenizer = yield* Tokenizer.Tokenizer
+ *   const tokens = yield* tokenizer.tokenize("Hello, world!")
+ *   return tokens.length
+ * })
+ * ```
+ *
  * @since 1.0.0
- * @category Tags
+ * @category Context
  */
 export declare class Tokenizer extends Tokenizer_base {
 }
 /**
+ * Tokenizer service interface providing text tokenization and truncation
+ * operations.
+ *
+ * This interface defines the core operations for converting text to tokens and
+ * managing content length within token limits for AI model compatibility.
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer, Prompt } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * const customTokenizer: Tokenizer.Service = {
+ *   tokenize: (input) =>
+ *     Effect.succeed(input.toString().split(' ').map((_, i) => i)),
+ *   truncate: (input, maxTokens) =>
+ *     Effect.succeed(Prompt.make(input.toString().slice(0, maxTokens * 5)))
+ * }
+ * ```
+ *
  * @since 1.0.0
+ * @category Models
  */
-export declare namespace Tokenizer {
+export interface Service {
     /**
-     * @since 1.0.0
-     * @category Models
+     * Converts text input into an array of token numbers.
      */
-    interface Service {
-        readonly tokenize: (input: AiInput.Raw) => Effect.Effect<Array<number>, AiError>;
-        readonly truncate: (input: AiInput.Raw, tokens: number) => Effect.Effect<AiInput.AiInput, AiError>;
-    }
+    readonly tokenize: (
+    /**
+     * The text input to tokenize.
+     */
+    input: Prompt.RawInput) => Effect.Effect<Array<number>, AiError.AiError>;
+    /**
+     * Truncates text input to fit within the specified token limit.
+     */
+    readonly truncate: (
+    /**
+     * The text input to truncate.
+     */
+    input: Prompt.RawInput, 
+    /**
+     * Maximum number of tokens to retain.
+     */
+    tokens: number) => Effect.Effect<Prompt.Prompt, AiError.AiError>;
 }
 /**
+ * Creates a Tokenizer service implementation from tokenization options.
+ *
+ * This function constructs a complete Tokenizer service by providing a
+ * tokenization function. The service handles both tokenization and
+ * truncation operations using the provided tokenizer.
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer, Prompt } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * // Simple word-based tokenizer
+ * const wordTokenizer = Tokenizer.make({
+ *   tokenize: (prompt) =>
+ *     Effect.succeed(
+ *       prompt.content
+ *         .flatMap(msg =>
+ *           typeof msg.content === "string"
+ *             ? msg.content.split(' ')
+ *             : msg.content.flatMap(part =>
+ *                 part.type === "text" ? part.text.split(' ') : []
+ *               )
+ *         )
+ *         .map((_, index) => index)
+ *     )
+ * })
+ * ```
+ *
  * @since 1.0.0
  * @category Constructors
  */
 export declare const make: (options: {
-    readonly tokenize: (content: AiInput.AiInput) => Effect.Effect<Array<number>, AiError>;
-}) => Tokenizer.Service;
+    /**
+     * Function to tokenize a prompt into an array of token numbers.
+     */
+    readonly tokenize: (
+    /**
+     * The prompt to tokenize.
+     */
+    content: Prompt.Prompt) => Effect.Effect<Array<number>, AiError.AiError>;
+}) => Service;
 export {};
 //# sourceMappingURL=Tokenizer.d.ts.map

package/dist/dts/Tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Tokenizer.d.ts","sourceRoot":"","sources":["../../src/Tokenizer.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;;AAEvC;;;GAGG;AACH,qBAAa,SAAU,SAAQ,cAG5B;CAAG;AAEN;;GAEG;AACH,MAAM,CAAC,OAAO,WAAW,SAAS,CAAC;IACjC;;;OAGG;IACH,UAAiB,OAAO;QACtB,QAAQ,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAA;QAChF,QAAQ,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;KACnG;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,IAAI,GAAI,SAAS;IAC5B,QAAQ,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAA;CACvF,KAAG,SAAS,CAAC,OAQV,CAAA"}
+{"version":3,"file":"Tokenizer.d.ts","sourceRoot":"","sources":["../../src/Tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,KAAK,OAAO,MAAM,cAAc,CAAA;AAC5C,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;;AAErC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,SAAU,SAAQ,cAG5B;CAAG;AAEN;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,OAAO;IACtB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC,QAAQ,KACnB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAClD;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC,QAAQ;IACtB;;OAEG;IACH,MAAM,EAAE,MAAM,KACX,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;CACnD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,eAAO,MAAM,IAAI,GAAI,SAAS;IAC5B;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,KACnB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;CACnD,KAAG,OAQA,CAAA"}