@effect/ai 0.26.0 → 0.27.0

package/dist/cjs/Telemetry.js

@@ -0,0 +1,176 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.addSpanAttributes = exports.addGenAIAnnotations = exports.CurrentSpanTransformer = void 0;
+var Context = _interopRequireWildcard(require("effect/Context"));
+var _Function = require("effect/Function");
+var Predicate = _interopRequireWildcard(require("effect/Predicate"));
+var String = _interopRequireWildcard(require("effect/String"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+/**
+ * The `Telemetry` module provides OpenTelemetry integration for operations
+ * performed against a large language model provider by defining telemetry
+ * attributes and utilities that follow the OpenTelemetry GenAI semantic
+ * conventions.
+ *
+ * @example
+ * ```ts
+ * import { Telemetry } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * // Add telemetry attributes to a span
+ * const addTelemetry = Effect.gen(function* () {
+ *   const span = yield* Effect.currentSpan
+ *
+ *   Telemetry.addGenAIAnnotations(span, {
+ *     system: "openai",
+ *     operation: { name: "chat" },
+ *     request: {
+ *       model: "gpt-4",
+ *       temperature: 0.7,
+ *       maxTokens: 1000
+ *     },
+ *     usage: {
+ *       inputTokens: 100,
+ *       outputTokens: 50
+ *     }
+ *   })
+ * })
+ * ```
+ *
+ * @since 1.0.0
+ */
+
+/**
+ * Creates a function to add attributes to a span with a given prefix and key transformation.
+ *
+ * This utility function is used internally to create specialized functions for adding
+ * different types of telemetry attributes to OpenTelemetry spans.
+ *
+ * @example
+ * ```ts
+ * import { Telemetry } from "@effect/ai"
+ * import { String, Tracer } from "effect"
+ *
+ * const addCustomAttributes = Telemetry.addSpanAttributes(
+ *   "custom.ai",
+ *   String.camelToSnake
+ * )
+ *
+ * // Usage with a span
+ * declare const span: Tracer.Span
+ * addCustomAttributes(span, {
+ *   modelName: "gpt-4",
+ *   maxTokens: 1000
+ * })
+ * // Results in attributes: "custom.ai.model_name" and "custom.ai.max_tokens"
+ * ```
+ *
+ * @since 1.0.0
+ * @category Utilities
+ */
+const addSpanAttributes = (
+/**
+ * The prefix to add to all attribute keys.
+ */
+keyPrefix,
+/**
+ * Function to transform attribute keys (e.g., camelCase to snake_case).
+ */
+transformKey) => (
+/**
+ * The OpenTelemetry span to add attributes to.
+ */
+span,
+/**
+ * The attributes to add to the span.
+ */
+attributes) => {
+  for (const [key, value] of Object.entries(attributes)) {
+    if (Predicate.isNotNullable(value)) {
+      span.attribute(`${keyPrefix}.${transformKey(key)}`, value);
+    }
+  }
+};
+exports.addSpanAttributes = addSpanAttributes;
+const addSpanBaseAttributes = /*#__PURE__*/addSpanAttributes("gen_ai", String.camelToSnake);
+const addSpanOperationAttributes = /*#__PURE__*/addSpanAttributes("gen_ai.operation", String.camelToSnake);
+const addSpanRequestAttributes = /*#__PURE__*/addSpanAttributes("gen_ai.request", String.camelToSnake);
+const addSpanResponseAttributes = /*#__PURE__*/addSpanAttributes("gen_ai.response", String.camelToSnake);
+const addSpanTokenAttributes = /*#__PURE__*/addSpanAttributes("gen_ai.token", String.camelToSnake);
+const addSpanUsageAttributes = /*#__PURE__*/addSpanAttributes("gen_ai.usage", String.camelToSnake);
+/**
+ * Applies GenAI telemetry attributes to an OpenTelemetry span.
+ *
+ * This function adds standardized GenAI attributes to a span following OpenTelemetry
+ * semantic conventions. It supports both curried and direct application patterns.
+ *
+ * **Note**: This function mutates the provided span in-place.
+ *
+ * @example
+ * ```ts
+ * import { Telemetry } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * const directUsage = Effect.gen(function* () {
+ *   const span = yield* Effect.currentSpan
+ *
+ *   Telemetry.addGenAIAnnotations(span, {
+ *     system: "openai",
+ *     request: { model: "gpt-4", temperature: 0.7 },
+ *     usage: { inputTokens: 100, outputTokens: 50 }
+ *   })
+ * })
+ * ```
+ *
+ * @since 1.0.0
+ * @category Utilities
+ */
+const addGenAIAnnotations = exports.addGenAIAnnotations = /*#__PURE__*/(0, _Function.dual)(2, (span, options) => {
+  addSpanBaseAttributes(span, {
+    system: options.system
+  });
+  if (Predicate.isNotNullable(options.operation)) addSpanOperationAttributes(span, options.operation);
+  if (Predicate.isNotNullable(options.request)) addSpanRequestAttributes(span, options.request);
+  if (Predicate.isNotNullable(options.response)) addSpanResponseAttributes(span, options.response);
+  if (Predicate.isNotNullable(options.token)) addSpanTokenAttributes(span, options.token);
+  if (Predicate.isNotNullable(options.usage)) addSpanUsageAttributes(span, options.usage);
+});
+/**
+ * Context tag for providing a span transformer to large langauge model
+ * operations.
+ *
+ * The CurrentSpanTransformer allows you to inject custom span transformation
+ * logic into AI operations, enabling application-specific telemetry and
+ * observability patterns.
+ *
+ * @example
+ * ```ts
+ * import { Telemetry } from "@effect/ai"
+ * import { Context, Effect } from "effect"
+ *
+ * // Create a custom span transformer
+ * const loggingTransformer: Telemetry.SpanTransformer = (options) => {
+ *   console.log(`AI request completed: ${options.model}`)
+ *   options.response.forEach((part, index) => {
+ *     console.log(`Part ${index}: ${part.type}`)
+ *   })
+ * }
+ *
+ * // Provide the transformer to your AI operations
+ * const program = myAIOperation.pipe(
+ *   Effect.provideService(
+ *     Telemetry.CurrentSpanTransformer,
+ *     Telemetry.CurrentSpanTransformer.of(loggingTransformer)
+ *   )
+ * )
+ * ```
+ *
+ * @since 1.0.0
+ * @category Context
+ */
+class CurrentSpanTransformer extends /*#__PURE__*/Context.Tag("@effect/ai/Telemetry/CurrentSpanTransformer")() {}
+exports.CurrentSpanTransformer = CurrentSpanTransformer;
+//# sourceMappingURL=Telemetry.js.map

package/dist/cjs/Telemetry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Telemetry.js","names":["Context","_interopRequireWildcard","require","_Function","Predicate","String","e","t","WeakMap","r","n","__esModule","o","i","f","__proto__","default","has","get","set","hasOwnProperty","call","Object","defineProperty","getOwnPropertyDescriptor","addSpanAttributes","keyPrefix","transformKey","span","attributes","key","value","entries","isNotNullable","attribute","exports","addSpanBaseAttributes","camelToSnake","addSpanOperationAttributes","addSpanRequestAttributes","addSpanResponseAttributes","addSpanTokenAttributes","addSpanUsageAttributes","addGenAIAnnotations","dual","options","system","operation","request","response","token","usage","CurrentSpanTransformer","Tag"],"sources":["../../src/Telemetry.ts"],"sourcesContent":[null],"mappings":";;;;;;AAiCA,IAAAA,OAAA,GAAAC,uBAAA,CAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AACA,IAAAE,SAAA,GAAAH,uBAAA,CAAAC,OAAA;AACA,IAAAG,MAAA,GAAAJ,uBAAA,CAAAC,OAAA;AAAuC,SAAAD,wBAAAK,CAAA,EAAAC,CAAA,6BAAAC,OAAA,MAAAC,CAAA,OAAAD,OAAA,IAAAE,CAAA,OAAAF,OAAA,YAAAP,uBAAA,YAAAA,CAAAK,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAK,UAAA,SAAAL,CAAA,MAAAM,CAAA,EAAAC,CAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,iBAAAA,CAAA,uBAAAA,CAAA,yBAAAA,CAAA,SAAAQ,CAAA,MAAAF,CAAA,GAAAL,CAAA,GAAAG,CAAA,GAAAD,CAAA,QAAAG,CAAA,CAAAK,GAAA,CAAAX,CAAA,UAAAM,CAAA,CAAAM,GAAA,CAAAZ,CAAA,GAAAM,CAAA,CAAAO,GAAA,CAAAb,CAAA,EAAAQ,CAAA,gBAAAP,CAAA,IAAAD,CAAA,gBAAAC,CAAA,OAAAa,cAAA,CAAAC,IAAA,CAAAf,CAAA,EAAAC,CAAA,OAAAM,CAAA,IAAAD,CAAA,GAAAU,MAAA,CAAAC,cAAA,KAAAD,MAAA,CAAAE,wBAAA,CAAAlB,CAAA,EAAAC,CAAA,OAAAM,CAAA,CAAAK,GAAA,IAAAL,CAAA,CAAAM,GAAA,IAAAP,CAAA,CAAAE,CAAA,EAAAP,CAAA,EAAAM,CAAA,IAAAC,CAAA,CAAAP,CAAA,IAAAD,CAAA,CAAAC,CAAA,WAAAO,CAAA,KAAAR,CAAA,EAAAC,CAAA;AApCvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqSA;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BO,MAAMkB,iBAAiB,GAAGA;AAC/B;;;AAGAC,SAAiB;AACjB;;;AAGAC,YAAqC,KAEvC;AACE;;;AAGAC,IAAU;AACV;;;AAGAC,UAAsB,KACd;EACR,KAAK,MAAM,CAACC,GAAG,EAAEC,KAAK,CAAC,IAAIT,MAAM,CAACU,OAAO,CAACH,UAAU,CAAC,EAAE;IACrD,IAAIzB,SAAS,CAAC6B,aAAa,CAACF,KAAK,CAAC,EAAE;MAClCH,IAAI,CAACM,SAAS,CAAC,GAAGR,SAAS,IAAIC,YAAY,CAACG,GAAG,CAAC,EAAE,EAAEC,KAAK,CAAC;IAC5D;EACF;AACF,CAAC;AAAAI,OAAA,CAAAV,iBAAA,GAAAA,iBAAA;AAED,MAAMW,qBAAqB,gBAAGX,iBAAiB,CAAC,QAAQ,EAAEpB,MAAM,CAACgC,YAAY,CAAiB;AAC9F,MAAMC,0BAA0B,gBAAGb,iBAAiB,CAAC,kBAAkB,EAAEpB,MAAM,CAACgC,YAAY,CAAsB;AAClH,MAAME,wBAAwB,gBAAGd,iBAAiB,CAAC,gBAAgB,EAAEpB,MAAM,CAACgC,YAAY,CAAoB;AAC5G,MAAMG,yBAAyB,gBAAGf,iBAAiB,CAAC,iBAAiB,EAAEpB,MAAM,CAACgC,YAAY,CAAqB;AAC/G,MAAMI,sBAAsB,gBAAGhB,iBAAiB,CAAC,cAAc,EAAEpB,MAAM,CAACgC,YAAY,CAAkB;AACtG,MAAMK,sBAAsB,gBAAGjB,iBAAiB,CAAC,cAAc,EAAEpB,MAAM,CAACgC,YAAY,CAAkB;AA4DtG;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BO,MAAMM,mBAAmB,GAAAR,OAAA,CAAAQ,mBAAA,gBA4E5B,IAAAC,cAAI,EAyDN,CAAC,EAAE,CAAChB,IAAI,EAAEiB,OAAO,KAAI;EACrBT,qBAAqB,CAACR,IAAI,EAAE;IAAEkB,MAAM,EAAED,OAAO,CAACC;EAAM,CAAE,CAAC;EACvD,IAAI1C,SAAS,CAAC6B,aAAa,CAACY,OAAO,CAACE,SAAS,CAAC,EAAET,0BAA0B,CAACV,IAAI,EAAEiB,OAAO,CAACE,SAAS,CAAC;EACnG,IAAI3C,SAAS,CAAC6B,aAAa,CAACY,OAAO,CAACG,OAAO,CAAC,EAAET,wBAAwB,CAACX,IAAI,EAAEiB,OAAO,CAACG,OAAO,CAAC;EAC7F,IAAI5C,SAAS,CAAC6B,aAAa,CAACY,OAAO,CAACI,QAAQ,CAAC,EAAET,yBAAyB,CAACZ,IAAI,EAAEiB,OAAO,CAACI,QAAQ,CAAC;EAChG,IAAI7C,SAAS,CAAC6B,aAAa,CAACY,OAAO,CAACK,KAAK,CAAC,EAAET,sBAAsB,CAACb,IAAI,EAAEiB,OAAO,CAACK,KAAK,CAAC;EACvF,IAAI9C,SAAS,CAAC6B,aAAa,CAACY,OAAO,CAACM,KAAK,CAAC,EAAET,sBAAsB,CAACd,IAAI,EAAEiB,OAAO,CAACM,KAAK,CAAC;AACzF,CAAC,CAAC;AAsCF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCM,MAAOC,sBAAuB,sBAAQpD,OAAO,CAACqD,GAAG,CAAC,6CAA6C,CAAC,EAGnG;AAAAlB,OAAA,CAAAiB,sBAAA,GAAAA,sBAAA","ignoreList":[]}

package/dist/cjs/Tokenizer.js

@@ -7,45 +7,124 @@ exports.make = exports.Tokenizer = void 0;
 var Context = _interopRequireWildcard(require("effect/Context"));
 var Effect = _interopRequireWildcard(require("effect/Effect"));
 var Predicate = _interopRequireWildcard(require("effect/Predicate"));
-var AiInput = _interopRequireWildcard(require("./AiInput.js"));
+var Prompt = _interopRequireWildcard(require("./Prompt.js"));
 function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
 /**
+ * The `Tokenizer` module provides tokenization and text truncation capabilities
+ * for large language model text processing workflows.
+ *
+ * This module offers services for converting text into tokens and truncating
+ * prompts based on token limits, essential for managing context length
+ * constraints in large language models.
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer, Prompt } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * const tokenizeText = Effect.gen(function* () {
+ *   const tokenizer = yield* Tokenizer.Tokenizer
+ *   const tokens = yield* tokenizer.tokenize("Hello, world!")
+ *   console.log(`Token count: ${tokens.length}`)
+ *   return tokens
+ * })
+ * ```
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer, Prompt } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * // Truncate a prompt to fit within token limits
+ * const truncatePrompt = Effect.gen(function* () {
+ *   const tokenizer = yield* Tokenizer.Tokenizer
+ *   const longPrompt = "This is a very long prompt..."
+ *   const truncated = yield* tokenizer.truncate(longPrompt, 100)
+ *   return truncated
+ * })
+ * ```
+ *
  * @since 1.0.0
  */
 
 /**
+ * The `Tokenizer` service tag for dependency injection.
+ *
+ * This tag provides access to tokenization functionality throughout your
+ * application, enabling token counting and prompt truncation capabilities.
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * const useTokenizer = Effect.gen(function* () {
+ *   const tokenizer = yield* Tokenizer.Tokenizer
+ *   const tokens = yield* tokenizer.tokenize("Hello, world!")
+ *   return tokens.length
+ * })
+ * ```
+ *
  * @since 1.0.0
- * @category Tags
+ * @category Context
  */
 class Tokenizer extends /*#__PURE__*/Context.Tag("@effect/ai/Tokenizer")() {}
 /**
+ * Creates a Tokenizer service implementation from tokenization options.
+ *
+ * This function constructs a complete Tokenizer service by providing a
+ * tokenization function. The service handles both tokenization and
+ * truncation operations using the provided tokenizer.
+ *
+ * @example
+ * ```ts
+ * import { Tokenizer, Prompt } from "@effect/ai"
+ * import { Effect } from "effect"
+ *
+ * // Simple word-based tokenizer
+ * const wordTokenizer = Tokenizer.make({
+ *   tokenize: (prompt) =>
+ *     Effect.succeed(
+ *       prompt.content
+ *         .flatMap(msg =>
+ *           typeof msg.content === "string"
+ *             ? msg.content.split(' ')
+ *             : msg.content.flatMap(part =>
+ *                 part.type === "text" ? part.text.split(' ') : []
+ *               )
+ *         )
+ *         .map((_, index) => index)
+ *     )
+ * })
+ * ```
+ *
  * @since 1.0.0
  * @category Constructors
  */
 exports.Tokenizer = Tokenizer;
 const make = options => Tokenizer.of({
   tokenize(input) {
-    return options.tokenize(AiInput.make(input));
+    return options.tokenize(Prompt.make(input));
   },
   truncate(input, tokens) {
-    return truncate(AiInput.make(input), options.tokenize, tokens);
+    return truncate(Prompt.make(input), options.tokenize, tokens);
   }
 });
 exports.make = make;
 const truncate = (self, tokenize, maxTokens) => Effect.suspend(() => {
   let count = 0;
-  let inputMessages = self.messages;
+  let inputMessages = self.content;
   let outputMessages = [];
   const loop = Effect.suspend(() => {
     const message = inputMessages[inputMessages.length - 1];
     if (Predicate.isUndefined(message)) {
-      return Effect.succeed(AiInput.make(outputMessages));
+      return Effect.succeed(Prompt.fromMessages(outputMessages));
     }
     inputMessages = inputMessages.slice(0, inputMessages.length - 1);
-    return Effect.flatMap(tokenize(AiInput.make(message)), tokens => {
+    return Effect.flatMap(tokenize(Prompt.fromMessages([message])), tokens => {
       count += tokens.length;
       if (count > maxTokens) {
-        return Effect.succeed(AiInput.make(outputMessages));
+        return Effect.succeed(Prompt.fromMessages(outputMessages));
       }
       outputMessages = [message, ...outputMessages];
       return loop;

package/dist/cjs/Tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"Tokenizer.js","names":["Context","_interopRequireWildcard","require","Effect","Predicate","AiInput","e","t","WeakMap","r","n","__esModule","o","i","f","__proto__","default","has","get","set","hasOwnProperty","call","Object","defineProperty","getOwnPropertyDescriptor","Tokenizer","Tag","exports","make","options","of","tokenize","input","truncate","tokens","self","maxTokens","suspend","count","inputMessages","messages","outputMessages","loop","message","length","isUndefined","succeed","slice","flatMap"],"sources":["../../src/Tokenizer.ts"],"sourcesContent":[null],"mappings":";;;;;;AAGA,IAAAA,OAAA,GAAAC,uBAAA,CAAAC,OAAA;AACA,IAAAC,MAAA,GAAAF,uBAAA,CAAAC,OAAA;AACA,IAAAE,SAAA,GAAAH,uBAAA,CAAAC,OAAA;AAEA,IAAAG,OAAA,GAAAJ,uBAAA,CAAAC,OAAA;AAAuC,SAAAD,wBAAAK,CAAA,EAAAC,CAAA,6BAAAC,OAAA,MAAAC,CAAA,OAAAD,OAAA,IAAAE,CAAA,OAAAF,OAAA,YAAAP,uBAAA,YAAAA,CAAAK,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAK,UAAA,SAAAL,CAAA,MAAAM,CAAA,EAAAC,CAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,iBAAAA,CAAA,uBAAAA,CAAA,yBAAAA,CAAA,SAAAQ,CAAA,MAAAF,CAAA,GAAAL,CAAA,GAAAG,CAAA,GAAAD,CAAA,QAAAG,CAAA,CAAAK,GAAA,CAAAX,CAAA,UAAAM,CAAA,CAAAM,GAAA,CAAAZ,CAAA,GAAAM,CAAA,CAAAO,GAAA,CAAAb,CAAA,EAAAQ,CAAA,gBAAAP,CAAA,IAAAD,CAAA,gBAAAC,CAAA,OAAAa,cAAA,CAAAC,IAAA,CAAAf,CAAA,EAAAC,CAAA,OAAAM,CAAA,IAAAD,CAAA,GAAAU,MAAA,CAAAC,cAAA,KAAAD,MAAA,CAAAE,wBAAA,CAAAlB,CAAA,EAAAC,CAAA,OAAAM,CAAA,CAAAK,GAAA,IAAAL,CAAA,CAAAM,GAAA,IAAAP,CAAA,CAAAE,CAAA,EAAAP,CAAA,EAAAM,CAAA,IAAAC,CAAA,CAAAP,CAAA,IAAAD,CAAA,CAAAC,CAAA,WAAAO,CAAA,KAAAR,CAAA,EAAAC,CAAA;AAPvC;;;;AASA;;;;AAIM,MAAOkB,SAAU,sBAAQzB,OAAO,CAAC0B,GAAG,CAAC,sBAAsB,CAAC,EAG/D;AAgBH;;;;AAAAC,OAAA,CAAAF,SAAA,GAAAA,SAAA;AAIO,MAAMG,IAAI,GAAIC,OAEpB,IACCJ,SAAS,CAACK,EAAE,CAAC;EACXC,QAAQA,CAACC,KAAK;IACZ,OAAOH,OAAO,CAACE,QAAQ,CAAC1B,OAAO,CAACuB,IAAI,CAACI,KAAK,CAAC,CAAC;EAC9C,CAAC;EACDC,QAAQA,CAACD,KAAK,EAAEE,MAAM;IACpB,OAAOD,QAAQ,CAAC5B,OAAO,CAACuB,IAAI,CAACI,KAAK,CAAC,EAAEH,OAAO,CAACE,QAAQ,EAAEG,MAAM,CAAC;EAChE;CACD,CAAC;AAAAP,OAAA,CAAAC,IAAA,GAAAA,IAAA;AAEJ,MAAMK,QAAQ,GAAGA,CACfE,IAAqB,EACrBJ,QAA2E,EAC3EK,SAAiB,KAEjBjC,MAAM,CAACkC,OAAO,CAAC,MAAK;EAClB,IAAIC,KAAK,GAAG,CAAC;EACb,IAAIC,aAAa,GAAGJ,IAAI,CAACK,QAAQ;EACjC,IAAIC,cAAc,GAA2B,EAAE;EAC/C,MAAMC,IAAI,GAA4CvC,MAAM,CAACkC,OAAO,CAAC,MAAK;IACxE,MAAMM,OAAO,GAAGJ,aAAa,CAACA,aAAa,CAACK,MAAM,GAAG,CAAC,CAAC;IACvD,IAAIxC,SAAS,CAACyC,WAAW,CAACF,OAAO,CAAC,EAAE;MAClC,OAAOxC,MAAM,CAAC2C,OAAO,CAACzC,OAAO,CAACuB,IAAI,CAACa,cAAc,CAAC,CAAC;IACrD;IACAF,aAAa,GAAGA,aAAa,CAACQ,KAAK,CAAC,CAAC,EAAER,aAAa,CAACK,MAAM,GAAG,CAAC,CAAC;IAChE,OAAOzC,MAAM,CAAC6C,OAAO,CAACjB,QAAQ,CAAC1B,OAAO,CAACuB,IAAI,CAACe,OAAO,CAAC,CAAC,EAAGT,MAAM,IAAI;MAChEI,KAAK,IAAIJ,MAAM,CAACU,MAAM;MACtB,IAAIN,KAAK,GAAGF,SAAS,EAAE;QACrB,OAAOjC,MAAM,CAAC2C,OAAO,CAACzC,OAAO,CAACuB,IAAI,CAACa,cAAc,CAAC,CAAC;MACrD;MACAA,cAAc,GAAG,CAACE,OAAO,EAAE,GAAGF,cAAc,CAAC;MAC7C,OAAOC,IAAI;IACb,CAAC,CAAC;EACJ,CAAC,CAAC;EACF,OAAOA,IAAI;AACb,CAAC,CAAC","ignoreList":[]}
+{"version":3,"file":"Tokenizer.js","names":["Context","_interopRequireWildcard","require","Effect","Predicate","Prompt","e","t","WeakMap","r","n","__esModule","o","i","f","__proto__","default","has","get","set","hasOwnProperty","call","Object","defineProperty","getOwnPropertyDescriptor","Tokenizer","Tag","exports","make","options","of","tokenize","input","truncate","tokens","self","maxTokens","suspend","count","inputMessages","content","outputMessages","loop","message","length","isUndefined","succeed","fromMessages","slice","flatMap"],"sources":["../../src/Tokenizer.ts"],"sourcesContent":[null],"mappings":";;;;;;AAqCA,IAAAA,OAAA,GAAAC,uBAAA,CAAAC,OAAA;AACA,IAAAC,MAAA,GAAAF,uBAAA,CAAAC,OAAA;AACA,IAAAE,SAAA,GAAAH,uBAAA,CAAAC,OAAA;AAEA,IAAAG,MAAA,GAAAJ,uBAAA,CAAAC,OAAA;AAAqC,SAAAD,wBAAAK,CAAA,EAAAC,CAAA,6BAAAC,OAAA,MAAAC,CAAA,OAAAD,OAAA,IAAAE,CAAA,OAAAF,OAAA,YAAAP,uBAAA,YAAAA,CAAAK,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAK,UAAA,SAAAL,CAAA,MAAAM,CAAA,EAAAC,CAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,iBAAAA,CAAA,uBAAAA,CAAA,yBAAAA,CAAA,SAAAQ,CAAA,MAAAF,CAAA,GAAAL,CAAA,GAAAG,CAAA,GAAAD,CAAA,QAAAG,CAAA,CAAAK,GAAA,CAAAX,CAAA,UAAAM,CAAA,CAAAM,GAAA,CAAAZ,CAAA,GAAAM,CAAA,CAAAO,GAAA,CAAAb,CAAA,EAAAQ,CAAA,gBAAAP,CAAA,IAAAD,CAAA,gBAAAC,CAAA,OAAAa,cAAA,CAAAC,IAAA,CAAAf,CAAA,EAAAC,CAAA,OAAAM,CAAA,IAAAD,CAAA,GAAAU,MAAA,CAAAC,cAAA,KAAAD,MAAA,CAAAE,wBAAA,CAAAlB,CAAA,EAAAC,CAAA,OAAAM,CAAA,CAAAK,GAAA,IAAAL,CAAA,CAAAM,GAAA,IAAAP,CAAA,CAAAE,CAAA,EAAAP,CAAA,EAAAM,CAAA,IAAAC,CAAA,CAAAP,CAAA,IAAAD,CAAA,CAAAC,CAAA,WAAAO,CAAA,KAAAR,CAAA,EAAAC,CAAA;AAzCrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2CA;;;;;;;;;;;;;;;;;;;;;AAqBM,MAAOkB,SAAU,sBAAQzB,OAAO,CAAC0B,GAAG,CAAC,sBAAsB,CAAC,EAG/D;AAkDH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAAC,OAAA,CAAAF,SAAA,GAAAA,SAAA;AAgCO,MAAMG,IAAI,GAAIC,OAUpB,IACCJ,SAAS,CAACK,EAAE,CAAC;EACXC,QAAQA,CAACC,KAAK;IACZ,OAAOH,OAAO,CAACE,QAAQ,CAAC1B,MAAM,CAACuB,IAAI,CAACI,KAAK,CAAC,CAAC;EAC7C,CAAC;EACDC,QAAQA,CAACD,KAAK,EAAEE,MAAM;IACpB,OAAOD,QAAQ,CAAC5B,MAAM,CAACuB,IAAI,CAACI,KAAK,CAAC,EAAEH,OAAO,CAACE,QAAQ,EAAEG,MAAM,CAAC;EAC/D;CACD,CAAC;AAAAP,OAAA,CAAAC,IAAA,GAAAA,IAAA;AAEJ,MAAMK,QAAQ,GAAGA,CACfE,IAAmB,EACnBJ,QAAiF,EACjFK,SAAiB,KAEjBjC,MAAM,CAACkC,OAAO,CAAC,MAAK;EAClB,IAAIC,KAAK,GAAG,CAAC;EACb,IAAIC,aAAa,GAAGJ,IAAI,CAACK,OAAO;EAChC,IAAIC,cAAc,GAA0B,EAAE;EAC9C,MAAMC,IAAI,GAAkDvC,MAAM,CAACkC,OAAO,CAAC,MAAK;IAC9E,MAAMM,OAAO,GAAGJ,aAAa,CAACA,aAAa,CAACK,MAAM,GAAG,CAAC,CAAC;IACvD,IAAIxC,SAAS,CAACyC,WAAW,CAACF,OAAO,CAAC,EAAE;MAClC,OAAOxC,MAAM,CAAC2C,OAAO,CAACzC,MAAM,CAAC0C,YAAY,CAACN,cAAc,CAAC,CAAC;IAC5D;IACAF,aAAa,GAAGA,aAAa,CAACS,KAAK,CAAC,CAAC,EAAET,aAAa,CAACK,MAAM,GAAG,CAAC,CAAC;IAChE,OAAOzC,MAAM,CAAC8C,OAAO,CAAClB,QAAQ,CAAC1B,MAAM,CAAC0C,YAAY,CAAC,CAACJ,OAAO,CAAC,CAAC,CAAC,EAAGT,MAAM,IAAI;MACzEI,KAAK,IAAIJ,MAAM,CAACU,MAAM;MACtB,IAAIN,KAAK,GAAGF,SAAS,EAAE;QACrB,OAAOjC,MAAM,CAAC2C,OAAO,CAACzC,MAAM,CAAC0C,YAAY,CAACN,cAAc,CAAC,CAAC;MAC5D;MACAA,cAAc,GAAG,CAACE,OAAO,EAAE,GAAGF,cAAc,CAAC;MAC7C,OAAOC,IAAI;IACb,CAAC,CAAC;EACJ,CAAC,CAAC;EACF,OAAOA,IAAI;AACb,CAAC,CAAC","ignoreList":[]}