@workglow/ai 0.0.99 → 0.0.101

package/dist/bun.js

@@ -13,6 +13,7 @@ import { globalServiceRegistry, WORKER_MANAGER } from "@workglow/util";
 class AiProviderRegistry {
   runFnRegistry = new Map;
   streamFnRegistry = new Map;
+  reactiveRunFnRegistry = new Map;
   providers = new Map;
   registerProvider(provider) {
     this.providers.set(provider.name, provider);
@@ -57,6 +58,27 @@ class AiProviderRegistry {
     const taskTypeMap = this.streamFnRegistry.get(taskType);
     return taskTypeMap?.get(modelProvider);
   }
+  registerAsWorkerReactiveRunFn(modelProvider, taskType) {
+    const reactiveFn = async (input, output, model) => {
+      const workerManager = globalServiceRegistry.get(WORKER_MANAGER);
+      return workerManager.callWorkerReactiveFunction(modelProvider, taskType, [
+        input,
+        output,
+        model
+      ]);
+    };
+    this.registerReactiveRunFn(modelProvider, taskType, reactiveFn);
+  }
+  registerReactiveRunFn(modelProvider, taskType, reactiveRunFn) {
+    if (!this.reactiveRunFnRegistry.has(taskType)) {
+      this.reactiveRunFnRegistry.set(taskType, new Map);
+    }
+    this.reactiveRunFnRegistry.get(taskType).set(modelProvider, reactiveRunFn);
+  }
+  getReactiveRunFn(modelProvider, taskType) {
+    const taskTypeMap = this.reactiveRunFnRegistry.get(taskType);
+    return taskTypeMap?.get(modelProvider);
+  }
   getDirectRunFn(modelProvider, taskType) {
     const taskTypeMap = this.runFnRegistry.get(taskType);
     const runFn = taskTypeMap?.get(modelProvider);
@@ -308,9 +330,11 @@ async function createDefaultQueue(providerName, concurrency) {
 class AiProvider {
   tasks;
   streamTasks;
-  constructor(tasks, streamTasks) {
+  reactiveTasks;
+  constructor(tasks, streamTasks, reactiveTasks) {
     this.tasks = tasks;
     this.streamTasks = streamTasks;
+    this.reactiveTasks = reactiveTasks;
   }
   get supportedTaskTypes() {
     return this.taskTypes;
@@ -321,6 +345,9 @@ class AiProvider {
   getStreamFn(taskType) {
     return this.streamTasks?.[taskType];
   }
+  getReactiveRunFn(taskType) {
+    return this.reactiveTasks?.[taskType];
+  }
   async register(options = { mode: "inline" }) {
     await this.onInitialize(options);
     if (options.mode === "worker") {
@@ -339,6 +366,7 @@ class AiProvider {
       for (const taskType of this.taskTypes) {
         registry.registerAsWorkerRunFn(this.name, taskType);
         registry.registerAsWorkerStreamFn(this.name, taskType);
+        registry.registerAsWorkerReactiveRunFn(this.name, taskType);
       }
     } else {
       for (const [taskType, fn] of Object.entries(this.tasks)) {
@@ -350,6 +378,11 @@ class AiProvider {
         }
       }
     }
+    if (this.reactiveTasks) {
+      for (const [taskType, fn] of Object.entries(this.reactiveTasks)) {
+        registry.registerReactiveRunFn(this.name, taskType, fn);
+      }
+    }
     registry.registerProvider(this);
     if (options.queue?.autoCreate !== false) {
       await this.createQueue(options.queue?.concurrency ?? 1);
@@ -367,6 +400,11 @@ class AiProvider {
         workerServer.registerStreamFunction(taskType, fn);
       }
     }
+    if (this.reactiveTasks) {
+      for (const [taskType, fn] of Object.entries(this.reactiveTasks)) {
+        workerServer.registerReactiveFunction(taskType, fn);
+      }
+    }
   }
   async onInitialize(_options) {}
   async dispose() {}
@@ -553,7 +591,7 @@ class AiTask extends JobQueueTask {
   static type = "AiTask";
   constructor(input = {}, config = {}) {
     const modelLabel = typeof input.model === "string" ? input.model : typeof input.model === "object" && input.model ? input.model.model_id || input.model.title || input.model.provider : undefined;
-    config.name ||= `${new.target.type || new.target.name}${modelLabel ? " with model " + modelLabel : ""}`;
+    config.title ||= `${new.target.type || new.target.name}${modelLabel ? " with model " + modelLabel : ""}`;
     super(input, config);
   }
   async getJobInput(input) {
@@ -576,7 +614,7 @@ class AiTask extends JobQueueTask {
     }
     const job = new AiJob({
       queueName: resolvedQueueName,
-      jobRunId: this.config.runnerId,
+      jobRunId: this.runConfig.runnerId,
       input: jobInput
     });
     return job;
@@ -585,6 +623,17 @@ class AiTask extends JobQueueTask {
     const model = input.model;
     return model?.provider;
   }
+  async executeReactive(input, output, context) {
+    const model = input.model;
+    if (model && typeof model === "object" && model.provider) {
+      const taskType = this.constructor.runtype ?? this.constructor.type;
+      const reactiveFn = getAiProviderRegistry().getReactiveRunFn(model.provider, taskType);
+      if (reactiveFn) {
+        return reactiveFn(input, output, model);
+      }
+    }
+    return super.executeReactive(input, output, context);
+  }
   async validateInput(input) {
     const inputSchema = this.inputSchema();
     if (typeof inputSchema === "boolean") {
@@ -1495,7 +1544,61 @@ var chunkVectorUpsert = (input, config) => {
 Workflow7.prototype.chunkVectorUpsert = CreateWorkflow7(ChunkVectorUpsertTask);
 
 // src/task/ContextBuilderTask.ts
-import { CreateWorkflow as CreateWorkflow8, Task as Task6, Workflow as Workflow8 } from "@workglow/task-graph";
+import { estimateTokens } from "@workglow/dataset";
+import {
+  CreateWorkflow as CreateWorkflow9,
+  Task as Task6,
+  Workflow as Workflow9
+} from "@workglow/task-graph";
+
+// src/task/CountTokensTask.ts
+import { CreateWorkflow as CreateWorkflow8, Workflow as Workflow8 } from "@workglow/task-graph";
+var modelSchema3 = TypeModel("model");
+var CountTokensInputSchema = {
+  type: "object",
+  properties: {
+    text: {
+      type: "string",
+      title: "Text",
+      description: "The text to count tokens for"
+    },
+    model: modelSchema3
+  },
+  required: ["text", "model"],
+  additionalProperties: false
+};
+var CountTokensOutputSchema = {
+  type: "object",
+  properties: {
+    count: {
+      type: "number",
+      title: "Token Count",
+      description: "The number of tokens in the text"
+    }
+  },
+  required: ["count"],
+  additionalProperties: false
+};
+
+class CountTokensTask extends AiTask {
+  static type = "CountTokensTask";
+  static category = "AI Text Model";
+  static title = "Count Tokens";
+  static description = "Counts the number of tokens in a text string using the model's tokenizer";
+  static cacheable = true;
+  static inputSchema() {
+    return CountTokensInputSchema;
+  }
+  static outputSchema() {
+    return CountTokensOutputSchema;
+  }
+}
+var countTokens = async (input, config) => {
+  return new CountTokensTask({}, config).run(input);
+};
+Workflow8.prototype.countTokens = CreateWorkflow8(CountTokensTask);
+
+// src/task/ContextBuilderTask.ts
 var ContextFormat = {
   SIMPLE: "simple",
   NUMBERED: "numbered",
@@ -1503,6 +1606,10 @@ var ContextFormat = {
   MARKDOWN: "markdown",
   JSON: "json"
 };
+var modelSchema4 = TypeModel("model", {
+  title: "Model",
+  description: "Model to use for token counting (optional, falls back to estimation)"
+});
 var inputSchema6 = {
   type: "object",
   properties: {
@@ -1542,6 +1649,13 @@ var inputSchema6 = {
       minimum: 0,
       default: 0
     },
+    maxTokens: {
+      type: "number",
+      title: "Max Tokens",
+      description: "Maximum number of tokens in context (0 = unlimited). Takes precedence over maxLength when set.",
+      minimum: 0,
+      default: 0
+    },
     includeMetadata: {
       type: "boolean",
       title: "Include Metadata",
@@ -1555,7 +1669,8 @@ var inputSchema6 = {
       default: `
 
 `
-    }
+    },
+    model: modelSchema4
   },
   required: ["chunks"],
   additionalProperties: false
@@ -1577,9 +1692,14 @@ var outputSchema6 = {
       type: "number",
       title: "Total Length",
       description: "Total length of context in characters"
+    },
+    totalTokens: {
+      type: "number",
+      title: "Total Tokens",
+      description: "Estimated token count of the context"
     }
   },
-  required: ["context", "chunksUsed", "totalLength"],
+  required: ["context", "chunksUsed", "totalLength", "totalTokens"],
   additionalProperties: false
 };
 
@@ -1595,49 +1715,77 @@ class ContextBuilderTask extends Task6 {
   static outputSchema() {
     return outputSchema6;
   }
-  async executeReactive(input, output) {
+  async executeReactive(input, _output, context) {
     const {
       chunks,
       metadata = [],
       scores = [],
       format = ContextFormat.SIMPLE,
       maxLength = 0,
+      maxTokens = 0,
       includeMetadata = false,
       separator = `
 
 `
     } = input;
-    let context = "";
+    let countFn = async (text) => estimateTokens(text);
+    if (input.model) {
+      const countTask = context.own(new CountTokensTask({ model: input.model }));
+      countFn = async (text) => {
+        try {
+          const result = await countTask.run({ text });
+          return result.count;
+        } catch (_err) {
+          return estimateTokens(text);
+        }
+      };
+    }
+    const useTokenBudget = maxTokens > 0;
+    let ctx = "";
     let chunksUsed = 0;
     for (let i = 0;i < chunks.length; i++) {
       const chunk = chunks[i];
       const meta = metadata[i];
       const score = scores[i];
       let formattedChunk = this.formatChunk(chunk, meta, score, i, format, includeMetadata);
-      if (maxLength > 0) {
-        const potentialLength = context.length + formattedChunk.length + separator.length;
-        if (potentialLength > maxLength) {
+      const prefix = chunksUsed > 0 ? separator : "";
+      const candidate = ctx + prefix + formattedChunk;
+      if (useTokenBudget) {
+        if (await countFn(candidate) > maxTokens) {
           if (chunksUsed === 0) {
-            const available = maxLength - context.length;
-            if (available > 100) {
+            let truncated = formattedChunk;
+            while (truncated.length > 10 && await countFn(truncated) > maxTokens) {
+              truncated = truncated.substring(0, Math.floor(truncated.length * 0.9));
+            }
+            if (truncated.length > 10) {
+              ctx = truncated.substring(0, truncated.length - 3) + "...";
+              chunksUsed++;
+            }
+          }
+          break;
+        }
+      } else if (maxLength > 0) {
+        if (candidate.length > maxLength) {
+          if (chunksUsed === 0) {
+            const available = maxLength - ctx.length;
+            if (available > 10) {
               formattedChunk = formattedChunk.substring(0, available - 3) + "...";
-              context += formattedChunk;
+              ctx += formattedChunk;
               chunksUsed++;
             }
           }
           break;
         }
       }
-      if (chunksUsed > 0) {
-        context += separator;
-      }
-      context += formattedChunk;
+      ctx = candidate;
       chunksUsed++;
     }
+    const totalTokens = await countFn(ctx);
     return {
-      context,
+      context: ctx,
       chunksUsed,
-      totalLength: context.length
+      totalLength: ctx.length,
+      totalTokens
     };
   }
   formatChunk(chunk, metadata, score, index, format, includeMetadata) {
@@ -1748,7 +1896,7 @@ class ContextBuilderTask extends Task6 {
 var contextBuilder = (input, config) => {
   return new ContextBuilderTask({}, config).run(input);
 };
-Workflow8.prototype.contextBuilder = CreateWorkflow8(ContextBuilderTask);
+Workflow9.prototype.contextBuilder = CreateWorkflow9(ContextBuilderTask);
 
 // src/task/DocumentEnricherTask.ts
 import {
@@ -1756,14 +1904,14 @@ import {
   hasChildren
 } from "@workglow/dataset";
 import {
-  CreateWorkflow as CreateWorkflow11,
+  CreateWorkflow as CreateWorkflow12,
   Task as Task7,
-  Workflow as Workflow11
+  Workflow as Workflow12
 } from "@workglow/task-graph";
 
 // src/task/TextNamedEntityRecognitionTask.ts
-import { CreateWorkflow as CreateWorkflow9, Workflow as Workflow9 } from "@workglow/task-graph";
-var modelSchema3 = TypeModel("model:TextNamedEntityRecognitionTask");
+import { CreateWorkflow as CreateWorkflow10, Workflow as Workflow10 } from "@workglow/task-graph";
+var modelSchema5 = TypeModel("model:TextNamedEntityRecognitionTask");
 var TextNamedEntityRecognitionInputSchema = {
   type: "object",
   properties: {
@@ -1782,7 +1930,7 @@ var TextNamedEntityRecognitionInputSchema = {
       "x-ui-group": "Configuration",
       "x-ui-group-open": false
     },
-    model: modelSchema3
+    model: modelSchema5
   },
   required: ["text", "model"],
   additionalProperties: false
@@ -1837,10 +1985,10 @@ class TextNamedEntityRecognitionTask extends AiTask {
 var textNamedEntityRecognition = (input, config) => {
   return new TextNamedEntityRecognitionTask({}, config).run(input);
 };
-Workflow9.prototype.textNamedEntityRecognition = CreateWorkflow9(TextNamedEntityRecognitionTask);
+Workflow10.prototype.textNamedEntityRecognition = CreateWorkflow10(TextNamedEntityRecognitionTask);
 
 // src/task/TextSummaryTask.ts
-import { CreateWorkflow as CreateWorkflow10, Workflow as Workflow10 } from "@workglow/task-graph";
+import { CreateWorkflow as CreateWorkflow11, Workflow as Workflow11 } from "@workglow/task-graph";
 
 // src/task/base/StreamingAiTask.ts
 import {
@@ -1853,7 +2001,7 @@ class StreamingAiTask extends AiTask {
     const queueName = await this.getDefaultQueueName(input);
     const job = new AiJob({
       queueName: queueName ?? this.type,
-      jobRunId: this.config.runnerId,
+      jobRunId: this.runConfig.runnerId,
       input: jobInput
     });
     const ports = getStreamingPorts(this.outputSchema());
@@ -1872,7 +2020,7 @@ class StreamingAiTask extends AiTask {
 }
 
 // src/task/TextSummaryTask.ts
-var modelSchema4 = TypeModel("model:TextSummaryTask");
+var modelSchema6 = TypeModel("model:TextSummaryTask");
 var TextSummaryInputSchema = {
   type: "object",
   properties: {
@@ -1881,7 +2029,7 @@ var TextSummaryInputSchema = {
       title: "Text",
       description: "The text to summarize"
     },
-    model: modelSchema4
+    model: modelSchema6
   },
   required: ["text", "model"],
   additionalProperties: false
@@ -1915,7 +2063,7 @@ class TextSummaryTask extends StreamingAiTask {
 var textSummary = async (input, config) => {
   return new TextSummaryTask({}, config).run(input);
 };
-Workflow10.prototype.textSummary = CreateWorkflow10(TextSummaryTask);
+Workflow11.prototype.textSummary = CreateWorkflow11(TextSummaryTask);
 
 // src/task/DocumentEnricherTask.ts
 var inputSchema7 = {
@@ -2146,15 +2294,15 @@ class DocumentEnricherTask extends Task7 {
 var documentEnricher = (input, config) => {
   return new DocumentEnricherTask({}, config).run(input);
 };
-Workflow11.prototype.documentEnricher = CreateWorkflow11(DocumentEnricherTask);
+Workflow12.prototype.documentEnricher = CreateWorkflow12(DocumentEnricherTask);
 
 // src/task/DownloadModelTask.ts
-import { CreateWorkflow as CreateWorkflow12, Workflow as Workflow12 } from "@workglow/task-graph";
-var modelSchema5 = TypeModel("model");
+import { CreateWorkflow as CreateWorkflow13, Workflow as Workflow13 } from "@workglow/task-graph";
+var modelSchema7 = TypeModel("model");
 var DownloadModelInputSchema = {
   type: "object",
   properties: {
-    model: modelSchema5
+    model: modelSchema7
   },
   required: ["model"],
   additionalProperties: false
@@ -2162,7 +2310,7 @@ var DownloadModelInputSchema = {
 var DownloadModelOutputSchema = {
   type: "object",
   properties: {
-    model: modelSchema5
+    model: modelSchema7
   },
   required: ["model"],
   additionalProperties: false
@@ -2205,11 +2353,11 @@ class DownloadModelTask extends AiTask {
 var downloadModel = (input, config) => {
   return new DownloadModelTask({}, config).run(input);
 };
-Workflow12.prototype.downloadModel = CreateWorkflow12(DownloadModelTask);
+Workflow13.prototype.downloadModel = CreateWorkflow13(DownloadModelTask);
 
 // src/task/FaceDetectorTask.ts
-import { CreateWorkflow as CreateWorkflow13, Workflow as Workflow13 } from "@workglow/task-graph";
-var modelSchema6 = TypeModel("model:FaceDetectorTask");
+import { CreateWorkflow as CreateWorkflow14, Workflow as Workflow14 } from "@workglow/task-graph";
+var modelSchema8 = TypeModel("model:FaceDetectorTask");
 var TypeBoundingBox2 = {
   type: "object",
   properties: {
@@ -2282,7 +2430,7 @@ var FaceDetectorInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema6,
+    model: modelSchema8,
     minDetectionConfidence: {
       type: "number",
       minimum: 0,
@@ -2336,11 +2484,11 @@ class FaceDetectorTask extends AiVisionTask {
 var faceDetector = (input, config) => {
   return new FaceDetectorTask({}, config).run(input);
 };
-Workflow13.prototype.faceDetector = CreateWorkflow13(FaceDetectorTask);
+Workflow14.prototype.faceDetector = CreateWorkflow14(FaceDetectorTask);
 
 // src/task/FaceLandmarkerTask.ts
-import { CreateWorkflow as CreateWorkflow14, Workflow as Workflow14 } from "@workglow/task-graph";
-var modelSchema7 = TypeModel("model:FaceLandmarkerTask");
+import { CreateWorkflow as CreateWorkflow15, Workflow as Workflow15 } from "@workglow/task-graph";
+var modelSchema9 = TypeModel("model:FaceLandmarkerTask");
 var TypeLandmark = {
   type: "object",
   properties: {
@@ -2412,7 +2560,7 @@ var FaceLandmarkerInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema7,
+    model: modelSchema9,
     numFaces: {
       type: "number",
       minimum: 1,
@@ -2498,11 +2646,11 @@ class FaceLandmarkerTask extends AiVisionTask {
 var faceLandmarker = (input, config) => {
   return new FaceLandmarkerTask({}, config).run(input);
 };
-Workflow14.prototype.faceLandmarker = CreateWorkflow14(FaceLandmarkerTask);
+Workflow15.prototype.faceLandmarker = CreateWorkflow15(FaceLandmarkerTask);
 
 // src/task/GestureRecognizerTask.ts
-import { CreateWorkflow as CreateWorkflow15, Workflow as Workflow15 } from "@workglow/task-graph";
-var modelSchema8 = TypeModel("model:GestureRecognizerTask");
+import { CreateWorkflow as CreateWorkflow16, Workflow as Workflow16 } from "@workglow/task-graph";
+var modelSchema10 = TypeModel("model:GestureRecognizerTask");
 var TypeLandmark2 = {
   type: "object",
   properties: {
@@ -2594,7 +2742,7 @@ var GestureRecognizerInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema8,
+    model: modelSchema10,
     numHands: {
       type: "number",
       minimum: 1,
@@ -2666,11 +2814,11 @@ class GestureRecognizerTask extends AiVisionTask {
 var gestureRecognizer = (input, config) => {
   return new GestureRecognizerTask({}, config).run(input);
 };
-Workflow15.prototype.gestureRecognizer = CreateWorkflow15(GestureRecognizerTask);
+Workflow16.prototype.gestureRecognizer = CreateWorkflow16(GestureRecognizerTask);
 
 // src/task/HandLandmarkerTask.ts
-import { CreateWorkflow as CreateWorkflow16, Workflow as Workflow16 } from "@workglow/task-graph";
-var modelSchema9 = TypeModel("model:HandLandmarkerTask");
+import { CreateWorkflow as CreateWorkflow17, Workflow as Workflow17 } from "@workglow/task-graph";
+var modelSchema11 = TypeModel("model:HandLandmarkerTask");
 var TypeLandmark3 = {
   type: "object",
   properties: {
@@ -2739,7 +2887,7 @@ var HandLandmarkerInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema9,
+    model: modelSchema11,
     numHands: {
       type: "number",
       minimum: 1,
@@ -2811,21 +2959,25 @@ class HandLandmarkerTask extends AiVisionTask {
 var handLandmarker = (input, config) => {
   return new HandLandmarkerTask({}, config).run(input);
 };
-Workflow16.prototype.handLandmarker = CreateWorkflow16(HandLandmarkerTask);
+Workflow17.prototype.handLandmarker = CreateWorkflow17(HandLandmarkerTask);
 
 // src/task/HierarchicalChunkerTask.ts
 import {
   ChunkNodeSchema as ChunkNodeSchema2,
-  estimateTokens,
+  estimateTokens as estimateTokens2,
   getChildren as getChildren2,
   hasChildren as hasChildren2
 } from "@workglow/dataset";
 import {
-  CreateWorkflow as CreateWorkflow17,
+  CreateWorkflow as CreateWorkflow18,
   Task as Task8,
-  Workflow as Workflow17
+  Workflow as Workflow18
 } from "@workglow/task-graph";
 import { uuid4 } from "@workglow/util";
+var modelSchema12 = TypeModel("model", {
+  title: "Model",
+  description: "Model to use for token counting"
+});
 var inputSchema8 = {
   type: "object",
   properties: {
@@ -2865,9 +3017,10 @@ var inputSchema8 = {
       title: "Chunking Strategy",
       description: "Strategy for chunking",
       default: "hierarchical"
-    }
+    },
+    model: modelSchema12
   },
-  required: [],
+  required: ["doc_id", "documentTree"],
   additionalProperties: false
 };
 var outputSchema8 = {
@@ -2933,11 +3086,23 @@ class HierarchicalChunkerTask extends Task8 {
       overlapTokens: overlap,
       reservedTokens
     };
+    let countFn = async (text) => estimateTokens2(text);
+    if (input.model) {
+      const countTask = context.own(new CountTokensTask({ model: input.model }));
+      countFn = async (text) => {
+        try {
+          const result = await countTask.run({ text });
+          return result.count;
+        } catch (_err) {
+          return estimateTokens2(text);
+        }
+      };
+    }
     const chunks = [];
     if (strategy === "hierarchical") {
-      await this.chunkHierarchically(root, [], doc_id, tokenBudget, chunks);
+      await this.chunkHierarchically(root, [], doc_id, tokenBudget, chunks, countFn);
     } else {
-      await this.chunkFlat(root, doc_id, tokenBudget, chunks);
+      await this.chunkFlat(root, doc_id, tokenBudget, chunks, countFn);
     }
     return {
       doc_id,
@@ -2946,24 +3111,30 @@ class HierarchicalChunkerTask extends Task8 {
       count: chunks.length
     };
   }
-  async chunkHierarchically(node, nodePath, doc_id, tokenBudget, chunks) {
+  async chunkHierarchically(node, nodePath, doc_id, tokenBudget, chunks, countFn) {
     const currentPath = [...nodePath, node.nodeId];
     if (!hasChildren2(node)) {
-      await this.chunkText(node.text, currentPath, doc_id, tokenBudget, chunks, node.nodeId);
+      await this.chunkText(node.text, currentPath, doc_id, tokenBudget, chunks, node.nodeId, countFn);
       return;
     }
     const children = getChildren2(node);
     for (const child of children) {
-      await this.chunkHierarchically(child, currentPath, doc_id, tokenBudget, chunks);
+      await this.chunkHierarchically(child, currentPath, doc_id, tokenBudget, chunks, countFn);
     }
   }
-  async chunkText(text, nodePath, doc_id, tokenBudget, chunks, leafNodeId) {
-    const maxChars = (tokenBudget.maxTokensPerChunk - tokenBudget.reservedTokens) * 4;
-    const overlapChars = tokenBudget.overlapTokens * 4;
-    if (estimateTokens(text) <= tokenBudget.maxTokensPerChunk - tokenBudget.reservedTokens) {
-      const chunkId = uuid4();
+  async chunkText(text, nodePath, doc_id, tokenBudget, chunks, leafNodeId, countFn) {
+    const maxTokens = tokenBudget.maxTokensPerChunk - tokenBudget.reservedTokens;
+    const overlapTokens = tokenBudget.overlapTokens;
+    if (maxTokens <= 0) {
+      throw new Error(`Invalid token budget: reservedTokens (${tokenBudget.reservedTokens}) must be less than maxTokensPerChunk (${tokenBudget.maxTokensPerChunk})`);
+    }
+    if (overlapTokens >= maxTokens) {
+      throw new Error(`Invalid token budget: overlapTokens (${overlapTokens}) must be less than effective maxTokens (${maxTokens})`);
+    }
+    const count = await countFn(text);
+    if (count <= maxTokens) {
       chunks.push({
-        chunkId,
+        chunkId: uuid4(),
         doc_id,
         text,
         nodePath,
@@ -2971,29 +3142,40 @@ class HierarchicalChunkerTask extends Task8 {
       });
       return;
     }
-    let chunkOrdinal = 0;
+    const findCharBoundary = async (startChar, targetTokens) => {
+      let lo = startChar;
+      let hi = Math.min(startChar + targetTokens * 6, text.length);
+      while (lo < hi) {
+        const mid = Math.ceil((lo + hi) / 2);
+        const count2 = await countFn(text.substring(startChar, mid));
+        if (count2 <= targetTokens) {
+          lo = mid;
+        } else {
+          hi = mid - 1;
+        }
+      }
+      return lo;
+    };
     let startOffset = 0;
     while (startOffset < text.length) {
-      const endOffset = Math.min(startOffset + maxChars, text.length);
-      const chunkText = text.substring(startOffset, endOffset);
-      const chunkId = uuid4();
+      const boundary = await findCharBoundary(startOffset, maxTokens);
+      const endOffset = Math.max(Math.min(boundary, text.length), startOffset + 1);
       chunks.push({
-        chunkId,
+        chunkId: uuid4(),
         doc_id,
-        text: chunkText,
+        text: text.substring(startOffset, endOffset),
         nodePath,
         depth: nodePath.length
       });
-      chunkOrdinal++;
-      startOffset += maxChars - overlapChars;
-      if (overlapChars >= maxChars) {
-        startOffset = endOffset;
-      }
+      if (endOffset >= text.length)
+        break;
+      const nextStart = await findCharBoundary(startOffset, maxTokens - overlapTokens);
+      startOffset = nextStart > startOffset ? nextStart : endOffset;
     }
   }
-  async chunkFlat(root, doc_id, tokenBudget, chunks) {
+  async chunkFlat(root, doc_id, tokenBudget, chunks, countFn) {
     const allText = this.collectAllText(root);
-    await this.chunkText(allText, [root.nodeId], doc_id, tokenBudget, chunks, root.nodeId);
+    await this.chunkText(allText, [root.nodeId], doc_id, tokenBudget, chunks, root.nodeId, countFn);
   }
   collectAllText(node) {
     const texts = [];
@@ -3015,7 +3197,7 @@ class HierarchicalChunkerTask extends Task8 {
 var hierarchicalChunker = (input, config) => {
   return new HierarchicalChunkerTask({}, config).run(input);
 };
-Workflow17.prototype.hierarchicalChunker = CreateWorkflow17(HierarchicalChunkerTask);
+Workflow18.prototype.hierarchicalChunker = CreateWorkflow18(HierarchicalChunkerTask);
 
 // src/task/HierarchyJoinTask.ts
 import {
@@ -3024,9 +3206,9 @@ import {
   TypeDocumentDataset
 } from "@workglow/dataset";
 import {
-  CreateWorkflow as CreateWorkflow18,
+  CreateWorkflow as CreateWorkflow19,
   Task as Task9,
-  Workflow as Workflow18
+  Workflow as Workflow19
 } from "@workglow/task-graph";
 var inputSchema9 = {
   type: "object",
@@ -3196,16 +3378,16 @@ class HierarchyJoinTask extends Task9 {
 var hierarchyJoin = (input, config) => {
   return new HierarchyJoinTask({}, config).run(input);
 };
-Workflow18.prototype.hierarchyJoin = CreateWorkflow18(HierarchyJoinTask);
+Workflow19.prototype.hierarchyJoin = CreateWorkflow19(HierarchyJoinTask);
 
 // src/task/ImageClassificationTask.ts
-import { CreateWorkflow as CreateWorkflow19, Workflow as Workflow19 } from "@workglow/task-graph";
-var modelSchema10 = TypeModel("model:ImageClassificationTask");
+import { CreateWorkflow as CreateWorkflow20, Workflow as Workflow20 } from "@workglow/task-graph";
+var modelSchema13 = TypeModel("model:ImageClassificationTask");
 var ImageClassificationInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema10,
+    model: modelSchema13,
     categories: {
       type: "array",
       items: {
@@ -3259,14 +3441,14 @@ class ImageClassificationTask extends AiVisionTask {
 var imageClassification = (input, config) => {
   return new ImageClassificationTask({}, config).run(input);
 };
-Workflow19.prototype.imageClassification = CreateWorkflow19(ImageClassificationTask);
+Workflow20.prototype.imageClassification = CreateWorkflow20(ImageClassificationTask);
 
 // src/task/ImageEmbeddingTask.ts
-import { CreateWorkflow as CreateWorkflow20, Workflow as Workflow20 } from "@workglow/task-graph";
+import { CreateWorkflow as CreateWorkflow21, Workflow as Workflow21 } from "@workglow/task-graph";
 import {
   TypedArraySchema as TypedArraySchema7
 } from "@workglow/util";
-var modelSchema11 = TypeModel("model:ImageEmbeddingTask");
+var modelSchema14 = TypeModel("model:ImageEmbeddingTask");
 var embeddingSchema = TypedArraySchema7({
   title: "Embedding",
   description: "The image embedding vector"
@@ -3275,7 +3457,7 @@ var ImageEmbeddingInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema11
+    model: modelSchema14
   },
   required: ["image", "model"],
   additionalProperties: false
@@ -3304,16 +3486,16 @@ class ImageEmbeddingTask extends AiVisionTask {
 var imageEmbedding = (input, config) => {
   return new ImageEmbeddingTask({}, config).run(input);
 };
-Workflow20.prototype.imageEmbedding = CreateWorkflow20(ImageEmbeddingTask);
+Workflow21.prototype.imageEmbedding = CreateWorkflow21(ImageEmbeddingTask);
 
 // src/task/ImageSegmentationTask.ts
-import { CreateWorkflow as CreateWorkflow21, Workflow as Workflow21 } from "@workglow/task-graph";
-var modelSchema12 = TypeModel("model:ImageSegmentationTask");
+import { CreateWorkflow as CreateWorkflow22, Workflow as Workflow22 } from "@workglow/task-graph";
+var modelSchema15 = TypeModel("model:ImageSegmentationTask");
 var ImageSegmentationInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema12,
+    model: modelSchema15,
     threshold: {
       type: "number",
       title: "Threshold",
@@ -3392,11 +3574,11 @@ class ImageSegmentationTask extends AiVisionTask {
 var imageSegmentation = (input, config) => {
   return new ImageSegmentationTask({}, config).run(input);
 };
-Workflow21.prototype.imageSegmentation = CreateWorkflow21(ImageSegmentationTask);
+Workflow22.prototype.imageSegmentation = CreateWorkflow22(ImageSegmentationTask);
 
 // src/task/ImageToTextTask.ts
-import { CreateWorkflow as CreateWorkflow22, Workflow as Workflow22 } from "@workglow/task-graph";
-var modelSchema13 = TypeModel("model:ImageToTextTask");
+import { CreateWorkflow as CreateWorkflow23, Workflow as Workflow23 } from "@workglow/task-graph";
+var modelSchema16 = TypeModel("model:ImageToTextTask");
 var generatedTextSchema = {
   type: "string",
   title: "Text",
@@ -3406,7 +3588,7 @@ var ImageToTextInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema13,
+    model: modelSchema16,
     maxTokens: {
       type: "number",
       title: "Max Tokens",
@@ -3447,11 +3629,11 @@ class ImageToTextTask extends AiVisionTask {
 var imageToText = (input, config) => {
   return new ImageToTextTask({}, config).run(input);
 };
-Workflow22.prototype.imageToText = CreateWorkflow22(ImageToTextTask);
+Workflow23.prototype.imageToText = CreateWorkflow23(ImageToTextTask);
 
 // src/task/ObjectDetectionTask.ts
-import { CreateWorkflow as CreateWorkflow23, Workflow as Workflow23 } from "@workglow/task-graph";
-var modelSchema14 = TypeModel("model:ObjectDetectionTask");
+import { CreateWorkflow as CreateWorkflow24, Workflow as Workflow24 } from "@workglow/task-graph";
+var modelSchema17 = TypeModel("model:ObjectDetectionTask");
 var detectionSchema = {
   type: "object",
   properties: {
@@ -3476,7 +3658,7 @@ var ObjectDetectionInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema14,
+    model: modelSchema17,
     labels: {
       type: "array",
       items: {
@@ -3530,11 +3712,11 @@ class ObjectDetectionTask extends AiVisionTask {
 var objectDetection = (input, config) => {
   return new ObjectDetectionTask({}, config).run(input);
 };
-Workflow23.prototype.objectDetection = CreateWorkflow23(ObjectDetectionTask);
+Workflow24.prototype.objectDetection = CreateWorkflow24(ObjectDetectionTask);
 
 // src/task/PoseLandmarkerTask.ts
-import { CreateWorkflow as CreateWorkflow24, Workflow as Workflow24 } from "@workglow/task-graph";
-var modelSchema15 = TypeModel("model:PoseLandmarkerTask");
+import { CreateWorkflow as CreateWorkflow25, Workflow as Workflow25 } from "@workglow/task-graph";
+var modelSchema18 = TypeModel("model:PoseLandmarkerTask");
 var TypePoseLandmark = {
   type: "object",
   properties: {
@@ -3613,7 +3795,7 @@ var PoseLandmarkerInputSchema = {
   type: "object",
   properties: {
     image: TypeImageInput,
-    model: modelSchema15,
+    model: modelSchema18,
     numPoses: {
       type: "number",
       minimum: 1,
@@ -3692,13 +3874,13 @@ class PoseLandmarkerTask extends AiVisionTask {
 var poseLandmarker = (input, config) => {
   return new PoseLandmarkerTask({}, config).run(input);
 };
-Workflow24.prototype.poseLandmarker = CreateWorkflow24(PoseLandmarkerTask);
+Workflow25.prototype.poseLandmarker = CreateWorkflow25(PoseLandmarkerTask);
 
 // src/task/QueryExpanderTask.ts
 import {
-  CreateWorkflow as CreateWorkflow25,
+  CreateWorkflow as CreateWorkflow26,
   Task as Task10,
-  Workflow as Workflow25
+  Workflow as Workflow26
 } from "@workglow/task-graph";
 var QueryExpansionMethod = {
   MULTI_QUERY: "multi-query",
@@ -3909,18 +4091,18 @@ class QueryExpanderTask extends Task10 {
 var queryExpander = (input, config) => {
   return new QueryExpanderTask({}, config).run(input);
 };
-Workflow25.prototype.queryExpander = CreateWorkflow25(QueryExpanderTask);
+Workflow26.prototype.queryExpander = CreateWorkflow26(QueryExpanderTask);
 
 // src/task/RerankerTask.ts
 import {
-  CreateWorkflow as CreateWorkflow27,
+  CreateWorkflow as CreateWorkflow28,
   Task as Task11,
-  Workflow as Workflow27
+  Workflow as Workflow28
 } from "@workglow/task-graph";
 
 // src/task/TextClassificationTask.ts
-import { CreateWorkflow as CreateWorkflow26, Workflow as Workflow26 } from "@workglow/task-graph";
-var modelSchema16 = TypeModel("model:TextClassificationTask");
+import { CreateWorkflow as CreateWorkflow27, Workflow as Workflow27 } from "@workglow/task-graph";
+var modelSchema19 = TypeModel("model:TextClassificationTask");
 var TextClassificationInputSchema = {
   type: "object",
   properties: {
@@ -3947,7 +4129,7 @@ var TextClassificationInputSchema = {
       description: "The maximum number of categories to return",
       "x-ui-group": "Configuration"
     },
-    model: modelSchema16
+    model: modelSchema19
   },
   required: ["text", "model"],
   additionalProperties: false
@@ -3997,7 +4179,7 @@ class TextClassificationTask extends AiTask {
 var textClassification = (input, config) => {
   return new TextClassificationTask({}, config).run(input);
 };
-Workflow26.prototype.textClassification = CreateWorkflow26(TextClassificationTask);
+Workflow27.prototype.textClassification = CreateWorkflow27(TextClassificationTask);
 
 // src/task/RerankerTask.ts
 var inputSchema11 = {
@@ -4221,14 +4403,14 @@ class RerankerTask extends Task11 {
 var reranker = (input, config) => {
   return new RerankerTask({}, config).run(input);
 };
-Workflow27.prototype.reranker = CreateWorkflow27(RerankerTask);
+Workflow28.prototype.reranker = CreateWorkflow28(RerankerTask);
 
 // src/task/StructuralParserTask.ts
 import { StructuralParser } from "@workglow/dataset";
 import {
-  CreateWorkflow as CreateWorkflow28,
+  CreateWorkflow as CreateWorkflow29,
   Task as Task12,
-  Workflow as Workflow28
+  Workflow as Workflow29
 } from "@workglow/task-graph";
 import { uuid4 as uuid42 } from "@workglow/util";
 var inputSchema12 = {
@@ -4330,13 +4512,13 @@ class StructuralParserTask extends Task12 {
 var structuralParser = (input, config) => {
   return new StructuralParserTask({}, config).run(input);
 };
-Workflow28.prototype.structuralParser = CreateWorkflow28(StructuralParserTask);
+Workflow29.prototype.structuralParser = CreateWorkflow29(StructuralParserTask);
 
 // src/task/TextChunkerTask.ts
 import {
-  CreateWorkflow as CreateWorkflow29,
+  CreateWorkflow as CreateWorkflow30,
   Task as Task13,
-  Workflow as Workflow29
+  Workflow as Workflow30
 } from "@workglow/task-graph";
 var ChunkingStrategy = {
   FIXED: "fixed",
@@ -4586,11 +4768,11 @@ class TextChunkerTask extends Task13 {
 var textChunker = (input, config) => {
   return new TextChunkerTask({}, config).run(input);
 };
-Workflow29.prototype.textChunker = CreateWorkflow29(TextChunkerTask);
+Workflow30.prototype.textChunker = CreateWorkflow30(TextChunkerTask);
 
 // src/task/TextFillMaskTask.ts
-import { CreateWorkflow as CreateWorkflow30, Workflow as Workflow30 } from "@workglow/task-graph";
-var modelSchema17 = TypeModel("model:TextFillMaskTask");
+import { CreateWorkflow as CreateWorkflow31, Workflow as Workflow31 } from "@workglow/task-graph";
+var modelSchema20 = TypeModel("model:TextFillMaskTask");
 var TextFillMaskInputSchema = {
   type: "object",
   properties: {
@@ -4599,7 +4781,7 @@ var TextFillMaskInputSchema = {
       title: "Text",
       description: "The text with a mask token to fill"
     },
-    model: modelSchema17
+    model: modelSchema20
   },
   required: ["text", "model"],
   additionalProperties: false
@@ -4654,21 +4836,21 @@ class TextFillMaskTask extends AiTask {
 var textFillMask = (input, config) => {
   return new TextFillMaskTask({}, config).run(input);
 };
-Workflow30.prototype.textFillMask = CreateWorkflow30(TextFillMaskTask);
+Workflow31.prototype.textFillMask = CreateWorkflow31(TextFillMaskTask);
 
 // src/task/TextGenerationTask.ts
-import { CreateWorkflow as CreateWorkflow31, Workflow as Workflow31 } from "@workglow/task-graph";
+import { CreateWorkflow as CreateWorkflow32, Workflow as Workflow32 } from "@workglow/task-graph";
 var generatedTextSchema2 = {
   type: "string",
   title: "Text",
   description: "The generated text",
   "x-stream": "append"
 };
-var modelSchema18 = TypeModel("model:TextGenerationTask");
+var modelSchema21 = TypeModel("model:TextGenerationTask");
 var TextGenerationInputSchema = {
   type: "object",
   properties: {
-    model: modelSchema18,
+    model: modelSchema21,
     prompt: {
       type: "string",
       title: "Prompt",
@@ -4742,11 +4924,11 @@ class TextGenerationTask extends StreamingAiTask {
 var textGeneration = (input, config) => {
   return new TextGenerationTask({}, config).run(input);
 };
-Workflow31.prototype.textGeneration = CreateWorkflow31(TextGenerationTask);
+Workflow32.prototype.textGeneration = CreateWorkflow32(TextGenerationTask);
 
 // src/task/TextLanguageDetectionTask.ts
-import { CreateWorkflow as CreateWorkflow32, Workflow as Workflow32 } from "@workglow/task-graph";
-var modelSchema19 = TypeModel("model:TextLanguageDetectionTask");
+import { CreateWorkflow as CreateWorkflow33, Workflow as Workflow33 } from "@workglow/task-graph";
+var modelSchema22 = TypeModel("model:TextLanguageDetectionTask");
 var TextLanguageDetectionInputSchema = {
   type: "object",
   properties: {
@@ -4763,7 +4945,7 @@ var TextLanguageDetectionInputSchema = {
       title: "Max Languages",
       description: "The maximum number of languages to return"
     },
-    model: modelSchema19
+    model: modelSchema22
   },
   required: ["text", "model"],
   additionalProperties: false
@@ -4813,10 +4995,10 @@ class TextLanguageDetectionTask extends AiTask {
 var textLanguageDetection = (input, config) => {
   return new TextLanguageDetectionTask({}, config).run(input);
 };
-Workflow32.prototype.textLanguageDetection = CreateWorkflow32(TextLanguageDetectionTask);
+Workflow33.prototype.textLanguageDetection = CreateWorkflow33(TextLanguageDetectionTask);
 
 // src/task/TextQuestionAnswerTask.ts
-import { CreateWorkflow as CreateWorkflow33, Workflow as Workflow33 } from "@workglow/task-graph";
+import { CreateWorkflow as CreateWorkflow34, Workflow as Workflow34 } from "@workglow/task-graph";
 var contextSchema = {
   type: "string",
   title: "Context",
@@ -4833,13 +5015,13 @@ var textSchema = {
   description: "The generated text",
   "x-stream": "append"
 };
-var modelSchema20 = TypeModel("model:TextQuestionAnswerTask");
+var modelSchema23 = TypeModel("model:TextQuestionAnswerTask");
 var TextQuestionAnswerInputSchema = {
   type: "object",
   properties: {
     context: contextSchema,
     question: questionSchema,
-    model: modelSchema20
+    model: modelSchema23
   },
   required: ["context", "question", "model"],
   additionalProperties: false
@@ -4868,11 +5050,11 @@ class TextQuestionAnswerTask extends StreamingAiTask {
 var textQuestionAnswer = (input, config) => {
   return new TextQuestionAnswerTask({}, config).run(input);
 };
-Workflow33.prototype.textQuestionAnswer = CreateWorkflow33(TextQuestionAnswerTask);
+Workflow34.prototype.textQuestionAnswer = CreateWorkflow34(TextQuestionAnswerTask);
 
 // src/task/TextRewriterTask.ts
-import { CreateWorkflow as CreateWorkflow34, Workflow as Workflow34 } from "@workglow/task-graph";
-var modelSchema21 = TypeModel("model:TextRewriterTask");
+import { CreateWorkflow as CreateWorkflow35, Workflow as Workflow35 } from "@workglow/task-graph";
+var modelSchema24 = TypeModel("model:TextRewriterTask");
 var TextRewriterInputSchema = {
   type: "object",
   properties: {
@@ -4886,7 +5068,7 @@ var TextRewriterInputSchema = {
       title: "Prompt",
       description: "The prompt to direct the rewriting"
     },
-    model: modelSchema21
+    model: modelSchema24
   },
   required: ["text", "prompt", "model"],
   additionalProperties: false
@@ -4920,11 +5102,11 @@ class TextRewriterTask extends StreamingAiTask {
 var textRewriter = (input, config) => {
   return new TextRewriterTask({}, config).run(input);
 };
-Workflow34.prototype.textRewriter = CreateWorkflow34(TextRewriterTask);
+Workflow35.prototype.textRewriter = CreateWorkflow35(TextRewriterTask);
 
 // src/task/TextTranslationTask.ts
-import { CreateWorkflow as CreateWorkflow35, Workflow as Workflow35 } from "@workglow/task-graph";
-var modelSchema22 = TypeModel("model:TextTranslationTask");
+import { CreateWorkflow as CreateWorkflow36, Workflow as Workflow36 } from "@workglow/task-graph";
+var modelSchema25 = TypeModel("model:TextTranslationTask");
 var translationTextSchema = {
   type: "string",
   title: "Text",
@@ -4951,7 +5133,7 @@ var TextTranslationInputSchema = {
       minLength: 2,
       maxLength: 2
     }),
-    model: modelSchema22
+    model: modelSchema25
   },
   required: ["text", "source_lang", "target_lang", "model"],
   additionalProperties: false
@@ -4986,13 +5168,13 @@ class TextTranslationTask extends StreamingAiTask {
 var textTranslation = (input, config) => {
   return new TextTranslationTask({}, config).run(input);
 };
-Workflow35.prototype.textTranslation = CreateWorkflow35(TextTranslationTask);
+Workflow36.prototype.textTranslation = CreateWorkflow36(TextTranslationTask);
 
 // src/task/TopicSegmenterTask.ts
 import {
-  CreateWorkflow as CreateWorkflow36,
+  CreateWorkflow as CreateWorkflow37,
   Task as Task14,
-  Workflow as Workflow36
+  Workflow as Workflow37
 } from "@workglow/task-graph";
 var SegmentationMethod = {
   HEURISTIC: "heuristic",
@@ -5273,15 +5455,15 @@ class TopicSegmenterTask extends Task14 {
 var topicSegmenter = (input, config) => {
   return new TopicSegmenterTask({}, config).run(input);
 };
-Workflow36.prototype.topicSegmenter = CreateWorkflow36(TopicSegmenterTask);
+Workflow37.prototype.topicSegmenter = CreateWorkflow37(TopicSegmenterTask);
 
 // src/task/UnloadModelTask.ts
-import { CreateWorkflow as CreateWorkflow37, Workflow as Workflow37 } from "@workglow/task-graph";
-var modelSchema23 = TypeModel("model");
+import { CreateWorkflow as CreateWorkflow38, Workflow as Workflow38 } from "@workglow/task-graph";
+var modelSchema26 = TypeModel("model");
 var UnloadModelInputSchema = {
   type: "object",
   properties: {
-    model: modelSchema23
+    model: modelSchema26
   },
   required: ["model"],
   additionalProperties: false
@@ -5289,7 +5471,7 @@ var UnloadModelInputSchema = {
 var UnloadModelOutputSchema = {
   type: "object",
   properties: {
-    model: modelSchema23
+    model: modelSchema26
   },
   required: ["model"],
   additionalProperties: false
@@ -5311,10 +5493,10 @@ class UnloadModelTask extends AiTask {
 var unloadModel = (input, config) => {
   return new UnloadModelTask({}, config).run(input);
 };
-Workflow37.prototype.unloadModel = CreateWorkflow37(UnloadModelTask);
+Workflow38.prototype.unloadModel = CreateWorkflow38(UnloadModelTask);
 
 // src/task/VectorQuantizeTask.ts
-import { CreateWorkflow as CreateWorkflow38, Task as Task15, Workflow as Workflow38 } from "@workglow/task-graph";
+import { CreateWorkflow as CreateWorkflow39, Task as Task15, Workflow as Workflow39 } from "@workglow/task-graph";
 import {
   normalizeNumberArray,
   TensorType,
@@ -5494,10 +5676,10 @@ class VectorQuantizeTask extends Task15 {
 var vectorQuantize = (input, config) => {
   return new VectorQuantizeTask({}, config).run(input);
 };
-Workflow38.prototype.vectorQuantize = CreateWorkflow38(VectorQuantizeTask);
+Workflow39.prototype.vectorQuantize = CreateWorkflow39(VectorQuantizeTask);
 
 // src/task/VectorSimilarityTask.ts
-import { CreateWorkflow as CreateWorkflow39, GraphAsTask, Workflow as Workflow39 } from "@workglow/task-graph";
+import { CreateWorkflow as CreateWorkflow40, GraphAsTask, Workflow as Workflow40 } from "@workglow/task-graph";
 import {
   cosineSimilarity,
   hammingSimilarity,
@@ -5603,13 +5785,14 @@ class VectorSimilarityTask extends GraphAsTask {
 var similarity = (input, config) => {
   return new VectorSimilarityTask({}, config).run(input);
 };
-Workflow39.prototype.similarity = CreateWorkflow39(VectorSimilarityTask);
+Workflow40.prototype.similarity = CreateWorkflow40(VectorSimilarityTask);
 
 // src/task/index.ts
 var registerAiTasks = () => {
   const tasks = [
     BackgroundRemovalTask,
     ChunkToVectorTask,
+    CountTokensTask,
     ContextBuilderTask,
     DocumentEnricherTask,
     ChunkRetrievalTask,
@@ -5691,6 +5874,7 @@ export {
   faceDetector,
   downloadModel,
   documentEnricher,
+  countTokens,
   contextBuilder,
   chunkVectorUpsert,
   chunkToVector,
@@ -5787,6 +5971,9 @@ export {
   FaceDetectorInputSchema,
   DownloadModelTask,
   DocumentEnricherTask,
+  CountTokensTask,
+  CountTokensOutputSchema,
+  CountTokensInputSchema,
   ContextFormat,
   ContextBuilderTask,
   ChunkingStrategy,
@@ -5804,4 +5991,4 @@ export {
   AiJob
 };
 
-//# debugId=E3FAD6BE2442A4DE64756E2164756E21
+//# debugId=8EB68A2DA4788C9664756E2164756E21