@workglow/task-graph 0.2.17 → 0.2.19

package/dist/common.d.ts

@@ -20,9 +20,12 @@ export * from "./task-graph/TransformRegistry";
 export * from "./task-graph/TransformTypes";
 export * from "./task-graph/transforms";
 export * from "./task-graph/autoConnect";
+export { registerRefcountablePredicate, asRefcountable, _resetRefcountablePredicatesForTests, type Refcountable, } from "./refcountable";
 export * from "./task";
 export * from "./storage/TaskGraphRepository";
 export * from "./storage/TaskGraphTabularRepository";
 export * from "./storage/TaskOutputRepository";
 export * from "./storage/TaskOutputTabularRepository";
+export { registerPortCodec, getPortCodec, _resetPortCodecsForTests } from "./storage/PortCodecRegistry";
+export type { PortCodec } from "./storage/PortCodecRegistry";
 //# sourceMappingURL=common.d.ts.map

package/dist/common.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,uBAAuB,CAAC;AACtC,cAAc,6BAA6B,CAAC;AAE5C,cAAc,oCAAoC,CAAC;AACnD,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAE7C,cAAc,0BAA0B,CAAC;AACzC,cAAc,kCAAkC,CAAC;AACjD,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AAEtC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AAEzC,cAAc,QAAQ,CAAC;AAEvB,cAAc,+BAA+B,CAAC;AAC9C,cAAc,sCAAsC,CAAC;AACrD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,uCAAuC,CAAC"}
+{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,uBAAuB,CAAC;AACtC,cAAc,6BAA6B,CAAC;AAE5C,cAAc,oCAAoC,CAAC;AACnD,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAE7C,cAAc,0BAA0B,CAAC;AACzC,cAAc,kCAAkC,CAAC;AACjD,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AAEtC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AAEzC,OAAO,EACL,6BAA6B,EAC7B,cAAc,EACd,oCAAoC,EACpC,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAExB,cAAc,QAAQ,CAAC;AAEvB,cAAc,+BAA+B,CAAC;AAC9C,cAAc,sCAAsC,CAAC;AACrD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,uCAAuC,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AACxG,YAAY,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC"}

package/dist/node.js

@@ -584,6 +584,10 @@ function computeGraphEntitlements(graph, options) {
 }
 // src/task/InputResolver.ts
 import { getInputResolvers } from "@workglow/util";
+function isPlainObject(value) {
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
 function getSchemaFormat(schema, visited = new WeakSet) {
   if (typeof schema !== "object" || schema === null)
     return;
@@ -671,6 +675,7 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
   for (const [key, propSchema] of Object.entries(properties)) {
     let value = resolved[key];
     const format = getSchemaFormat(propSchema);
+    let phase1Transformed = false;
     if (format) {
       let resolver = resolvers.get(format);
       if (!resolver) {
@@ -681,14 +686,18 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
         if (typeof value === "string") {
           value = await resolver(value, format, config.registry);
           resolved[key] = value;
+          phase1Transformed = true;
         } else if (Array.isArray(value) && value.some((item) => typeof item === "string")) {
           const results = await Promise.all(value.map((item) => typeof item === "string" ? resolver(item, format, config.registry) : item));
           value = results.filter((result) => result !== undefined);
           resolved[key] = value;
+          phase1Transformed = true;
         }
       }
     }
-    if (value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value)) {
+    const hasFormatResolver = format ? !!(resolvers.get(format) ?? resolvers.get(getFormatPrefix(format))) : false;
+    const skipPhase2 = hasFormatResolver && !phase1Transformed;
+    if (!skipPhase2 && value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value) && isPlainObject(value)) {
       const objectSchema = getObjectSchema(propSchema);
       if (objectSchema && !visited.has(objectSchema)) {
         visited.add(objectSchema);
@@ -1134,6 +1143,31 @@ import {
   uuid4 as uuid43
 } from "@workglow/util";
 
+// src/refcountable.ts
+var GLOBAL_KEY = Symbol.for("@workglow/task-graph/refcountable.predicates");
+var _g = globalThis;
+if (!Array.isArray(_g[GLOBAL_KEY])) {
+  _g[GLOBAL_KEY] = [];
+}
+var predicates = _g[GLOBAL_KEY];
+function registerRefcountablePredicate(p) {
+  predicates.push(p);
+}
+function asRefcountable(v) {
+  if (v === null || v === undefined)
+    return null;
+  if (typeof v !== "object")
+    return null;
+  for (const p of predicates) {
+    if (p(v))
+      return v;
+  }
+  return null;
+}
+function _resetRefcountablePredicatesForTests() {
+  predicates.length = 0;
+}
+
 // src/storage/TaskOutputRepository.ts
 import { createServiceToken as createServiceToken2, EventEmitter as EventEmitter2 } from "@workglow/util";
 var TASK_OUTPUT_REPOSITORY = createServiceToken2("taskgraph.taskOutputRepository");
@@ -1245,6 +1279,7 @@ import {
   globalServiceRegistry as globalServiceRegistry2,
   SpanStatusCode
 } from "@workglow/util";
+import { getPortCodec } from "@workglow/util";
 
 // src/task/StreamTypes.ts
 function getPortStreamMode(schema, portId) {
@@ -1348,6 +1383,42 @@ function hasStructuredOutput(schema) {
 }
 
 // src/task/TaskRunner.ts
+async function serializeOutputPorts(output, schema) {
+  if (!schema?.properties)
+    return output;
+  const out = { ...output };
+  for (const [key, prop] of Object.entries(schema.properties)) {
+    const codec = prop.format ? getPortCodec(prop.format) : undefined;
+    if (codec && out[key] !== undefined) {
+      out[key] = await codec.serialize(out[key]);
+    }
+  }
+  return out;
+}
+async function deserializeOutputPorts(output, schema) {
+  if (!schema?.properties)
+    return output;
+  const out = { ...output };
+  for (const [key, prop] of Object.entries(schema.properties)) {
+    const codec = prop.format ? getPortCodec(prop.format) : undefined;
+    if (codec && out[key] !== undefined) {
+      out[key] = await codec.deserialize(out[key]);
+    }
+  }
+  return out;
+}
+async function normalizeInputsForCacheKey(inputs, schema) {
+  if (!schema?.properties)
+    return inputs;
+  const out = { ...inputs };
+  for (const [key, prop] of Object.entries(schema.properties)) {
+    const codec = prop.format ? getPortCodec(prop.format) : undefined;
+    if (codec && out[key] !== undefined) {
+      out[key] = await codec.serialize(out[key]);
+    }
+  }
+  return out;
+}
 function hasRunConfig(i) {
   return i !== null && typeof i === "object" && "runConfig" in i;
 }
@@ -1364,6 +1435,7 @@ class TaskRunner {
   timeoutTimer;
   pendingTimeoutError;
   shouldAccumulate = true;
+  runWithPreviews = false;
   telemetrySpan;
   constructor(task) {
     this.task = task;
@@ -1403,9 +1475,13 @@ class TaskRunner {
           getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
         }
       }
+      const inputSchema = this.task.constructor.inputSchema();
+      const outputSchema = this.task.constructor.outputSchema();
+      const inputsForKey = this.outputCache ? await normalizeInputsForCacheKey(inputs, inputSchema) : inputs;
       if (this.task.cacheable) {
-        outputs = await this.outputCache?.getOutput(this.task.type, inputs);
-        if (outputs) {
+        const cached = await this.outputCache?.getOutput(this.task.type, inputsForKey);
+        if (cached !== undefined) {
+          outputs = await deserializeOutputPorts(cached, outputSchema);
           this.telemetrySpan?.addEvent("workglow.task.cache_hit");
           if (isStreamable) {
             this.task.runOutputData = outputs;
@@ -1424,7 +1500,8 @@ class TaskRunner {
           outputs = await this.executeTask(inputs);
         }
         if (this.task.cacheable && outputs !== undefined) {
-          await this.outputCache?.saveOutput(this.task.type, inputs, outputs);
+          const wireOutputs = await serializeOutputPorts(outputs, outputSchema);
+          await this.outputCache?.saveOutput(this.task.type, inputsForKey, wireOutputs);
         }
         this.task.runOutputData = outputs ?? {};
       }
@@ -1641,6 +1718,7 @@ class TaskRunner {
       this.outputCache = cache;
     }
     this.shouldAccumulate = config.shouldAccumulate !== false;
+    this.runWithPreviews = config.runWithPreviews === true;
     if (config.updateProgress) {
       this.updateProgress = config.updateProgress;
     }
@@ -2812,6 +2890,7 @@ class TaskGraphRunner {
   graph;
   outputCache;
   accumulateLeafOutputs = true;
+  runWithPreviews = false;
   registry = globalServiceRegistry3;
   resourceScope;
   abortController;
@@ -3046,6 +3125,24 @@ class TaskGraphRunner {
   }
   async pushOutputFromNodeToEdges(node, results) {
     const dataflows = this.graph.getTargetDataflows(node.id);
+    if (Object.keys(results).length > 0) {
+      const consumerCounts = new Map;
+      for (const dataflow of dataflows) {
+        if (dataflow.stream !== undefined)
+          continue;
+        const port = dataflow.sourceTaskPortId;
+        consumerCounts.set(port, (consumerCounts.get(port) ?? 0) + 1);
+      }
+      for (const [port, count] of consumerCounts) {
+        const extra = this.runWithPreviews ? count : count - 1;
+        if (extra <= 0)
+          continue;
+        const value = results[port];
+        const ref = asRefcountable(value);
+        if (ref)
+          ref.retain(extra);
+      }
+    }
     for (const dataflow of dataflows) {
       if (dataflow.stream !== undefined)
         continue;
@@ -3215,7 +3312,8 @@ class TaskGraphRunner {
       outputCache: this.outputCache ?? false,
       updateProgress: async (task2, progress, message, ...args) => await this.handleProgress(task2, progress, message, ...args),
       registry: this.registry,
-      resourceScope: this.resourceScope
+      resourceScope: this.resourceScope,
+      runWithPreviews: this.runWithPreviews
     });
     await this.pushOutputFromNodeToEdges(task, results);
     return {
@@ -3265,7 +3363,8 @@ class TaskGraphRunner {
         shouldAccumulate,
         updateProgress: async (task2, progress, message, ...args) => await this.handleProgress(task2, progress, message, ...args),
         registry: this.registry,
-        resourceScope: this.resourceScope
+        resourceScope: this.resourceScope,
+        runWithPreviews: this.runWithPreviews
       });
       await this.pushOutputFromNodeToEdges(task, results);
       return {
@@ -3340,6 +3439,17 @@ class TaskGraphRunner {
     }
   }
   resetTask(graph, task, runId) {
+    const previous = task.runOutputData;
+    if (previous) {
+      for (const port of Object.keys(previous)) {
+        const ref = asRefcountable(previous[port]);
+        if (!ref)
+          continue;
+        try {
+          ref.release();
+        } catch {}
+      }
+    }
     task.status = TaskStatus.PENDING;
     task.resetInputData();
     task.runOutputData = {};
@@ -3373,6 +3483,7 @@ class TaskGraphRunner {
       this.resourceScope = config.resourceScope;
     }
     this.accumulateLeafOutputs = config?.accumulateLeafOutputs !== false;
+    this.runWithPreviews = config?.runWithPreviews === true;
     if (config?.outputCache !== undefined) {
       if (typeof config.outputCache === "boolean") {
         if (config.outputCache === true) {
@@ -3472,6 +3583,7 @@ class TaskGraphRunner {
         throw new TaskConfigurationError(`Graph has ${taskCount} tasks, exceeding the limit of ${config.maxTasks}`);
       }
     }
+    this.runWithPreviews = false;
     this.previewScheduler.reset();
     this.previewRunning = true;
   }
@@ -3571,7 +3683,8 @@ class GraphAsTaskRunner extends TaskRunner {
       parentSignal: this.abortController?.signal,
       outputCache: this.outputCache,
       registry: this.registry,
-      resourceScope: this.resourceScope
+      resourceScope: this.resourceScope,
+      runWithPreviews: this.runWithPreviews
     });
     unsubscribe();
     return results;
@@ -4034,7 +4147,8 @@ class TaskGraph {
       registry: config?.registry,
       timeout: config?.timeout,
       maxTasks: config?.maxTasks,
-      resourceScope: config?.resourceScope
+      resourceScope: config?.resourceScope,
+      runWithPreviews: config?.runWithPreviews
     });
   }
   runPreview(input = {}, config = {}) {
@@ -4887,7 +5001,8 @@ class Workflow {
         parentSignal: this._abortController.signal,
         outputCache: this._outputCache,
         registry: config?.registry ?? this._registry,
-        resourceScope: config?.resourceScope
+        resourceScope: config?.resourceScope,
+        runWithPreviews: config?.runWithPreviews
       });
       const results = this.graph.mergeExecuteOutputsToRunOutput(output, PROPERTY_ARRAY);
       this.events.emit("complete");
@@ -8121,6 +8236,14 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
     this.emit("output_pruned");
   }
 }
+// src/storage/PortCodecRegistry.ts
+import {
+  registerPortCodec,
+  getPortCodec as getPortCodec2,
+  _resetPortCodecsForTests
+} from "@workglow/util";
+// src/node.ts
+registerRefcountablePredicate((v) => !!v && typeof v === "object" && ("backend" in v) && ("retain" in v) && ("release" in v) && ("materialize" in v));
 export {
   wrapSchemaInArray,
   whileTaskConfigSchema,
@@ -8143,6 +8266,8 @@ export {
   resolveIterationBound,
   resetMethodNameCache,
   removeIterationProperties,
+  registerRefcountablePredicate,
+  registerPortCodec,
   registerJobQueueFactory,
   registerBuiltInTransforms,
   registerBaseTasks,
@@ -8178,6 +8303,7 @@ export {
   getSchemaFormat,
   getProfileGrants,
   getPortStreamMode,
+  getPortCodec2 as getPortCodec,
   getOutputStreamMode,
   getObjectSchema,
   getObjectPortId,
@@ -8224,9 +8350,12 @@ export {
   calculateNodeDepths,
   buildIterationInputSchema,
   autoConnect,
+  asRefcountable,
   addIterationContextToSchema,
   addBoundaryNodesToGraphJson,
   addBoundaryNodesToDependencyJson,
+  _resetRefcountablePredicatesForTests,
+  _resetPortCodecsForTests,
   WorkflowError,
   Workflow,
   WhileTaskRunner,
@@ -8300,4 +8429,4 @@ export {
   BROWSER_GRANTS
 };
 
-//# debugId=8744F550EEB96ECE64756E2164756E21
+//# debugId=1770AD648331E38064756E2164756E21