@workglow/task-graph 0.2.17 → 0.2.18

package/dist/common.d.ts

@@ -20,9 +20,12 @@ export * from "./task-graph/TransformRegistry";
 export * from "./task-graph/TransformTypes";
 export * from "./task-graph/transforms";
 export * from "./task-graph/autoConnect";
+export { registerRefcountablePredicate, asRefcountable, _resetRefcountablePredicatesForTests, type Refcountable, } from "./refcountable";
 export * from "./task";
 export * from "./storage/TaskGraphRepository";
 export * from "./storage/TaskGraphTabularRepository";
 export * from "./storage/TaskOutputRepository";
 export * from "./storage/TaskOutputTabularRepository";
+export { registerPortCodec, getPortCodec, _resetPortCodecsForTests } from "./storage/PortCodecRegistry";
+export type { PortCodec } from "./storage/PortCodecRegistry";
 //# sourceMappingURL=common.d.ts.map

package/dist/common.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,uBAAuB,CAAC;AACtC,cAAc,6BAA6B,CAAC;AAE5C,cAAc,oCAAoC,CAAC;AACnD,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAE7C,cAAc,0BAA0B,CAAC;AACzC,cAAc,kCAAkC,CAAC;AACjD,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AAEtC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AAEzC,cAAc,QAAQ,CAAC;AAEvB,cAAc,+BAA+B,CAAC;AAC9C,cAAc,sCAAsC,CAAC;AACrD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,uCAAuC,CAAC"}
+{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,uBAAuB,CAAC;AACtC,cAAc,6BAA6B,CAAC;AAE5C,cAAc,oCAAoC,CAAC;AACnD,cAAc,iCAAiC,CAAC;AAChD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAE7C,cAAc,0BAA0B,CAAC;AACzC,cAAc,kCAAkC,CAAC;AACjD,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AAEtC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AAEzC,OAAO,EACL,6BAA6B,EAC7B,cAAc,EACd,oCAAoC,EACpC,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAExB,cAAc,QAAQ,CAAC;AAEvB,cAAc,+BAA+B,CAAC;AAC9C,cAAc,sCAAsC,CAAC;AACrD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,uCAAuC,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AACxG,YAAY,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC"}

package/dist/node.js

@@ -584,6 +584,10 @@ function computeGraphEntitlements(graph, options) {
 }
 // src/task/InputResolver.ts
 import { getInputResolvers } from "@workglow/util";
+function isPlainObject(value) {
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
 function getSchemaFormat(schema, visited = new WeakSet) {
   if (typeof schema !== "object" || schema === null)
     return;
@@ -671,6 +675,7 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
   for (const [key, propSchema] of Object.entries(properties)) {
     let value = resolved[key];
     const format = getSchemaFormat(propSchema);
+    let phase1Transformed = false;
     if (format) {
       let resolver = resolvers.get(format);
       if (!resolver) {
@@ -681,14 +686,18 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
         if (typeof value === "string") {
           value = await resolver(value, format, config.registry);
           resolved[key] = value;
+          phase1Transformed = true;
         } else if (Array.isArray(value) && value.some((item) => typeof item === "string")) {
           const results = await Promise.all(value.map((item) => typeof item === "string" ? resolver(item, format, config.registry) : item));
           value = results.filter((result) => result !== undefined);
           resolved[key] = value;
+          phase1Transformed = true;
         }
       }
     }
-    if (value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value)) {
+    const hasFormatResolver = format ? !!(resolvers.get(format) ?? resolvers.get(getFormatPrefix(format))) : false;
+    const skipPhase2 = hasFormatResolver && !phase1Transformed;
+    if (!skipPhase2 && value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value) && isPlainObject(value)) {
       const objectSchema = getObjectSchema(propSchema);
       if (objectSchema && !visited.has(objectSchema)) {
         visited.add(objectSchema);
@@ -1134,6 +1143,31 @@ import {
   uuid4 as uuid43
 } from "@workglow/util";
 
+// src/refcountable.ts
+var GLOBAL_KEY = Symbol.for("@workglow/task-graph/refcountable.predicates");
+var _g = globalThis;
+if (!Array.isArray(_g[GLOBAL_KEY])) {
+  _g[GLOBAL_KEY] = [];
+}
+var predicates = _g[GLOBAL_KEY];
+function registerRefcountablePredicate(p) {
+  predicates.push(p);
+}
+function asRefcountable(v) {
+  if (v === null || v === undefined)
+    return null;
+  if (typeof v !== "object")
+    return null;
+  for (const p of predicates) {
+    if (p(v))
+      return v;
+  }
+  return null;
+}
+function _resetRefcountablePredicatesForTests() {
+  predicates.length = 0;
+}
+
 // src/storage/TaskOutputRepository.ts
 import { createServiceToken as createServiceToken2, EventEmitter as EventEmitter2 } from "@workglow/util";
 var TASK_OUTPUT_REPOSITORY = createServiceToken2("taskgraph.taskOutputRepository");
@@ -1245,6 +1279,7 @@ import {
   globalServiceRegistry as globalServiceRegistry2,
   SpanStatusCode
 } from "@workglow/util";
+import { getPortCodec } from "@workglow/util";
 
 // src/task/StreamTypes.ts
 function getPortStreamMode(schema, portId) {
@@ -1348,6 +1383,42 @@ function hasStructuredOutput(schema) {
 }
 
 // src/task/TaskRunner.ts
+async function serializeOutputPorts(output, schema) {
+  if (!schema?.properties)
+    return output;
+  const out = { ...output };
+  for (const [key, prop] of Object.entries(schema.properties)) {
+    const codec = prop.format ? getPortCodec(prop.format) : undefined;
+    if (codec && out[key] !== undefined) {
+      out[key] = await codec.serialize(out[key]);
+    }
+  }
+  return out;
+}
+async function deserializeOutputPorts(output, schema) {
+  if (!schema?.properties)
+    return output;
+  const out = { ...output };
+  for (const [key, prop] of Object.entries(schema.properties)) {
+    const codec = prop.format ? getPortCodec(prop.format) : undefined;
+    if (codec && out[key] !== undefined) {
+      out[key] = await codec.deserialize(out[key]);
+    }
+  }
+  return out;
+}
+async function normalizeInputsForCacheKey(inputs, schema) {
+  if (!schema?.properties)
+    return inputs;
+  const out = { ...inputs };
+  for (const [key, prop] of Object.entries(schema.properties)) {
+    const codec = prop.format ? getPortCodec(prop.format) : undefined;
+    if (codec && out[key] !== undefined) {
+      out[key] = await codec.serialize(out[key]);
+    }
+  }
+  return out;
+}
 function hasRunConfig(i) {
   return i !== null && typeof i === "object" && "runConfig" in i;
 }
@@ -1403,9 +1474,13 @@ class TaskRunner {
           getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
         }
       }
+      const inputSchema = this.task.constructor.inputSchema();
+      const outputSchema = this.task.constructor.outputSchema();
+      const inputsForKey = this.outputCache ? await normalizeInputsForCacheKey(inputs, inputSchema) : inputs;
       if (this.task.cacheable) {
-        outputs = await this.outputCache?.getOutput(this.task.type, inputs);
-        if (outputs) {
+        const cached = await this.outputCache?.getOutput(this.task.type, inputsForKey);
+        if (cached !== undefined) {
+          outputs = await deserializeOutputPorts(cached, outputSchema);
           this.telemetrySpan?.addEvent("workglow.task.cache_hit");
           if (isStreamable) {
             this.task.runOutputData = outputs;
@@ -1424,7 +1499,8 @@ class TaskRunner {
           outputs = await this.executeTask(inputs);
         }
         if (this.task.cacheable && outputs !== undefined) {
-          await this.outputCache?.saveOutput(this.task.type, inputs, outputs);
+          const wireOutputs = await serializeOutputPorts(outputs, outputSchema);
+          await this.outputCache?.saveOutput(this.task.type, inputsForKey, wireOutputs);
         }
         this.task.runOutputData = outputs ?? {};
       }
@@ -3046,6 +3122,23 @@ class TaskGraphRunner {
   }
   async pushOutputFromNodeToEdges(node, results) {
     const dataflows = this.graph.getTargetDataflows(node.id);
+    if (Object.keys(results).length > 0) {
+      const consumerCounts = new Map;
+      for (const dataflow of dataflows) {
+        if (dataflow.stream !== undefined)
+          continue;
+        const port = dataflow.sourceTaskPortId;
+        consumerCounts.set(port, (consumerCounts.get(port) ?? 0) + 1);
+      }
+      for (const [port, count] of consumerCounts) {
+        if (count <= 1)
+          continue;
+        const value = results[port];
+        const ref = asRefcountable(value);
+        if (ref)
+          ref.retain(count - 1);
+      }
+    }
     for (const dataflow of dataflows) {
       if (dataflow.stream !== undefined)
         continue;
@@ -8121,6 +8214,14 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
     this.emit("output_pruned");
   }
 }
+// src/storage/PortCodecRegistry.ts
+import {
+  registerPortCodec,
+  getPortCodec as getPortCodec2,
+  _resetPortCodecsForTests
+} from "@workglow/util";
+// src/node.ts
+registerRefcountablePredicate((v) => !!v && typeof v === "object" && ("backend" in v) && ("retain" in v) && ("release" in v) && ("materialize" in v));
 export {
   wrapSchemaInArray,
   whileTaskConfigSchema,
@@ -8143,6 +8244,8 @@ export {
   resolveIterationBound,
   resetMethodNameCache,
   removeIterationProperties,
+  registerRefcountablePredicate,
+  registerPortCodec,
   registerJobQueueFactory,
   registerBuiltInTransforms,
   registerBaseTasks,
@@ -8178,6 +8281,7 @@ export {
   getSchemaFormat,
   getProfileGrants,
   getPortStreamMode,
+  getPortCodec2 as getPortCodec,
   getOutputStreamMode,
   getObjectSchema,
   getObjectPortId,
@@ -8224,9 +8328,12 @@ export {
   calculateNodeDepths,
   buildIterationInputSchema,
   autoConnect,
+  asRefcountable,
   addIterationContextToSchema,
   addBoundaryNodesToGraphJson,
   addBoundaryNodesToDependencyJson,
+  _resetRefcountablePredicatesForTests,
+  _resetPortCodecsForTests,
   WorkflowError,
   Workflow,
   WhileTaskRunner,
@@ -8300,4 +8407,4 @@ export {
   BROWSER_GRANTS
 };
 
-//# debugId=8744F550EEB96ECE64756E2164756E21
+//# debugId=3044ECF32B8CA32464756E2164756E21