@oomol-lab/open-flow 0.1.0-beta.77 → 0.1.0-beta.79

package/src/execution/common/events.ts

@@ -23,7 +23,7 @@ const nodeKinds = new Set([
   'javascript',
   'openapi',
   'http-request',
-  'dedupe',
+  'skip-duplicates',
   'llm',
   'value',
   'wait',
@@ -57,7 +57,7 @@ function nodeKind(
   | 'javascript'
   | 'openapi'
   | 'http-request'
-  | 'dedupe'
+  | 'skip-duplicates'
   | 'llm'
   | 'value'
   | 'wait'
@@ -78,7 +78,7 @@ function nodeKind(
     | 'javascript'
     | 'openapi'
     | 'http-request'
-    | 'dedupe'
+    | 'skip-duplicates'
     | 'llm'
     | 'value'
     | 'wait'
@@ -95,6 +95,7 @@ function nodeContext(
   event: Record<string, unknown>,
   flows: ReadonlyMap<string, string>,
   platformRunId: string,
+  identify: typeof opaqueId,
 ): {
   readonly executionId: Promise<string>
   readonly flowId: string
@@ -105,7 +106,7 @@ function nodeContext(
   const flowId = flows.get(rawRunId)
   if (flowId == null) throw new TypeError('Runtime node event preceded its run.started event.')
   return {
-    executionId: projectExecutionId(platformRunId, rawRunId, string(event.jobId, 'Runtime node event jobId')),
+    executionId: identify('execution', platformRunId, rawRunId, string(event.jobId, 'Runtime node event jobId')),
     flowId,
     nodeId: string(event.nodeId, 'Runtime node event nodeId'),
     rawRunId,
@@ -184,6 +185,17 @@ export function createEventProjector(platformRunId: string, nodeFailureCodes: Re
   const jobs = new Map<string, string>()
   const nodeProgressBuckets = new Map<string, number>()
   const runProgressBuckets = new Map<string, number>()
+  // Every event of a scope or execution repeats its identity, so each digest is computed once per projector.
+  const identities = new Map<string, Promise<string>>()
+  const identify: typeof opaqueId = (prefix, ...parts) => {
+    const key = JSON.stringify([prefix, ...parts])
+    let identity = identities.get(key)
+    if (identity == null) {
+      identity = opaqueId(prefix, ...parts)
+      identities.set(key, identity)
+    }
+    return identity
+  }
   return async (value) => {
     const event = object(value, 'Runtime event')
     const type = string(event.type, 'Runtime event type')
@@ -205,8 +217,8 @@ export function createEventProjector(platformRunId: string, nodeFailureCodes: Re
         kind: 'run.started',
         payload: {
           flowId,
-          ...(parentRunId === undefined ? {} : { parentScopeId: await opaqueId('scope', platformRunId, parentRunId) }),
-          scopeId: await opaqueId('scope', platformRunId, rawRunId),
+          ...(parentRunId === undefined ? {} : { parentScopeId: await identify('scope', platformRunId, parentRunId) }),
+          scopeId: await identify('scope', platformRunId, rawRunId),
         },
       }
     }
@@ -226,7 +238,7 @@ export function createEventProjector(platformRunId: string, nodeFailureCodes: Re
         payload: {
           flowId,
           progress: event.progress,
-          scopeId: await opaqueId('scope', platformRunId, rawRunId),
+          scopeId: await identify('scope', platformRunId, rawRunId),
         },
       }
     }
@@ -235,7 +247,7 @@ export function createEventProjector(platformRunId: string, nodeFailureCodes: Re
       const rawRunId = runId(event)
       if (type == 'run.failed' && rawRunId == platformRunId && event.failure != null) {
         const failure = object(event.failure, 'Runtime terminal failure')
-        const context = nodeContext(failure, flows, platformRunId)
+        const context = nodeContext(failure, flows, platformRunId, identify)
         return {
           kind: 'node.failed',
           payload: {
@@ -258,13 +270,13 @@ export function createEventProjector(platformRunId: string, nodeFailureCodes: Re
       type == 'node.completed' ||
       type == 'node.failed'
     ) {
-      const context = nodeContext(event, flows, platformRunId)
+      const context = nodeContext(event, flows, platformRunId, identify)
       const executionId = await context.executionId
       const payload = {
         executionId,
         flowId: context.flowId,
         nodeId: context.nodeId,
-        scopeId: await opaqueId('scope', platformRunId, context.rawRunId),
+        scopeId: await identify('scope', platformRunId, context.rawRunId),
         ...(event.systemOutputs === undefined ? {} : { systemOutputs: object(event.systemOutputs, 'Runtime system outputs') }),
         ...(event.handled === true ? { handled: true } : {}),
       }

package/src/execution/common/node-execution.ts

@@ -109,7 +109,7 @@ function nodeKind(
   | 'javascript'
   | 'openapi'
   | 'http-request'
-  | 'dedupe'
+  | 'skip-duplicates'
   | 'llm'
   | 'value'
   | 'wait'

package/src/execution/common/scheduler.ts

@@ -13,6 +13,7 @@ import * as Effect from 'effect/Effect'
 import * as FiberSet from 'effect/FiberSet'
 import * as Queue from 'effect/Queue'
 import { nanoid } from 'nanoid'
+import { z } from 'zod'
 import { isTriggerNode } from '../../flow/common/change.ts'
 import { nodeInputMappings } from '../../flow/common/condition.ts'
 import { nodeSystemOutputPorts, supportsNodeErrorHandling } from '../../flow/common/errorHandling.ts'
@@ -70,7 +71,7 @@ export type SchedulerEvent =
         | 'javascript'
         | 'openapi'
         | 'http-request'
-        | 'dedupe'
+        | 'skip-duplicates'
         | 'llm'
         | 'value'
         | 'wait'
@@ -217,7 +218,7 @@ export type RunLaunch =
       readonly resume?: never
     }
   | {
-      readonly resume: { readonly checkpoint: unknown }
+      readonly resume: { readonly checkpoint: unknown; readonly readyWorkflowRunIds?: readonly string[] }
       readonly bindingValues?: never
       readonly inputs?: never
       readonly trigger?: never
@@ -250,7 +251,7 @@ function runGraph(
   runId: string,
   launchInputs: Readonly<Record<string, Readonly<Record<string, JsonValue>>>> = {},
   trigger?: TriggerSeed,
-  resume?: { readonly checkpoint: FlowRunCheckpoint },
+  resume?: { readonly checkpoint: FlowRunCheckpoint; readonly readyWorkflowRunIds?: ReadonlySet<string> },
 ): Effect.Effect<FlowRunOutcome, Error> {
   return Effect.scoped(
     Effect.gen(function* () {
@@ -292,6 +293,8 @@ function runGraph(
       const changes = yield* Queue.unbounded<void>()
       let remainingMs = context.remainingMs
       let measuredAt = yield* Clock.currentTimeMillis
+      let nextCallCheckAt = measuredAt + (resume == null ? 1000 : 0)
+      let readyWorkflowRunIds = resume?.readyWorkflowRunIds
       let idleSince: number | undefined
       const agents: Record<string, FlowRunCheckpoint['agents'][string]> = { ...resume?.checkpoint.agents }
       const resolveInputs = (nodeId: string, node: ExecutableNode) => resolveNodeInputs(nodeId, node, dependencies, context.bindingValues, launch[nodeId])
@@ -546,15 +549,20 @@ function runGraph(
             ),
             now,
           )
-        for (const call of pendingCalls.values()) {
-          yield* observe(
-            call.nodeId,
-            call.jobId,
-            Effect.gen(function* () {
-              const result = yield* execute(call.nodeId, target.graph.nodes[call.nodeId] as ExecutableNode, call.jobId, call.input)
-              yield* settleNode(call.nodeId, call.jobId, result)
-            }),
-          ).pipe(Effect.catch(() => Effect.void))
+        if (pendingCalls.size > 0 && now >= nextCallCheckAt) {
+          for (const call of pendingCalls.values()) {
+            if (readyWorkflowRunIds != null && !readyWorkflowRunIds.has(call.childRunId)) continue
+            yield* observe(
+              call.nodeId,
+              call.jobId,
+              Effect.gen(function* () {
+                const result = yield* execute(call.nodeId, target.graph.nodes[call.nodeId] as ExecutableNode, call.jobId, call.input)
+                yield* settleNode(call.nodeId, call.jobId, result)
+              }),
+            ).pipe(Effect.catch(() => Effect.void))
+          }
+          readyWorkflowRunIds = undefined
+          nextCallCheckAt = (yield* Clock.currentTimeMillis) + 1000
         }
         const ready = dependencies.ready()
         for (const nodeId of ready) executeReady(nodeId)
@@ -592,7 +600,8 @@ function runGraph(
                 true,
               )
         let next = Effect.raceFirst(activity, decision)
-        if (pendingCalls.size > 0) next = Effect.raceFirst(next, Effect.sleep(1000).pipe(Effect.as({})))
+        if (pendingCalls.size > 0)
+          next = Effect.raceFirst(next, Effect.sleep(Math.max(0, nextCallCheckAt - (yield* Clock.currentTimeMillis))).pipe(Effect.as({})))
         if (left != null) next = Effect.raceFirst(next, Effect.sleep(Math.max(0, left)).pipe(Effect.as({})))
         const resolutions = yield* next
         applyResolutions(resolutions, yield* Clock.currentTimeMillis)
@@ -631,6 +640,11 @@ export function runFlow(prepared: PreparedFlow, options: FlowRunOptions): Effect
       return yield* Effect.fail(new Error('A resumed Flow Run cannot accept launch inputs, binding values, or a Trigger seed.'))
     }
     const decoded = options.resume == null ? undefined : decodeFlowRunCheckpoint(options.resume.checkpoint)
+    const readyWorkflowRunIds =
+      options.resume?.readyWorkflowRunIds === undefined ? undefined : new Set(z.array(z.string().min(1)).parse(options.resume.readyWorkflowRunIds))
+    if (readyWorkflowRunIds != null && [...readyWorkflowRunIds].some((id) => !decoded?.calls?.some((call) => call.childRunId == id))) {
+      return yield* Effect.fail(new Error('Ready workflow Run is not present in the checkpoint.'))
+    }
     const outputs = (nodeId: string, value: Readonly<Record<string, JsonValue>>) =>
       isResolutionNode(prepared.graph.nodes[nodeId]) && value.pending != null ? { ...value, pending: withWaitNotificationMessage(value.pending) } : value
     const checkpoint =
@@ -692,7 +706,7 @@ export function runFlow(prepared: PreparedFlow, options: FlowRunOptions): Effect
       options.runId,
       options.inputs,
       trigger,
-      checkpoint == null ? undefined : { checkpoint },
+      checkpoint == null ? undefined : { checkpoint, readyWorkflowRunIds },
     )
   })
   return program

package/src/flow/common/change.ts

@@ -167,7 +167,7 @@ export interface ConditionNode extends GraphNodeBase {
 
 export type ManagedTaskExecutor =
   | import('../../http/common/httpRequest.ts').HttpRequestExecutor
-  | import('../../state/common/state.ts').DedupeExecutor
+  | import('../../state/common/state.ts').SkipDuplicatesExecutor
   | WorkflowExecutor
   | import('../../decision/common/decision.ts').DecisionExecutor
   | import('../../openapi/common/openapi.ts').OpenApiExecutor

package/src/flow/common/changeSchema.ts

@@ -6,7 +6,7 @@ import { limitDecisionTask } from '../../decision/common/decision.ts'
 import { assertHttpAuthBindings } from '../../http/common/authBindings.ts'
 import { isHttpHost, maxHttpHosts } from '../../http/common/hosts.ts'
 import { httpRequestExecutorSchema } from '../../http/common/httpRequest.ts'
-import { dedupeExecutorSchema } from '../../state/common/state.ts'
+import { skipDuplicatesExecutorSchema } from '../../state/common/state.ts'
 import { checkJsonDepth } from './json.ts'
 import { retryConfigurationSchema } from './retry.ts'
 import { triggerScheduleSchema } from './triggerScheduleSchema.ts'
@@ -130,7 +130,7 @@ const managedFields = z.object({
     }),
     z.strictObject({ kind: z.literal('workflow'), flowId: text, revisionId: text }),
     httpRequestExecutorSchema,
-    dedupeExecutorSchema,
+    skipDuplicatesExecutorSchema,
   ]),
 })
 
@@ -729,10 +729,10 @@ function checkModelVersion(content: RevisionContent): void {
   if (
     content.modelVersion < 13 &&
     Object.values(content.document.graph.nodes).some(
-      (candidateNode) => candidateNode.kind === 'task' && 'executor' in candidateNode.task && candidateNode.task.executor.kind === 'dedupe',
+      (candidateNode) => candidateNode.kind === 'task' && 'executor' in candidateNode.task && candidateNode.task.executor.kind === 'skip-duplicates',
     )
   )
-    throw new Error('Deduplication requires Flow model v13.')
+    throw new Error('Skip Duplicates requires Flow model v13.')
   if (
     content.modelVersion < 12 &&
     Object.values(content.document.graph.nodes).some(

package/src/flow/common/errorHandling.ts

@@ -26,7 +26,7 @@ export const nodeErrorSchema = {
 } as const
 
 export function supportsNodeErrorHandling(node: GraphNode): boolean {
-  return supportsNodeRetry(node) || (node.kind === 'task' && 'executor' in node.task && node.task.executor.kind === 'dedupe')
+  return supportsNodeRetry(node) || (node.kind === 'task' && 'executor' in node.task && node.task.executor.kind === 'skip-duplicates')
 }
 
 export function nodeSystemOutputPorts(node: GraphNode): Readonly<Record<string, PortDefinition>> {

package/src/flow/common/semantics.ts

@@ -2,7 +2,7 @@ import { dequal } from 'dequal/lite'
 import { decisionTaskIssues } from '../../decision/common/decision.ts'
 import { httpRequestIssues, httpAuthHandles, httpRequestTask, httpRequestUrl } from '../../http/common/httpRequest.ts'
 import { openApiIssues, authHandles } from '../../openapi/common/openapi.ts'
-import { dedupeTask } from '../../state/common/state.ts'
+import { skipDuplicatesTask } from '../../state/common/state.ts'
 import { isTriggerNode } from './change.ts'
 import { nodeWorkflowReferences } from './workflowReferences.ts'
 export { renderPrompt } from './promptTemplate.ts'
@@ -274,14 +274,14 @@ export async function validateFlow(revision: RevisionContent, engine: EngineCont
       })
     for (const message of openApiIssues(task))
       checked.diagnostics.push({ code: 'openapi.config-invalid', column: 0, line: 1, message, path: `/document/graph/nodes/${nodeId}/task/executor` })
-    if (task.executor.kind === 'dedupe') {
-      const expected = dedupeTask(task.executor)
+    if (task.executor.kind === 'skip-duplicates') {
+      const expected = skipDuplicatesTask(task.executor)
       if (
         !dequal(httpPortContracts(task.inputs), httpPortContracts(expected.inputs)) ||
         !dequal(httpPortContracts(task.outputs), httpPortContracts(expected.outputs))
       )
         checked.diagnostics.push({
-          code: 'dedupe.ports-invalid',
+          code: 'skip-duplicates.ports-invalid',
           column: 0,
           line: 1,
           message: 'Deduplication ports must match the fixed contract.',
@@ -457,7 +457,9 @@ export async function prepareFlow(revision: RevisionContent, engineContract: str
   if (engine == null) return { kind: 'engine-unsupported' }
   if (
     !engine.flowState &&
-    Object.values(revision.document.graph.nodes).some((node) => node.kind === 'task' && 'executor' in node.task && node.task.executor.kind === 'dedupe')
+    Object.values(revision.document.graph.nodes).some(
+      (node) => node.kind === 'task' && 'executor' in node.task && node.task.executor.kind === 'skip-duplicates',
+    )
   )
     return { kind: 'engine-unsupported' }
   if (
@@ -527,11 +529,15 @@ export async function prepareFlow(revision: RevisionContent, engineContract: str
 
 export function createRuntimeProgram(prepared: PreparedFlow, entryModuleId: string, engineDigest: string): RuntimeProgram | undefined {
   if (!Object.hasOwn(prepared.modules, entryModuleId)) return
+  const modules = new Set([entryModuleId])
+  for (const id of modules) {
+    for (const imported of prepared.modules[id]!.imports) modules.add(imported)
+  }
   return {
     engineContract: prepared.engineContract,
     engineDigest,
     entryModuleId,
-    modules: prepared.modules,
+    modules: Object.fromEntries([...modules].map((id) => [id, prepared.modules[id]!])),
   }
 }