@effect/workflow 0.12.5 → 0.13.1

package/src/DurableDeferred.ts

@@ -9,7 +9,7 @@ import * as Effect from "effect/Effect"
 import * as Encoding from "effect/Encoding"
 import * as Exit from "effect/Exit"
 import { dual } from "effect/Function"
-import * as Option from "effect/Option"
+import * as Predicate from "effect/Predicate"
 import * as Schema from "effect/Schema"
 import type * as Activity from "./Activity.js"
 import * as Workflow from "./Workflow.js"
@@ -39,7 +39,7 @@ export interface DurableDeferred<
   readonly name: string
   readonly successSchema: Success
   readonly errorSchema: Error
-  readonly exitSchema: Schema.Exit<Success, Error, typeof Schema.Defect>
+  readonly exitSchema: Schema.ExitFromSelf<Success, Error, typeof Schema.Defect>
   readonly withActivityAttempt: Effect.Effect<DurableDeferred<Success, Error>>
 }
 
@@ -52,7 +52,7 @@ export interface Any {
   readonly name: string
   readonly successSchema: Schema.Schema.Any
   readonly errorSchema: Schema.Schema.All
-  readonly exitSchema: Schema.Exit<any, any, any>
+  readonly exitSchema: Schema.ExitFromSelf<any, any, any>
 }
 
 /**
@@ -70,7 +70,7 @@ export const make = <
   name,
   successSchema: options?.success ?? Schema.Void as any,
   errorSchema: options?.error ?? Schema.Never as any,
-  exitSchema: Schema.Exit({
+  exitSchema: Schema.ExitFromSelf({
     success: options?.success ?? Schema.Void as any,
     failure: options?.error ?? Schema.Never as any,
     defect: Schema.Defect
@@ -111,13 +111,14 @@ const await_: <Success extends Schema.Schema.Any, Error extends Schema.Schema.Al
 >(self: DurableDeferred<Success, Error>) {
   const engine = yield* EngineTag
   const instance = yield* InstanceTag
-  const oexit = yield* Workflow.wrapActivityResult(engine.deferredResult(self), Option.isNone)
-  if (Option.isNone(oexit)) {
+  const exit = yield* Workflow.wrapActivityResult(
+    engine.deferredResult(self),
+    Predicate.isUndefined
+  )
+  if (exit === undefined) {
     return yield* Workflow.suspend(instance)
   }
-  return yield* Effect.flatten(Effect.orDie(
-    Schema.decodeUnknown(self.exitSchema)(oexit.value)
-  ))
+  return yield* exit
 })
 
 export {
@@ -165,19 +166,15 @@ export const into: {
   Effect.contextWithEffect((context: Context.Context<WorkflowEngine | WorkflowInstance>) => {
     const engine = Context.get(context, EngineTag)
     const instance = Context.get(context, InstanceTag)
-    return Effect.onExit(
-      effect,
-      Effect.fnUntraced(function*(exit) {
-        if (instance.suspended) return
-        const encodedExit = yield* Effect.orDie(Schema.encode(self.exitSchema)(exit))
-        yield* engine.deferredDone({
-          workflowName: instance.workflow.name,
-          executionId: instance.executionId,
-          deferredName: self.name,
-          exit: encodedExit as any
-        })
+    return Effect.onExit(effect, (exit) => {
+      if (instance.suspended) return Effect.void
+      return engine.deferredDone(self, {
+        workflowName: instance.workflow.name,
+        executionId: instance.executionId,
+        deferredName: self.name,
+        exit
       })
-    )
+    })
   }))
 
 /**
@@ -218,11 +215,9 @@ export const raceAll = <
   })
   return Effect.gen(function*() {
     const engine = yield* EngineTag
-    const oexit = yield* Workflow.wrapActivityResult(engine.deferredResult(deferred), Option.isNone)
-    if (Option.isSome(oexit)) {
-      return yield* (Effect.flatten(Effect.orDie(
-        Schema.decodeUnknown(deferred.exitSchema)(oexit.value)
-      )) as Effect.Effect<any, any, any>)
+    const exit = yield* Workflow.wrapActivityResult(engine.deferredResult(deferred), Predicate.isUndefined)
+    if (exit) {
+      return yield* (Effect.flatten(exit) as Effect.Effect<any, any, any>)
     }
     return yield* into(Effect.raceAll(options.effects), deferred)
   })
@@ -443,14 +438,13 @@ export const done: {
   ) {
     const engine = yield* EngineTag
     const token = TokenParsed.fromString(options.token)
-    const exit = yield* Schema.encode(self.exitSchema)(options.exit)
-    yield* engine.deferredDone({
+    yield* engine.deferredDone(self, {
       workflowName: token.workflowName,
       executionId: token.executionId,
       deferredName: token.deferredName,
-      exit: exit as any
+      exit: options.exit
     })
-  }, Effect.orDie)
+  })
 )
 
 /**

package/src/DurableQueue.ts

@@ -0,0 +1,332 @@
+/**
+ * @since 1.0.0
+ */
+import * as PersistedQueue from "@effect/experimental/PersistedQueue"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as Schedule from "effect/Schedule"
+import * as Schema from "effect/Schema"
+import * as Activity from "./Activity.js"
+import * as DurableDeferred from "./DurableDeferred.js"
+import { makeHashDigest } from "./internal/crypto.js"
+import type * as WorkflowEngine from "./WorkflowEngine.js"
+
+/**
+ * @since 1.0.0
+ * @category Type IDs
+ */
+export type TypeId = "~@effect/workflow/DurableQueue"
+
+/**
+ * @since 1.0.0
+ * @category Type IDs
+ */
+export const TypeId: TypeId = "~@effect/workflow/DurableQueue"
+
+/**
+ * @since 1.0.0
+ * @category Models
+ */
+export interface DurableQueue<
+  Payload extends Schema.Schema.Any,
+  Success extends Schema.Schema.Any = typeof Schema.Void,
+  Error extends Schema.Schema.All = typeof Schema.Never
+> {
+  readonly [TypeId]: TypeId
+  readonly name: string
+  readonly payloadSchema: Payload
+  readonly idempotencyKey: (payload: Payload["Type"]) => string
+  readonly deferred: DurableDeferred.DurableDeferred<Success, Error>
+}
+
+/**
+ * A `DurableQueue` wraps a `PersistedQueue`, providing a way to wait for items
+ * to finish processing using a `DurableDeferred`.
+ *
+ * ```ts
+ * import { DurableQueue, Workflow } from "@effect/workflow"
+ * import { Effect, Schema } from "effect"
+ *
+ * // Define a DurableQueue that can be used to derive workers and offer items for
+ * // processing.
+ * const ApiQueue = DurableQueue.make({
+ *   name: "ApiQueue",
+ *   payload: {
+ *     id: Schema.String
+ *   },
+ *   success: Schema.Void,
+ *   error: Schema.Never,
+ *   idempotencyKey(payload) {
+ *     return payload.id
+ *   }
+ * })
+ *
+ * const MyWorkflow = Workflow.make({
+ *   name: "MyWorkflow",
+ *   payload: {
+ *     id: Schema.String
+ *   },
+ *   idempotencyKey: ({ id }) => id
+ * })
+ *
+ * const MyWorkflowLayer = MyWorkflow.toLayer(
+ *   Effect.fn(function*() {
+ *     // Add an item to the DurableQueue defined above.
+ *     //
+ *     // When the worker has finished processing the item, the workflow will
+ *     // resume.
+ *     //
+ *     yield* DurableQueue.process(ApiQueue, { id: "api-call-1" })
+ *
+ *     yield* Effect.log("Workflow succeeded!")
+ *   })
+ * )
+ *
+ * // Define a worker layer that can process items from the DurableQueue.
+ * const ApiWorker = DurableQueue.worker(
+ *   ApiQueue,
+ *   Effect.fn(function*({ id }) {
+ *     yield* Effect.log(`Worker processing API call with id: ${id}`)
+ *   }),
+ *   { concurrency: 5 } // Process up to 5 items concurrently
+ * )
+ * ```
+ *
+ * @since 1.0.0
+ * @category Constructors
+ */
+export const make = <
+  Payload extends Schema.Schema.Any | Schema.Struct.Fields,
+  Success extends Schema.Schema.Any = typeof Schema.Void,
+  Error extends Schema.Schema.All = typeof Schema.Never
+>(
+  options: {
+    readonly name: string
+    readonly payload: Payload
+    readonly idempotencyKey: (
+      payload: Payload extends Schema.Struct.Fields ? Schema.Struct<Payload>["Type"] : Payload["Type"]
+    ) => string
+    readonly success?: Success | undefined
+    readonly error?: Error | undefined
+  }
+): DurableQueue<
+  Payload extends Schema.Struct.Fields ? Schema.Struct<Payload> : Payload,
+  Success,
+  Error
+> => ({
+  [TypeId]: TypeId,
+  name: options.name,
+  payloadSchema: Schema.isSchema(options.payload) ? options.payload : Schema.Struct(options.payload) as any,
+  idempotencyKey: options.idempotencyKey as any,
+  deferred: DurableDeferred.make(`DurableQueue/${options.name}`, {
+    success: options.success,
+    error: options.error
+  })
+})
+
+const queueSchemas = new WeakMap<Schema.Schema.Any, Schema.Schema.Any>()
+const getQueueSchema = <Payload extends Schema.Schema.Any>(
+  payload: Payload
+): Schema.Struct<{
+  token: typeof Schema.String
+  payload: Payload
+  traceId: typeof Schema.String
+  spanId: typeof Schema.String
+  sampled: typeof Schema.Boolean
+}> => {
+  let schema = queueSchemas.get(payload)
+  if (!schema) {
+    schema = Schema.Struct({
+      token: Schema.String,
+      traceId: Schema.String,
+      spanId: Schema.String,
+      sampled: Schema.Boolean,
+      payload
+    })
+    queueSchemas.set(payload, schema)
+  }
+  return schema as any
+}
+
+/**
+ * @since 1.0.0
+ * @category Processing
+ */
+export const process: <
+  Payload extends Schema.Schema.Any,
+  Success extends Schema.Schema.Any,
+  Error extends Schema.Schema.All
+>(
+  self: DurableQueue<Payload, Success, Error>,
+  payload: Payload["Type"],
+  options?: {
+    readonly retrySchedule?: Schedule.Schedule<any, PersistedQueue.PersistedQueueError> | undefined
+  }
+) => Effect.Effect<
+  Success["Type"],
+  Error["Type"],
+  | WorkflowEngine.WorkflowEngine
+  | WorkflowEngine.WorkflowInstance
+  | PersistedQueue.PersistedQueueFactory
+  | Success["Context"]
+  | Error["Context"]
+  | Payload["Context"]
+> = Effect.fnUntraced(function*<
+  Payload extends Schema.Schema.Any,
+  Success extends Schema.Schema.Any,
+  Error extends Schema.Schema.All
+>(self: DurableQueue<Payload, Success, Error>, payload: Payload["Type"], options?: {
+  readonly retrySchedule?: Schedule.Schedule<any, PersistedQueue.PersistedQueueError> | undefined
+}) {
+  const key = yield* makeHashDigest(self.idempotencyKey(payload))
+
+  const deferred = DurableDeferred.make(`${self.deferred.name}/${key}`, {
+    success: self.deferred.successSchema,
+    error: self.deferred.errorSchema
+  })
+
+  yield* Activity.make({
+    name: `DurableQueue/${self.name}/${key}`,
+    execute: Effect.gen(function*() {
+      const span = yield* Effect.orDie(Effect.currentSpan)
+      const queue = yield* PersistedQueue.make({
+        name: `DurableQueue/${self.name}`,
+        schema: getQueueSchema(self.payloadSchema)
+      })
+      const token = yield* DurableDeferred.token(deferred)
+      yield* queue.offer({
+        token,
+        payload,
+        traceId: span.traceId,
+        spanId: span.spanId,
+        sampled: span.sampled
+      } as any).pipe(
+        Effect.tapErrorCause(Effect.logWarning),
+        Effect.catchTag("ParseError", Effect.die),
+        Effect.retry(options?.retrySchedule ?? defaultRetrySchedule),
+        Effect.orDie,
+        Effect.annotateLogs({
+          package: "@effect/workflow",
+          module: "DurableQueue",
+          fiber: "process",
+          queueName: self.name
+        })
+      )
+    })
+  })
+
+  return yield* DurableDeferred.await(deferred)
+})
+
+const defaultRetrySchedule = Schedule.exponential(500, 1.5).pipe(
+  Schedule.union(Schedule.spaced("1 minute"))
+)
+
+/**
+ * @since 1.0.0
+ * @category Worker
+ */
+export const makeWorker: <
+  Payload extends Schema.Schema.Any,
+  Success extends Schema.Schema.Any,
+  Error extends Schema.Schema.All,
+  R
+>(
+  self: DurableQueue<Payload, Success, Error>,
+  f: (payload: Payload["Type"]) => Effect.Effect<Success["Type"], Error["Type"], R>,
+  options?: { readonly concurrency?: number | undefined } | undefined
+) => Effect.Effect<
+  never,
+  never,
+  | WorkflowEngine.WorkflowEngine
+  | PersistedQueue.PersistedQueueFactory
+  | R
+  | Payload["Context"]
+  | Success["Context"]
+  | Error["Context"]
+> = Effect.fnUntraced(function*<
+  Payload extends Schema.Schema.Any,
+  Success extends Schema.Schema.Any,
+  Error extends Schema.Schema.All,
+  R
+>(
+  self: DurableQueue<Payload, Success, Error>,
+  f: (payload: Payload["Type"]) => Effect.Effect<Success["Type"], Error["Type"], R>,
+  options?: {
+    readonly concurrency?: number | undefined
+  }
+) {
+  const queue = yield* PersistedQueue.make({
+    name: `DurableQueue/${self.name}`,
+    schema: getQueueSchema(self.payloadSchema)
+  })
+  const concurrency = options?.concurrency ?? 1
+
+  const worker = queue.take((item_) => {
+    const item = item_ as any as {
+      token: DurableDeferred.Token
+      payload: Payload["Type"]
+      traceId: string
+      spanId: string
+      sampled: boolean
+    }
+    return f(item.payload).pipe(
+      Effect.exit,
+      Effect.flatMap((exit) =>
+        DurableDeferred.done(self.deferred, {
+          token: item.token,
+          exit
+        })
+      ),
+      Effect.asVoid,
+      Effect.withSpan(`DurableQueue/${self.name}/worker`, {
+        captureStackTrace: false,
+        parent: {
+          _tag: "ExternalSpan",
+          traceId: item.traceId,
+          spanId: item.spanId,
+          sampled: item.sampled,
+          context: Context.empty()
+        }
+      })
+    )
+  }).pipe(
+    Effect.catchAllCause(Effect.logWarning),
+    Effect.forever,
+    Effect.annotateLogs({
+      package: "@effect/workflow",
+      module: "DurableQueue",
+      fiber: "worker"
+    })
+  )
+
+  yield* Effect.replicateEffect(worker, concurrency, { concurrency, discard: true })
+  return yield* Effect.never
+})
+
+/**
+ * @since 1.0.0
+ * @category Worker
+ */
+export const worker: <
+  Payload extends Schema.Schema.Any,
+  Success extends Schema.Schema.Any,
+  Error extends Schema.Schema.All,
+  R
+>(
+  self: DurableQueue<Payload, Success, Error>,
+  f: (payload: Payload["Type"]) => Effect.Effect<Success["Type"], Error["Type"], R>,
+  options?: {
+    readonly concurrency?: number | undefined
+  } | undefined
+) => Layer.Layer<
+  never,
+  never,
+  | WorkflowEngine.WorkflowEngine
+  | PersistedQueue.PersistedQueueFactory
+  | R
+  | Payload["Context"]
+  | Success["Context"]
+  | Error["Context"]
+> = (self, f, options) => Layer.scopedDiscard(Effect.forkScoped(makeWorker(self, f, options)))

package/src/Workflow.ts

@@ -13,7 +13,7 @@ import * as Option from "effect/Option"
 import type { Pipeable } from "effect/Pipeable"
 import * as Predicate from "effect/Predicate"
 import * as PrimaryKey from "effect/PrimaryKey"
-import * as Schedule from "effect/Schedule"
+import type * as Schedule from "effect/Schedule"
 import * as Schema from "effect/Schema"
 import type * as AST from "effect/SchemaAST"
 import type * as Scope from "effect/Scope"
@@ -151,7 +151,7 @@ export interface Workflow<
       executionId: string
     ) => Effect.Effect<Success["Type"], Error["Type"], R>
   ) => Layer.Layer<
-    WorkflowEngine,
+    never,
     never,
     | WorkflowEngine
     | Exclude<R, WorkflowEngine | WorkflowInstance | Execution<Name> | Scope.Scope>
@@ -278,7 +278,6 @@ export const make = <
     readonly annotations?: Context.Context<never>
   }
 ): Workflow<Name, Payload extends Schema.Struct.Fields ? Schema.Struct<Payload> : Payload, Success, Error> => {
-  const suspendedRetrySchedule = options.suspendedRetrySchedule ?? defaultRetrySchedule
   const makeExecutionId = (payload: any) => makeHashDigest(`${options.name}-${options.idempotencyKey(payload)}`)
   const self: Workflow<Name, any, Success, Error> = {
     [TypeId]: TypeId,
@@ -305,61 +304,19 @@ export const make = <
         const engine = yield* EngineTag
         const executionId = yield* makeExecutionId(payload)
         yield* Effect.annotateCurrentSpan({ executionId })
-
-        const parentInstance = yield* Effect.serviceOption(InstanceTag)
-        let result: Result<unknown, unknown>
-        if (Option.isSome(parentInstance)) {
-          const instance = parentInstance.value
-          yield* Effect.addFinalizer(() => {
-            if (!instance.interrupted || result?._tag === "Complete") {
-              return Effect.void
-            }
-            return engine.interrupt(self, executionId)
-          })
-        }
-
-        if (opts?.discard) {
-          yield* engine.execute({
-            workflow: self,
-            executionId,
-            payload,
-            discard: true
-          })
-          return executionId
-        }
-        const run = engine.execute({
-          workflow: self,
+        return yield* engine.execute(self, {
           executionId,
           payload,
-          discard: false,
-          parent: Option.getOrUndefined(parentInstance)
+          discard: opts?.discard,
+          suspendedRetrySchedule: options.suspendedRetrySchedule
         })
-        if (Option.isSome(parentInstance)) {
-          result = yield* run
-          if (result._tag === "Suspended") {
-            return yield* suspend(parentInstance.value)
-          }
-          return yield* result.exit as Exit.Exit<Success["Type"], Error["Type"]>
-        }
-
-        let sleep: Effect.Effect<any> | undefined
-        while (true) {
-          result = yield* run
-          if (result._tag === "Complete") {
-            return yield* result.exit as Exit.Exit<Success["Type"], Error["Type"]>
-          }
-          sleep ??= (yield* Schedule.driver(suspendedRetrySchedule)).next(void 0).pipe(
-            Effect.catchAll(() => Effect.dieMessage(`${options.name}.execute: suspendedRetrySchedule exhausted`))
-          )
-          yield* sleep
-        }
       },
       Effect.withSpan(`${options.name}.execute`, { captureStackTrace: false })
     ),
     poll: Effect.fnUntraced(
       function*(executionId: string) {
         const engine = yield* EngineTag
-        return yield* engine.poll({ workflow: self, executionId })
+        return yield* engine.poll(self, executionId)
       },
       (effect, executionId) =>
         Effect.withSpan(effect, `${options.name}.poll`, {
@@ -390,14 +347,9 @@ export const make = <
         })
     ),
     toLayer: (execute) =>
-      Layer.scopedContext(Effect.gen(function*() {
-        const context = yield* Effect.context<WorkflowEngine>()
-        const engine = Context.get(context, EngineTag)
-        yield* engine.register(self, (payload, executionId) =>
-          Effect.suspend(() => execute(payload, executionId)).pipe(
-            Effect.mapInputContext((input) => Context.merge(context, input))
-          ) as any)
-        return EngineTag.context(engine)
+      Layer.scopedDiscard(Effect.gen(function*() {
+        const engine = yield* EngineTag
+        return yield* engine.register(self, execute)
       })) as any,
     executionId: (payload) => makeExecutionId(self.payloadSchema.make(payload)),
     withCompensation
@@ -406,10 +358,6 @@ export const make = <
   return self
 }
 
-const defaultRetrySchedule = Schedule.exponential(200, 1.5).pipe(
-  Schedule.union(Schedule.spaced(30000))
-)
-
 /**
  * @since 1.0.0
  * @category Constructors