effect 4.0.0-beta.65 → 4.0.0-beta.67

package/src/unstable/persistence/PersistedQueue.ts

@@ -1,4 +1,29 @@
 /**
+ * Schema-aware persisted queues for background work.
+ *
+ * A `PersistedQueue` stores JSON-encoded values in a named queue and lets
+ * workers `take` one value at a time inside a scoped processing window. It is
+ * useful for durable handoffs, background jobs, outbox-style integrations, and
+ * workloads where failed work should be retried across fibers, process
+ * restarts, or multiple workers sharing Redis or SQL.
+ *
+ * Delivery is at-least-once: a handler that fails, is interrupted, or loses its
+ * backing-store lock may see the same element again until `maxAttempts` is
+ * reached. Use stable custom ids when offering idempotent work, and choose ids
+ * that are collision-free for the backing store because stores can enforce
+ * uniqueness at the queue, prefix, or table level. Ordering is intentionally a
+ * store-level concern; retries, lock expiration, polling, and multiple workers
+ * can move entries behind newer work, so handlers should not rely on strict
+ * FIFO processing.
+ *
+ * Values are encoded and decoded with the supplied schema using the JSON codec,
+ * so schema services must be available when offering and taking values. Changing
+ * a queue name, schema, Redis prefix, SQL table, or id format is a persistence
+ * migration: old entries may decode differently, stop being visible, or collide
+ * with new entries. The memory store is process-local and volatile, while Redis
+ * and SQL stores use leases that should be tuned for the expected processing
+ * time.
+ *
  * @since 4.0.0
  */
 import type * as Arr from "../../Array.ts"
@@ -23,20 +48,30 @@ import type { SqlError } from "../sql/SqlError.ts"
 import * as Redis from "./Redis.ts"
 
 /**
- * @since 4.0.0
+ * Runtime type identifier for `PersistedQueue` values.
+ *
  * @category Type IDs
+ * @since 4.0.0
  */
 export const TypeId: TypeId = "~effect/persistence/PersistedQueue"
 
 /**
- * @since 4.0.0
+ * Type-level identifier used to brand `PersistedQueue` values.
+ *
  * @category Type IDs
+ * @since 4.0.0
  */
 export type TypeId = "~effect/persistence/PersistedQueue"
 
 /**
- * @since 4.0.0
+ * Persistent queue of schema-encoded values.
+ *
+ * `offer` enqueues values by id, and `take` processes one value at a time,
+ * marking it complete on success or retrying it until the maximum attempts is
+ * reached.
+ *
  * @category Models
+ * @since 4.0.0
  */
 export interface PersistedQueue<in out A, out R = never> {
   readonly [TypeId]: TypeId
@@ -72,8 +107,10 @@ export interface PersistedQueue<in out A, out R = never> {
 }
 
 /**
- * @since 4.0.0
+ * Service for constructing named `PersistedQueue` instances from schemas.
+ *
  * @category Factory
+ * @since 4.0.0
  */
 export class PersistedQueueFactory extends Context.Service<
   PersistedQueueFactory,
@@ -86,8 +123,11 @@ export class PersistedQueueFactory extends Context.Service<
 >()("effect/persistence/PersistedQueue/PersistedQueueFactory") {}
 
 /**
- * @since 4.0.0
+ * Accesses `PersistedQueueFactory` to create a named persisted queue for a
+ * schema.
+ *
  * @category Accessors
+ * @since 4.0.0
  */
 export const make = <S extends Schema.Top>(options: {
   readonly name: string
@@ -99,8 +139,14 @@ export const make = <S extends Schema.Top>(options: {
 > => PersistedQueueFactory.use((factory) => factory.make(options))
 
 /**
- * @since 4.0.0
+ * Creates a `PersistedQueueFactory` from the current `PersistedQueueStore`.
+ *
+ * Values are encoded and decoded with the supplied schema, automatically
+ * assigned an id when needed, and acknowledged or retried according to the
+ * `take` handler's exit.
+ *
  * @category Factory
+ * @since 4.0.0
  */
 export const makeFactory = Effect.gen(function*() {
   const store = yield* PersistedQueueStore
@@ -153,8 +199,10 @@ export const makeFactory = Effect.gen(function*() {
 })
 
 /**
- * @since 4.0.0
+ * Provides `PersistedQueueFactory` using the current `PersistedQueueStore`.
+ *
  * @category Factory
+ * @since 4.0.0
  */
 export const layer: Layer.Layer<
   PersistedQueueFactory,
@@ -163,20 +211,26 @@ export const layer: Layer.Layer<
 > = Layer.effect(PersistedQueueFactory, makeFactory)
 
 /**
- * @since 4.0.0
+ * Runtime type identifier for `PersistedQueueError`.
+ *
  * @category Errors
+ * @since 4.0.0
  */
 export const ErrorTypeId: ErrorTypeId = "~@effect/experimental/PersistedQueue/PersistedQueueError"
 
 /**
- * @since 4.0.0
+ * Type-level identifier used to brand `PersistedQueueError` values.
+ *
  * @category Errors
+ * @since 4.0.0
  */
 export type ErrorTypeId = "~@effect/experimental/PersistedQueue/PersistedQueueError"
 
 /**
- * @since 4.0.0
+ * Error raised by persisted queue store operations.
+ *
  * @category Errors
+ * @since 4.0.0
  */
 export class PersistedQueueError extends Schema.ErrorClass<PersistedQueueError>(
   "effect/persistence/PersistedQueue/PersistedQueueError"
@@ -186,14 +240,21 @@ export class PersistedQueueError extends Schema.ErrorClass<PersistedQueueError>(
   cause: Schema.optional(Schema.Defect)
 }) {
   /**
+   * Marks this value as a persisted queue error for runtime guards.
+   *
    * @since 4.0.0
    */
   readonly [ErrorTypeId]: ErrorTypeId = ErrorTypeId
 }
 
 /**
- * @since 4.0.0
+ * Low-level backing store service used by `PersistedQueue`.
+ *
+ * The store persists offered elements and returns taken elements in a scope so
+ * the finalizer can complete or retry them based on the processing exit.
+ *
  * @category Store
+ * @since 4.0.0
  */
 export class PersistedQueueStore extends Context.Service<
   PersistedQueueStore,
@@ -223,8 +284,13 @@ export class PersistedQueueStore extends Context.Service<
 >()("effect/persistence/PersistedQueue/PersistedQueueStore") {}
 
 /**
- * @since 4.0.0
+ * Provides an in-memory `PersistedQueueStore`.
+ *
+ * The store is process-local and volatile; failed takes are requeued until the
+ * configured maximum attempts is reached.
+ *
  * @category Store
+ * @since 4.0.0
  */
 export const layerStoreMemory: Layer.Layer<
   PersistedQueueStore
@@ -289,8 +355,14 @@ export const layerStoreMemory: Layer.Layer<
 })
 
 /**
- * @since 4.0.0
+ * Creates a Redis-backed `PersistedQueueStore`.
+ *
+ * The store uses Redis lists and hashes with worker locks, periodically
+ * refreshes locks while items are being processed, and moves exhausted items
+ * to a failed queue.
+ *
  * @category Store
+ * @since 4.0.0
  */
 export const makeStoreRedis = Effect.fnUntraced(function*(
   options?: {
@@ -648,8 +720,10 @@ end
 )
 
 /**
- * @since 4.0.0
+ * Provides a Redis-backed `PersistedQueueStore` using `makeStoreRedis`.
+ *
  * @category Store
+ * @since 4.0.0
  */
 export const layerStoreRedis: (
   options?: {
@@ -665,8 +739,14 @@ export const layerStoreRedis: (
 > = flow(makeStoreRedis, Layer.effect(PersistedQueueStore))
 
 /**
- * @since 4.0.0
+ * Creates a SQL-backed `PersistedQueueStore`.
+ *
+ * The store creates the queue table and indexes, acquires rows with
+ * per-worker locks, refreshes active locks while scoped takes are running, and
+ * retries or completes rows according to the processing exit.
+ *
  * @category Store
+ * @since 4.0.0
  */
 export const makeStoreSql: (
   options?: {
@@ -1093,8 +1173,10 @@ class QueueKey extends Data.Class<{
 }> {}
 
 /**
- * @since 4.0.0
+ * Provides a SQL-backed `PersistedQueueStore` using `makeStoreSql`.
+ *
  * @category Store
+ * @since 4.0.0
  */
 export const layerStoreSql: (
   options?: {

package/src/unstable/persistence/Persistence.ts

@@ -1,4 +1,27 @@
 /**
+ * Durable storage for encoded `Persistable` request results.
+ *
+ * The `Persistence` service creates scoped stores that read and write
+ * schema-encoded `Exit` values keyed by each request's `PrimaryKey`. It is the
+ * lower-level persistence layer used by `PersistedCache` and similar request
+ * workflows to reuse expensive or idempotent lookup results across fibers,
+ * process restarts, and workers that share a backing store.
+ *
+ * Each store is selected by a `storeId`, while each entry id comes from the
+ * request's primary key. Keep both stable and collision-free: changing the
+ * `storeId`, the primary-key format, or the success/error schemas is a
+ * persistence migration, because old entries may stop being found or fail to
+ * decode. Values are encoded with the request's success and error schemas using
+ * the JSON codec, so any required schema services must be available at store
+ * read and write time, and the backing value must stay JSON-compatible.
+ *
+ * TTLs are computed from the stored `Exit` and request key. Infinite TTLs are
+ * stored without an expiration, finite TTLs become backing-store expirations,
+ * and zero or negative TTLs skip the write entirely. Backing layers provide
+ * process-local memory, `KeyValueStore`, Redis, and SQL implementations; store
+ * ids are used as prefixes, table names, or SQL partitions and should be
+ * chosen with the target backing store in mind.
+ *
  * @since 4.0.0
  */
 import * as Arr from "../../Array.ts"
@@ -21,8 +44,10 @@ import * as Redis from "./Redis.ts"
 const ErrorTypeId = "~effect/persistence/Persistence/PersistenceError" as const
 
 /**
- * @since 4.0.0
+ * Error raised by persistence and backing-store operations.
+ *
  * @category errors
+ * @since 4.0.0
  */
 export class PersistenceError extends Schema.ErrorClass<PersistenceError>(ErrorTypeId)({
   _tag: Schema.tag("PersistenceError"),
@@ -30,14 +55,19 @@ export class PersistenceError extends Schema.ErrorClass<PersistenceError>(ErrorT
   cause: Schema.optional(Schema.Defect)
 }) {
   /**
+   * Marks this value as a persistence error for runtime guards.
+   *
    * @since 4.0.0
    */
   readonly [ErrorTypeId]: typeof ErrorTypeId = ErrorTypeId
 }
 
 /**
- * @since 4.0.0
+ * Service for creating scoped stores of persisted `Persistable` request
+ * results.
+ *
  * @category Models
+ * @since 4.0.0
  */
 export class Persistence extends Context.Service<Persistence, {
   readonly make: (options: {
@@ -47,8 +77,10 @@ export class Persistence extends Context.Service<Persistence, {
 }>()("effect/persistence/Persistence") {}
 
 /**
- * @since 4.0.0
+ * Typed store for persisted `Exit` values keyed by `Persistable` requests.
+ *
  * @category models
+ * @since 4.0.0
  */
 export interface PersistenceStore {
   readonly get: <A extends Schema.Top, E extends Schema.Top>(
@@ -79,16 +111,21 @@ export interface PersistenceStore {
 }
 
 /**
- * @since 4.0.0
+ * Service for creating raw backing stores for persistence store ids.
+ *
  * @category BackingPersistence
+ * @since 4.0.0
  */
 export class BackingPersistence extends Context.Service<BackingPersistence, {
   readonly make: (storeId: string) => Effect.Effect<BackingPersistenceStore, never, Scope.Scope>
 }>()("effect/persistence/BackingPersistence") {}
 
 /**
- * @since 4.0.0
+ * Raw persistence backing store for JSON-compatible objects with optional
+ * TTLs.
+ *
  * @category BackingPersistence
+ * @since 4.0.0
  */
 export interface BackingPersistenceStore {
   readonly get: (key: string) => Effect.Effect<object | undefined, PersistenceError>
@@ -108,8 +145,13 @@ export interface BackingPersistenceStore {
 }
 
 /**
- * @since 4.0.0
+ * Provides `Persistence` from `BackingPersistence`.
+ *
+ * The layer serializes and deserializes `Persistable` exits, applies
+ * per-entry TTLs, and skips writes whose TTL is zero or negative.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layer = Layer.effect(Persistence)(Effect.gen(function*() {
   const backing = yield* BackingPersistence
@@ -202,8 +244,12 @@ export const layer = Layer.effect(Persistence)(Effect.gen(function*() {
 }))
 
 /**
- * @since 4.0.0
+ * Provides an in-memory `BackingPersistence` grouped by store id.
+ *
+ * Entries are process-local and expire according to their stored TTL.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerBackingMemory: Layer.Layer<BackingPersistence> = Layer.sync(BackingPersistence)(
   () => {
@@ -249,8 +295,13 @@ export const layerBackingMemory: Layer.Layer<BackingPersistence> = Layer.sync(Ba
 )
 
 /**
- * @since 4.0.0
+ * Provides SQL-backed persistence using one table per store id.
+ *
+ * Each table is created if needed and stores JSON-encoded values with optional
+ * expiration timestamps.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerBackingSqlMultiTable: Layer.Layer<
   BackingPersistence,
@@ -457,8 +508,13 @@ export const layerBackingSqlMultiTable: Layer.Layer<
 }))
 
 /**
- * @since 4.0.0
+ * Provides SQL-backed persistence using a shared `effect_persistence` table.
+ *
+ * Rows are partitioned by `store_id` and store JSON-encoded values with
+ * optional expiration timestamps.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerBackingSql: Layer.Layer<
   BackingPersistence,
@@ -695,8 +751,13 @@ export const layerBackingSql: Layer.Layer<
 }))
 
 /**
- * @since 4.0.0
+ * Provides Redis-backed persistence.
+ *
+ * Each store id is used as a key prefix, values are JSON-encoded, and finite
+ * TTLs are stored with Redis expiration.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerBackingRedis: Layer.Layer<
   BackingPersistence,
@@ -849,8 +910,13 @@ end
 )
 
 /**
- * @since 4.0.0
+ * Provides `BackingPersistence` using a `KeyValueStore`.
+ *
+ * Each store id becomes a key prefix, and values are stored as JSON with
+ * optional expiration timestamps.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerBackingKvs: Layer.Layer<
   BackingPersistence,
@@ -945,46 +1011,61 @@ export const layerBackingKvs: Layer.Layer<
 }))
 
 /**
- * @since 4.0.0
+ * Provides `Persistence` backed by the current `KeyValueStore`.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerKvs: Layer.Layer<Persistence, never, KeyValueStore.KeyValueStore> = layer.pipe(
   Layer.provide(layerBackingKvs)
 )
 
 /**
- * @since 4.0.0
+ * Provides `Persistence` backed by process-local in-memory storage.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerMemory: Layer.Layer<Persistence> = layer.pipe(
   Layer.provide(layerBackingMemory)
 )
 
 /**
- * @since 4.0.0
+ * Provides `Persistence` backed by the current `Redis` service.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerRedis: Layer.Layer<Persistence, never, Redis.Redis> = layer.pipe(
   Layer.provide(layerBackingRedis)
 )
 
 /**
- * @since 4.0.0
+ * Provides `Persistence` backed by SQL with one table per store id.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerSqlMultiTable: Layer.Layer<Persistence, never, SqlClient.SqlClient> = layer.pipe(
   Layer.provide(layerBackingSqlMultiTable)
 )
 
 /**
- * @since 4.0.0
+ * Provides `Persistence` backed by SQL using a shared persistence table.
+ *
  * @category layers
+ * @since 4.0.0
  */
 export const layerSql: Layer.Layer<Persistence, never, SqlClient.SqlClient> = layer.pipe(
   Layer.provide(layerBackingSql)
 )
 
 /**
+ * Converts a TTL to an absolute expiration timestamp in milliseconds.
+ *
+ * Returns `null` for no TTL and uses `clock.currentTimeMillisUnsafe`, so it is
+ * intended for backing-store internals.
+ *
  * @since 4.0.0
  */
 export const unsafeTtlToExpires = (clock: Clock.Clock, ttl: Duration.Duration | undefined): number | null =>