@effect-app/infra 4.0.0-beta.281 → 4.0.0-beta.283

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/routing.ts CHANGED
@@ -2,9 +2,11 @@
2
2
  /* eslint-disable @typescript-eslint/no-unsafe-argument */
3
3
  /* eslint-disable @typescript-eslint/no-empty-object-type */
4
4
  /* eslint-disable @typescript-eslint/no-explicit-any */
5
+ import * as Array from "effect-app/Array"
5
6
  import type { NonEmptyReadonlyArray } from "effect-app/Array"
6
7
  import { getMeta } from "effect-app/client"
7
8
  import * as Config from "effect-app/Config"
9
+ import * as DataDependencies from "effect-app/DataDependencies"
8
10
  import * as Effect from "effect-app/Effect"
9
11
  import { type HttpHeaders } from "effect-app/http"
10
12
  import * as Layer from "effect-app/Layer"
@@ -454,10 +456,19 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
454
456
  // clients can invalidate queries even when the stream fails.
455
457
  const keysRef = Ref.makeUnsafe<ReadonlyArray<Invalidation.InvalidationKey>>([])
456
458
  const invalidationSet = Invalidation.makeInvalidationSet(keysRef)
459
+ const readsRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
460
+ const writesRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
461
+ const dependencyRecorder = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
462
+ const metadata = (keys: ReadonlyArray<Invalidation.InvalidationKey>) =>
463
+ Effect.map(
464
+ dependencyRecorder.drain,
465
+ (dataDependencies) => Invalidation.makeMetaData(keys, dataDependencies)
466
+ )
457
467
  return Stream.concat(
458
468
  (result as Stream.Stream<any, any, any>).pipe(
459
469
  Stream.map((item: any) => ({ _tag: "value" as const, value: item })),
460
470
  Stream.provideService(Invalidation.InvalidationSet, invalidationSet),
471
+ Stream.provideService(DataDependencies.DataDependencyRecorder, dependencyRecorder),
461
472
  // V3: after each value chunk, drain accumulated keys and emit a "metadata"
462
473
  // chunk if any keys were collected since the last drain. This lets clients
463
474
  // invalidate queries mid-stream without waiting for the "done" chunk.
@@ -465,13 +476,19 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
465
476
  Stream
466
477
  .fromEffect(
467
478
  Ref.getAndSet(keysRef, []).pipe(
468
- Effect.map((keys) =>
469
- keys.length > 0
470
- ? [
471
- valueChunk,
472
- { _tag: "metadata" as const, metadata: { invalidateQueries: keys } }
473
- ]
474
- : [valueChunk]
479
+ Effect.flatMap((keys) =>
480
+ metadata(keys).pipe(
481
+ Effect.map((meta) =>
482
+ Array.isReadonlyArrayNonEmpty(keys)
483
+ || Array.isReadonlyArrayNonEmpty(meta.dataDependencies.reads)
484
+ || Array.isReadonlyArrayNonEmpty(meta.dataDependencies.writes)
485
+ ? [
486
+ valueChunk,
487
+ { _tag: "metadata" as const, metadata: meta }
488
+ ]
489
+ : [valueChunk]
490
+ )
491
+ )
475
492
  )
476
493
  )
477
494
  )
@@ -482,11 +499,15 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
482
499
  Stream.fromEffect(
483
500
  Ref.get(keysRef).pipe(
484
501
  Effect.flatMap((keys) =>
485
- Effect.fail({
486
- _tag: "error" as const,
487
- error: err,
488
- metadata: { invalidateQueries: keys }
489
- })
502
+ metadata(keys).pipe(
503
+ Effect.flatMap((meta) =>
504
+ Effect.fail({
505
+ _tag: "error" as const,
506
+ error: err,
507
+ metadata: meta
508
+ })
509
+ )
510
+ )
490
511
  )
491
512
  )
492
513
  )
@@ -494,7 +515,11 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
494
515
  ),
495
516
  Stream.fromEffect(
496
517
  Ref.get(keysRef).pipe(
497
- Effect.map((keys) => ({ _tag: "done" as const, metadata: { invalidateQueries: keys } }))
518
+ Effect.flatMap((keys) =>
519
+ metadata(keys).pipe(
520
+ Effect.map((meta) => ({ _tag: "done" as const, metadata: meta }))
521
+ )
522
+ )
498
523
  )
499
524
  )
500
525
  )
@@ -511,6 +536,13 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
511
536
  })
512
537
  .pipe(Effect.andThen(result as Effect.Effect<unknown, unknown, unknown>))
513
538
 
539
+ const readsRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
540
+ const writesRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
541
+ const dependencyRecorder = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
542
+ effect = effect.pipe(
543
+ Effect.provideService(DataDependencies.DataDependencyRecorder, dependencyRecorder)
544
+ )
545
+
514
546
  // Commands: provide a request-scoped `InvalidationSet` and wrap both
515
547
  // success (`CommandResponseWithMetaData`) and handler-thrown failure
516
548
  // (`CommandFailureWithMetaData`) so the client receives accumulated
@@ -525,21 +557,42 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
525
557
  Effect.provideService(Invalidation.InvalidationSet, invalidationSet),
526
558
  Effect.flatMap((value) =>
527
559
  Ref.get(keysRef).pipe(
528
- Effect.map((keys) => ({ payload: value, metadata: { invalidateQueries: keys } }) as any)
560
+ Effect.flatMap((keys) =>
561
+ dependencyRecorder.get.pipe(
562
+ Effect.map((dataDependencies) =>
563
+ ({
564
+ payload: value,
565
+ metadata: Invalidation.makeMetaData(keys, dataDependencies)
566
+ }) as any
567
+ )
568
+ )
569
+ )
529
570
  )
530
571
  ),
531
572
  Effect.catch((err: any) =>
532
573
  Ref.get(keysRef).pipe(
533
574
  Effect.flatMap((keys) =>
534
- Effect.fail({
535
- _tag: "CommandFailureWithMetaData" as const,
536
- error: err,
537
- metadata: { invalidateQueries: keys }
538
- })
575
+ dependencyRecorder.get.pipe(
576
+ Effect.flatMap((dataDependencies) =>
577
+ Effect.fail({
578
+ _tag: "CommandFailureWithMetaData" as const,
579
+ error: err,
580
+ metadata: Invalidation.makeMetaData(keys, dataDependencies)
581
+ })
582
+ )
583
+ )
539
584
  )
540
585
  )
541
586
  )
542
587
  )
588
+ } else {
589
+ effect = effect.pipe(
590
+ Effect.flatMap((value) =>
591
+ dependencyRecorder.get.pipe(
592
+ Effect.map((dataDependencies) => ({ payload: value, metadata: { dataDependencies } }) as any)
593
+ )
594
+ )
595
+ )
543
596
  }
544
597
 
545
598
  return applyRequestTypeInterruptibility(resource.type, effect)
@@ -584,7 +637,7 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
584
637
  })
585
638
  : Rpc.make(resource._tag, {
586
639
  payload: resource,
587
- success: resource.success,
640
+ success: isStream ? resource.success : Invalidation.QueryResponseWithMetaData(resource.success),
588
641
  error: resource.error,
589
642
  stream: isStream
590
643
  }))
@@ -1,10 +1,12 @@
1
1
  import { describe, expect, it } from "@effect/vitest"
2
+ import * as DataDependencies from "effect-app/DataDependencies"
2
3
  import * as Effect from "effect-app/Effect"
3
4
  import * as Layer from "effect-app/Layer"
4
5
  import { makeRepo } from "effect-app/Model/Repository"
5
6
  import { RepositoryRegistryLive } from "effect-app/Model/Repository/Registry"
6
7
  import * as S from "effect-app/Schema"
7
8
  import { setupRequestContextFromCurrent } from "effect-app/setupRequest"
9
+ import * as Ref from "effect/Ref"
8
10
  import { MemoryStoreLive } from "../src/Store/Memory.js"
9
11
 
10
12
  class BatchItem extends S.Class<BatchItem>("BatchItem")({
@@ -59,4 +61,29 @@ describe("repository ext save/remove batching", () => {
59
61
  setupRequestContextFromCurrent(),
60
62
  Effect.provide(TestStoreLive)
61
63
  ))
64
+
65
+ it.effect("records repository read and write dependencies", () =>
66
+ Effect
67
+ .gen(function*() {
68
+ const readsRef = yield* Ref.make<DataDependencies.DataDependencies>([])
69
+ const writesRef = yield* Ref.make<DataDependencies.DataDependencies>([])
70
+ const recorder = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
71
+
72
+ yield* Effect
73
+ .gen(function*() {
74
+ const repo = yield* makeRepo("DependencyItem", BatchItem, {})
75
+ yield* repo.save(new BatchItem({ id: "1", label: "one" }))
76
+ yield* repo.all
77
+ yield* repo.find("1")
78
+ yield* repo.removeById("1")
79
+ })
80
+ .pipe(Effect.provideService(DataDependencies.DataDependencyRecorder, recorder))
81
+
82
+ expect(yield* Ref.get(readsRef)).toEqual([DataDependencies.repo("DependencyItem")])
83
+ expect(yield* Ref.get(writesRef)).toEqual([DataDependencies.repo("DependencyItem")])
84
+ })
85
+ .pipe(
86
+ setupRequestContextFromCurrent(),
87
+ Effect.provide(TestStoreLive)
88
+ ))
62
89
  })
@@ -14,12 +14,15 @@
14
14
  */
15
15
  import { NodeHttpServer } from "@effect/platform-node"
16
16
  import { expect, it } from "@effect/vitest"
17
- import { ApiClientFactory, InvalidationKeysFromServer, makeInvalidationKeysService, makeRpcClient } from "effect-app/client"
17
+ import { ApiClientFactory, DataDependencies, InvalidationKeysFromServer, InvalidStateError, makeInvalidationKeysService, makeRpcClient, OptimisticConcurrencyException } from "effect-app/client"
18
+ import * as Context from "effect-app/Context"
18
19
  import { HttpRouter, HttpServer } from "effect-app/http"
19
20
  import { DefaultGenericMiddlewares } from "effect-app/middleware"
21
+ import { makeRepo, RepositoryRegistryLive } from "effect-app/Model"
20
22
  import { Invalidation, MiddlewareMaker } from "effect-app/rpc"
21
23
  import * as S from "effect-app/Schema"
22
24
  import { TaggedErrorClass } from "effect-app/Schema"
25
+ import { setupRequestContextFromCurrent } from "effect-app/setupRequest"
23
26
  import * as Effect from "effect/Effect"
24
27
  import * as Exit from "effect/Exit"
25
28
  import * as Layer from "effect/Layer"
@@ -31,6 +34,7 @@ import { RpcSerialization } from "effect/unstable/rpc"
31
34
  import { createServer } from "http"
32
35
  import { makeRouter } from "../src/routing.js"
33
36
  import { DefaultGenericMiddlewaresLive } from "../src/routing/middleware.js"
37
+ import { MemoryStoreLive } from "../src/Store/Memory.js"
34
38
  import { AllowAnonymous, AllowAnonymousLive, RequestContextMap, RequireRoles, RequireRolesLive, SomeElseMiddleware, SomeElseMiddlewareLive, SomeService, Test, TestLive } from "./fixtures.js"
35
39
 
36
40
  // ---------------------------------------------------------------------------
@@ -99,14 +103,82 @@ class StreamWithKey extends Req.Command<StreamWithKey>()("StreamWithKey", {}, {
99
103
  success: S.Number
100
104
  }) {}
101
105
 
102
- const InvRsc = { DoNothing, DoWithDynamicKey, DoWithBothKeys, DoAndFail, StreamWithKey }
106
+ class StreamWithRepoWrite extends Req.Command<StreamWithRepoWrite>()("StreamWithRepoWrite", {}, {
107
+ stream: true,
108
+ allowAnonymous: true,
109
+ success: S.Number
110
+ }) {}
111
+
112
+ class RepoItem extends S.Class<RepoItem>("RepoItem")({
113
+ id: S.String,
114
+ label: S.String
115
+ }) {}
116
+
117
+ class GetRepoCount extends Req.Query<GetRepoCount>()("GetRepoCount", {}, {
118
+ allowAnonymous: true,
119
+ success: S.Number
120
+ }) {}
121
+
122
+ class SaveRepoItem extends Req.Command<SaveRepoItem>()("SaveRepoItem", {
123
+ id: S.String,
124
+ label: S.String
125
+ }, {
126
+ allowAnonymous: true,
127
+ error: S.Union([InvalidStateError, OptimisticConcurrencyException]),
128
+ success: S.Void
129
+ }) {}
130
+
131
+ class RepoItems extends Context.Service<RepoItems>()("RepoItems", {
132
+ make: makeRepo("RepoItem", RepoItem, {})
133
+ }) {
134
+ static Default = Layer.effect(this, this.make).pipe(
135
+ Layer.provide(Layer.merge(MemoryStoreLive, RepositoryRegistryLive))
136
+ )
137
+ }
138
+
139
+ // A second, unrelated repo — its writes must NOT invalidate a query that only read `RepoItem`.
140
+ class OtherItem extends S.Class<OtherItem>("OtherItem")({
141
+ id: S.String,
142
+ label: S.String
143
+ }) {}
144
+
145
+ class SaveOtherItem extends Req.Command<SaveOtherItem>()("SaveOtherItem", {
146
+ id: S.String,
147
+ label: S.String
148
+ }, {
149
+ allowAnonymous: true,
150
+ success: S.Void
151
+ }) {}
152
+
153
+ class OtherItems extends Context.Service<OtherItems>()("OtherItems", {
154
+ make: makeRepo("OtherItem", OtherItem, {})
155
+ }) {
156
+ static Default = Layer.effect(this, this.make).pipe(
157
+ Layer.provide(Layer.merge(MemoryStoreLive, RepositoryRegistryLive))
158
+ )
159
+ }
160
+
161
+ const InvRsc = {
162
+ DoNothing,
163
+ DoWithDynamicKey,
164
+ DoWithBothKeys,
165
+ DoAndFail,
166
+ StreamWithKey,
167
+ StreamWithRepoWrite,
168
+ GetRepoCount,
169
+ SaveRepoItem,
170
+ SaveOtherItem
171
+ }
103
172
 
104
173
  // ---------------------------------------------------------------------------
105
174
  // Controllers / router
106
175
  // ---------------------------------------------------------------------------
107
176
 
108
177
  const router = Router(InvRsc)({
178
+ dependencies: [RepoItems.Default, OtherItems.Default],
109
179
  *effect(match) {
180
+ const repo = yield* RepoItems
181
+ const otherRepo = yield* OtherItems
110
182
  return match({
111
183
  DoNothing: () => Effect.void,
112
184
  DoWithDynamicKey: Effect.fnUntraced(function*() {
@@ -125,7 +197,18 @@ const router = Router(InvRsc)({
125
197
  StreamWithKey: () =>
126
198
  Stream.fromIterable([1, 2, 3]).pipe(
127
199
  Stream.tap(() => Invalidation.InvalidationSet.use((_) => _.add(StreamKey)))
128
- )
200
+ ),
201
+ StreamWithRepoWrite: () =>
202
+ Stream.fromIterable([1, 2, 3]).pipe(
203
+ Stream.tap((n) =>
204
+ repo.save(new RepoItem({ id: String(n), label: "x" })).pipe(Effect.orDie, setupRequestContextFromCurrent())
205
+ )
206
+ ),
207
+ GetRepoCount: () => repo.all.pipe(Effect.map((_) => _.length), Effect.orDie, setupRequestContextFromCurrent()),
208
+ SaveRepoItem: ({ id, label }) =>
209
+ repo.save(new RepoItem({ id, label })).pipe(Effect.orDie, setupRequestContextFromCurrent()),
210
+ SaveOtherItem: ({ id, label }) =>
211
+ otherRepo.save(new OtherItem({ id, label })).pipe(Effect.orDie, setupRequestContextFromCurrent())
129
212
  })
130
213
  }
131
214
  })
@@ -171,6 +254,15 @@ const withCapture = <A, E, R>(eff: Effect.Effect<A, E, R>) =>
171
254
  return { result, keys: yield* Ref.get(ref) }
172
255
  })
173
256
 
257
+ const withDependencyCapture = <A, E, R>(eff: Effect.Effect<A, E, R>) =>
258
+ Effect.gen(function*() {
259
+ const readsRef = yield* Ref.make<DataDependencies.DataDependencies>([])
260
+ const writesRef = yield* Ref.make<DataDependencies.DataDependencies>([])
261
+ const svc = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
262
+ const result = yield* eff.pipe(Effect.provideService(DataDependencies.DataDependencyRecorder, svc), Effect.exit)
263
+ return { result, dependencies: yield* svc.get }
264
+ })
265
+
174
266
  // ---------------------------------------------------------------------------
175
267
  // Tests
176
268
  // ---------------------------------------------------------------------------
@@ -254,3 +346,75 @@ it.live(
254
346
  }, Effect.provide(TestLayer)),
255
347
  { timeout: 10_000 }
256
348
  )
349
+
350
+ it.live(
351
+ "stream: per-chunk repo writes are drained and forwarded to the client recorder",
352
+ Effect.fnUntraced(function*() {
353
+ const client = yield* ApiClientFactory.makeFor(Layer.empty)(InvRsc)
354
+ const readsRef = yield* Ref.make<DataDependencies.DataDependencies>([])
355
+ const writesRef = yield* Ref.make<DataDependencies.DataDependencies>([])
356
+ const svc = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
357
+ const values = yield* Stream.runCollect(client.StreamWithRepoWrite.handler()).pipe(
358
+ Effect.provideService(DataDependencies.DataDependencyRecorder, svc)
359
+ )
360
+ const writes = yield* Ref.get(writesRef)
361
+ expect(values).toStrictEqual([1, 2, 3])
362
+ // Each emitted value writes to RepoItem; routing drains the writes per chunk and the client
363
+ // accumulates them — the recorder dedupes, so a single RepoItem entry is recorded.
364
+ expect(writes).toStrictEqual([DataDependencies.repo("RepoItem")])
365
+ }, Effect.provide(TestLayer)),
366
+ { timeout: 10_000 }
367
+ )
368
+
369
+ it.live(
370
+ "repository dependencies flow through query and command metadata",
371
+ Effect.fnUntraced(function*() {
372
+ const client = yield* ApiClientFactory.makeFor(Layer.empty)(InvRsc)
373
+
374
+ const query = yield* withDependencyCapture(client.GetRepoCount.handler())
375
+ expect(Exit.isSuccess(query.result) && query.result.value).toBe(0)
376
+ expect(query.dependencies.reads).toStrictEqual([DataDependencies.repo("RepoItem")])
377
+ expect(query.dependencies.writes).toStrictEqual([])
378
+
379
+ const command = yield* withDependencyCapture(client.SaveRepoItem.handler({ id: "1", label: "one" }))
380
+ expect(Exit.isSuccess(command.result)).toBe(true)
381
+ expect(command.dependencies.reads).toStrictEqual([])
382
+ expect(command.dependencies.writes).toStrictEqual([DataDependencies.repo("RepoItem")])
383
+ }, Effect.provide(TestLayer)),
384
+ { timeout: 10_000 }
385
+ )
386
+
387
+ it.live(
388
+ "query invalidation: a command's writes invalidate exactly the queries whose reads intersect",
389
+ Effect.fnUntraced(function*() {
390
+ const client = yield* ApiClientFactory.makeFor(Layer.empty)(InvRsc)
391
+
392
+ // Client-side query registry: queryKey -> read dependencies forwarded by the server. Mirrors
393
+ // @effect-app/vue's `dependencyMetadata`; the derivation predicate (`intersects`) is the exact
394
+ // one the vue mutate engine uses to pick invalidation targets from a command's writes.
395
+ const queryReads = new Map<string, DataDependencies.DataDependencies>()
396
+ const invalidatedBy = (writes: DataDependencies.DataDependencies) =>
397
+ [...queryReads]
398
+ .filter(([, reads]) => DataDependencies.intersects(reads, writes))
399
+ .map(([key]) => key)
400
+
401
+ // Run the query through the real client; register its forwarded reads under its key.
402
+ const query = yield* withDependencyCapture(client.GetRepoCount.handler())
403
+ expect(Exit.isSuccess(query.result)).toBe(true)
404
+ queryReads.set("GetRepoCount", query.dependencies.reads)
405
+ // Before any command, nothing is invalidated.
406
+ expect(invalidatedBy([])).toStrictEqual([])
407
+
408
+ // A command writing the SAME repo the query read => the query is selected for invalidation.
409
+ const save = yield* withDependencyCapture(client.SaveRepoItem.handler({ id: "1", label: "one" }))
410
+ expect(Exit.isSuccess(save.result)).toBe(true)
411
+ expect(invalidatedBy(save.dependencies.writes)).toStrictEqual(["GetRepoCount"])
412
+
413
+ // A command writing an UNRELATED repo => the query is NOT invalidated (negative control).
414
+ const saveOther = yield* withDependencyCapture(client.SaveOtherItem.handler({ id: "2", label: "two" }))
415
+ expect(Exit.isSuccess(saveOther.result)).toBe(true)
416
+ expect(saveOther.dependencies.writes).toStrictEqual([DataDependencies.repo("OtherItem")])
417
+ expect(invalidatedBy(saveOther.dependencies.writes)).toStrictEqual([])
418
+ }, Effect.provide(TestLayer)),
419
+ { timeout: 10_000 }
420
+ )
@@ -1,186 +0,0 @@
1
- import { reportNonInterruptedFailure } from "@effect-app/infra/QueueMaker/errors"
2
- import { subMinutes } from "date-fns"
3
- import type { NonEmptyReadonlyArray } from "effect-app/Array"
4
- import * as Effect from "effect-app/Effect"
5
- import * as Option from "effect-app/Option"
6
- import { type QueueBase, QueueMeta } from "effect-app/QueueMaker"
7
- import * as S from "effect-app/Schema"
8
- import { type NonEmptyString255 } from "effect-app/Schema"
9
- import { getRequestContext, setupRequestContextWithCustomSpan } from "effect-app/setupRequest"
10
- import { pretty } from "effect-app/utils"
11
- import * as Fiber from "effect/Fiber"
12
- import * as Tracer from "effect/Tracer"
13
- import { SqlClient } from "effect/unstable/sql"
14
- import { InfraLogger } from "../logger.ts"
15
- import { messagingSpanArgs } from "../otel.ts"
16
- import { SQLModel } from "../SQL.ts"
17
-
18
- export const QueueId = S.Finite.pipe(S.brand("QueueId"))
19
- export type QueueId = typeof QueueId.Type
20
-
21
- // TODO: let the model track and Auto Generate versionColumn on every update instead
22
- export const makeSQLQueue = Effect.fnUntraced(function*<
23
- Evt extends { id: S.StringId; _tag: string },
24
- DrainEvt extends { id: S.StringId; _tag: string },
25
- EvtE,
26
- DrainEvtE
27
- >(
28
- queueName: NonEmptyString255,
29
- queueDrainName: NonEmptyString255,
30
- schema: S.Codec<Evt, EvtE>,
31
- drainSchema: S.Codec<DrainEvt, DrainEvtE>
32
- ) {
33
- const base = {
34
- id: SQLModel.Generated(QueueId),
35
- meta: SQLModel.JsonFromString(QueueMeta),
36
- name: S.NonEmptyString255,
37
- createdAt: SQLModel.DateTimeInsert,
38
- updatedAt: SQLModel.DateTimeUpdate,
39
- // TODO: at+owner
40
- processingAt: SQLModel.FieldOption(S.Date),
41
- finishedAt: SQLModel.FieldOption(S.Date),
42
- etag: S.String // TODO: use a SQLModel thing that auto updates it?
43
- // TODO: record locking.. / optimistic locking
44
- // rowVersion: SQLModel.DateTimeFromNumberWithNow
45
- }
46
- class Queue extends SQLModel.Class<Queue>("Queue")({
47
- body: SQLModel.JsonFromString(schema),
48
- ...base
49
- }) {}
50
- class Drain extends SQLModel.Class<Drain>("Drain")({
51
- body: SQLModel.JsonFromString(drainSchema),
52
- ...base
53
- }) {}
54
- const sql = yield* SqlClient.SqlClient
55
-
56
- const queueRepo = yield* SQLModel.makeRepository(Queue, {
57
- tableName: "queue",
58
- spanPrefix: "QueueRepo",
59
- idColumn: "id",
60
- versionColumn: "etag"
61
- })
62
-
63
- const drainRepo = yield* SQLModel.makeRepository(Drain, {
64
- tableName: "queue",
65
- spanPrefix: "DrainRepo",
66
- idColumn: "id",
67
- versionColumn: "etag"
68
- })
69
-
70
- const decodeDrain = S.decodeEffectConcurrently(Drain)
71
-
72
- const drain = Effect.gen(function*() {
73
- const limit = subMinutes(new Date(), 15)
74
- return yield* sql<typeof Drain.Encoded>`SELECT *
75
- FROM queue
76
- WHERE name = ${queueDrainName} AND finishedAt IS NULL AND (processingAt IS NULL OR processingAt < ${limit.getTime()})
77
- LIMIT 1`
78
- })
79
-
80
- const q = {
81
- offer: Effect.fnUntraced(function*(body: Evt, meta: typeof QueueMeta.Type) {
82
- yield* queueRepo.insertVoid(Queue.insert.make({
83
- body,
84
- meta,
85
- name: queueName,
86
- processingAt: Option.none(),
87
- finishedAt: Option.none(),
88
- etag: crypto.randomUUID()
89
- }))
90
- }),
91
- take: Effect.gen(function*() {
92
- while (true) {
93
- const [first] = yield* drain.pipe(Effect.withTracerEnabled(false)) // disable sql tracer otherwise we spam it..
94
- if (first) {
95
- const dec = yield* decodeDrain(first)
96
- const { createdAt, updatedAt, ...rest } = dec
97
- return yield* drainRepo.update(
98
- Drain.update.make({ ...rest, processingAt: Option.some(new Date()) }) // auto in lib , etag: crypto.randomUUID()
99
- )
100
- }
101
- if (first) return first
102
- yield* Effect.sleep(250)
103
- }
104
- }),
105
- finish: Effect.fn(function*({ createdAt, updatedAt, ...q }: Drain) {
106
- return yield* drainRepo.updateVoid(Drain.update.make({ ...q, finishedAt: Option.some(new Date()) })) // auto in lib , etag: crypto.randomUUID()
107
- })
108
- }
109
- const queue = {
110
- publish: Effect.fn(`publish ${queueName}`, {
111
- kind: "producer",
112
- attributes: {
113
- "messaging.system": "sql",
114
- "messaging.operation.name": "publish",
115
- "messaging.destination.name": queueName
116
- }
117
- })(function*(
118
- ...messages: NonEmptyReadonlyArray<Evt>
119
- ) {
120
- yield* Effect.annotateCurrentSpan({
121
- "messaging.batch.message_count": messages.length,
122
- "messaging.message.types": messages.map((_) => _._tag)
123
- })
124
- const requestContext = yield* getRequestContext
125
- yield* Effect.forEach(messages, (m) => q.offer(m, requestContext), { discard: true })
126
- }),
127
- drain: <DrainE, DrainR>(
128
- handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
129
- sessionId?: string
130
- ) => {
131
- const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
132
- const processMessage = Effect.fnUntraced(function*({ body, meta }: Drain) {
133
- let effect = InfraLogger
134
- .logDebug(`[${queueDrainName}] Processing incoming message`)
135
- .pipe(
136
- Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
137
- Effect.andThen(handleEvent(body)),
138
- silenceAndReportError,
139
- (_) => {
140
- const args = messagingSpanArgs({
141
- operation: "process",
142
- system: "sql",
143
- destination: queueDrainName,
144
- messageId: body.id,
145
- conversationId: sessionId,
146
- extra: { "messaging.message.type": body._tag, "messaging.message.body": body }
147
- }, "consumer")
148
- return setupRequestContextWithCustomSpan(
149
- _,
150
- meta,
151
- args.name,
152
- {
153
- captureStackTrace: false,
154
- kind: args.kind,
155
- attributes: args.attributes
156
- }
157
- )
158
- }
159
- )
160
- if (meta.span) {
161
- effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
162
- }
163
- return yield* effect
164
- })
165
-
166
- return Effect.fn(`receive ${queueDrainName}`, {
167
- kind: "consumer",
168
- attributes: {
169
- "messaging.system": "sql",
170
- "messaging.operation.name": "receive",
171
- "messaging.destination.name": queueDrainName,
172
- ...(sessionId !== undefined && { "messaging.message.conversation_id": sessionId })
173
- }
174
- })(function*() {
175
- const x = yield* q.take
176
- yield* processMessage(x).pipe(
177
- Effect.uninterruptible,
178
- Effect.forkChild,
179
- Effect.flatMap(Fiber.join),
180
- Effect.tap(q.finish(x))
181
- )
182
- }, (effect) => effect.pipe(silenceAndReportError, Effect.forever))()
183
- }
184
- }
185
- return queue as QueueBase<Evt, DrainEvt>
186
- })