@effect-app/infra 4.0.0-beta.281 → 4.0.0-beta.283
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/ClusterCosmos.d.ts +15 -15
- package/dist/MainFiberSet.d.ts +1 -0
- package/dist/MainFiberSet.d.ts.map +1 -1
- package/dist/MainFiberSet.js +14 -2
- package/dist/logger.d.ts +4 -4
- package/dist/routing/utils.js +2 -2
- package/dist/routing.d.ts.map +1 -1
- package/dist/routing.js +34 -12
- package/package.json +5 -5
- package/src/MainFiberSet.ts +18 -1
- package/src/routing/utils.ts +1 -1
- package/src/routing.ts +73 -20
- package/test/repository-ext.test.ts +27 -0
- package/test/rpc-e2e-invalidation.test.ts +167 -3
- package/src/QueueMaker/SQLQueue.ts +0 -186
- package/src/QueueMaker/memQueue.ts +0 -136
- package/src/QueueMaker/sbqueue.ts +0 -125
- package/src/RequestFiberSet.ts +0 -110
- package/src/ServiceBus.ts +0 -219
- package/src/memQueue.ts +0 -22
package/src/routing.ts
CHANGED
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
/* eslint-disable @typescript-eslint/no-unsafe-argument */
|
|
3
3
|
/* eslint-disable @typescript-eslint/no-empty-object-type */
|
|
4
4
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
5
|
+
import * as Array from "effect-app/Array"
|
|
5
6
|
import type { NonEmptyReadonlyArray } from "effect-app/Array"
|
|
6
7
|
import { getMeta } from "effect-app/client"
|
|
7
8
|
import * as Config from "effect-app/Config"
|
|
9
|
+
import * as DataDependencies from "effect-app/DataDependencies"
|
|
8
10
|
import * as Effect from "effect-app/Effect"
|
|
9
11
|
import { type HttpHeaders } from "effect-app/http"
|
|
10
12
|
import * as Layer from "effect-app/Layer"
|
|
@@ -454,10 +456,19 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
|
|
|
454
456
|
// clients can invalidate queries even when the stream fails.
|
|
455
457
|
const keysRef = Ref.makeUnsafe<ReadonlyArray<Invalidation.InvalidationKey>>([])
|
|
456
458
|
const invalidationSet = Invalidation.makeInvalidationSet(keysRef)
|
|
459
|
+
const readsRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
|
|
460
|
+
const writesRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
|
|
461
|
+
const dependencyRecorder = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
|
|
462
|
+
const metadata = (keys: ReadonlyArray<Invalidation.InvalidationKey>) =>
|
|
463
|
+
Effect.map(
|
|
464
|
+
dependencyRecorder.drain,
|
|
465
|
+
(dataDependencies) => Invalidation.makeMetaData(keys, dataDependencies)
|
|
466
|
+
)
|
|
457
467
|
return Stream.concat(
|
|
458
468
|
(result as Stream.Stream<any, any, any>).pipe(
|
|
459
469
|
Stream.map((item: any) => ({ _tag: "value" as const, value: item })),
|
|
460
470
|
Stream.provideService(Invalidation.InvalidationSet, invalidationSet),
|
|
471
|
+
Stream.provideService(DataDependencies.DataDependencyRecorder, dependencyRecorder),
|
|
461
472
|
// V3: after each value chunk, drain accumulated keys and emit a "metadata"
|
|
462
473
|
// chunk if any keys were collected since the last drain. This lets clients
|
|
463
474
|
// invalidate queries mid-stream without waiting for the "done" chunk.
|
|
@@ -465,13 +476,19 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
|
|
|
465
476
|
Stream
|
|
466
477
|
.fromEffect(
|
|
467
478
|
Ref.getAndSet(keysRef, []).pipe(
|
|
468
|
-
Effect.
|
|
469
|
-
keys.
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
479
|
+
Effect.flatMap((keys) =>
|
|
480
|
+
metadata(keys).pipe(
|
|
481
|
+
Effect.map((meta) =>
|
|
482
|
+
Array.isReadonlyArrayNonEmpty(keys)
|
|
483
|
+
|| Array.isReadonlyArrayNonEmpty(meta.dataDependencies.reads)
|
|
484
|
+
|| Array.isReadonlyArrayNonEmpty(meta.dataDependencies.writes)
|
|
485
|
+
? [
|
|
486
|
+
valueChunk,
|
|
487
|
+
{ _tag: "metadata" as const, metadata: meta }
|
|
488
|
+
]
|
|
489
|
+
: [valueChunk]
|
|
490
|
+
)
|
|
491
|
+
)
|
|
475
492
|
)
|
|
476
493
|
)
|
|
477
494
|
)
|
|
@@ -482,11 +499,15 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
|
|
|
482
499
|
Stream.fromEffect(
|
|
483
500
|
Ref.get(keysRef).pipe(
|
|
484
501
|
Effect.flatMap((keys) =>
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
502
|
+
metadata(keys).pipe(
|
|
503
|
+
Effect.flatMap((meta) =>
|
|
504
|
+
Effect.fail({
|
|
505
|
+
_tag: "error" as const,
|
|
506
|
+
error: err,
|
|
507
|
+
metadata: meta
|
|
508
|
+
})
|
|
509
|
+
)
|
|
510
|
+
)
|
|
490
511
|
)
|
|
491
512
|
)
|
|
492
513
|
)
|
|
@@ -494,7 +515,11 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
|
|
|
494
515
|
),
|
|
495
516
|
Stream.fromEffect(
|
|
496
517
|
Ref.get(keysRef).pipe(
|
|
497
|
-
Effect.
|
|
518
|
+
Effect.flatMap((keys) =>
|
|
519
|
+
metadata(keys).pipe(
|
|
520
|
+
Effect.map((meta) => ({ _tag: "done" as const, metadata: meta }))
|
|
521
|
+
)
|
|
522
|
+
)
|
|
498
523
|
)
|
|
499
524
|
)
|
|
500
525
|
)
|
|
@@ -511,6 +536,13 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
|
|
|
511
536
|
})
|
|
512
537
|
.pipe(Effect.andThen(result as Effect.Effect<unknown, unknown, unknown>))
|
|
513
538
|
|
|
539
|
+
const readsRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
|
|
540
|
+
const writesRef = Ref.makeUnsafe<DataDependencies.DataDependencies>([])
|
|
541
|
+
const dependencyRecorder = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
|
|
542
|
+
effect = effect.pipe(
|
|
543
|
+
Effect.provideService(DataDependencies.DataDependencyRecorder, dependencyRecorder)
|
|
544
|
+
)
|
|
545
|
+
|
|
514
546
|
// Commands: provide a request-scoped `InvalidationSet` and wrap both
|
|
515
547
|
// success (`CommandResponseWithMetaData`) and handler-thrown failure
|
|
516
548
|
// (`CommandFailureWithMetaData`) so the client receives accumulated
|
|
@@ -525,21 +557,42 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
|
|
|
525
557
|
Effect.provideService(Invalidation.InvalidationSet, invalidationSet),
|
|
526
558
|
Effect.flatMap((value) =>
|
|
527
559
|
Ref.get(keysRef).pipe(
|
|
528
|
-
Effect.
|
|
560
|
+
Effect.flatMap((keys) =>
|
|
561
|
+
dependencyRecorder.get.pipe(
|
|
562
|
+
Effect.map((dataDependencies) =>
|
|
563
|
+
({
|
|
564
|
+
payload: value,
|
|
565
|
+
metadata: Invalidation.makeMetaData(keys, dataDependencies)
|
|
566
|
+
}) as any
|
|
567
|
+
)
|
|
568
|
+
)
|
|
569
|
+
)
|
|
529
570
|
)
|
|
530
571
|
),
|
|
531
572
|
Effect.catch((err: any) =>
|
|
532
573
|
Ref.get(keysRef).pipe(
|
|
533
574
|
Effect.flatMap((keys) =>
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
575
|
+
dependencyRecorder.get.pipe(
|
|
576
|
+
Effect.flatMap((dataDependencies) =>
|
|
577
|
+
Effect.fail({
|
|
578
|
+
_tag: "CommandFailureWithMetaData" as const,
|
|
579
|
+
error: err,
|
|
580
|
+
metadata: Invalidation.makeMetaData(keys, dataDependencies)
|
|
581
|
+
})
|
|
582
|
+
)
|
|
583
|
+
)
|
|
539
584
|
)
|
|
540
585
|
)
|
|
541
586
|
)
|
|
542
587
|
)
|
|
588
|
+
} else {
|
|
589
|
+
effect = effect.pipe(
|
|
590
|
+
Effect.flatMap((value) =>
|
|
591
|
+
dependencyRecorder.get.pipe(
|
|
592
|
+
Effect.map((dataDependencies) => ({ payload: value, metadata: { dataDependencies } }) as any)
|
|
593
|
+
)
|
|
594
|
+
)
|
|
595
|
+
)
|
|
543
596
|
}
|
|
544
597
|
|
|
545
598
|
return applyRequestTypeInterruptibility(resource.type, effect)
|
|
@@ -584,7 +637,7 @@ export const makeRouter = <Live extends Layer.Layer<any, any, any> = Layer.Layer
|
|
|
584
637
|
})
|
|
585
638
|
: Rpc.make(resource._tag, {
|
|
586
639
|
payload: resource,
|
|
587
|
-
success: resource.success,
|
|
640
|
+
success: isStream ? resource.success : Invalidation.QueryResponseWithMetaData(resource.success),
|
|
588
641
|
error: resource.error,
|
|
589
642
|
stream: isStream
|
|
590
643
|
}))
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { describe, expect, it } from "@effect/vitest"
|
|
2
|
+
import * as DataDependencies from "effect-app/DataDependencies"
|
|
2
3
|
import * as Effect from "effect-app/Effect"
|
|
3
4
|
import * as Layer from "effect-app/Layer"
|
|
4
5
|
import { makeRepo } from "effect-app/Model/Repository"
|
|
5
6
|
import { RepositoryRegistryLive } from "effect-app/Model/Repository/Registry"
|
|
6
7
|
import * as S from "effect-app/Schema"
|
|
7
8
|
import { setupRequestContextFromCurrent } from "effect-app/setupRequest"
|
|
9
|
+
import * as Ref from "effect/Ref"
|
|
8
10
|
import { MemoryStoreLive } from "../src/Store/Memory.js"
|
|
9
11
|
|
|
10
12
|
class BatchItem extends S.Class<BatchItem>("BatchItem")({
|
|
@@ -59,4 +61,29 @@ describe("repository ext save/remove batching", () => {
|
|
|
59
61
|
setupRequestContextFromCurrent(),
|
|
60
62
|
Effect.provide(TestStoreLive)
|
|
61
63
|
))
|
|
64
|
+
|
|
65
|
+
it.effect("records repository read and write dependencies", () =>
|
|
66
|
+
Effect
|
|
67
|
+
.gen(function*() {
|
|
68
|
+
const readsRef = yield* Ref.make<DataDependencies.DataDependencies>([])
|
|
69
|
+
const writesRef = yield* Ref.make<DataDependencies.DataDependencies>([])
|
|
70
|
+
const recorder = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
|
|
71
|
+
|
|
72
|
+
yield* Effect
|
|
73
|
+
.gen(function*() {
|
|
74
|
+
const repo = yield* makeRepo("DependencyItem", BatchItem, {})
|
|
75
|
+
yield* repo.save(new BatchItem({ id: "1", label: "one" }))
|
|
76
|
+
yield* repo.all
|
|
77
|
+
yield* repo.find("1")
|
|
78
|
+
yield* repo.removeById("1")
|
|
79
|
+
})
|
|
80
|
+
.pipe(Effect.provideService(DataDependencies.DataDependencyRecorder, recorder))
|
|
81
|
+
|
|
82
|
+
expect(yield* Ref.get(readsRef)).toEqual([DataDependencies.repo("DependencyItem")])
|
|
83
|
+
expect(yield* Ref.get(writesRef)).toEqual([DataDependencies.repo("DependencyItem")])
|
|
84
|
+
})
|
|
85
|
+
.pipe(
|
|
86
|
+
setupRequestContextFromCurrent(),
|
|
87
|
+
Effect.provide(TestStoreLive)
|
|
88
|
+
))
|
|
62
89
|
})
|
|
@@ -14,12 +14,15 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { NodeHttpServer } from "@effect/platform-node"
|
|
16
16
|
import { expect, it } from "@effect/vitest"
|
|
17
|
-
import { ApiClientFactory, InvalidationKeysFromServer, makeInvalidationKeysService, makeRpcClient } from "effect-app/client"
|
|
17
|
+
import { ApiClientFactory, DataDependencies, InvalidationKeysFromServer, InvalidStateError, makeInvalidationKeysService, makeRpcClient, OptimisticConcurrencyException } from "effect-app/client"
|
|
18
|
+
import * as Context from "effect-app/Context"
|
|
18
19
|
import { HttpRouter, HttpServer } from "effect-app/http"
|
|
19
20
|
import { DefaultGenericMiddlewares } from "effect-app/middleware"
|
|
21
|
+
import { makeRepo, RepositoryRegistryLive } from "effect-app/Model"
|
|
20
22
|
import { Invalidation, MiddlewareMaker } from "effect-app/rpc"
|
|
21
23
|
import * as S from "effect-app/Schema"
|
|
22
24
|
import { TaggedErrorClass } from "effect-app/Schema"
|
|
25
|
+
import { setupRequestContextFromCurrent } from "effect-app/setupRequest"
|
|
23
26
|
import * as Effect from "effect/Effect"
|
|
24
27
|
import * as Exit from "effect/Exit"
|
|
25
28
|
import * as Layer from "effect/Layer"
|
|
@@ -31,6 +34,7 @@ import { RpcSerialization } from "effect/unstable/rpc"
|
|
|
31
34
|
import { createServer } from "http"
|
|
32
35
|
import { makeRouter } from "../src/routing.js"
|
|
33
36
|
import { DefaultGenericMiddlewaresLive } from "../src/routing/middleware.js"
|
|
37
|
+
import { MemoryStoreLive } from "../src/Store/Memory.js"
|
|
34
38
|
import { AllowAnonymous, AllowAnonymousLive, RequestContextMap, RequireRoles, RequireRolesLive, SomeElseMiddleware, SomeElseMiddlewareLive, SomeService, Test, TestLive } from "./fixtures.js"
|
|
35
39
|
|
|
36
40
|
// ---------------------------------------------------------------------------
|
|
@@ -99,14 +103,82 @@ class StreamWithKey extends Req.Command<StreamWithKey>()("StreamWithKey", {}, {
|
|
|
99
103
|
success: S.Number
|
|
100
104
|
}) {}
|
|
101
105
|
|
|
102
|
-
|
|
106
|
+
class StreamWithRepoWrite extends Req.Command<StreamWithRepoWrite>()("StreamWithRepoWrite", {}, {
|
|
107
|
+
stream: true,
|
|
108
|
+
allowAnonymous: true,
|
|
109
|
+
success: S.Number
|
|
110
|
+
}) {}
|
|
111
|
+
|
|
112
|
+
class RepoItem extends S.Class<RepoItem>("RepoItem")({
|
|
113
|
+
id: S.String,
|
|
114
|
+
label: S.String
|
|
115
|
+
}) {}
|
|
116
|
+
|
|
117
|
+
class GetRepoCount extends Req.Query<GetRepoCount>()("GetRepoCount", {}, {
|
|
118
|
+
allowAnonymous: true,
|
|
119
|
+
success: S.Number
|
|
120
|
+
}) {}
|
|
121
|
+
|
|
122
|
+
class SaveRepoItem extends Req.Command<SaveRepoItem>()("SaveRepoItem", {
|
|
123
|
+
id: S.String,
|
|
124
|
+
label: S.String
|
|
125
|
+
}, {
|
|
126
|
+
allowAnonymous: true,
|
|
127
|
+
error: S.Union([InvalidStateError, OptimisticConcurrencyException]),
|
|
128
|
+
success: S.Void
|
|
129
|
+
}) {}
|
|
130
|
+
|
|
131
|
+
class RepoItems extends Context.Service<RepoItems>()("RepoItems", {
|
|
132
|
+
make: makeRepo("RepoItem", RepoItem, {})
|
|
133
|
+
}) {
|
|
134
|
+
static Default = Layer.effect(this, this.make).pipe(
|
|
135
|
+
Layer.provide(Layer.merge(MemoryStoreLive, RepositoryRegistryLive))
|
|
136
|
+
)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// A second, unrelated repo — its writes must NOT invalidate a query that only read `RepoItem`.
|
|
140
|
+
class OtherItem extends S.Class<OtherItem>("OtherItem")({
|
|
141
|
+
id: S.String,
|
|
142
|
+
label: S.String
|
|
143
|
+
}) {}
|
|
144
|
+
|
|
145
|
+
class SaveOtherItem extends Req.Command<SaveOtherItem>()("SaveOtherItem", {
|
|
146
|
+
id: S.String,
|
|
147
|
+
label: S.String
|
|
148
|
+
}, {
|
|
149
|
+
allowAnonymous: true,
|
|
150
|
+
success: S.Void
|
|
151
|
+
}) {}
|
|
152
|
+
|
|
153
|
+
class OtherItems extends Context.Service<OtherItems>()("OtherItems", {
|
|
154
|
+
make: makeRepo("OtherItem", OtherItem, {})
|
|
155
|
+
}) {
|
|
156
|
+
static Default = Layer.effect(this, this.make).pipe(
|
|
157
|
+
Layer.provide(Layer.merge(MemoryStoreLive, RepositoryRegistryLive))
|
|
158
|
+
)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const InvRsc = {
|
|
162
|
+
DoNothing,
|
|
163
|
+
DoWithDynamicKey,
|
|
164
|
+
DoWithBothKeys,
|
|
165
|
+
DoAndFail,
|
|
166
|
+
StreamWithKey,
|
|
167
|
+
StreamWithRepoWrite,
|
|
168
|
+
GetRepoCount,
|
|
169
|
+
SaveRepoItem,
|
|
170
|
+
SaveOtherItem
|
|
171
|
+
}
|
|
103
172
|
|
|
104
173
|
// ---------------------------------------------------------------------------
|
|
105
174
|
// Controllers / router
|
|
106
175
|
// ---------------------------------------------------------------------------
|
|
107
176
|
|
|
108
177
|
const router = Router(InvRsc)({
|
|
178
|
+
dependencies: [RepoItems.Default, OtherItems.Default],
|
|
109
179
|
*effect(match) {
|
|
180
|
+
const repo = yield* RepoItems
|
|
181
|
+
const otherRepo = yield* OtherItems
|
|
110
182
|
return match({
|
|
111
183
|
DoNothing: () => Effect.void,
|
|
112
184
|
DoWithDynamicKey: Effect.fnUntraced(function*() {
|
|
@@ -125,7 +197,18 @@ const router = Router(InvRsc)({
|
|
|
125
197
|
StreamWithKey: () =>
|
|
126
198
|
Stream.fromIterable([1, 2, 3]).pipe(
|
|
127
199
|
Stream.tap(() => Invalidation.InvalidationSet.use((_) => _.add(StreamKey)))
|
|
128
|
-
)
|
|
200
|
+
),
|
|
201
|
+
StreamWithRepoWrite: () =>
|
|
202
|
+
Stream.fromIterable([1, 2, 3]).pipe(
|
|
203
|
+
Stream.tap((n) =>
|
|
204
|
+
repo.save(new RepoItem({ id: String(n), label: "x" })).pipe(Effect.orDie, setupRequestContextFromCurrent())
|
|
205
|
+
)
|
|
206
|
+
),
|
|
207
|
+
GetRepoCount: () => repo.all.pipe(Effect.map((_) => _.length), Effect.orDie, setupRequestContextFromCurrent()),
|
|
208
|
+
SaveRepoItem: ({ id, label }) =>
|
|
209
|
+
repo.save(new RepoItem({ id, label })).pipe(Effect.orDie, setupRequestContextFromCurrent()),
|
|
210
|
+
SaveOtherItem: ({ id, label }) =>
|
|
211
|
+
otherRepo.save(new OtherItem({ id, label })).pipe(Effect.orDie, setupRequestContextFromCurrent())
|
|
129
212
|
})
|
|
130
213
|
}
|
|
131
214
|
})
|
|
@@ -171,6 +254,15 @@ const withCapture = <A, E, R>(eff: Effect.Effect<A, E, R>) =>
|
|
|
171
254
|
return { result, keys: yield* Ref.get(ref) }
|
|
172
255
|
})
|
|
173
256
|
|
|
257
|
+
const withDependencyCapture = <A, E, R>(eff: Effect.Effect<A, E, R>) =>
|
|
258
|
+
Effect.gen(function*() {
|
|
259
|
+
const readsRef = yield* Ref.make<DataDependencies.DataDependencies>([])
|
|
260
|
+
const writesRef = yield* Ref.make<DataDependencies.DataDependencies>([])
|
|
261
|
+
const svc = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
|
|
262
|
+
const result = yield* eff.pipe(Effect.provideService(DataDependencies.DataDependencyRecorder, svc), Effect.exit)
|
|
263
|
+
return { result, dependencies: yield* svc.get }
|
|
264
|
+
})
|
|
265
|
+
|
|
174
266
|
// ---------------------------------------------------------------------------
|
|
175
267
|
// Tests
|
|
176
268
|
// ---------------------------------------------------------------------------
|
|
@@ -254,3 +346,75 @@ it.live(
|
|
|
254
346
|
}, Effect.provide(TestLayer)),
|
|
255
347
|
{ timeout: 10_000 }
|
|
256
348
|
)
|
|
349
|
+
|
|
350
|
+
it.live(
|
|
351
|
+
"stream: per-chunk repo writes are drained and forwarded to the client recorder",
|
|
352
|
+
Effect.fnUntraced(function*() {
|
|
353
|
+
const client = yield* ApiClientFactory.makeFor(Layer.empty)(InvRsc)
|
|
354
|
+
const readsRef = yield* Ref.make<DataDependencies.DataDependencies>([])
|
|
355
|
+
const writesRef = yield* Ref.make<DataDependencies.DataDependencies>([])
|
|
356
|
+
const svc = DataDependencies.makeDataDependencyRecorder(readsRef, writesRef)
|
|
357
|
+
const values = yield* Stream.runCollect(client.StreamWithRepoWrite.handler()).pipe(
|
|
358
|
+
Effect.provideService(DataDependencies.DataDependencyRecorder, svc)
|
|
359
|
+
)
|
|
360
|
+
const writes = yield* Ref.get(writesRef)
|
|
361
|
+
expect(values).toStrictEqual([1, 2, 3])
|
|
362
|
+
// Each emitted value writes to RepoItem; routing drains the writes per chunk and the client
|
|
363
|
+
// accumulates them — the recorder dedupes, so a single RepoItem entry is recorded.
|
|
364
|
+
expect(writes).toStrictEqual([DataDependencies.repo("RepoItem")])
|
|
365
|
+
}, Effect.provide(TestLayer)),
|
|
366
|
+
{ timeout: 10_000 }
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
it.live(
|
|
370
|
+
"repository dependencies flow through query and command metadata",
|
|
371
|
+
Effect.fnUntraced(function*() {
|
|
372
|
+
const client = yield* ApiClientFactory.makeFor(Layer.empty)(InvRsc)
|
|
373
|
+
|
|
374
|
+
const query = yield* withDependencyCapture(client.GetRepoCount.handler())
|
|
375
|
+
expect(Exit.isSuccess(query.result) && query.result.value).toBe(0)
|
|
376
|
+
expect(query.dependencies.reads).toStrictEqual([DataDependencies.repo("RepoItem")])
|
|
377
|
+
expect(query.dependencies.writes).toStrictEqual([])
|
|
378
|
+
|
|
379
|
+
const command = yield* withDependencyCapture(client.SaveRepoItem.handler({ id: "1", label: "one" }))
|
|
380
|
+
expect(Exit.isSuccess(command.result)).toBe(true)
|
|
381
|
+
expect(command.dependencies.reads).toStrictEqual([])
|
|
382
|
+
expect(command.dependencies.writes).toStrictEqual([DataDependencies.repo("RepoItem")])
|
|
383
|
+
}, Effect.provide(TestLayer)),
|
|
384
|
+
{ timeout: 10_000 }
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
it.live(
|
|
388
|
+
"query invalidation: a command's writes invalidate exactly the queries whose reads intersect",
|
|
389
|
+
Effect.fnUntraced(function*() {
|
|
390
|
+
const client = yield* ApiClientFactory.makeFor(Layer.empty)(InvRsc)
|
|
391
|
+
|
|
392
|
+
// Client-side query registry: queryKey -> read dependencies forwarded by the server. Mirrors
|
|
393
|
+
// @effect-app/vue's `dependencyMetadata`; the derivation predicate (`intersects`) is the exact
|
|
394
|
+
// one the vue mutate engine uses to pick invalidation targets from a command's writes.
|
|
395
|
+
const queryReads = new Map<string, DataDependencies.DataDependencies>()
|
|
396
|
+
const invalidatedBy = (writes: DataDependencies.DataDependencies) =>
|
|
397
|
+
[...queryReads]
|
|
398
|
+
.filter(([, reads]) => DataDependencies.intersects(reads, writes))
|
|
399
|
+
.map(([key]) => key)
|
|
400
|
+
|
|
401
|
+
// Run the query through the real client; register its forwarded reads under its key.
|
|
402
|
+
const query = yield* withDependencyCapture(client.GetRepoCount.handler())
|
|
403
|
+
expect(Exit.isSuccess(query.result)).toBe(true)
|
|
404
|
+
queryReads.set("GetRepoCount", query.dependencies.reads)
|
|
405
|
+
// Before any command, nothing is invalidated.
|
|
406
|
+
expect(invalidatedBy([])).toStrictEqual([])
|
|
407
|
+
|
|
408
|
+
// A command writing the SAME repo the query read => the query is selected for invalidation.
|
|
409
|
+
const save = yield* withDependencyCapture(client.SaveRepoItem.handler({ id: "1", label: "one" }))
|
|
410
|
+
expect(Exit.isSuccess(save.result)).toBe(true)
|
|
411
|
+
expect(invalidatedBy(save.dependencies.writes)).toStrictEqual(["GetRepoCount"])
|
|
412
|
+
|
|
413
|
+
// A command writing an UNRELATED repo => the query is NOT invalidated (negative control).
|
|
414
|
+
const saveOther = yield* withDependencyCapture(client.SaveOtherItem.handler({ id: "2", label: "two" }))
|
|
415
|
+
expect(Exit.isSuccess(saveOther.result)).toBe(true)
|
|
416
|
+
expect(saveOther.dependencies.writes).toStrictEqual([DataDependencies.repo("OtherItem")])
|
|
417
|
+
expect(invalidatedBy(saveOther.dependencies.writes)).toStrictEqual([])
|
|
418
|
+
}, Effect.provide(TestLayer)),
|
|
419
|
+
{ timeout: 10_000 }
|
|
420
|
+
)
|
|
@@ -1,186 +0,0 @@
|
|
|
1
|
-
import { reportNonInterruptedFailure } from "@effect-app/infra/QueueMaker/errors"
|
|
2
|
-
import { subMinutes } from "date-fns"
|
|
3
|
-
import type { NonEmptyReadonlyArray } from "effect-app/Array"
|
|
4
|
-
import * as Effect from "effect-app/Effect"
|
|
5
|
-
import * as Option from "effect-app/Option"
|
|
6
|
-
import { type QueueBase, QueueMeta } from "effect-app/QueueMaker"
|
|
7
|
-
import * as S from "effect-app/Schema"
|
|
8
|
-
import { type NonEmptyString255 } from "effect-app/Schema"
|
|
9
|
-
import { getRequestContext, setupRequestContextWithCustomSpan } from "effect-app/setupRequest"
|
|
10
|
-
import { pretty } from "effect-app/utils"
|
|
11
|
-
import * as Fiber from "effect/Fiber"
|
|
12
|
-
import * as Tracer from "effect/Tracer"
|
|
13
|
-
import { SqlClient } from "effect/unstable/sql"
|
|
14
|
-
import { InfraLogger } from "../logger.ts"
|
|
15
|
-
import { messagingSpanArgs } from "../otel.ts"
|
|
16
|
-
import { SQLModel } from "../SQL.ts"
|
|
17
|
-
|
|
18
|
-
export const QueueId = S.Finite.pipe(S.brand("QueueId"))
|
|
19
|
-
export type QueueId = typeof QueueId.Type
|
|
20
|
-
|
|
21
|
-
// TODO: let the model track and Auto Generate versionColumn on every update instead
|
|
22
|
-
export const makeSQLQueue = Effect.fnUntraced(function*<
|
|
23
|
-
Evt extends { id: S.StringId; _tag: string },
|
|
24
|
-
DrainEvt extends { id: S.StringId; _tag: string },
|
|
25
|
-
EvtE,
|
|
26
|
-
DrainEvtE
|
|
27
|
-
>(
|
|
28
|
-
queueName: NonEmptyString255,
|
|
29
|
-
queueDrainName: NonEmptyString255,
|
|
30
|
-
schema: S.Codec<Evt, EvtE>,
|
|
31
|
-
drainSchema: S.Codec<DrainEvt, DrainEvtE>
|
|
32
|
-
) {
|
|
33
|
-
const base = {
|
|
34
|
-
id: SQLModel.Generated(QueueId),
|
|
35
|
-
meta: SQLModel.JsonFromString(QueueMeta),
|
|
36
|
-
name: S.NonEmptyString255,
|
|
37
|
-
createdAt: SQLModel.DateTimeInsert,
|
|
38
|
-
updatedAt: SQLModel.DateTimeUpdate,
|
|
39
|
-
// TODO: at+owner
|
|
40
|
-
processingAt: SQLModel.FieldOption(S.Date),
|
|
41
|
-
finishedAt: SQLModel.FieldOption(S.Date),
|
|
42
|
-
etag: S.String // TODO: use a SQLModel thing that auto updates it?
|
|
43
|
-
// TODO: record locking.. / optimistic locking
|
|
44
|
-
// rowVersion: SQLModel.DateTimeFromNumberWithNow
|
|
45
|
-
}
|
|
46
|
-
class Queue extends SQLModel.Class<Queue>("Queue")({
|
|
47
|
-
body: SQLModel.JsonFromString(schema),
|
|
48
|
-
...base
|
|
49
|
-
}) {}
|
|
50
|
-
class Drain extends SQLModel.Class<Drain>("Drain")({
|
|
51
|
-
body: SQLModel.JsonFromString(drainSchema),
|
|
52
|
-
...base
|
|
53
|
-
}) {}
|
|
54
|
-
const sql = yield* SqlClient.SqlClient
|
|
55
|
-
|
|
56
|
-
const queueRepo = yield* SQLModel.makeRepository(Queue, {
|
|
57
|
-
tableName: "queue",
|
|
58
|
-
spanPrefix: "QueueRepo",
|
|
59
|
-
idColumn: "id",
|
|
60
|
-
versionColumn: "etag"
|
|
61
|
-
})
|
|
62
|
-
|
|
63
|
-
const drainRepo = yield* SQLModel.makeRepository(Drain, {
|
|
64
|
-
tableName: "queue",
|
|
65
|
-
spanPrefix: "DrainRepo",
|
|
66
|
-
idColumn: "id",
|
|
67
|
-
versionColumn: "etag"
|
|
68
|
-
})
|
|
69
|
-
|
|
70
|
-
const decodeDrain = S.decodeEffectConcurrently(Drain)
|
|
71
|
-
|
|
72
|
-
const drain = Effect.gen(function*() {
|
|
73
|
-
const limit = subMinutes(new Date(), 15)
|
|
74
|
-
return yield* sql<typeof Drain.Encoded>`SELECT *
|
|
75
|
-
FROM queue
|
|
76
|
-
WHERE name = ${queueDrainName} AND finishedAt IS NULL AND (processingAt IS NULL OR processingAt < ${limit.getTime()})
|
|
77
|
-
LIMIT 1`
|
|
78
|
-
})
|
|
79
|
-
|
|
80
|
-
const q = {
|
|
81
|
-
offer: Effect.fnUntraced(function*(body: Evt, meta: typeof QueueMeta.Type) {
|
|
82
|
-
yield* queueRepo.insertVoid(Queue.insert.make({
|
|
83
|
-
body,
|
|
84
|
-
meta,
|
|
85
|
-
name: queueName,
|
|
86
|
-
processingAt: Option.none(),
|
|
87
|
-
finishedAt: Option.none(),
|
|
88
|
-
etag: crypto.randomUUID()
|
|
89
|
-
}))
|
|
90
|
-
}),
|
|
91
|
-
take: Effect.gen(function*() {
|
|
92
|
-
while (true) {
|
|
93
|
-
const [first] = yield* drain.pipe(Effect.withTracerEnabled(false)) // disable sql tracer otherwise we spam it..
|
|
94
|
-
if (first) {
|
|
95
|
-
const dec = yield* decodeDrain(first)
|
|
96
|
-
const { createdAt, updatedAt, ...rest } = dec
|
|
97
|
-
return yield* drainRepo.update(
|
|
98
|
-
Drain.update.make({ ...rest, processingAt: Option.some(new Date()) }) // auto in lib , etag: crypto.randomUUID()
|
|
99
|
-
)
|
|
100
|
-
}
|
|
101
|
-
if (first) return first
|
|
102
|
-
yield* Effect.sleep(250)
|
|
103
|
-
}
|
|
104
|
-
}),
|
|
105
|
-
finish: Effect.fn(function*({ createdAt, updatedAt, ...q }: Drain) {
|
|
106
|
-
return yield* drainRepo.updateVoid(Drain.update.make({ ...q, finishedAt: Option.some(new Date()) })) // auto in lib , etag: crypto.randomUUID()
|
|
107
|
-
})
|
|
108
|
-
}
|
|
109
|
-
const queue = {
|
|
110
|
-
publish: Effect.fn(`publish ${queueName}`, {
|
|
111
|
-
kind: "producer",
|
|
112
|
-
attributes: {
|
|
113
|
-
"messaging.system": "sql",
|
|
114
|
-
"messaging.operation.name": "publish",
|
|
115
|
-
"messaging.destination.name": queueName
|
|
116
|
-
}
|
|
117
|
-
})(function*(
|
|
118
|
-
...messages: NonEmptyReadonlyArray<Evt>
|
|
119
|
-
) {
|
|
120
|
-
yield* Effect.annotateCurrentSpan({
|
|
121
|
-
"messaging.batch.message_count": messages.length,
|
|
122
|
-
"messaging.message.types": messages.map((_) => _._tag)
|
|
123
|
-
})
|
|
124
|
-
const requestContext = yield* getRequestContext
|
|
125
|
-
yield* Effect.forEach(messages, (m) => q.offer(m, requestContext), { discard: true })
|
|
126
|
-
}),
|
|
127
|
-
drain: <DrainE, DrainR>(
|
|
128
|
-
handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
|
|
129
|
-
sessionId?: string
|
|
130
|
-
) => {
|
|
131
|
-
const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
|
|
132
|
-
const processMessage = Effect.fnUntraced(function*({ body, meta }: Drain) {
|
|
133
|
-
let effect = InfraLogger
|
|
134
|
-
.logDebug(`[${queueDrainName}] Processing incoming message`)
|
|
135
|
-
.pipe(
|
|
136
|
-
Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
|
|
137
|
-
Effect.andThen(handleEvent(body)),
|
|
138
|
-
silenceAndReportError,
|
|
139
|
-
(_) => {
|
|
140
|
-
const args = messagingSpanArgs({
|
|
141
|
-
operation: "process",
|
|
142
|
-
system: "sql",
|
|
143
|
-
destination: queueDrainName,
|
|
144
|
-
messageId: body.id,
|
|
145
|
-
conversationId: sessionId,
|
|
146
|
-
extra: { "messaging.message.type": body._tag, "messaging.message.body": body }
|
|
147
|
-
}, "consumer")
|
|
148
|
-
return setupRequestContextWithCustomSpan(
|
|
149
|
-
_,
|
|
150
|
-
meta,
|
|
151
|
-
args.name,
|
|
152
|
-
{
|
|
153
|
-
captureStackTrace: false,
|
|
154
|
-
kind: args.kind,
|
|
155
|
-
attributes: args.attributes
|
|
156
|
-
}
|
|
157
|
-
)
|
|
158
|
-
}
|
|
159
|
-
)
|
|
160
|
-
if (meta.span) {
|
|
161
|
-
effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
|
|
162
|
-
}
|
|
163
|
-
return yield* effect
|
|
164
|
-
})
|
|
165
|
-
|
|
166
|
-
return Effect.fn(`receive ${queueDrainName}`, {
|
|
167
|
-
kind: "consumer",
|
|
168
|
-
attributes: {
|
|
169
|
-
"messaging.system": "sql",
|
|
170
|
-
"messaging.operation.name": "receive",
|
|
171
|
-
"messaging.destination.name": queueDrainName,
|
|
172
|
-
...(sessionId !== undefined && { "messaging.message.conversation_id": sessionId })
|
|
173
|
-
}
|
|
174
|
-
})(function*() {
|
|
175
|
-
const x = yield* q.take
|
|
176
|
-
yield* processMessage(x).pipe(
|
|
177
|
-
Effect.uninterruptible,
|
|
178
|
-
Effect.forkChild,
|
|
179
|
-
Effect.flatMap(Fiber.join),
|
|
180
|
-
Effect.tap(q.finish(x))
|
|
181
|
-
)
|
|
182
|
-
}, (effect) => effect.pipe(silenceAndReportError, Effect.forever))()
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
return queue as QueueBase<Evt, DrainEvt>
|
|
186
|
-
})
|