@starci/hfs 4.10.1 → 4.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +40 -1
- package/db/drift.mjs +19 -16
- package/db/image.mjs +13 -0
- package/db/migrate.mjs +3 -3
- package/db/postgres.mjs +1 -1
- package/package.json +2 -1
- package/runtime/knowledge/hfs/canon-pins.yaml +23 -8
- package/runtime/knowledge/hfs/rules.yaml +54 -13
- package/runtime/knowledge/hfs/slots.yaml +67 -36
- package/runtime/knowledge/patterns/be/event-bus.yaml +12 -10
- package/runtime/knowledge/patterns/be/idempotency.yaml +1 -1
- package/runtime/knowledge/patterns/be/jobs.yaml +3 -0
- package/runtime/knowledge/patterns/be/queues.yaml +14 -14
- package/runtime/knowledge/patterns/be/reactors.yaml +1 -1
- package/runtime/knowledge/patterns/be/realtime.yaml +1 -1
- package/runtime/modules/kernel/failure-codes.yaml +28 -18
- package/runtime/modules/schemas/application-stacks.schema.yaml +4 -2
- package/runtime/scripts/gates/starcistacks.mjs +6 -2
- package/runtime/scripts/guards/push-scan.mjs +1 -1
- package/runtime/scripts/hfs/ci-declaration.mjs +3 -2
- package/runtime/scripts/hfs/db-baseline-verb.mjs +94 -22
- package/runtime/scripts/hfs/db-errors.mjs +2 -0
- package/runtime/scripts/hfs/db-plan-verb.mjs +17 -5
- package/runtime/scripts/hfs/db-squash.mjs +514 -0
- package/runtime/scripts/hfs/declaration-shape.mjs +3 -15
- package/runtime/scripts/hfs/example-secrets.mjs +4 -3
- package/runtime/scripts/hfs/rule-params-shape.mjs +11 -9
- package/runtime/scripts/hfs/rules/a15-findings.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-config.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-constants.mjs +16 -1
- package/runtime/scripts/hfs/rules/database-queue-sources.mjs +2 -110
- package/runtime/scripts/hfs/rules/database-queues.mjs +53 -239
- package/runtime/scripts/hfs/rules/database-sql.mjs +2 -0
- package/runtime/scripts/hfs/rules/database-transaction.mjs +45 -0
- package/runtime/scripts/hfs/rules/deploy-scan.mjs +1 -1
- package/runtime/scripts/hfs/rules/edition.mjs +1 -1
- package/runtime/scripts/hfs/rules/env-parity.mjs +1 -1
- package/runtime/scripts/hfs/rules/lite-secrets.mjs +3 -3
- package/runtime/scripts/hfs/rules/script-hygiene.mjs +1 -1
- package/runtime/scripts/hfs/rules/secret-custody-only.mjs +2 -2
- package/runtime/scripts/hfs/rules/secrets.mjs +1 -1
- package/runtime/scripts/hfs/rules/stack-tooling.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks-deploy.mjs +2 -1
- package/runtime/scripts/hfs/rules/stacks-lite-backing.mjs +3 -4
- package/runtime/scripts/hfs/rules/stacks-runtime.mjs +2 -2
- package/runtime/scripts/hfs/rules/stacks-swarm.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks.mjs +3 -3
- package/runtime/scripts/hfs/sealed-members.mjs +10 -6
- package/runtime/scripts/hfs/secret-files.mjs +16 -0
- package/runtime/scripts/hfs/secret-init.mjs +120 -21
- package/runtime/scripts/hfs/secret-output.mjs +2 -1
- package/runtime/scripts/hfs/secret-prompt.mjs +128 -0
- package/runtime/scripts/hfs/secret-sops.mjs +93 -9
- package/runtime/scripts/hfs/secret-standard.mjs +315 -0
- package/runtime/scripts/hfs/secret-store.mjs +23 -9
- package/runtime/scripts/hfs/secret.mjs +110 -46
- package/runtime/scripts/hfs/slots.mjs +8 -1
- package/runtime/scripts/hfs/stack-deploy-config.mjs +20 -6
- package/runtime/scripts/hfs/stacks-gate.mjs +9 -2
- package/runtime/scripts/hfs/starcistacks-runtime.mjs +1 -1
- package/runtime/scripts/lib/sealed-member.mjs +36 -0
- package/runtime/scripts/lib/secret-patterns.mjs +1 -1
- package/runtime/scripts/lib/swarm-stack.mjs +4 -6
- package/scaffold/add-inbox-lite.mjs +2 -2
- package/scaffold/add-queue-lite.mjs +24 -5
- package/scaffold/app.mjs +2 -2
- package/src/main.mjs +5 -4
- package/stack/build.mjs +3 -1
- package/stack/config.mjs +2 -1
- package/stack/docker-transport.mjs +2 -1
- package/stack/remote.mjs +4 -3
- package/stack/sealed.mjs +9 -8
- package/stack/target-binding.mjs +2 -1
- package/sync/hygiene.mjs +1 -1
- package/sync/index.mjs +47 -12
- package/templates/app/db-bootstrap/db/bootstrap.sql +9 -14
- package/templates/app/db-image/starcistacks/images/db/Dockerfile +18 -0
- package/templates/app/gitignore +2 -1
- package/templates/app/gitignore-lite +3 -1
- package/templates/app/package-scripts/db.json +2 -1
- package/templates/app/skeleton-lite/.starcistacks/application-stacks.yaml +4 -2
- package/templates/app/skeleton-lite/.starcistacks/dev/README.md +1 -1
- package/templates/app/skeleton-lite/.starcistacks/dev/infra/swarm/stack.yaml +1 -1
- package/templates/app/skeleton-lite/.starcistacks/production/README.md +1 -1
- package/templates/app/skeleton-lite/.starcistacks/production/infra/swarm/stack.yaml +1 -1
- package/templates/app/skeleton-lite/README.md +2 -2
- package/templates/be/inbox/delivery-recorded.event.ts.tpl +7 -5
- package/templates/be/inbox/events.json.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.contracts.ts.tpl +58 -11
- package/templates/be/inbox/platform/event-bus/event-bus.decorators.ts.tpl +13 -1
- package/templates/be/inbox/platform/event-bus/event-bus.log-events.ts.tpl +9 -0
- package/templates/be/inbox/platform/event-bus/event-bus.module-definition.ts.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.module.ts.tpl +11 -5
- package/templates/be/inbox/platform/event-bus/event-bus.options.ts.tpl +9 -2
- package/templates/be/inbox/platform/event-bus/event-bus.port.ts.tpl +18 -5
- package/templates/be/inbox/platform/event-bus/event-bus.service.spec.ts.tpl +37 -16
- package/templates/be/inbox/platform/event-bus/event-bus.service.ts.tpl +33 -5
- package/templates/be/inbox/platform/event-bus/event-consumer.service.spec.ts.tpl +143 -0
- package/templates/be/inbox/platform/event-bus/event-consumer.service.ts.tpl +119 -0
- package/templates/be/inbox/platform/event-bus/index.ts.tpl +13 -4
- package/templates/be/inbox/platform/inbox/inbox.service.spec.ts.tpl +4 -4
- package/templates/be/inbox/platform/inbox/inbox.service.ts.tpl +4 -4
- package/templates/be/inbox/platform/migration.sql.tpl +97 -34
- package/templates/be/inbox/service.spec.ts.tpl +1 -1
- package/templates/be/inbox/webhook-door.ts.tpl +2 -2
- package/templates/be/patterns/jobs/platform/jobs.log-events.ts.tpl +2 -0
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.spec.ts.tpl +4 -4
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.ts.tpl +5 -5
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.spec.ts.tpl +65 -0
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.ts.tpl +78 -0
- package/templates/be/patterns/jobs/platform-lite/job-ticks.contracts.ts.tpl +16 -0
- package/templates/be/patterns/jobs/platform-lite/jobs.module.ts.tpl +40 -0
- package/templates/be/patterns/jobs/platform-lite/jobs.sql.tpl +44 -0
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.spec.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.spec.ts.tpl +2 -2
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.ts.tpl +2 -2
- package/templates/be/skeleton/apps/__app__/src/__app__.env.ts +4 -0
- package/templates/be/skeleton/apps/__app__/src/__app__.options.ts +3 -0
- package/templates/be/skeleton/apps/__app__/src/compose/platform.ts +1 -1
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.spec.ts +37 -2
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.ts +36 -3
- package/templates/be/skeleton/src/modules/platform/http/http.config.ts +11 -0
- package/templates/be/skeleton/src/modules/platform/http/http.decorators.ts +7 -0
- package/templates/be/skeleton/src/modules/platform/http/http.module-definition.ts +5 -3
- package/templates/be/skeleton/src/modules/platform/http/http.options.ts +9 -2
- package/templates/be/skeleton/src/modules/platform/http/index.ts +2 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.spec.ts +96 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.ts +93 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/index.ts +1 -1
- package/runtime/scripts/hfs/rules/database-queue-outbox.mjs +0 -102
- package/templates/be/skeleton-lite/src/modules/platform/database/database.functions.ts +0 -24
|
@@ -1,23 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The suffix of a consumer's dead-letter queue: a delivery past its consumer's maxDeliveries moves from `<topic>__<consumer>` to
|
|
3
|
+
* `<topic>__<consumer>__dead`, which an operator reads and replays (BE-EVENT-BUS-4).
|
|
4
|
+
*/
|
|
5
|
+
export const DEAD_QUEUE_SUFFIX = "__dead"
|
|
6
|
+
|
|
7
|
+
/** The deliveries a consumer gets before its message moves to the dead-letter queue, when its subscription names none. */
|
|
8
|
+
export const DEFAULT_MAX_DELIVERIES = 5
|
|
9
|
+
|
|
10
|
+
/** What every event payload carries: the business key a consumer's inbox claims, stable across redelivery and replay. */
|
|
11
|
+
export interface EventPayload {
|
|
12
|
+
/** The business identity of the fact (an order id, a receipt id): the consumer claims (consumer, dedupeKey) once. */
|
|
13
|
+
readonly dedupeKey: string
|
|
14
|
+
}
|
|
15
|
+
|
|
1
16
|
/** The base of each typed event in modules/events: its stable identity, name and payload. */
|
|
2
17
|
export abstract class BaseEvent {
|
|
3
|
-
/**
|
|
18
|
+
/** The stable identity of this event, carried for logs and tracing. */
|
|
4
19
|
abstract readonly eventId: string
|
|
5
|
-
/** The event name listed by the service's vendored event contract. */
|
|
20
|
+
/** The event name listed by the service's vendored event contract; it is the pgmq topic the event is sent to. */
|
|
6
21
|
abstract readonly eventName: string
|
|
7
|
-
/** The fact that a reactor
|
|
8
|
-
abstract readonly payload:
|
|
22
|
+
/** The fact that a reactor processes after commit, with its business dedupe key. */
|
|
23
|
+
abstract readonly payload: EventPayload
|
|
9
24
|
}
|
|
10
25
|
|
|
11
26
|
/** Parsing either returns the event or refuses a mismatched envelope. */
|
|
12
27
|
export type ParsedEvent<Event extends BaseEvent> = Event | null
|
|
13
28
|
|
|
14
|
-
/** The class side of a typed event, shared by publishers and
|
|
29
|
+
/** The class side of a typed event, shared by publishers and consumers. */
|
|
15
30
|
export interface EventClass<Event extends BaseEvent> {
|
|
16
31
|
/** The contract's event name. */
|
|
17
32
|
readonly eventName: string
|
|
18
33
|
/** The contract's payload version. */
|
|
19
34
|
readonly version: number
|
|
20
|
-
/** Reads the event from its
|
|
35
|
+
/** Reads the event from its envelope. */
|
|
21
36
|
parse(envelope: unknown): ParsedEvent<Event>
|
|
22
37
|
}
|
|
23
38
|
|
|
@@ -25,21 +40,25 @@ export interface EventClass<Event extends BaseEvent> {
|
|
|
25
40
|
export interface EventDelivery<Event extends BaseEvent> {
|
|
26
41
|
/** The request id carried explicitly from intake to delivery. */
|
|
27
42
|
readonly correlationId: string
|
|
28
|
-
/** The
|
|
43
|
+
/** The stable identity of the event. */
|
|
29
44
|
readonly eventId: string
|
|
45
|
+
/** The consumer's queue, `<topic>__<consumer>`: the first half of the inbox claim. */
|
|
46
|
+
readonly consumer: string
|
|
47
|
+
/** The payload's business key: the second half of the inbox claim. */
|
|
48
|
+
readonly dedupeKey: string
|
|
30
49
|
/** The typed event. */
|
|
31
50
|
readonly event: Event
|
|
32
|
-
/** The number of
|
|
51
|
+
/** The number of deliveries, including this one. */
|
|
33
52
|
readonly attempt: number
|
|
34
53
|
}
|
|
35
54
|
|
|
36
55
|
/** Explicit request context carried independently of the business payload. */
|
|
37
56
|
export interface EventMetadata {
|
|
38
|
-
/** The originating request or promoted
|
|
57
|
+
/** The originating request or promoted event identity. */
|
|
39
58
|
readonly correlationId: string
|
|
40
59
|
}
|
|
41
60
|
|
|
42
|
-
/** The typed event's
|
|
61
|
+
/** The typed event's envelope, sent once to its topic in the publisher's transaction and routed to every bound consumer queue. */
|
|
43
62
|
export interface EventEnvelope {
|
|
44
63
|
/** The originating request context, independent of the business payload. */
|
|
45
64
|
readonly metadata: EventMetadata
|
|
@@ -48,5 +67,33 @@ export interface EventEnvelope {
|
|
|
48
67
|
/** The contract's event name. */
|
|
49
68
|
readonly eventName: string
|
|
50
69
|
/** The data of the fact. */
|
|
51
|
-
readonly payload:
|
|
70
|
+
readonly payload: EventPayload
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** One message of a consumer's queue, as event_bus_read returns it. */
|
|
74
|
+
export interface EventMessage {
|
|
75
|
+
/** The pgmq message id within its queue. */
|
|
76
|
+
readonly msgId: string
|
|
77
|
+
/** How many times the message has been read, including this read. */
|
|
78
|
+
readonly readCt: number
|
|
79
|
+
/** The envelope as the publisher sent it; untrusted until parsed. */
|
|
80
|
+
readonly message: unknown
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** What a reactor's subscription may tune; every field is a literal of the subscription. */
|
|
84
|
+
export interface EventSubscriptionOptions {
|
|
85
|
+
/** The deliveries before the message moves to the dead-letter queue. Default DEFAULT_MAX_DELIVERIES. */
|
|
86
|
+
readonly maxDeliveries?: number
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** A delivery a consumer gave up on, as its dead-letter queue holds it. */
|
|
90
|
+
export interface DeadLetter {
|
|
91
|
+
/** The message id in the dead-letter queue: what requeue takes. */
|
|
92
|
+
readonly id: string
|
|
93
|
+
/** Why it was moved: its deliveries ran out, or its envelope was malformed. */
|
|
94
|
+
readonly reason: string
|
|
95
|
+
/** The deliveries it had. */
|
|
96
|
+
readonly readCount: number
|
|
97
|
+
/** The envelope as it was read. */
|
|
98
|
+
readonly message: unknown
|
|
52
99
|
}
|
|
@@ -1,9 +1,21 @@
|
|
|
1
1
|
import { injector } from "@modules/platform/composition"
|
|
2
2
|
import type { TypedParameterDecorator } from "@modules/platform/composition"
|
|
3
|
-
import
|
|
3
|
+
import { MODULE_OPTIONS_TOKEN } from "./event-bus.module-definition"
|
|
4
|
+
import type { EventBusOptions } from "./event-bus.options"
|
|
5
|
+
import type { EventBus, EventConsumerRegistry } from "./event-bus.port"
|
|
4
6
|
|
|
5
7
|
/** Token of the transactional EventBus publisher. */
|
|
6
8
|
export const EVENT_BUS: unique symbol = Symbol("platform.event-bus")
|
|
7
9
|
|
|
10
|
+
/** Token of the registry reactors subscribe their consumers to. */
|
|
11
|
+
export const EVENT_CONSUMER_REGISTRY: unique symbol = Symbol("platform.event-bus.consumers")
|
|
12
|
+
|
|
8
13
|
/** Injects the EventBus port. Parameter type: EventBus. */
|
|
9
14
|
export const InjectEventBus = (): TypedParameterDecorator<EventBus> => injector<EventBus>(EVENT_BUS)
|
|
15
|
+
|
|
16
|
+
/** Injects the consumer registry. Parameter type: EventConsumerRegistry. */
|
|
17
|
+
export const InjectEventConsumerRegistry = (): TypedParameterDecorator<EventConsumerRegistry> =>
|
|
18
|
+
injector<EventConsumerRegistry>(EVENT_CONSUMER_REGISTRY)
|
|
19
|
+
|
|
20
|
+
/** Injects the module's options. Parameter type: EventBusOptions. */
|
|
21
|
+
export const InjectEventBusOptions = (): TypedParameterDecorator<EventBusOptions> => injector<EventBusOptions>(MODULE_OPTIONS_TOKEN)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Log events of the event bus. */
|
|
2
|
+
export enum EventBusLogEvent {
|
|
3
|
+
/** A consumer threw; the message reappears after its visibility timeout. */
|
|
4
|
+
ConsumerFailed = "event-bus.consumer.failed",
|
|
5
|
+
/** A delivery ran out of deliveries or was malformed and moved to its consumer's dead-letter queue. */
|
|
6
|
+
DeliveryDead = "event-bus.delivery.dead",
|
|
7
|
+
/** A read of a consumer's queue failed; the loop reads again after its idle pause. */
|
|
8
|
+
ReadFailed = "event-bus.read.failed",
|
|
9
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ConfigurableModuleBuilder } from "@nestjs/common"
|
|
2
2
|
import type { EventBusOptions } from "./event-bus.options"
|
|
3
3
|
|
|
4
|
-
export const { ConfigurableModuleClass, OPTIONS_TYPE } = new ConfigurableModuleBuilder<EventBusOptions>()
|
|
4
|
+
export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE } = new ConfigurableModuleBuilder<EventBusOptions>()
|
|
5
5
|
.setExtras({ isGlobal: false }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
|
|
6
6
|
.build()
|
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
import { Module } from "@nestjs/common"
|
|
2
2
|
import type { DynamicModule } from "@nestjs/common"
|
|
3
|
-
import { EVENT_BUS } from "./event-bus.decorators"
|
|
3
|
+
import { EVENT_BUS, EVENT_CONSUMER_REGISTRY } from "./event-bus.decorators"
|
|
4
4
|
import { ConfigurableModuleClass, OPTIONS_TYPE } from "./event-bus.module-definition"
|
|
5
5
|
import { EventBusService } from "./event-bus.service"
|
|
6
|
+
import { EventConsumerService } from "./event-consumer.service"
|
|
6
7
|
|
|
7
8
|
@Module({})
|
|
8
|
-
/** Provides the
|
|
9
|
+
/** Provides the lite event bus on pgmq: the transactional publisher and the read loops of the consumer groups. */
|
|
9
10
|
export class EventBusModule extends ConfigurableModuleClass {
|
|
10
|
-
/** Registers the
|
|
11
|
+
/** Registers the capability once per app. */
|
|
11
12
|
static register(options: typeof OPTIONS_TYPE): DynamicModule {
|
|
12
13
|
const base = super.register(options)
|
|
13
14
|
return {
|
|
14
15
|
...base,
|
|
15
|
-
providers: [
|
|
16
|
-
|
|
16
|
+
providers: [
|
|
17
|
+
...(base.providers ?? []),
|
|
18
|
+
{ provide: EVENT_BUS, useClass: EventBusService },
|
|
19
|
+
EventConsumerService,
|
|
20
|
+
{ provide: EVENT_CONSUMER_REGISTRY, useExisting: EventConsumerService },
|
|
21
|
+
],
|
|
22
|
+
exports: [EVENT_BUS, EVENT_CONSUMER_REGISTRY],
|
|
17
23
|
}
|
|
18
24
|
}
|
|
19
25
|
}
|
|
@@ -1,2 +1,9 @@
|
|
|
1
|
-
/**
|
|
2
|
-
export interface EventBusOptions {
|
|
1
|
+
/** How the consumers of the app read their queues; every field has a default, so a bare registration works. */
|
|
2
|
+
export interface EventBusOptions {
|
|
3
|
+
/** Seconds a read message stays hidden: a consumer that fails or dies lets it reappear after this. Default 30. */
|
|
4
|
+
readonly visibilitySeconds?: number
|
|
5
|
+
/** The most messages one read takes from a queue. Default 10. */
|
|
6
|
+
readonly batch?: number
|
|
7
|
+
/** The pause between two reads of an empty queue, in milliseconds. Default 500. */
|
|
8
|
+
readonly idleMs?: number
|
|
9
|
+
}
|
|
@@ -1,16 +1,29 @@
|
|
|
1
1
|
import type { EntityManager } from "typeorm"
|
|
2
|
-
import type { BaseEvent, EventClass, EventDelivery } from "./event-bus.contracts"
|
|
2
|
+
import type { BaseEvent, DeadLetter, EventClass, EventDelivery, EventSubscriptionOptions } from "./event-bus.contracts"
|
|
3
3
|
|
|
4
|
-
/** The publishing side of the lite event bus
|
|
4
|
+
/** The publishing and operator side of the lite event bus on pgmq. */
|
|
5
5
|
export interface EventBus {
|
|
6
|
-
/**
|
|
6
|
+
/** Sends the typed event once to its topic in the caller's transaction; pgmq routes it to every bound consumer queue, and a rollback sends nothing. */
|
|
7
7
|
publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void>
|
|
8
|
+
/** The oldest deliveries the consumer of `queue` gave up on, at most `limit`, for an operator's cli command. */
|
|
9
|
+
deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>>
|
|
10
|
+
/** Sends one dead letter of `queue` back to that queue; false when it holds no dead letter with the id. */
|
|
11
|
+
requeue(queue: string, id: string): Promise<boolean>
|
|
8
12
|
}
|
|
9
13
|
|
|
10
|
-
/** A
|
|
14
|
+
/** A reactor's thin door: it reads one typed event and dispatches one command with the delivery's ids. */
|
|
11
15
|
export interface EventConsumer<Event extends BaseEvent> {
|
|
12
16
|
/** The event class the consumer reads. */
|
|
13
17
|
readonly event: EventClass<Event>
|
|
14
|
-
/** Applies one delivery
|
|
18
|
+
/** Applies one delivery; a throw leaves the message to reappear after its visibility timeout. */
|
|
15
19
|
handle(delivery: EventDelivery<Event>): Promise<void>
|
|
16
20
|
}
|
|
21
|
+
|
|
22
|
+
/** Where a reactor subscribes its consumer; reading starts when the app has bootstrapped. */
|
|
23
|
+
export interface EventConsumerRegistry {
|
|
24
|
+
/**
|
|
25
|
+
* Reads `queue` for `consumer`. The queue is a literal `<TOPIC>_<CONSUMER>_TOPIC_QUEUE` constant of platform/event-bus whose
|
|
26
|
+
* pgmq queue, dead-letter queue and topic binding a migration declares (R258); the subscription is static (R336).
|
|
27
|
+
*/
|
|
28
|
+
subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options?: EventSubscriptionOptions): void
|
|
29
|
+
}
|
|
@@ -1,27 +1,33 @@
|
|
|
1
1
|
import { Test } from "@nestjs/testing"
|
|
2
2
|
import { fakeTransaction, mockEntityManager } from "@starci/jest-preset"
|
|
3
|
+
import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
|
|
3
4
|
import { EventBusErrorCode } from "./errors/event-bus.error"
|
|
4
5
|
import { BaseEvent } from "./event-bus.contracts"
|
|
5
6
|
import { EventBusService } from "./event-bus.service"
|
|
6
7
|
|
|
7
|
-
// The
|
|
8
|
-
const
|
|
8
|
+
// The statements callDbFunction of platform/database sends for the bus functions of the migration.
|
|
9
|
+
const SEND = 'SELECT * FROM "public"."event_bus_send"($1, $2)'
|
|
10
|
+
const DEAD_LETTERS = 'SELECT * FROM "public"."event_bus_dead_letters"($1, $2)'
|
|
11
|
+
const REQUEUE = 'SELECT * FROM "public"."event_bus_requeue"($1, $2)'
|
|
12
|
+
const QUEUE = "spec_delivery_recorded__mailer"
|
|
9
13
|
|
|
10
14
|
class ReceiptRecordedEvent extends BaseEvent {
|
|
11
|
-
readonly eventId = "
|
|
15
|
+
readonly eventId = "event-1"
|
|
12
16
|
readonly eventName = "spec.delivery-recorded"
|
|
13
|
-
readonly payload = { receiptId: "receipt-1" }
|
|
17
|
+
readonly payload = { dedupeKey: "receipt-1", receiptId: "receipt-1" }
|
|
14
18
|
}
|
|
15
19
|
|
|
16
20
|
describe("EventBusService", () => {
|
|
17
|
-
const build = async () => {
|
|
18
|
-
const module = await Test.createTestingModule({
|
|
21
|
+
const build = async (primary = mockEntityManager({})) => {
|
|
22
|
+
const module = await Test.createTestingModule({
|
|
23
|
+
providers: [EventBusService, { provide: PRIMARY_ENTITY_MANAGER, useValue: primary }],
|
|
24
|
+
}).compile()
|
|
19
25
|
return module.get(EventBusService)
|
|
20
26
|
}
|
|
21
27
|
|
|
22
28
|
it.each(["", "unsafe\nid", "x".repeat(129)])("refuses invalid correlation before SQL: %s", async (correlationId) => {
|
|
23
29
|
const subject = await build()
|
|
24
|
-
const manager = mockEntityManager({ query: [
|
|
30
|
+
const manager = mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] })
|
|
25
31
|
|
|
26
32
|
await expect(subject.publish(new ReceiptRecordedEvent(), manager, correlationId)).rejects.toMatchObject({
|
|
27
33
|
code: EventBusErrorCode.CorrelationInvalid,
|
|
@@ -29,31 +35,46 @@ describe("EventBusService", () => {
|
|
|
29
35
|
expect(manager.query).not.toHaveBeenCalled()
|
|
30
36
|
})
|
|
31
37
|
|
|
32
|
-
it("
|
|
38
|
+
it("sends the typed envelope to the event's topic through the publisher's transaction", async () => {
|
|
33
39
|
const subject = await build()
|
|
34
|
-
const transaction = fakeTransaction(mockEntityManager({ query: [
|
|
40
|
+
const transaction = fakeTransaction(mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] }))
|
|
35
41
|
const event = new ReceiptRecordedEvent()
|
|
36
42
|
|
|
37
43
|
await transaction.em.transaction((manager) => subject.publish(event, manager, "request-1"))
|
|
38
44
|
|
|
39
|
-
expect(transaction.em.query).toHaveBeenCalledWith(
|
|
40
|
-
"receipt-1",
|
|
45
|
+
expect(transaction.em.query).toHaveBeenCalledWith(SEND, [
|
|
41
46
|
"spec.delivery-recorded",
|
|
42
47
|
JSON.stringify({ metadata: { correlationId: "request-1" }, eventId: event.eventId, eventName: event.eventName, payload: event.payload }),
|
|
43
|
-
"request-1",
|
|
44
48
|
])
|
|
45
49
|
expect(transaction.commits).toBe(1)
|
|
46
50
|
})
|
|
47
51
|
|
|
48
|
-
it("propagates
|
|
52
|
+
it("propagates a send failure so the publisher's transaction rolls back and nothing is sent", async () => {
|
|
49
53
|
const subject = await build()
|
|
50
|
-
const manager = mockEntityManager({ query: [
|
|
51
|
-
jest.spyOn(manager, "query").mockRejectedValue(new Error("
|
|
54
|
+
const manager = mockEntityManager({ query: [SEND, []] })
|
|
55
|
+
jest.spyOn(manager, "query").mockRejectedValue(new Error("pgmq unavailable"))
|
|
52
56
|
const transaction = fakeTransaction(manager)
|
|
53
57
|
|
|
54
|
-
await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("
|
|
58
|
+
await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("pgmq unavailable")
|
|
55
59
|
|
|
56
60
|
expect(transaction.commits).toBe(0)
|
|
57
61
|
expect(transaction.rollbacks).toBe(1)
|
|
58
62
|
})
|
|
63
|
+
|
|
64
|
+
it("lists a consumer's dead letters and requeues one by its dead-letter id", async () => {
|
|
65
|
+
const primary = mockEntityManager({
|
|
66
|
+
query: [
|
|
67
|
+
[DEAD_LETTERS, [{ msg_id: "7", message: { reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } } }]],
|
|
68
|
+
[REQUEUE, [{ event_bus_requeue: true }]],
|
|
69
|
+
],
|
|
70
|
+
})
|
|
71
|
+
const subject = await build(primary)
|
|
72
|
+
|
|
73
|
+
await expect(subject.deadLetters(QUEUE, 10)).resolves.toEqual([
|
|
74
|
+
{ id: "7", reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } },
|
|
75
|
+
])
|
|
76
|
+
await expect(subject.requeue(QUEUE, "7")).resolves.toBe(true)
|
|
77
|
+
expect(primary.query).toHaveBeenCalledWith(DEAD_LETTERS, [QUEUE, 10])
|
|
78
|
+
expect(primary.query).toHaveBeenCalledWith(REQUEUE, [QUEUE, "7"])
|
|
79
|
+
})
|
|
59
80
|
})
|
|
@@ -1,15 +1,28 @@
|
|
|
1
1
|
import { Injectable } from "@nestjs/common"
|
|
2
2
|
import type { EntityManager } from "typeorm"
|
|
3
3
|
import { isCorrelationId } from "@modules/platform/logging"
|
|
4
|
+
import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
|
|
4
5
|
import { EventBusError, EventBusErrorCode } from "./errors/event-bus.error"
|
|
5
|
-
import type { BaseEvent, EventEnvelope } from "./event-bus.contracts"
|
|
6
|
+
import type { BaseEvent, DeadLetter, EventEnvelope } from "./event-bus.contracts"
|
|
6
7
|
import type { EventBus } from "./event-bus.port"
|
|
7
|
-
|
|
8
|
+
|
|
9
|
+
/** One row of event_bus_dead_letters: pgmq returns the bigint id as text. */
|
|
10
|
+
interface DeadLetterRow {
|
|
11
|
+
readonly msg_id: string
|
|
12
|
+
readonly message: { readonly reason?: unknown; readonly readCount?: unknown; readonly message?: unknown }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** The one row event_bus_requeue answers. */
|
|
16
|
+
interface RequeueRow {
|
|
17
|
+
readonly event_bus_requeue: boolean | null
|
|
18
|
+
}
|
|
8
19
|
|
|
9
20
|
@Injectable()
|
|
10
|
-
/**
|
|
21
|
+
/** Sends a typed event once to its pgmq topic inside the publisher's transaction; no outbox and no relay stand between them. */
|
|
11
22
|
export class EventBusService implements EventBus {
|
|
12
|
-
|
|
23
|
+
constructor(@InjectPrimaryEntityManager() private readonly entityManager: EntityManager) {}
|
|
24
|
+
|
|
25
|
+
/** The send commits or rolls back with the change; pgmq copies it into every consumer queue bound to the event's name. */
|
|
13
26
|
async publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void> {
|
|
14
27
|
if (!isCorrelationId(correlationId)) throw new EventBusError({ code: EventBusErrorCode.CorrelationInvalid })
|
|
15
28
|
const envelope: EventEnvelope = {
|
|
@@ -18,6 +31,21 @@ export class EventBusService implements EventBus {
|
|
|
18
31
|
eventName: event.eventName,
|
|
19
32
|
payload: event.payload,
|
|
20
33
|
}
|
|
21
|
-
await callDbFunction(tx, "
|
|
34
|
+
await callDbFunction(tx, "public.event_bus_send", [event.eventName, JSON.stringify(envelope)])
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>> {
|
|
38
|
+
const rows = await callDbFunction<DeadLetterRow>(this.entityManager, "public.event_bus_dead_letters", [queue, limit])
|
|
39
|
+
return rows.map((row) => ({
|
|
40
|
+
id: row.msg_id,
|
|
41
|
+
reason: String(row.message.reason),
|
|
42
|
+
readCount: Number(row.message.readCount),
|
|
43
|
+
message: row.message.message,
|
|
44
|
+
}))
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async requeue(queue: string, id: string): Promise<boolean> {
|
|
48
|
+
const [row] = await callDbFunction<RequeueRow>(this.entityManager, "public.event_bus_requeue", [queue, id])
|
|
49
|
+
return row?.event_bus_requeue === true
|
|
22
50
|
}
|
|
23
51
|
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { Test } from "@nestjs/testing"
|
|
2
|
+
import { mockEntityManager } from "@starci/jest-preset"
|
|
3
|
+
import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
|
|
4
|
+
import { LOGGER } from "@modules/platform/logging"
|
|
5
|
+
import { BaseEvent } from "./event-bus.contracts"
|
|
6
|
+
import type { EventDelivery, ParsedEvent } from "./event-bus.contracts"
|
|
7
|
+
import { MODULE_OPTIONS_TOKEN } from "./event-bus.module-definition"
|
|
8
|
+
import type { EventConsumer } from "./event-bus.port"
|
|
9
|
+
import { EventConsumerService } from "./event-consumer.service"
|
|
10
|
+
|
|
11
|
+
// The statements callDbFunction of platform/database sends for the bus functions of the migration.
|
|
12
|
+
const READ = 'SELECT * FROM "public"."event_bus_read"($1, $2, $3)'
|
|
13
|
+
const ARCHIVE = 'SELECT * FROM "public"."event_bus_archive"($1, $2)'
|
|
14
|
+
const BURY = 'SELECT * FROM "public"."event_bus_bury"($1, $2, $3, $4, $5)'
|
|
15
|
+
const QUEUE = "spec_delivery_recorded__mailer"
|
|
16
|
+
|
|
17
|
+
class ReceiptRecordedEvent extends BaseEvent {
|
|
18
|
+
static readonly eventName = "spec.delivery-recorded"
|
|
19
|
+
static readonly version = 1
|
|
20
|
+
readonly eventName = ReceiptRecordedEvent.eventName
|
|
21
|
+
|
|
22
|
+
constructor(
|
|
23
|
+
readonly eventId: string,
|
|
24
|
+
readonly payload: { readonly dedupeKey: string; readonly receiptId: string },
|
|
25
|
+
) {
|
|
26
|
+
super()
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
static parse(envelope: unknown): ParsedEvent<ReceiptRecordedEvent> {
|
|
30
|
+
if (typeof envelope !== "object" || envelope === null || !("eventId" in envelope) || typeof envelope.eventId !== "string") return null
|
|
31
|
+
if (!("payload" in envelope) || typeof envelope.payload !== "object" || envelope.payload === null || !("dedupeKey" in envelope.payload)) return null
|
|
32
|
+
const dedupeKey = envelope.payload.dedupeKey
|
|
33
|
+
return typeof dedupeKey === "string" ? new ReceiptRecordedEvent(envelope.eventId, { dedupeKey, receiptId: dedupeKey }) : null
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const envelope = {
|
|
38
|
+
metadata: { correlationId: "request-1" },
|
|
39
|
+
eventId: "event-1",
|
|
40
|
+
eventName: "spec.delivery-recorded",
|
|
41
|
+
payload: { dedupeKey: "receipt-1", receiptId: "receipt-1" },
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
class RecordingConsumer implements EventConsumer<ReceiptRecordedEvent> {
|
|
45
|
+
readonly event = ReceiptRecordedEvent
|
|
46
|
+
readonly deliveries: Array<EventDelivery<ReceiptRecordedEvent>> = []
|
|
47
|
+
failing = false
|
|
48
|
+
|
|
49
|
+
handle(delivery: EventDelivery<ReceiptRecordedEvent>): Promise<void> {
|
|
50
|
+
if (this.failing) return Promise.reject(new Error("consumer down"))
|
|
51
|
+
this.deliveries.push(delivery)
|
|
52
|
+
return Promise.resolve()
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
describe("EventConsumerService", () => {
|
|
57
|
+
const logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }
|
|
58
|
+
|
|
59
|
+
const build = async (primary: ReturnType<typeof mockEntityManager>) => {
|
|
60
|
+
const module = await Test.createTestingModule({
|
|
61
|
+
providers: [
|
|
62
|
+
EventConsumerService,
|
|
63
|
+
{ provide: PRIMARY_ENTITY_MANAGER, useValue: primary },
|
|
64
|
+
{ provide: MODULE_OPTIONS_TOKEN, useValue: {} },
|
|
65
|
+
{ provide: LOGGER, useValue: logger },
|
|
66
|
+
],
|
|
67
|
+
}).compile()
|
|
68
|
+
return module.get(EventConsumerService)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
it("delivers a committed event once to the consumer with its inbox claim keys and archives it", async () => {
|
|
72
|
+
const primary = mockEntityManager({
|
|
73
|
+
query: [
|
|
74
|
+
[READ, [{ msg_id: "1", read_ct: 1, message: envelope }]],
|
|
75
|
+
[ARCHIVE, [{ event_bus_archive: true }]],
|
|
76
|
+
],
|
|
77
|
+
})
|
|
78
|
+
const subject = await build(primary)
|
|
79
|
+
const consumer = new RecordingConsumer()
|
|
80
|
+
|
|
81
|
+
await expect(subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })).resolves.toBe(1)
|
|
82
|
+
|
|
83
|
+
expect(consumer.deliveries).toEqual([
|
|
84
|
+
{ correlationId: "request-1", eventId: "event-1", consumer: QUEUE, dedupeKey: "receipt-1", event: expect.any(ReceiptRecordedEvent), attempt: 1 },
|
|
85
|
+
])
|
|
86
|
+
expect(primary.query).toHaveBeenCalledWith(READ, [QUEUE, 30, 10])
|
|
87
|
+
expect(primary.query).toHaveBeenCalledWith(ARCHIVE, [QUEUE, "1"])
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
it("leaves a failed delivery to reappear after its visibility timeout", async () => {
|
|
91
|
+
const primary = mockEntityManager({ query: [READ, [{ msg_id: "1", read_ct: 2, message: envelope }]] })
|
|
92
|
+
const subject = await build(primary)
|
|
93
|
+
const consumer = new RecordingConsumer()
|
|
94
|
+
consumer.failing = true
|
|
95
|
+
|
|
96
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
|
|
97
|
+
|
|
98
|
+
expect(primary.query).toHaveBeenCalledTimes(1)
|
|
99
|
+
expect(logger.error).toHaveBeenCalled()
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
it.each([
|
|
103
|
+
["max-deliveries", { msg_id: "1", read_ct: 4, message: envelope }],
|
|
104
|
+
["malformed", { msg_id: "1", read_ct: 1, message: { eventId: 42 } }],
|
|
105
|
+
["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, payload: { dedupeKey: " ", receiptId: "receipt-1" } } }],
|
|
106
|
+
["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, metadata: { correlationId: "unsafe\nid" } } }],
|
|
107
|
+
])("moves a delivery to the consumer's dead-letter queue: %s", async (reason, row) => {
|
|
108
|
+
const primary = mockEntityManager({ query: [[READ, [row]], [BURY, [{ event_bus_bury: true }]]] })
|
|
109
|
+
const subject = await build(primary)
|
|
110
|
+
const consumer = new RecordingConsumer()
|
|
111
|
+
|
|
112
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 3 })
|
|
113
|
+
|
|
114
|
+
expect(consumer.deliveries).toEqual([])
|
|
115
|
+
expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", row.read_ct, JSON.stringify(row.message), reason])
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
it("takes the default delivery budget of a subscription that names none", async () => {
|
|
119
|
+
const primary = mockEntityManager({ query: [[READ, [{ msg_id: "1", read_ct: 6, message: envelope }]], [BURY, [{ event_bus_bury: true }]]] })
|
|
120
|
+
const subject = await build(primary)
|
|
121
|
+
const consumer = new RecordingConsumer()
|
|
122
|
+
subject.subscribe(QUEUE, consumer)
|
|
123
|
+
|
|
124
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
|
|
125
|
+
|
|
126
|
+
expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", 6, JSON.stringify(envelope), "max-deliveries"])
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it("delivers a redelivery again, leaving its dedupe to the consumer's inbox claim", async () => {
|
|
130
|
+
const primary = mockEntityManager({
|
|
131
|
+
query: [
|
|
132
|
+
[READ, [{ msg_id: "1", read_ct: 2, message: envelope }]],
|
|
133
|
+
[ARCHIVE, [{ event_bus_archive: true }]],
|
|
134
|
+
],
|
|
135
|
+
})
|
|
136
|
+
const subject = await build(primary)
|
|
137
|
+
const consumer = new RecordingConsumer()
|
|
138
|
+
|
|
139
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
|
|
140
|
+
|
|
141
|
+
expect(consumer.deliveries.map((delivery) => [delivery.consumer, delivery.dedupeKey, delivery.attempt])).toEqual([[QUEUE, "receipt-1", 2]])
|
|
142
|
+
})
|
|
143
|
+
})
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { Injectable } from "@nestjs/common"
|
|
2
|
+
import type { OnApplicationBootstrap, OnApplicationShutdown } from "@nestjs/common"
|
|
3
|
+
import type { EntityManager } from "typeorm"
|
|
4
|
+
import { InjectLogger, isCorrelationId } from "@modules/platform/logging"
|
|
5
|
+
import type { Logger } from "@modules/platform/logging"
|
|
6
|
+
import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
|
|
7
|
+
import { InjectEventBusOptions } from "./event-bus.decorators"
|
|
8
|
+
import { DEFAULT_MAX_DELIVERIES } from "./event-bus.contracts"
|
|
9
|
+
import type { BaseEvent, EventMessage, EventSubscriptionOptions } from "./event-bus.contracts"
|
|
10
|
+
import { EventBusLogEvent } from "./event-bus.log-events"
|
|
11
|
+
import type { EventBusOptions } from "./event-bus.options"
|
|
12
|
+
import type { EventConsumer, EventConsumerRegistry } from "./event-bus.port"
|
|
13
|
+
|
|
14
|
+
/** One row of event_bus_read: pgmq returns the bigint id as text. */
|
|
15
|
+
interface MessageRow {
|
|
16
|
+
readonly msg_id: string
|
|
17
|
+
readonly read_ct: number
|
|
18
|
+
readonly message: unknown
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** One consumer: its queue `<topic>__<consumer>`, the door its deliveries go through and its delivery budget. */
|
|
22
|
+
export interface EventSubscription {
|
|
23
|
+
/** The consumer's pgmq queue. */
|
|
24
|
+
readonly queue: string
|
|
25
|
+
/** The thin door the consumer's deliveries go through. */
|
|
26
|
+
readonly consumer: EventConsumer<BaseEvent>
|
|
27
|
+
/** The deliveries before the message moves to `<queue>__dead`. */
|
|
28
|
+
readonly maxDeliveries: number
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The correlation id of an envelope: its metadata when present and valid, else a safe event id; null refuses the message. */
|
|
32
|
+
const correlationOf = (message: unknown, eventId: string): string | null => {
|
|
33
|
+
if (typeof message !== "object" || message === null || !("metadata" in message)) return isCorrelationId(eventId) ? eventId : null
|
|
34
|
+
const metadata = message.metadata
|
|
35
|
+
if (typeof metadata !== "object" || metadata === null || !("correlationId" in metadata)) return null
|
|
36
|
+
return isCorrelationId(metadata.correlationId) ? metadata.correlationId : null
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** True for a usable business dedupe key. */
|
|
40
|
+
const isDedupeKey = (value: unknown): value is string => typeof value === "string" && value.trim().length > 0 && value.length <= 256
|
|
41
|
+
|
|
42
|
+
@Injectable()
|
|
43
|
+
/**
|
|
44
|
+
* The consuming side of the lite event bus: each consumer reads its own pgmq queue with a visibility timeout, archives a message it
|
|
45
|
+
* handled, leaves a failed one to reappear, and moves one past its maxDeliveries, or malformed, to `<queue>__dead`. Delivery is at
|
|
46
|
+
* least once and unordered, so the command a consumer dispatches claims (consumer, dedupeKey) in the inbox first and reads current
|
|
47
|
+
* state rather than trusting the order of events (BE-EVENT-BUS-4).
|
|
48
|
+
*/
|
|
49
|
+
export class EventConsumerService implements EventConsumerRegistry, OnApplicationBootstrap, OnApplicationShutdown {
|
|
50
|
+
private readonly subscriptions: Array<EventSubscription> = []
|
|
51
|
+
private running = false
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
@InjectPrimaryEntityManager() private readonly entityManager: EntityManager,
|
|
55
|
+
@InjectEventBusOptions() private readonly options: EventBusOptions,
|
|
56
|
+
@InjectLogger() private readonly logger: Logger,
|
|
57
|
+
) {}
|
|
58
|
+
|
|
59
|
+
subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options: EventSubscriptionOptions = {}): void {
|
|
60
|
+
this.subscriptions.push({ queue, consumer, maxDeliveries: options.maxDeliveries ?? DEFAULT_MAX_DELIVERIES })
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
onApplicationBootstrap(): void {
|
|
64
|
+
this.running = true
|
|
65
|
+
for (const subscription of this.subscriptions) void this.loop(subscription)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
onApplicationShutdown(): void {
|
|
69
|
+
this.running = false
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Reads one batch of the consumer's queue and settles each message in order; answers how many it read. */
|
|
73
|
+
async drain(subscription: EventSubscription): Promise<number> {
|
|
74
|
+
const rows = await callDbFunction<MessageRow>(this.entityManager, "public.event_bus_read", [
|
|
75
|
+
subscription.queue,
|
|
76
|
+
this.options.visibilitySeconds ?? 30,
|
|
77
|
+
this.options.batch ?? 10,
|
|
78
|
+
])
|
|
79
|
+
for (const row of rows) await this.settle(subscription, { msgId: row.msg_id, readCt: row.read_ct, message: row.message })
|
|
80
|
+
return rows.length
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private async loop(subscription: EventSubscription): Promise<void> {
|
|
84
|
+
while (this.running) {
|
|
85
|
+
const read = await this.drain(subscription).catch((cause: unknown) => {
|
|
86
|
+
this.logger.error(EventBusLogEvent.ReadFailed, cause, { queue: subscription.queue })
|
|
87
|
+
return 0
|
|
88
|
+
})
|
|
89
|
+
if (read === 0) await new Promise((resolve) => setTimeout(resolve, this.options.idleMs ?? 500))
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private async settle(subscription: EventSubscription, message: EventMessage): Promise<void> {
|
|
94
|
+
const event = subscription.consumer.event.parse(message.message)
|
|
95
|
+
const correlationId = event === null ? null : correlationOf(message.message, event.eventId)
|
|
96
|
+
if (event === null || correlationId === null || !isDedupeKey(event.payload.dedupeKey)) return this.dead(subscription, message, "malformed", null)
|
|
97
|
+
if (message.readCt > subscription.maxDeliveries) return this.dead(subscription, message, "max-deliveries", event.eventName)
|
|
98
|
+
try {
|
|
99
|
+
await subscription.consumer.handle({
|
|
100
|
+
correlationId,
|
|
101
|
+
eventId: event.eventId,
|
|
102
|
+
consumer: subscription.queue,
|
|
103
|
+
dedupeKey: event.payload.dedupeKey,
|
|
104
|
+
event,
|
|
105
|
+
attempt: message.readCt,
|
|
106
|
+
})
|
|
107
|
+
} catch (cause) {
|
|
108
|
+
this.logger.error(EventBusLogEvent.ConsumerFailed, cause, { consumer: subscription.queue, event: event.eventName, attempt: message.readCt })
|
|
109
|
+
return
|
|
110
|
+
}
|
|
111
|
+
await callDbFunction(this.entityManager, "public.event_bus_archive", [subscription.queue, message.msgId])
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Moves the message to `<queue>__dead` and archives the original, in one call. */
|
|
115
|
+
private async dead(subscription: EventSubscription, message: EventMessage, reason: string, eventName: string | null): Promise<void> {
|
|
116
|
+
this.logger.warn(EventBusLogEvent.DeliveryDead, { consumer: subscription.queue, event: eventName, msgId: message.msgId, reason, attempt: message.readCt })
|
|
117
|
+
await callDbFunction(this.entityManager, "public.event_bus_bury", [subscription.queue, message.msgId, message.readCt, JSON.stringify(message.message), reason])
|
|
118
|
+
}
|
|
119
|
+
}
|