@starci/hfs 4.10.1 → 4.11.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/CHANGELOG.md +76 -0
  2. package/README.md +40 -1
  3. package/db/drift.mjs +19 -16
  4. package/db/image.mjs +13 -0
  5. package/db/migrate.mjs +3 -3
  6. package/db/postgres.mjs +1 -1
  7. package/package.json +2 -1
  8. package/runtime/knowledge/hfs/canon-pins.yaml +23 -8
  9. package/runtime/knowledge/hfs/rules.yaml +54 -13
  10. package/runtime/knowledge/hfs/slots.yaml +67 -36
  11. package/runtime/knowledge/patterns/be/event-bus.yaml +12 -10
  12. package/runtime/knowledge/patterns/be/idempotency.yaml +1 -1
  13. package/runtime/knowledge/patterns/be/jobs.yaml +3 -0
  14. package/runtime/knowledge/patterns/be/queues.yaml +14 -14
  15. package/runtime/knowledge/patterns/be/reactors.yaml +1 -1
  16. package/runtime/knowledge/patterns/be/realtime.yaml +1 -1
  17. package/runtime/modules/kernel/failure-codes.yaml +28 -18
  18. package/runtime/modules/schemas/application-stacks.schema.yaml +4 -2
  19. package/runtime/scripts/gates/starcistacks.mjs +6 -2
  20. package/runtime/scripts/guards/push-scan.mjs +1 -1
  21. package/runtime/scripts/hfs/ci-declaration.mjs +3 -2
  22. package/runtime/scripts/hfs/db-baseline-verb.mjs +94 -22
  23. package/runtime/scripts/hfs/db-errors.mjs +2 -0
  24. package/runtime/scripts/hfs/db-plan-verb.mjs +17 -5
  25. package/runtime/scripts/hfs/db-squash.mjs +514 -0
  26. package/runtime/scripts/hfs/declaration-shape.mjs +3 -15
  27. package/runtime/scripts/hfs/example-secrets.mjs +4 -3
  28. package/runtime/scripts/hfs/rule-params-shape.mjs +11 -9
  29. package/runtime/scripts/hfs/rules/a15-findings.mjs +1 -1
  30. package/runtime/scripts/hfs/rules/database-config.mjs +1 -1
  31. package/runtime/scripts/hfs/rules/database-constants.mjs +16 -1
  32. package/runtime/scripts/hfs/rules/database-queue-sources.mjs +2 -110
  33. package/runtime/scripts/hfs/rules/database-queues.mjs +53 -239
  34. package/runtime/scripts/hfs/rules/database-sql.mjs +2 -0
  35. package/runtime/scripts/hfs/rules/database-transaction.mjs +45 -0
  36. package/runtime/scripts/hfs/rules/deploy-scan.mjs +1 -1
  37. package/runtime/scripts/hfs/rules/edition.mjs +1 -1
  38. package/runtime/scripts/hfs/rules/env-parity.mjs +1 -1
  39. package/runtime/scripts/hfs/rules/lite-secrets.mjs +3 -3
  40. package/runtime/scripts/hfs/rules/script-hygiene.mjs +1 -1
  41. package/runtime/scripts/hfs/rules/secret-custody-only.mjs +2 -2
  42. package/runtime/scripts/hfs/rules/secrets.mjs +1 -1
  43. package/runtime/scripts/hfs/rules/stack-tooling.mjs +1 -1
  44. package/runtime/scripts/hfs/rules/stacks-deploy.mjs +2 -1
  45. package/runtime/scripts/hfs/rules/stacks-lite-backing.mjs +3 -4
  46. package/runtime/scripts/hfs/rules/stacks-runtime.mjs +2 -2
  47. package/runtime/scripts/hfs/rules/stacks-swarm.mjs +1 -1
  48. package/runtime/scripts/hfs/rules/stacks.mjs +3 -3
  49. package/runtime/scripts/hfs/sealed-members.mjs +10 -6
  50. package/runtime/scripts/hfs/secret-files.mjs +16 -0
  51. package/runtime/scripts/hfs/secret-init.mjs +120 -21
  52. package/runtime/scripts/hfs/secret-output.mjs +2 -1
  53. package/runtime/scripts/hfs/secret-prompt.mjs +128 -0
  54. package/runtime/scripts/hfs/secret-sops.mjs +93 -9
  55. package/runtime/scripts/hfs/secret-standard.mjs +315 -0
  56. package/runtime/scripts/hfs/secret-store.mjs +23 -9
  57. package/runtime/scripts/hfs/secret.mjs +110 -46
  58. package/runtime/scripts/hfs/slots.mjs +8 -1
  59. package/runtime/scripts/hfs/stack-deploy-config.mjs +20 -6
  60. package/runtime/scripts/hfs/stacks-gate.mjs +9 -2
  61. package/runtime/scripts/hfs/starcistacks-runtime.mjs +1 -1
  62. package/runtime/scripts/lib/sealed-member.mjs +36 -0
  63. package/runtime/scripts/lib/secret-patterns.mjs +1 -1
  64. package/runtime/scripts/lib/swarm-stack.mjs +4 -6
  65. package/scaffold/add-inbox-lite.mjs +2 -2
  66. package/scaffold/add-queue-lite.mjs +24 -5
  67. package/scaffold/app.mjs +2 -2
  68. package/src/main.mjs +5 -4
  69. package/stack/build.mjs +3 -1
  70. package/stack/config.mjs +2 -1
  71. package/stack/docker-transport.mjs +2 -1
  72. package/stack/remote.mjs +4 -3
  73. package/stack/sealed.mjs +9 -8
  74. package/stack/target-binding.mjs +2 -1
  75. package/sync/hygiene.mjs +1 -1
  76. package/sync/index.mjs +47 -12
  77. package/templates/app/db-bootstrap/db/bootstrap.sql +9 -14
  78. package/templates/app/db-image/starcistacks/images/db/Dockerfile +18 -0
  79. package/templates/app/gitignore +2 -1
  80. package/templates/app/gitignore-lite +3 -1
  81. package/templates/app/package-scripts/db.json +2 -1
  82. package/templates/app/skeleton-lite/.starcistacks/application-stacks.yaml +4 -2
  83. package/templates/app/skeleton-lite/.starcistacks/dev/README.md +1 -1
  84. package/templates/app/skeleton-lite/.starcistacks/dev/infra/swarm/stack.yaml +1 -1
  85. package/templates/app/skeleton-lite/.starcistacks/production/README.md +1 -1
  86. package/templates/app/skeleton-lite/.starcistacks/production/infra/swarm/stack.yaml +1 -1
  87. package/templates/app/skeleton-lite/README.md +2 -2
  88. package/templates/be/inbox/delivery-recorded.event.ts.tpl +7 -5
  89. package/templates/be/inbox/events.json.tpl +1 -1
  90. package/templates/be/inbox/platform/event-bus/event-bus.contracts.ts.tpl +58 -11
  91. package/templates/be/inbox/platform/event-bus/event-bus.decorators.ts.tpl +13 -1
  92. package/templates/be/inbox/platform/event-bus/event-bus.log-events.ts.tpl +9 -0
  93. package/templates/be/inbox/platform/event-bus/event-bus.module-definition.ts.tpl +1 -1
  94. package/templates/be/inbox/platform/event-bus/event-bus.module.ts.tpl +11 -5
  95. package/templates/be/inbox/platform/event-bus/event-bus.options.ts.tpl +9 -2
  96. package/templates/be/inbox/platform/event-bus/event-bus.port.ts.tpl +18 -5
  97. package/templates/be/inbox/platform/event-bus/event-bus.service.spec.ts.tpl +37 -16
  98. package/templates/be/inbox/platform/event-bus/event-bus.service.ts.tpl +33 -5
  99. package/templates/be/inbox/platform/event-bus/event-consumer.service.spec.ts.tpl +143 -0
  100. package/templates/be/inbox/platform/event-bus/event-consumer.service.ts.tpl +119 -0
  101. package/templates/be/inbox/platform/event-bus/index.ts.tpl +13 -4
  102. package/templates/be/inbox/platform/inbox/inbox.service.spec.ts.tpl +4 -4
  103. package/templates/be/inbox/platform/inbox/inbox.service.ts.tpl +4 -4
  104. package/templates/be/inbox/platform/migration.sql.tpl +97 -34
  105. package/templates/be/inbox/service.spec.ts.tpl +1 -1
  106. package/templates/be/inbox/webhook-door.ts.tpl +2 -2
  107. package/templates/be/patterns/jobs/platform/jobs.log-events.ts.tpl +2 -0
  108. package/templates/be/patterns/jobs/platform-lite/job-claim.service.spec.ts.tpl +4 -4
  109. package/templates/be/patterns/jobs/platform-lite/job-claim.service.ts.tpl +5 -5
  110. package/templates/be/patterns/jobs/platform-lite/job-tick.service.spec.ts.tpl +65 -0
  111. package/templates/be/patterns/jobs/platform-lite/job-tick.service.ts.tpl +78 -0
  112. package/templates/be/patterns/jobs/platform-lite/job-ticks.contracts.ts.tpl +16 -0
  113. package/templates/be/patterns/jobs/platform-lite/jobs.module.ts.tpl +40 -0
  114. package/templates/be/patterns/jobs/platform-lite/jobs.sql.tpl +44 -0
  115. package/templates/be/patterns/queues/platform-lite/queue-outbox.service.spec.ts.tpl +1 -1
  116. package/templates/be/patterns/queues/platform-lite/queue-outbox.service.ts.tpl +1 -1
  117. package/templates/be/patterns/queues/platform-lite/queue-relay.service.spec.ts.tpl +2 -2
  118. package/templates/be/patterns/queues/platform-lite/queue-relay.service.ts.tpl +2 -2
  119. package/templates/be/skeleton/apps/__app__/src/__app__.env.ts +4 -0
  120. package/templates/be/skeleton/apps/__app__/src/__app__.options.ts +3 -0
  121. package/templates/be/skeleton/apps/__app__/src/compose/platform.ts +1 -1
  122. package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.spec.ts +37 -2
  123. package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.ts +36 -3
  124. package/templates/be/skeleton/src/modules/platform/http/http.config.ts +11 -0
  125. package/templates/be/skeleton/src/modules/platform/http/http.decorators.ts +7 -0
  126. package/templates/be/skeleton/src/modules/platform/http/http.module-definition.ts +5 -3
  127. package/templates/be/skeleton/src/modules/platform/http/http.options.ts +9 -2
  128. package/templates/be/skeleton/src/modules/platform/http/index.ts +2 -0
  129. package/templates/be/skeleton-lite/src/modules/platform/database/database.client.spec.ts +96 -0
  130. package/templates/be/skeleton-lite/src/modules/platform/database/database.client.ts +93 -0
  131. package/templates/be/skeleton-lite/src/modules/platform/database/index.ts +1 -1
  132. package/runtime/scripts/hfs/rules/database-queue-outbox.mjs +0 -102
  133. package/templates/be/skeleton-lite/src/modules/platform/database/database.functions.ts +0 -24
@@ -1,23 +1,38 @@
1
+ /**
2
+ * The suffix of a consumer's dead-letter queue: a delivery past its consumer's maxDeliveries moves from `<topic>__<consumer>` to
3
+ * `<topic>__<consumer>__dead`, which an operator reads and replays (BE-EVENT-BUS-4).
4
+ */
5
+ export const DEAD_QUEUE_SUFFIX = "__dead"
6
+
7
+ /** The deliveries a consumer gets before its message moves to the dead-letter queue, when its subscription names none. */
8
+ export const DEFAULT_MAX_DELIVERIES = 5
9
+
10
+ /** What every event payload carries: the business key a consumer's inbox claims, stable across redelivery and replay. */
11
+ export interface EventPayload {
12
+ /** The business identity of the fact (an order id, a receipt id): the consumer claims (consumer, dedupeKey) once. */
13
+ readonly dedupeKey: string
14
+ }
15
+
1
16
  /** The base of each typed event in modules/events: its stable identity, name and payload. */
2
17
  export abstract class BaseEvent {
3
- /** A receiver uses this stable identity to deduplicate redelivery. */
18
+ /** The stable identity of this event, carried for logs and tracing. */
4
19
  abstract readonly eventId: string
5
- /** The event name listed by the service's vendored event contract. */
20
+ /** The event name listed by the service's vendored event contract; it is the pgmq topic the event is sent to. */
6
21
  abstract readonly eventName: string
7
- /** The fact that a reactor or job processes after commit. */
8
- abstract readonly payload: object
22
+ /** The fact that a reactor processes after commit, with its business dedupe key. */
23
+ abstract readonly payload: EventPayload
9
24
  }
10
25
 
11
26
  /** Parsing either returns the event or refuses a mismatched envelope. */
12
27
  export type ParsedEvent<Event extends BaseEvent> = Event | null
13
28
 
14
- /** The class side of a typed event, shared by publishers and later consumers. */
29
+ /** The class side of a typed event, shared by publishers and consumers. */
15
30
  export interface EventClass<Event extends BaseEvent> {
16
31
  /** The contract's event name. */
17
32
  readonly eventName: string
18
33
  /** The contract's payload version. */
19
34
  readonly version: number
20
- /** Reads the event from its durable envelope. */
35
+ /** Reads the event from its envelope. */
21
36
  parse(envelope: unknown): ParsedEvent<Event>
22
37
  }
23
38
 
@@ -25,21 +40,25 @@ export interface EventClass<Event extends BaseEvent> {
25
40
  export interface EventDelivery<Event extends BaseEvent> {
26
41
  /** The request id carried explicitly from intake to delivery. */
27
42
  readonly correlationId: string
28
- /** The consumer's idempotency key. */
43
+ /** The stable identity of the event. */
29
44
  readonly eventId: string
45
+ /** The consumer's queue, `<topic>__<consumer>`: the first half of the inbox claim. */
46
+ readonly consumer: string
47
+ /** The payload's business key: the second half of the inbox claim. */
48
+ readonly dedupeKey: string
30
49
  /** The typed event. */
31
50
  readonly event: Event
32
- /** The number of attempts, including this one. */
51
+ /** The number of deliveries, including this one. */
33
52
  readonly attempt: number
34
53
  }
35
54
 
36
55
  /** Explicit request context carried independently of the business payload. */
37
56
  export interface EventMetadata {
38
- /** The originating request or promoted legacy event identity. */
57
+ /** The originating request or promoted event identity. */
39
58
  readonly correlationId: string
40
59
  }
41
60
 
42
- /** The typed event's durable envelope, read only after the intake transaction commits. */
61
+ /** The typed event's envelope, sent once to its topic in the publisher's transaction and routed to every bound consumer queue. */
43
62
  export interface EventEnvelope {
44
63
  /** The originating request context, independent of the business payload. */
45
64
  readonly metadata: EventMetadata
@@ -48,5 +67,33 @@ export interface EventEnvelope {
48
67
  /** The contract's event name. */
49
68
  readonly eventName: string
50
69
  /** The data of the fact. */
51
- readonly payload: object
70
+ readonly payload: EventPayload
71
+ }
72
+
73
+ /** One message of a consumer's queue, as event_bus_read returns it. */
74
+ export interface EventMessage {
75
+ /** The pgmq message id within its queue. */
76
+ readonly msgId: string
77
+ /** How many times the message has been read, including this read. */
78
+ readonly readCt: number
79
+ /** The envelope as the publisher sent it; untrusted until parsed. */
80
+ readonly message: unknown
81
+ }
82
+
83
+ /** What a reactor's subscription may tune; every field is a literal of the subscription. */
84
+ export interface EventSubscriptionOptions {
85
+ /** The deliveries before the message moves to the dead-letter queue. Default DEFAULT_MAX_DELIVERIES. */
86
+ readonly maxDeliveries?: number
87
+ }
88
+
89
+ /** A delivery a consumer gave up on, as its dead-letter queue holds it. */
90
+ export interface DeadLetter {
91
+ /** The message id in the dead-letter queue: what requeue takes. */
92
+ readonly id: string
93
+ /** Why it was moved: its deliveries ran out, or its envelope was malformed. */
94
+ readonly reason: string
95
+ /** The deliveries it had. */
96
+ readonly readCount: number
97
+ /** The envelope as it was read. */
98
+ readonly message: unknown
52
99
  }
@@ -1,9 +1,21 @@
1
1
  import { injector } from "@modules/platform/composition"
2
2
  import type { TypedParameterDecorator } from "@modules/platform/composition"
3
- import type { EventBus } from "./event-bus.port"
3
+ import { MODULE_OPTIONS_TOKEN } from "./event-bus.module-definition"
4
+ import type { EventBusOptions } from "./event-bus.options"
5
+ import type { EventBus, EventConsumerRegistry } from "./event-bus.port"
4
6
 
5
7
  /** Token of the transactional EventBus publisher. */
6
8
  export const EVENT_BUS: unique symbol = Symbol("platform.event-bus")
7
9
 
10
+ /** Token of the registry reactors subscribe their consumers to. */
11
+ export const EVENT_CONSUMER_REGISTRY: unique symbol = Symbol("platform.event-bus.consumers")
12
+
8
13
  /** Injects the EventBus port. Parameter type: EventBus. */
9
14
  export const InjectEventBus = (): TypedParameterDecorator<EventBus> => injector<EventBus>(EVENT_BUS)
15
+
16
+ /** Injects the consumer registry. Parameter type: EventConsumerRegistry. */
17
+ export const InjectEventConsumerRegistry = (): TypedParameterDecorator<EventConsumerRegistry> =>
18
+ injector<EventConsumerRegistry>(EVENT_CONSUMER_REGISTRY)
19
+
20
+ /** Injects the module's options. Parameter type: EventBusOptions. */
21
+ export const InjectEventBusOptions = (): TypedParameterDecorator<EventBusOptions> => injector<EventBusOptions>(MODULE_OPTIONS_TOKEN)
@@ -0,0 +1,9 @@
1
+ /** Log events of the event bus. */
2
+ export enum EventBusLogEvent {
3
+ /** A consumer threw; the message reappears after its visibility timeout. */
4
+ ConsumerFailed = "event-bus.consumer.failed",
5
+ /** A delivery ran out of deliveries or was malformed and moved to its consumer's dead-letter queue. */
6
+ DeliveryDead = "event-bus.delivery.dead",
7
+ /** A read of a consumer's queue failed; the loop reads again after its idle pause. */
8
+ ReadFailed = "event-bus.read.failed",
9
+ }
@@ -1,6 +1,6 @@
1
1
  import { ConfigurableModuleBuilder } from "@nestjs/common"
2
2
  import type { EventBusOptions } from "./event-bus.options"
3
3
 
4
- export const { ConfigurableModuleClass, OPTIONS_TYPE } = new ConfigurableModuleBuilder<EventBusOptions>()
4
+ export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE } = new ConfigurableModuleBuilder<EventBusOptions>()
5
5
  .setExtras({ isGlobal: false }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
6
6
  .build()
@@ -1,19 +1,25 @@
1
1
  import { Module } from "@nestjs/common"
2
2
  import type { DynamicModule } from "@nestjs/common"
3
- import { EVENT_BUS } from "./event-bus.decorators"
3
+ import { EVENT_BUS, EVENT_CONSUMER_REGISTRY } from "./event-bus.decorators"
4
4
  import { ConfigurableModuleClass, OPTIONS_TYPE } from "./event-bus.module-definition"
5
5
  import { EventBusService } from "./event-bus.service"
6
+ import { EventConsumerService } from "./event-consumer.service"
6
7
 
7
8
  @Module({})
8
- /** Provides the transactional lite EventBus publisher, shared by domain intake services. */
9
+ /** Provides the lite event bus on pgmq: the transactional publisher and the read loops of the consumer groups. */
9
10
  export class EventBusModule extends ConfigurableModuleClass {
10
- /** Registers the publishing capability once per app. */
11
+ /** Registers the capability once per app. */
11
12
  static register(options: typeof OPTIONS_TYPE): DynamicModule {
12
13
  const base = super.register(options)
13
14
  return {
14
15
  ...base,
15
- providers: [...(base.providers ?? []), { provide: EVENT_BUS, useClass: EventBusService }],
16
- exports: [EVENT_BUS],
16
+ providers: [
17
+ ...(base.providers ?? []),
18
+ { provide: EVENT_BUS, useClass: EventBusService },
19
+ EventConsumerService,
20
+ { provide: EVENT_CONSUMER_REGISTRY, useExisting: EventConsumerService },
21
+ ],
22
+ exports: [EVENT_BUS, EVENT_CONSUMER_REGISTRY],
17
23
  }
18
24
  }
19
25
  }
@@ -1,2 +1,9 @@
1
- /** The lite publisher uses the caller's transaction and has no runtime options. */
2
- export interface EventBusOptions {}
1
+ /** How the consumers of the app read their queues; every field has a default, so a bare registration works. */
2
+ export interface EventBusOptions {
3
+ /** Seconds a read message stays hidden: a consumer that fails or dies lets it reappear after this. Default 30. */
4
+ readonly visibilitySeconds?: number
5
+ /** The most messages one read takes from a queue. Default 10. */
6
+ readonly batch?: number
7
+ /** The pause between two reads of an empty queue, in milliseconds. Default 500. */
8
+ readonly idleMs?: number
9
+ }
@@ -1,16 +1,29 @@
1
1
  import type { EntityManager } from "typeorm"
2
- import type { BaseEvent, EventClass, EventDelivery } from "./event-bus.contracts"
2
+ import type { BaseEvent, DeadLetter, EventClass, EventDelivery, EventSubscriptionOptions } from "./event-bus.contracts"
3
3
 
4
- /** The publishing side of the lite event bus; the reactor or job owns delivery of committed outbox rows. */
4
+ /** The publishing and operator side of the lite event bus on pgmq. */
5
5
  export interface EventBus {
6
- /** Writes the typed event in the caller's transaction; a rollback writes no durable event. */
6
+ /** Sends the typed event once to its topic in the caller's transaction; pgmq routes it to every bound consumer queue, and a rollback sends nothing. */
7
7
  publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void>
8
+ /** The oldest deliveries the consumer of `queue` gave up on, at most `limit`, for an operator's cli command. */
9
+ deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>>
10
+ /** Sends one dead letter of `queue` back to that queue; false when it holds no dead letter with the id. */
11
+ requeue(queue: string, id: string): Promise<boolean>
8
12
  }
9
13
 
10
- /** A later message transport receives one typed event and forwards its delivery id. */
14
+ /** A reactor's thin door: it reads one typed event and dispatches one command with the delivery's ids. */
11
15
  export interface EventConsumer<Event extends BaseEvent> {
12
16
  /** The event class the consumer reads. */
13
17
  readonly event: EventClass<Event>
14
- /** Applies one delivery after commit; a throw leaves the delivery available for retry. */
18
+ /** Applies one delivery; a throw leaves the message to reappear after its visibility timeout. */
15
19
  handle(delivery: EventDelivery<Event>): Promise<void>
16
20
  }
21
+
22
+ /** Where a reactor subscribes its consumer; reading starts when the app has bootstrapped. */
23
+ export interface EventConsumerRegistry {
24
+ /**
25
+ * Reads `queue` for `consumer`. The queue is a literal `<TOPIC>_<CONSUMER>_TOPIC_QUEUE` constant of platform/event-bus whose
26
+ * pgmq queue, dead-letter queue and topic binding a migration declares (R258); the subscription is static (R336).
27
+ */
28
+ subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options?: EventSubscriptionOptions): void
29
+ }
@@ -1,27 +1,33 @@
1
1
  import { Test } from "@nestjs/testing"
2
2
  import { fakeTransaction, mockEntityManager } from "@starci/jest-preset"
3
+ import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
3
4
  import { EventBusErrorCode } from "./errors/event-bus.error"
4
5
  import { BaseEvent } from "./event-bus.contracts"
5
6
  import { EventBusService } from "./event-bus.service"
6
7
 
7
- // The statement callDbFunction of platform/database sends for event_bus_write.
8
- const INSERT_OUTBOX_EVENT = "SELECT * FROM event_bus_write($1, $2, $3, $4)"
8
+ // The statements callDbFunction of platform/database sends for the bus functions of the migration.
9
+ const SEND = 'SELECT * FROM "public"."event_bus_send"($1, $2)'
10
+ const DEAD_LETTERS = 'SELECT * FROM "public"."event_bus_dead_letters"($1, $2)'
11
+ const REQUEUE = 'SELECT * FROM "public"."event_bus_requeue"($1, $2)'
12
+ const QUEUE = "spec_delivery_recorded__mailer"
9
13
 
10
14
  class ReceiptRecordedEvent extends BaseEvent {
11
- readonly eventId = "receipt-1"
15
+ readonly eventId = "event-1"
12
16
  readonly eventName = "spec.delivery-recorded"
13
- readonly payload = { receiptId: "receipt-1" }
17
+ readonly payload = { dedupeKey: "receipt-1", receiptId: "receipt-1" }
14
18
  }
15
19
 
16
20
  describe("EventBusService", () => {
17
- const build = async () => {
18
- const module = await Test.createTestingModule({ providers: [EventBusService] }).compile()
21
+ const build = async (primary = mockEntityManager({})) => {
22
+ const module = await Test.createTestingModule({
23
+ providers: [EventBusService, { provide: PRIMARY_ENTITY_MANAGER, useValue: primary }],
24
+ }).compile()
19
25
  return module.get(EventBusService)
20
26
  }
21
27
 
22
28
  it.each(["", "unsafe\nid", "x".repeat(129)])("refuses invalid correlation before SQL: %s", async (correlationId) => {
23
29
  const subject = await build()
24
- const manager = mockEntityManager({ query: [INSERT_OUTBOX_EVENT, []] })
30
+ const manager = mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] })
25
31
 
26
32
  await expect(subject.publish(new ReceiptRecordedEvent(), manager, correlationId)).rejects.toMatchObject({
27
33
  code: EventBusErrorCode.CorrelationInvalid,
@@ -29,31 +35,46 @@ describe("EventBusService", () => {
29
35
  expect(manager.query).not.toHaveBeenCalled()
30
36
  })
31
37
 
32
- it("writes the typed envelope through the receipt transaction", async () => {
38
+ it("sends the typed envelope to the event's topic through the publisher's transaction", async () => {
33
39
  const subject = await build()
34
- const transaction = fakeTransaction(mockEntityManager({ query: [INSERT_OUTBOX_EVENT, []] }))
40
+ const transaction = fakeTransaction(mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] }))
35
41
  const event = new ReceiptRecordedEvent()
36
42
 
37
43
  await transaction.em.transaction((manager) => subject.publish(event, manager, "request-1"))
38
44
 
39
- expect(transaction.em.query).toHaveBeenCalledWith(INSERT_OUTBOX_EVENT, [
40
- "receipt-1",
45
+ expect(transaction.em.query).toHaveBeenCalledWith(SEND, [
41
46
  "spec.delivery-recorded",
42
47
  JSON.stringify({ metadata: { correlationId: "request-1" }, eventId: event.eventId, eventName: event.eventName, payload: event.payload }),
43
- "request-1",
44
48
  ])
45
49
  expect(transaction.commits).toBe(1)
46
50
  })
47
51
 
48
- it("propagates an outbox failure so the receipt transaction rolls back", async () => {
52
+ it("propagates a send failure so the publisher's transaction rolls back and nothing is sent", async () => {
49
53
  const subject = await build()
50
- const manager = mockEntityManager({ query: [INSERT_OUTBOX_EVENT, []] })
51
- jest.spyOn(manager, "query").mockRejectedValue(new Error("outbox unavailable"))
54
+ const manager = mockEntityManager({ query: [SEND, []] })
55
+ jest.spyOn(manager, "query").mockRejectedValue(new Error("pgmq unavailable"))
52
56
  const transaction = fakeTransaction(manager)
53
57
 
54
- await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("outbox unavailable")
58
+ await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("pgmq unavailable")
55
59
 
56
60
  expect(transaction.commits).toBe(0)
57
61
  expect(transaction.rollbacks).toBe(1)
58
62
  })
63
+
64
+ it("lists a consumer's dead letters and requeues one by its dead-letter id", async () => {
65
+ const primary = mockEntityManager({
66
+ query: [
67
+ [DEAD_LETTERS, [{ msg_id: "7", message: { reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } } }]],
68
+ [REQUEUE, [{ event_bus_requeue: true }]],
69
+ ],
70
+ })
71
+ const subject = await build(primary)
72
+
73
+ await expect(subject.deadLetters(QUEUE, 10)).resolves.toEqual([
74
+ { id: "7", reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } },
75
+ ])
76
+ await expect(subject.requeue(QUEUE, "7")).resolves.toBe(true)
77
+ expect(primary.query).toHaveBeenCalledWith(DEAD_LETTERS, [QUEUE, 10])
78
+ expect(primary.query).toHaveBeenCalledWith(REQUEUE, [QUEUE, "7"])
79
+ })
59
80
  })
@@ -1,15 +1,28 @@
1
1
  import { Injectable } from "@nestjs/common"
2
2
  import type { EntityManager } from "typeorm"
3
3
  import { isCorrelationId } from "@modules/platform/logging"
4
+ import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
4
5
  import { EventBusError, EventBusErrorCode } from "./errors/event-bus.error"
5
- import type { BaseEvent, EventEnvelope } from "./event-bus.contracts"
6
+ import type { BaseEvent, DeadLetter, EventEnvelope } from "./event-bus.contracts"
6
7
  import type { EventBus } from "./event-bus.port"
7
- import { callDbFunction } from "@modules/platform/database"
8
+
9
+ /** One row of event_bus_dead_letters: pgmq returns the bigint id as text. */
10
+ interface DeadLetterRow {
11
+ readonly msg_id: string
12
+ readonly message: { readonly reason?: unknown; readonly readCount?: unknown; readonly message?: unknown }
13
+ }
14
+
15
+ /** The one row event_bus_requeue answers. */
16
+ interface RequeueRow {
17
+ readonly event_bus_requeue: boolean | null
18
+ }
8
19
 
9
20
  @Injectable()
10
- /** Writes a receipt event into the same transaction as its inbox receipt, without running any consumer. */
21
+ /** Sends a typed event once to its pgmq topic inside the publisher's transaction; no outbox and no relay stand between them. */
11
22
  export class EventBusService implements EventBus {
12
- /** The outbox commits or rolls back with the receipt; only a later reactor or job processes it. */
23
+ constructor(@InjectPrimaryEntityManager() private readonly entityManager: EntityManager) {}
24
+
25
+ /** The send commits or rolls back with the change; pgmq copies it into every consumer queue bound to the event's name. */
13
26
  async publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void> {
14
27
  if (!isCorrelationId(correlationId)) throw new EventBusError({ code: EventBusErrorCode.CorrelationInvalid })
15
28
  const envelope: EventEnvelope = {
@@ -18,6 +31,21 @@ export class EventBusService implements EventBus {
18
31
  eventName: event.eventName,
19
32
  payload: event.payload,
20
33
  }
21
- await callDbFunction(tx, "event_bus_write", [event.eventId, event.eventName, JSON.stringify(envelope), correlationId])
34
+ await callDbFunction(tx, "public.event_bus_send", [event.eventName, JSON.stringify(envelope)])
35
+ }
36
+
37
+ async deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>> {
38
+ const rows = await callDbFunction<DeadLetterRow>(this.entityManager, "public.event_bus_dead_letters", [queue, limit])
39
+ return rows.map((row) => ({
40
+ id: row.msg_id,
41
+ reason: String(row.message.reason),
42
+ readCount: Number(row.message.readCount),
43
+ message: row.message.message,
44
+ }))
45
+ }
46
+
47
+ async requeue(queue: string, id: string): Promise<boolean> {
48
+ const [row] = await callDbFunction<RequeueRow>(this.entityManager, "public.event_bus_requeue", [queue, id])
49
+ return row?.event_bus_requeue === true
22
50
  }
23
51
  }
@@ -0,0 +1,143 @@
1
+ import { Test } from "@nestjs/testing"
2
+ import { mockEntityManager } from "@starci/jest-preset"
3
+ import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
4
+ import { LOGGER } from "@modules/platform/logging"
5
+ import { BaseEvent } from "./event-bus.contracts"
6
+ import type { EventDelivery, ParsedEvent } from "./event-bus.contracts"
7
+ import { MODULE_OPTIONS_TOKEN } from "./event-bus.module-definition"
8
+ import type { EventConsumer } from "./event-bus.port"
9
+ import { EventConsumerService } from "./event-consumer.service"
10
+
11
+ // The statements callDbFunction of platform/database sends for the bus functions of the migration.
12
+ const READ = 'SELECT * FROM "public"."event_bus_read"($1, $2, $3)'
13
+ const ARCHIVE = 'SELECT * FROM "public"."event_bus_archive"($1, $2)'
14
+ const BURY = 'SELECT * FROM "public"."event_bus_bury"($1, $2, $3, $4, $5)'
15
+ const QUEUE = "spec_delivery_recorded__mailer"
16
+
17
+ class ReceiptRecordedEvent extends BaseEvent {
18
+ static readonly eventName = "spec.delivery-recorded"
19
+ static readonly version = 1
20
+ readonly eventName = ReceiptRecordedEvent.eventName
21
+
22
+ constructor(
23
+ readonly eventId: string,
24
+ readonly payload: { readonly dedupeKey: string; readonly receiptId: string },
25
+ ) {
26
+ super()
27
+ }
28
+
29
+ static parse(envelope: unknown): ParsedEvent<ReceiptRecordedEvent> {
30
+ if (typeof envelope !== "object" || envelope === null || !("eventId" in envelope) || typeof envelope.eventId !== "string") return null
31
+ if (!("payload" in envelope) || typeof envelope.payload !== "object" || envelope.payload === null || !("dedupeKey" in envelope.payload)) return null
32
+ const dedupeKey = envelope.payload.dedupeKey
33
+ return typeof dedupeKey === "string" ? new ReceiptRecordedEvent(envelope.eventId, { dedupeKey, receiptId: dedupeKey }) : null
34
+ }
35
+ }
36
+
37
+ const envelope = {
38
+ metadata: { correlationId: "request-1" },
39
+ eventId: "event-1",
40
+ eventName: "spec.delivery-recorded",
41
+ payload: { dedupeKey: "receipt-1", receiptId: "receipt-1" },
42
+ }
43
+
44
+ class RecordingConsumer implements EventConsumer<ReceiptRecordedEvent> {
45
+ readonly event = ReceiptRecordedEvent
46
+ readonly deliveries: Array<EventDelivery<ReceiptRecordedEvent>> = []
47
+ failing = false
48
+
49
+ handle(delivery: EventDelivery<ReceiptRecordedEvent>): Promise<void> {
50
+ if (this.failing) return Promise.reject(new Error("consumer down"))
51
+ this.deliveries.push(delivery)
52
+ return Promise.resolve()
53
+ }
54
+ }
55
+
56
+ describe("EventConsumerService", () => {
57
+ const logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }
58
+
59
+ const build = async (primary: ReturnType<typeof mockEntityManager>) => {
60
+ const module = await Test.createTestingModule({
61
+ providers: [
62
+ EventConsumerService,
63
+ { provide: PRIMARY_ENTITY_MANAGER, useValue: primary },
64
+ { provide: MODULE_OPTIONS_TOKEN, useValue: {} },
65
+ { provide: LOGGER, useValue: logger },
66
+ ],
67
+ }).compile()
68
+ return module.get(EventConsumerService)
69
+ }
70
+
71
+ it("delivers a committed event once to the consumer with its inbox claim keys and archives it", async () => {
72
+ const primary = mockEntityManager({
73
+ query: [
74
+ [READ, [{ msg_id: "1", read_ct: 1, message: envelope }]],
75
+ [ARCHIVE, [{ event_bus_archive: true }]],
76
+ ],
77
+ })
78
+ const subject = await build(primary)
79
+ const consumer = new RecordingConsumer()
80
+
81
+ await expect(subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })).resolves.toBe(1)
82
+
83
+ expect(consumer.deliveries).toEqual([
84
+ { correlationId: "request-1", eventId: "event-1", consumer: QUEUE, dedupeKey: "receipt-1", event: expect.any(ReceiptRecordedEvent), attempt: 1 },
85
+ ])
86
+ expect(primary.query).toHaveBeenCalledWith(READ, [QUEUE, 30, 10])
87
+ expect(primary.query).toHaveBeenCalledWith(ARCHIVE, [QUEUE, "1"])
88
+ })
89
+
90
+ it("leaves a failed delivery to reappear after its visibility timeout", async () => {
91
+ const primary = mockEntityManager({ query: [READ, [{ msg_id: "1", read_ct: 2, message: envelope }]] })
92
+ const subject = await build(primary)
93
+ const consumer = new RecordingConsumer()
94
+ consumer.failing = true
95
+
96
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
97
+
98
+ expect(primary.query).toHaveBeenCalledTimes(1)
99
+ expect(logger.error).toHaveBeenCalled()
100
+ })
101
+
102
+ it.each([
103
+ ["max-deliveries", { msg_id: "1", read_ct: 4, message: envelope }],
104
+ ["malformed", { msg_id: "1", read_ct: 1, message: { eventId: 42 } }],
105
+ ["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, payload: { dedupeKey: " ", receiptId: "receipt-1" } } }],
106
+ ["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, metadata: { correlationId: "unsafe\nid" } } }],
107
+ ])("moves a delivery to the consumer's dead-letter queue: %s", async (reason, row) => {
108
+ const primary = mockEntityManager({ query: [[READ, [row]], [BURY, [{ event_bus_bury: true }]]] })
109
+ const subject = await build(primary)
110
+ const consumer = new RecordingConsumer()
111
+
112
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 3 })
113
+
114
+ expect(consumer.deliveries).toEqual([])
115
+ expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", row.read_ct, JSON.stringify(row.message), reason])
116
+ })
117
+
118
+ it("takes the default delivery budget of a subscription that names none", async () => {
119
+ const primary = mockEntityManager({ query: [[READ, [{ msg_id: "1", read_ct: 6, message: envelope }]], [BURY, [{ event_bus_bury: true }]]] })
120
+ const subject = await build(primary)
121
+ const consumer = new RecordingConsumer()
122
+ subject.subscribe(QUEUE, consumer)
123
+
124
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
125
+
126
+ expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", 6, JSON.stringify(envelope), "max-deliveries"])
127
+ })
128
+
129
+ it("delivers a redelivery again, leaving its dedupe to the consumer's inbox claim", async () => {
130
+ const primary = mockEntityManager({
131
+ query: [
132
+ [READ, [{ msg_id: "1", read_ct: 2, message: envelope }]],
133
+ [ARCHIVE, [{ event_bus_archive: true }]],
134
+ ],
135
+ })
136
+ const subject = await build(primary)
137
+ const consumer = new RecordingConsumer()
138
+
139
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
140
+
141
+ expect(consumer.deliveries.map((delivery) => [delivery.consumer, delivery.dedupeKey, delivery.attempt])).toEqual([[QUEUE, "receipt-1", 2]])
142
+ })
143
+ })
@@ -0,0 +1,119 @@
1
+ import { Injectable } from "@nestjs/common"
2
+ import type { OnApplicationBootstrap, OnApplicationShutdown } from "@nestjs/common"
3
+ import type { EntityManager } from "typeorm"
4
+ import { InjectLogger, isCorrelationId } from "@modules/platform/logging"
5
+ import type { Logger } from "@modules/platform/logging"
6
+ import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
7
+ import { InjectEventBusOptions } from "./event-bus.decorators"
8
+ import { DEFAULT_MAX_DELIVERIES } from "./event-bus.contracts"
9
+ import type { BaseEvent, EventMessage, EventSubscriptionOptions } from "./event-bus.contracts"
10
+ import { EventBusLogEvent } from "./event-bus.log-events"
11
+ import type { EventBusOptions } from "./event-bus.options"
12
+ import type { EventConsumer, EventConsumerRegistry } from "./event-bus.port"
13
+
14
+ /** One row of event_bus_read: pgmq returns the bigint id as text. */
15
+ interface MessageRow {
16
+ readonly msg_id: string
17
+ readonly read_ct: number
18
+ readonly message: unknown
19
+ }
20
+
21
+ /** One consumer: its queue `<topic>__<consumer>`, the door its deliveries go through and its delivery budget. */
22
+ export interface EventSubscription {
23
+ /** The consumer's pgmq queue. */
24
+ readonly queue: string
25
+ /** The thin door the consumer's deliveries go through. */
26
+ readonly consumer: EventConsumer<BaseEvent>
27
+ /** The deliveries before the message moves to `<queue>__dead`. */
28
+ readonly maxDeliveries: number
29
+ }
30
+
31
+ /** The correlation id of an envelope: its metadata when present and valid, else a safe event id; null refuses the message. */
32
+ const correlationOf = (message: unknown, eventId: string): string | null => {
33
+ if (typeof message !== "object" || message === null || !("metadata" in message)) return isCorrelationId(eventId) ? eventId : null
34
+ const metadata = message.metadata
35
+ if (typeof metadata !== "object" || metadata === null || !("correlationId" in metadata)) return null
36
+ return isCorrelationId(metadata.correlationId) ? metadata.correlationId : null
37
+ }
38
+
39
+ /** True for a usable business dedupe key. */
40
+ const isDedupeKey = (value: unknown): value is string => typeof value === "string" && value.trim().length > 0 && value.length <= 256
41
+
42
+ @Injectable()
43
+ /**
44
+ * The consuming side of the lite event bus: each consumer reads its own pgmq queue with a visibility timeout, archives a message it
45
+ * handled, leaves a failed one to reappear, and moves one past its maxDeliveries, or malformed, to `<queue>__dead`. Delivery is at
46
+ * least once and unordered, so the command a consumer dispatches claims (consumer, dedupeKey) in the inbox first and reads current
47
+ * state rather than trusting the order of events (BE-EVENT-BUS-4).
48
+ */
49
+ export class EventConsumerService implements EventConsumerRegistry, OnApplicationBootstrap, OnApplicationShutdown {
50
+ private readonly subscriptions: Array<EventSubscription> = []
51
+ private running = false
52
+
53
+ constructor(
54
+ @InjectPrimaryEntityManager() private readonly entityManager: EntityManager,
55
+ @InjectEventBusOptions() private readonly options: EventBusOptions,
56
+ @InjectLogger() private readonly logger: Logger,
57
+ ) {}
58
+
59
+ subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options: EventSubscriptionOptions = {}): void {
60
+ this.subscriptions.push({ queue, consumer, maxDeliveries: options.maxDeliveries ?? DEFAULT_MAX_DELIVERIES })
61
+ }
62
+
63
+ onApplicationBootstrap(): void {
64
+ this.running = true
65
+ for (const subscription of this.subscriptions) void this.loop(subscription)
66
+ }
67
+
68
+ onApplicationShutdown(): void {
69
+ this.running = false
70
+ }
71
+
72
+ /** Reads one batch of the consumer's queue and settles each message in order; answers how many it read. */
73
+ async drain(subscription: EventSubscription): Promise<number> {
74
+ const rows = await callDbFunction<MessageRow>(this.entityManager, "public.event_bus_read", [
75
+ subscription.queue,
76
+ this.options.visibilitySeconds ?? 30,
77
+ this.options.batch ?? 10,
78
+ ])
79
+ for (const row of rows) await this.settle(subscription, { msgId: row.msg_id, readCt: row.read_ct, message: row.message })
80
+ return rows.length
81
+ }
82
+
83
+ private async loop(subscription: EventSubscription): Promise<void> {
84
+ while (this.running) {
85
+ const read = await this.drain(subscription).catch((cause: unknown) => {
86
+ this.logger.error(EventBusLogEvent.ReadFailed, cause, { queue: subscription.queue })
87
+ return 0
88
+ })
89
+ if (read === 0) await new Promise((resolve) => setTimeout(resolve, this.options.idleMs ?? 500))
90
+ }
91
+ }
92
+
93
+ private async settle(subscription: EventSubscription, message: EventMessage): Promise<void> {
94
+ const event = subscription.consumer.event.parse(message.message)
95
+ const correlationId = event === null ? null : correlationOf(message.message, event.eventId)
96
+ if (event === null || correlationId === null || !isDedupeKey(event.payload.dedupeKey)) return this.dead(subscription, message, "malformed", null)
97
+ if (message.readCt > subscription.maxDeliveries) return this.dead(subscription, message, "max-deliveries", event.eventName)
98
+ try {
99
+ await subscription.consumer.handle({
100
+ correlationId,
101
+ eventId: event.eventId,
102
+ consumer: subscription.queue,
103
+ dedupeKey: event.payload.dedupeKey,
104
+ event,
105
+ attempt: message.readCt,
106
+ })
107
+ } catch (cause) {
108
+ this.logger.error(EventBusLogEvent.ConsumerFailed, cause, { consumer: subscription.queue, event: event.eventName, attempt: message.readCt })
109
+ return
110
+ }
111
+ await callDbFunction(this.entityManager, "public.event_bus_archive", [subscription.queue, message.msgId])
112
+ }
113
+
114
+ /** Moves the message to `<queue>__dead` and archives the original, in one call. */
115
+ private async dead(subscription: EventSubscription, message: EventMessage, reason: string, eventName: string | null): Promise<void> {
116
+ this.logger.warn(EventBusLogEvent.DeliveryDead, { consumer: subscription.queue, event: eventName, msgId: message.msgId, reason, attempt: message.readCt })
117
+ await callDbFunction(this.entityManager, "public.event_bus_bury", [subscription.queue, message.msgId, message.readCt, JSON.stringify(message.message), reason])
118
+ }
119
+ }