@starci/hfs 4.11.0 → 4.11.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/CHANGELOG.md +85 -0
  2. package/README.md +43 -1
  3. package/db/drift.mjs +8 -6
  4. package/db/image.mjs +9 -4
  5. package/db/migrate.mjs +3 -3
  6. package/package.json +2 -1
  7. package/runtime/knowledge/hfs/canon-pins.yaml +13 -8
  8. package/runtime/knowledge/hfs/rules.yaml +30 -13
  9. package/runtime/knowledge/hfs/slots.yaml +63 -37
  10. package/runtime/knowledge/patterns/be/event-bus.yaml +12 -10
  11. package/runtime/knowledge/patterns/be/queues.yaml +2 -3
  12. package/runtime/knowledge/patterns/be/reactors.yaml +1 -1
  13. package/runtime/modules/kernel/failure-codes.yaml +17 -17
  14. package/runtime/modules/schemas/application-stacks.schema.yaml +4 -2
  15. package/runtime/scripts/api/sops/lib.mjs +30 -0
  16. package/runtime/scripts/api/sops/update-keys.mjs +20 -0
  17. package/runtime/scripts/gates/starcistacks.mjs +6 -2
  18. package/runtime/scripts/guards/push-scan.mjs +1 -1
  19. package/runtime/scripts/hfs/ci-declaration.mjs +3 -2
  20. package/runtime/scripts/hfs/db-baseline-verb.mjs +50 -10
  21. package/runtime/scripts/hfs/db-errors.mjs +2 -0
  22. package/runtime/scripts/hfs/db-plan-verb.mjs +17 -5
  23. package/runtime/scripts/hfs/db-squash.mjs +514 -0
  24. package/runtime/scripts/hfs/declaration-shape.mjs +3 -15
  25. package/runtime/scripts/hfs/example-secrets.mjs +4 -3
  26. package/runtime/scripts/hfs/rule-params-shape.mjs +11 -9
  27. package/runtime/scripts/hfs/rules/a15-findings.mjs +1 -1
  28. package/runtime/scripts/hfs/rules/database-config.mjs +1 -1
  29. package/runtime/scripts/hfs/rules/database-constants.mjs +16 -1
  30. package/runtime/scripts/hfs/rules/database-queue-sources.mjs +2 -110
  31. package/runtime/scripts/hfs/rules/database-queues.mjs +53 -239
  32. package/runtime/scripts/hfs/rules/database-sql.mjs +2 -0
  33. package/runtime/scripts/hfs/rules/database-transaction.mjs +45 -0
  34. package/runtime/scripts/hfs/rules/deploy-scan.mjs +1 -1
  35. package/runtime/scripts/hfs/rules/deploy-workflow.mjs +19 -1
  36. package/runtime/scripts/hfs/rules/edition.mjs +1 -1
  37. package/runtime/scripts/hfs/rules/env-parity.mjs +1 -1
  38. package/runtime/scripts/hfs/rules/lite-secrets.mjs +3 -3
  39. package/runtime/scripts/hfs/rules/script-hygiene.mjs +1 -1
  40. package/runtime/scripts/hfs/rules/secret-custody-only.mjs +2 -2
  41. package/runtime/scripts/hfs/rules/secrets.mjs +1 -1
  42. package/runtime/scripts/hfs/rules/stack-tooling.mjs +1 -1
  43. package/runtime/scripts/hfs/rules/stacks-deploy.mjs +2 -1
  44. package/runtime/scripts/hfs/rules/stacks-lite-backing.mjs +2 -2
  45. package/runtime/scripts/hfs/rules/stacks-runtime.mjs +2 -2
  46. package/runtime/scripts/hfs/rules/stacks-swarm.mjs +1 -1
  47. package/runtime/scripts/hfs/rules/stacks.mjs +3 -3
  48. package/runtime/scripts/hfs/sealed-members.mjs +10 -6
  49. package/runtime/scripts/hfs/secret-files.mjs +16 -0
  50. package/runtime/scripts/hfs/secret-init.mjs +120 -21
  51. package/runtime/scripts/hfs/secret-output.mjs +2 -1
  52. package/runtime/scripts/hfs/secret-prompt.mjs +128 -0
  53. package/runtime/scripts/hfs/secret-sops.mjs +108 -9
  54. package/runtime/scripts/hfs/secret-standard.mjs +369 -0
  55. package/runtime/scripts/hfs/secret-store.mjs +23 -9
  56. package/runtime/scripts/hfs/secret.mjs +140 -46
  57. package/runtime/scripts/hfs/slots.mjs +8 -1
  58. package/runtime/scripts/hfs/stack-deploy-config.mjs +31 -10
  59. package/runtime/scripts/hfs/stacks-gate.mjs +9 -2
  60. package/runtime/scripts/hfs/starcistacks-runtime.mjs +1 -1
  61. package/runtime/scripts/lib/sealed-member.mjs +36 -0
  62. package/runtime/scripts/lib/secret-patterns.mjs +1 -1
  63. package/runtime/scripts/lib/swarm-stack.mjs +2 -2
  64. package/scaffold/add-inbox-lite.mjs +2 -2
  65. package/scaffold/app.mjs +2 -2
  66. package/src/main.mjs +6 -4
  67. package/stack/age-install.mjs +4 -3
  68. package/stack/build.mjs +1 -1
  69. package/stack/config.mjs +2 -1
  70. package/stack/context.mjs +32 -2
  71. package/stack/docker-transport.mjs +2 -1
  72. package/stack/remote.mjs +8 -5
  73. package/stack/sealed.mjs +9 -8
  74. package/stack/server-key.mjs +296 -0
  75. package/stack/ssh-session.mjs +7 -1
  76. package/stack/target-binding.mjs +2 -1
  77. package/sync/hygiene.mjs +1 -1
  78. package/sync/index.mjs +36 -10
  79. package/templates/app/ci-workflows/github/workflows/images.yml +2 -0
  80. package/templates/app/ci-workflows/github/workflows/stack-deploy.yml +3 -1
  81. package/templates/app/ci-workflows/github/workflows/stack-key-restore.yml +58 -0
  82. package/templates/app/ci-workflows-lite/github/workflows/images.yml +2 -0
  83. package/templates/app/ci-workflows-lite/github/workflows/stack-deploy.yml +3 -1
  84. package/templates/app/ci-workflows-lite/github/workflows/stack-key-restore.yml +58 -0
  85. package/templates/app/db-bootstrap/db/bootstrap.sql +7 -4
  86. package/templates/app/db-image/starcistacks/images/db/Dockerfile +18 -0
  87. package/templates/app/gitignore +2 -1
  88. package/templates/app/gitignore-lite +3 -1
  89. package/templates/app/package-scripts/db.json +1 -1
  90. package/templates/app/skeleton-lite/.starcistacks/application-stacks.yaml +1 -1
  91. package/templates/app/skeleton-lite/.starcistacks/dev/README.md +1 -1
  92. package/templates/app/skeleton-lite/.starcistacks/production/README.md +1 -1
  93. package/templates/app/skeleton-lite/README.md +2 -2
  94. package/templates/be/inbox/delivery-recorded.event.ts.tpl +7 -5
  95. package/templates/be/inbox/events.json.tpl +1 -1
  96. package/templates/be/inbox/platform/event-bus/event-bus.contracts.ts.tpl +58 -11
  97. package/templates/be/inbox/platform/event-bus/event-bus.decorators.ts.tpl +13 -1
  98. package/templates/be/inbox/platform/event-bus/event-bus.log-events.ts.tpl +9 -0
  99. package/templates/be/inbox/platform/event-bus/event-bus.module-definition.ts.tpl +1 -1
  100. package/templates/be/inbox/platform/event-bus/event-bus.module.ts.tpl +11 -5
  101. package/templates/be/inbox/platform/event-bus/event-bus.options.ts.tpl +9 -2
  102. package/templates/be/inbox/platform/event-bus/event-bus.port.ts.tpl +18 -5
  103. package/templates/be/inbox/platform/event-bus/event-bus.service.spec.ts.tpl +37 -16
  104. package/templates/be/inbox/platform/event-bus/event-bus.service.ts.tpl +33 -5
  105. package/templates/be/inbox/platform/event-bus/event-consumer.service.spec.ts.tpl +143 -0
  106. package/templates/be/inbox/platform/event-bus/event-consumer.service.ts.tpl +119 -0
  107. package/templates/be/inbox/platform/event-bus/index.ts.tpl +13 -4
  108. package/templates/be/inbox/platform/inbox/inbox.service.spec.ts.tpl +4 -4
  109. package/templates/be/inbox/platform/inbox/inbox.service.ts.tpl +4 -4
  110. package/templates/be/inbox/platform/migration.sql.tpl +97 -34
  111. package/templates/be/inbox/service.spec.ts.tpl +1 -1
  112. package/templates/be/inbox/webhook-door.ts.tpl +2 -2
  113. package/templates/be/patterns/jobs/platform-lite/job-claim.service.spec.ts.tpl +4 -4
  114. package/templates/be/patterns/jobs/platform-lite/job-claim.service.ts.tpl +5 -5
  115. package/templates/be/patterns/jobs/platform-lite/job-tick.service.spec.ts.tpl +1 -1
  116. package/templates/be/patterns/jobs/platform-lite/job-tick.service.ts.tpl +1 -1
  117. package/templates/be/patterns/queues/platform-lite/queue-outbox.service.spec.ts.tpl +1 -1
  118. package/templates/be/patterns/queues/platform-lite/queue-outbox.service.ts.tpl +1 -1
  119. package/templates/be/patterns/queues/platform-lite/queue-relay.service.spec.ts.tpl +2 -2
  120. package/templates/be/patterns/queues/platform-lite/queue-relay.service.ts.tpl +2 -2
  121. package/templates/be/skeleton/apps/__app__/src/__app__.env.ts +4 -0
  122. package/templates/be/skeleton/apps/__app__/src/__app__.options.ts +3 -0
  123. package/templates/be/skeleton/apps/__app__/src/compose/platform.ts +1 -1
  124. package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.spec.ts +38 -3
  125. package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.ts +36 -4
  126. package/templates/be/skeleton/src/modules/platform/http/http.config.ts +11 -0
  127. package/templates/be/skeleton/src/modules/platform/http/http.decorators.ts +7 -0
  128. package/templates/be/skeleton/src/modules/platform/http/http.module-definition.ts +5 -3
  129. package/templates/be/skeleton/src/modules/platform/http/http.options.ts +9 -2
  130. package/templates/be/skeleton/src/modules/platform/http/index.ts +2 -0
  131. package/templates/be/skeleton-lite/apps/api/src/api.env.ts +4 -0
  132. package/templates/be/skeleton-lite/apps/api/src/api.options.ts +3 -0
  133. package/templates/be/skeleton-lite/apps/api/src/compose/platform.ts +1 -1
  134. package/templates/be/skeleton-lite/src/modules/platform/database/database.client.spec.ts +96 -0
  135. package/templates/be/skeleton-lite/src/modules/platform/database/database.client.ts +93 -0
  136. package/templates/be/skeleton-lite/src/modules/platform/database/index.ts +1 -1
  137. package/templates/be/skeleton-lite/src/tests/fixtures/builders/api.builder.ts +1 -0
  138. package/runtime/scripts/hfs/rules/database-queue-outbox.mjs +0 -102
  139. package/templates/app/db-image/db/Dockerfile +0 -8
  140. package/templates/be/skeleton-lite/src/modules/platform/database/database.functions.ts +0 -24
@@ -1,9 +1,21 @@
1
1
  import { injector } from "@modules/platform/composition"
2
2
  import type { TypedParameterDecorator } from "@modules/platform/composition"
3
- import type { EventBus } from "./event-bus.port"
3
+ import { MODULE_OPTIONS_TOKEN } from "./event-bus.module-definition"
4
+ import type { EventBusOptions } from "./event-bus.options"
5
+ import type { EventBus, EventConsumerRegistry } from "./event-bus.port"
4
6
 
5
7
  /** Token of the transactional EventBus publisher. */
6
8
  export const EVENT_BUS: unique symbol = Symbol("platform.event-bus")
7
9
 
10
+ /** Token of the registry reactors subscribe their consumers to. */
11
+ export const EVENT_CONSUMER_REGISTRY: unique symbol = Symbol("platform.event-bus.consumers")
12
+
8
13
  /** Injects the EventBus port. Parameter type: EventBus. */
9
14
  export const InjectEventBus = (): TypedParameterDecorator<EventBus> => injector<EventBus>(EVENT_BUS)
15
+
16
+ /** Injects the consumer registry. Parameter type: EventConsumerRegistry. */
17
+ export const InjectEventConsumerRegistry = (): TypedParameterDecorator<EventConsumerRegistry> =>
18
+ injector<EventConsumerRegistry>(EVENT_CONSUMER_REGISTRY)
19
+
20
+ /** Injects the module's options. Parameter type: EventBusOptions. */
21
+ export const InjectEventBusOptions = (): TypedParameterDecorator<EventBusOptions> => injector<EventBusOptions>(MODULE_OPTIONS_TOKEN)
@@ -0,0 +1,9 @@
1
+ /** Log events of the event bus. */
2
+ export enum EventBusLogEvent {
3
+ /** A consumer threw; the message reappears after its visibility timeout. */
4
+ ConsumerFailed = "event-bus.consumer.failed",
5
+ /** A delivery ran out of deliveries or was malformed and moved to its consumer's dead-letter queue. */
6
+ DeliveryDead = "event-bus.delivery.dead",
7
+ /** A read of a consumer's queue failed; the loop reads again after its idle pause. */
8
+ ReadFailed = "event-bus.read.failed",
9
+ }
@@ -1,6 +1,6 @@
1
1
  import { ConfigurableModuleBuilder } from "@nestjs/common"
2
2
  import type { EventBusOptions } from "./event-bus.options"
3
3
 
4
- export const { ConfigurableModuleClass, OPTIONS_TYPE } = new ConfigurableModuleBuilder<EventBusOptions>()
4
+ export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE } = new ConfigurableModuleBuilder<EventBusOptions>()
5
5
  .setExtras({ isGlobal: false }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
6
6
  .build()
@@ -1,19 +1,25 @@
1
1
  import { Module } from "@nestjs/common"
2
2
  import type { DynamicModule } from "@nestjs/common"
3
- import { EVENT_BUS } from "./event-bus.decorators"
3
+ import { EVENT_BUS, EVENT_CONSUMER_REGISTRY } from "./event-bus.decorators"
4
4
  import { ConfigurableModuleClass, OPTIONS_TYPE } from "./event-bus.module-definition"
5
5
  import { EventBusService } from "./event-bus.service"
6
+ import { EventConsumerService } from "./event-consumer.service"
6
7
 
7
8
  @Module({})
8
- /** Provides the transactional lite EventBus publisher, shared by domain intake services. */
9
+ /** Provides the lite event bus on pgmq: the transactional publisher and the read loops of the consumer groups. */
9
10
  export class EventBusModule extends ConfigurableModuleClass {
10
- /** Registers the publishing capability once per app. */
11
+ /** Registers the capability once per app. */
11
12
  static register(options: typeof OPTIONS_TYPE): DynamicModule {
12
13
  const base = super.register(options)
13
14
  return {
14
15
  ...base,
15
- providers: [...(base.providers ?? []), { provide: EVENT_BUS, useClass: EventBusService }],
16
- exports: [EVENT_BUS],
16
+ providers: [
17
+ ...(base.providers ?? []),
18
+ { provide: EVENT_BUS, useClass: EventBusService },
19
+ EventConsumerService,
20
+ { provide: EVENT_CONSUMER_REGISTRY, useExisting: EventConsumerService },
21
+ ],
22
+ exports: [EVENT_BUS, EVENT_CONSUMER_REGISTRY],
17
23
  }
18
24
  }
19
25
  }
@@ -1,2 +1,9 @@
1
- /** The lite publisher uses the caller's transaction and has no runtime options. */
2
- export interface EventBusOptions {}
1
+ /** How the consumers of the app read their queues; every field has a default, so a bare registration works. */
2
+ export interface EventBusOptions {
3
+ /** Seconds a read message stays hidden: a consumer that fails or dies lets it reappear after this. Default 30. */
4
+ readonly visibilitySeconds?: number
5
+ /** The most messages one read takes from a queue. Default 10. */
6
+ readonly batch?: number
7
+ /** The pause between two reads of an empty queue, in milliseconds. Default 500. */
8
+ readonly idleMs?: number
9
+ }
@@ -1,16 +1,29 @@
1
1
  import type { EntityManager } from "typeorm"
2
- import type { BaseEvent, EventClass, EventDelivery } from "./event-bus.contracts"
2
+ import type { BaseEvent, DeadLetter, EventClass, EventDelivery, EventSubscriptionOptions } from "./event-bus.contracts"
3
3
 
4
- /** The publishing side of the lite event bus; the reactor or job owns delivery of committed outbox rows. */
4
+ /** The publishing and operator side of the lite event bus on pgmq. */
5
5
  export interface EventBus {
6
- /** Writes the typed event in the caller's transaction; a rollback writes no durable event. */
6
+ /** Sends the typed event once to its topic in the caller's transaction; pgmq routes it to every bound consumer queue, and a rollback sends nothing. */
7
7
  publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void>
8
+ /** The oldest deliveries the consumer of `queue` gave up on, at most `limit`, for an operator's cli command. */
9
+ deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>>
10
+ /** Sends one dead letter of `queue` back to that queue; false when it holds no dead letter with the id. */
11
+ requeue(queue: string, id: string): Promise<boolean>
8
12
  }
9
13
 
10
- /** A later message transport receives one typed event and forwards its delivery id. */
14
+ /** A reactor's thin door: it reads one typed event and dispatches one command with the delivery's ids. */
11
15
  export interface EventConsumer<Event extends BaseEvent> {
12
16
  /** The event class the consumer reads. */
13
17
  readonly event: EventClass<Event>
14
- /** Applies one delivery after commit; a throw leaves the delivery available for retry. */
18
+ /** Applies one delivery; a throw leaves the message to reappear after its visibility timeout. */
15
19
  handle(delivery: EventDelivery<Event>): Promise<void>
16
20
  }
21
+
22
+ /** Where a reactor subscribes its consumer; reading starts when the app has bootstrapped. */
23
+ export interface EventConsumerRegistry {
24
+ /**
25
+ * Reads `queue` for `consumer`. The queue is a literal `<TOPIC>_<CONSUMER>_TOPIC_QUEUE` constant of platform/event-bus whose
26
+ * pgmq queue, dead-letter queue and topic binding a migration declares (R258); the subscription is static (R336).
27
+ */
28
+ subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options?: EventSubscriptionOptions): void
29
+ }
@@ -1,27 +1,33 @@
1
1
  import { Test } from "@nestjs/testing"
2
2
  import { fakeTransaction, mockEntityManager } from "@starci/jest-preset"
3
+ import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
3
4
  import { EventBusErrorCode } from "./errors/event-bus.error"
4
5
  import { BaseEvent } from "./event-bus.contracts"
5
6
  import { EventBusService } from "./event-bus.service"
6
7
 
7
- // The statement callDbFunction of platform/database sends for event_bus_write.
8
- const INSERT_OUTBOX_EVENT = "SELECT * FROM event_bus_write($1, $2, $3, $4)"
8
+ // The statements callDbFunction of platform/database sends for the bus functions of the migration.
9
+ const SEND = 'SELECT * FROM "public"."event_bus_send"($1, $2)'
10
+ const DEAD_LETTERS = 'SELECT * FROM "public"."event_bus_dead_letters"($1, $2)'
11
+ const REQUEUE = 'SELECT * FROM "public"."event_bus_requeue"($1, $2)'
12
+ const QUEUE = "spec_delivery_recorded__mailer"
9
13
 
10
14
  class ReceiptRecordedEvent extends BaseEvent {
11
- readonly eventId = "receipt-1"
15
+ readonly eventId = "event-1"
12
16
  readonly eventName = "spec.delivery-recorded"
13
- readonly payload = { receiptId: "receipt-1" }
17
+ readonly payload = { dedupeKey: "receipt-1", receiptId: "receipt-1" }
14
18
  }
15
19
 
16
20
  describe("EventBusService", () => {
17
- const build = async () => {
18
- const module = await Test.createTestingModule({ providers: [EventBusService] }).compile()
21
+ const build = async (primary = mockEntityManager({})) => {
22
+ const module = await Test.createTestingModule({
23
+ providers: [EventBusService, { provide: PRIMARY_ENTITY_MANAGER, useValue: primary }],
24
+ }).compile()
19
25
  return module.get(EventBusService)
20
26
  }
21
27
 
22
28
  it.each(["", "unsafe\nid", "x".repeat(129)])("refuses invalid correlation before SQL: %s", async (correlationId) => {
23
29
  const subject = await build()
24
- const manager = mockEntityManager({ query: [INSERT_OUTBOX_EVENT, []] })
30
+ const manager = mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] })
25
31
 
26
32
  await expect(subject.publish(new ReceiptRecordedEvent(), manager, correlationId)).rejects.toMatchObject({
27
33
  code: EventBusErrorCode.CorrelationInvalid,
@@ -29,31 +35,46 @@ describe("EventBusService", () => {
29
35
  expect(manager.query).not.toHaveBeenCalled()
30
36
  })
31
37
 
32
- it("writes the typed envelope through the receipt transaction", async () => {
38
+ it("sends the typed envelope to the event's topic through the publisher's transaction", async () => {
33
39
  const subject = await build()
34
- const transaction = fakeTransaction(mockEntityManager({ query: [INSERT_OUTBOX_EVENT, []] }))
40
+ const transaction = fakeTransaction(mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] }))
35
41
  const event = new ReceiptRecordedEvent()
36
42
 
37
43
  await transaction.em.transaction((manager) => subject.publish(event, manager, "request-1"))
38
44
 
39
- expect(transaction.em.query).toHaveBeenCalledWith(INSERT_OUTBOX_EVENT, [
40
- "receipt-1",
45
+ expect(transaction.em.query).toHaveBeenCalledWith(SEND, [
41
46
  "spec.delivery-recorded",
42
47
  JSON.stringify({ metadata: { correlationId: "request-1" }, eventId: event.eventId, eventName: event.eventName, payload: event.payload }),
43
- "request-1",
44
48
  ])
45
49
  expect(transaction.commits).toBe(1)
46
50
  })
47
51
 
48
- it("propagates an outbox failure so the receipt transaction rolls back", async () => {
52
+ it("propagates a send failure so the publisher's transaction rolls back and nothing is sent", async () => {
49
53
  const subject = await build()
50
- const manager = mockEntityManager({ query: [INSERT_OUTBOX_EVENT, []] })
51
- jest.spyOn(manager, "query").mockRejectedValue(new Error("outbox unavailable"))
54
+ const manager = mockEntityManager({ query: [SEND, []] })
55
+ jest.spyOn(manager, "query").mockRejectedValue(new Error("pgmq unavailable"))
52
56
  const transaction = fakeTransaction(manager)
53
57
 
54
- await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("outbox unavailable")
58
+ await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("pgmq unavailable")
55
59
 
56
60
  expect(transaction.commits).toBe(0)
57
61
  expect(transaction.rollbacks).toBe(1)
58
62
  })
63
+
64
+ it("lists a consumer's dead letters and requeues one by its dead-letter id", async () => {
65
+ const primary = mockEntityManager({
66
+ query: [
67
+ [DEAD_LETTERS, [{ msg_id: "7", message: { reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } } }]],
68
+ [REQUEUE, [{ event_bus_requeue: true }]],
69
+ ],
70
+ })
71
+ const subject = await build(primary)
72
+
73
+ await expect(subject.deadLetters(QUEUE, 10)).resolves.toEqual([
74
+ { id: "7", reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } },
75
+ ])
76
+ await expect(subject.requeue(QUEUE, "7")).resolves.toBe(true)
77
+ expect(primary.query).toHaveBeenCalledWith(DEAD_LETTERS, [QUEUE, 10])
78
+ expect(primary.query).toHaveBeenCalledWith(REQUEUE, [QUEUE, "7"])
79
+ })
59
80
  })
@@ -1,15 +1,28 @@
1
1
  import { Injectable } from "@nestjs/common"
2
2
  import type { EntityManager } from "typeorm"
3
3
  import { isCorrelationId } from "@modules/platform/logging"
4
+ import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
4
5
  import { EventBusError, EventBusErrorCode } from "./errors/event-bus.error"
5
- import type { BaseEvent, EventEnvelope } from "./event-bus.contracts"
6
+ import type { BaseEvent, DeadLetter, EventEnvelope } from "./event-bus.contracts"
6
7
  import type { EventBus } from "./event-bus.port"
7
- import { callDbFunction } from "@modules/platform/database"
8
+
9
+ /** One row of event_bus_dead_letters: pgmq returns the bigint id as text. */
10
+ interface DeadLetterRow {
11
+ readonly msg_id: string
12
+ readonly message: { readonly reason?: unknown; readonly readCount?: unknown; readonly message?: unknown }
13
+ }
14
+
15
+ /** The one row event_bus_requeue answers. */
16
+ interface RequeueRow {
17
+ readonly event_bus_requeue: boolean | null
18
+ }
8
19
 
9
20
  @Injectable()
10
- /** Writes a receipt event into the same transaction as its inbox receipt, without running any consumer. */
21
+ /** Sends a typed event once to its pgmq topic inside the publisher's transaction; no outbox and no relay stand between them. */
11
22
  export class EventBusService implements EventBus {
12
- /** The outbox commits or rolls back with the receipt; only a later reactor or job processes it. */
23
+ constructor(@InjectPrimaryEntityManager() private readonly entityManager: EntityManager) {}
24
+
25
+ /** The send commits or rolls back with the change; pgmq copies it into every consumer queue bound to the event's name. */
13
26
  async publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void> {
14
27
  if (!isCorrelationId(correlationId)) throw new EventBusError({ code: EventBusErrorCode.CorrelationInvalid })
15
28
  const envelope: EventEnvelope = {
@@ -18,6 +31,21 @@ export class EventBusService implements EventBus {
18
31
  eventName: event.eventName,
19
32
  payload: event.payload,
20
33
  }
21
- await callDbFunction(tx, "event_bus_write", [event.eventId, event.eventName, JSON.stringify(envelope), correlationId])
34
+ await callDbFunction(tx, "public.event_bus_send", [event.eventName, JSON.stringify(envelope)])
35
+ }
36
+
37
+ async deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>> {
38
+ const rows = await callDbFunction<DeadLetterRow>(this.entityManager, "public.event_bus_dead_letters", [queue, limit])
39
+ return rows.map((row) => ({
40
+ id: row.msg_id,
41
+ reason: String(row.message.reason),
42
+ readCount: Number(row.message.readCount),
43
+ message: row.message.message,
44
+ }))
45
+ }
46
+
47
+ async requeue(queue: string, id: string): Promise<boolean> {
48
+ const [row] = await callDbFunction<RequeueRow>(this.entityManager, "public.event_bus_requeue", [queue, id])
49
+ return row?.event_bus_requeue === true
22
50
  }
23
51
  }
@@ -0,0 +1,143 @@
1
+ import { Test } from "@nestjs/testing"
2
+ import { mockEntityManager } from "@starci/jest-preset"
3
+ import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
4
+ import { LOGGER } from "@modules/platform/logging"
5
+ import { BaseEvent } from "./event-bus.contracts"
6
+ import type { EventDelivery, ParsedEvent } from "./event-bus.contracts"
7
+ import { MODULE_OPTIONS_TOKEN } from "./event-bus.module-definition"
8
+ import type { EventConsumer } from "./event-bus.port"
9
+ import { EventConsumerService } from "./event-consumer.service"
10
+
11
+ // The statements callDbFunction of platform/database sends for the bus functions of the migration.
12
+ const READ = 'SELECT * FROM "public"."event_bus_read"($1, $2, $3)'
13
+ const ARCHIVE = 'SELECT * FROM "public"."event_bus_archive"($1, $2)'
14
+ const BURY = 'SELECT * FROM "public"."event_bus_bury"($1, $2, $3, $4, $5)'
15
+ const QUEUE = "spec_delivery_recorded__mailer"
16
+
17
+ class ReceiptRecordedEvent extends BaseEvent {
18
+ static readonly eventName = "spec.delivery-recorded"
19
+ static readonly version = 1
20
+ readonly eventName = ReceiptRecordedEvent.eventName
21
+
22
+ constructor(
23
+ readonly eventId: string,
24
+ readonly payload: { readonly dedupeKey: string; readonly receiptId: string },
25
+ ) {
26
+ super()
27
+ }
28
+
29
+ static parse(envelope: unknown): ParsedEvent<ReceiptRecordedEvent> {
30
+ if (typeof envelope !== "object" || envelope === null || !("eventId" in envelope) || typeof envelope.eventId !== "string") return null
31
+ if (!("payload" in envelope) || typeof envelope.payload !== "object" || envelope.payload === null || !("dedupeKey" in envelope.payload)) return null
32
+ const dedupeKey = envelope.payload.dedupeKey
33
+ return typeof dedupeKey === "string" ? new ReceiptRecordedEvent(envelope.eventId, { dedupeKey, receiptId: dedupeKey }) : null
34
+ }
35
+ }
36
+
37
+ const envelope = {
38
+ metadata: { correlationId: "request-1" },
39
+ eventId: "event-1",
40
+ eventName: "spec.delivery-recorded",
41
+ payload: { dedupeKey: "receipt-1", receiptId: "receipt-1" },
42
+ }
43
+
44
+ class RecordingConsumer implements EventConsumer<ReceiptRecordedEvent> {
45
+ readonly event = ReceiptRecordedEvent
46
+ readonly deliveries: Array<EventDelivery<ReceiptRecordedEvent>> = []
47
+ failing = false
48
+
49
+ handle(delivery: EventDelivery<ReceiptRecordedEvent>): Promise<void> {
50
+ if (this.failing) return Promise.reject(new Error("consumer down"))
51
+ this.deliveries.push(delivery)
52
+ return Promise.resolve()
53
+ }
54
+ }
55
+
56
+ describe("EventConsumerService", () => {
57
+ const logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }
58
+
59
+ const build = async (primary: ReturnType<typeof mockEntityManager>) => {
60
+ const module = await Test.createTestingModule({
61
+ providers: [
62
+ EventConsumerService,
63
+ { provide: PRIMARY_ENTITY_MANAGER, useValue: primary },
64
+ { provide: MODULE_OPTIONS_TOKEN, useValue: {} },
65
+ { provide: LOGGER, useValue: logger },
66
+ ],
67
+ }).compile()
68
+ return module.get(EventConsumerService)
69
+ }
70
+
71
+ it("delivers a committed event once to the consumer with its inbox claim keys and archives it", async () => {
72
+ const primary = mockEntityManager({
73
+ query: [
74
+ [READ, [{ msg_id: "1", read_ct: 1, message: envelope }]],
75
+ [ARCHIVE, [{ event_bus_archive: true }]],
76
+ ],
77
+ })
78
+ const subject = await build(primary)
79
+ const consumer = new RecordingConsumer()
80
+
81
+ await expect(subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })).resolves.toBe(1)
82
+
83
+ expect(consumer.deliveries).toEqual([
84
+ { correlationId: "request-1", eventId: "event-1", consumer: QUEUE, dedupeKey: "receipt-1", event: expect.any(ReceiptRecordedEvent), attempt: 1 },
85
+ ])
86
+ expect(primary.query).toHaveBeenCalledWith(READ, [QUEUE, 30, 10])
87
+ expect(primary.query).toHaveBeenCalledWith(ARCHIVE, [QUEUE, "1"])
88
+ })
89
+
90
+ it("leaves a failed delivery to reappear after its visibility timeout", async () => {
91
+ const primary = mockEntityManager({ query: [READ, [{ msg_id: "1", read_ct: 2, message: envelope }]] })
92
+ const subject = await build(primary)
93
+ const consumer = new RecordingConsumer()
94
+ consumer.failing = true
95
+
96
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
97
+
98
+ expect(primary.query).toHaveBeenCalledTimes(1)
99
+ expect(logger.error).toHaveBeenCalled()
100
+ })
101
+
102
+ it.each([
103
+ ["max-deliveries", { msg_id: "1", read_ct: 4, message: envelope }],
104
+ ["malformed", { msg_id: "1", read_ct: 1, message: { eventId: 42 } }],
105
+ ["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, payload: { dedupeKey: " ", receiptId: "receipt-1" } } }],
106
+ ["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, metadata: { correlationId: "unsafe\nid" } } }],
107
+ ])("moves a delivery to the consumer's dead-letter queue: %s", async (reason, row) => {
108
+ const primary = mockEntityManager({ query: [[READ, [row]], [BURY, [{ event_bus_bury: true }]]] })
109
+ const subject = await build(primary)
110
+ const consumer = new RecordingConsumer()
111
+
112
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 3 })
113
+
114
+ expect(consumer.deliveries).toEqual([])
115
+ expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", row.read_ct, JSON.stringify(row.message), reason])
116
+ })
117
+
118
+ it("takes the default delivery budget of a subscription that names none", async () => {
119
+ const primary = mockEntityManager({ query: [[READ, [{ msg_id: "1", read_ct: 6, message: envelope }]], [BURY, [{ event_bus_bury: true }]]] })
120
+ const subject = await build(primary)
121
+ const consumer = new RecordingConsumer()
122
+ subject.subscribe(QUEUE, consumer)
123
+
124
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
125
+
126
+ expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", 6, JSON.stringify(envelope), "max-deliveries"])
127
+ })
128
+
129
+ it("delivers a redelivery again, leaving its dedupe to the consumer's inbox claim", async () => {
130
+ const primary = mockEntityManager({
131
+ query: [
132
+ [READ, [{ msg_id: "1", read_ct: 2, message: envelope }]],
133
+ [ARCHIVE, [{ event_bus_archive: true }]],
134
+ ],
135
+ })
136
+ const subject = await build(primary)
137
+ const consumer = new RecordingConsumer()
138
+
139
+ await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
140
+
141
+ expect(consumer.deliveries.map((delivery) => [delivery.consumer, delivery.dedupeKey, delivery.attempt])).toEqual([[QUEUE, "receipt-1", 2]])
142
+ })
143
+ })
@@ -0,0 +1,119 @@
1
+ import { Injectable } from "@nestjs/common"
2
+ import type { OnApplicationBootstrap, OnApplicationShutdown } from "@nestjs/common"
3
+ import type { EntityManager } from "typeorm"
4
+ import { InjectLogger, isCorrelationId } from "@modules/platform/logging"
5
+ import type { Logger } from "@modules/platform/logging"
6
+ import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
7
+ import { InjectEventBusOptions } from "./event-bus.decorators"
8
+ import { DEFAULT_MAX_DELIVERIES } from "./event-bus.contracts"
9
+ import type { BaseEvent, EventMessage, EventSubscriptionOptions } from "./event-bus.contracts"
10
+ import { EventBusLogEvent } from "./event-bus.log-events"
11
+ import type { EventBusOptions } from "./event-bus.options"
12
+ import type { EventConsumer, EventConsumerRegistry } from "./event-bus.port"
13
+
14
+ /** One row of event_bus_read: pgmq returns the bigint id as text. */
15
+ interface MessageRow {
16
+ readonly msg_id: string
17
+ readonly read_ct: number
18
+ readonly message: unknown
19
+ }
20
+
21
+ /** One consumer: its queue `<topic>__<consumer>`, the door its deliveries go through and its delivery budget. */
22
+ export interface EventSubscription {
23
+ /** The consumer's pgmq queue. */
24
+ readonly queue: string
25
+ /** The thin door the consumer's deliveries go through. */
26
+ readonly consumer: EventConsumer<BaseEvent>
27
+ /** The deliveries before the message moves to `<queue>__dead`. */
28
+ readonly maxDeliveries: number
29
+ }
30
+
31
+ /** The correlation id of an envelope: its metadata when present and valid, else a safe event id; null refuses the message. */
32
+ const correlationOf = (message: unknown, eventId: string): string | null => {
33
+ if (typeof message !== "object" || message === null || !("metadata" in message)) return isCorrelationId(eventId) ? eventId : null
34
+ const metadata = message.metadata
35
+ if (typeof metadata !== "object" || metadata === null || !("correlationId" in metadata)) return null
36
+ return isCorrelationId(metadata.correlationId) ? metadata.correlationId : null
37
+ }
38
+
39
+ /** True for a usable business dedupe key. */
40
+ const isDedupeKey = (value: unknown): value is string => typeof value === "string" && value.trim().length > 0 && value.length <= 256
41
+
42
+ @Injectable()
43
+ /**
44
+ * The consuming side of the lite event bus: each consumer reads its own pgmq queue with a visibility timeout, archives a message it
45
+ * handled, leaves a failed one to reappear, and moves one past its maxDeliveries, or malformed, to `<queue>__dead`. Delivery is at
46
+ * least once and unordered, so the command a consumer dispatches claims (consumer, dedupeKey) in the inbox first and reads current
47
+ * state rather than trusting the order of events (BE-EVENT-BUS-4).
48
+ */
49
+ export class EventConsumerService implements EventConsumerRegistry, OnApplicationBootstrap, OnApplicationShutdown {
50
+ private readonly subscriptions: Array<EventSubscription> = []
51
+ private running = false
52
+
53
+ constructor(
54
+ @InjectPrimaryEntityManager() private readonly entityManager: EntityManager,
55
+ @InjectEventBusOptions() private readonly options: EventBusOptions,
56
+ @InjectLogger() private readonly logger: Logger,
57
+ ) {}
58
+
59
+ subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options: EventSubscriptionOptions = {}): void {
60
+ this.subscriptions.push({ queue, consumer, maxDeliveries: options.maxDeliveries ?? DEFAULT_MAX_DELIVERIES })
61
+ }
62
+
63
+ onApplicationBootstrap(): void {
64
+ this.running = true
65
+ for (const subscription of this.subscriptions) void this.loop(subscription)
66
+ }
67
+
68
+ onApplicationShutdown(): void {
69
+ this.running = false
70
+ }
71
+
72
+ /** Reads one batch of the consumer's queue and settles each message in order; answers how many it read. */
73
+ async drain(subscription: EventSubscription): Promise<number> {
74
+ const rows = await callDbFunction<MessageRow>(this.entityManager, "public.event_bus_read", [
75
+ subscription.queue,
76
+ this.options.visibilitySeconds ?? 30,
77
+ this.options.batch ?? 10,
78
+ ])
79
+ for (const row of rows) await this.settle(subscription, { msgId: row.msg_id, readCt: row.read_ct, message: row.message })
80
+ return rows.length
81
+ }
82
+
83
+ private async loop(subscription: EventSubscription): Promise<void> {
84
+ while (this.running) {
85
+ const read = await this.drain(subscription).catch((cause: unknown) => {
86
+ this.logger.error(EventBusLogEvent.ReadFailed, cause, { queue: subscription.queue })
87
+ return 0
88
+ })
89
+ if (read === 0) await new Promise((resolve) => setTimeout(resolve, this.options.idleMs ?? 500))
90
+ }
91
+ }
92
+
93
+ private async settle(subscription: EventSubscription, message: EventMessage): Promise<void> {
94
+ const event = subscription.consumer.event.parse(message.message)
95
+ const correlationId = event === null ? null : correlationOf(message.message, event.eventId)
96
+ if (event === null || correlationId === null || !isDedupeKey(event.payload.dedupeKey)) return this.dead(subscription, message, "malformed", null)
97
+ if (message.readCt > subscription.maxDeliveries) return this.dead(subscription, message, "max-deliveries", event.eventName)
98
+ try {
99
+ await subscription.consumer.handle({
100
+ correlationId,
101
+ eventId: event.eventId,
102
+ consumer: subscription.queue,
103
+ dedupeKey: event.payload.dedupeKey,
104
+ event,
105
+ attempt: message.readCt,
106
+ })
107
+ } catch (cause) {
108
+ this.logger.error(EventBusLogEvent.ConsumerFailed, cause, { consumer: subscription.queue, event: event.eventName, attempt: message.readCt })
109
+ return
110
+ }
111
+ await callDbFunction(this.entityManager, "public.event_bus_archive", [subscription.queue, message.msgId])
112
+ }
113
+
114
+ /** Moves the message to `<queue>__dead` and archives the original, in one call. */
115
+ private async dead(subscription: EventSubscription, message: EventMessage, reason: string, eventName: string | null): Promise<void> {
116
+ this.logger.warn(EventBusLogEvent.DeliveryDead, { consumer: subscription.queue, event: eventName, msgId: message.msgId, reason, attempt: message.readCt })
117
+ await callDbFunction(this.entityManager, "public.event_bus_bury", [subscription.queue, message.msgId, message.readCt, JSON.stringify(message.message), reason])
118
+ }
119
+ }
@@ -1,7 +1,16 @@
1
1
  export { EVENT_BUS_ERROR_KINDS } from "./errors/event-bus.error"
2
2
  export { EVENT_BUS_MESSAGES } from "./messages/event-bus.messages"
3
- export { BaseEvent } from "./event-bus.contracts"
4
- export type { EventClass, EventDelivery, EventEnvelope, ParsedEvent } from "./event-bus.contracts"
5
- export { EVENT_BUS, InjectEventBus } from "./event-bus.decorators"
3
+ export { BaseEvent, DEAD_QUEUE_SUFFIX, DEFAULT_MAX_DELIVERIES } from "./event-bus.contracts"
4
+ export type {
5
+ DeadLetter,
6
+ EventClass,
7
+ EventDelivery,
8
+ EventEnvelope,
9
+ EventPayload,
10
+ EventSubscriptionOptions,
11
+ ParsedEvent,
12
+ } from "./event-bus.contracts"
13
+ export { EVENT_BUS, EVENT_CONSUMER_REGISTRY, InjectEventBus, InjectEventConsumerRegistry } from "./event-bus.decorators"
6
14
  export { EventBusModule } from "./event-bus.module"
7
- export type { EventBus, EventConsumer } from "./event-bus.port"
15
+ export type { EventBusOptions } from "./event-bus.options"
16
+ export type { EventBus, EventConsumer, EventConsumerRegistry } from "./event-bus.port"
@@ -9,10 +9,10 @@ import { requestFingerprint } from "./request-fingerprint.policy"
9
9
  import { PostgresInbox } from "./inbox.service"
10
10
 
11
11
  // The statements callDbFunction of platform/database sends for each inbox function.
12
- const CLAIM_EVENT = "SELECT * FROM inbox_claim_event($1, $2)"
13
- const CLAIM_CLIENT = "SELECT * FROM inbox_claim_client($1, $2, $3, $4)"
14
- const SAVE_CLIENT_OUTCOME = "SELECT * FROM inbox_save_client_outcome($1, $2, $3)"
15
- const SWEEP_CLIENTS = "SELECT * FROM inbox_sweep_clients($1, $2)"
12
+ const CLAIM_EVENT = 'SELECT * FROM "public"."inbox_claim_event"($1, $2)'
13
+ const CLAIM_CLIENT = 'SELECT * FROM "public"."inbox_claim_client"($1, $2, $3, $4)'
14
+ const SAVE_CLIENT_OUTCOME = 'SELECT * FROM "public"."inbox_save_client_outcome"($1, $2, $3)'
15
+ const SWEEP_CLIENTS = 'SELECT * FROM "public"."inbox_sweep_clients"($1, $2)'
16
16
 
17
17
  const runClient = <T>(
18
18
  inbox: PostgresInbox,
@@ -54,7 +54,7 @@ export class PostgresInbox implements Inbox {
54
54
  if (!normalizedEventId) {
55
55
  throw new DatabaseError({ code: DatabaseErrorCode.WebhookDeliveryIdRequired })
56
56
  }
57
- const rows = await callDbFunction<InboxEventRow>(tx, "inbox_claim_event", [source, normalizedEventId])
57
+ const rows = await callDbFunction<InboxEventRow>(tx, "public.inbox_claim_event", [source, normalizedEventId])
58
58
  return rows.length > 0
59
59
  }
60
60
 
@@ -64,7 +64,7 @@ export class PostgresInbox implements Inbox {
64
64
  throw new InboxError({ code: InboxErrorCode.TransactionRequired })
65
65
  const source = clientSource(request)
66
66
  const fingerprint = requestFingerprint(request.request)
67
- const rows = await callDbFunction<ClaimClientRow<T>>(tx, "inbox_claim_client", [
67
+ const rows = await callDbFunction<ClaimClientRow<T>>(tx, "public.inbox_claim_client", [
68
68
  source,
69
69
  request.idempotencyKey,
70
70
  fingerprint,
@@ -84,7 +84,7 @@ export class PostgresInbox implements Inbox {
84
84
  } catch (cause) {
85
85
  throw new InboxError({ code: InboxErrorCode.OutcomeInvalid, cause })
86
86
  }
87
- const saved = await callDbFunction<SavedClientOutcomeRow<T>>(tx, "inbox_save_client_outcome", [
87
+ const saved = await callDbFunction<SavedClientOutcomeRow<T>>(tx, "public.inbox_save_client_outcome", [
88
88
  source,
89
89
  request.idempotencyKey,
90
90
  outcome,
@@ -96,7 +96,7 @@ export class PostgresInbox implements Inbox {
96
96
 
97
97
  /** A row lock and SKIP LOCKED keep retention from competing with a live retry or another sweep worker. */
98
98
  async sweepClients(cutoff: Date, tx: EntityManager): Promise<number> {
99
- const rows = await callDbFunction<InboxSweepRow>(tx, "inbox_sweep_clients", [cutoff, BATCH_ROWS])
99
+ const rows = await callDbFunction<InboxSweepRow>(tx, "public.inbox_sweep_clients", [cutoff, BATCH_ROWS])
100
100
  return rows[0]?.deleted_count ?? 0
101
101
  }
102
102
  }