@starci/hfs 4.11.0 → 4.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +40 -1
- package/db/drift.mjs +8 -6
- package/db/image.mjs +9 -4
- package/db/migrate.mjs +3 -3
- package/package.json +2 -1
- package/runtime/knowledge/hfs/canon-pins.yaml +13 -8
- package/runtime/knowledge/hfs/rules.yaml +29 -12
- package/runtime/knowledge/hfs/slots.yaml +48 -37
- package/runtime/knowledge/patterns/be/event-bus.yaml +12 -10
- package/runtime/knowledge/patterns/be/queues.yaml +2 -3
- package/runtime/knowledge/patterns/be/reactors.yaml +1 -1
- package/runtime/modules/kernel/failure-codes.yaml +17 -17
- package/runtime/modules/schemas/application-stacks.schema.yaml +4 -2
- package/runtime/scripts/gates/starcistacks.mjs +6 -2
- package/runtime/scripts/guards/push-scan.mjs +1 -1
- package/runtime/scripts/hfs/ci-declaration.mjs +3 -2
- package/runtime/scripts/hfs/db-baseline-verb.mjs +50 -10
- package/runtime/scripts/hfs/db-errors.mjs +2 -0
- package/runtime/scripts/hfs/db-plan-verb.mjs +17 -5
- package/runtime/scripts/hfs/db-squash.mjs +514 -0
- package/runtime/scripts/hfs/declaration-shape.mjs +3 -15
- package/runtime/scripts/hfs/example-secrets.mjs +4 -3
- package/runtime/scripts/hfs/rule-params-shape.mjs +11 -9
- package/runtime/scripts/hfs/rules/a15-findings.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-config.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-constants.mjs +16 -1
- package/runtime/scripts/hfs/rules/database-queue-sources.mjs +2 -110
- package/runtime/scripts/hfs/rules/database-queues.mjs +53 -239
- package/runtime/scripts/hfs/rules/database-sql.mjs +2 -0
- package/runtime/scripts/hfs/rules/database-transaction.mjs +45 -0
- package/runtime/scripts/hfs/rules/deploy-scan.mjs +1 -1
- package/runtime/scripts/hfs/rules/edition.mjs +1 -1
- package/runtime/scripts/hfs/rules/env-parity.mjs +1 -1
- package/runtime/scripts/hfs/rules/lite-secrets.mjs +3 -3
- package/runtime/scripts/hfs/rules/script-hygiene.mjs +1 -1
- package/runtime/scripts/hfs/rules/secret-custody-only.mjs +2 -2
- package/runtime/scripts/hfs/rules/secrets.mjs +1 -1
- package/runtime/scripts/hfs/rules/stack-tooling.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks-deploy.mjs +2 -1
- package/runtime/scripts/hfs/rules/stacks-lite-backing.mjs +2 -2
- package/runtime/scripts/hfs/rules/stacks-runtime.mjs +2 -2
- package/runtime/scripts/hfs/rules/stacks-swarm.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks.mjs +3 -3
- package/runtime/scripts/hfs/sealed-members.mjs +10 -6
- package/runtime/scripts/hfs/secret-files.mjs +16 -0
- package/runtime/scripts/hfs/secret-init.mjs +120 -21
- package/runtime/scripts/hfs/secret-output.mjs +2 -1
- package/runtime/scripts/hfs/secret-prompt.mjs +128 -0
- package/runtime/scripts/hfs/secret-sops.mjs +93 -9
- package/runtime/scripts/hfs/secret-standard.mjs +315 -0
- package/runtime/scripts/hfs/secret-store.mjs +23 -9
- package/runtime/scripts/hfs/secret.mjs +110 -46
- package/runtime/scripts/hfs/slots.mjs +8 -1
- package/runtime/scripts/hfs/stack-deploy-config.mjs +20 -6
- package/runtime/scripts/hfs/stacks-gate.mjs +9 -2
- package/runtime/scripts/hfs/starcistacks-runtime.mjs +1 -1
- package/runtime/scripts/lib/sealed-member.mjs +36 -0
- package/runtime/scripts/lib/secret-patterns.mjs +1 -1
- package/runtime/scripts/lib/swarm-stack.mjs +2 -2
- package/scaffold/add-inbox-lite.mjs +2 -2
- package/scaffold/app.mjs +2 -2
- package/src/main.mjs +5 -4
- package/stack/build.mjs +1 -1
- package/stack/config.mjs +2 -1
- package/stack/docker-transport.mjs +2 -1
- package/stack/remote.mjs +4 -3
- package/stack/sealed.mjs +9 -8
- package/stack/target-binding.mjs +2 -1
- package/sync/hygiene.mjs +1 -1
- package/sync/index.mjs +33 -10
- package/templates/app/db-bootstrap/db/bootstrap.sql +7 -4
- package/templates/app/db-image/starcistacks/images/db/Dockerfile +18 -0
- package/templates/app/gitignore +2 -1
- package/templates/app/gitignore-lite +3 -1
- package/templates/app/package-scripts/db.json +1 -1
- package/templates/app/skeleton-lite/.starcistacks/application-stacks.yaml +1 -1
- package/templates/app/skeleton-lite/.starcistacks/dev/README.md +1 -1
- package/templates/app/skeleton-lite/.starcistacks/production/README.md +1 -1
- package/templates/app/skeleton-lite/README.md +2 -2
- package/templates/be/inbox/delivery-recorded.event.ts.tpl +7 -5
- package/templates/be/inbox/events.json.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.contracts.ts.tpl +58 -11
- package/templates/be/inbox/platform/event-bus/event-bus.decorators.ts.tpl +13 -1
- package/templates/be/inbox/platform/event-bus/event-bus.log-events.ts.tpl +9 -0
- package/templates/be/inbox/platform/event-bus/event-bus.module-definition.ts.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.module.ts.tpl +11 -5
- package/templates/be/inbox/platform/event-bus/event-bus.options.ts.tpl +9 -2
- package/templates/be/inbox/platform/event-bus/event-bus.port.ts.tpl +18 -5
- package/templates/be/inbox/platform/event-bus/event-bus.service.spec.ts.tpl +37 -16
- package/templates/be/inbox/platform/event-bus/event-bus.service.ts.tpl +33 -5
- package/templates/be/inbox/platform/event-bus/event-consumer.service.spec.ts.tpl +143 -0
- package/templates/be/inbox/platform/event-bus/event-consumer.service.ts.tpl +119 -0
- package/templates/be/inbox/platform/event-bus/index.ts.tpl +13 -4
- package/templates/be/inbox/platform/inbox/inbox.service.spec.ts.tpl +4 -4
- package/templates/be/inbox/platform/inbox/inbox.service.ts.tpl +4 -4
- package/templates/be/inbox/platform/migration.sql.tpl +97 -34
- package/templates/be/inbox/service.spec.ts.tpl +1 -1
- package/templates/be/inbox/webhook-door.ts.tpl +2 -2
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.spec.ts.tpl +4 -4
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.ts.tpl +5 -5
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.spec.ts.tpl +1 -1
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.spec.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.spec.ts.tpl +2 -2
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.ts.tpl +2 -2
- package/templates/be/skeleton/apps/__app__/src/__app__.env.ts +4 -0
- package/templates/be/skeleton/apps/__app__/src/__app__.options.ts +3 -0
- package/templates/be/skeleton/apps/__app__/src/compose/platform.ts +1 -1
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.spec.ts +37 -2
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.ts +36 -3
- package/templates/be/skeleton/src/modules/platform/http/http.config.ts +11 -0
- package/templates/be/skeleton/src/modules/platform/http/http.decorators.ts +7 -0
- package/templates/be/skeleton/src/modules/platform/http/http.module-definition.ts +5 -3
- package/templates/be/skeleton/src/modules/platform/http/http.options.ts +9 -2
- package/templates/be/skeleton/src/modules/platform/http/index.ts +2 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.spec.ts +96 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.ts +93 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/index.ts +1 -1
- package/runtime/scripts/hfs/rules/database-queue-outbox.mjs +0 -102
- package/templates/app/db-image/db/Dockerfile +0 -8
- package/templates/be/skeleton-lite/src/modules/platform/database/database.functions.ts +0 -24
|
@@ -1,2 +1,9 @@
|
|
|
1
|
-
/**
|
|
2
|
-
export interface EventBusOptions {
|
|
1
|
+
/** How the consumers of the app read their queues; every field has a default, so a bare registration works. */
|
|
2
|
+
export interface EventBusOptions {
|
|
3
|
+
/** Seconds a read message stays hidden: a consumer that fails or dies lets it reappear after this. Default 30. */
|
|
4
|
+
readonly visibilitySeconds?: number
|
|
5
|
+
/** The most messages one read takes from a queue. Default 10. */
|
|
6
|
+
readonly batch?: number
|
|
7
|
+
/** The pause between two reads of an empty queue, in milliseconds. Default 500. */
|
|
8
|
+
readonly idleMs?: number
|
|
9
|
+
}
|
|
@@ -1,16 +1,29 @@
|
|
|
1
1
|
import type { EntityManager } from "typeorm"
|
|
2
|
-
import type { BaseEvent, EventClass, EventDelivery } from "./event-bus.contracts"
|
|
2
|
+
import type { BaseEvent, DeadLetter, EventClass, EventDelivery, EventSubscriptionOptions } from "./event-bus.contracts"
|
|
3
3
|
|
|
4
|
-
/** The publishing side of the lite event bus
|
|
4
|
+
/** The publishing and operator side of the lite event bus on pgmq. */
|
|
5
5
|
export interface EventBus {
|
|
6
|
-
/**
|
|
6
|
+
/** Sends the typed event once to its topic in the caller's transaction; pgmq routes it to every bound consumer queue, and a rollback sends nothing. */
|
|
7
7
|
publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void>
|
|
8
|
+
/** The oldest deliveries the consumer of `queue` gave up on, at most `limit`, for an operator's cli command. */
|
|
9
|
+
deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>>
|
|
10
|
+
/** Sends one dead letter of `queue` back to that queue; false when it holds no dead letter with the id. */
|
|
11
|
+
requeue(queue: string, id: string): Promise<boolean>
|
|
8
12
|
}
|
|
9
13
|
|
|
10
|
-
/** A
|
|
14
|
+
/** A reactor's thin door: it reads one typed event and dispatches one command with the delivery's ids. */
|
|
11
15
|
export interface EventConsumer<Event extends BaseEvent> {
|
|
12
16
|
/** The event class the consumer reads. */
|
|
13
17
|
readonly event: EventClass<Event>
|
|
14
|
-
/** Applies one delivery
|
|
18
|
+
/** Applies one delivery; a throw leaves the message to reappear after its visibility timeout. */
|
|
15
19
|
handle(delivery: EventDelivery<Event>): Promise<void>
|
|
16
20
|
}
|
|
21
|
+
|
|
22
|
+
/** Where a reactor subscribes its consumer; reading starts when the app has bootstrapped. */
|
|
23
|
+
export interface EventConsumerRegistry {
|
|
24
|
+
/**
|
|
25
|
+
* Reads `queue` for `consumer`. The queue is a literal `<TOPIC>_<CONSUMER>_TOPIC_QUEUE` constant of platform/event-bus whose
|
|
26
|
+
* pgmq queue, dead-letter queue and topic binding a migration declares (R258); the subscription is static (R336).
|
|
27
|
+
*/
|
|
28
|
+
subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options?: EventSubscriptionOptions): void
|
|
29
|
+
}
|
|
@@ -1,27 +1,33 @@
|
|
|
1
1
|
import { Test } from "@nestjs/testing"
|
|
2
2
|
import { fakeTransaction, mockEntityManager } from "@starci/jest-preset"
|
|
3
|
+
import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
|
|
3
4
|
import { EventBusErrorCode } from "./errors/event-bus.error"
|
|
4
5
|
import { BaseEvent } from "./event-bus.contracts"
|
|
5
6
|
import { EventBusService } from "./event-bus.service"
|
|
6
7
|
|
|
7
|
-
// The
|
|
8
|
-
const
|
|
8
|
+
// The statements callDbFunction of platform/database sends for the bus functions of the migration.
|
|
9
|
+
const SEND = 'SELECT * FROM "public"."event_bus_send"($1, $2)'
|
|
10
|
+
const DEAD_LETTERS = 'SELECT * FROM "public"."event_bus_dead_letters"($1, $2)'
|
|
11
|
+
const REQUEUE = 'SELECT * FROM "public"."event_bus_requeue"($1, $2)'
|
|
12
|
+
const QUEUE = "spec_delivery_recorded__mailer"
|
|
9
13
|
|
|
10
14
|
class ReceiptRecordedEvent extends BaseEvent {
|
|
11
|
-
readonly eventId = "
|
|
15
|
+
readonly eventId = "event-1"
|
|
12
16
|
readonly eventName = "spec.delivery-recorded"
|
|
13
|
-
readonly payload = { receiptId: "receipt-1" }
|
|
17
|
+
readonly payload = { dedupeKey: "receipt-1", receiptId: "receipt-1" }
|
|
14
18
|
}
|
|
15
19
|
|
|
16
20
|
describe("EventBusService", () => {
|
|
17
|
-
const build = async () => {
|
|
18
|
-
const module = await Test.createTestingModule({
|
|
21
|
+
const build = async (primary = mockEntityManager({})) => {
|
|
22
|
+
const module = await Test.createTestingModule({
|
|
23
|
+
providers: [EventBusService, { provide: PRIMARY_ENTITY_MANAGER, useValue: primary }],
|
|
24
|
+
}).compile()
|
|
19
25
|
return module.get(EventBusService)
|
|
20
26
|
}
|
|
21
27
|
|
|
22
28
|
it.each(["", "unsafe\nid", "x".repeat(129)])("refuses invalid correlation before SQL: %s", async (correlationId) => {
|
|
23
29
|
const subject = await build()
|
|
24
|
-
const manager = mockEntityManager({ query: [
|
|
30
|
+
const manager = mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] })
|
|
25
31
|
|
|
26
32
|
await expect(subject.publish(new ReceiptRecordedEvent(), manager, correlationId)).rejects.toMatchObject({
|
|
27
33
|
code: EventBusErrorCode.CorrelationInvalid,
|
|
@@ -29,31 +35,46 @@ describe("EventBusService", () => {
|
|
|
29
35
|
expect(manager.query).not.toHaveBeenCalled()
|
|
30
36
|
})
|
|
31
37
|
|
|
32
|
-
it("
|
|
38
|
+
it("sends the typed envelope to the event's topic through the publisher's transaction", async () => {
|
|
33
39
|
const subject = await build()
|
|
34
|
-
const transaction = fakeTransaction(mockEntityManager({ query: [
|
|
40
|
+
const transaction = fakeTransaction(mockEntityManager({ query: [SEND, [{ event_bus_send: 1 }]] }))
|
|
35
41
|
const event = new ReceiptRecordedEvent()
|
|
36
42
|
|
|
37
43
|
await transaction.em.transaction((manager) => subject.publish(event, manager, "request-1"))
|
|
38
44
|
|
|
39
|
-
expect(transaction.em.query).toHaveBeenCalledWith(
|
|
40
|
-
"receipt-1",
|
|
45
|
+
expect(transaction.em.query).toHaveBeenCalledWith(SEND, [
|
|
41
46
|
"spec.delivery-recorded",
|
|
42
47
|
JSON.stringify({ metadata: { correlationId: "request-1" }, eventId: event.eventId, eventName: event.eventName, payload: event.payload }),
|
|
43
|
-
"request-1",
|
|
44
48
|
])
|
|
45
49
|
expect(transaction.commits).toBe(1)
|
|
46
50
|
})
|
|
47
51
|
|
|
48
|
-
it("propagates
|
|
52
|
+
it("propagates a send failure so the publisher's transaction rolls back and nothing is sent", async () => {
|
|
49
53
|
const subject = await build()
|
|
50
|
-
const manager = mockEntityManager({ query: [
|
|
51
|
-
jest.spyOn(manager, "query").mockRejectedValue(new Error("
|
|
54
|
+
const manager = mockEntityManager({ query: [SEND, []] })
|
|
55
|
+
jest.spyOn(manager, "query").mockRejectedValue(new Error("pgmq unavailable"))
|
|
52
56
|
const transaction = fakeTransaction(manager)
|
|
53
57
|
|
|
54
|
-
await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("
|
|
58
|
+
await expect(transaction.em.transaction((tx) => subject.publish(new ReceiptRecordedEvent(), tx, "request-1"))).rejects.toThrow("pgmq unavailable")
|
|
55
59
|
|
|
56
60
|
expect(transaction.commits).toBe(0)
|
|
57
61
|
expect(transaction.rollbacks).toBe(1)
|
|
58
62
|
})
|
|
63
|
+
|
|
64
|
+
it("lists a consumer's dead letters and requeues one by its dead-letter id", async () => {
|
|
65
|
+
const primary = mockEntityManager({
|
|
66
|
+
query: [
|
|
67
|
+
[DEAD_LETTERS, [{ msg_id: "7", message: { reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } } }]],
|
|
68
|
+
[REQUEUE, [{ event_bus_requeue: true }]],
|
|
69
|
+
],
|
|
70
|
+
})
|
|
71
|
+
const subject = await build(primary)
|
|
72
|
+
|
|
73
|
+
await expect(subject.deadLetters(QUEUE, 10)).resolves.toEqual([
|
|
74
|
+
{ id: "7", reason: "max-deliveries", readCount: 6, message: { eventId: "event-1" } },
|
|
75
|
+
])
|
|
76
|
+
await expect(subject.requeue(QUEUE, "7")).resolves.toBe(true)
|
|
77
|
+
expect(primary.query).toHaveBeenCalledWith(DEAD_LETTERS, [QUEUE, 10])
|
|
78
|
+
expect(primary.query).toHaveBeenCalledWith(REQUEUE, [QUEUE, "7"])
|
|
79
|
+
})
|
|
59
80
|
})
|
|
@@ -1,15 +1,28 @@
|
|
|
1
1
|
import { Injectable } from "@nestjs/common"
|
|
2
2
|
import type { EntityManager } from "typeorm"
|
|
3
3
|
import { isCorrelationId } from "@modules/platform/logging"
|
|
4
|
+
import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
|
|
4
5
|
import { EventBusError, EventBusErrorCode } from "./errors/event-bus.error"
|
|
5
|
-
import type { BaseEvent, EventEnvelope } from "./event-bus.contracts"
|
|
6
|
+
import type { BaseEvent, DeadLetter, EventEnvelope } from "./event-bus.contracts"
|
|
6
7
|
import type { EventBus } from "./event-bus.port"
|
|
7
|
-
|
|
8
|
+
|
|
9
|
+
/** One row of event_bus_dead_letters: pgmq returns the bigint id as text. */
|
|
10
|
+
interface DeadLetterRow {
|
|
11
|
+
readonly msg_id: string
|
|
12
|
+
readonly message: { readonly reason?: unknown; readonly readCount?: unknown; readonly message?: unknown }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** The one row event_bus_requeue answers. */
|
|
16
|
+
interface RequeueRow {
|
|
17
|
+
readonly event_bus_requeue: boolean | null
|
|
18
|
+
}
|
|
8
19
|
|
|
9
20
|
@Injectable()
|
|
10
|
-
/**
|
|
21
|
+
/** Sends a typed event once to its pgmq topic inside the publisher's transaction; no outbox and no relay stand between them. */
|
|
11
22
|
export class EventBusService implements EventBus {
|
|
12
|
-
|
|
23
|
+
constructor(@InjectPrimaryEntityManager() private readonly entityManager: EntityManager) {}
|
|
24
|
+
|
|
25
|
+
/** The send commits or rolls back with the change; pgmq copies it into every consumer queue bound to the event's name. */
|
|
13
26
|
async publish(event: BaseEvent, tx: EntityManager, correlationId: string): Promise<void> {
|
|
14
27
|
if (!isCorrelationId(correlationId)) throw new EventBusError({ code: EventBusErrorCode.CorrelationInvalid })
|
|
15
28
|
const envelope: EventEnvelope = {
|
|
@@ -18,6 +31,21 @@ export class EventBusService implements EventBus {
|
|
|
18
31
|
eventName: event.eventName,
|
|
19
32
|
payload: event.payload,
|
|
20
33
|
}
|
|
21
|
-
await callDbFunction(tx, "
|
|
34
|
+
await callDbFunction(tx, "public.event_bus_send", [event.eventName, JSON.stringify(envelope)])
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async deadLetters(queue: string, limit: number): Promise<ReadonlyArray<DeadLetter>> {
|
|
38
|
+
const rows = await callDbFunction<DeadLetterRow>(this.entityManager, "public.event_bus_dead_letters", [queue, limit])
|
|
39
|
+
return rows.map((row) => ({
|
|
40
|
+
id: row.msg_id,
|
|
41
|
+
reason: String(row.message.reason),
|
|
42
|
+
readCount: Number(row.message.readCount),
|
|
43
|
+
message: row.message.message,
|
|
44
|
+
}))
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async requeue(queue: string, id: string): Promise<boolean> {
|
|
48
|
+
const [row] = await callDbFunction<RequeueRow>(this.entityManager, "public.event_bus_requeue", [queue, id])
|
|
49
|
+
return row?.event_bus_requeue === true
|
|
22
50
|
}
|
|
23
51
|
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { Test } from "@nestjs/testing"
|
|
2
|
+
import { mockEntityManager } from "@starci/jest-preset"
|
|
3
|
+
import { PRIMARY_ENTITY_MANAGER } from "@modules/platform/database"
|
|
4
|
+
import { LOGGER } from "@modules/platform/logging"
|
|
5
|
+
import { BaseEvent } from "./event-bus.contracts"
|
|
6
|
+
import type { EventDelivery, ParsedEvent } from "./event-bus.contracts"
|
|
7
|
+
import { MODULE_OPTIONS_TOKEN } from "./event-bus.module-definition"
|
|
8
|
+
import type { EventConsumer } from "./event-bus.port"
|
|
9
|
+
import { EventConsumerService } from "./event-consumer.service"
|
|
10
|
+
|
|
11
|
+
// The statements callDbFunction of platform/database sends for the bus functions of the migration.
|
|
12
|
+
const READ = 'SELECT * FROM "public"."event_bus_read"($1, $2, $3)'
|
|
13
|
+
const ARCHIVE = 'SELECT * FROM "public"."event_bus_archive"($1, $2)'
|
|
14
|
+
const BURY = 'SELECT * FROM "public"."event_bus_bury"($1, $2, $3, $4, $5)'
|
|
15
|
+
const QUEUE = "spec_delivery_recorded__mailer"
|
|
16
|
+
|
|
17
|
+
class ReceiptRecordedEvent extends BaseEvent {
|
|
18
|
+
static readonly eventName = "spec.delivery-recorded"
|
|
19
|
+
static readonly version = 1
|
|
20
|
+
readonly eventName = ReceiptRecordedEvent.eventName
|
|
21
|
+
|
|
22
|
+
constructor(
|
|
23
|
+
readonly eventId: string,
|
|
24
|
+
readonly payload: { readonly dedupeKey: string; readonly receiptId: string },
|
|
25
|
+
) {
|
|
26
|
+
super()
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
static parse(envelope: unknown): ParsedEvent<ReceiptRecordedEvent> {
|
|
30
|
+
if (typeof envelope !== "object" || envelope === null || !("eventId" in envelope) || typeof envelope.eventId !== "string") return null
|
|
31
|
+
if (!("payload" in envelope) || typeof envelope.payload !== "object" || envelope.payload === null || !("dedupeKey" in envelope.payload)) return null
|
|
32
|
+
const dedupeKey = envelope.payload.dedupeKey
|
|
33
|
+
return typeof dedupeKey === "string" ? new ReceiptRecordedEvent(envelope.eventId, { dedupeKey, receiptId: dedupeKey }) : null
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const envelope = {
|
|
38
|
+
metadata: { correlationId: "request-1" },
|
|
39
|
+
eventId: "event-1",
|
|
40
|
+
eventName: "spec.delivery-recorded",
|
|
41
|
+
payload: { dedupeKey: "receipt-1", receiptId: "receipt-1" },
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
class RecordingConsumer implements EventConsumer<ReceiptRecordedEvent> {
|
|
45
|
+
readonly event = ReceiptRecordedEvent
|
|
46
|
+
readonly deliveries: Array<EventDelivery<ReceiptRecordedEvent>> = []
|
|
47
|
+
failing = false
|
|
48
|
+
|
|
49
|
+
handle(delivery: EventDelivery<ReceiptRecordedEvent>): Promise<void> {
|
|
50
|
+
if (this.failing) return Promise.reject(new Error("consumer down"))
|
|
51
|
+
this.deliveries.push(delivery)
|
|
52
|
+
return Promise.resolve()
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
describe("EventConsumerService", () => {
|
|
57
|
+
const logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }
|
|
58
|
+
|
|
59
|
+
const build = async (primary: ReturnType<typeof mockEntityManager>) => {
|
|
60
|
+
const module = await Test.createTestingModule({
|
|
61
|
+
providers: [
|
|
62
|
+
EventConsumerService,
|
|
63
|
+
{ provide: PRIMARY_ENTITY_MANAGER, useValue: primary },
|
|
64
|
+
{ provide: MODULE_OPTIONS_TOKEN, useValue: {} },
|
|
65
|
+
{ provide: LOGGER, useValue: logger },
|
|
66
|
+
],
|
|
67
|
+
}).compile()
|
|
68
|
+
return module.get(EventConsumerService)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
it("delivers a committed event once to the consumer with its inbox claim keys and archives it", async () => {
|
|
72
|
+
const primary = mockEntityManager({
|
|
73
|
+
query: [
|
|
74
|
+
[READ, [{ msg_id: "1", read_ct: 1, message: envelope }]],
|
|
75
|
+
[ARCHIVE, [{ event_bus_archive: true }]],
|
|
76
|
+
],
|
|
77
|
+
})
|
|
78
|
+
const subject = await build(primary)
|
|
79
|
+
const consumer = new RecordingConsumer()
|
|
80
|
+
|
|
81
|
+
await expect(subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })).resolves.toBe(1)
|
|
82
|
+
|
|
83
|
+
expect(consumer.deliveries).toEqual([
|
|
84
|
+
{ correlationId: "request-1", eventId: "event-1", consumer: QUEUE, dedupeKey: "receipt-1", event: expect.any(ReceiptRecordedEvent), attempt: 1 },
|
|
85
|
+
])
|
|
86
|
+
expect(primary.query).toHaveBeenCalledWith(READ, [QUEUE, 30, 10])
|
|
87
|
+
expect(primary.query).toHaveBeenCalledWith(ARCHIVE, [QUEUE, "1"])
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
it("leaves a failed delivery to reappear after its visibility timeout", async () => {
|
|
91
|
+
const primary = mockEntityManager({ query: [READ, [{ msg_id: "1", read_ct: 2, message: envelope }]] })
|
|
92
|
+
const subject = await build(primary)
|
|
93
|
+
const consumer = new RecordingConsumer()
|
|
94
|
+
consumer.failing = true
|
|
95
|
+
|
|
96
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
|
|
97
|
+
|
|
98
|
+
expect(primary.query).toHaveBeenCalledTimes(1)
|
|
99
|
+
expect(logger.error).toHaveBeenCalled()
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
it.each([
|
|
103
|
+
["max-deliveries", { msg_id: "1", read_ct: 4, message: envelope }],
|
|
104
|
+
["malformed", { msg_id: "1", read_ct: 1, message: { eventId: 42 } }],
|
|
105
|
+
["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, payload: { dedupeKey: " ", receiptId: "receipt-1" } } }],
|
|
106
|
+
["malformed", { msg_id: "1", read_ct: 1, message: { ...envelope, metadata: { correlationId: "unsafe\nid" } } }],
|
|
107
|
+
])("moves a delivery to the consumer's dead-letter queue: %s", async (reason, row) => {
|
|
108
|
+
const primary = mockEntityManager({ query: [[READ, [row]], [BURY, [{ event_bus_bury: true }]]] })
|
|
109
|
+
const subject = await build(primary)
|
|
110
|
+
const consumer = new RecordingConsumer()
|
|
111
|
+
|
|
112
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 3 })
|
|
113
|
+
|
|
114
|
+
expect(consumer.deliveries).toEqual([])
|
|
115
|
+
expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", row.read_ct, JSON.stringify(row.message), reason])
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
it("takes the default delivery budget of a subscription that names none", async () => {
|
|
119
|
+
const primary = mockEntityManager({ query: [[READ, [{ msg_id: "1", read_ct: 6, message: envelope }]], [BURY, [{ event_bus_bury: true }]]] })
|
|
120
|
+
const subject = await build(primary)
|
|
121
|
+
const consumer = new RecordingConsumer()
|
|
122
|
+
subject.subscribe(QUEUE, consumer)
|
|
123
|
+
|
|
124
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
|
|
125
|
+
|
|
126
|
+
expect(primary.query).toHaveBeenCalledWith(BURY, [QUEUE, "1", 6, JSON.stringify(envelope), "max-deliveries"])
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it("delivers a redelivery again, leaving its dedupe to the consumer's inbox claim", async () => {
|
|
130
|
+
const primary = mockEntityManager({
|
|
131
|
+
query: [
|
|
132
|
+
[READ, [{ msg_id: "1", read_ct: 2, message: envelope }]],
|
|
133
|
+
[ARCHIVE, [{ event_bus_archive: true }]],
|
|
134
|
+
],
|
|
135
|
+
})
|
|
136
|
+
const subject = await build(primary)
|
|
137
|
+
const consumer = new RecordingConsumer()
|
|
138
|
+
|
|
139
|
+
await subject.drain({ queue: QUEUE, consumer, maxDeliveries: 5 })
|
|
140
|
+
|
|
141
|
+
expect(consumer.deliveries.map((delivery) => [delivery.consumer, delivery.dedupeKey, delivery.attempt])).toEqual([[QUEUE, "receipt-1", 2]])
|
|
142
|
+
})
|
|
143
|
+
})
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { Injectable } from "@nestjs/common"
|
|
2
|
+
import type { OnApplicationBootstrap, OnApplicationShutdown } from "@nestjs/common"
|
|
3
|
+
import type { EntityManager } from "typeorm"
|
|
4
|
+
import { InjectLogger, isCorrelationId } from "@modules/platform/logging"
|
|
5
|
+
import type { Logger } from "@modules/platform/logging"
|
|
6
|
+
import { callDbFunction, InjectPrimaryEntityManager } from "@modules/platform/database"
|
|
7
|
+
import { InjectEventBusOptions } from "./event-bus.decorators"
|
|
8
|
+
import { DEFAULT_MAX_DELIVERIES } from "./event-bus.contracts"
|
|
9
|
+
import type { BaseEvent, EventMessage, EventSubscriptionOptions } from "./event-bus.contracts"
|
|
10
|
+
import { EventBusLogEvent } from "./event-bus.log-events"
|
|
11
|
+
import type { EventBusOptions } from "./event-bus.options"
|
|
12
|
+
import type { EventConsumer, EventConsumerRegistry } from "./event-bus.port"
|
|
13
|
+
|
|
14
|
+
/** One row of event_bus_read: pgmq returns the bigint id as text. */
|
|
15
|
+
interface MessageRow {
|
|
16
|
+
readonly msg_id: string
|
|
17
|
+
readonly read_ct: number
|
|
18
|
+
readonly message: unknown
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** One consumer: its queue `<topic>__<consumer>`, the door its deliveries go through and its delivery budget. */
|
|
22
|
+
export interface EventSubscription {
|
|
23
|
+
/** The consumer's pgmq queue. */
|
|
24
|
+
readonly queue: string
|
|
25
|
+
/** The thin door the consumer's deliveries go through. */
|
|
26
|
+
readonly consumer: EventConsumer<BaseEvent>
|
|
27
|
+
/** The deliveries before the message moves to `<queue>__dead`. */
|
|
28
|
+
readonly maxDeliveries: number
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The correlation id of an envelope: its metadata when present and valid, else a safe event id; null refuses the message. */
|
|
32
|
+
const correlationOf = (message: unknown, eventId: string): string | null => {
|
|
33
|
+
if (typeof message !== "object" || message === null || !("metadata" in message)) return isCorrelationId(eventId) ? eventId : null
|
|
34
|
+
const metadata = message.metadata
|
|
35
|
+
if (typeof metadata !== "object" || metadata === null || !("correlationId" in metadata)) return null
|
|
36
|
+
return isCorrelationId(metadata.correlationId) ? metadata.correlationId : null
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** True for a usable business dedupe key. */
|
|
40
|
+
const isDedupeKey = (value: unknown): value is string => typeof value === "string" && value.trim().length > 0 && value.length <= 256
|
|
41
|
+
|
|
42
|
+
@Injectable()
|
|
43
|
+
/**
|
|
44
|
+
* The consuming side of the lite event bus: each consumer reads its own pgmq queue with a visibility timeout, archives a message it
|
|
45
|
+
* handled, leaves a failed one to reappear, and moves one past its maxDeliveries, or malformed, to `<queue>__dead`. Delivery is at
|
|
46
|
+
* least once and unordered, so the command a consumer dispatches claims (consumer, dedupeKey) in the inbox first and reads current
|
|
47
|
+
* state rather than trusting the order of events (BE-EVENT-BUS-4).
|
|
48
|
+
*/
|
|
49
|
+
export class EventConsumerService implements EventConsumerRegistry, OnApplicationBootstrap, OnApplicationShutdown {
|
|
50
|
+
private readonly subscriptions: Array<EventSubscription> = []
|
|
51
|
+
private running = false
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
@InjectPrimaryEntityManager() private readonly entityManager: EntityManager,
|
|
55
|
+
@InjectEventBusOptions() private readonly options: EventBusOptions,
|
|
56
|
+
@InjectLogger() private readonly logger: Logger,
|
|
57
|
+
) {}
|
|
58
|
+
|
|
59
|
+
subscribe<Event extends BaseEvent>(queue: string, consumer: EventConsumer<Event>, options: EventSubscriptionOptions = {}): void {
|
|
60
|
+
this.subscriptions.push({ queue, consumer, maxDeliveries: options.maxDeliveries ?? DEFAULT_MAX_DELIVERIES })
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
onApplicationBootstrap(): void {
|
|
64
|
+
this.running = true
|
|
65
|
+
for (const subscription of this.subscriptions) void this.loop(subscription)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
onApplicationShutdown(): void {
|
|
69
|
+
this.running = false
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Reads one batch of the consumer's queue and settles each message in order; answers how many it read. */
|
|
73
|
+
async drain(subscription: EventSubscription): Promise<number> {
|
|
74
|
+
const rows = await callDbFunction<MessageRow>(this.entityManager, "public.event_bus_read", [
|
|
75
|
+
subscription.queue,
|
|
76
|
+
this.options.visibilitySeconds ?? 30,
|
|
77
|
+
this.options.batch ?? 10,
|
|
78
|
+
])
|
|
79
|
+
for (const row of rows) await this.settle(subscription, { msgId: row.msg_id, readCt: row.read_ct, message: row.message })
|
|
80
|
+
return rows.length
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private async loop(subscription: EventSubscription): Promise<void> {
|
|
84
|
+
while (this.running) {
|
|
85
|
+
const read = await this.drain(subscription).catch((cause: unknown) => {
|
|
86
|
+
this.logger.error(EventBusLogEvent.ReadFailed, cause, { queue: subscription.queue })
|
|
87
|
+
return 0
|
|
88
|
+
})
|
|
89
|
+
if (read === 0) await new Promise((resolve) => setTimeout(resolve, this.options.idleMs ?? 500))
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private async settle(subscription: EventSubscription, message: EventMessage): Promise<void> {
|
|
94
|
+
const event = subscription.consumer.event.parse(message.message)
|
|
95
|
+
const correlationId = event === null ? null : correlationOf(message.message, event.eventId)
|
|
96
|
+
if (event === null || correlationId === null || !isDedupeKey(event.payload.dedupeKey)) return this.dead(subscription, message, "malformed", null)
|
|
97
|
+
if (message.readCt > subscription.maxDeliveries) return this.dead(subscription, message, "max-deliveries", event.eventName)
|
|
98
|
+
try {
|
|
99
|
+
await subscription.consumer.handle({
|
|
100
|
+
correlationId,
|
|
101
|
+
eventId: event.eventId,
|
|
102
|
+
consumer: subscription.queue,
|
|
103
|
+
dedupeKey: event.payload.dedupeKey,
|
|
104
|
+
event,
|
|
105
|
+
attempt: message.readCt,
|
|
106
|
+
})
|
|
107
|
+
} catch (cause) {
|
|
108
|
+
this.logger.error(EventBusLogEvent.ConsumerFailed, cause, { consumer: subscription.queue, event: event.eventName, attempt: message.readCt })
|
|
109
|
+
return
|
|
110
|
+
}
|
|
111
|
+
await callDbFunction(this.entityManager, "public.event_bus_archive", [subscription.queue, message.msgId])
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Moves the message to `<queue>__dead` and archives the original, in one call. */
|
|
115
|
+
private async dead(subscription: EventSubscription, message: EventMessage, reason: string, eventName: string | null): Promise<void> {
|
|
116
|
+
this.logger.warn(EventBusLogEvent.DeliveryDead, { consumer: subscription.queue, event: eventName, msgId: message.msgId, reason, attempt: message.readCt })
|
|
117
|
+
await callDbFunction(this.entityManager, "public.event_bus_bury", [subscription.queue, message.msgId, message.readCt, JSON.stringify(message.message), reason])
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
export { EVENT_BUS_ERROR_KINDS } from "./errors/event-bus.error"
|
|
2
2
|
export { EVENT_BUS_MESSAGES } from "./messages/event-bus.messages"
|
|
3
|
-
export { BaseEvent } from "./event-bus.contracts"
|
|
4
|
-
export type {
|
|
5
|
-
|
|
3
|
+
export { BaseEvent, DEAD_QUEUE_SUFFIX, DEFAULT_MAX_DELIVERIES } from "./event-bus.contracts"
|
|
4
|
+
export type {
|
|
5
|
+
DeadLetter,
|
|
6
|
+
EventClass,
|
|
7
|
+
EventDelivery,
|
|
8
|
+
EventEnvelope,
|
|
9
|
+
EventPayload,
|
|
10
|
+
EventSubscriptionOptions,
|
|
11
|
+
ParsedEvent,
|
|
12
|
+
} from "./event-bus.contracts"
|
|
13
|
+
export { EVENT_BUS, EVENT_CONSUMER_REGISTRY, InjectEventBus, InjectEventConsumerRegistry } from "./event-bus.decorators"
|
|
6
14
|
export { EventBusModule } from "./event-bus.module"
|
|
7
|
-
export type {
|
|
15
|
+
export type { EventBusOptions } from "./event-bus.options"
|
|
16
|
+
export type { EventBus, EventConsumer, EventConsumerRegistry } from "./event-bus.port"
|
|
@@ -9,10 +9,10 @@ import { requestFingerprint } from "./request-fingerprint.policy"
|
|
|
9
9
|
import { PostgresInbox } from "./inbox.service"
|
|
10
10
|
|
|
11
11
|
// The statements callDbFunction of platform/database sends for each inbox function.
|
|
12
|
-
const CLAIM_EVENT =
|
|
13
|
-
const CLAIM_CLIENT =
|
|
14
|
-
const SAVE_CLIENT_OUTCOME =
|
|
15
|
-
const SWEEP_CLIENTS =
|
|
12
|
+
const CLAIM_EVENT = 'SELECT * FROM "public"."inbox_claim_event"($1, $2)'
|
|
13
|
+
const CLAIM_CLIENT = 'SELECT * FROM "public"."inbox_claim_client"($1, $2, $3, $4)'
|
|
14
|
+
const SAVE_CLIENT_OUTCOME = 'SELECT * FROM "public"."inbox_save_client_outcome"($1, $2, $3)'
|
|
15
|
+
const SWEEP_CLIENTS = 'SELECT * FROM "public"."inbox_sweep_clients"($1, $2)'
|
|
16
16
|
|
|
17
17
|
const runClient = <T>(
|
|
18
18
|
inbox: PostgresInbox,
|
|
@@ -54,7 +54,7 @@ export class PostgresInbox implements Inbox {
|
|
|
54
54
|
if (!normalizedEventId) {
|
|
55
55
|
throw new DatabaseError({ code: DatabaseErrorCode.WebhookDeliveryIdRequired })
|
|
56
56
|
}
|
|
57
|
-
const rows = await callDbFunction<InboxEventRow>(tx, "inbox_claim_event", [source, normalizedEventId])
|
|
57
|
+
const rows = await callDbFunction<InboxEventRow>(tx, "public.inbox_claim_event", [source, normalizedEventId])
|
|
58
58
|
return rows.length > 0
|
|
59
59
|
}
|
|
60
60
|
|
|
@@ -64,7 +64,7 @@ export class PostgresInbox implements Inbox {
|
|
|
64
64
|
throw new InboxError({ code: InboxErrorCode.TransactionRequired })
|
|
65
65
|
const source = clientSource(request)
|
|
66
66
|
const fingerprint = requestFingerprint(request.request)
|
|
67
|
-
const rows = await callDbFunction<ClaimClientRow<T>>(tx, "inbox_claim_client", [
|
|
67
|
+
const rows = await callDbFunction<ClaimClientRow<T>>(tx, "public.inbox_claim_client", [
|
|
68
68
|
source,
|
|
69
69
|
request.idempotencyKey,
|
|
70
70
|
fingerprint,
|
|
@@ -84,7 +84,7 @@ export class PostgresInbox implements Inbox {
|
|
|
84
84
|
} catch (cause) {
|
|
85
85
|
throw new InboxError({ code: InboxErrorCode.OutcomeInvalid, cause })
|
|
86
86
|
}
|
|
87
|
-
const saved = await callDbFunction<SavedClientOutcomeRow<T>>(tx, "inbox_save_client_outcome", [
|
|
87
|
+
const saved = await callDbFunction<SavedClientOutcomeRow<T>>(tx, "public.inbox_save_client_outcome", [
|
|
88
88
|
source,
|
|
89
89
|
request.idempotencyKey,
|
|
90
90
|
outcome,
|
|
@@ -96,7 +96,7 @@ export class PostgresInbox implements Inbox {
|
|
|
96
96
|
|
|
97
97
|
/** A row lock and SKIP LOCKED keep retention from competing with a live retry or another sweep worker. */
|
|
98
98
|
async sweepClients(cutoff: Date, tx: EntityManager): Promise<number> {
|
|
99
|
-
const rows = await callDbFunction<InboxSweepRow>(tx, "inbox_sweep_clients", [cutoff, BATCH_ROWS])
|
|
99
|
+
const rows = await callDbFunction<InboxSweepRow>(tx, "public.inbox_sweep_clients", [cutoff, BATCH_ROWS])
|
|
100
100
|
return rows[0]?.deleted_count ?? 0
|
|
101
101
|
}
|
|
102
102
|
}
|