@starci/hfs 4.10.1 → 4.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +40 -1
- package/db/drift.mjs +19 -16
- package/db/image.mjs +13 -0
- package/db/migrate.mjs +3 -3
- package/db/postgres.mjs +1 -1
- package/package.json +2 -1
- package/runtime/knowledge/hfs/canon-pins.yaml +23 -8
- package/runtime/knowledge/hfs/rules.yaml +54 -13
- package/runtime/knowledge/hfs/slots.yaml +67 -36
- package/runtime/knowledge/patterns/be/event-bus.yaml +12 -10
- package/runtime/knowledge/patterns/be/idempotency.yaml +1 -1
- package/runtime/knowledge/patterns/be/jobs.yaml +3 -0
- package/runtime/knowledge/patterns/be/queues.yaml +14 -14
- package/runtime/knowledge/patterns/be/reactors.yaml +1 -1
- package/runtime/knowledge/patterns/be/realtime.yaml +1 -1
- package/runtime/modules/kernel/failure-codes.yaml +28 -18
- package/runtime/modules/schemas/application-stacks.schema.yaml +4 -2
- package/runtime/scripts/gates/starcistacks.mjs +6 -2
- package/runtime/scripts/guards/push-scan.mjs +1 -1
- package/runtime/scripts/hfs/ci-declaration.mjs +3 -2
- package/runtime/scripts/hfs/db-baseline-verb.mjs +94 -22
- package/runtime/scripts/hfs/db-errors.mjs +2 -0
- package/runtime/scripts/hfs/db-plan-verb.mjs +17 -5
- package/runtime/scripts/hfs/db-squash.mjs +514 -0
- package/runtime/scripts/hfs/declaration-shape.mjs +3 -15
- package/runtime/scripts/hfs/example-secrets.mjs +4 -3
- package/runtime/scripts/hfs/rule-params-shape.mjs +11 -9
- package/runtime/scripts/hfs/rules/a15-findings.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-config.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-constants.mjs +16 -1
- package/runtime/scripts/hfs/rules/database-queue-sources.mjs +2 -110
- package/runtime/scripts/hfs/rules/database-queues.mjs +53 -239
- package/runtime/scripts/hfs/rules/database-sql.mjs +2 -0
- package/runtime/scripts/hfs/rules/database-transaction.mjs +45 -0
- package/runtime/scripts/hfs/rules/deploy-scan.mjs +1 -1
- package/runtime/scripts/hfs/rules/edition.mjs +1 -1
- package/runtime/scripts/hfs/rules/env-parity.mjs +1 -1
- package/runtime/scripts/hfs/rules/lite-secrets.mjs +3 -3
- package/runtime/scripts/hfs/rules/script-hygiene.mjs +1 -1
- package/runtime/scripts/hfs/rules/secret-custody-only.mjs +2 -2
- package/runtime/scripts/hfs/rules/secrets.mjs +1 -1
- package/runtime/scripts/hfs/rules/stack-tooling.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks-deploy.mjs +2 -1
- package/runtime/scripts/hfs/rules/stacks-lite-backing.mjs +3 -4
- package/runtime/scripts/hfs/rules/stacks-runtime.mjs +2 -2
- package/runtime/scripts/hfs/rules/stacks-swarm.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks.mjs +3 -3
- package/runtime/scripts/hfs/sealed-members.mjs +10 -6
- package/runtime/scripts/hfs/secret-files.mjs +16 -0
- package/runtime/scripts/hfs/secret-init.mjs +120 -21
- package/runtime/scripts/hfs/secret-output.mjs +2 -1
- package/runtime/scripts/hfs/secret-prompt.mjs +128 -0
- package/runtime/scripts/hfs/secret-sops.mjs +93 -9
- package/runtime/scripts/hfs/secret-standard.mjs +315 -0
- package/runtime/scripts/hfs/secret-store.mjs +23 -9
- package/runtime/scripts/hfs/secret.mjs +110 -46
- package/runtime/scripts/hfs/slots.mjs +8 -1
- package/runtime/scripts/hfs/stack-deploy-config.mjs +20 -6
- package/runtime/scripts/hfs/stacks-gate.mjs +9 -2
- package/runtime/scripts/hfs/starcistacks-runtime.mjs +1 -1
- package/runtime/scripts/lib/sealed-member.mjs +36 -0
- package/runtime/scripts/lib/secret-patterns.mjs +1 -1
- package/runtime/scripts/lib/swarm-stack.mjs +4 -6
- package/scaffold/add-inbox-lite.mjs +2 -2
- package/scaffold/add-queue-lite.mjs +24 -5
- package/scaffold/app.mjs +2 -2
- package/src/main.mjs +5 -4
- package/stack/build.mjs +3 -1
- package/stack/config.mjs +2 -1
- package/stack/docker-transport.mjs +2 -1
- package/stack/remote.mjs +4 -3
- package/stack/sealed.mjs +9 -8
- package/stack/target-binding.mjs +2 -1
- package/sync/hygiene.mjs +1 -1
- package/sync/index.mjs +47 -12
- package/templates/app/db-bootstrap/db/bootstrap.sql +9 -14
- package/templates/app/db-image/starcistacks/images/db/Dockerfile +18 -0
- package/templates/app/gitignore +2 -1
- package/templates/app/gitignore-lite +3 -1
- package/templates/app/package-scripts/db.json +2 -1
- package/templates/app/skeleton-lite/.starcistacks/application-stacks.yaml +4 -2
- package/templates/app/skeleton-lite/.starcistacks/dev/README.md +1 -1
- package/templates/app/skeleton-lite/.starcistacks/dev/infra/swarm/stack.yaml +1 -1
- package/templates/app/skeleton-lite/.starcistacks/production/README.md +1 -1
- package/templates/app/skeleton-lite/.starcistacks/production/infra/swarm/stack.yaml +1 -1
- package/templates/app/skeleton-lite/README.md +2 -2
- package/templates/be/inbox/delivery-recorded.event.ts.tpl +7 -5
- package/templates/be/inbox/events.json.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.contracts.ts.tpl +58 -11
- package/templates/be/inbox/platform/event-bus/event-bus.decorators.ts.tpl +13 -1
- package/templates/be/inbox/platform/event-bus/event-bus.log-events.ts.tpl +9 -0
- package/templates/be/inbox/platform/event-bus/event-bus.module-definition.ts.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.module.ts.tpl +11 -5
- package/templates/be/inbox/platform/event-bus/event-bus.options.ts.tpl +9 -2
- package/templates/be/inbox/platform/event-bus/event-bus.port.ts.tpl +18 -5
- package/templates/be/inbox/platform/event-bus/event-bus.service.spec.ts.tpl +37 -16
- package/templates/be/inbox/platform/event-bus/event-bus.service.ts.tpl +33 -5
- package/templates/be/inbox/platform/event-bus/event-consumer.service.spec.ts.tpl +143 -0
- package/templates/be/inbox/platform/event-bus/event-consumer.service.ts.tpl +119 -0
- package/templates/be/inbox/platform/event-bus/index.ts.tpl +13 -4
- package/templates/be/inbox/platform/inbox/inbox.service.spec.ts.tpl +4 -4
- package/templates/be/inbox/platform/inbox/inbox.service.ts.tpl +4 -4
- package/templates/be/inbox/platform/migration.sql.tpl +97 -34
- package/templates/be/inbox/service.spec.ts.tpl +1 -1
- package/templates/be/inbox/webhook-door.ts.tpl +2 -2
- package/templates/be/patterns/jobs/platform/jobs.log-events.ts.tpl +2 -0
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.spec.ts.tpl +4 -4
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.ts.tpl +5 -5
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.spec.ts.tpl +65 -0
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.ts.tpl +78 -0
- package/templates/be/patterns/jobs/platform-lite/job-ticks.contracts.ts.tpl +16 -0
- package/templates/be/patterns/jobs/platform-lite/jobs.module.ts.tpl +40 -0
- package/templates/be/patterns/jobs/platform-lite/jobs.sql.tpl +44 -0
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.spec.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.spec.ts.tpl +2 -2
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.ts.tpl +2 -2
- package/templates/be/skeleton/apps/__app__/src/__app__.env.ts +4 -0
- package/templates/be/skeleton/apps/__app__/src/__app__.options.ts +3 -0
- package/templates/be/skeleton/apps/__app__/src/compose/platform.ts +1 -1
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.spec.ts +37 -2
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.ts +36 -3
- package/templates/be/skeleton/src/modules/platform/http/http.config.ts +11 -0
- package/templates/be/skeleton/src/modules/platform/http/http.decorators.ts +7 -0
- package/templates/be/skeleton/src/modules/platform/http/http.module-definition.ts +5 -3
- package/templates/be/skeleton/src/modules/platform/http/http.options.ts +9 -2
- package/templates/be/skeleton/src/modules/platform/http/index.ts +2 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.spec.ts +96 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.ts +93 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/index.ts +1 -1
- package/runtime/scripts/hfs/rules/database-queue-outbox.mjs +0 -102
- package/templates/be/skeleton-lite/src/modules/platform/database/database.functions.ts +0 -24
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { HttpSecurityOptions } from "@modules/platform/http-security"
|
|
2
2
|
import type { ServerOptions } from "@modules/platform/config"
|
|
3
3
|
import type { DatabaseConnectionConfig } from "@modules/platform/database"
|
|
4
|
+
import type { HttpOptions } from "@modules/platform/http"
|
|
4
5
|
|
|
5
6
|
/** Everything the {{app}} app needs from its environment, parsed by `{{app}}.env.ts` and handed to `AppModule.register`. */
|
|
6
7
|
export interface {{appPascal}}Options {
|
|
@@ -8,6 +9,8 @@ export interface {{appPascal}}Options {
|
|
|
8
9
|
readonly server: ServerOptions
|
|
9
10
|
/** The origin allowlist and rate limits. */
|
|
10
11
|
readonly httpSecurity: HttpSecurityOptions
|
|
12
|
+
/** What the outbound HTTP client trusts beyond Node's bundled certificate authorities. */
|
|
13
|
+
readonly http: HttpOptions
|
|
11
14
|
/** The primary database connection. */
|
|
12
15
|
readonly database: DatabaseConnectionConfig
|
|
13
16
|
}
|
|
@@ -30,7 +30,7 @@ export const platformModules = (options: {{appPascal}}Options): Array<DynamicMod
|
|
|
30
30
|
RetryModule.register({ isGlobal: true }),
|
|
31
31
|
IdsModule.register({ isGlobal: true }),
|
|
32
32
|
LoggingModule.register({ isGlobal: true }),
|
|
33
|
-
HttpModule.register({ isGlobal: true }),
|
|
33
|
+
HttpModule.register({ isGlobal: true, ...options.http }),
|
|
34
34
|
I18nModule.register({
|
|
35
35
|
isGlobal: true,
|
|
36
36
|
bundles: [HTTP_MESSAGES, ERRORS_MESSAGES, HTTP_SECURITY_MESSAGES, IDENTITY_MESSAGES],
|
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import { Test } from "@nestjs/testing"
|
|
2
2
|
import { mock } from "@starci/jest-preset"
|
|
3
|
+
import { Agent } from "undici"
|
|
3
4
|
import { HttpErrorCode } from "./errors/http.error"
|
|
4
5
|
import { FetchHttpClient } from "./fetch-http-client.client"
|
|
6
|
+
import { HTTP_OPTIONS } from "./http.decorators"
|
|
7
|
+
import type { HttpOptions } from "./http.options"
|
|
5
8
|
|
|
6
9
|
/** Asymmetric matchers read as `unknown`: jest types them `any`, which may not flow into a property or a typed argument. */
|
|
7
10
|
const anyOf = (type: unknown): unknown => expect.any(type)
|
|
8
11
|
const objectLike = (shape: Record<string, unknown>): unknown => expect.objectContaining(shape)
|
|
9
12
|
|
|
10
|
-
const build = async (): Promise<FetchHttpClient> => {
|
|
11
|
-
const moduleRef = await Test.createTestingModule({
|
|
13
|
+
const build = async (options: HttpOptions = {}): Promise<FetchHttpClient> => {
|
|
14
|
+
const moduleRef = await Test.createTestingModule({
|
|
15
|
+
providers: [FetchHttpClient, { provide: HTTP_OPTIONS, useValue: options }],
|
|
16
|
+
}).compile()
|
|
12
17
|
return moduleRef.get(FetchHttpClient)
|
|
13
18
|
}
|
|
14
19
|
|
|
@@ -233,3 +238,33 @@ describe("FetchHttpClient response stream failures", () => {
|
|
|
233
238
|
).rejects.toMatchObject({ code: HttpErrorCode.Timeout, cause: failure })
|
|
234
239
|
})
|
|
235
240
|
})
|
|
241
|
+
|
|
242
|
+
describe("FetchHttpClient trusted certificate authority", () => {
|
|
243
|
+
it("sends through fetch's own pool and has nothing to close when no CA is trusted", async () => {
|
|
244
|
+
const client = await build()
|
|
245
|
+
const response = responseOf()
|
|
246
|
+
response.text.mockResolvedValue("{}")
|
|
247
|
+
const send = jest.spyOn(global, "fetch").mockResolvedValue(response)
|
|
248
|
+
|
|
249
|
+
await client.request({ correlationId: "request-1", method: "GET", url: "https://service.test/orders", timeoutMs: 2500 })
|
|
250
|
+
|
|
251
|
+
expect(send.mock.calls[0]?.[1]).not.toHaveProperty("dispatcher")
|
|
252
|
+
await expect(client.onModuleDestroy()).resolves.toBeUndefined()
|
|
253
|
+
})
|
|
254
|
+
|
|
255
|
+
it("sends every request through one pool that trusts the CA, and closes it when the app stops", async () => {
|
|
256
|
+
const client = await build({ trustedCa: "the run's fake CA" })
|
|
257
|
+
const response = responseOf()
|
|
258
|
+
response.text.mockResolvedValue("{}")
|
|
259
|
+
const send = jest.spyOn(global, "fetch").mockResolvedValue(response)
|
|
260
|
+
|
|
261
|
+
await client.request({ correlationId: "request-1", method: "GET", url: "https://idp.test/jwks", timeoutMs: 2500 })
|
|
262
|
+
await client.request({ correlationId: "request-2", method: "GET", url: "https://service.test/orders", timeoutMs: 2500 })
|
|
263
|
+
|
|
264
|
+
const pool = send.mock.calls[0]?.[1]?.dispatcher
|
|
265
|
+
expect(pool).toBeInstanceOf(Agent)
|
|
266
|
+
expect(send.mock.calls[1]?.[1]?.dispatcher).toBe(pool)
|
|
267
|
+
await client.onModuleDestroy()
|
|
268
|
+
expect(pool).toHaveProperty("closed", true)
|
|
269
|
+
})
|
|
270
|
+
})
|
|
@@ -1,11 +1,34 @@
|
|
|
1
|
+
import { rootCertificates } from "node:tls"
|
|
1
2
|
import { Injectable } from "@nestjs/common"
|
|
3
|
+
import type { OnModuleDestroy } from "@nestjs/common"
|
|
4
|
+
import { Agent } from "undici"
|
|
2
5
|
import { isRecord } from "@modules/platform/primitives"
|
|
3
6
|
import { HttpError, HttpErrorCode } from "./errors/http.error"
|
|
7
|
+
import { InjectHttpOptions } from "./http.decorators"
|
|
8
|
+
import type { HttpOptions } from "./http.options"
|
|
4
9
|
import type { HttpClient, HttpRequest, HttpResponse } from "./http.port"
|
|
5
10
|
|
|
6
11
|
@Injectable()
|
|
7
12
|
/** The HttpClient adapter over the global `fetch`; the only place outside a spec that calls it. */
|
|
8
|
-
export class FetchHttpClient implements HttpClient {
|
|
13
|
+
export class FetchHttpClient implements HttpClient, OnModuleDestroy {
|
|
14
|
+
/**
|
|
15
|
+
* The connection pool that adds `trustedCa` to Node's bundled roots (the CA joins the trusted list, verification stays
|
|
16
|
+
* on); absent without one, so every request goes through fetch's own default pool as before.
|
|
17
|
+
*/
|
|
18
|
+
private readonly dispatcher: Agent | undefined
|
|
19
|
+
|
|
20
|
+
constructor(@InjectHttpOptions() options: HttpOptions) {
|
|
21
|
+
this.dispatcher =
|
|
22
|
+
options.trustedCa === undefined
|
|
23
|
+
? undefined
|
|
24
|
+
: new Agent({ connect: { ca: [...rootCertificates, options.trustedCa] } })
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Closes the pool of the trusted CA, if the app configured one, when the app stops. */
|
|
28
|
+
async onModuleDestroy(): Promise<void> {
|
|
29
|
+
await this.dispatcher?.close()
|
|
30
|
+
}
|
|
31
|
+
|
|
9
32
|
/** Sends `request` with its deadline and reads the body: parsed JSON, or the raw bytes when the request asks for them. */
|
|
10
33
|
async request(request: HttpRequest): Promise<HttpResponse> {
|
|
11
34
|
const deadline = AbortSignal.timeout(request.timeoutMs)
|
|
@@ -38,17 +61,27 @@ export class FetchHttpClient implements HttpClient {
|
|
|
38
61
|
private async send(request: HttpRequest, signal: AbortSignal): Promise<Response> {
|
|
39
62
|
try {
|
|
40
63
|
const body = this.bodyOf(request)
|
|
41
|
-
|
|
64
|
+
const init: RequestInit = {
|
|
42
65
|
method: request.method,
|
|
43
66
|
headers: this.headersOf(request),
|
|
44
67
|
...(body === undefined ? {} : { body }),
|
|
45
68
|
signal,
|
|
46
|
-
}
|
|
69
|
+
}
|
|
70
|
+
return await fetch(request.url, this.routed(init))
|
|
47
71
|
} catch (cause) {
|
|
48
72
|
throw this.failure(cause)
|
|
49
73
|
}
|
|
50
74
|
}
|
|
51
75
|
|
|
76
|
+
/**
|
|
77
|
+
* `init` sent through the trusted-CA pool when there is one. @types/node declares fetch over its own copy of undici's
|
|
78
|
+
* types, which the Agent of the undici package does not match nominally; `Object.assign` keeps the declared type of
|
|
79
|
+
* `init` and hands fetch the Agent itself.
|
|
80
|
+
*/
|
|
81
|
+
private routed(init: RequestInit): RequestInit {
|
|
82
|
+
return this.dispatcher === undefined ? init : Object.assign(init, { dispatcher: this.dispatcher })
|
|
83
|
+
}
|
|
84
|
+
|
|
52
85
|
/** The explicit id owns correlation; a copied traceparent would take precedence at the receiving logger. */
|
|
53
86
|
private headersOf(request: HttpRequest): Record<string, string> {
|
|
54
87
|
const extra = Object.fromEntries(
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { EnvSource } from "@modules/platform/config"
|
|
2
|
+
import type { HttpOptions } from "./http.options"
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Reads the http options: `HTTP_TRUSTED_CA` (or the file `HTTP_TRUSTED_CA_FILE` names) is the PEM of one more certificate
|
|
6
|
+
* authority the outbound client trusts beside Node's bundled roots; undeclared, the client trusts the bundled roots alone.
|
|
7
|
+
*/
|
|
8
|
+
export const parseHttpConfig = (env: EnvSource): HttpOptions => {
|
|
9
|
+
const trustedCa = env.optional("HTTP_TRUSTED_CA")
|
|
10
|
+
return trustedCa === undefined ? {} : { trustedCa }
|
|
11
|
+
}
|
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
import { injector } from "@modules/platform/composition"
|
|
2
2
|
import type { TypedParameterDecorator } from "@modules/platform/composition"
|
|
3
|
+
import type { HttpOptions } from "./http.options"
|
|
3
4
|
import type { HttpClient } from "./http.port"
|
|
4
5
|
|
|
6
|
+
/** Token of the http options, exported so a spec can provide it. */
|
|
7
|
+
export const HTTP_OPTIONS: unique symbol = Symbol("platform.http.options")
|
|
8
|
+
|
|
5
9
|
/** Token of the HttpClient port. */
|
|
6
10
|
export const HTTP_CLIENT: unique symbol = Symbol("platform.http.client")
|
|
7
11
|
|
|
12
|
+
/** Injects the options of the http capability. Parameter type: HttpOptions. */
|
|
13
|
+
export const InjectHttpOptions = (): TypedParameterDecorator<HttpOptions> => injector<HttpOptions>(HTTP_OPTIONS)
|
|
14
|
+
|
|
8
15
|
/** Injects the HttpClient port. Parameter type: HttpClient. */
|
|
9
16
|
export const InjectHttpClient = (): TypedParameterDecorator<HttpClient> => injector<HttpClient>(HTTP_CLIENT)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { ConfigurableModuleBuilder } from "@nestjs/common"
|
|
2
|
+
import { HTTP_OPTIONS } from "./http.decorators"
|
|
2
3
|
import type { HttpOptions } from "./http.options"
|
|
3
4
|
|
|
4
5
|
/** The configurable-module base of the http capability; `isGlobal` is decided by the app root. */
|
|
5
|
-
export const { ConfigurableModuleClass, OPTIONS_TYPE } =
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
export const { ConfigurableModuleClass, OPTIONS_TYPE } =
|
|
7
|
+
new ConfigurableModuleBuilder<HttpOptions>({ optionsInjectionToken: HTTP_OPTIONS })
|
|
8
|
+
.setExtras({ isGlobal: false }, (definition, extras) => ({ ...definition, global: extras.isGlobal }))
|
|
9
|
+
.build()
|
|
@@ -1,2 +1,9 @@
|
|
|
1
|
-
/** The http capability
|
|
2
|
-
export
|
|
1
|
+
/** The options of the http capability: what the outbound client trusts beyond Node's bundled certificate authorities. */
|
|
2
|
+
export interface HttpOptions {
|
|
3
|
+
/**
|
|
4
|
+
* PEM of one more certificate authority the client trusts, beside Node's bundled roots and never instead of them, so
|
|
5
|
+
* every other host is still verified as before (the test world passes its per-run fake CA, `w.fake.<name>.ca`).
|
|
6
|
+
* Absent, the client trusts the bundled roots alone. Verification is never turned off.
|
|
7
|
+
*/
|
|
8
|
+
readonly trustedCa?: string
|
|
9
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { Test } from "@nestjs/testing"
|
|
2
|
+
import { mockEntityManager } from "@starci/jest-preset"
|
|
3
|
+
import { advisoryXactLock, callDbFunction, notify } from "./database.client"
|
|
4
|
+
import { DatabaseError, DatabaseErrorCode } from "./errors/database.error"
|
|
5
|
+
|
|
6
|
+
const DATABASE_CLIENT = Symbol("database-client")
|
|
7
|
+
|
|
8
|
+
const setup = async () => {
|
|
9
|
+
const moduleRef = await Test.createTestingModule({
|
|
10
|
+
providers: [{ provide: DATABASE_CLIENT, useFactory: () => ({ advisoryXactLock, callDbFunction, notify }) }],
|
|
11
|
+
}).compile()
|
|
12
|
+
return moduleRef.get<{
|
|
13
|
+
readonly advisoryXactLock: typeof advisoryXactLock
|
|
14
|
+
readonly callDbFunction: typeof callDbFunction
|
|
15
|
+
readonly notify: typeof notify
|
|
16
|
+
}>(DATABASE_CLIENT)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** What `callDbFunction` throws for `name`, or null when it builds a call. */
|
|
20
|
+
const refusalOf = async (name: string): Promise<unknown> => {
|
|
21
|
+
const client = await setup()
|
|
22
|
+
try {
|
|
23
|
+
await client.callDbFunction(mockEntityManager({ query: ['SELECT * FROM "public"."never"()', []] }), name, [])
|
|
24
|
+
return null
|
|
25
|
+
} catch (error) {
|
|
26
|
+
return error
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
describe("callDbFunction", () => {
|
|
31
|
+
it("calls a schema-qualified function with each part quoted and the arguments as numbered parameters", async () => {
|
|
32
|
+
const client = await setup()
|
|
33
|
+
const manager = mockEntityManager({ query: ['SELECT * FROM "public"."job_claim"($1, $2)', [{ id: "job-1" }]] })
|
|
34
|
+
|
|
35
|
+
const rows = await client.callDbFunction<{ id: string }>(manager, "public.job_claim", ["mail", 3])
|
|
36
|
+
|
|
37
|
+
expect(rows).toEqual([{ id: "job-1" }])
|
|
38
|
+
expect(manager.query).toHaveBeenCalledWith('SELECT * FROM "public"."job_claim"($1, $2)', ["mail", 3])
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
it("calls a bare pg built-in of the canon's list in pg_catalog", async () => {
|
|
42
|
+
const client = await setup()
|
|
43
|
+
const manager = mockEntityManager({ query: ['SELECT * FROM "pg_catalog"."to_regproc"($1)', [{ to_regproc: null }]] })
|
|
44
|
+
|
|
45
|
+
const rows = await client.callDbFunction<{ to_regproc: string | null }>(manager, "to_regproc", ["public.job_claim"])
|
|
46
|
+
|
|
47
|
+
expect(rows).toEqual([{ to_regproc: null }])
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
it("calls a function of another schema with no arguments", async () => {
|
|
51
|
+
const client = await setup()
|
|
52
|
+
const manager = mockEntityManager({ query: ['SELECT * FROM "be_access"."chat_target_member"()', []] })
|
|
53
|
+
|
|
54
|
+
const rows = await client.callDbFunction(manager, "be_access.chat_target_member", [])
|
|
55
|
+
|
|
56
|
+
expect(rows).toEqual([])
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
it.each([
|
|
60
|
+
["a bare name that is not a listed built-in", "job_claim"],
|
|
61
|
+
["a pg_catalog name spelt out", "pg_catalog.pg_terminate_backend"],
|
|
62
|
+
["a quote inside a part", 'public."job_claim'],
|
|
63
|
+
["a second statement smuggled into the name", "public.job_claim(); SELECT 1; --"],
|
|
64
|
+
["an upper-case part", "public.Job_Claim"],
|
|
65
|
+
["three parts", "db.public.job_claim"],
|
|
66
|
+
["an empty schema", ".job_claim"],
|
|
67
|
+
["an empty name", ""],
|
|
68
|
+
])("refuses %s before any SQL runs", async (_case, name) => {
|
|
69
|
+
const refusal = await refusalOf(name)
|
|
70
|
+
|
|
71
|
+
expect(refusal).toBeInstanceOf(DatabaseError)
|
|
72
|
+
expect(refusal).toMatchObject({ code: DatabaseErrorCode.IdentifierRejected, params: { name } })
|
|
73
|
+
})
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
describe("advisoryXactLock", () => {
|
|
77
|
+
it("locks the 64-bit hash of the key for the transaction", async () => {
|
|
78
|
+
const client = await setup()
|
|
79
|
+
const manager = mockEntityManager({ query: ["SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [{}]] })
|
|
80
|
+
|
|
81
|
+
await expect(client.advisoryXactLock(manager, "jobs:mail")).resolves.toBeUndefined()
|
|
82
|
+
|
|
83
|
+
expect(manager.query).toHaveBeenCalledWith("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", ["jobs:mail"])
|
|
84
|
+
})
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
describe("notify", () => {
|
|
88
|
+
it("sends the payload on the channel", async () => {
|
|
89
|
+
const client = await setup()
|
|
90
|
+
const manager = mockEntityManager({ query: ["SELECT pg_notify($1, $2)", [{}]] })
|
|
91
|
+
|
|
92
|
+
await expect(client.notify(manager, "jobs", '{"id":"job-1"}')).resolves.toBeUndefined()
|
|
93
|
+
|
|
94
|
+
expect(manager.query).toHaveBeenCalledWith("SELECT pg_notify($1, $2)", ["jobs", '{"id":"job-1"}'])
|
|
95
|
+
})
|
|
96
|
+
})
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { EntityManager } from "typeorm"
|
|
2
|
+
import { DatabaseError, DatabaseErrorCode } from "./errors/database.error"
|
|
3
|
+
|
|
4
|
+
declare const functionPartBrand: unique symbol
|
|
5
|
+
declare const parameterListBrand: unique symbol
|
|
6
|
+
|
|
7
|
+
/** One part of a database function name that passed the identifier check: the only caller text a call statement holds. */
|
|
8
|
+
type DbFunctionPart = string & { readonly [functionPartBrand]: true }
|
|
9
|
+
|
|
10
|
+
/** The numbered parameters `$1, ..., $n` of one call, built from a count alone. */
|
|
11
|
+
type DbParameterList = string & { readonly [parameterListBrand]: true }
|
|
12
|
+
|
|
13
|
+
/** A function the call statement names: its schema and its name, each part checked. */
|
|
14
|
+
interface DbFunctionTarget {
|
|
15
|
+
readonly schema: DbFunctionPart
|
|
16
|
+
readonly name: DbFunctionPart
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** The pg built-ins the canon lists bare in ruleParams.be.dbFunctions; each is called in pg_catalog. */
|
|
20
|
+
const PG_BUILTINS: ReadonlySet<string> = new Set([
|
|
21
|
+
"pg_advisory_xact_lock",
|
|
22
|
+
"hashtextextended",
|
|
23
|
+
"pg_notify",
|
|
24
|
+
"to_regproc",
|
|
25
|
+
"gen_random_uuid",
|
|
26
|
+
])
|
|
27
|
+
|
|
28
|
+
/** The schema that holds the pg built-ins. */
|
|
29
|
+
const BUILTIN_SCHEMA = "pg_catalog"
|
|
30
|
+
|
|
31
|
+
/** One unquoted lower-case identifier: a part of this shape needs no escaping inside double quotes. */
|
|
32
|
+
const PART_PATTERN = /^[a-z_][a-z0-9_]*$/
|
|
33
|
+
|
|
34
|
+
function assertFunctionPart(_value: string): asserts _value is DbFunctionPart {
|
|
35
|
+
// The brand is compile-time only; `functionPart` is the single caller after checking the part.
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function assertParameterList(_value: string): asserts _value is DbParameterList {
|
|
39
|
+
// The brand is compile-time only; `parameterList` is the single caller.
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The refusal of a name that is not a callable database function. */
|
|
43
|
+
const rejected = (name: string): DatabaseError =>
|
|
44
|
+
new DatabaseError({ code: DatabaseErrorCode.IdentifierRejected, params: { name } })
|
|
45
|
+
|
|
46
|
+
/** Checks one part of a function name against the identifier shape and brands it. */
|
|
47
|
+
const functionPart = (part: string, name: string): DbFunctionPart => {
|
|
48
|
+
if (!PART_PATTERN.test(part)) throw rejected(name)
|
|
49
|
+
assertFunctionPart(part)
|
|
50
|
+
return part
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The function a name calls: `schema.name` for a function of a migration (never in pg_catalog), or a bare pg built-in of
|
|
55
|
+
* the canon's list, called in pg_catalog. Anything else is refused before any SQL is built.
|
|
56
|
+
*/
|
|
57
|
+
const functionTarget = (name: string): DbFunctionTarget => {
|
|
58
|
+
const parts = name.split(".")
|
|
59
|
+
if (parts.length === 1 && PG_BUILTINS.has(name)) {
|
|
60
|
+
return { schema: functionPart(BUILTIN_SCHEMA, name), name: functionPart(name, name) }
|
|
61
|
+
}
|
|
62
|
+
const [schema, fn] = parts
|
|
63
|
+
if (parts.length !== 2 || schema === undefined || fn === undefined || schema === BUILTIN_SCHEMA) throw rejected(name)
|
|
64
|
+
return { schema: functionPart(schema, name), name: functionPart(fn, name) }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The numbered parameter list of a call with `count` arguments. */
|
|
68
|
+
const parameterList = (count: number): DbParameterList => {
|
|
69
|
+
const list = Array.from({ length: count }, (_, index) => `$${String(index + 1)}`).join(", ")
|
|
70
|
+
assertParameterList(list)
|
|
71
|
+
return list
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Calls one database function TypeORM cannot express (a pg built-in or a SECURITY DEFINER function of a migration) with
|
|
76
|
+
* numbered parameters. The name is schema-qualified (`public.job_claim`) or a bare built-in of the canon's list, each part
|
|
77
|
+
* is checked and double-quoted, and the arguments are values, never text of the statement (R334, R68).
|
|
78
|
+
*/
|
|
79
|
+
export const callDbFunction = <T>(manager: EntityManager, name: string, args: ReadonlyArray<unknown>): Promise<ReadonlyArray<T>> => {
|
|
80
|
+
const target = functionTarget(name)
|
|
81
|
+
const parameters = parameterList(args.length)
|
|
82
|
+
return manager.query<Array<T>>(`SELECT * FROM "${target.schema}"."${target.name}"(${parameters})`, [...args])
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Takes a transaction-scoped advisory lock on the 64-bit hash of `key`; it is released when the manager's transaction ends. */
|
|
86
|
+
export const advisoryXactLock = async (manager: EntityManager, key: string): Promise<void> => {
|
|
87
|
+
await manager.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [key])
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Sends one NOTIFY on `channel` with `payload`, delivered when the manager's transaction commits. */
|
|
91
|
+
export const notify = async (manager: EntityManager, channel: string, payload: string): Promise<void> => {
|
|
92
|
+
await manager.query("SELECT pg_notify($1, $2)", [channel, payload])
|
|
93
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { DatabaseModule } from "./database.module"
|
|
2
|
-
export { advisoryXactLock, callDbFunction, notify } from "./database.
|
|
2
|
+
export { advisoryXactLock, callDbFunction, notify } from "./database.client"
|
|
3
3
|
export { DATABASE_PROBE, InjectDatabaseProbe, InjectPrimaryEntityManager, PRIMARY_ENTITY_MANAGER } from "./primary.decorators"
|
|
4
4
|
export { DatabaseProbeService } from "./database-probe.service"
|
|
5
5
|
export type { DatabaseConnectionOptions } from "./database.options"
|
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
// R258: an app whose queue is BullMQ on a declared Redis may keep its own outbox beside the canon `ruleParams.db.queueOutbox` table. The
|
|
2
|
-
// app declares it in hfs.json `sides.be.queueOutbox` ({table, enqueue, queueParameter}), and a pg_cron tick enqueues through the declared
|
|
3
|
-
// `enqueue` function only while that function's active definition does nothing but write the declared table with its queue column bound
|
|
4
|
-
// to the declared parameter. The checker reads the declaration and the function body; it names no app table or function.
|
|
5
|
-
import pg from 'libpg-query';
|
|
6
|
-
import { sqlStringParts as strings, walkSqlAst as walk, sqlParser } from '../../lib/sql-ast.mjs';
|
|
7
|
-
|
|
8
|
-
await pg.loadModule();
|
|
9
|
-
|
|
10
|
-
const sqlTree = sqlParser(pg);
|
|
11
|
-
const STATEMENTS = new Set(['PLpgSQL_stmt_block', 'PLpgSQL_stmt_execsql', 'PLpgSQL_stmt_if', 'PLpgSQL_stmt_assign', 'PLpgSQL_stmt_return']);
|
|
12
|
-
const OTHER_DML = ['UpdateStmt', 'DeleteStmt', 'MergeStmt'];
|
|
13
|
-
const qualified = (name) => name.includes('.') ? name : `public.${name}`;
|
|
14
|
-
|
|
15
|
-
/** The outbox an app declares, or null: it counts only while the back end declares the redis backing the relay hands rows to. */
|
|
16
|
-
export function queueOutboxDeclaration(declaration) {
|
|
17
|
-
const be = declaration?.sides?.be;
|
|
18
|
-
const declared = be?.queueOutbox;
|
|
19
|
-
if (!declared || !(be.backing ?? []).includes('redis')) return null;
|
|
20
|
-
return { table: qualified(declared.table), enqueue: qualified(declared.enqueue), parameter: declared.queueParameter };
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function plpgsqlStatements(fn, body) {
|
|
24
|
-
const parameters = (fn.parameters ?? []).map((item) => item.FunctionParameter?.name).filter(Boolean).map((name) => `"${name.replaceAll('"', '""')}" text`).join(',');
|
|
25
|
-
let tag = '$hfs_outbox$';
|
|
26
|
-
while (body.includes(tag)) tag = `${tag.slice(0, -1)}_$`;
|
|
27
|
-
let tree;
|
|
28
|
-
try { tree = pg.parsePlPgSQLSync(`create function hfs_probe(${parameters}) returns text language plpgsql as ${tag}${body}${tag};`); } catch { return null; }
|
|
29
|
-
const trees = [];
|
|
30
|
-
let safe = true;
|
|
31
|
-
walk(tree, (node) => {
|
|
32
|
-
for (const key of Object.keys(node)) if (key.startsWith('PLpgSQL_stmt_') && !STATEMENTS.has(key)) safe = false;
|
|
33
|
-
const expr = node.PLpgSQL_expr;
|
|
34
|
-
if (expr) trees.push(sqlTree(expr.parseMode === 0 ? expr.query : `select ${expr.query}`));
|
|
35
|
-
});
|
|
36
|
-
return safe ? trees : null;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function bodyStatements(fn) {
|
|
40
|
-
const option = (name) => (fn.options ?? []).find((item) => item.DefElem?.defname === name)?.DefElem?.arg;
|
|
41
|
-
const language = option('language')?.String?.sval ?? 'sql';
|
|
42
|
-
const body = strings(option('as')?.List?.items).join('');
|
|
43
|
-
if (language === 'plpgsql') return plpgsqlStatements(fn, body);
|
|
44
|
-
if (language !== 'sql') return null;
|
|
45
|
-
if (fn.sql_body) return (fn.sql_body.List ? fn.sql_body.List.items : [fn.sql_body]).map((stmt) => ({ stmts: [{ stmt }] }));
|
|
46
|
-
return sqlTree(body)?.stmts?.map((entry) => ({ stmts: [entry] })) ?? null;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/** True when the nodes hold no sub-select or other statement and call only functions `pureCall` admits. */
|
|
50
|
-
function pureNodes(nodes, pureCall) {
|
|
51
|
-
let ok = true;
|
|
52
|
-
walk(nodes, (node) => {
|
|
53
|
-
if (node.SubLink || node.SelectStmt || node.InsertStmt || OTHER_DML.some((key) => node[key])) ok = false;
|
|
54
|
-
if (node.FuncCall && !pureCall(node.FuncCall)) ok = false;
|
|
55
|
-
});
|
|
56
|
-
return ok;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/** `insert into <table> (..., queue, ...) values (..., <parameter>, ...) [on conflict .. do nothing] [returning ..]` with pure other values. */
|
|
60
|
-
function insertOk(declared, insert, pureCall) {
|
|
61
|
-
if (insert.withClause || `${insert.relation?.schemaname || 'public'}.${insert.relation?.relname}` !== declared.table) return false;
|
|
62
|
-
if (insert.onConflictClause && insert.onConflictClause.action !== 'ONCONFLICT_NOTHING') return false;
|
|
63
|
-
const columns = (insert.cols ?? []).map((item) => item.ResTarget?.name);
|
|
64
|
-
const rows = insert.selectStmt?.SelectStmt?.valuesLists ?? [];
|
|
65
|
-
const values = rows.length === 1 ? rows[0].List?.items ?? [] : [];
|
|
66
|
-
const at = columns.indexOf('queue');
|
|
67
|
-
if (at < 0 || new Set(columns).size !== columns.length || values.length !== columns.length) return false;
|
|
68
|
-
const bound = values[at].ColumnRef?.fields;
|
|
69
|
-
if (!bound || strings(bound).join('.') !== declared.parameter || bound.length !== 1) return false;
|
|
70
|
-
return pureNodes(values.filter((_, index) => index !== at), pureCall);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/** A read of the declared table with pure expressions, or an expression over variables alone. */
|
|
74
|
-
function readOk(declared, tree, pureCall) {
|
|
75
|
-
let ok = true;
|
|
76
|
-
walk(tree, (node) => {
|
|
77
|
-
if (OTHER_DML.some((key) => node[key]) || node.InsertStmt || node.SelectStmt?.intoClause) ok = false;
|
|
78
|
-
if (node.FuncCall && !pureCall(node.FuncCall)) ok = false;
|
|
79
|
-
if (node.RangeVar && `${node.RangeVar.schemaname || 'public'}.${node.RangeVar.relname}` !== declared.table) ok = false;
|
|
80
|
-
});
|
|
81
|
-
return ok;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
function statementOk(declared, tree, pureCall) {
|
|
85
|
-
const stmt = tree?.stmts?.length === 1 ? tree.stmts[0].stmt : null;
|
|
86
|
-
if (stmt?.InsertStmt) return insertOk(declared, stmt.InsertStmt, pureCall);
|
|
87
|
-
return Boolean(stmt?.SelectStmt) && readOk(declared, tree, pureCall);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* The position of the queue parameter when the active definition of the declared enqueue function only writes the declared outbox
|
|
92
|
-
* (inserts, reads of that table and pure expressions), or -1. `pureCall` admits a function call that does no work.
|
|
93
|
-
*/
|
|
94
|
-
export function outboxEnqueueQueueIndex(declared, entry, pureCall) {
|
|
95
|
-
const fn = entry?.node;
|
|
96
|
-
if (!fn) return -1;
|
|
97
|
-
const index = (fn.parameters ?? []).findIndex((item) => item.FunctionParameter?.name === declared.parameter);
|
|
98
|
-
const trees = bodyStatements(fn);
|
|
99
|
-
if (index < 0 || !trees?.length || !trees.every((tree) => statementOk(declared, tree, pureCall))) return -1;
|
|
100
|
-
const inserts = trees.filter((tree) => tree?.stmts?.[0]?.stmt?.InsertStmt).length;
|
|
101
|
-
return inserts === 1 ? index : -1;
|
|
102
|
-
}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import type { EntityManager } from "typeorm"
|
|
2
|
-
|
|
3
|
-
/** The database functions this app may call, each listed with its reason in ruleParams.be.dbFunctions or hfs.json sides.be.dbFunctions. */
|
|
4
|
-
const IDENTIFIER = /^[a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)?$/
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Calls one database function TypeORM cannot express (a pg built-in or a SECURITY DEFINER function of a migration) with numbered
|
|
8
|
-
* parameters; the function name is an identifier the canon lint holds to the listed set, never a value.
|
|
9
|
-
*/
|
|
10
|
-
export const callDbFunction = async <T>(manager: EntityManager, name: string, args: ReadonlyArray<unknown>): Promise<ReadonlyArray<T>> => {
|
|
11
|
-
if (!IDENTIFIER.test(name)) throw new TypeError(`${name} is not a database function name`)
|
|
12
|
-
const placeholders = args.map((_, index) => `$${String(index + 1)}`).join(", ")
|
|
13
|
-
return manager.query<Array<T>>(`SELECT * FROM ${name}(${placeholders})`, [...args])
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/** Takes a transaction-scoped advisory lock on the 64-bit hash of `key`; it is released when the manager's transaction ends. */
|
|
17
|
-
export const advisoryXactLock = async (manager: EntityManager, key: string): Promise<void> => {
|
|
18
|
-
await manager.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [key])
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/** Sends one NOTIFY on `channel` with `payload`, delivered when the manager's transaction commits. */
|
|
22
|
-
export const notify = async (manager: EntityManager, channel: string, payload: string): Promise<void> => {
|
|
23
|
-
await manager.query("SELECT pg_notify($1, $2)", [channel, payload])
|
|
24
|
-
}
|