@dudousxd/nestjs-catalog 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.controller.js +36 -2
- package/dist/catalog.pipeline.d.ts +156 -13
- package/dist/catalog.pipeline.js +92 -2
- package/dist/catalog.secrets.d.ts +283 -0
- package/dist/catalog.secrets.js +209 -0
- package/dist/catalog.workspace.d.ts +154 -0
- package/dist/catalog.workspace.js +59 -1
- package/dist/client.d.ts +13 -2
- package/dist/client.js +23 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +15 -2
- package/dist/transform-runner.d.ts +22 -5
- package/dist/transform-runner.js +126 -9
- package/package.json +1 -1
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Encrypting the credential that has to rest in `catalog_connection.config`.
|
|
3
|
+
*
|
|
4
|
+
* `config-secrets.ts` in the pipeline package closed the HTTP half of this: a
|
|
5
|
+
* connection URL is a password with an address attached, `config` is served by
|
|
6
|
+
* `GET pipeline/connections` under `catalog:read`, and the redaction stopped it
|
|
7
|
+
* travelling in a response. `MySqlPipelineStore` closed the write half: a
|
|
8
|
+
* password-bearing URL that is not already stored is refused, with a message
|
|
9
|
+
* naming `secretEnvVar`.
|
|
10
|
+
*
|
|
11
|
+
* Neither touches the thing this file is about. A redaction protects a reader
|
|
12
|
+
* coming through the API. It does nothing at all for a database dump, a read
|
|
13
|
+
* replica, a nightly backup, or anybody holding `SELECT` on the RDS instance —
|
|
14
|
+
* and for those, `config` is still a list of every password the catalog knows.
|
|
15
|
+
* `allowInlineCredentials` makes that population larger on purpose, which is
|
|
16
|
+
* precisely why encryption becomes worth building at the same time as the flag
|
|
17
|
+
* that needs it.
|
|
18
|
+
*
|
|
19
|
+
* ## A seam, not an implementation
|
|
20
|
+
*
|
|
21
|
+
* There is no cipher in this file and there must not be one. A library that
|
|
22
|
+
* shipped AES with a key read from an environment variable would have moved the
|
|
23
|
+
* problem from one column to one variable, and would have to answer for key
|
|
24
|
+
* rotation, per-environment separation, and an audit trail — all of which the
|
|
25
|
+
* host's KMS, Vault, or HSM already answers for, and none of which this package
|
|
26
|
+
* can answer for on its behalf. What is here is the shape of the question, so a
|
|
27
|
+
* provider can be written against it: {@link CatalogSecretVault}.
|
|
28
|
+
*
|
|
29
|
+
* The library never inspects a ciphertext, never chooses a key, and never
|
|
30
|
+
* decides an algorithm. It decides *which values* are secrets, *when* they are
|
|
31
|
+
* sealed, and *what happens when the vault cannot be reached* — three questions
|
|
32
|
+
* a provider should not have to have an opinion about.
|
|
33
|
+
*
|
|
34
|
+
* ## What the default does
|
|
35
|
+
*
|
|
36
|
+
* Refuses. {@link RefusingSecretVault} throws on `seal`, naming the token to
|
|
37
|
+
* bind. A default that quietly stored plaintext would be the worst shape
|
|
38
|
+
* available here: a host would turn `encryptCredentials` on, see saves succeed,
|
|
39
|
+
* and have exactly the same column contents as before with a docblock now
|
|
40
|
+
* claiming otherwise. Every other refusing default in this codebase exists for
|
|
41
|
+
* the same reason — `CATALOG_LOAD_EXPECTATIONS` unbound means incremental loads
|
|
42
|
+
* are refused rather than silently unpoliced.
|
|
43
|
+
*/
|
|
44
|
+
/**
|
|
45
|
+
* What a vault hands back. Stored verbatim; opaque to this library.
|
|
46
|
+
*
|
|
47
|
+
* Three fields and no fourth, because everything a rotation needs is here and
|
|
48
|
+
* nothing a reader could misuse is: the ciphertext says nothing without the
|
|
49
|
+
* vault, and the vault says nothing without the key. It is a plain JSON object
|
|
50
|
+
* so that it can sit in the same `json` column the plaintext sat in — no
|
|
51
|
+
* migration, no second table, and a column that may hold either form for as
|
|
52
|
+
* long as it takes a deployment to move over. {@link isSealedSecret} is what
|
|
53
|
+
* tells the two apart on the way out.
|
|
54
|
+
*/
|
|
55
|
+
export interface SealedSecret {
|
|
56
|
+
/** Which vault sealed it — `CatalogSecretVault.name`. */
|
|
57
|
+
vault: string;
|
|
58
|
+
/** Which key, in whatever form that vault names keys. */
|
|
59
|
+
keyId: string;
|
|
60
|
+
/**
|
|
61
|
+
* An opaque string the vault chose. **Not base64, and never decoded here.**
|
|
62
|
+
*
|
|
63
|
+
* This said "base64" first, and both provider implementations proved it
|
|
64
|
+
* wrong from opposite directions: HashiCorp Transit returns
|
|
65
|
+
* `vault:v1:<base64>`, which has two colons and is not decodable as base64 at
|
|
66
|
+
* all. The provider stores that verbatim and is right to — the `v1` is the
|
|
67
|
+
* row's only self-description of which key version sealed it, and
|
|
68
|
+
* `transit/rewrap` takes exactly that string back.
|
|
69
|
+
*
|
|
70
|
+
* The claim was harmless while nothing acted on it, which is what makes it
|
|
71
|
+
* worth correcting now rather than later: the day something takes it
|
|
72
|
+
* literally — a base64 `CHECK` constraint, a JSON-schema `contentEncoding`, a
|
|
73
|
+
* `Buffer.from(value, 'base64')` round trip — it breaks **silently**, because
|
|
74
|
+
* `Buffer.from` does not throw on input it cannot read. It returns different
|
|
75
|
+
* bytes.
|
|
76
|
+
*
|
|
77
|
+
* So the only rule is: non-empty, and given back to the vault exactly as it
|
|
78
|
+
* came. {@link isSealedSecret} checks that it is a non-empty string and must
|
|
79
|
+
* never be tightened past that — any narrower test encodes one vault's format
|
|
80
|
+
* as the contract and silently rejects the next one's rows.
|
|
81
|
+
*/
|
|
82
|
+
ciphertext: string;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Where a secret is being used, so a vault can scope or audit by it.
|
|
86
|
+
*
|
|
87
|
+
* Passed to `open` as well as to `seal`, and that is the interesting half: a
|
|
88
|
+
* KMS provider can put this in an encryption context so a ciphertext sealed for
|
|
89
|
+
* `connection/abc/url` cannot be replayed as `connector/xyz/url`, and a Vault
|
|
90
|
+
* provider can write an audit line naming the row somebody's credential was
|
|
91
|
+
* read for. Neither is possible if the seam only carries bytes.
|
|
92
|
+
*/
|
|
93
|
+
export interface SecretContext {
|
|
94
|
+
/** `connection` or `connector`. */
|
|
95
|
+
kind: string;
|
|
96
|
+
/**
|
|
97
|
+
* The row's id. Present and stable whenever `MySqlPipelineStore` is the
|
|
98
|
+
* caller, including on a first save.
|
|
99
|
+
*
|
|
100
|
+
* This said "absent on a first save", and both provider implementations
|
|
101
|
+
* independently left `id` out of their encryption context because of it —
|
|
102
|
+
* correctly, given what it said. A context that is absent on create and
|
|
103
|
+
* present on update seals under one context and opens under another, and the
|
|
104
|
+
* failure lands at the first connector run rather than at the save.
|
|
105
|
+
*
|
|
106
|
+
* So the store was changed rather than the providers: it mints the row's id
|
|
107
|
+
* *before* sealing instead of inside `em.create`, which costs one hoisted
|
|
108
|
+
* `randomUUID()` and makes the binding property real. A provider may now put
|
|
109
|
+
* this in an encryption context and get what the seam always claimed —
|
|
110
|
+
* `connection/abc/url` is not replayable as `connection/xyz/url`.
|
|
111
|
+
*
|
|
112
|
+
* Two things this does NOT make true, both worth stating before somebody
|
|
113
|
+
* relies on them:
|
|
114
|
+
*
|
|
115
|
+
* - It is still optional in the type, because a host calling a vault
|
|
116
|
+
* directly is not obliged to have a row. A provider that binds it must
|
|
117
|
+
* decide what an absent `id` means for its own callers.
|
|
118
|
+
* - **Rows sealed before this existed were sealed without it.** A provider
|
|
119
|
+
* that starts binding `id` on a deployment that already has sealed rows
|
|
120
|
+
* strands every one of them. Binding it is safe on a new deployment, and
|
|
121
|
+
* on an existing one only behind a re-seal.
|
|
122
|
+
*/
|
|
123
|
+
id?: string;
|
|
124
|
+
/** The config key being sealed — `url`, `password`. */
|
|
125
|
+
field: string;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The seam a host binds to make credentials unreadable in the catalog's own
|
|
129
|
+
* database.
|
|
130
|
+
*
|
|
131
|
+
* Both methods are async and both are expected to fail sometimes: they are
|
|
132
|
+
* network calls to something outside this process. What each failure *means*
|
|
133
|
+
* is not symmetric, and the store treats them differently — see
|
|
134
|
+
* {@link SecretSealFailedError} and {@link SecretOpenFailedError}.
|
|
135
|
+
*/
|
|
136
|
+
export interface CatalogSecretVault {
|
|
137
|
+
/**
|
|
138
|
+
* Stable, stored on every SealedSecret so a rotation can find its own.
|
|
139
|
+
*
|
|
140
|
+
* Stable across deployments and releases, not merely across a process: it is
|
|
141
|
+
* written into rows. Renaming it strands every row already sealed under the
|
|
142
|
+
* old name, which the store refuses loudly rather than guessing at — a bound
|
|
143
|
+
* vault whose name does not match the row's is not asked to try.
|
|
144
|
+
*/
|
|
145
|
+
readonly name: string;
|
|
146
|
+
seal(plaintext: string, context: SecretContext): Promise<SealedSecret>;
|
|
147
|
+
open(sealed: SealedSecret, context: SecretContext): Promise<string>;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The token a host binds one vault — or several — to.
|
|
151
|
+
*
|
|
152
|
+
* **Several is the supported shape, and it is what makes rotation possible.**
|
|
153
|
+
* Bind `CatalogSecretVault[]` and the store seals with the *first* and opens
|
|
154
|
+
* with whichever one's `name` matches the row. Moving from one vault to another
|
|
155
|
+
* is then: bind `[next, current]`, let saves reseal under `next`, and drop
|
|
156
|
+
* `current` when nothing is sealed under it any more. With a single vault that
|
|
157
|
+
* transition has no middle — the moment `next` is bound, every row sealed by
|
|
158
|
+
* `current` is unreadable — and a library that made rotation require an outage
|
|
159
|
+
* would be a library whose keys never got rotated.
|
|
160
|
+
*
|
|
161
|
+
* `@Optional()` where it is injected, so a host that binds nothing still boots.
|
|
162
|
+
* It then gets {@link RefusingSecretVault}, which costs nothing until something
|
|
163
|
+
* actually asks for a seal.
|
|
164
|
+
*/
|
|
165
|
+
export declare const CATALOG_SECRET_VAULT: unique symbol;
|
|
166
|
+
/**
|
|
167
|
+
* Whether a value that came back out of a JSON column is a sealed secret.
|
|
168
|
+
*
|
|
169
|
+
* A runtime guard and not a cast, because this is a `json` column: MikroORM
|
|
170
|
+
* hands back whatever is in it, and what is in it may have been written by a
|
|
171
|
+
* different release, by a hand-run `UPDATE`, or by a host's own seeder. The
|
|
172
|
+
* same argument `WorkflowRow.nodes` makes for being typed `unknown[]`.
|
|
173
|
+
*
|
|
174
|
+
* Deliberately strict about emptiness. A `{ vault: '', keyId: '', ciphertext:
|
|
175
|
+
* '' }` is not a sealed secret and must not be treated as one — treating it as
|
|
176
|
+
* one sends it to a vault that will refuse it, and the operator is told their
|
|
177
|
+
* vault is broken when the row is. An empty ciphertext falls through as an
|
|
178
|
+
* ordinary config value instead, which is what it is.
|
|
179
|
+
*
|
|
180
|
+
* Deliberately loose about everything else, and that is the harder half to
|
|
181
|
+
* hold. `ciphertext` is tested for "non-empty string" and must never be tested
|
|
182
|
+
* for more — not base64, not a length, not a prefix. Two vaults already
|
|
183
|
+
* disagree about its shape: KMS hands back base64, Transit hands back
|
|
184
|
+
* `vault:v1:<base64>`. Any tighter check would encode one of them as the
|
|
185
|
+
* contract and start silently refusing the other's rows, which surfaces as
|
|
186
|
+
* "this credential is not encrypted" for a value that is.
|
|
187
|
+
*
|
|
188
|
+
* What it cannot distinguish is a config value a host deliberately authored as
|
|
189
|
+
* an object with exactly these three non-empty string fields. Nothing prevents
|
|
190
|
+
* that and nothing sensible produces it; saying so is better than adding a
|
|
191
|
+
* marker field that would then have to be defended against forgery it also
|
|
192
|
+
* could not detect.
|
|
193
|
+
*/
|
|
194
|
+
export declare function isSealedSecret(value: unknown): value is SealedSecret;
|
|
195
|
+
/** The name {@link RefusingSecretVault} answers to. No row can carry it: it never seals. */
|
|
196
|
+
export declare const UNCONFIGURED_VAULT = "unconfigured";
|
|
197
|
+
/**
|
|
198
|
+
* The default: a vault that will not pretend.
|
|
199
|
+
*
|
|
200
|
+
* Bound whenever `CATALOG_SECRET_VAULT` is not, which is every host that has
|
|
201
|
+
* not thought about this. It costs nothing until `encryptCredentials` is turned
|
|
202
|
+
* on — the store only reaches a vault when there is something to seal or
|
|
203
|
+
* something sealed to open — so the refusal lands on the deployment that asked
|
|
204
|
+
* for encryption and did not say with what, and lands at the first save rather
|
|
205
|
+
* than at the first run.
|
|
206
|
+
*
|
|
207
|
+
* The alternative shapes were both considered and are both worse. A default
|
|
208
|
+
* that stored plaintext would make `encryptCredentials: true` a no-op with a
|
|
209
|
+
* reassuring name. A default that base64-encoded would be worse still: it looks
|
|
210
|
+
* sealed, `isSealedSecret` would agree, and a dump would be trivially reversed
|
|
211
|
+
* by anybody who noticed.
|
|
212
|
+
*/
|
|
213
|
+
export declare class RefusingSecretVault implements CatalogSecretVault {
|
|
214
|
+
readonly name = "unconfigured";
|
|
215
|
+
seal(_plaintext: string, context: SecretContext): Promise<SealedSecret>;
|
|
216
|
+
open(sealed: SealedSecret, context: SecretContext): Promise<string>;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Nothing is bound, or nothing bound answers to the name a row carries.
|
|
220
|
+
*
|
|
221
|
+
* `retryable = false`, and that field is not decoration. A connector run is
|
|
222
|
+
* dispatched as a durable step, and the dispatch boundary serialises a throw
|
|
223
|
+
* into `{message, code, retryable}` — `retryable` is the only field the engine
|
|
224
|
+
* reads on the way back in, and its predicate is `error?.retryable !== false`.
|
|
225
|
+
* A missing token will be exactly as missing on the third attempt as on the
|
|
226
|
+
* first, so this must not burn fifteen minutes of exponential backoff before
|
|
227
|
+
* saying so. `connector-run.steps.ts` documents that mechanism at length and
|
|
228
|
+
* this is the same use of it.
|
|
229
|
+
*/
|
|
230
|
+
export declare class SecretVaultNotConfiguredError extends Error {
|
|
231
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
232
|
+
readonly retryable = false;
|
|
233
|
+
readonly code = "secret_vault_not_configured";
|
|
234
|
+
constructor(message: string);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* A save could not seal a credential, so the save did not happen.
|
|
238
|
+
*
|
|
239
|
+
* Which is the correct outcome and worth being explicit about: the fallback a
|
|
240
|
+
* reasonable person reaches for — write it in plaintext and log a warning — is
|
|
241
|
+
* the one thing this feature exists to make impossible. A deployment that
|
|
242
|
+
* turned `encryptCredentials` on and then, during a vault outage, quietly wrote
|
|
243
|
+
* three passwords in the clear would have no way to find out which three.
|
|
244
|
+
*
|
|
245
|
+
* Deliberately not a `BadRequestException`. The caller's URL is fine; the vault
|
|
246
|
+
* is down. A 400 sends somebody to edit a field that is correct.
|
|
247
|
+
*/
|
|
248
|
+
export declare class SecretSealFailedError extends Error {
|
|
249
|
+
readonly code = "secret_seal_failed";
|
|
250
|
+
constructor(message: string, options?: {
|
|
251
|
+
cause?: unknown;
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* A read could not open a sealed credential.
|
|
256
|
+
*
|
|
257
|
+
* **This is the one whose class matters**, because a connector run is a durable
|
|
258
|
+
* step and the step decides whether to retry from the error it catches. Two
|
|
259
|
+
* things had to be got right here and both are load-bearing:
|
|
260
|
+
*
|
|
261
|
+
* - It is **not** a `NotFoundException` or a `BadRequestException`.
|
|
262
|
+
* `ConnectorRunSteps.runConnector` catches exactly those two around the
|
|
263
|
+
* runner and converts them to `UnavailableConnectorError`, which carries
|
|
264
|
+
* `retryable = false`. A vault that timed out would then be filed under
|
|
265
|
+
* `connector_unavailable` and never retried — and an operator filtering a
|
|
266
|
+
* failed-run list on that code would go looking for a connector somebody
|
|
267
|
+
* deleted. That is precisely the confusion `UnmetLoadExpectationError` was
|
|
268
|
+
* split into its own class to avoid.
|
|
269
|
+
* - `retryable` is **true by default**. A vault being briefly unreachable is
|
|
270
|
+
* the same kind of failure as a source being briefly unreachable, which is
|
|
271
|
+
* exactly what that step's `retries: 3, backoff exp 60s…900s` policy exists
|
|
272
|
+
* for. It is set false only when waiting provably cannot help: nothing is
|
|
273
|
+
* bound, or nothing bound answers to the name the row carries.
|
|
274
|
+
*/
|
|
275
|
+
export declare class SecretOpenFailedError extends Error {
|
|
276
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
277
|
+
readonly retryable: boolean;
|
|
278
|
+
readonly code = "secret_open_failed";
|
|
279
|
+
constructor(message: string, options: {
|
|
280
|
+
retryable: boolean;
|
|
281
|
+
cause?: unknown;
|
|
282
|
+
});
|
|
283
|
+
}
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Encrypting the credential that has to rest in `catalog_connection.config`.
|
|
4
|
+
*
|
|
5
|
+
* `config-secrets.ts` in the pipeline package closed the HTTP half of this: a
|
|
6
|
+
* connection URL is a password with an address attached, `config` is served by
|
|
7
|
+
* `GET pipeline/connections` under `catalog:read`, and the redaction stopped it
|
|
8
|
+
* travelling in a response. `MySqlPipelineStore` closed the write half: a
|
|
9
|
+
* password-bearing URL that is not already stored is refused, with a message
|
|
10
|
+
* naming `secretEnvVar`.
|
|
11
|
+
*
|
|
12
|
+
* Neither touches the thing this file is about. A redaction protects a reader
|
|
13
|
+
* coming through the API. It does nothing at all for a database dump, a read
|
|
14
|
+
* replica, a nightly backup, or anybody holding `SELECT` on the RDS instance —
|
|
15
|
+
* and for those, `config` is still a list of every password the catalog knows.
|
|
16
|
+
* `allowInlineCredentials` makes that population larger on purpose, which is
|
|
17
|
+
* precisely why encryption becomes worth building at the same time as the flag
|
|
18
|
+
* that needs it.
|
|
19
|
+
*
|
|
20
|
+
* ## A seam, not an implementation
|
|
21
|
+
*
|
|
22
|
+
* There is no cipher in this file and there must not be one. A library that
|
|
23
|
+
* shipped AES with a key read from an environment variable would have moved the
|
|
24
|
+
* problem from one column to one variable, and would have to answer for key
|
|
25
|
+
* rotation, per-environment separation, and an audit trail — all of which the
|
|
26
|
+
* host's KMS, Vault, or HSM already answers for, and none of which this package
|
|
27
|
+
* can answer for on its behalf. What is here is the shape of the question, so a
|
|
28
|
+
* provider can be written against it: {@link CatalogSecretVault}.
|
|
29
|
+
*
|
|
30
|
+
* The library never inspects a ciphertext, never chooses a key, and never
|
|
31
|
+
* decides an algorithm. It decides *which values* are secrets, *when* they are
|
|
32
|
+
* sealed, and *what happens when the vault cannot be reached* — three questions
|
|
33
|
+
* a provider should not have to have an opinion about.
|
|
34
|
+
*
|
|
35
|
+
* ## What the default does
|
|
36
|
+
*
|
|
37
|
+
* Refuses. {@link RefusingSecretVault} throws on `seal`, naming the token to
|
|
38
|
+
* bind. A default that quietly stored plaintext would be the worst shape
|
|
39
|
+
* available here: a host would turn `encryptCredentials` on, see saves succeed,
|
|
40
|
+
* and have exactly the same column contents as before with a docblock now
|
|
41
|
+
* claiming otherwise. Every other refusing default in this codebase exists for
|
|
42
|
+
* the same reason — `CATALOG_LOAD_EXPECTATIONS` unbound means incremental loads
|
|
43
|
+
* are refused rather than silently unpoliced.
|
|
44
|
+
*/
|
|
45
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
|
+
exports.SecretOpenFailedError = exports.SecretSealFailedError = exports.SecretVaultNotConfiguredError = exports.RefusingSecretVault = exports.UNCONFIGURED_VAULT = exports.CATALOG_SECRET_VAULT = void 0;
|
|
47
|
+
exports.isSealedSecret = isSealedSecret;
|
|
48
|
+
/**
|
|
49
|
+
* The token a host binds one vault — or several — to.
|
|
50
|
+
*
|
|
51
|
+
* **Several is the supported shape, and it is what makes rotation possible.**
|
|
52
|
+
* Bind `CatalogSecretVault[]` and the store seals with the *first* and opens
|
|
53
|
+
* with whichever one's `name` matches the row. Moving from one vault to another
|
|
54
|
+
* is then: bind `[next, current]`, let saves reseal under `next`, and drop
|
|
55
|
+
* `current` when nothing is sealed under it any more. With a single vault that
|
|
56
|
+
* transition has no middle — the moment `next` is bound, every row sealed by
|
|
57
|
+
* `current` is unreadable — and a library that made rotation require an outage
|
|
58
|
+
* would be a library whose keys never got rotated.
|
|
59
|
+
*
|
|
60
|
+
* `@Optional()` where it is injected, so a host that binds nothing still boots.
|
|
61
|
+
* It then gets {@link RefusingSecretVault}, which costs nothing until something
|
|
62
|
+
* actually asks for a seal.
|
|
63
|
+
*/
|
|
64
|
+
exports.CATALOG_SECRET_VAULT = Symbol('CATALOG_SECRET_VAULT');
|
|
65
|
+
/**
|
|
66
|
+
* Whether a value that came back out of a JSON column is a sealed secret.
|
|
67
|
+
*
|
|
68
|
+
* A runtime guard and not a cast, because this is a `json` column: MikroORM
|
|
69
|
+
* hands back whatever is in it, and what is in it may have been written by a
|
|
70
|
+
* different release, by a hand-run `UPDATE`, or by a host's own seeder. The
|
|
71
|
+
* same argument `WorkflowRow.nodes` makes for being typed `unknown[]`.
|
|
72
|
+
*
|
|
73
|
+
* Deliberately strict about emptiness. A `{ vault: '', keyId: '', ciphertext:
|
|
74
|
+
* '' }` is not a sealed secret and must not be treated as one — treating it as
|
|
75
|
+
* one sends it to a vault that will refuse it, and the operator is told their
|
|
76
|
+
* vault is broken when the row is. An empty ciphertext falls through as an
|
|
77
|
+
* ordinary config value instead, which is what it is.
|
|
78
|
+
*
|
|
79
|
+
* Deliberately loose about everything else, and that is the harder half to
|
|
80
|
+
* hold. `ciphertext` is tested for "non-empty string" and must never be tested
|
|
81
|
+
* for more — not base64, not a length, not a prefix. Two vaults already
|
|
82
|
+
* disagree about its shape: KMS hands back base64, Transit hands back
|
|
83
|
+
* `vault:v1:<base64>`. Any tighter check would encode one of them as the
|
|
84
|
+
* contract and start silently refusing the other's rows, which surfaces as
|
|
85
|
+
* "this credential is not encrypted" for a value that is.
|
|
86
|
+
*
|
|
87
|
+
* What it cannot distinguish is a config value a host deliberately authored as
|
|
88
|
+
* an object with exactly these three non-empty string fields. Nothing prevents
|
|
89
|
+
* that and nothing sensible produces it; saying so is better than adding a
|
|
90
|
+
* marker field that would then have to be defended against forgery it also
|
|
91
|
+
* could not detect.
|
|
92
|
+
*/
|
|
93
|
+
function isSealedSecret(value) {
|
|
94
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
95
|
+
return false;
|
|
96
|
+
const vault = Reflect.get(value, 'vault');
|
|
97
|
+
const keyId = Reflect.get(value, 'keyId');
|
|
98
|
+
const ciphertext = Reflect.get(value, 'ciphertext');
|
|
99
|
+
return (typeof vault === 'string' &&
|
|
100
|
+
vault.length > 0 &&
|
|
101
|
+
typeof keyId === 'string' &&
|
|
102
|
+
typeof ciphertext === 'string' &&
|
|
103
|
+
ciphertext.length > 0);
|
|
104
|
+
}
|
|
105
|
+
/** The name {@link RefusingSecretVault} answers to. No row can carry it: it never seals. */
|
|
106
|
+
exports.UNCONFIGURED_VAULT = 'unconfigured';
|
|
107
|
+
/**
|
|
108
|
+
* The default: a vault that will not pretend.
|
|
109
|
+
*
|
|
110
|
+
* Bound whenever `CATALOG_SECRET_VAULT` is not, which is every host that has
|
|
111
|
+
* not thought about this. It costs nothing until `encryptCredentials` is turned
|
|
112
|
+
* on — the store only reaches a vault when there is something to seal or
|
|
113
|
+
* something sealed to open — so the refusal lands on the deployment that asked
|
|
114
|
+
* for encryption and did not say with what, and lands at the first save rather
|
|
115
|
+
* than at the first run.
|
|
116
|
+
*
|
|
117
|
+
* The alternative shapes were both considered and are both worse. A default
|
|
118
|
+
* that stored plaintext would make `encryptCredentials: true` a no-op with a
|
|
119
|
+
* reassuring name. A default that base64-encoded would be worse still: it looks
|
|
120
|
+
* sealed, `isSealedSecret` would agree, and a dump would be trivially reversed
|
|
121
|
+
* by anybody who noticed.
|
|
122
|
+
*/
|
|
123
|
+
class RefusingSecretVault {
|
|
124
|
+
name = exports.UNCONFIGURED_VAULT;
|
|
125
|
+
seal(_plaintext, context) {
|
|
126
|
+
return Promise.reject(new SecretVaultNotConfiguredError(`encryptCredentials is on, so ${context.kind}.config.${context.field} has to be sealed, and no vault is bound to seal it with. Bind CATALOG_SECRET_VAULT to a CatalogSecretVault — an AWS KMS or HashiCorp Vault provider, or your own — or turn encryptCredentials off and decide what allowInlineCredentials should be instead.`));
|
|
127
|
+
}
|
|
128
|
+
open(sealed, context) {
|
|
129
|
+
// Reachable only through a row this instance did not write, since `seal`
|
|
130
|
+
// never returns — so the useful thing to say is the name the row is asking
|
|
131
|
+
// for, which is the one the operator has to go and bind.
|
|
132
|
+
return Promise.reject(new SecretVaultNotConfiguredError(`${context.kind}.config.${context.field} was sealed by the "${sealed.vault}" vault and no vault is bound to open it. Bind CATALOG_SECRET_VAULT to that provider; nothing else can read this value.`));
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
exports.RefusingSecretVault = RefusingSecretVault;
|
|
136
|
+
/**
|
|
137
|
+
* Nothing is bound, or nothing bound answers to the name a row carries.
|
|
138
|
+
*
|
|
139
|
+
* `retryable = false`, and that field is not decoration. A connector run is
|
|
140
|
+
* dispatched as a durable step, and the dispatch boundary serialises a throw
|
|
141
|
+
* into `{message, code, retryable}` — `retryable` is the only field the engine
|
|
142
|
+
* reads on the way back in, and its predicate is `error?.retryable !== false`.
|
|
143
|
+
* A missing token will be exactly as missing on the third attempt as on the
|
|
144
|
+
* first, so this must not burn fifteen minutes of exponential backoff before
|
|
145
|
+
* saying so. `connector-run.steps.ts` documents that mechanism at length and
|
|
146
|
+
* this is the same use of it.
|
|
147
|
+
*/
|
|
148
|
+
class SecretVaultNotConfiguredError extends Error {
|
|
149
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
150
|
+
retryable = false;
|
|
151
|
+
code = 'secret_vault_not_configured';
|
|
152
|
+
constructor(message) {
|
|
153
|
+
super(message);
|
|
154
|
+
this.name = 'SecretVaultNotConfiguredError';
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
exports.SecretVaultNotConfiguredError = SecretVaultNotConfiguredError;
|
|
158
|
+
/**
|
|
159
|
+
* A save could not seal a credential, so the save did not happen.
|
|
160
|
+
*
|
|
161
|
+
* Which is the correct outcome and worth being explicit about: the fallback a
|
|
162
|
+
* reasonable person reaches for — write it in plaintext and log a warning — is
|
|
163
|
+
* the one thing this feature exists to make impossible. A deployment that
|
|
164
|
+
* turned `encryptCredentials` on and then, during a vault outage, quietly wrote
|
|
165
|
+
* three passwords in the clear would have no way to find out which three.
|
|
166
|
+
*
|
|
167
|
+
* Deliberately not a `BadRequestException`. The caller's URL is fine; the vault
|
|
168
|
+
* is down. A 400 sends somebody to edit a field that is correct.
|
|
169
|
+
*/
|
|
170
|
+
class SecretSealFailedError extends Error {
|
|
171
|
+
code = 'secret_seal_failed';
|
|
172
|
+
constructor(message, options) {
|
|
173
|
+
super(message, options);
|
|
174
|
+
this.name = 'SecretSealFailedError';
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
exports.SecretSealFailedError = SecretSealFailedError;
|
|
178
|
+
/**
|
|
179
|
+
* A read could not open a sealed credential.
|
|
180
|
+
*
|
|
181
|
+
* **This is the one whose class matters**, because a connector run is a durable
|
|
182
|
+
* step and the step decides whether to retry from the error it catches. Two
|
|
183
|
+
* things had to be got right here and both are load-bearing:
|
|
184
|
+
*
|
|
185
|
+
* - It is **not** a `NotFoundException` or a `BadRequestException`.
|
|
186
|
+
* `ConnectorRunSteps.runConnector` catches exactly those two around the
|
|
187
|
+
* runner and converts them to `UnavailableConnectorError`, which carries
|
|
188
|
+
* `retryable = false`. A vault that timed out would then be filed under
|
|
189
|
+
* `connector_unavailable` and never retried — and an operator filtering a
|
|
190
|
+
* failed-run list on that code would go looking for a connector somebody
|
|
191
|
+
* deleted. That is precisely the confusion `UnmetLoadExpectationError` was
|
|
192
|
+
* split into its own class to avoid.
|
|
193
|
+
* - `retryable` is **true by default**. A vault being briefly unreachable is
|
|
194
|
+
* the same kind of failure as a source being briefly unreachable, which is
|
|
195
|
+
* exactly what that step's `retries: 3, backoff exp 60s…900s` policy exists
|
|
196
|
+
* for. It is set false only when waiting provably cannot help: nothing is
|
|
197
|
+
* bound, or nothing bound answers to the name the row carries.
|
|
198
|
+
*/
|
|
199
|
+
class SecretOpenFailedError extends Error {
|
|
200
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
201
|
+
retryable;
|
|
202
|
+
code = 'secret_open_failed';
|
|
203
|
+
constructor(message, options) {
|
|
204
|
+
super(message, { cause: options.cause });
|
|
205
|
+
this.name = 'SecretOpenFailedError';
|
|
206
|
+
this.retryable = options.retryable;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
exports.SecretOpenFailedError = SecretOpenFailedError;
|
|
@@ -11,6 +11,16 @@ export interface SavedQuery {
|
|
|
11
11
|
id: string;
|
|
12
12
|
name: string;
|
|
13
13
|
description?: string;
|
|
14
|
+
/**
|
|
15
|
+
* The statement, as it is now.
|
|
16
|
+
*
|
|
17
|
+
* Overwritten in place by {@link CatalogWorkspaceStore.updateSavedQuery}, and
|
|
18
|
+
* for a long time that was the end of it — a report that started answering
|
|
19
|
+
* differently left nothing to compare against, not even a version number. What
|
|
20
|
+
* it used to say is kept as {@link CatalogRevision}s now, read through
|
|
21
|
+
* {@link CatalogWorkspaceStore.listSavedQueryRevisions}; this field stays the
|
|
22
|
+
* one a run of the query executes.
|
|
23
|
+
*/
|
|
14
24
|
sql: string;
|
|
15
25
|
/** Free-form grouping, the way a folder would work without being one. */
|
|
16
26
|
folder?: string;
|
|
@@ -71,6 +81,122 @@ export interface SaveQueryInput {
|
|
|
71
81
|
visualization?: QueryVisualization;
|
|
72
82
|
shared?: boolean;
|
|
73
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* One recorded revision of something whose text a person edits.
|
|
86
|
+
*
|
|
87
|
+
* The same shape for a transform's code and for a saved query's SQL, and that is
|
|
88
|
+
* the point rather than a saving: the two are edited the same way and go wrong
|
|
89
|
+
* the same way — somebody changes the text, a load or a report starts coming out
|
|
90
|
+
* different, and the question afterwards is what the text used to say. `body` is
|
|
91
|
+
* named for that. `code` would have been a lie on half of its uses.
|
|
92
|
+
*
|
|
93
|
+
* ## What this exists to fix
|
|
94
|
+
*
|
|
95
|
+
* A {@link ConnectorRun} has always recorded `transformVersion`, so the catalog
|
|
96
|
+
* already knew *which* version produced a given load. What it did not keep was
|
|
97
|
+
* the text of that version: one row per transform, overwritten in place, the
|
|
98
|
+
* counter bumped and the previous code gone. A saved query had not even the
|
|
99
|
+
* counter. Meanwhile the runs list renders `code v3`, which reads as a reference
|
|
100
|
+
* to something retrievable — so an operator was told a version number, believed
|
|
101
|
+
* the source was recoverable, and it was not.
|
|
102
|
+
*
|
|
103
|
+
* The number on the run and the {@link version} here are **the same number**.
|
|
104
|
+
* That is the whole contract: "this load ran v3" becomes something a person can
|
|
105
|
+
* open.
|
|
106
|
+
*
|
|
107
|
+
* ## No `kind` field, deliberately
|
|
108
|
+
*
|
|
109
|
+
* A revision is always read through a route that already names its subject —
|
|
110
|
+
* `transforms/:id/revisions`, `saved-queries/:id/revisions` — so a discriminator
|
|
111
|
+
* here would be a field whose only possible value the caller had just supplied.
|
|
112
|
+
* The *store* keys by one, because one table holds both kinds; that is a storage
|
|
113
|
+
* concern and it stays in the store.
|
|
114
|
+
*
|
|
115
|
+
* ## What is NOT revisioned, and why
|
|
116
|
+
*
|
|
117
|
+
* A workflow graph, which has the identical "latest only" limitation and is
|
|
118
|
+
* deliberately left with it. See {@link CatalogWorkflow}, which makes the
|
|
119
|
+
* argument where somebody looking for the missing feature will find it.
|
|
120
|
+
*/
|
|
121
|
+
export interface CatalogRevision {
|
|
122
|
+
/**
|
|
123
|
+
* Stable id of this revision.
|
|
124
|
+
*
|
|
125
|
+
* Derived from the subject and the version rather than random — see
|
|
126
|
+
* `revisionKey` in the MikroORM store — so recording the same version twice
|
|
127
|
+
* replaces it instead of appending a second copy, and a screen may key a list
|
|
128
|
+
* on it across refetches.
|
|
129
|
+
*/
|
|
130
|
+
id: string;
|
|
131
|
+
/** What it belongs to — a transform id or a saved-query id. */
|
|
132
|
+
subjectId: string;
|
|
133
|
+
/** The version this revision IS. Matches `transformVersion` on a run. */
|
|
134
|
+
version: number;
|
|
135
|
+
/** The text as it was: the transform's code, or the query's SQL. */
|
|
136
|
+
body: string;
|
|
137
|
+
/**
|
|
138
|
+
* Who saved it.
|
|
139
|
+
*
|
|
140
|
+
* Exact for a transform, which is saved through a store method that is given
|
|
141
|
+
* the actor. **Approximate for a saved query**, whose update path is given
|
|
142
|
+
* none — `updateSavedQuery` takes an id and a patch, and `CatalogService`
|
|
143
|
+
* keeps the actor for the audit event it emits — so a saved query's revisions
|
|
144
|
+
* are attributed to the query's `createdBy`. That is who created it, not
|
|
145
|
+
* necessarily who last edited it, and it is recorded that way rather than
|
|
146
|
+
* invented: a name here that was picked to fill the field would be read as
|
|
147
|
+
* evidence. Threading the editor through `updateSavedQuery` is what would fix
|
|
148
|
+
* it, and it is a change to that method's contract rather than to this one.
|
|
149
|
+
*/
|
|
150
|
+
authoredBy: string;
|
|
151
|
+
authoredAt: string;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* How many revisions are kept per subject. Writing a newer one drops the oldest
|
|
155
|
+
* beyond this.
|
|
156
|
+
*
|
|
157
|
+
* ## Why there is a cap at all
|
|
158
|
+
*
|
|
159
|
+
* This is append-only text that grows forever, and it is the fourth append-only
|
|
160
|
+
* table in the bundled store. The other three earn their unboundedness and this
|
|
161
|
+
* one does not. An audit event and a connector run are each one small row per
|
|
162
|
+
* *thing that happened*, at a rate an operator can read off their own load
|
|
163
|
+
* schedule; staged rows are dropped the moment the run that produced them is
|
|
164
|
+
* finished with. A revision is neither: it grows with how often somebody edits,
|
|
165
|
+
* which nobody meters, and every row carries a whole code body rather than a
|
|
166
|
+
* counter. Unpredictable in rate *and* large per row is the combination worth
|
|
167
|
+
* bounding — and "this grows; here is the query to prune it" would have been a
|
|
168
|
+
* fourth unbounded table with a paragraph in front of it.
|
|
169
|
+
*
|
|
170
|
+
* ## Why a count per subject rather than an age bound
|
|
171
|
+
*
|
|
172
|
+
* An age bound keys the wrong thing. A transform edited twice in 2019 and relied
|
|
173
|
+
* on ever since would lose both revisions, while one edited daily keeps
|
|
174
|
+
* everything — exactly backwards. What makes a revision unrecoverable is being
|
|
175
|
+
* superseded, so that is what the bound counts.
|
|
176
|
+
*
|
|
177
|
+
* ## What it costs, stated rather than implied
|
|
178
|
+
*
|
|
179
|
+
* A run's `transformVersion` can name a revision that has been evicted: a
|
|
180
|
+
* transform saved more than this many times can no longer produce its earliest
|
|
181
|
+
* code. That loss is real. It is strictly smaller than the one it replaces —
|
|
182
|
+
* where every version but the newest was unrecoverable — and it is visible
|
|
183
|
+
* rather than silent, because a caller holding a version older than the oldest
|
|
184
|
+
* revision in the list can see that the list does not reach that far.
|
|
185
|
+
*
|
|
186
|
+
* At the cap a 4 KB body costs 200 KB per subject, so a thousand heavily-edited
|
|
187
|
+
* subjects cost roughly 200 MB. That is a ceiling, which is the point of having
|
|
188
|
+
* one.
|
|
189
|
+
*
|
|
190
|
+
* ## Why a constant and not a module option
|
|
191
|
+
*
|
|
192
|
+
* The number is part of what this table promises, and a console should be able
|
|
193
|
+
* to print "the last 50 are kept" without a round trip to ask which deployment
|
|
194
|
+
* it is talking to. A knob is also a promise to support every value of it,
|
|
195
|
+
* including the one that switches the feature off and is then reported as a bug.
|
|
196
|
+
* It becomes an option on the day there is a deployment it is wrong for, rather
|
|
197
|
+
* than in anticipation of one.
|
|
198
|
+
*/
|
|
199
|
+
export declare const CATALOG_REVISION_LIMIT = 50;
|
|
74
200
|
export interface Dashboard {
|
|
75
201
|
id: string;
|
|
76
202
|
name: string;
|
|
@@ -480,6 +606,26 @@ export interface CatalogWorkspaceStore {
|
|
|
480
606
|
* first and decides.
|
|
481
607
|
*/
|
|
482
608
|
deleteSavedQuery(id: string): Promise<boolean>;
|
|
609
|
+
/**
|
|
610
|
+
* Every SQL this saved query has ever been, newest first.
|
|
611
|
+
*
|
|
612
|
+
* **Optional**, and mixed in here rather than made a member every store must
|
|
613
|
+
* have, for the reason {@link CatalogWorkflowStore} gives about the same
|
|
614
|
+
* decision: a store written against the previous shape of this interface —
|
|
615
|
+
* including the routing proxy in the MikroORM package — implements
|
|
616
|
+
* `CatalogWorkspaceStore` today, and turning that into a compile error would
|
|
617
|
+
* be a breaking change for a feature that is purely additive.
|
|
618
|
+
* {@link supportsSavedQueryRevisions} is how a caller asks, so "this store
|
|
619
|
+
* keeps no revisions" is a sentence a route can say rather than a method that
|
|
620
|
+
* is missing at run time.
|
|
621
|
+
*
|
|
622
|
+
* An EMPTY list is a real answer and never an error: a query nobody has edited
|
|
623
|
+
* since this shipped may genuinely have nothing recorded. What the bundled
|
|
624
|
+
* store does about that — see `readRevisions` — is a store's decision, and a
|
|
625
|
+
* consumer must read "nothing recorded" as itself rather than as "nothing has
|
|
626
|
+
* changed".
|
|
627
|
+
*/
|
|
628
|
+
listSavedQueryRevisions?(id: string): Promise<CatalogRevision[]>;
|
|
483
629
|
listDashboards(): Promise<Dashboard[]>;
|
|
484
630
|
getDashboard(id: string): Promise<Dashboard | undefined>;
|
|
485
631
|
saveDashboard(input: {
|
|
@@ -498,5 +644,13 @@ export interface CatalogWorkspaceStore {
|
|
|
498
644
|
recordEvent(event: Omit<CatalogAuditEvent, 'id'>): Promise<void>;
|
|
499
645
|
listEvents(query: AuditQuery): Promise<CatalogAuditEvent[]>;
|
|
500
646
|
}
|
|
647
|
+
/**
|
|
648
|
+
* Whether this store keeps a saved query's history.
|
|
649
|
+
*
|
|
650
|
+
* Checks the method rather than a flag, the same way {@link isWorkspaceStore}
|
|
651
|
+
* and `supportsWorkflows` do: a flag is a claim and a method is the thing
|
|
652
|
+
* itself.
|
|
653
|
+
*/
|
|
654
|
+
export declare function supportsSavedQueryRevisions(store: CatalogWorkspaceStore): store is CatalogWorkspaceStore & Required<Pick<CatalogWorkspaceStore, 'listSavedQueryRevisions'>>;
|
|
501
655
|
export declare function isWorkspaceStore(store: unknown): store is CatalogWorkspaceStore;
|
|
502
656
|
export declare const CATALOG_WORKSPACE_STORE: unique symbol;
|