@dudousxd/nestjs-catalog 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.controller.js +39 -11
- package/dist/catalog.events.d.ts +79 -8
- package/dist/catalog.events.js +40 -1
- package/dist/catalog.overlay-store.d.ts +29 -0
- package/dist/catalog.overlay-store.js +45 -2
- package/dist/catalog.pipeline.d.ts +82 -2
- package/dist/catalog.pipeline.js +60 -2
- package/dist/catalog.registry.base.d.ts +15 -4
- package/dist/catalog.registry.d.ts +9 -3
- package/dist/catalog.registry.js +22 -4
- package/dist/catalog.secrets.d.ts +283 -0
- package/dist/catalog.secrets.js +209 -0
- package/dist/catalog.service.d.ts +20 -5
- package/dist/catalog.service.js +23 -8
- package/dist/client.d.ts +2 -1
- package/dist/client.js +9 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +14 -2
- package/dist/transform-runner.d.ts +22 -5
- package/dist/transform-runner.js +126 -9
- package/package.json +1 -1
package/dist/catalog.registry.js
CHANGED
|
@@ -179,7 +179,7 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
|
|
|
179
179
|
return this.entityClasses.get(name);
|
|
180
180
|
}
|
|
181
181
|
/** Tier-0 edit on a type. Never touches the database. */
|
|
182
|
-
async patchType(typeName, patch) {
|
|
182
|
+
async patchType(typeName, patch, curatedBy) {
|
|
183
183
|
const type = this.getType(typeName);
|
|
184
184
|
if (!type)
|
|
185
185
|
return undefined;
|
|
@@ -192,11 +192,15 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
|
|
|
192
192
|
(0, catalog_events_1.emitCatalog)('type.curated', {
|
|
193
193
|
typeName: type.name,
|
|
194
194
|
changed: Object.keys(rest),
|
|
195
|
+
// Through `curationActor` rather than passed straight in, even though the
|
|
196
|
+
// parameter is required: the callers this class actually has to survive are
|
|
197
|
+
// the ones the compiler never saw. See the note on that function.
|
|
198
|
+
principalId: (0, catalog_events_1.curationActor)(curatedBy),
|
|
195
199
|
});
|
|
196
200
|
return this.getType(type.name);
|
|
197
201
|
}
|
|
198
202
|
/** Tier-0 edit on a property. Never touches the database. */
|
|
199
|
-
async patchProperty(typeName, propertyName, patch) {
|
|
203
|
+
async patchProperty(typeName, propertyName, patch, curatedBy) {
|
|
200
204
|
const type = this.getType(typeName);
|
|
201
205
|
if (!type)
|
|
202
206
|
return undefined;
|
|
@@ -218,6 +222,7 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
|
|
|
218
222
|
typeName: type.name,
|
|
219
223
|
property: propertyName,
|
|
220
224
|
changed: Object.keys(patch),
|
|
225
|
+
principalId: (0, catalog_events_1.curationActor)(curatedBy),
|
|
221
226
|
});
|
|
222
227
|
return this.getType(type.name);
|
|
223
228
|
}
|
|
@@ -231,12 +236,18 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
|
|
|
231
236
|
*
|
|
232
237
|
* Emitted after the write, like the two patches above, so the trail says what
|
|
233
238
|
* happened rather than what was about to.
|
|
239
|
+
*
|
|
240
|
+
* The actor is applied here rather than inside {@link summariseOverlay}, which
|
|
241
|
+
* stays a pure function of the overlay. What was destroyed and who destroyed it
|
|
242
|
+
* are facts from two different places, and folding the principal into the
|
|
243
|
+
* summariser would mean the one function that must be callable with nothing but
|
|
244
|
+
* an old overlay suddenly needing the request as well.
|
|
234
245
|
*/
|
|
235
|
-
async resetOverlay() {
|
|
246
|
+
async resetOverlay(resetBy) {
|
|
236
247
|
const discarded = summariseOverlay(this.overlay);
|
|
237
248
|
this.overlay = { types: {} };
|
|
238
249
|
await this.persist();
|
|
239
|
-
(0, catalog_events_1.emitCatalog)('overlay.reset', discarded);
|
|
250
|
+
(0, catalog_events_1.emitCatalog)('overlay.reset', { ...discarded, principalId: (0, catalog_events_1.curationActor)(resetBy) });
|
|
240
251
|
}
|
|
241
252
|
async persist() {
|
|
242
253
|
await this.overlayStore.save(this.overlay);
|
|
@@ -390,6 +401,13 @@ exports.MikroOrmCatalogRegistry = MikroOrmCatalogRegistry = MikroOrmCatalogRegis
|
|
|
390
401
|
* order is forced: a caller has to hold the old overlay to call it, and cannot
|
|
391
402
|
* accidentally summarise the empty one it just installed.
|
|
392
403
|
*
|
|
404
|
+
* Everything the payload holds except the actor, stated as an `Omit` of the
|
|
405
|
+
* payload rather than a shape of its own. A hand-written interface here would be
|
|
406
|
+
* a second copy of the contract, free to fall behind the day a field is added —
|
|
407
|
+
* and the failure would be a summary silently missing a key that the type says
|
|
408
|
+
* is required. `principalId` is the caller's to supply because it is a fact about
|
|
409
|
+
* the request, not about the overlay.
|
|
410
|
+
*
|
|
393
411
|
* A type entry counts whatever it holds, including an entry that ended up empty.
|
|
394
412
|
* `buildType` treats a present entry as enrichment on the same terms, and the
|
|
395
413
|
* honest reading of one is "somebody patched this type" — which is exactly what
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Encrypting the credential that has to rest in `catalog_connection.config`.
|
|
3
|
+
*
|
|
4
|
+
* `config-secrets.ts` in the pipeline package closed the HTTP half of this: a
|
|
5
|
+
* connection URL is a password with an address attached, `config` is served by
|
|
6
|
+
* `GET pipeline/connections` under `catalog:read`, and the redaction stopped it
|
|
7
|
+
* travelling in a response. `MySqlPipelineStore` closed the write half: a
|
|
8
|
+
* password-bearing URL that is not already stored is refused, with a message
|
|
9
|
+
* naming `secretEnvVar`.
|
|
10
|
+
*
|
|
11
|
+
* Neither touches the thing this file is about. A redaction protects a reader
|
|
12
|
+
* coming through the API. It does nothing at all for a database dump, a read
|
|
13
|
+
* replica, a nightly backup, or anybody holding `SELECT` on the RDS instance —
|
|
14
|
+
* and for those, `config` is still a list of every password the catalog knows.
|
|
15
|
+
* `allowInlineCredentials` makes that population larger on purpose, which is
|
|
16
|
+
* precisely why encryption becomes worth building at the same time as the flag
|
|
17
|
+
* that needs it.
|
|
18
|
+
*
|
|
19
|
+
* ## A seam, not an implementation
|
|
20
|
+
*
|
|
21
|
+
* There is no cipher in this file and there must not be one. A library that
|
|
22
|
+
* shipped AES with a key read from an environment variable would have moved the
|
|
23
|
+
* problem from one column to one variable, and would have to answer for key
|
|
24
|
+
* rotation, per-environment separation, and an audit trail — all of which the
|
|
25
|
+
* host's KMS, Vault, or HSM already answers for, and none of which this package
|
|
26
|
+
* can answer for on its behalf. What is here is the shape of the question, so a
|
|
27
|
+
* provider can be written against it: {@link CatalogSecretVault}.
|
|
28
|
+
*
|
|
29
|
+
* The library never inspects a ciphertext, never chooses a key, and never
|
|
30
|
+
* decides an algorithm. It decides *which values* are secrets, *when* they are
|
|
31
|
+
* sealed, and *what happens when the vault cannot be reached* — three questions
|
|
32
|
+
* a provider should not have to have an opinion about.
|
|
33
|
+
*
|
|
34
|
+
* ## What the default does
|
|
35
|
+
*
|
|
36
|
+
* Refuses. {@link RefusingSecretVault} throws on `seal`, naming the token to
|
|
37
|
+
* bind. A default that quietly stored plaintext would be the worst shape
|
|
38
|
+
* available here: a host would turn `encryptCredentials` on, see saves succeed,
|
|
39
|
+
* and have exactly the same column contents as before with a docblock now
|
|
40
|
+
* claiming otherwise. Every other refusing default in this codebase exists for
|
|
41
|
+
* the same reason — `CATALOG_LOAD_EXPECTATIONS` unbound means incremental loads
|
|
42
|
+
* are refused rather than silently unpoliced.
|
|
43
|
+
*/
|
|
44
|
+
/**
|
|
45
|
+
* What a vault hands back. Stored verbatim; opaque to this library.
|
|
46
|
+
*
|
|
47
|
+
* Three fields and no fourth, because everything a rotation needs is here and
|
|
48
|
+
* nothing a reader could misuse is: the ciphertext says nothing without the
|
|
49
|
+
* vault, and the vault says nothing without the key. It is a plain JSON object
|
|
50
|
+
* so that it can sit in the same `json` column the plaintext sat in — no
|
|
51
|
+
* migration, no second table, and a column that may hold either form for as
|
|
52
|
+
* long as it takes a deployment to move over. {@link isSealedSecret} is what
|
|
53
|
+
* tells the two apart on the way out.
|
|
54
|
+
*/
|
|
55
|
+
export interface SealedSecret {
|
|
56
|
+
/** Which vault sealed it — `CatalogSecretVault.name`. */
|
|
57
|
+
vault: string;
|
|
58
|
+
/** Which key, in whatever form that vault names keys. */
|
|
59
|
+
keyId: string;
|
|
60
|
+
/**
|
|
61
|
+
* An opaque string the vault chose. **Not base64, and never decoded here.**
|
|
62
|
+
*
|
|
63
|
+
* This said "base64" first, and both provider implementations proved it
|
|
64
|
+
* wrong from opposite directions: HashiCorp Transit returns
|
|
65
|
+
* `vault:v1:<base64>`, which has two colons and is not decodable as base64 at
|
|
66
|
+
* all. The provider stores that verbatim and is right to — the `v1` is the
|
|
67
|
+
* row's only self-description of which key version sealed it, and
|
|
68
|
+
* `transit/rewrap` takes exactly that string back.
|
|
69
|
+
*
|
|
70
|
+
* The claim was harmless while nothing acted on it, which is what makes it
|
|
71
|
+
* worth correcting now rather than later: the day something takes it
|
|
72
|
+
* literally — a base64 `CHECK` constraint, a JSON-schema `contentEncoding`, a
|
|
73
|
+
* `Buffer.from(value, 'base64')` round trip — it breaks **silently**, because
|
|
74
|
+
* `Buffer.from` does not throw on input it cannot read. It returns different
|
|
75
|
+
* bytes.
|
|
76
|
+
*
|
|
77
|
+
* So the only rule is: non-empty, and given back to the vault exactly as it
|
|
78
|
+
* came. {@link isSealedSecret} checks that it is a non-empty string and must
|
|
79
|
+
* never be tightened past that — any narrower test encodes one vault's format
|
|
80
|
+
* as the contract and silently rejects the next one's rows.
|
|
81
|
+
*/
|
|
82
|
+
ciphertext: string;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Where a secret is being used, so a vault can scope or audit by it.
|
|
86
|
+
*
|
|
87
|
+
* Passed to `open` as well as to `seal`, and that is the interesting half: a
|
|
88
|
+
* KMS provider can put this in an encryption context so a ciphertext sealed for
|
|
89
|
+
* `connection/abc/url` cannot be replayed as `connector/xyz/url`, and a Vault
|
|
90
|
+
* provider can write an audit line naming the row somebody's credential was
|
|
91
|
+
* read for. Neither is possible if the seam only carries bytes.
|
|
92
|
+
*/
|
|
93
|
+
export interface SecretContext {
|
|
94
|
+
/** `connection` or `connector`. */
|
|
95
|
+
kind: string;
|
|
96
|
+
/**
|
|
97
|
+
* The row's id. Present and stable whenever `MySqlPipelineStore` is the
|
|
98
|
+
* caller, including on a first save.
|
|
99
|
+
*
|
|
100
|
+
* This said "absent on a first save", and both provider implementations
|
|
101
|
+
* independently left `id` out of their encryption context because of it —
|
|
102
|
+
* correctly, given what it said. A context that is absent on create and
|
|
103
|
+
* present on update seals under one context and opens under another, and the
|
|
104
|
+
* failure lands at the first connector run rather than at the save.
|
|
105
|
+
*
|
|
106
|
+
* So the store was changed rather than the providers: it mints the row's id
|
|
107
|
+
* *before* sealing instead of inside `em.create`, which costs one hoisted
|
|
108
|
+
* `randomUUID()` and makes the binding property real. A provider may now put
|
|
109
|
+
* this in an encryption context and get what the seam always claimed —
|
|
110
|
+
* `connection/abc/url` is not replayable as `connection/xyz/url`.
|
|
111
|
+
*
|
|
112
|
+
* Two things this does NOT make true, both worth stating before somebody
|
|
113
|
+
* relies on them:
|
|
114
|
+
*
|
|
115
|
+
* - It is still optional in the type, because a host calling a vault
|
|
116
|
+
* directly is not obliged to have a row. A provider that binds it must
|
|
117
|
+
* decide what an absent `id` means for its own callers.
|
|
118
|
+
* - **Rows sealed before this existed were sealed without it.** A provider
|
|
119
|
+
* that starts binding `id` on a deployment that already has sealed rows
|
|
120
|
+
* strands every one of them. Binding it is safe on a new deployment, and
|
|
121
|
+
* on an existing one only behind a re-seal.
|
|
122
|
+
*/
|
|
123
|
+
id?: string;
|
|
124
|
+
/** The config key being sealed — `url`, `password`. */
|
|
125
|
+
field: string;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The seam a host binds to make credentials unreadable in the catalog's own
|
|
129
|
+
* database.
|
|
130
|
+
*
|
|
131
|
+
* Both methods are async and both are expected to fail sometimes: they are
|
|
132
|
+
* network calls to something outside this process. What each failure *means*
|
|
133
|
+
* is not symmetric, and the store treats them differently — see
|
|
134
|
+
* {@link SecretSealFailedError} and {@link SecretOpenFailedError}.
|
|
135
|
+
*/
|
|
136
|
+
export interface CatalogSecretVault {
|
|
137
|
+
/**
|
|
138
|
+
* Stable, stored on every SealedSecret so a rotation can find its own.
|
|
139
|
+
*
|
|
140
|
+
* Stable across deployments and releases, not merely across a process: it is
|
|
141
|
+
* written into rows. Renaming it strands every row already sealed under the
|
|
142
|
+
* old name, which the store refuses loudly rather than guessing at — a bound
|
|
143
|
+
* vault whose name does not match the row's is not asked to try.
|
|
144
|
+
*/
|
|
145
|
+
readonly name: string;
|
|
146
|
+
seal(plaintext: string, context: SecretContext): Promise<SealedSecret>;
|
|
147
|
+
open(sealed: SealedSecret, context: SecretContext): Promise<string>;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The token a host binds one vault — or several — to.
|
|
151
|
+
*
|
|
152
|
+
* **Several is the supported shape, and it is what makes rotation possible.**
|
|
153
|
+
* Bind `CatalogSecretVault[]` and the store seals with the *first* and opens
|
|
154
|
+
* with whichever one's `name` matches the row. Moving from one vault to another
|
|
155
|
+
* is then: bind `[next, current]`, let saves reseal under `next`, and drop
|
|
156
|
+
* `current` when nothing is sealed under it any more. With a single vault that
|
|
157
|
+
* transition has no middle — the moment `next` is bound, every row sealed by
|
|
158
|
+
* `current` is unreadable — and a library that made rotation require an outage
|
|
159
|
+
* would be a library whose keys never got rotated.
|
|
160
|
+
*
|
|
161
|
+
* `@Optional()` where it is injected, so a host that binds nothing still boots.
|
|
162
|
+
* It then gets {@link RefusingSecretVault}, which costs nothing until something
|
|
163
|
+
* actually asks for a seal.
|
|
164
|
+
*/
|
|
165
|
+
export declare const CATALOG_SECRET_VAULT: unique symbol;
|
|
166
|
+
/**
|
|
167
|
+
* Whether a value that came back out of a JSON column is a sealed secret.
|
|
168
|
+
*
|
|
169
|
+
* A runtime guard and not a cast, because this is a `json` column: MikroORM
|
|
170
|
+
* hands back whatever is in it, and what is in it may have been written by a
|
|
171
|
+
* different release, by a hand-run `UPDATE`, or by a host's own seeder. The
|
|
172
|
+
* same argument `WorkflowRow.nodes` makes for being typed `unknown[]`.
|
|
173
|
+
*
|
|
174
|
+
* Deliberately strict about emptiness. A `{ vault: '', keyId: '', ciphertext:
|
|
175
|
+
* '' }` is not a sealed secret and must not be treated as one — treating it as
|
|
176
|
+
* one sends it to a vault that will refuse it, and the operator is told their
|
|
177
|
+
* vault is broken when the row is. An empty ciphertext falls through as an
|
|
178
|
+
* ordinary config value instead, which is what it is.
|
|
179
|
+
*
|
|
180
|
+
* Deliberately loose about everything else, and that is the harder half to
|
|
181
|
+
* hold. `ciphertext` is tested for "non-empty string" and must never be tested
|
|
182
|
+
* for more — not base64, not a length, not a prefix. Two vaults already
|
|
183
|
+
* disagree about its shape: KMS hands back base64, Transit hands back
|
|
184
|
+
* `vault:v1:<base64>`. Any tighter check would encode one of them as the
|
|
185
|
+
* contract and start silently refusing the other's rows, which surfaces as
|
|
186
|
+
* "this credential is not encrypted" for a value that is.
|
|
187
|
+
*
|
|
188
|
+
* What it cannot distinguish is a config value a host deliberately authored as
|
|
189
|
+
* an object with exactly these three non-empty string fields. Nothing prevents
|
|
190
|
+
* that and nothing sensible produces it; saying so is better than adding a
|
|
191
|
+
* marker field that would then have to be defended against forgery it also
|
|
192
|
+
* could not detect.
|
|
193
|
+
*/
|
|
194
|
+
export declare function isSealedSecret(value: unknown): value is SealedSecret;
|
|
195
|
+
/** The name {@link RefusingSecretVault} answers to. No row can carry it: it never seals. */
|
|
196
|
+
export declare const UNCONFIGURED_VAULT = "unconfigured";
|
|
197
|
+
/**
|
|
198
|
+
* The default: a vault that will not pretend.
|
|
199
|
+
*
|
|
200
|
+
* Bound whenever `CATALOG_SECRET_VAULT` is not, which is every host that has
|
|
201
|
+
* not thought about this. It costs nothing until `encryptCredentials` is turned
|
|
202
|
+
* on — the store only reaches a vault when there is something to seal or
|
|
203
|
+
* something sealed to open — so the refusal lands on the deployment that asked
|
|
204
|
+
* for encryption and did not say with what, and lands at the first save rather
|
|
205
|
+
* than at the first run.
|
|
206
|
+
*
|
|
207
|
+
* The alternative shapes were both considered and are both worse. A default
|
|
208
|
+
* that stored plaintext would make `encryptCredentials: true` a no-op with a
|
|
209
|
+
* reassuring name. A default that base64-encoded would be worse still: it looks
|
|
210
|
+
* sealed, `isSealedSecret` would agree, and a dump would be trivially reversed
|
|
211
|
+
* by anybody who noticed.
|
|
212
|
+
*/
|
|
213
|
+
export declare class RefusingSecretVault implements CatalogSecretVault {
|
|
214
|
+
readonly name = "unconfigured";
|
|
215
|
+
seal(_plaintext: string, context: SecretContext): Promise<SealedSecret>;
|
|
216
|
+
open(sealed: SealedSecret, context: SecretContext): Promise<string>;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Nothing is bound, or nothing bound answers to the name a row carries.
|
|
220
|
+
*
|
|
221
|
+
* `retryable = false`, and that field is not decoration. A connector run is
|
|
222
|
+
* dispatched as a durable step, and the dispatch boundary serialises a throw
|
|
223
|
+
* into `{message, code, retryable}` — `retryable` is the only field the engine
|
|
224
|
+
* reads on the way back in, and its predicate is `error?.retryable !== false`.
|
|
225
|
+
* A missing token will be exactly as missing on the third attempt as on the
|
|
226
|
+
* first, so this must not burn fifteen minutes of exponential backoff before
|
|
227
|
+
* saying so. `connector-run.steps.ts` documents that mechanism at length and
|
|
228
|
+
* this is the same use of it.
|
|
229
|
+
*/
|
|
230
|
+
export declare class SecretVaultNotConfiguredError extends Error {
|
|
231
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
232
|
+
readonly retryable = false;
|
|
233
|
+
readonly code = "secret_vault_not_configured";
|
|
234
|
+
constructor(message: string);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* A save could not seal a credential, so the save did not happen.
|
|
238
|
+
*
|
|
239
|
+
* Which is the correct outcome and worth being explicit about: the fallback a
|
|
240
|
+
* reasonable person reaches for — write it in plaintext and log a warning — is
|
|
241
|
+
* the one thing this feature exists to make impossible. A deployment that
|
|
242
|
+
* turned `encryptCredentials` on and then, during a vault outage, quietly wrote
|
|
243
|
+
* three passwords in the clear would have no way to find out which three.
|
|
244
|
+
*
|
|
245
|
+
* Deliberately not a `BadRequestException`. The caller's URL is fine; the vault
|
|
246
|
+
* is down. A 400 sends somebody to edit a field that is correct.
|
|
247
|
+
*/
|
|
248
|
+
export declare class SecretSealFailedError extends Error {
|
|
249
|
+
readonly code = "secret_seal_failed";
|
|
250
|
+
constructor(message: string, options?: {
|
|
251
|
+
cause?: unknown;
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* A read could not open a sealed credential.
|
|
256
|
+
*
|
|
257
|
+
* **This is the one whose class matters**, because a connector run is a durable
|
|
258
|
+
* step and the step decides whether to retry from the error it catches. Two
|
|
259
|
+
* things had to be got right here and both are load-bearing:
|
|
260
|
+
*
|
|
261
|
+
* - It is **not** a `NotFoundException` or a `BadRequestException`.
|
|
262
|
+
* `ConnectorRunSteps.runConnector` catches exactly those two around the
|
|
263
|
+
* runner and converts them to `UnavailableConnectorError`, which carries
|
|
264
|
+
* `retryable = false`. A vault that timed out would then be filed under
|
|
265
|
+
* `connector_unavailable` and never retried — and an operator filtering a
|
|
266
|
+
* failed-run list on that code would go looking for a connector somebody
|
|
267
|
+
* deleted. That is precisely the confusion `UnmetLoadExpectationError` was
|
|
268
|
+
* split into its own class to avoid.
|
|
269
|
+
* - `retryable` is **true by default**. A vault being briefly unreachable is
|
|
270
|
+
* the same kind of failure as a source being briefly unreachable, which is
|
|
271
|
+
* exactly what that step's `retries: 3, backoff exp 60s…900s` policy exists
|
|
272
|
+
* for. It is set false only when waiting provably cannot help: nothing is
|
|
273
|
+
* bound, or nothing bound answers to the name the row carries.
|
|
274
|
+
*/
|
|
275
|
+
export declare class SecretOpenFailedError extends Error {
|
|
276
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
277
|
+
readonly retryable: boolean;
|
|
278
|
+
readonly code = "secret_open_failed";
|
|
279
|
+
constructor(message: string, options: {
|
|
280
|
+
retryable: boolean;
|
|
281
|
+
cause?: unknown;
|
|
282
|
+
});
|
|
283
|
+
}
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Encrypting the credential that has to rest in `catalog_connection.config`.
|
|
4
|
+
*
|
|
5
|
+
* `config-secrets.ts` in the pipeline package closed the HTTP half of this: a
|
|
6
|
+
* connection URL is a password with an address attached, `config` is served by
|
|
7
|
+
* `GET pipeline/connections` under `catalog:read`, and the redaction stopped it
|
|
8
|
+
* travelling in a response. `MySqlPipelineStore` closed the write half: a
|
|
9
|
+
* password-bearing URL that is not already stored is refused, with a message
|
|
10
|
+
* naming `secretEnvVar`.
|
|
11
|
+
*
|
|
12
|
+
* Neither touches the thing this file is about. A redaction protects a reader
|
|
13
|
+
* coming through the API. It does nothing at all for a database dump, a read
|
|
14
|
+
* replica, a nightly backup, or anybody holding `SELECT` on the RDS instance —
|
|
15
|
+
* and for those, `config` is still a list of every password the catalog knows.
|
|
16
|
+
* `allowInlineCredentials` makes that population larger on purpose, which is
|
|
17
|
+
* precisely why encryption becomes worth building at the same time as the flag
|
|
18
|
+
* that needs it.
|
|
19
|
+
*
|
|
20
|
+
* ## A seam, not an implementation
|
|
21
|
+
*
|
|
22
|
+
* There is no cipher in this file and there must not be one. A library that
|
|
23
|
+
* shipped AES with a key read from an environment variable would have moved the
|
|
24
|
+
* problem from one column to one variable, and would have to answer for key
|
|
25
|
+
* rotation, per-environment separation, and an audit trail — all of which the
|
|
26
|
+
* host's KMS, Vault, or HSM already answers for, and none of which this package
|
|
27
|
+
* can answer for on its behalf. What is here is the shape of the question, so a
|
|
28
|
+
* provider can be written against it: {@link CatalogSecretVault}.
|
|
29
|
+
*
|
|
30
|
+
* The library never inspects a ciphertext, never chooses a key, and never
|
|
31
|
+
* decides an algorithm. It decides *which values* are secrets, *when* they are
|
|
32
|
+
* sealed, and *what happens when the vault cannot be reached* — three questions
|
|
33
|
+
* a provider should not have to have an opinion about.
|
|
34
|
+
*
|
|
35
|
+
* ## What the default does
|
|
36
|
+
*
|
|
37
|
+
* Refuses. {@link RefusingSecretVault} throws on `seal`, naming the token to
|
|
38
|
+
* bind. A default that quietly stored plaintext would be the worst shape
|
|
39
|
+
* available here: a host would turn `encryptCredentials` on, see saves succeed,
|
|
40
|
+
* and have exactly the same column contents as before with a docblock now
|
|
41
|
+
* claiming otherwise. Every other refusing default in this codebase exists for
|
|
42
|
+
* the same reason — `CATALOG_LOAD_EXPECTATIONS` unbound means incremental loads
|
|
43
|
+
* are refused rather than silently unpoliced.
|
|
44
|
+
*/
|
|
45
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
|
+
exports.SecretOpenFailedError = exports.SecretSealFailedError = exports.SecretVaultNotConfiguredError = exports.RefusingSecretVault = exports.UNCONFIGURED_VAULT = exports.CATALOG_SECRET_VAULT = void 0;
|
|
47
|
+
exports.isSealedSecret = isSealedSecret;
|
|
48
|
+
/**
|
|
49
|
+
* The token a host binds one vault — or several — to.
|
|
50
|
+
*
|
|
51
|
+
* **Several is the supported shape, and it is what makes rotation possible.**
|
|
52
|
+
* Bind `CatalogSecretVault[]` and the store seals with the *first* and opens
|
|
53
|
+
* with whichever one's `name` matches the row. Moving from one vault to another
|
|
54
|
+
* is then: bind `[next, current]`, let saves reseal under `next`, and drop
|
|
55
|
+
* `current` when nothing is sealed under it any more. With a single vault that
|
|
56
|
+
* transition has no middle — the moment `next` is bound, every row sealed by
|
|
57
|
+
* `current` is unreadable — and a library that made rotation require an outage
|
|
58
|
+
* would be a library whose keys never got rotated.
|
|
59
|
+
*
|
|
60
|
+
* `@Optional()` where it is injected, so a host that binds nothing still boots.
|
|
61
|
+
* It then gets {@link RefusingSecretVault}, which costs nothing until something
|
|
62
|
+
* actually asks for a seal.
|
|
63
|
+
*/
|
|
64
|
+
exports.CATALOG_SECRET_VAULT = Symbol('CATALOG_SECRET_VAULT');
|
|
65
|
+
/**
|
|
66
|
+
* Whether a value that came back out of a JSON column is a sealed secret.
|
|
67
|
+
*
|
|
68
|
+
* A runtime guard and not a cast, because this is a `json` column: MikroORM
|
|
69
|
+
* hands back whatever is in it, and what is in it may have been written by a
|
|
70
|
+
* different release, by a hand-run `UPDATE`, or by a host's own seeder. The
|
|
71
|
+
* same argument `WorkflowRow.nodes` makes for being typed `unknown[]`.
|
|
72
|
+
*
|
|
73
|
+
* Deliberately strict about emptiness. A `{ vault: '', keyId: '', ciphertext:
|
|
74
|
+
* '' }` is not a sealed secret and must not be treated as one — treating it as
|
|
75
|
+
* one sends it to a vault that will refuse it, and the operator is told their
|
|
76
|
+
* vault is broken when the row is. An empty ciphertext falls through as an
|
|
77
|
+
* ordinary config value instead, which is what it is.
|
|
78
|
+
*
|
|
79
|
+
* Deliberately loose about everything else, and that is the harder half to
|
|
80
|
+
* hold. `ciphertext` is tested for "non-empty string" and must never be tested
|
|
81
|
+
* for more — not base64, not a length, not a prefix. Two vaults already
|
|
82
|
+
* disagree about its shape: KMS hands back base64, Transit hands back
|
|
83
|
+
* `vault:v1:<base64>`. Any tighter check would encode one of them as the
|
|
84
|
+
* contract and start silently refusing the other's rows, which surfaces as
|
|
85
|
+
* "this credential is not encrypted" for a value that is.
|
|
86
|
+
*
|
|
87
|
+
* What it cannot distinguish is a config value a host deliberately authored as
|
|
88
|
+
* an object with exactly these three non-empty string fields. Nothing prevents
|
|
89
|
+
* that and nothing sensible produces it; saying so is better than adding a
|
|
90
|
+
* marker field that would then have to be defended against forgery it also
|
|
91
|
+
* could not detect.
|
|
92
|
+
*/
|
|
93
|
+
function isSealedSecret(value) {
|
|
94
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
95
|
+
return false;
|
|
96
|
+
const vault = Reflect.get(value, 'vault');
|
|
97
|
+
const keyId = Reflect.get(value, 'keyId');
|
|
98
|
+
const ciphertext = Reflect.get(value, 'ciphertext');
|
|
99
|
+
return (typeof vault === 'string' &&
|
|
100
|
+
vault.length > 0 &&
|
|
101
|
+
typeof keyId === 'string' &&
|
|
102
|
+
typeof ciphertext === 'string' &&
|
|
103
|
+
ciphertext.length > 0);
|
|
104
|
+
}
|
|
105
|
+
/** The name {@link RefusingSecretVault} answers to. No row can carry it: it never seals. */
|
|
106
|
+
exports.UNCONFIGURED_VAULT = 'unconfigured';
|
|
107
|
+
/**
|
|
108
|
+
* The default: a vault that will not pretend.
|
|
109
|
+
*
|
|
110
|
+
* Bound whenever `CATALOG_SECRET_VAULT` is not, which is every host that has
|
|
111
|
+
* not thought about this. It costs nothing until `encryptCredentials` is turned
|
|
112
|
+
* on — the store only reaches a vault when there is something to seal or
|
|
113
|
+
* something sealed to open — so the refusal lands on the deployment that asked
|
|
114
|
+
* for encryption and did not say with what, and lands at the first save rather
|
|
115
|
+
* than at the first run.
|
|
116
|
+
*
|
|
117
|
+
* The alternative shapes were both considered and are both worse. A default
|
|
118
|
+
* that stored plaintext would make `encryptCredentials: true` a no-op with a
|
|
119
|
+
* reassuring name. A default that base64-encoded would be worse still: it looks
|
|
120
|
+
* sealed, `isSealedSecret` would agree, and a dump would be trivially reversed
|
|
121
|
+
* by anybody who noticed.
|
|
122
|
+
*/
|
|
123
|
+
class RefusingSecretVault {
|
|
124
|
+
name = exports.UNCONFIGURED_VAULT;
|
|
125
|
+
seal(_plaintext, context) {
|
|
126
|
+
return Promise.reject(new SecretVaultNotConfiguredError(`encryptCredentials is on, so ${context.kind}.config.${context.field} has to be sealed, and no vault is bound to seal it with. Bind CATALOG_SECRET_VAULT to a CatalogSecretVault — an AWS KMS or HashiCorp Vault provider, or your own — or turn encryptCredentials off and decide what allowInlineCredentials should be instead.`));
|
|
127
|
+
}
|
|
128
|
+
open(sealed, context) {
|
|
129
|
+
// Reachable only through a row this instance did not write, since `seal`
|
|
130
|
+
// never returns — so the useful thing to say is the name the row is asking
|
|
131
|
+
// for, which is the one the operator has to go and bind.
|
|
132
|
+
return Promise.reject(new SecretVaultNotConfiguredError(`${context.kind}.config.${context.field} was sealed by the "${sealed.vault}" vault and no vault is bound to open it. Bind CATALOG_SECRET_VAULT to that provider; nothing else can read this value.`));
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
exports.RefusingSecretVault = RefusingSecretVault;
|
|
136
|
+
/**
|
|
137
|
+
* Nothing is bound, or nothing bound answers to the name a row carries.
|
|
138
|
+
*
|
|
139
|
+
* `retryable = false`, and that field is not decoration. A connector run is
|
|
140
|
+
* dispatched as a durable step, and the dispatch boundary serialises a throw
|
|
141
|
+
* into `{message, code, retryable}` — `retryable` is the only field the engine
|
|
142
|
+
* reads on the way back in, and its predicate is `error?.retryable !== false`.
|
|
143
|
+
* A missing token will be exactly as missing on the third attempt as on the
|
|
144
|
+
* first, so this must not burn fifteen minutes of exponential backoff before
|
|
145
|
+
* saying so. `connector-run.steps.ts` documents that mechanism at length and
|
|
146
|
+
* this is the same use of it.
|
|
147
|
+
*/
|
|
148
|
+
class SecretVaultNotConfiguredError extends Error {
|
|
149
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
150
|
+
retryable = false;
|
|
151
|
+
code = 'secret_vault_not_configured';
|
|
152
|
+
constructor(message) {
|
|
153
|
+
super(message);
|
|
154
|
+
this.name = 'SecretVaultNotConfiguredError';
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
exports.SecretVaultNotConfiguredError = SecretVaultNotConfiguredError;
|
|
158
|
+
/**
|
|
159
|
+
* A save could not seal a credential, so the save did not happen.
|
|
160
|
+
*
|
|
161
|
+
* Which is the correct outcome and worth being explicit about: the fallback a
|
|
162
|
+
* reasonable person reaches for — write it in plaintext and log a warning — is
|
|
163
|
+
* the one thing this feature exists to make impossible. A deployment that
|
|
164
|
+
* turned `encryptCredentials` on and then, during a vault outage, quietly wrote
|
|
165
|
+
* three passwords in the clear would have no way to find out which three.
|
|
166
|
+
*
|
|
167
|
+
* Deliberately not a `BadRequestException`. The caller's URL is fine; the vault
|
|
168
|
+
* is down. A 400 sends somebody to edit a field that is correct.
|
|
169
|
+
*/
|
|
170
|
+
class SecretSealFailedError extends Error {
|
|
171
|
+
code = 'secret_seal_failed';
|
|
172
|
+
constructor(message, options) {
|
|
173
|
+
super(message, options);
|
|
174
|
+
this.name = 'SecretSealFailedError';
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
exports.SecretSealFailedError = SecretSealFailedError;
|
|
178
|
+
/**
|
|
179
|
+
* A read could not open a sealed credential.
|
|
180
|
+
*
|
|
181
|
+
* **This is the one whose class matters**, because a connector run is a durable
|
|
182
|
+
* step and the step decides whether to retry from the error it catches. Two
|
|
183
|
+
* things had to be got right here and both are load-bearing:
|
|
184
|
+
*
|
|
185
|
+
* - It is **not** a `NotFoundException` or a `BadRequestException`.
|
|
186
|
+
* `ConnectorRunSteps.runConnector` catches exactly those two around the
|
|
187
|
+
* runner and converts them to `UnavailableConnectorError`, which carries
|
|
188
|
+
* `retryable = false`. A vault that timed out would then be filed under
|
|
189
|
+
* `connector_unavailable` and never retried — and an operator filtering a
|
|
190
|
+
* failed-run list on that code would go looking for a connector somebody
|
|
191
|
+
* deleted. That is precisely the confusion `UnmetLoadExpectationError` was
|
|
192
|
+
* split into its own class to avoid.
|
|
193
|
+
* - `retryable` is **true by default**. A vault being briefly unreachable is
|
|
194
|
+
* the same kind of failure as a source being briefly unreachable, which is
|
|
195
|
+
* exactly what that step's `retries: 3, backoff exp 60s…900s` policy exists
|
|
196
|
+
* for. It is set false only when waiting provably cannot help: nothing is
|
|
197
|
+
* bound, or nothing bound answers to the name the row carries.
|
|
198
|
+
*/
|
|
199
|
+
class SecretOpenFailedError extends Error {
|
|
200
|
+
/** The field the dispatch boundary serialises and the engine acts on. */
|
|
201
|
+
retryable;
|
|
202
|
+
code = 'secret_open_failed';
|
|
203
|
+
constructor(message, options) {
|
|
204
|
+
super(message, { cause: options.cause });
|
|
205
|
+
this.name = 'SecretOpenFailedError';
|
|
206
|
+
this.retryable = options.retryable;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
exports.SecretOpenFailedError = SecretOpenFailedError;
|
|
@@ -27,11 +27,26 @@ export declare class CatalogService {
|
|
|
27
27
|
getType(name: string): CatalogObjectTypeDef | undefined;
|
|
28
28
|
/** Nodes and edges, for drawing the model. */
|
|
29
29
|
getGraph(): CatalogGraph;
|
|
30
|
-
/**
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
30
|
+
/**
|
|
31
|
+
* Presentation-only. Never a schema change.
|
|
32
|
+
*
|
|
33
|
+
* @param curatedBy who is doing it, for the audit trail — the same required
|
|
34
|
+
* argument the sharing methods below take, and required for the same reason
|
|
35
|
+
* `deleteSavedQuery` gives. This facade forwards it rather than resolving it:
|
|
36
|
+
* a host writing its own controller has the request and this does not, and a
|
|
37
|
+
* default here would attribute every such host's curation to nobody.
|
|
38
|
+
*/
|
|
39
|
+
patchType(typeName: string, patch: Partial<CatalogOverlay['types'][string]>, curatedBy: string): Promise<CatalogObjectTypeDef | undefined>;
|
|
40
|
+
/** @param curatedBy who is doing it, for the audit trail. */
|
|
41
|
+
patchProperty(typeName: string, propertyName: string, patch: NonNullable<CatalogOverlay['types'][string]['properties']>[string], curatedBy: string): Promise<CatalogObjectTypeDef | undefined>;
|
|
42
|
+
/**
|
|
43
|
+
* Drops every runtime edit, where the registry supports it.
|
|
44
|
+
*
|
|
45
|
+
* @param resetBy who is doing it. The one act here that destroys curation in
|
|
46
|
+
* bulk, so it is the one whose actor is hardest to reconstruct afterwards —
|
|
47
|
+
* nothing versions an overlay, and after this there is nothing left to read.
|
|
48
|
+
*/
|
|
49
|
+
resetOverlay(resetBy: string): Promise<void>;
|
|
35
50
|
/** Columns a generic UI may render: visible, and not a blob. */
|
|
36
51
|
visibleColumns(type: CatalogObjectTypeDef): import("./catalog.types").CatalogPropertyDef[];
|
|
37
52
|
/**
|
package/dist/catalog.service.js
CHANGED
|
@@ -66,16 +66,31 @@ let CatalogService = class CatalogService {
|
|
|
66
66
|
getGraph() {
|
|
67
67
|
return this.registry.getGraph();
|
|
68
68
|
}
|
|
69
|
-
/**
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
/**
|
|
70
|
+
* Presentation-only. Never a schema change.
|
|
71
|
+
*
|
|
72
|
+
* @param curatedBy who is doing it, for the audit trail — the same required
|
|
73
|
+
* argument the sharing methods below take, and required for the same reason
|
|
74
|
+
* `deleteSavedQuery` gives. This facade forwards it rather than resolving it:
|
|
75
|
+
* a host writing its own controller has the request and this does not, and a
|
|
76
|
+
* default here would attribute every such host's curation to nobody.
|
|
77
|
+
*/
|
|
78
|
+
patchType(typeName, patch, curatedBy) {
|
|
79
|
+
return this.registry.patchType(typeName, patch, curatedBy);
|
|
72
80
|
}
|
|
73
|
-
|
|
74
|
-
|
|
81
|
+
/** @param curatedBy who is doing it, for the audit trail. */
|
|
82
|
+
patchProperty(typeName, propertyName, patch, curatedBy) {
|
|
83
|
+
return this.registry.patchProperty(typeName, propertyName, patch, curatedBy);
|
|
75
84
|
}
|
|
76
|
-
/**
|
|
77
|
-
|
|
78
|
-
|
|
85
|
+
/**
|
|
86
|
+
* Drops every runtime edit, where the registry supports it.
|
|
87
|
+
*
|
|
88
|
+
* @param resetBy who is doing it. The one act here that destroys curation in
|
|
89
|
+
* bulk, so it is the one whose actor is hardest to reconstruct afterwards —
|
|
90
|
+
* nothing versions an overlay, and after this there is nothing left to read.
|
|
91
|
+
*/
|
|
92
|
+
resetOverlay(resetBy) {
|
|
93
|
+
return this.registry.resetOverlay(resetBy);
|
|
79
94
|
}
|
|
80
95
|
/** Columns a generic UI may render: visible, and not a blob. */
|
|
81
96
|
visibleColumns(type) {
|
package/dist/client.d.ts
CHANGED
|
@@ -93,5 +93,6 @@ export { CONNECTOR_KINDS, isConnectorKind, isTransformLanguage, isWorkflowEdge,
|
|
|
93
93
|
* `WORKFLOW_NODE_ID_PATTERN` and `WORKFLOW_NODE_KINDS` are what a palette and an
|
|
94
94
|
* id field should be built from rather than from a second copy that drifts.
|
|
95
95
|
*/
|
|
96
|
-
export { validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, workflowGraphHash, workflowRunOrder, isWorkflowExecutionMode, isWorkflowNodeKind, } from './catalog.pipeline';
|
|
96
|
+
export { validateWorkflow, WORKFLOW_EXECUTION_MODES, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_STATUSES, workflowGraphHash, workflowRunOrder, isWorkflowExecutionMode, isWorkflowNodeKind, isWorkflowStatus, } from './catalog.pipeline';
|
|
97
|
+
export type { WorkflowStatus } from './catalog.pipeline';
|
|
97
98
|
export type { CatalogTrace, CatalogTraceList, CatalogTraceOutcome, CatalogTraceSpan, TraceQuery, } from './catalog.workspace';
|