@substrat-run/control-plane-api 0.139.0 → 0.141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +17 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +701 -160
- package/dist/api.js.map +1 -1
- package/dist/audited-change.d.ts +139 -0
- package/dist/audited-change.d.ts.map +1 -0
- package/dist/audited-change.js +214 -0
- package/dist/audited-change.js.map +1 -0
- package/dist/cf-observability.d.ts.map +1 -1
- package/dist/cf-observability.js +38 -1
- package/dist/cf-observability.js.map +1 -1
- package/dist/connect-links.d.ts +29 -0
- package/dist/connect-links.d.ts.map +1 -0
- package/dist/connect-links.js +139 -0
- package/dist/connect-links.js.map +1 -0
- package/dist/connect-url.d.ts +20 -1
- package/dist/connect-url.d.ts.map +1 -1
- package/dist/connect-url.js +1 -1
- package/dist/connect-url.js.map +1 -1
- package/dist/connection-relay.d.ts +24 -0
- package/dist/connection-relay.d.ts.map +1 -1
- package/dist/connection-relay.js +31 -12
- package/dist/connection-relay.js.map +1 -1
- package/dist/email-intent.d.ts +71 -0
- package/dist/email-intent.d.ts.map +1 -0
- package/dist/email-intent.js +186 -0
- package/dist/email-intent.js.map +1 -0
- package/dist/field-coverage-read.d.ts +44 -0
- package/dist/field-coverage-read.d.ts.map +1 -0
- package/dist/field-coverage-read.js +45 -0
- package/dist/field-coverage-read.js.map +1 -0
- package/dist/field-coverage-tally.d.ts +12 -0
- package/dist/field-coverage-tally.d.ts.map +1 -1
- package/dist/field-coverage-tally.js +55 -14
- package/dist/field-coverage-tally.js.map +1 -1
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -3
- package/dist/index.js.map +1 -1
- package/dist/observability.d.ts +13 -0
- package/dist/observability.d.ts.map +1 -1
- package/dist/platform-drain.d.ts +12 -2
- package/dist/platform-drain.d.ts.map +1 -1
- package/dist/platform-drain.js +15 -3
- package/dist/platform-drain.js.map +1 -1
- package/dist/reconcile.d.ts +17 -2
- package/dist/reconcile.d.ts.map +1 -1
- package/dist/reconcile.js +35 -1
- package/dist/reconcile.js.map +1 -1
- package/dist/scope-copy-cleanup.d.ts +76 -0
- package/dist/scope-copy-cleanup.d.ts.map +1 -0
- package/dist/scope-copy-cleanup.js +228 -0
- package/dist/scope-copy-cleanup.js.map +1 -0
- package/dist/staff-client.d.ts +3 -1
- package/dist/staff-client.d.ts.map +1 -1
- package/dist/staff-client.js +2 -0
- package/dist/staff-client.js.map +1 -1
- package/dist/vertical-client.d.ts +30 -2
- package/dist/vertical-client.d.ts.map +1 -1
- package/dist/vertical-client.js +100 -22
- package/dist/vertical-client.js.map +1 -1
- package/package.json +10 -10
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A change the control plane makes in ANOTHER store, audited around the call (#2064).
|
|
3
|
+
*
|
|
4
|
+
* The owner hand-over (#1665) and dashboard member management (#1150) both change state in the
|
|
5
|
+
* vertical's own deployment, so the admin log cannot record them in the same transaction. Each is
|
|
6
|
+
* written in two rows paired by `operationId`: an `intent` before the vertical is reached, then
|
|
7
|
+
* the outcome (`applied`, `refused` or `failed`). The two stores are separate, so no order makes
|
|
8
|
+
* the pair atomic. This one fails toward "an intent with no outcome", never toward "a change with
|
|
9
|
+
* no row":
|
|
10
|
+
*
|
|
11
|
+
* - the `intent` write failing stops the call, so nothing moves unaudited;
|
|
12
|
+
* - the `applied` write failing is reported to the caller as exactly that: the change happened
|
|
13
|
+
* and its row is missing (`unrecorded`), so nobody retries a change believing it failed;
|
|
14
|
+
* - the `refused`/`failed` write failing is NOT the caller's answer. The vertical's own status
|
|
15
|
+
* still is, because a correctly refused `403`/`409` turned into a `5xx` reads as retryable,
|
|
16
|
+
* and each retry would add an intent row. It is logged at error level with the operation id.
|
|
17
|
+
*
|
|
18
|
+
* Neither failure is silent any more, and neither is left for a reader to explain: an intent
|
|
19
|
+
* with no outcome is closed by `settleUnrecordedOutcomes` below, which the scheduled pass runs.
|
|
20
|
+
* Every control-plane flow of this shape goes through `auditedChange`, so the next one cannot
|
|
21
|
+
* leave the outcome write in a bare `.catch(() => undefined)` again. Its outcome write is the
|
|
22
|
+
* kernel's `recordAuditOutcome`, which the schedule and peer kill switches use inside the adapters
|
|
23
|
+
* too (#2089), and the settle below closes their intents by the same rule.
|
|
24
|
+
*/
|
|
25
|
+
import { type AdminLogEntry, type OpsFailureEntry, type PlatformActorId } from '@substrat-run/contracts';
|
|
26
|
+
import { UNRECORDED_OUTCOME_LOG, type HostAdmin } from '@substrat-run/kernel';
|
|
27
|
+
/** One row of an audited change, before the flow adds its own fields. */
|
|
28
|
+
export type AuditedRow<T> = {
|
|
29
|
+
phase: 'intent';
|
|
30
|
+
} | {
|
|
31
|
+
phase: 'applied';
|
|
32
|
+
result: T;
|
|
33
|
+
} | {
|
|
34
|
+
phase: 'refused' | 'failed';
|
|
35
|
+
error: string;
|
|
36
|
+
};
|
|
37
|
+
export interface AuditedChangeSpec<T> {
|
|
38
|
+
/** Names the flow in the error log line, e.g. `owner-transfer`. */
|
|
39
|
+
flow: string;
|
|
40
|
+
operationId: string;
|
|
41
|
+
/** Writes one row. Throws when the row could not be written. */
|
|
42
|
+
record: (row: AuditedRow<T>) => Promise<void>;
|
|
43
|
+
/** The call into the other store. */
|
|
44
|
+
run: () => Promise<T>;
|
|
45
|
+
/** Whether a throw from `run` is a refusal (nothing was written there) rather than a failure. */
|
|
46
|
+
refused: (error: unknown) => boolean;
|
|
47
|
+
/** Where an unwritten outcome is reported. Defaults to `console.error`. */
|
|
48
|
+
logError?: (message: string, fields: Record<string, unknown>) => void;
|
|
49
|
+
}
|
|
50
|
+
export type AuditedChange<T> =
|
|
51
|
+
/** Applied, and recorded. */
|
|
52
|
+
{
|
|
53
|
+
operationId: string;
|
|
54
|
+
result: T;
|
|
55
|
+
}
|
|
56
|
+
/** Applied, but the `applied` row could not be written: `unrecorded` is why. */
|
|
57
|
+
| {
|
|
58
|
+
operationId: string;
|
|
59
|
+
result: T;
|
|
60
|
+
unrecorded: string;
|
|
61
|
+
}
|
|
62
|
+
/** `run` threw; its error, untouched, for the caller to answer with. */
|
|
63
|
+
| {
|
|
64
|
+
operationId: string;
|
|
65
|
+
error: unknown;
|
|
66
|
+
};
|
|
67
|
+
/** The structured line both unwritten outcomes leave: the kernel's, re-exported where it was first. */
|
|
68
|
+
export { UNRECORDED_OUTCOME_LOG };
|
|
69
|
+
/**
|
|
70
|
+
* Run one audited change: the `intent` row (its failure propagates, and `run` is not called),
|
|
71
|
+
* the call, then the outcome row. See the module header for what each failure answers.
|
|
72
|
+
*/
|
|
73
|
+
export declare function auditedChange<T>(spec: AuditedChangeSpec<T>): Promise<AuditedChange<T>>;
|
|
74
|
+
export type SettleAdmin = Pick<HostAdmin, 'auditLog' | 'settleUnrecordedOutcome'>;
|
|
75
|
+
export interface SettleOptions {
|
|
76
|
+
admin: SettleAdmin;
|
|
77
|
+
actor: PlatformActorId;
|
|
78
|
+
now?: Date;
|
|
79
|
+
/**
|
|
80
|
+
* How old an intent must be before its missing outcome is called unknown. It must exceed
|
|
81
|
+
* `AUDITED_CALL_DEADLINE_MS`, the longest the vertical call between the two rows may run, so a
|
|
82
|
+
* live request is never settled; `settleUnrecordedOutcomes` refuses a window that does not.
|
|
83
|
+
*/
|
|
84
|
+
graceMs?: number;
|
|
85
|
+
/**
|
|
86
|
+
* How far back each pass looks. The pass runs every quarter hour, so a week is margin for a
|
|
87
|
+
* sweep that was down, not the expected lag; it bounds what each pass reads from a log that is
|
|
88
|
+
* never pruned.
|
|
89
|
+
*/
|
|
90
|
+
lookbackMs?: number;
|
|
91
|
+
logError?: LogError;
|
|
92
|
+
}
|
|
93
|
+
/** The structured line a stored id the contract would refuse leaves, once per read (#2064). */
|
|
94
|
+
export declare const MALFORMED_OPERATION_ID_LOG = "audit-operation-id-malformed";
|
|
95
|
+
type LogError = NonNullable<AuditedChangeSpec<unknown>['logError']>;
|
|
96
|
+
export interface SettleResult {
|
|
97
|
+
/** Intents this pass closed with an `unknown` row. */
|
|
98
|
+
settled: {
|
|
99
|
+
action: string;
|
|
100
|
+
operationId: string;
|
|
101
|
+
tenantId: string | null;
|
|
102
|
+
scopeId: string | null;
|
|
103
|
+
}[];
|
|
104
|
+
/** Per-intent write failures: the intent stays open, and the next pass tries again. */
|
|
105
|
+
errors: {
|
|
106
|
+
operationId: string;
|
|
107
|
+
error: string;
|
|
108
|
+
}[];
|
|
109
|
+
/** Rows whose stored operation id the contract refuses: passed over, by row id. */
|
|
110
|
+
malformed: string[];
|
|
111
|
+
}
|
|
112
|
+
/** An hour: sixty times the deadline on the call it waits out. */
|
|
113
|
+
export declare const SETTLE_GRACE_MS: number;
|
|
114
|
+
/**
|
|
115
|
+
* Close every audited change whose intent has no outcome row (#2064). Each intent older than
|
|
116
|
+
* the grace window and still without an outcome is handed to `HostAdmin.settleUnrecordedOutcome`,
|
|
117
|
+
* which writes the `unknown` row and its ops-failure row in ONE transaction, and only if no
|
|
118
|
+
* outcome exists by then. The scan here only nominates. The host's check is the one that holds,
|
|
119
|
+
* so two concurrent passes, or an outcome landing between the scan and the settle, still leave
|
|
120
|
+
* one outcome row. A real outcome written later supersedes `unknown` (kernel `audit-outcome.ts`).
|
|
121
|
+
*/
|
|
122
|
+
export declare function settleUnrecordedOutcomes(opts: SettleOptions): Promise<SettleResult>;
|
|
123
|
+
/** The structured line an operation with two real outcomes leaves: the audit's invariant broke. */
|
|
124
|
+
export declare const OUTCOME_CONFLICT_LOG = "audit-outcome-conflict";
|
|
125
|
+
/**
|
|
126
|
+
* The admin-log read surface's half of the priority rule (#2064). Each row of an audited change
|
|
127
|
+
* is annotated with the outcome its operation stands at: a real outcome beats `unknown` whatever
|
|
128
|
+
* the row order, and two real outcomes read `conflicting`. An `unknown` a real outcome beat is
|
|
129
|
+
* marked `superseded`. The rows are returned as written, so the raw history stays readable.
|
|
130
|
+
*/
|
|
131
|
+
export declare function withAuditedOutcomes(admin: Pick<HostAdmin, 'auditedOperations'>, actor: PlatformActorId, entries: AdminLogEntry[], logError?: AuditedChangeSpec<unknown>['logError']): Promise<AdminLogEntry[]>;
|
|
132
|
+
/**
|
|
133
|
+
* The digest's half (#2064): of the ops-failure rows the settle wrote for an `unknown` outcome,
|
|
134
|
+
* the ones whose operation has since recorded ONE real outcome, whatever the order the two rows
|
|
135
|
+
* landed in. The settle's row names the operation in `reference`. The digest drops these. A
|
|
136
|
+
* `conflicting` operation is kept and logged: it is the one a person must look at.
|
|
137
|
+
*/
|
|
138
|
+
export declare function supersededUnknowns(admin: Pick<HostAdmin, 'auditedOperations'>, actor: PlatformActorId, failures: readonly OpsFailureEntry[], logError?: AuditedChangeSpec<unknown>['logError']): Promise<Set<string>>;
|
|
139
|
+
//# sourceMappingURL=audited-change.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audited-change.d.ts","sourceRoot":"","sources":["../src/audited-change.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,EAAqC,KAAK,aAAa,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC5I,OAAO,EAEL,sBAAsB,EAQtB,KAAK,SAAS,EACf,MAAM,sBAAsB,CAAC;AAG9B,yEAAyE;AACzE,MAAM,MAAM,UAAU,CAAC,CAAC,IACpB;IAAE,KAAK,EAAE,QAAQ,CAAA;CAAE,GACnB;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,CAAC,CAAA;CAAE,GAC/B;IAAE,KAAK,EAAE,SAAS,GAAG,QAAQ,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,MAAM,EAAE,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,qCAAqC;IACrC,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC;IACtB,iGAAiG;IACjG,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IACrC,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACvE;AAED,MAAM,MAAM,aAAa,CAAC,CAAC;AACzB,6BAA6B;AAC3B;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,CAAC,CAAA;CAAE;AACpC,gFAAgF;GAC9E;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE;AACxD,wEAAwE;GACtE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAK5C,uGAAuG;AACvG,OAAO,EAAE,sBAAsB,EAAE,CAAC;AAElC;;;GAGG;AACH,wBAAsB,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAc5F;AAED,MAAM,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,yBAAyB,CAAC,CAAC;AAElF,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,eAAe,CAAC;IACvB,GAAG,CAAC,EAAE,IAAI,CAAC;IACX;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,+FAA+F;AAC/F,eAAO,MAAM,0BAA0B,iCAAiC,CAAC;AAyBzE,KAAK,QAAQ,GAAG,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;AAEpE,MAAM,WAAW,YAAY;IAC3B,sDAAsD;IACtD,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,EAAE,CAAC;IACpG,uFAAuF;IACvF,MAAM,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjD,mFAAmF;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,kEAAkE;AAClE,eAAO,MAAM,eAAe,QAAiB,CAAC;AAQ9C;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAgDzF;AAGD,mGAAmG;AACnG,eAAO,MAAM,oBAAoB,2BAA2B,CAAC;AAqB7D;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EAC3C,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,aAAa,EAAE,EACxB,QAAQ,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,GAChD,OAAO,CAAC,aAAa,EAAE,CAAC,CAkB1B;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EAC3C,KAAK,EAAE,eAAe,EACtB,QAAQ,EAAE,SAAS,eAAe,EAAE,EACpC,QAAQ,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,GAChD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAkBtB"}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A change the control plane makes in ANOTHER store, audited around the call (#2064).
|
|
3
|
+
*
|
|
4
|
+
* The owner hand-over (#1665) and dashboard member management (#1150) both change state in the
|
|
5
|
+
* vertical's own deployment, so the admin log cannot record them in the same transaction. Each is
|
|
6
|
+
* written in two rows paired by `operationId`: an `intent` before the vertical is reached, then
|
|
7
|
+
* the outcome (`applied`, `refused` or `failed`). The two stores are separate, so no order makes
|
|
8
|
+
* the pair atomic. This one fails toward "an intent with no outcome", never toward "a change with
|
|
9
|
+
* no row":
|
|
10
|
+
*
|
|
11
|
+
* - the `intent` write failing stops the call, so nothing moves unaudited;
|
|
12
|
+
* - the `applied` write failing is reported to the caller as exactly that: the change happened
|
|
13
|
+
* and its row is missing (`unrecorded`), so nobody retries a change believing it failed;
|
|
14
|
+
* - the `refused`/`failed` write failing is NOT the caller's answer. The vertical's own status
|
|
15
|
+
* still is, because a correctly refused `403`/`409` turned into a `5xx` reads as retryable,
|
|
16
|
+
* and each retry would add an intent row. It is logged at error level with the operation id.
|
|
17
|
+
*
|
|
18
|
+
* Neither failure is silent any more, and neither is left for a reader to explain: an intent
|
|
19
|
+
* with no outcome is closed by `settleUnrecordedOutcomes` below, which the scheduled pass runs.
|
|
20
|
+
* Every control-plane flow of this shape goes through `auditedChange`, so the next one cannot
|
|
21
|
+
* leave the outcome write in a bare `.catch(() => undefined)` again. Its outcome write is the
|
|
22
|
+
* kernel's `recordAuditOutcome`, which the schedule and peer kill switches use inside the adapters
|
|
23
|
+
* too (#2089), and the settle below closes their intents by the same rule.
|
|
24
|
+
*/
|
|
25
|
+
import { AUDIT_ERROR_MAX, auditOperationId } from '@substrat-run/contracts';
|
|
26
|
+
import { AUDITED_CHANGE_ACTIONS, UNRECORDED_OUTCOME_LOG, recordAuditOutcome, operationKeyOf, effectiveOutcomes, isSupersededOutcome, } from '@substrat-run/kernel';
|
|
27
|
+
import { AUDITED_CALL_DEADLINE_MS } from './vertical-client.js';
|
|
28
|
+
const messageOf = (e) => (e instanceof Error ? e.message : String(e));
|
|
29
|
+
const consoleError = (message, fields) => console.error(message, fields);
|
|
30
|
+
/** The structured line both unwritten outcomes leave: the kernel's, re-exported where it was first. */
|
|
31
|
+
export { UNRECORDED_OUTCOME_LOG };
|
|
32
|
+
/**
|
|
33
|
+
* Run one audited change: the `intent` row (its failure propagates, and `run` is not called),
|
|
34
|
+
* the call, then the outcome row. See the module header for what each failure answers.
|
|
35
|
+
*/
|
|
36
|
+
export async function auditedChange(spec) {
|
|
37
|
+
const { flow, operationId, record } = spec;
|
|
38
|
+
const logError = spec.logError ?? consoleError;
|
|
39
|
+
await record({ phase: 'intent' });
|
|
40
|
+
let result;
|
|
41
|
+
try {
|
|
42
|
+
result = await spec.run();
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
const phase = spec.refused(error) ? 'refused' : 'failed';
|
|
46
|
+
await recordAuditOutcome(() => record({ phase, error: messageOf(error).slice(0, AUDIT_ERROR_MAX) }), { flow, operationId, phase }, logError);
|
|
47
|
+
return { operationId, error };
|
|
48
|
+
}
|
|
49
|
+
const unrecorded = await recordAuditOutcome(() => record({ phase: 'applied', result }), { flow, operationId, phase: 'applied' }, logError);
|
|
50
|
+
return unrecorded === null ? { operationId, result } : { operationId, result, unrecorded };
|
|
51
|
+
}
|
|
52
|
+
/** The structured line a stored id the contract would refuse leaves, once per read (#2064). */
|
|
53
|
+
export const MALFORMED_OPERATION_ID_LOG = 'audit-operation-id-malformed';
|
|
54
|
+
/**
|
|
55
|
+
* The audited operation a STORED id names, or `malformed`, or null when it names none (another
|
|
56
|
+
* action, or no id). ONE predicate for every reader that turns a stored id into a resolved or
|
|
57
|
+
* enriched one: the admin-log annotation, the digest and the sweep's scan. It is the contract's
|
|
58
|
+
* own `auditOperationId`. A row written before the contract held ids to well-formed text may
|
|
59
|
+
* carry one it would now refuse. That row names no operation: it is returned raw, never
|
|
60
|
+
* enriched, settled or thrown on, and the reader counts it and logs it once.
|
|
61
|
+
*/
|
|
62
|
+
function storedOperationOf(action, operationId, where) {
|
|
63
|
+
if (!AUDITED_CHANGE_ACTIONS.includes(action) || operationId === undefined || operationId === null)
|
|
64
|
+
return null;
|
|
65
|
+
if (!auditOperationId.safeParse(operationId).success)
|
|
66
|
+
return 'malformed';
|
|
67
|
+
return { action, operationId: operationId, tenantId: where.tenantId, scopeId: where.scopeId };
|
|
68
|
+
}
|
|
69
|
+
/** One log line for every stored row a reader passed over, by row id. */
|
|
70
|
+
function reportMalformed(logError, reader, rows) {
|
|
71
|
+
if (rows.length > 0)
|
|
72
|
+
logError(MALFORMED_OPERATION_ID_LOG, { reader, count: rows.length, rows });
|
|
73
|
+
}
|
|
74
|
+
/** An hour: sixty times the deadline on the call it waits out. */
|
|
75
|
+
export const SETTLE_GRACE_MS = 60 * 60 * 1000;
|
|
76
|
+
const SETTLE_LOOKBACK_MS = 7 * 24 * 60 * 60 * 1000;
|
|
77
|
+
const PAGE = 500;
|
|
78
|
+
if (SETTLE_GRACE_MS <= AUDITED_CALL_DEADLINE_MS) {
|
|
79
|
+
throw new Error('SETTLE_GRACE_MS must exceed AUDITED_CALL_DEADLINE_MS, or a live call could be settled');
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Close every audited change whose intent has no outcome row (#2064). Each intent older than
|
|
83
|
+
* the grace window and still without an outcome is handed to `HostAdmin.settleUnrecordedOutcome`,
|
|
84
|
+
* which writes the `unknown` row and its ops-failure row in ONE transaction, and only if no
|
|
85
|
+
* outcome exists by then. The scan here only nominates. The host's check is the one that holds,
|
|
86
|
+
* so two concurrent passes, or an outcome landing between the scan and the settle, still leave
|
|
87
|
+
* one outcome row. A real outcome written later supersedes `unknown` (kernel `audit-outcome.ts`).
|
|
88
|
+
*/
|
|
89
|
+
export async function settleUnrecordedOutcomes(opts) {
|
|
90
|
+
const { admin, actor } = opts;
|
|
91
|
+
const now = (opts.now ?? new Date()).getTime();
|
|
92
|
+
const graceMs = opts.graceMs ?? SETTLE_GRACE_MS;
|
|
93
|
+
if (graceMs <= AUDITED_CALL_DEADLINE_MS) {
|
|
94
|
+
throw new Error(`a settle grace of ${graceMs} ms does not exceed the ${AUDITED_CALL_DEADLINE_MS} ms call deadline`);
|
|
95
|
+
}
|
|
96
|
+
const cutoff = new Date(now - graceMs).toISOString();
|
|
97
|
+
const since = new Date(now - (opts.lookbackMs ?? SETTLE_LOOKBACK_MS)).toISOString();
|
|
98
|
+
const intents = new Map();
|
|
99
|
+
const closed = new Set();
|
|
100
|
+
const malformed = [];
|
|
101
|
+
for (let cursor;;) {
|
|
102
|
+
const page = await admin.auditLog(actor, { action: [...AUDITED_CHANGE_ACTIONS], since, limit: PAGE, cursor });
|
|
103
|
+
for (const row of page) {
|
|
104
|
+
const after = row.after;
|
|
105
|
+
const operation = storedOperationOf(row.action, after?.operationId, row);
|
|
106
|
+
if (operation === 'malformed')
|
|
107
|
+
malformed.push(row.id);
|
|
108
|
+
if (operation === null || operation === 'malformed')
|
|
109
|
+
continue;
|
|
110
|
+
// The operation's whole identity, as every reader keys it.
|
|
111
|
+
const key = operationKeyOf(operation);
|
|
112
|
+
if (after?.phase === 'intent') {
|
|
113
|
+
if (row.at < cutoff)
|
|
114
|
+
intents.set(key, row);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
closed.add(key);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (page.length < PAGE)
|
|
121
|
+
break;
|
|
122
|
+
cursor = page[page.length - 1].id;
|
|
123
|
+
}
|
|
124
|
+
reportMalformed(opts.logError ?? consoleError, 'settle', malformed);
|
|
125
|
+
const result = { settled: [], errors: [], malformed };
|
|
126
|
+
const minutes = Math.round(graceMs / 60_000);
|
|
127
|
+
for (const [key, intent] of intents) {
|
|
128
|
+
if (closed.has(key))
|
|
129
|
+
continue;
|
|
130
|
+
const operationId = intent.after.operationId;
|
|
131
|
+
const error = `no outcome was recorded within ${minutes} minutes of the intent (${intent.at}); whether the change happened is not known to this log`;
|
|
132
|
+
try {
|
|
133
|
+
if (await admin.settleUnrecordedOutcome(actor, { intentId: intent.id, error })) {
|
|
134
|
+
result.settled.push({ action: intent.action, operationId, tenantId: intent.tenantId, scopeId: intent.scopeId });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch (e) {
|
|
138
|
+
result.errors.push({ operationId, error: messageOf(e) });
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return result;
|
|
142
|
+
}
|
|
143
|
+
/** The structured line an operation with two real outcomes leaves: the audit's invariant broke. */
|
|
144
|
+
export const OUTCOME_CONFLICT_LOG = 'audit-outcome-conflict';
|
|
145
|
+
/**
|
|
146
|
+
* Resolve `refs` to their effective outcomes (#2064): ONE batched read of exactly these
|
|
147
|
+
* operations' rows through the operation-id index, then the kernel's priority rule. An operation
|
|
148
|
+
* found `conflicting` is logged here, once per read, so a reader never hides it.
|
|
149
|
+
*/
|
|
150
|
+
async function resolveOperations(admin, actor, refs, logError = consoleError) {
|
|
151
|
+
if (refs.length === 0)
|
|
152
|
+
return new Map();
|
|
153
|
+
const effective = effectiveOutcomes(await admin.auditedOperations(actor, refs));
|
|
154
|
+
for (const [key, outcome] of effective) {
|
|
155
|
+
if (outcome.outcome === 'conflicting')
|
|
156
|
+
logError(OUTCOME_CONFLICT_LOG, { operation: JSON.parse(key), outcomes: outcome.real });
|
|
157
|
+
}
|
|
158
|
+
return effective;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* The admin-log read surface's half of the priority rule (#2064). Each row of an audited change
|
|
162
|
+
* is annotated with the outcome its operation stands at: a real outcome beats `unknown` whatever
|
|
163
|
+
* the row order, and two real outcomes read `conflicting`. An `unknown` a real outcome beat is
|
|
164
|
+
* marked `superseded`. The rows are returned as written, so the raw history stays readable.
|
|
165
|
+
*/
|
|
166
|
+
export async function withAuditedOutcomes(admin, actor, entries, logError) {
|
|
167
|
+
const operations = entries.map((row) => storedOperationOf(row.action, row.after?.operationId, row));
|
|
168
|
+
reportMalformed(logError ?? consoleError, 'admin-log', entries.filter((_, i) => operations[i] === 'malformed').map((row) => row.id));
|
|
169
|
+
const refs = operations.filter((o) => o !== null && o !== 'malformed');
|
|
170
|
+
if (refs.length === 0)
|
|
171
|
+
return entries;
|
|
172
|
+
const effective = await resolveOperations(admin, actor, refs, logError);
|
|
173
|
+
return entries.map((row, i) => {
|
|
174
|
+
const operation = operations[i];
|
|
175
|
+
if (operation === null || operation === 'malformed')
|
|
176
|
+
return row;
|
|
177
|
+
const outcome = effective.get(operationKeyOf(operation));
|
|
178
|
+
const phase = row.after.phase ?? null;
|
|
179
|
+
return {
|
|
180
|
+
...row,
|
|
181
|
+
audited: { operationId: operation.operationId, outcome: outcome?.outcome ?? 'pending', superseded: isSupersededOutcome(phase, outcome) },
|
|
182
|
+
};
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* The digest's half (#2064): of the ops-failure rows the settle wrote for an `unknown` outcome,
|
|
187
|
+
* the ones whose operation has since recorded ONE real outcome, whatever the order the two rows
|
|
188
|
+
* landed in. The settle's row names the operation in `reference`. The digest drops these. A
|
|
189
|
+
* `conflicting` operation is kept and logged: it is the one a person must look at.
|
|
190
|
+
*/
|
|
191
|
+
export async function supersededUnknowns(admin, actor, failures, logError) {
|
|
192
|
+
const unknowns = new Map();
|
|
193
|
+
const malformed = [];
|
|
194
|
+
for (const f of failures) {
|
|
195
|
+
if (f.stage !== 'outcome-unknown' || !f.operation.startsWith('audit.'))
|
|
196
|
+
continue;
|
|
197
|
+
const operation = storedOperationOf(f.operation.slice('audit.'.length), f.reference, f);
|
|
198
|
+
if (operation === 'malformed')
|
|
199
|
+
malformed.push(f.id);
|
|
200
|
+
else if (operation !== null)
|
|
201
|
+
unknowns.set(f.id, operation);
|
|
202
|
+
}
|
|
203
|
+
// A failure row passed over is kept in the digest: it is reported, never dropped as resolved.
|
|
204
|
+
reportMalformed(logError ?? consoleError, 'digest', malformed);
|
|
205
|
+
const effective = await resolveOperations(admin, actor, [...unknowns.values()], logError);
|
|
206
|
+
const superseded = new Set();
|
|
207
|
+
for (const [id, ref] of unknowns) {
|
|
208
|
+
const outcome = effective.get(operationKeyOf(ref));
|
|
209
|
+
if (outcome && outcome.real.length === 1)
|
|
210
|
+
superseded.add(id);
|
|
211
|
+
}
|
|
212
|
+
return superseded;
|
|
213
|
+
}
|
|
214
|
+
//# sourceMappingURL=audited-change.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audited-change.js","sourceRoot":"","sources":["../src/audited-change.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAkE,MAAM,yBAAyB,CAAC;AAC5I,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,cAAc,EACd,iBAAiB,EACjB,mBAAmB,GAKpB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AA8BhE,MAAM,SAAS,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACvF,MAAM,YAAY,GAAG,CAAC,OAAe,EAAE,MAA+B,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAE1G,uGAAuG;AACvG,OAAO,EAAE,sBAAsB,EAAE,CAAC;AAElC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAI,IAA0B;IAC/D,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC;IAC/C,MAAM,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClC,IAAI,MAAS,CAAC;IACd,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;QACzD,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC7I,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC3I,OAAO,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AAC7F,CAAC;AAuBD,+FAA+F;AAC/F,MAAM,CAAC,MAAM,0BAA0B,GAAG,8BAA8B,CAAC;AAEzE;;;;;;;GAOG;AACH,SAAS,iBAAiB,CACxB,MAAc,EACd,WAAoB,EACpB,KAA0D;IAE1D,IAAI,CAAE,sBAA4C,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACtI,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,OAAO;QAAE,OAAO,WAAW,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAqB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;AAC1G,CAAC;AAED,yEAAyE;AACzE,SAAS,eAAe,CAAC,QAAkB,EAAE,MAAc,EAAE,IAAc;IACzE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,0BAA0B,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAClG,CAAC;AAaD,kEAAkE;AAClE,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9C,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACnD,MAAM,IAAI,GAAG,GAAG,CAAC;AAEjB,IAAI,eAAe,IAAI,wBAAwB,EAAE,CAAC;IAChD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;AAC3G,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,IAAmB;IAChE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAC9B,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,eAAe,CAAC;IAChD,IAAI,OAAO,IAAI,wBAAwB,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,2BAA2B,wBAAwB,mBAAmB,CAAC,CAAC;IACtH,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAEpF,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgB,CAAC;IACvC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,MAA0B,IAAM,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,sBAAsB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9G,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,KAA0D,CAAC;YAC7E,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;YACzE,IAAI,SAAS,KAAK,WAAW;gBAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtD,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,WAAW;gBAAE,SAAS;YAC9D,2DAA2D;YAC3D,MAAM,GAAG,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,KAAK,EAAE,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,GAAG,CAAC,EAAE,GAAG,MAAM;oBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI;YAAE,MAAM;QAC9B,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;IACrC,CAAC;IAED,eAAe,CAAC,IAAI,CAAC,QAAQ,IAAI,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IACpE,MAAM,MAAM,GAAiB,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;IACpE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC;IAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC9B,MAAM,WAAW,GAAI,MAAM,CAAC,KAAiC,CAAC,WAAW,CAAC;QAC1E,MAAM,KAAK,GAAG,kCAAkC,OAAO,2BAA2B,MAAM,CAAC,EAAE,yDAAyD,CAAC;QACrJ,IAAI,CAAC;YACH,IAAI,MAAM,KAAK,CAAC,uBAAuB,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAC/E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YAClH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAGD,mGAAmG;AACnG,MAAM,CAAC,MAAM,oBAAoB,GAAG,wBAAwB,CAAC;AAE7D;;;;GAIG;AACH,KAAK,UAAU,iBAAiB,CAC9B,KAA2C,EAC3C,KAAsB,EACtB,IAA2B,EAC3B,QAAQ,GAAwD,YAAY;IAE5E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IACxC,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAChF,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,SAAS,EAAE,CAAC;QACvC,IAAI,OAAO,CAAC,OAAO,KAAK,aAAa;YAAE,QAAQ,CAAC,oBAAoB,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3I,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAA2C,EAC3C,KAAsB,EACtB,OAAwB,EACxB,QAAiD;IAEjD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACrC,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAG,GAAG,CAAC,KAA0C,EAAE,WAAW,EAAE,GAAG,CAAC,CACjG,CAAC;IACF,eAAe,CAAC,QAAQ,IAAI,YAAY,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACrI,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAA4B,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,WAAW,CAAC,CAAC;IACjG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IACxE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC;QACjC,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,WAAW;YAAE,OAAO,GAAG,CAAC;QAChE,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;QACzD,MAAM,KAAK,GAAI,GAAG,CAAC,KAA4B,CAAC,KAAK,IAAI,IAAI,CAAC;QAC9D,OAAO;YACL,GAAG,GAAG;YACN,OAAO,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,UAAU,EAAE,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE;SACzI,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAA2C,EAC3C,KAAsB,EACtB,QAAoC,EACpC,QAAiD;IAEjD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAC;IACxD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,KAAK,KAAK,iBAAiB,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,SAAS;QACjF,MAAM,SAAS,GAAG,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACxF,IAAI,SAAS,KAAK,WAAW;YAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC/C,IAAI,SAAS,KAAK,IAAI;YAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IAC7D,CAAC;IACD,8FAA8F;IAC9F,eAAe,CAAC,QAAQ,IAAI,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC1F,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cf-observability.d.ts","sourceRoot":"","sources":["../src/cf-observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAqE,KAAK,SAAS,EAAuB,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"cf-observability.d.ts","sourceRoot":"","sources":["../src/cf-observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAqE,KAAK,SAAS,EAAuB,MAAM,uBAAuB,CAAC;AAM/I,OAAO,KAAK,EACV,mBAAmB,EASpB,MAAM,oBAAoB,CAAC;AAG5B;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,kCAAkC,CAAC;AAsOnE;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC1B,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/D,OAAO,CAOT;AAyPD;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;;;;;;;OAgBG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;OASG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAuED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,sBAAsB,GAAG,mBAAmB,CAkzC7F"}
|
package/dist/cf-observability.js
CHANGED
|
@@ -4,7 +4,8 @@ import { cachedSource, cutOverSource } from './aggregate-source.js';
|
|
|
4
4
|
import { stableDeploymentRefFor } from './deploy.js';
|
|
5
5
|
import { aggregateReads } from './aggregate-reads.js';
|
|
6
6
|
import { serviceFamilyPattern } from './service-family.js';
|
|
7
|
-
import { FIELD_COVERAGE_ID_FIELD } from '@substrat-run/contracts';
|
|
7
|
+
import { FIELD_COVERAGE_ID_FIELD, ROUTER_SCRIPT_NAMES } from '@substrat-run/contracts';
|
|
8
|
+
import { readFieldCoverage } from './field-coverage-read.js';
|
|
8
9
|
import { TENANT_METRICS_LIMIT, REQUEST_FACET_KEYS, INVOCATION_KINDS, invocationKindOf } from './observability.js';
|
|
9
10
|
/**
|
|
10
11
|
* An HTTP method as RFC 9110 §9.1 defines it: a `token` (§5.6.2), one or more `tchar`.
|
|
@@ -931,6 +932,27 @@ export function createCfObservabilityReader(opts) {
|
|
|
931
932
|
async tenantLogs(input) {
|
|
932
933
|
return queryTenantLogs(input);
|
|
933
934
|
},
|
|
935
|
+
async fieldCoverage(input) {
|
|
936
|
+
const window = resolveObservabilityWindow(input);
|
|
937
|
+
const timeframe = { from: Date.parse(window.since), to: Date.parse(window.until) };
|
|
938
|
+
const reports = await queryAllFieldEvents([
|
|
939
|
+
{ key: 'tenantId', operation: 'eq', type: 'string', value: input.tenantId },
|
|
940
|
+
{ key: 'substrat', operation: 'eq', type: 'string', value: 'invocation' },
|
|
941
|
+
{ key: 'vertical', operation: 'eq', type: 'string', value: input.vertical },
|
|
942
|
+
{ key: FIELD_COVERAGE_ID_FIELD, operation: 'regex', type: 'string', value: '^[0-9A-HJKMNP-TV-Z]{26}$' },
|
|
943
|
+
...serviceFilter(input.services),
|
|
944
|
+
...(input.versionId ? [{ key: 'versionId', operation: 'eq', type: 'string', value: input.versionId }] : []),
|
|
945
|
+
], timeframe);
|
|
946
|
+
const router = (await Promise.all(ROUTER_SCRIPT_NAMES.map((service) => queryAllFieldEvents([
|
|
947
|
+
{ key: 'tenantId', operation: 'eq', type: 'string', value: input.tenantId },
|
|
948
|
+
{ key: 'router', operation: 'eq', type: 'string', value: 'request' },
|
|
949
|
+
{ key: 'vertical', operation: 'eq', type: 'string', value: input.vertical },
|
|
950
|
+
{ key: 'scopeId', operation: 'eq', type: 'string', value: input.scopeId },
|
|
951
|
+
{ key: FIELD_COVERAGE_ID_FIELD, operation: 'regex', type: 'string', value: '^[0-9A-HJKMNP-TV-Z]{26}$' },
|
|
952
|
+
{ key: '$metadata.service', operation: 'eq', type: 'string', value: service },
|
|
953
|
+
], timeframe)))).flat();
|
|
954
|
+
return readFieldCoverage(reports, router, input, window);
|
|
955
|
+
},
|
|
934
956
|
// #1746/#1877: the request histogram, facet counts and log patterns — computed once,
|
|
935
957
|
// from cubes, by `aggregateReads` over the (cached) telemetry source.
|
|
936
958
|
...aggregateReads(source),
|
|
@@ -1534,6 +1556,21 @@ export function createCfObservabilityReader(opts) {
|
|
|
1534
1556
|
const outer = json.result?.events;
|
|
1535
1557
|
return (Array.isArray(outer) ? outer : (outer?.events ?? []));
|
|
1536
1558
|
}
|
|
1559
|
+
/** Read a complete sampled span, or refuse it; a truncated page must never mean zero fields. */
|
|
1560
|
+
async function queryAllFieldEvents(filters, timeframe, depth = 0) {
|
|
1561
|
+
const page = await queryRaw(filters, timeframe, 2000);
|
|
1562
|
+
if (page.length < 2000)
|
|
1563
|
+
return page;
|
|
1564
|
+
if (depth >= 12 || timeframe.to - timeframe.from <= 1000) {
|
|
1565
|
+
throw new ControlPlaneError(503, 'field coverage window is too busy — choose a shorter window');
|
|
1566
|
+
}
|
|
1567
|
+
const middle = Math.floor((timeframe.from + timeframe.to) / 2);
|
|
1568
|
+
const [left, right] = await Promise.all([
|
|
1569
|
+
queryAllFieldEvents(filters, { from: timeframe.from, to: middle }, depth + 1),
|
|
1570
|
+
queryAllFieldEvents(filters, { from: middle, to: timeframe.to }, depth + 1),
|
|
1571
|
+
]);
|
|
1572
|
+
return [...left, ...right];
|
|
1573
|
+
}
|
|
1537
1574
|
/** One telemetry query — narrowed to a single service, or to none (the fleet view). */
|
|
1538
1575
|
async function queryEvents(service, level, search, hours, limit) {
|
|
1539
1576
|
const to = Date.now();
|