@substrat-run/adapter-cloudflare 0.139.0 → 0.140.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +4 -0
- package/dist/checker.js.map +1 -1
- package/dist/control-plane-do.d.ts +57 -6
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +213 -62
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +30 -7
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +247 -40
- package/dist/host.js.map +1 -1
- package/dist/live-reads.d.ts +30 -7
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +43 -9
- package/dist/live-reads.js.map +1 -1
- package/dist/route-resolver.d.ts +1 -1
- package/dist/route-resolver.d.ts.map +1 -1
- package/dist/route-resolver.js +7 -0
- package/dist/route-resolver.js.map +1 -1
- package/dist/scope-do.d.ts +29 -1
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +714 -217
- package/dist/scope-do.js.map +1 -1
- package/dist/sql.d.ts +2 -0
- package/dist/sql.d.ts.map +1 -1
- package/dist/sql.js +23 -1
- package/dist/sql.js.map +1 -1
- package/package.json +9 -9
package/dist/control-plane-do.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { DurableObject } from 'cloudflare:workers';
|
|
2
|
-
import { membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, MEMBERSHIP_FENCES_TABLE, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, memberAddedAudit, orgChangeBound, tenantCoverage, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, FINDINGS_DDL, createFindingRule, findingOfOpsFailure, findingOfSweepRun, pruneFindings, listFindingRules, listFindings, observeFinding, revokeFindingRule, setFindingStatus, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, PEER_SWITCHES_DDL, SWITCH_OWED_DDL, clearSwitchOwed, markSwitchOwed, scopeOwesSwitch, switchesOwedOf, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, recordSwitchedOff, recordSwitchedOn, recordWriteSuperseded, restoreSwitchRecord, scopesSwitchedOffFor, switchRecordStatesOf, switchRecordsOf, switchesBackfillSqlOf, switchesDdlOf, switchesTableExists, switchesTableOf, tenantHeldOf, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
|
|
2
|
+
import { membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, MEMBERSHIP_FENCES_TABLE, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, memberAddedAudit, orgChangeBound, tenantCoverage, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, FINDINGS_DDL, CONNECT_LINKS_DDL, consumeConnectLinkRow, insertConnectLink as insertConnectLinkRow, listConnectLinks as listConnectLinkRows, readConnectLink as readConnectLinkRow, restoreConnectLinkRow, revokeConnectLinkRow, createFindingRule, findingOfOpsFailure, findingOfSweepRun, pruneFindings, listFindingRules, listFindings, observeFinding, revokeFindingRule, setFindingStatus, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, PEER_SWITCHES_DDL, PEER_BINDINGS_DDL, SWITCH_OWED_DDL, clearSwitchOwed, markSwitchOwed, scopeOwesSwitch, switchesOwedOf, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, recordSwitchedOff, recordSwitchedOn, recordWriteSuperseded, restoreSwitchRecord, scopesSwitchedOffFor, switchRecordStatesOf, switchRecordsOf, switchesBackfillSqlOf, switchesDdlOf, switchesTableExists, switchesTableOf, tenantHeldOf, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, resolvePeerInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, SETTLE_INTENT_SQL, SETTLE_OUTCOME_SQL, settleOutcomeParamsOf, unknownOutcomeOf, ADMIN_LOG_INDEX_DDL, ADMIN_LOG_INDEXES_SQL, readAuditedOperations, } from '@substrat-run/kernel';
|
|
3
3
|
import { replyOf } from './do-reply.js';
|
|
4
|
-
import {
|
|
4
|
+
import { switchSqlOver } from './scope-do.js';
|
|
5
|
+
import { blankSqlComments, executableSqlStatements } from '@substrat-run/kernel';
|
|
5
6
|
import { doBuiltColumnsOf, doRedactionSql } from './sql.js';
|
|
6
7
|
import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, SCOPE_GATE_REASONS, substratError } from '@substrat-run/contracts';
|
|
7
8
|
/**
|
|
@@ -159,6 +160,7 @@ const DIRECTORY_DDL = `
|
|
|
159
160
|
serving_ref TEXT,
|
|
160
161
|
created_at TEXT NOT NULL
|
|
161
162
|
);
|
|
163
|
+
${PEER_BINDINGS_DDL}
|
|
162
164
|
CREATE TABLE IF NOT EXISTS hostnames (
|
|
163
165
|
hostname TEXT PRIMARY KEY,
|
|
164
166
|
tenant_id TEXT NOT NULL,
|
|
@@ -416,6 +418,9 @@ const DIRECTORY_DDL = `
|
|
|
416
418
|
updated_at TEXT NOT NULL,
|
|
417
419
|
PRIMARY KEY (connection_id, state_key)
|
|
418
420
|
);
|
|
421
|
+
-- A vertical's mailed connect links (connections.md §3.5.4) — kernel-owned DDL, so
|
|
422
|
+
-- both adapters build the same table.
|
|
423
|
+
${CONNECT_LINKS_DDL}
|
|
419
424
|
CREATE TABLE IF NOT EXISTS _substrat_identities (
|
|
420
425
|
provider TEXT NOT NULL,
|
|
421
426
|
external_id TEXT NOT NULL,
|
|
@@ -478,16 +483,12 @@ const DIRECTORY_DDL = `
|
|
|
478
483
|
on_behalf_of TEXT,
|
|
479
484
|
at TEXT NOT NULL
|
|
480
485
|
);
|
|
481
|
-
--
|
|
482
|
-
--
|
|
486
|
+
-- Every admin-log index, from the kernel's one list (#2064), which the legacy
|
|
487
|
+
-- rebuild runs again after its rename.
|
|
483
488
|
-- NOTE: keep every comment in this DDL free of semicolons. The constructor
|
|
484
489
|
-- splits this string on the statement separator, and a semicolon inside a
|
|
485
490
|
-- comment strands a comment-only fragment that execs as "no statement".
|
|
486
|
-
|
|
487
|
-
CREATE INDEX IF NOT EXISTS _substrat_admin_log_scope ON _substrat_admin_log (scope_id, id);
|
|
488
|
-
CREATE INDEX IF NOT EXISTS _substrat_admin_log_actor ON _substrat_admin_log (actor, id);
|
|
489
|
-
CREATE INDEX IF NOT EXISTS _substrat_admin_log_action ON _substrat_admin_log (action, id);
|
|
490
|
-
CREATE INDEX IF NOT EXISTS _substrat_admin_log_at ON _substrat_admin_log (at);
|
|
491
|
+
${ADMIN_LOG_INDEXES_SQL}
|
|
491
492
|
-- Operational failures (#559) - what the platform could NOT do, and why. NOT the
|
|
492
493
|
-- admin log (that is the never-swept compliance witness of successful mutations)
|
|
493
494
|
-- but retention-bounded telemetry, pruned on write after OPS_FAILURE_RETENTION_DAYS.
|
|
@@ -674,8 +675,8 @@ const SCOPE_COLUMNS_ADDED = [
|
|
|
674
675
|
* fragment and the DDL had parted company (its test holds it empty).
|
|
675
676
|
*/
|
|
676
677
|
export function planDirectoryDdl(ddl) {
|
|
677
|
-
const all =
|
|
678
|
-
const afterColumns =
|
|
678
|
+
const all = executableSqlStatements(ddl);
|
|
679
|
+
const afterColumns = executableSqlStatements(VERSION_MIGRATIONS_DDL);
|
|
679
680
|
const held = new Set(afterColumns);
|
|
680
681
|
return {
|
|
681
682
|
loop: all.filter((stmt) => !held.has(stmt)),
|
|
@@ -825,7 +826,7 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
825
826
|
this.buildDirectorySchema({ holdSwitchRecords: newRecords, holdFences: fencesAreNew });
|
|
826
827
|
for (const kind of newRecords) {
|
|
827
828
|
this.ctx.storage.transactionSync(() => {
|
|
828
|
-
for (const stmt of
|
|
829
|
+
for (const stmt of executableSqlStatements(switchesDdlOf(kind)))
|
|
829
830
|
this.sql.exec(stmt);
|
|
830
831
|
this.sql.exec(switchesBackfillSqlOf(kind));
|
|
831
832
|
});
|
|
@@ -833,7 +834,7 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
833
834
|
// #1184: the removal fence and its backfill from the admin log, together, the same way.
|
|
834
835
|
if (fencesAreNew) {
|
|
835
836
|
this.ctx.storage.transactionSync(() => {
|
|
836
|
-
for (const stmt of
|
|
837
|
+
for (const stmt of executableSqlStatements(MEMBERSHIP_FENCES_DDL))
|
|
837
838
|
this.sql.exec(stmt);
|
|
838
839
|
// Stamped now, inside this unit (`membershipFencesBackfillSql` says why).
|
|
839
840
|
this.sql.exec(membershipFencesBackfillSql(new Date().toISOString()));
|
|
@@ -907,7 +908,7 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
907
908
|
rebuildAtomically(statements) {
|
|
908
909
|
this.ctx.storage.transactionSync(() => {
|
|
909
910
|
for (const stmt of statements)
|
|
910
|
-
this.sql.exec(stmt);
|
|
911
|
+
this.sql.exec(blankSqlComments(stmt));
|
|
911
912
|
});
|
|
912
913
|
}
|
|
913
914
|
/**
|
|
@@ -972,6 +973,8 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
972
973
|
FROM _substrat_admin_log`,
|
|
973
974
|
'DROP TABLE _substrat_admin_log',
|
|
974
975
|
'ALTER TABLE _substrat_admin_log_new RENAME TO _substrat_admin_log',
|
|
976
|
+
// The DROP took every index on the table: rebuild them from the kernel's list (#2064).
|
|
977
|
+
...ADMIN_LOG_INDEX_DDL,
|
|
975
978
|
]);
|
|
976
979
|
}
|
|
977
980
|
/**
|
|
@@ -1292,6 +1295,7 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
1292
1295
|
'_substrat_system_switches', // #1674: the schedule switch's record, per scope
|
|
1293
1296
|
'_substrat_membership_fences', // #1184: the latest removal, per principal
|
|
1294
1297
|
'_substrat_peer_switches', // #2029: the peer switch's record, per scope
|
|
1298
|
+
'peer_bindings', // #1720: a tenant's explicit caller-to-target choices
|
|
1295
1299
|
'_substrat_switch_owed', // #2045: subjects owed a re-assert, per scope
|
|
1296
1300
|
'_substrat_findings', // #1748: the tenant's findings
|
|
1297
1301
|
'_substrat_finding_rules', // #1748: the tenant's suppress rules
|
|
@@ -1654,6 +1658,9 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
1654
1658
|
else {
|
|
1655
1659
|
this.sql.exec('UPDATE scopes SET status = ? WHERE scope_id = ?', to, scopeId);
|
|
1656
1660
|
}
|
|
1661
|
+
if (to === 'archived' || to === 'reaped') {
|
|
1662
|
+
this.sql.exec('UPDATE peer_bindings SET invalidated = 1 WHERE tenant_id = ? AND target_scope_id = ?', tenantId, scopeId);
|
|
1663
|
+
}
|
|
1657
1664
|
if (to === 'reaped') {
|
|
1658
1665
|
forgetSwitchesOf(this.kernelSql, scopeId);
|
|
1659
1666
|
this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
|
|
@@ -1761,21 +1768,74 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
1761
1768
|
forkedFrom: r.forked_from ?? null,
|
|
1762
1769
|
})), tenantId, vertical);
|
|
1763
1770
|
}
|
|
1771
|
+
/** Binding is read at call execution, so a queued invoke follows the current explicit choice. */
|
|
1772
|
+
resolvePeerInstance(tenantId, callerScopeId, vertical) {
|
|
1773
|
+
const rows = this.sql.exec(`SELECT scope_id, tenant_id, vertical, status, kind, forked_from FROM scopes
|
|
1774
|
+
WHERE tenant_id = ? AND vertical = ?`, tenantId, vertical).toArray();
|
|
1775
|
+
const binding = this.sql.exec('SELECT target_scope_id, invalidated FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? AND vertical = ?', tenantId, callerScopeId, vertical).toArray()[0];
|
|
1776
|
+
return resolvePeerInstanceFrom(rows.map((r) => ({
|
|
1777
|
+
id: r.scope_id,
|
|
1778
|
+
tenantId: r.tenant_id,
|
|
1779
|
+
vertical: r.vertical,
|
|
1780
|
+
status: r.status,
|
|
1781
|
+
kind: r.kind ?? '',
|
|
1782
|
+
forkedFrom: r.forked_from,
|
|
1783
|
+
})), tenantId, vertical, binding?.target_scope_id ?? null, binding?.invalidated === 1);
|
|
1784
|
+
}
|
|
1785
|
+
peerBinding(tenantId, callerScopeId, vertical) {
|
|
1786
|
+
return this.sql.exec('SELECT target_scope_id, invalidated FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? AND vertical = ?', tenantId, callerScopeId, vertical).toArray()[0];
|
|
1787
|
+
}
|
|
1788
|
+
peerBindings(tenantId, callerScopeId) {
|
|
1789
|
+
return this.sql.exec('SELECT vertical, target_scope_id, invalidated FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? ORDER BY vertical', tenantId, callerScopeId).toArray();
|
|
1790
|
+
}
|
|
1791
|
+
/** Directory write and endpoint checks share this DO's serialized unit. */
|
|
1792
|
+
setPeerBinding(tenantId, callerScopeId, vertical, targetScopeId) {
|
|
1793
|
+
const caller = this.sql.exec('SELECT tenant_id, status, kind, forked_from FROM scopes WHERE scope_id = ?', callerScopeId).toArray()[0];
|
|
1794
|
+
if (!caller || caller.tenant_id !== tenantId || caller.status !== 'active' || !isPrimaryScopeRow(caller)) {
|
|
1795
|
+
return { ok: false, code: 'not_found', message: 'calling app is not a live primary scope in this tenant' };
|
|
1796
|
+
}
|
|
1797
|
+
if (targetScopeId !== null) {
|
|
1798
|
+
const target = this.sql.exec('SELECT tenant_id, vertical, status, kind, forked_from FROM scopes WHERE scope_id = ?', targetScopeId).toArray()[0];
|
|
1799
|
+
if (!target || target.tenant_id !== tenantId || target.vertical !== vertical || target.status !== 'active' || !isPrimaryScopeRow(target)) {
|
|
1800
|
+
return { ok: false, code: 'not_found', message: 'target is not a live primary instance of this vertical in this tenant' };
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
const before = this.peerBinding(tenantId, callerScopeId, vertical);
|
|
1804
|
+
const previous = before?.target_scope_id ?? null;
|
|
1805
|
+
if (previous === targetScopeId && before?.invalidated !== 1)
|
|
1806
|
+
return { ok: true, changed: false, previous };
|
|
1807
|
+
if (targetScopeId === null) {
|
|
1808
|
+
this.sql.exec('DELETE FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? AND vertical = ?', tenantId, callerScopeId, vertical);
|
|
1809
|
+
}
|
|
1810
|
+
else {
|
|
1811
|
+
this.sql.exec(`INSERT INTO peer_bindings (tenant_id, caller_scope_id, vertical, target_scope_id)
|
|
1812
|
+
VALUES (?, ?, ?, ?) ON CONFLICT (tenant_id, caller_scope_id, vertical)
|
|
1813
|
+
DO UPDATE SET target_scope_id = excluded.target_scope_id, invalidated = 0`, tenantId, callerScopeId, vertical, targetScopeId);
|
|
1814
|
+
}
|
|
1815
|
+
return { ok: true, changed: true, previous };
|
|
1816
|
+
}
|
|
1764
1817
|
/**
|
|
1765
1818
|
* Everything the router needs to place ONE peer call (#1706), in one round trip: the
|
|
1766
1819
|
* CALLER's own scope record, and the target instance of `vertical` in that tenant with the
|
|
1767
1820
|
* script and dispatch parameters that instance runs under.
|
|
1768
1821
|
*
|
|
1769
|
-
* One
|
|
1822
|
+
* One directory RPC and two SQL reads, because this sits on a request path. The binding
|
|
1823
|
+
* joins the caller read, so even an unbound singleton adds no round trip. The caller's record is
|
|
1770
1824
|
* what decides whether it may call at all — live, primary, and the vertical it claims — and
|
|
1771
1825
|
* it is read HERE rather than inferred from routing, so this path does not inherit #1713.
|
|
1772
|
-
* The target side applies the kernel's one rule (`
|
|
1826
|
+
* The target side applies the kernel's one rule (`resolvePeerInstanceFrom`), the same
|
|
1773
1827
|
* rule the pure host and the local broker apply.
|
|
1774
1828
|
*/
|
|
1775
1829
|
peerCallTarget(tenantId, callerScopeId, callerVertical, vertical) {
|
|
1776
1830
|
const caller = this.sql
|
|
1777
|
-
.exec(`SELECT s.tenant_id, s.vertical, s.status, s.kind, s.forked_from,
|
|
1778
|
-
|
|
1831
|
+
.exec(`SELECT s.tenant_id, s.vertical, s.status, s.kind, s.forked_from,
|
|
1832
|
+
t.status AS tenant_status, pb.target_scope_id AS bound_target_scope_id,
|
|
1833
|
+
pb.invalidated AS binding_invalidated
|
|
1834
|
+
FROM scopes s
|
|
1835
|
+
LEFT JOIN tenants t ON t.tenant_id = s.tenant_id
|
|
1836
|
+
LEFT JOIN peer_bindings pb ON pb.tenant_id = s.tenant_id
|
|
1837
|
+
AND pb.caller_scope_id = s.scope_id AND pb.vertical = ?
|
|
1838
|
+
WHERE s.scope_id = ?`, vertical, callerScopeId)
|
|
1779
1839
|
.toArray()[0];
|
|
1780
1840
|
const candidates = this.sql
|
|
1781
1841
|
.exec(`SELECT s.scope_id, s.tenant_id, s.vertical, s.status, s.kind, s.forked_from,
|
|
@@ -1792,14 +1852,14 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
1792
1852
|
LEFT JOIN vertical_versions sv ON sv.id = vr.serving_version_id
|
|
1793
1853
|
WHERE s.tenant_id = ? AND s.vertical = ?`, tenantId, vertical)
|
|
1794
1854
|
.toArray();
|
|
1795
|
-
const resolution =
|
|
1855
|
+
const resolution = resolvePeerInstanceFrom(candidates.map((r) => ({
|
|
1796
1856
|
id: r.scope_id,
|
|
1797
1857
|
tenantId: r.tenant_id,
|
|
1798
1858
|
vertical: r.vertical,
|
|
1799
1859
|
status: r.status,
|
|
1800
1860
|
kind: r.kind ?? '',
|
|
1801
1861
|
forkedFrom: r.forked_from ?? null,
|
|
1802
|
-
})), tenantId, vertical);
|
|
1862
|
+
})), tenantId, vertical, caller?.bound_target_scope_id ?? null, caller?.binding_invalidated === 1);
|
|
1803
1863
|
const target = resolution.outcome === 'resolved'
|
|
1804
1864
|
? (candidates.find((r) => r.scope_id === resolution.instance.scopeId) ?? null)
|
|
1805
1865
|
: null;
|
|
@@ -2077,6 +2137,7 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
2077
2137
|
*/
|
|
2078
2138
|
deleteScopeDirectory(scopeId) {
|
|
2079
2139
|
this.ctx.storage.transactionSync(() => {
|
|
2140
|
+
this.sql.exec('UPDATE peer_bindings SET invalidated = 1 WHERE target_scope_id = ?', scopeId);
|
|
2080
2141
|
this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
|
|
2081
2142
|
forgetSwitchesOf(this.kernelSql, scopeId);
|
|
2082
2143
|
this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
|
|
@@ -2585,6 +2646,42 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
2585
2646
|
WHERE connection_id = ? AND revoked_at IS NULL`, at, id);
|
|
2586
2647
|
return true;
|
|
2587
2648
|
}
|
|
2649
|
+
// -- a vertical's mailed connect links (connections.md §3.5.4) ------------------
|
|
2650
|
+
// The kernel's statements (`connect-links.ts`), shared with the pure adapter. Each
|
|
2651
|
+
// runs inside this single-threaded object with no await between its read and its
|
|
2652
|
+
// write, which is what makes consume's single-use hold across racing callbacks here;
|
|
2653
|
+
// the UPDATE's own WHERE holds it again regardless. `now` is the coordinator's. A move
|
|
2654
|
+
// and the audit row it hands back commit in one unit: `audit` is the row the host
|
|
2655
|
+
// minted (actor, attribution), completed here from what the move changed.
|
|
2656
|
+
insertConnectLink(row, audit) {
|
|
2657
|
+
return this.auditedConnectLink(audit, (sql, write) => insertConnectLinkRow(sql, row, write));
|
|
2658
|
+
}
|
|
2659
|
+
readConnectLink(key) {
|
|
2660
|
+
return readConnectLinkRow(doRedactionSql(this.sql), key);
|
|
2661
|
+
}
|
|
2662
|
+
listConnectLinks(filter, now) {
|
|
2663
|
+
return listConnectLinkRows(doRedactionSql(this.sql), filter, now);
|
|
2664
|
+
}
|
|
2665
|
+
revokeConnectLink(key, audit) {
|
|
2666
|
+
return this.auditedConnectLink(audit, (sql, write) => revokeConnectLinkRow(sql, key, write));
|
|
2667
|
+
}
|
|
2668
|
+
consumeConnectLink(input, now, audit) {
|
|
2669
|
+
return this.auditedConnectLink(audit, (sql, write) => consumeConnectLinkRow(sql, input, now, write));
|
|
2670
|
+
}
|
|
2671
|
+
restoreConnectLink(key, now, audit) {
|
|
2672
|
+
return this.auditedConnectLink(audit, (sql, write) => restoreConnectLinkRow(sql, key, now, write));
|
|
2673
|
+
}
|
|
2674
|
+
auditedConnectLink(audit, run) {
|
|
2675
|
+
return this.ctx.storage.transactionSync(() => run(doRedactionSql(this.sql), (a) => this.recordAdmin({
|
|
2676
|
+
...audit,
|
|
2677
|
+
action: a.action,
|
|
2678
|
+
tenantId: a.target.tenantId,
|
|
2679
|
+
scopeId: a.target.scopeId,
|
|
2680
|
+
vertical: a.target.vertical,
|
|
2681
|
+
before: a.before,
|
|
2682
|
+
after: a.after,
|
|
2683
|
+
})));
|
|
2684
|
+
}
|
|
2588
2685
|
/**
|
|
2589
2686
|
* Record a connection grant directory-side (#592) — written by `grantToConnection`
|
|
2590
2687
|
* alongside the enforcement tuple, upserting in place on re-grant (the unique
|
|
@@ -2955,7 +3052,7 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
2955
3052
|
* #1713: the scopes whose lifecycle a deployment must hold, with their lifecycle as this
|
|
2956
3053
|
* directory holds it. A scope with a vertical (only a deployment serves one), past provisioning
|
|
2957
3054
|
* and not reaped (a reaped store must not be recreated by a delivery), under a tenant that is
|
|
2958
|
-
* not reaped. `scopeId` narrows it to one scope and `tenantId` to one tenant's scopes, the
|
|
3055
|
+
* not reaped, of a vertical that does not declare its lifecycle held at the router. `scopeId` narrows it to one scope and `tenantId` to one tenant's scopes, the
|
|
2959
3056
|
* fan-out of a tenant transition. With `drift`, only the scopes the heal sweep must deliver to:
|
|
2960
3057
|
* a receipt that differs from the directory, and every scope held now. Re-delivering a held
|
|
2961
3058
|
* scope every pass is what puts a hold back on a store a carry or a restore landed without it,
|
|
@@ -2966,6 +3063,10 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
2966
3063
|
's.vertical IS NOT NULL',
|
|
2967
3064
|
"s.status NOT IN ('provisioning', 'reaped')",
|
|
2968
3065
|
"t.status <> 'reaped'",
|
|
3066
|
+
// A vertical that declares its lifecycle held at the router (`lifecycleHold`) has no route
|
|
3067
|
+
// to deliver to and nothing a delivery would hold: none of its scopes is a target, for a
|
|
3068
|
+
// transition, the heal's drift, or the #2016 tenant-record ask alike.
|
|
3069
|
+
`COALESCE(CASE WHEN json_valid(v.install_spec) THEN json_extract(v.install_spec, '$.lifecycle') END, '') <> 'router'`,
|
|
2969
3070
|
];
|
|
2970
3071
|
const params = [];
|
|
2971
3072
|
if (filter.tenantId) {
|
|
@@ -3004,6 +3105,7 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
3004
3105
|
${epoch} AS epoch, COALESCE(rs.revision, 0) AS scope_rev, COALESCE(rt.revision, 0) AS tenant_rev
|
|
3005
3106
|
FROM scopes s
|
|
3006
3107
|
JOIN tenants t ON t.tenant_id = s.tenant_id
|
|
3108
|
+
LEFT JOIN verticals v ON v.slug = s.vertical
|
|
3007
3109
|
LEFT JOIN scope_lifecycle_receipts r ON r.scope_id = s.scope_id
|
|
3008
3110
|
LEFT JOIN scope_tenant_asks a ON a.scope_id = s.scope_id
|
|
3009
3111
|
LEFT JOIN lifecycle_revisions rs ON rs.subject = 'scope:' || s.scope_id
|
|
@@ -3055,50 +3157,99 @@ export class ControlPlaneDO extends DurableObject {
|
|
|
3055
3157
|
recordOpsFailure(row) {
|
|
3056
3158
|
// #1748: the evidence, its issue and its finding in ONE unit — a detector that throws takes
|
|
3057
3159
|
// the evidence with it, so the caller's retry observes it rather than losing the observation.
|
|
3058
|
-
this.ctx.storage.transactionSync(() =>
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
if (row.fingerprint !== null) {
|
|
3072
|
-
this.sql.exec(`INSERT INTO _substrat_issues
|
|
3073
|
-
(fingerprint, operation, stage, origin, code, status, seen_count, first_seen, last_seen, last_message, last_tenant_id, last_owner_kind, last_vertical, last_version, resolved_version, resolved_at)
|
|
3074
|
-
VALUES (?, ?, ?, ?, ?, 'new', 1, ?, ?, ?, ?, ?, ?, ?, NULL, NULL)
|
|
3075
|
-
ON CONFLICT (fingerprint) DO UPDATE SET
|
|
3076
|
-
seen_count = seen_count + 1,
|
|
3077
|
-
last_seen = excluded.last_seen,
|
|
3078
|
-
last_message = excluded.last_message,
|
|
3079
|
-
last_tenant_id = excluded.last_tenant_id,
|
|
3080
|
-
last_owner_kind = excluded.last_owner_kind,
|
|
3081
|
-
last_vertical = COALESCE(excluded.last_vertical, last_vertical),
|
|
3082
|
-
last_version = COALESCE(excluded.last_version, last_version),
|
|
3083
|
-
origin = COALESCE(excluded.origin, origin),
|
|
3084
|
-
status = CASE WHEN status = 'resolved' THEN 'regressed' ELSE status END`, row.fingerprint, row.operation, row.stage, row.origin, row.code, row.at, row.at, row.message, row.tenant_id, issueExemplarOwner(row.tenant_id), row.vertical, row.version);
|
|
3085
|
-
const issueHorizon = new Date(Date.now() - ISSUE_RETENTION_DAYS * 86_400_000).toISOString();
|
|
3086
|
-
this.sql.exec('DELETE FROM _substrat_issues WHERE last_seen < ?', issueHorizon);
|
|
3087
|
-
// #1748: the tenant's Recurring finding, over the tenant's own occurrences only.
|
|
3088
|
-
const finding = findingOfOpsFailure({
|
|
3089
|
-
tenantId: row.tenant_id,
|
|
3090
|
-
scopeId: row.scope_id,
|
|
3091
|
-
operation: row.operation,
|
|
3092
|
-
stage: row.stage,
|
|
3093
|
-
code: row.code,
|
|
3094
|
-
vertical: row.vertical,
|
|
3095
|
-
version: row.version,
|
|
3096
|
-
});
|
|
3097
|
-
if (finding)
|
|
3098
|
-
observeFinding(doRedactionSql(this.sql), finding, row.at);
|
|
3160
|
+
this.ctx.storage.transactionSync(() => this.insertOpsFailure(row));
|
|
3161
|
+
}
|
|
3162
|
+
/**
|
|
3163
|
+
* #2064: settle an audited-change intent with no outcome — read it, and only if no outcome row
|
|
3164
|
+
* exists for its operation, write the `unknown` row and its ops-failure row. One unit, so two
|
|
3165
|
+
* racing passes settle once and the digest row cannot be lost. See the kernel's `audit-outcome.ts`.
|
|
3166
|
+
*/
|
|
3167
|
+
settleUnrecordedOutcome(input) {
|
|
3168
|
+
return this.ctx.storage.transactionSync(() => {
|
|
3169
|
+
const row = this.sql.exec(SETTLE_INTENT_SQL, input.intentId).toArray()[0];
|
|
3170
|
+
const outcome = unknownOutcomeOf(row, input.intentId, input.error);
|
|
3171
|
+
if (this.sql.exec(SETTLE_OUTCOME_SQL, ...settleOutcomeParamsOf(outcome.operation)).toArray().length > 0) {
|
|
3172
|
+
return false;
|
|
3099
3173
|
}
|
|
3174
|
+
const at = new Date().toISOString();
|
|
3175
|
+
this.recordAdmin({
|
|
3176
|
+
id: ulid(),
|
|
3177
|
+
actor: input.actor,
|
|
3178
|
+
action: outcome.action,
|
|
3179
|
+
...outcome.target,
|
|
3180
|
+
before: null,
|
|
3181
|
+
after: outcome.after,
|
|
3182
|
+
at,
|
|
3183
|
+
});
|
|
3184
|
+
const failure = { ...outcome.failure, actor: input.actor };
|
|
3185
|
+
this.insertOpsFailure({
|
|
3186
|
+
id: ulid(),
|
|
3187
|
+
actor: input.actor,
|
|
3188
|
+
operation: failure.operation,
|
|
3189
|
+
stage: failure.stage ?? null,
|
|
3190
|
+
tenant_id: failure.tenantId ?? null,
|
|
3191
|
+
scope_id: failure.scopeId ?? null,
|
|
3192
|
+
vertical: failure.vertical ?? null,
|
|
3193
|
+
version: null,
|
|
3194
|
+
status: null,
|
|
3195
|
+
message: failure.message.slice(0, 2000),
|
|
3196
|
+
reference: failure.reference ?? null,
|
|
3197
|
+
origin: null,
|
|
3198
|
+
code: null,
|
|
3199
|
+
fingerprint: opsFailureFingerprint(failure),
|
|
3200
|
+
at,
|
|
3201
|
+
});
|
|
3202
|
+
return true;
|
|
3100
3203
|
});
|
|
3101
3204
|
}
|
|
3205
|
+
/** #2064: an audited operation's rows, by the operation-id index, in bounded batches. */
|
|
3206
|
+
auditedOperations(refs) {
|
|
3207
|
+
return readAuditedOperations((sql, params) => this.sql.exec(sql, ...params).toArray(), refs);
|
|
3208
|
+
}
|
|
3209
|
+
/** The body of `recordOpsFailure`, for a caller already inside a transaction. */
|
|
3210
|
+
insertOpsFailure(row) {
|
|
3211
|
+
this.sql.exec(`INSERT INTO _substrat_ops_failures
|
|
3212
|
+
(id, actor, operation, stage, tenant_id, scope_id, vertical, version, status, message, reference, origin, code, fingerprint, at)
|
|
3213
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.actor, row.operation, row.stage, row.tenant_id, row.scope_id, row.vertical, row.version, row.status, row.message, row.reference, row.origin, row.code, row.fingerprint, row.at);
|
|
3214
|
+
// Prune-on-write (#559): the retention lives HERE, not in a cron — every insert
|
|
3215
|
+
// pays for its own housekeeping, so the table stays bounded even on a deployment
|
|
3216
|
+
// whose scheduled pass is broken (the exact circumstance this table records).
|
|
3217
|
+
const horizon = new Date(Date.now() - OPS_FAILURE_RETENTION_DAYS * 86_400_000).toISOString();
|
|
3218
|
+
this.sql.exec('DELETE FROM _substrat_ops_failures WHERE at < ?', horizon);
|
|
3219
|
+
// The issues materialization (#1233): the group's counters live on their own
|
|
3220
|
+
// row, bumped in the same call, because the evidence self-prunes above and a
|
|
3221
|
+
// count must survive its own exemplars. A fresh arrival regresses a resolved
|
|
3222
|
+
// issue; an ignored one stays ignored — that is what ignoring means.
|
|
3223
|
+
if (row.fingerprint !== null) {
|
|
3224
|
+
this.sql.exec(`INSERT INTO _substrat_issues
|
|
3225
|
+
(fingerprint, operation, stage, origin, code, status, seen_count, first_seen, last_seen, last_message, last_tenant_id, last_owner_kind, last_vertical, last_version, resolved_version, resolved_at)
|
|
3226
|
+
VALUES (?, ?, ?, ?, ?, 'new', 1, ?, ?, ?, ?, ?, ?, ?, NULL, NULL)
|
|
3227
|
+
ON CONFLICT (fingerprint) DO UPDATE SET
|
|
3228
|
+
seen_count = seen_count + 1,
|
|
3229
|
+
last_seen = excluded.last_seen,
|
|
3230
|
+
last_message = excluded.last_message,
|
|
3231
|
+
last_tenant_id = excluded.last_tenant_id,
|
|
3232
|
+
last_owner_kind = excluded.last_owner_kind,
|
|
3233
|
+
last_vertical = COALESCE(excluded.last_vertical, last_vertical),
|
|
3234
|
+
last_version = COALESCE(excluded.last_version, last_version),
|
|
3235
|
+
origin = COALESCE(excluded.origin, origin),
|
|
3236
|
+
status = CASE WHEN status = 'resolved' THEN 'regressed' ELSE status END`, row.fingerprint, row.operation, row.stage, row.origin, row.code, row.at, row.at, row.message, row.tenant_id, issueExemplarOwner(row.tenant_id), row.vertical, row.version);
|
|
3237
|
+
const issueHorizon = new Date(Date.now() - ISSUE_RETENTION_DAYS * 86_400_000).toISOString();
|
|
3238
|
+
this.sql.exec('DELETE FROM _substrat_issues WHERE last_seen < ?', issueHorizon);
|
|
3239
|
+
// #1748: the tenant's Recurring finding, over the tenant's own occurrences only.
|
|
3240
|
+
const finding = findingOfOpsFailure({
|
|
3241
|
+
tenantId: row.tenant_id,
|
|
3242
|
+
scopeId: row.scope_id,
|
|
3243
|
+
operation: row.operation,
|
|
3244
|
+
stage: row.stage,
|
|
3245
|
+
code: row.code,
|
|
3246
|
+
vertical: row.vertical,
|
|
3247
|
+
version: row.version,
|
|
3248
|
+
});
|
|
3249
|
+
if (finding)
|
|
3250
|
+
observeFinding(doRedactionSql(this.sql), finding, row.at);
|
|
3251
|
+
}
|
|
3252
|
+
}
|
|
3102
3253
|
/**
|
|
3103
3254
|
* #1632: subject erasure's directory half — the failure text this directory keeps about a
|
|
3104
3255
|
* scope's subject. The kernel's walk, so the SQLite host runs the identical statements.
|