@tokenoftrust/storefront-runner 2.4.0 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/storefront/env.d.ts +5 -2
- package/apps/storefront/src/components/auth/AuthBadge.astro +7 -4
- package/apps/storefront/src/lib/activity/uiActor.ts +10 -9
- package/apps/storefront/src/lib/auth/adminApiGuard.ts +6 -6
- package/apps/storefront/src/lib/auth/adminEntry.ts +16 -9
- package/apps/storefront/src/lib/auth/operatorSession.ts +10 -3
- package/apps/storefront/src/middleware/controlPlane.ts +9 -8
- package/apps/storefront/test/pipeline/fake-preview-services.ts +249 -0
- package/apps/storefront/test/pipeline/harness.ts +28 -5
- package/apps/storefront/test/pipeline/pipeline-host.ts +30 -3
- package/apps/storefront/test/pipeline/synthetic-job.ts +120 -5
- package/apps/storefront/vitest.workers.config.cts +2 -0
- package/package.json +1 -1
package/apps/storefront/env.d.ts
CHANGED
|
@@ -206,14 +206,17 @@ declare global {
|
|
|
206
206
|
* gate once a valid, resource-bound `tot_session` cookie is present). Carries
|
|
207
207
|
* the opaque subject, email + role(s)/capability tot20 resolved on this tenant, so chrome
|
|
208
208
|
* can show "signed in as … — owner/developer/staff". Absent on open or
|
|
209
|
-
* password-gated hosts.
|
|
209
|
+
* password-gated hosts. Chrome only: authorization reads the session record for the
|
|
210
|
+
* routed tenant (lib/pipeline/admission-request), never this.
|
|
210
211
|
*/
|
|
211
212
|
viewer?: {
|
|
212
213
|
/** Opaque identity-provider subject from the verified session assertion. */
|
|
213
214
|
subject?: string;
|
|
214
215
|
email: string;
|
|
216
|
+
/** The viewer's standing on THIS host's tenant, as a one-element list; empty when none. */
|
|
215
217
|
roles: string[];
|
|
216
|
-
|
|
218
|
+
/** The viewer's standing on THIS host's tenant; null when the session holds none here. */
|
|
219
|
+
capability: string | null;
|
|
217
220
|
};
|
|
218
221
|
/** Cloudflare runtime (env bindings) — provided by the adapter on Workers. */
|
|
219
222
|
runtime?: { env: CloudflareEnv };
|
|
@@ -17,7 +17,8 @@ const CAP_LABEL: Record<string, string> = {
|
|
|
17
17
|
inviter: "Inviter",
|
|
18
18
|
staff: "ToT Staff",
|
|
19
19
|
};
|
|
20
|
-
|
|
20
|
+
// The capability is the viewer's standing on THIS store only; null = none here.
|
|
21
|
+
const label = viewer?.capability ? (CAP_LABEL[viewer.capability] ?? viewer.capability) : "";
|
|
21
22
|
---
|
|
22
23
|
|
|
23
24
|
{viewer && (
|
|
@@ -29,9 +30,11 @@ const label = viewer ? (CAP_LABEL[viewer.capability] ?? viewer.capability) : "";
|
|
|
29
30
|
<span aria-hidden="true">🔓</span>
|
|
30
31
|
<span class="text-[var(--color-muted)]">Signed in as</span>
|
|
31
32
|
<span class="font-semibold">{viewer.email}</span>
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
{label && (
|
|
34
|
+
<span class="rounded-full bg-[var(--color-primary)] px-2 py-0.5 font-semibold text-[var(--color-primary-contrast)]">
|
|
35
|
+
{label}
|
|
36
|
+
</span>
|
|
37
|
+
)}
|
|
35
38
|
<form method="post" action="/api/auth/logout" class="contents">
|
|
36
39
|
<button
|
|
37
40
|
type="submit"
|
|
@@ -49,6 +49,7 @@ import type {
|
|
|
49
49
|
} from "@tot/public-runtime";
|
|
50
50
|
import type { ActionKey } from "@tot/public-runtime";
|
|
51
51
|
import { readViewerSession } from "@/lib/auth/route";
|
|
52
|
+
import { sessionHostCapability } from "@/lib/auth/loginGate";
|
|
52
53
|
import { isStaffSession } from "@/lib/dashboard/staffAdmission";
|
|
53
54
|
import { readEnv } from "@/lib/env";
|
|
54
55
|
import { isOwnerCapability } from "@/lib/shared/capability";
|
|
@@ -57,7 +58,7 @@ import { hashActorId, recordActivity } from "./recordActivity";
|
|
|
57
58
|
/** The minimal viewer chrome the middleware stamps on `locals.viewer`. */
|
|
58
59
|
interface ViewerLike {
|
|
59
60
|
email?: string;
|
|
60
|
-
capability?: string;
|
|
61
|
+
capability?: string | null;
|
|
61
62
|
}
|
|
62
63
|
|
|
63
64
|
/**
|
|
@@ -107,14 +108,14 @@ export async function resolveUiActor(
|
|
|
107
108
|
// The host-bound `locals.viewer` chrome omits `staff[]`, so read the full session
|
|
108
109
|
// to tell a ToT-staff (admin) actor from a store owner (merchant). Best-effort.
|
|
109
110
|
const session = await readViewerSession(context).catch(() => null);
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
|
|
115
|
-
const viewer: ViewerLike | undefined =
|
|
116
|
-
(context.locals?.
|
|
117
|
-
|
|
111
|
+
// The signed-in identity is the host gate's `locals.viewer`, else the raw session (a
|
|
112
|
+
// route that never went through that gate, e.g. the /dev cockpit's rendezvous
|
|
113
|
+
// approval). Ownership is the session's standing on the ROUTED tenant only — a
|
|
114
|
+
// session that owns another store is not this store's merchant.
|
|
115
|
+
const email = context.locals?.viewer?.email ?? session?.email;
|
|
116
|
+
const viewer: ViewerLike | undefined = email
|
|
117
|
+
? { email, capability: sessionHostCapability(session, context.locals?.tenant?.appDomain ?? "") }
|
|
118
|
+
: undefined;
|
|
118
119
|
const kind = classifyActorKind(viewer, session);
|
|
119
120
|
const identifier = viewer?.email ?? input.owner ?? "system";
|
|
120
121
|
const id = await hashActorId(identifier, await readEnv("ACTIVITY_ACTOR_SALT"));
|
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
*
|
|
11
11
|
* no session → 401 sign_in_required
|
|
12
12
|
* session, but no standing for THIS tenant → 403 not_authorized
|
|
13
|
-
*
|
|
13
|
+
* (ToT staff without a live vendor selection for THIS tenant included)
|
|
14
|
+
* member/owner of THIS tenant, or staff holding a live selection for it → allowed
|
|
14
15
|
*
|
|
15
16
|
* The decision core is PURE (unit-testable); `requireAdminApiEntry` is the thin
|
|
16
17
|
* Astro/I-O wrapper the routes call.
|
|
@@ -28,9 +29,9 @@ export type AdminApiAccess =
|
|
|
28
29
|
* PURE fail-closed decision for a tenant-scoped admin API caller.
|
|
29
30
|
*
|
|
30
31
|
* @param requireOwner when true, only an owner/admin viewer OR a ToT-staff viewer
|
|
31
|
-
* passes (a plain team member is
|
|
32
|
-
* owner-level operational surfaces.
|
|
33
|
-
*
|
|
32
|
+
* holding a live, audited selection for THIS tenant passes (a plain team member is
|
|
33
|
+
* refused) — for routes that mutate/expose owner-level operational surfaces.
|
|
34
|
+
* Default (false) admits anyone with standing on the tenant.
|
|
34
35
|
*/
|
|
35
36
|
export function decideAdminApiAccess(input: {
|
|
36
37
|
record: SessionRecord | null | undefined;
|
|
@@ -42,9 +43,8 @@ export function decideAdminApiAccess(input: {
|
|
|
42
43
|
if (!record) return { ok: false, status: 401, reason: "sign_in_required" };
|
|
43
44
|
|
|
44
45
|
const entry = resolveAdminEntry({ record, hostResource, nowSeconds });
|
|
45
|
-
const staffScope = Array.isArray(record.staff) && record.staff.length > 0;
|
|
46
46
|
const authorized = requireOwner
|
|
47
|
-
? entry.isOwnerViewer ||
|
|
47
|
+
? entry.isOwnerViewer || entry.staffSelected
|
|
48
48
|
: entry.admitted;
|
|
49
49
|
if (!authorized) return { ok: false, status: 403, reason: "not_authorized" };
|
|
50
50
|
|
|
@@ -23,13 +23,15 @@
|
|
|
23
23
|
* owner is the ONLY source of `shipOnBehalf`/`apexCutover` here.
|
|
24
24
|
*
|
|
25
25
|
* The distinguishing logic: `admitted`
|
|
26
|
-
* separates an authenticated OWNER/member (a host-bound capability,
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
26
|
+
* separates an authenticated OWNER/member (a host-bound capability, which for ToT
|
|
27
|
+
* staff means a LIVE, audited vendor selection for THIS host) from an authenticated
|
|
28
|
+
* NON-member, staff without a selection for this store, and an anonymous visitor —
|
|
29
|
+
* none of whom act on this store's admin. Staff scope alone is a browsing
|
|
30
|
+
* admission at the login gate, never an admin one. `principal` labels the case.
|
|
30
31
|
*/
|
|
31
32
|
import type { SessionRecord } from "./session.js";
|
|
32
33
|
import { sessionHostCapability } from "./loginGate.js";
|
|
34
|
+
import { staffAdmits } from "@/lib/dashboard/tenantSelection";
|
|
33
35
|
import {
|
|
34
36
|
decideIsOwner,
|
|
35
37
|
isOwnerCapability,
|
|
@@ -49,11 +51,13 @@ export type AdminEntryPrincipal =
|
|
|
49
51
|
|
|
50
52
|
export interface AdminEntryDecision {
|
|
51
53
|
/**
|
|
52
|
-
* True when the viewer
|
|
53
|
-
* or a
|
|
54
|
-
* for
|
|
54
|
+
* True when the viewer holds standing on this tenant (membership, a session scoped
|
|
55
|
+
* to it, or a live staff selection for it). False for an authenticated non-member,
|
|
56
|
+
* staff without a selection for this store, and an anonymous visitor.
|
|
55
57
|
*/
|
|
56
58
|
admitted: boolean;
|
|
59
|
+
/** A ToT-staff viewer holding a LIVE, audited vendor selection for THIS host. */
|
|
60
|
+
staffSelected: boolean;
|
|
57
61
|
/** The owner/admin/member standing bound to this host, or null. */
|
|
58
62
|
hostCapability: string | null;
|
|
59
63
|
/** The acting viewer IS the tenant owner/admin (superset of ship-on-behalf). */
|
|
@@ -88,7 +92,10 @@ export function resolveAdminEntry(input: {
|
|
|
88
92
|
const { record, hostResource, nowSeconds, shipGrant } = input;
|
|
89
93
|
const hostCapability = sessionHostCapability(record ?? null, hostResource, nowSeconds);
|
|
90
94
|
const staffScope = Array.isArray(record?.staff) && record!.staff.length > 0;
|
|
91
|
-
const
|
|
95
|
+
const staffSelected =
|
|
96
|
+
staffScope && staffAdmits({ selection: record?.staffSelection, nowSeconds }, (hostResource ?? "").trim());
|
|
97
|
+
// Standing on THIS host only; a staff selection for this host is part of it.
|
|
98
|
+
const admitted = hostCapability !== null;
|
|
92
99
|
|
|
93
100
|
// Owner resolution — the OWNER VIEWER, not the bearer console: authorized
|
|
94
101
|
// (a member/staff viewer of THIS tenant) AND holding an owner/admin capability.
|
|
@@ -115,5 +122,5 @@ export function resolveAdminEntry(input: {
|
|
|
115
122
|
? "staff"
|
|
116
123
|
: "authenticated-non-member";
|
|
117
124
|
|
|
118
|
-
return { admitted, hostCapability, isOwnerViewer, shipCapabilities, principal };
|
|
125
|
+
return { admitted, staffSelected, hostCapability, isOwnerViewer, shipCapabilities, principal };
|
|
119
126
|
}
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
* viewer that lacks that capability is treated.
|
|
11
11
|
*/
|
|
12
12
|
import type { APIContext } from "astro";
|
|
13
|
+
import { readViewerSession } from "./route.js";
|
|
14
|
+
import { sessionHostCapability } from "./loginGate.js";
|
|
13
15
|
|
|
14
16
|
/** The host-bound viewer chrome the middleware sets on `locals.viewer`. */
|
|
15
17
|
type Viewer = NonNullable<APIContext["locals"]["viewer"]>;
|
|
@@ -89,9 +91,12 @@ export interface HostBoundSessionConfig<T> {
|
|
|
89
91
|
}
|
|
90
92
|
|
|
91
93
|
/**
|
|
92
|
-
* Two-path host-bound operator session: a signed-in
|
|
94
|
+
* Two-path host-bound operator session: a signed-in viewer on the
|
|
93
95
|
* middleware-resolved `locals.tenant` (path 1), else the Bearer gate (path 2).
|
|
94
|
-
* The tenant is ALWAYS the resolved host `appDomain` — never the request body
|
|
96
|
+
* The tenant is ALWAYS the resolved host `appDomain` — never the request body —
|
|
97
|
+
* and the viewer's standing is read from the session record FOR THAT TENANT
|
|
98
|
+
* (`sessionHostCapability`), never from the `locals.viewer` chrome, so standing the
|
|
99
|
+
* session holds on another store never authorizes here.
|
|
95
100
|
* When no viewer/tenant pair is present, or a present viewer is insufficient
|
|
96
101
|
* under a `fallThrough` policy, path 2 decides.
|
|
97
102
|
*/
|
|
@@ -102,7 +107,9 @@ export async function resolveHostBoundSession<T>(
|
|
|
102
107
|
const viewer = context.locals?.viewer;
|
|
103
108
|
const tenant = context.locals?.tenant?.appDomain;
|
|
104
109
|
if (viewer && tenant) {
|
|
105
|
-
|
|
110
|
+
const session = await readViewerSession(context).catch(() => null);
|
|
111
|
+
const standing = sessionHostCapability(session, tenant);
|
|
112
|
+
if (standing !== null && config.capabilities.has(standing)) {
|
|
106
113
|
return config.resolveViewer(viewer, tenant);
|
|
107
114
|
}
|
|
108
115
|
if (config.onInsufficientCapability.mode === "failClosed") {
|
|
@@ -196,12 +196,13 @@ export async function serveControlPlane(input: ControlPlaneInput): Promise<Respo
|
|
|
196
196
|
}
|
|
197
197
|
if (admitted && viewerRecord) {
|
|
198
198
|
// Surface the viewer's standing FOR THIS host's tenant as chrome, BOUND to
|
|
199
|
-
// the host
|
|
200
|
-
//
|
|
201
|
-
// the
|
|
202
|
-
//
|
|
203
|
-
//
|
|
204
|
-
//
|
|
199
|
+
// the host: a member sees their tenants[] role; the owner of their own tenant
|
|
200
|
+
// sees the resource-scoped session's capability; a ToT-staff viewer admitted
|
|
201
|
+
// via an explicit vendor selection sees the selection's capability. A viewer
|
|
202
|
+
// with no standing here (staff browsing without a selection) carries none —
|
|
203
|
+
// the session's standing on any other store never appears on this host.
|
|
204
|
+
// Authorization never reads this chrome: admission resolves standing from the
|
|
205
|
+
// session record for the routed tenant (lib/pipeline/admission-request).
|
|
205
206
|
//
|
|
206
207
|
// On an admin path, "the host" a staff viewer is really acting on is the
|
|
207
208
|
// ?asTenant= impersonation target (mirrors admin.astro/AdminPublishTab.astro's
|
|
@@ -221,8 +222,8 @@ export async function serveControlPlane(input: ControlPlaneInput): Promise<Respo
|
|
|
221
222
|
locals.viewer = {
|
|
222
223
|
...(viewerRecord.subject ? { subject: viewerRecord.subject } : {}),
|
|
223
224
|
email: viewerRecord.email,
|
|
224
|
-
roles: hostCapability ? [hostCapability] :
|
|
225
|
-
capability: hostCapability
|
|
225
|
+
roles: hostCapability ? [hostCapability] : [],
|
|
226
|
+
capability: hostCapability,
|
|
226
227
|
};
|
|
227
228
|
}
|
|
228
229
|
}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fake {@link PreviewJobServices} for the TenantPipeline fault harness: a forge, a version store
|
|
3
|
+
* and a release read model held in the Durable Object's own SQLite (`fake_*` tables), so every
|
|
4
|
+
* effect survives the evictions the tests inject and can be counted afterwards.
|
|
5
|
+
*
|
|
6
|
+
* Faults are configured per object (one object per tenant) through `fake_faults` rows:
|
|
7
|
+
* - `evict-after-forge-merge` the forge merges, then the object is evicted before the merge
|
|
8
|
+
* step records anything (the placeholder-sha hole)
|
|
9
|
+
* - `evict-before-build` evicted when the build step starts (after the merge was recorded)
|
|
10
|
+
* - `evict-at-file:<n>` evicted while the build processes file n
|
|
11
|
+
* - `slow-read` every build read fails with a typed transient timeout
|
|
12
|
+
* - `pointer-moves-to:<sha>` while the build runs, the preview webhook moves the shared preview
|
|
13
|
+
* to <sha>, a descendant of the commit being built
|
|
14
|
+
* - `evict-after-email:<id>` the follow-up's rebase of candidate <id> sends its author a conflict
|
|
15
|
+
* email, then the object is evicted
|
|
16
|
+
* - `reaccept-times-out` after the forge merged, the re-accept of a resumed merge answers a
|
|
17
|
+
* transient timeout instead of a refusal
|
|
18
|
+
* A fault fires `times` times (0 = every time); its firing is persisted before it acts.
|
|
19
|
+
*/
|
|
20
|
+
import type { DurableObjectState } from "cloudflare:workers";
|
|
21
|
+
import { ForgeToolError, ForgeTransportError } from "../../src/lib/forge/errors";
|
|
22
|
+
import type {
|
|
23
|
+
MembershipEntry,
|
|
24
|
+
MergeCandidate,
|
|
25
|
+
PreviewJobServices,
|
|
26
|
+
ReadModelTip,
|
|
27
|
+
} from "../../src/lib/pipeline/preview-jobs";
|
|
28
|
+
|
|
29
|
+
/** Files a fake build materializes. */
|
|
30
|
+
export const FAKE_FILE_COUNT = 250;
|
|
31
|
+
|
|
32
|
+
export interface FakeFault {
|
|
33
|
+
point: string;
|
|
34
|
+
times: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface FakeState {
|
|
38
|
+
merges: { changeId: string; sha: string; message: string }[];
|
|
39
|
+
fileRuns: { sha: string; file: number }[];
|
|
40
|
+
advances: { sha: string; membership: MembershipEntry[]; jobId: string; kind: string }[];
|
|
41
|
+
readModel: { sha: string; membership: MembershipEntry[]; green: boolean } | null;
|
|
42
|
+
previewPointer: string | null;
|
|
43
|
+
staged: string[];
|
|
44
|
+
refreshes: string[];
|
|
45
|
+
emails: string[];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function fakeTables(state: DurableObjectState): void {
|
|
49
|
+
const { sql } = state.storage;
|
|
50
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_open (change_id TEXT PRIMARY KEY, head_sha TEXT NOT NULL)`);
|
|
51
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_merges (seq INTEGER PRIMARY KEY AUTOINCREMENT, change_id TEXT, sha TEXT, message TEXT)`);
|
|
52
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_file_runs (seq INTEGER PRIMARY KEY AUTOINCREMENT, sha TEXT, file INTEGER)`);
|
|
53
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_advances (seq INTEGER PRIMARY KEY AUTOINCREMENT, sha TEXT, membership TEXT, job_id TEXT, kind TEXT)`);
|
|
54
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_read_model (k INTEGER PRIMARY KEY CHECK (k = 1), sha TEXT, membership TEXT, green INTEGER)`);
|
|
55
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_faults (point TEXT PRIMARY KEY, times INTEGER NOT NULL, fired INTEGER NOT NULL DEFAULT 0)`);
|
|
56
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_pointer (k INTEGER PRIMARY KEY CHECK (k = 1), sha TEXT)`);
|
|
57
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_ancestry (ancestor TEXT, descendant TEXT, PRIMARY KEY (ancestor, descendant))`);
|
|
58
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_events (seq INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT, subject TEXT)`);
|
|
59
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS fake_candidates (change_id TEXT PRIMARY KEY, pr INTEGER, conflicts INTEGER)`);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function configureFake(
|
|
63
|
+
state: DurableObjectState,
|
|
64
|
+
config: {
|
|
65
|
+
open?: { changeId: string; headSha: string }[];
|
|
66
|
+
faults?: FakeFault[];
|
|
67
|
+
/** Other open candidates a base advance rebuilds; `conflicts` ones email their author. */
|
|
68
|
+
rebaseCandidates?: { changeId: string; pr: number; conflicts?: boolean }[];
|
|
69
|
+
},
|
|
70
|
+
): void {
|
|
71
|
+
fakeTables(state);
|
|
72
|
+
const { sql } = state.storage;
|
|
73
|
+
for (const c of config.open ?? []) {
|
|
74
|
+
sql.exec(`INSERT OR REPLACE INTO fake_open (change_id, head_sha) VALUES (?, ?)`, c.changeId, c.headSha);
|
|
75
|
+
}
|
|
76
|
+
for (const f of config.faults ?? []) {
|
|
77
|
+
sql.exec(`INSERT OR REPLACE INTO fake_faults (point, times, fired) VALUES (?, ?, 0)`, f.point, f.times);
|
|
78
|
+
}
|
|
79
|
+
for (const c of config.rebaseCandidates ?? []) {
|
|
80
|
+
sql.exec(`INSERT OR REPLACE INTO fake_candidates (change_id, pr, conflicts) VALUES (?, ?, ?)`, c.changeId, c.pr, c.conflicts ? 1 : 0);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function readFakeState(state: DurableObjectState): FakeState {
|
|
85
|
+
fakeTables(state);
|
|
86
|
+
const { sql } = state.storage;
|
|
87
|
+
const rm = sql.exec<{ sha: string; membership: string; green: number }>(`SELECT * FROM fake_read_model`).toArray()[0];
|
|
88
|
+
const pointer = sql.exec<{ sha: string }>(`SELECT sha FROM fake_pointer`).toArray()[0];
|
|
89
|
+
const events = (kind: string) =>
|
|
90
|
+
sql.exec<{ subject: string }>(`SELECT subject FROM fake_events WHERE kind = ? ORDER BY seq`, kind).toArray().map((r) => r.subject);
|
|
91
|
+
return {
|
|
92
|
+
merges: sql
|
|
93
|
+
.exec<{ change_id: string; sha: string; message: string }>(`SELECT * FROM fake_merges ORDER BY seq`)
|
|
94
|
+
.toArray()
|
|
95
|
+
.map((r) => ({ changeId: r.change_id, sha: r.sha, message: r.message })),
|
|
96
|
+
fileRuns: sql.exec<{ sha: string; file: number }>(`SELECT sha, file FROM fake_file_runs ORDER BY seq`).toArray(),
|
|
97
|
+
advances: sql
|
|
98
|
+
.exec<{ sha: string; membership: string; job_id: string; kind: string }>(`SELECT * FROM fake_advances ORDER BY seq`)
|
|
99
|
+
.toArray()
|
|
100
|
+
.map((r) => ({ sha: r.sha, membership: JSON.parse(r.membership), jobId: r.job_id, kind: r.kind })),
|
|
101
|
+
readModel: rm ? { sha: rm.sha, membership: JSON.parse(rm.membership), green: rm.green === 1 } : null,
|
|
102
|
+
previewPointer: pointer?.sha ?? null,
|
|
103
|
+
staged: events("stage"),
|
|
104
|
+
refreshes: events("refresh"),
|
|
105
|
+
emails: events("email"),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function fakePreviewServices(state: DurableObjectState): PreviewJobServices {
|
|
110
|
+
fakeTables(state);
|
|
111
|
+
const { sql } = state.storage;
|
|
112
|
+
|
|
113
|
+
/** True when `point` should fire now; records the firing durably first. */
|
|
114
|
+
async function fires(point: string): Promise<boolean> {
|
|
115
|
+
const row = sql.exec<{ times: number; fired: number }>(`SELECT times, fired FROM fake_faults WHERE point = ?`, point).toArray()[0];
|
|
116
|
+
if (!row) return false;
|
|
117
|
+
if (row.times !== 0 && row.fired >= row.times) return false;
|
|
118
|
+
sql.exec(`UPDATE fake_faults SET fired = fired + 1 WHERE point = ?`, point);
|
|
119
|
+
await state.storage.sync();
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function evict(point: string): never {
|
|
124
|
+
state.abort(`synthetic eviction ${point}`);
|
|
125
|
+
throw new Error("abort() returned");
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const setPointer = (sha: string) =>
|
|
129
|
+
sql.exec(`INSERT INTO fake_pointer (k, sha) VALUES (1, ?) ON CONFLICT(k) DO UPDATE SET sha = excluded.sha`, sha);
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
async integrateCandidate(_repo, changeId, opts) {
|
|
133
|
+
const open = sql.exec<{ head_sha: string }>(`SELECT head_sha FROM fake_open WHERE change_id = ?`, changeId).toArray()[0];
|
|
134
|
+
if (!open) {
|
|
135
|
+
if (await fires("reaccept-times-out")) {
|
|
136
|
+
throw new ForgeTransportError("timed out reading the body", { transient: true, errorKind: "timeout" });
|
|
137
|
+
}
|
|
138
|
+
throw new ForgeToolError(`No open candidate "${changeId}".`, { data: { status: "not_found" } });
|
|
139
|
+
}
|
|
140
|
+
const n = sql.exec<{ n: number }>(`SELECT COUNT(*) AS n FROM fake_merges`).one().n;
|
|
141
|
+
const sha = `sha-${n + 1}-${changeId}`;
|
|
142
|
+
sql.exec(`DELETE FROM fake_open WHERE change_id = ?`, changeId);
|
|
143
|
+
sql.exec(`INSERT INTO fake_merges (change_id, sha, message) VALUES (?, ?, ?)`, changeId, sha, opts.message);
|
|
144
|
+
await state.storage.sync();
|
|
145
|
+
if (await fires("evict-after-forge-merge")) evict("after forge merge");
|
|
146
|
+
return { sha, integratedInto: "preview" };
|
|
147
|
+
},
|
|
148
|
+
|
|
149
|
+
async candidatePaths() {
|
|
150
|
+
return ["content/home.json"];
|
|
151
|
+
},
|
|
152
|
+
|
|
153
|
+
async findMergedPreviewCommit(_repo, paths, { tag }) {
|
|
154
|
+
if (paths.length === 0) return null;
|
|
155
|
+
const row = sql
|
|
156
|
+
.exec<{ sha: string; message: string }>(`SELECT sha, message FROM fake_merges ORDER BY seq DESC`)
|
|
157
|
+
.toArray()
|
|
158
|
+
.find((r) => r.message.split("\n", 1)[0]!.includes(tag));
|
|
159
|
+
return row?.sha ?? null;
|
|
160
|
+
},
|
|
161
|
+
|
|
162
|
+
async readModelTip(): Promise<ReadModelTip | null> {
|
|
163
|
+
const rm = sql.exec<{ sha: string; membership: string; green: number }>(`SELECT * FROM fake_read_model`).toArray()[0];
|
|
164
|
+
return rm ? { sha: rm.sha, membership: JSON.parse(rm.membership), green: rm.green === 1 } : null;
|
|
165
|
+
},
|
|
166
|
+
|
|
167
|
+
async materialize({ sha, resume }) {
|
|
168
|
+
if (await fires("evict-before-build")) evict("before build");
|
|
169
|
+
const moveTo = sql
|
|
170
|
+
.exec<{ point: string }>(`SELECT point FROM fake_faults WHERE point LIKE 'pointer-moves-to:%'`)
|
|
171
|
+
.toArray()[0]?.point;
|
|
172
|
+
if (moveTo && (await fires(moveTo))) {
|
|
173
|
+
const descendant = moveTo.slice("pointer-moves-to:".length);
|
|
174
|
+
sql.exec(`INSERT OR IGNORE INTO fake_ancestry (ancestor, descendant) VALUES (?, ?)`, sha, descendant);
|
|
175
|
+
setPointer(descendant);
|
|
176
|
+
}
|
|
177
|
+
if (await fires("slow-read")) {
|
|
178
|
+
return {
|
|
179
|
+
ok: false,
|
|
180
|
+
errors: ["fetch content/catalog: timed out"],
|
|
181
|
+
cause: new ForgeTransportError("timed out reading the body", { transient: true, errorKind: "timeout" }),
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
for (let file = 0; file < FAKE_FILE_COUNT; file += 1) {
|
|
185
|
+
const path = `content/file-${file}.json`;
|
|
186
|
+
if (resume.done(path)) continue;
|
|
187
|
+
if (await fires(`evict-at-file:${file}`)) evict(`at file ${file}`);
|
|
188
|
+
sql.exec(`INSERT INTO fake_file_runs (sha, file) VALUES (?, ?)`, sha, file);
|
|
189
|
+
await resume.record(path, { storedVersionId: `v-${sha}-${file}`, issues: [], skippedHere: false });
|
|
190
|
+
}
|
|
191
|
+
return { ok: true, versionId: sha, digest: `digest-${sha}` };
|
|
192
|
+
},
|
|
193
|
+
|
|
194
|
+
async evidence() {
|
|
195
|
+
return { promotable: true };
|
|
196
|
+
},
|
|
197
|
+
|
|
198
|
+
async previewPointer() {
|
|
199
|
+
return sql.exec<{ sha: string }>(`SELECT sha FROM fake_pointer`).toArray()[0]?.sha ?? null;
|
|
200
|
+
},
|
|
201
|
+
|
|
202
|
+
async isAncestor(_repo, ancestor, descendant) {
|
|
203
|
+
if (ancestor === descendant) return true;
|
|
204
|
+
return sql.exec(`SELECT 1 FROM fake_ancestry WHERE ancestor = ? AND descendant = ?`, ancestor, descendant).toArray().length > 0;
|
|
205
|
+
},
|
|
206
|
+
|
|
207
|
+
async advancePreview({ sha, membership, jobId, kind }) {
|
|
208
|
+
setPointer(sha);
|
|
209
|
+
sql.exec(
|
|
210
|
+
`INSERT INTO fake_advances (sha, membership, job_id, kind) VALUES (?, ?, ?, ?)`,
|
|
211
|
+
sha,
|
|
212
|
+
JSON.stringify(membership),
|
|
213
|
+
jobId,
|
|
214
|
+
kind,
|
|
215
|
+
);
|
|
216
|
+
sql.exec(
|
|
217
|
+
`INSERT INTO fake_read_model (k, sha, membership, green) VALUES (1, ?, ?, 1)
|
|
218
|
+
ON CONFLICT(k) DO UPDATE SET sha = excluded.sha, membership = excluded.membership, green = 1`,
|
|
219
|
+
sha,
|
|
220
|
+
JSON.stringify(membership),
|
|
221
|
+
);
|
|
222
|
+
},
|
|
223
|
+
|
|
224
|
+
async stageAggregate({ sha }) {
|
|
225
|
+
sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('stage', ?)`, sha);
|
|
226
|
+
},
|
|
227
|
+
|
|
228
|
+
async listRebaseCandidates({ acceptedChangeId }) {
|
|
229
|
+
return sql
|
|
230
|
+
.exec<{ change_id: string; pr: number }>(`SELECT change_id, pr FROM fake_candidates WHERE change_id != ? ORDER BY change_id`, acceptedChangeId)
|
|
231
|
+
.toArray()
|
|
232
|
+
.map((r) => ({ changeId: r.change_id, prNumber: r.pr }));
|
|
233
|
+
},
|
|
234
|
+
|
|
235
|
+
async rebaseCandidate({ candidate }) {
|
|
236
|
+
sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('refresh', ?)`, candidate.changeId);
|
|
237
|
+
const conflicts = sql.exec<{ conflicts: number }>(`SELECT conflicts FROM fake_candidates WHERE change_id = ?`, candidate.changeId).toArray()[0];
|
|
238
|
+
if (!conflicts?.conflicts) return { outcome: "refreshed" as const };
|
|
239
|
+
sql.exec(`INSERT INTO fake_events (kind, subject) VALUES ('email', ?)`, candidate.changeId);
|
|
240
|
+
await state.storage.sync();
|
|
241
|
+
if (await fires(`evict-after-email:${candidate.changeId}`)) evict(`after emailing ${candidate.changeId}`);
|
|
242
|
+
return { outcome: "conflict" as const, notified: true };
|
|
243
|
+
},
|
|
244
|
+
|
|
245
|
+
async prepareUndo(): Promise<MergeCandidate> {
|
|
246
|
+
throw new Error("prepareUndo is not exercised by the fault harness");
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
}
|
|
@@ -21,7 +21,7 @@ interface HarnessEnv {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
type TenantPipelineStub = {
|
|
24
|
-
[K in "submit" | "job" | "jobs" | "transitions" | "facts" | "harness" | "probeStaleWrites"]: TenantPipeline[K];
|
|
24
|
+
[K in "submit" | "job" | "jobs" | "transitions" | "facts" | "harness" | "probeStaleWrites" | "configureFake" | "fakeState" | "spans"]: TenantPipeline[K];
|
|
25
25
|
};
|
|
26
26
|
|
|
27
27
|
export const harnessEnv = env as unknown as HarnessEnv;
|
|
@@ -45,6 +45,23 @@ export interface Submitted {
|
|
|
45
45
|
state: string;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/** Submit an intent of any kind through a Worker's intake (its own isolate), timing the answer. */
|
|
49
|
+
export async function submitIntentVia(
|
|
50
|
+
tenant: string,
|
|
51
|
+
intent: { kind: string; input: unknown },
|
|
52
|
+
via: Fetcher = harnessEnv.PIPELINE_HOST,
|
|
53
|
+
): Promise<Submitted> {
|
|
54
|
+
const started = performance.now();
|
|
55
|
+
const res = await via.fetch(`https://pipeline.test/intents/${encodeURIComponent(tenant)}`, {
|
|
56
|
+
method: "POST",
|
|
57
|
+
headers: { "content-type": "application/json" },
|
|
58
|
+
body: JSON.stringify({ ...intent, actor: { type: "test", id: "harness" }, admissionDecisionId: null }),
|
|
59
|
+
});
|
|
60
|
+
const ms = performance.now() - started;
|
|
61
|
+
const body = (await res.json()) as { jobId: string; state: string };
|
|
62
|
+
return { status: res.status, jobId: body.jobId, ms, intake: res.headers.get("x-intake"), state: body.state };
|
|
63
|
+
}
|
|
64
|
+
|
|
48
65
|
export async function submit(
|
|
49
66
|
tenant: string,
|
|
50
67
|
faults: Fault[] = [],
|
|
@@ -68,7 +85,7 @@ export async function submit(
|
|
|
68
85
|
return { status: res.status, jobId: body.jobId, ms, intake: res.headers.get("x-intake"), state: body.state };
|
|
69
86
|
}
|
|
70
87
|
|
|
71
|
-
const TERMINAL = new Set<JobState>(["succeeded", "
|
|
88
|
+
const TERMINAL = new Set<JobState>(["succeeded", "needs-developer", "stuck"]);
|
|
72
89
|
|
|
73
90
|
/**
|
|
74
91
|
* Polls until every job is terminal. On every sample asserts the tenant never has more than one
|
|
@@ -100,14 +117,19 @@ export async function waitTerminal(tenant: string, jobIds: string[], timeoutMs =
|
|
|
100
117
|
}
|
|
101
118
|
}
|
|
102
119
|
|
|
103
|
-
const OUTCOME_NOTES = ["step-done", "checkpoint", "lease-expired"];
|
|
120
|
+
const OUTCOME_NOTES = ["step-done", "checkpoint", "lease-expired", "yielded"];
|
|
104
121
|
|
|
105
122
|
/**
|
|
106
123
|
* The job's transition log is a complete chain: created queued, started once, every step start
|
|
107
124
|
* has exactly one recorded outcome (or is superseded by a resume after an eviction), each step
|
|
108
125
|
* finished exactly once and in order, and it ends at `terminal` with nothing after it.
|
|
109
126
|
*/
|
|
110
|
-
export function assertCompleteLog(
|
|
127
|
+
export function assertCompleteLog(
|
|
128
|
+
log: TransitionView[],
|
|
129
|
+
terminal: JobState,
|
|
130
|
+
stepNames = ["s1", "s2", "s3", "s4", "s5"],
|
|
131
|
+
opts: { checkpointsOf?: { step: string; count: number } | null } = {},
|
|
132
|
+
): void {
|
|
111
133
|
expect(log.length).toBeGreaterThan(2);
|
|
112
134
|
expect([log[0]!.from, log[0]!.to]).toEqual(["none", "queued"]);
|
|
113
135
|
expect([log[1]!.from, log[1]!.to]).toEqual(["queued", "running"]);
|
|
@@ -136,7 +158,8 @@ export function assertCompleteLog(log: TransitionView[], terminal: JobState, ste
|
|
|
136
158
|
if (terminal === "succeeded") {
|
|
137
159
|
const done = log.filter((t) => t.note === "step-done").map((t) => t.step);
|
|
138
160
|
expect(done).toEqual(stepNames);
|
|
139
|
-
|
|
161
|
+
const checkpoints = opts.checkpointsOf === undefined ? { step: "s3", count: S3_CHUNKS - 1 } : opts.checkpointsOf;
|
|
162
|
+
if (checkpoints) expect(log.filter((t) => t.note === "checkpoint" && t.step === checkpoints.step)).toHaveLength(checkpoints.count);
|
|
140
163
|
}
|
|
141
164
|
}
|
|
142
165
|
|
|
@@ -2,17 +2,44 @@
|
|
|
2
2
|
* The Worker that hosts the TenantPipeline Durable Object for the harness, in its own isolate
|
|
3
3
|
* (as in production, where callers never share the object's isolate — and where an eviction
|
|
4
4
|
* restarts the object without touching its callers). It runs the production engine with the
|
|
5
|
-
* synthetic job kinds
|
|
5
|
+
* synthetic job kinds AND the production preview kinds (integrate / rebuild / revert) over fake
|
|
6
|
+
* services (`./fake-preview-services.ts`), and answers intents through `submitIntent`.
|
|
6
7
|
*/
|
|
7
8
|
import type { DurableObjectState } from "cloudflare:workers";
|
|
8
9
|
import { TenantPipeline as DeployedTenantPipeline } from "../../src/lib/pipeline/tenant-pipeline";
|
|
9
10
|
import { submitIntent, type TenantPipelineNamespace } from "../../src/lib/pipeline/intake";
|
|
10
11
|
import { StaleExecutionError, type JobKindRegistry, type PipelineIntent } from "../../src/lib/pipeline/job-runner";
|
|
11
|
-
import { abandonedContexts, syntheticJobKinds } from "./synthetic-job";
|
|
12
|
+
import { abandonedContexts, syntheticJobKinds, syntheticSpans } from "./synthetic-job";
|
|
13
|
+
import { registerPreviewJobKinds, type PreviewJobTiming } from "../../src/lib/pipeline/preview-jobs";
|
|
14
|
+
import { configureFake, fakePreviewServices, readFakeState, type FakeFault, type FakeState } from "./fake-preview-services";
|
|
15
|
+
|
|
16
|
+
/** The preview kinds' timing in the harness: the production shape, scaled to milliseconds. */
|
|
17
|
+
export const HARNESS_PREVIEW_TIMING: PreviewJobTiming = {
|
|
18
|
+
leaseMs: 1_000,
|
|
19
|
+
deadlineMs: 30_000,
|
|
20
|
+
maxAttemptsPerStep: 4,
|
|
21
|
+
retryBaseMs: 10,
|
|
22
|
+
defaultExpectedMs: 2_000,
|
|
23
|
+
};
|
|
12
24
|
|
|
13
25
|
export class TenantPipeline extends DeployedTenantPipeline {
|
|
14
26
|
protected override jobKinds(ctx: DurableObjectState): JobKindRegistry {
|
|
15
|
-
return syntheticJobKinds(ctx);
|
|
27
|
+
return registerPreviewJobKinds(syntheticJobKinds(ctx), fakePreviewServices(ctx), HARNESS_PREVIEW_TIMING);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Harness-only: open candidates and arm faults for the fake preview services. */
|
|
31
|
+
async configureFake(config: { open?: { changeId: string; headSha: string }[]; faults?: FakeFault[] }): Promise<void> {
|
|
32
|
+
configureFake(this.ctx, config);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Harness-only: execution spans of the cooperative-hang kind. */
|
|
36
|
+
async spans(jobId: string): Promise<{ started: number; ended: number | null }[]> {
|
|
37
|
+
return syntheticSpans(this.ctx, jobId);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Harness-only: what the fake forge, build and read model recorded. */
|
|
41
|
+
async fakeState(): Promise<FakeState> {
|
|
42
|
+
return readFakeState(this.ctx);
|
|
16
43
|
}
|
|
17
44
|
|
|
18
45
|
/** Harness-only reads of the synthetic step log. */
|
|
@@ -7,12 +7,14 @@
|
|
|
7
7
|
* execution is logged to `synthetic_runs` (to prove resume-not-restart) and every effect to
|
|
8
8
|
* `synthetic_effects` with INSERT OR IGNORE (the step is idempotent, as the engine requires).
|
|
9
9
|
*
|
|
10
|
-
* The intent carries a fault plan: throw / hang past the lease / evict the
|
|
10
|
+
* The intent carries a fault plan: throw a transient fault / hang past the lease / evict the
|
|
11
|
+
* object / refuse (a developer-class refusal) / crash (a platform-class error), at a step
|
|
11
12
|
* boundary (`before` the step's work, `after` it but before the engine records it) or `mid` s3
|
|
12
13
|
* (after its first chunk's checkpoint). A fault fires once unless `persistent`; its firing is
|
|
13
14
|
* persisted before it acts so it survives the eviction it causes.
|
|
14
15
|
*/
|
|
15
16
|
import type { DurableObjectState } from "cloudflare:workers";
|
|
17
|
+
import { PipelineRefusal } from "../../src/lib/pipeline/classify";
|
|
16
18
|
import {
|
|
17
19
|
InvalidJobInput,
|
|
18
20
|
JobKindRegistry,
|
|
@@ -23,7 +25,16 @@ import {
|
|
|
23
25
|
type StepOutcome,
|
|
24
26
|
} from "../../src/lib/pipeline/job-runner";
|
|
25
27
|
|
|
26
|
-
export type FaultMode = "throw" | "hang" | "evict";
|
|
28
|
+
export type FaultMode = "throw" | "hang" | "evict" | "refuse" | "crash";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The transient fault a `throw` raises: a typed connection reset, which `classify()` names
|
|
32
|
+
* `transient` — so the engine retries it, exactly as it retries a real network blip.
|
|
33
|
+
*/
|
|
34
|
+
export class SyntheticTransientFault extends Error {
|
|
35
|
+
override name = "SyntheticTransientFault";
|
|
36
|
+
readonly code = "ECONNRESET";
|
|
37
|
+
}
|
|
27
38
|
export interface Fault {
|
|
28
39
|
at: "before" | "after" | "mid";
|
|
29
40
|
step: number;
|
|
@@ -49,7 +60,7 @@ function parseInput(raw: unknown): SyntheticInput {
|
|
|
49
60
|
if (!Array.isArray(faults)) throw new InvalidJobInput("faults must be an array");
|
|
50
61
|
for (const f of faults as Fault[]) {
|
|
51
62
|
if (!["before", "after", "mid"].includes(f.at)) throw new InvalidJobInput("fault.at is invalid");
|
|
52
|
-
if (!["throw", "hang", "evict"].includes(f.mode)) throw new InvalidJobInput("fault.mode is invalid");
|
|
63
|
+
if (!["throw", "hang", "evict", "refuse", "crash"].includes(f.mode)) throw new InvalidJobInput("fault.mode is invalid");
|
|
53
64
|
if (!Number.isInteger(f.step) || f.step < 1 || f.step > 5) throw new InvalidJobInput("fault.step is invalid");
|
|
54
65
|
}
|
|
55
66
|
return { faults: faults as Fault[] };
|
|
@@ -81,7 +92,9 @@ function syntheticKind(state: DurableObjectState, kind: string, deadlineMs: numb
|
|
|
81
92
|
index,
|
|
82
93
|
);
|
|
83
94
|
await state.storage.sync();
|
|
84
|
-
if (fault.mode === "throw") throw new
|
|
95
|
+
if (fault.mode === "throw") throw new SyntheticTransientFault(`synthetic fault ${at} s${step}`);
|
|
96
|
+
if (fault.mode === "refuse") throw new PipelineRefusal("build_refused", [`synthetic refusal ${at} s${step}`]);
|
|
97
|
+
if (fault.mode === "crash") throw new Error(`synthetic crash ${at} s${step}`);
|
|
85
98
|
if (fault.mode === "evict") {
|
|
86
99
|
state.abort(`synthetic eviction ${at} s${step}`);
|
|
87
100
|
throw new Error("abort() returned");
|
|
@@ -148,9 +161,111 @@ function syntheticKind(state: DurableObjectState, kind: string, deadlineMs: numb
|
|
|
148
161
|
};
|
|
149
162
|
}
|
|
150
163
|
|
|
164
|
+
// ── Engine-behaviour kinds ──────────────────────────────────────────────────
|
|
165
|
+
|
|
166
|
+
/** s1 then an optional s2 that fails as the intent says (`crash` | `refuse` | `hang`). */
|
|
167
|
+
export const SYNTHETIC_OPTIONAL_TAIL_KIND = "synthetic-optional-tail";
|
|
168
|
+
/** One long step that works in chunks, checkpointing each, for longer than one execution may run. */
|
|
169
|
+
export const SYNTHETIC_LONG_STEP_KIND = "synthetic-long-step";
|
|
170
|
+
/** A step that hangs past its lease but honours its abort signal, then retries clean. */
|
|
171
|
+
export const SYNTHETIC_COOPERATIVE_HANG_KIND = "synthetic-cooperative-hang";
|
|
172
|
+
export const LONG_STEP_CHUNKS = 12;
|
|
173
|
+
export const LONG_STEP_CHUNK_MS = 100;
|
|
174
|
+
export const LONG_STEP_MAX_EXECUTION_MS = 450;
|
|
175
|
+
|
|
176
|
+
function engineKinds(state: DurableObjectState): JobKindDefinition<{ mode?: string }>[] {
|
|
177
|
+
const { sql } = state.storage;
|
|
178
|
+
sql.exec(`CREATE TABLE IF NOT EXISTS synthetic_spans (seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT, started INTEGER, ended INTEGER)`);
|
|
179
|
+
const base = { leaseMs: SYNTHETIC_LEASE_MS, deadlineMs: SYNTHETIC_DEADLINE_MS, retryBaseMs: 10, defaultExpectedMs: 1_000 };
|
|
180
|
+
const parse = (raw: unknown) => (raw && typeof raw === "object" ? (raw as { mode?: string }) : {});
|
|
181
|
+
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
182
|
+
return [
|
|
183
|
+
{
|
|
184
|
+
kind: SYNTHETIC_OPTIONAL_TAIL_KIND,
|
|
185
|
+
parseInput: parse,
|
|
186
|
+
steps: [
|
|
187
|
+
{ name: "s1", run: async () => ({ kind: "done" }) },
|
|
188
|
+
{
|
|
189
|
+
name: "s2",
|
|
190
|
+
optional: true,
|
|
191
|
+
run: async (ctx) => {
|
|
192
|
+
if (ctx.input.mode === "refuse") throw new PipelineRefusal("build_refused");
|
|
193
|
+
if (ctx.input.mode === "hang") await sleep(SYNTHETIC_LEASE_MS * 4);
|
|
194
|
+
throw new Error("optional tail crashed");
|
|
195
|
+
},
|
|
196
|
+
},
|
|
197
|
+
],
|
|
198
|
+
...base,
|
|
199
|
+
maxAttemptsPerStep: 2,
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
kind: SYNTHETIC_LONG_STEP_KIND,
|
|
203
|
+
parseInput: parse,
|
|
204
|
+
steps: [
|
|
205
|
+
{
|
|
206
|
+
name: "long",
|
|
207
|
+
run: async (ctx) => {
|
|
208
|
+
let next = (ctx.checkpoint as { next?: number } | null)?.next ?? 0;
|
|
209
|
+
while (next < LONG_STEP_CHUNKS) {
|
|
210
|
+
if (ctx.signal.aborted) throw new Error("aborted");
|
|
211
|
+
await sleep(LONG_STEP_CHUNK_MS);
|
|
212
|
+
next += 1;
|
|
213
|
+
ctx.saveCheckpoint({ next });
|
|
214
|
+
await ctx.renewLease();
|
|
215
|
+
}
|
|
216
|
+
return { kind: "done" };
|
|
217
|
+
},
|
|
218
|
+
},
|
|
219
|
+
],
|
|
220
|
+
...base,
|
|
221
|
+
// One attempt only: reaching the execution cap after checkpointing must not spend it.
|
|
222
|
+
maxAttemptsPerStep: 1,
|
|
223
|
+
maxExecutionMs: LONG_STEP_MAX_EXECUTION_MS,
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
kind: SYNTHETIC_COOPERATIVE_HANG_KIND,
|
|
227
|
+
parseInput: parse,
|
|
228
|
+
steps: [
|
|
229
|
+
{
|
|
230
|
+
name: "hang",
|
|
231
|
+
run: async (ctx) => {
|
|
232
|
+
const started = Date.now();
|
|
233
|
+
const seq = sql
|
|
234
|
+
.exec<{ seq: number }>(`INSERT INTO synthetic_spans (job_id, started) VALUES (?, ?) RETURNING seq`, ctx.jobId, started)
|
|
235
|
+
.one().seq;
|
|
236
|
+
const first = sql.exec<{ n: number }>(`SELECT COUNT(*) AS n FROM synthetic_spans WHERE job_id = ?`, ctx.jobId).one().n === 1;
|
|
237
|
+
try {
|
|
238
|
+
if (first) {
|
|
239
|
+
// Wait for the abort, then take a while to wind down (within one further lease).
|
|
240
|
+
await new Promise<void>((resolve) => ctx.signal.addEventListener("abort", () => resolve(), { once: true }));
|
|
241
|
+
await sleep(SYNTHETIC_LEASE_MS / 2);
|
|
242
|
+
throw new Error("stopped after abort");
|
|
243
|
+
}
|
|
244
|
+
return { kind: "done" };
|
|
245
|
+
} finally {
|
|
246
|
+
sql.exec(`UPDATE synthetic_spans SET ended = ? WHERE seq = ?`, Date.now(), seq);
|
|
247
|
+
}
|
|
248
|
+
},
|
|
249
|
+
},
|
|
250
|
+
],
|
|
251
|
+
...base,
|
|
252
|
+
maxAttemptsPerStep: 5,
|
|
253
|
+
},
|
|
254
|
+
];
|
|
255
|
+
}
|
|
256
|
+
|
|
151
257
|
export function syntheticJobKinds(state: DurableObjectState): JobKindRegistry {
|
|
152
258
|
harnessTables(state);
|
|
153
|
-
|
|
259
|
+
const registry = new JobKindRegistry()
|
|
154
260
|
.register(syntheticKind(state, SYNTHETIC_KIND, SYNTHETIC_DEADLINE_MS))
|
|
155
261
|
.register(syntheticKind(state, SYNTHETIC_SHORT_DEADLINE_KIND, SHORT_DEADLINE_MS));
|
|
262
|
+
for (const kind of engineKinds(state)) registry.register(kind as JobKindDefinition<unknown> as never);
|
|
263
|
+
return registry;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Execution spans of the cooperative-hang kind, for overlap checks. */
|
|
267
|
+
export function syntheticSpans(state: DurableObjectState, jobId: string): { started: number; ended: number | null }[] {
|
|
268
|
+
return state.storage.sql
|
|
269
|
+
.exec<{ started: number; ended: number | null }>(`SELECT started, ended FROM synthetic_spans WHERE job_id = ? ORDER BY seq`, jobId)
|
|
270
|
+
.toArray();
|
|
156
271
|
}
|
|
@@ -28,6 +28,8 @@ const hostModule = esbuild.buildSync({
|
|
|
28
28
|
const pipelineBinding = { TENANT_PIPELINE: { className: "TenantPipeline", scriptName: HOST } };
|
|
29
29
|
|
|
30
30
|
module.exports = defineWorkersConfig({
|
|
31
|
+
// The source tree's `@/` import alias (tsconfig `paths`), for modules the tests import directly.
|
|
32
|
+
resolve: { alias: { "@": path.join(__dirname, "src") } },
|
|
31
33
|
test: {
|
|
32
34
|
name: "pipeline",
|
|
33
35
|
include: ["test/pipeline/**/*.test.ts"],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tokenoftrust/storefront-runner",
|
|
3
|
-
"version": "2.4.
|
|
3
|
+
"version": "2.4.2",
|
|
4
4
|
"license": "SEE LICENSE IN LICENSE",
|
|
5
5
|
"description": "World-shareable storefront runner: multi-tenant renderer on Astro/Cloudflare. No control plane.",
|
|
6
6
|
"packageManager": "pnpm@11.9.0",
|