@kedge/cli 0.0.0-stage → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/bin/kedge.mjs +8 -0
- package/kedge-manifest.json +418 -0
- package/package.json +43 -3
- package/src/cf.ts +729 -0
- package/src/check.ts +45 -0
- package/src/cli-session.ts +57 -0
- package/src/code-lint.ts +339 -0
- package/src/commands.ts +959 -0
- package/src/compose.ts +637 -0
- package/src/d1-batch.ts +66 -0
- package/src/deploy.ts +1682 -0
- package/src/doctor.ts +701 -0
- package/src/fork.ts +67 -0
- package/src/index.ts +12 -0
- package/src/invariants.ts +168 -0
- package/src/keys-save.ts +298 -0
- package/src/local-state.ts +110 -0
- package/src/main.ts +470 -0
- package/src/manifest.ts +163 -0
- package/src/n1.ts +217 -0
- package/src/ops.ts +373 -0
- package/src/order.ts +7 -0
- package/src/output.ts +35 -0
- package/src/plugin-lint.ts +145 -0
- package/src/preview.ts +1038 -0
- package/src/schema.ts +72 -0
- package/src/smoke.ts +218 -0
- package/src/sqlite-verify.ts +333 -0
- package/src/store-api.ts +36 -0
- package/src/store.ts +132 -0
- package/src/tar.ts +125 -0
- package/src/target.ts +81 -0
- package/src/tz.ts +79 -0
- package/src/units.ts +185 -0
- package/src/upgrade.ts +235 -0
- package/src/worker-config.ts +81 -0
- package/README.md +0 -3
package/src/n1.ts
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
// Which pending migrations the previous Worker survives (docs/proposals/p0.8-cli-agent-kit.md §6.2).
|
|
2
|
+
// Rollback doesn't change "resources connected to your Worker", so every file
|
|
3
|
+
// an ordinary deploy applies must leave the running code working: by proof
|
|
4
|
+
// (core's release suites, a plugin generation's design) or by a statement
|
|
5
|
+
// someone wrote. Shape alone proves little: adding a column breaks a
|
|
6
|
+
// positional INSERT, and an expression index can throw on an old write.
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import type { PluginInstance } from "@kedge/core";
|
|
10
|
+
import { statementTexts } from "@kedge/core";
|
|
11
|
+
import { CUTOVER_MIGRATIONS } from "@kedge/db";
|
|
12
|
+
import type { Lock, LockEntry } from "./compose";
|
|
13
|
+
import type { Manifest } from "./manifest";
|
|
14
|
+
import { schemaOf } from "./schema";
|
|
15
|
+
|
|
16
|
+
export type Mode = "n1" | "cutover";
|
|
17
|
+
|
|
18
|
+
export interface Classified {
|
|
19
|
+
file: string;
|
|
20
|
+
source: string;
|
|
21
|
+
mode: Mode;
|
|
22
|
+
/** Why it's N−1: proof, design, shape, or the statement's text. */
|
|
23
|
+
because: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export class NotN1 extends Error {
|
|
27
|
+
constructor(readonly problems: string[]) {
|
|
28
|
+
super(`pending migrations the running Worker may not survive:\n ${problems.join("\n ")}`);
|
|
29
|
+
this.name = "NotN1";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const strip = (s: string) =>
|
|
34
|
+
s
|
|
35
|
+
.replace(/--[^\n]*/g, " ")
|
|
36
|
+
.replace(/\/\*[\s\S]*?\*\//g, " ")
|
|
37
|
+
.replace(/\s+/g, " ")
|
|
38
|
+
.trim();
|
|
39
|
+
/** A name as SQLite resolves it: unquoted, and case-blind, so `local_Cache` is `local_cache` (codex code r19). */
|
|
40
|
+
const unquote = (n: string) => n.replace(/^["`[]|["`\]]$/g, "").toLowerCase();
|
|
41
|
+
const NAME = String.raw`(?:"[^"]+"|\x60[^\x60]+\x60|\[[^\]]+\]|[A-Za-z_][\w$]*)`;
|
|
42
|
+
|
|
43
|
+
/** A plain column in an index: a name, maybe a direction or collation. No expression. */
|
|
44
|
+
const plainColumn = (c: string) =>
|
|
45
|
+
new RegExp(String.raw`^${NAME}(\s+COLLATE\s+\w+)?(\s+(ASC|DESC))?$`, "i").test(c.trim());
|
|
46
|
+
|
|
47
|
+
/** The object one statement creates, by kind and name, if it creates one. */
|
|
48
|
+
function createdBy(t: string): string | null {
|
|
49
|
+
const m = new RegExp(
|
|
50
|
+
String.raw`^CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?(?:UNIQUE\s+)?(TABLE|INDEX|TRIGGER|VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(${NAME})`,
|
|
51
|
+
"i",
|
|
52
|
+
).exec(strip(t));
|
|
53
|
+
return m ? `${(m[1] as string).toUpperCase()}:${unquote(m[2] as string)}` : null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Objects a file creates, by kind and name. */
|
|
57
|
+
function created(texts: readonly string[]): Set<string> {
|
|
58
|
+
return new Set(texts.map(createdBy).filter((k): k is string => k !== null));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Judges a file's statements in the order they run. A drop may take only what an earlier
|
|
63
|
+
* statement of the same file created and that didn't exist before the file: `DROP x; CREATE x`
|
|
64
|
+
* would otherwise pass as "its own new table" and destroy a live one (codex code r2).
|
|
65
|
+
*/
|
|
66
|
+
function judgeFile(
|
|
67
|
+
texts: readonly string[],
|
|
68
|
+
before: ReadonlySet<string>,
|
|
69
|
+
mayDrop: (kind: string, name: string) => boolean,
|
|
70
|
+
): ReturnType<typeof judge>[] {
|
|
71
|
+
const own = new Set<string>();
|
|
72
|
+
return texts.map((t) => {
|
|
73
|
+
const j = judge(t, own, mayDrop);
|
|
74
|
+
const made = createdBy(t);
|
|
75
|
+
if (made && !before.has(made)) own.add(made);
|
|
76
|
+
const dropped = /^DROP\s+(TABLE|INDEX|TRIGGER|VIEW)\s+(?:IF\s+EXISTS\s+)?(\S+)/i.exec(strip(t));
|
|
77
|
+
if (dropped) own.delete(`${(dropped[1] as string).toUpperCase()}:${unquote(dropped[2] as string)}`);
|
|
78
|
+
return j;
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Every object that exists before `file` runs, by kind and name: the lock's earlier files applied in order. */
|
|
83
|
+
function existingBefore(input: ClassifyInput, file: string): Set<string> {
|
|
84
|
+
const pending = new Set(input.pending.map((p) => p.file));
|
|
85
|
+
const texts: string[] = [];
|
|
86
|
+
for (const e of input.lock.entries) {
|
|
87
|
+
if (e.file === file) break;
|
|
88
|
+
texts.push(pending.has(e.file) ? input.text(e.file) : readFileSync(join(input.cwd, "migrations", e.file), "utf8"));
|
|
89
|
+
}
|
|
90
|
+
const schema = schemaOf(texts);
|
|
91
|
+
schema.db.close();
|
|
92
|
+
return new Set(
|
|
93
|
+
schema.objects.filter((o) => o.type !== "constraint").map((o) => `${o.type.toUpperCase()}:${o.name.toLowerCase()}`),
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** What one plugin or local statement is, for the shape rule and the drop rule. */
|
|
98
|
+
function judge(
|
|
99
|
+
stmt: string,
|
|
100
|
+
own: Set<string>,
|
|
101
|
+
mayDrop: (kind: string, name: string) => boolean,
|
|
102
|
+
): { ok: boolean; drop?: string; why: string } {
|
|
103
|
+
const s = strip(stmt);
|
|
104
|
+
if (/^CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?TABLE\b/i.test(s)) return { ok: true, why: "CREATE TABLE" };
|
|
105
|
+
const idx = new RegExp(
|
|
106
|
+
String.raw`^CREATE\s+(UNIQUE\s+)?INDEX\s+(?:IF\s+NOT\s+EXISTS\s+)?${NAME}\s+ON\s+${NAME}\s*\((.*)\)\s*(WHERE\b.*)?$`,
|
|
107
|
+
"i",
|
|
108
|
+
).exec(s);
|
|
109
|
+
if (idx) {
|
|
110
|
+
if (idx[1]) return { ok: false, why: "a unique index can refuse the running Worker's inserts" };
|
|
111
|
+
if (idx[3]) return { ok: false, why: "a partial index's predicate can throw on the running Worker's writes" };
|
|
112
|
+
if (!(idx[2] as string).split(",").every(plainColumn))
|
|
113
|
+
return { ok: false, why: "an expression index can throw on the running Worker's writes" };
|
|
114
|
+
return { ok: true, why: "a plain non-unique index" };
|
|
115
|
+
}
|
|
116
|
+
const drop = new RegExp(String.raw`^DROP\s+(TABLE|INDEX|TRIGGER|VIEW)\s+(?:IF\s+EXISTS\s+)?(${NAME})`, "i").exec(s);
|
|
117
|
+
if (drop) {
|
|
118
|
+
const kind = (drop[1] as string).toUpperCase();
|
|
119
|
+
const name = unquote(drop[2] as string);
|
|
120
|
+
if (own.has(`${kind}:${name}`) || mayDrop(kind, name)) return { ok: true, why: "drops what it may" };
|
|
121
|
+
return { ok: false, drop: `${kind} ${name}`, why: "drops something that exists before this file" };
|
|
122
|
+
}
|
|
123
|
+
if (/^ALTER\s+TABLE\b.*\b(RENAME|DROP\s+COLUMN)\b/i.test(s)) {
|
|
124
|
+
const t = new RegExp(String.raw`^ALTER\s+TABLE\s+(${NAME})`, "i").exec(s);
|
|
125
|
+
const name = unquote((t?.[1] as string) ?? "");
|
|
126
|
+
if (own.has(`TABLE:${name}`)) return { ok: true, why: "renames or drops within its own new table" };
|
|
127
|
+
return { ok: false, drop: `TABLE ${name}`, why: "renames or drops part of something that exists before this file" };
|
|
128
|
+
}
|
|
129
|
+
if (/^ALTER\s+TABLE\b.*\bADD\b/i.test(s))
|
|
130
|
+
return { ok: false, why: "a new column breaks the running Worker's positional INSERT" };
|
|
131
|
+
return { ok: false, why: `"${s.slice(0, 60)}" isn't a shape the running Worker can't trip` };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface ClassifyInput {
|
|
135
|
+
cwd: string;
|
|
136
|
+
lock: Lock;
|
|
137
|
+
pending: readonly LockEntry[];
|
|
138
|
+
manifest: Manifest | null;
|
|
139
|
+
// biome-ignore lint/suspicious/noExplicitAny: each plugin brings its own options type
|
|
140
|
+
plugins: readonly PluginInstance<any>[];
|
|
141
|
+
/** Each file's text, from the staged bytes the deploy applies (§6.2). */
|
|
142
|
+
text: (file: string) => string;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Classifies each pending file, or throws NotN1 naming each one that isn't. */
|
|
146
|
+
export function classify(input: ClassifyInput): Classified[] {
|
|
147
|
+
const out: Classified[] = [];
|
|
148
|
+
const problems: string[] = [];
|
|
149
|
+
const cutovers = new Set(CUTOVER_MIGRATIONS);
|
|
150
|
+
for (const e of input.pending) {
|
|
151
|
+
const text = input.text(e.file);
|
|
152
|
+
if (e.source.startsWith("core:0")) {
|
|
153
|
+
const f = e.source.slice("core:".length);
|
|
154
|
+
const mode = input.manifest?.migrations[f]?.mode ?? (cutovers.has(f) ? "cutover" : "n1");
|
|
155
|
+
out.push({
|
|
156
|
+
file: e.file,
|
|
157
|
+
source: e.source,
|
|
158
|
+
mode,
|
|
159
|
+
because:
|
|
160
|
+
mode === "cutover" ? "core marks it a cutover" : "core's release suites ran the previous release over it",
|
|
161
|
+
});
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
if (e.source.startsWith("core:")) {
|
|
165
|
+
// The pinned timezone's rules and the checkout-track fingerprint: core's generated files, N−1 by design.
|
|
166
|
+
out.push({ file: e.file, source: e.source, mode: "n1", because: "core generates it N−1 (p0.4 §4, p0.7 §6)" });
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
const [kind, plugin, ...rest] = e.source.split(":");
|
|
170
|
+
const id = rest.join(":");
|
|
171
|
+
const texts = statementTexts(text);
|
|
172
|
+
const before = existingBefore(input, e.file);
|
|
173
|
+
if (kind === "plugin" && (id.startsWith("generated:") || id.startsWith("entity:") || id === "install")) {
|
|
174
|
+
// The composer replacing what it owns: drops only of this plugin's earlier generated objects.
|
|
175
|
+
const earlier = new Set<string>();
|
|
176
|
+
for (const prior of input.lock.entries) {
|
|
177
|
+
if (prior.file === e.file) break;
|
|
178
|
+
if (prior.source.startsWith(`plugin:${plugin}:`))
|
|
179
|
+
for (const o of created(statementTexts(readFileSync(join(input.cwd, "migrations", prior.file), "utf8"))))
|
|
180
|
+
earlier.add(o);
|
|
181
|
+
}
|
|
182
|
+
const bad = judgeFile(texts, before, (k, n) => earlier.has(`${k}:${n}`)).filter((j) => j.drop);
|
|
183
|
+
if (bad.length) problems.push(`${e.file} (${e.source}): ${bad.map((b) => `${b.drop}: ${b.why}`).join("; ")}`);
|
|
184
|
+
else
|
|
185
|
+
out.push({
|
|
186
|
+
file: e.file,
|
|
187
|
+
source: e.source,
|
|
188
|
+
mode: "n1",
|
|
189
|
+
because: "a plugin generation: an older Worker's writes are fenced as PLUGIN_UPGRADING (p0.7 §6)",
|
|
190
|
+
});
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
const judged = judgeFile(texts, before, () => false);
|
|
194
|
+
const drops = judged.filter((j) => j.drop);
|
|
195
|
+
if (drops.length) {
|
|
196
|
+
problems.push(
|
|
197
|
+
`${e.file} (${e.source}): ${drops.map((d) => `${d.drop}: ${d.why}; only core's cutovers may, behind the fence`).join("; ")}`,
|
|
198
|
+
);
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
const statement =
|
|
202
|
+
/--\s*kedge:n-1\s+(\S.*)/.exec(text)?.[1]?.trim() ??
|
|
203
|
+
(kind === "plugin"
|
|
204
|
+
? input.plugins.find((p) => p.definition.name === plugin)?.definition.migrations?.find((m) => m.id === id)?.n1
|
|
205
|
+
: undefined);
|
|
206
|
+
const shaped = judged.filter((j) => !j.ok);
|
|
207
|
+
if (shaped.length === 0)
|
|
208
|
+
out.push({ file: e.file, source: e.source, mode: "n1", because: "shape: only new tables and plain indexes" });
|
|
209
|
+
else if (statement) out.push({ file: e.file, source: e.source, mode: "n1", because: `stated: ${statement}` });
|
|
210
|
+
else
|
|
211
|
+
problems.push(
|
|
212
|
+
`${e.file} (${e.source}): ${[...new Set(shaped.map((j) => j.why))].join("; ")}. If the running Worker survives it, say why in the file: -- kedge:n-1 <why>`,
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
if (problems.length) throw new NotN1(problems);
|
|
216
|
+
return out;
|
|
217
|
+
}
|
package/src/ops.ts
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
// The operations database (docs/proposals/p0.8-cli-agent-kit.md §3.1): a
|
|
2
|
+
// second D1 database per store, `<store>-ops`, that no Worker binds. It holds
|
|
3
|
+
// what a lost CI runner can't lose and a restore of the store can't rewind:
|
|
4
|
+
// who is running what on the store's production resources and on each preview
|
|
5
|
+
// (one operation per scope), every control-plane call before it's sent and
|
|
6
|
+
// its outcome, the deployment manifests, the hashes of applied migrations,
|
|
7
|
+
// the store's own resources, and the preview registry.
|
|
8
|
+
//
|
|
9
|
+
// Exclusion is a primary key and compare-and-set updates, read back with
|
|
10
|
+
// RETURNING, so of two callers one wins and a paused one finds its claim gone
|
|
11
|
+
// at its next step. Nothing here takes over on a timer: elapsed time doesn't
|
|
12
|
+
// prove a paused job is gone.
|
|
13
|
+
import { randomUUID } from "node:crypto";
|
|
14
|
+
import { hostname } from "node:os";
|
|
15
|
+
import type { Row } from "./cf";
|
|
16
|
+
|
|
17
|
+
export type Query = (sql: string, params?: readonly unknown[]) => Promise<Row[]>;
|
|
18
|
+
|
|
19
|
+
export const OPS_SCHEMA = [
|
|
20
|
+
`CREATE TABLE IF NOT EXISTS kedge_ops_meta (id INTEGER PRIMARY KEY CHECK (id = 1), store TEXT NOT NULL, created_at INTEGER NOT NULL)`,
|
|
21
|
+
// The store's choices made once (p0.10 §7: its jurisdiction), each written once and never changed.
|
|
22
|
+
`CREATE TABLE IF NOT EXISTS ops_setting (name TEXT PRIMARY KEY, value TEXT NOT NULL, created_at INTEGER NOT NULL)`,
|
|
23
|
+
`CREATE TABLE IF NOT EXISTS operation (
|
|
24
|
+
scope TEXT PRIMARY KEY, op_id TEXT NOT NULL UNIQUE, kind TEXT NOT NULL, plan TEXT NOT NULL, step TEXT NOT NULL,
|
|
25
|
+
progress TEXT NOT NULL DEFAULT '{}',
|
|
26
|
+
executor TEXT, executor_label TEXT, heartbeat_at INTEGER NOT NULL, started_at INTEGER NOT NULL)`,
|
|
27
|
+
`CREATE TABLE IF NOT EXISTS operation_log (
|
|
28
|
+
op_id TEXT PRIMARY KEY, scope TEXT NOT NULL, kind TEXT NOT NULL, plans TEXT NOT NULL, outcome TEXT NOT NULL,
|
|
29
|
+
started_at INTEGER NOT NULL, ended_at INTEGER NOT NULL)`,
|
|
30
|
+
`CREATE TABLE IF NOT EXISTS intent (
|
|
31
|
+
id TEXT PRIMARY KEY, op_id TEXT NOT NULL, scope TEXT NOT NULL, step TEXT NOT NULL, kind TEXT NOT NULL,
|
|
32
|
+
tag TEXT NOT NULL, detail TEXT NOT NULL,
|
|
33
|
+
state TEXT NOT NULL CHECK (state IN ('open', 'done', 'failed', 'unknown', 'not_executed')),
|
|
34
|
+
created_at INTEGER NOT NULL, closed_at INTEGER, resolved_by TEXT)`,
|
|
35
|
+
`CREATE INDEX IF NOT EXISTS intent_scope_state ON intent (scope, state)`,
|
|
36
|
+
`CREATE TABLE IF NOT EXISTS deployment (
|
|
37
|
+
id TEXT PRIMARY KEY, seq INTEGER NOT NULL UNIQUE, op_id TEXT NOT NULL, mode TEXT NOT NULL,
|
|
38
|
+
before_half TEXT NOT NULL, after_half TEXT, created_at INTEGER NOT NULL, completed_at INTEGER)`,
|
|
39
|
+
`CREATE TABLE IF NOT EXISTS applied (file TEXT PRIMARY KEY, sha256 TEXT NOT NULL, applied_at INTEGER NOT NULL)`,
|
|
40
|
+
`CREATE TABLE IF NOT EXISTS resource (
|
|
41
|
+
kind TEXT NOT NULL, name TEXT NOT NULL, id TEXT, state TEXT NOT NULL, created_at INTEGER NOT NULL,
|
|
42
|
+
PRIMARY KEY (kind, name))`,
|
|
43
|
+
`CREATE TABLE IF NOT EXISTS preview (
|
|
44
|
+
identity TEXT PRIMARY KEY, nonce TEXT NOT NULL, worker TEXT NOT NULL, database_name TEXT NOT NULL,
|
|
45
|
+
database_id TEXT, queue_name TEXT NOT NULL, queue_id TEXT, classes TEXT NOT NULL DEFAULT '[]', consumer_id TEXT,
|
|
46
|
+
state TEXT NOT NULL CHECK (state IN ('creating', 'bootstrapped', 'destroying', 'destroyed')),
|
|
47
|
+
credentials TEXT, applied TEXT NOT NULL DEFAULT '{}', lost_create INTEGER NOT NULL DEFAULT 0,
|
|
48
|
+
created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL)`,
|
|
49
|
+
// A destroyed generation whose create lost its answer, kept by its own names when the identity
|
|
50
|
+
// is reused, so the sweep still finds what a late create leaves for its 7 days (§3.1, codex code r2).
|
|
51
|
+
`CREATE TABLE IF NOT EXISTS preview_tombstone (
|
|
52
|
+
worker TEXT PRIMARY KEY, identity TEXT NOT NULL, database_name TEXT NOT NULL, queue_name TEXT NOT NULL,
|
|
53
|
+
classes TEXT NOT NULL DEFAULT '[]', destroyed_at INTEGER NOT NULL)`,
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
export class OperationHeld extends Error {
|
|
57
|
+
constructor(readonly held: OperationRow) {
|
|
58
|
+
super(
|
|
59
|
+
`${held.scope} is held by operation ${held.op_id} (${held.kind}, at "${held.step}", ${held.executor ? `run by ${held.executor_label ?? held.executor}` : "stopped"}, since ${new Date(held.started_at).toISOString()})`,
|
|
60
|
+
);
|
|
61
|
+
this.name = "OperationHeld";
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export class LostOwnership extends Error {
|
|
66
|
+
constructor(scope: string, step: string) {
|
|
67
|
+
super(
|
|
68
|
+
`${scope}: this process no longer holds its operation (at "${step}"); another took it over or it ended. Stopping.`,
|
|
69
|
+
);
|
|
70
|
+
this.name = "LostOwnership";
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface OperationRow {
|
|
75
|
+
scope: string;
|
|
76
|
+
op_id: string;
|
|
77
|
+
kind: string;
|
|
78
|
+
plan: string;
|
|
79
|
+
step: string;
|
|
80
|
+
/** What the operation has learned so far (ids, versions), so a resume continues from it. */
|
|
81
|
+
progress: string;
|
|
82
|
+
executor: string | null;
|
|
83
|
+
executor_label: string | null;
|
|
84
|
+
heartbeat_at: number;
|
|
85
|
+
started_at: number;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export interface IntentRow {
|
|
89
|
+
id: string;
|
|
90
|
+
op_id: string;
|
|
91
|
+
scope: string;
|
|
92
|
+
step: string;
|
|
93
|
+
kind: string;
|
|
94
|
+
tag: string;
|
|
95
|
+
detail: string;
|
|
96
|
+
state: "open" | "done" | "failed" | "unknown" | "not_executed";
|
|
97
|
+
created_at: number;
|
|
98
|
+
closed_at: number | null;
|
|
99
|
+
resolved_by: string | null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface DeploymentRow {
|
|
103
|
+
id: string;
|
|
104
|
+
seq: number;
|
|
105
|
+
op_id: string;
|
|
106
|
+
mode: string;
|
|
107
|
+
before_half: string;
|
|
108
|
+
after_half: string | null;
|
|
109
|
+
created_at: number;
|
|
110
|
+
completed_at: number | null;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Who this process is, for a refusal to name: the host and, in CI, the run. */
|
|
114
|
+
export function executorLabel(env: NodeJS.ProcessEnv = process.env): string {
|
|
115
|
+
const run = env.GITHUB_RUN_ID
|
|
116
|
+
? ` (GitHub run ${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/${env.GITHUB_RUN_ATTEMPT}` : ""})`
|
|
117
|
+
: "";
|
|
118
|
+
return `${hostname()}:${process.pid}${run}`;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export class OpsStore {
|
|
122
|
+
/** This process's identity: a random id, never reused. */
|
|
123
|
+
readonly executor = `ex_${randomUUID().replaceAll("-", "")}`;
|
|
124
|
+
constructor(
|
|
125
|
+
private readonly q: Query,
|
|
126
|
+
private readonly now: () => number = Date.now,
|
|
127
|
+
readonly label: string = executorLabel(),
|
|
128
|
+
) {}
|
|
129
|
+
|
|
130
|
+
async init(store: string): Promise<void> {
|
|
131
|
+
for (const s of OPS_SCHEMA) await this.q(s);
|
|
132
|
+
await this.q("INSERT INTO kedge_ops_meta (id, store, created_at) VALUES (1, ?, ?) ON CONFLICT (id) DO NOTHING", [
|
|
133
|
+
store,
|
|
134
|
+
this.now(),
|
|
135
|
+
]);
|
|
136
|
+
const [meta] = await this.q("SELECT store FROM kedge_ops_meta WHERE id = 1");
|
|
137
|
+
if (meta?.store !== store)
|
|
138
|
+
throw new Error(`this operations database belongs to ${String(meta?.store)}, not ${store}`);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** A setting written once; the value already recorded wins, and is what this answers. */
|
|
142
|
+
async settle(name: string, value: string): Promise<string> {
|
|
143
|
+
await this.q("INSERT INTO ops_setting (name, value, created_at) VALUES (?, ?, ?) ON CONFLICT (name) DO NOTHING", [
|
|
144
|
+
name,
|
|
145
|
+
value,
|
|
146
|
+
this.now(),
|
|
147
|
+
]);
|
|
148
|
+
const [r] = await this.q("SELECT value FROM ops_setting WHERE name = ?", [name]);
|
|
149
|
+
return String(r?.value);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async setting(name: string): Promise<string | null> {
|
|
153
|
+
const [r] = await this.q("SELECT value FROM ops_setting WHERE name = ?", [name]);
|
|
154
|
+
return r ? String(r.value) : null;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// ---- operations --------------------------------------------------------------------------------------
|
|
158
|
+
async current(scope: string): Promise<OperationRow | null> {
|
|
159
|
+
return ((await this.q("SELECT * FROM operation WHERE scope = ?", [scope]))[0] as OperationRow | undefined) ?? null;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Starts an operation, or refuses naming the one that holds the scope. */
|
|
163
|
+
async start(scope: string, kind: string, plan: unknown, step: string): Promise<string> {
|
|
164
|
+
const opId = `op_${randomUUID().replaceAll("-", "").slice(0, 20)}`;
|
|
165
|
+
const t = this.now();
|
|
166
|
+
const won = await this.q(
|
|
167
|
+
`INSERT INTO operation (scope, op_id, kind, plan, step, executor, executor_label, heartbeat_at, started_at)
|
|
168
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (scope) DO NOTHING RETURNING op_id`,
|
|
169
|
+
[scope, opId, kind, JSON.stringify(plan), step, this.executor, this.label, t, t],
|
|
170
|
+
);
|
|
171
|
+
if (won.length === 1) return opId;
|
|
172
|
+
const held = await this.current(scope);
|
|
173
|
+
if (!held) throw new Error(`${scope}: the operation vanished while starting; run again`);
|
|
174
|
+
throw new OperationHeld(held);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Records the step about to run, and what's been learned; refuses unless this process still holds the operation. */
|
|
178
|
+
async step(scope: string, opId: string, step: string, progress?: unknown): Promise<void> {
|
|
179
|
+
const ok = await this.q(
|
|
180
|
+
`UPDATE operation SET step = ?, heartbeat_at = ?${progress === undefined ? "" : ", progress = ?"}
|
|
181
|
+
WHERE scope = ? AND op_id = ? AND executor = ? RETURNING op_id`,
|
|
182
|
+
[step, this.now(), ...(progress === undefined ? [] : [JSON.stringify(progress)]), scope, opId, this.executor],
|
|
183
|
+
);
|
|
184
|
+
if (ok.length !== 1) throw new LostOwnership(scope, step);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** A clean stop: the operation stays, with nobody running it, for a resume. */
|
|
188
|
+
async stop(scope: string, opId: string): Promise<void> {
|
|
189
|
+
await this.q("UPDATE intent SET state = 'unknown' WHERE scope = ? AND op_id = ? AND state = 'open'", [scope, opId]);
|
|
190
|
+
await this.q(
|
|
191
|
+
"UPDATE operation SET executor = NULL, executor_label = NULL WHERE scope = ? AND op_id = ? AND executor = ?",
|
|
192
|
+
[scope, opId, this.executor],
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** Claims a stopped operation: of two resumes one wins, and a running one can't be resumed. */
|
|
197
|
+
async resume(scope: string, opId: string): Promise<OperationRow> {
|
|
198
|
+
const ok = await this.q(
|
|
199
|
+
"UPDATE operation SET executor = ?, executor_label = ?, heartbeat_at = ? WHERE scope = ? AND op_id = ? AND executor IS NULL RETURNING *",
|
|
200
|
+
[this.executor, this.label, this.now(), scope, opId],
|
|
201
|
+
);
|
|
202
|
+
if (ok.length === 1) return ok[0] as unknown as OperationRow;
|
|
203
|
+
const held = await this.current(scope);
|
|
204
|
+
if (!held || held.op_id !== opId) throw new Error(`${scope} has no operation ${opId}`);
|
|
205
|
+
throw new OperationHeld(held);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Takes over an operation whose executor is set, on someone's word that it
|
|
210
|
+
* was cancelled (--confirm-stopped). Its open intents become unknown: a call
|
|
211
|
+
* it had sent may still land, so the caller reconciles before acting.
|
|
212
|
+
*/
|
|
213
|
+
async takeOver(scope: string, opId: string, observed: string | null): Promise<OperationRow> {
|
|
214
|
+
// From the executor the caller saw, and only from it: two runners recovering the same dead
|
|
215
|
+
// one can't both win, and the loser stops before it sends anything (codex code r8).
|
|
216
|
+
const ok = await this.q(
|
|
217
|
+
`UPDATE operation SET executor = ?, executor_label = ?, heartbeat_at = ?
|
|
218
|
+
WHERE scope = ? AND op_id = ? AND executor IS ? RETURNING *`,
|
|
219
|
+
[this.executor, this.label, this.now(), scope, opId, observed],
|
|
220
|
+
);
|
|
221
|
+
if (ok.length !== 1) {
|
|
222
|
+
const held = await this.current(scope);
|
|
223
|
+
if (!held || held.op_id !== opId) throw new Error(`${scope} has no operation ${opId}`);
|
|
224
|
+
throw new OperationHeld(held);
|
|
225
|
+
}
|
|
226
|
+
await this.q("UPDATE intent SET state = 'unknown' WHERE scope = ? AND op_id = ? AND state = 'open'", [scope, opId]);
|
|
227
|
+
return ok[0] as unknown as OperationRow;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** A new plan under the same operation; the old one stays in its history (§6.2's --supersede). */
|
|
231
|
+
async supersede(scope: string, opId: string, plan: unknown, progress?: unknown): Promise<void> {
|
|
232
|
+
// The plan and what its progress may keep, in one statement: a crash between two writes would
|
|
233
|
+
// pair a new plan with the old plan's uploaded version (codex code r5).
|
|
234
|
+
const ok = await this.q(
|
|
235
|
+
`UPDATE operation SET plan = ?, heartbeat_at = ?${progress === undefined ? "" : ", progress = ?"}
|
|
236
|
+
WHERE scope = ? AND op_id = ? AND executor = ? RETURNING plan`,
|
|
237
|
+
[
|
|
238
|
+
JSON.stringify(plan),
|
|
239
|
+
this.now(),
|
|
240
|
+
...(progress === undefined ? [] : [JSON.stringify(progress)]),
|
|
241
|
+
scope,
|
|
242
|
+
opId,
|
|
243
|
+
this.executor,
|
|
244
|
+
],
|
|
245
|
+
);
|
|
246
|
+
if (ok.length !== 1) throw new LostOwnership(scope, "supersede");
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Ends the operation, logged with every plan it ran under. Two requests, so each is safe to
|
|
251
|
+
* repeat: the log row is written only by the owner and only once, and a finish whose release
|
|
252
|
+
* was lost after its log landed releases on the next try instead of failing on the log's key
|
|
253
|
+
* and holding the store for good (codex code r1).
|
|
254
|
+
*/
|
|
255
|
+
async finish(scope: string, opId: string, outcome: string, plans: readonly unknown[]): Promise<void> {
|
|
256
|
+
const row = await this.current(scope);
|
|
257
|
+
if (!row || row.op_id !== opId || row.executor !== this.executor) throw new LostOwnership(scope, "finish");
|
|
258
|
+
await this.q(
|
|
259
|
+
`INSERT INTO operation_log (op_id, scope, kind, plans, outcome, started_at, ended_at)
|
|
260
|
+
SELECT op_id, scope, kind, ?, ?, started_at, ? FROM operation WHERE scope = ? AND op_id = ? AND executor = ?
|
|
261
|
+
ON CONFLICT (op_id) DO NOTHING`,
|
|
262
|
+
[JSON.stringify(plans), outcome, this.now(), scope, opId, this.executor],
|
|
263
|
+
);
|
|
264
|
+
await this.q("DELETE FROM operation WHERE scope = ? AND op_id = ? AND executor = ?", [scope, opId, this.executor]);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// ---- intents (§3.1) ------------------------------------------------------------------------------------
|
|
268
|
+
async openIntent(
|
|
269
|
+
scope: string,
|
|
270
|
+
opId: string,
|
|
271
|
+
step: string,
|
|
272
|
+
kind: string,
|
|
273
|
+
tag: string,
|
|
274
|
+
detail: unknown,
|
|
275
|
+
): Promise<string> {
|
|
276
|
+
const id = `int_${randomUUID().replaceAll("-", "").slice(0, 20)}`;
|
|
277
|
+
// Recorded only by the operation's current executor: a runner that lost it sends nothing.
|
|
278
|
+
const ok = await this.q(
|
|
279
|
+
`INSERT INTO intent (id, op_id, scope, step, kind, tag, detail, state, created_at)
|
|
280
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, 'open', ? WHERE EXISTS
|
|
281
|
+
(SELECT 1 FROM operation WHERE scope = ? AND op_id = ? AND executor = ?) RETURNING id`,
|
|
282
|
+
[id, opId, scope, step, kind, tag, JSON.stringify(detail), this.now(), scope, opId, this.executor],
|
|
283
|
+
);
|
|
284
|
+
if (ok.length !== 1) throw new LostOwnership(scope, step);
|
|
285
|
+
return id;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
async closeIntent(
|
|
289
|
+
id: string,
|
|
290
|
+
state: "done" | "failed" | "unknown" | "not_executed",
|
|
291
|
+
resolvedBy: string,
|
|
292
|
+
): Promise<void> {
|
|
293
|
+
await this.q("UPDATE intent SET state = ?, closed_at = ?, resolved_by = ? WHERE id = ?", [
|
|
294
|
+
state,
|
|
295
|
+
this.now(),
|
|
296
|
+
resolvedBy,
|
|
297
|
+
id,
|
|
298
|
+
]);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** Calls whose outcome nobody knows: nothing in the scope runs while one exists. */
|
|
302
|
+
async unresolved(scope: string): Promise<IntentRow[]> {
|
|
303
|
+
return (await this.q("SELECT * FROM intent WHERE scope = ? AND state IN ('open', 'unknown') ORDER BY created_at", [
|
|
304
|
+
scope,
|
|
305
|
+
])) as unknown as IntentRow[];
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
async intents(scope: string, opId?: string): Promise<IntentRow[]> {
|
|
309
|
+
return (await this.q(`SELECT * FROM intent WHERE scope = ?${opId ? " AND op_id = ?" : ""} ORDER BY created_at`, [
|
|
310
|
+
scope,
|
|
311
|
+
...(opId ? [opId] : []),
|
|
312
|
+
])) as unknown as IntentRow[];
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
async intent(id: string): Promise<IntentRow | null> {
|
|
316
|
+
return ((await this.q("SELECT * FROM intent WHERE id = ?", [id]))[0] as IntentRow | undefined) ?? null;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// ---- deployments (§6.5) ----------------------------------------------------------------------------------
|
|
320
|
+
async deployments(): Promise<DeploymentRow[]> {
|
|
321
|
+
return (await this.q("SELECT * FROM deployment ORDER BY seq DESC")) as unknown as DeploymentRow[];
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
async addDeployment(opId: string, mode: string, before: unknown): Promise<string> {
|
|
325
|
+
const id = `dep_${randomUUID().replaceAll("-", "").slice(0, 20)}`;
|
|
326
|
+
await this.q(
|
|
327
|
+
`INSERT INTO deployment (id, seq, op_id, mode, before_half, created_at)
|
|
328
|
+
VALUES (?, (SELECT COALESCE(MAX(seq), 0) + 1 FROM deployment), ?, ?, ?, ?)`,
|
|
329
|
+
[id, opId, mode, JSON.stringify(before), this.now()],
|
|
330
|
+
);
|
|
331
|
+
return id;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
async completeDeployment(id: string, after: unknown): Promise<void> {
|
|
335
|
+
await this.q("UPDATE deployment SET after_half = ?, completed_at = ? WHERE id = ?", [
|
|
336
|
+
JSON.stringify(after),
|
|
337
|
+
this.now(),
|
|
338
|
+
id,
|
|
339
|
+
]);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// ---- applied migrations' hashes (§6.1) --------------------------------------------------------------------
|
|
343
|
+
async appliedHashes(): Promise<Map<string, string>> {
|
|
344
|
+
return new Map((await this.q("SELECT file, sha256 FROM applied")).map((r) => [String(r.file), String(r.sha256)]));
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
async recordApplied(file: string, sha256: string): Promise<void> {
|
|
348
|
+
await this.q("INSERT INTO applied (file, sha256, applied_at) VALUES (?, ?, ?) ON CONFLICT (file) DO NOTHING", [
|
|
349
|
+
file,
|
|
350
|
+
sha256,
|
|
351
|
+
this.now(),
|
|
352
|
+
]);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// ---- the store's own resources (§6.1) ------------------------------------------------------------------
|
|
356
|
+
async resource(kind: string, name: string): Promise<{ id: string | null; state: string } | null> {
|
|
357
|
+
const [r] = await this.q("SELECT id, state FROM resource WHERE kind = ? AND name = ?", [kind, name]);
|
|
358
|
+
return r ? { id: (r.id as string | null) ?? null, state: String(r.state) } : null;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/** Recorded before the resource is used (a Worker before its first upload). */
|
|
362
|
+
async recordResource(kind: string, name: string, id: string | null, state: string): Promise<void> {
|
|
363
|
+
await this.q(
|
|
364
|
+
`INSERT INTO resource (kind, name, id, state, created_at) VALUES (?, ?, ?, ?, ?)
|
|
365
|
+
ON CONFLICT (kind, name) DO UPDATE SET id = COALESCE(excluded.id, resource.id), state = excluded.state`,
|
|
366
|
+
[kind, name, id, state, this.now()],
|
|
367
|
+
);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
query(sql: string, params?: readonly unknown[]): Promise<Row[]> {
|
|
371
|
+
return this.q(sql, params);
|
|
372
|
+
}
|
|
373
|
+
}
|
package/src/order.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strings in code-point order. `localeCompare` follows the machine's ICU locale,
|
|
3
|
+
* which orders `_` differently under C.UTF-8 than under en_US, so a generated
|
|
4
|
+
* file or a hash sorted with it differs between two machines and `generate
|
|
5
|
+
* --check` fails on the one that didn't write it.
|
|
6
|
+
*/
|
|
7
|
+
export const byCodePoint = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
|
package/src/output.ts
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// `--json` (p0.10 §5): stdout carries one object when the command ends, and
|
|
2
|
+
// nothing else, so an agent parses it whole. Progress still reaches a person,
|
|
3
|
+
// on stderr: console.log is pointed there for the command's run, and every
|
|
4
|
+
// child the CLI starts (Wrangler, the store's checks and scenarios) writes its
|
|
5
|
+
// stdout there too, since a child's line on stdout would break the object.
|
|
6
|
+
import type { StdioOptions } from "node:child_process";
|
|
7
|
+
|
|
8
|
+
let reserved = false;
|
|
9
|
+
|
|
10
|
+
/** Runs `work` with stdout reserved for `emit` when `on`; nested runs keep the outer setting. */
|
|
11
|
+
export async function withJsonOutput<T>(on: boolean, work: () => Promise<T>): Promise<T> {
|
|
12
|
+
if (!on || reserved) return work();
|
|
13
|
+
const log = console.log;
|
|
14
|
+
const info = console.info;
|
|
15
|
+
reserved = true;
|
|
16
|
+
console.log = console.error;
|
|
17
|
+
console.info = console.error;
|
|
18
|
+
try {
|
|
19
|
+
return await work();
|
|
20
|
+
} finally {
|
|
21
|
+
reserved = false;
|
|
22
|
+
console.log = log;
|
|
23
|
+
console.info = info;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** The command's one result object. */
|
|
28
|
+
export function emit(result: Record<string, unknown>): void {
|
|
29
|
+
process.stdout.write(`${JSON.stringify(result)}\n`);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** A child's stdio: inherited, except that its stdout goes to stderr while stdout is reserved. */
|
|
33
|
+
export function childStdio(): StdioOptions {
|
|
34
|
+
return reserved ? ["inherit", 2, "inherit"] : "inherit";
|
|
35
|
+
}
|