@runeditrun/rer 0.2.24 → 0.2.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +16 -7
  2. package/cli/browser-runtime.mjs +65 -0
  3. package/cli/demo-host.mjs +31 -5
  4. package/cli/deploy-claim-request.mjs +5 -4
  5. package/cli/deploy-command.mjs +3 -3
  6. package/cli/deploy-inputs.mjs +2 -1
  7. package/cli/deploy.mjs +16 -4
  8. package/cli/export.mjs +1 -1
  9. package/cli/hosted-mail.mjs +23 -0
  10. package/cli/hosted-output.mjs +6 -2
  11. package/cli/hydration/browser.mjs +3 -9
  12. package/cli/inspect.mjs +3 -1
  13. package/cli/lifecycle.mjs +4 -1
  14. package/global/CHANGELOG.md +36 -0
  15. package/global/README.md +16 -7
  16. package/global/browser-runtime.mjs +45 -0
  17. package/global/cli/layout.mjs +8 -4
  18. package/global/main.mjs +13 -3
  19. package/install.sh +56 -3
  20. package/modules/container/README.md +11 -0
  21. package/modules/container/build.mjs +2 -1
  22. package/modules/container/demo-egress.mjs +15 -2
  23. package/modules/container/image-variables.mjs +27 -0
  24. package/modules/container/worker-engine.mjs +2 -2
  25. package/modules/delivery-producer/declarations.mjs +6 -1
  26. package/modules/delivery-producer/mail.mjs +66 -0
  27. package/modules/demo/label.mjs +5 -2
  28. package/modules/demo-host/README.md +9 -2
  29. package/modules/demo-host/certificate.mjs +56 -0
  30. package/modules/demo-host/measurement.mjs +19 -0
  31. package/modules/demo-host/operator.mjs +11 -6
  32. package/modules/demo-host/plan.mjs +1 -1
  33. package/modules/demo-host/worker.mjs +2 -0
  34. package/modules/deploy-kit/README.md +6 -4
  35. package/modules/deploy-kit/asset-health.mjs +28 -0
  36. package/modules/deploy-kit/d1-import.mjs +22 -4
  37. package/modules/deploy-kit/diagnostics.mjs +30 -0
  38. package/modules/deploy-kit/health.mjs +2 -23
  39. package/modules/deploy-kit/migration-ledger.mjs +6 -2
  40. package/modules/deploy-kit/migration-retry.mjs +91 -0
  41. package/modules/deploy-kit/migrations.mjs +9 -5
  42. package/modules/deploy-kit/owner-cli.mjs +6 -3
  43. package/modules/deploy-kit/owner-remove.mjs +87 -65
  44. package/modules/deploy-kit/provider-transport.mjs +28 -5
  45. package/modules/deploy-kit/provider.mjs +9 -6
  46. package/modules/deploy-kit/worker.mjs +5 -2
  47. package/modules/mail/README.md +18 -2
  48. package/modules/mail/src/adapters/rer/contract.mjs +5 -2
  49. package/modules/mail/src/adapters/rer/list-headers.mjs +44 -0
  50. package/modules/mail/src/core/validation.mjs +5 -2
  51. package/modules/mail/src/sender.mjs +1 -1
  52. package/modules/mail/src/unsubscribe/index.mjs +5 -5
  53. package/modules/operator-auth/README.md +10 -2
  54. package/modules/operator-auth/config.mjs +1 -0
  55. package/modules/operator-auth/conformance/presentation.mjs +2 -2
  56. package/modules/operator-auth/identity/email-link/confirmation.mjs +2 -1
  57. package/modules/operator-auth/identity/email-link/expiry.mjs +8 -0
  58. package/modules/operator-auth/identity/email-link/index.mjs +2 -1
  59. package/modules/operator-auth/identity/email-link/message.mjs +3 -2
  60. package/modules/operator-auth/index.d.mts +2 -0
  61. package/modules/portability/README.md +7 -2
  62. package/modules/portability/import-staging.mjs +45 -0
  63. package/modules/portability/import.mjs +8 -8
  64. package/modules/portability/storage.mjs +5 -2
  65. package/package.json +1 -1
@@ -0,0 +1,28 @@
1
+ import { nameAssetFailure } from './diagnostics.mjs';
2
+
3
+ /** Probe a real built file after identity health. A transient read gets one replay;
4
+ * a fresh edge missing a hashed file may settle within the existing route window. */
5
+ export async function servedAsset(origin, path, request, timeoutMs, {window, windowMs = 0, clock = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms))} = {}) {
6
+ const start = clock(), budget = Math.min(windowMs || timeoutMs, window?.remaining() ?? Infinity);
7
+ const deadline = AbortSignal.timeout(Math.max(1, Math.ceil(budget)));
8
+ let transient = 0, failure;
9
+ const fail = (code, status) => nameAssetFailure(new Error(code), [path], status);
10
+ for (;;) {
11
+ const remaining = Math.min(budget - (clock() - start), window?.remaining() ?? Infinity);
12
+ if (remaining <= 0 || deadline.aborted) throw failure ?? fail('asset-readiness-timeout');
13
+ let response;
14
+ try { response = await request(`${origin}/${path.split('/').map(encodeURIComponent).join('/')}`, {redirect:'manual',signal:AbortSignal.any([deadline,AbortSignal.timeout(Math.max(1, Math.ceil(Math.min(timeoutMs,remaining / (transient ? 1 : 2)))))]),headers:{'Cache-Control':'no-cache'}}); }
15
+ catch { failure = fail('asset-transport-failed'); }
16
+ if (response) {
17
+ await response.body?.cancel().catch(() => {});
18
+ if (response.status === 200 && !response.redirected) return {path,status:200};
19
+ failure = fail('asset-http-failed', response.status);
20
+ if (response.redirected || response.status !== 404 && (response.status < 500 || response.status > 599)) throw failure;
21
+ }
22
+ const propagating = response?.status === 404 && clock() - start + 2000 <= windowMs;
23
+ if (!propagating && (++transient > 1 || response?.status === 404)) throw failure;
24
+ const rest = Math.min(budget - (clock() - start), window?.remaining() ?? Infinity);
25
+ if (rest <= 0 || deadline.aborted) throw failure;
26
+ await sleep(Math.min(2000,rest));
27
+ }
28
+ }
@@ -1,9 +1,27 @@
1
1
  import { createHash } from 'node:crypto';
2
2
  import { assert, uuid } from './validation.mjs';
3
+ import { recordDeployFailure } from './diagnostics.mjs';
3
4
 
4
5
  // Wrangler's remote --file protocol. D1 imports execute the whole file atomically,
5
6
  // including trigger bodies; the query endpoint's semicolon splitting cannot.
6
- export async function importD1({ envelope, upload }, id, sql) {
7
+ // Cloudflare's own reason for an import that failed or did not finish, kept on the
8
+ // error as providerRefusal (rer deploy names it) and in the deploy diagnostic: its
9
+ // codes and messages without credentials, URLs (signed upload destinations) or
10
+ // control characters, each cut to 300 characters.
11
+ function importFailure(code, id, response, redact) {
12
+ const clean = text => redact(text).replace(/\bhttps?:\/\/\S+/gi, '[URL omitted]').replace(/[\x00-\x1f\x7f-\x9f]+/g, ' ').trim().slice(0, 300);
13
+ const said = [response?.error, ...(Array.isArray(response?.errors) ? response.errors : [])]
14
+ .map(item => typeof item === 'string' ? { message:item } : item)
15
+ .filter(item => Number.isSafeInteger(item?.code) || typeof item?.message === 'string' && item.message.trim()).slice(0, 3)
16
+ .map(item => ({ ...(Number.isSafeInteger(item.code) ? { code:item.code } : {}), ...(typeof item.message === 'string' ? { message:clean(item.message) } : {}) }));
17
+ const status = typeof response?.status === 'string' && /^[a-z_]{1,40}$/.test(response.status) ? response.status : undefined;
18
+ const errors = said.length ? said : status ? [{ message:`import status ${status}` }] : [];
19
+ const error = new Error(code);
20
+ error.providerRefusal = { operation:`POST /d1/database/${id}/import`, errors };
21
+ return recordDeployFailure(error, { providerCodes:errors.filter(item => item.code !== undefined).map(item => item.code), output:errors.map(item => item.message).filter(Boolean).join('\n') });
22
+ }
23
+
24
+ export async function importD1({ envelope, upload, redact = String }, id, sql) {
7
25
  assert(uuid(id) && typeof sql === 'string' && Buffer.byteLength(sql) <= 4 * 1024 * 1024, 'invalid-d1-import');
8
26
  const bytes = Buffer.from(sql), etag = createHash('md5').update(bytes).digest('hex');
9
27
  const post = async body => (await envelope(`/d1/database/${id}/import`, { method:'POST', body })).result;
@@ -15,12 +33,12 @@ export async function importD1({ envelope, upload }, id, sql) {
15
33
  }
16
34
  for (let polls = 0; polls < 100; polls++) {
17
35
  // REST places errors on the envelope; validate nested errors when supplied.
18
- assert(response?.success === true && response.type === 'import' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length), 'd1-import-failed');
36
+ if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) throw importFailure('d1-import-failed', id, response, redact);
19
37
  if (response.status === 'complete') {
20
- assert(Number.isSafeInteger(response.result?.num_queries) && response.result.num_queries > 0 && typeof response.result.final_bookmark === 'string' && response.result.final_bookmark.length > 0, 'd1-import-invalid-result');
38
+ if (!(Number.isSafeInteger(response.result?.num_queries) && response.result.num_queries > 0 && typeof response.result.final_bookmark === 'string' && response.result.final_bookmark.length > 0)) throw importFailure('d1-import-invalid-result', id, response, redact);
21
39
  return;
22
40
  }
23
- assert(response.status === 'active' && typeof response.at_bookmark === 'string' && response.at_bookmark.length > 0 && response.at_bookmark.length <= 1024, 'd1-import-uncertain');
41
+ if (!(response.status === 'active' && typeof response.at_bookmark === 'string' && response.at_bookmark.length > 0 && response.at_bookmark.length <= 1024)) throw importFailure('d1-import-uncertain', id, response, redact);
24
42
  response = await post({ action:'poll', current_bookmark:response.at_bookmark });
25
43
  }
26
44
  throw new Error('d1-import-poll-limit');
@@ -25,6 +25,24 @@ export function providerRefusal(error, redact = String) {
25
25
  const said = refusal.errors.map(item => [item?.code, typeof item?.message === 'string' ? redact(item.message).slice(0,300) : undefined].filter(part => part !== undefined && part !== '').join(' ')).filter(Boolean).slice(0,3);
26
26
  return `Cloudflare refused ${redact(refusal.operation).slice(0,200)}${said.length ? `: ${said.join('; ')}` : ''}`;
27
27
  }
28
+ /** What a migration that failed or stays refused means for the next run, from the
29
+ * error's own `migration` or `migrationClaim` (set by any copy of the kit) and its
30
+ * `d1:<name>` resource; undefined otherwise. */
31
+ export function migrationFailure(error, again = 'rer deploy') {
32
+ const safe = value => String(value).replace(/[^\w. -]/g,'?').slice(0,120);
33
+ const database = /^d1:([a-z0-9-]+)$/.exec(error?.resource ?? '')?.[1];
34
+ const claim = error?.migrationClaim;
35
+ if (typeof claim?.name === 'string') {
36
+ const name = safe(claim.name).replaceAll(' ','?');
37
+ const run = sql => `npx wrangler d1 execute ${database ?? '<database>'} --remote --command "${sql} WHERE name='${name}' AND state='applying'"`;
38
+ const present = (Array.isArray(claim.present) ? claim.present : []).slice(0,5).map(safe);
39
+ return `Migration ${name} in ${database ? `D1 ${database}` : 'its D1 database'} was started but never confirmed, and rer runs it again only when it can prove none of it applied. ${present.length
40
+ ? `Some of its changes are present (${present.join(', ')}). Undo them so the database is as it was before ${name}, then run ${again} again: rer checks again and runs it.`
41
+ : `It creates no table, index, view, trigger or column, so rer cannot check whether it applied. If none of its changes are present, delete its claim with ${run('DELETE FROM d1_migrations')}, then run ${again} again.`} If it did apply in full, record that instead with ${run("UPDATE d1_migrations SET state='applied'")}.`;
42
+ }
43
+ if (typeof error?.migration === 'string') return `Migration ${safe(error.migration)} did not apply; the next ${again} runs it again once rer proves none of it applied.`;
44
+ return undefined;
45
+ }
28
46
  /** Safe call identity travels with errors from independently vendored kits. */
29
47
  export function providerTransport(error, redact = String) {
30
48
  const detail = error?.providerTransport;
@@ -95,3 +113,15 @@ export function providerFailure(error, redact = String) {
95
113
  const clean = value => redact(String(value)).replace(/[\x00-\x1f\x7f-\x9f]+/g,' ').replace(/\b[a-f0-9]{32}\b/gi,'[REDACTED]').slice(0,300);
96
114
  return `Cloudflare call ${clean(call.operation)}: ${clean(call.reason ?? error.message)} (no Cloudflare error code or message supplied)`;
97
115
  }
116
+
117
+ // Asset paths are build inputs; the CLI still redacts them before display.
118
+ export function nameAssetFailure(error, paths, status = error.status ?? error.providerTransport?.status) {
119
+ error.assetFailure = {paths, status};
120
+ return error;
121
+ }
122
+ export function assetFailure(error, redact = String) {
123
+ const detail = error?.assetFailure;
124
+ if (!Array.isArray(detail?.paths)) return undefined;
125
+ const names = detail.paths.slice(0,8).map(path => redact(String(path)).replace(/[\x00-\x1f\x7f-\x9f]+/g,' ').slice(0,300));
126
+ return `Asset ${names.join(', ')}${detail.paths.length > 8 ? ` (+${detail.paths.length - 8} more)` : ''}: ${Number.isInteger(detail.status) ? `HTTP ${detail.status}` : 'no HTTP status (transport failure)'}`;
127
+ }
@@ -1,3 +1,4 @@
1
+ import { servedAsset } from './asset-health.mjs';
1
2
  import { readinessAPI } from './readiness-window.mjs';
2
3
  import { verifyServiceGrants } from './service-grants.mjs';
3
4
  import { assert, stable } from './validation.mjs';
@@ -47,32 +48,10 @@ export async function waitForDeploymentHealth(url, {
47
48
  throw healthError(`Deployment did not become healthy at ${url} within ${timeoutMs}ms; last result: ${lastResult}.`, lastResult === 'network error' ? 'health-transport-failed' : 'health-http-failed', lastResult);
48
49
  }
49
50
 
50
- /** An unauthenticated GET of a built file; anything but 200 means the browser gets no assets.
51
- * After an upload (`windowMs`), an edge still serving the previous version answers 404 for
52
- * the new build's hashed files: 404, 5xx and transport failures are retried within the window. */
53
- async function servedAsset(origin, path, request, timeoutMs, {window, windowMs = 0, clock = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve, ms))} = {}) {
54
- const start = clock();
55
- for (;;) {
56
- let response, failure;
57
- const remaining = window?.remaining() ?? timeoutMs;
58
- assert(remaining > 0, 'asset-readiness-timeout');
59
- try { response = await request(`${origin}/${path.split('/').map(encodeURIComponent).join('/')}`,{redirect:'manual',signal:AbortSignal.timeout(Math.max(1, Math.ceil(Math.min(timeoutMs,remaining)))),headers:{'Cache-Control':'no-cache'}}); }
60
- catch { failure = 'asset-transport-failed'; }
61
- if (response) {
62
- await response.body?.cancel().catch(() => {});
63
- if (response.status === 200 && !response.redirected) return {path,status:response.status};
64
- failure = 'asset-http-failed';
65
- if (response.redirected || response.status !== 404 && response.status < 500) throw new Error(failure);
66
- }
67
- if (clock() - start + 2000 > windowMs) throw new Error(failure);
68
- await sleep(Math.min(2000,window?.remaining() ?? 2000));
69
- }
70
- }
71
-
72
51
  /** Authenticated provider readback, then the unauthenticated account-derived /health.
73
52
  * Without `readiness`, /health gets the single bounded window of waitForDeploymentHealth.
74
53
  * With `readiness` (deploy after an upload) it gets the recorded 90-second wait of health-request.mjs.
75
- * `asset`, a built file's path, is then requested once and must answer 200. */
54
+ * `asset`, a built file's path, must answer 200 after at most one transient replay. */
76
55
  export async function verifyHealth({api,plan,receipt,fetch:request=globalThis.fetch,timeoutMs=30000,intervalMs=2000,now,sleep,readiness,asset}) {
77
56
  if (readiness?.window) api = readinessAPI(api, readiness.window);
78
57
  assert(receipt?.version === 1 && receipt.accountId === api.accountId && receipt.worker === plan.worker,'receipt-target-mismatch');
@@ -47,7 +47,7 @@ const same = (a, b) => a.length === b.length && a.every((row, index) => row.name
47
47
  export async function readLedger(api, id, migrations, query) {
48
48
  const read = async sql => (await query(api, id, sql))[0].results;
49
49
  const tables = (await read(`SELECT name FROM sqlite_master WHERE type='table' AND name IN ('${legacy}','${ledger}')`)).map(row => row.name);
50
- let old = [], applied = [], transition = '';
50
+ let old = [], applied = [], transition = '', claimed = false;
51
51
  if (tables.includes(legacy)) {
52
52
  const columns = await read(`SELECT name FROM pragma_table_info('${legacy}')`);
53
53
  assert(columns.length === 3 && columns.every(row => ['name','sha256','state'].includes(row.name)), 'invalid-migration-ledger');
@@ -59,6 +59,10 @@ export async function readLedger(api, id, migrations, query) {
59
59
  assert(['id','name','applied_at'].every(name => columns.includes(name)), 'invalid-migration-ledger');
60
60
  if (columns.includes('sha256') && columns.includes('state')) {
61
61
  applied = await read(`SELECT name, sha256, state FROM ${ledger} ORDER BY id`);
62
+ // The last row may claim this build's next migration, exact bytes, whose import
63
+ // did not complete; migrate re-runs it only on proof (migration-retry.mjs).
64
+ const last = applied.at(-1);
65
+ if (!tables.includes(legacy) && last?.state === 'applying' && last.name === migrations[applied.length - 1]?.name && last.sha256 === migrations[applied.length - 1].sha256) claimed = Boolean(applied.pop());
62
66
  assert(validPrefix(applied, migrations), 'changed-or-uncertain-migration-history');
63
67
  if (tables.includes(legacy)) assert(same(applied, old), 'changed-or-uncertain-migration-history');
64
68
  } else {
@@ -78,7 +82,7 @@ export async function readLedger(api, id, migrations, query) {
78
82
  applied = old;
79
83
  }
80
84
  if (tables.includes(legacy)) transition += `DROP TABLE ${legacy};`;
81
- return { applied, transition, exists:tables.includes(ledger) || tables.includes(legacy) };
85
+ return { applied, transition, claimed, exists:tables.includes(ledger) || tables.includes(legacy) };
82
86
  }
83
87
  function insert(row) {
84
88
  return `INSERT INTO ${ledger}(name,sha256,state) VALUES (${literal(row.name)},${literal(row.sha256)},'applied');\n`;
@@ -0,0 +1,91 @@
1
+ import { sqlTokens } from './sql-tokens.mjs';
2
+ import { literal } from './migration-ledger.mjs';
3
+
4
+ // A migration claimed `applying` whose import never completed is re-run only on
5
+ // proof that none of it applied. D1 imports the SQL and its completion marker as
6
+ // one atomic file, so one statement that fails when its effect already exists
7
+ // (CREATE without IF NOT EXISTS, ADD COLUMN, RENAME) and whose effect lasts to the
8
+ // end of the file proves it: its effect absent now means no attempt committed, and
9
+ // an earlier attempt that commits late makes the re-run fail whole instead of
10
+ // applying twice. Every such effect must be absent; any present one is refused.
11
+
12
+ const word = token => token?.kind === 'word' ? token.value.toUpperCase() : '';
13
+ const lower = token => token?.kind === 'word' || token?.kind === 'identifier' ? token.value.toLowerCase() : undefined;
14
+ const symbol = (token, value) => token?.kind === 'symbol' && token.value === value;
15
+
16
+ // Trigger bodies split here too; their parts are DML and add or remove no guard.
17
+ function statements(sql) {
18
+ const list = [[]];
19
+ for (const token of sqlTokens(sql)) symbol(token, ';') ? list.push([]) : list.at(-1).push(token);
20
+ return list.filter(statement => statement.length);
21
+ }
22
+
23
+ // [main.]name at index; anything in another schema is not checked.
24
+ function named(statement, index) {
25
+ if (!symbol(statement[index + 1], '.')) return { name:lower(statement[index]), next:index + 1 };
26
+ return { name:lower(statement[index]) === 'main' ? lower(statement[index + 2]) : undefined, next:index + 3 };
27
+ }
28
+
29
+ function create(statement) {
30
+ let index = 1;
31
+ if (['TEMP', 'TEMPORARY'].includes(word(statement[index]))) return undefined;
32
+ if (['UNIQUE', 'VIRTUAL'].includes(word(statement[index]))) index++;
33
+ const type = word(statement[index++]).toLowerCase();
34
+ if (!['table', 'index', 'view', 'trigger'].includes(type) || word(statement[index]) === 'IF') return undefined;
35
+ const target = named(statement, index);
36
+ if (!target.name) return undefined;
37
+ if (type === 'table' || type === 'view') return { type, object:target.name, table:target.name };
38
+ const on = statement.findIndex((token, at) => at >= target.next && word(token) === 'ON');
39
+ const table = on < 0 ? undefined : named(statement, on + 1).name;
40
+ return table && { type, object:target.name, table };
41
+ }
42
+
43
+ /** The effects of a migration whose absence proves it never applied. */
44
+ export function retryGuards(sql) {
45
+ let guards = [];
46
+ for (const statement of statements(sql)) {
47
+ const first = word(statement[0]);
48
+ if (first === 'CREATE') {
49
+ const guard = create(statement);
50
+ if (guard) guards.push(guard);
51
+ } else if (first === 'ALTER' && word(statement[1]) === 'TABLE') {
52
+ const { name:table, next } = named(statement, 2);
53
+ let index = next;
54
+ const action = word(statement[index++]);
55
+ if (word(statement[index]) === 'COLUMN') index++;
56
+ const column = lower(statement[index]);
57
+ if (!table) guards = [];
58
+ else if (action === 'ADD' && column) guards.push({ table, column });
59
+ else if (action === 'RENAME' && word(statement[next + 1]) === 'TO') {
60
+ const to = named(statement, next + 2).name;
61
+ guards = guards.filter(guard => guard.table !== table);
62
+ if (to) guards.push({ type:'table', object:to, table:to });
63
+ } else if (action === 'RENAME' && column && word(statement[index + 1]) === 'TO' && lower(statement[index + 2])) {
64
+ guards = guards.filter(guard => !(guard.table === table && guard.column === column));
65
+ guards.push({ table, column:lower(statement[index + 2]) });
66
+ } else if (action === 'DROP' && column) guards = guards.filter(guard => !(guard.table === table && guard.column === column));
67
+ else guards = [];
68
+ } else if (first === 'DROP') {
69
+ const type = word(statement[1]);
70
+ const { name } = named(statement, word(statement[2]) === 'IF' ? 4 : 2);
71
+ // A later drop removes the effect it would prove; an unreadable one proves nothing.
72
+ guards = name ? guards.filter(guard => guard.object !== name && !(['TABLE', 'VIEW'].includes(type) && guard.table === name)) : [];
73
+ }
74
+ }
75
+ return guards;
76
+ }
77
+
78
+ const label = guard => guard.column === undefined ? `${guard.type} ${guard.object}` : `column ${guard.table}.${guard.column}`;
79
+
80
+ /** Refuse unless the claimed migration's effects are provably absent: it then
81
+ * may run again. `read(sql)` returns one SELECT's rows. */
82
+ export async function assertRetryable(read, migration) {
83
+ const guards = retryGuards(migration.sql), present = [];
84
+ for (let start = 0; start < guards.length; start += 50) {
85
+ const part = guards.slice(start, start + 50).map((guard, index) => guard.column === undefined
86
+ ? `SELECT ${index} AS guard FROM sqlite_master WHERE type=${literal(guard.type)} AND lower(name)=${literal(guard.object)}`
87
+ : `SELECT ${index} AS guard FROM pragma_table_info(${literal(guard.table)}) WHERE lower(name)=${literal(guard.column)}`);
88
+ for (const row of await read(part.join(' UNION ALL '))) present.push(label(guards[start + row.guard]));
89
+ }
90
+ if (!guards.length || present.length) throw Object.assign(new Error('uncertain-migration-claim'), { migrationClaim:{ name:migration.name, present:[...new Set(present)] } });
91
+ }
@@ -4,6 +4,7 @@ import { regularBytes } from './artifact.mjs';
4
4
  import { assertMigrationSqlSafe } from './sql-safety.mjs';
5
5
  import { sqlTokens } from './sql-tokens.mjs';
6
6
  import { ledger, literal, createLedger, readLedger, recorded } from './migration-ledger.mjs';
7
+ import { assertRetryable } from './migration-retry.mjs';
7
8
  export function validateMigrations(migrations) {
8
9
  let previous = '';
9
10
  for (const migration of migrations) {
@@ -54,7 +55,7 @@ export async function query(api, id, sql) {
54
55
  }
55
56
  export async function migrate(api, id, migrations, { apply = true, upgrade = apply } = {}) {
56
57
  validateMigrations(migrations);
57
- const { applied, transition, exists } = await readLedger(api, id, migrations, query);
58
+ const { applied, transition, exists, claimed } = await readLedger(api, id, migrations, query);
58
59
  const pending = migrations.slice(applied.length);
59
60
  assert(apply || pending.length === 0,'pending-migrations');
60
61
  if (transition && upgrade) {
@@ -63,14 +64,17 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
63
64
  assert(!verified.transition && verified.applied.length === applied.length, 'migration-readback-mismatch');
64
65
  }
65
66
  if (pending.length && !exists) await query(api,id,createLedger);
66
- for (const migration of pending) {
67
- // Claim before execution. A lost response/partial provider failure is never automatically replayed.
68
- await query(api,id,`INSERT INTO ${ledger}(name,sha256,state) VALUES (${literal(migration.name)},${literal(migration.sha256)},'applying')`);
67
+ for (const [index, migration] of pending.entries()) {
68
+ // Claim before execution. A claim left by an import that did not complete runs
69
+ // again only when its effects are provably absent; otherwise it stays refused.
70
+ if (claimed && index === 0) await assertRetryable(async sql => (await query(api,id,sql))[0].results, migration);
71
+ else await query(api,id,`INSERT INTO ${ledger}(name,sha256,state) VALUES (${literal(migration.name)},${literal(migration.sha256)},'applying')`);
69
72
  const last = sqlTokens(migration.sql).at(-1);
70
73
  // A second terminator after trailing comments creates an empty D1 statement.
71
74
  // Keep the newline even when terminated: a final line comment must end.
72
75
  const separator = last?.kind === 'symbol' && last.value === ';' ? '' : '; ';
73
- await api.importSql(id,`${migration.sql}\n${separator}UPDATE ${ledger} SET state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(migration.sha256)};`);
76
+ await api.importSql(id,`${migration.sql}\n${separator}UPDATE ${ledger} SET state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(migration.sha256)};`)
77
+ .catch(error => { error.migration ??= migration.name; throw error; });
74
78
  }
75
79
  if (pending.length) {
76
80
  const final = (await query(api,id,`SELECT name, sha256, state FROM ${ledger} ORDER BY id`))[0].results;
@@ -8,7 +8,7 @@ import { assert, hexId } from './validation.mjs';
8
8
  import { regularBytes } from './artifact.mjs';
9
9
  import { packageBuild } from './prepare.mjs';
10
10
  import { cloudflare } from './provider.mjs';
11
- import { ownerTarget, ownerPlan, ownerDeploy, ownerHealth, settleOwnerState, sameDeployment, localOnlyBinding, withStateLock, readOwnerState, writeOwnerState } from './owner.mjs';
11
+ import { ownerTarget, ownerWorkerName, ownerPlan, ownerDeploy, ownerHealth, settleOwnerState, sameDeployment, localOnlyBinding, withStateLock, readOwnerState, writeOwnerState } from './owner.mjs';
12
12
  import { activeWorker } from './worker.mjs';
13
13
  import { ownerSecret } from './owner-secret.mjs';
14
14
  import { ownerRemove } from './owner-remove.mjs';
@@ -55,8 +55,11 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
55
55
  const deadline = AbortSignal.timeout(Math.max(1, Math.ceil(remaining)));
56
56
  return request(input, {...init,signal:init.signal ? AbortSignal.any([init.signal,deadline]) : deadline});
57
57
  };
58
- const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,...(command === 'remove' ? {maxRequests:Infinity} : {})}),{onCreated});
59
- const target = await ownerTarget(api,{project:declaration.project,environment:values.environment});
58
+ const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,...(command === 'remove' ? {maxRequests:Infinity,removal:true} : {})}),{onCreated});
59
+ // Removal needs the recorded account/name identity, not a workers.dev route.
60
+ const target = command === 'remove'
61
+ ? {accountId:api.accountId,project:declaration.project,environment:values.environment,worker:ownerWorkerName(declaration.project,values.environment)}
62
+ : await ownerTarget(api,{project:declaration.project,environment:values.environment});
60
63
  const statePath = values.state ? resolve(root,values.state) : join(root,'.rer','deploy',`${target.environment}.json`);
61
64
  if (command === 'remove') return {ok:true,...await ownerRemove({api,target,statePath}).catch(error => { throw api.annotateFailure(error,'remove'); })};
62
65
  if (command === 'secret') {
@@ -42,78 +42,100 @@ export async function ownerRemove({api: provider, target, statePath, clock, slee
42
42
  const api = withSettledLists(withScopeHints(provider));
43
43
  return withStateLock(statePath, async () => {
44
44
  const state = await readOwnerState(statePath, target);
45
- if (await settlePendingSecret(api, target.worker, state, {preflight:true})) await writeOwnerState(statePath, state);
46
- const owned = KINDS.flatMap(kind => Object.entries(state.owned[kind]).map(([name, entry]) => ({kind, name, entry})));
47
- assert(state.owned.worker || owned.length, 'nothing-to-remove');
48
- // Preflight, read-only.
49
- let worker = null;
50
- if (await workerSettings(api, target.worker) !== null) {
51
- if (!state.owned.worker) throw refuse('resource-name-collision', 'worker', target.worker);
52
- const expected = state.pendingUpload?.provider ?? state.receipt?.provider;
53
- if (!expected) throw refuse('uncertain-resource-create', 'worker', target.worker);
54
- const live = await activeWorker(api, target.worker);
55
- if (!sameDeployment(live, expected)) throw refuse('owned-resource-replaced', 'worker', target.worker);
56
- worker = target.worker;
57
- }
58
- assert(!state.pendingSecret, 'secret-change-unsettled');
59
- const listed = {}, doomed = [];
60
- for (const kind of KINDS) {
61
- if (!owned.some(item => item.kind === kind)) continue;
62
- listed[kind] = await resourceKinds[kind].list(api);
63
- for (const {name, entry} of owned.filter(item => item.kind === kind)) {
64
- const row = findResource(listed[kind], kind, name);
65
- if (!row) continue;
66
- if (kind === 'r2') requireBucketIdentity(name, entry);
67
- if (entry.status === 'pending') await confirmEmpty(api, target.worker, kind, name, resourceId(kind, row));
68
- else checkResourceIdentity(kind, name, row, entry);
69
- if (kind === 'r2' && entry.status === 'created' && entry.id) await checkBucket(api, name, entry);
70
- doomed.push({kind, name, entry, id:resourceId(kind, row)});
45
+ const remaining = new Set([...(state.owned.worker ? [`worker:${target.worker}`] : []), ...KINDS.flatMap(kind => Object.keys(state.owned[kind]).map(name => `${kind}:${name}`))]);
46
+ const removed = [], failed = [];
47
+ try {
48
+ if (await settlePendingSecret(api, target.worker, state, {preflight:true})) await writeOwnerState(statePath, state);
49
+ const owned = KINDS.flatMap(kind => Object.entries(state.owned[kind]).map(([name, entry]) => ({kind, name, entry})));
50
+ assert(state.owned.worker || owned.length, 'nothing-to-remove');
51
+ // Preflight, read-only.
52
+ let worker = null;
53
+ if (await workerSettings(api, target.worker) !== null) {
54
+ if (!state.owned.worker) throw refuse('resource-name-collision', 'worker', target.worker);
55
+ const expected = state.pendingUpload?.provider ?? state.receipt?.provider;
56
+ if (!expected) throw refuse('uncertain-resource-create', 'worker', target.worker);
57
+ const live = await activeWorker(api, target.worker);
58
+ if (!sameDeployment(live, expected)) throw refuse('owned-resource-replaced', 'worker', target.worker);
59
+ worker = target.worker;
60
+ } else remaining.delete(`worker:${target.worker}`);
61
+ assert(!state.pendingSecret, 'secret-change-unsettled');
62
+ const listed = {}, doomed = [];
63
+ for (const kind of KINDS) {
64
+ if (!owned.some(item => item.kind === kind)) continue;
65
+ listed[kind] = await resourceKinds[kind].list(api);
66
+ for (const {name, entry} of owned.filter(item => item.kind === kind)) {
67
+ const row = findResource(listed[kind], kind, name);
68
+ if (!row) { remaining.delete(`${kind}:${name}`); continue; }
69
+ if (kind === 'r2') requireBucketIdentity(name, entry);
70
+ if (entry.status === 'pending') await confirmEmpty(api, target.worker, kind, name, resourceId(kind, row));
71
+ else checkResourceIdentity(kind, name, row, entry);
72
+ if (kind === 'r2' && entry.status === 'created' && entry.id) await checkBucket(api, name, entry);
73
+ doomed.push({kind, name, entry, id:resourceId(kind, row)});
74
+ }
71
75
  }
72
- }
73
- const removed = [];
74
- // A Worker that consumes one of these queues is released first.
75
- for (const queue of doomed.filter(item => item.kind === 'queues')) {
76
- for (const consumer of await api.list(`/queues/${queue.id}/consumers`, {pagination:'single'})) {
77
- assert(hexId(consumer.consumer_id), 'invalid-consumer-id');
78
- await api.request(`/queues/${queue.id}/consumers/${consumer.consumer_id}`, {method:'DELETE'});
76
+ // A Worker that consumes one of these queues is released first.
77
+ for (const queue of doomed.filter(item => item.kind === 'queues')) {
78
+ for (const consumer of await api.list(`/queues/${queue.id}/consumers`, {pagination:'single'})) {
79
+ assert(hexId(consumer.consumer_id), 'invalid-consumer-id');
80
+ await api.request(`/queues/${queue.id}/consumers/${consumer.consumer_id}`, {method:'DELETE'});
81
+ }
79
82
  }
80
- }
81
- if (worker) { await deleteWorker({api, worker, ...(clock ? {clock} : {}), ...(sleep ? {sleep} : {})}); removed.push(`worker:${worker}`); }
82
- const failed = [];
83
- for (const item of doomed) {
84
- try {
85
- if (item.kind === 'r2') await removeBucket(api, item);
86
- else {
87
- await remove[item.kind](api, item.id, item.name);
88
- if (item.kind === 'queues') await waitForQueueRemoval(api,item,{clock,sleep});
83
+ if (worker) { await deleteWorker({api, worker, ...(clock ? {clock} : {}), ...(sleep ? {sleep} : {})}); removed.push(`worker:${worker}`); }
84
+ for (const item of doomed) {
85
+ try {
86
+ if (item.kind === 'r2') await removeBucket(api, item);
87
+ else {
88
+ await remove[item.kind](api, item.id, item.name);
89
+ if (item.kind === 'queues') await waitForQueueRemoval(api,item,{clock,sleep});
90
+ }
91
+ removed.push(`${item.kind}:${item.name}`);
89
92
  }
90
- removed.push(`${item.kind}:${item.name}`);
93
+ catch (error) { failed.push({resource:`${item.kind}:${item.name}`, code:error.message, error}); }
91
94
  }
92
- catch (error) { failed.push({resource:`${item.kind}:${item.name}`, code:error.message, error}); }
93
- }
94
- // Read back: nothing this environment created is still listed.
95
- const remaining = [];
96
- if (await workerSettings(api, target.worker) !== null) remaining.push(`worker:${target.worker}`);
97
- const namespaces = await api.list('/workers/durable_objects/namespaces');
98
- for (const row of namespaces) if (row.script === target.worker) remaining.push(`durable-object:${row.class}`);
99
- for (const kind of KINDS) {
100
- if (!listed[kind]) continue;
101
- const rows = await resourceKinds[kind].list(api);
102
- for (const {name} of owned.filter(item => item.kind === kind)) if (findResource(rows, kind, name)) remaining.push(`${kind}:${name}`);
103
- }
104
- if (remaining.length) {
105
- // Keep the record of what is left, so a later --remove finishes it.
106
- for (const kind of KINDS) for (const name of Object.keys(state.owned[kind])) if (!remaining.includes(`${kind}:${name}`)) delete state.owned[kind][name];
107
- if (!remaining.includes(`worker:${target.worker}`)) {
108
- state.owned.worker = null; state.receipt = null; delete state.pendingUpload;
109
- delete state.ownerClaim; delete state.retiredSecrets;
95
+ // Only complete, successful reads prove absence. Keep every unverified
96
+ // resource recorded even when its DELETE returned success.
97
+ let readFailure;
98
+ const readback = async read => { try { await read(); } catch (error) { readFailure ??= error; } };
99
+ await readback(async () => {
100
+ if (await workerSettings(api, target.worker) !== null) remaining.add(`worker:${target.worker}`);
101
+ else remaining.delete(`worker:${target.worker}`);
102
+ });
103
+ let namespacesKnown = false;
104
+ await readback(async () => {
105
+ const namespaces = await api.list('/workers/durable_objects/namespaces');
106
+ for (const row of namespaces) if (row.script === target.worker) remaining.add(`durable-object:${row.class}`);
107
+ namespacesKnown = true;
108
+ });
109
+ // Missing namespace readback cannot establish that the Worker's data is gone.
110
+ if (!namespacesKnown && state.owned.worker) remaining.add(`worker:${target.worker}`);
111
+ for (const kind of KINDS) {
112
+ if (!listed[kind]) continue;
113
+ await readback(async () => {
114
+ const rows = await resourceKinds[kind].list(api);
115
+ for (const {name} of owned.filter(item => item.kind === kind)) {
116
+ if (findResource(rows, kind, name)) remaining.add(`${kind}:${name}`);
117
+ else remaining.delete(`${kind}:${name}`);
118
+ }
119
+ });
120
+ }
121
+ if (remaining.size || readFailure) {
122
+ // Keep proof for every survivor or resource whose absence is unverified.
123
+ for (const kind of KINDS) for (const name of Object.keys(state.owned[kind])) if (!remaining.has(`${kind}:${name}`)) delete state.owned[kind][name];
124
+ if (!remaining.has(`worker:${target.worker}`) && ![...remaining].some(name => name.startsWith('durable-object:'))) {
125
+ state.owned.worker = null; state.receipt = null; delete state.pendingUpload;
126
+ delete state.ownerClaim; delete state.retiredSecrets;
127
+ }
128
+ await writeOwnerState(statePath, state);
129
+ throw readFailure ?? new Error('remove-incomplete');
110
130
  }
111
- await writeOwnerState(statePath, state);
112
- const error = new Error('remove-incomplete'); error.remaining = remaining; error.failed = failed; error.removed = removed.filter(resource => !remaining.includes(resource));
131
+ await rm(statePath, {force:true});
132
+ return {worker:target.worker, removed, remaining:[]};
133
+ } catch (error) {
134
+ error.remaining = [...remaining];
135
+ error.failed = failed;
136
+ error.removed = removed.filter(resource => !remaining.has(resource));
113
137
  throw error;
114
138
  }
115
- await rm(statePath, {force:true});
116
- return {worker:target.worker, removed, remaining:[]};
117
139
  });
118
140
  }
119
141
 
@@ -1,3 +1,5 @@
1
+ import { setTimeout as pause } from 'node:timers/promises';
2
+
1
3
  // A missing HTTP status does not prove that a write never reached Cloudflare.
2
4
  // Only these connection-establishment failures make replaying a write safe.
3
5
  const UNSENT = new Set(['ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', 'UND_ERR_CONNECT_TIMEOUT']);
@@ -9,11 +11,13 @@ function unsent(error, seen = new Set()) {
9
11
  return unsent(error.cause, seen);
10
12
  }
11
13
 
12
- /** Retry only before a response, once, sharing the original deadline and budget. */
13
- export async function providerResponse({send, method, timeoutMs, callerSignal, reserveRetry, failure}) {
14
+ /** Scoped safe replays share the original deadline and request budget. */
15
+ export async function providerResponse({send, method, timeoutMs, callerSignal, reserveRetry, failure, removal = false, replayableAsset = false, consume}) {
14
16
  const timeout = AbortSignal.timeout(timeoutMs);
15
17
  const signal = callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout;
16
18
  const read = method === 'GET' || method === 'HEAD';
19
+ const removalReplay = removal && ['GET','DELETE'].includes(method);
20
+ const maxAttempts = removalReplay ? 3 : 2;
17
21
  for (let attempt = 1; ; attempt++) {
18
22
  // Leave time for a second read when the first one times out. Caller deadlines
19
23
  // remain authoritative and are never reset by a retry.
@@ -21,12 +25,31 @@ export async function providerResponse({send, method, timeoutMs, callerSignal, r
21
25
  const timer = firstRead ? setTimeout(() => firstRead.abort(new DOMException('Provider read timed out','TimeoutError')), Math.max(1, Math.ceil(timeoutMs / 2))) : undefined;
22
26
  timer?.unref();
23
27
  const attemptSignal = firstRead ? AbortSignal.any([signal, firstRead.signal]) : signal;
24
- try { return {response:await send(attemptSignal),attempts:attempt}; }
28
+ let readingBody = false;
29
+ try {
30
+ const response = await send(attemptSignal);
31
+ clearTimeout(timer);
32
+ // A retried delete can observe the first attempt's committed deletion.
33
+ if (removalReplay && method === 'DELETE' && attempt > 1 && response.status === 404) {
34
+ void response.body?.cancel().catch(() => {});
35
+ return {response:new Response(null,{status:204}),attempts:attempt};
36
+ }
37
+ if ((removalReplay || replayableAsset) && response.status >= 500 && response.status <= 599 && attempt < maxAttempts && !signal.aborted && reserveRetry()) {
38
+ void response.body?.cancel().catch(() => {});
39
+ await pause(250 * 2 ** (attempt - 1),undefined,{signal});
40
+ continue;
41
+ }
42
+ readingBody = true;
43
+ const output = consume ? await consume(response,attempt) : undefined;
44
+ return {response,attempts:attempt,output};
45
+ }
25
46
  catch (cause) {
26
- if (attempt === 1 && !signal.aborted && (read || unsent(cause)) && reserveRetry()) continue;
47
+ const bodyReset = readingBody && replayableAsset && cause.message === 'provider-transport-failed';
48
+ if (attempt === 1 && !signal.aborted && (bodyReset || !readingBody && (read || replayableAsset || unsent(cause))) && reserveRetry()) continue;
49
+ if (readingBody) throw cause;
27
50
  throw failure(cause, attempt);
28
51
  }
29
- // Headers end the retry window; body consumption keeps the total deadline.
52
+ // Body consumption and scoped asset replays keep the total deadline.
30
53
  finally { clearTimeout(timer); }
31
54
  }
32
55
  }