@runeditrun/rer 0.2.23 → 0.2.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +16 -7
  2. package/cli/browser-runtime.mjs +65 -0
  3. package/cli/demo-host.mjs +1 -1
  4. package/cli/deploy-claim-request.mjs +5 -4
  5. package/cli/deploy-command.mjs +7 -5
  6. package/cli/deploy-inputs.mjs +2 -1
  7. package/cli/deploy-observation.mjs +20 -2
  8. package/cli/deploy-owner.mjs +12 -7
  9. package/cli/deploy.mjs +19 -6
  10. package/cli/export.mjs +1 -1
  11. package/cli/hydration/browser.mjs +3 -9
  12. package/cli/inspect.mjs +3 -1
  13. package/cli/lifecycle.mjs +4 -1
  14. package/cli/site.mjs +2 -2
  15. package/global/CHANGELOG.md +37 -0
  16. package/global/README.md +16 -7
  17. package/global/browser-runtime.mjs +45 -0
  18. package/global/cli/layout.mjs +8 -4
  19. package/global/main.mjs +13 -3
  20. package/install.sh +56 -3
  21. package/modules/container/README.md +6 -0
  22. package/modules/container/adapters/d1.mjs +1 -1
  23. package/modules/container/build.mjs +2 -1
  24. package/modules/container/image-variables.mjs +27 -0
  25. package/modules/container/transfer.mjs +0 -1
  26. package/modules/container/worker-engine.mjs +1 -1
  27. package/modules/delivery-producer/declarations.mjs +6 -1
  28. package/modules/delivery-producer/mail.mjs +66 -0
  29. package/modules/demo/label.mjs +5 -2
  30. package/modules/demo-host/README.md +29 -7
  31. package/modules/demo-host/container.mjs +19 -7
  32. package/modules/demo-host/placement.mjs +19 -5
  33. package/modules/demo-host/plan.mjs +11 -6
  34. package/modules/demo-host/start-script.mjs +2 -1
  35. package/modules/demo-host/startup.mjs +3 -3
  36. package/modules/demo-host/worker.mjs +27 -8
  37. package/modules/deploy-kit/README.md +41 -6
  38. package/modules/deploy-kit/asset-health.mjs +28 -0
  39. package/modules/deploy-kit/d1-import.mjs +22 -4
  40. package/modules/deploy-kit/diagnostics.mjs +46 -1
  41. package/modules/deploy-kit/health.mjs +2 -23
  42. package/modules/deploy-kit/migration-ledger.mjs +6 -2
  43. package/modules/deploy-kit/migration-retry.mjs +91 -0
  44. package/modules/deploy-kit/migrations.mjs +9 -5
  45. package/modules/deploy-kit/owner-cli.mjs +9 -6
  46. package/modules/deploy-kit/owner-remove.mjs +104 -64
  47. package/modules/deploy-kit/pagination.mjs +99 -0
  48. package/modules/deploy-kit/provider-transport.mjs +28 -5
  49. package/modules/deploy-kit/provider.mjs +38 -75
  50. package/modules/deploy-kit/r2-objects.mjs +6 -17
  51. package/modules/deploy-kit/site-readback.mjs +67 -0
  52. package/modules/deploy-kit/site.mjs +17 -31
  53. package/modules/deploy-kit/uploaded-worker.mjs +9 -3
  54. package/modules/deploy-kit/worker.mjs +5 -2
  55. package/modules/erasure/queue.mjs +0 -2
  56. package/modules/job-queue/cli.mjs +1 -1
  57. package/modules/operator-auth/README.md +10 -2
  58. package/modules/operator-auth/config.mjs +1 -0
  59. package/modules/operator-auth/conformance/presentation.mjs +2 -2
  60. package/modules/operator-auth/identity/email-link/confirmation.mjs +2 -1
  61. package/modules/operator-auth/identity/email-link/expiry.mjs +8 -0
  62. package/modules/operator-auth/identity/email-link/index.mjs +2 -1
  63. package/modules/operator-auth/identity/email-link/message.mjs +3 -2
  64. package/modules/operator-auth/identity/oidc/jwt.mjs +1 -1
  65. package/modules/operator-auth/index.d.mts +2 -0
  66. package/modules/portability/README.md +7 -2
  67. package/modules/portability/bridge.mjs +3 -3
  68. package/modules/portability/catalogue.mjs +1 -1
  69. package/modules/portability/cli-options.mjs +1 -1
  70. package/modules/portability/import-staging.mjs +45 -0
  71. package/modules/portability/import.mjs +8 -8
  72. package/modules/portability/legacy-files.mjs +1 -1
  73. package/modules/portability/legacy-safety.mjs +3 -3
  74. package/modules/portability/payloads.mjs +1 -1
  75. package/modules/portability/storage.mjs +5 -2
  76. package/modules/release/source.mjs +1 -1
  77. package/package.json +1 -1
@@ -1,10 +1,9 @@
1
1
  import { r2ObjectPages, r2ObjectPath } from './r2-objects.mjs';
2
2
  import { rm } from 'node:fs/promises';
3
- import { assert } from './validation.mjs';
3
+ import { assert, hexId } from './validation.mjs';
4
4
  import { resourceKinds, findResource } from './resources.mjs';
5
5
  import { activeWorker } from './worker.mjs';
6
6
  import { deleteWorker } from './teardown.mjs';
7
- import { workerConsumerScript } from './queue-consumer.mjs';
8
7
  import { query } from './migrations.mjs';
9
8
  import { KINDS, readOwnerState, writeOwnerState, withStateLock, withScopeHints, withSettledLists, workerSettings, sameDeployment, resourceId, checkResourceIdentity, refuse, settlePendingSecret } from './owner.mjs';
10
9
 
@@ -43,74 +42,100 @@ export async function ownerRemove({api: provider, target, statePath, clock, slee
43
42
  const api = withSettledLists(withScopeHints(provider));
44
43
  return withStateLock(statePath, async () => {
45
44
  const state = await readOwnerState(statePath, target);
46
- if (await settlePendingSecret(api, target.worker, state, {preflight:true})) await writeOwnerState(statePath, state);
47
- const owned = KINDS.flatMap(kind => Object.entries(state.owned[kind]).map(([name, entry]) => ({kind, name, entry})));
48
- assert(state.owned.worker || owned.length, 'nothing-to-remove');
49
- // Preflight, read-only.
50
- let worker = null;
51
- if (await workerSettings(api, target.worker) !== null) {
52
- if (!state.owned.worker) throw refuse('resource-name-collision', 'worker', target.worker);
53
- const expected = state.pendingUpload?.provider ?? state.receipt?.provider;
54
- if (!expected) throw refuse('uncertain-resource-create', 'worker', target.worker);
55
- const live = await activeWorker(api, target.worker);
56
- if (!sameDeployment(live, expected)) throw refuse('owned-resource-replaced', 'worker', target.worker);
57
- worker = target.worker;
58
- }
59
- assert(!state.pendingSecret, 'secret-change-unsettled');
60
- const listed = {}, doomed = [];
61
- for (const kind of KINDS) {
62
- if (!owned.some(item => item.kind === kind)) continue;
63
- listed[kind] = await resourceKinds[kind].list(api);
64
- for (const {name, entry} of owned.filter(item => item.kind === kind)) {
65
- const row = findResource(listed[kind], kind, name);
66
- if (!row) continue;
67
- if (kind === 'r2') requireBucketIdentity(name, entry);
68
- if (entry.status === 'pending') await confirmEmpty(api, target.worker, kind, name, resourceId(kind, row));
69
- else checkResourceIdentity(kind, name, row, entry);
70
- if (kind === 'r2' && entry.status === 'created' && entry.id) await checkBucket(api, name, entry);
71
- doomed.push({kind, name, entry, id:resourceId(kind, row)});
45
+ const remaining = new Set([...(state.owned.worker ? [`worker:${target.worker}`] : []), ...KINDS.flatMap(kind => Object.keys(state.owned[kind]).map(name => `${kind}:${name}`))]);
46
+ const removed = [], failed = [];
47
+ try {
48
+ if (await settlePendingSecret(api, target.worker, state, {preflight:true})) await writeOwnerState(statePath, state);
49
+ const owned = KINDS.flatMap(kind => Object.entries(state.owned[kind]).map(([name, entry]) => ({kind, name, entry})));
50
+ assert(state.owned.worker || owned.length, 'nothing-to-remove');
51
+ // Preflight, read-only.
52
+ let worker = null;
53
+ if (await workerSettings(api, target.worker) !== null) {
54
+ if (!state.owned.worker) throw refuse('resource-name-collision', 'worker', target.worker);
55
+ const expected = state.pendingUpload?.provider ?? state.receipt?.provider;
56
+ if (!expected) throw refuse('uncertain-resource-create', 'worker', target.worker);
57
+ const live = await activeWorker(api, target.worker);
58
+ if (!sameDeployment(live, expected)) throw refuse('owned-resource-replaced', 'worker', target.worker);
59
+ worker = target.worker;
60
+ } else remaining.delete(`worker:${target.worker}`);
61
+ assert(!state.pendingSecret, 'secret-change-unsettled');
62
+ const listed = {}, doomed = [];
63
+ for (const kind of KINDS) {
64
+ if (!owned.some(item => item.kind === kind)) continue;
65
+ listed[kind] = await resourceKinds[kind].list(api);
66
+ for (const {name, entry} of owned.filter(item => item.kind === kind)) {
67
+ const row = findResource(listed[kind], kind, name);
68
+ if (!row) { remaining.delete(`${kind}:${name}`); continue; }
69
+ if (kind === 'r2') requireBucketIdentity(name, entry);
70
+ if (entry.status === 'pending') await confirmEmpty(api, target.worker, kind, name, resourceId(kind, row));
71
+ else checkResourceIdentity(kind, name, row, entry);
72
+ if (kind === 'r2' && entry.status === 'created' && entry.id) await checkBucket(api, name, entry);
73
+ doomed.push({kind, name, entry, id:resourceId(kind, row)});
74
+ }
72
75
  }
73
- }
74
- const removed = [];
75
- // A Worker that consumes one of these queues is released first.
76
- for (const queue of doomed.filter(item => item.kind === 'queues')) {
77
- for (const consumer of await api.list(`/queues/${queue.id}/consumers`, {pagination:'single'})) {
78
- if (workerConsumerScript(consumer) === target.worker) await api.request(`/queues/${queue.id}/consumers/${consumer.consumer_id}`, {method:'DELETE'});
76
+ // A Worker that consumes one of these queues is released first.
77
+ for (const queue of doomed.filter(item => item.kind === 'queues')) {
78
+ for (const consumer of await api.list(`/queues/${queue.id}/consumers`, {pagination:'single'})) {
79
+ assert(hexId(consumer.consumer_id), 'invalid-consumer-id');
80
+ await api.request(`/queues/${queue.id}/consumers/${consumer.consumer_id}`, {method:'DELETE'});
81
+ }
79
82
  }
80
- }
81
- if (worker) { await deleteWorker({api, worker, ...(clock ? {clock} : {}), ...(sleep ? {sleep} : {})}); removed.push(`worker:${worker}`); }
82
- const failed = [];
83
- for (const item of doomed) {
84
- try {
85
- if (item.kind === 'r2') await removeBucket(api, item);
86
- else await remove[item.kind](api, item.id, item.name);
87
- removed.push(`${item.kind}:${item.name}`);
83
+ if (worker) { await deleteWorker({api, worker, ...(clock ? {clock} : {}), ...(sleep ? {sleep} : {})}); removed.push(`worker:${worker}`); }
84
+ for (const item of doomed) {
85
+ try {
86
+ if (item.kind === 'r2') await removeBucket(api, item);
87
+ else {
88
+ await remove[item.kind](api, item.id, item.name);
89
+ if (item.kind === 'queues') await waitForQueueRemoval(api,item,{clock,sleep});
90
+ }
91
+ removed.push(`${item.kind}:${item.name}`);
92
+ }
93
+ catch (error) { failed.push({resource:`${item.kind}:${item.name}`, code:error.message, error}); }
88
94
  }
89
- catch (error) { failed.push({resource:`${item.kind}:${item.name}`, code:error.message}); }
90
- }
91
- // Read back: nothing this environment created is still listed.
92
- const remaining = [];
93
- if (await workerSettings(api, target.worker) !== null) remaining.push(`worker:${target.worker}`);
94
- const namespaces = await api.list('/workers/durable_objects/namespaces');
95
- for (const row of namespaces) if (row.script === target.worker) remaining.push(`durable-object:${row.class}`);
96
- for (const kind of KINDS) {
97
- if (!listed[kind]) continue;
98
- const rows = await resourceKinds[kind].list(api);
99
- for (const {name} of owned.filter(item => item.kind === kind)) if (findResource(rows, kind, name)) remaining.push(`${kind}:${name}`);
100
- }
101
- if (remaining.length) {
102
- // Keep the record of what is left, so a later --remove finishes it.
103
- for (const kind of KINDS) for (const name of Object.keys(state.owned[kind])) if (!remaining.includes(`${kind}:${name}`)) delete state.owned[kind][name];
104
- if (!remaining.includes(`worker:${target.worker}`)) {
105
- state.owned.worker = null; state.receipt = null; delete state.pendingUpload;
106
- delete state.ownerClaim; delete state.retiredSecrets;
95
+ // Only complete, successful reads prove absence. Keep every unverified
96
+ // resource recorded even when its DELETE returned success.
97
+ let readFailure;
98
+ const readback = async read => { try { await read(); } catch (error) { readFailure ??= error; } };
99
+ await readback(async () => {
100
+ if (await workerSettings(api, target.worker) !== null) remaining.add(`worker:${target.worker}`);
101
+ else remaining.delete(`worker:${target.worker}`);
102
+ });
103
+ let namespacesKnown = false;
104
+ await readback(async () => {
105
+ const namespaces = await api.list('/workers/durable_objects/namespaces');
106
+ for (const row of namespaces) if (row.script === target.worker) remaining.add(`durable-object:${row.class}`);
107
+ namespacesKnown = true;
108
+ });
109
+ // Missing namespace readback cannot establish that the Worker's data is gone.
110
+ if (!namespacesKnown && state.owned.worker) remaining.add(`worker:${target.worker}`);
111
+ for (const kind of KINDS) {
112
+ if (!listed[kind]) continue;
113
+ await readback(async () => {
114
+ const rows = await resourceKinds[kind].list(api);
115
+ for (const {name} of owned.filter(item => item.kind === kind)) {
116
+ if (findResource(rows, kind, name)) remaining.add(`${kind}:${name}`);
117
+ else remaining.delete(`${kind}:${name}`);
118
+ }
119
+ });
120
+ }
121
+ if (remaining.size || readFailure) {
122
+ // Keep proof for every survivor or resource whose absence is unverified.
123
+ for (const kind of KINDS) for (const name of Object.keys(state.owned[kind])) if (!remaining.has(`${kind}:${name}`)) delete state.owned[kind][name];
124
+ if (!remaining.has(`worker:${target.worker}`) && ![...remaining].some(name => name.startsWith('durable-object:'))) {
125
+ state.owned.worker = null; state.receipt = null; delete state.pendingUpload;
126
+ delete state.ownerClaim; delete state.retiredSecrets;
127
+ }
128
+ await writeOwnerState(statePath, state);
129
+ throw readFailure ?? new Error('remove-incomplete');
107
130
  }
108
- await writeOwnerState(statePath, state);
109
- const error = new Error('remove-incomplete'); error.remaining = remaining; error.failed = failed; error.removed = removed.filter(resource => !remaining.includes(resource));
131
+ await rm(statePath, {force:true});
132
+ return {worker:target.worker, removed, remaining:[]};
133
+ } catch (error) {
134
+ error.remaining = [...remaining];
135
+ error.failed = failed;
136
+ error.removed = removed.filter(resource => !remaining.has(resource));
110
137
  throw error;
111
138
  }
112
- await rm(statePath, {force:true});
113
- return {worker:target.worker, removed, remaining:[]};
114
139
  });
115
140
  }
116
141
 
@@ -137,3 +162,18 @@ async function removeBucket(api, {name, entry}) {
137
162
  await checkBucket(api, name, entry);
138
163
  await api.request(`/r2/buckets/${name}`, {method:'DELETE'});
139
164
  }
165
+
166
+ // Queue deletion and its account listing can settle separately. Finish the
167
+ // readback within this removal invocation, without replaying the DELETE.
168
+ async function waitForQueueRemoval(api, item, {clock = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve,ms))}) {
169
+ const start = clock();
170
+ for (;;) {
171
+ if (!(await resourceKinds.queues.list(api)).some(row => row.queue_id === item.id)) return;
172
+ if (clock() - start >= 60000) {
173
+ const error = new Error('queue-removal-not-confirmed');
174
+ error.providerCall = {operation:'GET /queues',reason:'Cloudflare still lists the deleted queue after 60 seconds'};
175
+ throw error;
176
+ }
177
+ await sleep(500);
178
+ }
179
+ }
@@ -0,0 +1,99 @@
1
+ import { assert } from './validation.mjs';
2
+
3
+ // Request strategies describe the wire parameters only. Every endpoint uses
4
+ // this reader to decide whether a reply actually has another page.
5
+ export async function* providerPages(envelope, path, {field, pagination = 'page', zone = false, signal} = {}) {
6
+ assert(['page','cursor','r2-bounded','single'].includes(pagination), 'invalid-list-pagination');
7
+ assert(pagination !== 'r2-bounded' || path === '/r2/buckets' && field === 'buckets', 'invalid-r2-bounded-list');
8
+ const cursorMode = pagination === 'cursor' || pagination === 'r2-bounded';
9
+ const url = new URL(path, 'https://api.cloudflare.com');
10
+ const perPage = Number(url.searchParams.get('per_page') ?? url.searchParams.get('limit') ?? (cursorMode ? 1000 : pagination === 'single' ? 10000 : 100));
11
+ assert(Number.isSafeInteger(perPage) && perPage > 0, 'invalid-list-pagination');
12
+ let cursor, total, count = 0, pageMode = false;
13
+ const seen = new Set();
14
+ for (let page = 1; page <= 100; page++) {
15
+ const query = new URLSearchParams(url.searchParams);
16
+ if (cursor) {
17
+ query.delete('page');
18
+ if (pagination !== 'single' && !query.has('limit')) query.set('per_page',String(perPage));
19
+ query.set('cursor',cursor);
20
+ }
21
+ else if ((pageMode || !cursorMode) && (pagination !== 'single' || page > 1)) query.set('page',String(page));
22
+ if (pagination !== 'single' && !query.has('limit')) query.set('per_page',String(perPage));
23
+ const requestPath = url.pathname + (query.size ? `?${query}` : '');
24
+ let data, rows, next;
25
+ try {
26
+ data = await envelope(requestPath, {zone,signal});
27
+ rows = field ? data.result?.[field] : data.result;
28
+ assert(Array.isArray(rows), 'provider-invalid-list');
29
+ assert(rows.length <= perPage, 'provider-partial-list');
30
+ count += rows.length;
31
+ const info = data.result_info;
32
+ if (info == null) {
33
+ assert(rows.length < perPage, 'provider-incomplete-pagination');
34
+ assert(total === undefined || count === total, 'provider-partial-list');
35
+ } else {
36
+ assert(typeof info === 'object' && !Array.isArray(info), 'provider-incomplete-pagination');
37
+ if (info.per_page !== undefined) {
38
+ assert(Number.isSafeInteger(info.per_page) && info.per_page > 0 && info.per_page <= perPage, 'provider-incomplete-pagination');
39
+ assert(rows.length <= info.per_page, 'provider-partial-list');
40
+ }
41
+ if (info.count !== undefined) assert(info.count === rows.length, 'provider-partial-list');
42
+ if (info.page !== undefined) assert(info.page === page, 'provider-incomplete-pagination');
43
+ if (info.total_count !== undefined) {
44
+ assert(Number.isSafeInteger(info.total_count) && info.total_count >= 0, 'provider-incomplete-pagination');
45
+ total ??= info.total_count;
46
+ assert(total === info.total_count, 'provider-pagination-changed');
47
+ assert(count <= total, 'provider-partial-list');
48
+ }
49
+ if (info.total_pages !== undefined) assert(Number.isSafeInteger(info.total_pages) && info.total_pages >= 0 && (info.total_pages > 0 || count === 0), 'provider-incomplete-pagination');
50
+ if (info.is_truncated !== undefined) assert(typeof info.is_truncated === 'boolean', 'provider-incomplete-pagination');
51
+ assert(!info.delimited || Array.isArray(info.delimited) && !info.delimited.length, 'provider-incomplete-pagination');
52
+ assert(info.cursor == null || typeof info.cursor === 'string', 'provider-invalid-cursor');
53
+ const morePages = info.total_pages !== undefined && page < info.total_pages;
54
+ const moreRows = total !== undefined && count < total;
55
+ if (info.cursor) {
56
+ assert(info.is_truncated !== false, 'provider-invalid-cursor');
57
+ assert((info.total_pages === undefined || morePages) && (total === undefined || moreRows), 'provider-partial-list');
58
+ assert(info.cursor.length <= 4096 && /^[a-zA-Z0-9_+\/=-]+$/.test(info.cursor) && !seen.has(info.cursor), 'provider-invalid-cursor');
59
+ seen.add(info.cursor); next = info.cursor;
60
+ } else if (info.is_truncated === true) {
61
+ throw new Error('provider-incomplete-pagination');
62
+ } else if (cursorMode && moreRows && !morePages) {
63
+ throw new Error('provider-partial-list');
64
+ } else if (morePages || moreRows) {
65
+ assert(info.total_pages === undefined || morePages, 'provider-partial-list');
66
+ next = true;
67
+ } else {
68
+ assert(total === undefined || count === total, 'provider-partial-list');
69
+ }
70
+ if (next) assert(rows.length > 0 || morePages || typeof next === 'string', 'provider-partial-list');
71
+ }
72
+ } catch (error) {
73
+ // No response bodies, query values, account IDs or cursors in diagnostics.
74
+ error.providerCall ??= {operation:`GET ${url.pathname}`,reason:error.message};
75
+ throw error;
76
+ }
77
+ yield {rows,data};
78
+ if (!next) return;
79
+ cursor = typeof next === 'string' ? next : undefined;
80
+ pageMode = next === true;
81
+ }
82
+ const error = new Error('provider-pagination-limit');
83
+ error.providerCall = {operation:`GET ${url.pathname}`,reason:'Cloudflare continued advertising pages beyond the request limit'};
84
+ throw error;
85
+ }
86
+
87
+ // Keep result wrappers (versions/items, schedules, deployments) for existing kit
88
+ // callers while ensuring their GETs cannot bypass pagination. Query filters stay
89
+ // on every request; writes never pass through the reader.
90
+ export function providerListShape(path) {
91
+ const base = path.split('?')[0];
92
+ if (base === '/r2/buckets') return {field:'buckets',pagination:'cursor'};
93
+ if (/^\/r2\/buckets\/[^/]+\/objects$/.test(base) || /^\/storage\/kv\/namespaces\/[^/]+\/keys$/.test(base)) return {pagination:'cursor'};
94
+ const field = /\/workers\/scripts\/[^/]+\/(versions|deployments|schedules)$/.exec(base)?.[1];
95
+ if (field) return {field:field === 'versions' ? 'items' : field,...(field === 'schedules' ? {pagination:'single'} : {})};
96
+ if (['/d1/database','/storage/kv/namespaces','/queues','/workers/durable_objects/namespaces','/workers/domains','/zones','/containers/applications'].includes(base)) return {};
97
+ if (['/workers/scripts','/d1/database','/storage/kv/namespaces','/queues','/workers/durable_objects/namespaces','/workers/domains','/zones'].includes(base) || /^\/queues\/[^/]+\/consumers$/.test(base) || /^\/workers\/scripts\/[^/]+\/secrets$/.test(base) || /^\/zones\/[^/]+\/workers\/routes$/.test(base)) return {pagination:'single'};
98
+ return undefined;
99
+ }
@@ -1,3 +1,5 @@
1
+ import { setTimeout as pause } from 'node:timers/promises';
2
+
1
3
  // A missing HTTP status does not prove that a write never reached Cloudflare.
2
4
  // Only these connection-establishment failures make replaying a write safe.
3
5
  const UNSENT = new Set(['ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', 'UND_ERR_CONNECT_TIMEOUT']);
@@ -9,11 +11,13 @@ function unsent(error, seen = new Set()) {
9
11
  return unsent(error.cause, seen);
10
12
  }
11
13
 
12
- /** Retry only before a response, once, sharing the original deadline and budget. */
13
- export async function providerResponse({send, method, timeoutMs, callerSignal, reserveRetry, failure}) {
14
+ /** Scoped safe replays share the original deadline and request budget. */
15
+ export async function providerResponse({send, method, timeoutMs, callerSignal, reserveRetry, failure, removal = false, replayableAsset = false, consume}) {
14
16
  const timeout = AbortSignal.timeout(timeoutMs);
15
17
  const signal = callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout;
16
18
  const read = method === 'GET' || method === 'HEAD';
19
+ const removalReplay = removal && ['GET','DELETE'].includes(method);
20
+ const maxAttempts = removalReplay ? 3 : 2;
17
21
  for (let attempt = 1; ; attempt++) {
18
22
  // Leave time for a second read when the first one times out. Caller deadlines
19
23
  // remain authoritative and are never reset by a retry.
@@ -21,12 +25,31 @@ export async function providerResponse({send, method, timeoutMs, callerSignal, r
21
25
  const timer = firstRead ? setTimeout(() => firstRead.abort(new DOMException('Provider read timed out','TimeoutError')), Math.max(1, Math.ceil(timeoutMs / 2))) : undefined;
22
26
  timer?.unref();
23
27
  const attemptSignal = firstRead ? AbortSignal.any([signal, firstRead.signal]) : signal;
24
- try { return {response:await send(attemptSignal),attempts:attempt}; }
28
+ let readingBody = false;
29
+ try {
30
+ const response = await send(attemptSignal);
31
+ clearTimeout(timer);
32
+ // A retried delete can observe the first attempt's committed deletion.
33
+ if (removalReplay && method === 'DELETE' && attempt > 1 && response.status === 404) {
34
+ void response.body?.cancel().catch(() => {});
35
+ return {response:new Response(null,{status:204}),attempts:attempt};
36
+ }
37
+ if ((removalReplay || replayableAsset) && response.status >= 500 && response.status <= 599 && attempt < maxAttempts && !signal.aborted && reserveRetry()) {
38
+ void response.body?.cancel().catch(() => {});
39
+ await pause(250 * 2 ** (attempt - 1),undefined,{signal});
40
+ continue;
41
+ }
42
+ readingBody = true;
43
+ const output = consume ? await consume(response,attempt) : undefined;
44
+ return {response,attempts:attempt,output};
45
+ }
25
46
  catch (cause) {
26
- if (attempt === 1 && !signal.aborted && (read || unsent(cause)) && reserveRetry()) continue;
47
+ const bodyReset = readingBody && replayableAsset && cause.message === 'provider-transport-failed';
48
+ if (attempt === 1 && !signal.aborted && (bodyReset || !readingBody && (read || replayableAsset || unsent(cause))) && reserveRetry()) continue;
49
+ if (readingBody) throw cause;
27
50
  throw failure(cause, attempt);
28
51
  }
29
- // Headers end the retry window; body consumption keeps the total deadline.
52
+ // Body consumption and scoped asset replays keep the total deadline.
30
53
  finally { clearTimeout(timer); }
31
54
  }
32
55
  }
@@ -1,3 +1,4 @@
1
+ import { providerPages, providerListShape } from './pagination.mjs';
1
2
  import { deleteR2Object } from './r2-object-transport.mjs';
2
3
  import { isR2ObjectDelete } from './r2-objects.mjs';
3
4
  import { recordDeployFailure } from './diagnostics.mjs';
@@ -24,21 +25,23 @@ async function boundedText(response, limit = 4 * 1024 * 1024, onReadError) {
24
25
  export async function boundedJSON(response, limit) {
25
26
  try { return JSON.parse(await boundedText(response,limit)); } catch (error) { if (error instanceof SyntaxError) throw new Error('malformed-response'); throw error; }
26
27
  }
28
+ const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
27
29
  /** Only this fixed origin and the explicitly selected account ever receive the token. */
28
- export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500 }) {
30
+ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false }) {
29
31
  assert(hexId(accountId), 'invalid-cloudflare-account-id');
30
32
  assert(typeof token === 'string' && token.trim() && !/[\r\n]/.test(token), 'invalid-cloudflare-api-token');
31
33
  let requests = 0;
32
34
  // `zone` scope: the zone-level API (/zones/...), for a run site's zone lookup and redirect rule.
33
- async function envelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal } = {}) {
35
+ async function rawEnvelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal } = {}) {
34
36
  assert(typeof path === 'string' && (isR2ObjectDelete(path, method) || /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%')), 'invalid-provider-path');
35
37
  assert(!zone || /^\/zones(?:[/?]|$)/.test(path), 'invalid-provider-path');
36
38
  assert(++requests <= maxRequests, 'provider-request-budget');
37
39
  assert(!assetToken || path === '/workers/assets/upload?base64=true', 'invalid-asset-auth-target');
38
40
  // Asset-session credentials are discovered at runtime, outside the CLI env.
39
41
  // Preserve complete text for the CLI's remaining redaction before truncation.
40
- const redactCredentials = text => [accountId,token,assetToken].filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
41
- const redactAuth = text => /\/r2\/buckets\/[^/]+\/objects(?:[/?]|$)/.test(path) ? '[R2 object details omitted]' : /\/d1\/database\/[^/]+\/import$/.test(path) ? '[D1 import details omitted]' : redactCredentials(text);
42
+ const redactCredentials = text => redactValues(text,[accountId,token,assetToken]);
43
+ // D1 import text keeps Cloudflare's reason, never its signed upload destinations.
44
+ const redactAuth = text => /\/r2\/buckets\/[^/]+\/objects(?:[/?]|$)/.test(path) ? '[R2 object details omitted]' : /\/d1\/database\/[^/]+\/import$/.test(path) ? redactCredentials(text).replace(/\bhttps?:\/\/\S+/gi,'[URL omitted]') : redactCredentials(text);
42
45
  const operation = `${method} ${path.split('?')[0].replace(/(\/r2\/buckets\/[^/]+\/objects)\/.*/, '$1/[object]')}`;
43
46
  const endpoint = redactCredentials(`https://api.cloudflare.com/client/v4${zone ? '' : '/accounts/[REDACTED]'}${operation.slice(method.length + 1)}`);
44
47
  const transportFailure = (cause, attempts, status = null) => {
@@ -48,13 +51,14 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
48
51
  error.providerTransport = {operation:redactCredentials(operation),endpoint,attempts,status,reason:transportReason(cause)};
49
52
  return recordDeployFailure(error,{output:redactAuth(transportOutput(cause))});
50
53
  };
51
- const {response,attempts} = await providerResponse({method,timeoutMs,callerSignal,
54
+ const {response,output = ''} = await providerResponse({method,timeoutMs,callerSignal,removal,
55
+ replayableAsset:method === 'POST' && (/^\/workers\/scripts\/[^/]+\/assets-upload-session$/.test(path) || path === '/workers/assets/upload?base64=true'),
52
56
  reserveRetry:() => requests < maxRequests && (++requests, true),failure:transportFailure,
57
+ consume:(response,attempts) => response.body ? boundedText(response,undefined,cause => transportFailure(cause,attempts,response.status)) : '',
53
58
  send:signal => isR2ObjectDelete(path, method) ? objectDelete(`/client/v4/accounts/${accountId}${path}`, {headers:{Authorization:`Bearer ${token}`},signal}) : request(zone ? `https://api.cloudflare.com/client/v4${path}` : `https://api.cloudflare.com/client/v4/accounts/${accountId}${path}`, {
54
59
  method, redirect: 'error', signal, headers: { Authorization: `Bearer ${assetToken ?? token}`, ...(body && !(body instanceof FormData) ? { 'Content-Type':'application/json' } : {}) },
55
60
  ...(body === undefined ? {} : { body: body instanceof FormData ? body : JSON.stringify(body) }),
56
61
  })});
57
- const output = response.body ? await boundedText(response,undefined,cause => transportFailure(cause,attempts,response.status)) : '';
58
62
  // A successful DELETE may have no response body (including HTTP 200).
59
63
  // Nonempty replies and all other methods still require the API envelope.
60
64
  if (method === 'DELETE' && response.ok && output === '') return { success: true, result: null };
@@ -69,7 +73,7 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
69
73
  const bounded = text => redactAuth(text).replace(/[\x00-\x1f\x7f-\x9f]+/g,' ').trim().slice(0,300);
70
74
  const said = (Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code) || typeof x?.message === 'string').slice(0,3)
71
75
  .map(x => ({...(Number.isSafeInteger(x.code) ? {code:x.code} : {}),...(typeof x.message === 'string' ? {message:bounded(x.message)} : {})}));
72
- error.providerRefusal = {operation:`${method} ${path.split('?')[0].replace(/(\/r2\/buckets\/[^/]+\/objects)\/.*/, '$1/[object]')}`,errors:said.length || payload !== undefined || !output.trim() ? said : [{message:bounded(output.trim().split('\n')[0])}]};
76
+ error.providerRefusal = {operation:redactCredentials(operation),errors:said.length || payload !== undefined || !output.trim() ? said : [{message:bounded(output.trim().split('\n')[0])}]};
73
77
  throw recordDeployFailure(error,{providerCodes:(Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code)).map(x=>x.code).slice(0,8),
74
78
  output:redactAuth(Array.isArray(payload?.errors) ? payload.errors.filter(x=>typeof x?.message==='string').map(x=>x.message).join('\n') : output)});
75
79
  }
@@ -80,79 +84,38 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
80
84
  // D1 can fail inside a successful HTTP/API envelope. Read only typed
81
85
  // error fields; query results and metadata are never diagnostic output.
82
86
  const errors = failed.flatMap(row => [typeof row.error === 'string' ? {message:row.error} : row.error, ...(Array.isArray(row.errors) ? row.errors : [])]);
83
- throw recordDeployFailure(new Error('d1-query-failed'),{providerCodes:errors.filter(error=>Number.isSafeInteger(error?.code)).map(error=>error.code).slice(0,8),
87
+ const error = new Error('d1-query-failed');
88
+ error.providerRefusal = {operation:redactCredentials(operation),errors:errors.filter(item => Number.isSafeInteger(item?.code) || typeof item?.message === 'string').map(item => ({code:item.code,message:typeof item.message === 'string' ? redactAuth(item.message).slice(0,300) : undefined})).slice(0,3)};
89
+ throw recordDeployFailure(error,{providerCodes:errors.filter(error=>Number.isSafeInteger(error?.code)).map(error=>error.code).slice(0,8),
84
90
  output:redactAuth(errors.filter(error=>typeof error?.message==='string').map(error=>error.message).join('\n'))});
85
91
  }
86
92
  }
87
93
  return payload;
88
94
  }
89
- async function list(path, { field, pagination = 'page' } = {}) {
90
- if (pagination === 'single') {
91
- assert(path === '/workers/scripts' || /^\/queues\/[a-f0-9]{32}\/consumers$/.test(path),'invalid-single-list');
92
- const data = await envelope(path);
93
- assert(Array.isArray(data.result) && data.result.length <= 10000,'provider-invalid-list');
94
- const info = data.result_info;
95
- if (info != null) {
96
- assert(typeof info === 'object' && !Array.isArray(info),'provider-incomplete-pagination');
97
- for (const key of ['count','total_count']) if (Object.hasOwn(info,key)) assert(info[key] === data.result.length,'provider-partial-list');
98
- assert([undefined,null,''].includes(info.cursor) &&
99
- (info.total_pages === undefined || info.total_pages === 1 || info.total_pages === 0 && data.result.length === 0) &&
100
- (info.page === undefined || info.page === 1),'provider-incomplete-pagination');
101
- }
102
- return data.result;
95
+ async function envelope(path, options = {}) {
96
+ try { return await rawEnvelope(path,options); }
97
+ catch (error) {
98
+ error.providerCall ??= {operation:`${options.method ?? 'GET'} ${path.split('?')[0].replace(/(\/r2\/buckets\/[^/]+\/objects)\/.*/, '$1/[object]')}`,reason:error.message};
99
+ throw error;
103
100
  }
104
- assert(['page','cursor','r2-bounded'].includes(pagination),'invalid-list-pagination');
105
- const r2Bounded = pagination === 'r2-bounded';
106
- assert(!r2Bounded || path === '/r2/buckets' && field === 'buckets', 'invalid-r2-bounded-list');
107
- const result = []; const seen = new Set(); let cursor; let total;
108
- for (let page = 1; page <= 100; page++) {
109
- const suffix = r2Bounded || pagination === 'cursor' ? (path === '/r2/buckets' ? `?per_page=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}` : (cursor ? `?cursor=${encodeURIComponent(cursor)}` : '')) : `?page=${page}&per_page=100`;
110
- const data = await envelope(path + suffix); const rows = field ? data.result?.[field] : data.result;
111
- assert(Array.isArray(rows), 'provider-invalid-list');
112
- result.push(...rows); assert(result.length <= 10000, 'provider-list-too-large');
113
- if (r2Bounded) {
114
- const info = data.result_info;
115
- assert(rows.length <= 1000, 'provider-partial-list');
116
- // Only this endpoint's explicit bounded strategy uses the requested
117
- // limit when metadata is absent. Never invent metadata for other modes.
118
- if (info === undefined || info === null) {
119
- assert(rows.length < 1000, 'provider-incomplete-pagination');
120
- assert(total === undefined || result.length === total, 'provider-partial-list');
121
- return result;
122
- }
123
- assert(typeof info === 'object' && !Array.isArray(info) && Number.isSafeInteger(info.per_page) && info.per_page > 0 && info.per_page <= 1000, 'provider-incomplete-pagination');
124
- assert(rows.length <= info.per_page, 'provider-partial-list');
125
- if (Object.hasOwn(info, 'count')) assert(info.count === rows.length, 'provider-partial-list');
126
- if (Object.hasOwn(info, 'total_count')) {
127
- assert(Number.isSafeInteger(info.total_count) && info.total_count >= 0, 'provider-incomplete-pagination');
128
- total ??= info.total_count; assert(total === info.total_count, 'provider-pagination-changed');
129
- }
130
- assert(total === undefined || result.length <= total, 'provider-partial-list');
131
- assert(info.cursor === undefined || typeof info.cursor === 'string', 'provider-invalid-cursor');
132
- if (!info.cursor) {
133
- assert(rows.length < info.per_page && (total === undefined || result.length === total), 'provider-partial-list');
134
- return result;
135
- }
136
- assert(total === undefined || result.length < total, 'provider-partial-list');
137
- cursor = info.cursor; assert(/^[a-zA-Z0-9_+\/=-]+$/.test(cursor) && !seen.has(cursor), 'provider-invalid-cursor'); seen.add(cursor);
138
- } else if (pagination === 'cursor') {
139
- const info = data.result_info;
140
- // R2's terminal short page may explicitly report null pagination. The
141
- // requested documented limit supplies the bound; a full page is ambiguous.
142
- if (path === '/r2/buckets' && info === null) { assert(rows.length < 1000, 'provider-incomplete-pagination'); return result; }
143
- assert(info && Number.isSafeInteger(info.per_page) && info.per_page > 0, 'provider-incomplete-pagination');
144
- if (!info.cursor) { assert(rows.length < info.per_page, 'provider-partial-list'); return result; }
145
- cursor = info.cursor; assert(typeof cursor === 'string' && /^[a-zA-Z0-9_+\/=-]+$/.test(cursor) && !seen.has(cursor), 'provider-invalid-cursor'); seen.add(cursor);
146
- } else {
147
- const info = data.result_info;
148
- assert(info && info.page === page && Number.isSafeInteger(info.total_count) && info.total_count >= 0, 'provider-incomplete-pagination');
149
- total ??= info.total_count; assert(total === info.total_count, 'provider-pagination-changed');
150
- assert(result.length <= total, 'provider-partial-list');
151
- if (result.length === total) return result;
152
- assert(rows.length > 0 && (info.total_pages === undefined || page < info.total_pages), 'provider-partial-list');
153
- }
101
+ }
102
+ async function collect(path, options) {
103
+ let result, rows = [];
104
+ for await (const page of providerPages(envelope, path, options)) {
105
+ result ??= page.data.result;
106
+ rows.push(...page.rows);
107
+ assert(rows.length <= 10000, 'provider-list-too-large');
154
108
  }
155
- throw new Error('provider-pagination-limit');
109
+ return options?.field ? {...result,[options.field]:rows} : rows;
110
+ }
111
+ async function list(path, options = {}) {
112
+ const shape = {...providerListShape(path),...options};
113
+ const result = await collect(path,shape);
114
+ return shape.field ? result[shape.field] : result;
115
+ }
116
+ async function requestResult(path, options = {}) {
117
+ const shape = (!options.method || options.method === 'GET') && providerListShape(path);
118
+ return shape ? collect(path,{...shape,zone:options.zone,signal:options.signal}) : (await envelope(path,options)).result;
156
119
  }
157
120
  async function upload(url, bytes, etag) {
158
121
  const destination = importUploadURL(url);
@@ -167,6 +130,6 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
167
130
  assert(response.status === 200, 'd1-import-upload-failed');
168
131
  assert(response.headers.get('etag')?.replace(/^"|"$/g, '') === etag, 'd1-import-upload-etag-mismatch');
169
132
  }
170
- return { accountId, envelope, list, importSql(id, sql) { return importD1({envelope,upload}, id, sql); }, async request(path, options) { return (await envelope(path, options)).result; },
171
- async zoneRequest(path, options = {}) { return (await envelope(path, { ...options, zone: true })).result; } };
133
+ return { accountId, envelope, list, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult,
134
+ zoneRequest(path, options = {}) { return requestResult(path, { ...options, zone: true }); } };
172
135
  }
@@ -1,3 +1,4 @@
1
+ import { providerPages } from './pagination.mjs';
1
2
  import { assert } from './validation.mjs';
2
3
 
3
4
  // Cloudflare's object REST API requires literal slashes, with other reserved
@@ -17,24 +18,12 @@ export function isR2ObjectDelete(path, method) {
17
18
  // The API returns result: Object[] and result_info.is_truncated/cursor, not the
18
19
  // bucket-list shape. Yield bounded pages so removal never collects the bucket.
19
20
  export async function* r2ObjectPages(api, bucket) {
20
- const seen = new Set(); let cursor;
21
- for (;;) {
22
- const data = await api.envelope(`/r2/buckets/${bucket}/objects?per_page=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`);
23
- const rows = data.result, info = data.result_info;
24
- assert(Array.isArray(rows) && rows.length <= 1000 && rows.every(row => typeof row?.key === 'string' && row.key.length), 'invalid-r2-object-list');
21
+ try { for await (const {rows} of providerPages(api.envelope, `/r2/buckets/${bucket}/objects`, {pagination:'cursor'})) {
22
+ assert(rows.length <= 1000 && rows.every(row => typeof row?.key === 'string' && row.key.length), 'invalid-r2-object-list');
25
23
  assert(new Set(rows.map(row => row.key)).size === rows.length, 'invalid-r2-object-list');
26
- // Cloudflare may omit pagination (or return null) on any terminal page
27
- // shorter than the requested limit. A full page cannot prove completion.
28
- if (info == null) {
29
- assert(rows.length < 1000, 'provider-incomplete-pagination');
30
- yield rows; return;
31
- }
32
- assert(typeof info.is_truncated === 'boolean' && (!info.delimited || Array.isArray(info.delimited) && !info.delimited.length), 'provider-incomplete-pagination');
33
- if (info.is_truncated) {
34
- assert(rows.length > 0 && typeof info.cursor === 'string' && info.cursor.length > 0 && info.cursor.length <= 4096 && !seen.has(info.cursor), 'provider-invalid-cursor');
35
- cursor = info.cursor; seen.add(cursor);
36
- } else assert(info.cursor == null || info.cursor === '', 'provider-invalid-cursor');
37
24
  yield rows;
38
- if (!info.is_truncated) return;
25
+ } } catch (error) {
26
+ error.providerCall ??= {operation:`GET /r2/buckets/${bucket}/objects`,reason:error.message};
27
+ throw error;
39
28
  }
40
29
  }