@runeditrun/rer 0.2.23 → 0.2.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -7
- package/cli/browser-runtime.mjs +65 -0
- package/cli/demo-host.mjs +1 -1
- package/cli/deploy-claim-request.mjs +5 -4
- package/cli/deploy-command.mjs +7 -5
- package/cli/deploy-inputs.mjs +2 -1
- package/cli/deploy-observation.mjs +20 -2
- package/cli/deploy-owner.mjs +12 -7
- package/cli/deploy.mjs +19 -6
- package/cli/export.mjs +1 -1
- package/cli/hydration/browser.mjs +3 -9
- package/cli/inspect.mjs +3 -1
- package/cli/lifecycle.mjs +4 -1
- package/cli/site.mjs +2 -2
- package/global/CHANGELOG.md +37 -0
- package/global/README.md +16 -7
- package/global/browser-runtime.mjs +45 -0
- package/global/cli/layout.mjs +8 -4
- package/global/main.mjs +13 -3
- package/install.sh +56 -3
- package/modules/container/README.md +6 -0
- package/modules/container/adapters/d1.mjs +1 -1
- package/modules/container/build.mjs +2 -1
- package/modules/container/image-variables.mjs +27 -0
- package/modules/container/transfer.mjs +0 -1
- package/modules/container/worker-engine.mjs +1 -1
- package/modules/delivery-producer/declarations.mjs +6 -1
- package/modules/delivery-producer/mail.mjs +66 -0
- package/modules/demo/label.mjs +5 -2
- package/modules/demo-host/README.md +29 -7
- package/modules/demo-host/container.mjs +19 -7
- package/modules/demo-host/placement.mjs +19 -5
- package/modules/demo-host/plan.mjs +11 -6
- package/modules/demo-host/start-script.mjs +2 -1
- package/modules/demo-host/startup.mjs +3 -3
- package/modules/demo-host/worker.mjs +27 -8
- package/modules/deploy-kit/README.md +41 -6
- package/modules/deploy-kit/asset-health.mjs +28 -0
- package/modules/deploy-kit/d1-import.mjs +22 -4
- package/modules/deploy-kit/diagnostics.mjs +46 -1
- package/modules/deploy-kit/health.mjs +2 -23
- package/modules/deploy-kit/migration-ledger.mjs +6 -2
- package/modules/deploy-kit/migration-retry.mjs +91 -0
- package/modules/deploy-kit/migrations.mjs +9 -5
- package/modules/deploy-kit/owner-cli.mjs +9 -6
- package/modules/deploy-kit/owner-remove.mjs +104 -64
- package/modules/deploy-kit/pagination.mjs +99 -0
- package/modules/deploy-kit/provider-transport.mjs +28 -5
- package/modules/deploy-kit/provider.mjs +38 -75
- package/modules/deploy-kit/r2-objects.mjs +6 -17
- package/modules/deploy-kit/site-readback.mjs +67 -0
- package/modules/deploy-kit/site.mjs +17 -31
- package/modules/deploy-kit/uploaded-worker.mjs +9 -3
- package/modules/deploy-kit/worker.mjs +5 -2
- package/modules/erasure/queue.mjs +0 -2
- package/modules/job-queue/cli.mjs +1 -1
- package/modules/operator-auth/README.md +10 -2
- package/modules/operator-auth/config.mjs +1 -0
- package/modules/operator-auth/conformance/presentation.mjs +2 -2
- package/modules/operator-auth/identity/email-link/confirmation.mjs +2 -1
- package/modules/operator-auth/identity/email-link/expiry.mjs +8 -0
- package/modules/operator-auth/identity/email-link/index.mjs +2 -1
- package/modules/operator-auth/identity/email-link/message.mjs +3 -2
- package/modules/operator-auth/identity/oidc/jwt.mjs +1 -1
- package/modules/operator-auth/index.d.mts +2 -0
- package/modules/portability/README.md +7 -2
- package/modules/portability/bridge.mjs +3 -3
- package/modules/portability/catalogue.mjs +1 -1
- package/modules/portability/cli-options.mjs +1 -1
- package/modules/portability/import-staging.mjs +45 -0
- package/modules/portability/import.mjs +8 -8
- package/modules/portability/legacy-files.mjs +1 -1
- package/modules/portability/legacy-safety.mjs +3 -3
- package/modules/portability/payloads.mjs +1 -1
- package/modules/portability/storage.mjs +5 -2
- package/modules/release/source.mjs +1 -1
- package/package.json +1 -1
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { r2ObjectPages, r2ObjectPath } from './r2-objects.mjs';
|
|
2
2
|
import { rm } from 'node:fs/promises';
|
|
3
|
-
import { assert } from './validation.mjs';
|
|
3
|
+
import { assert, hexId } from './validation.mjs';
|
|
4
4
|
import { resourceKinds, findResource } from './resources.mjs';
|
|
5
5
|
import { activeWorker } from './worker.mjs';
|
|
6
6
|
import { deleteWorker } from './teardown.mjs';
|
|
7
|
-
import { workerConsumerScript } from './queue-consumer.mjs';
|
|
8
7
|
import { query } from './migrations.mjs';
|
|
9
8
|
import { KINDS, readOwnerState, writeOwnerState, withStateLock, withScopeHints, withSettledLists, workerSettings, sameDeployment, resourceId, checkResourceIdentity, refuse, settlePendingSecret } from './owner.mjs';
|
|
10
9
|
|
|
@@ -43,74 +42,100 @@ export async function ownerRemove({api: provider, target, statePath, clock, slee
|
|
|
43
42
|
const api = withSettledLists(withScopeHints(provider));
|
|
44
43
|
return withStateLock(statePath, async () => {
|
|
45
44
|
const state = await readOwnerState(statePath, target);
|
|
46
|
-
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
if (
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
listed
|
|
64
|
-
for (const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
45
|
+
const remaining = new Set([...(state.owned.worker ? [`worker:${target.worker}`] : []), ...KINDS.flatMap(kind => Object.keys(state.owned[kind]).map(name => `${kind}:${name}`))]);
|
|
46
|
+
const removed = [], failed = [];
|
|
47
|
+
try {
|
|
48
|
+
if (await settlePendingSecret(api, target.worker, state, {preflight:true})) await writeOwnerState(statePath, state);
|
|
49
|
+
const owned = KINDS.flatMap(kind => Object.entries(state.owned[kind]).map(([name, entry]) => ({kind, name, entry})));
|
|
50
|
+
assert(state.owned.worker || owned.length, 'nothing-to-remove');
|
|
51
|
+
// Preflight, read-only.
|
|
52
|
+
let worker = null;
|
|
53
|
+
if (await workerSettings(api, target.worker) !== null) {
|
|
54
|
+
if (!state.owned.worker) throw refuse('resource-name-collision', 'worker', target.worker);
|
|
55
|
+
const expected = state.pendingUpload?.provider ?? state.receipt?.provider;
|
|
56
|
+
if (!expected) throw refuse('uncertain-resource-create', 'worker', target.worker);
|
|
57
|
+
const live = await activeWorker(api, target.worker);
|
|
58
|
+
if (!sameDeployment(live, expected)) throw refuse('owned-resource-replaced', 'worker', target.worker);
|
|
59
|
+
worker = target.worker;
|
|
60
|
+
} else remaining.delete(`worker:${target.worker}`);
|
|
61
|
+
assert(!state.pendingSecret, 'secret-change-unsettled');
|
|
62
|
+
const listed = {}, doomed = [];
|
|
63
|
+
for (const kind of KINDS) {
|
|
64
|
+
if (!owned.some(item => item.kind === kind)) continue;
|
|
65
|
+
listed[kind] = await resourceKinds[kind].list(api);
|
|
66
|
+
for (const {name, entry} of owned.filter(item => item.kind === kind)) {
|
|
67
|
+
const row = findResource(listed[kind], kind, name);
|
|
68
|
+
if (!row) { remaining.delete(`${kind}:${name}`); continue; }
|
|
69
|
+
if (kind === 'r2') requireBucketIdentity(name, entry);
|
|
70
|
+
if (entry.status === 'pending') await confirmEmpty(api, target.worker, kind, name, resourceId(kind, row));
|
|
71
|
+
else checkResourceIdentity(kind, name, row, entry);
|
|
72
|
+
if (kind === 'r2' && entry.status === 'created' && entry.id) await checkBucket(api, name, entry);
|
|
73
|
+
doomed.push({kind, name, entry, id:resourceId(kind, row)});
|
|
74
|
+
}
|
|
72
75
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
// A Worker that consumes one of these queues is released first.
|
|
77
|
+
for (const queue of doomed.filter(item => item.kind === 'queues')) {
|
|
78
|
+
for (const consumer of await api.list(`/queues/${queue.id}/consumers`, {pagination:'single'})) {
|
|
79
|
+
assert(hexId(consumer.consumer_id), 'invalid-consumer-id');
|
|
80
|
+
await api.request(`/queues/${queue.id}/consumers/${consumer.consumer_id}`, {method:'DELETE'});
|
|
81
|
+
}
|
|
79
82
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
83
|
+
if (worker) { await deleteWorker({api, worker, ...(clock ? {clock} : {}), ...(sleep ? {sleep} : {})}); removed.push(`worker:${worker}`); }
|
|
84
|
+
for (const item of doomed) {
|
|
85
|
+
try {
|
|
86
|
+
if (item.kind === 'r2') await removeBucket(api, item);
|
|
87
|
+
else {
|
|
88
|
+
await remove[item.kind](api, item.id, item.name);
|
|
89
|
+
if (item.kind === 'queues') await waitForQueueRemoval(api,item,{clock,sleep});
|
|
90
|
+
}
|
|
91
|
+
removed.push(`${item.kind}:${item.name}`);
|
|
92
|
+
}
|
|
93
|
+
catch (error) { failed.push({resource:`${item.kind}:${item.name}`, code:error.message, error}); }
|
|
88
94
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (!remaining.
|
|
105
|
-
|
|
106
|
-
|
|
95
|
+
// Only complete, successful reads prove absence. Keep every unverified
|
|
96
|
+
// resource recorded even when its DELETE returned success.
|
|
97
|
+
let readFailure;
|
|
98
|
+
const readback = async read => { try { await read(); } catch (error) { readFailure ??= error; } };
|
|
99
|
+
await readback(async () => {
|
|
100
|
+
if (await workerSettings(api, target.worker) !== null) remaining.add(`worker:${target.worker}`);
|
|
101
|
+
else remaining.delete(`worker:${target.worker}`);
|
|
102
|
+
});
|
|
103
|
+
let namespacesKnown = false;
|
|
104
|
+
await readback(async () => {
|
|
105
|
+
const namespaces = await api.list('/workers/durable_objects/namespaces');
|
|
106
|
+
for (const row of namespaces) if (row.script === target.worker) remaining.add(`durable-object:${row.class}`);
|
|
107
|
+
namespacesKnown = true;
|
|
108
|
+
});
|
|
109
|
+
// Missing namespace readback cannot establish that the Worker's data is gone.
|
|
110
|
+
if (!namespacesKnown && state.owned.worker) remaining.add(`worker:${target.worker}`);
|
|
111
|
+
for (const kind of KINDS) {
|
|
112
|
+
if (!listed[kind]) continue;
|
|
113
|
+
await readback(async () => {
|
|
114
|
+
const rows = await resourceKinds[kind].list(api);
|
|
115
|
+
for (const {name} of owned.filter(item => item.kind === kind)) {
|
|
116
|
+
if (findResource(rows, kind, name)) remaining.add(`${kind}:${name}`);
|
|
117
|
+
else remaining.delete(`${kind}:${name}`);
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
if (remaining.size || readFailure) {
|
|
122
|
+
// Keep proof for every survivor or resource whose absence is unverified.
|
|
123
|
+
for (const kind of KINDS) for (const name of Object.keys(state.owned[kind])) if (!remaining.has(`${kind}:${name}`)) delete state.owned[kind][name];
|
|
124
|
+
if (!remaining.has(`worker:${target.worker}`) && ![...remaining].some(name => name.startsWith('durable-object:'))) {
|
|
125
|
+
state.owned.worker = null; state.receipt = null; delete state.pendingUpload;
|
|
126
|
+
delete state.ownerClaim; delete state.retiredSecrets;
|
|
127
|
+
}
|
|
128
|
+
await writeOwnerState(statePath, state);
|
|
129
|
+
throw readFailure ?? new Error('remove-incomplete');
|
|
107
130
|
}
|
|
108
|
-
await
|
|
109
|
-
|
|
131
|
+
await rm(statePath, {force:true});
|
|
132
|
+
return {worker:target.worker, removed, remaining:[]};
|
|
133
|
+
} catch (error) {
|
|
134
|
+
error.remaining = [...remaining];
|
|
135
|
+
error.failed = failed;
|
|
136
|
+
error.removed = removed.filter(resource => !remaining.has(resource));
|
|
110
137
|
throw error;
|
|
111
138
|
}
|
|
112
|
-
await rm(statePath, {force:true});
|
|
113
|
-
return {worker:target.worker, removed, remaining:[]};
|
|
114
139
|
});
|
|
115
140
|
}
|
|
116
141
|
|
|
@@ -137,3 +162,18 @@ async function removeBucket(api, {name, entry}) {
|
|
|
137
162
|
await checkBucket(api, name, entry);
|
|
138
163
|
await api.request(`/r2/buckets/${name}`, {method:'DELETE'});
|
|
139
164
|
}
|
|
165
|
+
|
|
166
|
+
// Queue deletion and its account listing can settle separately. Finish the
|
|
167
|
+
// readback within this removal invocation, without replaying the DELETE.
|
|
168
|
+
async function waitForQueueRemoval(api, item, {clock = () => performance.now(), sleep = ms => new Promise(resolve => setTimeout(resolve,ms))}) {
|
|
169
|
+
const start = clock();
|
|
170
|
+
for (;;) {
|
|
171
|
+
if (!(await resourceKinds.queues.list(api)).some(row => row.queue_id === item.id)) return;
|
|
172
|
+
if (clock() - start >= 60000) {
|
|
173
|
+
const error = new Error('queue-removal-not-confirmed');
|
|
174
|
+
error.providerCall = {operation:'GET /queues',reason:'Cloudflare still lists the deleted queue after 60 seconds'};
|
|
175
|
+
throw error;
|
|
176
|
+
}
|
|
177
|
+
await sleep(500);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { assert } from './validation.mjs';
|
|
2
|
+
|
|
3
|
+
// Request strategies describe the wire parameters only. Every endpoint uses
|
|
4
|
+
// this reader to decide whether a reply actually has another page.
|
|
5
|
+
export async function* providerPages(envelope, path, {field, pagination = 'page', zone = false, signal} = {}) {
|
|
6
|
+
assert(['page','cursor','r2-bounded','single'].includes(pagination), 'invalid-list-pagination');
|
|
7
|
+
assert(pagination !== 'r2-bounded' || path === '/r2/buckets' && field === 'buckets', 'invalid-r2-bounded-list');
|
|
8
|
+
const cursorMode = pagination === 'cursor' || pagination === 'r2-bounded';
|
|
9
|
+
const url = new URL(path, 'https://api.cloudflare.com');
|
|
10
|
+
const perPage = Number(url.searchParams.get('per_page') ?? url.searchParams.get('limit') ?? (cursorMode ? 1000 : pagination === 'single' ? 10000 : 100));
|
|
11
|
+
assert(Number.isSafeInteger(perPage) && perPage > 0, 'invalid-list-pagination');
|
|
12
|
+
let cursor, total, count = 0, pageMode = false;
|
|
13
|
+
const seen = new Set();
|
|
14
|
+
for (let page = 1; page <= 100; page++) {
|
|
15
|
+
const query = new URLSearchParams(url.searchParams);
|
|
16
|
+
if (cursor) {
|
|
17
|
+
query.delete('page');
|
|
18
|
+
if (pagination !== 'single' && !query.has('limit')) query.set('per_page',String(perPage));
|
|
19
|
+
query.set('cursor',cursor);
|
|
20
|
+
}
|
|
21
|
+
else if ((pageMode || !cursorMode) && (pagination !== 'single' || page > 1)) query.set('page',String(page));
|
|
22
|
+
if (pagination !== 'single' && !query.has('limit')) query.set('per_page',String(perPage));
|
|
23
|
+
const requestPath = url.pathname + (query.size ? `?${query}` : '');
|
|
24
|
+
let data, rows, next;
|
|
25
|
+
try {
|
|
26
|
+
data = await envelope(requestPath, {zone,signal});
|
|
27
|
+
rows = field ? data.result?.[field] : data.result;
|
|
28
|
+
assert(Array.isArray(rows), 'provider-invalid-list');
|
|
29
|
+
assert(rows.length <= perPage, 'provider-partial-list');
|
|
30
|
+
count += rows.length;
|
|
31
|
+
const info = data.result_info;
|
|
32
|
+
if (info == null) {
|
|
33
|
+
assert(rows.length < perPage, 'provider-incomplete-pagination');
|
|
34
|
+
assert(total === undefined || count === total, 'provider-partial-list');
|
|
35
|
+
} else {
|
|
36
|
+
assert(typeof info === 'object' && !Array.isArray(info), 'provider-incomplete-pagination');
|
|
37
|
+
if (info.per_page !== undefined) {
|
|
38
|
+
assert(Number.isSafeInteger(info.per_page) && info.per_page > 0 && info.per_page <= perPage, 'provider-incomplete-pagination');
|
|
39
|
+
assert(rows.length <= info.per_page, 'provider-partial-list');
|
|
40
|
+
}
|
|
41
|
+
if (info.count !== undefined) assert(info.count === rows.length, 'provider-partial-list');
|
|
42
|
+
if (info.page !== undefined) assert(info.page === page, 'provider-incomplete-pagination');
|
|
43
|
+
if (info.total_count !== undefined) {
|
|
44
|
+
assert(Number.isSafeInteger(info.total_count) && info.total_count >= 0, 'provider-incomplete-pagination');
|
|
45
|
+
total ??= info.total_count;
|
|
46
|
+
assert(total === info.total_count, 'provider-pagination-changed');
|
|
47
|
+
assert(count <= total, 'provider-partial-list');
|
|
48
|
+
}
|
|
49
|
+
if (info.total_pages !== undefined) assert(Number.isSafeInteger(info.total_pages) && info.total_pages >= 0 && (info.total_pages > 0 || count === 0), 'provider-incomplete-pagination');
|
|
50
|
+
if (info.is_truncated !== undefined) assert(typeof info.is_truncated === 'boolean', 'provider-incomplete-pagination');
|
|
51
|
+
assert(!info.delimited || Array.isArray(info.delimited) && !info.delimited.length, 'provider-incomplete-pagination');
|
|
52
|
+
assert(info.cursor == null || typeof info.cursor === 'string', 'provider-invalid-cursor');
|
|
53
|
+
const morePages = info.total_pages !== undefined && page < info.total_pages;
|
|
54
|
+
const moreRows = total !== undefined && count < total;
|
|
55
|
+
if (info.cursor) {
|
|
56
|
+
assert(info.is_truncated !== false, 'provider-invalid-cursor');
|
|
57
|
+
assert((info.total_pages === undefined || morePages) && (total === undefined || moreRows), 'provider-partial-list');
|
|
58
|
+
assert(info.cursor.length <= 4096 && /^[a-zA-Z0-9_+\/=-]+$/.test(info.cursor) && !seen.has(info.cursor), 'provider-invalid-cursor');
|
|
59
|
+
seen.add(info.cursor); next = info.cursor;
|
|
60
|
+
} else if (info.is_truncated === true) {
|
|
61
|
+
throw new Error('provider-incomplete-pagination');
|
|
62
|
+
} else if (cursorMode && moreRows && !morePages) {
|
|
63
|
+
throw new Error('provider-partial-list');
|
|
64
|
+
} else if (morePages || moreRows) {
|
|
65
|
+
assert(info.total_pages === undefined || morePages, 'provider-partial-list');
|
|
66
|
+
next = true;
|
|
67
|
+
} else {
|
|
68
|
+
assert(total === undefined || count === total, 'provider-partial-list');
|
|
69
|
+
}
|
|
70
|
+
if (next) assert(rows.length > 0 || morePages || typeof next === 'string', 'provider-partial-list');
|
|
71
|
+
}
|
|
72
|
+
} catch (error) {
|
|
73
|
+
// No response bodies, query values, account IDs or cursors in diagnostics.
|
|
74
|
+
error.providerCall ??= {operation:`GET ${url.pathname}`,reason:error.message};
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
yield {rows,data};
|
|
78
|
+
if (!next) return;
|
|
79
|
+
cursor = typeof next === 'string' ? next : undefined;
|
|
80
|
+
pageMode = next === true;
|
|
81
|
+
}
|
|
82
|
+
const error = new Error('provider-pagination-limit');
|
|
83
|
+
error.providerCall = {operation:`GET ${url.pathname}`,reason:'Cloudflare continued advertising pages beyond the request limit'};
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Keep result wrappers (versions/items, schedules, deployments) for existing kit
|
|
88
|
+
// callers while ensuring their GETs cannot bypass pagination. Query filters stay
|
|
89
|
+
// on every request; writes never pass through the reader.
|
|
90
|
+
export function providerListShape(path) {
|
|
91
|
+
const base = path.split('?')[0];
|
|
92
|
+
if (base === '/r2/buckets') return {field:'buckets',pagination:'cursor'};
|
|
93
|
+
if (/^\/r2\/buckets\/[^/]+\/objects$/.test(base) || /^\/storage\/kv\/namespaces\/[^/]+\/keys$/.test(base)) return {pagination:'cursor'};
|
|
94
|
+
const field = /\/workers\/scripts\/[^/]+\/(versions|deployments|schedules)$/.exec(base)?.[1];
|
|
95
|
+
if (field) return {field:field === 'versions' ? 'items' : field,...(field === 'schedules' ? {pagination:'single'} : {})};
|
|
96
|
+
if (['/d1/database','/storage/kv/namespaces','/queues','/workers/durable_objects/namespaces','/workers/domains','/zones','/containers/applications'].includes(base)) return {};
|
|
97
|
+
if (['/workers/scripts','/d1/database','/storage/kv/namespaces','/queues','/workers/durable_objects/namespaces','/workers/domains','/zones'].includes(base) || /^\/queues\/[^/]+\/consumers$/.test(base) || /^\/workers\/scripts\/[^/]+\/secrets$/.test(base) || /^\/zones\/[^/]+\/workers\/routes$/.test(base)) return {pagination:'single'};
|
|
98
|
+
return undefined;
|
|
99
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { setTimeout as pause } from 'node:timers/promises';
|
|
2
|
+
|
|
1
3
|
// A missing HTTP status does not prove that a write never reached Cloudflare.
|
|
2
4
|
// Only these connection-establishment failures make replaying a write safe.
|
|
3
5
|
const UNSENT = new Set(['ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', 'UND_ERR_CONNECT_TIMEOUT']);
|
|
@@ -9,11 +11,13 @@ function unsent(error, seen = new Set()) {
|
|
|
9
11
|
return unsent(error.cause, seen);
|
|
10
12
|
}
|
|
11
13
|
|
|
12
|
-
/**
|
|
13
|
-
export async function providerResponse({send, method, timeoutMs, callerSignal, reserveRetry, failure}) {
|
|
14
|
+
/** Scoped safe replays share the original deadline and request budget. */
|
|
15
|
+
export async function providerResponse({send, method, timeoutMs, callerSignal, reserveRetry, failure, removal = false, replayableAsset = false, consume}) {
|
|
14
16
|
const timeout = AbortSignal.timeout(timeoutMs);
|
|
15
17
|
const signal = callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout;
|
|
16
18
|
const read = method === 'GET' || method === 'HEAD';
|
|
19
|
+
const removalReplay = removal && ['GET','DELETE'].includes(method);
|
|
20
|
+
const maxAttempts = removalReplay ? 3 : 2;
|
|
17
21
|
for (let attempt = 1; ; attempt++) {
|
|
18
22
|
// Leave time for a second read when the first one times out. Caller deadlines
|
|
19
23
|
// remain authoritative and are never reset by a retry.
|
|
@@ -21,12 +25,31 @@ export async function providerResponse({send, method, timeoutMs, callerSignal, r
|
|
|
21
25
|
const timer = firstRead ? setTimeout(() => firstRead.abort(new DOMException('Provider read timed out','TimeoutError')), Math.max(1, Math.ceil(timeoutMs / 2))) : undefined;
|
|
22
26
|
timer?.unref();
|
|
23
27
|
const attemptSignal = firstRead ? AbortSignal.any([signal, firstRead.signal]) : signal;
|
|
24
|
-
|
|
28
|
+
let readingBody = false;
|
|
29
|
+
try {
|
|
30
|
+
const response = await send(attemptSignal);
|
|
31
|
+
clearTimeout(timer);
|
|
32
|
+
// A retried delete can observe the first attempt's committed deletion.
|
|
33
|
+
if (removalReplay && method === 'DELETE' && attempt > 1 && response.status === 404) {
|
|
34
|
+
void response.body?.cancel().catch(() => {});
|
|
35
|
+
return {response:new Response(null,{status:204}),attempts:attempt};
|
|
36
|
+
}
|
|
37
|
+
if ((removalReplay || replayableAsset) && response.status >= 500 && response.status <= 599 && attempt < maxAttempts && !signal.aborted && reserveRetry()) {
|
|
38
|
+
void response.body?.cancel().catch(() => {});
|
|
39
|
+
await pause(250 * 2 ** (attempt - 1),undefined,{signal});
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
readingBody = true;
|
|
43
|
+
const output = consume ? await consume(response,attempt) : undefined;
|
|
44
|
+
return {response,attempts:attempt,output};
|
|
45
|
+
}
|
|
25
46
|
catch (cause) {
|
|
26
|
-
|
|
47
|
+
const bodyReset = readingBody && replayableAsset && cause.message === 'provider-transport-failed';
|
|
48
|
+
if (attempt === 1 && !signal.aborted && (bodyReset || !readingBody && (read || replayableAsset || unsent(cause))) && reserveRetry()) continue;
|
|
49
|
+
if (readingBody) throw cause;
|
|
27
50
|
throw failure(cause, attempt);
|
|
28
51
|
}
|
|
29
|
-
//
|
|
52
|
+
// Body consumption and scoped asset replays keep the total deadline.
|
|
30
53
|
finally { clearTimeout(timer); }
|
|
31
54
|
}
|
|
32
55
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { providerPages, providerListShape } from './pagination.mjs';
|
|
1
2
|
import { deleteR2Object } from './r2-object-transport.mjs';
|
|
2
3
|
import { isR2ObjectDelete } from './r2-objects.mjs';
|
|
3
4
|
import { recordDeployFailure } from './diagnostics.mjs';
|
|
@@ -24,21 +25,23 @@ async function boundedText(response, limit = 4 * 1024 * 1024, onReadError) {
|
|
|
24
25
|
export async function boundedJSON(response, limit) {
|
|
25
26
|
try { return JSON.parse(await boundedText(response,limit)); } catch (error) { if (error instanceof SyntaxError) throw new Error('malformed-response'); throw error; }
|
|
26
27
|
}
|
|
28
|
+
const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
|
|
27
29
|
/** Only this fixed origin and the explicitly selected account ever receive the token. */
|
|
28
|
-
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500 }) {
|
|
30
|
+
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false }) {
|
|
29
31
|
assert(hexId(accountId), 'invalid-cloudflare-account-id');
|
|
30
32
|
assert(typeof token === 'string' && token.trim() && !/[\r\n]/.test(token), 'invalid-cloudflare-api-token');
|
|
31
33
|
let requests = 0;
|
|
32
34
|
// `zone` scope: the zone-level API (/zones/...), for a run site's zone lookup and redirect rule.
|
|
33
|
-
async function
|
|
35
|
+
async function rawEnvelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal } = {}) {
|
|
34
36
|
assert(typeof path === 'string' && (isR2ObjectDelete(path, method) || /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%')), 'invalid-provider-path');
|
|
35
37
|
assert(!zone || /^\/zones(?:[/?]|$)/.test(path), 'invalid-provider-path');
|
|
36
38
|
assert(++requests <= maxRequests, 'provider-request-budget');
|
|
37
39
|
assert(!assetToken || path === '/workers/assets/upload?base64=true', 'invalid-asset-auth-target');
|
|
38
40
|
// Asset-session credentials are discovered at runtime, outside the CLI env.
|
|
39
41
|
// Preserve complete text for the CLI's remaining redaction before truncation.
|
|
40
|
-
const redactCredentials = text => [accountId,token,assetToken]
|
|
41
|
-
|
|
42
|
+
const redactCredentials = text => redactValues(text,[accountId,token,assetToken]);
|
|
43
|
+
// D1 import text keeps Cloudflare's reason, never its signed upload destinations.
|
|
44
|
+
const redactAuth = text => /\/r2\/buckets\/[^/]+\/objects(?:[/?]|$)/.test(path) ? '[R2 object details omitted]' : /\/d1\/database\/[^/]+\/import$/.test(path) ? redactCredentials(text).replace(/\bhttps?:\/\/\S+/gi,'[URL omitted]') : redactCredentials(text);
|
|
42
45
|
const operation = `${method} ${path.split('?')[0].replace(/(\/r2\/buckets\/[^/]+\/objects)\/.*/, '$1/[object]')}`;
|
|
43
46
|
const endpoint = redactCredentials(`https://api.cloudflare.com/client/v4${zone ? '' : '/accounts/[REDACTED]'}${operation.slice(method.length + 1)}`);
|
|
44
47
|
const transportFailure = (cause, attempts, status = null) => {
|
|
@@ -48,13 +51,14 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
48
51
|
error.providerTransport = {operation:redactCredentials(operation),endpoint,attempts,status,reason:transportReason(cause)};
|
|
49
52
|
return recordDeployFailure(error,{output:redactAuth(transportOutput(cause))});
|
|
50
53
|
};
|
|
51
|
-
const {response,
|
|
54
|
+
const {response,output = ''} = await providerResponse({method,timeoutMs,callerSignal,removal,
|
|
55
|
+
replayableAsset:method === 'POST' && (/^\/workers\/scripts\/[^/]+\/assets-upload-session$/.test(path) || path === '/workers/assets/upload?base64=true'),
|
|
52
56
|
reserveRetry:() => requests < maxRequests && (++requests, true),failure:transportFailure,
|
|
57
|
+
consume:(response,attempts) => response.body ? boundedText(response,undefined,cause => transportFailure(cause,attempts,response.status)) : '',
|
|
53
58
|
send:signal => isR2ObjectDelete(path, method) ? objectDelete(`/client/v4/accounts/${accountId}${path}`, {headers:{Authorization:`Bearer ${token}`},signal}) : request(zone ? `https://api.cloudflare.com/client/v4${path}` : `https://api.cloudflare.com/client/v4/accounts/${accountId}${path}`, {
|
|
54
59
|
method, redirect: 'error', signal, headers: { Authorization: `Bearer ${assetToken ?? token}`, ...(body && !(body instanceof FormData) ? { 'Content-Type':'application/json' } : {}) },
|
|
55
60
|
...(body === undefined ? {} : { body: body instanceof FormData ? body : JSON.stringify(body) }),
|
|
56
61
|
})});
|
|
57
|
-
const output = response.body ? await boundedText(response,undefined,cause => transportFailure(cause,attempts,response.status)) : '';
|
|
58
62
|
// A successful DELETE may have no response body (including HTTP 200).
|
|
59
63
|
// Nonempty replies and all other methods still require the API envelope.
|
|
60
64
|
if (method === 'DELETE' && response.ok && output === '') return { success: true, result: null };
|
|
@@ -69,7 +73,7 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
69
73
|
const bounded = text => redactAuth(text).replace(/[\x00-\x1f\x7f-\x9f]+/g,' ').trim().slice(0,300);
|
|
70
74
|
const said = (Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code) || typeof x?.message === 'string').slice(0,3)
|
|
71
75
|
.map(x => ({...(Number.isSafeInteger(x.code) ? {code:x.code} : {}),...(typeof x.message === 'string' ? {message:bounded(x.message)} : {})}));
|
|
72
|
-
error.providerRefusal = {operation
|
|
76
|
+
error.providerRefusal = {operation:redactCredentials(operation),errors:said.length || payload !== undefined || !output.trim() ? said : [{message:bounded(output.trim().split('\n')[0])}]};
|
|
73
77
|
throw recordDeployFailure(error,{providerCodes:(Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code)).map(x=>x.code).slice(0,8),
|
|
74
78
|
output:redactAuth(Array.isArray(payload?.errors) ? payload.errors.filter(x=>typeof x?.message==='string').map(x=>x.message).join('\n') : output)});
|
|
75
79
|
}
|
|
@@ -80,79 +84,38 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
80
84
|
// D1 can fail inside a successful HTTP/API envelope. Read only typed
|
|
81
85
|
// error fields; query results and metadata are never diagnostic output.
|
|
82
86
|
const errors = failed.flatMap(row => [typeof row.error === 'string' ? {message:row.error} : row.error, ...(Array.isArray(row.errors) ? row.errors : [])]);
|
|
83
|
-
|
|
87
|
+
const error = new Error('d1-query-failed');
|
|
88
|
+
error.providerRefusal = {operation:redactCredentials(operation),errors:errors.filter(item => Number.isSafeInteger(item?.code) || typeof item?.message === 'string').map(item => ({code:item.code,message:typeof item.message === 'string' ? redactAuth(item.message).slice(0,300) : undefined})).slice(0,3)};
|
|
89
|
+
throw recordDeployFailure(error,{providerCodes:errors.filter(error=>Number.isSafeInteger(error?.code)).map(error=>error.code).slice(0,8),
|
|
84
90
|
output:redactAuth(errors.filter(error=>typeof error?.message==='string').map(error=>error.message).join('\n'))});
|
|
85
91
|
}
|
|
86
92
|
}
|
|
87
93
|
return payload;
|
|
88
94
|
}
|
|
89
|
-
async function
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const info = data.result_info;
|
|
95
|
-
if (info != null) {
|
|
96
|
-
assert(typeof info === 'object' && !Array.isArray(info),'provider-incomplete-pagination');
|
|
97
|
-
for (const key of ['count','total_count']) if (Object.hasOwn(info,key)) assert(info[key] === data.result.length,'provider-partial-list');
|
|
98
|
-
assert([undefined,null,''].includes(info.cursor) &&
|
|
99
|
-
(info.total_pages === undefined || info.total_pages === 1 || info.total_pages === 0 && data.result.length === 0) &&
|
|
100
|
-
(info.page === undefined || info.page === 1),'provider-incomplete-pagination');
|
|
101
|
-
}
|
|
102
|
-
return data.result;
|
|
95
|
+
async function envelope(path, options = {}) {
|
|
96
|
+
try { return await rawEnvelope(path,options); }
|
|
97
|
+
catch (error) {
|
|
98
|
+
error.providerCall ??= {operation:`${options.method ?? 'GET'} ${path.split('?')[0].replace(/(\/r2\/buckets\/[^/]+\/objects)\/.*/, '$1/[object]')}`,reason:error.message};
|
|
99
|
+
throw error;
|
|
103
100
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
assert(Array.isArray(rows), 'provider-invalid-list');
|
|
112
|
-
result.push(...rows); assert(result.length <= 10000, 'provider-list-too-large');
|
|
113
|
-
if (r2Bounded) {
|
|
114
|
-
const info = data.result_info;
|
|
115
|
-
assert(rows.length <= 1000, 'provider-partial-list');
|
|
116
|
-
// Only this endpoint's explicit bounded strategy uses the requested
|
|
117
|
-
// limit when metadata is absent. Never invent metadata for other modes.
|
|
118
|
-
if (info === undefined || info === null) {
|
|
119
|
-
assert(rows.length < 1000, 'provider-incomplete-pagination');
|
|
120
|
-
assert(total === undefined || result.length === total, 'provider-partial-list');
|
|
121
|
-
return result;
|
|
122
|
-
}
|
|
123
|
-
assert(typeof info === 'object' && !Array.isArray(info) && Number.isSafeInteger(info.per_page) && info.per_page > 0 && info.per_page <= 1000, 'provider-incomplete-pagination');
|
|
124
|
-
assert(rows.length <= info.per_page, 'provider-partial-list');
|
|
125
|
-
if (Object.hasOwn(info, 'count')) assert(info.count === rows.length, 'provider-partial-list');
|
|
126
|
-
if (Object.hasOwn(info, 'total_count')) {
|
|
127
|
-
assert(Number.isSafeInteger(info.total_count) && info.total_count >= 0, 'provider-incomplete-pagination');
|
|
128
|
-
total ??= info.total_count; assert(total === info.total_count, 'provider-pagination-changed');
|
|
129
|
-
}
|
|
130
|
-
assert(total === undefined || result.length <= total, 'provider-partial-list');
|
|
131
|
-
assert(info.cursor === undefined || typeof info.cursor === 'string', 'provider-invalid-cursor');
|
|
132
|
-
if (!info.cursor) {
|
|
133
|
-
assert(rows.length < info.per_page && (total === undefined || result.length === total), 'provider-partial-list');
|
|
134
|
-
return result;
|
|
135
|
-
}
|
|
136
|
-
assert(total === undefined || result.length < total, 'provider-partial-list');
|
|
137
|
-
cursor = info.cursor; assert(/^[a-zA-Z0-9_+\/=-]+$/.test(cursor) && !seen.has(cursor), 'provider-invalid-cursor'); seen.add(cursor);
|
|
138
|
-
} else if (pagination === 'cursor') {
|
|
139
|
-
const info = data.result_info;
|
|
140
|
-
// R2's terminal short page may explicitly report null pagination. The
|
|
141
|
-
// requested documented limit supplies the bound; a full page is ambiguous.
|
|
142
|
-
if (path === '/r2/buckets' && info === null) { assert(rows.length < 1000, 'provider-incomplete-pagination'); return result; }
|
|
143
|
-
assert(info && Number.isSafeInteger(info.per_page) && info.per_page > 0, 'provider-incomplete-pagination');
|
|
144
|
-
if (!info.cursor) { assert(rows.length < info.per_page, 'provider-partial-list'); return result; }
|
|
145
|
-
cursor = info.cursor; assert(typeof cursor === 'string' && /^[a-zA-Z0-9_+\/=-]+$/.test(cursor) && !seen.has(cursor), 'provider-invalid-cursor'); seen.add(cursor);
|
|
146
|
-
} else {
|
|
147
|
-
const info = data.result_info;
|
|
148
|
-
assert(info && info.page === page && Number.isSafeInteger(info.total_count) && info.total_count >= 0, 'provider-incomplete-pagination');
|
|
149
|
-
total ??= info.total_count; assert(total === info.total_count, 'provider-pagination-changed');
|
|
150
|
-
assert(result.length <= total, 'provider-partial-list');
|
|
151
|
-
if (result.length === total) return result;
|
|
152
|
-
assert(rows.length > 0 && (info.total_pages === undefined || page < info.total_pages), 'provider-partial-list');
|
|
153
|
-
}
|
|
101
|
+
}
|
|
102
|
+
async function collect(path, options) {
|
|
103
|
+
let result, rows = [];
|
|
104
|
+
for await (const page of providerPages(envelope, path, options)) {
|
|
105
|
+
result ??= page.data.result;
|
|
106
|
+
rows.push(...page.rows);
|
|
107
|
+
assert(rows.length <= 10000, 'provider-list-too-large');
|
|
154
108
|
}
|
|
155
|
-
|
|
109
|
+
return options?.field ? {...result,[options.field]:rows} : rows;
|
|
110
|
+
}
|
|
111
|
+
async function list(path, options = {}) {
|
|
112
|
+
const shape = {...providerListShape(path),...options};
|
|
113
|
+
const result = await collect(path,shape);
|
|
114
|
+
return shape.field ? result[shape.field] : result;
|
|
115
|
+
}
|
|
116
|
+
async function requestResult(path, options = {}) {
|
|
117
|
+
const shape = (!options.method || options.method === 'GET') && providerListShape(path);
|
|
118
|
+
return shape ? collect(path,{...shape,zone:options.zone,signal:options.signal}) : (await envelope(path,options)).result;
|
|
156
119
|
}
|
|
157
120
|
async function upload(url, bytes, etag) {
|
|
158
121
|
const destination = importUploadURL(url);
|
|
@@ -167,6 +130,6 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
167
130
|
assert(response.status === 200, 'd1-import-upload-failed');
|
|
168
131
|
assert(response.headers.get('etag')?.replace(/^"|"$/g, '') === etag, 'd1-import-upload-etag-mismatch');
|
|
169
132
|
}
|
|
170
|
-
return { accountId, envelope, list, importSql(id, sql) { return importD1({envelope,upload}, id, sql); },
|
|
171
|
-
|
|
133
|
+
return { accountId, envelope, list, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult,
|
|
134
|
+
zoneRequest(path, options = {}) { return requestResult(path, { ...options, zone: true }); } };
|
|
172
135
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { providerPages } from './pagination.mjs';
|
|
1
2
|
import { assert } from './validation.mjs';
|
|
2
3
|
|
|
3
4
|
// Cloudflare's object REST API requires literal slashes, with other reserved
|
|
@@ -17,24 +18,12 @@ export function isR2ObjectDelete(path, method) {
|
|
|
17
18
|
// The API returns result: Object[] and result_info.is_truncated/cursor, not the
|
|
18
19
|
// bucket-list shape. Yield bounded pages so removal never collects the bucket.
|
|
19
20
|
export async function* r2ObjectPages(api, bucket) {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const data = await api.envelope(`/r2/buckets/${bucket}/objects?per_page=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`);
|
|
23
|
-
const rows = data.result, info = data.result_info;
|
|
24
|
-
assert(Array.isArray(rows) && rows.length <= 1000 && rows.every(row => typeof row?.key === 'string' && row.key.length), 'invalid-r2-object-list');
|
|
21
|
+
try { for await (const {rows} of providerPages(api.envelope, `/r2/buckets/${bucket}/objects`, {pagination:'cursor'})) {
|
|
22
|
+
assert(rows.length <= 1000 && rows.every(row => typeof row?.key === 'string' && row.key.length), 'invalid-r2-object-list');
|
|
25
23
|
assert(new Set(rows.map(row => row.key)).size === rows.length, 'invalid-r2-object-list');
|
|
26
|
-
// Cloudflare may omit pagination (or return null) on any terminal page
|
|
27
|
-
// shorter than the requested limit. A full page cannot prove completion.
|
|
28
|
-
if (info == null) {
|
|
29
|
-
assert(rows.length < 1000, 'provider-incomplete-pagination');
|
|
30
|
-
yield rows; return;
|
|
31
|
-
}
|
|
32
|
-
assert(typeof info.is_truncated === 'boolean' && (!info.delimited || Array.isArray(info.delimited) && !info.delimited.length), 'provider-incomplete-pagination');
|
|
33
|
-
if (info.is_truncated) {
|
|
34
|
-
assert(rows.length > 0 && typeof info.cursor === 'string' && info.cursor.length > 0 && info.cursor.length <= 4096 && !seen.has(info.cursor), 'provider-invalid-cursor');
|
|
35
|
-
cursor = info.cursor; seen.add(cursor);
|
|
36
|
-
} else assert(info.cursor == null || info.cursor === '', 'provider-invalid-cursor');
|
|
37
24
|
yield rows;
|
|
38
|
-
|
|
25
|
+
} } catch (error) {
|
|
26
|
+
error.providerCall ??= {operation:`GET /r2/buckets/${bucket}/objects`,reason:error.message};
|
|
27
|
+
throw error;
|
|
39
28
|
}
|
|
40
29
|
}
|