@runeditrun/rer 0.2.84 → 0.2.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -24
- package/cli/README.md +17 -1
- package/cli/bounded.mjs +1 -1
- package/cli/business/carry.mjs +149 -0
- package/cli/business/command.mjs +6 -1
- package/cli/business/connect.mjs +1 -1
- package/cli/business/local-commands.mjs +2 -0
- package/cli/business/opt-ins.mjs +8 -4
- package/cli/business/questions.mjs +4 -1
- package/cli/business/state.mjs +1 -1
- package/cli/business/supervisor.mjs +7 -1
- package/cli/container-transfer.mjs +2 -2
- package/cli/contract.mjs +6 -6
- package/cli/demo-native.mjs +2 -1
- package/cli/deploy-command.mjs +6 -4
- package/cli/deploy.mjs +9 -7
- package/cli/errors.mjs +1 -1
- package/cli/export.mjs +1 -1
- package/cli/human-error.mjs +20 -0
- package/cli/lifecycle.mjs +23 -5
- package/cli/main.mjs +1 -1
- package/cli/managed-data-schema.mjs +114 -0
- package/cli/managed-data-zip.mjs +116 -0
- package/cli/managed-data.mjs +110 -0
- package/cli/managed-server.mjs +25 -14
- package/cli/native-owner-entry.mjs +86 -0
- package/cli/output.mjs +3 -2
- package/cli/parse.mjs +2 -2
- package/cli/portability-empty.mjs +59 -0
- package/cli/portability-migrations.mjs +7 -5
- package/cli/portability-policy.mjs +5 -2
- package/cli/portability-worker.mjs +15 -6
- package/cli/portability.mjs +30 -8
- package/cli/ready-summary.mjs +9 -0
- package/cli/run-transfers.mjs +37 -0
- package/cli/runtime-front-door.mjs +4 -4
- package/cli/server-proxy.mjs +4 -3
- package/cli/server-registry.mjs +13 -8
- package/cli/site.mjs +6 -1
- package/cli/transfers.mjs +17 -4
- package/edition-authoring/base-copies.mjs +55 -7
- package/foundation/runtime/contract-docs.mjs +3 -2
- package/foundation/runtime/run-data.d.mts +1 -0
- package/foundation/runtime/run-data.mjs +2 -0
- package/global/CHANGELOG.md +33 -0
- package/global/README.md +21 -24
- package/global/cli/command.mjs +8 -2
- package/global/cli/install.mjs +4 -5
- package/global/cli/path-link.mjs +3 -2
- package/global/cli/repo-command.mjs +3 -7
- package/global/launch/dev.mjs +12 -2
- package/global/launch/facts.mjs +2 -2
- package/global/launch/render.mjs +16 -9
- package/global/launch/state.mjs +4 -4
- package/global/main.mjs +12 -7
- package/install.sh +17 -6
- package/modules/container/README.md +6 -1
- package/modules/container/adapters/d1-limits.mjs +286 -0
- package/modules/container/adapters/d1-sqlite.mjs +95 -0
- package/modules/container/adapters/d1-workerd-preload.mjs +19 -0
- package/modules/container/adapters/d1-workerd-setup.mjs +26 -0
- package/modules/container/adapters/d1-workerd.mjs +5 -0
- package/modules/container/adapters/sqlite-worker.mjs +5 -3
- package/modules/container/build.mjs +1 -0
- package/modules/container/image-variables.mjs +8 -3
- package/modules/container/transfer.mjs +14 -4
- package/modules/container/worker-engine.mjs +3 -3
- package/modules/demo-host/build.mjs +1 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/demo-host/native/sql.mjs +7 -2
- package/modules/deploy-kit/README.md +22 -2
- package/modules/deploy-kit/d1-import-diagnostics.mjs +56 -0
- package/modules/deploy-kit/d1-import.mjs +17 -5
- package/modules/deploy-kit/diagnostics.mjs +29 -3
- package/modules/deploy-kit/owner-cli.mjs +20 -7
- package/modules/deploy-kit/provider-transport.mjs +19 -4
- package/modules/deploy-kit/provider.mjs +57 -22
- package/modules/deploy-kit/readiness-window.mjs +1 -1
- package/modules/deploy-kit/site.mjs +8 -3
- package/modules/deploy-kit/worker.mjs +4 -4
- package/modules/import-kit/src/d1/README.md +23 -4
- package/modules/import-kit/src/d1/atomic-prefix.ts +2 -2
- package/modules/import-kit/src/d1/budget.ts +15 -7
- package/modules/import-kit/src/d1/column-layout.ts +5 -0
- package/modules/import-kit/src/d1/creation.ts +15 -8
- package/modules/import-kit/src/d1/database.ts +3 -3
- package/modules/import-kit/src/d1/erasure-pages.ts +2 -1
- package/modules/import-kit/src/d1/erasure.ts +2 -2
- package/modules/import-kit/src/d1/jobs.ts +10 -6
- package/modules/import-kit/src/d1/plans.ts +14 -8
- package/modules/import-kit/src/d1/retirement.ts +2 -2
- package/modules/import-kit/src/d1/sets.ts +1 -3
- package/modules/import-kit/src/d1/staged-creation.ts +3 -3
- package/modules/import-kit/src/door/limits.ts +1 -1
- package/modules/import-kit/src/door/refusals.ts +1 -1
- package/modules/portability/README.md +112 -4
- package/modules/portability/backfills.mjs +9 -8
- package/modules/portability/bridge.mjs +12 -5
- package/modules/portability/catalogue.mjs +12 -5
- package/modules/portability/files.mjs +4 -3
- package/modules/portability/http.mjs +3 -3
- package/modules/portability/import.mjs +18 -9
- package/modules/portability/index.d.mts +4 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +3 -2
- package/modules/portability/legacy-safety.mjs +11 -2
- package/modules/portability/migration-history.mjs +2 -2
- package/modules/portability/progress.mjs +1 -0
- package/modules/portability/restore-empty.mjs +12 -0
- package/modules/release/github-verify.mjs +11 -2
- package/onboarding/adapter.mjs +35 -6
- package/onboarding/app-fetch.mjs +6 -4
- package/onboarding/development-runtime.mjs +19 -0
- package/onboarding/local/operation.mjs +45 -27
- package/onboarding/loopback-fetch.mjs +16 -0
- package/onboarding/operator-installer.mjs +2 -4
- package/onboarding/process-diagnostics.mjs +6 -2
- package/onboarding/production-runtime.mjs +4 -2
- package/onboarding/readiness.mjs +1 -0
- package/package.json +1 -1
- package/scripts/publish-check/public_hygiene.py +1 -1
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { Miniflare, convertV4MiniflareOptions, coupleWebSocket } from '
|
|
1
|
+
import { Miniflare, convertV4MiniflareOptions, coupleWebSocket } from './adapters/d1-workerd.mjs';
|
|
2
2
|
import { readdir } from 'node:fs/promises';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { rootCertificates } from 'node:tls';
|
|
5
5
|
import { ensurePortabilitySchema } from './portability-schema.mjs';
|
|
6
6
|
import { migrate } from './migrations.mjs';
|
|
7
7
|
async function files(root) { const result = []; for (const item of await readdir(root, { withFileTypes: true })) { const path = join(root, item.name); if (item.isDirectory()) result.push(...await files(path)); else if (/\.(m?js)$/.test(item.name)) result.push(path); } return result; }
|
|
8
|
-
export async function workerEngine({ root, data, manifest, settings, edge }) {
|
|
8
|
+
export async function workerEngine({ root, data, manifest, settings, edge, applyMigrations=true }) {
|
|
9
9
|
const config = manifest.config, directory = join(root, 'dist/server'), entry = join(root, manifest.entry);
|
|
10
10
|
const modules = [entry, ...(await files(directory)).filter(path => path !== entry)].map(path => ({ type: 'ESModule', path }));
|
|
11
11
|
const app = { name:'app', modules, modulesRoot:directory, compatibilityDate:config.compatibility_date, compatibilityFlags:config.compatibility_flags,
|
|
@@ -23,7 +23,7 @@ export async function workerEngine({ root, data, manifest, settings, edge }) {
|
|
|
23
23
|
serviceBindings:{ NETWORK:{ network:{ allow:['public'],tlsOptions:{ trustBrowserCas:true,trustedCertificates:[...rootCertificates] } } } } }],
|
|
24
24
|
}));
|
|
25
25
|
await mf.ready; const databases = {}, objects = {};
|
|
26
|
-
for (const [binding, plan] of Object.entries(manifest.migrations)) { const db = await mf.getD1Database(binding, 'app'); databases[binding] = db; await migrate(db, plan); await ensurePortabilitySchema(db, manifest.portabilityMigrations); }
|
|
26
|
+
for (const [binding, plan] of Object.entries(manifest.migrations)) { const db = await mf.getD1Database(binding, 'app'); databases[binding] = db; if(applyMigrations){await migrate(db, plan); await ensurePortabilitySchema(db, manifest.portabilityMigrations);} }
|
|
27
27
|
for (const bucket of config.r2_buckets ?? []) objects[bucket.binding] = await mf.getR2Bucket(bucket.binding, 'app');
|
|
28
28
|
const worker = await mf.getWorker('app');
|
|
29
29
|
return { mf, databases, objects, app:worker, coupleWebSocket,
|
|
@@ -33,6 +33,7 @@ export async function buildDemo(root, output, declaration) {
|
|
|
33
33
|
for (const binding of bindings.objects) if (!entryExports.get(entry)?.has(binding.class_name)) throw new Error(`Loader demo cannot bind ${binding.name}: missing Durable Object export ${binding.class_name}`);
|
|
34
34
|
if (!emitted.has(entry)) throw new Error('Demo Worker entry was not emitted by this build');
|
|
35
35
|
await checkBuildVariables(root, workerConfig.vars ?? {}, declaration.secrets ?? []);
|
|
36
|
+
await writeFile(join(root, 'dist/server/wrangler.json'), JSON.stringify(workerConfig) + '\n');
|
|
36
37
|
const { unstable_splitSqlQuery: split } = require('wrangler');
|
|
37
38
|
const migrations = Object.fromEntries((workerConfig.d1_databases ?? []).map(db => [db.binding, []]));
|
|
38
39
|
const inventory = withRewrites(await demoMigrationInventory(root, { config: workerConfig, configFile: 'dist/server/wrangler.json' }), await readMigrationRewrites(root));
|
|
@@ -31,7 +31,8 @@ export async function prepareNative(folder) {
|
|
|
31
31
|
const entry = `../app/${main}`;
|
|
32
32
|
modules['_rer/isolate.js'] = { js: (await readFile(new URL('./isolate.mjs', import.meta.url), 'utf8')).replace('__RER_APP_ENTRY__', entry)
|
|
33
33
|
+ objectExports(bindings) };
|
|
34
|
-
for (const file of ISOLATE_FILES) modules[`_rer/${file}.js`] = { js: await readFile(new URL(`./${file}.mjs`, import.meta.url), 'utf8') };
|
|
34
|
+
for (const file of ISOLATE_FILES) modules[`_rer/${file}.js`] = { js: (await readFile(new URL(`./${file}.mjs`, import.meta.url), 'utf8')).replace('../../container/adapters/d1-limits.mjs', './d1-limits.js') };
|
|
35
|
+
modules['_rer/d1-limits.js'] = { js: await readFile(new URL('../../container/adapters/d1-limits.mjs', import.meta.url), 'utf8') };
|
|
35
36
|
const assets = await collectFiles(join(app, 'dist/client'));
|
|
36
37
|
await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
|
|
37
38
|
const theme = await prepareTheme(folder, assets);
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
import { d1Preflight, d1ResultPreflight, createD1RowGuard, d1ColumnCount } from '../../container/adapters/d1-limits.mjs';
|
|
2
|
+
const guardRows = createD1RowGuard();
|
|
1
3
|
export function sqlite(storage) {
|
|
2
4
|
let lastRowId = 0;
|
|
3
5
|
function query(query, args = []) {
|
|
6
|
+
d1Preflight(query, args, table => d1ColumnCount(table, sql => storage.sql.exec(sql).toArray()));
|
|
4
7
|
const started = performance.now();
|
|
5
8
|
const read = /^\s*(?:SELECT|EXPLAIN)\b/i.test(query);
|
|
6
9
|
const before = read ? 0 : storage.sql.exec('SELECT total_changes() AS n').one().n;
|
|
@@ -8,11 +11,13 @@ export function sqlite(storage) {
|
|
|
8
11
|
const columns = cursor.columnNames;
|
|
9
12
|
const rows = [...cursor.raw()].map(row => row.map(value => value instanceof ArrayBuffer ? [...new Uint8Array(value)] : value));
|
|
10
13
|
const after = read ? { n: 0, id: lastRowId } : storage.sql.exec('SELECT total_changes() AS n, last_insert_rowid() AS id').one();
|
|
14
|
+
d1ResultPreflight(rows);
|
|
15
|
+
if (after.n !== before) guardRows(storage.sql, sql => ({ get: () => storage.sql.exec(sql).toArray()[0], all: () => storage.sql.exec(sql).toArray() }), undefined, query);
|
|
11
16
|
lastRowId = after.id;
|
|
12
17
|
return { success: true, results: rows.map(row => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), columns, rows,
|
|
13
18
|
meta: { duration: performance.now() - started, changes: after.n - before, last_row_id: after.id, changed_db: after.n !== before, rows_read: cursor.rowsRead, rows_written: cursor.rowsWritten,
|
|
14
19
|
size_after: storage.sql.databaseSize, served_by: 'rer-native-demo', served_by_primary: true } };
|
|
15
20
|
}
|
|
16
|
-
return { query, batch: statements => storage.transactionSync(() => statements.map(statement => query(statement.query, statement.args))),
|
|
17
|
-
exec(query) { const started = performance.now(); storage.sql.exec(query).toArray(); return { count: query.split('\n').filter(line => line.trim()).length, duration: performance.now() - started }; } };
|
|
21
|
+
return { query: (sql, args) => storage.transactionSync(() => query(sql, args)), batch: statements => storage.transactionSync(() => statements.map(statement => query(statement.query, statement.args))),
|
|
22
|
+
exec(query) { return storage.transactionSync(() => { d1Preflight(query, [], table => d1ColumnCount(table, sql => storage.sql.exec(sql).toArray())); const started = performance.now(); storage.sql.exec(query).toArray(); guardRows(storage.sql, sql => ({ get: () => storage.sql.exec(sql).toArray()[0], all: () => storage.sql.exec(sql).toArray() }), undefined, query); return { count: query.split('\n').filter(line => line.trim()).length, duration: performance.now() - started }; }); } };
|
|
18
23
|
}
|
|
@@ -72,7 +72,7 @@ node <kit>/owner-cli.mjs deploy --declaration deploy.json --environment producti
|
|
|
72
72
|
node <kit>/owner-cli.mjs health --declaration deploy.json --environment production
|
|
73
73
|
```
|
|
74
74
|
|
|
75
|
-
The owner deployment declaration may set top-level `"productDirectory": ".rer/deploy/product"` for a product staged by the build or deploy hook. It must name a normalized checkout-relative subdirectory: no absolute path, backslash, empty, `.` or `..` segment. The directory must exist after the build or hook, with no symlink on its path. rer reads this field and passes `--product` only to the kit's packaging commands (`
|
|
75
|
+
The owner deployment declaration may set top-level `"productDirectory": ".rer/deploy/product"` for a product staged by the build or deploy hook. It must name a normalized checkout-relative subdirectory: no absolute path, backslash, empty, `.` or `..` segment. The directory must exist after the build or hook, with no symlink on its path. rer reads this field and passes `--product` only to the kit's packaging commands (`local-plan`, `deploy` and `health`). A public dry run runs the build lifecycle instead of the custom upload hook. A hook that invokes the kit itself still supplies `--product` on its own invocation; the wrapper reads the declaration for its own steps. Omitting the field keeps packaging at the checkout root. State, root ownership, secret changes and removal keep their existing locations and behavior; dry run needs no credentials and never reads target secrets.
|
|
76
76
|
|
|
77
77
|
The seed declares bindings, never names:
|
|
78
78
|
|
|
@@ -113,11 +113,24 @@ A refused provider call also carries `providerRefusal` on the thrown error: the
|
|
|
113
113
|
|
|
114
114
|
A transport failure carries `providerTransport`: the operation, sanitized endpoint, attempt count, whether an HTTP status was received and a structural network reason. Both standard and custom-hook `rer deploy` failures name the call and retain the same-input recovery command. Endpoint details omit URL credentials, queries, account ids and R2 object keys; signed D1 import destinations are withheld.
|
|
115
115
|
|
|
116
|
+
Run site failures retain the same error diagnostic in human output and in
|
|
117
|
+
`siteDeploy` on both JSON failure events. `providerRequest` gives the failed
|
|
118
|
+
step, method, endpoint path with account and Worker-content identifiers replaced
|
|
119
|
+
by placeholders, elapsed milliseconds, effective per-call timeout limit, and a
|
|
120
|
+
network cause class or HTTP status. Response excerpts omit URLs and credentials
|
|
121
|
+
before being bounded to 2 KiB of UTF-8. An HTTP refusal keeps its status even
|
|
122
|
+
when reading its body or preparing an existing safe replay reaches the deadline.
|
|
123
|
+
The provider limit remains 30 seconds per call, shared across its existing
|
|
124
|
+
replays and body reads; the remaining site readiness window can shorten it.
|
|
125
|
+
Asset upload buckets each get a separate call and name their batch position.
|
|
126
|
+
File count alone does not establish a required transfer time, so this diagnostic
|
|
127
|
+
change adds no retries and does not increase any timeout.
|
|
128
|
+
|
|
116
129
|
### rer deploy: inputs, the hosted owner, secret changes and removal
|
|
117
130
|
|
|
118
131
|
`rer deploy` runs these steps for a seed whose adapter uses `ownerDeploy` from `cli/deploy.mjs`.
|
|
119
132
|
|
|
120
|
-
- **Dry run.** Application `rer deploy --dry-run`
|
|
133
|
+
- **Dry run.** Application `rer deploy --dry-run` runs the local build lifecycle and calls `owner-cli.mjs local-plan` to validate and print target names, binding names and migration names. It needs no credentials, makes no Cloudflare requests and changes nothing remotely. Missing local secret inputs are guidance for the real deploy; target secrets, permissions and remote readiness remain unchecked. The lower-level kit `dry-run` command retains its credentialed target-secret checks.
|
|
121
134
|
|
|
122
135
|
- **Inputs.** Declared secrets come from the environment, then `--env-file FILE` (an owned 0600 file of `KEY=VALUE` lines), which wins. For a secret the Worker does not hold yet, rer supplies the setup manifest's owner email (`emailField`) from the local setup's private settings, and for each secret field the manifest generates (`generate`) the value rer keeps for this environment in `.rer/deploy/<environment>.secrets.env` (0600, ignored by Git), else a fresh random value, which rer keeps once the Worker holds it (the one-time setup token is always fresh and never kept); a value the Worker already holds is kept. The build runs with the checkout's local settings files moved aside, and a `dist/` holding a `.dev.vars*`, `.env*` or `*.env` file is refused by name before any provider request. A still-missing secret stops the run before any write with a message that names it, unless its declaration says `"optional": true`: then the deploy binds nothing for it and reports it as not set, and `--set-secret` adds it later.
|
|
123
136
|
- **Hosted owner.** After deploy and health pass, rer claims the setup owner on the hosted app: it POSTs the setup token to the seed's standard owner step (`hooks.owner`, by default `/auth/bootstrap`). 303 means claimed; `409 already_initialized` means an owner was already claimed and is kept. Then rer deletes the token secret from the Worker, records the new Worker version in the state file, and lists the token in `retiredSecrets`, so later plans no longer declare it. The build fingerprint still names it, and the recorded claim stays as recorded, so an untouched repeat deploy writes nothing. A Worker that still holds a token rer did not upload in this run (an older deploy) first gets a fresh token through the same recorded secret change. Dependency approvals that need the token run before the claim. A seed with an owner step function, an identity-provider claim, or no declared token secret claims its owner its own way.
|
|
@@ -251,6 +264,13 @@ No Single Redirect (dynamic redirect ruleset) permission is needed. Base's refer
|
|
|
251
264
|
|
|
252
265
|
## Migration safety
|
|
253
266
|
|
|
267
|
+
Failed D1 imports carry `d1Import` on the error and in both `rer deploy --json`
|
|
268
|
+
failure events: `step` (`init`, `upload`, `ingest`, or `poll`), HTTP `status`
|
|
269
|
+
(null for transport failures), provider `errors`, and a redacted `body` excerpt
|
|
270
|
+
of at most 2 KiB of UTF-8. Transport failures also keep `causeCode`. The upload
|
|
271
|
+
refusal code remains `d1-import-upload-failed`. Credentials and signed URLs are
|
|
272
|
+
removed before excerpts are bounded; HTTP refusals are not retried.
|
|
273
|
+
|
|
254
274
|
SQL files have ordered unique names and SHA-256 identities. `d1_migrations` holds controller history with ordered IDs, names, timestamps, SHA-256 hashes and states. Native archives already exclude this installation table. Standard Wrangler name-only inserts remain possible, but their missing hashes/states block subsequent Base deployment pending reviewed adoption. Every applied entry must name a migration in the desired set, once, with its unchanged bytes (or bytes a reviewed rewrite replaced); entries may skip migrations, because a release can add migrations that sort below ones an install already applied. The desired set's unapplied migrations then run in name order, as Wrangler applies them, and a ledger that holds an entry the build lacks, changed bytes or an unconfirmed claim is refused. A new migration gets an `applying` claim before its SQL and completion marker are submitted together through the [Cloudflare D1 SQL import API](https://developers.cloudflare.com/api/resources/d1/subresources/database/methods/import/), using the same whole-file protocol as Wrangler to preserve compound trigger bodies and atomic SQL/completion. Uploads use bounded HTTPS R2 destinations, without account credentials; polling is bounded and signed URLs are never diagnostic output. A failed import keeps Cloudflare's own codes and messages on the error (`providerRefusal`, which `rer deploy` prints, with credentials and URLs removed and each message cut to 300 characters) and names the migration it was running.
|
|
255
275
|
|
|
256
276
|
An import that did not complete leaves its `applying` claim. The next deploy of the same migration bytes runs it again only when it proves none of it applied (`migration-retry.mjs`): the import is atomic, so the migration's effects that would make a second run fail (a CREATE without IF NOT EXISTS, ADD COLUMN, RENAME) and that last to the end of the file must all be absent. Then no attempt committed, and one that commits late makes the re-run fail whole instead of applying twice. A name the file drops or renames away existed before it ran, so a table it rebuilds under the same name, and that table's indexes, are not checked. A migration with no such effect, or with any of them present, stays refused as `uncertain-migration-claim`; the message names the migration, what is present, and the exact step: undo what is present and deploy again, or confirm by hand and delete or complete its claim with `wrangler d1 execute`. The kit never re-runs a migration whose effects may be present, rolls back user data, or marks an unknown outcome successful. Any other unconfirmed or changed history is still `changed-or-uncertain-migration-history`. Readback checks every final identity.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { transportReason } from './provider-transport.mjs';
|
|
2
|
+
|
|
3
|
+
// Import replies only need a bounded diagnostic prefix. A large refusal must
|
|
4
|
+
// keep its HTTP status rather than becoming a generic response-size failure.
|
|
5
|
+
export async function importResponseText(response, onReadError) {
|
|
6
|
+
if (!response.body) return '';
|
|
7
|
+
const reader = response.body.getReader(), chunks = []; let length = 0;
|
|
8
|
+
try {
|
|
9
|
+
while (length < 64 * 1024) {
|
|
10
|
+
let part;
|
|
11
|
+
try { part = await reader.read(); } catch (cause) { throw onReadError(cause); }
|
|
12
|
+
if (part.done) break;
|
|
13
|
+
const bytes = part.value.subarray(0,64 * 1024-length);
|
|
14
|
+
chunks.push(bytes); length += bytes.length;
|
|
15
|
+
}
|
|
16
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
17
|
+
} finally { await reader.cancel().catch(() => {}); }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Redact complete provider text before bounding UTF-8 bytes. In particular a
|
|
21
|
+
// credential straddling the excerpt boundary must never become a public prefix.
|
|
22
|
+
export function importText(text, redact = String, limit = 2048) {
|
|
23
|
+
const safe = redact(String(text)).replace(/\\\//g, '/').replace(/\bhttps?:\/\/[^\s<>"']+/gi, '[URL omitted]')
|
|
24
|
+
.replace(/[\x00-\x1f\x7f-\x9f]+/g, ' ').trim();
|
|
25
|
+
let end = Math.min(safe.length, limit);
|
|
26
|
+
while (Buffer.byteLength(safe.slice(0, end)) > limit) end--;
|
|
27
|
+
// Do not leave a split surrogate at the end of an excerpt.
|
|
28
|
+
if (/[\uD800-\uDBFF]/.test(safe[end - 1] ?? '')) end--;
|
|
29
|
+
return safe.slice(0, end);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function importDiagnostic(error, {step, status = null, body = '', payload, redact = String, cause}) {
|
|
33
|
+
const xmlCode = /<Code>([^<]{1,120})<\/Code>/.exec(body)?.[1];
|
|
34
|
+
const xmlMessage = /<Message>([^<]*)<\/Message>/.exec(body)?.[1];
|
|
35
|
+
const items = [payload?.error, ...(Array.isArray(payload?.errors) ? payload.errors : []),
|
|
36
|
+
...(xmlCode || xmlMessage ? [{code:xmlCode,message:xmlMessage}] : [])];
|
|
37
|
+
const hasCode = code => Number.isSafeInteger(code) || typeof code === 'string' && /^[A-Za-z0-9_-]{1,120}$/.test(code);
|
|
38
|
+
const errors = items.map(item => typeof item === 'string' ? {message:item} : item)
|
|
39
|
+
.filter(item => hasCode(item?.code) || typeof item?.message === 'string').slice(0, 3)
|
|
40
|
+
.map(item => ({...(hasCode(item.code) ? {code:typeof item.code === 'number' ? item.code : importText(item.code,redact,120)} : {}),
|
|
41
|
+
...(typeof item.message === 'string' ? {message:importText(item.message, redact, 300)} : {})}));
|
|
42
|
+
error.status = status;
|
|
43
|
+
error.d1Import = {step, status, errors, body:importText(payload === undefined ? body : JSON.stringify(payload), redact),
|
|
44
|
+
...(cause ? {causeCode:transportReason(cause)} : {})};
|
|
45
|
+
return error;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function importFailureText(error, redact = String) {
|
|
49
|
+
const detail = error?.d1Import;
|
|
50
|
+
if (!detail) return undefined;
|
|
51
|
+
const said = detail.errors.map(item => [item.code, item.message].filter(value => value !== undefined).join(' ')).join('; ');
|
|
52
|
+
return `D1 import ${detail.step}: ${detail.status === null ? 'no HTTP status' : `HTTP ${detail.status}`}`
|
|
53
|
+
+ (detail.causeCode ? ` (${detail.causeCode})` : '')
|
|
54
|
+
+ (said ? `; ${importText(said, redact, 1000)}` : '')
|
|
55
|
+
+ (detail.body ? `; response: ${importText(detail.body, redact)}` : '');
|
|
56
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { assert, uuid } from './validation.mjs';
|
|
3
3
|
import { recordDeployFailure } from './diagnostics.mjs';
|
|
4
|
+
import { importDiagnostic } from './d1-import-diagnostics.mjs';
|
|
4
5
|
|
|
5
6
|
export const D1_IMPORT_POLL_LIMIT = 100;
|
|
6
7
|
|
|
@@ -26,7 +27,18 @@ function importFailure(code, id, response, redact) {
|
|
|
26
27
|
export async function importD1({ envelope, upload, redact = String }, id, sql) {
|
|
27
28
|
assert(uuid(id) && typeof sql === 'string' && Buffer.byteLength(sql) <= 4 * 1024 * 1024, 'invalid-d1-import');
|
|
28
29
|
const bytes = Buffer.from(sql), etag = createHash('md5').update(bytes).digest('hex');
|
|
29
|
-
|
|
30
|
+
let exchange, step;
|
|
31
|
+
const post = async body => {
|
|
32
|
+
step = body.action; exchange = undefined;
|
|
33
|
+
try {
|
|
34
|
+
return (await envelope(`/d1/database/${id}/import`, { method:'POST', body, onResponse:value => { exchange = value; } })).result;
|
|
35
|
+
} catch (error) {
|
|
36
|
+
let payload; try { payload = JSON.parse(exchange?.body); } catch { /* Non-JSON refusals still keep their excerpt. */ }
|
|
37
|
+
throw recordDeployFailure(importDiagnostic(error,{step,status:exchange?.status ?? error.status ?? error.providerTransport?.status ?? null,
|
|
38
|
+
body:exchange?.body,payload,redact,cause:error.providerTransport ? {code:error.providerTransport.reason,name:error.providerTransport.reason} : undefined}), {});
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
const fail = code => importDiagnostic(importFailure(code, id, response, redact),{step,status:exchange?.status ?? null,body:exchange?.body,payload:response,redact});
|
|
30
42
|
let response = await post({ action:'init', etag });
|
|
31
43
|
if (response && Object.hasOwn(response, 'upload_url')) {
|
|
32
44
|
assert(typeof response.filename === 'string' && response.filename.length > 0 && response.filename.length <= 1024 && !/[\x00-\x1f\x7f]/.test(response.filename), 'invalid-d1-import-filename');
|
|
@@ -35,15 +47,15 @@ export async function importD1({ envelope, upload, redact = String }, id, sql) {
|
|
|
35
47
|
}
|
|
36
48
|
for (let polls = 0; polls < D1_IMPORT_POLL_LIMIT; polls++) {
|
|
37
49
|
// REST places errors on the envelope; validate nested errors when supplied.
|
|
38
|
-
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) throw
|
|
50
|
+
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) throw fail('d1-import-failed');
|
|
39
51
|
if (response.status === 'complete') {
|
|
40
|
-
if (!(Number.isSafeInteger(response.result?.num_queries) && response.result.num_queries > 0 && typeof response.result.final_bookmark === 'string' && response.result.final_bookmark.length > 0)) throw
|
|
52
|
+
if (!(Number.isSafeInteger(response.result?.num_queries) && response.result.num_queries > 0 && typeof response.result.final_bookmark === 'string' && response.result.final_bookmark.length > 0)) throw fail('d1-import-invalid-result');
|
|
41
53
|
return;
|
|
42
54
|
}
|
|
43
|
-
if (!(response.status === 'active' && typeof response.at_bookmark === 'string' && response.at_bookmark.length > 0 && response.at_bookmark.length <= 1024)) throw
|
|
55
|
+
if (!(response.status === 'active' && typeof response.at_bookmark === 'string' && response.at_bookmark.length > 0 && response.at_bookmark.length <= 1024)) throw fail('d1-import-uncertain');
|
|
44
56
|
response = await post({ action:'poll', current_bookmark:response.at_bookmark });
|
|
45
57
|
}
|
|
46
|
-
throw
|
|
58
|
+
throw fail('d1-import-poll-limit');
|
|
47
59
|
}
|
|
48
60
|
|
|
49
61
|
export function importUploadURL(value) {
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { importFailureText, importText } from './d1-import-diagnostics.mjs';
|
|
2
|
+
|
|
1
3
|
// Provider text is private until the CLI's existing credential redactor approves
|
|
2
4
|
// it. Identity, rather than mutable Error properties, carries this diagnostic.
|
|
3
5
|
const failures = new WeakMap();
|
|
@@ -11,18 +13,32 @@ export function deployDiagnostic(error, redact) {
|
|
|
11
13
|
const clean = value => (redact ?? String)(String(value)).replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f]/g,'').replace(/\b[a-f0-9]{32}\b/gi,'[REDACTED]');
|
|
12
14
|
const failureCode = detail.code ?? error.message;
|
|
13
15
|
const code = /^[a-zA-Z0-9_-]{1,120}$/.test(failureCode) ? clean(failureCode) : 'deploy-failed';
|
|
14
|
-
return {code,step:detail.step ?? 'deploy',created:(detail.created ?? []).map(clean),
|
|
16
|
+
return {code,step:clean(detail.step ?? 'deploy'),created:(detail.created ?? []).map(clean),
|
|
17
|
+
...(detail.providerRequest ? {providerRequest:{...detail.providerRequest,endpoint:clean(detail.providerRequest.endpoint),step:clean(detail.step ?? 'deploy'),body:redact ? providerText(detail.output ?? '',redact) : '(provider output withheld; rer deploy provides redacted details)'}} : {}),
|
|
15
18
|
...(typeof error.resource==='string' && /^(?:d1|r2|kv|queues|worker|service):[a-z0-9-]+$/.test(error.resource) ? {resource:clean(error.resource)} : {}),
|
|
16
19
|
...(detail.providerCodes?.length ? {providerCodes:detail.providerCodes} : {}),
|
|
20
|
+
...(error.d1Import ? {d1Import:{...error.d1Import,body:importText(error.d1Import.body,redact),
|
|
21
|
+
errors:error.d1Import.errors.map(item => ({...item,...(item.message ? {message:importText(item.message,redact,300)} : {})}))}} : {}),
|
|
17
22
|
...(redact && providerFailure(error,redact) ? {provider:providerFailure(error,redact)} : {}),
|
|
18
23
|
output:(detail.output ? (redact ? clean(detail.output) : '(provider output withheld; rer deploy provides redacted details)') : '(no provider output)').split(/\r?\n/).slice(0,8).map(line=>line.slice(0,500)).join('\n')};
|
|
19
24
|
}
|
|
25
|
+
// Reuse the D1 upload retention boundary: redact complete text, omit URLs,
|
|
26
|
+
// then bound UTF-8 bytes so a credential cannot straddle the public excerpt.
|
|
27
|
+
export function providerText(text, redact = String, limit = 2048) {
|
|
28
|
+
const safe = redact(String(text)).replace(/\\\//g, '/').replace(/\bhttps?:\/\/[^\s<>"']+/gi, '[URL omitted]')
|
|
29
|
+
.replace(/[\x00-\x1f\x7f-\x9f]+/g, ' ').replace(/\b[a-f0-9]{32}\b/gi,'[REDACTED]').trim();
|
|
30
|
+
let end = Math.min(safe.length, limit);
|
|
31
|
+
while (Buffer.byteLength(safe.slice(0, end)) > limit) end--;
|
|
32
|
+
if (/[\uD800-\uDBFF]/.test(safe[end - 1] ?? '')) end--;
|
|
33
|
+
return safe.slice(0, end);
|
|
34
|
+
}
|
|
35
|
+
|
|
20
36
|
/** One line naming the provider call that was refused and what Cloudflare said, from
|
|
21
37
|
* the error's own `providerRefusal` (set by any copy of the kit); undefined otherwise. */
|
|
22
38
|
export function providerRefusal(error, redact = String) {
|
|
23
39
|
const refusal = error?.providerRefusal;
|
|
24
40
|
if (typeof refusal?.operation !== 'string' || !Array.isArray(refusal.errors)) return undefined;
|
|
25
|
-
const said = refusal.errors.map(item => [item?.code, typeof item?.message === 'string' ?
|
|
41
|
+
const said = refusal.errors.map(item => [item?.code, typeof item?.message === 'string' ? providerText(item.message,redact,300) : undefined].filter(part => part !== undefined && part !== '').join(' ')).filter(Boolean).slice(0,3);
|
|
26
42
|
return `Cloudflare refused ${redact(refusal.operation).slice(0,200)}${said.length ? `: ${said.join('; ')}` : ''}`;
|
|
27
43
|
}
|
|
28
44
|
/** What a migration that failed or stays refused means for the next run, from the
|
|
@@ -101,7 +117,6 @@ export function deploymentDiagnostics(api, {onCreated} = {}) {
|
|
|
101
117
|
},
|
|
102
118
|
annotateFailure(error, step = currentStep) {
|
|
103
119
|
const prior = failures.get(error);
|
|
104
|
-
if (currentCall) error.providerCall ??= {operation:currentCall,reason:error.message};
|
|
105
120
|
// Node filesystem/CLI codes are structural; their messages can hold paths
|
|
106
121
|
// and inputs, so retain them privately for the caller's redaction boundary.
|
|
107
122
|
const cliCode = typeof error.code === 'string' && /^(?:E[A-Z0-9_]{1,63}|health-[a-z-]+)$/.test(error.code) ? error.code : undefined;
|
|
@@ -113,6 +128,17 @@ export function deploymentDiagnostics(api, {onCreated} = {}) {
|
|
|
113
128
|
|
|
114
129
|
/** All provider failure paths share the same safe, portable rendering. */
|
|
115
130
|
export function providerFailure(error, redact = String) {
|
|
131
|
+
const imported = importFailureText(error,redact);
|
|
132
|
+
if (imported) return imported;
|
|
133
|
+
const retained = failures.get(error)?.providerRequest;
|
|
134
|
+
if (retained) {
|
|
135
|
+
const step = providerText(failures.get(error).step,redact,200);
|
|
136
|
+
const cause = retained.cause === 'timeout' ? `timeout after ${retained.timeoutMs / 1000} s (limit ${retained.timeoutMs} ms)` : retained.cause === 'connection timeout' ? `connection timeout (per-call limit ${retained.timeoutMs} ms)` : retained.cause;
|
|
137
|
+
const body = providerText(failures.get(error).output ?? '',redact,300);
|
|
138
|
+
const outcome = retained.status === null ? cause : `HTTP ${retained.status}${retained.cause === 'HTTP refusal' ? '' : `; ${cause}`}`;
|
|
139
|
+
const specific = providerRefusal(error,redact) ?? providerTransport(error,redact);
|
|
140
|
+
return `${step}: ${retained.method} ${providerText(retained.endpoint,redact,300)}; elapsed ${retained.elapsedMs} ms; ${outcome}${specific ? `; ${specific}` : ''}${body ? `; ${body}` : ''}`;
|
|
141
|
+
}
|
|
116
142
|
const specific = providerRefusal(error,redact) ?? providerTransport(error,redact);
|
|
117
143
|
if (!specific && error?.cause instanceof Error) return providerFailure(error.cause,redact);
|
|
118
144
|
if (specific) return specific;
|
|
@@ -15,10 +15,11 @@ import { ownerRemove } from './owner-remove.mjs';
|
|
|
15
15
|
import { ownerSignin } from './owner-signin.mjs';
|
|
16
16
|
import { ownerRekey } from './owner-rekey.mjs';
|
|
17
17
|
import { refuseLocalModels } from './local-only.mjs';
|
|
18
|
+
import { validateMigrations } from './migrations.mjs';
|
|
18
19
|
import { publicOrigin } from './public-origin.mjs';
|
|
19
20
|
|
|
20
21
|
export const cronNotice = 'This Worker has a Cron Trigger. Cloudflare may take up to about two hours to start its first scheduled run on some accounts.';
|
|
21
|
-
const usage = 'usage: owner-cli.mjs deploy|dry-run|health|remove|inspect|record-owner|signin|rekey --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
|
|
22
|
+
const usage = 'usage: owner-cli.mjs deploy|dry-run|local-plan|health|remove|inspect|record-owner|signin|rekey --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
|
|
22
23
|
|
|
23
24
|
/** The owner's committed source and existing local build. Run the seed's build first. */
|
|
24
25
|
async function ownerArtifact(root, product) {
|
|
@@ -34,13 +35,15 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
34
35
|
try { parsed = parseArgs({args:argv,allowPositionals:true,strict:true,options:{declaration:{type:'string'},environment:{type:'string'},product:{type:'string',default:''},state:{type:'string'},set:{type:'string',multiple:true},unset:{type:'string',multiple:true},outcome:{type:'string'}}}); }
|
|
35
36
|
catch { throw new Error('usage'); }
|
|
36
37
|
const {positionals:[command,...rest],values} = parsed;
|
|
37
|
-
if (!['deploy','dry-run','health','secret','remove','inspect','record-owner','signin','rekey'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
|
|
38
|
+
if (!['deploy','dry-run','local-plan','health','secret','remove','inspect','record-owner','signin','rekey'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
|
|
38
39
|
if ((command === 'secret') !== (values.set !== undefined || values.unset !== undefined) || values.set !== undefined && values.unset !== undefined) throw new Error('usage');
|
|
39
40
|
if (values.outcome !== undefined && command !== 'record-owner' || command === 'record-owner' && !['claimed','kept'].includes(values.outcome)) throw new Error('usage');
|
|
40
41
|
await refuseLocalModels(resolve(cwd));
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
if (command !== 'local-plan') {
|
|
43
|
+
const missing = ['CLOUDFLARE_ACCOUNT_ID','CLOUDFLARE_API_TOKEN'].filter(key => typeof env[key] !== 'string' || !env[key].trim());
|
|
44
|
+
if (missing.length) { const error = new Error('missing-input'); error.inputs = missing; throw error; }
|
|
45
|
+
assert(hexId(env.CLOUDFLARE_ACCOUNT_ID) && !/[\r\n]/.test(env.CLOUDFLARE_API_TOKEN),'invalid-cloudflare-input');
|
|
46
|
+
}
|
|
44
47
|
const root = resolve(cwd);
|
|
45
48
|
let declaration;
|
|
46
49
|
try { declaration = JSON.parse(await regularBytes(resolve(root,values.declaration),256 * 1024)); }
|
|
@@ -49,10 +52,20 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
49
52
|
// Validate even direct kit invocations before any provider request or write.
|
|
50
53
|
publicOrigin(declaration, env);
|
|
51
54
|
let artifact;
|
|
52
|
-
if (command === 'dry-run') {
|
|
55
|
+
if (command === 'dry-run' || command === 'local-plan') {
|
|
53
56
|
artifact = await packageBuild({root,source:{sha:'0'.repeat(40),tree:'0'.repeat(40)},runIdentity:{id:'0',attempt:'1'},productDirectory:values.product});
|
|
54
57
|
const worker = ownerWorkerName(declaration.project,values.environment);
|
|
55
|
-
ownerPlan({project:declaration.project,environment:values.environment,worker,origin:`https://${worker}.example.invalid`},declaration,artifact,{localOnlyKeys,env});
|
|
58
|
+
const plan = ownerPlan({project:declaration.project,environment:values.environment,worker,origin:`https://${worker}.example.invalid`},declaration,artifact,{localOnlyKeys,env});
|
|
59
|
+
if (command === 'local-plan') for (const item of plan.d1) validateMigrations(item.migrations);
|
|
60
|
+
// Print names only: configuration values and migration SQL stay local.
|
|
61
|
+
// Return before constructing a provider client or resolving an account.
|
|
62
|
+
if (command === 'local-plan') return {ok:true,dryRun:true,deployed:false,plan:{
|
|
63
|
+
worker:plan.worker,
|
|
64
|
+
d1:plan.d1.map(({binding,name,migrations}) => ({binding,name,migrations:migrations.map(item => item.name)})),
|
|
65
|
+
r2:plan.r2,kv:plan.kv ?? [],queues:plan.queues,durableObjects:plan.durableObjects,services:plan.services ?? [],
|
|
66
|
+
secrets:plan.secrets.map(({binding,optional}) => ({binding,optional:optional ?? false})),
|
|
67
|
+
vars:Object.keys(plan.vars),crons:plan.crons,
|
|
68
|
+
}};
|
|
56
69
|
}
|
|
57
70
|
// Removal reads identity before each object DELETE and may span many pages.
|
|
58
71
|
// Bound each response, not the owner's bucket size; failures retain ownership.
|
|
@@ -25,9 +25,9 @@ export async function providerResponse({send, method, timeoutMs, callerSignal, r
|
|
|
25
25
|
const timer = firstRead ? setTimeout(() => firstRead.abort(new DOMException('Provider read timed out','TimeoutError')), Math.max(1, Math.ceil(timeoutMs / 2))) : undefined;
|
|
26
26
|
timer?.unref();
|
|
27
27
|
const attemptSignal = firstRead ? AbortSignal.any([signal, firstRead.signal]) : signal;
|
|
28
|
-
let readingBody = false;
|
|
28
|
+
let readingBody = false, response;
|
|
29
29
|
try {
|
|
30
|
-
|
|
30
|
+
response = await send(attemptSignal);
|
|
31
31
|
clearTimeout(timer);
|
|
32
32
|
// A retried delete can observe the first attempt's committed deletion.
|
|
33
33
|
if (removalReplay && method === 'DELETE' && attempt > 1 && response.status === 404) {
|
|
@@ -40,14 +40,17 @@ export async function providerResponse({send, method, timeoutMs, callerSignal, r
|
|
|
40
40
|
continue;
|
|
41
41
|
}
|
|
42
42
|
readingBody = true;
|
|
43
|
-
const output = consume ? await consume(response,attempt) : undefined;
|
|
43
|
+
const output = consume ? await consume(response,attempt,attemptSignal) : undefined;
|
|
44
44
|
return {response,attempts:attempt,output};
|
|
45
45
|
}
|
|
46
46
|
catch (cause) {
|
|
47
|
+
// A received HTTP refusal remains an HTTP refusal if the shared deadline
|
|
48
|
+
// expires while preparing its existing safe replay.
|
|
49
|
+
if (response && !response.ok && signal.aborted) return {response,attempts:attempt,output:'[response retry deadline expired]'};
|
|
47
50
|
const bodyReset = readingBody && replayableAsset && cause.message === 'provider-transport-failed';
|
|
48
51
|
if (attempt === 1 && !signal.aborted && (bodyReset || !readingBody && (read || replayableAsset || unsent(cause))) && reserveRetry()) continue;
|
|
49
52
|
if (readingBody) throw cause;
|
|
50
|
-
throw failure(cause, attempt);
|
|
53
|
+
throw failure(timeout.aborted ? timeout.reason : cause, attempt);
|
|
51
54
|
}
|
|
52
55
|
// Body consumption and scoped asset replays keep the total deadline.
|
|
53
56
|
finally { clearTimeout(timer); }
|
|
@@ -77,3 +80,15 @@ export function transportReason(error) {
|
|
|
77
80
|
}
|
|
78
81
|
return 'network error';
|
|
79
82
|
}
|
|
83
|
+
|
|
84
|
+
/** Structural classes only; raw network messages stay behind the redactor. */
|
|
85
|
+
export function transportClass(error) {
|
|
86
|
+
const reason = transportReason(error);
|
|
87
|
+
if (reason === 'TimeoutError') return 'timeout';
|
|
88
|
+
if (reason === 'UND_ERR_CONNECT_TIMEOUT') return 'connection timeout';
|
|
89
|
+
if (/ECONNRESET|EPIPE|UND_ERR_SOCKET/.test(reason)) return 'connection reset';
|
|
90
|
+
if (/ENOTFOUND|EAI_AGAIN/.test(reason)) return 'DNS failure';
|
|
91
|
+
if (/TLS|CERT|SSL/.test(reason)) return 'TLS failure';
|
|
92
|
+
if (reason === 'AbortError') return 'cancelled';
|
|
93
|
+
return 'network failure';
|
|
94
|
+
}
|
|
@@ -1,32 +1,44 @@
|
|
|
1
1
|
import { providerPages, providerListShape } from './pagination.mjs';
|
|
2
2
|
import { deleteR2Object } from './r2-object-transport.mjs';
|
|
3
3
|
import { isR2ObjectDelete } from './r2-objects.mjs';
|
|
4
|
-
import { recordDeployFailure } from './diagnostics.mjs';
|
|
4
|
+
import { recordDeployFailure, providerText } from './diagnostics.mjs';
|
|
5
5
|
import { D1_IMPORT_POLL_LIMIT, importD1, importUploadURL } from './d1-import.mjs';
|
|
6
|
+
import { importDiagnostic, importResponseText } from './d1-import-diagnostics.mjs';
|
|
6
7
|
import { assert, hexId, uuid } from './validation.mjs';
|
|
7
|
-
import { providerResponse, transportOutput, transportReason } from './provider-transport.mjs';
|
|
8
|
+
import { providerResponse, transportOutput, transportReason, transportClass } from './provider-transport.mjs';
|
|
8
9
|
|
|
9
10
|
// A gateway/server error can arrive after a write. An explicit API refusal cannot.
|
|
10
11
|
export const definiteProviderRefusal = error => error.status === 200 || error.status >= 400 && error.status < 500;
|
|
11
12
|
|
|
12
|
-
async function boundedBytes(response, limit, onReadError) {
|
|
13
|
+
async function boundedBytes(response, limit, onReadError, prefix = false) {
|
|
13
14
|
assert(response.body, 'empty-response');
|
|
14
15
|
const reader = response.body.getReader(); const chunks = []; let length = 0;
|
|
15
16
|
try { while (true) {
|
|
16
17
|
let chunk;
|
|
17
18
|
// Only a failed stream read is transport failure; retain size/shape codes.
|
|
18
|
-
try { chunk = await reader.read(); } catch (cause) {
|
|
19
|
-
|
|
19
|
+
try { chunk = await reader.read(); } catch (cause) {
|
|
20
|
+
if (prefix) { chunks.push(Buffer.from(` [response read failed: ${transportReason(cause)}]`)); break; }
|
|
21
|
+
throw onReadError ? onReadError(cause) : cause;
|
|
22
|
+
}
|
|
23
|
+
const { done } = chunk; if (done) break;
|
|
24
|
+
const value = prefix ? chunk.value.subarray(0,limit-length) : chunk.value;
|
|
20
25
|
length += value.length; assert(length <= limit, 'response-too-large'); chunks.push(value);
|
|
26
|
+
if (prefix && length >= limit) break;
|
|
21
27
|
} }
|
|
22
28
|
finally { await reader.cancel().catch(() => {}); }
|
|
23
29
|
return Buffer.concat(chunks);
|
|
24
30
|
}
|
|
25
|
-
const boundedText = async (response, limit = 4 * 1024 * 1024, onReadError) => (await boundedBytes(response, limit, onReadError)).toString('utf8');
|
|
31
|
+
const boundedText = async (response, limit = 4 * 1024 * 1024, onReadError, prefix) => (await boundedBytes(response, limit, onReadError, prefix)).toString('utf8');
|
|
26
32
|
export async function boundedJSON(response, limit) {
|
|
27
33
|
try { return JSON.parse(await boundedText(response,limit)); } catch (error) { if (error instanceof SyntaxError) throw new Error('malformed-response'); throw error; }
|
|
28
34
|
}
|
|
29
|
-
const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>
|
|
35
|
+
const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>{
|
|
36
|
+
value = value.split(secret).join('[REDACTED]');
|
|
37
|
+
// A bounded response may end partway through a known credential. Withhold
|
|
38
|
+
// that suffix too, before the shorter public excerpt is selected.
|
|
39
|
+
const at = secret.length >= 8 ? value.lastIndexOf(secret.slice(0,8)) : -1;
|
|
40
|
+
return at >= 0 && secret.startsWith(value.slice(at)) ? value.slice(0,at)+'[REDACTED]' : value;
|
|
41
|
+
},text);
|
|
30
42
|
/** Only this fixed origin and the explicitly selected account ever receive the token. */
|
|
31
43
|
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false, budgetMigrations = false, requestSignal = () => undefined }) {
|
|
32
44
|
assert(hexId(accountId), 'invalid-cloudflare-account-id');
|
|
@@ -50,7 +62,7 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
50
62
|
maxRequests = limit;
|
|
51
63
|
}
|
|
52
64
|
// `zone` scope: the zone-level API (/zones/...), for a run site's zone lookup and redirect rule.
|
|
53
|
-
async function rawEnvelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal } = {}) {
|
|
65
|
+
async function rawEnvelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal, step, deadlineMs, onResponse } = {}) {
|
|
54
66
|
assert(typeof path === 'string' && (isR2ObjectDelete(path, method) || /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%')), 'invalid-provider-path');
|
|
55
67
|
assert(!zone || /^\/zones(?:[/?]|$)/.test(path), 'invalid-provider-path');
|
|
56
68
|
assert(++requests <= maxRequests, 'provider-request-budget');
|
|
@@ -62,21 +74,29 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
62
74
|
const redactAuth = text => /\/r2\/buckets\/[^/]+\/objects(?:[/?]|$)/.test(path) ? '[R2 object details omitted]' : /\/d1\/database\/[^/]+\/import$/.test(path) ? redactCredentials(text).replace(/\bhttps?:\/\/\S+/gi,'[URL omitted]') : redactCredentials(text);
|
|
63
75
|
const operation = `${method} ${path.split('?')[0].replace(/(\/r2\/buckets\/[^/]+\/objects)\/.*/, '$1/[object]')}`;
|
|
64
76
|
const endpoint = redactCredentials(`https://api.cloudflare.com/client/v4${zone ? '' : '/accounts/[REDACTED]'}${operation.slice(method.length + 1)}`);
|
|
77
|
+
const started = performance.now(), limitMs = Math.max(1,Math.ceil(Math.min(timeoutMs,deadlineMs ?? timeoutMs)));
|
|
78
|
+
const detail = (status, cause) => ({method,endpoint:endpoint.replace('https://api.cloudflare.com',''),status,elapsedMs:Math.round(performance.now()-started),timeoutMs:limitMs,cause});
|
|
79
|
+
const failureStep = step ?? `${method} ${operation.slice(method.length + 1)}`;
|
|
65
80
|
const transportFailure = (cause, attempts, status = null) => {
|
|
66
81
|
const error = new Error('provider-transport-failed');
|
|
67
82
|
// Portable across vendored kit copies used by custom deploy hooks. Raw
|
|
68
83
|
// network messages stay private until the CLI applies its redactor.
|
|
69
84
|
error.providerTransport = {operation:redactCredentials(operation),endpoint,attempts,status,reason:transportReason(cause)};
|
|
70
|
-
return recordDeployFailure(error,{output:
|
|
85
|
+
return recordDeployFailure(error,{step:failureStep,providerRequest:detail(status,transportClass(cause)),output:providerText(transportOutput(cause),redactAuth,Infinity)});
|
|
71
86
|
};
|
|
72
|
-
const {response,output = ''} = await providerResponse({method,timeoutMs,callerSignal:deadlineSignal(callerSignal),removal,
|
|
87
|
+
const {response,output = ''} = await providerResponse({method,timeoutMs:limitMs,callerSignal:deadlineSignal(callerSignal),removal,
|
|
73
88
|
replayableAsset:method === 'POST' && (/^\/workers\/scripts\/[^/]+\/assets-upload-session$/.test(path) || path === '/workers/assets/upload?base64=true'),
|
|
74
89
|
reserveRetry:() => requests < maxRequests && (++requests, true),failure:transportFailure,
|
|
75
|
-
consume:(response,attempts) =>
|
|
90
|
+
consume:(response,attempts,signal) => {
|
|
91
|
+
onResponse?.({status:response.status,body:''});
|
|
92
|
+
const failedRead = cause => transportFailure(signal.aborted ? signal.reason : cause,attempts,response.status);
|
|
93
|
+
return onResponse && !response.ok ? importResponseText(response,failedRead) : response.body ? boundedText(response,response.ok ? undefined : 64 * 1024,failedRead,!response.ok) : '';
|
|
94
|
+
},
|
|
76
95
|
send:signal => isR2ObjectDelete(path, method) ? objectDelete(`/client/v4/accounts/${accountId}${path}`, {headers:{Authorization:`Bearer ${token}`},signal}) : request(zone ? `https://api.cloudflare.com/client/v4${path}` : `https://api.cloudflare.com/client/v4/accounts/${accountId}${path}`, {
|
|
77
96
|
method, redirect: 'error', signal, headers: { Authorization: `Bearer ${assetToken ?? token}`, ...(body && !(body instanceof FormData) ? { 'Content-Type':'application/json' } : {}) },
|
|
78
97
|
...(body === undefined ? {} : { body: body instanceof FormData ? body : JSON.stringify(body) }),
|
|
79
98
|
})});
|
|
99
|
+
onResponse?.({status:response.status, body:output});
|
|
80
100
|
// A successful DELETE may have no response body (including HTTP 200).
|
|
81
101
|
// Nonempty replies and all other methods still require the API envelope.
|
|
82
102
|
if (method === 'DELETE' && response.ok && output === '') return { success: true, result: null };
|
|
@@ -92,8 +112,8 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
92
112
|
const said = (Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code) || typeof x?.message === 'string').slice(0,3)
|
|
93
113
|
.map(x => ({...(Number.isSafeInteger(x.code) ? {code:x.code} : {}),...(typeof x.message === 'string' ? {message:bounded(x.message)} : {})}));
|
|
94
114
|
error.providerRefusal = {operation:redactCredentials(operation),errors:said.length || payload !== undefined || !output.trim() ? said : [{message:bounded(output.trim().split('\n')[0])}]};
|
|
95
|
-
throw recordDeployFailure(error,{providerCodes:(Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code)).map(x=>x.code).slice(0,8),
|
|
96
|
-
output:
|
|
115
|
+
throw recordDeployFailure(error,{step:failureStep,providerRequest:detail(response.status,'HTTP refusal'),providerCodes:(Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code)).map(x=>x.code).slice(0,8),
|
|
116
|
+
output:providerText(/^\/d1\/database\//.test(path) && Array.isArray(payload?.errors) ? payload.errors.filter(x=>typeof x?.message==='string').map(x=>x.message).join('\n') : output,redactAuth,Infinity)});
|
|
97
117
|
}
|
|
98
118
|
assert(payload?.success === true && Object.hasOwn(payload,'result') && (!payload.errors || Array.isArray(payload.errors) && !payload.errors.length), 'provider-unsuccessful-or-malformed');
|
|
99
119
|
if (/^\/d1\/database\/[^/]+\/query$/.test(path) && Array.isArray(payload.result)) {
|
|
@@ -140,11 +160,18 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
140
160
|
assert(/^[a-z0-9-]{1,63}$/.test(worker) && /^[0-9a-f-]{36}$/.test(versionId), 'invalid-provider-path');
|
|
141
161
|
assert(++requests <= maxRequests, 'provider-request-budget');
|
|
142
162
|
const operation = 'GET /workers/scripts/[worker]/content/v2';
|
|
163
|
+
const endpoint = '/client/v4/accounts/[REDACTED]/workers/scripts/[worker]/content/v2', started = performance.now();
|
|
164
|
+
const retain = (error,status,cause,output) => recordDeployFailure(error,{step:'reading Worker content',
|
|
165
|
+
providerRequest:{method:'GET',endpoint,status,elapsedMs:Math.round(performance.now()-started),timeoutMs,cause},
|
|
166
|
+
output:providerText(output,text => redactValues(text,[accountId,token]),Infinity)});
|
|
167
|
+
const failure = (cause,attempts,status = null) => {
|
|
168
|
+
const reason = transportReason(cause);
|
|
169
|
+
return retain(Object.assign(new Error('provider-transport-failed'),{providerTransport:{operation,endpoint,attempts,status,reason}}),status,transportClass(cause),transportOutput(cause));
|
|
170
|
+
};
|
|
143
171
|
const {response,output} = await providerResponse({method:'GET',timeoutMs,callerSignal:deadlineSignal(),reserveRetry:() => requests < maxRequests && (++requests, true),
|
|
144
172
|
send:signal => request(`https://api.cloudflare.com/client/v4/accounts/${accountId}/workers/scripts/${worker}/content/v2?version=${versionId}`, {method:'GET',redirect:'error',signal,headers:{Authorization:`Bearer ${token}`}}),
|
|
145
|
-
consume:response => response.body ? boundedBytes(response, 64 * 1024 * 1024) : Buffer.alloc(0),
|
|
146
|
-
|
|
147
|
-
if (!response.ok) throw Object.assign(new Error(`provider-http-${response.status}`), {status:response.status,providerCall:{operation,reason:`provider-http-${response.status}`}});
|
|
173
|
+
consume:(response,attempts,signal) => response.body ? boundedBytes(response,response.ok ? 64 * 1024 * 1024 : 64 * 1024,cause => failure(signal.aborted ? signal.reason : cause,attempts,response.status),!response.ok) : Buffer.alloc(0),failure});
|
|
174
|
+
if (!response.ok) throw retain(Object.assign(new Error(`provider-http-${response.status}`), {status:response.status}),response.status,'HTTP refusal',output.toString('utf8'));
|
|
148
175
|
const type = response.headers.get('content-type') ?? '', entry = response.headers.get('cf-entrypoint');
|
|
149
176
|
assert(type.startsWith('multipart/form-data') && typeof entry === 'string' && entry, 'invalid-worker-content');
|
|
150
177
|
const parts = await new Response(output, {headers:{'content-type':type}}).formData();
|
|
@@ -154,15 +181,23 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
154
181
|
async function upload(url, bytes, etag) {
|
|
155
182
|
const destination = importUploadURL(url);
|
|
156
183
|
assert(++requests <= maxRequests, 'provider-request-budget');
|
|
157
|
-
const
|
|
184
|
+
const redact = text => redactValues(text,[accountId,token,destination]);
|
|
185
|
+
const failure = (cause, attempts, status = null) => recordDeployFailure(importDiagnostic(Object.assign(new Error('d1-import-upload-transport-failed'),
|
|
186
|
+
{providerTransport:{operation:'PUT D1 import upload',endpoint:'https://[REDACTED].r2.cloudflarestorage.com/[D1 import upload]',attempts,status,reason:transportReason(cause)}}),
|
|
187
|
+
{step:'upload',status,redact,cause}), {});
|
|
188
|
+
const {response,output = ''} = await providerResponse({method:'PUT',timeoutMs,callerSignal:deadlineSignal(),
|
|
158
189
|
reserveRetry:() => requests < maxRequests && (++requests, true),
|
|
159
190
|
send:signal => request(destination, { method:'PUT', redirect:'error', signal, headers:{'Content-Length':String(bytes.length)}, body:bytes }),
|
|
160
|
-
failure
|
|
191
|
+
failure,
|
|
192
|
+
consume:async (response, attempts) => {
|
|
193
|
+
if (response.status === 200) { await response.body?.cancel().catch(() => {}); return ''; }
|
|
194
|
+
return importResponseText(response,cause => failure(cause,attempts,response.status));
|
|
195
|
+
},
|
|
161
196
|
});
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
197
|
+
let payload; try { payload = JSON.parse(output); } catch { /* R2 may return plain text or XML. */ }
|
|
198
|
+
const fail = code => recordDeployFailure(importDiagnostic(new Error(code),{step:'upload',status:response.status,body:output,payload,redact}), {});
|
|
199
|
+
if (response.status !== 200) throw fail('d1-import-upload-failed');
|
|
200
|
+
if (response.headers.get('etag')?.replace(/^"|"$/g, '') !== etag) throw fail('d1-import-upload-etag-mismatch');
|
|
166
201
|
}
|
|
167
202
|
return { accountId, envelope, list, reserveMigrationRequests, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult, workerContent,
|
|
168
203
|
zoneRequest(path, options = {}) { return requestResult(path, { ...options, zone: true }); } };
|
|
@@ -27,6 +27,6 @@ export function readinessAPI(api, window, failure = 'deployment-readiness-timeou
|
|
|
27
27
|
const remaining = window.remaining();
|
|
28
28
|
assert(remaining > 0, failure);
|
|
29
29
|
const deadline = window.timeout(Math.max(1, Math.ceil(remaining)));
|
|
30
|
-
return api.request(path, {...options, signal:options.signal ? AbortSignal.any([options.signal,deadline]) : deadline});
|
|
30
|
+
return api.request(path, {...options, deadlineMs:remaining, signal:options.signal ? AbortSignal.any([options.signal,deadline]) : deadline});
|
|
31
31
|
}};
|
|
32
32
|
}
|