@runeditrun/rer 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/bounded.mjs +1 -1
- package/cli/deploy.mjs +1 -1
- package/cli/errors.mjs +4 -0
- package/cli/transfers.mjs +1 -0
- package/global/CHANGELOG.md +20 -0
- package/modules/deploy-kit/README.md +5 -1
- package/modules/deploy-kit/owner-cli.mjs +3 -1
- package/modules/deploy-kit/owner-remove.mjs +38 -8
- package/modules/deploy-kit/provider.mjs +7 -5
- package/modules/deploy-kit/r2-object-transport.mjs +19 -0
- package/modules/deploy-kit/r2-objects.mjs +34 -0
- package/modules/job-queue/load-admission.mjs +38 -0
- package/modules/mail/package.json +2 -1
- package/modules/mail/pnpm-lock.yaml +1 -0
- package/modules/operator-auth/README.md +4 -1
- package/modules/operator-auth/identity/email-link/index.mjs +6 -2
- package/modules/owner-claim/README.md +8 -6
- package/modules/owner-claim/transaction.mjs +4 -4
- package/modules/portability/README.md +1 -0
- package/modules/portability/cli.mjs +11 -17
- package/modules/portability/format.mjs +2 -2
- package/modules/portability/http-client.mjs +97 -0
- package/modules/portability/lines.mjs +3 -1
- package/modules/portability/storage.mjs +4 -1
- package/onboarding/commands.mjs +2 -1
- package/onboarding/queue.mjs +8 -1
- package/onboarding/queued-entry.mjs +1 -1
- package/package.json +1 -1
package/cli/bounded.mjs
CHANGED
|
@@ -24,7 +24,7 @@ export async function boundedCommand(project,options,argv,out){
|
|
|
24
24
|
const {createCommands}=await shared('commands.mjs');
|
|
25
25
|
store=await createCommands(directory,{logs:[]},{terminal:()=>{}});
|
|
26
26
|
result=await queuedCommand(process.execPath,[fileURLToPath(new URL('../bin/rer.mjs',import.meta.url)),...childArgs],{
|
|
27
|
-
cwd:project.root,signal:cancellation.signal,stdio:[!options.json&&!options.nonInteractive?'inherit':'ignore','pipe','pipe'],originalCI:policy.ci,env:{...process.env,...policy.environment,RER_BOUNDED_OUTPUT:'1'},
|
|
27
|
+
cwd:project.root,verb:options.verb,signal:cancellation.signal,stdio:[!options.json&&!options.nonInteractive?'inherit':'ignore','pipe','pipe'],originalCI:policy.ci,env:{...process.env,...policy.environment,RER_BOUNDED_OUTPUT:'1'},
|
|
28
28
|
label:await queueLabel(project.root,options.verb,'workflow'),report:text=>out.progress(store.redact(text)+'\n'),onWaiting:options.json?details=>out.event('waiting',details):undefined,
|
|
29
29
|
spawn:(command,args,native)=>{
|
|
30
30
|
const child=store.spawn('Running bounded workflow',command,args,native);
|
package/cli/deploy.mjs
CHANGED
|
@@ -109,7 +109,7 @@ function failureText({ error, step, created, again, retry, secretChange, deploye
|
|
|
109
109
|
const at = step === 'deploy' ? diagnostic?.step ?? 'prepare deploy' : step;
|
|
110
110
|
const input = /^missing-secret-([A-Z][A-Z0-9_]*)$/.exec(code)?.[1];
|
|
111
111
|
if (input) return `Deploy needs ${input}, which the deploy declaration lists as a secret and the Worker does not hold it. Set ${input} in the environment or in an --env-file file (an owned 0600 file of KEY=VALUE lines), then run the same command again. Nothing was uploaded.${created.length ? ` Resources created this attempt: ${created.join(', ')}.` : ''}`;
|
|
112
|
-
if (code === 'remove-incomplete') return `Removal stopped with ${error.remaining.join(', ')} still present${error.failed.length ? ` (${error.failed.map(item => `${item.resource}: ${item.code}`).join('; ')})` : ''}. Removed: ${error.removed.join(', ') || 'nothing'}.
|
|
112
|
+
if (code === 'remove-incomplete') return `Removal stopped with ${error.remaining.join(', ')} still present${error.failed.length ? ` (${error.failed.map(item => `${item.resource}: ${item.code}`).join('; ')})` : ''}. Removed: ${error.removed.join(', ') || 'nothing'}. Confirmed owned R2 buckets are emptied automatically; unproved buckets are never purged. Run ${again} --remove --yes again to finish.`;
|
|
113
113
|
if (code === 'nothing-to-remove') return `This checkout has no rer deployment record for this environment, so --remove deletes nothing. Run it from the checkout that deployed it.`;
|
|
114
114
|
if (code === 'owner-token-retirement-failed') return `The application is deployed at ${deployed?.origin}; owner ${error.owner}; setup token retirement failed (${redact(String(error.cause))}). Run ${again} --unset-secret ${error.tokenKey ?? 'SETUP_TOKEN'} using the same environment credentials, or run ${again} again to finish.`;
|
|
115
115
|
if (code === 'owner-claim-failed') return `The application is deployed at ${deployed?.origin}, but claiming its owner failed: ${redact(String(error.cause))}. Owner claim and setup token retirement are not confirmed; run ${again} again using the same environment credentials to finish.`;
|
package/cli/errors.mjs
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { transferDiagnostic } from '../modules/portability/http-client.mjs';
|
|
2
|
+
|
|
1
3
|
export class RerError extends Error {
|
|
2
4
|
constructor(code, message, exitCode, details = {}) { super(message); Object.assign(this, { code, exitCode, ...details }); }
|
|
3
5
|
}
|
|
@@ -7,6 +9,8 @@ export const refuse = message => new RerError('safety_refusal', message, 4);
|
|
|
7
9
|
|
|
8
10
|
export async function lifecycleError(error, store) {
|
|
9
11
|
if(error instanceof RerError)return error;
|
|
12
|
+
const transfer=transferDiagnostic(error);
|
|
13
|
+
if(transfer)return new RerError('transfer_failed',transfer,1);
|
|
10
14
|
const {shared}=await import('./shared.mjs');
|
|
11
15
|
const {approvedFailure}=await shared('failures.mjs');
|
|
12
16
|
const approved=approvedFailure(error);
|
package/cli/transfers.mjs
CHANGED
|
@@ -22,6 +22,7 @@ export async function transfer(project, options, signal, custody) {
|
|
|
22
22
|
// A later archive of the same installation carries its latest erasure journal.
|
|
23
23
|
const journal = options.journal ? await contained(project.root, options.journal, { privateFile: true }) : undefined;
|
|
24
24
|
await project.adapter.prepareData?.(project.root, custody.store, signal);
|
|
25
|
+
await custody.store.step(`${target === 'remote' ? 'Hosted' : 'Local'} ${direction}`);
|
|
25
26
|
// Only now can the seed's Vite configuration load in a checkout that never built.
|
|
26
27
|
await located.unserved(); await legacyOutput?.unserved();
|
|
27
28
|
const result=await api[direction]({ root: project.root, target, path, journal, origin: options.origin, sessionFile, signal, platform:options.platform, legacyRecovery, ...custody });
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.10
|
|
4
|
+
|
|
5
|
+
- First-owner bootstrap and signed owner claims use a client-IP rate budget, so junk setup-token requests from another client cannot block the correct claim. Repeated wrong tokens from one client remain limited.
|
|
6
|
+
|
|
7
|
+
### Upgrade notes
|
|
8
|
+
|
|
9
|
+
- Seeds receive this auth runtime fix with their next Base refresh. Updating the rer package alone does not replace seed-owned runtime copies.
|
|
10
|
+
|
|
11
|
+
## 0.2.9
|
|
12
|
+
|
|
13
|
+
- `rer deploy --remove --yes` empties each R2 bucket whose saved creation identity proves rer owns it, deletes the bucket, and verifies removal. Buckets with uncertain or replaced ownership are refused; interrupted deletion retains the deployment record so removal can resume.
|
|
14
|
+
- A failed hosted export names the export step, job ID, HTTP status and returned machine error code. A successful pnpm preparation is no longer blamed for a later export failure.
|
|
15
|
+
- The waiting-for-owner Lobby shows setup-token guidance only for an edition that declares `manualSetupTokenClaim: true`; automatic owner claims need no token from the visitor.
|
|
16
|
+
- Base's shared and exported dependency graphs carry sharp 0.35.5, undici 7.29.1, source-map-js 1.2.2, patched esbuild edges at 0.28.1 or newer, and Playwright 1.63.0.
|
|
17
|
+
|
|
18
|
+
### Upgrade notes
|
|
19
|
+
|
|
20
|
+
- Removal's existing `--yes` confirmation includes permanently deleting objects in buckets rer proves it created. Preserve an export before removing a deployment whose data you need.
|
|
21
|
+
- A seed using manual setup-token owner claims sets `manualSetupTokenClaim: true` in its edition identity. Seeds receive the shared runtime and dependency updates with their next Base refresh; updating the rer package alone does not refresh seed-owned runtime copies or dependency locks.
|
|
22
|
+
|
|
3
23
|
## 0.2.8
|
|
4
24
|
|
|
5
25
|
- `rer deploy --demo` resolves Wrangler through its exported package manifest and gives the Cloudflare Container application a valid explicit name. This fixes deployments refused by package export restrictions or an application name containing consecutive dashes.
|
|
@@ -116,7 +116,11 @@ A refused provider call also carries `providerRefusal` on the thrown error: the
|
|
|
116
116
|
- **Inputs.** Declared secrets come from the environment, then `--env-file FILE` (an owned 0600 file of `KEY=VALUE` lines), which wins. For a secret the Worker does not hold yet, rer supplies the setup manifest's owner email (`emailField`) from the local setup's private settings, and a fresh random value for each secret field the manifest generates (`generate`), such as the session secret and the setup token; a value the Worker already holds is kept. A still-missing secret stops the run before any write with a message that names it.
|
|
117
117
|
- **Hosted owner.** After deploy and health pass, rer claims the setup owner on the hosted app: it POSTs the setup token to the seed's standard owner step (`hooks.owner`, by default `/auth/bootstrap`). 303 means claimed; `409 already_initialized` means an owner was already claimed and is kept. Then rer deletes the token secret from the Worker, records the new Worker version in the state file, and lists the token in `retiredSecrets`, so later plans no longer declare it. The build fingerprint still names it, and the recorded claim stays as recorded, so an untouched repeat deploy writes nothing. A Worker that still holds a token rer did not upload in this run (an older deploy) first gets a fresh token through the same recorded secret change. Dependency approvals that need the token run before the claim. A seed with an owner step function, an identity-provider claim, or no declared token secret claims its owner its own way.
|
|
118
118
|
- **Secret changes.** `rer deploy --set-secret NAME` (value from the environment or `--env-file`, under the declaration's environment name) and `--unset-secret NAME` change one declared secret of the recorded Worker without a build. Before writing, rer records the starting version and the exact live names of all other secrets, never their values. Accepted or ambiguous writes retain that intent until readback confirms the change; explicit refusals clear it. An ambiguous failure, or a later command's preflight, can clear an unapplied intent only when the live deployment still matches the receipt and the newest listed version matches it too. A surviving intent blocks further mutations until consistency returns; retry the same command then. Recovery requires the recorded immediate successor (or the recorded confirming rotation's successor), the receipt's script etag and fingerprint, the intended binding's presence, and an unchanged set of other secret names. Ambiguous rotations require a confirming PUT before reporting success. Older intents without names may recover only from the exact starting version's historical bindings, with its id, etag and fingerprint verified; unavailable or malformed historical bindings refuse adoption. Each historical binding must have an exact provider type supported by this kit; unrecognized types cannot prove that a name was not a secret. Added, removed or renamed other secrets refuse deploy and removal before writes. Values remain opaque: these checks cannot distinguish an otherwise identical outside change to the pending binding.
|
|
119
|
-
- **Removal.** `rer deploy --remove --yes` deletes what the state file says rer created for this environment: the Worker (schedules first; its Durable Objects go with it) and each D1, R2, KV and queue that is still the one rer created, after checking all of them before the first delete. A replaced or uncertain resource refuses the whole removal, except a
|
|
119
|
+
- **Removal.** `rer deploy --remove --yes` deletes what the state file says rer created for this environment: the Worker (schedules first; its Durable Objects go with it) and each D1, R2, KV and queue that is still the one rer created, after checking all of them before the first delete. A replaced or uncertain resource refuses the whole removal, except a D1 or KV whose create answer was lost: under this environment's derived name it is deleted once confirmed empty (a D1 with no tables, a KV namespace with no keys); one holding data is refused as `uncertain-resource-not-empty`, and an uncertain queue, whose backlog rer cannot read, as `uncertain-resource-unverifiable` (delete it by hand if it is the empty one from the interrupted run).
|
|
120
|
+
|
|
121
|
+
R2 removal requires a saved `created` identity (the bucket's creation date). Pending buckets or buckets missing that identity are refused as `uncertain-resource-create`, and same-named replacements as `owned-resource-replaced`, even if empty. For a confirmed bucket, removal automatically lists and deletes its objects before deleting the bucket, checking its identity again before every object delete and the bucket delete. Serialize outside bucket deletion and recreation with removal: the provider cannot make a delete conditional on that identity.
|
|
122
|
+
|
|
123
|
+
Removal reads back that no recorded resource remains before deleting the state file. On partial failure, the state file retains the remaining resources so a later `--remove --yes` can finish. Resources rer did not create are never touched.
|
|
120
124
|
|
|
121
125
|
### If the state file is lost
|
|
122
126
|
|
|
@@ -42,7 +42,9 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
42
42
|
try { declaration = JSON.parse(await regularBytes(resolve(root,values.declaration),256 * 1024)); }
|
|
43
43
|
catch (error) { throw new Error(error.code === 'ENOENT' ? 'missing-owner-declaration' : 'invalid-owner-declaration'); }
|
|
44
44
|
assert(typeof declaration?.project === 'string','missing-owner-project');
|
|
45
|
-
|
|
45
|
+
// Removal reads identity before each object DELETE and may span many pages.
|
|
46
|
+
// Bound each response, not the owner's bucket size; failures retain ownership.
|
|
47
|
+
const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,...(command === 'remove' ? {maxRequests:Infinity} : {})}),{onCreated});
|
|
46
48
|
const target = await ownerTarget(api,{project:declaration.project,environment:values.environment});
|
|
47
49
|
const statePath = values.state ? resolve(root,values.state) : join(root,'.rer','deploy',`${target.environment}.json`);
|
|
48
50
|
if (command === 'remove') return {ok:true,...await ownerRemove({api,target,statePath}).catch(error => { throw api.annotateFailure(error,'remove'); })};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { r2ObjectPages, r2ObjectPath } from './r2-objects.mjs';
|
|
1
2
|
import { rm } from 'node:fs/promises';
|
|
2
3
|
import { assert } from './validation.mjs';
|
|
3
4
|
import { resourceKinds, findResource } from './resources.mjs';
|
|
@@ -9,16 +10,14 @@ import { KINDS, readOwnerState, writeOwnerState, withStateLock, withScopeHints,
|
|
|
9
10
|
|
|
10
11
|
const remove = {
|
|
11
12
|
d1: (api, id) => api.request(`/d1/database/${id}`, {method:'DELETE'}),
|
|
12
|
-
r2: (api, _id, name) => api.request(`/r2/buckets/${name}`, {method:'DELETE'}),
|
|
13
13
|
kv: (api, id) => api.request(`/storage/kv/namespaces/${id}`, {method:'DELETE'}),
|
|
14
14
|
queues: (api, id) => api.request(`/queues/${id}`, {method:'DELETE'}),
|
|
15
15
|
};
|
|
16
16
|
|
|
17
17
|
/** A resource whose create response was lost (still `pending` in the state file) may be
|
|
18
18
|
* rer's or someone else's. It is removed only under this environment's derived name and
|
|
19
|
-
* once it is confirmed empty: a D1 with no tables
|
|
20
|
-
*
|
|
21
|
-
* be read, so an uncertain queue refuses with its own code. */
|
|
19
|
+
* once it is confirmed empty: a D1 with no tables or a KV namespace with no keys. R2 always
|
|
20
|
+
* requires a confirmed creation identity. A queue's backlog cannot be read, so an uncertain queue refuses with its own code. */
|
|
22
21
|
async function confirmEmpty(api, worker, kind, name, id) {
|
|
23
22
|
if (!name.startsWith(`${worker}-`)) throw refuse('uncertain-resource-create', kind, name);
|
|
24
23
|
if (kind === 'queues') throw refuse('uncertain-resource-unverifiable', kind, name);
|
|
@@ -38,7 +37,8 @@ async function confirmEmpty(api, worker, kind, name, id) {
|
|
|
38
37
|
* each only while it is still the one rer created. Read back that each is gone, then
|
|
39
38
|
* drop the state file. Nothing else in the account is touched. Ownership is checked for
|
|
40
39
|
* everything before the first delete; a replaced or uncertain resource refuses it all,
|
|
41
|
-
* except an uncertain one confirmed empty (see confirmEmpty).
|
|
40
|
+
* except an uncertain one confirmed empty (see confirmEmpty). R2 object purges
|
|
41
|
+
* additionally require the saved creation timestamp and recheck it before each delete. */
|
|
42
42
|
export async function ownerRemove({api: provider, target, statePath, clock, sleep}) {
|
|
43
43
|
const api = withSettledLists(withScopeHints(provider));
|
|
44
44
|
return withStateLock(statePath, async () => {
|
|
@@ -64,9 +64,11 @@ export async function ownerRemove({api: provider, target, statePath, clock, slee
|
|
|
64
64
|
for (const {name, entry} of owned.filter(item => item.kind === kind)) {
|
|
65
65
|
const row = findResource(listed[kind], kind, name);
|
|
66
66
|
if (!row) continue;
|
|
67
|
+
if (kind === 'r2') requireBucketIdentity(name, entry);
|
|
67
68
|
if (entry.status === 'pending') await confirmEmpty(api, target.worker, kind, name, resourceId(kind, row));
|
|
68
69
|
else checkResourceIdentity(kind, name, row, entry);
|
|
69
|
-
|
|
70
|
+
if (kind === 'r2' && entry.status === 'created' && entry.id) await checkBucket(api, name, entry);
|
|
71
|
+
doomed.push({kind, name, entry, id:resourceId(kind, row)});
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
74
|
const removed = [];
|
|
@@ -79,7 +81,11 @@ export async function ownerRemove({api: provider, target, statePath, clock, slee
|
|
|
79
81
|
if (worker) { await deleteWorker({api, worker, ...(clock ? {clock} : {}), ...(sleep ? {sleep} : {})}); removed.push(`worker:${worker}`); }
|
|
80
82
|
const failed = [];
|
|
81
83
|
for (const item of doomed) {
|
|
82
|
-
try {
|
|
84
|
+
try {
|
|
85
|
+
if (item.kind === 'r2') await removeBucket(api, item);
|
|
86
|
+
else await remove[item.kind](api, item.id, item.name);
|
|
87
|
+
removed.push(`${item.kind}:${item.name}`);
|
|
88
|
+
}
|
|
83
89
|
catch (error) { failed.push({resource:`${item.kind}:${item.name}`, code:error.message}); }
|
|
84
90
|
}
|
|
85
91
|
// Read back: nothing this environment created is still listed.
|
|
@@ -100,10 +106,34 @@ export async function ownerRemove({api: provider, target, statePath, clock, slee
|
|
|
100
106
|
delete state.ownerClaim; delete state.retiredSecrets;
|
|
101
107
|
}
|
|
102
108
|
await writeOwnerState(statePath, state);
|
|
103
|
-
const error = new Error('remove-incomplete'); error.remaining = remaining; error.failed = failed; error.removed = removed;
|
|
109
|
+
const error = new Error('remove-incomplete'); error.remaining = remaining; error.failed = failed; error.removed = removed.filter(resource => !remaining.includes(resource));
|
|
104
110
|
throw error;
|
|
105
111
|
}
|
|
106
112
|
await rm(statePath, {force:true});
|
|
107
113
|
return {worker:target.worker, removed, remaining:[]};
|
|
108
114
|
});
|
|
109
115
|
}
|
|
116
|
+
|
|
117
|
+
async function checkBucket(api, name, entry) {
|
|
118
|
+
const row = await api.request(`/r2/buckets/${name}`);
|
|
119
|
+
if (row?.name !== name) throw refuse('owned-resource-replaced', 'r2', name);
|
|
120
|
+
checkResourceIdentity('r2', name, row, entry);
|
|
121
|
+
}
|
|
122
|
+
function requireBucketIdentity(name, entry) {
|
|
123
|
+
if (entry.status !== 'created' || typeof entry.id !== 'string' || !entry.id) throw refuse('uncertain-resource-create', 'r2', name);
|
|
124
|
+
}
|
|
125
|
+
async function removeBucket(api, {name, entry}) {
|
|
126
|
+
requireBucketIdentity(name, entry);
|
|
127
|
+
await checkBucket(api, name, entry);
|
|
128
|
+
for await (const rows of r2ObjectPages(api, name)) {
|
|
129
|
+
for (const {key} of rows) {
|
|
130
|
+
// Preflight may precede Worker teardown. Re-read before each destructive
|
|
131
|
+
// request too. The provider has no conditional bucket-identity delete;
|
|
132
|
+
// out-of-band delete/recreate operations must be serialized by the owner.
|
|
133
|
+
await checkBucket(api, name, entry);
|
|
134
|
+
await api.request(r2ObjectPath(name, key), {method:'DELETE'});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
await checkBucket(api, name, entry);
|
|
138
|
+
await api.request(`/r2/buckets/${name}`, {method:'DELETE'});
|
|
139
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { deleteR2Object } from './r2-object-transport.mjs';
|
|
2
|
+
import { isR2ObjectDelete } from './r2-objects.mjs';
|
|
1
3
|
import { recordDeployFailure } from './diagnostics.mjs';
|
|
2
4
|
import { importD1, importUploadURL } from './d1-import.mjs';
|
|
3
5
|
import { assert, hexId } from './validation.mjs';
|
|
@@ -30,23 +32,23 @@ function transportOutput(error) {
|
|
|
30
32
|
return lines.filter(Boolean).join('\n');
|
|
31
33
|
}
|
|
32
34
|
/** Only this fixed origin and the explicitly selected account ever receive the token. */
|
|
33
|
-
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, timeoutMs = 30000, maxRequests = 500 }) {
|
|
35
|
+
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500 }) {
|
|
34
36
|
assert(hexId(accountId), 'invalid-cloudflare-account-id');
|
|
35
37
|
assert(typeof token === 'string' && token.trim() && !/[\r\n]/.test(token), 'invalid-cloudflare-api-token');
|
|
36
38
|
let requests = 0;
|
|
37
39
|
// `zone` scope: the zone-level API (/zones/...), for a run site's zone lookup and redirect rule.
|
|
38
40
|
async function envelope(path, { method = 'GET', body, assetToken, zone = false } = {}) {
|
|
39
|
-
assert(typeof path === 'string' && /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%'), 'invalid-provider-path');
|
|
41
|
+
assert(typeof path === 'string' && (isR2ObjectDelete(path, method) || /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%')), 'invalid-provider-path');
|
|
40
42
|
assert(!zone || /^\/zones(?:[/?]|$)/.test(path), 'invalid-provider-path');
|
|
41
43
|
assert(++requests <= maxRequests, 'provider-request-budget');
|
|
42
44
|
assert(!assetToken || path === '/workers/assets/upload?base64=true', 'invalid-asset-auth-target');
|
|
43
45
|
// Asset-session credentials are discovered at runtime, outside the CLI env.
|
|
44
46
|
// Preserve complete text for the CLI's remaining redaction before truncation.
|
|
45
|
-
const redactAuth = text => /\/d1\/database\/[^/]+\/import$/.test(path) ? '[D1 import details omitted]' : [accountId,token,assetToken].filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
|
|
47
|
+
const redactAuth = text => /\/r2\/buckets\/[^/]+\/objects(?:[/?]|$)/.test(path) ? '[R2 object details omitted]' : /\/d1\/database\/[^/]+\/import$/.test(path) ? '[D1 import details omitted]' : [accountId,token,assetToken].filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
|
|
46
48
|
const transportFailure = cause => recordDeployFailure(new Error('provider-transport-failed'),{output:redactAuth(transportOutput(cause))});
|
|
47
49
|
const signal = AbortSignal.timeout(timeoutMs);
|
|
48
50
|
let response;
|
|
49
|
-
try { response = await request(zone ? `https://api.cloudflare.com/client/v4${path}` : `https://api.cloudflare.com/client/v4/accounts/${accountId}${path}`, {
|
|
51
|
+
try { response = isR2ObjectDelete(path, method) ? await objectDelete(`/client/v4/accounts/${accountId}${path}`, {headers:{Authorization:`Bearer ${token}`},signal}) : await request(zone ? `https://api.cloudflare.com/client/v4${path}` : `https://api.cloudflare.com/client/v4/accounts/${accountId}${path}`, {
|
|
50
52
|
method, redirect: 'error', signal, headers: { Authorization: `Bearer ${assetToken ?? token}`, ...(body && !(body instanceof FormData) ? { 'Content-Type':'application/json' } : {}) },
|
|
51
53
|
...(body === undefined ? {} : { body: body instanceof FormData ? body : JSON.stringify(body) }),
|
|
52
54
|
}); } catch (cause) { throw transportFailure(cause); }
|
|
@@ -62,7 +64,7 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
62
64
|
const bounded = text => redactAuth(text).replace(/[\x00-\x1f\x7f-\x9f]+/g,' ').trim().slice(0,300);
|
|
63
65
|
const said = (Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code) || typeof x?.message === 'string').slice(0,3)
|
|
64
66
|
.map(x => ({...(Number.isSafeInteger(x.code) ? {code:x.code} : {}),...(typeof x.message === 'string' ? {message:bounded(x.message)} : {})}));
|
|
65
|
-
error.providerRefusal = {operation:`${method} ${path.split('?')[0]}`,errors:said.length || payload !== undefined || !output.trim() ? said : [{message:bounded(output.trim().split('\n')[0])}]};
|
|
67
|
+
error.providerRefusal = {operation:`${method} ${path.split('?')[0].replace(/(\/r2\/buckets\/[^/]+\/objects)\/.*/, '$1/[object]')}`,errors:said.length || payload !== undefined || !output.trim() ? said : [{message:bounded(output.trim().split('\n')[0])}]};
|
|
66
68
|
throw recordDeployFailure(error,{providerCodes:(Array.isArray(payload?.errors) ? payload.errors : []).filter(x => Number.isSafeInteger(x?.code)).map(x=>x.code).slice(0,8),
|
|
67
69
|
output:redactAuth(Array.isArray(payload?.errors) ? payload.errors.filter(x=>typeof x?.message==='string').map(x=>x.message).join('\n') : output)});
|
|
68
70
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import https from 'node:https';
|
|
2
|
+
import { Readable } from 'node:stream';
|
|
3
|
+
|
|
4
|
+
// fetch normalizes dot segments, even percent-encoded ones. Object keys are
|
|
5
|
+
// opaque: send the validated account path verbatim through Node's HTTP client.
|
|
6
|
+
// This transport never follows redirects and never selects another origin.
|
|
7
|
+
export function deleteR2Object(path, {headers, signal}) {
|
|
8
|
+
return new Promise((resolve, reject) => {
|
|
9
|
+
const request = https.request({hostname:'api.cloudflare.com', port:443, path, method:'DELETE', headers, signal}, response => {
|
|
10
|
+
try {
|
|
11
|
+
const empty = [204,205,304].includes(response.statusCode);
|
|
12
|
+
if (empty) response.resume();
|
|
13
|
+
resolve(new Response(empty ? null : Readable.toWeb(response), {status:response.statusCode, headers:response.headers}));
|
|
14
|
+
} catch (error) { response.destroy(); reject(error); }
|
|
15
|
+
});
|
|
16
|
+
request.on('error', reject);
|
|
17
|
+
request.end();
|
|
18
|
+
});
|
|
19
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { assert } from './validation.mjs';
|
|
2
|
+
|
|
3
|
+
// Cloudflare's object REST API requires literal slashes, with other reserved
|
|
4
|
+
// characters encoded. The provider sends this path without URL normalization.
|
|
5
|
+
export function r2ObjectPath(bucket, key) {
|
|
6
|
+
assert(typeof key === 'string' && key.length > 0, 'unsafe-r2-object-key');
|
|
7
|
+
const encoded = key.split('/').map(part => (part === '.' || part === '..' ? part.replaceAll('.', '%2E') : encodeURIComponent(part)).replace(/[!'()*]/g, c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)).join('/');
|
|
8
|
+
return `/r2/buckets/${bucket}/objects/${encoded}`;
|
|
9
|
+
}
|
|
10
|
+
export function isR2ObjectDelete(path, method) {
|
|
11
|
+
if (method !== 'DELETE') return false;
|
|
12
|
+
const match = /^\/r2\/buckets\/([a-z0-9][a-z0-9-]{1,62}[a-z0-9])\/objects\/(.+)$/.exec(path);
|
|
13
|
+
if (!match) return false;
|
|
14
|
+
try { return r2ObjectPath(match[1], decodeURIComponent(match[2])) === path; } catch { return false; }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// The API returns result: Object[] and result_info.is_truncated/cursor, not the
|
|
18
|
+
// bucket-list shape. Yield bounded pages so removal never collects the bucket.
|
|
19
|
+
export async function* r2ObjectPages(api, bucket) {
|
|
20
|
+
const seen = new Set(); let cursor;
|
|
21
|
+
for (;;) {
|
|
22
|
+
const data = await api.envelope(`/r2/buckets/${bucket}/objects?per_page=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`);
|
|
23
|
+
const rows = data.result, info = data.result_info;
|
|
24
|
+
assert(Array.isArray(rows) && rows.length <= 1000 && rows.every(row => typeof row?.key === 'string' && row.key.length), 'invalid-r2-object-list');
|
|
25
|
+
assert(info && typeof info.is_truncated === 'boolean' && (!info.delimited || Array.isArray(info.delimited) && !info.delimited.length), 'provider-incomplete-pagination');
|
|
26
|
+
assert(new Set(rows.map(row => row.key)).size === rows.length, 'invalid-r2-object-list');
|
|
27
|
+
if (info.is_truncated) {
|
|
28
|
+
assert(rows.length > 0 && typeof info.cursor === 'string' && info.cursor.length > 0 && info.cursor.length <= 4096 && !seen.has(info.cursor), 'provider-invalid-cursor');
|
|
29
|
+
cursor = info.cursor; seen.add(cursor);
|
|
30
|
+
}
|
|
31
|
+
yield rows;
|
|
32
|
+
if (!info.is_truncated) return;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { loadavg } from 'node:os';
|
|
2
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
3
|
+
import { configuration } from './config.mjs';
|
|
4
|
+
import { signalExit } from './command.mjs';
|
|
5
|
+
|
|
6
|
+
// CI and descendants sharing a live lease need no second queue slot, but each
|
|
7
|
+
// verification command still waits for load before starting its own deadlines.
|
|
8
|
+
// Once admitted, a running command is never suspended or polled here.
|
|
9
|
+
export async function waitForLoad({ signal, onWaiting, report } = {}) {
|
|
10
|
+
const controller = new AbortController();
|
|
11
|
+
const interrupt = () => controller.abort('SIGINT'), terminate = () => controller.abort('SIGTERM');
|
|
12
|
+
const external = () => controller.abort(signal.reason === 'SIGINT' ? 'SIGINT' : 'SIGTERM');
|
|
13
|
+
process.on('SIGINT', interrupt); process.on('SIGTERM', terminate);
|
|
14
|
+
signal?.addEventListener('abort', external, { once: true });
|
|
15
|
+
if (signal?.aborted) external();
|
|
16
|
+
try {
|
|
17
|
+
const config = await configuration();
|
|
18
|
+
let reportedAt;
|
|
19
|
+
while (!controller.signal.aborted) {
|
|
20
|
+
const load = loadavg()[0];
|
|
21
|
+
if (Number.isFinite(load) && load <= config.loadThreshold) return null;
|
|
22
|
+
if (reportedAt === undefined || Date.now() - reportedAt >= 15_000) {
|
|
23
|
+
const reason = Number.isFinite(load)
|
|
24
|
+
? `the 1-minute load is ${load.toFixed(1)} and verification commands start at ${config.loadThreshold} or below (change with RER_JOBS_LOAD_THRESHOLD)`
|
|
25
|
+
: 'this computer\'s load could not be read';
|
|
26
|
+
const detail = { reason, reasonCodes: [Number.isFinite(load) ? 'load' : 'resources_unavailable'], resources: { load }, limits: { loadThreshold: config.loadThreshold } };
|
|
27
|
+
if (onWaiting) await onWaiting(detail);
|
|
28
|
+
else report?.(`Waiting to start verification command: ${reason}. Running commands finish uninterrupted.`);
|
|
29
|
+
reportedAt = Date.now();
|
|
30
|
+
}
|
|
31
|
+
await delay(config.pollMs, undefined, { signal: controller.signal }).catch(error => { if (error.name !== 'AbortError') throw error; });
|
|
32
|
+
}
|
|
33
|
+
return { code: signalExit(controller.signal.reason), signal: controller.signal.reason };
|
|
34
|
+
} finally {
|
|
35
|
+
process.off('SIGINT', interrupt); process.off('SIGTERM', terminate);
|
|
36
|
+
signal?.removeEventListener('abort', external);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -285,7 +285,10 @@ an invitation whose issuer still has authority.
|
|
|
285
285
|
Durable fixed-window limits pair the client-IP resolver with a normalized email;
|
|
286
286
|
defaults are 30 per IP and 5 per email per 15 minutes. `limitWindowSeconds`,
|
|
287
287
|
`ipLimit` and `emailLimit` configure those defaults. Bootstrap, magic link,
|
|
288
|
-
invitation, confirmation and review use separate scopes.
|
|
288
|
+
invitation, confirmation and review use separate scopes. Bootstrap uses only
|
|
289
|
+
the client-IP budget: its configured owner email is shared by every caller,
|
|
290
|
+
so wrong setup tokens from one client cannot exhaust another client's budget.
|
|
291
|
+
The same bootstrap policy applies to signed owner claims. Confirmation limits bind
|
|
289
292
|
the presented opaque token instead of the address. `clientIp` is mandatory;
|
|
290
293
|
unavailable addresses fail closed (503). Header trust belongs to the deployment
|
|
291
294
|
boundary. Do not trust X-Forwarded-For.
|
|
@@ -24,10 +24,14 @@ export function createEmailLinkLifecycle({ config: input, clock = Date.now, mail
|
|
|
24
24
|
if ((scope === 'magic' || scope === 'bootstrap') && await isLocalCli(request, config)) return;
|
|
25
25
|
const ip = await clientIp(request);
|
|
26
26
|
if (typeof ip !== 'string' || !ip || ip.length > 128) deny('client_ip_unavailable', 503);
|
|
27
|
-
|
|
27
|
+
// Bootstrap's email is the configured owner, shared by every caller. An
|
|
28
|
+
// unauthenticated caller must not spend another client's setup budget.
|
|
29
|
+
const subjects = [`${scope}:ip:${ip}`];
|
|
30
|
+
if (scope !== 'bootstrap') subjects.push(`${scope}:email:${email}`);
|
|
31
|
+
const keys = await Promise.all(subjects.map(subject => hash('rate', subject)));
|
|
28
32
|
const counts = await store.limits(keys, clock(), config.limitWindowSeconds * 1000);
|
|
29
33
|
if (!Array.isArray(counts) || counts.length !== keys.length || counts.some(value => !Number.isSafeInteger(value) || value < 0)) deny('invalid_proof_persistence', 503);
|
|
30
|
-
if (counts[0] > config.ipLimit || counts[1] > config.emailLimit) deny('rate_limited', 429);
|
|
34
|
+
if (counts[0] > config.ipLimit || (scope !== 'bootstrap' && counts[1] > config.emailLimit)) deny('rate_limited', 429);
|
|
31
35
|
}
|
|
32
36
|
async function newToken(kind, returnTo) {
|
|
33
37
|
const token = opaqueToken();
|
|
@@ -109,12 +109,14 @@ COOP `same-origin` to these two cooperating screens.
|
|
|
109
109
|
|
|
110
110
|
## Atomic admission and checks
|
|
111
111
|
|
|
112
|
-
Operator-auth normally commits rate
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
112
|
+
Operator-auth normally commits its client-IP rate limit before bootstrap. This
|
|
113
|
+
budget is per client, so wrong setup tokens from another client cannot block a
|
|
114
|
+
valid signed claim. A request-local D1 adapter reads the actual existing counter
|
|
115
|
+
and projects the real UPSERT result without writing. It defers that unchanged
|
|
116
|
+
UPSERT until the real bootstrap's unchanged three-statement batch. One D1
|
|
117
|
+
transaction inserts the single-use claim, checks an empty auth store, runs the
|
|
118
|
+
original limit and bootstrap SQL, then checks owner/session identity and the
|
|
119
|
+
actual client-IP threshold before committing.
|
|
118
120
|
A SQL CHECK failure or racing claim rolls the entire batch back. No application
|
|
119
121
|
mock, substitute owner/session creation, in-memory lock or retry is involved.
|
|
120
122
|
The adapter accepts only that exact SQL protocol and never serves other auth APIs.
|
|
@@ -36,7 +36,7 @@ export function bootstrapTransaction(db, claim, config, clock) {
|
|
|
36
36
|
async batch(statements) {
|
|
37
37
|
if (committed || !Array.isArray(statements) || statements.some(s => !owned.has(s))) return drift();
|
|
38
38
|
if (!limits) {
|
|
39
|
-
if (statements.length !==
|
|
39
|
+
if (statements.length !== 1 || statements.some(s => normalize(s.sql) !== LIMIT || s.args.length !== 2)) return drift();
|
|
40
40
|
limits = statements;
|
|
41
41
|
// Read the exact counters which the real UPSERT will increment. Auth's
|
|
42
42
|
// configured limit check still runs; repeat it on actual transaction
|
|
@@ -61,9 +61,9 @@ export function bootstrapTransaction(db, claim, config, clock) {
|
|
|
61
61
|
EXISTS(SELECT 1 FROM oa_operators WHERE id=? AND email=? AND role='owner' AND status='active') AND
|
|
62
62
|
EXISTS(SELECT 1 FROM oa_sessions WHERE id=? AND operator_id=?) AND
|
|
63
63
|
${claim.identity ? `EXISTS(SELECT 1 FROM rer_dependency_link WHERE interface='oidc-provider@1' AND issuer=? AND client_id=? AND metadata=? AND needs_relink=0 AND linked_at=?) AND EXISTS(SELECT 1 FROM rer_dependency_installation WHERE singleton=1 AND sector_id=?) AND` : ''}
|
|
64
|
-
${pin.sql} AND (SELECT count FROM oa_limits WHERE key=?)<=?
|
|
64
|
+
${pin.sql} AND (SELECT count FROM oa_limits WHERE key=?)<=?
|
|
65
65
|
THEN 1 ELSE 2 END WHERE jti=?`).bind(owner, owner, claim.email, session, owner,
|
|
66
|
-
...(claim.identity?[claim.identity.issuer,claim.identity.clientId,claim.identity.metadata,claim.identity.linkedAt,claim.installationId]:[]), ...pin.args, limits[0].args[0], config.ipLimit,
|
|
66
|
+
...(claim.identity?[claim.identity.issuer,claim.identity.clientId,claim.identity.metadata,claim.identity.linkedAt,claim.installationId]:[]), ...pin.args, limits[0].args[0], config.ipLimit, claim.jti);
|
|
67
67
|
try {
|
|
68
68
|
const identity = claim.identity ? [
|
|
69
69
|
db.prepare('INSERT INTO oc_idp_grants(issuer,client_id,jti,subject,sector_id) VALUES(?,?,?,?,?)').bind(claim.identity.issuer,claim.identity.clientId,claim.jti,claim.identity.subject,claim.installationId),
|
|
@@ -76,7 +76,7 @@ export function bootstrapTransaction(db, claim, config, clock) {
|
|
|
76
76
|
] : [];
|
|
77
77
|
const results = await db.batch([start, ...[...limits, ...statements].map(s => db.prepare(s.sql).bind(...s.args)), ...identity, finish]);
|
|
78
78
|
committed = true;
|
|
79
|
-
return results.slice(
|
|
79
|
+
return results.slice(1 + limits.length, 1 + limits.length + statements.length);
|
|
80
80
|
} catch {
|
|
81
81
|
// Includes unique/empty-store fences and final CHECK failure. D1 has
|
|
82
82
|
// rolled the whole batch back, including every original limit UPSERT.
|
|
@@ -436,6 +436,7 @@ modules/portability/import.mjs
|
|
|
436
436
|
modules/portability/payloads.mjs
|
|
437
437
|
modules/portability/journal.mjs
|
|
438
438
|
modules/portability/http.mjs
|
|
439
|
+
modules/portability/http-client.mjs
|
|
439
440
|
modules/portability/cli.mjs
|
|
440
441
|
modules/portability/cli-options.mjs
|
|
441
442
|
modules/portability/schema/portability.sql
|
|
@@ -5,29 +5,23 @@ import { pathToFileURL, fileURLToPath } from 'node:url';
|
|
|
5
5
|
import { options, cookieFile, absent, privateFile, checkpoint, promote } from './cli-options.mjs';
|
|
6
6
|
import { readArchive } from './format.mjs';
|
|
7
7
|
import { canonicalJson } from './hash.mjs';
|
|
8
|
+
import { transferClient, transferDiagnostic } from './http-client.mjs';
|
|
8
9
|
export async function main(args) {
|
|
9
10
|
const opts=options([...args]),cookie=await cookieFile(opts['session-file']);
|
|
10
11
|
const endpoint=opts.origin+opts.basePath;
|
|
11
|
-
|
|
12
|
-
let response;
|
|
13
|
-
try{response=await fetch(endpoint+'/'+path,{method,redirect:'error',headers:{Cookie:cookie,Origin:opts.origin,...(body===undefined?{}:{'Content-Type':'application/json'})},body:body===undefined?undefined:JSON.stringify(body)});}
|
|
14
|
-
catch{throw new Error('The transfer request failed.');}
|
|
15
|
-
if(!response.ok) {await response.body?.cancel();throw new Error('The transfer was refused (HTTP '+response.status+').');}
|
|
16
|
-
return response;
|
|
17
|
-
}
|
|
18
|
-
const get=async(...args)=>(await request(...args)).json();
|
|
12
|
+
const client=transferClient({endpoint,origin:opts.origin,cookie}),{get}=client;
|
|
19
13
|
const shell=value=>"'"+value.replaceAll("'","'\\''")+"'";
|
|
20
14
|
const resumeCommand=(id)=>'node '+shell(fileURLToPath(import.meta.url))+' '+opts.command+' --origin '+shell(opts.origin)+' --session-file '+shell(opts['session-file'])+' '+(opts.command==='export'?'--out '+shell(opts.out):'--from '+shell(opts.from))+' --resume '+id;
|
|
21
15
|
if(opts.command==='export') {
|
|
22
16
|
await absent(opts.out);
|
|
23
|
-
const id=opts.resume??(await get('exports','POST',{})).id;
|
|
17
|
+
const id=client.jobId(opts.resume??(await get('exports','POST',{})).id,'exports');
|
|
24
18
|
await checkpoint(opts.out+'.transfer.json',{version:1,kind:'export',id,origin:opts.origin});
|
|
25
19
|
process.stdout.write('Job '+id+'\nResume: '+resumeCommand(id)+'\n');
|
|
26
|
-
let
|
|
27
|
-
while(status.phase==='building')
|
|
28
|
-
if(status.phase!=='complete')throw
|
|
29
|
-
const
|
|
30
|
-
try{for await(const record of
|
|
20
|
+
let path='exports/'+id,status=await get(path);
|
|
21
|
+
while(status.phase==='building'){path='exports/'+id+'/advance';status=await get(path,'POST',{expectedPart:status.completed,maxParts:16});}
|
|
22
|
+
if(status.phase!=='complete')throw client.failedJob(path,status);
|
|
23
|
+
const records=await client.download(id),file=await privateFile(opts.out+'.partial',Boolean(opts.resume));
|
|
24
|
+
try{for await(const record of records)await file.writeFile(canonicalJson(record)+'\n');await file.sync();}
|
|
31
25
|
finally{await file.close();}
|
|
32
26
|
await promote(opts.out+'.partial',opts.out);process.stdout.write('Export complete.\n');return;
|
|
33
27
|
}
|
|
@@ -36,7 +30,7 @@ export async function main(args) {
|
|
|
36
30
|
// Verify every record even when earlier uploads are already acknowledged.
|
|
37
31
|
for await(const record of readArchive(source)) {
|
|
38
32
|
if(record.kind==='manifest') {
|
|
39
|
-
id
|
|
33
|
+
id=client.jobId(id??(await get('imports','POST',{confirmed:true,manifest:record})).id,'imports');
|
|
40
34
|
await checkpoint(opts.from+'.transfer.json',{version:1,kind:'import',id,origin:opts.origin});
|
|
41
35
|
process.stdout.write('Job '+id+'\nResume: '+resumeCommand(id)+'\n');
|
|
42
36
|
}else if(record.kind==='part')await get('imports/'+id+'/parts/'+record.index,'PUT',record);
|
|
@@ -50,7 +44,7 @@ export async function main(args) {
|
|
|
50
44
|
if(opts.command==='abort')status=await get(collection+'/'+opts.resume+'/abort','POST',{});
|
|
51
45
|
process.stdout.write(JSON.stringify(status)+'\n');
|
|
52
46
|
}
|
|
53
|
-
if(process.argv[1] && existsSync(process.argv[1]) && import.meta.url===pathToFileURL(realpathSync(process.argv[1])).href)main(process.argv.slice(2)).catch(
|
|
47
|
+
if(process.argv[1] && existsSync(process.argv[1]) && import.meta.url===pathToFileURL(realpathSync(process.argv[1])).href)main(process.argv.slice(2)).catch(error=>{
|
|
54
48
|
// Errors may contain server diagnostics or filesystem paths. Do not echo them.
|
|
55
|
-
process.stderr.write('Transfer failed: check the origin, absolute paths, output, session file and job status
|
|
49
|
+
process.stderr.write((transferDiagnostic(error)??'Transfer failed: check the origin, absolute paths, output, session file and job status.')+'\n');process.exitCode=1;
|
|
56
50
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { archiveLines } from './lines.mjs';
|
|
2
2
|
import { canonicalJson, sha256, StreamHash, unbase64, utf8 } from './hash.mjs';
|
|
3
|
-
import { insist } from './storage.mjs';
|
|
3
|
+
import { insist, PortabilityError } from './storage.mjs';
|
|
4
4
|
import { validateRow } from './catalogue.mjs';
|
|
5
5
|
export const FORMAT='runeditrun-portability';
|
|
6
6
|
export const MAX_LINE_BYTES=24*1024*1024;
|
|
@@ -107,7 +107,7 @@ async function* verifiedRecords(source,maxLineBytes,partBytes,validateManifest,r
|
|
|
107
107
|
let manifest,verifier,archive,index=0,closed=false;
|
|
108
108
|
for await(const line of archiveLines(source,maxLineBytes)) {
|
|
109
109
|
insist(!closed,'data_after_closing');
|
|
110
|
-
let record;try{record=JSON.parse(line);}catch{throw new
|
|
110
|
+
let record;try{record=JSON.parse(line);}catch{throw new PortabilityError('archive_json');}
|
|
111
111
|
if(!manifest) {await validateManifest(record);manifest=record;verifier=new ContentVerifier(manifest);archive=new ArchiveVerifier(manifest);yield record;continue;}
|
|
112
112
|
if(record.kind==='closing') {
|
|
113
113
|
insist(index===manifest.partCount && record.partCount===index,'part_count');
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { readArchive } from './format.mjs';
|
|
2
|
+
import { portabilityErrorCode } from './storage.mjs';
|
|
3
|
+
|
|
4
|
+
const jobPattern = /^[a-zA-Z0-9-]{1,100}$/;
|
|
5
|
+
const stages = new Set(['create', 'status', 'advance', 'download', 'parts', 'finalize', 'abort']);
|
|
6
|
+
const publicValue = (value, cookie) => ![cookie, cookie.slice(cookie.indexOf('=') + 1)].filter(Boolean).some(secret => value.includes(secret));
|
|
7
|
+
const jobId = (value, cookie) => typeof value === 'string' && jobPattern.test(value) && !value.includes('\n') && publicValue(value, cookie) ? value : undefined;
|
|
8
|
+
|
|
9
|
+
// The runner and a seed adapter can import independent maintained copies. Share
|
|
10
|
+
// identity approval within their realm, exposing only the bounded constructor
|
|
11
|
+
// and lookup, never the map or a way to approve an arbitrary Error/message.
|
|
12
|
+
const diagnosticKey = Symbol.for('runeditrun.portability.diagnostics.v1');
|
|
13
|
+
if (!Object.hasOwn(globalThis, diagnosticKey)) Object.defineProperty(globalThis, diagnosticKey, { value: createDiagnostics() });
|
|
14
|
+
const diagnostics = globalThis[diagnosticKey];
|
|
15
|
+
export const transferDiagnostic = error => diagnostics.get(error);
|
|
16
|
+
function createDiagnostics() {
|
|
17
|
+
const approved = new WeakMap();
|
|
18
|
+
function failure(path, status, body, cookie) {
|
|
19
|
+
const [collection, id, action] = path.split('/');
|
|
20
|
+
const operation = collection === 'exports' ? 'export' : 'import';
|
|
21
|
+
const step = stages.has(action) ? action : id ? 'status' : 'create';
|
|
22
|
+
const details = [];
|
|
23
|
+
if (jobId(id, cookie)) details.push('job ' + id);
|
|
24
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) details.push('HTTP ' + status);
|
|
25
|
+
const raw = body?.code ?? body?.error?.code ?? body?.error;
|
|
26
|
+
// Portability codes may append table, key or row details after a colon.
|
|
27
|
+
// Retain only the bounded machine identifier, never those private details.
|
|
28
|
+
const code = typeof raw === 'string' && publicValue(raw, cookie) ? raw.split(':', 1)[0] : undefined;
|
|
29
|
+
if (code && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(code) && !code.includes('\n') && publicValue(code, cookie)) details.push('code ' + code);
|
|
30
|
+
else if (Number.isSafeInteger(raw) && raw >= 0 && publicValue(String(raw), cookie)) details.push('code ' + raw);
|
|
31
|
+
const message = `Hosted ${operation} ${step} failed${details.length ? ' (' + details.join('; ') + ')' : ''}.`;
|
|
32
|
+
const error = new Error(message);
|
|
33
|
+
approved.set(error, message);
|
|
34
|
+
return error;
|
|
35
|
+
}
|
|
36
|
+
return Object.freeze({ failure, get: error => approved.get(error) });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function transferClient({ endpoint, origin, cookie }) {
|
|
40
|
+
const responses = new WeakMap();
|
|
41
|
+
const failure = (path, status, body) => diagnostics.failure(path, status, body, cookie);
|
|
42
|
+
async function request(path, method = 'GET', body) {
|
|
43
|
+
let response;
|
|
44
|
+
try {
|
|
45
|
+
response = await fetch(endpoint + '/' + path, { method, redirect: 'error',
|
|
46
|
+
headers: { Cookie: cookie, Origin: origin, ...(body === undefined ? {} : { 'Content-Type': 'application/json' }) },
|
|
47
|
+
body: body === undefined ? undefined : JSON.stringify(body) });
|
|
48
|
+
} catch { throw failure(path); }
|
|
49
|
+
if (!response.ok) throw failure(path, response.status, await errorBody(response));
|
|
50
|
+
return response;
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
async download(id) {
|
|
54
|
+
const path = 'exports/' + id + '/download', response = await request(path);
|
|
55
|
+
// Consumption and verification happen after the HTTP request succeeds.
|
|
56
|
+
// Preserve that response's context without trusting a stream's Error.
|
|
57
|
+
return (async function* () {
|
|
58
|
+
try { yield* readArchive(response.body); }
|
|
59
|
+
catch (error) { throw failure(path, response.status, { code: portabilityErrorCode(error) }); }
|
|
60
|
+
})();
|
|
61
|
+
},
|
|
62
|
+
async get(...args) {
|
|
63
|
+
const response = await request(...args);
|
|
64
|
+
let body;
|
|
65
|
+
try { body = await response.json(); } catch { throw failure(args[0], response.status); }
|
|
66
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)) throw failure(args[0], response.status);
|
|
67
|
+
responses.set(body, response.status);
|
|
68
|
+
return body;
|
|
69
|
+
},
|
|
70
|
+
jobId(value, collection) {
|
|
71
|
+
if (!jobId(value, cookie)) throw failure(collection);
|
|
72
|
+
return value;
|
|
73
|
+
},
|
|
74
|
+
failedJob(path, body) { return failure(path, responses.get(body), body); },
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Error responses can be arbitrary provider HTML or huge private diagnostics.
|
|
79
|
+
// Read a bounded JSON envelope solely to recover its machine code.
|
|
80
|
+
async function errorBody(response) {
|
|
81
|
+
const reader = response.body?.getReader();
|
|
82
|
+
if (!reader) return;
|
|
83
|
+
try {
|
|
84
|
+
let size = 0; const chunks = [];
|
|
85
|
+
for (;;) {
|
|
86
|
+
const { value, done } = await reader.read();
|
|
87
|
+
if (done) break;
|
|
88
|
+
size += value.length;
|
|
89
|
+
if (size > 16384) return;
|
|
90
|
+
chunks.push(value);
|
|
91
|
+
}
|
|
92
|
+
const bytes = new Uint8Array(size); let offset = 0;
|
|
93
|
+
for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; }
|
|
94
|
+
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
|
|
95
|
+
} catch { return; }
|
|
96
|
+
finally { try { await reader.cancel(); } catch {} reader.releaseLock(); }
|
|
97
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { PortabilityError } from './storage.mjs';
|
|
2
|
+
|
|
1
3
|
/** Read one bounded UTF-8 line at a time; never materialize the complete archive. */
|
|
2
4
|
export async function* archiveLines(source, maxLineBytes) {
|
|
3
5
|
if (!Number.isSafeInteger(maxLineBytes) || maxLineBytes < 1) throw new Error('Invalid archive line limit.');
|
|
@@ -15,7 +17,7 @@ export async function* archiveLines(source, maxLineBytes) {
|
|
|
15
17
|
}
|
|
16
18
|
function line() {
|
|
17
19
|
// Invalid UTF-8 is malformed archive text: the same constant diagnostic as invalid JSON.
|
|
18
|
-
let value; try { value = decoder.decode(buffer.subarray(0, bytes)); } catch { throw new
|
|
20
|
+
let value; try { value = decoder.decode(buffer.subarray(0, bytes)); } catch { throw new PortabilityError('archive_json'); }
|
|
19
21
|
bytes = 0;
|
|
20
22
|
return value.endsWith('\r') ? value.slice(0, -1) : value;
|
|
21
23
|
}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { canonicalJson } from './hash.mjs';
|
|
2
|
+
const errorCodes = new WeakMap();
|
|
3
|
+
// A verifier's original machine code remains independent of mutable properties.
|
|
4
|
+
export const portabilityErrorCode = error => errorCodes.get(error);
|
|
2
5
|
export class PortabilityError extends Error {
|
|
3
|
-
constructor(code, status=400) { super(code); this.code=code; this.status=status; }
|
|
6
|
+
constructor(code, status=400) { super(code); this.code=code; this.status=status; errorCodes.set(this,code); }
|
|
4
7
|
}
|
|
5
8
|
export function insist(condition, code, status=400) { if (!condition) throw new PortabilityError(code,status); }
|
|
6
9
|
export const quote = name => '"'+name.replaceAll('"','""')+'"';
|
package/onboarding/commands.mjs
CHANGED
|
@@ -141,7 +141,8 @@ export async function createCommands(directory,state,{terminal=text=>writeSync(1
|
|
|
141
141
|
async run(label,command,args=[],options={}) {
|
|
142
142
|
if(queuePolicy && options.heavy !== false) {
|
|
143
143
|
let queuedChild;
|
|
144
|
-
const
|
|
144
|
+
const commandSignal=signal&&options.signal?AbortSignal.any([signal,options.signal]):options.signal||signal;
|
|
145
|
+
const result = await queuedCommand(command,args,{...options,verb:queuePolicy.verb,signal:commandSignal,originalCI:policy.ci,
|
|
145
146
|
label:await queueLabel(options.cwd??process.cwd(),queuePolicy.verb,options.phase??'command'),report,onWaiting,
|
|
146
147
|
spawn:(program,argv,native)=>(queuedChild=spawnCommand(label,program,argv,{...native,onChild:options.onChild})),cleanup:async(child,result)=>{await stopCommand(child);await options.afterCommand?.(result);}});
|
|
147
148
|
if(result){if(result.code!==0)throw failure(label,result,captures.get(queuedChild));return result;}
|
package/onboarding/queue.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { readFile, realpath } from 'node:fs/promises';
|
|
|
5
5
|
const portable = basename(dirname(fileURLToPath(import.meta.url))) === 'base-onboarding';
|
|
6
6
|
const queueURL = new URL(portable ? '../base-cli/modules/job-queue/' : '../modules/job-queue/', import.meta.url);
|
|
7
7
|
export const heavyVerbs = new Set(['install', 'setup', 'build', 'verify', 'dev', 'preview', 'deploy', 'release', 'export', 'import', 'installer']);
|
|
8
|
+
export const verificationBoundary = verb => verb === 'release' || verb === 'verify' || verb?.startsWith('verify:');
|
|
8
9
|
export async function queueLabel(root, verb, phase) {
|
|
9
10
|
root = await realpath(root);
|
|
10
11
|
let project='project';
|
|
@@ -35,7 +36,13 @@ export async function queuedCommand(command, args, options) {
|
|
|
35
36
|
// Keep the human recipe relative while retaining the executable argv for tools.
|
|
36
37
|
const statusCommandText = [basename(process.execPath), relative(options.cwd ?? process.cwd(), statusCommand[1]), ...statusCommand.slice(2)].map(shellQuote).join(' ');
|
|
37
38
|
// Read the original caller's decision before an install synthesizes CI=true.
|
|
38
|
-
if (options.originalCI === true || await liveMembership())
|
|
39
|
+
if (options.originalCI === true || await liveMembership()) {
|
|
40
|
+
if (verificationBoundary(options.verb)) {
|
|
41
|
+
const { waitForLoad } = await import(new URL('load-admission.mjs', queueURL));
|
|
42
|
+
return waitForLoad(options);
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
39
46
|
const queue = await createQueue();
|
|
40
47
|
try {
|
|
41
48
|
const result = await queue.run([command, ...args], { ...options, kind: 'heavy', maxWait: 1200,
|
|
@@ -16,7 +16,7 @@ export async function queuedEntrypoint(url,verb='verify') {
|
|
|
16
16
|
const policy=await commandPolicy();
|
|
17
17
|
try{
|
|
18
18
|
const result=await queuedCommand(process.execPath,[...process.execArgv,fileURLToPath(url),...process.argv.slice(2)],{
|
|
19
|
-
cwd:process.cwd(),originalCI:policy.ci,label:await queueLabel(process.cwd(),verb,'suite'),report:text=>store.report(text),env:{...process.env,...policy.environment},
|
|
19
|
+
cwd:process.cwd(),verb,originalCI:policy.ci,label:await queueLabel(process.cwd(),verb,'suite'),report:text=>store.report(text),env:{...process.env,...policy.environment},
|
|
20
20
|
spawn:(command,args,options)=>store.spawn(verb,command,args,options),cleanup:child=>store.stop(child)});
|
|
21
21
|
return result?.code??null;
|
|
22
22
|
}finally{await policy.close();}
|