@runeditrun/rer 0.2.47 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/dev-identity.mjs +1 -1
- package/cli/main.mjs +7 -1
- package/cli/managed-server.mjs +1 -1
- package/cli/portability-progress.mjs +19 -0
- package/cli/portability-worker.mjs +13 -3
- package/cli/portability.mjs +4 -3
- package/cli/release.mjs +1 -1
- package/foundation/runtime/seed-contract.mjs +16 -3
- package/global/CHANGELOG.md +13 -0
- package/global/launch/facts.mjs +17 -10
- package/modules/demo-host/README.md +5 -2
- package/modules/demo-host/build.mjs +2 -2
- package/modules/demo-host/native/config.mjs +2 -1
- package/modules/demo-host/native/prepare.mjs +3 -2
- package/modules/demo-host/native/warm.mjs +19 -7
- package/modules/demo-host/native/worker.mjs +3 -1
- package/modules/job-queue/identity.mjs +5 -4
- package/modules/operator-auth/dev/index.mjs +7 -4
- package/modules/portability/README.md +1 -1
- package/modules/portability/export.mjs +2 -2
- package/modules/portability/files.mjs +25 -6
- package/modules/portability/format.mjs +3 -1
- package/modules/portability/import.mjs +8 -0
- package/modules/portability/index.d.mts +2 -2
- package/modules/portability/index.mjs +2 -2
- package/modules/portability/progress.mjs +54 -0
- package/modules/release/catalogue.mjs +5 -25
- package/package.json +1 -1
package/cli/dev-identity.mjs
CHANGED
|
@@ -61,7 +61,7 @@ export async function resolvePersona(url, key, as, signal) {
|
|
|
61
61
|
const { appFetch } = await shared('app-fetch.mjs');
|
|
62
62
|
const response = await appFetch(url + '/__dev/personas', { headers: { [DEV_KEY_HEADER]: key }, signal: AbortSignal.any([signal, AbortSignal.timeout(10000)].filter(Boolean)) });
|
|
63
63
|
if (!response.ok) { await response.body?.cancel(); throw refuse(`The application did not serve dev identity (HTTP ${response.status}). Check its local-only entry.`); }
|
|
64
|
-
const personas = await response.json(), persona = personas.find(entry => entry.as === as);
|
|
64
|
+
const personas = await response.json(), email = as.toLowerCase(), persona = personas.find(entry => entry.as === as || (as.includes('@') && entry.email?.toLowerCase() === email));
|
|
65
65
|
if (persona) return persona;
|
|
66
66
|
// The people rer can sign in as: each persona with its email, then every
|
|
67
67
|
// invited or active person in this checkout's local database by email.
|
package/cli/main.mjs
CHANGED
|
@@ -66,11 +66,17 @@ export async function main(argv = process.argv.slice(2)) {
|
|
|
66
66
|
else result = await (await import('./lifecycle.mjs')).runCommand(project, options, out);
|
|
67
67
|
out.result(true, 0, result); return 0;
|
|
68
68
|
} catch (error) {
|
|
69
|
-
const failure = error instanceof RerError ? error :
|
|
69
|
+
const failure = error instanceof RerError ? error : unexpectedFailure(error);
|
|
70
70
|
out.error(failure); out.result(false, failure.exitCode, failure.childExitCode === undefined ? {} : { childExitCode: failure.childExitCode, signal: failure.signal }); return failure.exitCode;
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
// A failure rer did not classify keeps its own first line; the hint is the next step.
|
|
75
|
+
function unexpectedFailure(error) {
|
|
76
|
+
const message = String(error?.message ?? error).split('\n')[0].trim().slice(0, 300);
|
|
77
|
+
return new RerError('command_failed', `${message || 'Command failed.'}\nUnexpected failure: run the command again, and if it repeats inspect the project operating documentation.`, 1);
|
|
78
|
+
}
|
|
79
|
+
|
|
74
80
|
function showHelp(options,out) {
|
|
75
81
|
if(options.json){out.event('help',{usage:help(options.verb)});out.result(true,0);}else out.human(help(options.verb));
|
|
76
82
|
return 0;
|
package/cli/managed-server.mjs
CHANGED
|
@@ -175,5 +175,5 @@ async function devReady(url,key,options,signal){
|
|
|
175
175
|
const as=options.as??'owner',persona=await resolvePersona(url,key,as,signal);
|
|
176
176
|
// The owner persona names the owner the session will actually have (the
|
|
177
177
|
// remaining owner once the setup owner has left), or none when none is left.
|
|
178
|
-
return {identity:true,signedInAs:persona.email??null,ownerLeft:as==='owner'&&!persona.email,openUrl:openUrl(url,key,{path:'/__dev/key',as})};
|
|
178
|
+
return {identity:true,signedInAs:persona.email??null,ownerLeft:as==='owner'&&!persona.email,openUrl:openUrl(url,key,{path:'/__dev/key',as:persona.as})};
|
|
179
179
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { lstat, readdir } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
// Bytes under a directory, for the first progress line's estimate. Missing or vanishing files count as zero.
|
|
5
|
+
export async function stateBytes(directory) {
|
|
6
|
+
let total = 0;
|
|
7
|
+
for (const entry of await readdir(directory, { withFileTypes: true }).catch(() => [])) {
|
|
8
|
+
const path = join(directory, entry.name);
|
|
9
|
+
total += entry.isDirectory() ? await stateBytes(path) : (await lstat(path).catch(() => ({ size: 0 }))).size;
|
|
10
|
+
}
|
|
11
|
+
return total;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// The class and code of an unexpected failure, when each is a plain identifier. Its message can hold row data.
|
|
15
|
+
export function nativeFailure(error) {
|
|
16
|
+
const name = typeof error?.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(error.name) ? error.name : 'unknown';
|
|
17
|
+
const code = typeof error?.code === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(error.code) ? error.code : Number.isSafeInteger(error?.code) ? String(error.code) : undefined;
|
|
18
|
+
return { class: name, ...(code ? { code } : {}) };
|
|
19
|
+
}
|
|
@@ -6,9 +6,11 @@ import { legacyTargetState, requireLegacyEmpty } from '../modules/portability/le
|
|
|
6
6
|
import { migrationReplay } from '../modules/portability/migration-history.mjs';
|
|
7
7
|
import { appliedMigrations as appliedMigrationNames } from '../modules/portability/bridge.mjs';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
-
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
9
|
+
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
10
10
|
import { createRequire } from 'node:module';
|
|
11
11
|
import { createPortability, PortabilityError, readArchive } from '../modules/portability/index.mjs';
|
|
12
|
+
import { transferProgress } from '../modules/portability/progress.mjs';
|
|
13
|
+
import { nativeFailure, stateBytes } from './portability-progress.mjs';
|
|
12
14
|
import { parseJsonc } from './jsonc.mjs';
|
|
13
15
|
import { loadPolicy } from './portability-policy.mjs';
|
|
14
16
|
|
|
@@ -33,8 +35,12 @@ async function dataConfiguration(root) {
|
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
async function run(direction, { root, path, journal, binding, buckets, policy, policyModule, transient, legacyRecovery }) {
|
|
38
|
+
// Lines for the person go to the CLI over IPC, which prints them as the command's progress.
|
|
39
|
+
const progress = transferProgress(line => process.send({ progress: line }));
|
|
40
|
+
if (!legacyRecovery && ['export', 'import'].includes(direction)) progress({ phase: 'start', direction, bytes: direction === 'export' ? await stateBytes(join(root, '.wrangler/state')) : (await stat(path)).size });
|
|
36
41
|
// Validate all inputs before opening bindings or acquiring a restore lock.
|
|
37
42
|
if (direction === 'import') {
|
|
43
|
+
progress({ phase: 'checking', direction });
|
|
38
44
|
for await (const record of fileRecords(path,legacyRecovery?readLegacyArchive:readArchive)) {}
|
|
39
45
|
if(journal)for await(const record of fileRecords(journal)) {}
|
|
40
46
|
}
|
|
@@ -56,7 +62,9 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
56
62
|
await ensurePortabilitySchema(proxy.env[binding], await portabilityPlan(unstable_splitSqlQuery));
|
|
57
63
|
if(legacyRecovery)return await recoverLegacyArchive(options,path,legacyRecovery.output,transient,legacyRecovery.acknowledged);
|
|
58
64
|
const service = await createPortability(options);
|
|
59
|
-
|
|
65
|
+
// This checkout's operation lock means no other import can be running: staging left behind is a crashed restore's.
|
|
66
|
+
if (direction === 'import' && await service.imports.abortAbandoned()) process.send({ progress: 'Cleaned up an unfinished earlier import: a restore that stopped part-way left its staged data here. Its staging is removed and this import starts from the beginning.' });
|
|
67
|
+
return await (direction === 'export' ? exportArchive(service,path,transient,progress) : importArchive(service,path,transient,journal,progress));
|
|
60
68
|
} finally { await proxy?.dispose(); await rm(configPath, { force: true }); }
|
|
61
69
|
}
|
|
62
70
|
|
|
@@ -73,6 +81,8 @@ catch (error) {
|
|
|
73
81
|
const migration = error instanceof PortabilityError && error.code.startsWith('schema_mismatch:migration_data:') ? error.code.slice('schema_mismatch:migration_data:'.length) : undefined;
|
|
74
82
|
// A populated destination names its table or bucket: schema names, never row data.
|
|
75
83
|
const occupied = error instanceof PortabilityError && /^(target|bucket)_not_empty:/.test(error.code) ? error.code.slice(error.code.indexOf(':') + 1) : undefined;
|
|
84
|
+
// Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
|
|
85
|
+
const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
|
|
76
86
|
process.exitCode = exitCode;
|
|
77
|
-
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), exitCode });
|
|
87
|
+
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), ...(native ? { native } : {}), exitCode });
|
|
78
88
|
}
|
package/cli/portability.mjs
CHANGED
|
@@ -73,17 +73,18 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
73
73
|
const child = store.spawn(`${direction === 'policy-preflight' ? 'Checking archive policy' : direction === 'legacy-preflight' ? 'Checking empty recovery destination' : direction === 'export' ? 'Exporting' : 'Restoring'} local installation data`, process.execPath,
|
|
74
74
|
['--import', notice, worker, direction, JSON.stringify(input)], { cwd: input.root, signal, env: { ...process.env, WRANGLER_SEND_METRICS: 'false' }, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
|
|
75
75
|
let receipt;
|
|
76
|
-
|
|
76
|
+
// Progress lines come first, as {progress}; the last message is the receipt.
|
|
77
|
+
child.on('message', message => { if (typeof message?.progress === 'string') store.report(message.progress); else receipt = message; });
|
|
77
78
|
const closed = new Promise(resolve => child.once('close', (code, signal) => resolve({ code, signal })));
|
|
78
79
|
await new Promise(resolve => { child.once('exit', resolve); child.once('error', resolve); });
|
|
79
80
|
await store.stop(child);
|
|
80
81
|
const status = await closed;
|
|
81
|
-
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed',
|
|
82
|
+
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed', `Local portability ended before it finished (${status.signal ? `signal ${status.signal}` : `exit code ${status.code}`}). Run the same command again; it starts over${direction === 'import' ? ' and first cleans up anything this attempt staged' : ''}.`, 1);
|
|
82
83
|
if (status.code !== 0 || !receipt.ok) throw new RerError(receipt.code ?? 'transfer_failed', receipt.reason==='legacy_restored_export_failed'
|
|
83
84
|
? 'Legacy data was restored and audited, but its v2 export failed. Preserve this installation, resolve the output problem, then run rer export --output FILE. Do not reset it to retry recovery.'
|
|
84
85
|
: receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
|
|
85
86
|
: receipt.occupied ? `Restore refused (${receipt.reason}): ${receipt.reason === 'bucket_not_empty' ? `storage bucket ${receipt.occupied} in this installation already holds files` : `table ${receipt.occupied} in this installation already holds rows its migrations did not insert`}, and a restore only goes into an empty installation. To replace this installation's data with the archive, keep a copy with rer export --output exports/before-restore.rer, then run rer reset --yes and rer import --input ${relative(input.root, input.path)} --yes again.`
|
|
86
|
-
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'})
|
|
87
|
+
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
|
|
87
88
|
delete receipt.ok;
|
|
88
89
|
return receipt;
|
|
89
90
|
}
|
package/cli/release.mjs
CHANGED
|
@@ -67,7 +67,7 @@ export async function dispatchRelease(args) {
|
|
|
67
67
|
console.log(JSON.stringify(receipt, null, 2));
|
|
68
68
|
return receipt.status === 'BLOCKED' ? 1 : 0;
|
|
69
69
|
} catch (error) {
|
|
70
|
-
console.error(error?.
|
|
70
|
+
console.error(error?.message || 'Release failed during argument, source or artifact validation. No release completed.');
|
|
71
71
|
return 2;
|
|
72
72
|
}
|
|
73
73
|
}
|
|
@@ -120,6 +120,19 @@ function areaGroup(headings, area) {
|
|
|
120
120
|
return 'published'
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
+
// A clause's prose, on one line: what follows its definition on that line, then
|
|
124
|
+
// the lines below it up to the next heading or definition. A bullet's body also
|
|
125
|
+
// ends at the first unindented line; a bold paragraph's or heading's runs on.
|
|
126
|
+
function bodyText(lines, line, form, rest) {
|
|
127
|
+
const body = [rest]
|
|
128
|
+
for (let next = line; next < lines.length; next++) {
|
|
129
|
+
const text = lines[next]
|
|
130
|
+
if (heading.test(text) || definitions.some(pattern => pattern.test(text)) || (form === 0 && text.trim() && !/^[\t ]/.test(text))) break
|
|
131
|
+
body.push(text)
|
|
132
|
+
}
|
|
133
|
+
return body.join(' ').replace(/\s+/gu, ' ').trim()
|
|
134
|
+
}
|
|
135
|
+
|
|
123
136
|
// Clause definitions everywhere except the Status section and retired-ID
|
|
124
137
|
// sections. The area is the nearest heading above that is not a clause.
|
|
125
138
|
// A clause's own "*Status:* partial." line, kept beside its text, is recorded
|
|
@@ -132,7 +145,7 @@ function readDefinitions(lines, headings, skipped, problems) {
|
|
|
132
145
|
const marker = /^\s*(```|~~~)/.exec(text)?.[1]
|
|
133
146
|
if (marker) { fence = fence === marker ? undefined : fence ?? marker; return }
|
|
134
147
|
if (fence || skipped.some(([start, end]) => line >= start && line <= end)) { current = undefined; return }
|
|
135
|
-
const
|
|
148
|
+
const form = definitions.findIndex(pattern => pattern.test(text)), match = form < 0 ? null : definitions[form].exec(text)
|
|
136
149
|
if (!match) {
|
|
137
150
|
if (heading.test(text)) current = undefined
|
|
138
151
|
const stated = /^\*{1,2}Status:\*{1,2}\s*(done|partial|missing)\b/.exec(text)
|
|
@@ -143,7 +156,7 @@ function readDefinitions(lines, headings, skipped, problems) {
|
|
|
143
156
|
const [, clause, title] = match
|
|
144
157
|
if (defined.has(clause)) { problems.push({ line, message: `${clause} is defined twice (first at line ${defined.get(clause).line}).` }); return }
|
|
145
158
|
if (!area) { problems.push({ line, message: `${clause} is defined above every section heading, so it has no area.` }); return }
|
|
146
|
-
current = { id: clause, title: cleanTitle(title), area: area.text, line }
|
|
159
|
+
current = { id: clause, title: cleanTitle(title), area: area.text, line, text: bodyText(lines, line, form, text.slice(match[0].length)) }
|
|
147
160
|
defined.set(clause, current)
|
|
148
161
|
const group = areaGroup(headings, area), known = areas.get(area.text)
|
|
149
162
|
if (known && known.group !== group) problems.push({ line: area.line, message: `Area "${area.text}" is a ${group} area here but a ${known.group} area at line ${known.line}; give one of them its own name.` })
|
|
@@ -256,7 +269,7 @@ export function parseContract(text, { file = 'CONTRACT.md' } = {}) {
|
|
|
256
269
|
if (state === 'done' && !evidence.length) problems.push({ line: rowLine, message: `${clause} is done, so its Evidence names the repository path that proves it, in backticks.` })
|
|
257
270
|
if (state === 'partial' && !evidence.length) problems.push({ line: rowLine, message: `${clause} is partial, so its Evidence names the repository path that backs it, in backticks.` })
|
|
258
271
|
if ((state === 'partial' || state === 'missing') && !gap) problems.push({ line: rowLine, message: `${clause} is ${state}, so its Gap says what is missing.` })
|
|
259
|
-
if (definition && kinds.includes(kind) && statuses.includes(state)) clauses.push({ id: clause, title, kind, status: state, evidence, gap, area: definition.area, line: definition.line, row: rowLine })
|
|
272
|
+
if (definition && kinds.includes(kind) && statuses.includes(state)) clauses.push({ id: clause, title, kind, status: state, evidence, gap, area: definition.area, text: definition.text, line: definition.line, row: rowLine })
|
|
260
273
|
}
|
|
261
274
|
}
|
|
262
275
|
line = end
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.49
|
|
4
|
+
|
|
5
|
+
- **Large imports no longer fail with D1's request limit.** The import kit sized each D1 batch from the data statements alone and left out the authority, progress, scope and return checks that `atomic()` adds to the same request. A large import could therefore build one request just over D1's limit and stop with 409 `IMPORT_D1_REQUEST_LIMIT`: Nook's 3,547-entry Slack import stopped at checkpoint 51, where the estimate said 524 statements and the request held 573 against 572 remaining. Each batch is now sized with `atomicCount()`, the same count `atomic()` sends. The apply, reverse and fenced-reversal planners share it, because undo had the same gap. Nook's import now applies all 3,547 entries in 78 saved batches (#230).
|
|
6
|
+
- **A native demo serves every built chunk as immutable.** The demo worker now sends `cache-control: public, max-age=31536000, immutable` for every hashed chunk the seed's build produced, taken from its Vite manifest or else its recorded `assetsDir`, as well as for the existing `/assets/` default. Before, seeds that build elsewhere, such as Lamp with `/app-assets/`, had their chunks revalidated on every use, so the start page's prefetch barely helped (#227).
|
|
7
|
+
- **`rer release` names the real validation error.** A validation failure now prints its own message, such as "CONTRACT.md must contain published clauses", in place of the generic "Release failed during argument, source or artifact validation", and still exits 2. The release catalogue reads clauses through `parseContract`, the same parser `rer check` and the run site use, so published clauses written as headings or bold paragraphs release just like the bullet form. `rer dev --as <email>` also matches a built-in persona by its address, so a person invited at a persona's address signs in as that invited person with the invited role (#231).
|
|
8
|
+
- Base's TanStack Start migration notes add one harness line for test queues: keep the producers, drop the consumers, and name a single opt-in for tests that need real consumers (#225).
|
|
9
|
+
|
|
10
|
+
## 0.2.48
|
|
11
|
+
|
|
12
|
+
- **The job queue no longer aborts or strands a waiting command under load.** Before, when another queued job's owner did not answer its 3 s identity probe, the waiting build, verify or deploy stopped with "Command failed; inspect the project operating documentation." A probe timeout now counts as "the owner may still be alive", so the waiting job samples again instead of giving up. The same holds for the queue lock and for nested-command membership, which had left a job queued behind its own parent until the queue's maximum wait. A recorded failed install is retried only when `pnpm-lock.yaml` or `package.json` changed since it failed. An error that is not one of rer's own now prints its message and a hint instead of the generic line (#219).
|
|
13
|
+
- **The demo start page warms the chunks the seed actually built.** The warm list now comes from the build output: the client Vite manifest when the build wrote one, else the build's `assetsDir` (recorded in `demo.json` from the loaded Vite config), else `assets`. Seeds that build to a non-default directory, such as Lamp's `/app-assets/`, had an empty warm list; Lamp's is now 32 chunks with the console's routes first (#223).
|
|
14
|
+
- **`rer export` and `rer import` show progress, and a crashed import cleans up after itself.** Export prints the expected time, then parts built, parts written and bytes with the time left (one line every 3 s, plus each phase edge). Import prints parts staged. An import that died part-way leaves staging that the next import removes, saying "Cleaned up an unfinished earlier import". An unexpected failure prints its error class or code and "Run the same command again; it starts over." (#224).
|
|
15
|
+
|
|
3
16
|
## 0.2.47
|
|
4
17
|
|
|
5
18
|
- **The publication scan resolves GitHub repositories in bulk.** Before, the scan looked up each GitHub repository a text names with its own REST call. Now it collects every reference first and resolves them together, in one aliased GraphQL query per 100 repositories, which also draws on GitHub's separate GraphQL budget. A repository counts as public or private only on a complete answer that is not disabled or locked. It counts as missing only when GitHub returns NOT_FOUND for exactly that name. Any other answer for a repository, and any batch that is not a complete HTTP 200 GraphQL document, falls back to the earlier strict REST lookup for that repository. Nothing is cached on disk. A run-oncall scan now makes 1 gh call instead of 19, and its report is byte-identical (#216).
|
package/global/launch/facts.mjs
CHANGED
|
@@ -20,17 +20,24 @@ export async function describeCheckout(root, entry, env, signal) {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
// Install when the last install
|
|
24
|
-
// pnpm writes node_modules/.modules.yaml only once dependencies
|
|
25
|
-
// built, so a stopped install leaves it absent. An install this
|
|
26
|
-
// is finished only when its record says so: that also covers the
|
|
27
|
-
// own install hooks, which pnpm runs after writing the manifest.
|
|
23
|
+
// Install when the last install was killed, nothing is linked, or the lockfile
|
|
24
|
+
// changed since. pnpm writes node_modules/.modules.yaml only once dependencies
|
|
25
|
+
// are linked and built, so a stopped install leaves it absent. An install this
|
|
26
|
+
// launcher ran is finished only when its record says so: that also covers the
|
|
27
|
+
// project's own install hooks, which pnpm runs after writing the manifest. A
|
|
28
|
+
// recorded failed install is retried only when the lockfile or package.json is
|
|
29
|
+
// newer than both that attempt and what is installed; dependencies repaired
|
|
30
|
+
// since are current, and the next launch uses them.
|
|
28
31
|
export async function needsInstall(root, last) {
|
|
29
|
-
if (last
|
|
30
|
-
const
|
|
31
|
-
if (
|
|
32
|
-
const
|
|
33
|
-
|
|
32
|
+
if (last?.state === 'running') return true;
|
|
33
|
+
const linked = await mtime(join(root, 'node_modules/.modules.yaml'));
|
|
34
|
+
if (linked === undefined) return true;
|
|
35
|
+
const since = Math.max(linked, last?.at ?? 0);
|
|
36
|
+
for (const file of last?.state === 'failed' ? ['pnpm-lock.yaml', 'package.json'] : ['pnpm-lock.yaml']) {
|
|
37
|
+
const changed = await mtime(join(root, file));
|
|
38
|
+
if (changed !== undefined && changed > since) return true;
|
|
39
|
+
}
|
|
40
|
+
return false;
|
|
34
41
|
}
|
|
35
42
|
|
|
36
43
|
// A checkout is set up when its pin's describe says so (`local.prepared`, from
|
|
@@ -13,8 +13,11 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
|
|
|
13
13
|
- `worker.mjs`, `pages.mjs`, `start-script.mjs`: Turnstile start, progress,
|
|
14
14
|
cancellation and visitor routing. Static assets are served at the edge.
|
|
15
15
|
- `native/warm.mjs`: the start page's prefetch list, built from `dist/client`
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
(the Vite manifest if the build wrote one, else the build's `assetsDir`, else
|
|
17
|
+
`assets`) and TanStack Start's route manifest, the console's views first,
|
|
18
|
+
capped by compressed size. The same built chunks (`bundle.immutable`) are
|
|
19
|
+
what `native/worker.mjs` serves with an immutable `cache-control`, besides
|
|
20
|
+
hashed files under `/assets/`.
|
|
18
21
|
- `capacity.mjs`: concurrent visitor admission and operator diagnostics. This
|
|
19
22
|
is used by the native host as well as retained during legacy upgrades.
|
|
20
23
|
- `upgrade.mjs`: verifies ownership, removes the old Container application and
|
|
@@ -40,9 +40,9 @@ export async function buildDemo(root, output, declaration) {
|
|
|
40
40
|
await rm(output, { recursive: true, force: true });
|
|
41
41
|
await mkdir(join(output, 'app'), { recursive: true });
|
|
42
42
|
await cp(join(root, 'dist'), join(output, 'app/dist'), { recursive: true });
|
|
43
|
-
// Where the console is mounted orders what the start page prefetches (native/warm.mjs).
|
|
43
|
+
// Where the console is mounted orders what the start page prefetches, and the build's assetsDir says where its chunks are (native/warm.mjs).
|
|
44
44
|
const { operatorPath } = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8').catch(() => '{}')).deploy ?? {};
|
|
45
|
-
await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath }) + '\n');
|
|
45
|
+
await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath, assetsDir: loaded.config.build?.assetsDir }) + '\n');
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
if (process.argv[1] === import.meta.filename) {
|
|
@@ -2,7 +2,8 @@ import { parseCron } from './cron.mjs';
|
|
|
2
2
|
|
|
3
3
|
export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'r2-store', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
|
|
4
4
|
// Hashed build output: served immutable, and what the start page may prefetch.
|
|
5
|
-
export const
|
|
5
|
+
export const hashedAsset = (assetsDir = 'assets') => new RegExp(`^/${assetsDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/.*-[\\w-]{8,}\\.[\\w]+$`);
|
|
6
|
+
export const HASHED_ASSET = hashedAsset();
|
|
6
7
|
export const ISOLATE_FILES = ['bindings', 'namespace', 'r2-client', 'd1', 'sql', 'seed-object'];
|
|
7
8
|
export function nativeBindings(config) {
|
|
8
9
|
for (const kind of ['kv_namespaces', 'services', 'workflows', 'ai', 'vectorize', 'hyperdrive', 'analytics_engine_datasets', 'browser', 'images', 'send_email', 'dispatch_namespaces', 'mtls_certificates', 'pipelines', 'secrets_store_secrets', 'unsafe', 'vpc_services', 'vpc_networks', 'ratelimits', 'worker_loaders', 'containers', 'ai_search_namespaces', 'ai_search', 'agent_memory', 'connect', 'artifacts', 'unsafe_hello_world', 'flagship', 'logfwdr', 'version_metadata']) {
|
|
@@ -4,7 +4,7 @@ import { createHash } from 'node:crypto';
|
|
|
4
4
|
import { collectBuildModules } from '../../deploy-kit/build-modules.mjs';
|
|
5
5
|
import { collectFiles, moduleType } from '../../deploy-kit/artifact.mjs';
|
|
6
6
|
import { nativeBindings, ISOLATE_FILES, objectExports } from './config.mjs';
|
|
7
|
-
import { warmList } from './warm.mjs';
|
|
7
|
+
import { warmList, builtChunks } from './warm.mjs';
|
|
8
8
|
|
|
9
9
|
// The Worker Loader module kind for each upload type deploy-kit gives a build
|
|
10
10
|
// module, so a demo carries what a normal deploy uploads. Binary kinds travel
|
|
@@ -34,7 +34,8 @@ export async function prepareNative(folder) {
|
|
|
34
34
|
const bundle = { compatibilityDate: config.compatibility_date, compatibilityFlags: [...new Set([...(config.compatibility_flags ?? []), 'nodejs_compat'])], modules,
|
|
35
35
|
bindings, migrations: manifest.migrations, vars: config.vars ?? {}, crons: config.triggers?.crons ?? [],
|
|
36
36
|
assetPaths: assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path),
|
|
37
|
-
|
|
37
|
+
immutable: [...builtChunks(assets, manifest.assetsDir)].sort(),
|
|
38
|
+
warm: (await warmList(assets, root, manifest.operatorPath, manifest.assetsDir)).paths };
|
|
38
39
|
bundle.digest = createHash('sha256').update(JSON.stringify(bundle)).digest('hex');
|
|
39
40
|
await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
|
|
40
41
|
await writeFile(join(folder, 'modules/demo-host/native/bundle.mjs'), `export default ${JSON.stringify(bundle)};\n`);
|
|
@@ -2,7 +2,7 @@ import { readdir } from 'node:fs/promises';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { pathToFileURL } from 'node:url';
|
|
4
4
|
import { gzipSync } from 'node:zlib';
|
|
5
|
-
import {
|
|
5
|
+
import { hashedAsset } from './config.mjs';
|
|
6
6
|
|
|
7
7
|
// The start page prefetches the console's hashed JS and CSS while the visitor
|
|
8
8
|
// waits for the human check and sign-in, so each view's first visit finds its
|
|
@@ -16,10 +16,22 @@ const segments = path => path.split('/').filter(Boolean);
|
|
|
16
16
|
const dynamic = path => segments(path).filter(part => part.startsWith('$')).length;
|
|
17
17
|
const within = (path, root) => path === root || path.startsWith(root.endsWith('/') ? root : root + '/');
|
|
18
18
|
|
|
19
|
+
// The hashed JS and CSS the client build produced: the files its Vite manifest
|
|
20
|
+
// names (build.manifest), else everything under the configured build.assetsDir
|
|
21
|
+
// (Vite's default is 'assets'). Never source maps or unhashed files.
|
|
22
|
+
const MANIFESTS = ['.vite/manifest.json', 'manifest.json'];
|
|
23
|
+
export function builtChunks(files, assetsDir) {
|
|
24
|
+
const manifest = files.find(file => MANIFESTS.includes(file.path));
|
|
25
|
+
const paths = (manifest
|
|
26
|
+
? Object.values(JSON.parse(Buffer.from(manifest.base64, 'base64'))).flatMap(entry => [entry.file, ...(entry.css ?? [])]).map(path => '/' + path)
|
|
27
|
+
: files.map(file => '/' + file.path).filter(path => hashedAsset(assetsDir).test(path)));
|
|
28
|
+
return new Set(paths.filter(path => /-[\w-]{8,}\.(js|css)$/.test(path)));
|
|
29
|
+
}
|
|
30
|
+
|
|
19
31
|
// TanStack Start's route manifest from the server build: each route's CSS and
|
|
20
32
|
// the chunks it preloads.
|
|
21
|
-
async function startRoutes(server) {
|
|
22
|
-
const directory = join(server,
|
|
33
|
+
async function startRoutes(server, assetsDir) {
|
|
34
|
+
const directory = join(server, assetsDir);
|
|
23
35
|
const file = (await readdir(directory).catch(() => [])).find(name => /^_tanstack-start-manifest_v-[\w-]+\.js$/.test(name));
|
|
24
36
|
if (!file) return {};
|
|
25
37
|
const { tsrStartManifest } = await import(pathToFileURL(join(directory, file)));
|
|
@@ -36,10 +48,10 @@ function routeOrder(routes, consolePath) {
|
|
|
36
48
|
}
|
|
37
49
|
|
|
38
50
|
/** The ordered, size-capped paths the start page prefetches, from dist/client's files and the server build. */
|
|
39
|
-
export async function warmList(files, server, consolePath = '/app') {
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
const routes = await startRoutes(server);
|
|
51
|
+
export async function warmList(files, server, consolePath = '/app', assetsDir = 'assets') {
|
|
52
|
+
const chunks = builtChunks(files, assetsDir);
|
|
53
|
+
const sizes = new Map(files.filter(file => chunks.has('/' + file.path)).map(file => ['/' + file.path, gzipSync(Buffer.from(file.base64, 'base64')).length]));
|
|
54
|
+
const routes = await startRoutes(server, assetsDir);
|
|
43
55
|
const ordered = new Set();
|
|
44
56
|
for (const id of routeOrder(routes, consolePath)) for (const path of [...(routes[id].css ?? []), ...(routes[id].preloads ?? [])]) if (sizes.has(path)) ordered.add(path);
|
|
45
57
|
for (const path of [...sizes.keys()].sort((a, b) => sizes.get(a) - sizes.get(b))) ordered.add(path);
|
|
@@ -7,6 +7,8 @@ export { NativeDemo } from './visitor.mjs';
|
|
|
7
7
|
export { DemoCapacity } from '../capacity.mjs';
|
|
8
8
|
|
|
9
9
|
const assets = new Set(bundle.assetPaths);
|
|
10
|
+
// The seed's built chunks (its Vite manifest, else its assetsDir) are hashed wherever the build put them.
|
|
11
|
+
const chunks = new Set(bundle.immutable);
|
|
10
12
|
export default {
|
|
11
13
|
async fetch(request, env, ctx) {
|
|
12
14
|
const url = new URL(request.url);
|
|
@@ -14,7 +16,7 @@ export default {
|
|
|
14
16
|
if (['GET', 'HEAD'].includes(request.method) && assets.has(url.pathname)) {
|
|
15
17
|
const response = await env.ASSETS.fetch(request);
|
|
16
18
|
const headers = new Headers(response.headers);
|
|
17
|
-
if (HASHED_ASSET.test(url.pathname)) headers.set('cache-control', 'public, max-age=31536000, immutable');
|
|
19
|
+
if (chunks.has(url.pathname) || HASHED_ASSET.test(url.pathname)) headers.set('cache-control', 'public, max-age=31536000, immutable');
|
|
18
20
|
headers.set('x-rer-demo-assets', 'edge');
|
|
19
21
|
return new Response(response.body, { status: response.status, headers });
|
|
20
22
|
}
|
|
@@ -31,14 +31,15 @@ export async function alive(owner) {
|
|
|
31
31
|
return new Promise((resolve, reject) => {
|
|
32
32
|
const connection = createConnection(owner.socket);
|
|
33
33
|
let body = '';
|
|
34
|
-
connection.setTimeout(3000, () => connection.destroy(Error('Owner identity probe timed out
|
|
34
|
+
connection.setTimeout(3000, () => connection.destroy(Object.assign(Error('Owner identity probe timed out'), { code: 'PROBE_TIMEOUT' })));
|
|
35
35
|
connection.on('data', data => { body += data; if (body.length > 128) connection.destroy(Error('Invalid owner identity')); });
|
|
36
36
|
connection.on('end', () => resolve(body === owner.token));
|
|
37
37
|
connection.on('error', async error => {
|
|
38
38
|
if (['ENOENT', 'ECONNREFUSED'].includes(error.code)) resolve(false);
|
|
39
|
-
else if (error.code
|
|
40
|
-
// Listener shutdown can reset a probe after the initial PID check
|
|
41
|
-
//
|
|
39
|
+
else if (['ECONNRESET', 'PROBE_TIMEOUT'].includes(error.code)) {
|
|
40
|
+
// Listener shutdown can reset a probe after the initial PID check, and a
|
|
41
|
+
// busy owner can answer late. Neither proves death: reclaim only a process
|
|
42
|
+
// that is gone; otherwise retain the owner until the next sample.
|
|
42
43
|
try {
|
|
43
44
|
const after = await processInfo(owner.pid);
|
|
44
45
|
resolve(Boolean(after && !after.zombie && after.start === owner.start));
|
|
@@ -130,15 +130,18 @@ async function enter(request, gate, { config, store, hash, current, everyone, ow
|
|
|
130
130
|
reply.set('location', '/');
|
|
131
131
|
return new Response(null, { status: 303, headers: reply });
|
|
132
132
|
}
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
const value = await auth.oneClick.enter(request, { as, returnTo: url.searchParams.get('returnTo') ?? '/app' });
|
|
133
|
+
const oneClick = async email => {
|
|
134
|
+
const value = await auth.oneClick.enter(request, { as: email, returnTo: url.searchParams.get('returnTo') ?? '/app' });
|
|
136
135
|
reply.append('set-cookie', value.cookie);
|
|
137
136
|
reply.set('location', value.returnTo);
|
|
138
137
|
return new Response(null, { status: 303, headers: reply });
|
|
139
|
-
}
|
|
138
|
+
};
|
|
139
|
+
const persona = (await everyone()).find(p => p.as === as);
|
|
140
|
+
if (!persona) return oneClick(as);
|
|
140
141
|
if (!persona.email) return text(409, 'No active owner remains in this app. Sign in another way, or run rer reset --yes and rer setup for a new owner.');
|
|
141
142
|
let operator = await store.byEmail(persona.email);
|
|
143
|
+
// Someone invited at a persona's address is that person, with the role they were invited to; accepting the invite signs them in.
|
|
144
|
+
if (operator?.status === 'invited') return oneClick(operator.email);
|
|
142
145
|
// Personas are real operators, created only here, in local dev.
|
|
143
146
|
if (!operator && persona.role) {
|
|
144
147
|
if (persona.role === 'owner') {
|
|
@@ -121,7 +121,7 @@ idempotence and recovery; D1 cannot roll back Durable Object or external state.
|
|
|
121
121
|
The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
|
|
122
122
|
closing record. `readArchive(stream)` uses the shared bounded, fatal UTF-8 line
|
|
123
123
|
reader. The default maximum line size is 24 MiB; callers can set a smaller bound.
|
|
124
|
-
No runtime or CLI materializes the whole archive. Readers enforce a 16 MiB
|
|
124
|
+
No runtime or CLI materializes the whole archive. The manifest opens the archive and holds the final part count and payload hashes, so an export builds all its parts in the job before any byte is written; `exportArchive` and `importArchive` take an optional `progress` callback (`progress.mjs` turns its events into throttled lines with a time-left estimate). Readers enforce a 16 MiB
|
|
125
125
|
canonical UTF-8 part limit; `readArchive(stream,{partBytes})` can lower it.
|
|
126
126
|
Import uploads enforce the policy's part limit before any staging write.
|
|
127
127
|
A part has at most 1,000 rows;
|
|
@@ -96,7 +96,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
96
96
|
if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
|
|
97
97
|
const before=row.completed,items=await nextItems(context,row);
|
|
98
98
|
if(items) {
|
|
99
|
-
const part=await makePart(before,items)
|
|
99
|
+
const part=await makePart(before,items),size=await checkPart(part,before,row.metadata.partBytes);row.completed++;row.metadata.bytes=(row.metadata.bytes??0)+size+1;
|
|
100
100
|
try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
|
|
101
101
|
catch(e){if((await job(context.db,id)).completed===before)throw e;}
|
|
102
102
|
if(expectedPart!==undefined)expectedPart++;
|
|
@@ -114,7 +114,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
114
114
|
return exportStatus(context,id);
|
|
115
115
|
}
|
|
116
116
|
export async function exportStatus(context,id) {
|
|
117
|
-
const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,contentHash:row.metadata.contentHash??null};
|
|
117
|
+
const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,bytes:row.metadata.bytes??0,contentHash:row.metadata.contentHash??null};
|
|
118
118
|
}
|
|
119
119
|
export async function downloadExport(context,id) {
|
|
120
120
|
const row=await job(context.db,id,'export');insist(row.phase==='complete','export_incomplete',409);
|
|
@@ -4,6 +4,7 @@ import { dirname } from 'node:path';
|
|
|
4
4
|
import { Readable } from 'node:stream';
|
|
5
5
|
import { randomUUID } from 'node:crypto';
|
|
6
6
|
import { readArchive } from './index.mjs';
|
|
7
|
+
import { canonicalJson } from './hash.mjs';
|
|
7
8
|
export async function* fileRecords(path,reader=readArchive) {
|
|
8
9
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
9
10
|
try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
|
|
@@ -19,23 +20,35 @@ function receipt(manifest, closing, transient) {
|
|
|
19
20
|
requiredConfiguration: manifest.requiredConfiguration,
|
|
20
21
|
};
|
|
21
22
|
}
|
|
22
|
-
|
|
23
|
+
// `progress` receives {phase, parts, total?, bytes?} events; the CLI turns them into lines (progress.mjs).
|
|
24
|
+
export async function exportArchive(service, path, transient, progress) {
|
|
23
25
|
let id, temporary, file;
|
|
24
26
|
try {
|
|
25
27
|
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
26
28
|
temporary = path + '.' + randomUUID() + '.partial';
|
|
27
29
|
file = await open(temporary, 'wx', 0o600);
|
|
28
30
|
id = await service.exports.create();
|
|
29
|
-
|
|
31
|
+
// The manifest leads the archive and holds the final part count and payload hashes, so
|
|
32
|
+
// nothing can be written until every part is built (export.mjs advanceExport). One part per
|
|
33
|
+
// advance keeps the progress line moving; a part is at most 16 MiB.
|
|
34
|
+
let status;
|
|
35
|
+
do {
|
|
36
|
+
status = await service.exports.advance(id, { maxParts: 1 });
|
|
37
|
+
progress?.({ phase: 'building', parts: status.completed, bytes: status.bytes, ...(status.phase === 'complete' ? { total: status.total } : {}) });
|
|
38
|
+
} while (status.phase === 'building');
|
|
30
39
|
const reader = (await service.exports.download(id)).getReader();
|
|
31
40
|
try {
|
|
32
|
-
|
|
41
|
+
// One chunk per record: the manifest, then each part, then the closing record.
|
|
42
|
+
for (let record = 0, written = 0;; record++) {
|
|
33
43
|
const { done, value } = await reader.read(); if (done) break;
|
|
34
44
|
let offset = 0;
|
|
35
45
|
while (offset < value.length) offset += (await file.write(value, offset)).bytesWritten;
|
|
46
|
+
written += value.length;
|
|
47
|
+
if (record > 0 && record <= status.total) progress?.({ phase: 'writing', parts: record, total: status.total, bytes: written });
|
|
36
48
|
}
|
|
37
49
|
} finally { await reader.cancel(); reader.releaseLock(); }
|
|
38
50
|
await file.sync(); await file.close(); file = undefined;
|
|
51
|
+
progress?.({ phase: 'checking', direction: 'export' });
|
|
39
52
|
let manifest, closing;
|
|
40
53
|
for await (const record of fileRecords(temporary)) { if (record.kind === 'manifest') manifest = record; if (record.kind === 'closing') closing = record; }
|
|
41
54
|
await link(temporary, path);
|
|
@@ -46,19 +59,25 @@ export async function exportArchive(service, path, transient) {
|
|
|
46
59
|
if (id) await service.exports.abort(id);
|
|
47
60
|
}
|
|
48
61
|
}
|
|
49
|
-
export async function importArchive(service, path, transient, journal) {
|
|
50
|
-
let id, manifest, closing, complete = false;
|
|
62
|
+
export async function importArchive(service, path, transient, journal, progress) {
|
|
63
|
+
let id, manifest, closing, complete = false, staged = 0;
|
|
51
64
|
try {
|
|
52
65
|
// Validate both complete archives before creating any restore state.
|
|
53
66
|
let later;
|
|
67
|
+
progress?.({ phase: 'checking', direction: 'import' });
|
|
54
68
|
for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
|
|
55
69
|
if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
|
|
56
70
|
}
|
|
57
71
|
for await (const record of fileRecords(path)) {
|
|
58
72
|
if (record.kind === 'manifest') { manifest = record; id = await service.imports.create(manifest); }
|
|
59
|
-
else if (record.kind === 'part')
|
|
73
|
+
else if (record.kind === 'part') {
|
|
74
|
+
await service.imports.uploadPart(id, record.index, record);
|
|
75
|
+
staged += Buffer.byteLength(canonicalJson(record)) + 1;
|
|
76
|
+
progress?.({ phase: 'staging', parts: record.index + 1, total: manifest.partCount, bytes: staged });
|
|
77
|
+
}
|
|
60
78
|
else closing = record;
|
|
61
79
|
}
|
|
80
|
+
progress?.({ phase: 'finishing', total: manifest.partCount });
|
|
62
81
|
const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
|
|
63
82
|
return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}) };
|
|
64
83
|
} finally { if (id && !complete) await service.imports.abort(id); }
|
|
@@ -11,9 +11,11 @@ export async function makePart(index,items) {
|
|
|
11
11
|
export async function checkPart(part,index,partBytes=MAX_PART_BYTES) {
|
|
12
12
|
insist(part?.kind==='part' && part.index===index && Array.isArray(part.items) && part.items.length>0 && Object.keys(part).sort().join(',')==='index,items,kind,sha256','part_order');
|
|
13
13
|
insist(Number.isSafeInteger(partBytes) && partBytes>0 && partBytes<=MAX_PART_BYTES,'invalid_part_limit');
|
|
14
|
-
|
|
14
|
+
const size=utf8(canonicalJson(part)).length;
|
|
15
|
+
insist(size<=partBytes,'part_byte_limit',413);
|
|
15
16
|
insist(part.sha256===await sha256(canonicalJson({kind:part.kind,index:part.index,items:part.items})),'part_checksum');
|
|
16
17
|
insist(part.items.filter(x=>x.kind==='row').length<=1000,'part_row_limit');
|
|
18
|
+
return size;
|
|
17
19
|
}
|
|
18
20
|
export async function checkManifest(m) {
|
|
19
21
|
insist(m?.kind==='manifest' && m.format===FORMAT,'archive_format');
|
|
@@ -197,3 +197,11 @@ export async function abortImport(context,id) {
|
|
|
197
197
|
await context.db.batch([save(context.db,row),statement(context.db,'UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1 AND import_job=?',id)]);
|
|
198
198
|
return importStatus(context,id);
|
|
199
199
|
}
|
|
200
|
+
// A restore's staging outlives a worker that died mid-restore (a crashed process or workerd) and refuses
|
|
201
|
+
// every later import as running. Only a caller that owns the destination, as the local CLI does under the
|
|
202
|
+
// checkout's operation lock, may call this: a hosted service can have a live import.
|
|
203
|
+
export async function abortAbandonedImport(context) {
|
|
204
|
+
const id=(await state(context.db)).import_job;
|
|
205
|
+
if(!id)return null;
|
|
206
|
+
await abortImport(context,id);return id;
|
|
207
|
+
}
|
|
@@ -28,7 +28,7 @@ export interface ArchivePolicy {
|
|
|
28
28
|
extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
|
|
29
29
|
partBytes?: number;
|
|
30
30
|
}
|
|
31
|
-
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
|
|
31
|
+
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
|
|
32
32
|
/** Later additive migrations an older archive was restored across. */
|
|
33
33
|
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
|
|
34
34
|
/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
|
|
@@ -36,7 +36,7 @@ export interface MigrationReplay {names:string[];statements(name:string):string[
|
|
|
36
36
|
export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
|
|
37
37
|
export interface Portability {
|
|
38
38
|
exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
|
|
39
|
-
imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>};
|
|
39
|
+
imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
|
|
40
40
|
status(id:string):Promise<Status>;actor(id:string):Promise<string>;
|
|
41
41
|
erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
|
|
42
42
|
redactionStatements():Promise<D1PreparedStatement[]>;
|
|
@@ -2,7 +2,7 @@ import { policy, catalogue } from './catalogue.mjs';
|
|
|
2
2
|
import { wireDatabase } from '../d1-transport/index.mjs';
|
|
3
3
|
import { insist, job } from './storage.mjs';
|
|
4
4
|
import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
|
|
5
|
-
import { createImport, uploadPart, finalizeImport, importStatus, abortImport } from './import.mjs';
|
|
5
|
+
import { createImport, uploadPart, finalizeImport, importStatus, abortImport, abortAbandonedImport } from './import.mjs';
|
|
6
6
|
import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
|
|
7
7
|
export { archiveLines } from './lines.mjs';
|
|
8
8
|
export { excludedTables, excludedTable } from './catalogue.mjs';
|
|
@@ -17,7 +17,7 @@ export async function createPortability({db,buckets={},policy:input,migrations})
|
|
|
17
17
|
const context={db,buckets,config,migrations};
|
|
18
18
|
return {
|
|
19
19
|
exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
|
|
20
|
-
imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id)},
|
|
20
|
+
imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
|
|
21
21
|
status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
|
|
22
22
|
actor:async id=>(await job(db,id)).actor,
|
|
23
23
|
erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Human progress for a long export or import. The caller sends events; this turns
|
|
2
|
+
// them into at most one line every few seconds, with an estimate of the time left
|
|
3
|
+
// once a phase has two samples. The first and last line of every phase always show.
|
|
4
|
+
// Rates are bytes per second measured on a 1.2 GB local install on a loaded machine
|
|
5
|
+
// (export 1725 s, import 2234 s); a quiet machine is faster.
|
|
6
|
+
const RATES = { export: 700_000, import: 550_000 };
|
|
7
|
+
const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'];
|
|
8
|
+
export function formatBytes(bytes) {
|
|
9
|
+
let value = bytes, unit = 0;
|
|
10
|
+
while (value >= 1000 && unit < UNITS.length - 1) { value /= 1000; unit++; }
|
|
11
|
+
return `${unit && value < 100 ? value.toFixed(1) : Math.round(value)} ${UNITS[unit]}`;
|
|
12
|
+
}
|
|
13
|
+
export function formatDuration(seconds) {
|
|
14
|
+
if (seconds < 45) return 'under a minute';
|
|
15
|
+
const minutes = Math.max(1, Math.round(seconds / 60));
|
|
16
|
+
return minutes < 90 ? `about ${minutes} min` : `about ${Math.floor(minutes / 60)} h ${minutes % 60} min`;
|
|
17
|
+
}
|
|
18
|
+
const left = seconds => `, ${formatDuration(seconds)} left`;
|
|
19
|
+
export function transferProgress(report, { interval = 3000, now = () => performance.now() } = {}) {
|
|
20
|
+
let phase, first, printed = -Infinity, expected;
|
|
21
|
+
const line = event => {
|
|
22
|
+
const bytes = event.bytes === undefined ? '' : ` (${formatBytes(event.bytes)}${event.phase === 'building' && expected ? ` of about ${formatBytes(expected)}` : ''})`;
|
|
23
|
+
const parts = event.total === undefined ? `${event.parts} part${event.parts === 1 ? '' : 's'} so far` : `${event.parts} of ${event.total} parts`;
|
|
24
|
+
// Seconds left, from the rate since the phase's first sample.
|
|
25
|
+
const eta = () => {
|
|
26
|
+
const seconds = (now() - first.time) / 1000;
|
|
27
|
+
if (event === first.event || seconds <= 0 || event.parts === event.total) return '';
|
|
28
|
+
if (event.total !== undefined && event.parts > first.event.parts) return left((event.total - event.parts) * seconds / (event.parts - first.event.parts));
|
|
29
|
+
if (event.phase === 'building' && expected && event.bytes > first.event.bytes && event.bytes < expected) return left((expected - event.bytes) * seconds / (event.bytes - first.event.bytes));
|
|
30
|
+
return '';
|
|
31
|
+
};
|
|
32
|
+
switch (event.phase) {
|
|
33
|
+
case 'building': return `Building the archive: ${parts}${bytes}${eta()}.`;
|
|
34
|
+
case 'writing': return `Writing the archive: ${parts}${bytes}${eta()}.`;
|
|
35
|
+
case 'staging': return `Staging the restore: ${parts}${bytes}${eta()}.`;
|
|
36
|
+
case 'checking': return event.direction === 'export' ? 'Checking the written archive.' : 'Checking the whole archive before anything is restored.';
|
|
37
|
+
case 'finishing': return `All ${event.total} parts are staged; verifying them and publishing the restore. This can take several minutes.`;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
return event => {
|
|
41
|
+
if (event.phase === 'start') {
|
|
42
|
+
const noun = event.direction === 'export' ? `Exporting about ${formatBytes(event.bytes)} of local data` : `Restoring a ${formatBytes(event.bytes)} archive`;
|
|
43
|
+
if (event.direction === 'export') expected = event.bytes;
|
|
44
|
+
report(`${noun}. ${event.bytes / RATES[event.direction] < 45 ? 'This takes under a minute.' : `An archive this size usually takes ${formatDuration(event.bytes / RATES[event.direction])}; a busy machine takes longer. Progress follows every few seconds.`}`);
|
|
45
|
+
printed = now();
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const time = now(), edge = event.phase !== phase;
|
|
49
|
+
if (edge) { phase = event.phase; first = { time, event }; }
|
|
50
|
+
// A phase's last event (all parts built, written or staged) prints whatever the clock says.
|
|
51
|
+
const last = event.total !== undefined && event.parts === event.total;
|
|
52
|
+
if (edge || last || time - printed >= interval) { report(line(event)); printed = time; }
|
|
53
|
+
};
|
|
54
|
+
}
|
|
@@ -1,36 +1,16 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
3
3
|
import { resolve, dirname } from 'node:path';
|
|
4
|
-
import {
|
|
4
|
+
import { parseContract } from '../../foundation/runtime/seed-contract.mjs';
|
|
5
5
|
|
|
6
6
|
export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
7
7
|
export const json = value => JSON.stringify(value, null, 2) + '\n';
|
|
8
8
|
|
|
9
|
+
// The listing's clauses are the contract's published ones, read by the parser
|
|
10
|
+
// `rer check` and the run site use: a malformed contract throws its ContractError.
|
|
9
11
|
function clauses(text) {
|
|
10
|
-
const result =
|
|
11
|
-
|
|
12
|
-
const finish = () => {
|
|
13
|
-
if (!current) return;
|
|
14
|
-
const body = current.body.join(' ').replace(/\s+/gu, ' ').trim();
|
|
15
|
-
if (body) result.push({ id: current.id, title: current.title, text: body });
|
|
16
|
-
current = null;
|
|
17
|
-
};
|
|
18
|
-
const clauseLine = new RegExp(`^- \\*\\*(${clauseIdSource})\\s+[—–-]\\s+(.+?)\\.?\\*\\*[ \\t]*(.*)$`);
|
|
19
|
-
for (const line of text.split(/\r\n|\n|\r/)) {
|
|
20
|
-
const match = clauseLine.exec(line);
|
|
21
|
-
if (match) {
|
|
22
|
-
finish();
|
|
23
|
-
current = { id: match[1], title: match[2], body: [match[3]] };
|
|
24
|
-
continue;
|
|
25
|
-
}
|
|
26
|
-
if (!current) continue;
|
|
27
|
-
if (!line.trim() || /^[\t ]/.test(line)) {
|
|
28
|
-
current.body.push(line);
|
|
29
|
-
continue;
|
|
30
|
-
}
|
|
31
|
-
finish();
|
|
32
|
-
}
|
|
33
|
-
finish();
|
|
12
|
+
const result = parseContract(text).clauses.filter(clause => clause.kind === 'published' && clause.text)
|
|
13
|
+
.sort((a, b) => a.line - b.line).map(({ id, title, text }) => ({ id, title, text }));
|
|
34
14
|
if (!result.length) throw Error('CONTRACT.md must contain published clauses');
|
|
35
15
|
return result;
|
|
36
16
|
}
|