@runeditrun/rer 0.2.47 → 0.2.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/main.mjs +7 -1
- package/cli/portability-progress.mjs +19 -0
- package/cli/portability-worker.mjs +13 -3
- package/cli/portability.mjs +4 -3
- package/global/CHANGELOG.md +6 -0
- package/global/launch/facts.mjs +17 -10
- package/modules/demo-host/README.md +3 -2
- package/modules/demo-host/build.mjs +2 -2
- package/modules/demo-host/native/config.mjs +2 -1
- package/modules/demo-host/native/prepare.mjs +1 -1
- package/modules/demo-host/native/warm.mjs +19 -7
- package/modules/job-queue/identity.mjs +5 -4
- package/modules/portability/README.md +1 -1
- package/modules/portability/export.mjs +2 -2
- package/modules/portability/files.mjs +25 -6
- package/modules/portability/format.mjs +3 -1
- package/modules/portability/import.mjs +8 -0
- package/modules/portability/index.d.mts +2 -2
- package/modules/portability/index.mjs +2 -2
- package/modules/portability/progress.mjs +54 -0
- package/package.json +1 -1
package/cli/main.mjs
CHANGED
|
@@ -66,11 +66,17 @@ export async function main(argv = process.argv.slice(2)) {
|
|
|
66
66
|
else result = await (await import('./lifecycle.mjs')).runCommand(project, options, out);
|
|
67
67
|
out.result(true, 0, result); return 0;
|
|
68
68
|
} catch (error) {
|
|
69
|
-
const failure = error instanceof RerError ? error :
|
|
69
|
+
const failure = error instanceof RerError ? error : unexpectedFailure(error);
|
|
70
70
|
out.error(failure); out.result(false, failure.exitCode, failure.childExitCode === undefined ? {} : { childExitCode: failure.childExitCode, signal: failure.signal }); return failure.exitCode;
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
// A failure rer did not classify keeps its own first line; the hint is the next step.
|
|
75
|
+
function unexpectedFailure(error) {
|
|
76
|
+
const message = String(error?.message ?? error).split('\n')[0].trim().slice(0, 300);
|
|
77
|
+
return new RerError('command_failed', `${message || 'Command failed.'}\nUnexpected failure: run the command again, and if it repeats inspect the project operating documentation.`, 1);
|
|
78
|
+
}
|
|
79
|
+
|
|
74
80
|
function showHelp(options,out) {
|
|
75
81
|
if(options.json){out.event('help',{usage:help(options.verb)});out.result(true,0);}else out.human(help(options.verb));
|
|
76
82
|
return 0;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { lstat, readdir } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
// Bytes under a directory, for the first progress line's estimate. Missing or vanishing files count as zero.
|
|
5
|
+
export async function stateBytes(directory) {
|
|
6
|
+
let total = 0;
|
|
7
|
+
for (const entry of await readdir(directory, { withFileTypes: true }).catch(() => [])) {
|
|
8
|
+
const path = join(directory, entry.name);
|
|
9
|
+
total += entry.isDirectory() ? await stateBytes(path) : (await lstat(path).catch(() => ({ size: 0 }))).size;
|
|
10
|
+
}
|
|
11
|
+
return total;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// The class and code of an unexpected failure, when each is a plain identifier. Its message can hold row data.
|
|
15
|
+
export function nativeFailure(error) {
|
|
16
|
+
const name = typeof error?.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(error.name) ? error.name : 'unknown';
|
|
17
|
+
const code = typeof error?.code === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(error.code) ? error.code : Number.isSafeInteger(error?.code) ? String(error.code) : undefined;
|
|
18
|
+
return { class: name, ...(code ? { code } : {}) };
|
|
19
|
+
}
|
|
@@ -6,9 +6,11 @@ import { legacyTargetState, requireLegacyEmpty } from '../modules/portability/le
|
|
|
6
6
|
import { migrationReplay } from '../modules/portability/migration-history.mjs';
|
|
7
7
|
import { appliedMigrations as appliedMigrationNames } from '../modules/portability/bridge.mjs';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
-
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
9
|
+
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
10
10
|
import { createRequire } from 'node:module';
|
|
11
11
|
import { createPortability, PortabilityError, readArchive } from '../modules/portability/index.mjs';
|
|
12
|
+
import { transferProgress } from '../modules/portability/progress.mjs';
|
|
13
|
+
import { nativeFailure, stateBytes } from './portability-progress.mjs';
|
|
12
14
|
import { parseJsonc } from './jsonc.mjs';
|
|
13
15
|
import { loadPolicy } from './portability-policy.mjs';
|
|
14
16
|
|
|
@@ -33,8 +35,12 @@ async function dataConfiguration(root) {
|
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
async function run(direction, { root, path, journal, binding, buckets, policy, policyModule, transient, legacyRecovery }) {
|
|
38
|
+
// Lines for the person go to the CLI over IPC, which prints them as the command's progress.
|
|
39
|
+
const progress = transferProgress(line => process.send({ progress: line }));
|
|
40
|
+
if (!legacyRecovery && ['export', 'import'].includes(direction)) progress({ phase: 'start', direction, bytes: direction === 'export' ? await stateBytes(join(root, '.wrangler/state')) : (await stat(path)).size });
|
|
36
41
|
// Validate all inputs before opening bindings or acquiring a restore lock.
|
|
37
42
|
if (direction === 'import') {
|
|
43
|
+
progress({ phase: 'checking', direction });
|
|
38
44
|
for await (const record of fileRecords(path,legacyRecovery?readLegacyArchive:readArchive)) {}
|
|
39
45
|
if(journal)for await(const record of fileRecords(journal)) {}
|
|
40
46
|
}
|
|
@@ -56,7 +62,9 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
56
62
|
await ensurePortabilitySchema(proxy.env[binding], await portabilityPlan(unstable_splitSqlQuery));
|
|
57
63
|
if(legacyRecovery)return await recoverLegacyArchive(options,path,legacyRecovery.output,transient,legacyRecovery.acknowledged);
|
|
58
64
|
const service = await createPortability(options);
|
|
59
|
-
|
|
65
|
+
// This checkout's operation lock means no other import can be running: staging left behind is a crashed restore's.
|
|
66
|
+
if (direction === 'import' && await service.imports.abortAbandoned()) process.send({ progress: 'Cleaned up an unfinished earlier import: a restore that stopped part-way left its staged data here. Its staging is removed and this import starts from the beginning.' });
|
|
67
|
+
return await (direction === 'export' ? exportArchive(service,path,transient,progress) : importArchive(service,path,transient,journal,progress));
|
|
60
68
|
} finally { await proxy?.dispose(); await rm(configPath, { force: true }); }
|
|
61
69
|
}
|
|
62
70
|
|
|
@@ -73,6 +81,8 @@ catch (error) {
|
|
|
73
81
|
const migration = error instanceof PortabilityError && error.code.startsWith('schema_mismatch:migration_data:') ? error.code.slice('schema_mismatch:migration_data:'.length) : undefined;
|
|
74
82
|
// A populated destination names its table or bucket: schema names, never row data.
|
|
75
83
|
const occupied = error instanceof PortabilityError && /^(target|bucket)_not_empty:/.test(error.code) ? error.code.slice(error.code.indexOf(':') + 1) : undefined;
|
|
84
|
+
// Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
|
|
85
|
+
const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
|
|
76
86
|
process.exitCode = exitCode;
|
|
77
|
-
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), exitCode });
|
|
87
|
+
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), ...(native ? { native } : {}), exitCode });
|
|
78
88
|
}
|
package/cli/portability.mjs
CHANGED
|
@@ -73,17 +73,18 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
73
73
|
const child = store.spawn(`${direction === 'policy-preflight' ? 'Checking archive policy' : direction === 'legacy-preflight' ? 'Checking empty recovery destination' : direction === 'export' ? 'Exporting' : 'Restoring'} local installation data`, process.execPath,
|
|
74
74
|
['--import', notice, worker, direction, JSON.stringify(input)], { cwd: input.root, signal, env: { ...process.env, WRANGLER_SEND_METRICS: 'false' }, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
|
|
75
75
|
let receipt;
|
|
76
|
-
|
|
76
|
+
// Progress lines come first, as {progress}; the last message is the receipt.
|
|
77
|
+
child.on('message', message => { if (typeof message?.progress === 'string') store.report(message.progress); else receipt = message; });
|
|
77
78
|
const closed = new Promise(resolve => child.once('close', (code, signal) => resolve({ code, signal })));
|
|
78
79
|
await new Promise(resolve => { child.once('exit', resolve); child.once('error', resolve); });
|
|
79
80
|
await store.stop(child);
|
|
80
81
|
const status = await closed;
|
|
81
|
-
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed',
|
|
82
|
+
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed', `Local portability ended before it finished (${status.signal ? `signal ${status.signal}` : `exit code ${status.code}`}). Run the same command again; it starts over${direction === 'import' ? ' and first cleans up anything this attempt staged' : ''}.`, 1);
|
|
82
83
|
if (status.code !== 0 || !receipt.ok) throw new RerError(receipt.code ?? 'transfer_failed', receipt.reason==='legacy_restored_export_failed'
|
|
83
84
|
? 'Legacy data was restored and audited, but its v2 export failed. Preserve this installation, resolve the output problem, then run rer export --output FILE. Do not reset it to retry recovery.'
|
|
84
85
|
: receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
|
|
85
86
|
: receipt.occupied ? `Restore refused (${receipt.reason}): ${receipt.reason === 'bucket_not_empty' ? `storage bucket ${receipt.occupied} in this installation already holds files` : `table ${receipt.occupied} in this installation already holds rows its migrations did not insert`}, and a restore only goes into an empty installation. To replace this installation's data with the archive, keep a copy with rer export --output exports/before-restore.rer, then run rer reset --yes and rer import --input ${relative(input.root, input.path)} --yes again.`
|
|
86
|
-
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'})
|
|
87
|
+
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
|
|
87
88
|
delete receipt.ok;
|
|
88
89
|
return receipt;
|
|
89
90
|
}
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.48
|
|
4
|
+
|
|
5
|
+
- **The job queue no longer aborts or strands a waiting command under load.** Before, when another queued job's owner did not answer its 3 s identity probe, the waiting build, verify or deploy stopped with "Command failed; inspect the project operating documentation." A probe timeout now counts as "the owner may still be alive", so the waiting job samples again instead of giving up. The same holds for the queue lock and for nested-command membership, which had left a job queued behind its own parent until the queue's maximum wait. A recorded failed install is retried only when `pnpm-lock.yaml` or `package.json` changed since it failed. An error that is not one of rer's own now prints its message and a hint instead of the generic line (#219).
|
|
6
|
+
- **The demo start page warms the chunks the seed actually built.** The warm list now comes from the build output: the client Vite manifest when the build wrote one, else the build's `assetsDir` (recorded in `demo.json` from the loaded Vite config), else `assets`. Seeds that build to a non-default directory, such as Lamp's `/app-assets/`, had an empty warm list; Lamp's is now 32 chunks with the console's routes first (#223).
|
|
7
|
+
- **`rer export` and `rer import` show progress, and a crashed import cleans up after itself.** Export prints the expected time, then parts built, parts written and bytes with the time left (one line every 3 s, plus each phase edge). Import prints parts staged. An import that died part-way leaves staging that the next import removes, saying "Cleaned up an unfinished earlier import". An unexpected failure prints its error class or code and "Run the same command again; it starts over." (#224).
|
|
8
|
+
|
|
3
9
|
## 0.2.47
|
|
4
10
|
|
|
5
11
|
- **The publication scan resolves GitHub repositories in bulk.** Before, the scan looked up each GitHub repository a text names with its own REST call. Now it collects every reference first and resolves them together, in one aliased GraphQL query per 100 repositories, which also draws on GitHub's separate GraphQL budget. A repository counts as public or private only on a complete answer that is not disabled or locked. It counts as missing only when GitHub returns NOT_FOUND for exactly that name. Any other answer for a repository, and any batch that is not a complete HTTP 200 GraphQL document, falls back to the earlier strict REST lookup for that repository. Nothing is cached on disk. A run-oncall scan now makes 1 gh call instead of 19, and its report is byte-identical (#216).
|
package/global/launch/facts.mjs
CHANGED
|
@@ -20,17 +20,24 @@ export async function describeCheckout(root, entry, env, signal) {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
// Install when the last install
|
|
24
|
-
// pnpm writes node_modules/.modules.yaml only once dependencies
|
|
25
|
-
// built, so a stopped install leaves it absent. An install this
|
|
26
|
-
// is finished only when its record says so: that also covers the
|
|
27
|
-
// own install hooks, which pnpm runs after writing the manifest.
|
|
23
|
+
// Install when the last install was killed, nothing is linked, or the lockfile
|
|
24
|
+
// changed since. pnpm writes node_modules/.modules.yaml only once dependencies
|
|
25
|
+
// are linked and built, so a stopped install leaves it absent. An install this
|
|
26
|
+
// launcher ran is finished only when its record says so: that also covers the
|
|
27
|
+
// project's own install hooks, which pnpm runs after writing the manifest. A
|
|
28
|
+
// recorded failed install is retried only when the lockfile or package.json is
|
|
29
|
+
// newer than both that attempt and what is installed; dependencies repaired
|
|
30
|
+
// since are current, and the next launch uses them.
|
|
28
31
|
export async function needsInstall(root, last) {
|
|
29
|
-
if (last
|
|
30
|
-
const
|
|
31
|
-
if (
|
|
32
|
-
const
|
|
33
|
-
|
|
32
|
+
if (last?.state === 'running') return true;
|
|
33
|
+
const linked = await mtime(join(root, 'node_modules/.modules.yaml'));
|
|
34
|
+
if (linked === undefined) return true;
|
|
35
|
+
const since = Math.max(linked, last?.at ?? 0);
|
|
36
|
+
for (const file of last?.state === 'failed' ? ['pnpm-lock.yaml', 'package.json'] : ['pnpm-lock.yaml']) {
|
|
37
|
+
const changed = await mtime(join(root, file));
|
|
38
|
+
if (changed !== undefined && changed > since) return true;
|
|
39
|
+
}
|
|
40
|
+
return false;
|
|
34
41
|
}
|
|
35
42
|
|
|
36
43
|
// A checkout is set up when its pin's describe says so (`local.prepared`, from
|
|
@@ -13,8 +13,9 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
|
|
|
13
13
|
- `worker.mjs`, `pages.mjs`, `start-script.mjs`: Turnstile start, progress,
|
|
14
14
|
cancellation and visitor routing. Static assets are served at the edge.
|
|
15
15
|
- `native/warm.mjs`: the start page's prefetch list, built from `dist/client`
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
(the Vite manifest if the build wrote one, else the build's `assetsDir`, else
|
|
17
|
+
`assets`) and TanStack Start's route manifest, the console's views first,
|
|
18
|
+
capped by compressed size.
|
|
18
19
|
- `capacity.mjs`: concurrent visitor admission and operator diagnostics. This
|
|
19
20
|
is used by the native host as well as retained during legacy upgrades.
|
|
20
21
|
- `upgrade.mjs`: verifies ownership, removes the old Container application and
|
|
@@ -40,9 +40,9 @@ export async function buildDemo(root, output, declaration) {
|
|
|
40
40
|
await rm(output, { recursive: true, force: true });
|
|
41
41
|
await mkdir(join(output, 'app'), { recursive: true });
|
|
42
42
|
await cp(join(root, 'dist'), join(output, 'app/dist'), { recursive: true });
|
|
43
|
-
// Where the console is mounted orders what the start page prefetches (native/warm.mjs).
|
|
43
|
+
// Where the console is mounted orders what the start page prefetches, and the build's assetsDir says where its chunks are (native/warm.mjs).
|
|
44
44
|
const { operatorPath } = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8').catch(() => '{}')).deploy ?? {};
|
|
45
|
-
await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath }) + '\n');
|
|
45
|
+
await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath, assetsDir: loaded.config.build?.assetsDir }) + '\n');
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
if (process.argv[1] === import.meta.filename) {
|
|
@@ -2,7 +2,8 @@ import { parseCron } from './cron.mjs';
|
|
|
2
2
|
|
|
3
3
|
export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'r2-store', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
|
|
4
4
|
// Hashed build output: served immutable, and what the start page may prefetch.
|
|
5
|
-
export const
|
|
5
|
+
export const hashedAsset = (assetsDir = 'assets') => new RegExp(`^/${assetsDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/.*-[\\w-]{8,}\\.[\\w]+$`);
|
|
6
|
+
export const HASHED_ASSET = hashedAsset();
|
|
6
7
|
export const ISOLATE_FILES = ['bindings', 'namespace', 'r2-client', 'd1', 'sql', 'seed-object'];
|
|
7
8
|
export function nativeBindings(config) {
|
|
8
9
|
for (const kind of ['kv_namespaces', 'services', 'workflows', 'ai', 'vectorize', 'hyperdrive', 'analytics_engine_datasets', 'browser', 'images', 'send_email', 'dispatch_namespaces', 'mtls_certificates', 'pipelines', 'secrets_store_secrets', 'unsafe', 'vpc_services', 'vpc_networks', 'ratelimits', 'worker_loaders', 'containers', 'ai_search_namespaces', 'ai_search', 'agent_memory', 'connect', 'artifacts', 'unsafe_hello_world', 'flagship', 'logfwdr', 'version_metadata']) {
|
|
@@ -34,7 +34,7 @@ export async function prepareNative(folder) {
|
|
|
34
34
|
const bundle = { compatibilityDate: config.compatibility_date, compatibilityFlags: [...new Set([...(config.compatibility_flags ?? []), 'nodejs_compat'])], modules,
|
|
35
35
|
bindings, migrations: manifest.migrations, vars: config.vars ?? {}, crons: config.triggers?.crons ?? [],
|
|
36
36
|
assetPaths: assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path),
|
|
37
|
-
warm: (await warmList(assets, root, manifest.operatorPath)).paths };
|
|
37
|
+
warm: (await warmList(assets, root, manifest.operatorPath, manifest.assetsDir)).paths };
|
|
38
38
|
bundle.digest = createHash('sha256').update(JSON.stringify(bundle)).digest('hex');
|
|
39
39
|
await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
|
|
40
40
|
await writeFile(join(folder, 'modules/demo-host/native/bundle.mjs'), `export default ${JSON.stringify(bundle)};\n`);
|
|
@@ -2,7 +2,7 @@ import { readdir } from 'node:fs/promises';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { pathToFileURL } from 'node:url';
|
|
4
4
|
import { gzipSync } from 'node:zlib';
|
|
5
|
-
import {
|
|
5
|
+
import { hashedAsset } from './config.mjs';
|
|
6
6
|
|
|
7
7
|
// The start page prefetches the console's hashed JS and CSS while the visitor
|
|
8
8
|
// waits for the human check and sign-in, so each view's first visit finds its
|
|
@@ -16,10 +16,22 @@ const segments = path => path.split('/').filter(Boolean);
|
|
|
16
16
|
const dynamic = path => segments(path).filter(part => part.startsWith('$')).length;
|
|
17
17
|
const within = (path, root) => path === root || path.startsWith(root.endsWith('/') ? root : root + '/');
|
|
18
18
|
|
|
19
|
+
// The hashed JS and CSS the client build produced: the files its Vite manifest
|
|
20
|
+
// names (build.manifest), else everything under the configured build.assetsDir
|
|
21
|
+
// (Vite's default is 'assets'). Never source maps or unhashed files.
|
|
22
|
+
const MANIFESTS = ['.vite/manifest.json', 'manifest.json'];
|
|
23
|
+
function builtChunks(files, assetsDir) {
|
|
24
|
+
const manifest = files.find(file => MANIFESTS.includes(file.path));
|
|
25
|
+
const paths = (manifest
|
|
26
|
+
? Object.values(JSON.parse(Buffer.from(manifest.base64, 'base64'))).flatMap(entry => [entry.file, ...(entry.css ?? [])]).map(path => '/' + path)
|
|
27
|
+
: files.map(file => '/' + file.path).filter(path => hashedAsset(assetsDir).test(path)));
|
|
28
|
+
return new Set(paths.filter(path => /-[\w-]{8,}\.(js|css)$/.test(path)));
|
|
29
|
+
}
|
|
30
|
+
|
|
19
31
|
// TanStack Start's route manifest from the server build: each route's CSS and
|
|
20
32
|
// the chunks it preloads.
|
|
21
|
-
async function startRoutes(server) {
|
|
22
|
-
const directory = join(server,
|
|
33
|
+
async function startRoutes(server, assetsDir) {
|
|
34
|
+
const directory = join(server, assetsDir);
|
|
23
35
|
const file = (await readdir(directory).catch(() => [])).find(name => /^_tanstack-start-manifest_v-[\w-]+\.js$/.test(name));
|
|
24
36
|
if (!file) return {};
|
|
25
37
|
const { tsrStartManifest } = await import(pathToFileURL(join(directory, file)));
|
|
@@ -36,10 +48,10 @@ function routeOrder(routes, consolePath) {
|
|
|
36
48
|
}
|
|
37
49
|
|
|
38
50
|
/** The ordered, size-capped paths the start page prefetches, from dist/client's files and the server build. */
|
|
39
|
-
export async function warmList(files, server, consolePath = '/app') {
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
const routes = await startRoutes(server);
|
|
51
|
+
export async function warmList(files, server, consolePath = '/app', assetsDir = 'assets') {
|
|
52
|
+
const chunks = builtChunks(files, assetsDir);
|
|
53
|
+
const sizes = new Map(files.filter(file => chunks.has('/' + file.path)).map(file => ['/' + file.path, gzipSync(Buffer.from(file.base64, 'base64')).length]));
|
|
54
|
+
const routes = await startRoutes(server, assetsDir);
|
|
43
55
|
const ordered = new Set();
|
|
44
56
|
for (const id of routeOrder(routes, consolePath)) for (const path of [...(routes[id].css ?? []), ...(routes[id].preloads ?? [])]) if (sizes.has(path)) ordered.add(path);
|
|
45
57
|
for (const path of [...sizes.keys()].sort((a, b) => sizes.get(a) - sizes.get(b))) ordered.add(path);
|
|
@@ -31,14 +31,15 @@ export async function alive(owner) {
|
|
|
31
31
|
return new Promise((resolve, reject) => {
|
|
32
32
|
const connection = createConnection(owner.socket);
|
|
33
33
|
let body = '';
|
|
34
|
-
connection.setTimeout(3000, () => connection.destroy(Error('Owner identity probe timed out
|
|
34
|
+
connection.setTimeout(3000, () => connection.destroy(Object.assign(Error('Owner identity probe timed out'), { code: 'PROBE_TIMEOUT' })));
|
|
35
35
|
connection.on('data', data => { body += data; if (body.length > 128) connection.destroy(Error('Invalid owner identity')); });
|
|
36
36
|
connection.on('end', () => resolve(body === owner.token));
|
|
37
37
|
connection.on('error', async error => {
|
|
38
38
|
if (['ENOENT', 'ECONNREFUSED'].includes(error.code)) resolve(false);
|
|
39
|
-
else if (error.code
|
|
40
|
-
// Listener shutdown can reset a probe after the initial PID check
|
|
41
|
-
//
|
|
39
|
+
else if (['ECONNRESET', 'PROBE_TIMEOUT'].includes(error.code)) {
|
|
40
|
+
// Listener shutdown can reset a probe after the initial PID check, and a
|
|
41
|
+
// busy owner can answer late. Neither proves death: reclaim only a process
|
|
42
|
+
// that is gone; otherwise retain the owner until the next sample.
|
|
42
43
|
try {
|
|
43
44
|
const after = await processInfo(owner.pid);
|
|
44
45
|
resolve(Boolean(after && !after.zombie && after.start === owner.start));
|
|
@@ -121,7 +121,7 @@ idempotence and recovery; D1 cannot roll back Durable Object or external state.
|
|
|
121
121
|
The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
|
|
122
122
|
closing record. `readArchive(stream)` uses the shared bounded, fatal UTF-8 line
|
|
123
123
|
reader. The default maximum line size is 24 MiB; callers can set a smaller bound.
|
|
124
|
-
No runtime or CLI materializes the whole archive. Readers enforce a 16 MiB
|
|
124
|
+
No runtime or CLI materializes the whole archive. The manifest opens the archive and holds the final part count and payload hashes, so an export builds all its parts in the job before any byte is written; `exportArchive` and `importArchive` take an optional `progress` callback (`progress.mjs` turns its events into throttled lines with a time-left estimate). Readers enforce a 16 MiB
|
|
125
125
|
canonical UTF-8 part limit; `readArchive(stream,{partBytes})` can lower it.
|
|
126
126
|
Import uploads enforce the policy's part limit before any staging write.
|
|
127
127
|
A part has at most 1,000 rows;
|
|
@@ -96,7 +96,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
96
96
|
if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
|
|
97
97
|
const before=row.completed,items=await nextItems(context,row);
|
|
98
98
|
if(items) {
|
|
99
|
-
const part=await makePart(before,items)
|
|
99
|
+
const part=await makePart(before,items),size=await checkPart(part,before,row.metadata.partBytes);row.completed++;row.metadata.bytes=(row.metadata.bytes??0)+size+1;
|
|
100
100
|
try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
|
|
101
101
|
catch(e){if((await job(context.db,id)).completed===before)throw e;}
|
|
102
102
|
if(expectedPart!==undefined)expectedPart++;
|
|
@@ -114,7 +114,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
114
114
|
return exportStatus(context,id);
|
|
115
115
|
}
|
|
116
116
|
export async function exportStatus(context,id) {
|
|
117
|
-
const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,contentHash:row.metadata.contentHash??null};
|
|
117
|
+
const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,bytes:row.metadata.bytes??0,contentHash:row.metadata.contentHash??null};
|
|
118
118
|
}
|
|
119
119
|
export async function downloadExport(context,id) {
|
|
120
120
|
const row=await job(context.db,id,'export');insist(row.phase==='complete','export_incomplete',409);
|
|
@@ -4,6 +4,7 @@ import { dirname } from 'node:path';
|
|
|
4
4
|
import { Readable } from 'node:stream';
|
|
5
5
|
import { randomUUID } from 'node:crypto';
|
|
6
6
|
import { readArchive } from './index.mjs';
|
|
7
|
+
import { canonicalJson } from './hash.mjs';
|
|
7
8
|
export async function* fileRecords(path,reader=readArchive) {
|
|
8
9
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
9
10
|
try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
|
|
@@ -19,23 +20,35 @@ function receipt(manifest, closing, transient) {
|
|
|
19
20
|
requiredConfiguration: manifest.requiredConfiguration,
|
|
20
21
|
};
|
|
21
22
|
}
|
|
22
|
-
|
|
23
|
+
// `progress` receives {phase, parts, total?, bytes?} events; the CLI turns them into lines (progress.mjs).
|
|
24
|
+
export async function exportArchive(service, path, transient, progress) {
|
|
23
25
|
let id, temporary, file;
|
|
24
26
|
try {
|
|
25
27
|
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
26
28
|
temporary = path + '.' + randomUUID() + '.partial';
|
|
27
29
|
file = await open(temporary, 'wx', 0o600);
|
|
28
30
|
id = await service.exports.create();
|
|
29
|
-
|
|
31
|
+
// The manifest leads the archive and holds the final part count and payload hashes, so
|
|
32
|
+
// nothing can be written until every part is built (export.mjs advanceExport). One part per
|
|
33
|
+
// advance keeps the progress line moving; a part is at most 16 MiB.
|
|
34
|
+
let status;
|
|
35
|
+
do {
|
|
36
|
+
status = await service.exports.advance(id, { maxParts: 1 });
|
|
37
|
+
progress?.({ phase: 'building', parts: status.completed, bytes: status.bytes, ...(status.phase === 'complete' ? { total: status.total } : {}) });
|
|
38
|
+
} while (status.phase === 'building');
|
|
30
39
|
const reader = (await service.exports.download(id)).getReader();
|
|
31
40
|
try {
|
|
32
|
-
|
|
41
|
+
// One chunk per record: the manifest, then each part, then the closing record.
|
|
42
|
+
for (let record = 0, written = 0;; record++) {
|
|
33
43
|
const { done, value } = await reader.read(); if (done) break;
|
|
34
44
|
let offset = 0;
|
|
35
45
|
while (offset < value.length) offset += (await file.write(value, offset)).bytesWritten;
|
|
46
|
+
written += value.length;
|
|
47
|
+
if (record > 0 && record <= status.total) progress?.({ phase: 'writing', parts: record, total: status.total, bytes: written });
|
|
36
48
|
}
|
|
37
49
|
} finally { await reader.cancel(); reader.releaseLock(); }
|
|
38
50
|
await file.sync(); await file.close(); file = undefined;
|
|
51
|
+
progress?.({ phase: 'checking', direction: 'export' });
|
|
39
52
|
let manifest, closing;
|
|
40
53
|
for await (const record of fileRecords(temporary)) { if (record.kind === 'manifest') manifest = record; if (record.kind === 'closing') closing = record; }
|
|
41
54
|
await link(temporary, path);
|
|
@@ -46,19 +59,25 @@ export async function exportArchive(service, path, transient) {
|
|
|
46
59
|
if (id) await service.exports.abort(id);
|
|
47
60
|
}
|
|
48
61
|
}
|
|
49
|
-
export async function importArchive(service, path, transient, journal) {
|
|
50
|
-
let id, manifest, closing, complete = false;
|
|
62
|
+
export async function importArchive(service, path, transient, journal, progress) {
|
|
63
|
+
let id, manifest, closing, complete = false, staged = 0;
|
|
51
64
|
try {
|
|
52
65
|
// Validate both complete archives before creating any restore state.
|
|
53
66
|
let later;
|
|
67
|
+
progress?.({ phase: 'checking', direction: 'import' });
|
|
54
68
|
for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
|
|
55
69
|
if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
|
|
56
70
|
}
|
|
57
71
|
for await (const record of fileRecords(path)) {
|
|
58
72
|
if (record.kind === 'manifest') { manifest = record; id = await service.imports.create(manifest); }
|
|
59
|
-
else if (record.kind === 'part')
|
|
73
|
+
else if (record.kind === 'part') {
|
|
74
|
+
await service.imports.uploadPart(id, record.index, record);
|
|
75
|
+
staged += Buffer.byteLength(canonicalJson(record)) + 1;
|
|
76
|
+
progress?.({ phase: 'staging', parts: record.index + 1, total: manifest.partCount, bytes: staged });
|
|
77
|
+
}
|
|
60
78
|
else closing = record;
|
|
61
79
|
}
|
|
80
|
+
progress?.({ phase: 'finishing', total: manifest.partCount });
|
|
62
81
|
const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
|
|
63
82
|
return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}) };
|
|
64
83
|
} finally { if (id && !complete) await service.imports.abort(id); }
|
|
@@ -11,9 +11,11 @@ export async function makePart(index,items) {
|
|
|
11
11
|
export async function checkPart(part,index,partBytes=MAX_PART_BYTES) {
|
|
12
12
|
insist(part?.kind==='part' && part.index===index && Array.isArray(part.items) && part.items.length>0 && Object.keys(part).sort().join(',')==='index,items,kind,sha256','part_order');
|
|
13
13
|
insist(Number.isSafeInteger(partBytes) && partBytes>0 && partBytes<=MAX_PART_BYTES,'invalid_part_limit');
|
|
14
|
-
|
|
14
|
+
const size=utf8(canonicalJson(part)).length;
|
|
15
|
+
insist(size<=partBytes,'part_byte_limit',413);
|
|
15
16
|
insist(part.sha256===await sha256(canonicalJson({kind:part.kind,index:part.index,items:part.items})),'part_checksum');
|
|
16
17
|
insist(part.items.filter(x=>x.kind==='row').length<=1000,'part_row_limit');
|
|
18
|
+
return size;
|
|
17
19
|
}
|
|
18
20
|
export async function checkManifest(m) {
|
|
19
21
|
insist(m?.kind==='manifest' && m.format===FORMAT,'archive_format');
|
|
@@ -197,3 +197,11 @@ export async function abortImport(context,id) {
|
|
|
197
197
|
await context.db.batch([save(context.db,row),statement(context.db,'UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1 AND import_job=?',id)]);
|
|
198
198
|
return importStatus(context,id);
|
|
199
199
|
}
|
|
200
|
+
// A restore's staging outlives a worker that died mid-restore (a crashed process or workerd) and refuses
|
|
201
|
+
// every later import as running. Only a caller that owns the destination, as the local CLI does under the
|
|
202
|
+
// checkout's operation lock, may call this: a hosted service can have a live import.
|
|
203
|
+
export async function abortAbandonedImport(context) {
|
|
204
|
+
const id=(await state(context.db)).import_job;
|
|
205
|
+
if(!id)return null;
|
|
206
|
+
await abortImport(context,id);return id;
|
|
207
|
+
}
|
|
@@ -28,7 +28,7 @@ export interface ArchivePolicy {
|
|
|
28
28
|
extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
|
|
29
29
|
partBytes?: number;
|
|
30
30
|
}
|
|
31
|
-
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
|
|
31
|
+
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
|
|
32
32
|
/** Later additive migrations an older archive was restored across. */
|
|
33
33
|
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
|
|
34
34
|
/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
|
|
@@ -36,7 +36,7 @@ export interface MigrationReplay {names:string[];statements(name:string):string[
|
|
|
36
36
|
export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
|
|
37
37
|
export interface Portability {
|
|
38
38
|
exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
|
|
39
|
-
imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>};
|
|
39
|
+
imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
|
|
40
40
|
status(id:string):Promise<Status>;actor(id:string):Promise<string>;
|
|
41
41
|
erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
|
|
42
42
|
redactionStatements():Promise<D1PreparedStatement[]>;
|
|
@@ -2,7 +2,7 @@ import { policy, catalogue } from './catalogue.mjs';
|
|
|
2
2
|
import { wireDatabase } from '../d1-transport/index.mjs';
|
|
3
3
|
import { insist, job } from './storage.mjs';
|
|
4
4
|
import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
|
|
5
|
-
import { createImport, uploadPart, finalizeImport, importStatus, abortImport } from './import.mjs';
|
|
5
|
+
import { createImport, uploadPart, finalizeImport, importStatus, abortImport, abortAbandonedImport } from './import.mjs';
|
|
6
6
|
import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
|
|
7
7
|
export { archiveLines } from './lines.mjs';
|
|
8
8
|
export { excludedTables, excludedTable } from './catalogue.mjs';
|
|
@@ -17,7 +17,7 @@ export async function createPortability({db,buckets={},policy:input,migrations})
|
|
|
17
17
|
const context={db,buckets,config,migrations};
|
|
18
18
|
return {
|
|
19
19
|
exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
|
|
20
|
-
imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id)},
|
|
20
|
+
imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
|
|
21
21
|
status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
|
|
22
22
|
actor:async id=>(await job(db,id)).actor,
|
|
23
23
|
erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Human progress for a long export or import. The caller sends events; this turns
|
|
2
|
+
// them into at most one line every few seconds, with an estimate of the time left
|
|
3
|
+
// once a phase has two samples. The first and last line of every phase always show.
|
|
4
|
+
// Rates are bytes per second measured on a 1.2 GB local install on a loaded machine
|
|
5
|
+
// (export 1725 s, import 2234 s); a quiet machine is faster.
|
|
6
|
+
const RATES = { export: 700_000, import: 550_000 };
|
|
7
|
+
const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'];
|
|
8
|
+
export function formatBytes(bytes) {
|
|
9
|
+
let value = bytes, unit = 0;
|
|
10
|
+
while (value >= 1000 && unit < UNITS.length - 1) { value /= 1000; unit++; }
|
|
11
|
+
return `${unit && value < 100 ? value.toFixed(1) : Math.round(value)} ${UNITS[unit]}`;
|
|
12
|
+
}
|
|
13
|
+
export function formatDuration(seconds) {
|
|
14
|
+
if (seconds < 45) return 'under a minute';
|
|
15
|
+
const minutes = Math.max(1, Math.round(seconds / 60));
|
|
16
|
+
return minutes < 90 ? `about ${minutes} min` : `about ${Math.floor(minutes / 60)} h ${minutes % 60} min`;
|
|
17
|
+
}
|
|
18
|
+
const left = seconds => `, ${formatDuration(seconds)} left`;
|
|
19
|
+
export function transferProgress(report, { interval = 3000, now = () => performance.now() } = {}) {
|
|
20
|
+
let phase, first, printed = -Infinity, expected;
|
|
21
|
+
const line = event => {
|
|
22
|
+
const bytes = event.bytes === undefined ? '' : ` (${formatBytes(event.bytes)}${event.phase === 'building' && expected ? ` of about ${formatBytes(expected)}` : ''})`;
|
|
23
|
+
const parts = event.total === undefined ? `${event.parts} part${event.parts === 1 ? '' : 's'} so far` : `${event.parts} of ${event.total} parts`;
|
|
24
|
+
// Seconds left, from the rate since the phase's first sample.
|
|
25
|
+
const eta = () => {
|
|
26
|
+
const seconds = (now() - first.time) / 1000;
|
|
27
|
+
if (event === first.event || seconds <= 0 || event.parts === event.total) return '';
|
|
28
|
+
if (event.total !== undefined && event.parts > first.event.parts) return left((event.total - event.parts) * seconds / (event.parts - first.event.parts));
|
|
29
|
+
if (event.phase === 'building' && expected && event.bytes > first.event.bytes && event.bytes < expected) return left((expected - event.bytes) * seconds / (event.bytes - first.event.bytes));
|
|
30
|
+
return '';
|
|
31
|
+
};
|
|
32
|
+
switch (event.phase) {
|
|
33
|
+
case 'building': return `Building the archive: ${parts}${bytes}${eta()}.`;
|
|
34
|
+
case 'writing': return `Writing the archive: ${parts}${bytes}${eta()}.`;
|
|
35
|
+
case 'staging': return `Staging the restore: ${parts}${bytes}${eta()}.`;
|
|
36
|
+
case 'checking': return event.direction === 'export' ? 'Checking the written archive.' : 'Checking the whole archive before anything is restored.';
|
|
37
|
+
case 'finishing': return `All ${event.total} parts are staged; verifying them and publishing the restore. This can take several minutes.`;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
return event => {
|
|
41
|
+
if (event.phase === 'start') {
|
|
42
|
+
const noun = event.direction === 'export' ? `Exporting about ${formatBytes(event.bytes)} of local data` : `Restoring a ${formatBytes(event.bytes)} archive`;
|
|
43
|
+
if (event.direction === 'export') expected = event.bytes;
|
|
44
|
+
report(`${noun}. ${event.bytes / RATES[event.direction] < 45 ? 'This takes under a minute.' : `An archive this size usually takes ${formatDuration(event.bytes / RATES[event.direction])}; a busy machine takes longer. Progress follows every few seconds.`}`);
|
|
45
|
+
printed = now();
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const time = now(), edge = event.phase !== phase;
|
|
49
|
+
if (edge) { phase = event.phase; first = { time, event }; }
|
|
50
|
+
// A phase's last event (all parts built, written or staged) prints whatever the clock says.
|
|
51
|
+
const last = event.total !== undefined && event.parts === event.total;
|
|
52
|
+
if (edge || last || time - printed >= interval) { report(line(event)); printed = time; }
|
|
53
|
+
};
|
|
54
|
+
}
|