@runeditrun/rer 0.2.46 → 0.2.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/cli/main.mjs CHANGED
@@ -66,11 +66,17 @@ export async function main(argv = process.argv.slice(2)) {
66
66
  else result = await (await import('./lifecycle.mjs')).runCommand(project, options, out);
67
67
  out.result(true, 0, result); return 0;
68
68
  } catch (error) {
69
- const failure = error instanceof RerError ? error : new RerError('command_failed', 'Command failed; inspect the project operating documentation.', 1);
69
+ const failure = error instanceof RerError ? error : unexpectedFailure(error);
70
70
  out.error(failure); out.result(false, failure.exitCode, failure.childExitCode === undefined ? {} : { childExitCode: failure.childExitCode, signal: failure.signal }); return failure.exitCode;
71
71
  }
72
72
  }
73
73
 
74
+ // A failure rer did not classify keeps its own first line; the hint is the next step.
75
+ function unexpectedFailure(error) {
76
+ const message = String(error?.message ?? error).split('\n')[0].trim().slice(0, 300);
77
+ return new RerError('command_failed', `${message || 'Command failed.'}\nUnexpected failure: run the command again, and if it repeats inspect the project operating documentation.`, 1);
78
+ }
79
+
74
80
  function showHelp(options,out) {
75
81
  if(options.json){out.event('help',{usage:help(options.verb)});out.result(true,0);}else out.human(help(options.verb));
76
82
  return 0;
@@ -0,0 +1,19 @@
1
+ import { lstat, readdir } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+
4
+ // Bytes under a directory, for the first progress line's estimate. Missing or vanishing files count as zero.
5
+ export async function stateBytes(directory) {
6
+ let total = 0;
7
+ for (const entry of await readdir(directory, { withFileTypes: true }).catch(() => [])) {
8
+ const path = join(directory, entry.name);
9
+ total += entry.isDirectory() ? await stateBytes(path) : (await lstat(path).catch(() => ({ size: 0 }))).size;
10
+ }
11
+ return total;
12
+ }
13
+
14
+ // The class and code of an unexpected failure, when each is a plain identifier. Its message can hold row data.
15
+ export function nativeFailure(error) {
16
+ const name = typeof error?.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(error.name) ? error.name : 'unknown';
17
+ const code = typeof error?.code === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(error.code) ? error.code : Number.isSafeInteger(error?.code) ? String(error.code) : undefined;
18
+ return { class: name, ...(code ? { code } : {}) };
19
+ }
@@ -6,9 +6,11 @@ import { legacyTargetState, requireLegacyEmpty } from '../modules/portability/le
6
6
  import { migrationReplay } from '../modules/portability/migration-history.mjs';
7
7
  import { appliedMigrations as appliedMigrationNames } from '../modules/portability/bridge.mjs';
8
8
  import { join } from 'node:path';
9
- import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
9
+ import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
10
10
  import { createRequire } from 'node:module';
11
11
  import { createPortability, PortabilityError, readArchive } from '../modules/portability/index.mjs';
12
+ import { transferProgress } from '../modules/portability/progress.mjs';
13
+ import { nativeFailure, stateBytes } from './portability-progress.mjs';
12
14
  import { parseJsonc } from './jsonc.mjs';
13
15
  import { loadPolicy } from './portability-policy.mjs';
14
16
 
@@ -33,8 +35,12 @@ async function dataConfiguration(root) {
33
35
  }
34
36
 
35
37
  async function run(direction, { root, path, journal, binding, buckets, policy, policyModule, transient, legacyRecovery }) {
38
+ // Lines for the person go to the CLI over IPC, which prints them as the command's progress.
39
+ const progress = transferProgress(line => process.send({ progress: line }));
40
+ if (!legacyRecovery && ['export', 'import'].includes(direction)) progress({ phase: 'start', direction, bytes: direction === 'export' ? await stateBytes(join(root, '.wrangler/state')) : (await stat(path)).size });
36
41
  // Validate all inputs before opening bindings or acquiring a restore lock.
37
42
  if (direction === 'import') {
43
+ progress({ phase: 'checking', direction });
38
44
  for await (const record of fileRecords(path,legacyRecovery?readLegacyArchive:readArchive)) {}
39
45
  if(journal)for await(const record of fileRecords(journal)) {}
40
46
  }
@@ -56,7 +62,9 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
56
62
  await ensurePortabilitySchema(proxy.env[binding], await portabilityPlan(unstable_splitSqlQuery));
57
63
  if(legacyRecovery)return await recoverLegacyArchive(options,path,legacyRecovery.output,transient,legacyRecovery.acknowledged);
58
64
  const service = await createPortability(options);
59
- return await (direction === 'export' ? exportArchive(service,path,transient) : importArchive(service,path,transient,journal));
65
+ // This checkout's operation lock means no other import can be running: staging left behind is a crashed restore's.
66
+ if (direction === 'import' && await service.imports.abortAbandoned()) process.send({ progress: 'Cleaned up an unfinished earlier import: a restore that stopped part-way left its staged data here. Its staging is removed and this import starts from the beginning.' });
67
+ return await (direction === 'export' ? exportArchive(service,path,transient,progress) : importArchive(service,path,transient,journal,progress));
60
68
  } finally { await proxy?.dispose(); await rm(configPath, { force: true }); }
61
69
  }
62
70
 
@@ -73,6 +81,8 @@ catch (error) {
73
81
  const migration = error instanceof PortabilityError && error.code.startsWith('schema_mismatch:migration_data:') ? error.code.slice('schema_mismatch:migration_data:'.length) : undefined;
74
82
  // A populated destination names its table or bucket: schema names, never row data.
75
83
  const occupied = error instanceof PortabilityError && /^(target|bucket)_not_empty:/.test(error.code) ? error.code.slice(error.code.indexOf(':') + 1) : undefined;
84
+ // Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
85
+ const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
76
86
  process.exitCode = exitCode;
77
- await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), exitCode });
87
+ await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), ...(native ? { native } : {}), exitCode });
78
88
  }
@@ -73,17 +73,18 @@ async function nativeWorker(store,direction,input,signal) {
73
73
  const child = store.spawn(`${direction === 'policy-preflight' ? 'Checking archive policy' : direction === 'legacy-preflight' ? 'Checking empty recovery destination' : direction === 'export' ? 'Exporting' : 'Restoring'} local installation data`, process.execPath,
74
74
  ['--import', notice, worker, direction, JSON.stringify(input)], { cwd: input.root, signal, env: { ...process.env, WRANGLER_SEND_METRICS: 'false' }, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
75
75
  let receipt;
76
- child.once('message', message => { receipt = message; });
76
+ // Progress lines come first, as {progress}; the last message is the receipt.
77
+ child.on('message', message => { if (typeof message?.progress === 'string') store.report(message.progress); else receipt = message; });
77
78
  const closed = new Promise(resolve => child.once('close', (code, signal) => resolve({ code, signal })));
78
79
  await new Promise(resolve => { child.once('exit', resolve); child.once('error', resolve); });
79
80
  await store.stop(child);
80
81
  const status = await closed;
81
- if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed', 'Local portability did not return a complete receipt.', 1);
82
+ if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed', `Local portability ended before it finished (${status.signal ? `signal ${status.signal}` : `exit code ${status.code}`}). Run the same command again; it starts over${direction === 'import' ? ' and first cleans up anything this attempt staged' : ''}.`, 1);
82
83
  if (status.code !== 0 || !receipt.ok) throw new RerError(receipt.code ?? 'transfer_failed', receipt.reason==='legacy_restored_export_failed'
83
84
  ? 'Legacy data was restored and audited, but its v2 export failed. Preserve this installation, resolve the output problem, then run rer export --output FILE. Do not reset it to retry recovery.'
84
85
  : receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
85
86
  : receipt.occupied ? `Restore refused (${receipt.reason}): ${receipt.reason === 'bucket_not_empty' ? `storage bucket ${receipt.occupied} in this installation already holds files` : `table ${receipt.occupied} in this installation already holds rows its migrations did not insert`}, and a restore only goes into an empty installation. To replace this installation's data with the archive, keep a copy with rer export --output exports/before-restore.rer, then run rer reset --yes and rer import --input ${relative(input.root, input.path)} --yes again.`
86
- : `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}).`, receipt.exitCode ?? 1);
87
+ : `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
87
88
  delete receipt.ok;
88
89
  return receipt;
89
90
  }
@@ -1,5 +1,16 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.48
4
+
5
+ - **The job queue no longer aborts or strands a waiting command under load.** Before, when another queued job's owner did not answer its 3 s identity probe, the waiting build, verify or deploy stopped with "Command failed; inspect the project operating documentation." A probe timeout now counts as "the owner may still be alive", so the waiting job samples again instead of giving up. The same holds for the queue lock and for nested-command membership, which had left a job queued behind its own parent until the queue's maximum wait. A recorded failed install is retried only when `pnpm-lock.yaml` or `package.json` changed since it failed. An error that is not one of rer's own now prints its message and a hint instead of the generic line (#219).
6
+ - **The demo start page warms the chunks the seed actually built.** The warm list now comes from the build output: the client Vite manifest when the build wrote one, else the build's `assetsDir` (recorded in `demo.json` from the loaded Vite config), else `assets`. Seeds that build to a non-default directory, such as Lamp's `/app-assets/`, had an empty warm list; Lamp's is now 32 chunks with the console's routes first (#223).
7
+ - **`rer export` and `rer import` show progress, and a crashed import cleans up after itself.** Export prints the expected time, then parts built, parts written and bytes with the time left (one line every 3 s, plus each phase edge). Import prints parts staged. An import that died part-way leaves staging that the next import removes, saying "Cleaned up an unfinished earlier import". An unexpected failure prints its error class or code and "Run the same command again; it starts over." (#224).
8
+
9
+ ## 0.2.47
10
+
11
+ - **The publication scan resolves GitHub repositories in bulk.** Before, the scan looked up each GitHub repository a text names with its own REST call. Now it collects every reference first and resolves them together, in one aliased GraphQL query per 100 repositories, which also draws on GitHub's separate GraphQL budget. A repository counts as public or private only on a complete answer that is not disabled or locked. It counts as missing only when GitHub returns NOT_FOUND for exactly that name. Any other answer for a repository, and any batch that is not a complete HTTP 200 GraphQL document, falls back to the earlier strict REST lookup for that repository. Nothing is cached on disk. A run-oncall scan now makes 1 gh call instead of 19, and its report is byte-identical (#216).
12
+ - **Plainer terminal lines for someone new.** Queue lines now name the seed by its `seed.json` name, else its folder name, and never by its project or package id. `rer run` and the build step run package scripts with `pnpm run --silent`, so pnpm's `> package@version script` banner is gone and the script's own output and errors still show. Step 1 of the Run page now says the installer does not edit your shell files. It also says that the line it may end with puts rer on your PATH now and appends the same setting to your shell's startup file for new terminals (#215).
13
+
3
14
  ## 0.2.46
4
15
 
5
16
  - **`rer companion` pairs your computer with a Run Edit Run app and runs its change requests here.** The local companion is now a `rer` verb: `map` approves one Git checkout whose `origin` is exactly the app's GitHub repository, `pair` redeems the one-use code from the app's Devices page, `codex-auth` or `claude-auth` sets up a dedicated executor home, `commands` approves setup and verify argv lists, and `run` takes jobs from the app. Each job starts from the exact commit the app names in a separate detached checkout, and the provider runs inside a macOS `sandbox-exec` boundary with no GitHub, Cloudflare or MCP credentials; on other systems `run` refuses before it asks for work. It makes outbound HTTPS requests only and opens no local server. State lives in `~/.rer-local` (`RER_LOCAL_HOME` picks another folder), the device key in a 0600 file. `status` prints the paired device, mapped projects, active job and pending events, never the key. Revoke a device in the app under Settings › Devices (#202).
@@ -20,17 +20,24 @@ export async function describeCheckout(root, entry, env, signal) {
20
20
  }
21
21
  }
22
22
 
23
- // Install when the last install did not finish or the lockfile changed since.
24
- // pnpm writes node_modules/.modules.yaml only once dependencies are linked and
25
- // built, so a stopped install leaves it absent. An install this launcher ran
26
- // is finished only when its record says so: that also covers the project's
27
- // own install hooks, which pnpm runs after writing the manifest.
23
+ // Install when the last install was killed, nothing is linked, or the lockfile
24
+ // changed since. pnpm writes node_modules/.modules.yaml only once dependencies
25
+ // are linked and built, so a stopped install leaves it absent. An install this
26
+ // launcher ran is finished only when its record says so: that also covers the
27
+ // project's own install hooks, which pnpm runs after writing the manifest. A
28
+ // recorded failed install is retried only when the lockfile or package.json is
29
+ // newer than both that attempt and what is installed; dependencies repaired
30
+ // since are current, and the next launch uses them.
28
31
  export async function needsInstall(root, last) {
29
- if (last && last.state !== 'done') return true;
30
- const manifest = await mtime(join(root, 'node_modules/.modules.yaml'));
31
- if (manifest === undefined) return true;
32
- const lock = await mtime(join(root, 'pnpm-lock.yaml'));
33
- return lock !== undefined && Math.max(manifest, last?.at ?? 0) < lock;
32
+ if (last?.state === 'running') return true;
33
+ const linked = await mtime(join(root, 'node_modules/.modules.yaml'));
34
+ if (linked === undefined) return true;
35
+ const since = Math.max(linked, last?.at ?? 0);
36
+ for (const file of last?.state === 'failed' ? ['pnpm-lock.yaml', 'package.json'] : ['pnpm-lock.yaml']) {
37
+ const changed = await mtime(join(root, file));
38
+ if (changed !== undefined && changed > since) return true;
39
+ }
40
+ return false;
34
41
  }
35
42
 
36
43
  // A checkout is set up when its pin's describe says so (`local.prepared`, from
package/global/run.mjs CHANGED
@@ -1,6 +1,7 @@
1
1
  // rer run SCRIPT [ARGS]: `pnpm run SCRIPT ARGS` in a checkout, with the pnpm
2
2
  // and Node it pins. rer keeps both in its own cache, off PATH, so a project's
3
3
  // documented `pnpm run ...` works as `rer run ...` right after rer install.
4
+ // --silent drops pnpm's "> package@version script" banner; the script's own output and errors stay.
4
5
  // A script that runs Playwright gets the checkout's browser first, on first need.
5
6
  import { spawn } from 'node:child_process';
6
7
  import { readFile } from 'node:fs/promises';
@@ -52,7 +53,7 @@ export async function runScript(args) {
52
53
  const code = await (await import('./browser-runtime.mjs')).browserPreflight(target.root, ['run']);
53
54
  if (code !== undefined) return code;
54
55
  }
55
- const child = spawn('pnpm', ['run', ...args], { cwd: target.root, env: process.env, stdio: 'inherit' });
56
+ const child = spawn('pnpm', ['run', '--silent', ...args], { cwd: target.root, env: process.env, stdio: 'inherit' });
56
57
  const forward = new Map(['SIGINT', 'SIGTERM', 'SIGHUP'].map(signal => [signal, () => child.kill(signal)]));
57
58
  for (const [signal, handler] of forward) process.on(signal, handler);
58
59
  try {
@@ -13,8 +13,9 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
13
13
  - `worker.mjs`, `pages.mjs`, `start-script.mjs`: Turnstile start, progress,
14
14
  cancellation and visitor routing. Static assets are served at the edge.
15
15
  - `native/warm.mjs`: the start page's prefetch list, built from `dist/client`
16
- and TanStack Start's route manifest, the console's views first, capped by
17
- compressed size.
16
+ (the Vite manifest if the build wrote one, else the build's `assetsDir`, else
17
+ `assets`) and TanStack Start's route manifest, the console's views first,
18
+ capped by compressed size.
18
19
  - `capacity.mjs`: concurrent visitor admission and operator diagnostics. This
19
20
  is used by the native host as well as retained during legacy upgrades.
20
21
  - `upgrade.mjs`: verifies ownership, removes the old Container application and
@@ -40,9 +40,9 @@ export async function buildDemo(root, output, declaration) {
40
40
  await rm(output, { recursive: true, force: true });
41
41
  await mkdir(join(output, 'app'), { recursive: true });
42
42
  await cp(join(root, 'dist'), join(output, 'app/dist'), { recursive: true });
43
- // Where the console is mounted orders what the start page prefetches (native/warm.mjs).
43
+ // Where the console is mounted orders what the start page prefetches, and the build's assetsDir says where its chunks are (native/warm.mjs).
44
44
  const { operatorPath } = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8').catch(() => '{}')).deploy ?? {};
45
- await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath }) + '\n');
45
+ await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath, assetsDir: loaded.config.build?.assetsDir }) + '\n');
46
46
  }
47
47
 
48
48
  if (process.argv[1] === import.meta.filename) {
@@ -2,7 +2,8 @@ import { parseCron } from './cron.mjs';
2
2
 
3
3
  export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'r2-store', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
4
4
  // Hashed build output: served immutable, and what the start page may prefetch.
5
- export const HASHED_ASSET = /^\/assets\/.*-[\w-]{8,}\.[\w]+$/;
5
+ export const hashedAsset = (assetsDir = 'assets') => new RegExp(`^/${assetsDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/.*-[\\w-]{8,}\\.[\\w]+$`);
6
+ export const HASHED_ASSET = hashedAsset();
6
7
  export const ISOLATE_FILES = ['bindings', 'namespace', 'r2-client', 'd1', 'sql', 'seed-object'];
7
8
  export function nativeBindings(config) {
8
9
  for (const kind of ['kv_namespaces', 'services', 'workflows', 'ai', 'vectorize', 'hyperdrive', 'analytics_engine_datasets', 'browser', 'images', 'send_email', 'dispatch_namespaces', 'mtls_certificates', 'pipelines', 'secrets_store_secrets', 'unsafe', 'vpc_services', 'vpc_networks', 'ratelimits', 'worker_loaders', 'containers', 'ai_search_namespaces', 'ai_search', 'agent_memory', 'connect', 'artifacts', 'unsafe_hello_world', 'flagship', 'logfwdr', 'version_metadata']) {
@@ -34,7 +34,7 @@ export async function prepareNative(folder) {
34
34
  const bundle = { compatibilityDate: config.compatibility_date, compatibilityFlags: [...new Set([...(config.compatibility_flags ?? []), 'nodejs_compat'])], modules,
35
35
  bindings, migrations: manifest.migrations, vars: config.vars ?? {}, crons: config.triggers?.crons ?? [],
36
36
  assetPaths: assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path),
37
- warm: (await warmList(assets, root, manifest.operatorPath)).paths };
37
+ warm: (await warmList(assets, root, manifest.operatorPath, manifest.assetsDir)).paths };
38
38
  bundle.digest = createHash('sha256').update(JSON.stringify(bundle)).digest('hex');
39
39
  await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
40
40
  await writeFile(join(folder, 'modules/demo-host/native/bundle.mjs'), `export default ${JSON.stringify(bundle)};\n`);
@@ -2,7 +2,7 @@ import { readdir } from 'node:fs/promises';
2
2
  import { join } from 'node:path';
3
3
  import { pathToFileURL } from 'node:url';
4
4
  import { gzipSync } from 'node:zlib';
5
- import { HASHED_ASSET } from './config.mjs';
5
+ import { hashedAsset } from './config.mjs';
6
6
 
7
7
  // The start page prefetches the console's hashed JS and CSS while the visitor
8
8
  // waits for the human check and sign-in, so each view's first visit finds its
@@ -16,10 +16,22 @@ const segments = path => path.split('/').filter(Boolean);
16
16
  const dynamic = path => segments(path).filter(part => part.startsWith('$')).length;
17
17
  const within = (path, root) => path === root || path.startsWith(root.endsWith('/') ? root : root + '/');
18
18
 
19
+ // The hashed JS and CSS the client build produced: the files its Vite manifest
20
+ // names (build.manifest), else everything under the configured build.assetsDir
21
+ // (Vite's default is 'assets'). Never source maps or unhashed files.
22
+ const MANIFESTS = ['.vite/manifest.json', 'manifest.json'];
23
+ function builtChunks(files, assetsDir) {
24
+ const manifest = files.find(file => MANIFESTS.includes(file.path));
25
+ const paths = (manifest
26
+ ? Object.values(JSON.parse(Buffer.from(manifest.base64, 'base64'))).flatMap(entry => [entry.file, ...(entry.css ?? [])]).map(path => '/' + path)
27
+ : files.map(file => '/' + file.path).filter(path => hashedAsset(assetsDir).test(path)));
28
+ return new Set(paths.filter(path => /-[\w-]{8,}\.(js|css)$/.test(path)));
29
+ }
30
+
19
31
  // TanStack Start's route manifest from the server build: each route's CSS and
20
32
  // the chunks it preloads.
21
- async function startRoutes(server) {
22
- const directory = join(server, 'assets');
33
+ async function startRoutes(server, assetsDir) {
34
+ const directory = join(server, assetsDir);
23
35
  const file = (await readdir(directory).catch(() => [])).find(name => /^_tanstack-start-manifest_v-[\w-]+\.js$/.test(name));
24
36
  if (!file) return {};
25
37
  const { tsrStartManifest } = await import(pathToFileURL(join(directory, file)));
@@ -36,10 +48,10 @@ function routeOrder(routes, consolePath) {
36
48
  }
37
49
 
38
50
  /** The ordered, size-capped paths the start page prefetches, from dist/client's files and the server build. */
39
- export async function warmList(files, server, consolePath = '/app') {
40
- const sizes = new Map(files.map(file => ['/' + file.path, file]).filter(([path]) => HASHED_ASSET.test(path) && /\.(js|css)$/.test(path))
41
- .map(([path, file]) => [path, gzipSync(Buffer.from(file.base64, 'base64')).length]));
42
- const routes = await startRoutes(server);
51
+ export async function warmList(files, server, consolePath = '/app', assetsDir = 'assets') {
52
+ const chunks = builtChunks(files, assetsDir);
53
+ const sizes = new Map(files.filter(file => chunks.has('/' + file.path)).map(file => ['/' + file.path, gzipSync(Buffer.from(file.base64, 'base64')).length]));
54
+ const routes = await startRoutes(server, assetsDir);
43
55
  const ordered = new Set();
44
56
  for (const id of routeOrder(routes, consolePath)) for (const path of [...(routes[id].css ?? []), ...(routes[id].preloads ?? [])]) if (sizes.has(path)) ordered.add(path);
45
57
  for (const path of [...sizes.keys()].sort((a, b) => sizes.get(a) - sizes.get(b))) ordered.add(path);
@@ -31,14 +31,15 @@ export async function alive(owner) {
31
31
  return new Promise((resolve, reject) => {
32
32
  const connection = createConnection(owner.socket);
33
33
  let body = '';
34
- connection.setTimeout(3000, () => connection.destroy(Error('Owner identity probe timed out; refusing admission')));
34
+ connection.setTimeout(3000, () => connection.destroy(Object.assign(Error('Owner identity probe timed out'), { code: 'PROBE_TIMEOUT' })));
35
35
  connection.on('data', data => { body += data; if (body.length > 128) connection.destroy(Error('Invalid owner identity')); });
36
36
  connection.on('end', () => resolve(body === owner.token));
37
37
  connection.on('error', async error => {
38
38
  if (['ENOENT', 'ECONNREFUSED'].includes(error.code)) resolve(false);
39
- else if (error.code === 'ECONNRESET') {
40
- // Listener shutdown can reset a probe after the initial PID check.
41
- // Reclaim proven death; otherwise retain the owner until the next sample.
39
+ else if (['ECONNRESET', 'PROBE_TIMEOUT'].includes(error.code)) {
40
+ // Listener shutdown can reset a probe after the initial PID check, and a
41
+ // busy owner can answer late. Neither proves death: reclaim only a process
42
+ // that is gone; otherwise retain the owner until the next sample.
42
43
  try {
43
44
  const after = await processInfo(owner.pid);
44
45
  resolve(Boolean(after && !after.zombie && after.start === owner.start));
@@ -121,7 +121,7 @@ idempotence and recovery; D1 cannot roll back Durable Object or external state.
121
121
  The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
122
122
  closing record. `readArchive(stream)` uses the shared bounded, fatal UTF-8 line
123
123
  reader. The default maximum line size is 24 MiB; callers can set a smaller bound.
124
- No runtime or CLI materializes the whole archive. Readers enforce a 16 MiB
124
+ No runtime or CLI materializes the whole archive. The manifest opens the archive and holds the final part count and payload hashes, so an export builds all its parts in the job before any byte is written; `exportArchive` and `importArchive` take an optional `progress` callback (`progress.mjs` turns its events into throttled lines with a time-left estimate). Readers enforce a 16 MiB
125
125
  canonical UTF-8 part limit; `readArchive(stream,{partBytes})` can lower it.
126
126
  Import uploads enforce the policy's part limit before any staging write.
127
127
  A part has at most 1,000 rows;
@@ -96,7 +96,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
96
96
  if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
97
97
  const before=row.completed,items=await nextItems(context,row);
98
98
  if(items) {
99
- const part=await makePart(before,items);await checkPart(part,before,row.metadata.partBytes);row.completed++;
99
+ const part=await makePart(before,items),size=await checkPart(part,before,row.metadata.partBytes);row.completed++;row.metadata.bytes=(row.metadata.bytes??0)+size+1;
100
100
  try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
101
101
  catch(e){if((await job(context.db,id)).completed===before)throw e;}
102
102
  if(expectedPart!==undefined)expectedPart++;
@@ -114,7 +114,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
114
114
  return exportStatus(context,id);
115
115
  }
116
116
  export async function exportStatus(context,id) {
117
- const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,contentHash:row.metadata.contentHash??null};
117
+ const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,bytes:row.metadata.bytes??0,contentHash:row.metadata.contentHash??null};
118
118
  }
119
119
  export async function downloadExport(context,id) {
120
120
  const row=await job(context.db,id,'export');insist(row.phase==='complete','export_incomplete',409);
@@ -4,6 +4,7 @@ import { dirname } from 'node:path';
4
4
  import { Readable } from 'node:stream';
5
5
  import { randomUUID } from 'node:crypto';
6
6
  import { readArchive } from './index.mjs';
7
+ import { canonicalJson } from './hash.mjs';
7
8
  export async function* fileRecords(path,reader=readArchive) {
8
9
  const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
9
10
  try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
@@ -19,23 +20,35 @@ function receipt(manifest, closing, transient) {
19
20
  requiredConfiguration: manifest.requiredConfiguration,
20
21
  };
21
22
  }
22
- export async function exportArchive(service, path, transient) {
23
+ // `progress` receives {phase, parts, total?, bytes?} events; the CLI turns them into lines (progress.mjs).
24
+ export async function exportArchive(service, path, transient, progress) {
23
25
  let id, temporary, file;
24
26
  try {
25
27
  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
26
28
  temporary = path + '.' + randomUUID() + '.partial';
27
29
  file = await open(temporary, 'wx', 0o600);
28
30
  id = await service.exports.create();
29
- while ((await service.exports.advance(id, { maxParts: 16 })).phase === 'building') {}
31
+ // The manifest leads the archive and holds the final part count and payload hashes, so
32
+ // nothing can be written until every part is built (export.mjs advanceExport). One part per
33
+ // advance keeps the progress line moving; a part is at most 16 MiB.
34
+ let status;
35
+ do {
36
+ status = await service.exports.advance(id, { maxParts: 1 });
37
+ progress?.({ phase: 'building', parts: status.completed, bytes: status.bytes, ...(status.phase === 'complete' ? { total: status.total } : {}) });
38
+ } while (status.phase === 'building');
30
39
  const reader = (await service.exports.download(id)).getReader();
31
40
  try {
32
- for (;;) {
41
+ // One chunk per record: the manifest, then each part, then the closing record.
42
+ for (let record = 0, written = 0;; record++) {
33
43
  const { done, value } = await reader.read(); if (done) break;
34
44
  let offset = 0;
35
45
  while (offset < value.length) offset += (await file.write(value, offset)).bytesWritten;
46
+ written += value.length;
47
+ if (record > 0 && record <= status.total) progress?.({ phase: 'writing', parts: record, total: status.total, bytes: written });
36
48
  }
37
49
  } finally { await reader.cancel(); reader.releaseLock(); }
38
50
  await file.sync(); await file.close(); file = undefined;
51
+ progress?.({ phase: 'checking', direction: 'export' });
39
52
  let manifest, closing;
40
53
  for await (const record of fileRecords(temporary)) { if (record.kind === 'manifest') manifest = record; if (record.kind === 'closing') closing = record; }
41
54
  await link(temporary, path);
@@ -46,19 +59,25 @@ export async function exportArchive(service, path, transient) {
46
59
  if (id) await service.exports.abort(id);
47
60
  }
48
61
  }
49
- export async function importArchive(service, path, transient, journal) {
50
- let id, manifest, closing, complete = false;
62
+ export async function importArchive(service, path, transient, journal, progress) {
63
+ let id, manifest, closing, complete = false, staged = 0;
51
64
  try {
52
65
  // Validate both complete archives before creating any restore state.
53
66
  let later;
67
+ progress?.({ phase: 'checking', direction: 'import' });
54
68
  for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
55
69
  if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
56
70
  }
57
71
  for await (const record of fileRecords(path)) {
58
72
  if (record.kind === 'manifest') { manifest = record; id = await service.imports.create(manifest); }
59
- else if (record.kind === 'part') await service.imports.uploadPart(id, record.index, record);
73
+ else if (record.kind === 'part') {
74
+ await service.imports.uploadPart(id, record.index, record);
75
+ staged += Buffer.byteLength(canonicalJson(record)) + 1;
76
+ progress?.({ phase: 'staging', parts: record.index + 1, total: manifest.partCount, bytes: staged });
77
+ }
60
78
  else closing = record;
61
79
  }
80
+ progress?.({ phase: 'finishing', total: manifest.partCount });
62
81
  const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
63
82
  return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}) };
64
83
  } finally { if (id && !complete) await service.imports.abort(id); }
@@ -11,9 +11,11 @@ export async function makePart(index,items) {
11
11
  export async function checkPart(part,index,partBytes=MAX_PART_BYTES) {
12
12
  insist(part?.kind==='part' && part.index===index && Array.isArray(part.items) && part.items.length>0 && Object.keys(part).sort().join(',')==='index,items,kind,sha256','part_order');
13
13
  insist(Number.isSafeInteger(partBytes) && partBytes>0 && partBytes<=MAX_PART_BYTES,'invalid_part_limit');
14
- insist(utf8(canonicalJson(part)).length<=partBytes,'part_byte_limit',413);
14
+ const size=utf8(canonicalJson(part)).length;
15
+ insist(size<=partBytes,'part_byte_limit',413);
15
16
  insist(part.sha256===await sha256(canonicalJson({kind:part.kind,index:part.index,items:part.items})),'part_checksum');
16
17
  insist(part.items.filter(x=>x.kind==='row').length<=1000,'part_row_limit');
18
+ return size;
17
19
  }
18
20
  export async function checkManifest(m) {
19
21
  insist(m?.kind==='manifest' && m.format===FORMAT,'archive_format');
@@ -197,3 +197,11 @@ export async function abortImport(context,id) {
197
197
  await context.db.batch([save(context.db,row),statement(context.db,'UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1 AND import_job=?',id)]);
198
198
  return importStatus(context,id);
199
199
  }
200
+ // A restore's staging outlives a worker that died mid-restore (a crashed process or workerd) and refuses
201
+ // every later import as running. Only a caller that owns the destination, as the local CLI does under the
202
+ // checkout's operation lock, may call this: a hosted service can have a live import.
203
+ export async function abortAbandonedImport(context) {
204
+ const id=(await state(context.db)).import_job;
205
+ if(!id)return null;
206
+ await abortImport(context,id);return id;
207
+ }
@@ -28,7 +28,7 @@ export interface ArchivePolicy {
28
28
  extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
29
29
  partBytes?: number;
30
30
  }
31
- export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
31
+ export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
32
32
  /** Later additive migrations an older archive was restored across. */
33
33
  export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
34
34
  /** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
@@ -36,7 +36,7 @@ export interface MigrationReplay {names:string[];statements(name:string):string[
36
36
  export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
37
37
  export interface Portability {
38
38
  exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
39
- imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>};
39
+ imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
40
40
  status(id:string):Promise<Status>;actor(id:string):Promise<string>;
41
41
  erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
42
42
  redactionStatements():Promise<D1PreparedStatement[]>;
@@ -2,7 +2,7 @@ import { policy, catalogue } from './catalogue.mjs';
2
2
  import { wireDatabase } from '../d1-transport/index.mjs';
3
3
  import { insist, job } from './storage.mjs';
4
4
  import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
5
- import { createImport, uploadPart, finalizeImport, importStatus, abortImport } from './import.mjs';
5
+ import { createImport, uploadPart, finalizeImport, importStatus, abortImport, abortAbandonedImport } from './import.mjs';
6
6
  import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
7
7
  export { archiveLines } from './lines.mjs';
8
8
  export { excludedTables, excludedTable } from './catalogue.mjs';
@@ -17,7 +17,7 @@ export async function createPortability({db,buckets={},policy:input,migrations})
17
17
  const context={db,buckets,config,migrations};
18
18
  return {
19
19
  exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
20
- imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id)},
20
+ imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
21
21
  status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
22
22
  actor:async id=>(await job(db,id)).actor,
23
23
  erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
@@ -0,0 +1,54 @@
1
+ // Human progress for a long export or import. The caller sends events; this turns
2
+ // them into at most one line every few seconds, with an estimate of the time left
3
+ // once a phase has two samples. The first and last line of every phase always show.
4
+ // Rates are bytes per second measured on a 1.2 GB local install on a loaded machine
5
+ // (export 1725 s, import 2234 s); a quiet machine is faster.
6
+ const RATES = { export: 700_000, import: 550_000 };
7
+ const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'];
8
+ export function formatBytes(bytes) {
9
+ let value = bytes, unit = 0;
10
+ while (value >= 1000 && unit < UNITS.length - 1) { value /= 1000; unit++; }
11
+ return `${unit && value < 100 ? value.toFixed(1) : Math.round(value)} ${UNITS[unit]}`;
12
+ }
13
+ export function formatDuration(seconds) {
14
+ if (seconds < 45) return 'under a minute';
15
+ const minutes = Math.max(1, Math.round(seconds / 60));
16
+ return minutes < 90 ? `about ${minutes} min` : `about ${Math.floor(minutes / 60)} h ${minutes % 60} min`;
17
+ }
18
+ const left = seconds => `, ${formatDuration(seconds)} left`;
19
+ export function transferProgress(report, { interval = 3000, now = () => performance.now() } = {}) {
20
+ let phase, first, printed = -Infinity, expected;
21
+ const line = event => {
22
+ const bytes = event.bytes === undefined ? '' : ` (${formatBytes(event.bytes)}${event.phase === 'building' && expected ? ` of about ${formatBytes(expected)}` : ''})`;
23
+ const parts = event.total === undefined ? `${event.parts} part${event.parts === 1 ? '' : 's'} so far` : `${event.parts} of ${event.total} parts`;
24
+ // Seconds left, from the rate since the phase's first sample.
25
+ const eta = () => {
26
+ const seconds = (now() - first.time) / 1000;
27
+ if (event === first.event || seconds <= 0 || event.parts === event.total) return '';
28
+ if (event.total !== undefined && event.parts > first.event.parts) return left((event.total - event.parts) * seconds / (event.parts - first.event.parts));
29
+ if (event.phase === 'building' && expected && event.bytes > first.event.bytes && event.bytes < expected) return left((expected - event.bytes) * seconds / (event.bytes - first.event.bytes));
30
+ return '';
31
+ };
32
+ switch (event.phase) {
33
+ case 'building': return `Building the archive: ${parts}${bytes}${eta()}.`;
34
+ case 'writing': return `Writing the archive: ${parts}${bytes}${eta()}.`;
35
+ case 'staging': return `Staging the restore: ${parts}${bytes}${eta()}.`;
36
+ case 'checking': return event.direction === 'export' ? 'Checking the written archive.' : 'Checking the whole archive before anything is restored.';
37
+ case 'finishing': return `All ${event.total} parts are staged; verifying them and publishing the restore. This can take several minutes.`;
38
+ }
39
+ };
40
+ return event => {
41
+ if (event.phase === 'start') {
42
+ const noun = event.direction === 'export' ? `Exporting about ${formatBytes(event.bytes)} of local data` : `Restoring a ${formatBytes(event.bytes)} archive`;
43
+ if (event.direction === 'export') expected = event.bytes;
44
+ report(`${noun}. ${event.bytes / RATES[event.direction] < 45 ? 'This takes under a minute.' : `An archive this size usually takes ${formatDuration(event.bytes / RATES[event.direction])}; a busy machine takes longer. Progress follows every few seconds.`}`);
45
+ printed = now();
46
+ return;
47
+ }
48
+ const time = now(), edge = event.phase !== phase;
49
+ if (edge) { phase = event.phase; first = { time, event }; }
50
+ // A phase's last event (all parts built, written or staged) prints whatever the clock says.
51
+ const last = event.total !== undefined && event.parts === event.total;
52
+ if (edge || last || time - printed >= interval) { report(line(event)); printed = time; }
53
+ };
54
+ }
@@ -19,7 +19,7 @@ export function operatorInstaller({ database, features, prepare, prepareData })
19
19
  keepExisting: true, originKey: 'AUTH_ORIGIN',
20
20
  values: ({ origin }) => ({ AUTH_LOCAL: 'true', AUTH_ORIGIN: origin, MAIL_PROVIDER: 'development', MAIL_FROM: 'auth@localhost.test' }),
21
21
  },
22
- build: ['pnpm', ['run', 'build:app']],
22
+ build: ['pnpm', ['run', '--silent', 'build:app']],
23
23
  migrate: context => migrateOperator(context, database),
24
24
  // Setup uses the local entry for owner-side dependency approval without mail.
25
25
  start: developmentTarget,
@@ -6,16 +6,16 @@ const portable = basename(dirname(fileURLToPath(import.meta.url))) === 'base-onb
6
6
  const queueURL = new URL(portable ? '../base-cli/modules/job-queue/' : '../modules/job-queue/', import.meta.url);
7
7
  export const heavyVerbs = new Set(['install', 'setup', 'build', 'verify', 'dev', 'preview', 'deploy', 'release', 'export', 'import', 'installer']);
8
8
  export const verificationBoundary = verb => verb === 'release' || verb === 'verify' || verb?.startsWith('verify:');
9
+ // The seed's display name (seed.json `name`, else its folder name) for terminal lines,
10
+ // never its package or project id; the hash tells two checkouts of one seed apart.
9
11
  export async function queueLabel(root, verb, phase) {
10
12
  root = await realpath(root);
11
- let project='project';
12
- for(const file of ['OPERATIONS.json','package.json']){
13
- try{const value=JSON.parse(await readFile(join(root,file),'utf8')),name=value.project??value.name;
14
- if(typeof name==='string'&&/^@?[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)?$/.test(name)){project=name;break;}
15
- }catch{}
16
- }
17
-
18
- return `${project} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
13
+ let name = basename(root);
14
+ try {
15
+ const value = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8')).name;
16
+ if (typeof value === 'string' && value.trim() && !/[\p{Cc}]/u.test(value) && value.length <= 80) name = value.trim();
17
+ } catch {}
18
+ return `${name} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
19
19
  }
20
20
 
21
21
  function shellQuote(value) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.46",
3
+ "version": "0.2.48",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -11,7 +11,11 @@ entropy or English stopword filters. Both assignment rules go through the same
11
11
  location-bound Source/module identity classification; provider and environment findings
12
12
  remain independent.
13
13
  `tree_scan.py` applies the kept text and licence policy. `history_scan.py`
14
- assigns release, working-tree and historical scopes. The Node entry point bounds
14
+ assigns release, working-tree and historical scopes. Inspection returns each
15
+ text's repository references instead of resolving them; `history_scan.py`
16
+ resolves the whole scan's references at once through `github_refs.py`
17
+ (GraphQL batches, a REST lookup for any repository a batch leaves in doubt) and
18
+ then adds their rules. The Node entry point bounds
15
19
  the whole operation and stops its private process group on timeout/cancellation.
16
20
 
17
21
  Git message records carry their kind explicitly. Only commit records enable the
@@ -200,14 +200,18 @@ def response_constant(value):
200
200
  raise ValueError('nonstandard API JSON constant')
201
201
 
202
202
 
203
- def visibility(output, exit_code):
203
+ def response(output):
204
+ """Split `gh api --include` output into its HTTP status and strict JSON document."""
204
205
  headers, body = re.split(r'\r?\n\r?\n', output, maxsplit=1)
205
206
  status = re.fullmatch(r'HTTP/(?:1\.[01]|2(?:\.0)?|3(?:\.0)?) (\d{3})(?: [^\r\n]*)?',
206
207
  headers.splitlines()[0])
207
208
  if not status:
208
- return 'incomplete'
209
- code = int(status.group(1))
210
- document = json.loads(body, object_pairs_hook=response_object, parse_constant=response_constant)
209
+ raise ValueError('no HTTP status line')
210
+ return int(status.group(1)), json.loads(body, object_pairs_hook=response_object, parse_constant=response_constant)
211
+
212
+
213
+ def visibility(output, exit_code):
214
+ code, document = response(output)
211
215
  if not isinstance(document, dict):
212
216
  return 'incomplete'
213
217
  if code == 200 and exit_code == 0 and type(document.get('private')) is bool:
@@ -221,22 +225,89 @@ def visibility(output, exit_code):
221
225
  return 'incomplete'
222
226
 
223
227
 
228
+ BATCH = 100
229
+ FIELDS = frozenset({'isPrivate', 'isDisabled', 'isLocked'})
230
+
231
+
232
+ def bulk_query(repositories):
233
+ """One aliased GraphQL query; owners and names travel as variables, never as query text."""
234
+ variables, fields, arguments = [], [], []
235
+ for index, repository in enumerate(repositories):
236
+ owner, name = repository.split('/')
237
+ variables.append(f'$o{index}:String!,$n{index}:String!')
238
+ fields.append(f'r{index}:repository(owner:$o{index},name:$n{index}){{{" ".join(sorted(FIELDS))}}}')
239
+ arguments += ['-f', f'o{index}={owner}', '-f', f'n{index}={name}']
240
+ return ['-f', 'query=query(' + ','.join(variables) + '){' + ' '.join(fields) + '}', *arguments]
241
+
242
+
243
+ def bulk_visibility(repositories, output, exit_code):
244
+ """Return only the verdicts the batch states without doubt.
245
+
246
+ A repository left out takes today's single REST lookup: a disabled or locked
247
+ repository, an error other than GitHub's own not-found for exactly that name
248
+ (no access through SAML or an IP allow list, a rate limit), or any response
249
+ that is not a complete HTTP 200 GraphQL document for exactly this batch.
250
+ """
251
+ code, document = response(output)
252
+ aliases = {f'r{index}': repository for index, repository in enumerate(repositories)}
253
+ if code != 200 or not isinstance(document, dict) or not set(document) <= {'data', 'errors'}:
254
+ return {}
255
+ data, errors = document.get('data'), document.get('errors', [])
256
+ # gh exits 1 exactly when the document carries errors.
257
+ if not isinstance(data, dict) or set(data) != set(aliases) or not isinstance(errors, list) or exit_code != (1 if errors else 0):
258
+ return {}
259
+ reported = {}
260
+ for error in errors:
261
+ path = error.get('path') if isinstance(error, dict) else None
262
+ if not (isinstance(path, list) and len(path) == 1 and path[0] in aliases):
263
+ return {}
264
+ reported.setdefault(path[0], []).append(error)
265
+ states = {}
266
+ for alias, repository in aliases.items():
267
+ node, problems = data[alias], reported.get(alias, [])
268
+ if (not problems and isinstance(node, dict) and set(node) == FIELDS and type(node['isPrivate']) is bool and
269
+ node['isDisabled'] is False and node['isLocked'] is False):
270
+ states[repository] = 'private' if node['isPrivate'] else 'public'
271
+ elif (node is None and len(problems) == 1 and set(problems[0]) <= {'type', 'path', 'locations', 'message'} and
272
+ problems[0].get('type') == 'NOT_FOUND' and
273
+ problems[0].get('message') == f"Could not resolve to a Repository with the name '{repository}'."):
274
+ states[repository] = 'missing'
275
+ return states
276
+
277
+
278
+ def gh(arguments):
279
+ # No disk cache, auth arguments, diagnostic output or response metadata.
280
+ # Disable gh debug logging even if inherited from the calling shell.
281
+ env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
282
+ env['GH_PROMPT_DISABLED'] = '1'
283
+ response = subprocess.run(['gh', 'api', '--hostname', 'github.com', '--include', *arguments],
284
+ env=env, capture_output=True, timeout=60)
285
+ return response.stdout.decode('utf-8'), response.returncode
286
+
287
+
224
288
  class GitHubResolver:
289
+ """Verdicts for one run only: no disk cache can carry a stale or planted answer."""
225
290
  def __init__(self, spool=None):
226
291
  self.cache = {}
227
292
 
293
+ def resolve_all(self, repositories):
294
+ """Resolve in GraphQL batches; a repository its batch leaves in doubt takes the REST lookup."""
295
+ pending = sorted(set(repositories) - self.cache.keys())
296
+ for start in range(0, len(pending), BATCH):
297
+ batch = pending[start:start + BATCH]
298
+ try:
299
+ states = bulk_visibility(batch, *gh(['graphql', *bulk_query(batch)]))
300
+ except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
301
+ states = {}
302
+ for repository in batch:
303
+ self.cache[repository] = states[repository] if repository in states else self.resolve(repository)
304
+ return {repository: self.cache[repository] for repository in repositories}
305
+
228
306
  def resolve(self, repository):
229
307
  if repository in self.cache:
230
308
  return self.cache[repository]
231
- # No disk cache, auth arguments, diagnostic output or response metadata.
232
- # Disable gh debug logging even if inherited from the calling shell.
233
- env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
234
- env['GH_PROMPT_DISABLED'] = '1'
235
309
  try:
236
- response = subprocess.run(['gh', 'api', '--hostname', 'github.com', '--include', 'repos/' + repository],
237
- env=env, capture_output=True, timeout=60)
238
- output = response.stdout.decode('utf-8')
239
- result = visibility(output, response.returncode)
310
+ result = visibility(*gh(['repos/' + repository]))
240
311
  except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
241
312
  result = 'incomplete'
242
313
  self.cache[repository] = result
@@ -11,6 +11,7 @@ from licenses import EXCEPTIONS, exceptions, license_rules, registry
11
11
  from reviewed import REVIEWED, load_reviewed
12
12
  from report import new_report, Findings, finish
13
13
  from tree_scan import context, text_rules, working_files, working_regular
14
+ from public_hygiene import reference_rules
14
15
  from content_identities import ContentIdentities, candidate as identity_candidate
15
16
  from checksum_maps import ChecksumMaps
16
17
 
@@ -36,7 +37,6 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
36
37
  add(INVALID, 'edition.json', history.commit)
37
38
  config['pendingDestination'] = None
38
39
  recorded = registry() + exceptions(history.selected(EXCEPTIONS))
39
- resolver = GitHubResolver()
40
40
  public_blobs = {e['oid'] for c in history.public_commits for e in history.entries(c)}
41
41
  # A file byte-identical to the same path on the public base is already public.
42
42
  public_paths = {e['path']: e['oid'] for e in history.entries(history.public_commit)
@@ -45,6 +45,8 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
45
45
  report['history']['newlyPublicCommits'] = [c for c in history.commits if c not in history.public_commits]
46
46
  seen_trees, seen_blobs, inspected = set(), set(), {}
47
47
  entries_by_commit, dependencies = {}, {}
48
+ # Repository references wait here so that the whole scan resolves them in bulk.
49
+ deferred = []
48
50
  release_license_good = not license_rules(history.selected('LICENSE'), recorded)
49
51
  for commit in history.commits:
50
52
  entries = history.entries(commit)
@@ -74,11 +76,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
74
76
  local = dependencies[pair] & config_tree['localPaths']
75
77
  key = (path, oid, signature, local)
76
78
  if key not in inspected:
77
- inspected[key] = text_rules(history.blob(oid), path, config_tree, resolver, hashes)
78
- rules, provenance = inspected[key]
79
+ inspected[key] = text_rules(history.blob(oid), path, config_tree, hashes)
80
+ rules, provenance, references = inspected[key]
79
81
  entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
80
82
  for rule in rules:
81
83
  add(rule, path, commit, oid, entry_scope)
84
+ if references:
85
+ deferred.append((references, path, commit, oid, entry_scope))
82
86
  if provenance:
83
87
  add('provenance.declared-source', path, commit, oid, entry_scope)
84
88
  report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
@@ -94,11 +98,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
94
98
  # their known immutable identity; new bytes have no history identity.
95
99
  entry = entries_by_commit[history.commit].get(path)
96
100
  oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
97
- rules, _ = text_rules(data, path, selected_context, resolver, hashes)
101
+ rules, _, references = text_rules(data, path, selected_context, hashes)
98
102
  public_oid = public_paths.get(path)
99
103
  scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
100
104
  for rule in rules:
101
105
  add(rule, path, history.commit, oid, scope)
106
+ if references:
107
+ deferred.append((references, path, history.commit, oid, scope))
102
108
  if path == 'LICENSE':
103
109
  for rule in license_rules(data, recorded):
104
110
  add(rule, path, history.commit, oid, 'working-tree')
@@ -159,11 +165,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
159
165
  for message_type, commit, data in history.messages():
160
166
  path = '@' + message_type + '-message'
161
167
  report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
162
- rules, _ = text_rules(data, path, selected_context, resolver, hashes,
163
- commit_message=message_type == 'commit')
168
+ rules, _, references = text_rules(data, path, selected_context, hashes,
169
+ commit_message=message_type == 'commit')
164
170
  scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
165
171
  for rule in rules:
166
172
  add(rule, path, commit, scope=scope)
173
+ if references:
174
+ deferred.append((references, path, commit, None, scope))
167
175
  messages.append(data + b'\n')
168
176
  length = (data + b'\n').count(b'\n')
169
177
  lines.append((count, count + length, path, commit, scope))
@@ -172,6 +180,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
172
180
  line = hit.get('StartLine', 0)
173
181
  location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
174
182
  add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
183
+ states = GitHubResolver().resolve_all({repository for references, *_ in deferred for repository, _, _ in references})
184
+ for references, path, commit, oid, scope in deferred:
185
+ for rule in reference_rules(references, states):
186
+ add(rule, path, commit, oid, scope)
175
187
  except PublicBaseError:
176
188
  add('scan.public-base-rejected', '.')
177
189
  except InspectionError as error:
@@ -153,32 +153,33 @@ def without_spans(data, spans):
153
153
  return b' '.join(kept)
154
154
 
155
155
 
156
- def inspect(data, path, config, resolver, *, commit_message=False):
156
+ def inspect(data, path, config, *, commit_message=False):
157
157
  chunks = list(text_chunks(data))
158
158
  # Image and video samples are not text: they can spell an address or a
159
159
  # path by chance. Comments, XMP, tags and other metadata remain text.
160
160
  spans = sample_spans(data) or []
161
161
  # Names in a C2PA manifest's signing certificates are certificate metadata.
162
162
  spans += certificate_spans(data)
163
- rules, provenance = set(), False
163
+ rules, provenance, references = set(), False, set()
164
164
  for start, end, fields in chunks:
165
165
  spans.append((start, end))
166
166
  for field in fields:
167
167
  # A metadata value is text, not a standalone source manifest or
168
168
  # trust bundle, even when the containing image has a JSON name.
169
- field_rules, field_provenance = inspect_text(field, '@png-text', config, resolver, binary_content=False)
169
+ field_rules, field_provenance, field_references = inspect_text(field, '@png-text', config, binary_content=False)
170
170
  rules.update(field_rules)
171
171
  provenance |= field_provenance
172
+ references |= field_references
172
173
  # Keep the original binary classification for content outside text chunks.
173
174
  # CRC-adjacent bytes must not also create a synthetic raw mailbox match.
174
- raw_rules, raw_provenance = inspect_text(without_spans(data, spans), path, config, resolver,
175
- commit_message=commit_message, binary_content=binary(data))
175
+ raw_rules, raw_provenance, raw_references = inspect_text(without_spans(data, spans), path, config,
176
+ commit_message=commit_message, binary_content=binary(data))
176
177
  rules.update(raw_rules)
177
178
  provenance |= raw_provenance
178
- return rules, provenance
179
+ return rules, provenance, frozenset(references | raw_references)
179
180
 
180
181
 
181
- def inspect_text(data, path, config, resolver, *, commit_message=False, binary_content=None):
182
+ def inspect_text(data, path, config, *, commit_message=False, binary_content=None):
182
183
  # NUL removal covers UTF-16/32 ASCII content, matching the secret scanner.
183
184
  text = decoded_text(data)
184
185
  rules = set()
@@ -197,21 +198,30 @@ def inspect_text(data, path, config, resolver, *, commit_message=False, binary_c
197
198
  rules.add('hygiene.email')
198
199
  ref_text, provenance = provenance_text(text, path, config.get('declaredSources', frozenset()))
199
200
  stated, ambiguous = reference_sets(ref_text, path, config.get('localPaths', frozenset()), config.get('projectIdentity'))
200
- for repository in sorted(stated | ambiguous):
201
+ references = set()
202
+ for repository in stated | ambiguous:
201
203
  # A future source identity does not assert current GitHub visibility.
202
204
  # Explicit release destinations still take the strict visibility path.
203
205
  if (repository in config.get('intendedPublicRepositories', frozenset()) and
204
206
  repository != config.get('pendingDestination')):
205
207
  continue
206
- state = resolver.resolve(repository)
208
+ references.add((repository, repository in ambiguous, repository == config.get('pendingDestination')))
209
+ return rules, provenance, frozenset(references)
210
+
211
+
212
+ def reference_rules(references, states):
213
+ """Rules for (repository, ambiguous, pending destination) references once their states are known."""
214
+ rules = set()
215
+ for repository, ambiguous, pending in references:
216
+ state = states[repository]
207
217
  # A bare word/word with no GitHub context names a repository only when
208
218
  # one exists privately; that no repository answers to it proves nothing.
209
- if state == 'missing' and repository in ambiguous and repository != config.get('pendingDestination'):
219
+ if state == 'missing' and ambiguous and not pending:
210
220
  continue
211
- if state == 'missing' and repository == config.get('pendingDestination'):
221
+ if state == 'missing' and pending:
212
222
  rules.add(PENDING_DESTINATION)
213
223
  elif state in ('private', 'missing'):
214
224
  rules.add('hygiene.github-' + state)
215
225
  elif state != 'public':
216
226
  rules.add('scan.github-incomplete')
217
- return rules, provenance
227
+ return rules
@@ -37,15 +37,16 @@ def context(history, commit, config):
37
37
  'trustBundles': trust}, invalid
38
38
 
39
39
 
40
- def text_rules(data, path, config, resolver, env_hashes, *, commit_message=False):
41
- rules, provenance = inspect(data, path, config, resolver, commit_message=commit_message)
42
- path_rules, _ = inspect(path.encode('utf-8', 'surrogateescape'), '@path', config, resolver)
40
+ def text_rules(data, path, config, env_hashes, *, commit_message=False):
41
+ """Rules, provenance and the repository references whose rules wait for one bulk resolution."""
42
+ rules, provenance, references = inspect(data, path, config, commit_message=commit_message)
43
+ path_rules, _, path_references = inspect(path.encode('utf-8', 'surrogateescape'), '@path', config)
43
44
  rules.update(path_rules)
44
45
  if env_hashes and (environment_match(data, env_hashes) or environment_match(path.encode(), env_hashes)):
45
46
  rules.add('secret.env-match')
46
47
  if data.startswith(b'version https://git-lfs.github.com/spec/v1'):
47
48
  rules.add('scan.lfs-uninspected')
48
- return rules, provenance
49
+ return rules, provenance, references | path_references
49
50
 
50
51
 
51
52
  def working_files(repo):