@runeditrun/rer 0.2.46 → 0.2.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/main.mjs +7 -1
- package/cli/portability-progress.mjs +19 -0
- package/cli/portability-worker.mjs +13 -3
- package/cli/portability.mjs +4 -3
- package/global/CHANGELOG.md +11 -0
- package/global/launch/facts.mjs +17 -10
- package/global/run.mjs +2 -1
- package/modules/demo-host/README.md +3 -2
- package/modules/demo-host/build.mjs +2 -2
- package/modules/demo-host/native/config.mjs +2 -1
- package/modules/demo-host/native/prepare.mjs +1 -1
- package/modules/demo-host/native/warm.mjs +19 -7
- package/modules/job-queue/identity.mjs +5 -4
- package/modules/portability/README.md +1 -1
- package/modules/portability/export.mjs +2 -2
- package/modules/portability/files.mjs +25 -6
- package/modules/portability/format.mjs +3 -1
- package/modules/portability/import.mjs +8 -0
- package/modules/portability/index.d.mts +2 -2
- package/modules/portability/index.mjs +2 -2
- package/modules/portability/progress.mjs +54 -0
- package/onboarding/operator-installer.mjs +1 -1
- package/onboarding/queue.mjs +8 -8
- package/package.json +1 -1
- package/scripts/publish-check/README.md +5 -1
- package/scripts/publish-check/github_refs.py +83 -12
- package/scripts/publish-check/history_scan.py +18 -6
- package/scripts/publish-check/public_hygiene.py +22 -12
- package/scripts/publish-check/tree_scan.py +5 -4
package/cli/main.mjs
CHANGED
|
@@ -66,11 +66,17 @@ export async function main(argv = process.argv.slice(2)) {
|
|
|
66
66
|
else result = await (await import('./lifecycle.mjs')).runCommand(project, options, out);
|
|
67
67
|
out.result(true, 0, result); return 0;
|
|
68
68
|
} catch (error) {
|
|
69
|
-
const failure = error instanceof RerError ? error :
|
|
69
|
+
const failure = error instanceof RerError ? error : unexpectedFailure(error);
|
|
70
70
|
out.error(failure); out.result(false, failure.exitCode, failure.childExitCode === undefined ? {} : { childExitCode: failure.childExitCode, signal: failure.signal }); return failure.exitCode;
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
// A failure rer did not classify keeps its own first line; the hint is the next step.
|
|
75
|
+
function unexpectedFailure(error) {
|
|
76
|
+
const message = String(error?.message ?? error).split('\n')[0].trim().slice(0, 300);
|
|
77
|
+
return new RerError('command_failed', `${message || 'Command failed.'}\nUnexpected failure: run the command again, and if it repeats inspect the project operating documentation.`, 1);
|
|
78
|
+
}
|
|
79
|
+
|
|
74
80
|
function showHelp(options,out) {
|
|
75
81
|
if(options.json){out.event('help',{usage:help(options.verb)});out.result(true,0);}else out.human(help(options.verb));
|
|
76
82
|
return 0;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { lstat, readdir } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
// Bytes under a directory, for the first progress line's estimate. Missing or vanishing files count as zero.
|
|
5
|
+
export async function stateBytes(directory) {
|
|
6
|
+
let total = 0;
|
|
7
|
+
for (const entry of await readdir(directory, { withFileTypes: true }).catch(() => [])) {
|
|
8
|
+
const path = join(directory, entry.name);
|
|
9
|
+
total += entry.isDirectory() ? await stateBytes(path) : (await lstat(path).catch(() => ({ size: 0 }))).size;
|
|
10
|
+
}
|
|
11
|
+
return total;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// The class and code of an unexpected failure, when each is a plain identifier. Its message can hold row data.
|
|
15
|
+
export function nativeFailure(error) {
|
|
16
|
+
const name = typeof error?.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(error.name) ? error.name : 'unknown';
|
|
17
|
+
const code = typeof error?.code === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(error.code) ? error.code : Number.isSafeInteger(error?.code) ? String(error.code) : undefined;
|
|
18
|
+
return { class: name, ...(code ? { code } : {}) };
|
|
19
|
+
}
|
|
@@ -6,9 +6,11 @@ import { legacyTargetState, requireLegacyEmpty } from '../modules/portability/le
|
|
|
6
6
|
import { migrationReplay } from '../modules/portability/migration-history.mjs';
|
|
7
7
|
import { appliedMigrations as appliedMigrationNames } from '../modules/portability/bridge.mjs';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
-
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
9
|
+
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
10
10
|
import { createRequire } from 'node:module';
|
|
11
11
|
import { createPortability, PortabilityError, readArchive } from '../modules/portability/index.mjs';
|
|
12
|
+
import { transferProgress } from '../modules/portability/progress.mjs';
|
|
13
|
+
import { nativeFailure, stateBytes } from './portability-progress.mjs';
|
|
12
14
|
import { parseJsonc } from './jsonc.mjs';
|
|
13
15
|
import { loadPolicy } from './portability-policy.mjs';
|
|
14
16
|
|
|
@@ -33,8 +35,12 @@ async function dataConfiguration(root) {
|
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
async function run(direction, { root, path, journal, binding, buckets, policy, policyModule, transient, legacyRecovery }) {
|
|
38
|
+
// Lines for the person go to the CLI over IPC, which prints them as the command's progress.
|
|
39
|
+
const progress = transferProgress(line => process.send({ progress: line }));
|
|
40
|
+
if (!legacyRecovery && ['export', 'import'].includes(direction)) progress({ phase: 'start', direction, bytes: direction === 'export' ? await stateBytes(join(root, '.wrangler/state')) : (await stat(path)).size });
|
|
36
41
|
// Validate all inputs before opening bindings or acquiring a restore lock.
|
|
37
42
|
if (direction === 'import') {
|
|
43
|
+
progress({ phase: 'checking', direction });
|
|
38
44
|
for await (const record of fileRecords(path,legacyRecovery?readLegacyArchive:readArchive)) {}
|
|
39
45
|
if(journal)for await(const record of fileRecords(journal)) {}
|
|
40
46
|
}
|
|
@@ -56,7 +62,9 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
56
62
|
await ensurePortabilitySchema(proxy.env[binding], await portabilityPlan(unstable_splitSqlQuery));
|
|
57
63
|
if(legacyRecovery)return await recoverLegacyArchive(options,path,legacyRecovery.output,transient,legacyRecovery.acknowledged);
|
|
58
64
|
const service = await createPortability(options);
|
|
59
|
-
|
|
65
|
+
// This checkout's operation lock means no other import can be running: staging left behind is a crashed restore's.
|
|
66
|
+
if (direction === 'import' && await service.imports.abortAbandoned()) process.send({ progress: 'Cleaned up an unfinished earlier import: a restore that stopped part-way left its staged data here. Its staging is removed and this import starts from the beginning.' });
|
|
67
|
+
return await (direction === 'export' ? exportArchive(service,path,transient,progress) : importArchive(service,path,transient,journal,progress));
|
|
60
68
|
} finally { await proxy?.dispose(); await rm(configPath, { force: true }); }
|
|
61
69
|
}
|
|
62
70
|
|
|
@@ -73,6 +81,8 @@ catch (error) {
|
|
|
73
81
|
const migration = error instanceof PortabilityError && error.code.startsWith('schema_mismatch:migration_data:') ? error.code.slice('schema_mismatch:migration_data:'.length) : undefined;
|
|
74
82
|
// A populated destination names its table or bucket: schema names, never row data.
|
|
75
83
|
const occupied = error instanceof PortabilityError && /^(target|bucket)_not_empty:/.test(error.code) ? error.code.slice(error.code.indexOf(':') + 1) : undefined;
|
|
84
|
+
// Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
|
|
85
|
+
const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
|
|
76
86
|
process.exitCode = exitCode;
|
|
77
|
-
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), exitCode });
|
|
87
|
+
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(migration ? { migration } : {}), ...(occupied ? { occupied } : {}), ...(native ? { native } : {}), exitCode });
|
|
78
88
|
}
|
package/cli/portability.mjs
CHANGED
|
@@ -73,17 +73,18 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
73
73
|
const child = store.spawn(`${direction === 'policy-preflight' ? 'Checking archive policy' : direction === 'legacy-preflight' ? 'Checking empty recovery destination' : direction === 'export' ? 'Exporting' : 'Restoring'} local installation data`, process.execPath,
|
|
74
74
|
['--import', notice, worker, direction, JSON.stringify(input)], { cwd: input.root, signal, env: { ...process.env, WRANGLER_SEND_METRICS: 'false' }, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
|
|
75
75
|
let receipt;
|
|
76
|
-
|
|
76
|
+
// Progress lines come first, as {progress}; the last message is the receipt.
|
|
77
|
+
child.on('message', message => { if (typeof message?.progress === 'string') store.report(message.progress); else receipt = message; });
|
|
77
78
|
const closed = new Promise(resolve => child.once('close', (code, signal) => resolve({ code, signal })));
|
|
78
79
|
await new Promise(resolve => { child.once('exit', resolve); child.once('error', resolve); });
|
|
79
80
|
await store.stop(child);
|
|
80
81
|
const status = await closed;
|
|
81
|
-
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed',
|
|
82
|
+
if (!receipt || typeof receipt !== 'object') throw new RerError('transfer_failed', `Local portability ended before it finished (${status.signal ? `signal ${status.signal}` : `exit code ${status.code}`}). Run the same command again; it starts over${direction === 'import' ? ' and first cleans up anything this attempt staged' : ''}.`, 1);
|
|
82
83
|
if (status.code !== 0 || !receipt.ok) throw new RerError(receipt.code ?? 'transfer_failed', receipt.reason==='legacy_restored_export_failed'
|
|
83
84
|
? 'Legacy data was restored and audited, but its v2 export failed. Preserve this installation, resolve the output problem, then run rer export --output FILE. Do not reset it to retry recovery.'
|
|
84
85
|
: receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
|
|
85
86
|
: receipt.occupied ? `Restore refused (${receipt.reason}): ${receipt.reason === 'bucket_not_empty' ? `storage bucket ${receipt.occupied} in this installation already holds files` : `table ${receipt.occupied} in this installation already holds rows its migrations did not insert`}, and a restore only goes into an empty installation. To replace this installation's data with the archive, keep a copy with rer export --output exports/before-restore.rer, then run rer reset --yes and rer import --input ${relative(input.root, input.path)} --yes again.`
|
|
86
|
-
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'})
|
|
87
|
+
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
|
|
87
88
|
delete receipt.ok;
|
|
88
89
|
return receipt;
|
|
89
90
|
}
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.48
|
|
4
|
+
|
|
5
|
+
- **The job queue no longer aborts or strands a waiting command under load.** Before, when another queued job's owner did not answer its 3 s identity probe, the waiting build, verify or deploy stopped with "Command failed; inspect the project operating documentation." A probe timeout now counts as "the owner may still be alive", so the waiting job samples again instead of giving up. The same holds for the queue lock and for nested-command membership, which had left a job queued behind its own parent until the queue's maximum wait. A recorded failed install is retried only when `pnpm-lock.yaml` or `package.json` changed since it failed. An error that is not one of rer's own now prints its message and a hint instead of the generic line (#219).
|
|
6
|
+
- **The demo start page warms the chunks the seed actually built.** The warm list now comes from the build output: the client Vite manifest when the build wrote one, else the build's `assetsDir` (recorded in `demo.json` from the loaded Vite config), else `assets`. Seeds that build to a non-default directory, such as Lamp's `/app-assets/`, had an empty warm list; Lamp's is now 32 chunks with the console's routes first (#223).
|
|
7
|
+
- **`rer export` and `rer import` show progress, and a crashed import cleans up after itself.** Export prints the expected time, then parts built, parts written and bytes with the time left (one line every 3 s, plus each phase edge). Import prints parts staged. An import that died part-way leaves staging that the next import removes, saying "Cleaned up an unfinished earlier import". An unexpected failure prints its error class or code and "Run the same command again; it starts over." (#224).
|
|
8
|
+
|
|
9
|
+
## 0.2.47
|
|
10
|
+
|
|
11
|
+
- **The publication scan resolves GitHub repositories in bulk.** Before, the scan looked up each GitHub repository a text names with its own REST call. Now it collects every reference first and resolves them together, in one aliased GraphQL query per 100 repositories, which also draws on GitHub's separate GraphQL budget. A repository counts as public or private only on a complete answer that is not disabled or locked. It counts as missing only when GitHub returns NOT_FOUND for exactly that name. Any other answer for a repository, and any batch that is not a complete HTTP 200 GraphQL document, falls back to the earlier strict REST lookup for that repository. Nothing is cached on disk. A run-oncall scan now makes 1 gh call instead of 19, and its report is byte-identical (#216).
|
|
12
|
+
- **Plainer terminal lines for someone new.** Queue lines now name the seed by its `seed.json` name, else its folder name, and never by its project or package id. `rer run` and the build step run package scripts with `pnpm run --silent`, so pnpm's `> package@version script` banner is gone and the script's own output and errors still show. Step 1 of the Run page now says the installer does not edit your shell files. It also says that the line it may end with puts rer on your PATH now and appends the same setting to your shell's startup file for new terminals (#215).
|
|
13
|
+
|
|
3
14
|
## 0.2.46
|
|
4
15
|
|
|
5
16
|
- **`rer companion` pairs your computer with a Run Edit Run app and runs its change requests here.** The local companion is now a `rer` verb: `map` approves one Git checkout whose `origin` is exactly the app's GitHub repository, `pair` redeems the one-use code from the app's Devices page, `codex-auth` or `claude-auth` sets up a dedicated executor home, `commands` approves setup and verify argv lists, and `run` takes jobs from the app. Each job starts from the exact commit the app names in a separate detached checkout, and the provider runs inside a macOS `sandbox-exec` boundary with no GitHub, Cloudflare or MCP credentials; on other systems `run` refuses before it asks for work. It makes outbound HTTPS requests only and opens no local server. State lives in `~/.rer-local` (`RER_LOCAL_HOME` picks another folder), the device key in a 0600 file. `status` prints the paired device, mapped projects, active job and pending events, never the key. Revoke a device in the app under Settings › Devices (#202).
|
package/global/launch/facts.mjs
CHANGED
|
@@ -20,17 +20,24 @@ export async function describeCheckout(root, entry, env, signal) {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
// Install when the last install
|
|
24
|
-
// pnpm writes node_modules/.modules.yaml only once dependencies
|
|
25
|
-
// built, so a stopped install leaves it absent. An install this
|
|
26
|
-
// is finished only when its record says so: that also covers the
|
|
27
|
-
// own install hooks, which pnpm runs after writing the manifest.
|
|
23
|
+
// Install when the last install was killed, nothing is linked, or the lockfile
|
|
24
|
+
// changed since. pnpm writes node_modules/.modules.yaml only once dependencies
|
|
25
|
+
// are linked and built, so a stopped install leaves it absent. An install this
|
|
26
|
+
// launcher ran is finished only when its record says so: that also covers the
|
|
27
|
+
// project's own install hooks, which pnpm runs after writing the manifest. A
|
|
28
|
+
// recorded failed install is retried only when the lockfile or package.json is
|
|
29
|
+
// newer than both that attempt and what is installed; dependencies repaired
|
|
30
|
+
// since are current, and the next launch uses them.
|
|
28
31
|
export async function needsInstall(root, last) {
|
|
29
|
-
if (last
|
|
30
|
-
const
|
|
31
|
-
if (
|
|
32
|
-
const
|
|
33
|
-
|
|
32
|
+
if (last?.state === 'running') return true;
|
|
33
|
+
const linked = await mtime(join(root, 'node_modules/.modules.yaml'));
|
|
34
|
+
if (linked === undefined) return true;
|
|
35
|
+
const since = Math.max(linked, last?.at ?? 0);
|
|
36
|
+
for (const file of last?.state === 'failed' ? ['pnpm-lock.yaml', 'package.json'] : ['pnpm-lock.yaml']) {
|
|
37
|
+
const changed = await mtime(join(root, file));
|
|
38
|
+
if (changed !== undefined && changed > since) return true;
|
|
39
|
+
}
|
|
40
|
+
return false;
|
|
34
41
|
}
|
|
35
42
|
|
|
36
43
|
// A checkout is set up when its pin's describe says so (`local.prepared`, from
|
package/global/run.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// rer run SCRIPT [ARGS]: `pnpm run SCRIPT ARGS` in a checkout, with the pnpm
|
|
2
2
|
// and Node it pins. rer keeps both in its own cache, off PATH, so a project's
|
|
3
3
|
// documented `pnpm run ...` works as `rer run ...` right after rer install.
|
|
4
|
+
// --silent drops pnpm's "> package@version script" banner; the script's own output and errors stay.
|
|
4
5
|
// A script that runs Playwright gets the checkout's browser first, on first need.
|
|
5
6
|
import { spawn } from 'node:child_process';
|
|
6
7
|
import { readFile } from 'node:fs/promises';
|
|
@@ -52,7 +53,7 @@ export async function runScript(args) {
|
|
|
52
53
|
const code = await (await import('./browser-runtime.mjs')).browserPreflight(target.root, ['run']);
|
|
53
54
|
if (code !== undefined) return code;
|
|
54
55
|
}
|
|
55
|
-
const child = spawn('pnpm', ['run', ...args], { cwd: target.root, env: process.env, stdio: 'inherit' });
|
|
56
|
+
const child = spawn('pnpm', ['run', '--silent', ...args], { cwd: target.root, env: process.env, stdio: 'inherit' });
|
|
56
57
|
const forward = new Map(['SIGINT', 'SIGTERM', 'SIGHUP'].map(signal => [signal, () => child.kill(signal)]));
|
|
57
58
|
for (const [signal, handler] of forward) process.on(signal, handler);
|
|
58
59
|
try {
|
|
@@ -13,8 +13,9 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
|
|
|
13
13
|
- `worker.mjs`, `pages.mjs`, `start-script.mjs`: Turnstile start, progress,
|
|
14
14
|
cancellation and visitor routing. Static assets are served at the edge.
|
|
15
15
|
- `native/warm.mjs`: the start page's prefetch list, built from `dist/client`
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
(the Vite manifest if the build wrote one, else the build's `assetsDir`, else
|
|
17
|
+
`assets`) and TanStack Start's route manifest, the console's views first,
|
|
18
|
+
capped by compressed size.
|
|
18
19
|
- `capacity.mjs`: concurrent visitor admission and operator diagnostics. This
|
|
19
20
|
is used by the native host as well as retained during legacy upgrades.
|
|
20
21
|
- `upgrade.mjs`: verifies ownership, removes the old Container application and
|
|
@@ -40,9 +40,9 @@ export async function buildDemo(root, output, declaration) {
|
|
|
40
40
|
await rm(output, { recursive: true, force: true });
|
|
41
41
|
await mkdir(join(output, 'app'), { recursive: true });
|
|
42
42
|
await cp(join(root, 'dist'), join(output, 'app/dist'), { recursive: true });
|
|
43
|
-
// Where the console is mounted orders what the start page prefetches (native/warm.mjs).
|
|
43
|
+
// Where the console is mounted orders what the start page prefetches, and the build's assetsDir says where its chunks are (native/warm.mjs).
|
|
44
44
|
const { operatorPath } = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8').catch(() => '{}')).deploy ?? {};
|
|
45
|
-
await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath }) + '\n');
|
|
45
|
+
await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath, assetsDir: loaded.config.build?.assetsDir }) + '\n');
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
if (process.argv[1] === import.meta.filename) {
|
|
@@ -2,7 +2,8 @@ import { parseCron } from './cron.mjs';
|
|
|
2
2
|
|
|
3
3
|
export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'r2-store', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
|
|
4
4
|
// Hashed build output: served immutable, and what the start page may prefetch.
|
|
5
|
-
export const
|
|
5
|
+
export const hashedAsset = (assetsDir = 'assets') => new RegExp(`^/${assetsDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/.*-[\\w-]{8,}\\.[\\w]+$`);
|
|
6
|
+
export const HASHED_ASSET = hashedAsset();
|
|
6
7
|
export const ISOLATE_FILES = ['bindings', 'namespace', 'r2-client', 'd1', 'sql', 'seed-object'];
|
|
7
8
|
export function nativeBindings(config) {
|
|
8
9
|
for (const kind of ['kv_namespaces', 'services', 'workflows', 'ai', 'vectorize', 'hyperdrive', 'analytics_engine_datasets', 'browser', 'images', 'send_email', 'dispatch_namespaces', 'mtls_certificates', 'pipelines', 'secrets_store_secrets', 'unsafe', 'vpc_services', 'vpc_networks', 'ratelimits', 'worker_loaders', 'containers', 'ai_search_namespaces', 'ai_search', 'agent_memory', 'connect', 'artifacts', 'unsafe_hello_world', 'flagship', 'logfwdr', 'version_metadata']) {
|
|
@@ -34,7 +34,7 @@ export async function prepareNative(folder) {
|
|
|
34
34
|
const bundle = { compatibilityDate: config.compatibility_date, compatibilityFlags: [...new Set([...(config.compatibility_flags ?? []), 'nodejs_compat'])], modules,
|
|
35
35
|
bindings, migrations: manifest.migrations, vars: config.vars ?? {}, crons: config.triggers?.crons ?? [],
|
|
36
36
|
assetPaths: assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path),
|
|
37
|
-
warm: (await warmList(assets, root, manifest.operatorPath)).paths };
|
|
37
|
+
warm: (await warmList(assets, root, manifest.operatorPath, manifest.assetsDir)).paths };
|
|
38
38
|
bundle.digest = createHash('sha256').update(JSON.stringify(bundle)).digest('hex');
|
|
39
39
|
await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
|
|
40
40
|
await writeFile(join(folder, 'modules/demo-host/native/bundle.mjs'), `export default ${JSON.stringify(bundle)};\n`);
|
|
@@ -2,7 +2,7 @@ import { readdir } from 'node:fs/promises';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { pathToFileURL } from 'node:url';
|
|
4
4
|
import { gzipSync } from 'node:zlib';
|
|
5
|
-
import {
|
|
5
|
+
import { hashedAsset } from './config.mjs';
|
|
6
6
|
|
|
7
7
|
// The start page prefetches the console's hashed JS and CSS while the visitor
|
|
8
8
|
// waits for the human check and sign-in, so each view's first visit finds its
|
|
@@ -16,10 +16,22 @@ const segments = path => path.split('/').filter(Boolean);
|
|
|
16
16
|
const dynamic = path => segments(path).filter(part => part.startsWith('$')).length;
|
|
17
17
|
const within = (path, root) => path === root || path.startsWith(root.endsWith('/') ? root : root + '/');
|
|
18
18
|
|
|
19
|
+
// The hashed JS and CSS the client build produced: the files its Vite manifest
|
|
20
|
+
// names (build.manifest), else everything under the configured build.assetsDir
|
|
21
|
+
// (Vite's default is 'assets'). Never source maps or unhashed files.
|
|
22
|
+
const MANIFESTS = ['.vite/manifest.json', 'manifest.json'];
|
|
23
|
+
function builtChunks(files, assetsDir) {
|
|
24
|
+
const manifest = files.find(file => MANIFESTS.includes(file.path));
|
|
25
|
+
const paths = (manifest
|
|
26
|
+
? Object.values(JSON.parse(Buffer.from(manifest.base64, 'base64'))).flatMap(entry => [entry.file, ...(entry.css ?? [])]).map(path => '/' + path)
|
|
27
|
+
: files.map(file => '/' + file.path).filter(path => hashedAsset(assetsDir).test(path)));
|
|
28
|
+
return new Set(paths.filter(path => /-[\w-]{8,}\.(js|css)$/.test(path)));
|
|
29
|
+
}
|
|
30
|
+
|
|
19
31
|
// TanStack Start's route manifest from the server build: each route's CSS and
|
|
20
32
|
// the chunks it preloads.
|
|
21
|
-
async function startRoutes(server) {
|
|
22
|
-
const directory = join(server,
|
|
33
|
+
async function startRoutes(server, assetsDir) {
|
|
34
|
+
const directory = join(server, assetsDir);
|
|
23
35
|
const file = (await readdir(directory).catch(() => [])).find(name => /^_tanstack-start-manifest_v-[\w-]+\.js$/.test(name));
|
|
24
36
|
if (!file) return {};
|
|
25
37
|
const { tsrStartManifest } = await import(pathToFileURL(join(directory, file)));
|
|
@@ -36,10 +48,10 @@ function routeOrder(routes, consolePath) {
|
|
|
36
48
|
}
|
|
37
49
|
|
|
38
50
|
/** The ordered, size-capped paths the start page prefetches, from dist/client's files and the server build. */
|
|
39
|
-
export async function warmList(files, server, consolePath = '/app') {
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
const routes = await startRoutes(server);
|
|
51
|
+
export async function warmList(files, server, consolePath = '/app', assetsDir = 'assets') {
|
|
52
|
+
const chunks = builtChunks(files, assetsDir);
|
|
53
|
+
const sizes = new Map(files.filter(file => chunks.has('/' + file.path)).map(file => ['/' + file.path, gzipSync(Buffer.from(file.base64, 'base64')).length]));
|
|
54
|
+
const routes = await startRoutes(server, assetsDir);
|
|
43
55
|
const ordered = new Set();
|
|
44
56
|
for (const id of routeOrder(routes, consolePath)) for (const path of [...(routes[id].css ?? []), ...(routes[id].preloads ?? [])]) if (sizes.has(path)) ordered.add(path);
|
|
45
57
|
for (const path of [...sizes.keys()].sort((a, b) => sizes.get(a) - sizes.get(b))) ordered.add(path);
|
|
@@ -31,14 +31,15 @@ export async function alive(owner) {
|
|
|
31
31
|
return new Promise((resolve, reject) => {
|
|
32
32
|
const connection = createConnection(owner.socket);
|
|
33
33
|
let body = '';
|
|
34
|
-
connection.setTimeout(3000, () => connection.destroy(Error('Owner identity probe timed out
|
|
34
|
+
connection.setTimeout(3000, () => connection.destroy(Object.assign(Error('Owner identity probe timed out'), { code: 'PROBE_TIMEOUT' })));
|
|
35
35
|
connection.on('data', data => { body += data; if (body.length > 128) connection.destroy(Error('Invalid owner identity')); });
|
|
36
36
|
connection.on('end', () => resolve(body === owner.token));
|
|
37
37
|
connection.on('error', async error => {
|
|
38
38
|
if (['ENOENT', 'ECONNREFUSED'].includes(error.code)) resolve(false);
|
|
39
|
-
else if (error.code
|
|
40
|
-
// Listener shutdown can reset a probe after the initial PID check
|
|
41
|
-
//
|
|
39
|
+
else if (['ECONNRESET', 'PROBE_TIMEOUT'].includes(error.code)) {
|
|
40
|
+
// Listener shutdown can reset a probe after the initial PID check, and a
|
|
41
|
+
// busy owner can answer late. Neither proves death: reclaim only a process
|
|
42
|
+
// that is gone; otherwise retain the owner until the next sample.
|
|
42
43
|
try {
|
|
43
44
|
const after = await processInfo(owner.pid);
|
|
44
45
|
resolve(Boolean(after && !after.zombie && after.start === owner.start));
|
|
@@ -121,7 +121,7 @@ idempotence and recovery; D1 cannot roll back Durable Object or external state.
|
|
|
121
121
|
The UTF-8 JSON-lines archive has a manifest, indexed checksummed parts and a
|
|
122
122
|
closing record. `readArchive(stream)` uses the shared bounded, fatal UTF-8 line
|
|
123
123
|
reader. The default maximum line size is 24 MiB; callers can set a smaller bound.
|
|
124
|
-
No runtime or CLI materializes the whole archive. Readers enforce a 16 MiB
|
|
124
|
+
No runtime or CLI materializes the whole archive. The manifest opens the archive and holds the final part count and payload hashes, so an export builds all its parts in the job before any byte is written; `exportArchive` and `importArchive` take an optional `progress` callback (`progress.mjs` turns its events into throttled lines with a time-left estimate). Readers enforce a 16 MiB
|
|
125
125
|
canonical UTF-8 part limit; `readArchive(stream,{partBytes})` can lower it.
|
|
126
126
|
Import uploads enforce the policy's part limit before any staging write.
|
|
127
127
|
A part has at most 1,000 rows;
|
|
@@ -96,7 +96,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
96
96
|
if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
|
|
97
97
|
const before=row.completed,items=await nextItems(context,row);
|
|
98
98
|
if(items) {
|
|
99
|
-
const part=await makePart(before,items)
|
|
99
|
+
const part=await makePart(before,items),size=await checkPart(part,before,row.metadata.partBytes);row.completed++;row.metadata.bytes=(row.metadata.bytes??0)+size+1;
|
|
100
100
|
try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
|
|
101
101
|
catch(e){if((await job(context.db,id)).completed===before)throw e;}
|
|
102
102
|
if(expectedPart!==undefined)expectedPart++;
|
|
@@ -114,7 +114,7 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
114
114
|
return exportStatus(context,id);
|
|
115
115
|
}
|
|
116
116
|
export async function exportStatus(context,id) {
|
|
117
|
-
const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,contentHash:row.metadata.contentHash??null};
|
|
117
|
+
const row=await job(context.db,id,'export');return {id,kind:'export',phase:row.phase,completed:row.completed,total:row.phase==='complete'?row.completed:null,bytes:row.metadata.bytes??0,contentHash:row.metadata.contentHash??null};
|
|
118
118
|
}
|
|
119
119
|
export async function downloadExport(context,id) {
|
|
120
120
|
const row=await job(context.db,id,'export');insist(row.phase==='complete','export_incomplete',409);
|
|
@@ -4,6 +4,7 @@ import { dirname } from 'node:path';
|
|
|
4
4
|
import { Readable } from 'node:stream';
|
|
5
5
|
import { randomUUID } from 'node:crypto';
|
|
6
6
|
import { readArchive } from './index.mjs';
|
|
7
|
+
import { canonicalJson } from './hash.mjs';
|
|
7
8
|
export async function* fileRecords(path,reader=readArchive) {
|
|
8
9
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
9
10
|
try { yield* reader(Readable.toWeb(file.createReadStream({ autoClose: false }))); }
|
|
@@ -19,23 +20,35 @@ function receipt(manifest, closing, transient) {
|
|
|
19
20
|
requiredConfiguration: manifest.requiredConfiguration,
|
|
20
21
|
};
|
|
21
22
|
}
|
|
22
|
-
|
|
23
|
+
// `progress` receives {phase, parts, total?, bytes?} events; the CLI turns them into lines (progress.mjs).
|
|
24
|
+
export async function exportArchive(service, path, transient, progress) {
|
|
23
25
|
let id, temporary, file;
|
|
24
26
|
try {
|
|
25
27
|
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
26
28
|
temporary = path + '.' + randomUUID() + '.partial';
|
|
27
29
|
file = await open(temporary, 'wx', 0o600);
|
|
28
30
|
id = await service.exports.create();
|
|
29
|
-
|
|
31
|
+
// The manifest leads the archive and holds the final part count and payload hashes, so
|
|
32
|
+
// nothing can be written until every part is built (export.mjs advanceExport). One part per
|
|
33
|
+
// advance keeps the progress line moving; a part is at most 16 MiB.
|
|
34
|
+
let status;
|
|
35
|
+
do {
|
|
36
|
+
status = await service.exports.advance(id, { maxParts: 1 });
|
|
37
|
+
progress?.({ phase: 'building', parts: status.completed, bytes: status.bytes, ...(status.phase === 'complete' ? { total: status.total } : {}) });
|
|
38
|
+
} while (status.phase === 'building');
|
|
30
39
|
const reader = (await service.exports.download(id)).getReader();
|
|
31
40
|
try {
|
|
32
|
-
|
|
41
|
+
// One chunk per record: the manifest, then each part, then the closing record.
|
|
42
|
+
for (let record = 0, written = 0;; record++) {
|
|
33
43
|
const { done, value } = await reader.read(); if (done) break;
|
|
34
44
|
let offset = 0;
|
|
35
45
|
while (offset < value.length) offset += (await file.write(value, offset)).bytesWritten;
|
|
46
|
+
written += value.length;
|
|
47
|
+
if (record > 0 && record <= status.total) progress?.({ phase: 'writing', parts: record, total: status.total, bytes: written });
|
|
36
48
|
}
|
|
37
49
|
} finally { await reader.cancel(); reader.releaseLock(); }
|
|
38
50
|
await file.sync(); await file.close(); file = undefined;
|
|
51
|
+
progress?.({ phase: 'checking', direction: 'export' });
|
|
39
52
|
let manifest, closing;
|
|
40
53
|
for await (const record of fileRecords(temporary)) { if (record.kind === 'manifest') manifest = record; if (record.kind === 'closing') closing = record; }
|
|
41
54
|
await link(temporary, path);
|
|
@@ -46,19 +59,25 @@ export async function exportArchive(service, path, transient) {
|
|
|
46
59
|
if (id) await service.exports.abort(id);
|
|
47
60
|
}
|
|
48
61
|
}
|
|
49
|
-
export async function importArchive(service, path, transient, journal) {
|
|
50
|
-
let id, manifest, closing, complete = false;
|
|
62
|
+
export async function importArchive(service, path, transient, journal, progress) {
|
|
63
|
+
let id, manifest, closing, complete = false, staged = 0;
|
|
51
64
|
try {
|
|
52
65
|
// Validate both complete archives before creating any restore state.
|
|
53
66
|
let later;
|
|
67
|
+
progress?.({ phase: 'checking', direction: 'import' });
|
|
54
68
|
for (const file of [path, journal].filter(Boolean)) for await (const record of fileRecords(file)) {
|
|
55
69
|
if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
|
|
56
70
|
}
|
|
57
71
|
for await (const record of fileRecords(path)) {
|
|
58
72
|
if (record.kind === 'manifest') { manifest = record; id = await service.imports.create(manifest); }
|
|
59
|
-
else if (record.kind === 'part')
|
|
73
|
+
else if (record.kind === 'part') {
|
|
74
|
+
await service.imports.uploadPart(id, record.index, record);
|
|
75
|
+
staged += Buffer.byteLength(canonicalJson(record)) + 1;
|
|
76
|
+
progress?.({ phase: 'staging', parts: record.index + 1, total: manifest.partCount, bytes: staged });
|
|
77
|
+
}
|
|
60
78
|
else closing = record;
|
|
61
79
|
}
|
|
80
|
+
progress?.({ phase: 'finishing', total: manifest.partCount });
|
|
62
81
|
const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
|
|
63
82
|
return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}) };
|
|
64
83
|
} finally { if (id && !complete) await service.imports.abort(id); }
|
|
@@ -11,9 +11,11 @@ export async function makePart(index,items) {
|
|
|
11
11
|
export async function checkPart(part,index,partBytes=MAX_PART_BYTES) {
|
|
12
12
|
insist(part?.kind==='part' && part.index===index && Array.isArray(part.items) && part.items.length>0 && Object.keys(part).sort().join(',')==='index,items,kind,sha256','part_order');
|
|
13
13
|
insist(Number.isSafeInteger(partBytes) && partBytes>0 && partBytes<=MAX_PART_BYTES,'invalid_part_limit');
|
|
14
|
-
|
|
14
|
+
const size=utf8(canonicalJson(part)).length;
|
|
15
|
+
insist(size<=partBytes,'part_byte_limit',413);
|
|
15
16
|
insist(part.sha256===await sha256(canonicalJson({kind:part.kind,index:part.index,items:part.items})),'part_checksum');
|
|
16
17
|
insist(part.items.filter(x=>x.kind==='row').length<=1000,'part_row_limit');
|
|
18
|
+
return size;
|
|
17
19
|
}
|
|
18
20
|
export async function checkManifest(m) {
|
|
19
21
|
insist(m?.kind==='manifest' && m.format===FORMAT,'archive_format');
|
|
@@ -197,3 +197,11 @@ export async function abortImport(context,id) {
|
|
|
197
197
|
await context.db.batch([save(context.db,row),statement(context.db,'UPDATE rer_portability_state SET import_job=NULL,applying=0 WHERE id=1 AND import_job=?',id)]);
|
|
198
198
|
return importStatus(context,id);
|
|
199
199
|
}
|
|
200
|
+
// A restore's staging outlives a worker that died mid-restore (a crashed process or workerd) and refuses
|
|
201
|
+
// every later import as running. Only a caller that owns the destination, as the local CLI does under the
|
|
202
|
+
// checkout's operation lock, may call this: a hosted service can have a live import.
|
|
203
|
+
export async function abortAbandonedImport(context) {
|
|
204
|
+
const id=(await state(context.db)).import_job;
|
|
205
|
+
if(!id)return null;
|
|
206
|
+
await abortImport(context,id);return id;
|
|
207
|
+
}
|
|
@@ -28,7 +28,7 @@ export interface ArchivePolicy {
|
|
|
28
28
|
extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
|
|
29
29
|
partBytes?: number;
|
|
30
30
|
}
|
|
31
|
-
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
|
|
31
|
+
export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
|
|
32
32
|
/** Later additive migrations an older archive was restored across. */
|
|
33
33
|
export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
|
|
34
34
|
/** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
|
|
@@ -36,7 +36,7 @@ export interface MigrationReplay {names:string[];statements(name:string):string[
|
|
|
36
36
|
export interface PortabilityOptions {db:D1Database;buckets?:Record<string,R2Bucket>;policy:ArchivePolicy;migrations?:()=>Promise<MigrationReplay>}
|
|
37
37
|
export interface Portability {
|
|
38
38
|
exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
|
|
39
|
-
imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>};
|
|
39
|
+
imports: {create(manifest:Manifest,actor?:string):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
|
|
40
40
|
status(id:string):Promise<Status>;actor(id:string):Promise<string>;
|
|
41
41
|
erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
|
|
42
42
|
redactionStatements():Promise<D1PreparedStatement[]>;
|
|
@@ -2,7 +2,7 @@ import { policy, catalogue } from './catalogue.mjs';
|
|
|
2
2
|
import { wireDatabase } from '../d1-transport/index.mjs';
|
|
3
3
|
import { insist, job } from './storage.mjs';
|
|
4
4
|
import { createExport, advanceExport, exportStatus, downloadExport, abortExport } from './export.mjs';
|
|
5
|
-
import { createImport, uploadPart, finalizeImport, importStatus, abortImport } from './import.mjs';
|
|
5
|
+
import { createImport, uploadPart, finalizeImport, importStatus, abortImport, abortAbandonedImport } from './import.mjs';
|
|
6
6
|
import { erasureStatements, redactionStatements, invalidateTransfersStatements } from './journal.mjs';
|
|
7
7
|
export { archiveLines } from './lines.mjs';
|
|
8
8
|
export { excludedTables, excludedTable } from './catalogue.mjs';
|
|
@@ -17,7 +17,7 @@ export async function createPortability({db,buckets={},policy:input,migrations})
|
|
|
17
17
|
const context={db,buckets,config,migrations};
|
|
18
18
|
return {
|
|
19
19
|
exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
|
|
20
|
-
imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id)},
|
|
20
|
+
imports:{create:(manifest,actor)=>createImport(context,manifest,actor),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
|
|
21
21
|
status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
|
|
22
22
|
actor:async id=>(await job(db,id)).actor,
|
|
23
23
|
erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Human progress for a long export or import. The caller sends events; this turns
|
|
2
|
+
// them into at most one line every few seconds, with an estimate of the time left
|
|
3
|
+
// once a phase has two samples. The first and last line of every phase always show.
|
|
4
|
+
// Rates are bytes per second measured on a 1.2 GB local install on a loaded machine
|
|
5
|
+
// (export 1725 s, import 2234 s); a quiet machine is faster.
|
|
6
|
+
const RATES = { export: 700_000, import: 550_000 };
|
|
7
|
+
const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'];
|
|
8
|
+
export function formatBytes(bytes) {
|
|
9
|
+
let value = bytes, unit = 0;
|
|
10
|
+
while (value >= 1000 && unit < UNITS.length - 1) { value /= 1000; unit++; }
|
|
11
|
+
return `${unit && value < 100 ? value.toFixed(1) : Math.round(value)} ${UNITS[unit]}`;
|
|
12
|
+
}
|
|
13
|
+
export function formatDuration(seconds) {
|
|
14
|
+
if (seconds < 45) return 'under a minute';
|
|
15
|
+
const minutes = Math.max(1, Math.round(seconds / 60));
|
|
16
|
+
return minutes < 90 ? `about ${minutes} min` : `about ${Math.floor(minutes / 60)} h ${minutes % 60} min`;
|
|
17
|
+
}
|
|
18
|
+
const left = seconds => `, ${formatDuration(seconds)} left`;
|
|
19
|
+
export function transferProgress(report, { interval = 3000, now = () => performance.now() } = {}) {
|
|
20
|
+
let phase, first, printed = -Infinity, expected;
|
|
21
|
+
const line = event => {
|
|
22
|
+
const bytes = event.bytes === undefined ? '' : ` (${formatBytes(event.bytes)}${event.phase === 'building' && expected ? ` of about ${formatBytes(expected)}` : ''})`;
|
|
23
|
+
const parts = event.total === undefined ? `${event.parts} part${event.parts === 1 ? '' : 's'} so far` : `${event.parts} of ${event.total} parts`;
|
|
24
|
+
// Seconds left, from the rate since the phase's first sample.
|
|
25
|
+
const eta = () => {
|
|
26
|
+
const seconds = (now() - first.time) / 1000;
|
|
27
|
+
if (event === first.event || seconds <= 0 || event.parts === event.total) return '';
|
|
28
|
+
if (event.total !== undefined && event.parts > first.event.parts) return left((event.total - event.parts) * seconds / (event.parts - first.event.parts));
|
|
29
|
+
if (event.phase === 'building' && expected && event.bytes > first.event.bytes && event.bytes < expected) return left((expected - event.bytes) * seconds / (event.bytes - first.event.bytes));
|
|
30
|
+
return '';
|
|
31
|
+
};
|
|
32
|
+
switch (event.phase) {
|
|
33
|
+
case 'building': return `Building the archive: ${parts}${bytes}${eta()}.`;
|
|
34
|
+
case 'writing': return `Writing the archive: ${parts}${bytes}${eta()}.`;
|
|
35
|
+
case 'staging': return `Staging the restore: ${parts}${bytes}${eta()}.`;
|
|
36
|
+
case 'checking': return event.direction === 'export' ? 'Checking the written archive.' : 'Checking the whole archive before anything is restored.';
|
|
37
|
+
case 'finishing': return `All ${event.total} parts are staged; verifying them and publishing the restore. This can take several minutes.`;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
return event => {
|
|
41
|
+
if (event.phase === 'start') {
|
|
42
|
+
const noun = event.direction === 'export' ? `Exporting about ${formatBytes(event.bytes)} of local data` : `Restoring a ${formatBytes(event.bytes)} archive`;
|
|
43
|
+
if (event.direction === 'export') expected = event.bytes;
|
|
44
|
+
report(`${noun}. ${event.bytes / RATES[event.direction] < 45 ? 'This takes under a minute.' : `An archive this size usually takes ${formatDuration(event.bytes / RATES[event.direction])}; a busy machine takes longer. Progress follows every few seconds.`}`);
|
|
45
|
+
printed = now();
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const time = now(), edge = event.phase !== phase;
|
|
49
|
+
if (edge) { phase = event.phase; first = { time, event }; }
|
|
50
|
+
// A phase's last event (all parts built, written or staged) prints whatever the clock says.
|
|
51
|
+
const last = event.total !== undefined && event.parts === event.total;
|
|
52
|
+
if (edge || last || time - printed >= interval) { report(line(event)); printed = time; }
|
|
53
|
+
};
|
|
54
|
+
}
|
|
@@ -19,7 +19,7 @@ export function operatorInstaller({ database, features, prepare, prepareData })
|
|
|
19
19
|
keepExisting: true, originKey: 'AUTH_ORIGIN',
|
|
20
20
|
values: ({ origin }) => ({ AUTH_LOCAL: 'true', AUTH_ORIGIN: origin, MAIL_PROVIDER: 'development', MAIL_FROM: 'auth@localhost.test' }),
|
|
21
21
|
},
|
|
22
|
-
build: ['pnpm', ['run', 'build:app']],
|
|
22
|
+
build: ['pnpm', ['run', '--silent', 'build:app']],
|
|
23
23
|
migrate: context => migrateOperator(context, database),
|
|
24
24
|
// Setup uses the local entry for owner-side dependency approval without mail.
|
|
25
25
|
start: developmentTarget,
|
package/onboarding/queue.mjs
CHANGED
|
@@ -6,16 +6,16 @@ const portable = basename(dirname(fileURLToPath(import.meta.url))) === 'base-onb
|
|
|
6
6
|
const queueURL = new URL(portable ? '../base-cli/modules/job-queue/' : '../modules/job-queue/', import.meta.url);
|
|
7
7
|
export const heavyVerbs = new Set(['install', 'setup', 'build', 'verify', 'dev', 'preview', 'deploy', 'release', 'export', 'import', 'installer']);
|
|
8
8
|
export const verificationBoundary = verb => verb === 'release' || verb === 'verify' || verb?.startsWith('verify:');
|
|
9
|
+
// The seed's display name (seed.json `name`, else its folder name) for terminal lines,
|
|
10
|
+
// never its package or project id; the hash tells two checkouts of one seed apart.
|
|
9
11
|
export async function queueLabel(root, verb, phase) {
|
|
10
12
|
root = await realpath(root);
|
|
11
|
-
let
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
return `${project} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
|
|
13
|
+
let name = basename(root);
|
|
14
|
+
try {
|
|
15
|
+
const value = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8')).name;
|
|
16
|
+
if (typeof value === 'string' && value.trim() && !/[\p{Cc}]/u.test(value) && value.length <= 80) name = value.trim();
|
|
17
|
+
} catch {}
|
|
18
|
+
return `${name} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
function shellQuote(value) {
|
package/package.json
CHANGED
|
@@ -11,7 +11,11 @@ entropy or English stopword filters. Both assignment rules go through the same
|
|
|
11
11
|
location-bound Source/module identity classification; provider and environment findings
|
|
12
12
|
remain independent.
|
|
13
13
|
`tree_scan.py` applies the kept text and licence policy. `history_scan.py`
|
|
14
|
-
assigns release, working-tree and historical scopes.
|
|
14
|
+
assigns release, working-tree and historical scopes. Inspection returns each
|
|
15
|
+
text's repository references instead of resolving them; `history_scan.py`
|
|
16
|
+
resolves the whole scan's references at once through `github_refs.py`
|
|
17
|
+
(GraphQL batches, a REST lookup for any repository a batch leaves in doubt) and
|
|
18
|
+
then adds their rules. The Node entry point bounds
|
|
15
19
|
the whole operation and stops its private process group on timeout/cancellation.
|
|
16
20
|
|
|
17
21
|
Git message records carry their kind explicitly. Only commit records enable the
|
|
@@ -200,14 +200,18 @@ def response_constant(value):
|
|
|
200
200
|
raise ValueError('nonstandard API JSON constant')
|
|
201
201
|
|
|
202
202
|
|
|
203
|
-
def
|
|
203
|
+
def response(output):
|
|
204
|
+
"""Split `gh api --include` output into its HTTP status and strict JSON document."""
|
|
204
205
|
headers, body = re.split(r'\r?\n\r?\n', output, maxsplit=1)
|
|
205
206
|
status = re.fullmatch(r'HTTP/(?:1\.[01]|2(?:\.0)?|3(?:\.0)?) (\d{3})(?: [^\r\n]*)?',
|
|
206
207
|
headers.splitlines()[0])
|
|
207
208
|
if not status:
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
209
|
+
raise ValueError('no HTTP status line')
|
|
210
|
+
return int(status.group(1)), json.loads(body, object_pairs_hook=response_object, parse_constant=response_constant)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def visibility(output, exit_code):
|
|
214
|
+
code, document = response(output)
|
|
211
215
|
if not isinstance(document, dict):
|
|
212
216
|
return 'incomplete'
|
|
213
217
|
if code == 200 and exit_code == 0 and type(document.get('private')) is bool:
|
|
@@ -221,22 +225,89 @@ def visibility(output, exit_code):
|
|
|
221
225
|
return 'incomplete'
|
|
222
226
|
|
|
223
227
|
|
|
228
|
+
BATCH = 100
|
|
229
|
+
FIELDS = frozenset({'isPrivate', 'isDisabled', 'isLocked'})
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def bulk_query(repositories):
|
|
233
|
+
"""One aliased GraphQL query; owners and names travel as variables, never as query text."""
|
|
234
|
+
variables, fields, arguments = [], [], []
|
|
235
|
+
for index, repository in enumerate(repositories):
|
|
236
|
+
owner, name = repository.split('/')
|
|
237
|
+
variables.append(f'$o{index}:String!,$n{index}:String!')
|
|
238
|
+
fields.append(f'r{index}:repository(owner:$o{index},name:$n{index}){{{" ".join(sorted(FIELDS))}}}')
|
|
239
|
+
arguments += ['-f', f'o{index}={owner}', '-f', f'n{index}={name}']
|
|
240
|
+
return ['-f', 'query=query(' + ','.join(variables) + '){' + ' '.join(fields) + '}', *arguments]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def bulk_visibility(repositories, output, exit_code):
|
|
244
|
+
"""Return only the verdicts the batch states without doubt.
|
|
245
|
+
|
|
246
|
+
A repository left out takes today's single REST lookup: a disabled or locked
|
|
247
|
+
repository, an error other than GitHub's own not-found for exactly that name
|
|
248
|
+
(no access through SAML or an IP allow list, a rate limit), or any response
|
|
249
|
+
that is not a complete HTTP 200 GraphQL document for exactly this batch.
|
|
250
|
+
"""
|
|
251
|
+
code, document = response(output)
|
|
252
|
+
aliases = {f'r{index}': repository for index, repository in enumerate(repositories)}
|
|
253
|
+
if code != 200 or not isinstance(document, dict) or not set(document) <= {'data', 'errors'}:
|
|
254
|
+
return {}
|
|
255
|
+
data, errors = document.get('data'), document.get('errors', [])
|
|
256
|
+
# gh exits 1 exactly when the document carries errors.
|
|
257
|
+
if not isinstance(data, dict) or set(data) != set(aliases) or not isinstance(errors, list) or exit_code != (1 if errors else 0):
|
|
258
|
+
return {}
|
|
259
|
+
reported = {}
|
|
260
|
+
for error in errors:
|
|
261
|
+
path = error.get('path') if isinstance(error, dict) else None
|
|
262
|
+
if not (isinstance(path, list) and len(path) == 1 and path[0] in aliases):
|
|
263
|
+
return {}
|
|
264
|
+
reported.setdefault(path[0], []).append(error)
|
|
265
|
+
states = {}
|
|
266
|
+
for alias, repository in aliases.items():
|
|
267
|
+
node, problems = data[alias], reported.get(alias, [])
|
|
268
|
+
if (not problems and isinstance(node, dict) and set(node) == FIELDS and type(node['isPrivate']) is bool and
|
|
269
|
+
node['isDisabled'] is False and node['isLocked'] is False):
|
|
270
|
+
states[repository] = 'private' if node['isPrivate'] else 'public'
|
|
271
|
+
elif (node is None and len(problems) == 1 and set(problems[0]) <= {'type', 'path', 'locations', 'message'} and
|
|
272
|
+
problems[0].get('type') == 'NOT_FOUND' and
|
|
273
|
+
problems[0].get('message') == f"Could not resolve to a Repository with the name '{repository}'."):
|
|
274
|
+
states[repository] = 'missing'
|
|
275
|
+
return states
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def gh(arguments):
|
|
279
|
+
# No disk cache, auth arguments, diagnostic output or response metadata.
|
|
280
|
+
# Disable gh debug logging even if inherited from the calling shell.
|
|
281
|
+
env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
|
|
282
|
+
env['GH_PROMPT_DISABLED'] = '1'
|
|
283
|
+
response = subprocess.run(['gh', 'api', '--hostname', 'github.com', '--include', *arguments],
|
|
284
|
+
env=env, capture_output=True, timeout=60)
|
|
285
|
+
return response.stdout.decode('utf-8'), response.returncode
|
|
286
|
+
|
|
287
|
+
|
|
224
288
|
class GitHubResolver:
|
|
289
|
+
"""Verdicts for one run only: no disk cache can carry a stale or planted answer."""
|
|
225
290
|
def __init__(self, spool=None):
|
|
226
291
|
self.cache = {}
|
|
227
292
|
|
|
293
|
+
def resolve_all(self, repositories):
|
|
294
|
+
"""Resolve in GraphQL batches; a repository its batch leaves in doubt takes the REST lookup."""
|
|
295
|
+
pending = sorted(set(repositories) - self.cache.keys())
|
|
296
|
+
for start in range(0, len(pending), BATCH):
|
|
297
|
+
batch = pending[start:start + BATCH]
|
|
298
|
+
try:
|
|
299
|
+
states = bulk_visibility(batch, *gh(['graphql', *bulk_query(batch)]))
|
|
300
|
+
except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
|
|
301
|
+
states = {}
|
|
302
|
+
for repository in batch:
|
|
303
|
+
self.cache[repository] = states[repository] if repository in states else self.resolve(repository)
|
|
304
|
+
return {repository: self.cache[repository] for repository in repositories}
|
|
305
|
+
|
|
228
306
|
def resolve(self, repository):
|
|
229
307
|
if repository in self.cache:
|
|
230
308
|
return self.cache[repository]
|
|
231
|
-
# No disk cache, auth arguments, diagnostic output or response metadata.
|
|
232
|
-
# Disable gh debug logging even if inherited from the calling shell.
|
|
233
|
-
env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
|
|
234
|
-
env['GH_PROMPT_DISABLED'] = '1'
|
|
235
309
|
try:
|
|
236
|
-
|
|
237
|
-
env=env, capture_output=True, timeout=60)
|
|
238
|
-
output = response.stdout.decode('utf-8')
|
|
239
|
-
result = visibility(output, response.returncode)
|
|
310
|
+
result = visibility(*gh(['repos/' + repository]))
|
|
240
311
|
except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
|
|
241
312
|
result = 'incomplete'
|
|
242
313
|
self.cache[repository] = result
|
|
@@ -11,6 +11,7 @@ from licenses import EXCEPTIONS, exceptions, license_rules, registry
|
|
|
11
11
|
from reviewed import REVIEWED, load_reviewed
|
|
12
12
|
from report import new_report, Findings, finish
|
|
13
13
|
from tree_scan import context, text_rules, working_files, working_regular
|
|
14
|
+
from public_hygiene import reference_rules
|
|
14
15
|
from content_identities import ContentIdentities, candidate as identity_candidate
|
|
15
16
|
from checksum_maps import ChecksumMaps
|
|
16
17
|
|
|
@@ -36,7 +37,6 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
36
37
|
add(INVALID, 'edition.json', history.commit)
|
|
37
38
|
config['pendingDestination'] = None
|
|
38
39
|
recorded = registry() + exceptions(history.selected(EXCEPTIONS))
|
|
39
|
-
resolver = GitHubResolver()
|
|
40
40
|
public_blobs = {e['oid'] for c in history.public_commits for e in history.entries(c)}
|
|
41
41
|
# A file byte-identical to the same path on the public base is already public.
|
|
42
42
|
public_paths = {e['path']: e['oid'] for e in history.entries(history.public_commit)
|
|
@@ -45,6 +45,8 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
45
45
|
report['history']['newlyPublicCommits'] = [c for c in history.commits if c not in history.public_commits]
|
|
46
46
|
seen_trees, seen_blobs, inspected = set(), set(), {}
|
|
47
47
|
entries_by_commit, dependencies = {}, {}
|
|
48
|
+
# Repository references wait here so that the whole scan resolves them in bulk.
|
|
49
|
+
deferred = []
|
|
48
50
|
release_license_good = not license_rules(history.selected('LICENSE'), recorded)
|
|
49
51
|
for commit in history.commits:
|
|
50
52
|
entries = history.entries(commit)
|
|
@@ -74,11 +76,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
74
76
|
local = dependencies[pair] & config_tree['localPaths']
|
|
75
77
|
key = (path, oid, signature, local)
|
|
76
78
|
if key not in inspected:
|
|
77
|
-
inspected[key] = text_rules(history.blob(oid), path, config_tree,
|
|
78
|
-
rules, provenance = inspected[key]
|
|
79
|
+
inspected[key] = text_rules(history.blob(oid), path, config_tree, hashes)
|
|
80
|
+
rules, provenance, references = inspected[key]
|
|
79
81
|
entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
|
|
80
82
|
for rule in rules:
|
|
81
83
|
add(rule, path, commit, oid, entry_scope)
|
|
84
|
+
if references:
|
|
85
|
+
deferred.append((references, path, commit, oid, entry_scope))
|
|
82
86
|
if provenance:
|
|
83
87
|
add('provenance.declared-source', path, commit, oid, entry_scope)
|
|
84
88
|
report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
|
|
@@ -94,11 +98,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
94
98
|
# their known immutable identity; new bytes have no history identity.
|
|
95
99
|
entry = entries_by_commit[history.commit].get(path)
|
|
96
100
|
oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
|
|
97
|
-
rules, _ = text_rules(data, path, selected_context,
|
|
101
|
+
rules, _, references = text_rules(data, path, selected_context, hashes)
|
|
98
102
|
public_oid = public_paths.get(path)
|
|
99
103
|
scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
|
|
100
104
|
for rule in rules:
|
|
101
105
|
add(rule, path, history.commit, oid, scope)
|
|
106
|
+
if references:
|
|
107
|
+
deferred.append((references, path, history.commit, oid, scope))
|
|
102
108
|
if path == 'LICENSE':
|
|
103
109
|
for rule in license_rules(data, recorded):
|
|
104
110
|
add(rule, path, history.commit, oid, 'working-tree')
|
|
@@ -159,11 +165,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
159
165
|
for message_type, commit, data in history.messages():
|
|
160
166
|
path = '@' + message_type + '-message'
|
|
161
167
|
report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
|
|
162
|
-
rules, _ = text_rules(data, path, selected_context,
|
|
163
|
-
|
|
168
|
+
rules, _, references = text_rules(data, path, selected_context, hashes,
|
|
169
|
+
commit_message=message_type == 'commit')
|
|
164
170
|
scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
|
|
165
171
|
for rule in rules:
|
|
166
172
|
add(rule, path, commit, scope=scope)
|
|
173
|
+
if references:
|
|
174
|
+
deferred.append((references, path, commit, None, scope))
|
|
167
175
|
messages.append(data + b'\n')
|
|
168
176
|
length = (data + b'\n').count(b'\n')
|
|
169
177
|
lines.append((count, count + length, path, commit, scope))
|
|
@@ -172,6 +180,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
172
180
|
line = hit.get('StartLine', 0)
|
|
173
181
|
location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
|
|
174
182
|
add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
|
|
183
|
+
states = GitHubResolver().resolve_all({repository for references, *_ in deferred for repository, _, _ in references})
|
|
184
|
+
for references, path, commit, oid, scope in deferred:
|
|
185
|
+
for rule in reference_rules(references, states):
|
|
186
|
+
add(rule, path, commit, oid, scope)
|
|
175
187
|
except PublicBaseError:
|
|
176
188
|
add('scan.public-base-rejected', '.')
|
|
177
189
|
except InspectionError as error:
|
|
@@ -153,32 +153,33 @@ def without_spans(data, spans):
|
|
|
153
153
|
return b' '.join(kept)
|
|
154
154
|
|
|
155
155
|
|
|
156
|
-
def inspect(data, path, config,
|
|
156
|
+
def inspect(data, path, config, *, commit_message=False):
|
|
157
157
|
chunks = list(text_chunks(data))
|
|
158
158
|
# Image and video samples are not text: they can spell an address or a
|
|
159
159
|
# path by chance. Comments, XMP, tags and other metadata remain text.
|
|
160
160
|
spans = sample_spans(data) or []
|
|
161
161
|
# Names in a C2PA manifest's signing certificates are certificate metadata.
|
|
162
162
|
spans += certificate_spans(data)
|
|
163
|
-
rules, provenance = set(), False
|
|
163
|
+
rules, provenance, references = set(), False, set()
|
|
164
164
|
for start, end, fields in chunks:
|
|
165
165
|
spans.append((start, end))
|
|
166
166
|
for field in fields:
|
|
167
167
|
# A metadata value is text, not a standalone source manifest or
|
|
168
168
|
# trust bundle, even when the containing image has a JSON name.
|
|
169
|
-
field_rules, field_provenance = inspect_text(field, '@png-text', config,
|
|
169
|
+
field_rules, field_provenance, field_references = inspect_text(field, '@png-text', config, binary_content=False)
|
|
170
170
|
rules.update(field_rules)
|
|
171
171
|
provenance |= field_provenance
|
|
172
|
+
references |= field_references
|
|
172
173
|
# Keep the original binary classification for content outside text chunks.
|
|
173
174
|
# CRC-adjacent bytes must not also create a synthetic raw mailbox match.
|
|
174
|
-
raw_rules, raw_provenance = inspect_text(without_spans(data, spans), path, config,
|
|
175
|
-
|
|
175
|
+
raw_rules, raw_provenance, raw_references = inspect_text(without_spans(data, spans), path, config,
|
|
176
|
+
commit_message=commit_message, binary_content=binary(data))
|
|
176
177
|
rules.update(raw_rules)
|
|
177
178
|
provenance |= raw_provenance
|
|
178
|
-
return rules, provenance
|
|
179
|
+
return rules, provenance, frozenset(references | raw_references)
|
|
179
180
|
|
|
180
181
|
|
|
181
|
-
def inspect_text(data, path, config,
|
|
182
|
+
def inspect_text(data, path, config, *, commit_message=False, binary_content=None):
|
|
182
183
|
# NUL removal covers UTF-16/32 ASCII content, matching the secret scanner.
|
|
183
184
|
text = decoded_text(data)
|
|
184
185
|
rules = set()
|
|
@@ -197,21 +198,30 @@ def inspect_text(data, path, config, resolver, *, commit_message=False, binary_c
|
|
|
197
198
|
rules.add('hygiene.email')
|
|
198
199
|
ref_text, provenance = provenance_text(text, path, config.get('declaredSources', frozenset()))
|
|
199
200
|
stated, ambiguous = reference_sets(ref_text, path, config.get('localPaths', frozenset()), config.get('projectIdentity'))
|
|
200
|
-
|
|
201
|
+
references = set()
|
|
202
|
+
for repository in stated | ambiguous:
|
|
201
203
|
# A future source identity does not assert current GitHub visibility.
|
|
202
204
|
# Explicit release destinations still take the strict visibility path.
|
|
203
205
|
if (repository in config.get('intendedPublicRepositories', frozenset()) and
|
|
204
206
|
repository != config.get('pendingDestination')):
|
|
205
207
|
continue
|
|
206
|
-
|
|
208
|
+
references.add((repository, repository in ambiguous, repository == config.get('pendingDestination')))
|
|
209
|
+
return rules, provenance, frozenset(references)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def reference_rules(references, states):
|
|
213
|
+
"""Rules for (repository, ambiguous, pending destination) references once their states are known."""
|
|
214
|
+
rules = set()
|
|
215
|
+
for repository, ambiguous, pending in references:
|
|
216
|
+
state = states[repository]
|
|
207
217
|
# A bare word/word with no GitHub context names a repository only when
|
|
208
218
|
# one exists privately; that no repository answers to it proves nothing.
|
|
209
|
-
if state == 'missing' and
|
|
219
|
+
if state == 'missing' and ambiguous and not pending:
|
|
210
220
|
continue
|
|
211
|
-
if state == 'missing' and
|
|
221
|
+
if state == 'missing' and pending:
|
|
212
222
|
rules.add(PENDING_DESTINATION)
|
|
213
223
|
elif state in ('private', 'missing'):
|
|
214
224
|
rules.add('hygiene.github-' + state)
|
|
215
225
|
elif state != 'public':
|
|
216
226
|
rules.add('scan.github-incomplete')
|
|
217
|
-
return rules
|
|
227
|
+
return rules
|
|
@@ -37,15 +37,16 @@ def context(history, commit, config):
|
|
|
37
37
|
'trustBundles': trust}, invalid
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def text_rules(data, path, config,
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
def text_rules(data, path, config, env_hashes, *, commit_message=False):
|
|
41
|
+
"""Rules, provenance and the repository references whose rules wait for one bulk resolution."""
|
|
42
|
+
rules, provenance, references = inspect(data, path, config, commit_message=commit_message)
|
|
43
|
+
path_rules, _, path_references = inspect(path.encode('utf-8', 'surrogateescape'), '@path', config)
|
|
43
44
|
rules.update(path_rules)
|
|
44
45
|
if env_hashes and (environment_match(data, env_hashes) or environment_match(path.encode(), env_hashes)):
|
|
45
46
|
rules.add('secret.env-match')
|
|
46
47
|
if data.startswith(b'version https://git-lfs.github.com/spec/v1'):
|
|
47
48
|
rules.add('scan.lfs-uninspected')
|
|
48
|
-
return rules, provenance
|
|
49
|
+
return rules, provenance, references | path_references
|
|
49
50
|
|
|
50
51
|
|
|
51
52
|
def working_files(repo):
|