@runeditrun/rer 0.2.57 → 0.2.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/cli/contract.mjs CHANGED
@@ -11,7 +11,7 @@ const definitions = {
11
11
  build: [['--site'], 'Production build with private local settings excluded and restored. --site instead writes the static run site (homepage, /run, docs) to its declared folder and prints that folder.'],
12
12
  verify: [[], 'Run the complete internal verification target.'],
13
13
  reset: [['--yes'], 'Confirmed removal of the declared local inventory only.'],
14
- deploy: [['--owner-name', '--environment', '--dry-run', '--site', '--demo', '--host', '--env-file', '--image', '--bind-address', '--http-port', '--https-port', '--remove', '--yes', '--set-secret', '--unset-secret', '--rotate-secret'], 'Owner-account deployment and health; dry run performs local validation/build only. Builds and deploy hooks run with the checkout\'s local settings files moved aside, and a build output holding one (.dev.vars, .env) is refused. Declared secrets come from the environment or --env-file (an owned 0600 KEY=VALUE file); a new Worker gets the local setup owner and fresh generated secrets when they are not given. rer keeps the secrets it generates in .rer/deploy/ENVIRONMENT.secrets.env (0600) and reuses them on every redeploy; it refuses to generate a replacement for one a live Worker holds, and --rotate-secret NAME[,NAME] replaces them on purpose. After a healthy deploy rer claims the setup owner on the hosted app with the saved setup owner name and the one-time setup token, then removes the token. If the installer declares an owner-name field and no name is saved, deploy asks once and saves it; --non-interactive requires --owner-name NAME before any upload. --set-secret NAME[,NAME] (each value from the environment or --env-file; repeat the flag or join names with commas) and --unset-secret NAME[,NAME] change those secrets of the deployed Worker without a build, all of them in one new Worker version or none, and record that version; a missing value refuses before any change. --remove --yes deletes what this environment\'s deploy record says rer created (Worker, D1, R2, KV, queues), reads back that each is gone, and touches nothing else. --site deploys the run site instead: an assets-only Worker on its custom domain with www redirected; with --dry-run it builds the site and prints the plan with no provider request. --demo deploys the hosted demo instead (deploy-demo.json): a front Worker on demo.<domain> giving each visitor their own signed-in, throwaway native Worker isolate of this seed with real public reads/checks and simulated external writes; --dry-run writes it to .rer/demo-host with no provider request; --demo --remove --yes deletes that demo (its Worker, demo.<domain> Custom Domain, each only while the record its deploy left proves it the demo\'s), reads back that each is gone, and is a no-op once it is.'],
14
+ deploy: [['--owner-name', '--environment', '--dry-run', '--site', '--demo', '--host', '--env-file', '--image', '--bind-address', '--http-port', '--https-port', '--remove', '--yes', '--set-secret', '--unset-secret', '--rotate-secret'], 'Owner-account deployment and health; dry run validates/builds locally and reads target secret bindings with Cloudflare credentials, without provider writes. Missing required secrets fail as in deploy; missing credentials or unreadable target secrets also fail. Builds and deploy hooks run with the checkout\'s local settings files moved aside, and a build output holding one (.dev.vars, .env) is refused. Declared secrets come from the environment or --env-file (an owned 0600 KEY=VALUE file); a new Worker gets the local setup owner and fresh generated secrets when they are not given. rer keeps the secrets it generates in .rer/deploy/ENVIRONMENT.secrets.env (0600) and reuses them on every redeploy; it refuses to generate a replacement for one a live Worker holds, and --rotate-secret NAME[,NAME] replaces them on purpose. After a healthy deploy rer claims the setup owner on the hosted app with the saved setup owner name and the one-time setup token, then removes the token. If the installer declares an owner-name field and no name is saved, deploy asks once and saves it; --non-interactive requires --owner-name NAME before any upload. --set-secret NAME[,NAME] (each value from the environment or --env-file; repeat the flag or join names with commas) and --unset-secret NAME[,NAME] change those secrets of the deployed Worker without a build, all of them in one new Worker version or none, and record that version; a missing value refuses before any change. --remove --yes deletes what this environment\'s deploy record says rer created (Worker, D1, R2, KV, queues), reads back that each is gone, and touches nothing else. --site deploys the run site instead: an assets-only Worker on its custom domain with www redirected; with --dry-run it builds the site and prints the plan with no provider request. --demo deploys the hosted demo instead (deploy-demo.json): a front Worker on demo.<domain> giving each visitor their own signed-in, throwaway native Worker isolate of this seed with real public reads/checks and simulated external writes; --dry-run writes it to .rer/demo-host with no provider request; --demo --remove --yes deletes that demo (its Worker, demo.<domain> Custom Domain, each only while the record its deploy left proves it the demo\'s), reads back that each is gone, and is a no-op once it is.'],
15
15
  export: [['--target', '--output', '--origin', '--session-file'], 'Complete supported archive of the local copy on this computer, with integrity and exclusions, written to --output. The hosted install cannot be exported yet: --origin or --target remote refuses (exit 2) before any effect and names the command that saves the hosted database.'],
16
16
  import: [['--target', '--input', '--journal', '--origin', '--session-file', '--yes', '--legacy-v1', '--acknowledge-unverified-metadata', '--output'], 'Confirmed atomic restore into an empty destination; --journal LATER_ARCHIVE also fences rows that installation erased since. Backup-only local v1 recovery requires --legacy-v1 --acknowledge-unverified-metadata and a new --output v2 archive; --journal requires normal v2 import.'],
17
17
  doctor: [[], 'Read-only installed tool, configuration and state diagnostics.'],
@@ -4,6 +4,7 @@ import { readFile, readdir } from 'node:fs/promises';
4
4
  import { contained } from './project.mjs';
5
5
  import { invalid, refuse, RerError } from './errors.mjs';
6
6
  import { ownerDeploy } from './deploy.mjs';
7
+ import { secretFailureMessage } from './deploy-secret-error.mjs';
7
8
  import { deployEnvironment, deployAgain } from './deploy-inputs.mjs';
8
9
  import { claimHostedOwner } from './deploy-owner.mjs';
9
10
  import { deployOwnerInput } from './deploy-owner-input.mjs';
@@ -78,11 +79,11 @@ export async function deployCommand(context) {
78
79
  if (local) throw refuse(`${path} names the local-only key ${local} as a deploy binding or secret. Remove it; local dev keys never reach a deployment.`);
79
80
  deployGuide(declaration);
80
81
  const settings = adapter.hooks?.secrets ?? adapter.settings;
81
- // What a dry run went ahead without, listed when it completes.
82
+ // Required secrets have been checked; retain the production sign-in guidance.
82
83
  const dryRun = async result => {
83
84
  const production = await deployEnvironment({ root, options, declaration, setupManifest, settings, localOnlyKeys });
84
85
  const { signIn } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
85
- const needed = neededForRealDeploy({ declaration, env: production.env, signIn });
86
+ const needed = neededForRealDeploy({ declaration, env: production.env, signIn, secretsChecked: true });
86
87
  reportNeeded(store, needed);
87
88
  return needed.length && result && typeof result === 'object' ? { ...result, neededForRealDeploy: needed } : result;
88
89
  };
@@ -134,7 +135,10 @@ export async function deployCommand(context) {
134
135
  }
135
136
  hookReturned = true;
136
137
  await refuseLocalEnvOutput(root, adapter.devIdentity?.output, { uploaded: !options.dryRun });
137
- if (options.dryRun) return dryRun(deployed);
138
+ if (options.dryRun) {
139
+ await command('dry-run');
140
+ return dryRun(deployed);
141
+ }
138
142
  await rememberHookState(root, environment, deployed?.state);
139
143
  hookRecord = await hookState(root, environment);
140
144
  const origin = hookOrigin(deployed, deployAgain(options));
@@ -163,6 +167,9 @@ export async function deployCommand(context) {
163
167
  } catch (error) {
164
168
  const again = deployAgain(options);
165
169
  const { commandFailure } = await shared('failures.mjs');
170
+ const secrets = secretFailureMessage(error);
171
+ if (secrets) throw commandFailure(secrets);
172
+ if (options.dryRun && error.message === 'missing-input') throw new RerError('missing_input', `Dry run cannot read the target secret bindings without ${error.inputs.join(' and ')}. Set the Cloudflare credentials in the environment or an owned 0600 --env-file, then run the same command again.`, 3);
166
173
  if (error.hookFailure) throw commandFailure(store.redact?.(redaction.redact(error.message)) ?? redaction.redact(error.message));
167
174
  if (error instanceof RerError && !deployed?.deployed && !recorded) { error.message = store.redact?.(redaction.redact(error.message)) ?? redaction.redact(error.message); throw error; }
168
175
  const outcome = error.owner ?? claimed ?? observedClaims.at(-1)?.outcome;
@@ -29,22 +29,24 @@ function credentialHelp(declaration) {
29
29
  return [`Create an account API token at ${tokenPage} with these account permissions: ${needed.join(', ')}. Copy your account ID from the Cloudflare dashboard.`];
30
30
  }
31
31
 
32
- export function missingCredentialsMessage(declaration, env) {
32
+ export function missingCredentialsMessage(declaration, env, { dryRun = false, signIn } = {}) {
33
33
  const absent = cloudflareCredentials.filter(name => !env[name]?.trim());
34
- return [`${absent.join(' and ')} ${absent.length > 1 ? 'are' : 'is'} required to deploy to your Cloudflare account. Nothing was deployed.`,
34
+ return [...(dryRun ? ['Dry run cannot read the target secret bindings without Cloudflare credentials.'] : []),
35
+ `${absent.join(' and ')} ${absent.length > 1 ? 'are' : 'is'} required to deploy to your Cloudflare account. Nothing was deployed.`,
35
36
  ...credentialHelp(declaration),
36
- 'Set them in your environment or an owned 0600 --env-file, then run the same deploy command again.'].join('\n');
37
+ 'Set them in your environment or an owned 0600 --env-file, then run the same deploy command again.',
38
+ ...(dryRun ? ['Needed for a real deploy (the dry run could not check the target):', ...neededForRealDeploy({declaration,env,signIn}).map(item => ` - ${item}`)] : [])].join('\n');
37
39
  }
38
40
 
39
- // What a dry run went ahead without: the Cloudflare values, each required
40
- // declared secret not supplied, and production sign-in's missing mail inputs.
41
- export function neededForRealDeploy({ declaration, env, signIn }) {
41
+ // Setup guidance and failed dry-run inputs. After target resolution, held
42
+ // secrets no longer count as missing local inputs.
43
+ export function neededForRealDeploy({ declaration, env, signIn, secretsChecked = false }) {
42
44
  const declared = (Array.isArray(declaration.secrets) ? declaration.secrets : []).filter(item => item?.environment);
43
45
  // Sign-in names its inputs by binding; a person sets the environment name the declaration maps to it.
44
46
  const mail = signIn && declared.reduce((text, item) => item.binding === item.environment ? text : text.replace(new RegExp(`\\b${item.binding}\\b`, 'g'), item.environment), signIn);
45
47
  const listed = name => cloudflareCredentials.includes(name) || mail && new RegExp(`\\b${name}\\b`).test(mail);
46
- const secrets = [...new Set(declared.filter(item => !item.optional && !env[item.environment]?.trim()).map(item => item.environment))].filter(name => !listed(name));
47
- const credentials = cloudflareCredentials.filter(name => !env[name]?.trim());
48
+ const secrets = secretsChecked ? [] : [...new Set(declared.filter(item => !item.optional && !env[item.environment]?.trim()).map(item => item.environment))].filter(name => !listed(name));
49
+ const credentials = secretsChecked ? [] : cloudflareCredentials.filter(name => !env[name]?.trim());
48
50
  return [
49
51
  ...(credentials.length ? [`${credentials.join(' and ')} (${credentialHelp(declaration).join(' ')})`] : []),
50
52
  ...secrets,
@@ -0,0 +1,8 @@
1
+ // Both deploy paths render the shared resolver's complete missing-input list.
2
+ export function secretFailureMessage(error) {
3
+ if (error.message === 'token-scope-missing-workers') return `Cannot read the target's secret list; required secret bindings could not be checked. The Cloudflare token needs Workers Scripts permission (${error.message}). Nothing was uploaded.`;
4
+ const first = /^missing-secret-([A-Z][A-Z0-9_]*)$/.exec(error.message)?.[1];
5
+ if (!first) return;
6
+ const names = error.missingSecrets?.map(({binding,environment}) => binding === environment ? binding : `${binding} (${environment})`) ?? [first];
7
+ return `Deploy needs ${names.join(', ')}, which the deploy declaration lists as ${names.length === 1 ? 'a secret' : 'secrets'} and the Worker does not hold. Set ${names.length === 1 ? 'its value' : 'their values'} in the named environment inputs or in an --env-file file (an owned 0600 file of KEY=VALUE lines), then run the same command again. Nothing was uploaded.`;
8
+ }
package/cli/deploy.mjs CHANGED
@@ -5,8 +5,8 @@ import { shared } from './shared.mjs';
5
5
  import { settingKinds } from './deploy-settings.mjs';
6
6
  import { contained } from './project.mjs';
7
7
  import { missing, invalid, refuse, RerError } from './errors.mjs';
8
- import { ownerPlan, ownerWorkerName, localOnlyBinding } from '../modules/deploy-kit/owner.mjs';
9
- import { packageBuild } from '../modules/deploy-kit/prepare.mjs';
8
+ import { localOnlyBinding } from '../modules/deploy-kit/owner.mjs';
9
+ import { secretFailureMessage } from './deploy-secret-error.mjs';
10
10
  import { refuseLocalModels } from '../modules/deploy-kit/local-only.mjs';
11
11
  import { deployEnvironment, deployAgain } from './deploy-inputs.mjs';
12
12
  import { claimHostedOwner } from './deploy-owner.mjs';
@@ -38,31 +38,20 @@ export function ownerDeploy(declarationPath, lifecycle) {
38
38
  const recordedState = change.length ? await deploymentRecord(root, options) : undefined;
39
39
  // A dry run reads the settings file too, so a refused key fails here and not only on the real deploy.
40
40
  if (options.dryRun && options.envFile) await deployEnvironment({ root, options, declaration, localOnlyKeys });
41
- let production, ownerInput;
41
+ let production, ownerInput, signIn;
42
42
  if (!change.length) {
43
43
  production = await deployEnvironment({ root, options, declaration, setupManifest, settings: lifecycle.settings, localOnlyKeys });
44
- await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed, adapter: lifecycle, declarationPath, options });
44
+ ({signIn} = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed, adapter: lifecycle, declarationPath, options }) ?? {});
45
45
  if (!options.dryRun) committedSource(root, options);
46
46
  ownerInput = await deployOwnerInput({ root, adapter, setupSeed, options, signal });
47
47
  await withoutLocalSettings(root, { settings: lifecycle.settings, operationToken }, () => lifecycle.run(root, {}, store, () => {}, signal, { verb: 'build', operationToken }));
48
48
  await checkBuild();
49
49
  await refuseLocalEnvOutput(root);
50
50
  }
51
- if (options.dryRun) {
52
- // Locally validate the real declaration/artifact without ownerTarget's
53
- // account lookup. This synthetic target is validation data, not a provider.
54
- const worker = ownerWorkerName(declaration.project, environment);
55
- const artifact = await packageBuild({ root, source: { sha: '0'.repeat(40), tree: '0'.repeat(40) }, runIdentity: { id: '0', attempt: '1' }, productDirectory: '' });
56
- ownerPlan({ project: declaration.project, environment, worker, origin: `https://${worker}.example.invalid` }, declaration, artifact, { localOnlyKeys, env: production.env });
57
- const optional = (Array.isArray(declaration.secrets) ? declaration.secrets : []).filter(item => item?.optional && !production.env[item.environment]?.trim()).map(item => item.binding);
58
- if (optional.length) store.report(`deploy: optional secrets not supplied: ${optional.join(', ')}. A Worker that does not hold one deploys without it.`);
59
- store.report('deploy: dry run complete; no deployment occurred and no provider request was made.');
60
- return { dryRun: true, deployed: false };
61
- }
62
51
  const { env, supplied, generated, reused, rotated } = await deployEnvironment({ root, options, declaration, setupManifest, settings: lifecycle.settings, localOnlyKeys });
63
52
  if (options.setSecret || options.unsetSecret) secretChangeInputs(declaration, options, env, again);
64
53
  const kept = keptSecrets(declaration, setupManifest);
65
- if (cloudflareCredentials.some(name => !env[name]?.trim())) throw missing(missingCredentialsMessage(declaration, env));
54
+ if (cloudflareCredentials.some(name => !env[name]?.trim())) throw missing(missingCredentialsMessage(declaration, env, {dryRun:options.dryRun,signIn}));
66
55
  const { main } = await import('../modules/deploy-kit/owner-cli.mjs');
67
56
  const { createRedaction } = await shared('redaction.mjs');
68
57
  const redaction = createRedaction();
@@ -74,6 +63,12 @@ export function ownerDeploy(declarationPath, lifecycle) {
74
63
  const created = [];
75
64
  let step = 'deploy', deployed;
76
65
  try {
66
+ if (options.dryRun) {
67
+ const {optionalUnset} = await owner('dry-run');
68
+ if (optionalUnset.length) store.report(`deploy: optional secrets not supplied: ${optionalUnset.join(', ')}. A Worker that does not hold one deploys without it.`);
69
+ store.report('deploy: dry run complete; target secret bindings checked; no deployment occurred.');
70
+ return { dryRun: true, deployed: false };
71
+ }
77
72
  // A custom hook's own state file, when its deploy named one.
78
73
  const state = change.length ? recordedState : await hookState(root, environment);
79
74
  if (options.remove) {
@@ -164,8 +159,8 @@ function failureText({ error, step, created, again, retry, secretChange, deploye
164
159
  const diagnostic = deployDiagnostic(error, redact);
165
160
  const code = diagnostic?.code ?? (/^[a-zA-Z0-9_-]{1,120}$/.test(error.message) ? redact(error.message) : 'deploy-failed');
166
161
  const at = step === 'deploy' ? diagnostic?.step ?? 'prepare deploy' : step;
167
- const input = /^missing-secret-([A-Z][A-Z0-9_]*)$/.exec(code)?.[1];
168
- if (input) return `Deploy needs ${input}, which the deploy declaration lists as a secret and the Worker does not hold it. Set ${input} in the environment or in an --env-file file (an owned 0600 file of KEY=VALUE lines), then run the same command again. Nothing was uploaded.${created.length ? ` Resources created this attempt: ${created.join(', ')}.` : ''}`;
162
+ const secrets = secretFailureMessage(error);
163
+ if (secrets) return secrets;
169
164
  if (code === 'nothing-to-remove') return `The deployment record selected by ${again} has no resources that rer owns, so --remove deletes nothing.`;
170
165
  if (code === 'owner-token-retirement-failed') return `The application is deployed at ${deployed?.origin}; owner ${error.owner}; setup token retirement failed (${redact(String(error.cause))}${providerFailure(error,redact) ? `; ${providerFailure(error,redact)}` : ''}). Run ${again} --unset-secret ${error.tokenKey ?? 'SETUP_TOKEN'} using the same environment credentials, or run ${again} again to finish.`;
171
166
  if (code === 'owner-claim-failed') return `The application is deployed at ${deployed?.origin}, but claiming its owner failed: ${redact(String(error.cause))}. Owner claim and setup token retirement are not confirmed; run ${again} again using the same environment credentials to finish.`;
@@ -1,5 +1,14 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.59
4
+
5
+ - **Release publication scans complete on large seeds and partial clones.** `rer release` hydrates missing history blobs in one fetch and inspects the staged release annotation in the same scan. Orbit finishes in 82 seconds instead of exceeding the 300-second budget, with identical findings. A seed must adopt the new rer through its pin for its release command to use it (#278).
6
+ - **Native demos decode binary modules when needed.** WebAssembly and data modules decode lazily inside the Worker Loader factory with native base64 support, reducing host startup work while preserving their bytes (#280).
7
+
8
+ ## 0.2.58
9
+
10
+ - **Deploy dry runs validate required target secrets.** `rer deploy --dry-run` uses the real deploy secret-binding resolver and refuses with the same exit and message when required bindings are missing. It names every missing binding and its environment input, accepts secrets already held by the Worker, and makes no target writes. Missing credentials or read permission also refuse (#275).
11
+
3
12
  ## 0.2.57
4
13
 
5
14
  - **Large release scans finish within the publication budget.** The JSON identity pass tracks UTF-8 byte offsets incrementally. Orbit's scan finishes in 79 seconds instead of exceeding the 300-second deadline, with identical findings (#270).
@@ -1,16 +1,21 @@
1
1
  import { WorkerEntrypoint } from 'cloudflare:workers';
2
+ import { Buffer } from 'node:buffer';
2
3
  import demoEgress from '../../container/demo-egress.mjs';
3
4
  import bundle from './bundle.mjs';
4
5
 
5
- // wasm and data modules arrive as base64; the Loader takes their bytes.
6
- const bytes = text => Uint8Array.from(atob(text), char => char.charCodeAt(0)).buffer;
7
- const modules = Object.fromEntries(Object.entries(bundle.modules).map(([name, module]) =>
6
+ // Decode only when a Loader factory needs the modules, once per host isolate.
7
+ const bytes = text => {
8
+ const buffer = Buffer.from(text, 'base64');
9
+ return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
10
+ };
11
+ let modules;
12
+ const loaderModules = () => modules ??= Object.fromEntries(Object.entries(bundle.modules).map(([name, module]) =>
8
13
  [name, 'wasm' in module ? { wasm: bytes(module.wasm) } : 'data' in module ? { data: bytes(module.data) } : module]));
9
14
 
10
15
  export function loaded(env, ctx, id, runtime) {
11
16
  return env.LOADER.get(`${bundle.digest}:${id}`, () => ({
12
17
  compatibilityDate: bundle.compatibilityDate, compatibilityFlags: bundle.compatibilityFlags,
13
- mainModule: '_rer/isolate.js', modules,
18
+ mainModule: '_rer/isolate.js', modules: loaderModules(),
14
19
  globalOutbound: ctx.exports.DemoEgress({ props: { id } }),
15
20
  env: { ...runtime, RER_NATIVE_CONFIG: bundle.bindings, RER_NATIVE_HOST: ctx.exports.HostBridge({ props: { id, key: runtime.RER_NATIVE_KEY } }), [bundle.bindings.assets]: ctx.exports.DemoAssets({}) },
16
21
  }));
@@ -115,6 +115,8 @@ A transport failure carries `providerTransport`: the operation, sanitized endpoi
115
115
 
116
116
  `rer deploy` runs these steps for a seed whose adapter uses `ownerDeploy` from `cli/deploy.mjs`.
117
117
 
118
+ - **Dry run.** Application `rer deploy --dry-run` builds and validates locally, then calls `owner-cli.mjs dry-run` to read the selected target and resolve required secrets with the same `secretBindings` function as deploy. All missing required bindings and their environment names are reported together, with identical nonzero exit and error in human and JSON output. Held secrets are inherited, supplied values are accepted without upload, absent optional secrets stay unset, and recorded retired secrets stay retired. Cloudflare credentials are required; a failed secret-list read fails the command. Dry run does not settle pending secret changes, write deployment state, upload, change secrets or create resources. An unsettled secret change must be completed by its original command first.
119
+
118
120
  - **Inputs.** Declared secrets come from the environment, then `--env-file FILE` (an owned 0600 file of `KEY=VALUE` lines), which wins. For a secret the Worker does not hold yet, rer supplies the setup manifest's owner email (`emailField`) from the local setup's private settings, and for each secret field the manifest generates (`generate`) the value rer keeps for this environment in `.rer/deploy/<environment>.secrets.env` (0600, ignored by Git), else a fresh random value, which rer keeps once the Worker holds it (the one-time setup token is always fresh and never kept); a value the Worker already holds is kept. The build runs with the checkout's local settings files moved aside, and a `dist/` holding a `.dev.vars*`, `.env*` or `*.env` file is refused by name before any provider request. A still-missing secret stops the run before any write with a message that names it, unless its declaration says `"optional": true`: then the deploy binds nothing for it and reports it as not set, and `--set-secret` adds it later.
119
121
  - **Hosted owner.** After deploy and health pass, rer claims the setup owner on the hosted app: it POSTs the setup token to the seed's standard owner step (`hooks.owner`, by default `/auth/bootstrap`). 303 means claimed; `409 already_initialized` means an owner was already claimed and is kept. Then rer deletes the token secret from the Worker, records the new Worker version in the state file, and lists the token in `retiredSecrets`, so later plans no longer declare it. The build fingerprint still names it, and the recorded claim stays as recorded, so an untouched repeat deploy writes nothing. A Worker that still holds a token rer did not upload in this run (an older deploy) first gets a fresh token through the same recorded secret change. Dependency approvals that need the token run before the claim. A seed with an owner step function, an identity-provider claim, or no declared token secret claims its owner its own way.
120
122
  - **Several secrets at once.** `rer deploy --set-secret MAIL_API_KEY --set-secret MAIL_FROM` or `rer deploy --set-secret MAIL_API_KEY,MAIL_FROM` (likewise `--unset-secret A,B`) changes all of them in one new Worker version or none: a group of secrets is never half set. Every named secret must be declared and, for `--set-secret`, have its value in the environment or `--env-file`; otherwise rer names each missing one and changes nothing. One secret uses Cloudflare's secret call. Several are one copy of the live version through the Versions API (its own modules, assets and compatibility settings, every other binding inherited), then deployed at 100%: Cloudflare's per-secret calls each make a version, and its settings call can add secrets but not remove them (measured 17 October 2026). The copy carries a marker rer records first, so a copy whose answer was lost is recognised, and one uploaded but not deployed is deployed by the same command run again.
@@ -8,8 +8,8 @@ import { assert, hexId } from './validation.mjs';
8
8
  import { regularBytes } from './artifact.mjs';
9
9
  import { packageBuild } from './prepare.mjs';
10
10
  import { cloudflare } from './provider.mjs';
11
- import { ownerTarget, ownerWorkerName, ownerPlan, ownerDeploy, ownerHealth, settleOwnerState, sameDeployment, localOnlyBinding, withStateLock, readOwnerState, writeOwnerState } from './owner.mjs';
12
- import { activeWorker } from './worker.mjs';
11
+ import { ownerTarget, ownerWorkerName, ownerPlan, ownerDeploy, ownerHealth, settleOwnerState, sameDeployment, localOnlyBinding, withStateLock, readOwnerState, writeOwnerState, workerSettings, withScopeHints } from './owner.mjs';
12
+ import { activeWorker, secretBindings } from './worker.mjs';
13
13
  import { ownerSecret } from './owner-secret.mjs';
14
14
  import { ownerRemove } from './owner-remove.mjs';
15
15
  import { ownerSignin } from './owner-signin.mjs';
@@ -17,7 +17,7 @@ import { refuseLocalModels } from './local-only.mjs';
17
17
  import { publicOrigin } from './public-origin.mjs';
18
18
 
19
19
  export const cronNotice = 'This Worker has a Cron Trigger. Cloudflare may take up to about two hours to start its first scheduled run on some accounts.';
20
- const usage = 'usage: owner-cli.mjs deploy|health|remove|inspect|record-owner|signin --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
20
+ const usage = 'usage: owner-cli.mjs deploy|dry-run|health|remove|inspect|record-owner|signin --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
21
21
 
22
22
  /** The owner's committed source and existing local build. Run the seed's build first. */
23
23
  async function ownerArtifact(root, product) {
@@ -33,7 +33,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
33
33
  try { parsed = parseArgs({args:argv,allowPositionals:true,strict:true,options:{declaration:{type:'string'},environment:{type:'string'},product:{type:'string',default:''},state:{type:'string'},set:{type:'string',multiple:true},unset:{type:'string',multiple:true},outcome:{type:'string'}}}); }
34
34
  catch { throw new Error('usage'); }
35
35
  const {positionals:[command,...rest],values} = parsed;
36
- if (!['deploy','health','secret','remove','inspect','record-owner','signin'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
36
+ if (!['deploy','dry-run','health','secret','remove','inspect','record-owner','signin'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
37
37
  if ((command === 'secret') !== (values.set !== undefined || values.unset !== undefined) || values.set !== undefined && values.unset !== undefined) throw new Error('usage');
38
38
  if (values.outcome !== undefined && command !== 'record-owner' || command === 'record-owner' && !['claimed','kept'].includes(values.outcome)) throw new Error('usage');
39
39
  await refuseLocalModels(resolve(cwd));
@@ -47,6 +47,12 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
47
47
  assert(typeof declaration?.project === 'string','missing-owner-project');
48
48
  // Validate even direct kit invocations before any provider request or write.
49
49
  publicOrigin(declaration, env);
50
+ let artifact;
51
+ if (command === 'dry-run') {
52
+ artifact = await packageBuild({root,source:{sha:'0'.repeat(40),tree:'0'.repeat(40)},runIdentity:{id:'0',attempt:'1'},productDirectory:values.product});
53
+ const worker = ownerWorkerName(declaration.project,values.environment);
54
+ ownerPlan({project:declaration.project,environment:values.environment,worker,origin:`https://${worker}.example.invalid`},declaration,artifact,{localOnlyKeys,env});
55
+ }
50
56
  // Removal reads identity before each object DELETE and may span many pages.
51
57
  // Bound each response, not the owner's bucket size; failures retain ownership.
52
58
  let requestWindow = propagation?.uploaded ? propagation.window : undefined;
@@ -74,7 +80,9 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
74
80
  return {ok:true,...await ownerSecret({api,target,statePath,changes,fetch,healthOptions}).catch(error => { throw api.annotateFailure(error,'secrets'); })};
75
81
  }
76
82
  // A secret rer retired (the spent setup token) is no longer part of the plan.
77
- const state = await settleOwnerState({api,target,statePath});
83
+ // A dry run only reads state: settling an interrupted change may write it.
84
+ const state = command === 'dry-run' ? await readOwnerState(statePath,target) : await settleOwnerState({api,target,statePath});
85
+ if (command === 'dry-run') assert(!state.pendingSecret, 'secret-change-unsettled');
78
86
  const {retiredSecrets = []} = state;
79
87
  if (command === 'record-owner') return withStateLock(statePath, async () => {
80
88
  const current = await readOwnerState(statePath, target);
@@ -97,8 +105,14 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
97
105
  return {ok:true, origin:target.origin, versionId:live.versionId, retiredSecrets, ownerClaim:state.ownerClaim, liveSecrets:rows.map(row => row.name)};
98
106
  }
99
107
  const current = retiredSecrets.length && Array.isArray(declaration.secrets) ? {...declaration,secrets:declaration.secrets.filter(item => !retiredSecrets.includes(item?.binding))} : declaration;
100
- const artifact = await ownerArtifact(root,values.product);
108
+ artifact ??= await ownerArtifact(root,values.product);
101
109
  const plan = ownerPlan(target,current,artifact,{localOnlyKeys,env});
110
+ if (command === 'dry-run') {
111
+ const checked = withScopeHints(api);
112
+ const exists = await workerSettings(checked,target.worker) !== null;
113
+ const {unset} = await secretBindings(checked,target.worker,plan,env,exists);
114
+ return {ok:true,dryRun:true,deployed:false,optionalUnset:unset};
115
+ }
102
116
  if (command === 'health') {
103
117
  const readiness = {...(healthOptions?.now ? {clock:healthOptions.now} : {}), ...(healthOptions?.sleep ? {sleep:healthOptions.sleep} : {}),
104
118
  ...healthOptions?.readiness,single:true,totalMs:30000,
@@ -130,10 +144,10 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
130
144
  /** Prints refusal codes and resource names only; never inputs, tokens or provider bodies. */
131
145
  export function report(error) {
132
146
  const diagnostic = deployDiagnostic(error);
133
- if (diagnostic) return {ok:false,...diagnostic};
147
+ if (diagnostic) return {ok:false,...diagnostic,...(error.missingSecrets ? {missingSecrets:error.missingSecrets} : {})};
134
148
  if (error.message === 'missing-input') return {ok:false,code:'missing-input',inputs:error.inputs};
135
149
  if (error.message === 'usage') return {ok:false,code:'usage',usage};
136
- return {ok:false,code:/^[a-zA-Z0-9_-]{1,120}$/.test(error.message) ? error.message : 'owner-deploy-failed',...(error.resource ? {resource:error.resource} : {})};
150
+ return {ok:false,code:/^[a-zA-Z0-9_-]{1,120}$/.test(error.message) ? error.message : 'owner-deploy-failed',...(error.resource ? {resource:error.resource} : {}),...(error.missingSecrets ? {missingSecrets:error.missingSecrets} : {})};
137
151
  }
138
152
  if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
139
153
  try { console.log(JSON.stringify(await main())); }
@@ -49,10 +49,16 @@ export async function secretBindings(api, worker, plan, env, exists) {
49
49
  const supplied = item => typeof env[item.environment] === 'string' && env[item.environment].trim();
50
50
  // An optional secret neither held nor supplied stays unbound; the app treats it as not connected.
51
51
  const unset = plan.secrets.filter(item => item.optional && !names.includes(item.binding) && !supplied(item)).map(item => item.binding);
52
+ const missingSecrets = plan.secrets.filter(item => !item.optional && !names.includes(item.binding) && !supplied(item))
53
+ .map(({binding,environment}) => ({binding,environment}));
54
+ if (missingSecrets.length) {
55
+ const error = new Error(`missing-secret-${missingSecrets[0].environment}`);
56
+ error.missingSecrets = missingSecrets;
57
+ throw error;
58
+ }
52
59
  for (const item of plan.secrets) {
53
60
  if (names.includes(item.binding)) bindings.push({type:'inherit',name:item.binding});
54
61
  else if (!unset.includes(item.binding)) {
55
- assert(supplied(item), `missing-secret-${item.environment}`);
56
62
  bindings.push({type:'secret_text',name:item.binding,text:env[item.environment]});
57
63
  }
58
64
  }
@@ -5,7 +5,7 @@ import { inspectSource, newOutput, git } from './source.mjs';
5
5
  import { verifySource } from './source-verification.mjs';
6
6
  import { catalogue, writeCatalogue, sha256, json } from './catalogue.mjs';
7
7
  import { publicExport, cleanHistory, annotate, releaseMessage } from './export.mjs';
8
- import { publicationScan, validatePublicationDestination } from './publication-scan.mjs';
8
+ import { publicationScan, sourcePublicationScan, validatePublicationDestination } from './publication-scan.mjs';
9
9
  import { publicDestination, requirePublicDestination, remotePublicHead, publicChild, assertPublicTree } from './public-child.mjs';
10
10
  import { writePublishHandoff } from './publish-handoff.mjs';
11
11
  import { releasePublicBase } from './public-base.mjs';
@@ -53,28 +53,9 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
53
53
  // when available, retaining the selected source commit and tag identity.
54
54
  const exported = destination ? publicChild(source, temporary, destination, publicParent, Boolean(tag))
55
55
  : clean ? cleanHistory(source, temporary, Boolean(tag)) : null;
56
- const scanRepo = exported?.repo ?? source.repo;
57
- const scanRef = exported?.commit ?? source.commit;
58
- let report = publicationScan(scanRepo, scanRef, envFile, project, repository, scanBase);
59
- if (!exported && tag && report.status === 'PASS') {
60
- // Stage the exact final annotation before any source tag or distributable
61
- // artifact exists, in a fresh clone of the selected commit: verification
62
- // left build output in its checkout, which is not release source. Source
63
- // history was scanned above, including refs a checkout clone leaves out.
64
- const candidate = resolve(temporary, 'annotation');
65
- git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
66
- git(candidate, 'checkout', '--detach', source.commit);
67
- const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
68
- annotate(candidate, source, source.commit, staged ? `${source.tag}-publication-candidate` : source.tag);
69
- const annotations = publicationScan(candidate, source.commit, envFile, project, repository, scanBase);
70
- report.annotationInspection = annotations;
71
- if (annotations.status !== 'PASS') {
72
- report.status = annotations.status;
73
- report.complete = annotations.complete;
74
- report.guarantees.historyScanned = annotations.complete;
75
- report.findings.push(...annotations.findings);
76
- }
77
- }
56
+ let report = exported
57
+ ? publicationScan(exported.repo, exported.commit, envFile, project, repository, scanBase)
58
+ : sourcePublicationScan(source, temporary, envFile, project, repository, scanBase, Boolean(tag));
78
59
  if (destination) {
79
60
  assertPublicTree(exported.repo, exported.commit, source.tree);
80
61
  if (report.status === 'PASS' && (report.complete !== true || report.guarantees?.historyScanned !== true ||
@@ -1,5 +1,9 @@
1
1
  import { execFileSync } from 'node:child_process';
2
2
  import { resolve } from 'node:path';
3
+ import { writeFileSync } from 'node:fs';
4
+ import { git } from './source.mjs';
5
+ import { annotate } from './export.mjs';
6
+ import { runningVersion } from '../../cli/version.mjs';
3
7
 
4
8
  export function validatePublicationDestination(source, project, repository) {
5
9
  try {
@@ -9,14 +13,42 @@ export function validatePublicationDestination(source, project, repository) {
9
13
  } catch { throw Error('Publication destination must agree with the committed edition and project identities'); }
10
14
  }
11
15
 
12
- export function publicationScan(repo, ref, envFile, project, repository, publicBase) {
16
+ export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
17
+ const started = performance.now();
18
+ let report;
13
19
  try {
14
- return JSON.parse(execFileSync(process.execPath,
20
+ report = JSON.parse(execFileSync(process.execPath,
15
21
  [resolve(import.meta.dirname, '../../scripts/publish-check.mjs'), '--repo', repo, '--ref', ref,
16
- ...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : [])],
22
+ ...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : []),
23
+ ...(annotationFile ? ['--annotation-file', annotationFile] : [])],
17
24
  { stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 128 * 1024 * 1024 }));
18
25
  } catch (error) {
19
- try { return JSON.parse(error.stdout); }
26
+ try { report = JSON.parse(error.stdout); }
20
27
  catch { throw Error('Publication inspection did not produce a report'); }
21
28
  }
29
+ report.invocation = { caller: 'release', scans: 1, stagedAnnotation: Boolean(annotationFile), version: runningVersion(),
30
+ seconds: (performance.now() - started) / 1000 };
31
+ return report;
32
+ }
33
+
34
+ // This is the ordinary release gate, also usable without installing/building a
35
+ // seed. The scanner sees the original checkout and every source ref; only the
36
+ // new annotation is read from the clean candidate, never verification output.
37
+ export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged) {
38
+ const started = performance.now();
39
+ let annotationFile;
40
+ if (tagged) {
41
+ const candidate = resolve(temporary, 'annotation');
42
+ git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
43
+ git(candidate, 'checkout', '--detach', source.commit);
44
+ const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
45
+ const name = staged ? `${source.tag}-publication-candidate` : source.tag;
46
+ annotate(candidate, source, source.commit, name);
47
+ annotationFile = resolve(temporary, 'annotation.tag');
48
+ writeFileSync(annotationFile, git(candidate, 'cat-file', 'tag', `refs/tags/${name}`), { mode: 0o600 });
49
+ }
50
+ const stagingSeconds = (performance.now() - started) / 1000;
51
+ const report = publicationScan(source.repo, source.commit, envFile, project, repository, publicBase, annotationFile);
52
+ Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
53
+ return report;
22
54
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.57",
3
+ "version": "0.2.59",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -4,6 +4,20 @@ See [setup, command and scope](../publish-check-README.md).
4
4
 
5
5
  `git_history.py` delegates revision resolution, traversal, tree inventory and
6
6
  blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
7
+ Before any tree inventory asks for blob sizes, `history_hydration.py` counts
8
+ missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
9
+ from its promisor remote. A failed fetch or remaining missing object makes the
10
+ inspection incomplete. `history.hydration` reports the missing counts, fetch
11
+ count and elapsed time; subsequent history reads disable lazy fetching.
12
+
13
+ Reports include implementation version, Python version and a digest of the
14
+ Python scanner source, plus total and per-phase wall times and call counts.
15
+ The release gate uses this same scanner entry point. When comparing it with a
16
+ standalone run, compare implementation digests as well as source trees:
17
+ `rer` inside a checkout runs the rer version that checkout pins, even when a
18
+ newer launcher started it. A tagged exact-source release stages its actual Git tag object in an isolated
19
+ clone and includes its message in the same full source scan. `annotationInspection`
20
+ is the staged annotation subset of that result; no second history walk is needed.
7
21
  `gitleaks.py` runs the pinned scanner in git, directory and stdin modes and keeps
8
22
  its report private until translating it into the value-free v3 envelope.
9
23
  Hexadecimal credential assignments also use a Gitleaks rule without generic
@@ -20,12 +20,12 @@ def main():
20
20
  parser = SafeParser(description='Scan working files and full history with system Git and pinned Gitleaks.')
21
21
  parser.add_argument('--repo', required=True)
22
22
  parser.add_argument('--ref', default='HEAD')
23
- for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir'):
23
+ for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir', 'annotation-file'):
24
24
  parser.add_argument('--' + option)
25
25
  parser.add_argument('--timeout-seconds', type=seconds, default=300)
26
26
  args = parser.parse_args()
27
27
  report = scan(args.repo, args.ref, args.public_base, args.env_file, args.project, args.repository,
28
- cache_dir=args.cache_dir, timeout=args.timeout_seconds)
28
+ cache_dir=args.cache_dir, timeout=args.timeout_seconds, annotation_file=args.annotation_file)
29
29
  print(json.dumps(report))
30
30
  return {'PASS': 0, 'FAIL': 1, 'INCOMPLETE': 2}[report['status']]
31
31
 
@@ -39,6 +39,9 @@ class History:
39
39
  raise PublicBaseError('public-base.fetch-failed') from None
40
40
  public_revisions = [self.public_commit, *sorted(self.public_tags)] if public_base else []
41
41
  self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
42
+ from history_hydration import prepare
43
+ self.hydration = prepare(self)
44
+ self.env['GIT_NO_LAZY_FETCH'] = '1'
42
45
  self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
43
46
  self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
44
47
  self.trees = {}
@@ -41,7 +41,7 @@ def inspect(binary, mode, repo=None, revisions=None, data=None, timeout=300, red
41
41
  if repo is not None:
42
42
  args.append(repo)
43
43
  env = {k: v for k, v in os.environ.items() if not k.startswith(('GITLEAKS_', 'GIT_'))}
44
- env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0')
44
+ env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0', GIT_NO_LAZY_FETCH='1')
45
45
  try:
46
46
  result = subprocess.run(args, input=data, capture_output=True, env=env, timeout=timeout)
47
47
  if result.returncode not in (0, 1):
@@ -0,0 +1,28 @@
1
+ """Fetch missing promised history in one pack before inspecting any blobs."""
2
+ from time import monotonic
3
+ from git_history import InspectionError
4
+
5
+
6
+ def prepare(history):
7
+ started = monotonic()
8
+ def missing():
9
+ return sum(line.startswith(b'?') for line in
10
+ history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines())
11
+ before = missing()
12
+ fetches = 0
13
+ if before:
14
+ # A blob:none clone already owns the commits and trees. ls-tree -l and
15
+ # cat-file would otherwise ask its promisor for one blob at a time.
16
+ remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
17
+ names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
18
+ for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
19
+ if len(names) != 1:
20
+ raise InspectionError('history.promisor-ambiguous')
21
+ history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
22
+ '--no-recurse-submodules', '--no-write-fetch-head', '--', names[0])
23
+ fetches = 1
24
+ after = missing() if before else 0
25
+ if after:
26
+ raise InspectionError('history.missing-blob')
27
+ return dict(missingBefore=before, missingAfter=after, fetches=fetches,
28
+ seconds=round(monotonic() - started, 6))
@@ -1,5 +1,7 @@
1
1
  """Scan current files and full history using native Git and pinned Gitleaks."""
2
2
  from pathlib import Path
3
+ from scan_timings import Timings
4
+ import staged_annotation
3
5
  from git_history import History, InspectionError
4
6
  from public_base import PublicBaseError
5
7
  from gitleaks import executable, inspect as secrets, rule as secret_rule
@@ -16,15 +18,18 @@ from content_identities import ContentIdentities, candidate as identity_candidat
16
18
  from checksum_maps import ChecksumMaps
17
19
 
18
20
 
19
- def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300):
21
+ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300, annotation_file=None):
20
22
  report = new_report(public_base is not None)
21
- history = None
23
+ timings = Timings(report)
24
+ history = staged = None
22
25
  add = Findings(report, {}, frozenset()).add
23
26
  try:
24
27
  hashes = load_hashes(env_file)
25
28
  report['guarantees']['environmentChecked'] = env_file is not None
26
29
  binary = executable()
27
- history = History(repo, ref, public_base)
30
+ history = timings.call('history.prepare', History, repo, ref, public_base)
31
+ report['history']['hydration'] = history.hydration
32
+ staged = staged_annotation.load(annotation_file, history.commit)
28
33
  report['source'] = dict(commit=history.commit, tree=history.commit_tree(history.commit).decode())
29
34
  report['publicBase'] = dict(ref=history.public_label, commit=history.public_commit) if public_base else None
30
35
  config = load_config(history.selected(CONFIG))
@@ -54,7 +59,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
54
59
  tree = history.commit_tree(commit).decode()
55
60
  seen_trees.add(tree)
56
61
  scope = 'release-tree' if commit == history.commit else 'pre-existing' if commit in history.public_commits else 'newly-public'
57
- config_tree, invalid = context(history, commit, config)
62
+ config_tree, invalid = timings.call('history.context', context, history, commit, config)
58
63
  for path in invalid:
59
64
  add(INVALID if (project is not None or repository is not None) and path == 'edition.json' else 'scan.identity-invalid', path, commit, scope=scope)
60
65
  license_entry = entries_by_commit[commit].get('LICENSE')
@@ -76,7 +81,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
76
81
  local = dependencies[pair] & config_tree['localPaths']
77
82
  key = (path, oid, signature, local)
78
83
  if key not in inspected:
79
- inspected[key] = text_rules(history.blob(oid), path, config_tree, hashes)
84
+ inspected[key] = timings.call('history.text', text_rules, history.blob(oid), path, config_tree, hashes)
80
85
  rules, provenance, references = inspected[key]
81
86
  entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
82
87
  for rule in rules:
@@ -86,7 +91,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
86
91
  if provenance:
87
92
  add('provenance.declared-source', path, commit, oid, entry_scope)
88
93
  report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
89
- selected_context, _ = context(history, history.commit, config)
94
+ selected_context, _ = timings.call('working-tree.context', context, history, history.commit, config)
90
95
  working_paths = []
91
96
  working_identities = {}
92
97
  for path, data in working_files(Path(history.repo)):
@@ -98,7 +103,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
98
103
  # their known immutable identity; new bytes have no history identity.
99
104
  entry = entries_by_commit[history.commit].get(path)
100
105
  oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
101
- rules, _, references = text_rules(data, path, selected_context, hashes)
106
+ rules, _, references = timings.call('working-tree.text', text_rules, data, path, selected_context, hashes)
102
107
  public_oid = public_paths.get(path)
103
108
  scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
104
109
  for rule in rules:
@@ -110,11 +115,11 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
110
115
  add(rule, path, history.commit, oid, 'working-tree')
111
116
  if not (Path(history.repo) / 'LICENSE').is_file():
112
117
  add('license.missing', 'LICENSE', scope='working-tree')
113
- identities = {'working-tree': ContentIdentities(working_identities,
118
+ identities = {'working-tree': timings.call('identities.prepare', ContentIdentities, working_identities,
114
119
  lambda path: working_regular(Path(history.repo), path))}
115
120
  checksums = {}
116
121
  for mode in ('git', 'dir'):
117
- for hit in secrets(binary, mode, history.repo, history.revisions, timeout=timeout, redact=False):
122
+ for hit in timings.call('gitleaks.' + mode, secrets, binary, mode, history.repo, history.revisions, timeout=timeout, redact=False):
118
123
  path, commit = hit.get('File', '.'), hit.get('Commit') or history.commit
119
124
  if mode == 'dir' and Path(path).is_absolute():
120
125
  path = Path(path).resolve().relative_to(Path(history.repo).resolve()).as_posix()
@@ -133,12 +138,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
133
138
  entry = scoped_entries.get(path)
134
139
  return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
135
140
  if identity_scope not in checksums:
136
- checksums[identity_scope] = ChecksumMaps(
141
+ checksums[identity_scope] = timings.call('checksums.prepare', ChecksumMaps,
137
142
  working_paths if mode == 'dir' else scoped_entries,
138
143
  (lambda path: working_regular(Path(history.repo), path)) if mode == 'dir' else read_regular)
139
144
  if identity_candidate(path):
140
145
  if identity_scope not in identities:
141
- identities[identity_scope] = ContentIdentities({
146
+ identities[identity_scope] = timings.call('identities.prepare', ContentIdentities, {
142
147
  p: history.blob(e['oid']) for p, e in scoped_entries.items()
143
148
  if identity_candidate(p) and e['kind'] == 'blob' and e['mode'] in ('100644', '100755')}, read_regular)
144
149
  if identities[identity_scope].matches(path, hit):
@@ -162,10 +167,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
162
167
  scope = 'working-tree' if path in working_paths else 'pre-existing' if commit in history.public_commits else 'newly-public'
163
168
  lines.append((count, count + length, path, commit, scope))
164
169
  count += length
165
- for message_type, commit, data in history.messages():
170
+ for message_type, commit, data in staged_annotation.messages(history, staged):
166
171
  path = '@' + message_type + '-message'
167
172
  report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
168
- rules, _, references = text_rules(data, path, selected_context, hashes,
173
+ rules, _, references = timings.call('messages.text', text_rules, data, path, selected_context, hashes,
169
174
  commit_message=message_type == 'commit')
170
175
  scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
171
176
  for rule in rules:
@@ -176,11 +181,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
176
181
  length = (data + b'\n').count(b'\n')
177
182
  lines.append((count, count + length, path, commit, scope))
178
183
  count += length
179
- for hit in secrets(binary, 'stdin', data=b''.join(messages), timeout=timeout):
184
+ for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=b''.join(messages), timeout=timeout):
180
185
  line = hit.get('StartLine', 0)
181
186
  location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
182
187
  add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
183
- states = GitHubResolver().resolve_all({repository for references, *_ in deferred for repository, _, _ in references})
188
+ states = timings.call('github.references', GitHubResolver().resolve_all,
189
+ {repository for references, *_ in deferred for repository, _, _ in references})
184
190
  for references, path, commit, oid, scope in deferred:
185
191
  for rule in reference_rules(references, states):
186
192
  add(rule, path, commit, oid, scope)
@@ -197,4 +203,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
197
203
  finally:
198
204
  if history:
199
205
  history.close()
200
- return finish(report)
206
+ timings.finish()
207
+ finish(report)
208
+ staged_annotation.summary(report, staged)
209
+ return report
@@ -0,0 +1,46 @@
1
+ """Numeric scan costs and implementation identity, without source file values."""
2
+ import hashlib
3
+ import json
4
+ from pathlib import Path
5
+ import platform
6
+ from time import monotonic
7
+
8
+
9
+ def implementation():
10
+ directory = Path(__file__).parent
11
+ digest = hashlib.sha256()
12
+ for path in sorted(directory.glob('*.py')):
13
+ if path.name.startswith(('test', 'coverage_')):
14
+ continue
15
+ digest.update(path.name.encode() + b'\0' + path.read_bytes())
16
+ root = directory.parent.parent
17
+ for path, key in ((root / 'inventory.json', 'cliVersion'),
18
+ (root / 'package.json', 'version'), (root / 'global/package.json', 'version')):
19
+ if path.is_file():
20
+ version = json.loads(path.read_text()).get(key)
21
+ if version:
22
+ break
23
+ else:
24
+ version = None
25
+ return dict(version=version, sha256=digest.hexdigest(), python=platform.python_version())
26
+
27
+
28
+ class Timings:
29
+ def __init__(self, report):
30
+ self.started = monotonic()
31
+ self.values = report['timings'] = dict(totalSeconds=0, phases={})
32
+ report['implementation'] = implementation()
33
+
34
+ def call(self, name, function, *args, **kwargs):
35
+ started = monotonic()
36
+ try:
37
+ return function(*args, **kwargs)
38
+ finally:
39
+ phase = self.values['phases'].setdefault(name, dict(calls=0, seconds=0))
40
+ phase['calls'] += 1
41
+ phase['seconds'] += monotonic() - started
42
+
43
+ def finish(self):
44
+ self.values['totalSeconds'] = round(monotonic() - self.started, 6)
45
+ for phase in self.values['phases'].values():
46
+ phase['seconds'] = round(phase['seconds'], 6)
@@ -0,0 +1,39 @@
1
+ """Include the actual staged Git tag in the source scan's message policy."""
2
+ import hashlib
3
+ from pathlib import Path
4
+ from git_history import InspectionError
5
+ from report import new_report, finish
6
+
7
+
8
+ def load(path, commit):
9
+ if path is None:
10
+ return None
11
+ raw = Path(path).read_bytes()
12
+ header, separator, message = raw.partition(b'\n\n')
13
+ if not separator or not header.startswith(('object ' + commit + '\ntype commit\n').encode()):
14
+ raise InspectionError('scan.annotation-invalid')
15
+ digest = hashlib.sha1 if len(commit) == 40 else hashlib.sha256
16
+ oid = digest(b'tag ' + str(len(raw)).encode() + b'\0' + raw).hexdigest()
17
+ return dict(oid=oid, message=message)
18
+
19
+
20
+ def messages(history, staged):
21
+ yield from history.messages()
22
+ if staged:
23
+ yield 'tag', staged['oid'], staged['message']
24
+
25
+
26
+ def summary(report, staged):
27
+ if not staged:
28
+ return
29
+ annotation = new_report(report.get('publicBase') is not None)
30
+ annotation.update(source=report['source'], publicBase=report.get('publicBase'),
31
+ implementation=report['implementation'])
32
+ annotation['history'].update(scope='staged-annotation', tagMessages=1)
33
+ annotation['guarantees']['environmentChecked'] = report['guarantees']['environmentChecked']
34
+ for bucket in ('findings', 'preExisting', 'warnings', 'fixtures', 'reviewed', 'info'):
35
+ annotation[bucket] = [{**item, 'commit': staged['oid'], 'commits': [staged['oid']]}
36
+ for item in report[bucket] if staged['oid'] in item['commits']]
37
+ if not report['complete']:
38
+ annotation['findings'].append(dict(rule='scan.source-incomplete', path='.'))
39
+ report['annotationInspection'] = finish(annotation)
@@ -3,6 +3,7 @@ import { spawn } from 'node:child_process';
3
3
  import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs';
4
4
  import { tmpdir } from 'node:os';
5
5
  import { join } from 'node:path';
6
+ import { runningVersion } from '../cli/version.mjs';
6
7
 
7
8
  const args = process.argv.slice(2);
8
9
  let seconds = 300;
@@ -54,5 +55,7 @@ if (reason) report = {
54
55
  history: { commits: 0, trees: 0, blobs: 0 },
55
56
  guarantees: { historyScanned: false, visualRecognition: false, environmentChecked: false },
56
57
  };
58
+ report.implementation ??= {};
59
+ report.implementation.version = runningVersion();
57
60
  process.stdout.write(JSON.stringify(report) + '\n');
58
61
  process.exitCode = code;