@runeditrun/rer 0.2.57 → 0.2.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/contract.mjs +1 -1
- package/cli/deploy-command.mjs +10 -3
- package/cli/deploy-needs.mjs +10 -8
- package/cli/deploy-secret-error.mjs +8 -0
- package/cli/deploy.mjs +13 -18
- package/global/CHANGELOG.md +9 -0
- package/modules/demo-host/native/loader.mjs +9 -4
- package/modules/deploy-kit/README.md +2 -0
- package/modules/deploy-kit/owner-cli.mjs +22 -8
- package/modules/deploy-kit/worker.mjs +7 -1
- package/modules/release/produce.mjs +4 -23
- package/modules/release/publication-scan.mjs +36 -4
- package/package.json +1 -1
- package/scripts/publish-check/README.md +14 -0
- package/scripts/publish-check/cli.py +2 -2
- package/scripts/publish-check/git_history.py +3 -0
- package/scripts/publish-check/gitleaks.py +1 -1
- package/scripts/publish-check/history_hydration.py +28 -0
- package/scripts/publish-check/history_scan.py +25 -16
- package/scripts/publish-check/scan_timings.py +46 -0
- package/scripts/publish-check/staged_annotation.py +39 -0
- package/scripts/publish-check.mjs +3 -0
package/cli/contract.mjs
CHANGED
|
@@ -11,7 +11,7 @@ const definitions = {
|
|
|
11
11
|
build: [['--site'], 'Production build with private local settings excluded and restored. --site instead writes the static run site (homepage, /run, docs) to its declared folder and prints that folder.'],
|
|
12
12
|
verify: [[], 'Run the complete internal verification target.'],
|
|
13
13
|
reset: [['--yes'], 'Confirmed removal of the declared local inventory only.'],
|
|
14
|
-
deploy: [['--owner-name', '--environment', '--dry-run', '--site', '--demo', '--host', '--env-file', '--image', '--bind-address', '--http-port', '--https-port', '--remove', '--yes', '--set-secret', '--unset-secret', '--rotate-secret'], 'Owner-account deployment and health; dry run
|
|
14
|
+
deploy: [['--owner-name', '--environment', '--dry-run', '--site', '--demo', '--host', '--env-file', '--image', '--bind-address', '--http-port', '--https-port', '--remove', '--yes', '--set-secret', '--unset-secret', '--rotate-secret'], 'Owner-account deployment and health; dry run validates/builds locally and reads target secret bindings with Cloudflare credentials, without provider writes. Missing required secrets fail as in deploy; missing credentials or unreadable target secrets also fail. Builds and deploy hooks run with the checkout\'s local settings files moved aside, and a build output holding one (.dev.vars, .env) is refused. Declared secrets come from the environment or --env-file (an owned 0600 KEY=VALUE file); a new Worker gets the local setup owner and fresh generated secrets when they are not given. rer keeps the secrets it generates in .rer/deploy/ENVIRONMENT.secrets.env (0600) and reuses them on every redeploy; it refuses to generate a replacement for one a live Worker holds, and --rotate-secret NAME[,NAME] replaces them on purpose. After a healthy deploy rer claims the setup owner on the hosted app with the saved setup owner name and the one-time setup token, then removes the token. If the installer declares an owner-name field and no name is saved, deploy asks once and saves it; --non-interactive requires --owner-name NAME before any upload. --set-secret NAME[,NAME] (each value from the environment or --env-file; repeat the flag or join names with commas) and --unset-secret NAME[,NAME] change those secrets of the deployed Worker without a build, all of them in one new Worker version or none, and record that version; a missing value refuses before any change. --remove --yes deletes what this environment\'s deploy record says rer created (Worker, D1, R2, KV, queues), reads back that each is gone, and touches nothing else. --site deploys the run site instead: an assets-only Worker on its custom domain with www redirected; with --dry-run it builds the site and prints the plan with no provider request. --demo deploys the hosted demo instead (deploy-demo.json): a front Worker on demo.<domain> giving each visitor their own signed-in, throwaway native Worker isolate of this seed with real public reads/checks and simulated external writes; --dry-run writes it to .rer/demo-host with no provider request; --demo --remove --yes deletes that demo (its Worker, demo.<domain> Custom Domain, each only while the record its deploy left proves it the demo\'s), reads back that each is gone, and is a no-op once it is.'],
|
|
15
15
|
export: [['--target', '--output', '--origin', '--session-file'], 'Complete supported archive of the local copy on this computer, with integrity and exclusions, written to --output. The hosted install cannot be exported yet: --origin or --target remote refuses (exit 2) before any effect and names the command that saves the hosted database.'],
|
|
16
16
|
import: [['--target', '--input', '--journal', '--origin', '--session-file', '--yes', '--legacy-v1', '--acknowledge-unverified-metadata', '--output'], 'Confirmed atomic restore into an empty destination; --journal LATER_ARCHIVE also fences rows that installation erased since. Backup-only local v1 recovery requires --legacy-v1 --acknowledge-unverified-metadata and a new --output v2 archive; --journal requires normal v2 import.'],
|
|
17
17
|
doctor: [[], 'Read-only installed tool, configuration and state diagnostics.'],
|
package/cli/deploy-command.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import { readFile, readdir } from 'node:fs/promises';
|
|
|
4
4
|
import { contained } from './project.mjs';
|
|
5
5
|
import { invalid, refuse, RerError } from './errors.mjs';
|
|
6
6
|
import { ownerDeploy } from './deploy.mjs';
|
|
7
|
+
import { secretFailureMessage } from './deploy-secret-error.mjs';
|
|
7
8
|
import { deployEnvironment, deployAgain } from './deploy-inputs.mjs';
|
|
8
9
|
import { claimHostedOwner } from './deploy-owner.mjs';
|
|
9
10
|
import { deployOwnerInput } from './deploy-owner-input.mjs';
|
|
@@ -78,11 +79,11 @@ export async function deployCommand(context) {
|
|
|
78
79
|
if (local) throw refuse(`${path} names the local-only key ${local} as a deploy binding or secret. Remove it; local dev keys never reach a deployment.`);
|
|
79
80
|
deployGuide(declaration);
|
|
80
81
|
const settings = adapter.hooks?.secrets ?? adapter.settings;
|
|
81
|
-
//
|
|
82
|
+
// Required secrets have been checked; retain the production sign-in guidance.
|
|
82
83
|
const dryRun = async result => {
|
|
83
84
|
const production = await deployEnvironment({ root, options, declaration, setupManifest, settings, localOnlyKeys });
|
|
84
85
|
const { signIn } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
|
|
85
|
-
const needed = neededForRealDeploy({ declaration, env: production.env, signIn });
|
|
86
|
+
const needed = neededForRealDeploy({ declaration, env: production.env, signIn, secretsChecked: true });
|
|
86
87
|
reportNeeded(store, needed);
|
|
87
88
|
return needed.length && result && typeof result === 'object' ? { ...result, neededForRealDeploy: needed } : result;
|
|
88
89
|
};
|
|
@@ -134,7 +135,10 @@ export async function deployCommand(context) {
|
|
|
134
135
|
}
|
|
135
136
|
hookReturned = true;
|
|
136
137
|
await refuseLocalEnvOutput(root, adapter.devIdentity?.output, { uploaded: !options.dryRun });
|
|
137
|
-
if (options.dryRun)
|
|
138
|
+
if (options.dryRun) {
|
|
139
|
+
await command('dry-run');
|
|
140
|
+
return dryRun(deployed);
|
|
141
|
+
}
|
|
138
142
|
await rememberHookState(root, environment, deployed?.state);
|
|
139
143
|
hookRecord = await hookState(root, environment);
|
|
140
144
|
const origin = hookOrigin(deployed, deployAgain(options));
|
|
@@ -163,6 +167,9 @@ export async function deployCommand(context) {
|
|
|
163
167
|
} catch (error) {
|
|
164
168
|
const again = deployAgain(options);
|
|
165
169
|
const { commandFailure } = await shared('failures.mjs');
|
|
170
|
+
const secrets = secretFailureMessage(error);
|
|
171
|
+
if (secrets) throw commandFailure(secrets);
|
|
172
|
+
if (options.dryRun && error.message === 'missing-input') throw new RerError('missing_input', `Dry run cannot read the target secret bindings without ${error.inputs.join(' and ')}. Set the Cloudflare credentials in the environment or an owned 0600 --env-file, then run the same command again.`, 3);
|
|
166
173
|
if (error.hookFailure) throw commandFailure(store.redact?.(redaction.redact(error.message)) ?? redaction.redact(error.message));
|
|
167
174
|
if (error instanceof RerError && !deployed?.deployed && !recorded) { error.message = store.redact?.(redaction.redact(error.message)) ?? redaction.redact(error.message); throw error; }
|
|
168
175
|
const outcome = error.owner ?? claimed ?? observedClaims.at(-1)?.outcome;
|
package/cli/deploy-needs.mjs
CHANGED
|
@@ -29,22 +29,24 @@ function credentialHelp(declaration) {
|
|
|
29
29
|
return [`Create an account API token at ${tokenPage} with these account permissions: ${needed.join(', ')}. Copy your account ID from the Cloudflare dashboard.`];
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
export function missingCredentialsMessage(declaration, env) {
|
|
32
|
+
export function missingCredentialsMessage(declaration, env, { dryRun = false, signIn } = {}) {
|
|
33
33
|
const absent = cloudflareCredentials.filter(name => !env[name]?.trim());
|
|
34
|
-
return [
|
|
34
|
+
return [...(dryRun ? ['Dry run cannot read the target secret bindings without Cloudflare credentials.'] : []),
|
|
35
|
+
`${absent.join(' and ')} ${absent.length > 1 ? 'are' : 'is'} required to deploy to your Cloudflare account. Nothing was deployed.`,
|
|
35
36
|
...credentialHelp(declaration),
|
|
36
|
-
'Set them in your environment or an owned 0600 --env-file, then run the same deploy command again.'
|
|
37
|
+
'Set them in your environment or an owned 0600 --env-file, then run the same deploy command again.',
|
|
38
|
+
...(dryRun ? ['Needed for a real deploy (the dry run could not check the target):', ...neededForRealDeploy({declaration,env,signIn}).map(item => ` - ${item}`)] : [])].join('\n');
|
|
37
39
|
}
|
|
38
40
|
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
export function neededForRealDeploy({ declaration, env, signIn }) {
|
|
41
|
+
// Setup guidance and failed dry-run inputs. After target resolution, held
|
|
42
|
+
// secrets no longer count as missing local inputs.
|
|
43
|
+
export function neededForRealDeploy({ declaration, env, signIn, secretsChecked = false }) {
|
|
42
44
|
const declared = (Array.isArray(declaration.secrets) ? declaration.secrets : []).filter(item => item?.environment);
|
|
43
45
|
// Sign-in names its inputs by binding; a person sets the environment name the declaration maps to it.
|
|
44
46
|
const mail = signIn && declared.reduce((text, item) => item.binding === item.environment ? text : text.replace(new RegExp(`\\b${item.binding}\\b`, 'g'), item.environment), signIn);
|
|
45
47
|
const listed = name => cloudflareCredentials.includes(name) || mail && new RegExp(`\\b${name}\\b`).test(mail);
|
|
46
|
-
const secrets = [...new Set(declared.filter(item => !item.optional && !env[item.environment]?.trim()).map(item => item.environment))].filter(name => !listed(name));
|
|
47
|
-
const credentials = cloudflareCredentials.filter(name => !env[name]?.trim());
|
|
48
|
+
const secrets = secretsChecked ? [] : [...new Set(declared.filter(item => !item.optional && !env[item.environment]?.trim()).map(item => item.environment))].filter(name => !listed(name));
|
|
49
|
+
const credentials = secretsChecked ? [] : cloudflareCredentials.filter(name => !env[name]?.trim());
|
|
48
50
|
return [
|
|
49
51
|
...(credentials.length ? [`${credentials.join(' and ')} (${credentialHelp(declaration).join(' ')})`] : []),
|
|
50
52
|
...secrets,
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// Both deploy paths render the shared resolver's complete missing-input list.
|
|
2
|
+
export function secretFailureMessage(error) {
|
|
3
|
+
if (error.message === 'token-scope-missing-workers') return `Cannot read the target's secret list; required secret bindings could not be checked. The Cloudflare token needs Workers Scripts permission (${error.message}). Nothing was uploaded.`;
|
|
4
|
+
const first = /^missing-secret-([A-Z][A-Z0-9_]*)$/.exec(error.message)?.[1];
|
|
5
|
+
if (!first) return;
|
|
6
|
+
const names = error.missingSecrets?.map(({binding,environment}) => binding === environment ? binding : `${binding} (${environment})`) ?? [first];
|
|
7
|
+
return `Deploy needs ${names.join(', ')}, which the deploy declaration lists as ${names.length === 1 ? 'a secret' : 'secrets'} and the Worker does not hold. Set ${names.length === 1 ? 'its value' : 'their values'} in the named environment inputs or in an --env-file file (an owned 0600 file of KEY=VALUE lines), then run the same command again. Nothing was uploaded.`;
|
|
8
|
+
}
|
package/cli/deploy.mjs
CHANGED
|
@@ -5,8 +5,8 @@ import { shared } from './shared.mjs';
|
|
|
5
5
|
import { settingKinds } from './deploy-settings.mjs';
|
|
6
6
|
import { contained } from './project.mjs';
|
|
7
7
|
import { missing, invalid, refuse, RerError } from './errors.mjs';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
8
|
+
import { localOnlyBinding } from '../modules/deploy-kit/owner.mjs';
|
|
9
|
+
import { secretFailureMessage } from './deploy-secret-error.mjs';
|
|
10
10
|
import { refuseLocalModels } from '../modules/deploy-kit/local-only.mjs';
|
|
11
11
|
import { deployEnvironment, deployAgain } from './deploy-inputs.mjs';
|
|
12
12
|
import { claimHostedOwner } from './deploy-owner.mjs';
|
|
@@ -38,31 +38,20 @@ export function ownerDeploy(declarationPath, lifecycle) {
|
|
|
38
38
|
const recordedState = change.length ? await deploymentRecord(root, options) : undefined;
|
|
39
39
|
// A dry run reads the settings file too, so a refused key fails here and not only on the real deploy.
|
|
40
40
|
if (options.dryRun && options.envFile) await deployEnvironment({ root, options, declaration, localOnlyKeys });
|
|
41
|
-
let production, ownerInput;
|
|
41
|
+
let production, ownerInput, signIn;
|
|
42
42
|
if (!change.length) {
|
|
43
43
|
production = await deployEnvironment({ root, options, declaration, setupManifest, settings: lifecycle.settings, localOnlyKeys });
|
|
44
|
-
await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed, adapter: lifecycle, declarationPath, options });
|
|
44
|
+
({signIn} = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed, adapter: lifecycle, declarationPath, options }) ?? {});
|
|
45
45
|
if (!options.dryRun) committedSource(root, options);
|
|
46
46
|
ownerInput = await deployOwnerInput({ root, adapter, setupSeed, options, signal });
|
|
47
47
|
await withoutLocalSettings(root, { settings: lifecycle.settings, operationToken }, () => lifecycle.run(root, {}, store, () => {}, signal, { verb: 'build', operationToken }));
|
|
48
48
|
await checkBuild();
|
|
49
49
|
await refuseLocalEnvOutput(root);
|
|
50
50
|
}
|
|
51
|
-
if (options.dryRun) {
|
|
52
|
-
// Locally validate the real declaration/artifact without ownerTarget's
|
|
53
|
-
// account lookup. This synthetic target is validation data, not a provider.
|
|
54
|
-
const worker = ownerWorkerName(declaration.project, environment);
|
|
55
|
-
const artifact = await packageBuild({ root, source: { sha: '0'.repeat(40), tree: '0'.repeat(40) }, runIdentity: { id: '0', attempt: '1' }, productDirectory: '' });
|
|
56
|
-
ownerPlan({ project: declaration.project, environment, worker, origin: `https://${worker}.example.invalid` }, declaration, artifact, { localOnlyKeys, env: production.env });
|
|
57
|
-
const optional = (Array.isArray(declaration.secrets) ? declaration.secrets : []).filter(item => item?.optional && !production.env[item.environment]?.trim()).map(item => item.binding);
|
|
58
|
-
if (optional.length) store.report(`deploy: optional secrets not supplied: ${optional.join(', ')}. A Worker that does not hold one deploys without it.`);
|
|
59
|
-
store.report('deploy: dry run complete; no deployment occurred and no provider request was made.');
|
|
60
|
-
return { dryRun: true, deployed: false };
|
|
61
|
-
}
|
|
62
51
|
const { env, supplied, generated, reused, rotated } = await deployEnvironment({ root, options, declaration, setupManifest, settings: lifecycle.settings, localOnlyKeys });
|
|
63
52
|
if (options.setSecret || options.unsetSecret) secretChangeInputs(declaration, options, env, again);
|
|
64
53
|
const kept = keptSecrets(declaration, setupManifest);
|
|
65
|
-
if (cloudflareCredentials.some(name => !env[name]?.trim())) throw missing(missingCredentialsMessage(declaration, env));
|
|
54
|
+
if (cloudflareCredentials.some(name => !env[name]?.trim())) throw missing(missingCredentialsMessage(declaration, env, {dryRun:options.dryRun,signIn}));
|
|
66
55
|
const { main } = await import('../modules/deploy-kit/owner-cli.mjs');
|
|
67
56
|
const { createRedaction } = await shared('redaction.mjs');
|
|
68
57
|
const redaction = createRedaction();
|
|
@@ -74,6 +63,12 @@ export function ownerDeploy(declarationPath, lifecycle) {
|
|
|
74
63
|
const created = [];
|
|
75
64
|
let step = 'deploy', deployed;
|
|
76
65
|
try {
|
|
66
|
+
if (options.dryRun) {
|
|
67
|
+
const {optionalUnset} = await owner('dry-run');
|
|
68
|
+
if (optionalUnset.length) store.report(`deploy: optional secrets not supplied: ${optionalUnset.join(', ')}. A Worker that does not hold one deploys without it.`);
|
|
69
|
+
store.report('deploy: dry run complete; target secret bindings checked; no deployment occurred.');
|
|
70
|
+
return { dryRun: true, deployed: false };
|
|
71
|
+
}
|
|
77
72
|
// A custom hook's own state file, when its deploy named one.
|
|
78
73
|
const state = change.length ? recordedState : await hookState(root, environment);
|
|
79
74
|
if (options.remove) {
|
|
@@ -164,8 +159,8 @@ function failureText({ error, step, created, again, retry, secretChange, deploye
|
|
|
164
159
|
const diagnostic = deployDiagnostic(error, redact);
|
|
165
160
|
const code = diagnostic?.code ?? (/^[a-zA-Z0-9_-]{1,120}$/.test(error.message) ? redact(error.message) : 'deploy-failed');
|
|
166
161
|
const at = step === 'deploy' ? diagnostic?.step ?? 'prepare deploy' : step;
|
|
167
|
-
const
|
|
168
|
-
if (
|
|
162
|
+
const secrets = secretFailureMessage(error);
|
|
163
|
+
if (secrets) return secrets;
|
|
169
164
|
if (code === 'nothing-to-remove') return `The deployment record selected by ${again} has no resources that rer owns, so --remove deletes nothing.`;
|
|
170
165
|
if (code === 'owner-token-retirement-failed') return `The application is deployed at ${deployed?.origin}; owner ${error.owner}; setup token retirement failed (${redact(String(error.cause))}${providerFailure(error,redact) ? `; ${providerFailure(error,redact)}` : ''}). Run ${again} --unset-secret ${error.tokenKey ?? 'SETUP_TOKEN'} using the same environment credentials, or run ${again} again to finish.`;
|
|
171
166
|
if (code === 'owner-claim-failed') return `The application is deployed at ${deployed?.origin}, but claiming its owner failed: ${redact(String(error.cause))}. Owner claim and setup token retirement are not confirmed; run ${again} again using the same environment credentials to finish.`;
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.59
|
|
4
|
+
|
|
5
|
+
- **Release publication scans complete on large seeds and partial clones.** `rer release` hydrates missing history blobs in one fetch and inspects the staged release annotation in the same scan. Orbit finishes in 82 seconds instead of exceeding the 300-second budget, with identical findings. A seed must adopt the new rer through its pin for its release command to use it (#278).
|
|
6
|
+
- **Native demos decode binary modules when needed.** WebAssembly and data modules decode lazily inside the Worker Loader factory with native base64 support, reducing host startup work while preserving their bytes (#280).
|
|
7
|
+
|
|
8
|
+
## 0.2.58
|
|
9
|
+
|
|
10
|
+
- **Deploy dry runs validate required target secrets.** `rer deploy --dry-run` uses the real deploy secret-binding resolver and refuses with the same exit and message when required bindings are missing. It names every missing binding and its environment input, accepts secrets already held by the Worker, and makes no target writes. Missing credentials or read permission also refuse (#275).
|
|
11
|
+
|
|
3
12
|
## 0.2.57
|
|
4
13
|
|
|
5
14
|
- **Large release scans finish within the publication budget.** The JSON identity pass tracks UTF-8 byte offsets incrementally. Orbit's scan finishes in 79 seconds instead of exceeding the 300-second deadline, with identical findings (#270).
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
import { WorkerEntrypoint } from 'cloudflare:workers';
|
|
2
|
+
import { Buffer } from 'node:buffer';
|
|
2
3
|
import demoEgress from '../../container/demo-egress.mjs';
|
|
3
4
|
import bundle from './bundle.mjs';
|
|
4
5
|
|
|
5
|
-
//
|
|
6
|
-
const bytes = text =>
|
|
7
|
-
const
|
|
6
|
+
// Decode only when a Loader factory needs the modules, once per host isolate.
|
|
7
|
+
const bytes = text => {
|
|
8
|
+
const buffer = Buffer.from(text, 'base64');
|
|
9
|
+
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
|
|
10
|
+
};
|
|
11
|
+
let modules;
|
|
12
|
+
const loaderModules = () => modules ??= Object.fromEntries(Object.entries(bundle.modules).map(([name, module]) =>
|
|
8
13
|
[name, 'wasm' in module ? { wasm: bytes(module.wasm) } : 'data' in module ? { data: bytes(module.data) } : module]));
|
|
9
14
|
|
|
10
15
|
export function loaded(env, ctx, id, runtime) {
|
|
11
16
|
return env.LOADER.get(`${bundle.digest}:${id}`, () => ({
|
|
12
17
|
compatibilityDate: bundle.compatibilityDate, compatibilityFlags: bundle.compatibilityFlags,
|
|
13
|
-
mainModule: '_rer/isolate.js', modules,
|
|
18
|
+
mainModule: '_rer/isolate.js', modules: loaderModules(),
|
|
14
19
|
globalOutbound: ctx.exports.DemoEgress({ props: { id } }),
|
|
15
20
|
env: { ...runtime, RER_NATIVE_CONFIG: bundle.bindings, RER_NATIVE_HOST: ctx.exports.HostBridge({ props: { id, key: runtime.RER_NATIVE_KEY } }), [bundle.bindings.assets]: ctx.exports.DemoAssets({}) },
|
|
16
21
|
}));
|
|
@@ -115,6 +115,8 @@ A transport failure carries `providerTransport`: the operation, sanitized endpoi
|
|
|
115
115
|
|
|
116
116
|
`rer deploy` runs these steps for a seed whose adapter uses `ownerDeploy` from `cli/deploy.mjs`.
|
|
117
117
|
|
|
118
|
+
- **Dry run.** Application `rer deploy --dry-run` builds and validates locally, then calls `owner-cli.mjs dry-run` to read the selected target and resolve required secrets with the same `secretBindings` function as deploy. All missing required bindings and their environment names are reported together, with identical nonzero exit and error in human and JSON output. Held secrets are inherited, supplied values are accepted without upload, absent optional secrets stay unset, and recorded retired secrets stay retired. Cloudflare credentials are required; a failed secret-list read fails the command. Dry run does not settle pending secret changes, write deployment state, upload, change secrets or create resources. An unsettled secret change must be completed by its original command first.
|
|
119
|
+
|
|
118
120
|
- **Inputs.** Declared secrets come from the environment, then `--env-file FILE` (an owned 0600 file of `KEY=VALUE` lines), which wins. For a secret the Worker does not hold yet, rer supplies the setup manifest's owner email (`emailField`) from the local setup's private settings, and for each secret field the manifest generates (`generate`) the value rer keeps for this environment in `.rer/deploy/<environment>.secrets.env` (0600, ignored by Git), else a fresh random value, which rer keeps once the Worker holds it (the one-time setup token is always fresh and never kept); a value the Worker already holds is kept. The build runs with the checkout's local settings files moved aside, and a `dist/` holding a `.dev.vars*`, `.env*` or `*.env` file is refused by name before any provider request. A still-missing secret stops the run before any write with a message that names it, unless its declaration says `"optional": true`: then the deploy binds nothing for it and reports it as not set, and `--set-secret` adds it later.
|
|
119
121
|
- **Hosted owner.** After deploy and health pass, rer claims the setup owner on the hosted app: it POSTs the setup token to the seed's standard owner step (`hooks.owner`, by default `/auth/bootstrap`). 303 means claimed; `409 already_initialized` means an owner was already claimed and is kept. Then rer deletes the token secret from the Worker, records the new Worker version in the state file, and lists the token in `retiredSecrets`, so later plans no longer declare it. The build fingerprint still names it, and the recorded claim stays as recorded, so an untouched repeat deploy writes nothing. A Worker that still holds a token rer did not upload in this run (an older deploy) first gets a fresh token through the same recorded secret change. Dependency approvals that need the token run before the claim. A seed with an owner step function, an identity-provider claim, or no declared token secret claims its owner its own way.
|
|
120
122
|
- **Several secrets at once.** `rer deploy --set-secret MAIL_API_KEY --set-secret MAIL_FROM` or `rer deploy --set-secret MAIL_API_KEY,MAIL_FROM` (likewise `--unset-secret A,B`) changes all of them in one new Worker version or none: a group of secrets is never half set. Every named secret must be declared and, for `--set-secret`, have its value in the environment or `--env-file`; otherwise rer names each missing one and changes nothing. One secret uses Cloudflare's secret call. Several are one copy of the live version through the Versions API (its own modules, assets and compatibility settings, every other binding inherited), then deployed at 100%: Cloudflare's per-secret calls each make a version, and its settings call can add secrets but not remove them (measured 17 October 2026). The copy carries a marker rer records first, so a copy whose answer was lost is recognised, and one uploaded but not deployed is deployed by the same command run again.
|
|
@@ -8,8 +8,8 @@ import { assert, hexId } from './validation.mjs';
|
|
|
8
8
|
import { regularBytes } from './artifact.mjs';
|
|
9
9
|
import { packageBuild } from './prepare.mjs';
|
|
10
10
|
import { cloudflare } from './provider.mjs';
|
|
11
|
-
import { ownerTarget, ownerWorkerName, ownerPlan, ownerDeploy, ownerHealth, settleOwnerState, sameDeployment, localOnlyBinding, withStateLock, readOwnerState, writeOwnerState } from './owner.mjs';
|
|
12
|
-
import { activeWorker } from './worker.mjs';
|
|
11
|
+
import { ownerTarget, ownerWorkerName, ownerPlan, ownerDeploy, ownerHealth, settleOwnerState, sameDeployment, localOnlyBinding, withStateLock, readOwnerState, writeOwnerState, workerSettings, withScopeHints } from './owner.mjs';
|
|
12
|
+
import { activeWorker, secretBindings } from './worker.mjs';
|
|
13
13
|
import { ownerSecret } from './owner-secret.mjs';
|
|
14
14
|
import { ownerRemove } from './owner-remove.mjs';
|
|
15
15
|
import { ownerSignin } from './owner-signin.mjs';
|
|
@@ -17,7 +17,7 @@ import { refuseLocalModels } from './local-only.mjs';
|
|
|
17
17
|
import { publicOrigin } from './public-origin.mjs';
|
|
18
18
|
|
|
19
19
|
export const cronNotice = 'This Worker has a Cron Trigger. Cloudflare may take up to about two hours to start its first scheduled run on some accounts.';
|
|
20
|
-
const usage = 'usage: owner-cli.mjs deploy|health|remove|inspect|record-owner|signin --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
|
|
20
|
+
const usage = 'usage: owner-cli.mjs deploy|dry-run|health|remove|inspect|record-owner|signin --declaration <file> --environment <name> [--product <dir>] [--state <file>] [--outcome claimed|kept] | secret ... --set NAME [--set NAME]|--unset NAME [--unset NAME]';
|
|
21
21
|
|
|
22
22
|
/** The owner's committed source and existing local build. Run the seed's build first. */
|
|
23
23
|
async function ownerArtifact(root, product) {
|
|
@@ -33,7 +33,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
33
33
|
try { parsed = parseArgs({args:argv,allowPositionals:true,strict:true,options:{declaration:{type:'string'},environment:{type:'string'},product:{type:'string',default:''},state:{type:'string'},set:{type:'string',multiple:true},unset:{type:'string',multiple:true},outcome:{type:'string'}}}); }
|
|
34
34
|
catch { throw new Error('usage'); }
|
|
35
35
|
const {positionals:[command,...rest],values} = parsed;
|
|
36
|
-
if (!['deploy','health','secret','remove','inspect','record-owner','signin'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
|
|
36
|
+
if (!['deploy','dry-run','health','secret','remove','inspect','record-owner','signin'].includes(command) || rest.length || !values.declaration || !values.environment) throw new Error('usage');
|
|
37
37
|
if ((command === 'secret') !== (values.set !== undefined || values.unset !== undefined) || values.set !== undefined && values.unset !== undefined) throw new Error('usage');
|
|
38
38
|
if (values.outcome !== undefined && command !== 'record-owner' || command === 'record-owner' && !['claimed','kept'].includes(values.outcome)) throw new Error('usage');
|
|
39
39
|
await refuseLocalModels(resolve(cwd));
|
|
@@ -47,6 +47,12 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
47
47
|
assert(typeof declaration?.project === 'string','missing-owner-project');
|
|
48
48
|
// Validate even direct kit invocations before any provider request or write.
|
|
49
49
|
publicOrigin(declaration, env);
|
|
50
|
+
let artifact;
|
|
51
|
+
if (command === 'dry-run') {
|
|
52
|
+
artifact = await packageBuild({root,source:{sha:'0'.repeat(40),tree:'0'.repeat(40)},runIdentity:{id:'0',attempt:'1'},productDirectory:values.product});
|
|
53
|
+
const worker = ownerWorkerName(declaration.project,values.environment);
|
|
54
|
+
ownerPlan({project:declaration.project,environment:values.environment,worker,origin:`https://${worker}.example.invalid`},declaration,artifact,{localOnlyKeys,env});
|
|
55
|
+
}
|
|
50
56
|
// Removal reads identity before each object DELETE and may span many pages.
|
|
51
57
|
// Bound each response, not the owner's bucket size; failures retain ownership.
|
|
52
58
|
let requestWindow = propagation?.uploaded ? propagation.window : undefined;
|
|
@@ -74,7 +80,9 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
74
80
|
return {ok:true,...await ownerSecret({api,target,statePath,changes,fetch,healthOptions}).catch(error => { throw api.annotateFailure(error,'secrets'); })};
|
|
75
81
|
}
|
|
76
82
|
// A secret rer retired (the spent setup token) is no longer part of the plan.
|
|
77
|
-
|
|
83
|
+
// A dry run only reads state: settling an interrupted change may write it.
|
|
84
|
+
const state = command === 'dry-run' ? await readOwnerState(statePath,target) : await settleOwnerState({api,target,statePath});
|
|
85
|
+
if (command === 'dry-run') assert(!state.pendingSecret, 'secret-change-unsettled');
|
|
78
86
|
const {retiredSecrets = []} = state;
|
|
79
87
|
if (command === 'record-owner') return withStateLock(statePath, async () => {
|
|
80
88
|
const current = await readOwnerState(statePath, target);
|
|
@@ -97,8 +105,14 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
97
105
|
return {ok:true, origin:target.origin, versionId:live.versionId, retiredSecrets, ownerClaim:state.ownerClaim, liveSecrets:rows.map(row => row.name)};
|
|
98
106
|
}
|
|
99
107
|
const current = retiredSecrets.length && Array.isArray(declaration.secrets) ? {...declaration,secrets:declaration.secrets.filter(item => !retiredSecrets.includes(item?.binding))} : declaration;
|
|
100
|
-
|
|
108
|
+
artifact ??= await ownerArtifact(root,values.product);
|
|
101
109
|
const plan = ownerPlan(target,current,artifact,{localOnlyKeys,env});
|
|
110
|
+
if (command === 'dry-run') {
|
|
111
|
+
const checked = withScopeHints(api);
|
|
112
|
+
const exists = await workerSettings(checked,target.worker) !== null;
|
|
113
|
+
const {unset} = await secretBindings(checked,target.worker,plan,env,exists);
|
|
114
|
+
return {ok:true,dryRun:true,deployed:false,optionalUnset:unset};
|
|
115
|
+
}
|
|
102
116
|
if (command === 'health') {
|
|
103
117
|
const readiness = {...(healthOptions?.now ? {clock:healthOptions.now} : {}), ...(healthOptions?.sleep ? {sleep:healthOptions.sleep} : {}),
|
|
104
118
|
...healthOptions?.readiness,single:true,totalMs:30000,
|
|
@@ -130,10 +144,10 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
130
144
|
/** Prints refusal codes and resource names only; never inputs, tokens or provider bodies. */
|
|
131
145
|
export function report(error) {
|
|
132
146
|
const diagnostic = deployDiagnostic(error);
|
|
133
|
-
if (diagnostic) return {ok:false,...diagnostic};
|
|
147
|
+
if (diagnostic) return {ok:false,...diagnostic,...(error.missingSecrets ? {missingSecrets:error.missingSecrets} : {})};
|
|
134
148
|
if (error.message === 'missing-input') return {ok:false,code:'missing-input',inputs:error.inputs};
|
|
135
149
|
if (error.message === 'usage') return {ok:false,code:'usage',usage};
|
|
136
|
-
return {ok:false,code:/^[a-zA-Z0-9_-]{1,120}$/.test(error.message) ? error.message : 'owner-deploy-failed',...(error.resource ? {resource:error.resource} : {})};
|
|
150
|
+
return {ok:false,code:/^[a-zA-Z0-9_-]{1,120}$/.test(error.message) ? error.message : 'owner-deploy-failed',...(error.resource ? {resource:error.resource} : {}),...(error.missingSecrets ? {missingSecrets:error.missingSecrets} : {})};
|
|
137
151
|
}
|
|
138
152
|
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
|
139
153
|
try { console.log(JSON.stringify(await main())); }
|
|
@@ -49,10 +49,16 @@ export async function secretBindings(api, worker, plan, env, exists) {
|
|
|
49
49
|
const supplied = item => typeof env[item.environment] === 'string' && env[item.environment].trim();
|
|
50
50
|
// An optional secret neither held nor supplied stays unbound; the app treats it as not connected.
|
|
51
51
|
const unset = plan.secrets.filter(item => item.optional && !names.includes(item.binding) && !supplied(item)).map(item => item.binding);
|
|
52
|
+
const missingSecrets = plan.secrets.filter(item => !item.optional && !names.includes(item.binding) && !supplied(item))
|
|
53
|
+
.map(({binding,environment}) => ({binding,environment}));
|
|
54
|
+
if (missingSecrets.length) {
|
|
55
|
+
const error = new Error(`missing-secret-${missingSecrets[0].environment}`);
|
|
56
|
+
error.missingSecrets = missingSecrets;
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
52
59
|
for (const item of plan.secrets) {
|
|
53
60
|
if (names.includes(item.binding)) bindings.push({type:'inherit',name:item.binding});
|
|
54
61
|
else if (!unset.includes(item.binding)) {
|
|
55
|
-
assert(supplied(item), `missing-secret-${item.environment}`);
|
|
56
62
|
bindings.push({type:'secret_text',name:item.binding,text:env[item.environment]});
|
|
57
63
|
}
|
|
58
64
|
}
|
|
@@ -5,7 +5,7 @@ import { inspectSource, newOutput, git } from './source.mjs';
|
|
|
5
5
|
import { verifySource } from './source-verification.mjs';
|
|
6
6
|
import { catalogue, writeCatalogue, sha256, json } from './catalogue.mjs';
|
|
7
7
|
import { publicExport, cleanHistory, annotate, releaseMessage } from './export.mjs';
|
|
8
|
-
import { publicationScan, validatePublicationDestination } from './publication-scan.mjs';
|
|
8
|
+
import { publicationScan, sourcePublicationScan, validatePublicationDestination } from './publication-scan.mjs';
|
|
9
9
|
import { publicDestination, requirePublicDestination, remotePublicHead, publicChild, assertPublicTree } from './public-child.mjs';
|
|
10
10
|
import { writePublishHandoff } from './publish-handoff.mjs';
|
|
11
11
|
import { releasePublicBase } from './public-base.mjs';
|
|
@@ -53,28 +53,9 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
53
53
|
// when available, retaining the selected source commit and tag identity.
|
|
54
54
|
const exported = destination ? publicChild(source, temporary, destination, publicParent, Boolean(tag))
|
|
55
55
|
: clean ? cleanHistory(source, temporary, Boolean(tag)) : null;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
if (!exported && tag && report.status === 'PASS') {
|
|
60
|
-
// Stage the exact final annotation before any source tag or distributable
|
|
61
|
-
// artifact exists, in a fresh clone of the selected commit: verification
|
|
62
|
-
// left build output in its checkout, which is not release source. Source
|
|
63
|
-
// history was scanned above, including refs a checkout clone leaves out.
|
|
64
|
-
const candidate = resolve(temporary, 'annotation');
|
|
65
|
-
git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
|
|
66
|
-
git(candidate, 'checkout', '--detach', source.commit);
|
|
67
|
-
const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
|
|
68
|
-
annotate(candidate, source, source.commit, staged ? `${source.tag}-publication-candidate` : source.tag);
|
|
69
|
-
const annotations = publicationScan(candidate, source.commit, envFile, project, repository, scanBase);
|
|
70
|
-
report.annotationInspection = annotations;
|
|
71
|
-
if (annotations.status !== 'PASS') {
|
|
72
|
-
report.status = annotations.status;
|
|
73
|
-
report.complete = annotations.complete;
|
|
74
|
-
report.guarantees.historyScanned = annotations.complete;
|
|
75
|
-
report.findings.push(...annotations.findings);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
56
|
+
let report = exported
|
|
57
|
+
? publicationScan(exported.repo, exported.commit, envFile, project, repository, scanBase)
|
|
58
|
+
: sourcePublicationScan(source, temporary, envFile, project, repository, scanBase, Boolean(tag));
|
|
78
59
|
if (destination) {
|
|
79
60
|
assertPublicTree(exported.repo, exported.commit, source.tree);
|
|
80
61
|
if (report.status === 'PASS' && (report.complete !== true || report.guarantees?.historyScanned !== true ||
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { execFileSync } from 'node:child_process';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
+
import { writeFileSync } from 'node:fs';
|
|
4
|
+
import { git } from './source.mjs';
|
|
5
|
+
import { annotate } from './export.mjs';
|
|
6
|
+
import { runningVersion } from '../../cli/version.mjs';
|
|
3
7
|
|
|
4
8
|
export function validatePublicationDestination(source, project, repository) {
|
|
5
9
|
try {
|
|
@@ -9,14 +13,42 @@ export function validatePublicationDestination(source, project, repository) {
|
|
|
9
13
|
} catch { throw Error('Publication destination must agree with the committed edition and project identities'); }
|
|
10
14
|
}
|
|
11
15
|
|
|
12
|
-
export function publicationScan(repo, ref, envFile, project, repository, publicBase) {
|
|
16
|
+
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
|
|
17
|
+
const started = performance.now();
|
|
18
|
+
let report;
|
|
13
19
|
try {
|
|
14
|
-
|
|
20
|
+
report = JSON.parse(execFileSync(process.execPath,
|
|
15
21
|
[resolve(import.meta.dirname, '../../scripts/publish-check.mjs'), '--repo', repo, '--ref', ref,
|
|
16
|
-
...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : [])
|
|
22
|
+
...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : []),
|
|
23
|
+
...(annotationFile ? ['--annotation-file', annotationFile] : [])],
|
|
17
24
|
{ stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 128 * 1024 * 1024 }));
|
|
18
25
|
} catch (error) {
|
|
19
|
-
try {
|
|
26
|
+
try { report = JSON.parse(error.stdout); }
|
|
20
27
|
catch { throw Error('Publication inspection did not produce a report'); }
|
|
21
28
|
}
|
|
29
|
+
report.invocation = { caller: 'release', scans: 1, stagedAnnotation: Boolean(annotationFile), version: runningVersion(),
|
|
30
|
+
seconds: (performance.now() - started) / 1000 };
|
|
31
|
+
return report;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// This is the ordinary release gate, also usable without installing/building a
|
|
35
|
+
// seed. The scanner sees the original checkout and every source ref; only the
|
|
36
|
+
// new annotation is read from the clean candidate, never verification output.
|
|
37
|
+
export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged) {
|
|
38
|
+
const started = performance.now();
|
|
39
|
+
let annotationFile;
|
|
40
|
+
if (tagged) {
|
|
41
|
+
const candidate = resolve(temporary, 'annotation');
|
|
42
|
+
git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
|
|
43
|
+
git(candidate, 'checkout', '--detach', source.commit);
|
|
44
|
+
const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
|
|
45
|
+
const name = staged ? `${source.tag}-publication-candidate` : source.tag;
|
|
46
|
+
annotate(candidate, source, source.commit, name);
|
|
47
|
+
annotationFile = resolve(temporary, 'annotation.tag');
|
|
48
|
+
writeFileSync(annotationFile, git(candidate, 'cat-file', 'tag', `refs/tags/${name}`), { mode: 0o600 });
|
|
49
|
+
}
|
|
50
|
+
const stagingSeconds = (performance.now() - started) / 1000;
|
|
51
|
+
const report = publicationScan(source.repo, source.commit, envFile, project, repository, publicBase, annotationFile);
|
|
52
|
+
Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
|
|
53
|
+
return report;
|
|
22
54
|
}
|
package/package.json
CHANGED
|
@@ -4,6 +4,20 @@ See [setup, command and scope](../publish-check-README.md).
|
|
|
4
4
|
|
|
5
5
|
`git_history.py` delegates revision resolution, traversal, tree inventory and
|
|
6
6
|
blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
|
|
7
|
+
Before any tree inventory asks for blob sizes, `history_hydration.py` counts
|
|
8
|
+
missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
|
|
9
|
+
from its promisor remote. A failed fetch or remaining missing object makes the
|
|
10
|
+
inspection incomplete. `history.hydration` reports the missing counts, fetch
|
|
11
|
+
count and elapsed time; subsequent history reads disable lazy fetching.
|
|
12
|
+
|
|
13
|
+
Reports include implementation version, Python version and a digest of the
|
|
14
|
+
Python scanner source, plus total and per-phase wall times and call counts.
|
|
15
|
+
The release gate uses this same scanner entry point. When comparing it with a
|
|
16
|
+
standalone run, compare implementation digests as well as source trees:
|
|
17
|
+
`rer` inside a checkout runs the rer version that checkout pins, even when a
|
|
18
|
+
newer launcher started it. A tagged exact-source release stages its actual Git tag object in an isolated
|
|
19
|
+
clone and includes its message in the same full source scan. `annotationInspection`
|
|
20
|
+
is the staged annotation subset of that result; no second history walk is needed.
|
|
7
21
|
`gitleaks.py` runs the pinned scanner in git, directory and stdin modes and keeps
|
|
8
22
|
its report private until translating it into the value-free v3 envelope.
|
|
9
23
|
Hexadecimal credential assignments also use a Gitleaks rule without generic
|
|
@@ -20,12 +20,12 @@ def main():
|
|
|
20
20
|
parser = SafeParser(description='Scan working files and full history with system Git and pinned Gitleaks.')
|
|
21
21
|
parser.add_argument('--repo', required=True)
|
|
22
22
|
parser.add_argument('--ref', default='HEAD')
|
|
23
|
-
for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir'):
|
|
23
|
+
for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir', 'annotation-file'):
|
|
24
24
|
parser.add_argument('--' + option)
|
|
25
25
|
parser.add_argument('--timeout-seconds', type=seconds, default=300)
|
|
26
26
|
args = parser.parse_args()
|
|
27
27
|
report = scan(args.repo, args.ref, args.public_base, args.env_file, args.project, args.repository,
|
|
28
|
-
cache_dir=args.cache_dir, timeout=args.timeout_seconds)
|
|
28
|
+
cache_dir=args.cache_dir, timeout=args.timeout_seconds, annotation_file=args.annotation_file)
|
|
29
29
|
print(json.dumps(report))
|
|
30
30
|
return {'PASS': 0, 'FAIL': 1, 'INCOMPLETE': 2}[report['status']]
|
|
31
31
|
|
|
@@ -39,6 +39,9 @@ class History:
|
|
|
39
39
|
raise PublicBaseError('public-base.fetch-failed') from None
|
|
40
40
|
public_revisions = [self.public_commit, *sorted(self.public_tags)] if public_base else []
|
|
41
41
|
self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
|
|
42
|
+
from history_hydration import prepare
|
|
43
|
+
self.hydration = prepare(self)
|
|
44
|
+
self.env['GIT_NO_LAZY_FETCH'] = '1'
|
|
42
45
|
self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
|
|
43
46
|
self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
|
|
44
47
|
self.trees = {}
|
|
@@ -41,7 +41,7 @@ def inspect(binary, mode, repo=None, revisions=None, data=None, timeout=300, red
|
|
|
41
41
|
if repo is not None:
|
|
42
42
|
args.append(repo)
|
|
43
43
|
env = {k: v for k, v in os.environ.items() if not k.startswith(('GITLEAKS_', 'GIT_'))}
|
|
44
|
-
env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0')
|
|
44
|
+
env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0', GIT_NO_LAZY_FETCH='1')
|
|
45
45
|
try:
|
|
46
46
|
result = subprocess.run(args, input=data, capture_output=True, env=env, timeout=timeout)
|
|
47
47
|
if result.returncode not in (0, 1):
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Fetch missing promised history in one pack before inspecting any blobs."""
|
|
2
|
+
from time import monotonic
|
|
3
|
+
from git_history import InspectionError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def prepare(history):
|
|
7
|
+
started = monotonic()
|
|
8
|
+
def missing():
|
|
9
|
+
return sum(line.startswith(b'?') for line in
|
|
10
|
+
history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines())
|
|
11
|
+
before = missing()
|
|
12
|
+
fetches = 0
|
|
13
|
+
if before:
|
|
14
|
+
# A blob:none clone already owns the commits and trees. ls-tree -l and
|
|
15
|
+
# cat-file would otherwise ask its promisor for one blob at a time.
|
|
16
|
+
remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
|
|
17
|
+
names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
|
|
18
|
+
for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
|
|
19
|
+
if len(names) != 1:
|
|
20
|
+
raise InspectionError('history.promisor-ambiguous')
|
|
21
|
+
history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
|
|
22
|
+
'--no-recurse-submodules', '--no-write-fetch-head', '--', names[0])
|
|
23
|
+
fetches = 1
|
|
24
|
+
after = missing() if before else 0
|
|
25
|
+
if after:
|
|
26
|
+
raise InspectionError('history.missing-blob')
|
|
27
|
+
return dict(missingBefore=before, missingAfter=after, fetches=fetches,
|
|
28
|
+
seconds=round(monotonic() - started, 6))
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Scan current files and full history using native Git and pinned Gitleaks."""
|
|
2
2
|
from pathlib import Path
|
|
3
|
+
from scan_timings import Timings
|
|
4
|
+
import staged_annotation
|
|
3
5
|
from git_history import History, InspectionError
|
|
4
6
|
from public_base import PublicBaseError
|
|
5
7
|
from gitleaks import executable, inspect as secrets, rule as secret_rule
|
|
@@ -16,15 +18,18 @@ from content_identities import ContentIdentities, candidate as identity_candidat
|
|
|
16
18
|
from checksum_maps import ChecksumMaps
|
|
17
19
|
|
|
18
20
|
|
|
19
|
-
def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300):
|
|
21
|
+
def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300, annotation_file=None):
|
|
20
22
|
report = new_report(public_base is not None)
|
|
21
|
-
|
|
23
|
+
timings = Timings(report)
|
|
24
|
+
history = staged = None
|
|
22
25
|
add = Findings(report, {}, frozenset()).add
|
|
23
26
|
try:
|
|
24
27
|
hashes = load_hashes(env_file)
|
|
25
28
|
report['guarantees']['environmentChecked'] = env_file is not None
|
|
26
29
|
binary = executable()
|
|
27
|
-
history = History
|
|
30
|
+
history = timings.call('history.prepare', History, repo, ref, public_base)
|
|
31
|
+
report['history']['hydration'] = history.hydration
|
|
32
|
+
staged = staged_annotation.load(annotation_file, history.commit)
|
|
28
33
|
report['source'] = dict(commit=history.commit, tree=history.commit_tree(history.commit).decode())
|
|
29
34
|
report['publicBase'] = dict(ref=history.public_label, commit=history.public_commit) if public_base else None
|
|
30
35
|
config = load_config(history.selected(CONFIG))
|
|
@@ -54,7 +59,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
54
59
|
tree = history.commit_tree(commit).decode()
|
|
55
60
|
seen_trees.add(tree)
|
|
56
61
|
scope = 'release-tree' if commit == history.commit else 'pre-existing' if commit in history.public_commits else 'newly-public'
|
|
57
|
-
config_tree, invalid =
|
|
62
|
+
config_tree, invalid = timings.call('history.context', context, history, commit, config)
|
|
58
63
|
for path in invalid:
|
|
59
64
|
add(INVALID if (project is not None or repository is not None) and path == 'edition.json' else 'scan.identity-invalid', path, commit, scope=scope)
|
|
60
65
|
license_entry = entries_by_commit[commit].get('LICENSE')
|
|
@@ -76,7 +81,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
76
81
|
local = dependencies[pair] & config_tree['localPaths']
|
|
77
82
|
key = (path, oid, signature, local)
|
|
78
83
|
if key not in inspected:
|
|
79
|
-
inspected[key] =
|
|
84
|
+
inspected[key] = timings.call('history.text', text_rules, history.blob(oid), path, config_tree, hashes)
|
|
80
85
|
rules, provenance, references = inspected[key]
|
|
81
86
|
entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
|
|
82
87
|
for rule in rules:
|
|
@@ -86,7 +91,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
86
91
|
if provenance:
|
|
87
92
|
add('provenance.declared-source', path, commit, oid, entry_scope)
|
|
88
93
|
report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
|
|
89
|
-
selected_context, _ = context
|
|
94
|
+
selected_context, _ = timings.call('working-tree.context', context, history, history.commit, config)
|
|
90
95
|
working_paths = []
|
|
91
96
|
working_identities = {}
|
|
92
97
|
for path, data in working_files(Path(history.repo)):
|
|
@@ -98,7 +103,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
98
103
|
# their known immutable identity; new bytes have no history identity.
|
|
99
104
|
entry = entries_by_commit[history.commit].get(path)
|
|
100
105
|
oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
|
|
101
|
-
rules, _, references = text_rules
|
|
106
|
+
rules, _, references = timings.call('working-tree.text', text_rules, data, path, selected_context, hashes)
|
|
102
107
|
public_oid = public_paths.get(path)
|
|
103
108
|
scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
|
|
104
109
|
for rule in rules:
|
|
@@ -110,11 +115,11 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
110
115
|
add(rule, path, history.commit, oid, 'working-tree')
|
|
111
116
|
if not (Path(history.repo) / 'LICENSE').is_file():
|
|
112
117
|
add('license.missing', 'LICENSE', scope='working-tree')
|
|
113
|
-
identities = {'working-tree': ContentIdentities
|
|
118
|
+
identities = {'working-tree': timings.call('identities.prepare', ContentIdentities, working_identities,
|
|
114
119
|
lambda path: working_regular(Path(history.repo), path))}
|
|
115
120
|
checksums = {}
|
|
116
121
|
for mode in ('git', 'dir'):
|
|
117
|
-
for hit in secrets
|
|
122
|
+
for hit in timings.call('gitleaks.' + mode, secrets, binary, mode, history.repo, history.revisions, timeout=timeout, redact=False):
|
|
118
123
|
path, commit = hit.get('File', '.'), hit.get('Commit') or history.commit
|
|
119
124
|
if mode == 'dir' and Path(path).is_absolute():
|
|
120
125
|
path = Path(path).resolve().relative_to(Path(history.repo).resolve()).as_posix()
|
|
@@ -133,12 +138,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
133
138
|
entry = scoped_entries.get(path)
|
|
134
139
|
return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
|
|
135
140
|
if identity_scope not in checksums:
|
|
136
|
-
checksums[identity_scope] = ChecksumMaps
|
|
141
|
+
checksums[identity_scope] = timings.call('checksums.prepare', ChecksumMaps,
|
|
137
142
|
working_paths if mode == 'dir' else scoped_entries,
|
|
138
143
|
(lambda path: working_regular(Path(history.repo), path)) if mode == 'dir' else read_regular)
|
|
139
144
|
if identity_candidate(path):
|
|
140
145
|
if identity_scope not in identities:
|
|
141
|
-
identities[identity_scope] = ContentIdentities
|
|
146
|
+
identities[identity_scope] = timings.call('identities.prepare', ContentIdentities, {
|
|
142
147
|
p: history.blob(e['oid']) for p, e in scoped_entries.items()
|
|
143
148
|
if identity_candidate(p) and e['kind'] == 'blob' and e['mode'] in ('100644', '100755')}, read_regular)
|
|
144
149
|
if identities[identity_scope].matches(path, hit):
|
|
@@ -162,10 +167,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
162
167
|
scope = 'working-tree' if path in working_paths else 'pre-existing' if commit in history.public_commits else 'newly-public'
|
|
163
168
|
lines.append((count, count + length, path, commit, scope))
|
|
164
169
|
count += length
|
|
165
|
-
for message_type, commit, data in
|
|
170
|
+
for message_type, commit, data in staged_annotation.messages(history, staged):
|
|
166
171
|
path = '@' + message_type + '-message'
|
|
167
172
|
report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
|
|
168
|
-
rules, _, references = text_rules
|
|
173
|
+
rules, _, references = timings.call('messages.text', text_rules, data, path, selected_context, hashes,
|
|
169
174
|
commit_message=message_type == 'commit')
|
|
170
175
|
scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
|
|
171
176
|
for rule in rules:
|
|
@@ -176,11 +181,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
176
181
|
length = (data + b'\n').count(b'\n')
|
|
177
182
|
lines.append((count, count + length, path, commit, scope))
|
|
178
183
|
count += length
|
|
179
|
-
for hit in secrets
|
|
184
|
+
for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=b''.join(messages), timeout=timeout):
|
|
180
185
|
line = hit.get('StartLine', 0)
|
|
181
186
|
location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
|
|
182
187
|
add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
|
|
183
|
-
states = GitHubResolver().resolve_all
|
|
188
|
+
states = timings.call('github.references', GitHubResolver().resolve_all,
|
|
189
|
+
{repository for references, *_ in deferred for repository, _, _ in references})
|
|
184
190
|
for references, path, commit, oid, scope in deferred:
|
|
185
191
|
for rule in reference_rules(references, states):
|
|
186
192
|
add(rule, path, commit, oid, scope)
|
|
@@ -197,4 +203,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
197
203
|
finally:
|
|
198
204
|
if history:
|
|
199
205
|
history.close()
|
|
200
|
-
|
|
206
|
+
timings.finish()
|
|
207
|
+
finish(report)
|
|
208
|
+
staged_annotation.summary(report, staged)
|
|
209
|
+
return report
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Numeric scan costs and implementation identity, without source file values."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import platform
|
|
6
|
+
from time import monotonic
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def implementation():
|
|
10
|
+
directory = Path(__file__).parent
|
|
11
|
+
digest = hashlib.sha256()
|
|
12
|
+
for path in sorted(directory.glob('*.py')):
|
|
13
|
+
if path.name.startswith(('test', 'coverage_')):
|
|
14
|
+
continue
|
|
15
|
+
digest.update(path.name.encode() + b'\0' + path.read_bytes())
|
|
16
|
+
root = directory.parent.parent
|
|
17
|
+
for path, key in ((root / 'inventory.json', 'cliVersion'),
|
|
18
|
+
(root / 'package.json', 'version'), (root / 'global/package.json', 'version')):
|
|
19
|
+
if path.is_file():
|
|
20
|
+
version = json.loads(path.read_text()).get(key)
|
|
21
|
+
if version:
|
|
22
|
+
break
|
|
23
|
+
else:
|
|
24
|
+
version = None
|
|
25
|
+
return dict(version=version, sha256=digest.hexdigest(), python=platform.python_version())
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Timings:
|
|
29
|
+
def __init__(self, report):
|
|
30
|
+
self.started = monotonic()
|
|
31
|
+
self.values = report['timings'] = dict(totalSeconds=0, phases={})
|
|
32
|
+
report['implementation'] = implementation()
|
|
33
|
+
|
|
34
|
+
def call(self, name, function, *args, **kwargs):
|
|
35
|
+
started = monotonic()
|
|
36
|
+
try:
|
|
37
|
+
return function(*args, **kwargs)
|
|
38
|
+
finally:
|
|
39
|
+
phase = self.values['phases'].setdefault(name, dict(calls=0, seconds=0))
|
|
40
|
+
phase['calls'] += 1
|
|
41
|
+
phase['seconds'] += monotonic() - started
|
|
42
|
+
|
|
43
|
+
def finish(self):
|
|
44
|
+
self.values['totalSeconds'] = round(monotonic() - self.started, 6)
|
|
45
|
+
for phase in self.values['phases'].values():
|
|
46
|
+
phase['seconds'] = round(phase['seconds'], 6)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Include the actual staged Git tag in the source scan's message policy."""
|
|
2
|
+
import hashlib
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from git_history import InspectionError
|
|
5
|
+
from report import new_report, finish
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load(path, commit):
|
|
9
|
+
if path is None:
|
|
10
|
+
return None
|
|
11
|
+
raw = Path(path).read_bytes()
|
|
12
|
+
header, separator, message = raw.partition(b'\n\n')
|
|
13
|
+
if not separator or not header.startswith(('object ' + commit + '\ntype commit\n').encode()):
|
|
14
|
+
raise InspectionError('scan.annotation-invalid')
|
|
15
|
+
digest = hashlib.sha1 if len(commit) == 40 else hashlib.sha256
|
|
16
|
+
oid = digest(b'tag ' + str(len(raw)).encode() + b'\0' + raw).hexdigest()
|
|
17
|
+
return dict(oid=oid, message=message)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def messages(history, staged):
|
|
21
|
+
yield from history.messages()
|
|
22
|
+
if staged:
|
|
23
|
+
yield 'tag', staged['oid'], staged['message']
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def summary(report, staged):
|
|
27
|
+
if not staged:
|
|
28
|
+
return
|
|
29
|
+
annotation = new_report(report.get('publicBase') is not None)
|
|
30
|
+
annotation.update(source=report['source'], publicBase=report.get('publicBase'),
|
|
31
|
+
implementation=report['implementation'])
|
|
32
|
+
annotation['history'].update(scope='staged-annotation', tagMessages=1)
|
|
33
|
+
annotation['guarantees']['environmentChecked'] = report['guarantees']['environmentChecked']
|
|
34
|
+
for bucket in ('findings', 'preExisting', 'warnings', 'fixtures', 'reviewed', 'info'):
|
|
35
|
+
annotation[bucket] = [{**item, 'commit': staged['oid'], 'commits': [staged['oid']]}
|
|
36
|
+
for item in report[bucket] if staged['oid'] in item['commits']]
|
|
37
|
+
if not report['complete']:
|
|
38
|
+
annotation['findings'].append(dict(rule='scan.source-incomplete', path='.'))
|
|
39
|
+
report['annotationInspection'] = finish(annotation)
|
|
@@ -3,6 +3,7 @@ import { spawn } from 'node:child_process';
|
|
|
3
3
|
import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
+
import { runningVersion } from '../cli/version.mjs';
|
|
6
7
|
|
|
7
8
|
const args = process.argv.slice(2);
|
|
8
9
|
let seconds = 300;
|
|
@@ -54,5 +55,7 @@ if (reason) report = {
|
|
|
54
55
|
history: { commits: 0, trees: 0, blobs: 0 },
|
|
55
56
|
guarantees: { historyScanned: false, visualRecognition: false, environmentChecked: false },
|
|
56
57
|
};
|
|
58
|
+
report.implementation ??= {};
|
|
59
|
+
report.implementation.version = runningVersion();
|
|
57
60
|
process.stdout.write(JSON.stringify(report) + '\n');
|
|
58
61
|
process.exitCode = code;
|