@runeditrun/rer 0.2.58 → 0.2.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/global/CHANGELOG.md +5 -0
- package/modules/demo-host/native/loader.mjs +9 -4
- package/modules/release/produce.mjs +4 -23
- package/modules/release/publication-scan.mjs +36 -4
- package/package.json +1 -1
- package/scripts/publish-check/README.md +14 -0
- package/scripts/publish-check/cli.py +2 -2
- package/scripts/publish-check/git_history.py +3 -0
- package/scripts/publish-check/gitleaks.py +1 -1
- package/scripts/publish-check/history_hydration.py +28 -0
- package/scripts/publish-check/history_scan.py +25 -16
- package/scripts/publish-check/scan_timings.py +46 -0
- package/scripts/publish-check/staged_annotation.py +39 -0
- package/scripts/publish-check.mjs +3 -0
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.59
|
|
4
|
+
|
|
5
|
+
- **Release publication scans complete on large seeds and partial clones.** `rer release` hydrates missing history blobs in one fetch and inspects the staged release annotation in the same scan. Orbit finishes in 82 seconds instead of exceeding the 300-second budget, with identical findings. A seed must adopt the new rer through its pin for its release command to use it (#278).
|
|
6
|
+
- **Native demos decode binary modules when needed.** WebAssembly and data modules decode lazily inside the Worker Loader factory with native base64 support, reducing host startup work while preserving their bytes (#280).
|
|
7
|
+
|
|
3
8
|
## 0.2.58
|
|
4
9
|
|
|
5
10
|
- **Deploy dry runs validate required target secrets.** `rer deploy --dry-run` uses the real deploy secret-binding resolver and refuses with the same exit and message when required bindings are missing. It names every missing binding and its environment input, accepts secrets already held by the Worker, and makes no target writes. Missing credentials or read permission also refuse (#275).
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
import { WorkerEntrypoint } from 'cloudflare:workers';
|
|
2
|
+
import { Buffer } from 'node:buffer';
|
|
2
3
|
import demoEgress from '../../container/demo-egress.mjs';
|
|
3
4
|
import bundle from './bundle.mjs';
|
|
4
5
|
|
|
5
|
-
//
|
|
6
|
-
const bytes = text =>
|
|
7
|
-
const
|
|
6
|
+
// Decode only when a Loader factory needs the modules, once per host isolate.
|
|
7
|
+
const bytes = text => {
|
|
8
|
+
const buffer = Buffer.from(text, 'base64');
|
|
9
|
+
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
|
|
10
|
+
};
|
|
11
|
+
let modules;
|
|
12
|
+
const loaderModules = () => modules ??= Object.fromEntries(Object.entries(bundle.modules).map(([name, module]) =>
|
|
8
13
|
[name, 'wasm' in module ? { wasm: bytes(module.wasm) } : 'data' in module ? { data: bytes(module.data) } : module]));
|
|
9
14
|
|
|
10
15
|
export function loaded(env, ctx, id, runtime) {
|
|
11
16
|
return env.LOADER.get(`${bundle.digest}:${id}`, () => ({
|
|
12
17
|
compatibilityDate: bundle.compatibilityDate, compatibilityFlags: bundle.compatibilityFlags,
|
|
13
|
-
mainModule: '_rer/isolate.js', modules,
|
|
18
|
+
mainModule: '_rer/isolate.js', modules: loaderModules(),
|
|
14
19
|
globalOutbound: ctx.exports.DemoEgress({ props: { id } }),
|
|
15
20
|
env: { ...runtime, RER_NATIVE_CONFIG: bundle.bindings, RER_NATIVE_HOST: ctx.exports.HostBridge({ props: { id, key: runtime.RER_NATIVE_KEY } }), [bundle.bindings.assets]: ctx.exports.DemoAssets({}) },
|
|
16
21
|
}));
|
|
@@ -5,7 +5,7 @@ import { inspectSource, newOutput, git } from './source.mjs';
|
|
|
5
5
|
import { verifySource } from './source-verification.mjs';
|
|
6
6
|
import { catalogue, writeCatalogue, sha256, json } from './catalogue.mjs';
|
|
7
7
|
import { publicExport, cleanHistory, annotate, releaseMessage } from './export.mjs';
|
|
8
|
-
import { publicationScan, validatePublicationDestination } from './publication-scan.mjs';
|
|
8
|
+
import { publicationScan, sourcePublicationScan, validatePublicationDestination } from './publication-scan.mjs';
|
|
9
9
|
import { publicDestination, requirePublicDestination, remotePublicHead, publicChild, assertPublicTree } from './public-child.mjs';
|
|
10
10
|
import { writePublishHandoff } from './publish-handoff.mjs';
|
|
11
11
|
import { releasePublicBase } from './public-base.mjs';
|
|
@@ -53,28 +53,9 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
53
53
|
// when available, retaining the selected source commit and tag identity.
|
|
54
54
|
const exported = destination ? publicChild(source, temporary, destination, publicParent, Boolean(tag))
|
|
55
55
|
: clean ? cleanHistory(source, temporary, Boolean(tag)) : null;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
if (!exported && tag && report.status === 'PASS') {
|
|
60
|
-
// Stage the exact final annotation before any source tag or distributable
|
|
61
|
-
// artifact exists, in a fresh clone of the selected commit: verification
|
|
62
|
-
// left build output in its checkout, which is not release source. Source
|
|
63
|
-
// history was scanned above, including refs a checkout clone leaves out.
|
|
64
|
-
const candidate = resolve(temporary, 'annotation');
|
|
65
|
-
git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
|
|
66
|
-
git(candidate, 'checkout', '--detach', source.commit);
|
|
67
|
-
const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
|
|
68
|
-
annotate(candidate, source, source.commit, staged ? `${source.tag}-publication-candidate` : source.tag);
|
|
69
|
-
const annotations = publicationScan(candidate, source.commit, envFile, project, repository, scanBase);
|
|
70
|
-
report.annotationInspection = annotations;
|
|
71
|
-
if (annotations.status !== 'PASS') {
|
|
72
|
-
report.status = annotations.status;
|
|
73
|
-
report.complete = annotations.complete;
|
|
74
|
-
report.guarantees.historyScanned = annotations.complete;
|
|
75
|
-
report.findings.push(...annotations.findings);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
56
|
+
let report = exported
|
|
57
|
+
? publicationScan(exported.repo, exported.commit, envFile, project, repository, scanBase)
|
|
58
|
+
: sourcePublicationScan(source, temporary, envFile, project, repository, scanBase, Boolean(tag));
|
|
78
59
|
if (destination) {
|
|
79
60
|
assertPublicTree(exported.repo, exported.commit, source.tree);
|
|
80
61
|
if (report.status === 'PASS' && (report.complete !== true || report.guarantees?.historyScanned !== true ||
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { execFileSync } from 'node:child_process';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
+
import { writeFileSync } from 'node:fs';
|
|
4
|
+
import { git } from './source.mjs';
|
|
5
|
+
import { annotate } from './export.mjs';
|
|
6
|
+
import { runningVersion } from '../../cli/version.mjs';
|
|
3
7
|
|
|
4
8
|
export function validatePublicationDestination(source, project, repository) {
|
|
5
9
|
try {
|
|
@@ -9,14 +13,42 @@ export function validatePublicationDestination(source, project, repository) {
|
|
|
9
13
|
} catch { throw Error('Publication destination must agree with the committed edition and project identities'); }
|
|
10
14
|
}
|
|
11
15
|
|
|
12
|
-
export function publicationScan(repo, ref, envFile, project, repository, publicBase) {
|
|
16
|
+
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
|
|
17
|
+
const started = performance.now();
|
|
18
|
+
let report;
|
|
13
19
|
try {
|
|
14
|
-
|
|
20
|
+
report = JSON.parse(execFileSync(process.execPath,
|
|
15
21
|
[resolve(import.meta.dirname, '../../scripts/publish-check.mjs'), '--repo', repo, '--ref', ref,
|
|
16
|
-
...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : [])
|
|
22
|
+
...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : []),
|
|
23
|
+
...(annotationFile ? ['--annotation-file', annotationFile] : [])],
|
|
17
24
|
{ stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 128 * 1024 * 1024 }));
|
|
18
25
|
} catch (error) {
|
|
19
|
-
try {
|
|
26
|
+
try { report = JSON.parse(error.stdout); }
|
|
20
27
|
catch { throw Error('Publication inspection did not produce a report'); }
|
|
21
28
|
}
|
|
29
|
+
report.invocation = { caller: 'release', scans: 1, stagedAnnotation: Boolean(annotationFile), version: runningVersion(),
|
|
30
|
+
seconds: (performance.now() - started) / 1000 };
|
|
31
|
+
return report;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// This is the ordinary release gate, also usable without installing/building a
|
|
35
|
+
// seed. The scanner sees the original checkout and every source ref; only the
|
|
36
|
+
// new annotation is read from the clean candidate, never verification output.
|
|
37
|
+
export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged) {
|
|
38
|
+
const started = performance.now();
|
|
39
|
+
let annotationFile;
|
|
40
|
+
if (tagged) {
|
|
41
|
+
const candidate = resolve(temporary, 'annotation');
|
|
42
|
+
git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
|
|
43
|
+
git(candidate, 'checkout', '--detach', source.commit);
|
|
44
|
+
const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
|
|
45
|
+
const name = staged ? `${source.tag}-publication-candidate` : source.tag;
|
|
46
|
+
annotate(candidate, source, source.commit, name);
|
|
47
|
+
annotationFile = resolve(temporary, 'annotation.tag');
|
|
48
|
+
writeFileSync(annotationFile, git(candidate, 'cat-file', 'tag', `refs/tags/${name}`), { mode: 0o600 });
|
|
49
|
+
}
|
|
50
|
+
const stagingSeconds = (performance.now() - started) / 1000;
|
|
51
|
+
const report = publicationScan(source.repo, source.commit, envFile, project, repository, publicBase, annotationFile);
|
|
52
|
+
Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
|
|
53
|
+
return report;
|
|
22
54
|
}
|
package/package.json
CHANGED
|
@@ -4,6 +4,20 @@ See [setup, command and scope](../publish-check-README.md).
|
|
|
4
4
|
|
|
5
5
|
`git_history.py` delegates revision resolution, traversal, tree inventory and
|
|
6
6
|
blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
|
|
7
|
+
Before any tree inventory asks for blob sizes, `history_hydration.py` counts
|
|
8
|
+
missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
|
|
9
|
+
from its promisor remote. A failed fetch or remaining missing object makes the
|
|
10
|
+
inspection incomplete. `history.hydration` reports the missing counts, fetch
|
|
11
|
+
count and elapsed time; subsequent history reads disable lazy fetching.
|
|
12
|
+
|
|
13
|
+
Reports include implementation version, Python version and a digest of the
|
|
14
|
+
Python scanner source, plus total and per-phase wall times and call counts.
|
|
15
|
+
The release gate uses this same scanner entry point. When comparing it with a
|
|
16
|
+
standalone run, compare implementation digests as well as source trees:
|
|
17
|
+
`rer` inside a checkout runs the rer version that checkout pins, even when a
|
|
18
|
+
newer launcher started it. A tagged exact-source release stages its actual Git tag object in an isolated
|
|
19
|
+
clone and includes its message in the same full source scan. `annotationInspection`
|
|
20
|
+
is the staged annotation subset of that result; no second history walk is needed.
|
|
7
21
|
`gitleaks.py` runs the pinned scanner in git, directory and stdin modes and keeps
|
|
8
22
|
its report private until translating it into the value-free v3 envelope.
|
|
9
23
|
Hexadecimal credential assignments also use a Gitleaks rule without generic
|
|
@@ -20,12 +20,12 @@ def main():
|
|
|
20
20
|
parser = SafeParser(description='Scan working files and full history with system Git and pinned Gitleaks.')
|
|
21
21
|
parser.add_argument('--repo', required=True)
|
|
22
22
|
parser.add_argument('--ref', default='HEAD')
|
|
23
|
-
for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir'):
|
|
23
|
+
for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir', 'annotation-file'):
|
|
24
24
|
parser.add_argument('--' + option)
|
|
25
25
|
parser.add_argument('--timeout-seconds', type=seconds, default=300)
|
|
26
26
|
args = parser.parse_args()
|
|
27
27
|
report = scan(args.repo, args.ref, args.public_base, args.env_file, args.project, args.repository,
|
|
28
|
-
cache_dir=args.cache_dir, timeout=args.timeout_seconds)
|
|
28
|
+
cache_dir=args.cache_dir, timeout=args.timeout_seconds, annotation_file=args.annotation_file)
|
|
29
29
|
print(json.dumps(report))
|
|
30
30
|
return {'PASS': 0, 'FAIL': 1, 'INCOMPLETE': 2}[report['status']]
|
|
31
31
|
|
|
@@ -39,6 +39,9 @@ class History:
|
|
|
39
39
|
raise PublicBaseError('public-base.fetch-failed') from None
|
|
40
40
|
public_revisions = [self.public_commit, *sorted(self.public_tags)] if public_base else []
|
|
41
41
|
self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
|
|
42
|
+
from history_hydration import prepare
|
|
43
|
+
self.hydration = prepare(self)
|
|
44
|
+
self.env['GIT_NO_LAZY_FETCH'] = '1'
|
|
42
45
|
self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
|
|
43
46
|
self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
|
|
44
47
|
self.trees = {}
|
|
@@ -41,7 +41,7 @@ def inspect(binary, mode, repo=None, revisions=None, data=None, timeout=300, red
|
|
|
41
41
|
if repo is not None:
|
|
42
42
|
args.append(repo)
|
|
43
43
|
env = {k: v for k, v in os.environ.items() if not k.startswith(('GITLEAKS_', 'GIT_'))}
|
|
44
|
-
env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0')
|
|
44
|
+
env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0', GIT_NO_LAZY_FETCH='1')
|
|
45
45
|
try:
|
|
46
46
|
result = subprocess.run(args, input=data, capture_output=True, env=env, timeout=timeout)
|
|
47
47
|
if result.returncode not in (0, 1):
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Fetch missing promised history in one pack before inspecting any blobs."""
|
|
2
|
+
from time import monotonic
|
|
3
|
+
from git_history import InspectionError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def prepare(history):
|
|
7
|
+
started = monotonic()
|
|
8
|
+
def missing():
|
|
9
|
+
return sum(line.startswith(b'?') for line in
|
|
10
|
+
history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines())
|
|
11
|
+
before = missing()
|
|
12
|
+
fetches = 0
|
|
13
|
+
if before:
|
|
14
|
+
# A blob:none clone already owns the commits and trees. ls-tree -l and
|
|
15
|
+
# cat-file would otherwise ask its promisor for one blob at a time.
|
|
16
|
+
remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
|
|
17
|
+
names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
|
|
18
|
+
for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
|
|
19
|
+
if len(names) != 1:
|
|
20
|
+
raise InspectionError('history.promisor-ambiguous')
|
|
21
|
+
history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
|
|
22
|
+
'--no-recurse-submodules', '--no-write-fetch-head', '--', names[0])
|
|
23
|
+
fetches = 1
|
|
24
|
+
after = missing() if before else 0
|
|
25
|
+
if after:
|
|
26
|
+
raise InspectionError('history.missing-blob')
|
|
27
|
+
return dict(missingBefore=before, missingAfter=after, fetches=fetches,
|
|
28
|
+
seconds=round(monotonic() - started, 6))
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Scan current files and full history using native Git and pinned Gitleaks."""
|
|
2
2
|
from pathlib import Path
|
|
3
|
+
from scan_timings import Timings
|
|
4
|
+
import staged_annotation
|
|
3
5
|
from git_history import History, InspectionError
|
|
4
6
|
from public_base import PublicBaseError
|
|
5
7
|
from gitleaks import executable, inspect as secrets, rule as secret_rule
|
|
@@ -16,15 +18,18 @@ from content_identities import ContentIdentities, candidate as identity_candidat
|
|
|
16
18
|
from checksum_maps import ChecksumMaps
|
|
17
19
|
|
|
18
20
|
|
|
19
|
-
def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300):
|
|
21
|
+
def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300, annotation_file=None):
|
|
20
22
|
report = new_report(public_base is not None)
|
|
21
|
-
|
|
23
|
+
timings = Timings(report)
|
|
24
|
+
history = staged = None
|
|
22
25
|
add = Findings(report, {}, frozenset()).add
|
|
23
26
|
try:
|
|
24
27
|
hashes = load_hashes(env_file)
|
|
25
28
|
report['guarantees']['environmentChecked'] = env_file is not None
|
|
26
29
|
binary = executable()
|
|
27
|
-
history = History
|
|
30
|
+
history = timings.call('history.prepare', History, repo, ref, public_base)
|
|
31
|
+
report['history']['hydration'] = history.hydration
|
|
32
|
+
staged = staged_annotation.load(annotation_file, history.commit)
|
|
28
33
|
report['source'] = dict(commit=history.commit, tree=history.commit_tree(history.commit).decode())
|
|
29
34
|
report['publicBase'] = dict(ref=history.public_label, commit=history.public_commit) if public_base else None
|
|
30
35
|
config = load_config(history.selected(CONFIG))
|
|
@@ -54,7 +59,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
54
59
|
tree = history.commit_tree(commit).decode()
|
|
55
60
|
seen_trees.add(tree)
|
|
56
61
|
scope = 'release-tree' if commit == history.commit else 'pre-existing' if commit in history.public_commits else 'newly-public'
|
|
57
|
-
config_tree, invalid =
|
|
62
|
+
config_tree, invalid = timings.call('history.context', context, history, commit, config)
|
|
58
63
|
for path in invalid:
|
|
59
64
|
add(INVALID if (project is not None or repository is not None) and path == 'edition.json' else 'scan.identity-invalid', path, commit, scope=scope)
|
|
60
65
|
license_entry = entries_by_commit[commit].get('LICENSE')
|
|
@@ -76,7 +81,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
76
81
|
local = dependencies[pair] & config_tree['localPaths']
|
|
77
82
|
key = (path, oid, signature, local)
|
|
78
83
|
if key not in inspected:
|
|
79
|
-
inspected[key] =
|
|
84
|
+
inspected[key] = timings.call('history.text', text_rules, history.blob(oid), path, config_tree, hashes)
|
|
80
85
|
rules, provenance, references = inspected[key]
|
|
81
86
|
entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
|
|
82
87
|
for rule in rules:
|
|
@@ -86,7 +91,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
86
91
|
if provenance:
|
|
87
92
|
add('provenance.declared-source', path, commit, oid, entry_scope)
|
|
88
93
|
report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
|
|
89
|
-
selected_context, _ = context
|
|
94
|
+
selected_context, _ = timings.call('working-tree.context', context, history, history.commit, config)
|
|
90
95
|
working_paths = []
|
|
91
96
|
working_identities = {}
|
|
92
97
|
for path, data in working_files(Path(history.repo)):
|
|
@@ -98,7 +103,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
98
103
|
# their known immutable identity; new bytes have no history identity.
|
|
99
104
|
entry = entries_by_commit[history.commit].get(path)
|
|
100
105
|
oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
|
|
101
|
-
rules, _, references = text_rules
|
|
106
|
+
rules, _, references = timings.call('working-tree.text', text_rules, data, path, selected_context, hashes)
|
|
102
107
|
public_oid = public_paths.get(path)
|
|
103
108
|
scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
|
|
104
109
|
for rule in rules:
|
|
@@ -110,11 +115,11 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
110
115
|
add(rule, path, history.commit, oid, 'working-tree')
|
|
111
116
|
if not (Path(history.repo) / 'LICENSE').is_file():
|
|
112
117
|
add('license.missing', 'LICENSE', scope='working-tree')
|
|
113
|
-
identities = {'working-tree': ContentIdentities
|
|
118
|
+
identities = {'working-tree': timings.call('identities.prepare', ContentIdentities, working_identities,
|
|
114
119
|
lambda path: working_regular(Path(history.repo), path))}
|
|
115
120
|
checksums = {}
|
|
116
121
|
for mode in ('git', 'dir'):
|
|
117
|
-
for hit in secrets
|
|
122
|
+
for hit in timings.call('gitleaks.' + mode, secrets, binary, mode, history.repo, history.revisions, timeout=timeout, redact=False):
|
|
118
123
|
path, commit = hit.get('File', '.'), hit.get('Commit') or history.commit
|
|
119
124
|
if mode == 'dir' and Path(path).is_absolute():
|
|
120
125
|
path = Path(path).resolve().relative_to(Path(history.repo).resolve()).as_posix()
|
|
@@ -133,12 +138,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
133
138
|
entry = scoped_entries.get(path)
|
|
134
139
|
return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
|
|
135
140
|
if identity_scope not in checksums:
|
|
136
|
-
checksums[identity_scope] = ChecksumMaps
|
|
141
|
+
checksums[identity_scope] = timings.call('checksums.prepare', ChecksumMaps,
|
|
137
142
|
working_paths if mode == 'dir' else scoped_entries,
|
|
138
143
|
(lambda path: working_regular(Path(history.repo), path)) if mode == 'dir' else read_regular)
|
|
139
144
|
if identity_candidate(path):
|
|
140
145
|
if identity_scope not in identities:
|
|
141
|
-
identities[identity_scope] = ContentIdentities
|
|
146
|
+
identities[identity_scope] = timings.call('identities.prepare', ContentIdentities, {
|
|
142
147
|
p: history.blob(e['oid']) for p, e in scoped_entries.items()
|
|
143
148
|
if identity_candidate(p) and e['kind'] == 'blob' and e['mode'] in ('100644', '100755')}, read_regular)
|
|
144
149
|
if identities[identity_scope].matches(path, hit):
|
|
@@ -162,10 +167,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
162
167
|
scope = 'working-tree' if path in working_paths else 'pre-existing' if commit in history.public_commits else 'newly-public'
|
|
163
168
|
lines.append((count, count + length, path, commit, scope))
|
|
164
169
|
count += length
|
|
165
|
-
for message_type, commit, data in
|
|
170
|
+
for message_type, commit, data in staged_annotation.messages(history, staged):
|
|
166
171
|
path = '@' + message_type + '-message'
|
|
167
172
|
report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
|
|
168
|
-
rules, _, references = text_rules
|
|
173
|
+
rules, _, references = timings.call('messages.text', text_rules, data, path, selected_context, hashes,
|
|
169
174
|
commit_message=message_type == 'commit')
|
|
170
175
|
scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
|
|
171
176
|
for rule in rules:
|
|
@@ -176,11 +181,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
176
181
|
length = (data + b'\n').count(b'\n')
|
|
177
182
|
lines.append((count, count + length, path, commit, scope))
|
|
178
183
|
count += length
|
|
179
|
-
for hit in secrets
|
|
184
|
+
for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=b''.join(messages), timeout=timeout):
|
|
180
185
|
line = hit.get('StartLine', 0)
|
|
181
186
|
location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
|
|
182
187
|
add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
|
|
183
|
-
states = GitHubResolver().resolve_all
|
|
188
|
+
states = timings.call('github.references', GitHubResolver().resolve_all,
|
|
189
|
+
{repository for references, *_ in deferred for repository, _, _ in references})
|
|
184
190
|
for references, path, commit, oid, scope in deferred:
|
|
185
191
|
for rule in reference_rules(references, states):
|
|
186
192
|
add(rule, path, commit, oid, scope)
|
|
@@ -197,4 +203,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
197
203
|
finally:
|
|
198
204
|
if history:
|
|
199
205
|
history.close()
|
|
200
|
-
|
|
206
|
+
timings.finish()
|
|
207
|
+
finish(report)
|
|
208
|
+
staged_annotation.summary(report, staged)
|
|
209
|
+
return report
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Numeric scan costs and implementation identity, without source file values."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import platform
|
|
6
|
+
from time import monotonic
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def implementation():
|
|
10
|
+
directory = Path(__file__).parent
|
|
11
|
+
digest = hashlib.sha256()
|
|
12
|
+
for path in sorted(directory.glob('*.py')):
|
|
13
|
+
if path.name.startswith(('test', 'coverage_')):
|
|
14
|
+
continue
|
|
15
|
+
digest.update(path.name.encode() + b'\0' + path.read_bytes())
|
|
16
|
+
root = directory.parent.parent
|
|
17
|
+
for path, key in ((root / 'inventory.json', 'cliVersion'),
|
|
18
|
+
(root / 'package.json', 'version'), (root / 'global/package.json', 'version')):
|
|
19
|
+
if path.is_file():
|
|
20
|
+
version = json.loads(path.read_text()).get(key)
|
|
21
|
+
if version:
|
|
22
|
+
break
|
|
23
|
+
else:
|
|
24
|
+
version = None
|
|
25
|
+
return dict(version=version, sha256=digest.hexdigest(), python=platform.python_version())
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Timings:
|
|
29
|
+
def __init__(self, report):
|
|
30
|
+
self.started = monotonic()
|
|
31
|
+
self.values = report['timings'] = dict(totalSeconds=0, phases={})
|
|
32
|
+
report['implementation'] = implementation()
|
|
33
|
+
|
|
34
|
+
def call(self, name, function, *args, **kwargs):
|
|
35
|
+
started = monotonic()
|
|
36
|
+
try:
|
|
37
|
+
return function(*args, **kwargs)
|
|
38
|
+
finally:
|
|
39
|
+
phase = self.values['phases'].setdefault(name, dict(calls=0, seconds=0))
|
|
40
|
+
phase['calls'] += 1
|
|
41
|
+
phase['seconds'] += monotonic() - started
|
|
42
|
+
|
|
43
|
+
def finish(self):
|
|
44
|
+
self.values['totalSeconds'] = round(monotonic() - self.started, 6)
|
|
45
|
+
for phase in self.values['phases'].values():
|
|
46
|
+
phase['seconds'] = round(phase['seconds'], 6)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Include the actual staged Git tag in the source scan's message policy."""
|
|
2
|
+
import hashlib
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from git_history import InspectionError
|
|
5
|
+
from report import new_report, finish
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load(path, commit):
|
|
9
|
+
if path is None:
|
|
10
|
+
return None
|
|
11
|
+
raw = Path(path).read_bytes()
|
|
12
|
+
header, separator, message = raw.partition(b'\n\n')
|
|
13
|
+
if not separator or not header.startswith(('object ' + commit + '\ntype commit\n').encode()):
|
|
14
|
+
raise InspectionError('scan.annotation-invalid')
|
|
15
|
+
digest = hashlib.sha1 if len(commit) == 40 else hashlib.sha256
|
|
16
|
+
oid = digest(b'tag ' + str(len(raw)).encode() + b'\0' + raw).hexdigest()
|
|
17
|
+
return dict(oid=oid, message=message)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def messages(history, staged):
|
|
21
|
+
yield from history.messages()
|
|
22
|
+
if staged:
|
|
23
|
+
yield 'tag', staged['oid'], staged['message']
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def summary(report, staged):
|
|
27
|
+
if not staged:
|
|
28
|
+
return
|
|
29
|
+
annotation = new_report(report.get('publicBase') is not None)
|
|
30
|
+
annotation.update(source=report['source'], publicBase=report.get('publicBase'),
|
|
31
|
+
implementation=report['implementation'])
|
|
32
|
+
annotation['history'].update(scope='staged-annotation', tagMessages=1)
|
|
33
|
+
annotation['guarantees']['environmentChecked'] = report['guarantees']['environmentChecked']
|
|
34
|
+
for bucket in ('findings', 'preExisting', 'warnings', 'fixtures', 'reviewed', 'info'):
|
|
35
|
+
annotation[bucket] = [{**item, 'commit': staged['oid'], 'commits': [staged['oid']]}
|
|
36
|
+
for item in report[bucket] if staged['oid'] in item['commits']]
|
|
37
|
+
if not report['complete']:
|
|
38
|
+
annotation['findings'].append(dict(rule='scan.source-incomplete', path='.'))
|
|
39
|
+
report['annotationInspection'] = finish(annotation)
|
|
@@ -3,6 +3,7 @@ import { spawn } from 'node:child_process';
|
|
|
3
3
|
import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
+
import { runningVersion } from '../cli/version.mjs';
|
|
6
7
|
|
|
7
8
|
const args = process.argv.slice(2);
|
|
8
9
|
let seconds = 300;
|
|
@@ -54,5 +55,7 @@ if (reason) report = {
|
|
|
54
55
|
history: { commits: 0, trees: 0, blobs: 0 },
|
|
55
56
|
guarantees: { historyScanned: false, visualRecognition: false, environmentChecked: false },
|
|
56
57
|
};
|
|
58
|
+
report.implementation ??= {};
|
|
59
|
+
report.implementation.version = runningVersion();
|
|
57
60
|
process.stdout.write(JSON.stringify(report) + '\n');
|
|
58
61
|
process.exitCode = code;
|