@clear-capabilities/agentic-security-scanner 0.172.0 → 0.173.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/bin/agentic-security.js +37 -0
- package/dist/119.index.js +197 -0
- package/dist/128.index.js +129 -0
- package/dist/1293.index.js +5689 -0
- package/dist/1336.index.js +101 -101
- package/dist/1351.index.js +133 -0
- package/dist/1387.index.js +271 -0
- package/dist/1437.index.js +23 -0
- package/dist/1445.index.js +110 -0
- package/dist/1505.index.js +151 -0
- package/dist/1557.index.js +293 -0
- package/dist/1621.index.js +311 -0
- package/dist/1751.index.js +160 -0
- package/dist/1765.index.js +727 -0
- package/dist/1861.index.js +690 -0
- package/dist/1898.index.js +116 -0
- package/dist/1906.index.js +433 -0
- package/dist/1920.index.js +1 -291
- package/dist/208.index.js +1631 -0
- package/dist/2199.index.js +0 -239
- package/dist/2223.index.js +159 -0
- package/dist/2530.index.js +152 -0
- package/dist/2565.index.js +116 -0
- package/dist/2574.index.js +127 -0
- package/dist/2620.index.js +345 -0
- package/dist/2773.index.js +98 -0
- package/dist/2775.index.js +32 -0
- package/dist/2966.index.js +171 -0
- package/dist/2972.index.js +87 -0
- package/dist/2991.index.js +131 -0
- package/dist/3063.index.js +833 -0
- package/dist/3075.index.js +227 -0
- package/dist/3121.index.js +297 -0
- package/dist/3314.index.js +263 -0
- package/dist/3480.index.js +1822 -0
- package/dist/3481.index.js +24 -0
- package/dist/3506.index.js +268 -0
- package/dist/3533.index.js +591 -0
- package/dist/3653.index.js +450 -0
- package/dist/3822.index.js +131 -0
- package/dist/3868.index.js +292 -0
- package/dist/3917.index.js +152 -0
- package/dist/3918.index.js +192 -0
- package/dist/4090.index.js +288 -0
- package/dist/4121.index.js +1190 -0
- package/dist/4142.index.js +0 -239
- package/dist/4336.index.js +436 -0
- package/dist/4387.index.js +532 -0
- package/dist/4413.index.js +302 -0
- package/dist/4476.index.js +4510 -0
- package/dist/4491.index.js +133 -0
- package/dist/4538.index.js +185 -0
- package/dist/4684.index.js +319 -0
- package/dist/4740.index.js +165 -0
- package/dist/4780.index.js +209 -0
- package/dist/4802.index.js +225 -0
- package/dist/4812.index.js +379 -0
- package/dist/4835.index.js +126 -0
- package/dist/494.index.js +226 -0
- package/dist/4991.index.js +549 -0
- package/dist/5199.index.js +305 -0
- package/dist/5216.index.js +109 -0
- package/dist/5439.index.js +301 -0
- package/dist/5443.index.js +97 -0
- package/dist/5496.index.js +377 -0
- package/dist/550.index.js +2 -1
- package/dist/5801.index.js +588 -0
- package/dist/5858.index.js +223 -0
- package/dist/5933.index.js +143 -0
- package/dist/601.index.js +23 -0
- package/dist/6173.index.js +273 -0
- package/dist/6291.index.js +1871 -0
- package/dist/6312.index.js +666 -0
- package/dist/634.index.js +16182 -0
- package/dist/644.index.js +87 -0
- package/dist/6508.index.js +283 -0
- package/dist/6560.index.js +645 -0
- package/dist/6576.index.js +422 -0
- package/dist/6579.index.js +271 -0
- package/dist/6745.index.js +305 -0
- package/dist/6818.index.js +21642 -0
- package/dist/6972.index.js +167 -0
- package/dist/6995.index.js +163 -0
- package/dist/7249.index.js +191 -0
- package/dist/7372.index.js +1197 -0
- package/dist/7405.index.js +23 -6
- package/dist/7546.index.js +541 -0
- package/dist/76.index.js +448 -0
- package/dist/7879.index.js +163 -0
- package/dist/8025.index.js +130 -0
- package/dist/8054.index.js +248 -0
- package/dist/8097.index.js +201 -0
- package/dist/8126.index.js +191 -0
- package/dist/8178.index.js +78 -0
- package/dist/818.index.js +212 -0
- package/dist/8382.index.js +93 -0
- package/dist/8408.index.js +261 -0
- package/dist/8540.index.js +174 -0
- package/dist/8635.index.js +802 -0
- package/dist/8688.index.js +410 -0
- package/dist/8699.index.js +779 -0
- package/dist/8891.index.js +457 -0
- package/dist/8916.index.js +885 -0
- package/dist/9146.index.js +170 -0
- package/dist/9168.index.js +841 -0
- package/dist/9292.index.js +369 -0
- package/dist/9382.index.js +338 -48
- package/dist/9422.index.js +866 -0
- package/dist/9440.index.js +453 -0
- package/dist/9479.index.js +152 -0
- package/dist/9481.index.js +220 -0
- package/dist/9539.index.js +100 -0
- package/dist/9547.index.js +103 -0
- package/dist/9560.index.js +41 -61
- package/dist/9692.index.js +90 -0
- package/dist/9800.index.js +213 -1
- package/dist/9899.index.js +138 -0
- package/dist/9981.index.js +206 -0
- package/dist/999.index.js +117 -0
- package/dist/agentic-security.mjs +28 -28
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/package.json +31 -6
- package/src/context-intelligence/CLAUDE.md +4 -0
- package/src/context-intelligence/adapters/haskell-hie.js +28 -1
- package/src/context-intelligence/candidates.js +2 -1
- package/src/context-intelligence/canonical.js +25 -0
- package/src/context-intelligence/capabilities.js +2 -1
- package/src/context-intelligence/cli.js +16 -10
- package/src/context-intelligence/config.js +4 -1
- package/src/context-intelligence/controls.js +108 -10
- package/src/context-intelligence/counterexample.js +200 -0
- package/src/context-intelligence/cross-route.js +111 -0
- package/src/context-intelligence/entrypoints.js +2 -2
- package/src/context-intelligence/evaluation/index.js +11 -4
- package/src/context-intelligence/evaluation/scripted-reader.js +12 -2
- package/src/context-intelligence/findings.js +166 -0
- package/src/context-intelligence/invariants.js +197 -0
- package/src/context-intelligence/queue.js +6 -1
- package/src/context-intelligence/reachability.js +10 -4
- package/src/context-intelligence/review.js +129 -16
- package/src/dataflow/access-paths.js +11 -0
- package/src/dataflow/analysis-limits.js +66 -0
- package/src/dataflow/engine.js +272 -20
- package/src/dataflow/service-edges.js +174 -0
- package/src/dataflow/summaries.js +7 -1
- package/src/discovery/judge.js +2 -1
- package/src/engine.js +82 -14
- package/src/fix/apply-fix-service.js +55 -8
- package/src/fix/batch-journal.js +84 -0
- package/src/fix/exploit-replay.js +51 -0
- package/src/fix/footprint.js +65 -0
- package/src/fix/patch-policy.js +49 -0
- package/src/fix/secret-persistence.js +39 -0
- package/src/fix/test-commands.js +84 -0
- package/src/fix/verification-adapters.js +97 -0
- package/src/fix/verification-record.js +243 -0
- package/src/fix/verify-patch.js +89 -0
- package/src/ir/CLAUDE.md +5 -0
- package/src/ir/callgraph.js +23 -1
- package/src/ir/module-resolver.js +315 -0
- package/src/ir/parser-js.js +21 -2
- package/src/ir/semantic-graph.js +245 -0
- package/src/lineage/build-attested-correlation.js +63 -0
- package/src/lineage/deployment-identity.js +76 -0
- package/src/mcp/tools.js +13 -34
- package/src/pipeline/canonical-output.js +64 -0
- package/src/pipeline/detector-runner.js +7 -0
- package/src/pipeline/scan-health.js +14 -0
- package/src/pipeline/supervised-analyzers.js +125 -0
- package/src/pipeline/supervised-child.js +17 -0
- package/src/pipeline/telemetry.js +79 -0
- package/src/posture/accuracy-scorecard.js +98 -9
- package/src/posture/artifact-registry.js +5 -0
- package/src/posture/assurance/evidence-registry.js +215 -0
- package/src/posture/assurance/release-package-verify.js +135 -0
- package/src/posture/assurance/release-package.js +118 -0
- package/src/posture/assurance/release-verdict.js +94 -0
- package/src/posture/assurance/waiver.js +200 -0
- package/src/posture/compliance-truth.js +100 -0
- package/src/posture/evaluation/scan-child.js +3 -1
- package/src/posture/evaluation/why-missed.js +3 -1
- package/src/posture/exploitability-evidence.js +141 -0
- package/src/posture/fix-verify.js +4 -2
- package/src/posture/fleet/exposure-ranking.js +115 -0
- package/src/posture/fleet/service-chain-graph.js +150 -0
- package/src/posture/graph-reconciliation.js +154 -0
- package/src/posture/provenance/coverage.js +103 -0
- package/src/posture/provenance/git-evidence.js +84 -0
- package/src/posture/provenance/origin-resolver.js +7 -1
- package/src/posture/provenance/predicate-replay.js +14 -4
- package/src/posture/runtime-correlation.js +61 -15
- package/src/posture/runtime-explanation.js +116 -0
- package/src/posture/runtime-trace-ingest.js +194 -0
- package/src/posture/scan-checkpoint.js +38 -10
- package/src/posture/waf-ingest.js +7 -2
- package/src/report/index.js +4 -0
- package/src/report/oscal.js +38 -0
- package/src/report/perspectives.js +150 -0
- package/src/sast/_comment-strip.js +8 -1
- package/src/sast/ruby.js +26 -0
- package/src/util/string-memo.js +38 -0
|
@@ -0,0 +1,1822 @@
|
|
|
1
|
+
export const id = 3480;
|
|
2
|
+
export const ids = [3480];
|
|
3
|
+
export const modules = {
|
|
4
|
+
|
|
5
|
+
/***/ 64130:
|
|
6
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
// EXPORTS
|
|
10
|
+
__webpack_require__.d(__webpack_exports__, {
|
|
11
|
+
mt: () => (/* binding */ findSecret)
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
// UNUSED EXPORTS: BLOB_DIR, BUNDLE_LIMITS, BUNDLE_SCHEMA, INDEX_FILE, ROLES, bundleDigestOf, checkLogicalName, exportBundle, importBundle, verifyBundle
|
|
15
|
+
|
|
16
|
+
// EXTERNAL MODULE: external "node:fs"
|
|
17
|
+
var external_node_fs_ = __webpack_require__(73024);
|
|
18
|
+
// EXTERNAL MODULE: external "node:path"
|
|
19
|
+
var external_node_path_ = __webpack_require__(76760);
|
|
20
|
+
// EXTERNAL MODULE: ./src/posture/evidence-bundle.js
|
|
21
|
+
var evidence_bundle = __webpack_require__(32876);
|
|
22
|
+
// EXTERNAL MODULE: ./src/posture/assurance/identity.js
|
|
23
|
+
var identity = __webpack_require__(91026);
|
|
24
|
+
// EXTERNAL MODULE: ./src/llm-validator/redact.js
|
|
25
|
+
var redact = __webpack_require__(98039);
|
|
26
|
+
// EXTERNAL MODULE: ./src/posture/assurance/schema-kit.js
|
|
27
|
+
var schema_kit = __webpack_require__(99666);
|
|
28
|
+
;// CONCATENATED MODULE: ./src/posture/portfolio/manifest.js
|
|
29
|
+
// Release assurance manifest (X-701).
|
|
30
|
+
//
|
|
31
|
+
// A manifest binds one release claim to the exact identities it is about, and says out loud what was NOT checked. It is a
|
|
32
|
+
// closed-world record built on the assurance schema kit (assurance/schema-kit.js) and the deterministic identity helpers
|
|
33
|
+
// (assurance/identity.js), the same pair the release-evidence contract uses, so a manifest id is a refactor-stable hash over an
|
|
34
|
+
// allowlist of semantic fields and never covers a clock.
|
|
35
|
+
//
|
|
36
|
+
// What it binds (X-701.AC01): the subject repository and commit, every dependency revision, every build artifact digest, the scan
|
|
37
|
+
// scope, the graph snapshot digest, the invariant versions and the verification receipts the claim rests on.
|
|
38
|
+
//
|
|
39
|
+
// What it discloses (X-701.AC02): every mandatory check lands in exactly one of completed / incomplete / unsupported / waived, and
|
|
40
|
+
// the exact blocking-finding policy (id, digest, minimum blocking severity) is part of the record. A waiver names who approved it and
|
|
41
|
+
// why; an incomplete or unsupported check names what is missing.
|
|
42
|
+
//
|
|
43
|
+
// What validation REFUSES (X-701.AC03): evidence that is not bound to a declared receipt, a receipt whose commit is not the commit of
|
|
44
|
+
// the repository it claims to be about, a mandatory check that appears nowhere, and a coverage block that disagrees with the check
|
|
45
|
+
// lists. Residual risks and coverage are preserved as machine-readable fields, never folded into prose.
|
|
46
|
+
//
|
|
47
|
+
// Pure: no fs, no clock, no network. Synthetic fixtures set `synthetic: true`; that flag is carried, never inferred.
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
const MANIFEST_SCHEMA = 'agentic-security/release-assurance-manifest';
|
|
53
|
+
const MANIFEST_ID_PREFIX = 'ram';
|
|
54
|
+
const BLOCKING_SEVERITIES = Object.freeze(['critical', 'high', 'medium', 'low']);
|
|
55
|
+
const CHECK_GROUPS = Object.freeze(['completed', 'incomplete', 'unsupported', 'waived']);
|
|
56
|
+
|
|
57
|
+
// Hard ceilings: a manifest is attacker-influenced input at verify time.
|
|
58
|
+
const MANIFEST_LIMITS = Object.freeze({ maxChecks: 512, maxReceipts: 2048, maxArtifacts: 512, maxDependencies: 512, maxInvariants: 512, maxRisks: 256 });
|
|
59
|
+
|
|
60
|
+
const ALLOWED = (/* unused pure expression or super */ null && ([
|
|
61
|
+
'schema', 'schemaVersion', 'id', 'synthetic', 'subject', 'dependencies', 'artifacts', 'scope', 'graphSnapshot', 'invariantVersions',
|
|
62
|
+
'verificationReceipts', 'checks', 'policy', 'findings', 'residualRisks', 'coverage', 'complete', 'createdAt',
|
|
63
|
+
]));
|
|
64
|
+
const REQUIRED = (/* unused pure expression or super */ null && ([
|
|
65
|
+
'schema', 'schemaVersion', 'id', 'subject', 'dependencies', 'artifacts', 'scope', 'graphSnapshot', 'invariantVersions',
|
|
66
|
+
'verificationReceipts', 'checks', 'policy', 'findings', 'residualRisks', 'coverage', 'complete',
|
|
67
|
+
]));
|
|
68
|
+
const ID_FIELDS = Object.freeze([
|
|
69
|
+
'subject', 'dependencies', 'artifacts', 'scope', 'graphSnapshot', 'invariantVersions', 'verificationReceipts', 'checks', 'policy',
|
|
70
|
+
'findings', 'residualRisks', 'coverage', 'complete',
|
|
71
|
+
]);
|
|
72
|
+
|
|
73
|
+
const manifestId = (m) => semanticId(MANIFEST_ID_PREFIX, m, ID_FIELDS);
|
|
74
|
+
/** The digest a signature binds: the whole manifest, key-order independent. */
|
|
75
|
+
const manifest_manifestDigest = (m) => digestOf(m);
|
|
76
|
+
|
|
77
|
+
/** Counts derived from the check lists. The stored `coverage` must equal this, so it cannot drift from the lists it summarizes. */
|
|
78
|
+
function deriveCoverage(checks, scope) {
|
|
79
|
+
const n = (g) => (Array.isArray(checks?.[g]) ? checks[g].length : 0);
|
|
80
|
+
const mandatory = Array.isArray(scope?.mandatory) ? scope.mandatory.length : 0;
|
|
81
|
+
return {
|
|
82
|
+
mandatory, completed: n('completed'), incomplete: n('incomplete'), unsupported: n('unsupported'), waived: n('waived'),
|
|
83
|
+
complete: mandatory > 0 && n('incomplete') === 0 && n('unsupported') === 0 && n('completed') + n('waived') === mandatory,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Build a manifest from the bound facts; coverage, completeness and id are computed, never supplied. */
|
|
88
|
+
function buildManifest(f) {
|
|
89
|
+
const checks = {
|
|
90
|
+
completed: f.checks?.completed ?? [], incomplete: f.checks?.incomplete ?? [], unsupported: f.checks?.unsupported ?? [], waived: f.checks?.waived ?? [],
|
|
91
|
+
};
|
|
92
|
+
const m = {
|
|
93
|
+
schema: MANIFEST_SCHEMA, schemaVersion: SCHEMA_VERSION,
|
|
94
|
+
...(f.synthetic === true ? { synthetic: true } : {}),
|
|
95
|
+
subject: f.subject, dependencies: f.dependencies ?? [], artifacts: f.artifacts ?? [], scope: f.scope,
|
|
96
|
+
graphSnapshot: f.graphSnapshot, invariantVersions: f.invariantVersions ?? [], verificationReceipts: f.verificationReceipts ?? [],
|
|
97
|
+
checks, policy: f.policy, findings: f.findings, residualRisks: f.residualRisks ?? [],
|
|
98
|
+
coverage: deriveCoverage(checks, f.scope),
|
|
99
|
+
};
|
|
100
|
+
m.complete = m.coverage.complete;
|
|
101
|
+
m.id = manifestId(m);
|
|
102
|
+
return m;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function checkList(ctx, p, list, max) {
|
|
106
|
+
if (!Array.isArray(list)) { ctx.err('BAD_TYPE', p, 'must be an array'); return false; }
|
|
107
|
+
if (list.length > max) { ctx.err('RULE_VIOLATION', p, `has ${list.length} entries; the limit is ${max}`); return false; }
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function closedKeys(ctx, p, obj, allowed, what) {
|
|
112
|
+
for (const k of Object.keys(obj)) if (!allowed.includes(k)) ctx.err('UNKNOWN_FIELD', `${p}.${k}`, `not part of ${what}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function isCommitLike(v) { return typeof v === 'string' && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(v); }
|
|
116
|
+
|
|
117
|
+
function manifest_validateManifest(m) {
|
|
118
|
+
const g = guardObject(m);
|
|
119
|
+
const ctx = g.ctx;
|
|
120
|
+
if (!g.ok) return result(ctx);
|
|
121
|
+
if (!checkHeader(ctx, m, MANIFEST_SCHEMA)) return result(ctx);
|
|
122
|
+
checkFields(ctx, m, ALLOWED, REQUIRED);
|
|
123
|
+
if (m.synthetic !== undefined && m.synthetic !== true) ctx.err('BAD_TYPE', 'synthetic', 'must be true when present');
|
|
124
|
+
|
|
125
|
+
// subject
|
|
126
|
+
const s = m.subject;
|
|
127
|
+
const repoCommits = new Map();
|
|
128
|
+
if (!isPlainObject(s)) ctx.err('BAD_TYPE', 'subject', 'must be an object');
|
|
129
|
+
else {
|
|
130
|
+
closedKeys(ctx, 'subject', s, ['repository', 'commit', 'bundleDigest', 'policyDigest'], 'subject');
|
|
131
|
+
checkString(ctx, 'subject.repository', s.repository);
|
|
132
|
+
checkCommit(ctx, 'subject.commit', s.commit ?? null, { nullable: false });
|
|
133
|
+
checkDigest(ctx, 'subject.bundleDigest', s.bundleDigest ?? null);
|
|
134
|
+
checkDigest(ctx, 'subject.policyDigest', s.policyDigest ?? null);
|
|
135
|
+
if (typeof s.repository === 'string') repoCommits.set(s.repository, s.commit);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// dependencies
|
|
139
|
+
if (checkList(ctx, 'dependencies', m.dependencies, MANIFEST_LIMITS.maxDependencies)) {
|
|
140
|
+
m.dependencies.forEach((d, i) => {
|
|
141
|
+
const p = `dependencies[${i}]`;
|
|
142
|
+
if (!isPlainObject(d)) { ctx.err('BAD_TYPE', p, 'must be an object'); return; }
|
|
143
|
+
closedKeys(ctx, p, d, ['name', 'revision'], 'a dependency');
|
|
144
|
+
if (checkString(ctx, `${p}.name`, d.name)) {
|
|
145
|
+
if (repoCommits.has(d.name)) ctx.err('DUPLICATE_ID', `${p}.name`, `dependency '${d.name}' is declared twice or collides with the subject repository`);
|
|
146
|
+
else repoCommits.set(d.name, d.revision);
|
|
147
|
+
}
|
|
148
|
+
checkString(ctx, `${p}.revision`, d.revision);
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// artifacts
|
|
153
|
+
if (checkList(ctx, 'artifacts', m.artifacts, MANIFEST_LIMITS.maxArtifacts)) {
|
|
154
|
+
if (m.artifacts.length === 0) ctx.err('RULE_VIOLATION', 'artifacts', 'a release manifest must bind at least one build artifact digest');
|
|
155
|
+
m.artifacts.forEach((a, i) => {
|
|
156
|
+
const p = `artifacts[${i}]`;
|
|
157
|
+
if (!isPlainObject(a)) { ctx.err('BAD_TYPE', p, 'must be an object'); return; }
|
|
158
|
+
closedKeys(ctx, p, a, ['name', 'digest'], 'an artifact');
|
|
159
|
+
checkString(ctx, `${p}.name`, a.name);
|
|
160
|
+
checkDigest(ctx, `${p}.digest`, a.digest);
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// scope
|
|
165
|
+
const mandatory = new Set();
|
|
166
|
+
if (!isPlainObject(m.scope)) ctx.err('BAD_TYPE', 'scope', 'must be an object');
|
|
167
|
+
else {
|
|
168
|
+
closedKeys(ctx, 'scope', m.scope, ['description', 'mandatory'], 'scope');
|
|
169
|
+
checkString(ctx, 'scope.description', m.scope.description);
|
|
170
|
+
if (checkList(ctx, 'scope.mandatory', m.scope.mandatory, MANIFEST_LIMITS.maxChecks)) {
|
|
171
|
+
if (m.scope.mandatory.length === 0) ctx.err('RULE_VIOLATION', 'scope.mandatory', 'a manifest must name at least one mandatory check');
|
|
172
|
+
m.scope.mandatory.forEach((id, i) => {
|
|
173
|
+
if (!checkString(ctx, `scope.mandatory[${i}]`, id)) return;
|
|
174
|
+
if (mandatory.has(id)) ctx.err('DUPLICATE_ID', `scope.mandatory[${i}]`, `duplicate mandatory check '${id}'`);
|
|
175
|
+
mandatory.add(id);
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// graph snapshot
|
|
181
|
+
if (!isPlainObject(m.graphSnapshot)) ctx.err('BAD_TYPE', 'graphSnapshot', 'must be an object');
|
|
182
|
+
else {
|
|
183
|
+
closedKeys(ctx, 'graphSnapshot', m.graphSnapshot, ['digest'], 'graphSnapshot');
|
|
184
|
+
checkDigest(ctx, 'graphSnapshot.digest', m.graphSnapshot.digest ?? null);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (checkList(ctx, 'invariantVersions', m.invariantVersions, MANIFEST_LIMITS.maxInvariants)) {
|
|
188
|
+
m.invariantVersions.forEach((v, i) => {
|
|
189
|
+
const p = `invariantVersions[${i}]`;
|
|
190
|
+
if (!isPlainObject(v)) { ctx.err('BAD_TYPE', p, 'must be an object'); return; }
|
|
191
|
+
closedKeys(ctx, p, v, ['id', 'version'], 'an invariant version');
|
|
192
|
+
checkString(ctx, `${p}.id`, v.id); checkString(ctx, `${p}.version`, v.version);
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// receipts: each must be about a declared repository and carry THAT repository's commit
|
|
197
|
+
const receipts = new Map();
|
|
198
|
+
if (checkList(ctx, 'verificationReceipts', m.verificationReceipts, MANIFEST_LIMITS.maxReceipts)) {
|
|
199
|
+
m.verificationReceipts.forEach((r, i) => {
|
|
200
|
+
const p = `verificationReceipts[${i}]`;
|
|
201
|
+
if (!isPlainObject(r)) { ctx.err('BAD_TYPE', p, 'must be an object'); return; }
|
|
202
|
+
closedKeys(ctx, p, r, ['id', 'digest', 'repository', 'commit'], 'a receipt');
|
|
203
|
+
if (checkString(ctx, `${p}.id`, r.id)) {
|
|
204
|
+
if (receipts.has(r.id)) ctx.err('DUPLICATE_ID', `${p}.id`, `duplicate receipt '${r.id}'`);
|
|
205
|
+
receipts.set(r.id, r);
|
|
206
|
+
}
|
|
207
|
+
checkDigest(ctx, `${p}.digest`, r.digest);
|
|
208
|
+
checkCommit(ctx, `${p}.commit`, r.commit ?? null, { nullable: false });
|
|
209
|
+
if (checkString(ctx, `${p}.repository`, r.repository)) {
|
|
210
|
+
if (!repoCommits.has(r.repository)) ctx.err('DANGLING_REF', `${p}.repository`, `receipt is about '${r.repository}', which is neither the subject repository nor a declared dependency`);
|
|
211
|
+
else if (isCommitLike(repoCommits.get(r.repository)) && r.commit !== repoCommits.get(r.repository)) {
|
|
212
|
+
ctx.err('RULE_VIOLATION', `${p}.commit`, `commit mismatch: receipt commit ${String(r.commit).slice(0, 12)} is not the bound commit ${String(repoCommits.get(r.repository)).slice(0, 12)} of '${r.repository}'`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// checks
|
|
219
|
+
const seenIds = new Map();
|
|
220
|
+
if (!isPlainObject(m.checks)) ctx.err('BAD_TYPE', 'checks', 'must be an object');
|
|
221
|
+
else {
|
|
222
|
+
closedKeys(ctx, 'checks', m.checks, CHECK_GROUPS, 'a check group');
|
|
223
|
+
for (const grp of CHECK_GROUPS) {
|
|
224
|
+
if (!checkList(ctx, `checks.${grp}`, m.checks[grp], MANIFEST_LIMITS.maxChecks)) continue;
|
|
225
|
+
m.checks[grp].forEach((c, i) => {
|
|
226
|
+
const p = `checks.${grp}[${i}]`;
|
|
227
|
+
if (!isPlainObject(c)) { ctx.err('BAD_TYPE', p, 'must be an object'); return; }
|
|
228
|
+
closedKeys(ctx, p, c, ['id', 'statement', 'evidenceRefs', 'gaps', 'reason', 'approvedBy'], 'a check');
|
|
229
|
+
if (checkString(ctx, `${p}.id`, c.id)) {
|
|
230
|
+
if (seenIds.has(c.id)) ctx.err('DUPLICATE_ID', `${p}.id`, `check '${c.id}' is already listed under ${seenIds.get(c.id)}`);
|
|
231
|
+
else seenIds.set(c.id, grp);
|
|
232
|
+
if (mandatory.size && !mandatory.has(c.id)) ctx.err('RULE_VIOLATION', `${p}.id`, `check '${c.id}' is not part of the declared mandatory scope`);
|
|
233
|
+
}
|
|
234
|
+
checkString(ctx, `${p}.statement`, c.statement);
|
|
235
|
+
const refs = c.evidenceRefs;
|
|
236
|
+
if (!Array.isArray(refs) || !refs.every((x) => typeof x === 'string')) ctx.err('BAD_TYPE', `${p}.evidenceRefs`, 'must be an array of receipt ids');
|
|
237
|
+
else {
|
|
238
|
+
if (grp === 'completed' && refs.length === 0) ctx.err('RULE_VIOLATION', `${p}.evidenceRefs`, 'a completed check must cite evidence');
|
|
239
|
+
refs.forEach((ref, j) => { if (!receipts.has(ref)) ctx.err('DANGLING_REF', `${p}.evidenceRefs[${j}]`, `unbound evidence: '${ref}' is not a declared verification receipt`); });
|
|
240
|
+
}
|
|
241
|
+
const gapsOk = Array.isArray(c.gaps) && c.gaps.every((x) => typeof x === 'string' && x.trim());
|
|
242
|
+
if (grp === 'incomplete' || grp === 'unsupported') {
|
|
243
|
+
if (!gapsOk || c.gaps.length === 0) ctx.err('RULE_VIOLATION', `${p}.gaps`, `a ${grp} check must say what is missing`);
|
|
244
|
+
} else if (c.gaps !== undefined && (!gapsOk || c.gaps.length > 0)) ctx.err('RULE_VIOLATION', `${p}.gaps`, `a ${grp} check cannot list gaps`);
|
|
245
|
+
if (grp === 'waived') { checkString(ctx, `${p}.reason`, c.reason); checkString(ctx, `${p}.approvedBy`, c.approvedBy); }
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
for (const id of mandatory) if (!seenIds.has(id)) ctx.err('RULE_VIOLATION', 'checks', `omitted mandatory scope: '${id}' is not completed, incomplete, unsupported or waived`);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// policy
|
|
252
|
+
if (!isPlainObject(m.policy)) ctx.err('BAD_TYPE', 'policy', 'must be an object');
|
|
253
|
+
else {
|
|
254
|
+
closedKeys(ctx, 'policy', m.policy, ['id', 'digest', 'blockingSeverity'], 'policy');
|
|
255
|
+
checkString(ctx, 'policy.id', m.policy.id);
|
|
256
|
+
checkDigest(ctx, 'policy.digest', m.policy.digest);
|
|
257
|
+
checkEnum(ctx, 'policy.blockingSeverity', m.policy.blockingSeverity, BLOCKING_SEVERITIES);
|
|
258
|
+
if (isPlainObject(s) && isDigest(m.policy.digest) && m.policy.digest !== s.policyDigest) ctx.err('RULE_VIOLATION', 'policy.digest', 'policy digest does not match subject.policyDigest');
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// findings summary
|
|
262
|
+
if (!isPlainObject(m.findings)) ctx.err('BAD_TYPE', 'findings', 'must be an object');
|
|
263
|
+
else {
|
|
264
|
+
closedKeys(ctx, 'findings', m.findings, ['total', 'blocking'], 'findings');
|
|
265
|
+
for (const k of ['total', 'blocking']) if (!Number.isInteger(m.findings[k]) || m.findings[k] < 0) ctx.err('BAD_TYPE', `findings.${k}`, 'must be a non-negative integer');
|
|
266
|
+
if (Number.isInteger(m.findings.total) && Number.isInteger(m.findings.blocking) && m.findings.blocking > m.findings.total) ctx.err('RULE_VIOLATION', 'findings.blocking', 'blocking cannot exceed total');
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// residual risks
|
|
270
|
+
if (checkList(ctx, 'residualRisks', m.residualRisks, MANIFEST_LIMITS.maxRisks)) {
|
|
271
|
+
m.residualRisks.forEach((r, i) => {
|
|
272
|
+
const p = `residualRisks[${i}]`;
|
|
273
|
+
if (!isPlainObject(r)) { ctx.err('BAD_TYPE', p, 'must be an object'); return; }
|
|
274
|
+
closedKeys(ctx, p, r, ['id', 'statement', 'severity'], 'a residual risk');
|
|
275
|
+
checkString(ctx, `${p}.id`, r.id); checkString(ctx, `${p}.statement`, r.statement);
|
|
276
|
+
checkEnum(ctx, `${p}.severity`, r.severity, [...BLOCKING_SEVERITIES, 'info']);
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// coverage and completeness must agree with the lists
|
|
281
|
+
if (isPlainObject(m.checks) && isPlainObject(m.scope)) {
|
|
282
|
+
const want = deriveCoverage(m.checks, m.scope);
|
|
283
|
+
if (!isPlainObject(m.coverage) || digestOf(m.coverage) !== digestOf(want)) ctx.err('RULE_VIOLATION', 'coverage', `coverage disagrees with the check lists (expected ${JSON.stringify(want)})`);
|
|
284
|
+
if (typeof m.complete !== 'boolean') ctx.err('BAD_TYPE', 'complete', 'must be a boolean');
|
|
285
|
+
else if (m.complete !== want.complete) ctx.err('RULE_VIOLATION', 'complete', `claimed complete=${m.complete} but the check lists support complete=${want.complete}`);
|
|
286
|
+
}
|
|
287
|
+
checkId(ctx, m, manifestId(m));
|
|
288
|
+
return result(ctx);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
;// CONCATENATED MODULE: ./src/posture/portfolio/bundle.js
|
|
292
|
+
// Portable, content-addressed evidence bundle and its OFFLINE verifier (X-702).
|
|
293
|
+
//
|
|
294
|
+
// A bundle is a directory:
|
|
295
|
+
//
|
|
296
|
+
// bundle.json the index: schema, manifest digest, every entry (role, logical name, digest, size), replay prerequisites
|
|
297
|
+
// blobs/<sha256 hex> one file per entry, named by the digest of its canonical bytes
|
|
298
|
+
//
|
|
299
|
+
// Everything a reviewer needs to check a release claim is in it (X-702.AC01): the assurance manifest, the SANITIZED findings, finding
|
|
300
|
+
// provenance, replay manifests, the dependency and toolchain identities, and the verification receipts the manifest cites. Nothing in it
|
|
301
|
+
// refers to the agent conversation that produced the work.
|
|
302
|
+
//
|
|
303
|
+
// The verifier (X-702.AC02) uses node:fs and node:crypto only. It never opens a socket and never executes anything from the bundle:
|
|
304
|
+
// it re-hashes every blob against the index, validates the manifest through manifest.js, checks that every receipt the manifest cites
|
|
305
|
+
// is present as a blob, and then DISCLOSES what a runtime replay would need (`replay.prerequisites`) without attempting one. A verified
|
|
306
|
+
// bundle therefore proves its contents are unmodified; it does not prove a replay would still reproduce today.
|
|
307
|
+
//
|
|
308
|
+
// Export and import enforce limits (X-702.AC03): entry count, per-blob and total bytes, logical-name shape (no absolute path, no `..`,
|
|
309
|
+
// no unusual characters), no symlinks, and a secret filter. The filter first REDACTS known secret shapes in findings and provenance, then
|
|
310
|
+
// refuses to write a blob in which any secret shape survives; the importer refuses a bundle that carries one. A missing blob, a modified
|
|
311
|
+
// blob or a modified index fails integrity validation; none is ever repaired or skipped silently.
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
const BUNDLE_SCHEMA = 'agentic-security/portable-evidence-bundle';
|
|
321
|
+
const BUNDLE_VERSION = '1.0.0';
|
|
322
|
+
const INDEX_FILE = 'bundle.json';
|
|
323
|
+
const BLOB_DIR = 'blobs';
|
|
324
|
+
const ROLES = Object.freeze(['manifest', 'findings', 'provenance', 'replay-manifest', 'toolchain', 'receipt']);
|
|
325
|
+
const REQUIRED_ROLES = Object.freeze(['manifest', 'findings', 'provenance', 'toolchain']);
|
|
326
|
+
|
|
327
|
+
const BUNDLE_LIMITS = Object.freeze({ maxEntries: 4096, maxBlobBytes: 4 * 1024 * 1024, maxTotalBytes: 64 * 1024 * 1024, maxIndexBytes: 8 * 1024 * 1024, maxNameLength: 200, maxDepth: 24, maxString: 8192 });
|
|
328
|
+
|
|
329
|
+
const NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)*$/;
|
|
330
|
+
const DIGEST_RE = /^sha256:[0-9a-f]{64}$/;
|
|
331
|
+
|
|
332
|
+
// ---------------------------------------------------------------- secret filter
|
|
333
|
+
|
|
334
|
+
const SECRET_PATTERNS = [
|
|
335
|
+
/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----/,
|
|
336
|
+
/\bAKIA[0-9A-Z]{16}\b/,
|
|
337
|
+
/\bgh[pousr]_[A-Za-z0-9]{30,}\b/,
|
|
338
|
+
/\bgithub_pat_[A-Za-z0-9_]{30,}\b/,
|
|
339
|
+
/\bxox[abprs]-[A-Za-z0-9-]{10,}\b/,
|
|
340
|
+
/\bsk-[A-Za-z0-9_-]{20,}\b/,
|
|
341
|
+
/\bBearer\s+[A-Za-z0-9._~+/=-]{20,}/,
|
|
342
|
+
/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/,
|
|
343
|
+
/\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)\b["']?\s*[:=]\s*["']?[^\s"',;]{8,}/i,
|
|
344
|
+
];
|
|
345
|
+
const PLACEHOLDER = '[REDACTED-SECRET]';
|
|
346
|
+
|
|
347
|
+
/** The first secret shape found in a text, or null. */
|
|
348
|
+
function findSecret(text) {
|
|
349
|
+
if (typeof text !== 'string') return null;
|
|
350
|
+
for (const re of SECRET_PATTERNS) { const m = re.exec(text); if (m && !m[0].includes(PLACEHOLDER)) return m[0].slice(0, 12) + '...'; }
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Replace every known secret shape with a placeholder. */
|
|
355
|
+
function redactText(text) {
|
|
356
|
+
let out = String(text);
|
|
357
|
+
for (const re of SECRET_PATTERNS) out = out.replace(new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g'), PLACEHOLDER);
|
|
358
|
+
return redactSecrets(out).text;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function sanitizeValue(v, depth = 0) {
|
|
362
|
+
if (depth > BUNDLE_LIMITS.maxDepth) throw new Error('evidence is nested deeper than the limit');
|
|
363
|
+
if (typeof v === 'string') return redactText(v.length > BUNDLE_LIMITS.maxString ? v.slice(0, BUNDLE_LIMITS.maxString) : v);
|
|
364
|
+
if (Array.isArray(v)) return v.map((x) => sanitizeValue(x, depth + 1));
|
|
365
|
+
if (v && typeof v === 'object') return Object.fromEntries(Object.keys(v).sort().map((k) => [k, sanitizeValue(v[k], depth + 1)]));
|
|
366
|
+
return v;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// A finding leaves the machine as a closed set of fields. Source snippets, taint traces and raw evidence stay behind.
|
|
370
|
+
const FINDING_FIELDS = (/* unused pure expression or super */ null && (['id', 'stableId', 'severity', 'file', 'line', 'vuln', 'cwe', 'family', 'parser', 'confidence', 'confidenceTier', 'description', 'remediation', 'unreachable', 'findingProvenance']));
|
|
371
|
+
|
|
372
|
+
function sanitizeFindings(findings) {
|
|
373
|
+
return (Array.isArray(findings) ? findings : []).map((f) => {
|
|
374
|
+
const out = {};
|
|
375
|
+
for (const k of FINDING_FIELDS) if (f && f[k] !== undefined) out[k] = sanitizeValue(f[k]);
|
|
376
|
+
return out;
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// ---------------------------------------------------------------- names and paths
|
|
381
|
+
|
|
382
|
+
function checkLogicalName(name) {
|
|
383
|
+
if (typeof name !== 'string' || !name) return 'empty name';
|
|
384
|
+
if (name.length > BUNDLE_LIMITS.maxNameLength) return `name longer than ${BUNDLE_LIMITS.maxNameLength}`;
|
|
385
|
+
if (name.includes('\0') || name.includes('\\')) return 'name contains a NUL or backslash';
|
|
386
|
+
if (path.isAbsolute(name) || name.startsWith('/')) return 'absolute path';
|
|
387
|
+
if (name.split('/').some((s) => s === '..' || s === '.')) return 'path traversal';
|
|
388
|
+
if (!NAME_RE.test(name)) return 'name has characters outside the allowed set';
|
|
389
|
+
return null;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
function blobFile(dir, digest) { return path.join(dir, BLOB_DIR, digest.slice('sha256:'.length)); }
|
|
393
|
+
|
|
394
|
+
function writeAtomic(file, bytes) {
|
|
395
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
396
|
+
fs.writeFileSync(tmp, bytes, { flag: 'wx', mode: 0o644 });
|
|
397
|
+
fs.renameSync(tmp, file);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// ---------------------------------------------------------------- replay prerequisites
|
|
401
|
+
|
|
402
|
+
const BASE_PREREQUISITES = Object.freeze([
|
|
403
|
+
{ code: 'source-at-commit', statement: 'the exact source revision named by the manifest subject must be checked out' },
|
|
404
|
+
{ code: 'toolchain-identities', statement: 'the toolchain identities recorded in the bundle must be installed at the recorded versions' },
|
|
405
|
+
{ code: 'confinement-backend', statement: 'a runtime replay needs a confinement backend; isolation-required replays are only advertised on Linux and enforcement there is unverified in this build' },
|
|
406
|
+
]);
|
|
407
|
+
|
|
408
|
+
function replayPrerequisites(replayManifests) {
|
|
409
|
+
const seen = new Set(BASE_PREREQUISITES.map((p) => p.code));
|
|
410
|
+
const out = BASE_PREREQUISITES.map((p) => ({ ...p }));
|
|
411
|
+
for (const rm of replayManifests) {
|
|
412
|
+
for (const p of Array.isArray(rm?.prerequisites) ? rm.prerequisites : []) {
|
|
413
|
+
const code = typeof p === 'string' ? p : p?.code;
|
|
414
|
+
if (typeof code !== 'string' || seen.has(code)) continue;
|
|
415
|
+
seen.add(code);
|
|
416
|
+
out.push({ code, statement: typeof p === 'object' && typeof p.statement === 'string' ? p.statement : code, source: rm.id ?? null });
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return out;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// ---------------------------------------------------------------- export
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Write a bundle to `outDir` (must not exist or must be empty). Throws on a limit, name, manifest or secret violation, and writes the
|
|
426
|
+
* index LAST so a half-written directory has no index and never verifies.
|
|
427
|
+
*
|
|
428
|
+
* @param {object} a
|
|
429
|
+
* @param {object} a.manifest a valid release assurance manifest
|
|
430
|
+
* @param {object[]} a.findings raw findings; sanitized here
|
|
431
|
+
* @param {object} a.provenance finding provenance and dependency identities
|
|
432
|
+
* @param {object[]} [a.replayManifests] `{ id, prerequisites?, ... }`
|
|
433
|
+
* @param {object} a.toolchain toolchain identities
|
|
434
|
+
* @param {Array<{id:string, content:*}>} [a.receipts] the verification receipt contents; each digest must equal the manifest's
|
|
435
|
+
*/
|
|
436
|
+
function exportBundle({ outDir, manifest, findings = [], provenance = {}, replayManifests = [], toolchain = {}, receipts = [] }) {
|
|
437
|
+
const v = validateManifest(manifest);
|
|
438
|
+
if (!v.ok) throw new Error(`refusing to export: the manifest is invalid (${v.errors[0].code} ${v.errors[0].path})`);
|
|
439
|
+
if (fs.existsSync(outDir) && fs.readdirSync(outDir).length > 0) throw new Error('refusing to export into a non-empty directory');
|
|
440
|
+
|
|
441
|
+
const items = [];
|
|
442
|
+
const add = (role, name, value, sanitize = true) => {
|
|
443
|
+
const bad = checkLogicalName(name);
|
|
444
|
+
if (bad) throw new Error(`refusing to export '${name}': ${bad}`);
|
|
445
|
+
const body = sanitize ? sanitizeValue(value) : value;
|
|
446
|
+
const bytes = Buffer.from(canonicalJson(body));
|
|
447
|
+
if (bytes.length > BUNDLE_LIMITS.maxBlobBytes) throw new Error(`refusing to export '${name}': ${bytes.length} bytes exceeds the ${BUNDLE_LIMITS.maxBlobBytes} byte limit`);
|
|
448
|
+
const hit = findSecret(bytes.toString('utf8'));
|
|
449
|
+
if (hit) throw new Error(`refusing to export '${name}': a secret shape survived redaction (${hit})`);
|
|
450
|
+
items.push({ role, name, digest: digestOfBytes(bytes), size: bytes.length, bytes });
|
|
451
|
+
};
|
|
452
|
+
add('manifest', 'manifest.json', manifest, false);
|
|
453
|
+
add('findings', 'findings.json', sanitizeFindings(findings), false);
|
|
454
|
+
add('provenance', 'provenance.json', provenance);
|
|
455
|
+
add('toolchain', 'toolchain.json', toolchain);
|
|
456
|
+
replayManifests.forEach((rm, i) => add('replay-manifest', `replay/${String(rm?.id ?? i).replace(/[^A-Za-z0-9._-]/g, '_')}.json`, rm));
|
|
457
|
+
const byId = new Map(receipts.map((r) => [r.id, r]));
|
|
458
|
+
for (const r of manifest.verificationReceipts) {
|
|
459
|
+
const given = byId.get(r.id);
|
|
460
|
+
if (!given) throw new Error(`refusing to export: receipt '${r.id}' cited by the manifest was not supplied`);
|
|
461
|
+
if (digestOf(given.content) !== r.digest) throw new Error(`refusing to export: receipt '${r.id}' content does not match the digest the manifest binds`);
|
|
462
|
+
add('receipt', `receipts/${r.id.replace(/[^A-Za-z0-9._-]/g, '_')}.json`, given.content);
|
|
463
|
+
}
|
|
464
|
+
if (items.length > BUNDLE_LIMITS.maxEntries) throw new Error('refusing to export: too many entries');
|
|
465
|
+
const total = items.reduce((n, i) => n + i.size, 0);
|
|
466
|
+
if (total > BUNDLE_LIMITS.maxTotalBytes) throw new Error('refusing to export: total size exceeds the limit');
|
|
467
|
+
const names = new Set();
|
|
468
|
+
for (const i of items) { if (names.has(i.name)) throw new Error(`refusing to export: duplicate name '${i.name}'`); names.add(i.name); }
|
|
469
|
+
|
|
470
|
+
fs.mkdirSync(path.join(outDir, BLOB_DIR), { recursive: true });
|
|
471
|
+
for (const i of items) {
|
|
472
|
+
const f = blobFile(outDir, i.digest);
|
|
473
|
+
if (!fs.existsSync(f)) writeAtomic(f, i.bytes);
|
|
474
|
+
}
|
|
475
|
+
const entries = items.map(({ role, name, digest, size }) => ({ role, name, digest, size })).sort((a, b) => (a.name < b.name ? -1 : 1));
|
|
476
|
+
const index = indexFor(entries, manifest, replayManifests);
|
|
477
|
+
writeAtomic(path.join(outDir, INDEX_FILE), Buffer.from(JSON.stringify(index, null, 2)));
|
|
478
|
+
return { bundleDigest: index.bundleDigest, manifestDigest: index.manifestDigest, entries: entries.length, totalBytes: total };
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function indexFor(entries, manifest, replayManifests) {
|
|
482
|
+
const body = {
|
|
483
|
+
schema: BUNDLE_SCHEMA, schemaVersion: BUNDLE_VERSION,
|
|
484
|
+
...(manifest.synthetic === true ? { synthetic: true } : {}),
|
|
485
|
+
manifestId: manifest.id, manifestDigest: manifestDigest(manifest), entries,
|
|
486
|
+
replay: { attempted: false, prerequisites: replayPrerequisites(replayManifests) },
|
|
487
|
+
limits: BUNDLE_LIMITS,
|
|
488
|
+
};
|
|
489
|
+
return { ...body, bundleDigest: bundleDigestOf(body) };
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/** The identity of a bundle: the manifest digest plus every entry's digest, independent of file order. */
|
|
493
|
+
function bundleDigestOf(index) {
|
|
494
|
+
return digestOf({ manifestDigest: index.manifestDigest, entries: index.entries.map((e) => [e.role, e.name, e.digest, e.size]) });
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// ---------------------------------------------------------------- verify (offline)
|
|
498
|
+
|
|
499
|
+
function readBounded(file, max) {
|
|
500
|
+
const st = fs.lstatSync(file);
|
|
501
|
+
if (st.isSymbolicLink() || !st.isFile()) throw Object.assign(new Error('not a regular file'), { code: 'NOT_REGULAR' });
|
|
502
|
+
if (st.size > max) throw Object.assign(new Error(`${st.size} bytes exceeds ${max}`), { code: 'TOO_LARGE' });
|
|
503
|
+
const fd = fs.openSync(file, 'r');
|
|
504
|
+
try {
|
|
505
|
+
const buf = Buffer.alloc(Math.min(st.size, max) + 1);
|
|
506
|
+
const n = fs.readSync(fd, buf, 0, buf.length, 0);
|
|
507
|
+
if (n > max) throw Object.assign(new Error('grew past the limit while reading'), { code: 'TOO_LARGE' });
|
|
508
|
+
return buf.subarray(0, n);
|
|
509
|
+
} finally { fs.closeSync(fd); }
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/**
|
|
513
|
+
* Verify a bundle directory using only the local filesystem. Returns `{ ok, errors, manifest, entries, replay, network:false }`.
|
|
514
|
+
* `errors` carry a stable `code`: BAD_INDEX, BAD_ENTRY, MISSING_BLOB, DIGEST_MISMATCH, SIZE_MISMATCH, TOO_LARGE, NOT_REGULAR,
|
|
515
|
+
* BUNDLE_DIGEST_MISMATCH, MANIFEST_INVALID, MANIFEST_MISMATCH, MISSING_EVIDENCE, MISSING_ROLE, SECRET_IN_BLOB, LIMIT.
|
|
516
|
+
*/
|
|
517
|
+
function verifyBundle(dir) {
|
|
518
|
+
const errors = [];
|
|
519
|
+
const err = (code, message, extra = {}) => errors.push({ code, message, ...extra });
|
|
520
|
+
const out = (extra = {}) => ({ ok: errors.length === 0, errors, network: false, manifest: null, entries: [], replay: { attempted: false, prerequisites: [], statement: 'replay was not attempted: this verifier is offline and executes nothing from the bundle' }, ...extra });
|
|
521
|
+
|
|
522
|
+
let index;
|
|
523
|
+
try { index = JSON.parse(readBounded(path.join(dir, INDEX_FILE), BUNDLE_LIMITS.maxIndexBytes).toString('utf8')); } catch (e) {
|
|
524
|
+
err('BAD_INDEX', `cannot read ${INDEX_FILE}: ${e.code || e.message}`); return out();
|
|
525
|
+
}
|
|
526
|
+
if (!index || index.schema !== BUNDLE_SCHEMA || !Array.isArray(index.entries) || typeof index.bundleDigest !== 'string') { err('BAD_INDEX', 'not a portable evidence bundle index'); return out(); }
|
|
527
|
+
if (index.entries.length > BUNDLE_LIMITS.maxEntries) { err('LIMIT', 'too many entries'); return out(); }
|
|
528
|
+
|
|
529
|
+
const names = new Set();
|
|
530
|
+
let total = 0;
|
|
531
|
+
const loaded = new Map();
|
|
532
|
+
for (const e of index.entries) {
|
|
533
|
+
if (!e || !ROLES.includes(e.role) || !DIGEST_RE.test(e.digest ?? '') || !Number.isInteger(e.size) || e.size < 0) { err('BAD_ENTRY', 'malformed entry', { entry: e?.name ?? null }); continue; }
|
|
534
|
+
const bad = checkLogicalName(e.name);
|
|
535
|
+
if (bad) { err('BAD_ENTRY', `entry name rejected: ${bad}`, { entry: String(e.name).slice(0, 60) }); continue; }
|
|
536
|
+
if (names.has(e.name)) { err('BAD_ENTRY', 'duplicate entry name', { entry: e.name }); continue; }
|
|
537
|
+
names.add(e.name);
|
|
538
|
+
if (e.size > BUNDLE_LIMITS.maxBlobBytes) { err('TOO_LARGE', `entry exceeds the per-blob limit`, { entry: e.name }); continue; }
|
|
539
|
+
total += e.size;
|
|
540
|
+
let bytes;
|
|
541
|
+
try { bytes = readBounded(blobFile(dir, e.digest), BUNDLE_LIMITS.maxBlobBytes); } catch (x) {
|
|
542
|
+
err(x.code === 'ENOENT' ? 'MISSING_BLOB' : (x.code || 'MISSING_BLOB'), `blob for '${e.name}' is unavailable: ${x.code || x.message}`, { entry: e.name }); continue;
|
|
543
|
+
}
|
|
544
|
+
if (bytes.length !== e.size) { err('SIZE_MISMATCH', `blob size differs from the index`, { entry: e.name }); continue; }
|
|
545
|
+
if (digestOfBytes(bytes) !== e.digest) { err('DIGEST_MISMATCH', `blob for '${e.name}' was modified after export`, { entry: e.name }); continue; }
|
|
546
|
+
const hit = findSecret(bytes.toString('utf8'));
|
|
547
|
+
if (hit) err('SECRET_IN_BLOB', `'${e.name}' carries a secret shape`, { entry: e.name });
|
|
548
|
+
loaded.set(e.name, { e, bytes });
|
|
549
|
+
}
|
|
550
|
+
if (total > BUNDLE_LIMITS.maxTotalBytes) err('LIMIT', 'total size exceeds the limit');
|
|
551
|
+
for (const role of REQUIRED_ROLES) if (!index.entries.some((e) => e?.role === role)) err('MISSING_ROLE', `required role '${role}' is absent`);
|
|
552
|
+
if (index.entries.every((e) => e && DIGEST_RE.test(e.digest ?? '')) && bundleDigestOf(index) !== index.bundleDigest) err('BUNDLE_DIGEST_MISMATCH', 'the index does not match its recorded bundle digest');
|
|
553
|
+
|
|
554
|
+
// manifest
|
|
555
|
+
let manifest = null;
|
|
556
|
+
const mEntry = index.entries.find((e) => e?.role === 'manifest');
|
|
557
|
+
if (mEntry && loaded.has(mEntry.name)) {
|
|
558
|
+
try { manifest = JSON.parse(loaded.get(mEntry.name).bytes.toString('utf8')); } catch { err('MANIFEST_INVALID', 'manifest is not JSON'); }
|
|
559
|
+
if (manifest) {
|
|
560
|
+
const v = validateManifest(manifest);
|
|
561
|
+
if (!v.ok) err('MANIFEST_INVALID', `manifest fails validation: ${v.errors[0].code} ${v.errors[0].path}`, { detail: v.errors.slice(0, 5) });
|
|
562
|
+
else {
|
|
563
|
+
if (manifestDigest(manifest) !== index.manifestDigest) err('MANIFEST_MISMATCH', 'the manifest is not the one the index was built for');
|
|
564
|
+
const have = new Set(index.entries.filter((e) => e?.role === 'receipt').map((e) => e.digest));
|
|
565
|
+
for (const r of manifest.verificationReceipts) if (!have.has(r.digest)) err('MISSING_EVIDENCE', `receipt '${r.id}' cited by the manifest is not in the bundle`, { receipt: r.id });
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
const replayManifests = [];
|
|
570
|
+
for (const [, { e, bytes }] of loaded) if (e.role === 'replay-manifest') { try { replayManifests.push(JSON.parse(bytes.toString('utf8'))); } catch { /* integrity already checked; unparsable replay manifest adds no prerequisites */ } }
|
|
571
|
+
return out({
|
|
572
|
+
manifest: errors.length === 0 ? manifest : null,
|
|
573
|
+
entries: [...loaded.values()].map(({ e }) => ({ role: e.role, name: e.name, digest: e.digest })),
|
|
574
|
+
bundleDigest: index.bundleDigest,
|
|
575
|
+
replay: { attempted: false, prerequisites: replayPrerequisites(replayManifests), statement: 'replay was not attempted: this verifier is offline and executes nothing from the bundle' },
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
// ---------------------------------------------------------------- import
|
|
580
|
+
|
|
581
|
+
/**
|
|
582
|
+
* Verify `from`, then materialize its entries under `to` using their logical names. Verification failure writes nothing. Every
|
|
583
|
+
* destination is resolved and confined to `to`; an existing destination is never overwritten.
|
|
584
|
+
*/
|
|
585
|
+
function importBundle({ from, to }) {
|
|
586
|
+
const v = verifyBundle(from);
|
|
587
|
+
if (!v.ok) return { ok: false, errors: v.errors, written: [] };
|
|
588
|
+
const index = JSON.parse(fs.readFileSync(path.join(from, INDEX_FILE), 'utf8'));
|
|
589
|
+
const root = path.resolve(to);
|
|
590
|
+
const plan = [];
|
|
591
|
+
for (const e of index.entries) {
|
|
592
|
+
const dest = path.resolve(root, e.name);
|
|
593
|
+
if (dest !== root && !dest.startsWith(root + path.sep)) return { ok: false, errors: [{ code: 'BAD_ENTRY', message: `entry '${e.name}' escapes the destination` }], written: [] };
|
|
594
|
+
if (fs.existsSync(dest)) return { ok: false, errors: [{ code: 'DESTINATION_EXISTS', message: `'${e.name}' already exists at the destination` }], written: [] };
|
|
595
|
+
plan.push({ e, dest });
|
|
596
|
+
}
|
|
597
|
+
fs.mkdirSync(root, { recursive: true });
|
|
598
|
+
const written = [];
|
|
599
|
+
for (const { e, dest } of plan) {
|
|
600
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
601
|
+
writeAtomic(dest, readBounded(blobFile(from, e.digest), BUNDLE_LIMITS.maxBlobBytes));
|
|
602
|
+
written.push(e.name);
|
|
603
|
+
}
|
|
604
|
+
return { ok: true, errors: [], written, bundleDigest: v.bundleDigest, replay: v.replay };
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
/***/ }),
|
|
609
|
+
|
|
610
|
+
/***/ 83480:
|
|
611
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
612
|
+
|
|
613
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
614
|
+
/* harmony export */ TJ: () => (/* binding */ buildProgressView)
|
|
615
|
+
/* harmony export */ });
|
|
616
|
+
/* unused harmony exports PROGRESS_SCHEMA, PROGRESS_VERSION, FINDINGS_SCHEMA, aggregateFindings, reviewItemsFromInvariantCoverage, reviewItemsFromBoundaryContexts, portfolioProgressFields, attachPortfolioProgress */
|
|
617
|
+
/* harmony import */ var _assurance_config_js__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(20622);
|
|
618
|
+
/* harmony import */ var _bundle_js__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(64130);
|
|
619
|
+
/* harmony import */ var _wording_js__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(33695);
|
|
620
|
+
/* harmony import */ var _scheduler_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(98406);
|
|
621
|
+
/* harmony import */ var _work_units_js__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(42284);
|
|
622
|
+
// Coverage-aware portfolio progress and review views (X-707).
|
|
623
|
+
//
|
|
624
|
+
// ONE projection for every surface, in the pattern of lineage/deployment/projection.js: the CLI (`agentic-security portfolio progress`),
|
|
625
|
+
// the MCP tool (`portfolio_progress`) and the fleet summary all call `buildProgressView`, so none formats its own progress text and the
|
|
626
|
+
// surfaces cannot disagree. It is a VIEW: it reads a verified store and ledger and invents nothing.
|
|
627
|
+
//
|
|
628
|
+
// What it keeps apart (X-707.AC01), because adding them together is how a portfolio looks finished when it is not:
|
|
629
|
+
// verified units whose result is current and complete. The only count that is progress.
|
|
630
|
+
// problems blocked, failed, canceled and stale units, each listed with its reason. A stale unit is one whose earlier result was
|
|
631
|
+
// invalidated by a changed input; it is pending again and its old result does not count.
|
|
632
|
+
// coverage per repository, how many of its units are verified, and whether the repository is fully verified, partial or not started.
|
|
633
|
+
// budget per limit: the limit (or `null` when none is enforced), what is used, what in-flight attempts hold, what remains.
|
|
634
|
+
// review what a human still has to look at: blocked units, contracts awaiting approval, boundaries that could not be resolved.
|
|
635
|
+
//
|
|
636
|
+
// Aggregate findings (X-707.AC02) deduplicate by STABLE IDENTITY only. Two findings with the same `stableId` are one aggregate finding
|
|
637
|
+
// with every occurrence kept (repository, environment, release, commit, file, line, severity). The affected releases are listed
|
|
638
|
+
// separately, per finding and per release. A finding with no stable id is never merged by guesswork: it stays its own row, listed apart.
|
|
639
|
+
//
|
|
640
|
+
// Liveness (X-707.AC03). A worker's last sign of life is the newest of its heartbeat and its lease events. Beyond the declared
|
|
641
|
+
// `HEARTBEAT.staleAfterMs` it is `stale` and stays in the view with its age: a stale worker is never dropped from the report to make it
|
|
642
|
+
// look healthier, and its output is irrelevant (nothing here reads stdout). A controller that has finished is reported as FINISHED, which
|
|
643
|
+
// is not the same as every unit verified, which is not the same as every repository passing; the view says each separately and never
|
|
644
|
+
// states that the portfolio passed.
|
|
645
|
+
//
|
|
646
|
+
// Strings from the store (failure reasons) are checked for secret shapes before they are shown. Pure given its inputs: no clock (`now`
|
|
647
|
+
// is an argument), no randomness, fixed key order.
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
const PROGRESS_SCHEMA = 'agentic-security/portfolio-progress';
|
|
656
|
+
const PROGRESS_VERSION = '1.0.0';
|
|
657
|
+
const FINDINGS_SCHEMA = 'agentic-security/portfolio-findings';
|
|
658
|
+
const SEVERITY_RANK = Object.freeze({ info: 0, low: 1, medium: 2, high: 3, critical: 4 });
|
|
659
|
+
const MAX_STRING = 300;
|
|
660
|
+
const MAX_LIST = 200;
|
|
661
|
+
|
|
662
|
+
const safe = (s) => {
|
|
663
|
+
const t = String(s ?? '').slice(0, MAX_STRING);
|
|
664
|
+
return (0,_bundle_js__WEBPACK_IMPORTED_MODULE_1__/* .findSecret */ .mt)(t) ? '[withheld: secret-shaped text]' : t;
|
|
665
|
+
};
|
|
666
|
+
const byKey = (k) => (a, b) => (a[k] < b[k] ? -1 : a[k] > b[k] ? 1 : 0);
|
|
667
|
+
const cap = (list) => ({ items: list.slice(0, MAX_LIST), truncated: list.length > MAX_LIST, total: list.length });
|
|
668
|
+
|
|
669
|
+
// ---------------------------------------------------------------- aggregate findings (X-707.AC02)
|
|
670
|
+
|
|
671
|
+
/**
|
|
672
|
+
* @param {Array<{ repository: string, environment?: string, release?: string, commit?: string, evidenceRef?: string, finding: object }>} entries
|
|
673
|
+
* each occurrence of a finding in one repository/environment/release
|
|
674
|
+
*/
|
|
675
|
+
function aggregateFindings(entries, { blockingSeverity = 'high' } = {}) {
|
|
676
|
+
const rank = (s) => SEVERITY_RANK[s] ?? -1;
|
|
677
|
+
const groups = new Map();
|
|
678
|
+
const unidentified = [];
|
|
679
|
+
let occurrences = 0;
|
|
680
|
+
for (const e of Array.isArray(entries) ? entries : []) {
|
|
681
|
+
const f = e?.finding;
|
|
682
|
+
if (!e || typeof e.repository !== 'string' || !f || typeof f !== 'object') continue;
|
|
683
|
+
occurrences += 1;
|
|
684
|
+
const occ = {
|
|
685
|
+
repository: e.repository, environment: e.environment ?? null, release: e.release ?? null, commit: e.commit ?? null,
|
|
686
|
+
file: typeof f.file === 'string' ? f.file : null, line: Number.isInteger(f.line) ? f.line : null, severity: f.severity ?? null, evidenceRef: e.evidenceRef ?? null,
|
|
687
|
+
};
|
|
688
|
+
if (typeof f.stableId !== 'string' || !f.stableId) { unidentified.push({ ...occ, vuln: safe(f.vuln), cwe: f.cwe ?? null, note: 'no stable id: not merged with any other finding' }); continue; }
|
|
689
|
+
if (!groups.has(f.stableId)) groups.set(f.stableId, { identity: f.stableId, vuln: safe(f.vuln), cwe: f.cwe ?? null, family: f.family ?? null, occurrences: [] });
|
|
690
|
+
groups.get(f.stableId).occurrences.push(occ);
|
|
691
|
+
}
|
|
692
|
+
const items = [...groups.values()].sort(byKey('identity')).map((g) => {
|
|
693
|
+
g.occurrences.sort((a, b) => `${a.repository}|${a.environment}|${a.release}|${a.file}|${a.line}`.localeCompare(`${b.repository}|${b.environment}|${b.release}|${b.file}|${b.line}`));
|
|
694
|
+
const sev = g.occurrences.reduce((m, o) => (rank(o.severity) > rank(m) ? o.severity : m), null);
|
|
695
|
+
const uniq = (k) => [...new Set(g.occurrences.map((o) => o[k]).filter((v) => v !== null))].sort();
|
|
696
|
+
return { ...g, severity: sev, severities: uniq('severity'), repositories: uniq('repository'), environments: uniq('environment'), affectedReleases: uniq('release'), occurrenceCount: g.occurrences.length };
|
|
697
|
+
});
|
|
698
|
+
const byRelease = {};
|
|
699
|
+
for (const it of items) for (const r of it.affectedReleases) (byRelease[r] ??= []).push(it.identity);
|
|
700
|
+
const bySeverity = {};
|
|
701
|
+
for (const it of items) bySeverity[it.severity ?? 'unknown'] = (bySeverity[it.severity ?? 'unknown'] ?? 0) + 1;
|
|
702
|
+
const blockingRepos = new Set();
|
|
703
|
+
for (const it of items) for (const o of it.occurrences) if (rank(o.severity) >= rank(blockingSeverity)) blockingRepos.add(o.repository);
|
|
704
|
+
for (const o of unidentified) if (rank(o.severity) >= rank(blockingSeverity)) blockingRepos.add(o.repository);
|
|
705
|
+
return {
|
|
706
|
+
schema: FINDINGS_SCHEMA, schemaVersion: PROGRESS_VERSION, occurrences, unique: items.length, blockingSeverity,
|
|
707
|
+
items, unidentified: unidentified.sort((a, b) => `${a.repository}|${a.file}|${a.line}`.localeCompare(`${b.repository}|${b.file}|${b.line}`)),
|
|
708
|
+
byRelease: Object.fromEntries(Object.keys(byRelease).sort().map((k) => [k, byRelease[k].sort()])),
|
|
709
|
+
bySeverity: Object.fromEntries(Object.keys(bySeverity).sort().map((k) => [k, bySeverity[k]])),
|
|
710
|
+
repositoriesWithBlockingFindings: [...blockingRepos].sort(),
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
// ---------------------------------------------------------------- review queue sources
|
|
715
|
+
|
|
716
|
+
/** Review items from a business coverage report (posture/invariants/coverage.js): contracts that are not approved. */
|
|
717
|
+
function reviewItemsFromInvariantCoverage(coverage) {
|
|
718
|
+
const out = [];
|
|
719
|
+
for (const i of coverage?.invariants ?? []) {
|
|
720
|
+
if (i.state !== 'approved') out.push({ kind: 'contract-approval', id: i.id ?? i.key, repository: null, reason: `contract '${safe(i.key)}' is ${safe(i.state)}: a violation of it is advisory until a reviewer approves it`, source: 'invariant-coverage' });
|
|
721
|
+
}
|
|
722
|
+
return out;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/** Review items from boundary contexts (lineage/deployment/projection.js): findings whose deployment exposure is unresolved or unbound. */
|
|
726
|
+
function reviewItemsFromBoundaryContexts(contexts) {
|
|
727
|
+
const out = [];
|
|
728
|
+
for (const c of Array.isArray(contexts) ? contexts : []) {
|
|
729
|
+
if (!c || typeof c !== 'object') continue;
|
|
730
|
+
const unresolved = c.exposure?.state === 'unresolved';
|
|
731
|
+
const unbound = c.binding && c.binding.status !== 'bound';
|
|
732
|
+
if (unresolved || unbound) out.push({ kind: 'boundary-resolution', id: c.id ?? null, repository: c.finding?.repository ?? null, reason: unresolved ? 'deployment reachability is unresolved: a person must decide whether the path is real' : 'the finding is not bound to a deployed service', source: 'deployment-boundaries' });
|
|
733
|
+
}
|
|
734
|
+
return out;
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
// ---------------------------------------------------------------- the view
|
|
738
|
+
|
|
739
|
+
const lastSignal = (u, ledger) => {
|
|
740
|
+
const attemptId = u.lease?.attemptId;
|
|
741
|
+
let at = 0;
|
|
742
|
+
for (const e of u.events) if (e.attemptId === attemptId && typeof e.at === 'number' && e.at > at) at = e.at;
|
|
743
|
+
const hb = attemptId ? ledger.heartbeats?.[attemptId]?.at : null;
|
|
744
|
+
return Math.max(at, typeof hb === 'number' ? hb : 0);
|
|
745
|
+
};
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* @param {object} p
|
|
749
|
+
* @param {object} p.store a verified portfolio store
|
|
750
|
+
* @param {object} [p.ledger] the scheduler ledger (usage, reservations, heartbeats, cancellations)
|
|
751
|
+
* @param {object} [p.budgets] the budgets the scheduler runs under
|
|
752
|
+
* @param {number} p.now milliseconds
|
|
753
|
+
* @param {Array} [p.reviewItems]
|
|
754
|
+
* @param {Array} [p.findings] occurrences for `aggregateFindings`
|
|
755
|
+
* @param {string} [p.blockingSeverity]
|
|
756
|
+
* @param {{ intervalMs: number, staleAfterMs: number }} [p.heartbeat]
|
|
757
|
+
*/
|
|
758
|
+
function buildProgressView({ store, ledger = null, budgets = null, now, reviewItems = [], findings = null, blockingSeverity = 'high', heartbeat = _scheduler_js__WEBPACK_IMPORTED_MODULE_3__/* .HEARTBEAT */ .rU } = {}) {
|
|
759
|
+
if (!store || typeof store.units !== 'object') return { ok: false, errors: [{ code: 'NO_STORE', message: 'a verified portfolio store is required' }] };
|
|
760
|
+
if (!Number.isFinite(now)) return { ok: false, errors: [{ code: 'NO_NOW', message: 'now (milliseconds) is required; the view reads no clock' }] };
|
|
761
|
+
const led = ledger ?? { usage: { portfolio: {}, repositories: {} }, reservations: {}, heartbeats: {}, cancellations: [], overruns: [] };
|
|
762
|
+
const units = Object.values(store.units).sort(byKey('id'));
|
|
763
|
+
const ref = (u) => ({ unitId: u.id, repository: u.repository, taskType: u.taskType });
|
|
764
|
+
|
|
765
|
+
const count = Object.fromEntries(_work_units_js__WEBPACK_IMPORTED_MODULE_4__/* .UNIT_STATES */ .Yc.map((s) => [s, 0]));
|
|
766
|
+
for (const u of units) count[u.state] += 1;
|
|
767
|
+
const lastEvent = (u, type) => [...u.events].reverse().find((e) => e.type === type);
|
|
768
|
+
const blocked = units.filter((u) => u.state === 'blocked').map((u) => ({ ...ref(u), reason: safe(u.blockedReason) }));
|
|
769
|
+
const failed = units.filter((u) => u.state === 'failed').map((u) => ({ ...ref(u), retryCount: u.retryCount, reason: safe(lastEvent(u, 'failed')?.reason ?? (lastEvent(u, 'expired') ? 'lease expired' : 'failed')) }));
|
|
770
|
+
const canceled = units.filter((u) => u.state === 'canceled').map((u) => ({ ...ref(u), reason: safe(lastEvent(u, 'canceled')?.reason) }));
|
|
771
|
+
const stale = units.filter((u) => u.stale.length && u.state !== 'verified').map((u) => ({ ...ref(u), state: u.state, staleResults: u.stale.length, reason: 'an earlier result was invalidated by a changed input and does not count', changedDimensions: u.stale.at(-1)?.changedDimensions ?? null }));
|
|
772
|
+
|
|
773
|
+
// workers: every lease holder, live or stale
|
|
774
|
+
const workers = units.filter((u) => u.state === 'leased' || u.state === 'running').map((u) => {
|
|
775
|
+
const last = lastSignal(u, led);
|
|
776
|
+
const age = Math.max(0, now - last);
|
|
777
|
+
return { ...ref(u), holder: u.lease.holder, attemptId: u.lease.attemptId, state: u.state, lastSignalAt: last || null, ageMs: age, status: !last ? 'no-heartbeat' : age > heartbeat.staleAfterMs ? 'stale' : 'live', leaseExpiresAt: u.lease.expiresAt };
|
|
778
|
+
}).sort(byKey('attemptId'));
|
|
779
|
+
|
|
780
|
+
// coverage by repository
|
|
781
|
+
const repos = new Map();
|
|
782
|
+
for (const u of units) {
|
|
783
|
+
if (!repos.has(u.repository)) repos.set(u.repository, { repository: u.repository, commit: u.commit, units: 0, verified: 0, problems: 0, inFlight: 0, pending: 0 });
|
|
784
|
+
const r = repos.get(u.repository);
|
|
785
|
+
r.units += 1;
|
|
786
|
+
if (u.state === 'verified') r.verified += 1;
|
|
787
|
+
else if (u.state === 'blocked' || u.state === 'failed' || u.state === 'canceled') r.problems += 1;
|
|
788
|
+
else if (u.state === 'leased' || u.state === 'running') r.inFlight += 1;
|
|
789
|
+
else r.pending += 1;
|
|
790
|
+
}
|
|
791
|
+
const coverageRepos = [...repos.values()].sort(byKey('repository')).map((r) => ({ ...r, status: r.verified === r.units ? 'fully-verified' : r.verified > 0 ? 'partial' : r.problems > 0 ? 'incomplete' : 'not-started' }));
|
|
792
|
+
const fully = coverageRepos.filter((r) => r.status === 'fully-verified').length;
|
|
793
|
+
|
|
794
|
+
// budget
|
|
795
|
+
const reserved = { portfolio: {}, repositories: {} };
|
|
796
|
+
for (const r of Object.values(led.reservations ?? {})) {
|
|
797
|
+
reserved.portfolio.count = (reserved.portfolio.count ?? 0) + 1;
|
|
798
|
+
for (const d of _scheduler_js__WEBPACK_IMPORTED_MODULE_3__/* .DIMENSIONS */ .Yf) reserved.portfolio[d] = (reserved.portfolio[d] ?? 0) + r[d];
|
|
799
|
+
reserved.repositories[r.repository] ??= { count: 0 };
|
|
800
|
+
reserved.repositories[r.repository].count += 1;
|
|
801
|
+
for (const d of _scheduler_js__WEBPACK_IMPORTED_MODULE_3__/* .DIMENSIONS */ .Yf) reserved.repositories[r.repository][d] = (reserved.repositories[r.repository][d] ?? 0) + r[d];
|
|
802
|
+
}
|
|
803
|
+
const dimView = (limits, used, held) => {
|
|
804
|
+
const o = {};
|
|
805
|
+
for (const d of [..._scheduler_js__WEBPACK_IMPORTED_MODULE_3__/* .DIMENSIONS */ .Yf, 'concurrency']) {
|
|
806
|
+
const limit = Number.isFinite(limits?.[d]) ? limits[d] : null;
|
|
807
|
+
const u = d === 'concurrency' ? (held?.count ?? 0) : (used?.[d] ?? 0);
|
|
808
|
+
const h = d === 'concurrency' ? 0 : (held?.[d] ?? 0);
|
|
809
|
+
o[d] = { limit, used: u, reserved: h, remaining: limit === null ? null : Math.max(0, limit - u - h), enforced: limit !== null };
|
|
810
|
+
}
|
|
811
|
+
return o;
|
|
812
|
+
};
|
|
813
|
+
const repoLimits = (name) => ({ ...(budgets?.repositories?.default ?? {}), ...(budgets?.repositories?.[name] ?? {}) });
|
|
814
|
+
const budget = {
|
|
815
|
+
declared: !!budgets,
|
|
816
|
+
portfolio: dimView(budgets?.portfolio, led.usage?.portfolio, reserved.portfolio),
|
|
817
|
+
repositories: Object.fromEntries(coverageRepos.map((r) => [r.repository, dimView(repoLimits(r.repository), led.usage?.repositories?.[r.repository], reserved.repositories[r.repository])])),
|
|
818
|
+
overruns: (led.overruns ?? []).length,
|
|
819
|
+
};
|
|
820
|
+
|
|
821
|
+
// review queue
|
|
822
|
+
const review = [
|
|
823
|
+
...blocked.map((b) => ({ kind: 'blocked-unit', id: b.unitId, repository: b.repository, reason: b.reason, source: 'portfolio' })),
|
|
824
|
+
...(Array.isArray(reviewItems) ? reviewItems : []).map((i) => ({ kind: String(i.kind ?? 'review'), id: i.id ?? null, repository: i.repository ?? null, reason: safe(i.reason), source: String(i.source ?? 'supplied') })),
|
|
825
|
+
];
|
|
826
|
+
const reviewByKind = {};
|
|
827
|
+
for (const r of review) reviewByKind[r.kind] = (reviewByKind[r.kind] ?? 0) + 1;
|
|
828
|
+
|
|
829
|
+
const aggregate = Array.isArray(findings) ? aggregateFindings(findings, { blockingSeverity }) : null;
|
|
830
|
+
const inFlight = count.leased + count.running;
|
|
831
|
+
const total = units.length;
|
|
832
|
+
const controllerFinished = inFlight === 0 && count.pending === 0;
|
|
833
|
+
const allVerified = total > 0 && count.verified === total;
|
|
834
|
+
const nonVerified = total - count.verified;
|
|
835
|
+
const completion = {
|
|
836
|
+
controllerFinished,
|
|
837
|
+
allUnitsVerified: allVerified,
|
|
838
|
+
repositoriesFullyVerified: { n: fully, of: coverageRepos.length },
|
|
839
|
+
findingsSupplied: aggregate !== null,
|
|
840
|
+
repositoriesWithBlockingFindings: aggregate ? aggregate.repositoriesWithBlockingFindings : null,
|
|
841
|
+
passAssessment: 'not-implied',
|
|
842
|
+
statement: controllerFinished && !allVerified
|
|
843
|
+
? `The controller has finished, but ${nonVerified} of ${total} unit(s) are not verified (${count.blocked} blocked, ${count.failed} failed, ${count.canceled} canceled${count.pending ? `, ${count.pending} pending` : ''}). Finished does not mean every repository was checked, and it does not mean any repository passed.`
|
|
844
|
+
: controllerFinished && allVerified
|
|
845
|
+
? `Every planned unit is verified (${total} of ${total}) in ${fully} of ${coverageRepos.length} repositories. That means the planned checks completed; it does not mean every repository passed${aggregate ? `: ${aggregate.repositoriesWithBlockingFindings.length} repositor${aggregate.repositoriesWithBlockingFindings.length === 1 ? 'y has' : 'ies have'} ${blockingSeverity}-or-worse findings in the supplied results` : ' (findings were not supplied to this view)'}.`
|
|
846
|
+
: `The controller has not finished: ${count.verified} of ${total} unit(s) verified, ${inFlight} in flight, ${count.pending} pending.`,
|
|
847
|
+
};
|
|
848
|
+
|
|
849
|
+
const staleWorkers = workers.filter((w) => w.status !== 'live');
|
|
850
|
+
const lines = [
|
|
851
|
+
`Portfolio progress (${store.synthetic ? 'SYNTHETIC; ' : ''}as of ${now}): ${count.verified}/${total} unit(s) verified; ${fully}/${coverageRepos.length} repositor${coverageRepos.length === 1 ? 'y' : 'ies'} fully verified`,
|
|
852
|
+
` Not verified: ${count.blocked} blocked, ${count.failed} failed, ${count.canceled} canceled, ${stale.length} stale, ${count.pending} pending, ${inFlight} in flight`,
|
|
853
|
+
` Workers: ${workers.length - staleWorkers.length} live, ${staleWorkers.length} stale or silent (stale after ${heartbeat.staleAfterMs} ms without a heartbeat)`,
|
|
854
|
+
` Budget: ${budgets ? _scheduler_js__WEBPACK_IMPORTED_MODULE_3__/* .DIMENSIONS */ .Yf.filter((d) => budget.portfolio[d].enforced).map((d) => `${d} ${budget.portfolio[d].remaining} left`).join(', ') || 'no portfolio limit enforced' : 'not declared to this view'}`,
|
|
855
|
+
` Pending human review: ${review.length}${review.length ? ` (${Object.keys(reviewByKind).sort().map((k) => `${k}: ${reviewByKind[k]}`).join(', ')})` : ''}`,
|
|
856
|
+
` ${completion.statement}`,
|
|
857
|
+
` ${_wording_js__WEBPACK_IMPORTED_MODULE_2__/* .NOT_A_GUARANTEE */ .zi}`,
|
|
858
|
+
];
|
|
859
|
+
return {
|
|
860
|
+
ok: true,
|
|
861
|
+
view: {
|
|
862
|
+
schema: PROGRESS_SCHEMA, schemaVersion: PROGRESS_VERSION, ...(store.synthetic ? { synthetic: true } : {}), planId: store.planId, asOf: now,
|
|
863
|
+
heartbeat: { intervalMs: heartbeat.intervalMs, staleAfterMs: heartbeat.staleAfterMs },
|
|
864
|
+
units: { total, verified: count.verified, pending: count.pending, inFlight, blocked: cap(blocked), failed: cap(failed), canceled: cap(canceled), stale: cap(stale), counts: count },
|
|
865
|
+
workers: { items: workers, stale: staleWorkers.length, live: workers.length - staleWorkers.length },
|
|
866
|
+
coverage: { repositories: coverageRepos, fullyVerified: fully, total: coverageRepos.length },
|
|
867
|
+
budget,
|
|
868
|
+
review: { pending: review.length, byKind: Object.fromEntries(Object.keys(reviewByKind).sort().map((k) => [k, reviewByKind[k]])), ...cap(review) },
|
|
869
|
+
cancellations: (led.cancellations ?? []).map((c) => ({ scope: c.scope, reason: safe(c.reason), at: c.at })),
|
|
870
|
+
findings: aggregate,
|
|
871
|
+
completion,
|
|
872
|
+
lines,
|
|
873
|
+
},
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
/** The additive field a surface asks for: `{ portfolioProgress }`, or `{}` when the feature is off, so flag-off output is unchanged. */
|
|
878
|
+
function portfolioProgressFields({ config, ...input } = {}) {
|
|
879
|
+
if (!config || featureStatus(config, FEATURE_ID).status !== 'ok') return {};
|
|
880
|
+
const r = buildProgressView(input);
|
|
881
|
+
return r.ok ? { portfolioProgress: r.view } : { portfolioProgressErrors: r.errors };
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/** Attach the view to a fleet rollup. Flag off: the SAME object comes back untouched. */
|
|
885
|
+
function attachPortfolioProgress(rollup, input) {
|
|
886
|
+
const extra = portfolioProgressFields(input);
|
|
887
|
+
return Object.keys(extra).length ? { ...rollup, ...extra } : rollup;
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
/***/ }),
|
|
892
|
+
|
|
893
|
+
/***/ 98406:
|
|
894
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
895
|
+
|
|
896
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
897
|
+
/* harmony export */ GN: () => (/* binding */ ledgerPathFor),
|
|
898
|
+
/* harmony export */ SC: () => (/* binding */ readLedger),
|
|
899
|
+
/* harmony export */ Yf: () => (/* binding */ DIMENSIONS),
|
|
900
|
+
/* harmony export */ rU: () => (/* binding */ HEARTBEAT)
|
|
901
|
+
/* harmony export */ });
|
|
902
|
+
/* unused harmony exports LEDGER_SCHEMA, WEIGHT_RANGE, SKIP_CODES, validateBudgets, newLedger, selectUnit, scheduleNext, settleAttempt, heartbeat, routingBudgetFor, spendBoundOf, chargeModelSpend, createCancelScope, runScheduled */
|
|
903
|
+
/* harmony import */ var node_fs__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(73024);
|
|
904
|
+
/* harmony import */ var node_path__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(76760);
|
|
905
|
+
/* harmony import */ var _sandbox_supervise_js__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(81129);
|
|
906
|
+
/* harmony import */ var _work_units_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(42284);
|
|
907
|
+
// Bounded portfolio scheduler: budgets, fairness and scoped cancellation (X-706).
|
|
908
|
+
//
|
|
909
|
+
// work-units.js gives durable, leased units but hands out the next pending unit in id order, with no notion of cost and no notion of
|
|
910
|
+
// who is waiting. This module decides WHICH unit is leased next, and whether ANY unit may be leased at all, from five limits declared
|
|
911
|
+
// at two levels (the whole portfolio, and each repository): wall time, provider spend, concurrency, request count and storage.
|
|
912
|
+
//
|
|
913
|
+
// ADMISSION (X-706.AC01). The check happens BEFORE the lease, inside the same locked store transaction that creates the lease, so two
|
|
914
|
+
// processes cannot both squeeze under a limit. A unit must carry an ESTIMATE of what it can cost (an unknown cost is never zero: a unit
|
|
915
|
+
// with no estimate is not leased, and is reported `no-estimate`). Admission counts what has been used, plus what in-flight attempts have
|
|
916
|
+
// RESERVED, plus this unit's estimate. A refusal is one of two kinds, and they are reported separately:
|
|
917
|
+
// at-capacity would fit once an in-flight attempt settles (a wait)
|
|
918
|
+
// exhausted would not fit even with nothing in flight (will not recover without a larger budget)
|
|
919
|
+
// An attempt that ends without a usage report (expired, failed, canceled) is charged its whole reservation, because what it spent is
|
|
920
|
+
// unknown. A reported overrun of the estimate is recorded and shrinks what remains; it is detected, not rolled back. Wall time is also
|
|
921
|
+
// enforced while the unit runs: the executor's signal aborts at the unit's wall estimate (capped by what the portfolio has left).
|
|
922
|
+
//
|
|
923
|
+
// FAIRNESS (X-706.AC02). Priority is a WEIGHT, not an order. Among repositories that currently have an admissible unit, the next lease
|
|
924
|
+
// goes to the one with the lowest `leasesGranted / weight` (weights are integers 1..8, default 1; ties break on repository name, and
|
|
925
|
+
// within a repository on unit id). Consequences, each tested:
|
|
926
|
+
// - no starvation: a repository that is admissible is served within a bounded number of leases whatever the others hold, so a
|
|
927
|
+
// repository with 300 units cannot hold back one with 1;
|
|
928
|
+
// - priority shifts the SHARE, it never removes it: weight 8 against weight 1 is served about eight times as often, and the
|
|
929
|
+
// weight-1 repository is still served;
|
|
930
|
+
// - a repository that was blocked for a long time is owed service and is served first when it is admissible again (a bounded burst);
|
|
931
|
+
// - a blocked repository (declared blocked, over its own budget, or with every unit blocked) is SKIPPED, never waited on. The skip
|
|
932
|
+
// reason is returned per repository, so a stall is visible instead of silent.
|
|
933
|
+
// A unit whose estimate cannot fit the remaining budget is never leased and never blocks a smaller sibling.
|
|
934
|
+
//
|
|
935
|
+
// CANCELLATION (X-706.AC03). A cancellation names a scope (one unit, one repository, or everything) and is recorded in the ledger, so a
|
|
936
|
+
// process that did not receive the call still honours it. For attempts running in THIS process the scope's AbortController is aborted;
|
|
937
|
+
// work started through `ctx.spawn` runs under sandbox/supervise.js, which terminates the whole process tree and reports survivors. A
|
|
938
|
+
// lease is RELEASED only when the attempt has settled and nothing survived; otherwise it is left to EXPIRE (so no other worker can start
|
|
939
|
+
// the same unit while a descendant may still be running) and the report says so. Verified units and their receipts are never touched.
|
|
940
|
+
// The report is always `incomplete` with a reason when anything in scope was not verified.
|
|
941
|
+
//
|
|
942
|
+
// SPEND (routing). `routingBudgetFor` gives the routing policy the remaining dollars (portfolio and repository, less reservations), and
|
|
943
|
+
// `chargeModelSpend` refuses a routing decision that is blocked, has no upper cost bound, or would exceed the unit's reservation.
|
|
944
|
+
// No provider is called here; the executor does that through an injected transport.
|
|
945
|
+
//
|
|
946
|
+
// Heartbeats are recorded with the lease renewal. Nothing here reads a clock except where a function takes `clock`/`now`.
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
const LEDGER_SCHEMA = 'agentic-security/portfolio-ledger';
|
|
954
|
+
const LEDGER_VERSION = 1;
|
|
955
|
+
const DIMENSIONS = Object.freeze(['wallMs', 'spendUsd', 'requests', 'storageBytes']);
|
|
956
|
+
const WEIGHT_RANGE = Object.freeze({ min: 1, max: 8, default: 1 });
|
|
957
|
+
const HEARTBEAT = Object.freeze({ intervalMs: 5000, staleAfterMs: 15000 });
|
|
958
|
+
const SKIP_CODES = Object.freeze(['repository-blocked', 'no-estimate', 'at-capacity', 'exhausted', 'canceled']);
|
|
959
|
+
const MAX_LEDGER_BYTES = 16 * 1024 * 1024;
|
|
960
|
+
|
|
961
|
+
const zero = () => ({ wallMs: 0, spendUsd: 0, requests: 0, storageBytes: 0 });
|
|
962
|
+
const isNum = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0;
|
|
963
|
+
|
|
964
|
+
// ---------------------------------------------------------------- budgets
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Budgets: `{ portfolio: { concurrency, wallMs, spendUsd, requests, storageBytes }, repositories?: { default?, [name]: partial limits },
|
|
968
|
+
* priorities?: { [name]: weight } }`. Every portfolio limit is REQUIRED (an unlimited portfolio is a decision to make out loud, not a
|
|
969
|
+
* default); a repository limit that is absent is not enforced for that repository and the view says so.
|
|
970
|
+
*/
|
|
971
|
+
function validateBudgets(b) {
|
|
972
|
+
const errors = [];
|
|
973
|
+
const bad = (p, message) => errors.push({ code: 'BAD_BUDGETS', path: p, message });
|
|
974
|
+
if (!b || typeof b !== 'object') return { ok: false, errors: [{ code: 'BAD_BUDGETS', path: '', message: 'not an object' }] };
|
|
975
|
+
const lim = (o, p, required) => {
|
|
976
|
+
if (!o || typeof o !== 'object') { if (required) bad(p, 'required'); return; }
|
|
977
|
+
if (o.concurrency !== undefined && !(Number.isInteger(o.concurrency) && o.concurrency >= 1 && o.concurrency <= 64)) bad(`${p}.concurrency`, 'an integer in 1..64');
|
|
978
|
+
else if (required && o.concurrency === undefined) bad(`${p}.concurrency`, 'required');
|
|
979
|
+
for (const d of DIMENSIONS) {
|
|
980
|
+
if (o[d] === undefined) { if (required) bad(`${p}.${d}`, 'required'); } else if (!isNum(o[d])) bad(`${p}.${d}`, 'a finite non-negative number');
|
|
981
|
+
}
|
|
982
|
+
};
|
|
983
|
+
lim(b.portfolio, 'portfolio', true);
|
|
984
|
+
for (const [name, o] of Object.entries(b.repositories ?? {})) lim(o, `repositories.${name}`, false);
|
|
985
|
+
for (const [name, w] of Object.entries(b.priorities ?? {})) {
|
|
986
|
+
if (!(Number.isInteger(w) && w >= WEIGHT_RANGE.min && w <= WEIGHT_RANGE.max)) bad(`priorities.${name}`, `an integer weight in ${WEIGHT_RANGE.min}..${WEIGHT_RANGE.max}`);
|
|
987
|
+
}
|
|
988
|
+
return { ok: errors.length === 0, errors };
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
const limitsFor = (budgets, repo) => ({ ...(budgets.repositories?.default ?? {}), ...(budgets.repositories?.[repo] ?? {}) });
|
|
992
|
+
const weightOf = (budgets, repo) => budgets.priorities?.[repo] ?? WEIGHT_RANGE.default;
|
|
993
|
+
|
|
994
|
+
// ---------------------------------------------------------------- ledger
|
|
995
|
+
|
|
996
|
+
const ledgerPathFor = (storeFile) => `${storeFile}.ledger.json`;
|
|
997
|
+
|
|
998
|
+
function newLedger() {
|
|
999
|
+
return { schema: LEDGER_SCHEMA, version: LEDGER_VERSION, usage: { portfolio: zero(), repositories: {} }, reservations: {}, grants: {}, heartbeats: {}, cancellations: [], overruns: [], settled: {} };
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
function readLedger(storeFile) {
|
|
1003
|
+
const file = ledgerPathFor(storeFile);
|
|
1004
|
+
let raw;
|
|
1005
|
+
try {
|
|
1006
|
+
const st = node_fs__WEBPACK_IMPORTED_MODULE_0__.lstatSync(file);
|
|
1007
|
+
if (st.isSymbolicLink() || !st.isFile() || st.size > MAX_LEDGER_BYTES) throw (0,_work_units_js__WEBPACK_IMPORTED_MODULE_3__/* .typedError */ .Zz)('LEDGER_CORRUPT', 'ledger is not a regular file within the size limit');
|
|
1008
|
+
raw = node_fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(file, 'utf8');
|
|
1009
|
+
} catch (e) { if (e.code === 'ENOENT') return newLedger(); throw e; }
|
|
1010
|
+
let l;
|
|
1011
|
+
try { l = JSON.parse(raw); } catch { throw (0,_work_units_js__WEBPACK_IMPORTED_MODULE_3__/* .typedError */ .Zz)('LEDGER_CORRUPT', 'ledger is not valid JSON'); }
|
|
1012
|
+
if (!l || l.schema !== LEDGER_SCHEMA || l.version !== LEDGER_VERSION || typeof l.reservations !== 'object' || typeof l.usage !== 'object') throw (0,_work_units_js__WEBPACK_IMPORTED_MODULE_3__/* .typedError */ .Zz)('LEDGER_CORRUPT', 'not a portfolio ledger of a supported version');
|
|
1013
|
+
return l;
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
function writeLedger(storeFile, ledger) {
|
|
1017
|
+
const file = ledgerPathFor(storeFile);
|
|
1018
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
1019
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
1020
|
+
fs.writeFileSync(tmp, JSON.stringify(ledger), { mode: 0o600 });
|
|
1021
|
+
fs.renameSync(tmp, file);
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
const repoUsage = (ledger, repo) => (ledger.usage.repositories[repo] ??= zero());
|
|
1025
|
+
const add = (a, b) => { for (const d of DIMENSIONS) a[d] += b[d] ?? 0; };
|
|
1026
|
+
|
|
1027
|
+
/** Charge an attempt's reservation to usage and release it. `usage` null charges the whole reservation (what it spent is unknown). Idempotent. */
|
|
1028
|
+
function settle(ledger, attemptId, usage, how) {
|
|
1029
|
+
const r = ledger.reservations[attemptId];
|
|
1030
|
+
if (!r) return null;
|
|
1031
|
+
const charge = {};
|
|
1032
|
+
const over = [];
|
|
1033
|
+
for (const d of DIMENSIONS) {
|
|
1034
|
+
charge[d] = usage && isNum(usage[d]) ? usage[d] : r[d];
|
|
1035
|
+
if (usage && isNum(usage[d]) && usage[d] > r[d]) over.push(d);
|
|
1036
|
+
}
|
|
1037
|
+
add(ledger.usage.portfolio, charge); add(repoUsage(ledger, r.repository), charge);
|
|
1038
|
+
if (over.length) ledger.overruns.push({ attemptId, unitId: r.unitId, repository: r.repository, dimensions: over });
|
|
1039
|
+
delete ledger.reservations[attemptId];
|
|
1040
|
+
delete ledger.heartbeats[attemptId];
|
|
1041
|
+
ledger.settled[attemptId] = how;
|
|
1042
|
+
return { charge, over };
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
const inScope = (unit, scope) => !!scope && (scope.all === true || (scope.unitId && scope.unitId === unit.id) || (scope.repository && scope.repository === unit.repository));
|
|
1046
|
+
|
|
1047
|
+
/** Settle reservations whose lease ended and cancel pending units inside a recorded cancellation scope. Returns what changed. */
|
|
1048
|
+
function reconcile(store, ledger, now) {
|
|
1049
|
+
const recovered = recoverExpired(store, now);
|
|
1050
|
+
const settledNow = [];
|
|
1051
|
+
for (const [attemptId, r] of Object.entries(ledger.reservations)) {
|
|
1052
|
+
const u = store.units[r.unitId];
|
|
1053
|
+
if (!u || !u.lease || u.lease.attemptId !== attemptId) { settle(ledger, attemptId, null, 'lease-ended'); settledNow.push(attemptId); }
|
|
1054
|
+
}
|
|
1055
|
+
const canceled = [];
|
|
1056
|
+
for (const c of ledger.cancellations) {
|
|
1057
|
+
for (const u of Object.values(store.units)) {
|
|
1058
|
+
if ((u.state === 'pending' || u.state === 'blocked') && inScope(u, c.scope)) { cancelUnits(store, { unitId: u.id, reason: c.reason, now }); canceled.push(u.id); }
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
return { recovered, settled: settledNow, canceled };
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
// ---------------------------------------------------------------- admission and selection
|
|
1065
|
+
|
|
1066
|
+
function checkLimits(limits, used, flight, flightCount, est, scope) {
|
|
1067
|
+
const out = [];
|
|
1068
|
+
if (Number.isInteger(limits.concurrency) && flightCount >= limits.concurrency) out.push({ code: 'at-capacity', scope, dimension: 'concurrency', limit: limits.concurrency, inFlight: flightCount });
|
|
1069
|
+
for (const d of DIMENSIONS) {
|
|
1070
|
+
if (!isNum(limits[d])) continue;
|
|
1071
|
+
if (used[d] + est[d] > limits[d]) out.push({ code: 'exhausted', scope, dimension: d, limit: limits[d], used: used[d], needed: est[d] });
|
|
1072
|
+
else if (used[d] + flight[d] + est[d] > limits[d]) out.push({ code: 'at-capacity', scope, dimension: d, limit: limits[d], used: used[d], reserved: flight[d], needed: est[d] });
|
|
1073
|
+
}
|
|
1074
|
+
return out;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
function normalizeEstimate(raw) {
|
|
1078
|
+
if (!raw || typeof raw !== 'object') return null;
|
|
1079
|
+
const e = {};
|
|
1080
|
+
for (const d of DIMENSIONS) { if (!isNum(raw[d])) return null; e[d] = raw[d]; }
|
|
1081
|
+
return e.wallMs > 0 ? e : null;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
/** What each repository and the portfolio currently have reserved by in-flight attempts. */
|
|
1085
|
+
function flightOf(ledger) {
|
|
1086
|
+
const portfolio = { ...zero(), count: 0 };
|
|
1087
|
+
const repos = {};
|
|
1088
|
+
for (const r of Object.values(ledger.reservations)) {
|
|
1089
|
+
const rr = (repos[r.repository] ??= { ...zero(), count: 0 });
|
|
1090
|
+
for (const d of DIMENSIONS) { portfolio[d] += r[d]; rr[d] += r[d]; }
|
|
1091
|
+
portfolio.count += 1; rr.count += 1;
|
|
1092
|
+
}
|
|
1093
|
+
return { portfolio, repos };
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/**
|
|
1097
|
+
* Pure selection over a reconciled store and ledger: the unit that would be leased next, or why none can be.
|
|
1098
|
+
* @returns {{ unit: object|null, estimate: object|null, skipped: Array, idle: boolean, waiting: boolean }}
|
|
1099
|
+
*/
|
|
1100
|
+
function selectUnit({ store, ledger, budgets, estimateOf, blockedRepositories = [] }) {
|
|
1101
|
+
const flight = flightOf(ledger);
|
|
1102
|
+
const pendingByRepo = new Map();
|
|
1103
|
+
for (const u of Object.values(store.units).sort((a, b) => (a.id < b.id ? -1 : 1))) {
|
|
1104
|
+
if (u.state === 'pending') { if (!pendingByRepo.has(u.repository)) pendingByRepo.set(u.repository, []); pendingByRepo.get(u.repository).push(u); }
|
|
1105
|
+
}
|
|
1106
|
+
const skipped = [];
|
|
1107
|
+
const blocked = new Set(blockedRepositories);
|
|
1108
|
+
const candidates = [];
|
|
1109
|
+
for (const [repo, units] of [...pendingByRepo].sort(([a], [b]) => (a < b ? -1 : 1))) {
|
|
1110
|
+
if (blocked.has(repo)) { skipped.push({ repository: repo, code: 'repository-blocked', detail: 'declared blocked; its units are skipped and independent repositories continue' }); continue; }
|
|
1111
|
+
const repoFlight = flight.repos[repo] ?? { ...zero(), count: 0 };
|
|
1112
|
+
const repoLimits = limitsFor(budgets, repo);
|
|
1113
|
+
let pick = null; const unitSkips = [];
|
|
1114
|
+
for (const u of units) {
|
|
1115
|
+
const est = normalizeEstimate(estimateOf(u));
|
|
1116
|
+
if (!est) { unitSkips.push({ repository: repo, unitId: u.id, code: 'no-estimate', detail: 'no cost estimate, and an unknown cost is never treated as zero' }); continue; }
|
|
1117
|
+
const refusals = [
|
|
1118
|
+
...checkLimits(budgets.portfolio, ledger.usage.portfolio, flight.portfolio, flight.portfolio.count, est, 'portfolio'),
|
|
1119
|
+
...checkLimits(repoLimits, repoUsage(ledger, repo), repoFlight, repoFlight.count, est, 'repository'),
|
|
1120
|
+
];
|
|
1121
|
+
if (refusals.length) {
|
|
1122
|
+
const code = refusals.some((r) => r.code === 'exhausted') ? 'exhausted' : 'at-capacity';
|
|
1123
|
+
unitSkips.push({ repository: repo, unitId: u.id, code, detail: refusals.map((r) => `${r.scope} ${r.dimension}`).join(', '), refusals });
|
|
1124
|
+
continue;
|
|
1125
|
+
}
|
|
1126
|
+
pick = { unit: u, estimate: est };
|
|
1127
|
+
break;
|
|
1128
|
+
}
|
|
1129
|
+
skipped.push(...unitSkips.filter((s) => !pick || s.unitId !== pick.unit.id));
|
|
1130
|
+
if (pick) candidates.push({ repo, ...pick });
|
|
1131
|
+
}
|
|
1132
|
+
let chosen = null;
|
|
1133
|
+
for (const c of candidates) {
|
|
1134
|
+
const score = (ledger.grants[c.repo] ?? 0) / weightOf(budgets, c.repo);
|
|
1135
|
+
if (!chosen || score < chosen.score) chosen = { ...c, score };
|
|
1136
|
+
}
|
|
1137
|
+
const idle = pendingByRepo.size === 0;
|
|
1138
|
+
return { unit: chosen ? chosen.unit : null, estimate: chosen ? chosen.estimate : null, skipped, idle, waiting: !chosen && skipped.some((s) => s.code === 'at-capacity') };
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
/**
|
|
1142
|
+
* Admit and lease the next unit in ONE locked transaction. Returns `{ lease, ... }` or `{ lease: null, idle, waiting, skipped }`.
|
|
1143
|
+
* A repeat `requestId` from the same holder returns the existing lease and reserves nothing more.
|
|
1144
|
+
*/
|
|
1145
|
+
function scheduleNext(file, { budgets, estimateOf, holder, now, ttlMs, requestId = null, blockedRepositories = [] }) {
|
|
1146
|
+
const bv = validateBudgets(budgets);
|
|
1147
|
+
if (!bv.ok) throw typedError('BAD_BUDGETS', `invalid budgets: ${bv.errors[0].path} ${bv.errors[0].message}`, { errors: bv.errors });
|
|
1148
|
+
return mutateStore(file, (store) => {
|
|
1149
|
+
const ledger = readLedger(file);
|
|
1150
|
+
const changed = reconcile(store, ledger, now);
|
|
1151
|
+
const finish = (out) => { writeLedger(file, ledger); return { ...out, reconciled: changed }; };
|
|
1152
|
+
if (requestId) {
|
|
1153
|
+
for (const u of Object.values(store.units)) {
|
|
1154
|
+
if (u.lease && u.lease.requestId === requestId && u.lease.holder === holder) {
|
|
1155
|
+
return finish({ lease: { unitId: u.id, attemptId: u.lease.attemptId, expiresAt: u.lease.expiresAt, repository: u.repository, duplicate: true }, idle: false, waiting: false, skipped: [] });
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
const sel = selectUnit({ store, ledger, budgets, estimateOf, blockedRepositories });
|
|
1160
|
+
if (!sel.unit) return finish({ lease: null, idle: sel.idle, waiting: sel.waiting, skipped: sel.skipped });
|
|
1161
|
+
const l = leaseUnit(store, { holder, now, ttlMs, unitId: sel.unit.id, requestId });
|
|
1162
|
+
ledger.reservations[l.attemptId] = { unitId: sel.unit.id, repository: sel.unit.repository, holder, at: now, ...sel.estimate, used: { spendUsd: 0, requests: 0 } };
|
|
1163
|
+
ledger.grants[sel.unit.repository] = (ledger.grants[sel.unit.repository] ?? 0) + 1;
|
|
1164
|
+
return finish({ lease: { ...l, repository: sel.unit.repository, reservation: sel.estimate }, idle: false, waiting: false, skipped: sel.skipped });
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
/** Settle an attempt's reservation (idempotent). `usage` null charges the whole reservation. */
|
|
1169
|
+
function settleAttempt(file, { attemptId, usage = null, how = 'reported' }) {
|
|
1170
|
+
return mutateStore(file, () => {
|
|
1171
|
+
const ledger = readLedger(file);
|
|
1172
|
+
const r = settle(ledger, attemptId, usage, how);
|
|
1173
|
+
writeLedger(file, ledger);
|
|
1174
|
+
return r;
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
/** Renew the lease and record a heartbeat in one transaction. */
|
|
1179
|
+
function heartbeat(file, { unitId, attemptId, now, ttlMs }) {
|
|
1180
|
+
return mutateStore(file, (store) => {
|
|
1181
|
+
const ledger = readLedger(file);
|
|
1182
|
+
const r = renewLease(store, { unitId, attemptId, now, ttlMs });
|
|
1183
|
+
if (r.ok) {
|
|
1184
|
+
const u = store.units[unitId];
|
|
1185
|
+
ledger.heartbeats[attemptId] = { holder: u.lease.holder, unitId, repository: u.repository, at: now };
|
|
1186
|
+
writeLedger(file, ledger);
|
|
1187
|
+
}
|
|
1188
|
+
return r;
|
|
1189
|
+
});
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
// ---------------------------------------------------------------- routing and spend
|
|
1193
|
+
|
|
1194
|
+
/** What the routing policy may spend for a task in `repository` right now: the smaller of the portfolio and repository remainders, less reservations. */
|
|
1195
|
+
function routingBudgetFor({ budgets, ledger, repository }) {
|
|
1196
|
+
const flight = flightOf(ledger);
|
|
1197
|
+
const left = (limit, used, reserved) => (isNum(limit) ? Math.max(0, limit - used - reserved) : Infinity);
|
|
1198
|
+
const p = left(budgets.portfolio.spendUsd, ledger.usage.portfolio.spendUsd, flight.portfolio.spendUsd);
|
|
1199
|
+
const r = left(limitsFor(budgets, repository).spendUsd, repoUsage(ledger, repository).spendUsd, (flight.repos[repository] ?? zero()).spendUsd);
|
|
1200
|
+
const remainingUsd = Math.min(p, r);
|
|
1201
|
+
return { remainingUsd: Number.isFinite(remainingUsd) ? remainingUsd : 0 };
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
/** The dollar upper bound of a routing decision, or null when the decision selects nothing or carries no bound. Unknown is never free. */
|
|
1205
|
+
function spendBoundOf(decision) {
|
|
1206
|
+
if (!decision || (decision.status !== 'routed' && decision.status !== 'fallback') || !decision.selected) return null;
|
|
1207
|
+
const c = decision.expectedBounds?.costUsd;
|
|
1208
|
+
const bound = Math.max(isNum(c?.upperBound) ? c.upperBound : 0, isNum(c?.measuredMedian) ? c.measuredMedian : 0);
|
|
1209
|
+
return bound > 0 ? bound : null;
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
/** Charge one model request (a routing decision) against the attempt's reservation. Refuses when blocked, unbounded or over the reservation. */
|
|
1213
|
+
function chargeModelSpend(file, { attemptId, decision }) {
|
|
1214
|
+
return mutateStore(file, () => {
|
|
1215
|
+
const ledger = readLedger(file);
|
|
1216
|
+
const r = ledger.reservations[attemptId];
|
|
1217
|
+
if (!r) return { ok: false, code: 'no-reservation', reason: 'this attempt holds no reservation (it ended, or was never admitted)' };
|
|
1218
|
+
const bound = spendBoundOf(decision);
|
|
1219
|
+
if (bound === null) return { ok: false, code: 'no-spend-bound', reason: 'the routing decision selects nothing or carries no cost bound; an unknown cost is not charged as zero and the request is not made' };
|
|
1220
|
+
if (r.used.requests + 1 > r.requests) return { ok: false, code: 'request-limit', reason: `the unit reserved ${r.requests} request(s)` };
|
|
1221
|
+
if (r.used.spendUsd + bound > r.spendUsd) return { ok: false, code: 'unit-spend-exceeded', reason: `needs up to ${bound} against ${r.spendUsd - r.used.spendUsd} remaining in the unit's reservation` };
|
|
1222
|
+
r.used.spendUsd += bound; r.used.requests += 1;
|
|
1223
|
+
writeLedger(file, ledger);
|
|
1224
|
+
return { ok: true, charged: bound };
|
|
1225
|
+
});
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
// ---------------------------------------------------------------- cancellation
|
|
1229
|
+
|
|
1230
|
+
/** Tracks the attempts running in this process so a scope can be cancelled. */
|
|
1231
|
+
function createCancelScope() {
|
|
1232
|
+
const running = new Map();
|
|
1233
|
+
return {
|
|
1234
|
+
running,
|
|
1235
|
+
register(attemptId, entry) { running.set(attemptId, entry); },
|
|
1236
|
+
unregister(attemptId) { running.delete(attemptId); },
|
|
1237
|
+
/**
|
|
1238
|
+
* Cancel everything in `scope` ({ unitId } | { repository } | { all: true }).
|
|
1239
|
+
* @returns {Promise<object>} an `incomplete` report with the reason, what was released and what is left to expire
|
|
1240
|
+
*/
|
|
1241
|
+
async cancel({ file, scope, reason, now, waitMs = 5000 }) {
|
|
1242
|
+
if (!scope || !(scope.all === true || scope.unitId || scope.repository)) throw typedError('BAD_SCOPE', 'a cancellation needs a scope: unitId, repository or all');
|
|
1243
|
+
const why = String(reason ?? 'canceled').slice(0, 300);
|
|
1244
|
+
mutateStore(file, (store) => {
|
|
1245
|
+
const ledger = readLedger(file);
|
|
1246
|
+
ledger.cancellations.push({ scope: { ...scope }, reason: why, at: now });
|
|
1247
|
+
reconcile(store, ledger, now);
|
|
1248
|
+
writeLedger(file, ledger);
|
|
1249
|
+
});
|
|
1250
|
+
const mine = [...running.entries()].filter(([, e]) => inScope({ id: e.unitId, repository: e.repository }, scope));
|
|
1251
|
+
for (const [, e] of mine) e.controller.abort(Object.assign(new Error(why), { code: 'CANCELED' }));
|
|
1252
|
+
await Promise.race([Promise.all(mine.map(([, e]) => e.settled)), new Promise((r) => setTimeout(r, waitMs))]);
|
|
1253
|
+
return mutateStore(file, (store) => {
|
|
1254
|
+
const ledger = readLedger(file);
|
|
1255
|
+
const report = { status: 'nothing-to-cancel', incomplete: false, reason: why, scope: { ...scope }, canceled: [], leasesReleased: [], leasesLeftToExpire: [], verifiedPreserved: [], survivors: [] };
|
|
1256
|
+
for (const id of Object.keys(store.units).sort()) {
|
|
1257
|
+
const u = store.units[id];
|
|
1258
|
+
if (!inScope(u, scope)) continue;
|
|
1259
|
+
if (u.state === 'verified') { report.verifiedPreserved.push(id); continue; }
|
|
1260
|
+
if (u.state === 'leased' || u.state === 'running') {
|
|
1261
|
+
const attemptId = u.lease.attemptId;
|
|
1262
|
+
const e = running.get(attemptId) ?? mine.find(([a]) => a === attemptId)?.[1];
|
|
1263
|
+
if (e && e.finished === true && e.survivors.length === 0) {
|
|
1264
|
+
cancelUnits(store, { unitId: id, reason: why, now });
|
|
1265
|
+
settle(ledger, attemptId, null, 'canceled');
|
|
1266
|
+
report.leasesReleased.push(id); report.canceled.push(id);
|
|
1267
|
+
} else {
|
|
1268
|
+
report.leasesLeftToExpire.push({ unitId: id, attemptId, expiresAt: u.lease.expiresAt, why: e ? (e.survivors.length ? 'a descendant survived termination' : 'the attempt did not settle in time') : 'the attempt runs in another process' });
|
|
1269
|
+
if (e?.survivors.length) report.survivors.push({ unitId: id, pids: e.survivors });
|
|
1270
|
+
}
|
|
1271
|
+
} else if (u.state === 'canceled') report.canceled.push(id);
|
|
1272
|
+
}
|
|
1273
|
+
if (report.canceled.length || report.leasesLeftToExpire.length) { report.status = 'incomplete'; report.incomplete = true; }
|
|
1274
|
+
writeLedger(file, ledger);
|
|
1275
|
+
return report;
|
|
1276
|
+
});
|
|
1277
|
+
},
|
|
1278
|
+
};
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
// ---------------------------------------------------------------- the driver
|
|
1282
|
+
|
|
1283
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
1284
|
+
|
|
1285
|
+
/**
|
|
1286
|
+
* Drive a store with budgets. `executor(unit, ctx)` returns `{ resultDigest, dependencies, usage? }` where `usage` may report
|
|
1287
|
+
* `{ spendUsd, requests, storageBytes }` (wall time is measured). `ctx` carries `attemptId`, `signal`, `heartbeat()`, `reservation`,
|
|
1288
|
+
* `spawn(bin, args, opts)` (supervised: the process tree ends with the attempt) and `requestModel(decision)` (charges the routing decision).
|
|
1289
|
+
*
|
|
1290
|
+
* With `backend`, the backend is probed before every lease; an unavailable one stops the run with `stopped: 'blocked'` and the typed probe
|
|
1291
|
+
* result in `blocked`. @returns {Promise<{ progress: object|null, stopped: string, order: string[], skipped: Array, blocked?: object }>}
|
|
1292
|
+
*/
|
|
1293
|
+
async function runScheduled({
|
|
1294
|
+
file, budgets, estimateOf, executor, holder = 'worker', clock = () => Date.now(), ttlMs, maxLeases = Infinity, cancelScope = null,
|
|
1295
|
+
blockedRepositories = [], heartbeatMs = HEARTBEAT.intervalMs, pollMs = 25, maxWaitMs = 2000, workers = null, backend = null,
|
|
1296
|
+
}) {
|
|
1297
|
+
const bv = validateBudgets(budgets);
|
|
1298
|
+
if (!bv.ok) throw typedError('BAD_BUDGETS', `invalid budgets: ${bv.errors[0].path} ${bv.errors[0].message}`, { errors: bv.errors });
|
|
1299
|
+
const order = [];
|
|
1300
|
+
let leased = 0; let stopped = 'drained'; let lastSkipped = []; let blocked = null;
|
|
1301
|
+
const limit = Math.max(1, Math.min(workers ?? budgets.portfolio.concurrency, 16));
|
|
1302
|
+
let inFlight = 0;
|
|
1303
|
+
|
|
1304
|
+
async function worker(n) {
|
|
1305
|
+
let waited = 0;
|
|
1306
|
+
for (;;) {
|
|
1307
|
+
if (leased >= maxLeases) { stopped = 'max-leases'; return; }
|
|
1308
|
+
// a shared backend that went away stops the run: nothing is written anywhere else instead (X-708.AC03)
|
|
1309
|
+
if (backend) { const p = backend.probe(); if (!p.ok) { stopped = 'blocked'; blocked = p; return; } }
|
|
1310
|
+
let r;
|
|
1311
|
+
try { r = scheduleNext(file, { budgets, estimateOf, holder: `${holder}-${n}`, now: clock(), ttlMs, blockedRepositories }); } catch (e) {
|
|
1312
|
+
if (backend && !backend.probe().ok) { stopped = 'blocked'; blocked = backend.probe(); return; }
|
|
1313
|
+
throw e;
|
|
1314
|
+
}
|
|
1315
|
+
lastSkipped = r.skipped;
|
|
1316
|
+
if (!r.lease) {
|
|
1317
|
+
if (r.idle) return;
|
|
1318
|
+
if (r.waiting) {
|
|
1319
|
+
if (inFlight === 0 && waited >= maxWaitMs) { stopped = 'waiting-timeout'; return; }
|
|
1320
|
+
await sleep(pollMs); waited += pollMs; continue;
|
|
1321
|
+
}
|
|
1322
|
+
stopped = 'nothing-schedulable'; return;
|
|
1323
|
+
}
|
|
1324
|
+
waited = 0;
|
|
1325
|
+
leased += 1; inFlight += 1;
|
|
1326
|
+
order.push(r.lease.unitId);
|
|
1327
|
+
try { await runOne(r.lease); } catch (e) {
|
|
1328
|
+
// the attempt could not record its outcome because the backend is gone: its lease is left to expire, never re-homed
|
|
1329
|
+
if (backend && !backend.probe().ok) { stopped = 'blocked'; blocked = backend.probe(); return; }
|
|
1330
|
+
throw e;
|
|
1331
|
+
} finally { inFlight -= 1; }
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
async function runOne(lease) {
|
|
1336
|
+
const { unitId, attemptId } = lease;
|
|
1337
|
+
const controller = new AbortController();
|
|
1338
|
+
const entry = { unitId, repository: lease.repository, controller, finished: false, survivors: [], terminations: [], settled: null };
|
|
1339
|
+
let resolveSettled; entry.settled = new Promise((r) => { resolveSettled = r; });
|
|
1340
|
+
cancelScope?.register(attemptId, entry);
|
|
1341
|
+
const startedAt = clock();
|
|
1342
|
+
let timer = null; let beat = null;
|
|
1343
|
+
try {
|
|
1344
|
+
mutateStore(file, (s) => startUnit(s, { unitId, attemptId, now: clock() }));
|
|
1345
|
+
const unit = readStore(file).units[unitId];
|
|
1346
|
+
const wallCap = lease.reservation.wallMs;
|
|
1347
|
+
timer = setTimeout(() => controller.abort(Object.assign(new Error('wall-time budget for the unit was reached'), { code: 'WALL_TIMEOUT' })), wallCap);
|
|
1348
|
+
const doBeat = () => { try { heartbeat(file, { unitId, attemptId, now: clock(), ttlMs }); } catch { /* the lease is gone; the result will be refused */ } };
|
|
1349
|
+
beat = setInterval(doBeat, heartbeatMs); doBeat();
|
|
1350
|
+
const ctx = {
|
|
1351
|
+
attemptId, signal: controller.signal, reservation: lease.reservation, heartbeat: doBeat,
|
|
1352
|
+
async spawn(bin, args, opts = {}) {
|
|
1353
|
+
const r = await superviseSpawn(bin, args, { ...opts, signal: controller.signal });
|
|
1354
|
+
entry.terminations.push(r.termination);
|
|
1355
|
+
if (r.termination?.survivors?.length) entry.survivors.push(...r.termination.survivors);
|
|
1356
|
+
return r;
|
|
1357
|
+
},
|
|
1358
|
+
requestModel: (decision) => chargeModelSpend(file, { attemptId, decision }),
|
|
1359
|
+
};
|
|
1360
|
+
let res = null; let err = null;
|
|
1361
|
+
try { res = await executor(unit, ctx); } catch (e) { err = e; }
|
|
1362
|
+
clearTimeout(timer); clearInterval(beat);
|
|
1363
|
+
const canceled = controller.signal.aborted && controller.signal.reason?.code === 'CANCELED';
|
|
1364
|
+
if (canceled) return; // cancel() performs the store transition once this attempt has settled
|
|
1365
|
+
const timedOut = controller.signal.aborted && controller.signal.reason?.code === 'WALL_TIMEOUT';
|
|
1366
|
+
const spent = mutateStore(file, (s) => {
|
|
1367
|
+
const ledger = readLedger(file);
|
|
1368
|
+
const tracked = ledger.reservations[attemptId]?.used ?? { spendUsd: 0, requests: 0 };
|
|
1369
|
+
const failed = err || !res || timedOut;
|
|
1370
|
+
// a failed attempt spent an unknown amount: it is charged its reservation; a reported success is charged what it reported
|
|
1371
|
+
const usage = failed ? null : { wallMs: Math.max(1, clock() - startedAt), spendUsd: tracked.spendUsd, requests: tracked.requests, ...(res.usage ?? {}) };
|
|
1372
|
+
settle(ledger, attemptId, usage, failed ? 'failed' : 'reported');
|
|
1373
|
+
writeLedger(file, ledger);
|
|
1374
|
+
if (failed) failUnit(s, { unitId, attemptId, reason: timedOut ? 'wall-time budget reached' : err ? err.message : 'executor returned nothing', now: clock() });
|
|
1375
|
+
else completeUnit(s, { unitId, attemptId, resultDigest: res.resultDigest, dependencies: res.dependencies, now: clock() });
|
|
1376
|
+
return true;
|
|
1377
|
+
});
|
|
1378
|
+
void spent;
|
|
1379
|
+
} finally {
|
|
1380
|
+
clearTimeout(timer); clearInterval(beat);
|
|
1381
|
+
entry.finished = true;
|
|
1382
|
+
resolveSettled();
|
|
1383
|
+
cancelScope?.unregister(attemptId);
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
await Promise.all(Array.from({ length: limit }, (_, i) => worker(i)));
|
|
1388
|
+
let progress = null;
|
|
1389
|
+
try { progress = progressOf(readStore(file)); } catch (e) { if (!blocked) throw e; }
|
|
1390
|
+
return { progress, stopped, order, skipped: lastSkipped, ...(blocked ? { blocked } : {}) };
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
/***/ }),
|
|
1396
|
+
|
|
1397
|
+
/***/ 42284:
|
|
1398
|
+
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
|
|
1399
|
+
|
|
1400
|
+
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
|
|
1401
|
+
/* harmony export */ H9: () => (/* binding */ verifyStore),
|
|
1402
|
+
/* harmony export */ Yc: () => (/* binding */ UNIT_STATES),
|
|
1403
|
+
/* harmony export */ Zz: () => (/* binding */ typedError),
|
|
1404
|
+
/* harmony export */ uz: () => (/* binding */ readStore)
|
|
1405
|
+
/* harmony export */ });
|
|
1406
|
+
/* unused harmony exports DEPENDENCY_DIMENSIONS, PORTFOLIO_LIMITS, planPortfolio, appendEvent, newStore, recoverExpired, leaseUnit, startUnit, renewLease, checkDependencies, completeUnit, failUnit, blockUnit, unblockUnit, cancelUnits, progressOf, scopedResults, fullyVerifiedRepositories, openStore, mutateStore, runPortfolio */
|
|
1407
|
+
/* harmony import */ var node_crypto__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(77598);
|
|
1408
|
+
/* harmony import */ var node_fs__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(73024);
|
|
1409
|
+
/* harmony import */ var node_path__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(76760);
|
|
1410
|
+
/* harmony import */ var _evidence_bundle_js__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(32876);
|
|
1411
|
+
/* harmony import */ var _assurance_identity_js__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(91026);
|
|
1412
|
+
/* harmony import */ var _assurance_schema_kit_js__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(99666);
|
|
1413
|
+
// Durable, leased portfolio work units (X-704).
|
|
1414
|
+
//
|
|
1415
|
+
// fleet.js already gives a many-repository run isolation, bounded concurrency and a per-repository completed marker. What it cannot do
|
|
1416
|
+
// is say WHICH unit of work was in flight when a process died, hand that unit to another worker without doing it twice, or refuse to
|
|
1417
|
+
// count a half-finished attempt as progress. This module adds that, in the same local-first shape: one JSON file, written atomically,
|
|
1418
|
+
// no server, no network.
|
|
1419
|
+
//
|
|
1420
|
+
// PLAN `planPortfolio` decomposes the AUTHORIZED repositories into units, one per (repository, exact commit, task type). A unit id
|
|
1421
|
+
// is a hash of those three and its required inputs, so the same plan always yields the same ids (X-704.AC01); a repository
|
|
1422
|
+
// that is not on the authorization list is excluded and disclosed, never silently planned.
|
|
1423
|
+
// STATES pending, leased, running, verified, blocked, failed, canceled. A lease has an expiry; an expired lease is recovered into
|
|
1424
|
+
// pending (or failed once retries are spent). Every transition appends an event to the unit's attempt chain; the chain is
|
|
1425
|
+
// hash-linked, so an attempt record can be read and verified but not rewritten (X-704.AC02).
|
|
1426
|
+
// PROGRESS only a unit in `verified` with a recorded result counts. A leased or running unit, a failed attempt, an expired lease and a
|
|
1427
|
+
// duplicate delivery of a completion all leave the count where it was (X-704.AC03).
|
|
1428
|
+
//
|
|
1429
|
+
// Operations are idempotent by attempt id: delivering the same `complete` twice records one verification; delivering a completion for a
|
|
1430
|
+
// stale attempt (the lease expired and the unit moved on) is refused, because a zombie worker's result must not displace its successor's.
|
|
1431
|
+
//
|
|
1432
|
+
// The clock is an argument everywhere (`now`, milliseconds), so recovery is testable without sleeping and nothing here reads a clock.
|
|
1433
|
+
// Persistence is `mutateStore`: an exclusive lock file, read, verify, change, write to a temporary file, rename. A store that fails
|
|
1434
|
+
// verification is an error, never silently reset to empty.
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
|
|
1440
|
+
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
const STORE_SCHEMA = 'agentic-security/portfolio-store';
|
|
1444
|
+
const PLAN_SCHEMA = 'agentic-security/portfolio-plan';
|
|
1445
|
+
const STORE_VERSION = 1;
|
|
1446
|
+
const UNIT_STATES = Object.freeze(['pending', 'leased', 'running', 'verified', 'blocked', 'failed', 'canceled']);
|
|
1447
|
+
const TASK_TYPES = Object.freeze(['sast-scan', 'sca-reachability', 'boundary-graph', 'invariant-scenarios', 'verification-replay', 'attestation']);
|
|
1448
|
+
const DEPENDENCY_DIMENSIONS = Object.freeze(['code', 'policy', 'graph', 'invariant', 'oracle', 'toolchain']);
|
|
1449
|
+
|
|
1450
|
+
const PORTFOLIO_LIMITS = Object.freeze({
|
|
1451
|
+
maxUnits: 5000, maxEventsPerUnit: 400, maxStoreBytes: 32 * 1024 * 1024, defaultLeaseMs: 5 * 60_000, maxLeaseMs: 6 * 60 * 60_000, defaultMaxRetries: 2, maxRetries: 10,
|
|
1452
|
+
lockStaleMs: 30_000, lockWaitMs: 2000,
|
|
1453
|
+
});
|
|
1454
|
+
|
|
1455
|
+
const DIGEST_RE = /^sha256:[0-9a-f]{64}$/;
|
|
1456
|
+
|
|
1457
|
+
// ---------------------------------------------------------------- plan
|
|
1458
|
+
|
|
1459
|
+
const unitIdOf = (u) => `wu:${node_crypto__WEBPACK_IMPORTED_MODULE_0__.createHash('sha256').update((0,_evidence_bundle_js__WEBPACK_IMPORTED_MODULE_3__/* .canonicalJson */ .dj)({ repository: u.repository, commit: u.commit, taskType: u.taskType, requiredInputs: u.requiredInputs })).digest('hex').slice(0, 16)}`;
|
|
1460
|
+
|
|
1461
|
+
/**
|
|
1462
|
+
* Decompose authorized repositories into stable work units.
|
|
1463
|
+
*
|
|
1464
|
+
* @param {object} a
|
|
1465
|
+
* @param {Array<{name:string, commit:string}>} a.repositories candidates, each pinned to an exact commit
|
|
1466
|
+
* @param {string[]} a.authorized the repository names the operator has authorized; anything else is excluded and disclosed
|
|
1467
|
+
* @param {string[]} a.taskTypes task types to run per repository (from TASK_TYPES)
|
|
1468
|
+
* @param {Object<string,string[]>} [a.requiredInputs] per task type, the named inputs the unit needs (e.g. ['source','policy'])
|
|
1469
|
+
*/
|
|
1470
|
+
function planPortfolio({ repositories = [], authorized = [], taskTypes = [], requiredInputs = {}, synthetic = false } = {}) {
|
|
1471
|
+
const errors = [];
|
|
1472
|
+
const rejected = [];
|
|
1473
|
+
const allow = new Set(authorized);
|
|
1474
|
+
for (const t of taskTypes) if (!TASK_TYPES.includes(t)) errors.push({ code: 'UNKNOWN_TASK_TYPE', message: `'${t}' is not a task type` });
|
|
1475
|
+
if (taskTypes.length === 0) errors.push({ code: 'NO_TASKS', message: 'a plan needs at least one task type' });
|
|
1476
|
+
const seen = new Set();
|
|
1477
|
+
const units = [];
|
|
1478
|
+
for (const r of repositories) {
|
|
1479
|
+
if (!r || typeof r.name !== 'string' || !r.name) { errors.push({ code: 'BAD_REPOSITORY', message: 'repository needs a name' }); continue; }
|
|
1480
|
+
if (!allow.has(r.name)) { rejected.push({ repository: r.name, reason: 'not-authorized' }); continue; }
|
|
1481
|
+
if (!isCommit(r.commit)) { errors.push({ code: 'UNBOUND_REVISION', message: `repository '${r.name}' is not pinned to an exact commit` }); continue; }
|
|
1482
|
+
if (seen.has(r.name)) { errors.push({ code: 'DUPLICATE_REPOSITORY', message: `repository '${r.name}' is listed twice` }); continue; }
|
|
1483
|
+
seen.add(r.name);
|
|
1484
|
+
for (const taskType of taskTypes) {
|
|
1485
|
+
const inputs = [...new Set(requiredInputs[taskType] ?? [])].sort();
|
|
1486
|
+
const u = { repository: r.name, commit: r.commit, taskType, requiredInputs: inputs };
|
|
1487
|
+
units.push({ id: unitIdOf(u), ...u });
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
if (units.length > PORTFOLIO_LIMITS.maxUnits) errors.push({ code: 'TOO_MANY_UNITS', message: `plan has ${units.length} units; the limit is ${PORTFOLIO_LIMITS.maxUnits}` });
|
|
1491
|
+
units.sort((a, b) => (a.id < b.id ? -1 : 1));
|
|
1492
|
+
const body = { schema: PLAN_SCHEMA, schemaVersion: '1.0.0', ...(synthetic ? { synthetic: true } : {}), units, rejected: rejected.sort((a, b) => (a.repository < b.repository ? -1 : 1)) };
|
|
1493
|
+
return { ok: errors.length === 0, errors, plan: errors.length === 0 ? { ...body, id: `pplan:${digestOf(units).slice(7, 23)}` } : null };
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
// ---------------------------------------------------------------- state
|
|
1497
|
+
|
|
1498
|
+
const eventDigest = (e) => (0,_assurance_identity_js__WEBPACK_IMPORTED_MODULE_4__/* .digestOf */ .ol)(e);
|
|
1499
|
+
|
|
1500
|
+
function appendEvent(unit, ev) {
|
|
1501
|
+
if (unit.events.length >= PORTFOLIO_LIMITS.maxEventsPerUnit) throw typedError('EVENT_LIMIT', `unit ${unit.id} reached ${PORTFOLIO_LIMITS.maxEventsPerUnit} recorded events`);
|
|
1502
|
+
const prev = unit.events.length ? unit.events[unit.events.length - 1].digest : null;
|
|
1503
|
+
const body = { seq: unit.events.length + 1, prev, ...ev };
|
|
1504
|
+
unit.events.push({ ...body, digest: eventDigest(body) });
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
function typedError(code, message, extra = {}) { return Object.assign(new Error(message), { code, ...extra }); }
|
|
1508
|
+
|
|
1509
|
+
/** A new, empty store for a plan. */
|
|
1510
|
+
function newStore(plan) {
|
|
1511
|
+
if (!plan || plan.schema !== PLAN_SCHEMA) throw typedError('BAD_PLAN', 'not a portfolio plan');
|
|
1512
|
+
const units = {};
|
|
1513
|
+
for (const u of plan.units) {
|
|
1514
|
+
units[u.id] = { ...u, state: 'pending', retryCount: 0, maxRetries: PORTFOLIO_LIMITS.defaultMaxRetries, generation: 0, attemptSeq: 0, lease: null, result: null, stale: [], blockedReason: null, events: [] };
|
|
1515
|
+
}
|
|
1516
|
+
return { schema: STORE_SCHEMA, version: STORE_VERSION, planId: plan.id, ...(plan.synthetic ? { synthetic: true } : {}), units };
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
/** Verify the store's internal consistency: event chains, state/lease/result agreement. Returns `{ ok, errors }`. */
|
|
1520
|
+
function verifyStore(store) {
|
|
1521
|
+
const errors = [];
|
|
1522
|
+
if (!store || store.schema !== STORE_SCHEMA || store.version !== STORE_VERSION || typeof store.units !== 'object' || !store.units) return { ok: false, errors: [{ code: 'BAD_STORE', message: 'not a portfolio store of a supported version' }] };
|
|
1523
|
+
for (const [id, u] of Object.entries(store.units)) {
|
|
1524
|
+
if (u.id !== id || unitIdOf(u) !== id) errors.push({ code: 'ID_MISMATCH', unit: id, message: 'unit identity does not match its repository, commit, task type and inputs' });
|
|
1525
|
+
if (!UNIT_STATES.includes(u.state)) errors.push({ code: 'BAD_STATE', unit: id, message: `unknown state '${u.state}'` });
|
|
1526
|
+
let prev = null;
|
|
1527
|
+
(u.events ?? []).forEach((e, i) => {
|
|
1528
|
+
const { digest, ...body } = e;
|
|
1529
|
+
if (e.seq !== i + 1 || e.prev !== prev || eventDigest(body) !== digest) errors.push({ code: 'EVENT_CHAIN_BROKEN', unit: id, message: `event ${i + 1} was altered or removed` });
|
|
1530
|
+
prev = digest;
|
|
1531
|
+
});
|
|
1532
|
+
if ((u.state === 'leased' || u.state === 'running') !== !!u.lease) errors.push({ code: 'LEASE_STATE_MISMATCH', unit: id, message: 'a lease exists exactly when the unit is leased or running' });
|
|
1533
|
+
if (u.state === 'verified' && !(u.result && DIGEST_RE.test(u.result.resultDigest ?? ''))) errors.push({ code: 'VERIFIED_WITHOUT_RESULT', unit: id, message: 'a verified unit must carry a result digest' });
|
|
1534
|
+
if (u.state !== 'verified' && u.result) errors.push({ code: 'RESULT_WITHOUT_VERIFIED', unit: id, message: 'only a verified unit may carry a current result' });
|
|
1535
|
+
}
|
|
1536
|
+
return { ok: errors.length === 0, errors };
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
// ---------------------------------------------------------------- transitions (pure on a store object)
|
|
1540
|
+
|
|
1541
|
+
const unitOf = (store, id) => {
|
|
1542
|
+
const u = store.units[id];
|
|
1543
|
+
if (!u) throw typedError('UNKNOWN_UNIT', `no such unit '${id}'`);
|
|
1544
|
+
return u;
|
|
1545
|
+
};
|
|
1546
|
+
|
|
1547
|
+
function clampLease(ttlMs) {
|
|
1548
|
+
const t = ttlMs ?? PORTFOLIO_LIMITS.defaultLeaseMs;
|
|
1549
|
+
if (!Number.isFinite(t) || t < 1) throw typedError('BAD_LEASE', 'lease duration must be a positive number of milliseconds');
|
|
1550
|
+
return Math.min(t, PORTFOLIO_LIMITS.maxLeaseMs);
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
function spendAttempt(u, ev, now) {
|
|
1554
|
+
// an attempt ended without a verified result: it costs a retry. Out of retries is a terminal failure.
|
|
1555
|
+
u.lease = null;
|
|
1556
|
+
u.retryCount += 1;
|
|
1557
|
+
appendEvent(u, { ...ev, at: now });
|
|
1558
|
+
u.state = u.retryCount > u.maxRetries ? 'failed' : 'pending';
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
/** Recover every unit whose lease has expired. Returns the ids recovered. */
|
|
1562
|
+
function recoverExpired(store, now) {
|
|
1563
|
+
const out = [];
|
|
1564
|
+
for (const id of Object.keys(store.units).sort()) {
|
|
1565
|
+
const u = store.units[id];
|
|
1566
|
+
if ((u.state === 'leased' || u.state === 'running') && u.lease && u.lease.expiresAt <= now) {
|
|
1567
|
+
const attemptId = u.lease.attemptId;
|
|
1568
|
+
spendAttempt(u, { type: 'expired', attemptId, holder: u.lease.holder }, now);
|
|
1569
|
+
out.push(id);
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
return out;
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
/**
|
|
1576
|
+
* Lease the next pending unit (or a named one) to `holder`. A repeat of the same `requestId` from the same holder returns the same lease
|
|
1577
|
+
* instead of leasing a second unit (duplicate delivery of the request). Returns `null` when nothing is leasable.
|
|
1578
|
+
*/
|
|
1579
|
+
function leaseUnit(store, { holder, now, ttlMs, unitId = null, requestId = null, skip = () => false }) {
|
|
1580
|
+
if (typeof holder !== 'string' || !holder) throw typedError('BAD_HOLDER', 'a lease needs a holder');
|
|
1581
|
+
recoverExpired(store, now);
|
|
1582
|
+
if (requestId) {
|
|
1583
|
+
for (const u of Object.values(store.units)) {
|
|
1584
|
+
if (u.lease && u.lease.requestId === requestId && u.lease.holder === holder) return { unitId: u.id, attemptId: u.lease.attemptId, expiresAt: u.lease.expiresAt, duplicate: true };
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
const candidates = unitId ? [unitOf(store, unitId)] : Object.values(store.units).sort((a, b) => (a.id < b.id ? -1 : 1));
|
|
1588
|
+
const u = candidates.find((c) => c.state === 'pending' && !skip(c));
|
|
1589
|
+
if (!u) return null;
|
|
1590
|
+
u.attemptSeq += 1;
|
|
1591
|
+
const attemptId = `${u.id}#${u.generation}.${u.attemptSeq}`;
|
|
1592
|
+
const expiresAt = now + clampLease(ttlMs);
|
|
1593
|
+
u.lease = { holder, attemptId, expiresAt, requestId };
|
|
1594
|
+
u.state = 'leased';
|
|
1595
|
+
appendEvent(u, { type: 'leased', attemptId, holder, at: now, expiresAt });
|
|
1596
|
+
return { unitId: u.id, attemptId, expiresAt, duplicate: false };
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
function liveAttempt(store, unitId, attemptId, now) {
|
|
1600
|
+
recoverExpired(store, now);
|
|
1601
|
+
const u = unitOf(store, unitId);
|
|
1602
|
+
if (!u.lease || u.lease.attemptId !== attemptId) return { u, ok: false };
|
|
1603
|
+
return { u, ok: true };
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
/** leased -> running. Idempotent for the same attempt. */
|
|
1607
|
+
function startUnit(store, { unitId, attemptId, now }) {
|
|
1608
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
1609
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'this attempt does not hold the lease (it expired, or the unit moved on)' };
|
|
1610
|
+
if (u.state === 'running') return { ok: true, duplicate: true };
|
|
1611
|
+
u.state = 'running';
|
|
1612
|
+
appendEvent(u, { type: 'started', attemptId, at: now });
|
|
1613
|
+
return { ok: true, duplicate: false };
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
/** Extend a live lease. */
|
|
1617
|
+
function renewLease(store, { unitId, attemptId, now, ttlMs }) {
|
|
1618
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
1619
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'the lease is no longer held' };
|
|
1620
|
+
u.lease.expiresAt = now + clampLease(ttlMs);
|
|
1621
|
+
appendEvent(u, { type: 'renewed', attemptId, at: now, expiresAt: u.lease.expiresAt });
|
|
1622
|
+
return { ok: true, expiresAt: u.lease.expiresAt };
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
/** A recorded result's dependency digests, one per dimension. All six are required: a result without them cannot be reused. */
|
|
1626
|
+
function checkDependencies(deps) {
|
|
1627
|
+
const missing = DEPENDENCY_DIMENSIONS.filter((d) => !DIGEST_RE.test(deps?.[d] ?? ''));
|
|
1628
|
+
return missing;
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
/**
|
|
1632
|
+
* Record a verified result. Accepted only from the live attempt, with all six dependency digests. A second delivery of the same
|
|
1633
|
+
* completion is acknowledged and changes nothing; a completion from a stale attempt is refused.
|
|
1634
|
+
*/
|
|
1635
|
+
function completeUnit(store, { unitId, attemptId, resultDigest, dependencies, now }) {
|
|
1636
|
+
const u0 = unitOf(store, unitId);
|
|
1637
|
+
if (u0.state === 'verified' && u0.result?.attemptId === attemptId) return { ok: true, duplicate: true, counted: false };
|
|
1638
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
1639
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'this attempt does not hold the lease (it expired, or the unit moved on); the result is discarded' };
|
|
1640
|
+
if (!DIGEST_RE.test(resultDigest ?? '')) return { ok: false, code: 'bad-result', reason: 'a result needs a sha256 digest' };
|
|
1641
|
+
const missing = checkDependencies(dependencies);
|
|
1642
|
+
if (missing.length) return { ok: false, code: 'unbound-dependencies', reason: `result is missing dependency digests: ${missing.join(', ')}` };
|
|
1643
|
+
const deps = Object.fromEntries(DEPENDENCY_DIMENSIONS.map((d) => [d, dependencies[d]]));
|
|
1644
|
+
u.lease = null;
|
|
1645
|
+
u.result = { attemptId, resultDigest, dependencies: deps, dependencyDigest: digestOf(deps), generation: u.generation };
|
|
1646
|
+
u.state = 'verified';
|
|
1647
|
+
appendEvent(u, { type: 'verified', attemptId, at: now, resultDigest, dependencyDigest: u.result.dependencyDigest });
|
|
1648
|
+
return { ok: true, duplicate: false, counted: true };
|
|
1649
|
+
}
|
|
1650
|
+
|
|
1651
|
+
/** The live attempt ended in failure. Costs a retry; out of retries the unit is terminally `failed`. */
|
|
1652
|
+
function failUnit(store, { unitId, attemptId, reason, now }) {
|
|
1653
|
+
const u0 = unitOf(store, unitId);
|
|
1654
|
+
const dup = u0.events.some((e) => e.type === 'failed' && e.attemptId === attemptId);
|
|
1655
|
+
if (dup) return { ok: true, duplicate: true };
|
|
1656
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
1657
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'this attempt does not hold the lease' };
|
|
1658
|
+
spendAttempt(u, { type: 'failed', attemptId, holder: u.lease?.holder ?? null, reason: String(reason ?? 'failed').slice(0, 300) }, now);
|
|
1659
|
+
return { ok: true, duplicate: false, state: u.state };
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
/** Park a unit that cannot proceed (a missing input). Not counted; independent units still run. */
|
|
1663
|
+
function blockUnit(store, { unitId, reason, now }) {
|
|
1664
|
+
const u = unitOf(store, unitId);
|
|
1665
|
+
if (!['pending', 'leased', 'running'].includes(u.state)) return { ok: false, code: 'bad-state', reason: `cannot block a ${u.state} unit` };
|
|
1666
|
+
u.lease = null; u.state = 'blocked'; u.blockedReason = String(reason ?? 'blocked').slice(0, 300);
|
|
1667
|
+
appendEvent(u, { type: 'blocked', reason: u.blockedReason, at: now });
|
|
1668
|
+
return { ok: true };
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
function unblockUnit(store, { unitId, now }) {
|
|
1672
|
+
const u = unitOf(store, unitId);
|
|
1673
|
+
if (u.state !== 'blocked') return { ok: false, code: 'bad-state', reason: `unit is ${u.state}, not blocked` };
|
|
1674
|
+
u.state = 'pending'; u.blockedReason = null;
|
|
1675
|
+
appendEvent(u, { type: 'unblocked', at: now });
|
|
1676
|
+
return { ok: true };
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
/** Cancel every unit that is not verified or terminally failed (or one named unit). Leases are released; verified receipts and failure records are kept. */
|
|
1680
|
+
function cancelUnits(store, { unitId = null, reason, now }) {
|
|
1681
|
+
const canceled = [];
|
|
1682
|
+
for (const id of Object.keys(store.units).sort()) {
|
|
1683
|
+
if (unitId && id !== unitId) continue;
|
|
1684
|
+
const u = store.units[id];
|
|
1685
|
+
if (u.state === 'verified' || u.state === 'canceled' || u.state === 'failed') continue; // receipts and terminal failures keep their own status
|
|
1686
|
+
u.lease = null; u.state = 'canceled';
|
|
1687
|
+
appendEvent(u, { type: 'canceled', reason: String(reason ?? 'canceled').slice(0, 300), at: now });
|
|
1688
|
+
canceled.push(id);
|
|
1689
|
+
}
|
|
1690
|
+
return { ok: true, canceled };
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
// ---------------------------------------------------------------- progress
|
|
1694
|
+
|
|
1695
|
+
/** Counts by state. `verified` is the only count that is progress. */
|
|
1696
|
+
function progressOf(store) {
|
|
1697
|
+
const by = Object.fromEntries(UNIT_STATES.map((s) => [s, 0]));
|
|
1698
|
+
let staleResults = 0;
|
|
1699
|
+
for (const u of Object.values(store.units)) { by[u.state] += 1; staleResults += u.stale.length; }
|
|
1700
|
+
const total = Object.keys(store.units).length;
|
|
1701
|
+
return { total, ...by, completed: by.verified, remaining: total - by.verified - by.canceled, staleResults, done: total > 0 && by.verified === total };
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
/** The current scoped results: verified units only, keyed by unit id. Two runs converged exactly when these are equal. */
|
|
1705
|
+
function scopedResults(store) {
|
|
1706
|
+
return Object.fromEntries(Object.values(store.units).filter((u) => u.state === 'verified').sort((a, b) => (a.id < b.id ? -1 : 1)).map((u) => [u.id, u.result.resultDigest]));
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
/** Repositories whose every unit is verified and current: what `runFleet({ portfolioVerified })` may skip. */
|
|
1710
|
+
function fullyVerifiedRepositories(store) {
|
|
1711
|
+
const by = new Map();
|
|
1712
|
+
for (const u of Object.values(store.units)) by.set(u.repository, (by.get(u.repository) ?? true) && u.state === 'verified');
|
|
1713
|
+
return [...by].filter(([, ok]) => ok).map(([r]) => r).sort();
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
// ---------------------------------------------------------------- persistence
|
|
1717
|
+
|
|
1718
|
+
function sleepSync(ms) { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }
|
|
1719
|
+
|
|
1720
|
+
function withLock(file, fn) {
|
|
1721
|
+
const lock = `${file}.lock`;
|
|
1722
|
+
const start = Date.now();
|
|
1723
|
+
for (;;) {
|
|
1724
|
+
try { fs.writeFileSync(lock, String(process.pid), { flag: 'wx' }); break; } catch (e) {
|
|
1725
|
+
if (e.code !== 'EEXIST') throw e;
|
|
1726
|
+
try { if (Date.now() - fs.statSync(lock).mtimeMs > PORTFOLIO_LIMITS.lockStaleMs) { fs.unlinkSync(lock); continue; } } catch { continue; }
|
|
1727
|
+
if (Date.now() - start > PORTFOLIO_LIMITS.lockWaitMs) throw typedError('LOCK_TIMEOUT', 'could not take the portfolio store lock');
|
|
1728
|
+
sleepSync(5);
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
try { return fn(); } finally { try { fs.unlinkSync(lock); } catch { /* already gone */ } }
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
/** Read and verify a store. Throws `STORE_CORRUPT` rather than returning a guess. */
|
|
1735
|
+
function readStore(file) {
|
|
1736
|
+
let raw;
|
|
1737
|
+
try {
|
|
1738
|
+
const st = node_fs__WEBPACK_IMPORTED_MODULE_1__.lstatSync(file);
|
|
1739
|
+
if (st.isSymbolicLink() || !st.isFile() || st.size > PORTFOLIO_LIMITS.maxStoreBytes) throw typedError('STORE_CORRUPT', 'store is not a regular file within the size limit');
|
|
1740
|
+
raw = node_fs__WEBPACK_IMPORTED_MODULE_1__.readFileSync(file, 'utf8');
|
|
1741
|
+
} catch (e) { if (e.code === 'ENOENT') return null; throw e; }
|
|
1742
|
+
let store;
|
|
1743
|
+
try { store = JSON.parse(raw); } catch { throw typedError('STORE_CORRUPT', 'store is not valid JSON'); }
|
|
1744
|
+
const v = verifyStore(store);
|
|
1745
|
+
if (!v.ok) throw typedError('STORE_CORRUPT', `store failed verification: ${v.errors[0].code} ${v.errors[0].unit ?? ''}`, { errors: v.errors });
|
|
1746
|
+
return store;
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
function writeStore(file, store) {
|
|
1750
|
+
const text = JSON.stringify(store);
|
|
1751
|
+
if (text.length > PORTFOLIO_LIMITS.maxStoreBytes) throw typedError('STORE_TOO_LARGE', 'store would exceed the size limit');
|
|
1752
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
1753
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
1754
|
+
fs.writeFileSync(tmp, text, { mode: 0o600 });
|
|
1755
|
+
fs.renameSync(tmp, file);
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
/**
|
|
1759
|
+
* Open (or create) the durable store for a plan. An existing store for a DIFFERENT plan is an error: resuming the wrong portfolio would
|
|
1760
|
+
* silently mix two sets of results.
|
|
1761
|
+
*/
|
|
1762
|
+
function openStore(file, plan) {
|
|
1763
|
+
return withLock(file, () => {
|
|
1764
|
+
const existing = readStore(file);
|
|
1765
|
+
if (existing) {
|
|
1766
|
+
if (existing.planId !== plan.id) throw typedError('PLAN_MISMATCH', `the store at this path belongs to plan ${existing.planId}, not ${plan.id}`);
|
|
1767
|
+
return existing;
|
|
1768
|
+
}
|
|
1769
|
+
const fresh = newStore(plan);
|
|
1770
|
+
writeStore(file, fresh);
|
|
1771
|
+
return fresh;
|
|
1772
|
+
});
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
/** Locked read-modify-write. `fn(store)` mutates and returns a value; the store is verified before it is written back. */
|
|
1776
|
+
function mutateStore(file, fn) {
|
|
1777
|
+
return withLock(file, () => {
|
|
1778
|
+
const store = readStore(file);
|
|
1779
|
+
if (!store) throw typedError('NO_STORE', 'no portfolio store at this path');
|
|
1780
|
+
const out = fn(store);
|
|
1781
|
+
const v = verifyStore(store);
|
|
1782
|
+
if (!v.ok) throw typedError('STORE_CORRUPT', `refusing to write a store that fails verification: ${v.errors[0].code}`, { errors: v.errors });
|
|
1783
|
+
writeStore(file, store);
|
|
1784
|
+
return out;
|
|
1785
|
+
});
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
// ---------------------------------------------------------------- runner
|
|
1789
|
+
|
|
1790
|
+
/**
|
|
1791
|
+
* Drive a store to completion with bounded concurrency. Mirrors runFleet's isolation: an executor that throws FAILS its unit (and costs
|
|
1792
|
+
* a retry), it never takes the run down. `executor(unit, ctx)` returns `{ resultDigest, dependencies }`.
|
|
1793
|
+
*
|
|
1794
|
+
* `maxUnits` stops after that many units have been leased, for interruption tests and for time-boxed runs. Returns the progress.
|
|
1795
|
+
*/
|
|
1796
|
+
async function runPortfolio({ file, executor, holder = 'worker', clock = () => Date.now(), ttlMs, concurrency = 2, maxUnits = Infinity, skip = () => false }) {
|
|
1797
|
+
let leased = 0;
|
|
1798
|
+
const limit = Math.max(1, Math.min(concurrency, 16));
|
|
1799
|
+
async function worker(n) {
|
|
1800
|
+
for (;;) {
|
|
1801
|
+
if (leased >= maxUnits) return;
|
|
1802
|
+
const lease = mutateStore(file, (s) => leaseUnit(s, { holder: `${holder}-${n}`, now: clock(), ttlMs, skip }));
|
|
1803
|
+
if (!lease) return;
|
|
1804
|
+
leased += 1;
|
|
1805
|
+
mutateStore(file, (s) => startUnit(s, { unitId: lease.unitId, attemptId: lease.attemptId, now: clock() }));
|
|
1806
|
+
const unit = readStore(file).units[lease.unitId];
|
|
1807
|
+
let res = null; let err = null;
|
|
1808
|
+
try { res = await executor(unit, { attemptId: lease.attemptId }); } catch (e) { err = e; }
|
|
1809
|
+
mutateStore(file, (s) => {
|
|
1810
|
+
if (err || !res) failUnit(s, { unitId: lease.unitId, attemptId: lease.attemptId, reason: err ? err.message : 'executor returned nothing', now: clock() });
|
|
1811
|
+
else completeUnit(s, { unitId: lease.unitId, attemptId: lease.attemptId, resultDigest: res.resultDigest, dependencies: res.dependencies, now: clock() });
|
|
1812
|
+
});
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
await Promise.all(Array.from({ length: limit }, (_, i) => worker(i)));
|
|
1816
|
+
return progressOf(readStore(file));
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
|
|
1820
|
+
/***/ })
|
|
1821
|
+
|
|
1822
|
+
};
|