@clear-capabilities/agentic-security-scanner 0.161.0 → 0.163.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/bin/agentic-security.js +98 -2
- package/dist/1340.index.js +1871 -0
- package/dist/3477.index.js +727 -0
- package/dist/3499.index.js +23 -3
- package/dist/4113.index.js +59 -8
- package/dist/5014.index.js +833 -0
- package/dist/5756.index.js +2 -2
- package/dist/6257.index.js +1 -1
- package/dist/6907.index.js +248 -0
- package/dist/6994.index.js +1 -1
- package/dist/7033.index.js +33 -5
- package/dist/7439.index.js +1822 -0
- package/dist/7838.index.js +38 -2
- package/dist/8798.index.js +1 -1
- package/dist/9207.index.js +5 -0
- package/dist/9560.index.js +2336 -27
- package/dist/957.index.js +1 -1
- package/dist/{1905.index.js → 9999.index.js} +306 -13
- package/dist/agentic-security.mjs +12 -12
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/package.json +39 -7
- package/src/capabilities/CLAUDE.md +114 -0
- package/src/capabilities/address.js +73 -0
- package/src/capabilities/attack-coverage.js +93 -0
- package/src/capabilities/decide.js +256 -0
- package/src/capabilities/delegate.js +69 -0
- package/src/capabilities/egress.js +20 -0
- package/src/capabilities/hook-advice.js +50 -0
- package/src/capabilities/index.js +32 -0
- package/src/capabilities/manifest.js +344 -0
- package/src/capabilities/outbound.js +189 -0
- package/src/capabilities/paths.js +57 -0
- package/src/capabilities/probes.js +211 -0
- package/src/capabilities/proxy.js +182 -0
- package/src/capabilities/reasons.js +67 -0
- package/src/capabilities/receipts.js +356 -0
- package/src/capabilities/records.js +58 -0
- package/src/capabilities/recovery.js +367 -0
- package/src/capabilities/report.js +81 -0
- package/src/capabilities/runner.js +236 -0
- package/src/capabilities/secrets.js +26 -0
- package/src/capabilities/tool-gate.js +119 -0
- package/src/capabilities/tool-registry.js +60 -0
- package/src/dataflow/CLAUDE.md +29 -0
- package/src/dataflow/catalog.js +21 -0
- package/src/dataflow/engine.js +20 -0
- package/src/dataflow/guard-dominance.js +352 -0
- package/src/discovery/CLAUDE.md +18 -0
- package/src/discovery/index.js +43 -1
- package/src/discovery/lenses.js +18 -0
- package/src/discovery/memory.js +7 -5
- package/src/engine.js +80 -15
- package/src/fix/apply-fix-service.js +33 -3
- package/src/integrations/index.js +5 -2
- package/src/ir/CLAUDE.md +14 -0
- package/src/ir/implicit-handler-params.js +74 -0
- package/src/ir/parser-cs.js +20 -0
- package/src/ir/parser-go.js +30 -13
- package/src/ir/parser-java.js +11 -0
- package/src/language/nix-eval-isolation.js +33 -5
- package/src/lineage/CLAUDE.md +31 -0
- package/src/lineage/deployment/adapter-cloud.js +202 -0
- package/src/lineage/deployment/adapter-compose.js +90 -0
- package/src/lineage/deployment/adapter-kit.js +142 -0
- package/src/lineage/deployment/adapter-kubernetes.js +247 -0
- package/src/lineage/deployment/attack-paths.js +443 -0
- package/src/lineage/deployment/boundaries-run.js +223 -0
- package/src/lineage/deployment/boundary-graph.js +567 -0
- package/src/lineage/deployment/cross-repo.js +253 -0
- package/src/lineage/deployment/drift.js +383 -0
- package/src/lineage/deployment/ingest.js +221 -0
- package/src/lineage/deployment/projection.js +244 -0
- package/src/lineage/deployment/trace-correlation.js +541 -0
- package/src/lineage/source-registry.js +19 -0
- package/src/mcp/CLAUDE.md +14 -3
- package/src/mcp/invariant-tools.js +64 -0
- package/src/mcp/portfolio-tools.js +79 -0
- package/src/mcp/redact.js +15 -0
- package/src/mcp/server.js +24 -3
- package/src/mcp/tools.js +30 -5
- package/src/posture/CLAUDE.md +505 -0
- package/src/posture/accuracy-scorecard.js +95 -4
- package/src/posture/assurance/baseline-inventory.js +218 -0
- package/src/posture/assurance/baseline.js +322 -0
- package/src/posture/assurance/bounded-io.js +145 -0
- package/src/posture/assurance/config.js +299 -0
- package/src/posture/assurance/contracts.js +315 -0
- package/src/posture/assurance/identity.js +62 -0
- package/src/posture/assurance/migrations.js +259 -0
- package/src/posture/assurance/rollback.js +214 -0
- package/src/posture/assurance/rollout.js +200 -0
- package/src/posture/assurance/schema-kit.js +111 -0
- package/src/posture/assurance/verification-record.js +258 -0
- package/src/posture/attestation.js +5 -0
- package/src/posture/autopilot.js +28 -1
- package/src/posture/business-logic.js +26 -4
- package/src/posture/evaluation/bakeoff.js +148 -0
- package/src/posture/evaluation/custody.js +200 -0
- package/src/posture/evaluation/deployment-ablation.js +416 -0
- package/src/posture/evaluation/dev-cases.js +65 -0
- package/src/posture/evaluation/economics.js +85 -0
- package/src/posture/evaluation/gates.js +161 -0
- package/src/posture/evaluation/grouping.js +101 -0
- package/src/posture/evaluation/interval.js +132 -0
- package/src/posture/evaluation/invariant-ablation.js +325 -0
- package/src/posture/evaluation/labels.js +200 -0
- package/src/posture/evaluation/protocol.js +308 -0
- package/src/posture/evaluation/release-gate.js +209 -0
- package/src/posture/evaluation/report.js +158 -0
- package/src/posture/evaluation/runner.js +273 -0
- package/src/posture/evaluation/scan-child.js +80 -0
- package/src/posture/evaluation/score.js +174 -0
- package/src/posture/evaluation/synthetic.js +90 -0
- package/src/posture/evaluation/why-missed.js +219 -0
- package/src/posture/evidence-bundle.js +93 -7
- package/src/posture/execution-proof.js +11 -1
- package/src/posture/fix-verify-loop.js +20 -6
- package/src/posture/fix-verify.js +37 -2
- package/src/posture/fleet.js +11 -2
- package/src/posture/invariants/cli.js +95 -0
- package/src/posture/invariants/coverage.js +164 -0
- package/src/posture/invariants/export.js +178 -0
- package/src/posture/invariants/expressions.js +46 -0
- package/src/posture/invariants/lifecycle.js +289 -0
- package/src/posture/invariants/project-input.js +55 -0
- package/src/posture/invariants/repair.js +340 -0
- package/src/posture/invariants/run.js +45 -0
- package/src/posture/invariants/scenarios.js +311 -0
- package/src/posture/invariants/schema.js +167 -0
- package/src/posture/invariants/shrink.js +407 -0
- package/src/posture/invariants/state-assertions.js +165 -0
- package/src/posture/logic-claims.js +12 -0
- package/src/posture/model-routing.js +40 -0
- package/src/posture/oracles/adapters.js +531 -0
- package/src/posture/oracles/conformance.js +276 -0
- package/src/posture/oracles/oracle.js +453 -0
- package/src/posture/oracles/registry.js +76 -0
- package/src/posture/oracles/scenario-classes.js +223 -0
- package/src/posture/portfolio/backend.js +279 -0
- package/src/posture/portfolio/bundle.js +314 -0
- package/src/posture/portfolio/cli.js +142 -0
- package/src/posture/portfolio/manifest.js +263 -0
- package/src/posture/portfolio/progress.js +267 -0
- package/src/posture/portfolio/resume.js +130 -0
- package/src/posture/portfolio/retention.js +207 -0
- package/src/posture/portfolio/scheduler.js +488 -0
- package/src/posture/portfolio/signing.js +155 -0
- package/src/posture/portfolio/wording.js +59 -0
- package/src/posture/portfolio/work-units.js +405 -0
- package/src/posture/replay/replay.js +250 -0
- package/src/posture/routing/adapter.js +140 -0
- package/src/posture/routing/calibration.js +278 -0
- package/src/posture/routing/canary.js +141 -0
- package/src/posture/routing/control.js +35 -0
- package/src/posture/routing/decide.js +211 -0
- package/src/posture/routing/drift.js +137 -0
- package/src/posture/routing/economics.js +185 -0
- package/src/posture/routing/feedback.js +207 -0
- package/src/posture/routing/index.js +31 -0
- package/src/posture/routing/outcomes.js +281 -0
- package/src/posture/routing/promotion.js +291 -0
- package/src/posture/routing/receipts.js +59 -0
- package/src/posture/routing/report.js +142 -0
- package/src/posture/routing/shadow.js +53 -0
- package/src/posture/specification-mining.js +26 -0
- package/src/posture/verification/advisory-state.js +71 -0
- package/src/posture/verification/emit.js +257 -0
- package/src/posture/verification/hypothesis-promotion.js +211 -0
- package/src/posture/verification/patch-negative.js +226 -0
- package/src/posture/verification/patch-promotion.js +113 -0
- package/src/posture/verification/projection.js +183 -0
- package/src/posture/verification/repair-records.js +124 -0
- package/src/posture/verifier.js +16 -0
- package/src/report/index.js +100 -1
- package/src/sandbox/CLAUDE.md +51 -0
- package/src/sandbox/backend-namespace.js +7 -0
- package/src/sandbox/backend-userspace.js +151 -34
- package/src/sandbox/control-probes.js +215 -0
- package/src/sandbox/supervise.js +201 -0
- package/src/sandbox/trust-boundary.js +113 -0
- package/src/sandbox/trust-domains.js +171 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Bounded assurance wording (X-703.AC03).
|
|
2
|
+
//
|
|
3
|
+
// A scan that found nothing blocking has said something narrow: the checks that ran, on the code and policy it names, found nothing the
|
|
4
|
+
// policy blocks. It has not said the software is safe, secure or free of vulnerabilities, and a signature on the result does not turn
|
|
5
|
+
// it into independent certification. Unconditional "Safe to deploy" wording over-claims in exactly the case where a check did not
|
|
6
|
+
// run, so with the `portfolio-assurance` feature on the verdict becomes the headline below followed by the scope and the gaps.
|
|
7
|
+
//
|
|
8
|
+
// The old wording stays the default. Existing fixtures and docs pin it; the change is additive behind the feature flag
|
|
9
|
+
// (assurance/config.js, `portfolio-assurance`, default off), so a flag-off run is byte-identical to before.
|
|
10
|
+
|
|
11
|
+
import { resolveAssuranceConfig, featureStatus } from '../assurance/config.js';
|
|
12
|
+
|
|
13
|
+
export const FEATURE_ID = 'portfolio-assurance';
|
|
14
|
+
export const HEADLINE = 'No blocking findings in completed supported checks';
|
|
15
|
+
export const BLOCKED_HEADLINE = 'Blocking findings present in completed supported checks';
|
|
16
|
+
const INCOMPLETE_HEADLINE = 'Blocking-finding checks did not complete';
|
|
17
|
+
export const NOT_A_GUARANTEE = 'This does not mean the software is safe or free of vulnerabilities, and a signature on it is not independent certification.';
|
|
18
|
+
|
|
19
|
+
/** Is the bounded wording on? An explicit boolean option wins; otherwise the assurance configuration (kill switch, environment, project file) decides. */
|
|
20
|
+
export function portfolioAssuranceEnabled({ option, env = process.env, scanRoot } = {}) {
|
|
21
|
+
if (typeof option === 'boolean') return option;
|
|
22
|
+
try {
|
|
23
|
+
return featureStatus(resolveAssuranceConfig({ scanRoot, env }), FEATURE_ID).status === 'ok';
|
|
24
|
+
} catch { return false; }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The bounded statement for a validated manifest: headline, scope and gaps, in that order. `signature` is the optional result of
|
|
29
|
+
* `verifyAssuranceClaim`; it can only ADD the trust-basis disclosure, never soften the headline.
|
|
30
|
+
*/
|
|
31
|
+
export function assuranceStatement(manifest, { signature = null } = {}) {
|
|
32
|
+
const cov = manifest.coverage;
|
|
33
|
+
const blocking = manifest.findings.blocking;
|
|
34
|
+
let headline = HEADLINE;
|
|
35
|
+
if (blocking > 0) headline = `${BLOCKED_HEADLINE}: ${blocking}`;
|
|
36
|
+
else if (!cov.complete) headline = `${HEADLINE}, but the mandatory scope is not fully covered`;
|
|
37
|
+
const scope = `Scope: ${cov.completed} of ${cov.mandatory} mandatory checks completed, ${cov.waived} waived, ${cov.incomplete} incomplete, ${cov.unsupported} unsupported. ${manifest.scope.description}`;
|
|
38
|
+
const gaps = [];
|
|
39
|
+
for (const c of manifest.checks.incomplete) gaps.push(`incomplete: ${c.id} (${c.gaps.join('; ')})`);
|
|
40
|
+
for (const c of manifest.checks.unsupported) gaps.push(`unsupported: ${c.id} (${c.gaps.join('; ')})`);
|
|
41
|
+
for (const c of manifest.checks.waived) gaps.push(`waived: ${c.id} (${c.reason}; approved by ${c.approvedBy})`);
|
|
42
|
+
for (const r of manifest.residualRisks) gaps.push(`residual risk: ${r.id} (${r.statement})`);
|
|
43
|
+
const lines = [headline, scope, gaps.length ? `Gaps: ${gaps.join(' | ')}` : 'Gaps: none recorded in the manifest', `Policy: ${manifest.policy.id} (blocking severity ${manifest.policy.blockingSeverity} and above)`];
|
|
44
|
+
if (manifest.synthetic === true) lines.push('Synthetic fixture: this describes no real release.');
|
|
45
|
+
if (signature) {
|
|
46
|
+
lines.push(signature.ok
|
|
47
|
+
? `Signature: valid under trust basis ${signature.trustBasis} (self-issued, signer ${signature.signer}). It shows the manifest is unmodified; it is not independent certification.`
|
|
48
|
+
: `Signature: NOT valid (${signature.reason}). Treat this statement as unverified.`);
|
|
49
|
+
}
|
|
50
|
+
lines.push(NOT_A_GUARANTEE);
|
|
51
|
+
return { headline, scope, gaps, blocking, complete: cov.complete, text: lines.join('\n') };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Headline for the one-screen scan verdict, from what a plain scan knows (no manifest). */
|
|
55
|
+
export function scanVerdictWording({ clean, scanIncomplete, actionableCount }) {
|
|
56
|
+
if (clean) return { icon: '✅', headline: HEADLINE, tier: 'ok' };
|
|
57
|
+
if (actionableCount === 0 && scanIncomplete) return { icon: '⚠️', headline: INCOMPLETE_HEADLINE, tier: 'incomplete' };
|
|
58
|
+
return { icon: '❌', headline: `${BLOCKED_HEADLINE}: ${actionableCount}`, tier: 'blocked' };
|
|
59
|
+
}
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
// Durable, leased portfolio work units (X-704).
|
|
2
|
+
//
|
|
3
|
+
// fleet.js already gives a many-repository run isolation, bounded concurrency and a per-repository completed marker. What it cannot do
|
|
4
|
+
// is say WHICH unit of work was in flight when a process died, hand that unit to another worker without doing it twice, or refuse to
|
|
5
|
+
// count a half-finished attempt as progress. This module adds that, in the same local-first shape: one JSON file, written atomically,
|
|
6
|
+
// no server, no network.
|
|
7
|
+
//
|
|
8
|
+
// PLAN `planPortfolio` decomposes the AUTHORIZED repositories into units, one per (repository, exact commit, task type). A unit id
|
|
9
|
+
// is a hash of those three and its required inputs, so the same plan always yields the same ids (X-704.AC01); a repository
|
|
10
|
+
// that is not on the authorization list is excluded and disclosed, never silently planned.
|
|
11
|
+
// STATES pending, leased, running, verified, blocked, failed, canceled. A lease has an expiry; an expired lease is recovered into
|
|
12
|
+
// pending (or failed once retries are spent). Every transition appends an event to the unit's attempt chain; the chain is
|
|
13
|
+
// hash-linked, so an attempt record can be read and verified but not rewritten (X-704.AC02).
|
|
14
|
+
// PROGRESS only a unit in `verified` with a recorded result counts. A leased or running unit, a failed attempt, an expired lease and a
|
|
15
|
+
// duplicate delivery of a completion all leave the count where it was (X-704.AC03).
|
|
16
|
+
//
|
|
17
|
+
// Operations are idempotent by attempt id: delivering the same `complete` twice records one verification; delivering a completion for a
|
|
18
|
+
// stale attempt (the lease expired and the unit moved on) is refused, because a zombie worker's result must not displace its successor's.
|
|
19
|
+
//
|
|
20
|
+
// The clock is an argument everywhere (`now`, milliseconds), so recovery is testable without sleeping and nothing here reads a clock.
|
|
21
|
+
// Persistence is `mutateStore`: an exclusive lock file, read, verify, change, write to a temporary file, rename. A store that fails
|
|
22
|
+
// verification is an error, never silently reset to empty.
|
|
23
|
+
|
|
24
|
+
import * as crypto from 'node:crypto';
|
|
25
|
+
import * as fs from 'node:fs';
|
|
26
|
+
import * as path from 'node:path';
|
|
27
|
+
import { canonicalJson } from '../evidence-bundle.js';
|
|
28
|
+
import { digestOf } from '../assurance/identity.js';
|
|
29
|
+
import { isCommit } from '../assurance/schema-kit.js';
|
|
30
|
+
|
|
31
|
+
const STORE_SCHEMA = 'agentic-security/portfolio-store';
|
|
32
|
+
const PLAN_SCHEMA = 'agentic-security/portfolio-plan';
|
|
33
|
+
const STORE_VERSION = 1;
|
|
34
|
+
export const UNIT_STATES = Object.freeze(['pending', 'leased', 'running', 'verified', 'blocked', 'failed', 'canceled']);
|
|
35
|
+
const TASK_TYPES = Object.freeze(['sast-scan', 'sca-reachability', 'boundary-graph', 'invariant-scenarios', 'verification-replay', 'attestation']);
|
|
36
|
+
export const DEPENDENCY_DIMENSIONS = Object.freeze(['code', 'policy', 'graph', 'invariant', 'oracle', 'toolchain']);
|
|
37
|
+
|
|
38
|
+
export const PORTFOLIO_LIMITS = Object.freeze({
|
|
39
|
+
maxUnits: 5000, maxEventsPerUnit: 400, maxStoreBytes: 32 * 1024 * 1024, defaultLeaseMs: 5 * 60_000, maxLeaseMs: 6 * 60 * 60_000, defaultMaxRetries: 2, maxRetries: 10,
|
|
40
|
+
lockStaleMs: 30_000, lockWaitMs: 2000,
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const DIGEST_RE = /^sha256:[0-9a-f]{64}$/;
|
|
44
|
+
|
|
45
|
+
// ---------------------------------------------------------------- plan
|
|
46
|
+
|
|
47
|
+
const unitIdOf = (u) => `wu:${crypto.createHash('sha256').update(canonicalJson({ repository: u.repository, commit: u.commit, taskType: u.taskType, requiredInputs: u.requiredInputs })).digest('hex').slice(0, 16)}`;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Decompose authorized repositories into stable work units.
|
|
51
|
+
*
|
|
52
|
+
* @param {object} a
|
|
53
|
+
* @param {Array<{name:string, commit:string}>} a.repositories candidates, each pinned to an exact commit
|
|
54
|
+
* @param {string[]} a.authorized the repository names the operator has authorized; anything else is excluded and disclosed
|
|
55
|
+
* @param {string[]} a.taskTypes task types to run per repository (from TASK_TYPES)
|
|
56
|
+
* @param {Object<string,string[]>} [a.requiredInputs] per task type, the named inputs the unit needs (e.g. ['source','policy'])
|
|
57
|
+
*/
|
|
58
|
+
export function planPortfolio({ repositories = [], authorized = [], taskTypes = [], requiredInputs = {}, synthetic = false } = {}) {
|
|
59
|
+
const errors = [];
|
|
60
|
+
const rejected = [];
|
|
61
|
+
const allow = new Set(authorized);
|
|
62
|
+
for (const t of taskTypes) if (!TASK_TYPES.includes(t)) errors.push({ code: 'UNKNOWN_TASK_TYPE', message: `'${t}' is not a task type` });
|
|
63
|
+
if (taskTypes.length === 0) errors.push({ code: 'NO_TASKS', message: 'a plan needs at least one task type' });
|
|
64
|
+
const seen = new Set();
|
|
65
|
+
const units = [];
|
|
66
|
+
for (const r of repositories) {
|
|
67
|
+
if (!r || typeof r.name !== 'string' || !r.name) { errors.push({ code: 'BAD_REPOSITORY', message: 'repository needs a name' }); continue; }
|
|
68
|
+
if (!allow.has(r.name)) { rejected.push({ repository: r.name, reason: 'not-authorized' }); continue; }
|
|
69
|
+
if (!isCommit(r.commit)) { errors.push({ code: 'UNBOUND_REVISION', message: `repository '${r.name}' is not pinned to an exact commit` }); continue; }
|
|
70
|
+
if (seen.has(r.name)) { errors.push({ code: 'DUPLICATE_REPOSITORY', message: `repository '${r.name}' is listed twice` }); continue; }
|
|
71
|
+
seen.add(r.name);
|
|
72
|
+
for (const taskType of taskTypes) {
|
|
73
|
+
const inputs = [...new Set(requiredInputs[taskType] ?? [])].sort();
|
|
74
|
+
const u = { repository: r.name, commit: r.commit, taskType, requiredInputs: inputs };
|
|
75
|
+
units.push({ id: unitIdOf(u), ...u });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (units.length > PORTFOLIO_LIMITS.maxUnits) errors.push({ code: 'TOO_MANY_UNITS', message: `plan has ${units.length} units; the limit is ${PORTFOLIO_LIMITS.maxUnits}` });
|
|
79
|
+
units.sort((a, b) => (a.id < b.id ? -1 : 1));
|
|
80
|
+
const body = { schema: PLAN_SCHEMA, schemaVersion: '1.0.0', ...(synthetic ? { synthetic: true } : {}), units, rejected: rejected.sort((a, b) => (a.repository < b.repository ? -1 : 1)) };
|
|
81
|
+
return { ok: errors.length === 0, errors, plan: errors.length === 0 ? { ...body, id: `pplan:${digestOf(units).slice(7, 23)}` } : null };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// ---------------------------------------------------------------- state
|
|
85
|
+
|
|
86
|
+
const eventDigest = (e) => digestOf(e);
|
|
87
|
+
|
|
88
|
+
export function appendEvent(unit, ev) {
|
|
89
|
+
if (unit.events.length >= PORTFOLIO_LIMITS.maxEventsPerUnit) throw typedError('EVENT_LIMIT', `unit ${unit.id} reached ${PORTFOLIO_LIMITS.maxEventsPerUnit} recorded events`);
|
|
90
|
+
const prev = unit.events.length ? unit.events[unit.events.length - 1].digest : null;
|
|
91
|
+
const body = { seq: unit.events.length + 1, prev, ...ev };
|
|
92
|
+
unit.events.push({ ...body, digest: eventDigest(body) });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function typedError(code, message, extra = {}) { return Object.assign(new Error(message), { code, ...extra }); }
|
|
96
|
+
|
|
97
|
+
/** A new, empty store for a plan. */
|
|
98
|
+
export function newStore(plan) {
|
|
99
|
+
if (!plan || plan.schema !== PLAN_SCHEMA) throw typedError('BAD_PLAN', 'not a portfolio plan');
|
|
100
|
+
const units = {};
|
|
101
|
+
for (const u of plan.units) {
|
|
102
|
+
units[u.id] = { ...u, state: 'pending', retryCount: 0, maxRetries: PORTFOLIO_LIMITS.defaultMaxRetries, generation: 0, attemptSeq: 0, lease: null, result: null, stale: [], blockedReason: null, events: [] };
|
|
103
|
+
}
|
|
104
|
+
return { schema: STORE_SCHEMA, version: STORE_VERSION, planId: plan.id, ...(plan.synthetic ? { synthetic: true } : {}), units };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Verify the store's internal consistency: event chains, state/lease/result agreement. Returns `{ ok, errors }`. */
|
|
108
|
+
export function verifyStore(store) {
|
|
109
|
+
const errors = [];
|
|
110
|
+
if (!store || store.schema !== STORE_SCHEMA || store.version !== STORE_VERSION || typeof store.units !== 'object' || !store.units) return { ok: false, errors: [{ code: 'BAD_STORE', message: 'not a portfolio store of a supported version' }] };
|
|
111
|
+
for (const [id, u] of Object.entries(store.units)) {
|
|
112
|
+
if (u.id !== id || unitIdOf(u) !== id) errors.push({ code: 'ID_MISMATCH', unit: id, message: 'unit identity does not match its repository, commit, task type and inputs' });
|
|
113
|
+
if (!UNIT_STATES.includes(u.state)) errors.push({ code: 'BAD_STATE', unit: id, message: `unknown state '${u.state}'` });
|
|
114
|
+
let prev = null;
|
|
115
|
+
(u.events ?? []).forEach((e, i) => {
|
|
116
|
+
const { digest, ...body } = e;
|
|
117
|
+
if (e.seq !== i + 1 || e.prev !== prev || eventDigest(body) !== digest) errors.push({ code: 'EVENT_CHAIN_BROKEN', unit: id, message: `event ${i + 1} was altered or removed` });
|
|
118
|
+
prev = digest;
|
|
119
|
+
});
|
|
120
|
+
if ((u.state === 'leased' || u.state === 'running') !== !!u.lease) errors.push({ code: 'LEASE_STATE_MISMATCH', unit: id, message: 'a lease exists exactly when the unit is leased or running' });
|
|
121
|
+
if (u.state === 'verified' && !(u.result && DIGEST_RE.test(u.result.resultDigest ?? ''))) errors.push({ code: 'VERIFIED_WITHOUT_RESULT', unit: id, message: 'a verified unit must carry a result digest' });
|
|
122
|
+
if (u.state !== 'verified' && u.result) errors.push({ code: 'RESULT_WITHOUT_VERIFIED', unit: id, message: 'only a verified unit may carry a current result' });
|
|
123
|
+
}
|
|
124
|
+
return { ok: errors.length === 0, errors };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// ---------------------------------------------------------------- transitions (pure on a store object)
|
|
128
|
+
|
|
129
|
+
const unitOf = (store, id) => {
|
|
130
|
+
const u = store.units[id];
|
|
131
|
+
if (!u) throw typedError('UNKNOWN_UNIT', `no such unit '${id}'`);
|
|
132
|
+
return u;
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
function clampLease(ttlMs) {
|
|
136
|
+
const t = ttlMs ?? PORTFOLIO_LIMITS.defaultLeaseMs;
|
|
137
|
+
if (!Number.isFinite(t) || t < 1) throw typedError('BAD_LEASE', 'lease duration must be a positive number of milliseconds');
|
|
138
|
+
return Math.min(t, PORTFOLIO_LIMITS.maxLeaseMs);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function spendAttempt(u, ev, now) {
|
|
142
|
+
// an attempt ended without a verified result: it costs a retry. Out of retries is a terminal failure.
|
|
143
|
+
u.lease = null;
|
|
144
|
+
u.retryCount += 1;
|
|
145
|
+
appendEvent(u, { ...ev, at: now });
|
|
146
|
+
u.state = u.retryCount > u.maxRetries ? 'failed' : 'pending';
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Recover every unit whose lease has expired. Returns the ids recovered. */
|
|
150
|
+
export function recoverExpired(store, now) {
|
|
151
|
+
const out = [];
|
|
152
|
+
for (const id of Object.keys(store.units).sort()) {
|
|
153
|
+
const u = store.units[id];
|
|
154
|
+
if ((u.state === 'leased' || u.state === 'running') && u.lease && u.lease.expiresAt <= now) {
|
|
155
|
+
const attemptId = u.lease.attemptId;
|
|
156
|
+
spendAttempt(u, { type: 'expired', attemptId, holder: u.lease.holder }, now);
|
|
157
|
+
out.push(id);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return out;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Lease the next pending unit (or a named one) to `holder`. A repeat of the same `requestId` from the same holder returns the same lease
|
|
165
|
+
* instead of leasing a second unit (duplicate delivery of the request). Returns `null` when nothing is leasable.
|
|
166
|
+
*/
|
|
167
|
+
export function leaseUnit(store, { holder, now, ttlMs, unitId = null, requestId = null, skip = () => false }) {
|
|
168
|
+
if (typeof holder !== 'string' || !holder) throw typedError('BAD_HOLDER', 'a lease needs a holder');
|
|
169
|
+
recoverExpired(store, now);
|
|
170
|
+
if (requestId) {
|
|
171
|
+
for (const u of Object.values(store.units)) {
|
|
172
|
+
if (u.lease && u.lease.requestId === requestId && u.lease.holder === holder) return { unitId: u.id, attemptId: u.lease.attemptId, expiresAt: u.lease.expiresAt, duplicate: true };
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const candidates = unitId ? [unitOf(store, unitId)] : Object.values(store.units).sort((a, b) => (a.id < b.id ? -1 : 1));
|
|
176
|
+
const u = candidates.find((c) => c.state === 'pending' && !skip(c));
|
|
177
|
+
if (!u) return null;
|
|
178
|
+
u.attemptSeq += 1;
|
|
179
|
+
const attemptId = `${u.id}#${u.generation}.${u.attemptSeq}`;
|
|
180
|
+
const expiresAt = now + clampLease(ttlMs);
|
|
181
|
+
u.lease = { holder, attemptId, expiresAt, requestId };
|
|
182
|
+
u.state = 'leased';
|
|
183
|
+
appendEvent(u, { type: 'leased', attemptId, holder, at: now, expiresAt });
|
|
184
|
+
return { unitId: u.id, attemptId, expiresAt, duplicate: false };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function liveAttempt(store, unitId, attemptId, now) {
|
|
188
|
+
recoverExpired(store, now);
|
|
189
|
+
const u = unitOf(store, unitId);
|
|
190
|
+
if (!u.lease || u.lease.attemptId !== attemptId) return { u, ok: false };
|
|
191
|
+
return { u, ok: true };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** leased -> running. Idempotent for the same attempt. */
|
|
195
|
+
export function startUnit(store, { unitId, attemptId, now }) {
|
|
196
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
197
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'this attempt does not hold the lease (it expired, or the unit moved on)' };
|
|
198
|
+
if (u.state === 'running') return { ok: true, duplicate: true };
|
|
199
|
+
u.state = 'running';
|
|
200
|
+
appendEvent(u, { type: 'started', attemptId, at: now });
|
|
201
|
+
return { ok: true, duplicate: false };
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Extend a live lease. */
|
|
205
|
+
export function renewLease(store, { unitId, attemptId, now, ttlMs }) {
|
|
206
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
207
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'the lease is no longer held' };
|
|
208
|
+
u.lease.expiresAt = now + clampLease(ttlMs);
|
|
209
|
+
appendEvent(u, { type: 'renewed', attemptId, at: now, expiresAt: u.lease.expiresAt });
|
|
210
|
+
return { ok: true, expiresAt: u.lease.expiresAt };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** A recorded result's dependency digests, one per dimension. All six are required: a result without them cannot be reused. */
|
|
214
|
+
export function checkDependencies(deps) {
|
|
215
|
+
const missing = DEPENDENCY_DIMENSIONS.filter((d) => !DIGEST_RE.test(deps?.[d] ?? ''));
|
|
216
|
+
return missing;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Record a verified result. Accepted only from the live attempt, with all six dependency digests. A second delivery of the same
|
|
221
|
+
* completion is acknowledged and changes nothing; a completion from a stale attempt is refused.
|
|
222
|
+
*/
|
|
223
|
+
export function completeUnit(store, { unitId, attemptId, resultDigest, dependencies, now }) {
|
|
224
|
+
const u0 = unitOf(store, unitId);
|
|
225
|
+
if (u0.state === 'verified' && u0.result?.attemptId === attemptId) return { ok: true, duplicate: true, counted: false };
|
|
226
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
227
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'this attempt does not hold the lease (it expired, or the unit moved on); the result is discarded' };
|
|
228
|
+
if (!DIGEST_RE.test(resultDigest ?? '')) return { ok: false, code: 'bad-result', reason: 'a result needs a sha256 digest' };
|
|
229
|
+
const missing = checkDependencies(dependencies);
|
|
230
|
+
if (missing.length) return { ok: false, code: 'unbound-dependencies', reason: `result is missing dependency digests: ${missing.join(', ')}` };
|
|
231
|
+
const deps = Object.fromEntries(DEPENDENCY_DIMENSIONS.map((d) => [d, dependencies[d]]));
|
|
232
|
+
u.lease = null;
|
|
233
|
+
u.result = { attemptId, resultDigest, dependencies: deps, dependencyDigest: digestOf(deps), generation: u.generation };
|
|
234
|
+
u.state = 'verified';
|
|
235
|
+
appendEvent(u, { type: 'verified', attemptId, at: now, resultDigest, dependencyDigest: u.result.dependencyDigest });
|
|
236
|
+
return { ok: true, duplicate: false, counted: true };
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** The live attempt ended in failure. Costs a retry; out of retries the unit is terminally `failed`. */
|
|
240
|
+
export function failUnit(store, { unitId, attemptId, reason, now }) {
|
|
241
|
+
const u0 = unitOf(store, unitId);
|
|
242
|
+
const dup = u0.events.some((e) => e.type === 'failed' && e.attemptId === attemptId);
|
|
243
|
+
if (dup) return { ok: true, duplicate: true };
|
|
244
|
+
const { u, ok } = liveAttempt(store, unitId, attemptId, now);
|
|
245
|
+
if (!ok) return { ok: false, code: 'stale-attempt', reason: 'this attempt does not hold the lease' };
|
|
246
|
+
spendAttempt(u, { type: 'failed', attemptId, holder: u.lease?.holder ?? null, reason: String(reason ?? 'failed').slice(0, 300) }, now);
|
|
247
|
+
return { ok: true, duplicate: false, state: u.state };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** Park a unit that cannot proceed (a missing input). Not counted; independent units still run. */
|
|
251
|
+
export function blockUnit(store, { unitId, reason, now }) {
|
|
252
|
+
const u = unitOf(store, unitId);
|
|
253
|
+
if (!['pending', 'leased', 'running'].includes(u.state)) return { ok: false, code: 'bad-state', reason: `cannot block a ${u.state} unit` };
|
|
254
|
+
u.lease = null; u.state = 'blocked'; u.blockedReason = String(reason ?? 'blocked').slice(0, 300);
|
|
255
|
+
appendEvent(u, { type: 'blocked', reason: u.blockedReason, at: now });
|
|
256
|
+
return { ok: true };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export function unblockUnit(store, { unitId, now }) {
|
|
260
|
+
const u = unitOf(store, unitId);
|
|
261
|
+
if (u.state !== 'blocked') return { ok: false, code: 'bad-state', reason: `unit is ${u.state}, not blocked` };
|
|
262
|
+
u.state = 'pending'; u.blockedReason = null;
|
|
263
|
+
appendEvent(u, { type: 'unblocked', at: now });
|
|
264
|
+
return { ok: true };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Cancel every unit that is not verified or terminally failed (or one named unit). Leases are released; verified receipts and failure records are kept. */
|
|
268
|
+
export function cancelUnits(store, { unitId = null, reason, now }) {
|
|
269
|
+
const canceled = [];
|
|
270
|
+
for (const id of Object.keys(store.units).sort()) {
|
|
271
|
+
if (unitId && id !== unitId) continue;
|
|
272
|
+
const u = store.units[id];
|
|
273
|
+
if (u.state === 'verified' || u.state === 'canceled' || u.state === 'failed') continue; // receipts and terminal failures keep their own status
|
|
274
|
+
u.lease = null; u.state = 'canceled';
|
|
275
|
+
appendEvent(u, { type: 'canceled', reason: String(reason ?? 'canceled').slice(0, 300), at: now });
|
|
276
|
+
canceled.push(id);
|
|
277
|
+
}
|
|
278
|
+
return { ok: true, canceled };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ---------------------------------------------------------------- progress
|
|
282
|
+
|
|
283
|
+
/** Counts by state. `verified` is the only count that is progress. */
|
|
284
|
+
export function progressOf(store) {
|
|
285
|
+
const by = Object.fromEntries(UNIT_STATES.map((s) => [s, 0]));
|
|
286
|
+
let staleResults = 0;
|
|
287
|
+
for (const u of Object.values(store.units)) { by[u.state] += 1; staleResults += u.stale.length; }
|
|
288
|
+
const total = Object.keys(store.units).length;
|
|
289
|
+
return { total, ...by, completed: by.verified, remaining: total - by.verified - by.canceled, staleResults, done: total > 0 && by.verified === total };
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** The current scoped results: verified units only, keyed by unit id. Two runs converged exactly when these are equal. */
|
|
293
|
+
export function scopedResults(store) {
|
|
294
|
+
return Object.fromEntries(Object.values(store.units).filter((u) => u.state === 'verified').sort((a, b) => (a.id < b.id ? -1 : 1)).map((u) => [u.id, u.result.resultDigest]));
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Repositories whose every unit is verified and current: what `runFleet({ portfolioVerified })` may skip. */
|
|
298
|
+
export function fullyVerifiedRepositories(store) {
|
|
299
|
+
const by = new Map();
|
|
300
|
+
for (const u of Object.values(store.units)) by.set(u.repository, (by.get(u.repository) ?? true) && u.state === 'verified');
|
|
301
|
+
return [...by].filter(([, ok]) => ok).map(([r]) => r).sort();
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// ---------------------------------------------------------------- persistence
|
|
305
|
+
|
|
306
|
+
function sleepSync(ms) { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }
|
|
307
|
+
|
|
308
|
+
function withLock(file, fn) {
|
|
309
|
+
const lock = `${file}.lock`;
|
|
310
|
+
const start = Date.now();
|
|
311
|
+
for (;;) {
|
|
312
|
+
try { fs.writeFileSync(lock, String(process.pid), { flag: 'wx' }); break; } catch (e) {
|
|
313
|
+
if (e.code !== 'EEXIST') throw e;
|
|
314
|
+
try { if (Date.now() - fs.statSync(lock).mtimeMs > PORTFOLIO_LIMITS.lockStaleMs) { fs.unlinkSync(lock); continue; } } catch { continue; }
|
|
315
|
+
if (Date.now() - start > PORTFOLIO_LIMITS.lockWaitMs) throw typedError('LOCK_TIMEOUT', 'could not take the portfolio store lock');
|
|
316
|
+
sleepSync(5);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
try { return fn(); } finally { try { fs.unlinkSync(lock); } catch { /* already gone */ } }
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** Read and verify a store. Throws `STORE_CORRUPT` rather than returning a guess. */
|
|
323
|
+
export function readStore(file) {
|
|
324
|
+
let raw;
|
|
325
|
+
try {
|
|
326
|
+
const st = fs.lstatSync(file);
|
|
327
|
+
if (st.isSymbolicLink() || !st.isFile() || st.size > PORTFOLIO_LIMITS.maxStoreBytes) throw typedError('STORE_CORRUPT', 'store is not a regular file within the size limit');
|
|
328
|
+
raw = fs.readFileSync(file, 'utf8');
|
|
329
|
+
} catch (e) { if (e.code === 'ENOENT') return null; throw e; }
|
|
330
|
+
let store;
|
|
331
|
+
try { store = JSON.parse(raw); } catch { throw typedError('STORE_CORRUPT', 'store is not valid JSON'); }
|
|
332
|
+
const v = verifyStore(store);
|
|
333
|
+
if (!v.ok) throw typedError('STORE_CORRUPT', `store failed verification: ${v.errors[0].code} ${v.errors[0].unit ?? ''}`, { errors: v.errors });
|
|
334
|
+
return store;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function writeStore(file, store) {
|
|
338
|
+
const text = JSON.stringify(store);
|
|
339
|
+
if (text.length > PORTFOLIO_LIMITS.maxStoreBytes) throw typedError('STORE_TOO_LARGE', 'store would exceed the size limit');
|
|
340
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
341
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
342
|
+
fs.writeFileSync(tmp, text, { mode: 0o600 });
|
|
343
|
+
fs.renameSync(tmp, file);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Open (or create) the durable store for a plan. An existing store for a DIFFERENT plan is an error: resuming the wrong portfolio would
|
|
348
|
+
* silently mix two sets of results.
|
|
349
|
+
*/
|
|
350
|
+
export function openStore(file, plan) {
|
|
351
|
+
return withLock(file, () => {
|
|
352
|
+
const existing = readStore(file);
|
|
353
|
+
if (existing) {
|
|
354
|
+
if (existing.planId !== plan.id) throw typedError('PLAN_MISMATCH', `the store at this path belongs to plan ${existing.planId}, not ${plan.id}`);
|
|
355
|
+
return existing;
|
|
356
|
+
}
|
|
357
|
+
const fresh = newStore(plan);
|
|
358
|
+
writeStore(file, fresh);
|
|
359
|
+
return fresh;
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** Locked read-modify-write. `fn(store)` mutates and returns a value; the store is verified before it is written back. */
|
|
364
|
+
export function mutateStore(file, fn) {
|
|
365
|
+
return withLock(file, () => {
|
|
366
|
+
const store = readStore(file);
|
|
367
|
+
if (!store) throw typedError('NO_STORE', 'no portfolio store at this path');
|
|
368
|
+
const out = fn(store);
|
|
369
|
+
const v = verifyStore(store);
|
|
370
|
+
if (!v.ok) throw typedError('STORE_CORRUPT', `refusing to write a store that fails verification: ${v.errors[0].code}`, { errors: v.errors });
|
|
371
|
+
writeStore(file, store);
|
|
372
|
+
return out;
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// ---------------------------------------------------------------- runner
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Drive a store to completion with bounded concurrency. Mirrors runFleet's isolation: an executor that throws FAILS its unit (and costs
|
|
380
|
+
* a retry), it never takes the run down. `executor(unit, ctx)` returns `{ resultDigest, dependencies }`.
|
|
381
|
+
*
|
|
382
|
+
* `maxUnits` stops after that many units have been leased, for interruption tests and for time-boxed runs. Returns the progress.
|
|
383
|
+
*/
|
|
384
|
+
export async function runPortfolio({ file, executor, holder = 'worker', clock = () => Date.now(), ttlMs, concurrency = 2, maxUnits = Infinity, skip = () => false }) {
|
|
385
|
+
let leased = 0;
|
|
386
|
+
const limit = Math.max(1, Math.min(concurrency, 16));
|
|
387
|
+
async function worker(n) {
|
|
388
|
+
for (;;) {
|
|
389
|
+
if (leased >= maxUnits) return;
|
|
390
|
+
const lease = mutateStore(file, (s) => leaseUnit(s, { holder: `${holder}-${n}`, now: clock(), ttlMs, skip }));
|
|
391
|
+
if (!lease) return;
|
|
392
|
+
leased += 1;
|
|
393
|
+
mutateStore(file, (s) => startUnit(s, { unitId: lease.unitId, attemptId: lease.attemptId, now: clock() }));
|
|
394
|
+
const unit = readStore(file).units[lease.unitId];
|
|
395
|
+
let res = null; let err = null;
|
|
396
|
+
try { res = await executor(unit, { attemptId: lease.attemptId }); } catch (e) { err = e; }
|
|
397
|
+
mutateStore(file, (s) => {
|
|
398
|
+
if (err || !res) failUnit(s, { unitId: lease.unitId, attemptId: lease.attemptId, reason: err ? err.message : 'executor returned nothing', now: clock() });
|
|
399
|
+
else completeUnit(s, { unitId: lease.unitId, attemptId: lease.attemptId, resultDigest: res.resultDigest, dependencies: res.dependencies, now: clock() });
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
await Promise.all(Array.from({ length: limit }, (_, i) => worker(i)));
|
|
404
|
+
return progressOf(readStore(file));
|
|
405
|
+
}
|