@clear-capabilities/agentic-security-scanner 0.174.0 → 0.176.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/bin/agentic-security.js +199 -28
  3. package/dist/2066.index.js +850 -0
  4. package/dist/2199.index.js +1 -1
  5. package/dist/2584.index.js +3332 -0
  6. package/dist/2923.index.js +13 -0
  7. package/dist/2946.index.js +81 -0
  8. package/dist/3196.index.js +27 -0
  9. package/dist/3727.index.js +292 -0
  10. package/dist/4168.index.js +509 -0
  11. package/dist/4281.index.js +993 -0
  12. package/dist/4978bec8fdf5c733dc85.json +283 -0
  13. package/dist/5099.index.js +96 -0
  14. package/dist/5616.index.js +1471 -0
  15. package/dist/5620.index.js +2370 -0
  16. package/dist/5817.index.js +714 -0
  17. package/dist/6333.index.js +98 -0
  18. package/dist/6655.index.js +3221 -0
  19. package/dist/7838.index.js +446 -974
  20. package/dist/818.index.js +4 -3
  21. package/dist/83.index.js +159 -0
  22. package/dist/8551.index.js +82 -0
  23. package/dist/8633.index.js +93 -0
  24. package/dist/8714.index.js +76 -76
  25. package/dist/8825.index.js +84 -0
  26. package/dist/932.index.js +186 -0
  27. package/dist/9560.index.js +358 -19
  28. package/dist/9800.index.js +6 -1
  29. package/dist/agentic-security.mjs +10 -10
  30. package/dist/agentic-security.mjs.sha256 +1 -1
  31. package/dist/df1e8683e7eda7b2b79c.js +29 -0
  32. package/package.json +7 -6
  33. package/src/artifact/CLAUDE.md +4 -0
  34. package/src/artifact/artifact-evidence.js +63 -0
  35. package/src/artifact/cosign.js +61 -0
  36. package/src/artifact/engine-source.js +72 -0
  37. package/src/artifact/oci-layout.js +47 -0
  38. package/src/artifact/registry.js +213 -0
  39. package/src/capabilities/tool-registry.js +2 -0
  40. package/src/language/haskell-security-rules.js +1 -1
  41. package/src/language/nixos-hardening.js +37 -0
  42. package/src/lineage/CLAUDE.md +4 -0
  43. package/src/lineage/identity-token.js +11 -0
  44. package/src/lineage/observation-adapters.js +13 -0
  45. package/src/lineage/runtime-edge-states.js +139 -0
  46. package/src/lineage/runtime-explorer-view.js +82 -0
  47. package/src/lineage/runtime-observation.js +11 -0
  48. package/src/mcp/CLAUDE.md +3 -1
  49. package/src/mcp/proof-tools.js +132 -0
  50. package/src/mcp/server.js +1 -1
  51. package/src/mcp/tools.js +3 -1
  52. package/src/posture/CLAUDE.md +23 -0
  53. package/src/posture/artifact-registry.js +4 -0
  54. package/src/posture/dnr/attack-facts.js +31 -0
  55. package/src/posture/dnr/cli.js +138 -0
  56. package/src/posture/dnr/detection-ir.js +123 -0
  57. package/src/posture/dnr/exposure-window.js +134 -0
  58. package/src/posture/dnr/hunt.js +84 -0
  59. package/src/posture/dnr/log-reader.js +189 -0
  60. package/src/posture/dnr/matchers.js +90 -0
  61. package/src/posture/dnr/outputs.js +262 -0
  62. package/src/posture/dnr/payload-classes.js +120 -0
  63. package/src/posture/dnr/redact.js +25 -0
  64. package/src/posture/dnr/store.js +57 -0
  65. package/src/posture/dnr/validate.js +64 -0
  66. package/src/posture/dnr/view.js +51 -0
  67. package/src/posture/evaluation/lab/arm-ollama.js +88 -0
  68. package/src/posture/evaluation/lab/arms.js +142 -0
  69. package/src/posture/evaluation/lab/cli.js +135 -0
  70. package/src/posture/evaluation/lab/dataset.js +139 -0
  71. package/src/posture/evaluation/lab/materialize.js +44 -0
  72. package/src/posture/evaluation/lab/neutralize.js +144 -0
  73. package/src/posture/evaluation/lab/report.js +176 -0
  74. package/src/posture/evaluation/lab/run.js +225 -0
  75. package/src/posture/evaluation/lab/score.js +135 -0
  76. package/src/posture/evaluation/lab/scoreboard.js +48 -0
  77. package/src/posture/evaluation/lab/stats.js +41 -0
  78. package/src/posture/evaluation/lab/store.js +53 -0
  79. package/src/posture/fleet/cli.js +248 -63
  80. package/src/posture/fleet/discover-edges.js +135 -0
  81. package/src/posture/fleet/enterprise-graph.js +148 -0
  82. package/src/posture/fleet/fleet-graph.js +9 -4
  83. package/src/posture/fleet/fleet-manifest.js +50 -5
  84. package/src/posture/fleet/job-runner.js +153 -0
  85. package/src/posture/fleet/job-store.js +136 -0
  86. package/src/posture/fleet/org-health.js +56 -0
  87. package/src/posture/fleet/scan-cache.js +112 -0
  88. package/src/posture/fleet/scan-child.js +29 -0
  89. package/src/posture/proof-fabric/cli.js +34 -6
  90. package/src/posture/proof-fabric/container.js +57 -0
  91. package/src/posture/proof-fabric/fix-proof.js +2 -1
  92. package/src/posture/proof-fabric/generate-python.js +129 -0
  93. package/src/posture/proof-fabric/generate.js +213 -0
  94. package/src/posture/proof-fabric/reproducer.js +3 -0
  95. package/src/posture/proof-fabric/runner.js +20 -11
  96. package/src/posture/proof-fabric/siblings.js +77 -0
  97. package/src/posture/routing/policy-router.js +103 -0
  98. package/src/posture/rule-compiler/cli.js +64 -6
  99. package/src/posture/rule-compiler/corpus.js +10 -2
  100. package/src/posture/rule-compiler/mutation.js +72 -1
  101. package/src/posture/rule-compiler/promotion.js +21 -5
  102. package/src/posture/rule-compiler/proposal.js +132 -0
  103. package/src/posture/rule-compiler/store.js +16 -0
@@ -0,0 +1,148 @@
1
+ // The enterprise graph (PRD 4, requirements 6 to 8): the repositories, APIs, queues, databases, external providers and trust zones
2
+ // of an organization, with every edge carrying the evidence it came from.
3
+ //
4
+ // Two kinds of edge, never confused:
5
+ // * `depends-on` edges come from each repository's own package manifest (the declared name and the spec it pins). A flaw in the
6
+ // dependency can affect the dependent. They are NOT movement an attacker makes.
7
+ // * REACH edges are what an attacker can move along: a call from one service to another, a producer to a queue, a queue to its
8
+ // consumer, a service to a database or an external provider, and the Internet to anything declared Internet-facing. Those come
9
+ // from the fleet manifest, which is operator-declared. They are labelled `source: fleet-manifest` and are exactly as good as the
10
+ // operator's declaration; nothing here pretends to have discovered them. (Code-derived call and queue edges are what the Data
11
+ // Flow Explorer's per-repository graph provides; joining that in is not built.)
12
+ //
13
+ // A graph edge without evidence is a bug, and `validateEnterpriseGraph` says so: "every graph edge traceable to source evidence".
14
+ import { createHash } from 'node:crypto';
15
+
16
+ const ENTERPRISE_GRAPH_SCHEMA = 'agentic-security/enterprise-graph';
17
+ export const INTERNET = 'internet';
18
+ const MAX_PATHS = 200;
19
+ export const MAX_DEPTH = 8;
20
+ const AUTH_CWES = new Set(['CWE-287', 'CWE-306', 'CWE-285', 'CWE-862', 'CWE-863', 'CWE-384', 'CWE-613', 'CWE-639', 'CWE-290', 'CWE-294', 'CWE-308', 'CWE-1390']);
21
+
22
+ const edgeId = (type, from, to) => `edge:${type}:${from}:${to}`;
23
+ const hash = (v) => createHash('sha256').update(JSON.stringify(v)).digest('hex');
24
+
25
+ export function buildEnterpriseGraph({ manifest, dependencyGraph, discovered = null }) {
26
+ const nodes = new Map();
27
+ const edges = new Map();
28
+ const addNode = (n) => nodes.set(n.id, n);
29
+ const addEdge = (type, from, to, evidence, reach) => {
30
+ const id = edgeId(type, from, to);
31
+ const e = edges.get(id) ?? { id, type, from, to, reach, evidence: [] };
32
+ e.evidence.push(evidence);
33
+ edges.set(id, e);
34
+ };
35
+ addNode({ id: INTERNET, type: 'internet', kind: 'internet', internetFacing: false, environment: null, dataClasses: [], trustZone: 'internet', required: false });
36
+ for (const r of manifest.repositories) addNode({ id: r.id, type: 'repository', kind: r.kind, internetFacing: r.internetFacing, environment: r.environment, dataClasses: r.dataClasses, trustZone: r.trustZone, required: r.required });
37
+ for (const r of manifest.resources ?? []) addNode({ id: r.id, type: r.kind, kind: r.kind, internetFacing: r.internetFacing, environment: r.environment, dataClasses: r.dataClasses, trustZone: r.trustZone, required: false });
38
+
39
+ const ev = (repo, field, value) => ({ source: 'fleet-manifest', repo, field, value });
40
+ for (const n of [...nodes.values()]) if (n.internetFacing) addEdge('exposes', INTERNET, n.id, ev(n.id, 'internetFacing', true), true);
41
+ for (const r of manifest.repositories) {
42
+ for (const t of r.calls) addEdge('calls', r.id, t, ev(r.id, 'calls', t), true);
43
+ for (const t of r.produces) addEdge('produces', r.id, t, ev(r.id, 'produces', t), true);
44
+ for (const t of r.consumes) addEdge('consumes', t, r.id, ev(r.id, 'consumes', t), true);
45
+ for (const t of r.reads) addEdge('reads', r.id, t, ev(r.id, 'reads', t), true);
46
+ for (const t of r.writes) addEdge('writes', r.id, t, ev(r.id, 'writes', t), true);
47
+ for (const t of r.providers) addEdge('uses-provider', r.id, t, ev(r.id, 'providers', t), true);
48
+ }
49
+ for (const d of dependencyGraph?.edges ?? []) addEdge('depends-on', d.from, d.to, d.evidence?.[0] ?? { source: 'package-manifest', repo: d.from, file: null, declared: d.via, spec: null }, false);
50
+ // Edges found in the code. They share an id with a declared edge of the same type and endpoints, so the evidence list shows both; an
51
+ // edge with ONLY discovered evidence is marked `discovered: true` so a query can say it rests on a code literal, not on a declaration.
52
+ for (const d of discovered?.edges ?? []) for (const evidence of d.evidence) addEdge(d.type, d.from, d.to, evidence, true);
53
+ for (const e of edges.values()) e.discovered = e.evidence.every((x) => x.source === 'discovered-code-literal');
54
+ return { schema: ENTERPRISE_GRAPH_SCHEMA, version: 1, organization: manifest.organization, nodes: [...nodes.values()], edges: [...edges.values()], digest: null };
55
+ }
56
+
57
+ export function finalizeGraph(graph) { return { ...graph, digest: hash({ n: graph.nodes, e: graph.edges }) }; }
58
+
59
+ /** Every edge has evidence, every endpoint exists. Returns a list of problems. */
60
+ export function validateEnterpriseGraph(g) {
61
+ const errs = [];
62
+ const ids = new Set((g?.nodes ?? []).map((n) => n.id));
63
+ for (const e of g?.edges ?? []) {
64
+ if (!Array.isArray(e.evidence) || !e.evidence.length || !e.evidence.every((x) => x && typeof x.source === 'string')) errs.push(`edge ${e.id} has no source evidence`);
65
+ if (!ids.has(e.from) || !ids.has(e.to)) errs.push(`edge ${e.id} points at a node that is not in the graph`);
66
+ }
67
+ return errs;
68
+ }
69
+
70
+ const nodeOf = (g, id) => g.nodes.find((n) => n.id === id);
71
+
72
+ /** Repositories that match a "vulnerable code" selector, and everything that depends on them. */
73
+ export function vulnerableSet({ graph, findingsByRepo = new Map(), selector }) {
74
+ const direct = new Set(); const why = new Map();
75
+ if (selector.repo) { direct.add(selector.repo); why.set(selector.repo, 'named'); }
76
+ for (const [repo, findings] of findingsByRepo) {
77
+ const hit = (findings ?? []).find((f) => (selector.family === 'auth' ? (AUTH_CWES.has(String(f.cwe)) || /auth/i.test(String(f.family ?? ''))) : selector.family ? new RegExp(selector.family, 'i').test(String(f.family ?? f.vuln ?? '')) : false) || (selector.cwe && selector.cwe.includes(String(f.cwe))));
78
+ if (hit) { direct.add(repo); why.set(repo, `finding ${hit.id ?? hit.stableId ?? ''} ${hit.cwe ?? ''}`.trim()); }
79
+ }
80
+ const incoming = new Map();
81
+ for (const e of graph.edges) if (e.type === 'depends-on') { if (!incoming.has(e.to)) incoming.set(e.to, []); incoming.get(e.to).push(e.from); }
82
+ const all = new Set(direct); const stack = [...direct];
83
+ while (stack.length) { const c = stack.pop(); for (const f of incoming.get(c) ?? []) if (!all.has(f)) { all.add(f); why.set(f, `depends on ${c}`); stack.push(f); } }
84
+ return { direct, all, why };
85
+ }
86
+
87
+ /**
88
+ * Attack paths: simple paths from the Internet to any node holding `toDataClass`, along reach edges only. A path is "involving"
89
+ * when it passes through a repository in `involving`. Bounded in depth and count; `truncated` says when a bound was hit.
90
+ */
91
+ export function attackPaths({ graph, toDataClass, from = INTERNET, involving = null, maxDepth = MAX_DEPTH, maxPaths = MAX_PATHS }) {
92
+ const out = new Map();
93
+ for (const e of graph.edges) if (e.reach) { if (!out.has(e.from)) out.set(e.from, []); out.get(e.from).push(e); }
94
+ const results = []; let truncated = false;
95
+ const targets = new Set(graph.nodes.filter((n) => n.dataClasses.includes(toDataClass)).map((n) => n.id));
96
+ const walk = (id, nodePath, edgePath) => {
97
+ if (results.length >= maxPaths) { truncated = true; return; }
98
+ if (targets.has(id) && nodePath.length > 1) {
99
+ const inv = involving ? nodePath.filter((n) => involving.all.has(n)) : [];
100
+ if (!involving || inv.length) results.push({ nodes: [...nodePath], edges: edgePath.map((e) => e.id), involving: inv, discoveredEdges: edgePath.filter((e) => e.discovered).map((e) => e.id), evidence: edgePath.flatMap((e) => e.evidence.map((x) => ({ edge: e.id, ...x }))) });
101
+ }
102
+ if (nodePath.length > maxDepth) { truncated = true; return; }
103
+ for (const e of out.get(id) ?? []) if (!nodePath.includes(e.to)) walk(e.to, [...nodePath, e.to], [...edgePath, e]);
104
+ };
105
+ if (nodeOf(graph, from)) walk(from, [from], []);
106
+ return { dataClass: toDataClass, targets: [...targets], paths: results, truncated, bounds: { maxDepth, maxPaths } };
107
+ }
108
+
109
+ /** The shared-component report: who depends on `component` (transitively), and how many of them are exposed, in production, or hold regulated data. */
110
+ export function componentBlastRadius({ graph, component, version = null }) {
111
+ if (!nodeOf(graph, component)) return null;
112
+ const incoming = new Map();
113
+ for (const e of graph.edges) if (e.type === 'depends-on') { if (!incoming.has(e.to)) incoming.set(e.to, []); incoming.get(e.to).push(e); }
114
+ const found = new Map(); const queue = [{ id: component, path: [component], vm: 'not-asked' }];
115
+ while (queue.length) {
116
+ const { id, path, vm: parentVm } = queue.shift();
117
+ for (const e of incoming.get(id) ?? []) {
118
+ if (found.has(e.from)) continue;
119
+ const spec = e.evidence.map((x) => x.spec).find((x) => x != null) ?? null;
120
+ // only the dependents that pin the component itself can be judged against a version; one reached THROUGH another repository
121
+ // inherits that repository's status, and is not reached at all when the repository in between was ruled out
122
+ const direct = id === component;
123
+ const vm = direct ? (version == null ? 'not-asked' : spec == null ? 'unknown' : String(spec).includes(version) ? 'matched' : 'mismatch') : parentVm;
124
+ found.set(e.from, { id: e.from, depth: path.length, path: [...path, e.from], spec, versionMatch: vm, inherited: !direct, evidence: e.evidence });
125
+ if (vm !== 'mismatch') queue.push({ id: e.from, path: [...path, e.from], vm });
126
+ }
127
+ }
128
+ const all = [...found.values()].sort((a, b) => a.depth - b.depth || a.id.localeCompare(b.id));
129
+ const counted = all.filter((d) => d.versionMatch !== 'mismatch');
130
+ const attrs = (d) => nodeOf(graph, d.id);
131
+ const classes = {};
132
+ for (const d of counted) for (const c of attrs(d).dataClasses) classes[c] = (classes[c] ?? 0) + 1;
133
+ return {
134
+ component, version, dependents: all,
135
+ counts: { repositories: counted.length, internetFacing: counted.filter((d) => attrs(d).internetFacing).length, production: counted.filter((d) => attrs(d).environment === 'production').length, byDataClass: classes },
136
+ excludedByVersion: all.filter((d) => d.versionMatch === 'mismatch').map((d) => d.id),
137
+ unknownVersion: counted.filter((d) => d.versionMatch === 'unknown').map((d) => d.id),
138
+ };
139
+ }
140
+
141
+ export function renderBlastRadius(r) {
142
+ const n = (k, one, many) => `${k} ${k === 1 ? one : many}`;
143
+ const L = [`${r.component}${r.version ? `@${r.version}` : ''}`, '', 'Used by:', n(r.counts.repositories, 'repository', 'repositories'), n(r.counts.internetFacing, 'Internet-facing service', 'Internet-facing services'), n(r.counts.production, 'production app', 'production apps')];
144
+ for (const [c, k] of Object.entries(r.counts.byDataClass).sort()) L.push(n(k, `${c.toUpperCase()}-scoped service`, `${c.toUpperCase()}-scoped services`));
145
+ if (r.excludedByVersion.length) L.push('', `Excluded, their pinned version does not match: ${r.excludedByVersion.join(', ')}`);
146
+ if (r.unknownVersion.length) L.push('', `Counted, but their pinned version could not be read: ${r.unknownVersion.join(', ')}`);
147
+ return L.join('\n');
148
+ }
@@ -27,8 +27,10 @@ function readBounded(p, maxBytes = 1024 * 1024) {
27
27
  function parsePackageJson(text) {
28
28
  let j;
29
29
  try { j = JSON.parse(text); } catch { return { name: null, deps: new Set() }; }
30
+ const specs = new Map();
31
+ for (const block of [j.peerDependencies, j.devDependencies, j.dependencies]) for (const [k, v] of Object.entries(block || {})) if (typeof v === 'string') specs.set(k, v.slice(0, 200));
30
32
  const deps = new Set([...Object.keys(j.dependencies || {}), ...Object.keys(j.devDependencies || {}), ...Object.keys(j.peerDependencies || {})]);
31
- return { name: typeof j.name === 'string' ? j.name : null, deps };
33
+ return { name: typeof j.name === 'string' ? j.name : null, deps, specs, version: typeof j.version === 'string' ? j.version : null };
32
34
  }
33
35
 
34
36
  function parseGoMod(text) {
@@ -85,15 +87,18 @@ export function readRepoManifest(repoPath) {
85
87
  let name = null;
86
88
  const deps = new Set();
87
89
  const ecosystems = new Set();
90
+ const specs = new Map();
91
+ const files = new Map();
88
92
  for (const { file, ecosystem } of MANIFESTS) {
89
93
  const text = readBounded(path.join(repoPath, file));
90
94
  if (text == null) continue;
91
95
  const parsed = PARSERS[file](text);
92
96
  if (parsed.name && !name) name = parsed.name;
93
- for (const d of parsed.deps) deps.add(d);
97
+ for (const d of parsed.deps) { deps.add(d); if (!files.has(d)) files.set(d, file); }
98
+ for (const [d, spec] of parsed.specs ?? []) specs.set(d, spec);
94
99
  ecosystems.add(ecosystem);
95
100
  }
96
- return { name, deps, ecosystems: [...ecosystems] };
101
+ return { name, deps, ecosystems: [...ecosystems], specs, files };
97
102
  }
98
103
 
99
104
  /**
@@ -116,7 +121,7 @@ export function buildFleetDependencyGraph(repos) {
116
121
  for (const dep of r.manifest.deps) {
117
122
  const targets = byName.get(dep);
118
123
  if (!targets) continue;
119
- for (const to of targets) if (to !== id) edges.push({ from: id, to, via: dep });
124
+ for (const to of targets) if (to !== id) edges.push({ from: id, to, via: dep, evidence: [{ source: 'package-manifest', repo: id, file: r.manifest.files.get(dep) ?? null, declared: dep, spec: r.manifest.specs.get(dep) ?? null }] });
120
125
  }
121
126
  }
122
127
  const nodes = [...manifests.values()].map((r) => ({ id: r.id, path: r.path, declaredName: r.manifest.name, ecosystems: r.manifest.ecosystems }));
@@ -14,7 +14,13 @@ import fs from 'node:fs';
14
14
  import path from 'node:path';
15
15
  import { load as loadYaml } from '../../util/yaml.js';
16
16
 
17
- const FLEET_MANIFEST_SCHEMA_KEYS = Object.freeze(['organization', 'repositories', 'workers', 'perRepoTimeout']);
17
+ const FLEET_MANIFEST_SCHEMA_KEYS = Object.freeze(['organization', 'repositories', 'workers', 'perRepoTimeout', 'retries', 'resources']);
18
+ const REPO_KEYS = Object.freeze(['id', 'path', 'required', 'kind', 'internetFacing', 'environment', 'dataClasses', 'trustZone', 'calls', 'produces', 'consumes', 'reads', 'writes', 'providers', 'aliases']);
19
+ const RESOURCE_KEYS = Object.freeze(['id', 'kind', 'dataClasses', 'trustZone', 'internetFacing', 'environment', 'aliases']);
20
+ const REPO_KINDS = Object.freeze(['service', 'library', 'frontend', 'worker', 'gateway']);
21
+ const RESOURCE_KINDS = Object.freeze(['queue', 'database', 'external-provider', 'gateway']);
22
+ export const ENVIRONMENTS = Object.freeze(['production', 'staging', 'development']);
23
+ const NAME_RE = /^[a-z][a-z0-9-]{0,63}$/;
18
24
  const DURATION_RE = /^(\d+)(s|m|h)$/;
19
25
 
20
26
  export function parseDuration(s, fallbackMs) {
@@ -34,9 +40,13 @@ export function validateFleetManifest(doc, { baseDir = '.' } = {}) {
34
40
  if (!Array.isArray(doc.repositories) || doc.repositories.length === 0) errors.push('repositories must be a non-empty array');
35
41
  const seen = new Set();
36
42
  const repos = [];
43
+ const idList = (v, where) => { if (v === undefined) return []; if (!Array.isArray(v) || !v.every((x) => typeof x === 'string' && NAME_RE.test(x))) { errors.push(`${where} must be a list of ids`); return []; } return v; };
44
+ const aliasList = (v, where) => { if (v === undefined) return []; if (!Array.isArray(v) || v.length > 16 || !v.every((x) => typeof x === 'string' && /^[a-z0-9][a-z0-9._-]{0,99}$/.test(x))) { errors.push(`${where} must be a list of up to 16 lower-case names or hostnames`); return []; } return v; };
45
+ const classes = (v, where) => { if (v === undefined) return []; if (!Array.isArray(v) || !v.every((x) => typeof x === 'string' && /^[a-z][a-z0-9-]{0,30}$/.test(x))) { errors.push(`${where} must be a list of data class names such as pci or pii`); return []; } return v; };
37
46
  for (const [i, r] of (Array.isArray(doc.repositories) ? doc.repositories : []).entries()) {
38
47
  if (!r || typeof r !== 'object') { errors.push(`repositories[${i}] must be a mapping`); continue; }
39
- if (typeof r.id !== 'string' || !/^[a-z][a-z0-9-]{0,63}$/.test(r.id)) { errors.push(`repositories[${i}].id must be a lowercase kebab-case string`); continue; }
48
+ for (const k of Object.keys(r)) if (!REPO_KEYS.includes(k)) errors.push(`repositories[${i}]: unknown key ${k}`);
49
+ if (typeof r.id !== 'string' || !NAME_RE.test(r.id)) { errors.push(`repositories[${i}].id must be a lowercase kebab-case string`); continue; }
40
50
  if (seen.has(r.id)) { errors.push(`duplicate repository id: ${r.id}`); continue; }
41
51
  seen.add(r.id);
42
52
  if (typeof r.path !== 'string' || !r.path) { errors.push(`repositories[${i}].path is required`); continue; }
@@ -44,16 +54,51 @@ export function validateFleetManifest(doc, { baseDir = '.' } = {}) {
44
54
  let isDir = false;
45
55
  try { isDir = fs.statSync(abs).isDirectory(); } catch { /* reported below */ }
46
56
  if (!isDir) errors.push(`repositories[${i}].path does not resolve to a directory: ${r.path}`);
47
- repos.push({ id: r.id, path: abs });
57
+ if (r.kind !== undefined && !REPO_KINDS.includes(r.kind)) errors.push(`repositories[${i}].kind must be one of ${REPO_KINDS.join(', ')}`);
58
+ if (r.environment !== undefined && !ENVIRONMENTS.includes(r.environment)) errors.push(`repositories[${i}].environment must be one of ${ENVIRONMENTS.join(', ')}`);
59
+ for (const k of ['required', 'internetFacing']) if (r[k] !== undefined && typeof r[k] !== 'boolean') errors.push(`repositories[${i}].${k} must be true or false`);
60
+ if (r.trustZone !== undefined && !(typeof r.trustZone === 'string' && NAME_RE.test(r.trustZone))) errors.push(`repositories[${i}].trustZone must be a name`);
61
+ repos.push({
62
+ id: r.id, path: abs, required: r.required !== false, kind: r.kind ?? null, internetFacing: r.internetFacing === true, environment: r.environment ?? null,
63
+ dataClasses: classes(r.dataClasses, `repositories[${i}].dataClasses`), trustZone: r.trustZone ?? null,
64
+ aliases: aliasList(r.aliases, `repositories[${i}].aliases`), calls: idList(r.calls, `repositories[${i}].calls`), produces: idList(r.produces, `repositories[${i}].produces`), consumes: idList(r.consumes, `repositories[${i}].consumes`),
65
+ reads: idList(r.reads, `repositories[${i}].reads`), writes: idList(r.writes, `repositories[${i}].writes`), providers: idList(r.providers, `repositories[${i}].providers`),
66
+ });
67
+ }
68
+ const resources = [];
69
+ const resIds = new Set();
70
+ if (doc.resources !== undefined && !Array.isArray(doc.resources)) errors.push('resources must be a list');
71
+ for (const [i, r] of (Array.isArray(doc.resources) ? doc.resources : []).entries()) {
72
+ if (!r || typeof r !== 'object') { errors.push(`resources[${i}] must be a mapping`); continue; }
73
+ for (const k of Object.keys(r)) if (!RESOURCE_KEYS.includes(k)) errors.push(`resources[${i}]: unknown key ${k}`);
74
+ if (typeof r.id !== 'string' || !NAME_RE.test(r.id)) { errors.push(`resources[${i}].id must be a lowercase kebab-case string`); continue; }
75
+ if (seen.has(r.id) || resIds.has(r.id)) { errors.push(`resource id ${r.id} collides with another repository or resource`); continue; }
76
+ resIds.add(r.id);
77
+ if (!RESOURCE_KINDS.includes(r.kind)) errors.push(`resources[${i}].kind must be one of ${RESOURCE_KINDS.join(', ')}`);
78
+ if (r.environment !== undefined && !ENVIRONMENTS.includes(r.environment)) errors.push(`resources[${i}].environment must be one of ${ENVIRONMENTS.join(', ')}`);
79
+ if (r.internetFacing !== undefined && typeof r.internetFacing !== 'boolean') errors.push(`resources[${i}].internetFacing must be true or false`);
80
+ resources.push({ id: r.id, kind: r.kind, aliases: aliasList(r.aliases, `resources[${i}].aliases`), dataClasses: classes(r.dataClasses, `resources[${i}].dataClasses`), trustZone: r.trustZone ?? null, internetFacing: r.internetFacing === true, environment: r.environment ?? null });
81
+ }
82
+ const known = new Set([...seen, ...resIds]);
83
+ for (const r of repos) {
84
+ for (const [field, kinds] of [['calls', ['repo', 'gateway']], ['produces', ['queue']], ['consumes', ['queue']], ['reads', ['database']], ['writes', ['database']], ['providers', ['external-provider']]]) {
85
+ for (const target of r[field]) {
86
+ if (!known.has(target)) { errors.push(`repository ${r.id}: ${field} names ${target}, which is neither a repository nor a resource`); continue; }
87
+ const res = resources.find((x) => x.id === target);
88
+ const ok = res ? kinds.includes(res.kind) : kinds.includes('repo');
89
+ if (!ok) errors.push(`repository ${r.id}: ${field} cannot point at ${target}${res ? ` (a ${res.kind})` : ' (a repository)'}`);
90
+ }
91
+ }
48
92
  }
49
93
  if (doc.workers != null && !(Number.isInteger(doc.workers) && doc.workers >= 1 && doc.workers <= 32)) errors.push('workers must be an integer in 1..32');
94
+ if (doc.retries != null && !(Number.isInteger(doc.retries) && doc.retries >= 0 && doc.retries <= 5)) errors.push('retries must be an integer in 0..5');
50
95
  if (doc.perRepoTimeout != null && parseDuration(doc.perRepoTimeout, null) == null) errors.push(`perRepoTimeout is not a valid duration: ${doc.perRepoTimeout}`);
51
96
  if (errors.length) return { ok: false, errors };
52
97
  return {
53
98
  ok: true, errors: [],
54
99
  manifest: {
55
- organization: doc.organization, repositories: repos,
56
- workers: doc.workers ?? 4, perRepoTimeoutMs: parseDuration(doc.perRepoTimeout, 20 * 60000),
100
+ organization: doc.organization, repositories: repos, resources,
101
+ workers: doc.workers ?? 4, retries: doc.retries ?? 1, perRepoTimeoutMs: parseDuration(doc.perRepoTimeout, 20 * 60000),
57
102
  },
58
103
  };
59
104
  }
@@ -0,0 +1,153 @@
1
+ // The fleet scan controller (PRD 4, requirements 2 to 5). It turns a manifest into jobs in the job store and runs them with bounded
2
+ // workers, each scan in its own child process under a hard timeout, so one hung repository costs one worker for one timeout and
3
+ // nothing else. Every state change is in the store, which is what makes `fleet resume` an ordinary continuation.
4
+ //
5
+ // plan fingerprint each repository, derive its cache key, enqueue one idempotent job per repository
6
+ // run claim -> (cache hit | scan in a supervised child, heartbeating) -> complete | fail-with-retry
7
+ // recover an expired lease, or a lease held by a dead process on this host, goes back in the queue
8
+ //
9
+ // A worker that loses its lease (it stopped heartbeating long enough for another worker to take the job) is fenced out: its late
10
+ // result is refused by the store and its scan is aborted. Results are written per attempt (`<repo>.<epoch>.json`) and the job row
11
+ // points at the winning one, so a retry can never produce two sets of findings for one repository.
12
+ import fs from 'node:fs';
13
+ import os from 'node:os';
14
+ import path from 'node:path';
15
+ import { createHash } from 'node:crypto';
16
+ import { fileURLToPath } from 'node:url';
17
+ import { superviseSpawn } from '../../sandbox/supervise.js';
18
+ import { statePath } from '../state-dir.js';
19
+ import { repoFingerprint, scannerIdentity, cacheKey, readCache, writeCache } from './scan-cache.js';
20
+ import { BEGIN, END } from './scan-child.js';
21
+
22
+ const SCAN_CHILD = fileURLToPath(new URL('./scan-child.js', import.meta.url));
23
+ const sha = (s) => createHash('sha256').update(s).digest('hex');
24
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
25
+ export const DEFAULTS = Object.freeze({ leaseMs: 30_000, heartbeatMs: 10_000, retryDelayMs: 2_000, pollMs: 100 });
26
+
27
+ /** The default scan: one supervised child process per repository. Throws `{code: 'TIMEOUT'|'ERROR', message}`. */
28
+ export async function childScan(repoPath, { timeoutMs, signal }) {
29
+ const r = await superviseSpawn(process.execPath, [SCAN_CHILD, repoPath], { cwd: repoPath, env: { ...process.env, AGENTIC_SECURITY_NO_STATE: '1' }, timeoutMs, signal, maxOutputBytes: 96 * 1024 * 1024 });
30
+ const fail = (code, message) => Object.assign(new Error(message), { code });
31
+ if (r.timedOut) throw fail('TIMEOUT', `no result within ${Math.round(timeoutMs / 1000)}s`);
32
+ if (r.cancelled) throw fail('CANCELLED', 'cancelled');
33
+ if (r.spawnError) throw fail('ERROR', `could not start the scan: ${r.spawnError}`);
34
+ if (r.outputCapped) throw fail('ERROR', 'the scan produced more output than allowed');
35
+ if (r.exitCode !== 0) throw fail('ERROR', `scan exited ${r.exitCode}: ${String(r.stderr).trim().split('\n').pop()?.slice(0, 200) ?? ''}`);
36
+ const a = r.stdout.indexOf(BEGIN); const b = r.stdout.indexOf(END);
37
+ if (a < 0 || b < a) throw fail('ERROR', 'the scan produced no result block');
38
+ try { return JSON.parse(r.stdout.slice(a + BEGIN.length, b)); } catch (e) { throw fail('ERROR', `unreadable scan result: ${e.message}`); }
39
+ }
40
+
41
+ const hostId = () => os.hostname();
42
+ const pidAlive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } };
43
+
44
+ /** Free jobs held by workers on THIS host whose process no longer exists. Other hosts are handled by lease expiry alone. */
45
+ export function reclaimDeadWorkers(store, runId, { host = hostId(), alive = pidAlive, self = null } = {}) {
46
+ let freed = 0;
47
+ const seen = new Set();
48
+ for (const j of store.listJobs(runId)) {
49
+ if (j.status !== 'leased' || !j.worker || seen.has(j.worker) || j.worker === self) continue;
50
+ const m = /^(.*):(\d+):\d+$/.exec(j.worker);
51
+ if (!m || m[1] !== host) continue;
52
+ seen.add(j.worker);
53
+ if (!alive(Number(m[2]))) freed += store.reclaimWorker(runId, j.worker);
54
+ }
55
+ return freed;
56
+ }
57
+
58
+ export function planRun({ manifest, manifestHash, store, runId = null, config = {}, now = Date.now(), identity = scannerIdentity(), fingerprint = repoFingerprint, retries = manifest.retries, meta = {} }) {
59
+ const scanConfig = { provenance: false, ...config };
60
+ const prints = manifest.repositories.map((r) => ({ r, fp: fingerprint(r.path) }));
61
+ const configHash = sha(JSON.stringify(scanConfig));
62
+ const id = runId ?? `frun-${sha(JSON.stringify({ manifestHash, configHash, identity, t: now, prints: prints.map((p) => [p.r.id, p.fp.content]) })).slice(0, 16)}`;
63
+ store.begin({ runId: id, organization: manifest.organization, manifestHash, configHash, scannerIdentity: identity, createdAt: now, meta: { workers: manifest.workers, perRepoTimeoutMs: manifest.perRepoTimeoutMs, retries, ...meta } });
64
+ const created = store.enqueue(id, prints.map(({ r, fp }) => ({ jobId: `${id}:${r.id}`, repoId: r.id, commit: fp.commit, cacheKey: cacheKey({ fingerprint: fp, config: scanConfig, identity }), timeoutMs: manifest.perRepoTimeoutMs, maxAttempts: retries + 1, required: r.required })), { now });
65
+ return { runId: id, created, identity };
66
+ }
67
+
68
+ /**
69
+ * Run (or continue) a planned run until no job is queued or leased by a live worker. Returns when the work is done, the signal
70
+ * aborts, or `maxWaitMs` passes with only other workers' leases outstanding.
71
+ */
72
+ export async function runJobs({ manifest, store, runId, root, scan = childScan, workers = manifest.workers, leaseMs = DEFAULTS.leaseMs, heartbeatMs = DEFAULTS.heartbeatMs, retryDelayMs = DEFAULTS.retryDelayMs, pollMs = DEFAULTS.pollMs, maxWaitMs = null, signal, now = () => Date.now(), onEvent = () => {}, cacheDir = null, artifactDir = null, useCache = true }) {
73
+ const repoById = new Map(manifest.repositories.map((r) => [r.id, r]));
74
+ const cacheD = cacheDir ?? statePath(root, 'fleet', 'cache');
75
+ const artD = artifactDir ?? statePath(root, 'fleet', 'artifacts', runId);
76
+ fs.mkdirSync(artD, { recursive: true, mode: 0o700 });
77
+ const deadline = maxWaitMs ?? (manifest.perRepoTimeoutMs * 3 + leaseMs);
78
+ const writeArtifact = (job, payload) => {
79
+ const text = JSON.stringify({ schema: 'agentic-security/fleet-scan-artifact', version: 1, runId, repoId: job.repoId, epoch: job.epoch, commit: job.commit, ...payload });
80
+ const file = path.join(artD, `${job.repoId}.${job.epoch}.json`);
81
+ fs.writeFileSync(file, text, { mode: 0o600 });
82
+ return { file, sha256: sha(text) };
83
+ };
84
+
85
+ async function worker(slot) {
86
+ const id = `${hostId()}:${process.pid}:${slot}`;
87
+ let idleSince = null;
88
+ for (;;) {
89
+ if (signal?.aborted) return;
90
+ store.reclaimExpired(runId, { now: now() });
91
+ const job = store.claim(runId, { worker: id, leaseMs, now: now() });
92
+ if (!job) {
93
+ const open = store.listJobs(runId).filter((j) => j.status === 'queued' || j.status === 'leased');
94
+ if (!open.length) return;
95
+ idleSince ??= now();
96
+ if (now() - idleSince > deadline) return;
97
+ await sleep(pollMs);
98
+ continue;
99
+ }
100
+ idleSince = null;
101
+ const repo = repoById.get(job.repoId);
102
+ onEvent({ type: 'start', repo: job.repoId, attempt: job.attempts });
103
+ const cached = useCache ? readCache(cacheD, job.cacheKey) : null;
104
+ if (cached) {
105
+ const a = writeArtifact(job, { entry: cached.entry, findings: cached.findings, findingsTruncated: !!cached.findingsTruncated, cached: true });
106
+ const status = cached.entry?.scanHealth && cached.entry.scanHealth.status !== 'complete' ? 'partial' : 'succeeded';
107
+ if (store.complete(job.jobId, job.epoch, { status, artifactSha256: a.sha256, summary: { total: cached.entry?.total ?? null, cached: true, artifact: path.basename(a.file) }, now: now() })) onEvent({ type: 'done', repo: job.repoId, cached: true, status });
108
+ continue;
109
+ }
110
+ const ac = new AbortController();
111
+ const onAbort = () => ac.abort();
112
+ signal?.addEventListener('abort', onAbort, { once: true });
113
+ let lost = false;
114
+ const hb = setInterval(() => { if (!store.heartbeat(job.jobId, job.epoch, { now: now(), leaseMs })) { lost = true; ac.abort(); } }, heartbeatMs);
115
+ try {
116
+ const out = await scan(repo.path, { timeoutMs: job.timeoutMs, signal: ac.signal });
117
+ clearInterval(hb);
118
+ if (lost) { onEvent({ type: 'fenced', repo: job.repoId }); continue; }
119
+ if (!out?.entry) throw Object.assign(new Error('the scan returned no entry'), { code: 'ERROR' });
120
+ if (out.entry.ok === false) throw Object.assign(new Error(out.entry.error || 'the scan failed'), { code: 'ERROR' });
121
+ const a = writeArtifact(job, { entry: out.entry, findings: out.findings ?? [], findingsTruncated: !!out.findingsTruncated });
122
+ const status = out.entry.scanHealth && out.entry.scanHealth.status !== 'complete' ? 'partial' : 'succeeded';
123
+ if (store.complete(job.jobId, job.epoch, { status, artifactSha256: a.sha256, summary: { total: out.entry.total ?? null, cached: false, artifact: path.basename(a.file) }, now: now() })) {
124
+ if (useCache && status === 'succeeded') writeCache(cacheD, job.cacheKey, { entry: out.entry, findings: out.findings ?? [], findingsTruncated: !!out.findingsTruncated });
125
+ onEvent({ type: 'done', repo: job.repoId, cached: false, status });
126
+ } else onEvent({ type: 'fenced', repo: job.repoId });
127
+ } catch (e) {
128
+ clearInterval(hb);
129
+ if (lost || e?.code === 'CANCELLED') { if (e?.code === 'CANCELLED' && !lost) store.fail(job.jobId, job.epoch, { error: 'cancelled', now: now(), retryDelayMs: 0 }); continue; }
130
+ const r = store.fail(job.jobId, job.epoch, { error: e?.message ?? String(e), timedOut: e?.code === 'TIMEOUT', now: now(), retryDelayMs });
131
+ onEvent({ type: 'failed', repo: job.repoId, error: e?.message ?? String(e), timedOut: e?.code === 'TIMEOUT', next: r?.state ?? null });
132
+ } finally { clearInterval(hb); signal?.removeEventListener('abort', onAbort); }
133
+ }
134
+ }
135
+ await Promise.all(Array.from({ length: Math.max(1, Math.min(workers, 32)) }, (_, i) => worker(i)));
136
+ return { aborted: !!signal?.aborted };
137
+ }
138
+
139
+ /** Load the artifacts the jobs point at, verifying each against the hash the store recorded. A missing or altered artifact is reported, not used. */
140
+ export function loadArtifacts({ store, runId, root, artifactDir = null }) {
141
+ const dir = artifactDir ?? statePath(root, 'fleet', 'artifacts', runId);
142
+ const out = new Map(); const problems = [];
143
+ for (const j of store.listJobs(runId)) {
144
+ if (!['succeeded', 'partial'].includes(j.status)) continue;
145
+ const file = path.join(dir, j.summary?.artifact ?? `${j.repoId}.${j.epoch}.json`);
146
+ try {
147
+ const text = fs.readFileSync(file, 'utf8');
148
+ if (sha(text) !== j.artifactSha256) { problems.push({ repoId: j.repoId, problem: 'artifact does not match the recorded hash' }); continue; }
149
+ out.set(j.repoId, JSON.parse(text));
150
+ } catch { problems.push({ repoId: j.repoId, problem: 'artifact is missing or unreadable' }); }
151
+ }
152
+ return { artifacts: out, problems };
153
+ }
@@ -0,0 +1,136 @@
1
+ // The fleet job store (PRD 4, requirements 2 and 3): jobs with a lease, a heartbeat, a retry count, a timeout and a result hash,
2
+ // behind one small interface so the storage can change without the scheduler noticing.
3
+ //
4
+ // createSqliteJobStore({ file }) local deployment, one file, no server (node:sqlite)
5
+ // createMemoryJobStore() the reference implementation; the conformance tests run the SAME assertions against both,
6
+ // which is what shows the interface does not secretly depend on SQLite. A PostgreSQL or a
7
+ // queue-backed store would be a third implementation of the same methods.
8
+ //
9
+ // Semantics that matter (and are tested on both stores):
10
+ // * IDEMPOTENT: a job's id is derived from its run, repository, commit and configuration, and `enqueue` of an existing id does
11
+ // nothing. Enqueueing a run twice does not make two jobs.
12
+ // * LEASED: `claim` hands one queued job to one worker, increments a fencing `epoch`, and starts a lease. A worker that stops
13
+ // heartbeating loses the job when the lease expires; `reclaimExpired` puts it back in the queue.
14
+ // * FENCED: `heartbeat`, `complete` and `fail` must present the epoch they were given. A worker that lost its lease (and whose
15
+ // job another worker now holds) is refused, so a slow zombie can neither extend a lease it no longer has nor overwrite the
16
+ // result of the worker that replaced it. That is what keeps a retry from duplicating findings.
17
+ // * BOUNDED RETRIES: a failure puts the job back in the queue until `maxAttempts`, then it is `failed` (or `timed_out`),
18
+ // terminal, and visible in the organization's health. A job is never silently dropped.
19
+ import fs from 'node:fs';
20
+ import path from 'node:path';
21
+
22
+ export const JOB_STATUSES = Object.freeze(['queued', 'leased', 'succeeded', 'partial', 'failed', 'timed_out', 'cancelled']);
23
+ const TERMINAL = new Set(['succeeded', 'partial', 'failed', 'timed_out', 'cancelled']);
24
+ export const isTerminal = (s) => TERMINAL.has(s);
25
+
26
+ const jobOf = (r) => (r ? {
27
+ jobId: r.job_id, runId: r.run_id, repoId: r.repo_id, commit: r.commit_sha ?? null, cacheKey: r.cache_key ?? null, status: r.status, worker: r.worker ?? null,
28
+ epoch: r.epoch, leaseExpiresAt: r.lease_expires_at ?? null, heartbeatAt: r.heartbeat_at ?? null, attempts: r.attempts, maxAttempts: r.max_attempts,
29
+ timeoutMs: r.timeout_ms, nextAttemptAt: r.next_attempt_at ?? 0, artifactSha256: r.artifact_sha256 ?? null, summary: r.summary ? JSON.parse(r.summary) : null,
30
+ error: r.error ?? null, createdAt: r.created_at, startedAt: r.started_at ?? null, finishedAt: r.finished_at ?? null, required: !!r.required,
31
+ } : null);
32
+
33
+ // ------------------------------------------------------------------------------------------------------- memory
34
+ export function createMemoryJobStore() {
35
+ const runs = new Map(); const jobs = new Map();
36
+ const all = (runId) => [...jobs.values()].filter((j) => j.runId === runId);
37
+ const store = {
38
+ kind: 'memory',
39
+ begin(run) { if (!runs.has(run.runId)) runs.set(run.runId, { ...run }); return runs.get(run.runId); },
40
+ getRun: (runId) => runs.get(runId) ?? null,
41
+ listRuns: () => [...runs.values()].sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt))),
42
+ enqueue(runId, list, { now = Date.now() } = {}) {
43
+ let created = 0;
44
+ for (const j of list) { if (jobs.has(j.jobId)) continue; jobs.set(j.jobId, { runId, repoId: j.repoId, jobId: j.jobId, commit: j.commit ?? null, cacheKey: j.cacheKey ?? null, status: 'queued', worker: null, epoch: 0, leaseExpiresAt: null, heartbeatAt: null, attempts: 0, maxAttempts: j.maxAttempts ?? 2, timeoutMs: j.timeoutMs, nextAttemptAt: 0, artifactSha256: null, summary: null, error: null, createdAt: now, startedAt: null, finishedAt: null, required: j.required !== false }); created++; }
45
+ return created;
46
+ },
47
+ claim(runId, { worker, leaseMs, now = Date.now() }) {
48
+ const j = all(runId).filter((x) => x.status === 'queued' && x.nextAttemptAt <= now).sort((a, b) => a.createdAt - b.createdAt || a.repoId.localeCompare(b.repoId))[0];
49
+ if (!j) return null;
50
+ Object.assign(j, { status: 'leased', worker, epoch: j.epoch + 1, attempts: j.attempts + 1, leaseExpiresAt: now + leaseMs, heartbeatAt: now, startedAt: now, error: null });
51
+ return { ...j };
52
+ },
53
+ heartbeat(jobId, epoch, { now = Date.now(), leaseMs }) { const j = jobs.get(jobId); if (!j || j.status !== 'leased' || j.epoch !== epoch) return false; j.heartbeatAt = now; j.leaseExpiresAt = now + leaseMs; return true; },
54
+ complete(jobId, epoch, { status = 'succeeded', artifactSha256 = null, summary = null, now = Date.now() }) {
55
+ const j = jobs.get(jobId); if (!j || j.status !== 'leased' || j.epoch !== epoch) return false;
56
+ Object.assign(j, { status, artifactSha256, summary, finishedAt: now, leaseExpiresAt: null, error: null }); return true;
57
+ },
58
+ fail(jobId, epoch, { error, timedOut = false, now = Date.now(), retryDelayMs = 0 }) {
59
+ const j = jobs.get(jobId); if (!j || j.status !== 'leased' || j.epoch !== epoch) return null;
60
+ j.error = String(error ?? '').slice(0, 500); j.leaseExpiresAt = null; j.worker = null;
61
+ if (j.attempts < j.maxAttempts) { j.status = 'queued'; j.nextAttemptAt = now + retryDelayMs; return { state: 'queued', attempts: j.attempts }; }
62
+ j.status = timedOut ? 'timed_out' : 'failed'; j.finishedAt = now; return { state: j.status, attempts: j.attempts };
63
+ },
64
+ reclaimExpired(runId, { now = Date.now() }) { let n = 0; for (const j of all(runId)) if (j.status === 'leased' && j.leaseExpiresAt != null && j.leaseExpiresAt <= now) { j.status = 'queued'; j.worker = null; j.leaseExpiresAt = null; j.nextAttemptAt = 0; j.error = 'lease expired'; n++; } return n; },
65
+ reclaimWorker(runId, worker) { let n = 0; for (const j of all(runId)) if (j.status === 'leased' && j.worker === worker) { j.status = 'queued'; j.worker = null; j.leaseExpiresAt = null; j.nextAttemptAt = 0; j.error = 'worker is gone'; n++; } return n; },
66
+ cancel(runId, { now = Date.now() }) { let n = 0; for (const j of all(runId)) if (!TERMINAL.has(j.status)) { j.status = 'cancelled'; j.finishedAt = now; j.leaseExpiresAt = null; n++; } return n; },
67
+ retime(runId, timeoutMs) { let n = 0; for (const j of all(runId)) if (j.status === 'queued') { j.timeoutMs = timeoutMs; n++; } return n; },
68
+ requeueTerminal(runId, statuses) { let n = 0; for (const j of all(runId)) if (statuses.includes(j.status)) { Object.assign(j, { status: 'queued', worker: null, attempts: 0, nextAttemptAt: 0, leaseExpiresAt: null, finishedAt: null, error: null }); n++; } return n; },
69
+ listJobs: (runId) => all(runId).map((j) => ({ ...j })).sort((a, b) => a.repoId.localeCompare(b.repoId)),
70
+ getJob: (jobId) => (jobs.get(jobId) ? { ...jobs.get(jobId) } : null),
71
+ close() {},
72
+ };
73
+ return store;
74
+ }
75
+
76
+ // ------------------------------------------------------------------------------------------------------- sqlite
77
+ const SCHEMA = `
78
+ CREATE TABLE IF NOT EXISTS fleet_runs (run_id TEXT PRIMARY KEY, organization TEXT NOT NULL, manifest_hash TEXT NOT NULL, config_hash TEXT NOT NULL, scanner_identity TEXT, created_at INTEGER NOT NULL, meta TEXT);
79
+ CREATE TABLE IF NOT EXISTS fleet_jobs (
80
+ job_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES fleet_runs(run_id), repo_id TEXT NOT NULL, commit_sha TEXT, cache_key TEXT,
81
+ status TEXT NOT NULL, worker TEXT, epoch INTEGER NOT NULL DEFAULT 0, lease_expires_at INTEGER, heartbeat_at INTEGER,
82
+ attempts INTEGER NOT NULL DEFAULT 0, max_attempts INTEGER NOT NULL, timeout_ms INTEGER NOT NULL, next_attempt_at INTEGER NOT NULL DEFAULT 0,
83
+ artifact_sha256 TEXT, summary TEXT, error TEXT, created_at INTEGER NOT NULL, started_at INTEGER, finished_at INTEGER, required INTEGER NOT NULL DEFAULT 1
84
+ );
85
+ CREATE INDEX IF NOT EXISTS fleet_jobs_run_status ON fleet_jobs (run_id, status);
86
+ `;
87
+
88
+ export async function createSqliteJobStore({ file }) {
89
+ const { DatabaseSync } = await import('node:sqlite');
90
+ if (file !== ':memory:') fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
91
+ const db = new DatabaseSync(file);
92
+ db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000; PRAGMA foreign_keys = ON;');
93
+ db.exec(SCHEMA);
94
+ const tx = (fn) => { db.exec('BEGIN IMMEDIATE'); try { const r = fn(); db.exec('COMMIT'); return r; } catch (e) { try { db.exec('ROLLBACK'); } catch { /* already rolled back */ } throw e; } };
95
+ const q = (sql, ...p) => db.prepare(sql).all(...p);
96
+ const one = (sql, ...p) => db.prepare(sql).get(...p);
97
+ const run = (sql, ...p) => db.prepare(sql).run(...p);
98
+ return {
99
+ kind: 'sqlite', file,
100
+ begin(r) { run('INSERT OR IGNORE INTO fleet_runs (run_id, organization, manifest_hash, config_hash, scanner_identity, created_at, meta) VALUES (?,?,?,?,?,?,?)', r.runId, r.organization, r.manifestHash, r.configHash, r.scannerIdentity ?? null, r.createdAt, JSON.stringify(r.meta ?? {})); return this.getRun(r.runId); },
101
+ getRun(runId) { const r = one('SELECT * FROM fleet_runs WHERE run_id = ?', runId); return r ? { runId: r.run_id, organization: r.organization, manifestHash: r.manifest_hash, configHash: r.config_hash, scannerIdentity: r.scanner_identity, createdAt: r.created_at, meta: JSON.parse(r.meta ?? '{}') } : null; },
102
+ listRuns() { return q('SELECT * FROM fleet_runs ORDER BY created_at DESC').map((r) => ({ runId: r.run_id, organization: r.organization, manifestHash: r.manifest_hash, createdAt: r.created_at })); },
103
+ enqueue(runId, list, { now = Date.now() } = {}) {
104
+ return tx(() => { let created = 0; for (const j of list) { const r = run('INSERT OR IGNORE INTO fleet_jobs (job_id, run_id, repo_id, commit_sha, cache_key, status, max_attempts, timeout_ms, created_at, required) VALUES (?,?,?,?,?,?,?,?,?,?)', j.jobId, runId, j.repoId, j.commit ?? null, j.cacheKey ?? null, 'queued', j.maxAttempts ?? 2, j.timeoutMs, now, j.required === false ? 0 : 1); created += Number(r.changes); } return created; });
105
+ },
106
+ claim(runId, { worker, leaseMs, now = Date.now() }) {
107
+ return tx(() => {
108
+ const j = one("SELECT job_id FROM fleet_jobs WHERE run_id = ? AND status = 'queued' AND next_attempt_at <= ? ORDER BY created_at, repo_id LIMIT 1", runId, now);
109
+ if (!j) return null;
110
+ run("UPDATE fleet_jobs SET status = 'leased', worker = ?, epoch = epoch + 1, attempts = attempts + 1, lease_expires_at = ?, heartbeat_at = ?, started_at = ?, error = NULL WHERE job_id = ?", worker, now + leaseMs, now, now, j.job_id);
111
+ return jobOf(one('SELECT * FROM fleet_jobs WHERE job_id = ?', j.job_id));
112
+ });
113
+ },
114
+ heartbeat(jobId, epoch, { now = Date.now(), leaseMs }) { return Number(run("UPDATE fleet_jobs SET heartbeat_at = ?, lease_expires_at = ? WHERE job_id = ? AND status = 'leased' AND epoch = ?", now, now + leaseMs, jobId, epoch).changes) === 1; },
115
+ complete(jobId, epoch, { status = 'succeeded', artifactSha256 = null, summary = null, now = Date.now() }) { return Number(run("UPDATE fleet_jobs SET status = ?, artifact_sha256 = ?, summary = ?, finished_at = ?, lease_expires_at = NULL, error = NULL WHERE job_id = ? AND status = 'leased' AND epoch = ?", status, artifactSha256, summary ? JSON.stringify(summary) : null, now, jobId, epoch).changes) === 1; },
116
+ fail(jobId, epoch, { error, timedOut = false, now = Date.now(), retryDelayMs = 0 }) {
117
+ return tx(() => {
118
+ const j = one("SELECT attempts, max_attempts FROM fleet_jobs WHERE job_id = ? AND status = 'leased' AND epoch = ?", jobId, epoch);
119
+ if (!j) return null;
120
+ const msg = String(error ?? '').slice(0, 500);
121
+ if (j.attempts < j.max_attempts) { run("UPDATE fleet_jobs SET status = 'queued', worker = NULL, lease_expires_at = NULL, next_attempt_at = ?, error = ? WHERE job_id = ?", now + retryDelayMs, msg, jobId); return { state: 'queued', attempts: j.attempts }; }
122
+ const st = timedOut ? 'timed_out' : 'failed';
123
+ run('UPDATE fleet_jobs SET status = ?, worker = NULL, lease_expires_at = NULL, finished_at = ?, error = ? WHERE job_id = ?', st, now, msg, jobId);
124
+ return { state: st, attempts: j.attempts };
125
+ });
126
+ },
127
+ reclaimExpired(runId, { now = Date.now() }) { return Number(run("UPDATE fleet_jobs SET status = 'queued', worker = NULL, lease_expires_at = NULL, next_attempt_at = 0, error = 'lease expired' WHERE run_id = ? AND status = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ?", runId, now).changes); },
128
+ reclaimWorker(runId, worker) { return Number(run("UPDATE fleet_jobs SET status = 'queued', worker = NULL, lease_expires_at = NULL, next_attempt_at = 0, error = 'worker is gone' WHERE run_id = ? AND status = 'leased' AND worker = ?", runId, worker).changes); },
129
+ cancel(runId, { now = Date.now() }) { return Number(run("UPDATE fleet_jobs SET status = 'cancelled', finished_at = ?, lease_expires_at = NULL WHERE run_id = ? AND status IN ('queued','leased')", now, runId).changes); },
130
+ retime(runId, timeoutMs) { return Number(run("UPDATE fleet_jobs SET timeout_ms = ? WHERE run_id = ? AND status = 'queued'", timeoutMs, runId).changes); },
131
+ requeueTerminal(runId, statuses) { return Number(run(`UPDATE fleet_jobs SET status = 'queued', worker = NULL, attempts = 0, next_attempt_at = 0, lease_expires_at = NULL, finished_at = NULL, error = NULL WHERE run_id = ? AND status IN (${statuses.map(() => '?').join(',')})`, runId, ...statuses).changes); },
132
+ listJobs: (runId) => q('SELECT * FROM fleet_jobs WHERE run_id = ? ORDER BY repo_id', runId).map(jobOf),
133
+ getJob: (jobId) => jobOf(one('SELECT * FROM fleet_jobs WHERE job_id = ?', jobId)),
134
+ close() { try { db.close(); } catch { /* already closed */ } },
135
+ };
136
+ }