@ngockhoale/ukit 3.0.8 → 3.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +18 -1
  2. package/manifests/documentation.yaml +11 -0
  3. package/package.json +1 -1
  4. package/scripts/audit/decision-coverage.mjs +29 -2
  5. package/scripts/bench/data-foundation.mjs +52 -3
  6. package/scripts/bench/decision-runtime-baseline.mjs +427 -0
  7. package/scripts/bench/decision-runtime-metrics.mjs +67 -0
  8. package/scripts/bench/decision-runtime-variant.mjs +626 -0
  9. package/scripts/bench/memory-ablation.mjs +495 -0
  10. package/scripts/bench/memory-baseline.mjs +596 -0
  11. package/scripts/bench/memory-bench.mjs +661 -0
  12. package/scripts/bench/memory-canary.mjs +321 -0
  13. package/scripts/bench/memory-corpus.mjs +354 -0
  14. package/scripts/bench/memory-gate.mjs +389 -0
  15. package/scripts/bench/memory-metrics.mjs +179 -0
  16. package/scripts/bench/parallel-agents.mjs +33 -11
  17. package/scripts/bench/recorder-overhead.mjs +204 -0
  18. package/scripts/bench/sqlite-spike.mjs +451 -0
  19. package/scripts/measure-decision-gateway.mjs +306 -0
  20. package/scripts/perf/audit-perf.mjs +35 -17
  21. package/src/bug/triageBug.js +4 -3
  22. package/src/cli/commands/memory.js +357 -63
  23. package/src/context/detectProjectContext.js +11 -1
  24. package/src/core/agentRuntime/adapters.js +254 -0
  25. package/src/core/agentRuntime/artifacts.js +192 -0
  26. package/src/core/agentRuntime/completionGate.js +176 -0
  27. package/src/core/agentRuntime/context.js +149 -0
  28. package/src/core/agentRuntime/contract.js +247 -0
  29. package/src/core/agentRuntime/diagnostics.js +244 -0
  30. package/src/core/agentRuntime/evaluation.js +163 -0
  31. package/src/core/agentRuntime/eventStore.js +404 -0
  32. package/src/core/agentRuntime/liveness.js +60 -0
  33. package/src/core/agentRuntime/planCompiler.js +322 -0
  34. package/src/core/agentRuntime/promotion.js +53 -0
  35. package/src/core/agentRuntime/qualityComparison.js +112 -0
  36. package/src/core/agentRuntime/recovery.js +266 -0
  37. package/src/core/agentRuntime/resourcePolicy.js +78 -0
  38. package/src/core/agentRuntime/runtimeSupport.js +237 -0
  39. package/src/core/agentRuntime/supervisor.js +565 -0
  40. package/src/core/agentRuntime/vmEngine.js +621 -0
  41. package/src/core/codeintel/analogy.js +3 -2
  42. package/src/core/experiments/dynamicWorkflow.js +17 -2
  43. package/src/core/fileOps.js +21 -3
  44. package/src/core/memory/deltaOverlays.js +75 -30
  45. package/src/core/memory/learningCandidates.js +93 -48
  46. package/src/core/memory/memoryFlags.js +83 -0
  47. package/src/core/memory/memoryFreshness.js +190 -0
  48. package/src/core/memory/memoryHit.js +144 -0
  49. package/src/core/memory/migrate.js +69 -189
  50. package/src/core/memory/migrateMapping.js +232 -0
  51. package/src/core/memory/mutateMemory.js +323 -0
  52. package/src/core/memory/policy.js +96 -0
  53. package/src/core/memory/projectIdentity.js +266 -0
  54. package/src/core/memory/recordIndex.js +178 -0
  55. package/src/core/memory/recordStore.js +133 -20
  56. package/src/core/memory/records.js +144 -6
  57. package/src/core/memory/retrieval.js +259 -125
  58. package/src/core/memory/store.js +16 -5
  59. package/src/core/memory/storeBackup.js +226 -0
  60. package/src/core/memory/storeV2.js +63 -26
  61. package/src/core/memory/storeV2Loader.js +30 -12
  62. package/src/core/memory/userMemory.js +38 -20
  63. package/src/core/memory/writeClassification.js +161 -0
  64. package/src/core/memory/writeGuard.js +129 -0
  65. package/src/core/observability/adapters/hookTelemetryAdapter.js +90 -0
  66. package/src/core/observability/analytics/cohorts.js +148 -0
  67. package/src/core/observability/analytics/storeDigest.js +163 -0
  68. package/src/core/observability/evaluation/experimentPlan.js +95 -0
  69. package/src/core/observability/evaluation/findings.js +99 -0
  70. package/src/core/observability/evaluation/optimizationKnowledge.js +10 -1
  71. package/src/core/observability/evaluation/perturbation.js +273 -0
  72. package/src/core/observability/evaluation/replay.js +7 -1
  73. package/src/core/observability/evaluation/scorecard.js +23 -3
  74. package/src/core/observability/rollout.js +11 -7
  75. package/src/core/observability/schema/compatibility.js +135 -0
  76. package/src/core/observability/schema/registry.js +99 -0
  77. package/src/core/observability/schema/validate.js +7 -0
  78. package/src/core/observability/support/import.js +53 -9
  79. package/src/core/observability/support/paths.js +13 -3
  80. package/src/core/observability/support/projector.js +148 -12
  81. package/src/core/output/index.js +12 -2
  82. package/src/core/runtimeConfig.js +83 -0
  83. package/src/core/runtimePaths.js +3 -0
  84. package/src/core/sensitiveValueScanner.js +40 -0
  85. package/src/core/token/index.js +40 -3
  86. package/src/decision/client.js +37 -13
  87. package/src/decision/protocol.js +1 -1
  88. package/src/decision/registry.js +5 -3
  89. package/src/decision/runtimeDecide.js +242 -0
  90. package/src/decision/runtimeFilter.js +150 -0
  91. package/src/decision/runtimeScheduler.js +239 -0
  92. package/src/index/buildIndex.js +13 -12
  93. package/src/index/queryIndex.js +35 -14
  94. package/src/index/relatedTests.js +50 -8
  95. package/src/index/resolveContext.js +9 -4
  96. package/src/manifest/selectItems.js +7 -3
  97. package/src/render/instructionRenderer.js +17 -5
  98. package/template_project/.claude/ukit/index/lib/index-core.mjs +94 -39
  99. package/template_project/.claude/ukit/index/route-task.mjs +121 -19
  100. package/template_project/.claude/ukit/index/unic-decision.mjs +28 -13
  101. package/template_project/.claude/ukit/runtime/memory-flags.mjs +51 -0
  102. package/template_project/.claude/ukit/runtime/memory-freshness.mjs +155 -0
  103. package/template_project/.claude/ukit/runtime/memory-policy.mjs +286 -0
  104. package/template_project/.claude/ukit/runtime/output-compression.mjs +3 -0
  105. package/template_project/.claude/ukit/runtime/reinject-context.mjs +145 -14
@@ -0,0 +1,161 @@
1
+ // writeClassification — deterministic classification of incoming writes
2
+ // (SPEC §5 FR-020..FR-022). Pure, no I/O: same candidate + existing set
3
+ // always yields the same {action, targetId?, reason}. Wired into
4
+ // mutateMemory op:'add' as the default classifyAdd so duplicates,
5
+ // enrichments, supersessions, and conflicts resolve predictably; conflicts
6
+ // and user-edited records land in a bounded REVIEW queue, never
7
+ // auto-authoritative.
8
+ //
9
+ // Normalization matches store.js normalizePatternText: lowercase +
10
+ // whitespace-collapse + trim.
11
+
12
+ import { MAX_EVIDENCE } from './records.js';
13
+
14
+ export const REVIEW_QUEUE_MAX = 64;
15
+
16
+ const USER_WRITERS = new Set(['user', 'cli']);
17
+ const NEGATION_PREFIXES = ['not ', 'never '];
18
+
19
+ function normalizeText(text) {
20
+ return String(text ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
21
+ }
22
+
23
+ function candidateProjectId(candidate) {
24
+ return candidate?.project_id ?? candidate?.projectId ?? null;
25
+ }
26
+
27
+ function candidateCreatedBy(candidate) {
28
+ return candidate?.created_by ?? candidate?.createdBy ?? null;
29
+ }
30
+
31
+ function candidateEvidence(candidate) {
32
+ return Array.isArray(candidate?.evidence) ? candidate.evidence : [];
33
+ }
34
+
35
+ function evidenceKey(entry) {
36
+ return `${entry?.kind ?? ''}:${entry?.locator ?? ''}`;
37
+ }
38
+
39
+ function hasDelta(candidate, match) {
40
+ const existingKeys = new Set((match.evidence ?? []).map(evidenceKey));
41
+ if (candidateEvidence(candidate).some((e) => !existingKeys.has(evidenceKey(e)))) {
42
+ return true;
43
+ }
44
+ const provenance = candidate?.provenance;
45
+ if (provenance != null && provenance !== match.provenance) return true;
46
+ const confidence = candidate?.confidence;
47
+ if (typeof confidence === 'number' && confidence !== match.confidence) return true;
48
+ return false;
49
+ }
50
+
51
+ function isUserEdited(record) {
52
+ return USER_WRITERS.has(record?.created_by) || record?.meta?.userEdited === true;
53
+ }
54
+
55
+ function stripNegation(text) {
56
+ for (const prefix of NEGATION_PREFIXES) {
57
+ if (text.startsWith(prefix)) return text.slice(prefix.length);
58
+ }
59
+ return text;
60
+ }
61
+
62
+ function isNegationFlip(a, b) {
63
+ if (a === b) return false;
64
+ return stripNegation(a) === b || stripNegation(b) === a;
65
+ }
66
+
67
+ function sameKey(candidate, record) {
68
+ return normalizeText(candidate?.text) === normalizeText(record?.text)
69
+ && (candidate?.scope ?? 'repo') === record?.scope
70
+ && candidateProjectId(candidate) === (record?.project_id ?? null);
71
+ }
72
+
73
+ /**
74
+ * classifyWrite(candidate, existing, {now} = {}) →
75
+ * {action:'ADD'|'ENRICH'|'SUPERSEDE'|'NOOP'|'REVIEW', targetId?, reason}
76
+ * `existing` is the record list inside the locked write. Pure + deterministic.
77
+ */
78
+ export function classifyWrite(candidate, existing = [], { now } = {}) {
79
+ const records = Array.isArray(existing) ? existing : [];
80
+ const active = records.filter((r) => r?.status === 'active');
81
+ const candidateId = candidate?.id ?? candidate?.recordId ?? null;
82
+
83
+ // (c) explicit supersession — validated before key matching so a malformed
84
+ // supersedes list can never silently fall through to ADD/ENRICH.
85
+ const supersedes = Array.isArray(candidate?.supersedes) ? candidate.supersedes : [];
86
+ if (supersedes.length > 0) {
87
+ const targets = [];
88
+ for (const id of supersedes) {
89
+ const target = records.find((r) => r.id === id);
90
+ if (!target || target.id === candidateId || target.superseded_by != null
91
+ || target.status === 'archived') {
92
+ return { action: 'REVIEW', reason: 'supersede-invalid' };
93
+ }
94
+ targets.push(target);
95
+ }
96
+ // (f) user-edited targets are never silently superseded.
97
+ const writer = candidateCreatedBy(candidate);
98
+ if (targets.some((t) => isUserEdited(t) && t.created_by !== writer)) {
99
+ return { action: 'REVIEW', reason: 'user-edit-conflict' };
100
+ }
101
+ return { action: 'SUPERSEDE', targetId: targets[0].id, targetIds: targets.map((t) => t.id) };
102
+ }
103
+
104
+ const keyMatch = active.find((r) => sameKey(candidate, r));
105
+ if (keyMatch) {
106
+ // (f) same-key writes against a user/cli-authored record from a different
107
+ // writer go to REVIEW — user edits are never silently enriched away.
108
+ const writer = candidateCreatedBy(candidate);
109
+ if (isUserEdited(keyMatch) && writer != null && keyMatch.created_by !== writer) {
110
+ return { action: 'REVIEW', targetId: keyMatch.id, reason: 'user-edit-conflict' };
111
+ }
112
+ // (a) identical key, no new evidence/provenance/confidence → NOOP.
113
+ if (!hasDelta(candidate, keyMatch)) {
114
+ return { action: 'NOOP', targetId: keyMatch.id };
115
+ }
116
+ // (b) same key with new evidence/provenance/confidence → ENRICH.
117
+ return { action: 'ENRICH', targetId: keyMatch.id };
118
+ }
119
+
120
+ const candidateText = normalizeText(candidate?.text);
121
+ const sameText = active.filter((r) => normalizeText(r?.text) === candidateText);
122
+
123
+ // (d) same normalized text under a different scope/project_id → ambiguous.
124
+ if (sameText.length > 0) {
125
+ return { action: 'REVIEW', reason: 'scope-ambiguous' };
126
+ }
127
+
128
+ // (e) contradiction heuristic — same meta.signature/meta.subject with a
129
+ // different value/text, or a negation-only text delta.
130
+ const signature = candidate?.meta?.signature ?? candidate?.signature;
131
+ const subject = candidate?.meta?.subject ?? candidate?.subject;
132
+ for (const record of active) {
133
+ const sameHandle = (signature != null && record?.meta?.signature === signature)
134
+ || (subject != null && record?.meta?.subject === subject);
135
+ if (sameHandle && normalizeText(record.text) !== candidateText) {
136
+ return { action: 'REVIEW', reason: 'contradiction' };
137
+ }
138
+ if (isNegationFlip(candidateText, normalizeText(record?.text))) {
139
+ return { action: 'REVIEW', reason: 'contradiction' };
140
+ }
141
+ }
142
+
143
+ return { action: 'ADD' };
144
+ }
145
+
146
+ /**
147
+ * mergeEvidence(existing, incoming) → evidence[] — union by kind:locator,
148
+ * bounded at MAX_EVIDENCE (existing entries keep precedence).
149
+ */
150
+ export function mergeEvidence(existing = [], incoming = []) {
151
+ const merged = [...(Array.isArray(existing) ? existing : [])];
152
+ const seen = new Set(merged.map(evidenceKey));
153
+ for (const entry of Array.isArray(incoming) ? incoming : []) {
154
+ const key = evidenceKey(entry);
155
+ if (seen.has(key)) continue;
156
+ seen.add(key);
157
+ merged.push(entry);
158
+ if (merged.length >= MAX_EVIDENCE) break;
159
+ }
160
+ return merged.slice(0, MAX_EVIDENCE);
161
+ }
@@ -0,0 +1,129 @@
1
+ /**
2
+ * writeGuard.js (C58 TASK-002 — SPEC §5 FR-004/005/006, §14)
3
+ *
4
+ * Redact/reject engine shared by every memory write entry point.
5
+ * - guardWritePayload: reject-mode for interactive/single writes.
6
+ * - redactWritePayload: redact-mode for bulk lanes (migration/export) so a
7
+ * secret-bearing record is sanitized instead of silently dropped.
8
+ *
9
+ * Deep-scans payload.text, every string leaf of payload.meta, and
10
+ * payload.evidence[*].locator / .fingerprint. Leak-safety contract: return
11
+ * values carry labels only — never a matched value, excerpt, or hash.
12
+ * Never throws; non-string/non-object input is a clean passthrough.
13
+ */
14
+
15
+ import {
16
+ scanText,
17
+ redactText,
18
+ isSensitiveDataGateEnabled,
19
+ loadSensitiveAllowlist,
20
+ } from '../sensitiveValueScanner.js';
21
+ import { resolveMemoryStage } from './memoryFlags.js';
22
+ function* stringLeaves(value) {
23
+ if (typeof value === 'string') {
24
+ yield value;
25
+ } else if (Array.isArray(value)) {
26
+ for (const item of value) yield* stringLeaves(item);
27
+ } else if (value && typeof value === 'object') {
28
+ for (const key of Object.keys(value)) yield* stringLeaves(value[key]);
29
+ }
30
+ }
31
+
32
+ function* payloadStrings(payload) {
33
+ if (typeof payload.text === 'string') yield payload.text;
34
+ if (payload.meta && typeof payload.meta === 'object') {
35
+ yield* stringLeaves(payload.meta);
36
+ }
37
+ if (Array.isArray(payload.evidence)) {
38
+ for (const ev of payload.evidence) {
39
+ if (!ev || typeof ev !== 'object') continue;
40
+ if (typeof ev.locator === 'string') yield ev.locator;
41
+ if (typeof ev.fingerprint === 'string') yield ev.fingerprint;
42
+ }
43
+ }
44
+ }
45
+
46
+ function scanOptions(config) {
47
+ return {
48
+ allowlistHashes: loadSensitiveAllowlist(config),
49
+ gateEnabled: isSensitiveDataGateEnabled(config),
50
+ };
51
+ }
52
+
53
+ /**
54
+ * Reject-mode gate: returns {ok:true} when the payload carries no
55
+ * high-confidence secret values, else {ok:false, reason:'secret-detected',
56
+ * labels}. Gate off or non-object input → {ok:true}.
57
+ */
58
+ export function guardWritePayload(payload, { config, projectId } = {}) {
59
+ try {
60
+ if (!payload || typeof payload !== 'object') return { ok: true };
61
+ // M06 rollout flag (SPEC §5 FR-003): writer stage 'off' denies the write
62
+ // at the guard too — defense-in-depth behind mutateMemory's own consult.
63
+ // Only consulted when a memoryV2 block exists; absent config (user lane,
64
+ // legacy callers) keeps the guard purely a secret gate.
65
+ if (config?.memoryV2 && typeof config.memoryV2 === 'object'
66
+ && resolveMemoryStage(config, 'writer', { projectId }) === 'off') {
67
+ return { ok: false, reason: 'writer-disabled' };
68
+ }
69
+ const opts = scanOptions(config);
70
+ const labels = new Set();
71
+ for (const text of payloadStrings(payload)) {
72
+ const res = scanText(text, opts);
73
+ for (const label of res.labels) labels.add(label);
74
+ }
75
+ if (labels.size === 0) return { ok: true };
76
+ return { ok: false, reason: 'secret-detected', labels: [...labels] };
77
+ } catch {
78
+ return { ok: true };
79
+ }
80
+ }
81
+
82
+ /**
83
+ * Redact-mode gate: returns a new payload with every matched secret span
84
+ * replaced by `[REDACTED:<label>]`. Input is never mutated. Gate off or
85
+ * non-object input → `{payload, redacted:false, labels:[]}` passthrough.
86
+ */
87
+ export function redactWritePayload(payload, { config } = {}) {
88
+ try {
89
+ if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
90
+ return { payload, redacted: false, labels: [] };
91
+ }
92
+ const opts = scanOptions(config);
93
+ if (opts.gateEnabled === false) return { payload, redacted: false, labels: [] };
94
+
95
+ const labels = new Set();
96
+ let redacted = false;
97
+ const redactValue = (value) => {
98
+ if (typeof value === 'string') {
99
+ const res = redactText(value, opts);
100
+ for (const label of res.labels) labels.add(label);
101
+ if (res.redacted) redacted = true;
102
+ return res.text;
103
+ }
104
+ if (Array.isArray(value)) return value.map(redactValue);
105
+ if (value && typeof value === 'object') {
106
+ const out = {};
107
+ for (const key of Object.keys(value)) out[key] = redactValue(value[key]);
108
+ return out;
109
+ }
110
+ return value;
111
+ };
112
+
113
+ const next = { ...payload };
114
+ if (typeof payload.text === 'string') next.text = redactValue(payload.text);
115
+ if (payload.meta && typeof payload.meta === 'object') next.meta = redactValue(payload.meta);
116
+ if (Array.isArray(payload.evidence)) {
117
+ next.evidence = payload.evidence.map((ev) => {
118
+ if (!ev || typeof ev !== 'object') return ev;
119
+ const out = { ...ev };
120
+ if (typeof ev.locator === 'string') out.locator = redactValue(ev.locator);
121
+ if (typeof ev.fingerprint === 'string') out.fingerprint = redactValue(ev.fingerprint);
122
+ return out;
123
+ });
124
+ }
125
+ return { payload: next, redacted, labels: [...labels] };
126
+ } catch {
127
+ return { payload, redacted: false, labels: [] };
128
+ }
129
+ }
@@ -0,0 +1,90 @@
1
+ // hookTelemetryAdapter.js (TASK-001, SPEC §2 G2-FR02/FR05, DF-FR08) — the
2
+ // minimal observer seam over existing hook latency telemetry. Rows are
3
+ // produced today by `template_project/.claude/ukit/runtime/hook-telemetry.mjs`
4
+ // (`buildTimingRow`, TELEMETRY_VERSION 1) and by `hook-chain-runner.mjs`.
5
+ // This adapter is a PURE, read-only transform: it never reads files, never
6
+ // mutates the owner event, and never wires hooks itself — callers pass one
7
+ // row through and receive a sanitized SemanticRecord (or null) to feed
8
+ // `recorder.emit()` inside the hook deadline.
9
+ //
10
+ // adaptHookTelemetryEvent(event, ctx?) → SemanticRecord | null
11
+ //
12
+ // Only allowlisted metadata crosses the seam: hook identity (event, hook
13
+ // path, tool name), outcome, and measured timings. tool_use_id, session ids,
14
+ // payload bodies, and any field outside the support allowlist never reach a
15
+ // record.
16
+ //
17
+ // DF-FR08: hook rows carry no host usage, so `payload.resource` is always
18
+ // `{ value: null, source: 'UNKNOWN' }` — unknown is never 0, never invented.
19
+ // An unmeasurable elapsed window (`elapsedMs: null`) stays `duration_ms:
20
+ // null` and marks `telemetry_complete: false`; the optional stage split maps
21
+ // to `queue_ms` only when the owner actually measured it.
22
+
23
+ import {
24
+ isPlainObject,
25
+ asString,
26
+ asIsoTime,
27
+ createAdapterContext,
28
+ emitRecord,
29
+ } from './common.js';
30
+
31
+ const TELEMETRY_ROW_VERSION = 1;
32
+
33
+ // Owner outcome vocabulary that means the hook run did not complete
34
+ // successfully. Everything else (ok, allow, pass, skip, ...) is a completed
35
+ // invocation — the raw outcome string still travels in payload.outcome.
36
+ const FAILED_OUTCOMES = new Set([
37
+ 'error',
38
+ 'fail',
39
+ 'failed',
40
+ 'timeout',
41
+ 'deny',
42
+ 'denied',
43
+ 'block',
44
+ 'blocked',
45
+ 'crash',
46
+ 'killed',
47
+ ]);
48
+
49
+ function finiteMs(value) {
50
+ return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
51
+ }
52
+
53
+ export function adaptHookTelemetryEvent(event, ctx) {
54
+ if (!isPlainObject(event)) return null;
55
+ // Version gate: rows from a newer owner format are unallowlisted input —
56
+ // adapting them would guess at field meanings, so they never persist.
57
+ if (event.v !== TELEMETRY_ROW_VERSION) return null;
58
+
59
+ const hook = asString(event.hook);
60
+ const hookEvent = asString(event.hookEvent ?? event.event);
61
+ const toolName = asString(event.toolName ?? event.tool);
62
+ // A row that names nothing observable (no hook, event, or tool) carries no
63
+ // safe provenance — drop it rather than emit an anonymous record.
64
+ if (!hook && !hookEvent && !toolName) return null;
65
+
66
+ const durationMs = finiteMs(event.elapsedMs);
67
+ const stageMs = finiteMs(event.stageMs);
68
+ const outcome = asString(event.outcome) ?? 'ok';
69
+
70
+ const context = ctx && typeof ctx === 'object' && typeof ctx.now === 'function'
71
+ ? ctx
72
+ : createAdapterContext({ writerId: 'adapter-hook-telemetry' });
73
+
74
+ const payload = {
75
+ operation: hookEvent,
76
+ name: hook,
77
+ tool_name: toolName,
78
+ outcome,
79
+ duration_ms: durationMs,
80
+ resource: { value: null, source: 'UNKNOWN' },
81
+ telemetry_complete: durationMs !== null,
82
+ };
83
+ if (stageMs !== null) payload.queue_ms = stageMs;
84
+
85
+ return emitRecord(context, {
86
+ semantic_name: FAILED_OUTCOMES.has(outcome.toLowerCase()) ? 'tool.failed' : 'tool.completed',
87
+ wall_time_utc: asIsoTime(event.ts),
88
+ payload,
89
+ });
90
+ }
@@ -0,0 +1,148 @@
1
+ /**
2
+ * cohorts.js (C73 TASK-001, SPEC §2 G3-FR04/G3-FR05, §4) — cohort projection.
3
+ *
4
+ * cohortize(summaries) → Map<key, cohort>
5
+ *
6
+ * Groups TraceSummary rows into honest cohorts. Cohort key components:
7
+ * operation — operation family of the trace's first root span
8
+ * model_version — summary.metric_version
9
+ * token_source — sorted unique resource sources ('+'-joined when mixed)
10
+ * telemetry — 'complete' | 'incomplete'
11
+ * task_type — summary.task_type (appended last: key format stays
12
+ * additive for existing consumers)
13
+ *
14
+ * Any missing or unattributable field maps to the literal key `UNKNOWN` —
15
+ * never dropped, never coerced to 0. A trace with no resource values lands
16
+ * in a `token_source=UNKNOWN` cohort that still carries its own denominator
17
+ * (records + traces) so unknown usage is visible, not silently 0.
18
+ *
19
+ * Each cohort carries:
20
+ * { key, operation, model_version, token_source, telemetry, task_type,
21
+ * count,
22
+ * denominators: { records, traces },
23
+ * aggregates: { resource_total, resource_numeric, resource_unknown,
24
+ * critical_path_max, dropped_count, retries },
25
+ * coverage: { spans, open_spans, orphan_spans, duplicate_ends } }
26
+ *
27
+ * Pure function: no IO, no clock, no randomness. Iteration order of the
28
+ * returned Map is sorted cohort keys — deterministic regardless of input
29
+ * order (summaries are sorted by trace_id before grouping).
30
+ */
31
+
32
+ const UNKNOWN = 'UNKNOWN';
33
+
34
+ function isPlainObject(value) {
35
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
36
+ }
37
+
38
+ function num(value) {
39
+ return typeof value === 'number' && Number.isFinite(value) ? value : 0;
40
+ }
41
+
42
+ /** Operation family: the first root span wins; unattributable → UNKNOWN. */
43
+ function operationOf(summary) {
44
+ const spans = Array.isArray(summary.spans) ? summary.spans : [];
45
+ if (spans.length === 0) return UNKNOWN;
46
+ const ids = new Set(spans.map((s) => s && s.span_id));
47
+ const root = spans.find(
48
+ (s) => !s || typeof s.parent_span_id !== 'string' || !ids.has(s.parent_span_id),
49
+ );
50
+ const family = (root || spans[0]) && (root || spans[0]).family;
51
+ return typeof family === 'string' && family.length > 0 ? family : UNKNOWN;
52
+ }
53
+
54
+ /** Token source: sorted unique resource.sources keys; none → UNKNOWN. */
55
+ function tokenSourceOf(summary) {
56
+ const sources = isPlainObject(summary.resource) && isPlainObject(summary.resource.sources)
57
+ ? Object.keys(summary.resource.sources).filter((k) => summary.resource.sources[k] > 0).sort()
58
+ : [];
59
+ return sources.length > 0 ? sources.join('+') : UNKNOWN;
60
+ }
61
+
62
+ function modelVersionOf(summary) {
63
+ return typeof summary.metric_version === 'string' && summary.metric_version.length > 0
64
+ ? summary.metric_version
65
+ : UNKNOWN;
66
+ }
67
+ function taskTypeOf(summary) {
68
+ return typeof summary.task_type === 'string' && summary.task_type.length > 0
69
+ ? summary.task_type
70
+ : UNKNOWN;
71
+ }
72
+
73
+ export function cohortize(summaries) {
74
+ const input = Array.isArray(summaries) ? summaries.filter(isPlainObject) : [];
75
+ const ordered = input
76
+ .slice()
77
+ .sort((a, b) => String(a.trace_id).localeCompare(String(b.trace_id)));
78
+
79
+ const byKey = new Map();
80
+ for (const summary of ordered) {
81
+ const operation = operationOf(summary);
82
+ const model_version = modelVersionOf(summary);
83
+ const token_source = tokenSourceOf(summary);
84
+ const telemetry = summary.telemetry_complete === true ? 'complete' : 'incomplete';
85
+ const task_type = taskTypeOf(summary);
86
+ const key = `operation=${operation}|model_version=${model_version}|token_source=${token_source}|telemetry=${telemetry}|task_type=${task_type}`;
87
+
88
+ let cohort = byKey.get(key);
89
+ if (!cohort) {
90
+ cohort = {
91
+ key,
92
+ operation,
93
+ model_version,
94
+ token_source,
95
+ telemetry,
96
+ task_type,
97
+ count: 0,
98
+ denominators: { records: 0, traces: 0 },
99
+ aggregates: {
100
+ resource_total: null,
101
+ resource_numeric: 0,
102
+ resource_unknown: 0,
103
+ critical_path_max: null,
104
+ dropped_count: 0,
105
+ retries: 0,
106
+ },
107
+ coverage: { spans: 0, open_spans: 0, orphan_spans: 0, duplicate_ends: 0 },
108
+ };
109
+ byKey.set(key, cohort);
110
+ }
111
+
112
+ const cov = isPlainObject(summary.coverage) ? summary.coverage : {};
113
+ const res = isPlainObject(summary.resource) ? summary.resource : {};
114
+ const retries = isPlainObject(summary.retries) ? summary.retries : {};
115
+ const drops = isPlainObject(summary.drops) ? summary.drops : {};
116
+
117
+ cohort.count += 1;
118
+ cohort.denominators.records += num(cov.records);
119
+ cohort.denominators.traces += 1;
120
+
121
+ if (typeof res.total_value === 'number' && Number.isFinite(res.total_value)) {
122
+ cohort.aggregates.resource_total = (cohort.aggregates.resource_total || 0) + res.total_value;
123
+ }
124
+ cohort.aggregates.resource_numeric += num(res.numeric_count);
125
+ cohort.aggregates.resource_unknown += num(res.unknown_count);
126
+ if (typeof summary.critical_path_ms === 'number' && Number.isFinite(summary.critical_path_ms)) {
127
+ if (
128
+ cohort.aggregates.critical_path_max === null ||
129
+ summary.critical_path_ms > cohort.aggregates.critical_path_max
130
+ ) {
131
+ cohort.aggregates.critical_path_max = summary.critical_path_ms;
132
+ }
133
+ }
134
+ cohort.aggregates.dropped_count += num(drops.dropped_count);
135
+ cohort.aggregates.retries += num(retries.retries);
136
+
137
+ cohort.coverage.spans += num(cov.spans);
138
+ cohort.coverage.open_spans += num(cov.open_spans);
139
+ cohort.coverage.orphan_spans += num(cov.orphan_spans);
140
+ cohort.coverage.duplicate_ends += num(cov.duplicate_ends);
141
+ }
142
+
143
+ // Deterministic iteration: sorted cohort keys only.
144
+ const sorted = new Map(
145
+ [...byKey.entries()].sort((a, b) => a[0].localeCompare(b[0])),
146
+ );
147
+ return sorted;
148
+ }
@@ -0,0 +1,163 @@
1
+ /**
2
+ * storeDigest.js (C73 TASK-001, SPEC §2 G3-FR02/G3-FR03, §4) — store-level
3
+ * deterministic digest over a rebuildIndex result.
4
+ *
5
+ * renderStoreDigest(rebuildResult, opts?) → string
6
+ * digestStoreHash(rebuildResult) → sha256 hex
7
+ *
8
+ * The store digest is the W3 byte-identity artifact: one bounded markdown
9
+ * document over the whole rebuild result — a store header (records read,
10
+ * traces, coverage gaps, ok flag), one compact fingerprint line per trace,
11
+ * and a cohort table where every aggregate carries numerator/denominator
12
+ * and coverage. Unknowns render as `unknown`, never `0`.
13
+ *
14
+ * digestStoreHash canonicalizes the rebuild result (stable key order,
15
+ * trace order sorted by trace_id) and hashes it — two rebuilds over the
16
+ * same retained facts MUST produce the same hex string.
17
+ *
18
+ * Contract:
19
+ * - Hard cap: output never exceeds STORE_DIGEST_MAX_BYTES (default 16KB;
20
+ * opts.maxBytes overrides). Overflow is truncated deterministically
21
+ * with an explicit marker — never silent.
22
+ * - Deterministic: no clock, no randomness; sorted trace order and
23
+ * sorted cohort keys only.
24
+ * - Pure/read-only: no fs, no writes.
25
+ */
26
+
27
+ import crypto from 'node:crypto';
28
+
29
+ import { cohortize } from './cohorts.js';
30
+
31
+ export const STORE_DIGEST_MAX_BYTES = 16 * 1024;
32
+
33
+ const MAX_TRACE_ROWS = 256;
34
+ const MAX_COHORT_ROWS = 64;
35
+
36
+ function isPlainObject(value) {
37
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
38
+ }
39
+
40
+ function fmtNum(value) {
41
+ return typeof value === 'number' && Number.isFinite(value) ? String(value) : 'unknown';
42
+ }
43
+
44
+ /** Stable stringify: object keys sorted at every depth, arrays in order. */
45
+ function stableStringify(value) {
46
+ if (value === null || typeof value !== 'object') return JSON.stringify(value);
47
+ if (Array.isArray(value)) {
48
+ return `[${value.map((item) => stableStringify(item)).join(',')}]`;
49
+ }
50
+ const keys = Object.keys(value).sort();
51
+ const parts = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`);
52
+ return `{${parts.join(',')}}`;
53
+ }
54
+
55
+ function canonicalResult(rebuildResult) {
56
+ const r = isPlainObject(rebuildResult) ? rebuildResult : {};
57
+ const summaries = (Array.isArray(r.summaries) ? r.summaries : [])
58
+ .slice()
59
+ .sort((a, b) => String(a && a.trace_id).localeCompare(String(b && b.trace_id)));
60
+ return { summaries, coverage: isPlainObject(r.coverage) ? r.coverage : {}, ok: r.ok === true };
61
+ }
62
+
63
+ /**
64
+ * sha256 over the canonical serialization of { summaries, coverage, ok }.
65
+ * Trace order is normalized by trace_id and object key order is stable, so
66
+ * byte-identity of the hash is byte-identity of the rebuild, not of the
67
+ * in-memory shape.
68
+ */
69
+ export function digestStoreHash(rebuildResult) {
70
+ const canonical = stableStringify(canonicalResult(rebuildResult));
71
+ return crypto.createHash('sha256').update(canonical, 'utf8').digest('hex');
72
+ }
73
+
74
+ function traceLine(summary) {
75
+ const s = isPlainObject(summary) ? summary : {};
76
+ const cov = isPlainObject(s.coverage) ? s.coverage : {};
77
+ const res = isPlainObject(s.resource) ? s.resource : {};
78
+ const drops = isPlainObject(s.drops) ? s.drops : {};
79
+ return (
80
+ `- ${s.trace_id || 'unknown'}` +
81
+ ` | spans=${fmtNum(cov.spans)}` +
82
+ ` | critical_path_ms=${fmtNum(s.critical_path_ms)}` +
83
+ ` | resource_total=${fmtNum(res.total_value)} (numeric=${fmtNum(res.numeric_count)}, unknown=${fmtNum(res.unknown_count)})` +
84
+ ` | drops=${fmtNum(drops.dropped_count)}` +
85
+ ` | telemetry=${s.telemetry_complete === true ? 'complete' : 'incomplete'}`
86
+ );
87
+ }
88
+
89
+ function cohortRow(cohort) {
90
+ return (
91
+ `| ${cohort.operation} | ${cohort.model_version} | ${cohort.token_source}` +
92
+ ` | ${cohort.telemetry} | ${cohort.denominators.traces}` +
93
+ ` | ${cohort.denominators.records}` +
94
+ ` | ${fmtNum(cohort.aggregates.resource_total)}` +
95
+ ` | ${cohort.aggregates.resource_numeric}/${cohort.aggregates.resource_unknown}` +
96
+ ` | ${fmtNum(cohort.aggregates.critical_path_max)}` +
97
+ ` | ${cohort.aggregates.dropped_count}` +
98
+ ` | ${cohort.coverage.open_spans} |`
99
+ );
100
+ }
101
+
102
+ export function renderStoreDigest(rebuildResult, opts = {}) {
103
+ const r = canonicalResult(rebuildResult);
104
+ const cov = r.coverage;
105
+ const maxBytes = Number.isInteger(opts.maxBytes) && opts.maxBytes > 0
106
+ ? opts.maxBytes
107
+ : STORE_DIGEST_MAX_BYTES;
108
+
109
+ const head = [
110
+ '# store digest',
111
+ `ok: ${r.ok ? 'true' : 'false'}`,
112
+ `records_read: ${fmtNum(cov.records_read)}`,
113
+ `traces: ${fmtNum(cov.traces)}`,
114
+ `untraced_records: ${fmtNum(cov.untraced_records)}`,
115
+ '',
116
+ '## coverage gaps',
117
+ `corrupt_lines: ${fmtNum(cov.corrupt_lines)}`,
118
+ `partial_tail_bytes: ${fmtNum(cov.partial_tail_bytes)}`,
119
+ `quarantined: ${fmtNum(cov.quarantined_segments)}`,
120
+ `expired_segments: ${fmtNum(cov.expired_segments)}`,
121
+ `cursor_missed: ${cov.cursor_missed === true ? 'true' : 'false'}`,
122
+ `degraded: ${Array.isArray(cov.degraded) ? cov.degraded.length : 0}`,
123
+ '',
124
+ ];
125
+
126
+ const summaries = r.summaries;
127
+ const shown = summaries.slice(0, MAX_TRACE_ROWS);
128
+ const traceLines = ['## traces', `summaries: ${summaries.length} total, ${shown.length} shown`];
129
+ for (const summary of shown) traceLines.push(traceLine(summary));
130
+ if (summaries.length > shown.length) {
131
+ traceLines.push(`- … truncated: ${summaries.length - shown.length} trace(s) omitted (digest cap)`);
132
+ }
133
+ traceLines.push('');
134
+
135
+ const cohorts = [...cohortize(summaries).values()];
136
+ const shownCohorts = cohorts.slice(0, MAX_COHORT_ROWS);
137
+ const cohortLines = [
138
+ '## cohorts',
139
+ `cohorts: ${cohorts.length} total`,
140
+ '| operation | model_version | token_source | telemetry | traces | records | resource_total | resource(numeric/unknown) | critical_path_max | dropped | open_spans |',
141
+ ];
142
+ for (const cohort of shownCohorts) cohortLines.push(cohortRow(cohort));
143
+ if (cohorts.length > shownCohorts.length) {
144
+ cohortLines.push(`- … truncated: ${cohorts.length - shownCohorts.length} cohort(s) omitted (digest cap)`);
145
+ }
146
+
147
+ let out = [...head, ...traceLines, ...cohortLines].join('\n');
148
+
149
+ // Hard cap: deterministic tail truncation with an explicit marker.
150
+ if (Buffer.byteLength(out, 'utf8') > maxBytes) {
151
+ const marker = '\n[truncated: store digest exceeded byte cap]\n';
152
+ const budget = maxBytes - Buffer.byteLength(marker, 'utf8');
153
+ let lo = 0;
154
+ let hi = out.length;
155
+ while (lo < hi) {
156
+ const mid = (lo + hi + 1) >> 1;
157
+ if (Buffer.byteLength(out.slice(0, mid), 'utf8') <= budget) lo = mid;
158
+ else hi = mid - 1;
159
+ }
160
+ out = out.slice(0, lo) + marker;
161
+ }
162
+ return out;
163
+ }