@ngockhoale/ukit 3.0.12 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +1 -0
  3. package/manifests/documentation.yaml +12 -0
  4. package/manifests/platform.full.yaml +24 -0
  5. package/package.json +1 -1
  6. package/scripts/bench/data-foundation.mjs +368 -50
  7. package/src/cli/commands/doctor.js +232 -3
  8. package/src/cli/commands/feedback.js +64 -1
  9. package/src/cli/commands/install.js +18 -0
  10. package/src/cli/commands/memory.js +42 -37
  11. package/src/cli/commands/telemetry.js +460 -0
  12. package/src/cli/index.js +7 -0
  13. package/src/core/agentRuntime/adapters.js +83 -2
  14. package/src/core/agentRuntime/diagnostics.js +104 -0
  15. package/src/core/agentRuntime/supervisor.js +137 -0
  16. package/src/core/agentRuntime/telemetry.js +204 -0
  17. package/src/core/memory/memoryEmit.js +131 -0
  18. package/src/core/memory/memoryHit.js +1 -1
  19. package/src/core/memory/migrate.js +18 -11
  20. package/src/core/memory/migrateMapping.js +15 -7
  21. package/src/core/memory/mutateMemory.js +22 -4
  22. package/src/core/memory/recordIndex.js +10 -3
  23. package/src/core/memory/recordStore.js +28 -3
  24. package/src/core/memory/retrieval.js +79 -38
  25. package/src/core/memory/store.js +37 -37
  26. package/src/core/memory/storeV2.js +28 -25
  27. package/src/core/memory/storeV2Loader.js +2 -2
  28. package/src/core/observability/adapters/ingest.js +576 -0
  29. package/src/core/observability/analytics/anomalies.js +415 -0
  30. package/src/core/observability/analytics/summary.js +16 -1
  31. package/src/core/observability/emit/config.js +69 -1
  32. package/src/core/observability/emit/crash.js +434 -0
  33. package/src/core/observability/emit/lifecycle.js +349 -0
  34. package/src/core/observability/emit/recorder.js +135 -9
  35. package/src/core/observability/evaluation/aiPacket.js +52 -10
  36. package/src/core/observability/evaluation/outcomes.js +95 -0
  37. package/src/core/observability/evaluation/runner.js +225 -0
  38. package/src/core/observability/privacy/allowlist.js +23 -3
  39. package/src/core/observability/schema/compatibility.js +48 -3
  40. package/src/core/observability/schema/constants.js +5 -0
  41. package/src/core/observability/schema/registry.js +57 -0
  42. package/src/core/observability/schema/validate.js +68 -6
  43. package/src/core/observability/segments/internal.js +42 -8
  44. package/src/core/observability/segments/readSegments.js +35 -1
  45. package/src/core/observability/segments/recovery.js +3 -2
  46. package/src/core/observability/segments/retention.js +137 -33
  47. package/src/core/observability/support/projector.js +88 -18
  48. package/src/core/observability/support/provision.js +160 -0
  49. package/src/core/observability/support/renderer.js +2 -2
  50. package/src/core/observability/support/schedule.js +174 -0
  51. package/src/decision/registry.js +144 -0
  52. package/src/decision/reviewVerdict.js +309 -0
  53. package/template_project/.claude/agents/code-reviewer.md +25 -1
  54. package/template_project/.claude/agents/ukit-small-task-maintainer.md +16 -0
  55. package/template_project/.claude/commands/ukit/handoff-fullstack.md +2 -0
  56. package/template_project/.claude/commands/ukit/handoff-review.md +12 -0
  57. package/template_project/.claude/hooks/auto-allow-bash.sh +7 -1
  58. package/template_project/.claude/hooks/auto-prune-bash.sh +16 -7
  59. package/template_project/.claude/hooks/verification-guard.sh +13 -4
  60. package/template_project/.claude/ukit/index/review-verdict.mjs +592 -0
  61. package/template_project/.claude/ukit/index/sidecar-decision.mjs +595 -0
  62. package/template_project/.claude/ukit/index/unic-decision.mjs +10 -1
  63. package/template_project/.claude/ukit/runtime/async-lock.mjs +26 -0
  64. package/template_project/.codex/settings.json +3 -0
  65. package/template_project/.omp/agents/code-reviewer.md +25 -1
  66. package/template_project/.omp/agents/ukit-small-task-maintainer.md +16 -0
@@ -0,0 +1,349 @@
1
+ /**
2
+ * lifecycle.js (TASK-005, SPEC §5 DF2-FR05, §6, §10) — the live recorder
3
+ * for a `ukit` process.
4
+ *
5
+ * getRecorder({ projectRoot, config, recorderOpts? }) → recorder
6
+ * startLifecycle({ projectRoot, config, recorder?, flushIntervalMs?,
7
+ * exitDeadlineMs?, registerCrashCapture?,
8
+ * projectionIntervalMs?, refreshSupport? }) → { stop() }
9
+ * recorderHealth({ projectRoot? }?) → counters
10
+ * resetForTests()
11
+ *
12
+ * Layout under the project (all bounded, covered by the existing `/.ukit/`
13
+ * gitignore rule — no new ignore machinery):
14
+ *
15
+ * .ukit/storage/observability/ observability root (crashes/)
16
+ * .ukit/storage/observability/segments/ segment root (active.jsonl + sealed)
17
+ *
18
+ * Contracts (SPEC §8):
19
+ * - getRecorder memoizes one recorder per (segment root, boot_id): the
20
+ * memo key is the resolved projectRoot and a recorder lives for the
21
+ * process boot, so the pair is one entry. The config object supplied
22
+ * on the FIRST call for a root is the one the recorder resolves
23
+ * against — pass the live runtime config object so stage flips (the
24
+ * kill switch) take effect; later calls with different config objects
25
+ * do not re-create the recorder.
26
+ * - Missing/invalid projectRoot → a memoized DISABLED recorder: emit()
27
+ * → { status:'dropped', reason:'recorder-disabled' }, flush() a typed
28
+ * no-op, health() zeroed. Callers (CLI, ingest, doctor) never special-
29
+ * case a null recorder.
30
+ * - startLifecycle wires an unref'd interval flush (default 30s, no
31
+ * deadline — reliability over latency), a `beforeExit` flush bounded
32
+ * by exitDeadlineMs (default 2s, runs at most once per start), and
33
+ * crash capture under the observability root. stop() clears every
34
+ * handle and awaits any in-flight flush so no timer/handler can leak
35
+ * or pin the event loop.
36
+ * - DF2-FR12 (TASK-012): startLifecycle also wires a second unref'd
37
+ * interval (default 15min — `observability.projector.min_interval_ms`
38
+ * lengthens it; `projectionIntervalMs` overrides wholesale for tests;
39
+ * the stamp gate in schedule.js always enforces the ≥15min floor
40
+ * between writes) calling the support-projection seam:
41
+ * `refreshSupport` when injected, else `maybeRefreshSupport` from
42
+ * ../support/schedule.js via dynamic import so the emit hot path
43
+ * never eagerly loads the projector graph. Refresh ticks are
44
+ * serialized by projectionInFlight and awaited by stop() like the
45
+ * flush lane.
46
+ * - Crash capture is stage-gated at start: with stage 'off' nothing is
47
+ * written anywhere — "off" is the kill switch for disk too. (A later
48
+ * runtime promotion mid-process does not retro-install capture; the
49
+ * stage is read at startLifecycle time.)
50
+ * - The exit flush is serialized through recorder.flush() itself, so a
51
+ * slow writer can only consume the deadline — the process never hangs
52
+ * on telemetry. The beforeExit handler self-guards against re-entry
53
+ * (beforeExit re-emits whenever an async handler keeps the loop alive,
54
+ * so an unguarded handler would flush forever and prevent exit).
55
+ * - recorderHealth() merges the recorder's own counters with lifecycle
56
+ * state and flat aliases: emitted (=accepted into queue), dropped
57
+ * (all drop counters summed), flushed (=written to disk). All values
58
+ * are 'unknown'-friendly: absent recorder → zeroed counters.
59
+ *
60
+ * No LLM, no daemon, no sync fs on the hot path. The module never throws.
61
+ */
62
+
63
+ import path from 'node:path';
64
+
65
+ import { createRecorder } from './recorder.js';
66
+ import { registerCrashCapture } from './crash.js';
67
+ import { resolveStage } from './config.js';
68
+
69
+ const DEFAULT_FLUSH_INTERVAL_MS = 30_000;
70
+ const DEFAULT_EXIT_DEADLINE_MS = 2_000;
71
+
72
+ // Mirrors MIN_PROJECTION_INTERVAL_MS in ../support/schedule.js — kept local
73
+ // so the emit hot path never eagerly loads the projector graph (the refresh
74
+ // seam itself is reached via dynamic import inside the timer tick).
75
+ const PROJECTION_FLOOR_INTERVAL_MS = 15 * 60 * 1000;
76
+
77
+ function projectionIntervalOf(config, overrideMs) {
78
+ if (Number.isFinite(overrideMs) && overrideMs > 0) return overrideMs;
79
+ const node = config && typeof config === 'object' ? config.observability : undefined;
80
+ const projector = node && typeof node === 'object' && !Array.isArray(node)
81
+ ? node.projector
82
+ : undefined;
83
+ const raw = projector && typeof projector === 'object' && !Array.isArray(projector)
84
+ ? projector.min_interval_ms
85
+ : undefined;
86
+ const interval = Number.isFinite(raw) && raw > 0 ? raw : PROJECTION_FLOOR_INTERVAL_MS;
87
+ return Math.max(interval, PROJECTION_FLOOR_INTERVAL_MS);
88
+ }
89
+
90
+ /** `<projectRoot>/.ukit/storage/observability` — crashes + diagnostics. */
91
+ export function observabilityRoot(projectRoot) {
92
+ return path.join(projectRoot, '.ukit', 'storage', 'observability');
93
+ }
94
+
95
+ /** `<projectRoot>/.ukit/storage/observability/segments` — recorder root. */
96
+ export function segmentsRoot(projectRoot) {
97
+ return path.join(observabilityRoot(projectRoot), 'segments');
98
+ }
99
+
100
+ function isNonEmptyString(value) {
101
+ return typeof value === 'string' && value.length > 0;
102
+ }
103
+
104
+ // --- disabled recorder (SPEC §8: missing root → emit→dropped/recorder-disabled)
105
+
106
+ function makeDisabledRecorder() {
107
+ const noop = () => {};
108
+ const health = () => ({
109
+ stage: 'off',
110
+ queue_depth: 0,
111
+ queue_bound: 0,
112
+ accepted: 0,
113
+ written: 0,
114
+ dropped_stage_off: 0,
115
+ dropped_invalid: 0,
116
+ dropped_sanitized: 0,
117
+ dropped_oldest: 0,
118
+ dropped_write: 0,
119
+ write_failures: 0,
120
+ dropped_sampled: 0,
121
+ invalid_spans: 0,
122
+ flushes: 0,
123
+ last_flush: null,
124
+ last_error: null,
125
+ });
126
+ return {
127
+ emit: () => ({ status: 'dropped', reason: 'recorder-disabled' }),
128
+ startSpan: () => ({ ended: true }),
129
+ endSpan: noop,
130
+ flush: async () => ({ status: 'ok', written: 0, dropped: 0, remaining: 0, elapsed_ms: 0 }),
131
+ health,
132
+ // Contract marker so downstream surfaces can distinguish a disabled
133
+ // recorder from a staged one without guessing from counters.
134
+ disabled: true,
135
+ };
136
+ }
137
+
138
+ // --- module state ------------------------------------------------------------
139
+
140
+ const recorders = new Map(); // projectRoot → recorder
141
+ const DISABLED_KEY = Symbol('disabled');
142
+ let disabledRecorder = null;
143
+ let lastProjectRoot = null;
144
+
145
+ const lifecycle = {
146
+ running: false,
147
+ interval: null,
148
+ clearIntervalFn: null,
149
+ beforeExit: null,
150
+ capture: null,
151
+ recorder: null,
152
+ inFlight: null,
153
+ exitFlushed: false,
154
+ projection: null,
155
+ projectionInFlight: null,
156
+ };
157
+
158
+ export function getRecorder({ projectRoot, config, recorderOpts } = {}) {
159
+ if (!isNonEmptyString(projectRoot)) {
160
+ if (disabledRecorder === null) disabledRecorder = makeDisabledRecorder();
161
+ return disabledRecorder;
162
+ }
163
+ const key = path.resolve(projectRoot);
164
+ lastProjectRoot = key;
165
+ const existing = recorders.get(key);
166
+ if (existing) return existing;
167
+ const opts = recorderOpts && typeof recorderOpts === 'object' ? recorderOpts : {};
168
+ const recorder = createRecorder({
169
+ ...opts,
170
+ root: segmentsRoot(key),
171
+ config,
172
+ });
173
+ recorders.set(key, recorder);
174
+ return recorder;
175
+ }
176
+
177
+ function detachLifecycle() {
178
+ if (lifecycle.interval !== null) {
179
+ (lifecycle.clearIntervalFn || clearInterval)(lifecycle.interval);
180
+ }
181
+ clearInterval(lifecycle.projection);
182
+ if (lifecycle.beforeExit !== null) {
183
+ process.removeListener('beforeExit', lifecycle.beforeExit);
184
+ }
185
+ if (lifecycle.capture && typeof lifecycle.capture.dispose === 'function') {
186
+ try { lifecycle.capture.dispose(); } catch {}
187
+ }
188
+ lifecycle.interval = null;
189
+ lifecycle.clearIntervalFn = null;
190
+ lifecycle.projection = null;
191
+ lifecycle.projectionInFlight = null;
192
+ lifecycle.beforeExit = null;
193
+ lifecycle.capture = null;
194
+ lifecycle.recorder = null;
195
+ lifecycle.exitFlushed = false;
196
+ lifecycle.running = false;
197
+ }
198
+
199
+ export function startLifecycle({
200
+ projectRoot,
201
+ config,
202
+ recorder,
203
+ flushIntervalMs,
204
+ exitDeadlineMs,
205
+ registerCrashCapture: captureImpl,
206
+ projectionIntervalMs,
207
+ refreshSupport,
208
+ } = {}) {
209
+ // Idempotent restart: a second start replaces the wiring rather than
210
+ // stacking listeners/timers.
211
+ detachLifecycle();
212
+ const rec = recorder || getRecorder({ projectRoot, config });
213
+ if (isNonEmptyString(projectRoot)) lastProjectRoot = path.resolve(projectRoot);
214
+
215
+ const intervalMs = Number.isFinite(flushIntervalMs) && flushIntervalMs > 0
216
+ ? flushIntervalMs
217
+ : DEFAULT_FLUSH_INTERVAL_MS;
218
+ const deadlineMs = Number.isFinite(exitDeadlineMs) && exitDeadlineMs > 0
219
+ ? exitDeadlineMs
220
+ : DEFAULT_EXIT_DEADLINE_MS;
221
+ const installCapture = typeof captureImpl === 'function' ? captureImpl : registerCrashCapture;
222
+
223
+ // Reliability lane: the periodic flush has no deadline — it drains at
224
+ // the writer's own pace and is serialized with any other in-flight pump.
225
+ const onTick = () => {
226
+ const run = rec.flush();
227
+ lifecycle.inFlight = run;
228
+ run.finally(() => {
229
+ if (lifecycle.inFlight === run) lifecycle.inFlight = null;
230
+ });
231
+ // fire-and-forget: pump() never rejects, belt-and-suspenders anyway
232
+ run.catch(() => {});
233
+ };
234
+ const interval = setInterval(onTick, intervalMs);
235
+ if (typeof interval.unref === 'function') interval.unref();
236
+
237
+ // Exit lane: bounded flush. beforeExit re-fires while an async handler
238
+ // keeps work alive, so run it at most once — a slow writer must not
239
+ // turn telemetry into an exit hang.
240
+ const onBeforeExit = () => {
241
+ if (lifecycle.exitFlushed) return;
242
+ lifecycle.exitFlushed = true;
243
+ const run = rec.flush({ deadlineMs });
244
+ lifecycle.inFlight = run;
245
+ run.finally(() => {
246
+ if (lifecycle.inFlight === run) lifecycle.inFlight = null;
247
+ });
248
+ run.catch(() => {});
249
+ };
250
+ process.on('beforeExit', onBeforeExit);
251
+
252
+ // Crash capture is diagnostics, not emission — but stage 'off' means
253
+ // zero disk writes (the kill switch extends to disk), so it installs
254
+ // only when the stage is live at start time.
255
+ let capture = null;
256
+ if (resolveStage(config) !== 'off') {
257
+ try {
258
+ const ids = rec.health();
259
+ capture = installCapture({
260
+ root: observabilityRoot(path.resolve(projectRoot)),
261
+ writerId: ids.writer_id,
262
+ bootId: ids.boot_id,
263
+ });
264
+ } catch {
265
+ capture = null; // best-effort: capture failure never breaks lifecycle
266
+ }
267
+ }
268
+
269
+ // DF2-FR12 scheduled support refresh: unref'd timer on the same lifecycle
270
+ // — the emit hot path stays free of the projector graph because the
271
+ // default seam is a dynamic import resolved on the first tick. Ticks
272
+ // overlap-guard via projectionInFlight (a slow projection skips the next
273
+ // tick rather than stacking concurrent writes).
274
+ const projectionMs = projectionIntervalOf(config, projectionIntervalMs);
275
+ const refreshImpl =
276
+ typeof refreshSupport === 'function'
277
+ ? refreshSupport
278
+ : async (opts) => {
279
+ const { maybeRefreshSupport } = await import('../support/schedule.js');
280
+ return maybeRefreshSupport(opts);
281
+ };
282
+ const onProjectionTick = () => {
283
+ if (lifecycle.projectionInFlight) return;
284
+ const run = Promise.resolve()
285
+ .then(() => refreshImpl({ projectRoot, config, recorder: rec }))
286
+ .catch(() => {});
287
+ lifecycle.projectionInFlight = run;
288
+ run.finally(() => {
289
+ if (lifecycle.projectionInFlight === run) lifecycle.projectionInFlight = null;
290
+ });
291
+ };
292
+ const projection = setInterval(onProjectionTick, projectionMs);
293
+ if (typeof projection.unref === 'function') projection.unref();
294
+
295
+ lifecycle.running = true;
296
+ lifecycle.interval = interval;
297
+ lifecycle.clearIntervalFn = clearInterval;
298
+ lifecycle.beforeExit = onBeforeExit;
299
+ lifecycle.capture = capture;
300
+ lifecycle.recorder = rec;
301
+ lifecycle.projection = projection;
302
+
303
+ async function stop() {
304
+ const pending = lifecycle.inFlight;
305
+ const pendingProjection = lifecycle.projectionInFlight;
306
+ detachLifecycle();
307
+ if (pending) {
308
+ try { await pending; } catch {}
309
+ }
310
+ if (pendingProjection) {
311
+ try { await pendingProjection; } catch {}
312
+ }
313
+ }
314
+
315
+ return { stop };
316
+ }
317
+
318
+ export function recorderHealth({ projectRoot } = {}) {
319
+ const key = isNonEmptyString(projectRoot)
320
+ ? path.resolve(projectRoot)
321
+ : lastProjectRoot;
322
+ const base = key && recorders.get(key) ? recorders.get(key).health() : makeDisabledRecorder().health();
323
+ const dropped = (base.dropped_stage_off || 0)
324
+ + (base.dropped_invalid || 0)
325
+ + (base.dropped_sanitized || 0)
326
+ + (base.dropped_oldest || 0)
327
+ + (base.dropped_write || 0)
328
+ + (base.dropped_sampled || 0);
329
+ return {
330
+ ...base,
331
+ emitted: base.accepted || 0,
332
+ dropped,
333
+ flushed: base.written || 0,
334
+ running: lifecycle.running === true,
335
+ project_root: key,
336
+ // Async handle for awaiting a lifecycle-triggered drain (tests, CLI
337
+ // status right after collect) — lifecycle state, not a counter.
338
+ in_flight: lifecycle.inFlight,
339
+ };
340
+ }
341
+
342
+ /** Test/reset seam: detach every handle and drop all memoized recorders. */
343
+ export function resetForTests() {
344
+ detachLifecycle();
345
+ lifecycle.inFlight = null;
346
+ recorders.clear();
347
+ disabledRecorder = null;
348
+ lastProjectRoot = null;
349
+ }
@@ -6,11 +6,11 @@
6
6
  * → { emit, startSpan, endSpan, flush, health }
7
7
  *
8
8
  * Pipeline per emit(): stage gate → fill missing envelope fields →
9
- * validateSemanticRecord → sanitizeObserved → bounded in-memory queue.
10
- * Nothing reaches disk from emit(); flush() drains the queue through the
11
- * segment store's appendRecord (TASK-006) — the recorder composes the
12
- * writer, it never reimplements writes.
13
- *
9
+ * validateSemanticRecord → sanitizeObserved → sampling decider → bounded
10
+ * in-memory queue. Nothing reaches disk from emit(); flush() drains the
11
+ * queue through the segment store's appendRecord (TASK-006) — the recorder
12
+ * composes the writer, it never reimplements writes.
13
+
14
14
  * Guarantees (DF-FR04):
15
15
  * - emit() is synchronous, non-blocking, and never throws — every failure
16
16
  * is a typed { status: 'dropped', reason } plus a health() counter.
@@ -25,6 +25,22 @@
25
25
  * - No LLM, no sync fs, no daemon. Stage is read live per emit, so
26
26
  * config.observability.stage = 'off' is an immediate kill switch.
27
27
  *
28
+ * DF2-FR02: when `observability.sampling` resolves to a policy (see
29
+ * emit/config.js), every kept record is stamped
30
+ * `sampling:{policy_version, rate}` with its EFFECTIVE keep rate and a
31
+ * seeded-bucket decider evicts records pre-queue:
32
+ * `bucket = murmurHash(record_id, seed) % 10000`, kept iff
33
+ * `bucket < round(rate * 10000)` — same record_id + seed always lands the
34
+ * same decision, and `samplingSeed` is injectable for tests. `critical`
35
+ * importance is never sampled (effective rate 1 regardless of config).
36
+ * Sampled drops never emit per-record telemetry on the hot path: they
37
+ * accumulate into `pending_sampled` and flush() enqueues ONE
38
+ * telemetry.dropped{reason_code:'SAMPLED_OUT', dropped_count} aggregate per
39
+ * flush; a malformed config latch surfaces once per boot as
40
+ * telemetry.dropped{reason_code:'CONFIG_INVALID'}. Self-records are built
41
+ * through the same validate/sanitize path but are never recursively
42
+ * sampled.
43
+ *
28
44
  * DF-FR08: model.completed/model.failed records emitted without
29
45
  * payload.resource are stamped { value: null, source: 'UNKNOWN' } —
30
46
  * host-blind stays explicitly unknown, never 0, never invented.
@@ -37,9 +53,11 @@ import { sanitizeObserved } from '../privacy/sanitizeObserved.js';
37
53
  import { SEMANTIC_REGISTRY } from '../schema/registry.js';
38
54
  import { SCHEMA_VERSION } from '../schema/constants.js';
39
55
  import { appendRecord } from '../segments/retention.js';
40
- import { resolveStage } from './config.js';
56
+ import { resolveStage, resolveSampling, samplingConfigInvalid } from './config.js';
41
57
 
42
58
  const DEFAULT_QUEUE_SIZE = 1024;
59
+ const SAMPLING_BUCKETS = 10000;
60
+ const SAMPLING_SEED = 0;
43
61
  const SPAN_STATUSES = new Set(['completed', 'failed', 'blocked']);
44
62
  // Semantic names whose payload carries typed resource usage (DF-FR08).
45
63
  const RESOURCE_SEMANTICS = new Set(['model.completed', 'model.failed']);
@@ -48,6 +66,47 @@ function isPlainObject(value) {
48
66
  return value !== null && typeof value === 'object' && !Array.isArray(value);
49
67
  }
50
68
 
69
+ // DF2-FR02 decider: murmur-like seeded hash of record_id → bucket in
70
+ // [0, SAMPLING_BUCKETS). Pure integer math — no allocation, no IO, and the
71
+ // same (record_id, seed) always yields the same bucket across hosts so a
72
+ // sampling run is replayable from the recorded ids alone.
73
+ function samplingBucket(recordId, seed) {
74
+ let h = (seed >>> 0) ^ recordId.length;
75
+ for (let i = 0; i < recordId.length; i++) {
76
+ h = Math.imul(h ^ recordId.charCodeAt(i), 0x5bd1e995);
77
+ h ^= h >>> 13;
78
+ }
79
+ h ^= h >>> 15;
80
+ h = Math.imul(h, 0x5bd1e995);
81
+ h ^= h >>> 15;
82
+ return (h >>> 0) % SAMPLING_BUCKETS;
83
+ }
84
+
85
+ // 'critical' is never sampled (DF2-FR02): effective keep rate is 1 even
86
+ // when the policy carries rates.critical below 1 — a misconfigured
87
+ // critical rate can never shed the records kept for crash diagnosis.
88
+ function effectiveSamplingRate(policy, importance) {
89
+ if (importance === 'critical') return 1;
90
+ const rate = policy.rates[importance];
91
+ return typeof rate === 'number' ? rate : 1;
92
+ }
93
+
94
+ function keptBySampling(recordId, rate, seed) {
95
+ if (rate >= 1) return true;
96
+ if (rate <= 0) return false;
97
+ if (typeof recordId !== 'string' || recordId.length === 0) return true;
98
+ return samplingBucket(recordId, seed) < Math.round(rate * SAMPLING_BUCKETS);
99
+ }
100
+
101
+ function stampSampling(candidate, policy) {
102
+ if (policy !== null && isPlainObject(candidate)) {
103
+ candidate.sampling = {
104
+ policy_version: policy.policy_version,
105
+ rate: effectiveSamplingRate(policy, candidate.importance),
106
+ };
107
+ }
108
+ }
109
+
51
110
  function defaultClock() {
52
111
  return {
53
112
  nowMs: () => Date.now(),
@@ -56,10 +115,11 @@ function defaultClock() {
56
115
  };
57
116
  }
58
117
 
59
- export function createRecorder({ root, config, queueSize, writer, clock } = {}) {
118
+ export function createRecorder({ root, config, queueSize, writer, clock, samplingSeed } = {}) {
60
119
  const bound = Number.isInteger(queueSize) && queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE;
61
120
  const write = typeof writer === 'function' ? writer : (record) => appendRecord(root, record);
62
121
  const time = clock && typeof clock.nowNs === 'function' ? clock : defaultClock();
122
+ const seed = Number.isFinite(samplingSeed) ? samplingSeed >>> 0 : SAMPLING_SEED;
63
123
 
64
124
  const state = {
65
125
  queue: [],
@@ -76,6 +136,11 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
76
136
  dropped_write: 0,
77
137
  write_failures: 0,
78
138
  invalid_spans: 0,
139
+ dropped_sampled: 0,
140
+ pending_sampled: 0,
141
+ config_invalid_reported: false,
142
+ flushes: 0,
143
+ last_flush: null,
79
144
  last_error: null,
80
145
  };
81
146
 
@@ -120,7 +185,9 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
120
185
  state.dropped_stage_off += 1;
121
186
  return { status: 'dropped', reason: 'stage-off' };
122
187
  }
188
+ const policy = resolveSampling(config);
123
189
  const candidate = isPlainObject(record) ? fillEnvelope(record) : record;
190
+ stampSampling(candidate, policy);
124
191
  const validation = validateSemanticRecord(candidate);
125
192
  if (!validation.ok) {
126
193
  state.dropped_invalid += 1;
@@ -131,6 +198,11 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
131
198
  state.dropped_sanitized += 1;
132
199
  return { status: 'dropped', reason: 'sanitize-rejected' };
133
200
  }
201
+ if (policy !== null && !keptBySampling(candidate.record_id, candidate.sampling.rate, seed)) {
202
+ state.dropped_sampled += 1;
203
+ state.pending_sampled += 1;
204
+ return { status: 'dropped', reason: 'sampled-out' };
205
+ }
134
206
  let evicted = false;
135
207
  if (state.queue.length >= bound) {
136
208
  state.queue.shift();
@@ -148,6 +220,25 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
148
220
  }
149
221
  }
150
222
 
223
+ // Recorder-originated telemetry (drop aggregates, config faults) goes
224
+ // through the same fill/validate/sanitize path as caller records but
225
+ // bypasses the decider — a self-record is never recursively sampled. On a
226
+ // full queue the oldest record is evicted for it, same policy as emit().
227
+ function queueSelfRecord(record, policy) {
228
+ const candidate = fillEnvelope(record);
229
+ stampSampling(candidate, policy);
230
+ const validation = validateSemanticRecord(candidate);
231
+ if (!validation.ok) return false;
232
+ const clean = sanitizeObserved(candidate);
233
+ if (!clean.ok) return false;
234
+ if (state.queue.length >= bound) {
235
+ state.queue.shift();
236
+ state.dropped_oldest += 1;
237
+ }
238
+ state.queue.push(clean.record);
239
+ return true;
240
+ }
241
+
151
242
  function startSpan(traceId, operation) {
152
243
  const trace_id = typeof traceId === 'string' && traceId.length > 0
153
244
  ? traceId
@@ -198,7 +289,11 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
198
289
  .then(() => write(record))
199
290
  .then((r) => (r && r.ok ? 'ok' : (r && r.reason) || 'write-failed'))
200
291
  .catch(() => 'write-failed');
201
- if (!Number.isFinite(budgetMs) || budgetMs <= 0) return 'deadline';
292
+ if (!Number.isFinite(budgetMs) || budgetMs <= 0) {
293
+ // No deadline (Infinity) → await the write unbounded; a non-positive
294
+ // budget reports deadline without starting another write race.
295
+ return Number.isFinite(budgetMs) ? 'deadline' : writePromise;
296
+ }
202
297
  let timer;
203
298
  try {
204
299
  return await Promise.race([
@@ -266,10 +361,36 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
266
361
  /* pump() never rejects; belt-and-suspenders */
267
362
  }
268
363
  }
364
+ // Drain pending self-telemetry into the same flush: ONE aggregate per
365
+ // reason — sampled drops are rate-limited to a single queued record per
366
+ // flush (never recursively sampled), and the malformed-config latch is
367
+ // reported once per boot. The aggregate record itself stays queued if
368
+ // its write fails, so the count is never silently lost before landing.
369
+ const policyAtFlush = resolveSampling(config);
370
+ if (state.pending_sampled > 0) {
371
+ const count = state.pending_sampled;
372
+ if (queueSelfRecord({
373
+ semantic_name: 'telemetry.dropped',
374
+ payload: { reason_code: 'SAMPLED_OUT', dropped_count: count },
375
+ }, policyAtFlush)) {
376
+ state.pending_sampled = 0;
377
+ }
378
+ }
379
+ if (samplingConfigInvalid() && !state.config_invalid_reported) {
380
+ if (queueSelfRecord({
381
+ semantic_name: 'telemetry.dropped',
382
+ payload: { reason_code: 'CONFIG_INVALID', dropped_count: 1 },
383
+ }, policyAtFlush)) {
384
+ state.config_invalid_reported = true;
385
+ }
386
+ }
269
387
  const run = pump(deadlineMs);
270
388
  state.pumping = run;
271
389
  try {
272
- return await run;
390
+ const result = await run;
391
+ state.flushes += 1;
392
+ state.last_flush = result;
393
+ return result;
273
394
  } finally {
274
395
  if (state.pumping === run) state.pumping = null;
275
396
  }
@@ -277,6 +398,10 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
277
398
 
278
399
  function health() {
279
400
  return {
401
+ boot_id: state.boot_id,
402
+ writer_id: state.writer_id,
403
+ flushes: state.flushes,
404
+ last_flush: state.last_flush,
280
405
  stage: resolveStage(config),
281
406
  queue_depth: state.queue.length,
282
407
  queue_bound: bound,
@@ -288,6 +413,7 @@ export function createRecorder({ root, config, queueSize, writer, clock } = {})
288
413
  dropped_oldest: state.dropped_oldest,
289
414
  dropped_write: state.dropped_write,
290
415
  write_failures: state.write_failures,
416
+ dropped_sampled: state.dropped_sampled,
291
417
  invalid_spans: state.invalid_spans,
292
418
  last_error: state.last_error,
293
419
  };