@manyoars/oars 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -29
- package/dist/cli.d.ts +4 -1
- package/dist/cli.js +102 -32
- package/dist/commands/login.d.ts +1 -1
- package/dist/commands/login.js +1 -1
- package/dist/gate/hook-main.js +13 -1
- package/dist/gate/hook-process.d.ts +11 -1
- package/dist/gate/hook-process.js +50 -2
- package/dist/gate/hook-record.d.ts +3 -0
- package/dist/gate/hook-record.js +8 -0
- package/dist/gate/hook.d.ts +2 -0
- package/dist/gate/hook.js +3 -0
- package/dist/gate/replay.js +2 -5
- package/dist/gate/report.js +1 -1
- package/dist/gate/transcripts.js +1 -1
- package/dist/hook-bin.js +27 -2
- package/dist/issuer.d.ts +7 -9
- package/dist/issuer.js +10 -32
- package/dist/main.js +3 -0
- package/dist/sessions/actions.js +78 -0
- package/dist/sessions/allow.d.ts +10 -1
- package/dist/sessions/allow.js +29 -9
- package/dist/sessions/baseline.d.ts +1 -1
- package/dist/sessions/baseline.js +6 -4
- package/dist/sessions/call-shape.d.ts +19 -0
- package/dist/sessions/call-shape.js +85 -0
- package/dist/sessions/cell-increments.d.ts +5 -1
- package/dist/sessions/cell-increments.js +84 -50
- package/dist/sessions/cells.d.ts +33 -3
- package/dist/sessions/cells.js +89 -9
- package/dist/sessions/codex.js +93 -27
- package/dist/sessions/command.d.ts +11 -1
- package/dist/sessions/command.js +64 -18
- package/dist/sessions/coverage.d.ts +19 -0
- package/dist/sessions/coverage.js +56 -0
- package/dist/sessions/cursor.js +2 -2
- package/dist/sessions/derive.js +72 -6
- package/dist/sessions/exclusion.d.ts +6 -0
- package/dist/sessions/exclusion.js +8 -0
- package/dist/sessions/extract.d.ts +4 -2
- package/dist/sessions/extract.js +58 -26
- package/dist/sessions/hook-calls.js +1 -1
- package/dist/sessions/hook-index.d.ts +14 -1
- package/dist/sessions/hook-index.js +10 -3
- package/dist/sessions/hooks.d.ts +5 -1
- package/dist/sessions/hooks.js +133 -22
- package/dist/sessions/identity.d.ts +8 -3
- package/dist/sessions/identity.js +45 -15
- package/dist/sessions/install.d.ts +18 -2
- package/dist/sessions/install.js +53 -8
- package/dist/sessions/mcp.d.ts +3 -2
- package/dist/sessions/mcp.js +8 -4
- package/dist/sessions/motif.d.ts +2 -1
- package/dist/sessions/motif.js +2 -2
- package/dist/sessions/opencode.d.ts +33 -0
- package/dist/sessions/opencode.js +319 -0
- package/dist/sessions/otel.d.ts +60 -0
- package/dist/sessions/otel.js +494 -0
- package/dist/sessions/package-books.d.ts +48 -0
- package/dist/sessions/package-books.js +120 -0
- package/dist/sessions/package-resolve.d.ts +2 -0
- package/dist/sessions/package-resolve.js +3 -1
- package/dist/sessions/pi.d.ts +36 -0
- package/dist/sessions/pi.js +353 -0
- package/dist/sessions/pool-books.d.ts +21 -0
- package/dist/sessions/pool-books.js +75 -0
- package/dist/sessions/pool-choices.d.ts +5 -0
- package/dist/sessions/pool-choices.js +11 -0
- package/dist/sessions/pool-command.d.ts +76 -5
- package/dist/sessions/pool-command.js +263 -136
- package/dist/sessions/pool-nightly-deps.d.ts +3 -0
- package/dist/sessions/pool-nightly-deps.js +11 -0
- package/dist/sessions/pool-nightly.d.ts +10 -0
- package/dist/sessions/pool-nightly.js +85 -0
- package/dist/sessions/pool-outbox.d.ts +15 -2
- package/dist/sessions/pool-outbox.js +79 -26
- package/dist/sessions/pool-print.d.ts +1 -1
- package/dist/sessions/pool-raw.d.ts +27 -0
- package/dist/sessions/pool-raw.js +68 -0
- package/dist/sessions/pool-replica.d.ts +35 -0
- package/dist/sessions/pool-replica.js +91 -0
- package/dist/sessions/pool-send-lock.d.ts +20 -0
- package/dist/sessions/pool-send-lock.js +87 -0
- package/dist/sessions/pool-sent.d.ts +4 -0
- package/dist/sessions/pool-sent.js +26 -0
- package/dist/sessions/pool-totals.d.ts +10 -0
- package/dist/sessions/pool-totals.js +21 -0
- package/dist/sessions/pool-wire.d.ts +240 -30
- package/dist/sessions/pool-wire.js +695 -215
- package/dist/sessions/public-vocabulary.d.ts +2 -3
- package/dist/sessions/public-vocabulary.js +2 -9
- package/dist/sessions/query.d.ts +16 -4
- package/dist/sessions/query.js +32 -10
- package/dist/sessions/read-cost.js +14 -3
- package/dist/sessions/read-recovery-edge.d.ts +2 -0
- package/dist/sessions/read-recovery-edge.js +26 -0
- package/dist/sessions/read.d.ts +8 -1
- package/dist/sessions/read.js +6 -3
- package/dist/sessions/recall-bundle.d.ts +12 -0
- package/dist/sessions/recall-bundle.js +33 -0
- package/dist/sessions/recall-frames.d.ts +18 -0
- package/dist/sessions/recall-frames.js +44 -0
- package/dist/sessions/recall-next.d.ts +145 -0
- package/dist/sessions/recall-next.js +241 -0
- package/dist/sessions/recall-object.d.ts +2 -0
- package/dist/sessions/recall-object.js +1 -0
- package/dist/sessions/recall-push.d.ts +2 -2
- package/dist/sessions/recall-push.js +3 -51
- package/dist/sessions/recall-reads.d.ts +7 -0
- package/dist/sessions/recall-reads.js +7 -0
- package/dist/sessions/recovery-calls.d.ts +21 -0
- package/dist/sessions/recovery-calls.js +25 -0
- package/dist/sessions/redact.d.ts +11 -0
- package/dist/sessions/redact.js +59 -0
- package/dist/sessions/schema.d.ts +59 -9
- package/dist/sessions/schema.js +22 -3
- package/dist/sessions/session-shape.d.ts +6 -3
- package/dist/sessions/session-shape.js +19 -6
- package/dist/sessions/sources.js +75 -1
- package/dist/sessions/store.d.ts +13 -1
- package/dist/sessions/store.js +36 -16
- package/dist/sessions/survival.d.ts +2 -0
- package/dist/sessions/survival.js +2 -2
- package/dist/sessions/transcript-tail.d.ts +14 -0
- package/dist/sessions/transcript-tail.js +64 -0
- package/dist/sessions/vscode-copilot.js +24 -7
- package/dist/telemetry/command.d.ts +8 -0
- package/dist/telemetry/command.js +85 -0
- package/dist/telemetry/consent.d.ts +17 -0
- package/dist/telemetry/consent.js +63 -0
- package/dist/telemetry/crash.d.ts +8 -0
- package/dist/telemetry/crash.js +52 -0
- package/dist/telemetry/emit.d.ts +31 -0
- package/dist/telemetry/emit.js +42 -0
- package/dist/telemetry/fields.d.ts +16 -0
- package/dist/telemetry/fields.js +26 -0
- package/dist/telemetry/health.d.ts +21 -0
- package/dist/telemetry/health.js +108 -0
- package/dist/telemetry/hook-emit.d.ts +4 -0
- package/dist/telemetry/hook-emit.js +18 -0
- package/dist/telemetry/measures.d.ts +32 -0
- package/dist/telemetry/measures.js +128 -0
- package/dist/telemetry/observe.d.ts +13 -0
- package/dist/telemetry/observe.js +51 -0
- package/dist/telemetry/read-events.d.ts +9 -0
- package/dist/telemetry/read-events.js +17 -0
- package/dist/telemetry/record.d.ts +17 -0
- package/dist/telemetry/record.js +26 -0
- package/dist/telemetry/report.d.ts +77 -0
- package/dist/telemetry/report.js +188 -0
- package/dist/telemetry/schema.d.ts +40 -0
- package/dist/telemetry/schema.js +187 -0
- package/dist/telemetry/send-main.d.ts +1 -0
- package/dist/telemetry/send-main.js +11 -0
- package/dist/telemetry/send.d.ts +36 -0
- package/dist/telemetry/send.js +172 -0
- package/dist/telemetry/spool.d.ts +17 -0
- package/dist/telemetry/spool.js +70 -0
- package/dist/telemetry/wire.d.ts +25 -0
- package/dist/telemetry/wire.js +76 -0
- package/package.json +2 -2
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// What the nightly run tells telemetry about the health of an install: a snapshot of the session store (counts, sizes, verify results, the versions it
|
|
2
|
+
// holds) and what the run cost. Counts, bytes, and closed words only; the schema checks them again before anything reaches the spool.
|
|
3
|
+
import { readdir, stat } from 'node:fs/promises';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { CAPTURE_VERSION, OLDEST_READABLE_CAPTURE } from '../sessions/schema.js';
|
|
6
|
+
import { readSteps } from '../sessions/steps.js';
|
|
7
|
+
import { frozenSessions, loadStore, paths, readSkeletons } from '../sessions/store.js';
|
|
8
|
+
import { LITERAL, publicModel, publicVersion } from '../sessions/public-vocabulary.js';
|
|
9
|
+
import { verifyAll, verifyCaptured } from '../sessions/verify.js';
|
|
10
|
+
import { emitEvent, recordsEvents } from './emit.js';
|
|
11
|
+
import { spoolEvents } from './read-events.js';
|
|
12
|
+
const MAX_LIST = 32;
|
|
13
|
+
const OTHER = 'other';
|
|
14
|
+
const WEEK_MS = 7 * 86_400_000;
|
|
15
|
+
/**
|
|
16
|
+
* The distinct values seen, each kept when the pool wire's own public vocabulary admits it and written as `other` when it does not, sorted with `other`
|
|
17
|
+
* last. More than the schema's list length folds the tail into `other`.
|
|
18
|
+
*/
|
|
19
|
+
export const closedList = (values, kind) => {
|
|
20
|
+
const visible = kind === 'version' ? publicVersion : publicModel;
|
|
21
|
+
const kept = new Set();
|
|
22
|
+
let other = false;
|
|
23
|
+
for (const value of values) {
|
|
24
|
+
const shown = visible(value);
|
|
25
|
+
if (shown === null || shown === LITERAL || shown !== value)
|
|
26
|
+
other = true;
|
|
27
|
+
else
|
|
28
|
+
kept.add(value);
|
|
29
|
+
}
|
|
30
|
+
const sorted = [...kept].sort();
|
|
31
|
+
const room = MAX_LIST - 1;
|
|
32
|
+
if (sorted.length > room)
|
|
33
|
+
return [...sorted.slice(0, room), OTHER];
|
|
34
|
+
return other ? [...sorted, OTHER] : sorted;
|
|
35
|
+
};
|
|
36
|
+
/** The distinct whole-number versions among the stamps, ascending; a stamp that is not a whole number names no version and is left out. */
|
|
37
|
+
export const versionsHeld = (stamps) => [...new Set(stamps.map(Number).filter((stamp) => Number.isSafeInteger(stamp) && stamp >= 0))].sort((a, b) => a - b).slice(-MAX_LIST);
|
|
38
|
+
/** Every byte under a folder; a folder that is not there holds none. */
|
|
39
|
+
const bytesUnder = async (dir) => {
|
|
40
|
+
let entries;
|
|
41
|
+
try {
|
|
42
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
46
|
+
return 0;
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
let total = 0;
|
|
50
|
+
for (const entry of entries)
|
|
51
|
+
total += entry.isDirectory() ? await bytesUnder(join(dir, entry.name)) : (await stat(join(dir, entry.name))).size;
|
|
52
|
+
return total;
|
|
53
|
+
};
|
|
54
|
+
/** The bytes the last seven days of sends put on the wire, read from telemetry's own `wire-send` events: nothing is asked of the pool. */
|
|
55
|
+
const egressBytes = (dir, now) => spoolEvents(dir, 'wire-send').reduce((total, { at, data }) => (data.sent === 1 && typeof data.bytes === 'number' && Date.parse(at) >= now - WEEK_MS ? total + data.bytes : total), 0);
|
|
56
|
+
/** The extractor version each pipeline step last ran at: the log is append-only, so older rows name versions the store no longer holds. A tie on `at` goes to the later row. */
|
|
57
|
+
const lastExtractorVersions = (steps) => {
|
|
58
|
+
const latest = new Map();
|
|
59
|
+
for (const step of steps) {
|
|
60
|
+
const held = latest.get(step.step);
|
|
61
|
+
if (held === undefined || Date.parse(step.at) >= Date.parse(held.at))
|
|
62
|
+
latest.set(step.step, step);
|
|
63
|
+
}
|
|
64
|
+
return [...latest.values()].map((step) => step.ev);
|
|
65
|
+
};
|
|
66
|
+
const storeFields = async (runtime) => {
|
|
67
|
+
const { storeDir } = runtime;
|
|
68
|
+
const { skeletons, unreadable } = await readSkeletons(storeDir, undefined, true);
|
|
69
|
+
const store = await loadStore(storeDir);
|
|
70
|
+
const steps = await readSteps(storeDir);
|
|
71
|
+
const captured = [...store.sessions, ...store.subagents];
|
|
72
|
+
const frozen = await frozenSessions(runtime, captured.filter((session) => Number(session.cv) >= OLDEST_READABLE_CAPTURE && Number(session.cv) < Number(CAPTURE_VERSION)));
|
|
73
|
+
const results = [...verifyAll(skeletons, skeletons), ...verifyCaptured(captured, store.tools, frozen)];
|
|
74
|
+
return {
|
|
75
|
+
sessions: store.sessions.length,
|
|
76
|
+
subagents: store.subagents.length,
|
|
77
|
+
skeletons: skeletons.length,
|
|
78
|
+
unparseable: unreadable + captured.reduce((sum, session) => sum + session.badLines, 0),
|
|
79
|
+
rawBytes: await bytesUnder(paths(storeDir).raw),
|
|
80
|
+
verifyChecked: results.length,
|
|
81
|
+
verifyFailed: results.filter((result) => result.violations > 0).length,
|
|
82
|
+
// What the store holds: the `cv` of each captured session and the `ev` each pipeline step last ran at. A store with a session behind this build lists two.
|
|
83
|
+
captureVersions: versionsHeld(captured.map((session) => session.cv)),
|
|
84
|
+
extractorVersions: versionsHeld(lastExtractorVersions(steps)),
|
|
85
|
+
harnessVersions: closedList(captured.flatMap((session) => session.versions), 'version'),
|
|
86
|
+
models: closedList(captured.flatMap((session) => Object.keys(session.models)), 'model'),
|
|
87
|
+
};
|
|
88
|
+
};
|
|
89
|
+
/** The store-health event of one nightly run. A store that cannot be read is recorded as that and nothing else; it never stops the run. */
|
|
90
|
+
export const recordStoreHealth = async (telemetry, runtime) => {
|
|
91
|
+
// The pass reads every skeleton and stats every raw file, so an install with telemetry off must not start it.
|
|
92
|
+
if (!recordsEvents(telemetry))
|
|
93
|
+
return;
|
|
94
|
+
try {
|
|
95
|
+
const fields = await storeFields(runtime);
|
|
96
|
+
// The outbox is an extra: a folder that cannot be read leaves the two figures out and the snapshot stands.
|
|
97
|
+
const extras = await bytesUnder(join(runtime.storeDir, 'pool-outbox')).then((outboxBytes) => ({ outboxBytes, egressBytes: egressBytes(telemetry.telemetry?.dir ?? '', telemetry.now()) }), () => ({}));
|
|
98
|
+
emitEvent(telemetry, 'store-health', { ...fields, ...extras });
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
emitEvent(telemetry, 'store-health', { readFailed: 1 });
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
/** What a nightly run cost: the whole run, the deposit's parse, the process's peak memory, and whether it failed. */
|
|
105
|
+
export const recordRunCost = (telemetry, cost) => {
|
|
106
|
+
const peak = telemetry.peakRssMb ?? (() => Math.round(process.resourceUsage().maxRSS / 1024));
|
|
107
|
+
emitEvent(telemetry, 'run-cost', { totalMs: Math.max(0, cost.totalMs), parseMs: Math.max(0, cost.parseMs), peakRssMb: peak(), failed: cost.failed ? 1 : 0 });
|
|
108
|
+
};
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type Emit } from './emit.js';
|
|
2
|
+
/** The version in the package.json at `manifest`; `0.0.0` when it cannot be read (the schema writes a version it cannot read as `unknown`). */
|
|
3
|
+
export declare const versionAt: (manifest: URL) => string;
|
|
4
|
+
export declare const hookEmitter: (env: NodeJS.ProcessEnv, platform: NodeJS.Platform, home: string, manifest: URL, now: () => number) => Emit;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// The emitter a hook process builds for itself. A hook never loads the CLI (`main.ts` reads the version and the folders), so this reads the same two things for itself:
|
|
2
|
+
// the config folder the CLI uses, and the version in this package's `package.json`, read only when an event is about to be written.
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { defaultConfigDir } from '../credentials.js';
|
|
6
|
+
import { emitFor } from './emit.js';
|
|
7
|
+
/** The version in the package.json at `manifest`; `0.0.0` when it cannot be read (the schema writes a version it cannot read as `unknown`). */
|
|
8
|
+
export const versionAt = (manifest) => {
|
|
9
|
+
try {
|
|
10
|
+
const parsed = JSON.parse(readFileSync(manifest, 'utf8'));
|
|
11
|
+
const version = typeof parsed === 'object' && parsed !== null ? Reflect.get(parsed, 'version') : undefined;
|
|
12
|
+
return typeof version === 'string' ? version : '0.0.0';
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return '0.0.0';
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
export const hookEmitter = (env, platform, home, manifest, now) => emitFor({ env, telemetry: { dir: join(defaultConfigDir(env, platform, home), 'oars', 'telemetry'), platform }, version: () => versionAt(manifest), now });
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** A latency in a cell, or a yes (1) or no (0) that a rate is made from. */
|
|
2
|
+
export type Kind = 'latency' | 'rate';
|
|
3
|
+
export interface Measure {
|
|
4
|
+
metric: string;
|
|
5
|
+
/** What the value groups by: the harness and harness version a hook ran under (`claude-code/2.1.5`), or the oars version for a command. */
|
|
6
|
+
dimension: 'harnessVersion' | 'cli';
|
|
7
|
+
value: string;
|
|
8
|
+
kind: Kind;
|
|
9
|
+
/** Milliseconds for a latency; 1 or 0 for a rate. */
|
|
10
|
+
x: number;
|
|
11
|
+
}
|
|
12
|
+
/** The windows the report shows and the pool serves, in days. */
|
|
13
|
+
export declare const WINDOWS: readonly number[];
|
|
14
|
+
/** The hooks that exist; any other name in a batch is not measured. */
|
|
15
|
+
export declare const HOOK_NAMES: readonly string[];
|
|
16
|
+
export declare const OTHER = "other";
|
|
17
|
+
export declare const UNRECORDED = "unrecorded";
|
|
18
|
+
/** What one event measures; an event of a type nothing measures, or that is not an object, measures nothing. */
|
|
19
|
+
export declare const measuresOf: (event: unknown) => Measure[];
|
|
20
|
+
/** Every cell is named by these three together. */
|
|
21
|
+
export declare const cellKey: (measure: Pick<Measure, "metric" | "dimension" | "value">) => string;
|
|
22
|
+
/**
|
|
23
|
+
* The bucket edges a latency histogram counts in, each 1.5 times the one before (1, 2, 3, 4, 6, 9, ...) up to a day. The pool keeps a histogram, not the latencies, so its percentile is the
|
|
24
|
+
* upper edge of the bucket that holds it: within a factor of 1.5 of the true one.
|
|
25
|
+
*/
|
|
26
|
+
export declare const LATENCY_EDGES: readonly number[];
|
|
27
|
+
/** The index of the first edge at or above the latency. */
|
|
28
|
+
export declare const bucketOf: (milliseconds: number) => number;
|
|
29
|
+
/** The p-th percentile (0 to 1) of a histogram given as [bucket index, count] pairs, as that bucket's upper edge. */
|
|
30
|
+
export declare const histogramPercentile: (buckets: readonly (readonly [number, number])[], p: number) => number;
|
|
31
|
+
/** The nearest-rank p-th percentile (0 to 1) of numbers; 0 for none. */
|
|
32
|
+
export declare const percentile: (values: readonly number[], p: number) => number;
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// What telemetry measures, in one place: the CLI's own report and the pool's rollup both turn an event into the same `Measure`s, so a user's number and the fleet's are the same number.
|
|
2
|
+
// A measure is one observation in a cell: a latency in milliseconds, or a yes/no that rates are made from. The cell is (metric, dimension, value): `hook.pre-tool.latency` at harness
|
|
3
|
+
// `claude-code`, version `2.1.5`. The reader is tolerant on purpose: the pool reads batches kept whole from any client, so a value that does not fit is skipped, never trusted.
|
|
4
|
+
import { isHarness } from '../sessions/pool-wire.js';
|
|
5
|
+
/** The windows the report shows and the pool serves, in days. */
|
|
6
|
+
export const WINDOWS = [7, 30];
|
|
7
|
+
const DAY_MS = 86_400_000;
|
|
8
|
+
/** The hooks that exist; any other name in a batch is not measured. */
|
|
9
|
+
export const HOOK_NAMES = ['session-start', 'prompt', 'pre-tool', 'post-tool', 'stop', 'nightly', 'gate'];
|
|
10
|
+
/** A group value as the program writes it: a version or a word, short, with no space, slash, or quote. Anything else is `other`. */
|
|
11
|
+
const VALUE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,31}$/;
|
|
12
|
+
const COMMAND = /^[a-z][A-Za-z0-9-]{0,31}$/;
|
|
13
|
+
export const OTHER = 'other';
|
|
14
|
+
export const UNRECORDED = 'unrecorded';
|
|
15
|
+
const valueOf = (raw) => {
|
|
16
|
+
if (raw === undefined)
|
|
17
|
+
return UNRECORDED;
|
|
18
|
+
return typeof raw === 'string' && VALUE.test(raw) ? raw : OTHER;
|
|
19
|
+
};
|
|
20
|
+
const isObject = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
21
|
+
const fieldOf = (value, key) => (isObject(value) ? Reflect.get(value, key) : undefined);
|
|
22
|
+
/** A whole number of milliseconds a day long at most, or undefined. */
|
|
23
|
+
const millis = (value) => (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 && value <= DAY_MS ? value : undefined);
|
|
24
|
+
const latency = (metric, dimension, value, x) => ({ metric, dimension, value, kind: 'latency', x });
|
|
25
|
+
const rate = (metric, dimension, value, yes) => ({ metric, dimension, value, kind: 'rate', x: yes ? 1 : 0 });
|
|
26
|
+
/**
|
|
27
|
+
* The value of a hook cell: the harness and its version, `claude-code/2.1.5`, so one version number under two harnesses is two cells. A harness off the closed list is `other`; an event written
|
|
28
|
+
* before the field existed has none and reads `unrecorded`, never dropped.
|
|
29
|
+
*/
|
|
30
|
+
const hookCellOf = (harness, version) => {
|
|
31
|
+
const named = harness === undefined ? UNRECORDED : typeof harness === 'string' && isHarness(harness) ? harness : OTHER;
|
|
32
|
+
return `${named}/${valueOf(version)}`;
|
|
33
|
+
};
|
|
34
|
+
/** The measures of a `hook` event: its latency, and whether it found something. A hook that failed is neither a hit nor a miss. */
|
|
35
|
+
const hookMeasures = (data) => {
|
|
36
|
+
const name = fieldOf(data, 'name');
|
|
37
|
+
if (typeof name !== 'string' || !HOOK_NAMES.includes(name))
|
|
38
|
+
return [];
|
|
39
|
+
const version = hookCellOf(fieldOf(data, 'harness'), fieldOf(data, 'harnessVersion'));
|
|
40
|
+
const found = [];
|
|
41
|
+
const took = millis(fieldOf(data, 'latencyMs'));
|
|
42
|
+
if (took !== undefined)
|
|
43
|
+
found.push(latency(`hook.${name}.latency`, 'harnessVersion', version, took));
|
|
44
|
+
const outcome = fieldOf(data, 'outcome');
|
|
45
|
+
// `held` found something and held it back for the unexposed arm, so it counts as a find.
|
|
46
|
+
if (outcome === 'hit' || outcome === 'held')
|
|
47
|
+
found.push(rate(`hook.${name}.hit`, 'harnessVersion', version, true));
|
|
48
|
+
else if (outcome === 'miss')
|
|
49
|
+
found.push(rate(`hook.${name}.hit`, 'harnessVersion', version, false));
|
|
50
|
+
return found;
|
|
51
|
+
};
|
|
52
|
+
/** The measures of a `command` event: how long it took, and whether it failed, under the first name of its path. */
|
|
53
|
+
const commandMeasures = (event, data) => {
|
|
54
|
+
const path = fieldOf(data, 'path');
|
|
55
|
+
const first = Array.isArray(path) && typeof path[0] === 'string' && COMMAND.test(path[0]) ? path[0] : 'none';
|
|
56
|
+
const version = valueOf(fieldOf(event, 'cli'));
|
|
57
|
+
const found = [];
|
|
58
|
+
const took = millis(fieldOf(data, 'durationMs'));
|
|
59
|
+
if (took !== undefined)
|
|
60
|
+
found.push(latency(`command.${first}.latency`, 'cli', version, took));
|
|
61
|
+
const exit = fieldOf(data, 'exitCode');
|
|
62
|
+
if (typeof exit === 'number' && Number.isSafeInteger(exit) && exit >= 0 && exit <= 255)
|
|
63
|
+
found.push(rate(`command.${first}.failure`, 'cli', version, exit !== 0));
|
|
64
|
+
return found;
|
|
65
|
+
};
|
|
66
|
+
/** The measures of a `store-health` event: whether the nightly could read the store, and whether it found a session that does not verify. */
|
|
67
|
+
const storeMeasures = (event, data) => {
|
|
68
|
+
const version = valueOf(fieldOf(event, 'cli'));
|
|
69
|
+
const unread = fieldOf(data, 'readFailed') === 1;
|
|
70
|
+
const found = [rate('store.readFailure', 'cli', version, unread)];
|
|
71
|
+
const failed = fieldOf(data, 'verifyFailed');
|
|
72
|
+
if (!unread && typeof failed === 'number' && Number.isSafeInteger(failed) && failed >= 0)
|
|
73
|
+
found.push(rate('store.verifyFailure', 'cli', version, failed > 0));
|
|
74
|
+
return found;
|
|
75
|
+
};
|
|
76
|
+
/** What one event measures; an event of a type nothing measures, or that is not an object, measures nothing. */
|
|
77
|
+
export const measuresOf = (event) => {
|
|
78
|
+
const type = fieldOf(event, 'type');
|
|
79
|
+
const data = fieldOf(event, 'data');
|
|
80
|
+
switch (type) {
|
|
81
|
+
case 'hook':
|
|
82
|
+
return hookMeasures(data);
|
|
83
|
+
case 'command':
|
|
84
|
+
return commandMeasures(event, data);
|
|
85
|
+
case 'store-health':
|
|
86
|
+
return storeMeasures(event, data);
|
|
87
|
+
default:
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
/** Every cell is named by these three together. */
|
|
92
|
+
export const cellKey = (measure) => `${measure.metric}\u0000${measure.dimension}\u0000${measure.value}`;
|
|
93
|
+
/**
|
|
94
|
+
* The bucket edges a latency histogram counts in, each 1.5 times the one before (1, 2, 3, 4, 6, 9, ...) up to a day. The pool keeps a histogram, not the latencies, so its percentile is the
|
|
95
|
+
* upper edge of the bucket that holds it: within a factor of 1.5 of the true one.
|
|
96
|
+
*/
|
|
97
|
+
export const LATENCY_EDGES = (() => {
|
|
98
|
+
const edges = [];
|
|
99
|
+
for (let step = 0;; step += 1) {
|
|
100
|
+
const edge = Math.ceil(1.5 ** step);
|
|
101
|
+
if (edges.at(-1) !== edge)
|
|
102
|
+
edges.push(edge);
|
|
103
|
+
if (edge >= DAY_MS)
|
|
104
|
+
return edges;
|
|
105
|
+
}
|
|
106
|
+
})();
|
|
107
|
+
/** The index of the first edge at or above the latency. */
|
|
108
|
+
export const bucketOf = (milliseconds) => LATENCY_EDGES.findIndex((edge) => milliseconds <= edge);
|
|
109
|
+
/** The p-th percentile (0 to 1) of a histogram given as [bucket index, count] pairs, as that bucket's upper edge. */
|
|
110
|
+
export const histogramPercentile = (buckets, p) => {
|
|
111
|
+
const sorted = [...buckets].sort((a, b) => a[0] - b[0]);
|
|
112
|
+
const total = sorted.reduce((sum, [, count]) => sum + count, 0);
|
|
113
|
+
const target = Math.max(1, Math.ceil(p * total));
|
|
114
|
+
let seen = 0;
|
|
115
|
+
for (const [index, count] of sorted) {
|
|
116
|
+
seen += count;
|
|
117
|
+
if (seen >= target)
|
|
118
|
+
return LATENCY_EDGES[index] ?? DAY_MS;
|
|
119
|
+
}
|
|
120
|
+
return 0;
|
|
121
|
+
};
|
|
122
|
+
/** The nearest-rank p-th percentile (0 to 1) of numbers; 0 for none. */
|
|
123
|
+
export const percentile = (values, p) => {
|
|
124
|
+
if (values.length === 0)
|
|
125
|
+
return 0;
|
|
126
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
127
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))] ?? 0;
|
|
128
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Command } from 'commander';
|
|
2
|
+
import type { RunDeps } from '../cli.js';
|
|
3
|
+
import type { ExitCode } from '../errors.js';
|
|
4
|
+
export interface Observed {
|
|
5
|
+
/** The command path, such as `['pool', 'fetch']`; empty when no command ran (`--help`, `--version`, a usage error). */
|
|
6
|
+
path: string[];
|
|
7
|
+
/** The long names (as commander keys them) of the options given on the command line. */
|
|
8
|
+
flags: string[];
|
|
9
|
+
}
|
|
10
|
+
/** Called by commander just before an action runs. */
|
|
11
|
+
export declare const observeCommand: (observed: Observed, action: Command) => void;
|
|
12
|
+
/** Prints the notice on the first run that is not a hook, then records the event. Never throws and never changes what the command did. */
|
|
13
|
+
export declare const finishTelemetry: (deps: RunDeps, argv: string[], observed: Observed, startedAt: number, exit: ExitCode) => void;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { consentOf, markNoticeShown, NOTICE } from './consent.js';
|
|
2
|
+
import { recordEvent } from './record.js';
|
|
3
|
+
import { startDailySend } from './send.js';
|
|
4
|
+
import { errorClassOf } from './schema.js';
|
|
5
|
+
const pathOf = (command) => {
|
|
6
|
+
const names = [];
|
|
7
|
+
let step = command;
|
|
8
|
+
// The root program has no parent and is not part of the path.
|
|
9
|
+
while (step.parent !== null) {
|
|
10
|
+
names.unshift(step.name());
|
|
11
|
+
step = step.parent;
|
|
12
|
+
}
|
|
13
|
+
return names;
|
|
14
|
+
};
|
|
15
|
+
/** Called by commander just before an action runs. */
|
|
16
|
+
export const observeCommand = (observed, action) => {
|
|
17
|
+
observed.path = pathOf(action);
|
|
18
|
+
const given = action.optsWithGlobals();
|
|
19
|
+
observed.flags = Object.keys(given).filter((key) => action.getOptionValueSourceWithGlobals(key) === 'cli');
|
|
20
|
+
};
|
|
21
|
+
const isHook = (argv, observed) => {
|
|
22
|
+
const gateHook = (words) => words[0] === 'gate' && words[1] === 'hook';
|
|
23
|
+
return gateHook(observed.path) || gateHook(argv);
|
|
24
|
+
};
|
|
25
|
+
/** Prints the notice on the first run that is not a hook, then records the event. Never throws and never changes what the command did. */
|
|
26
|
+
export const finishTelemetry = (deps, argv, observed, startedAt, exit) => {
|
|
27
|
+
try {
|
|
28
|
+
const runtime = deps.telemetry;
|
|
29
|
+
if (runtime === undefined)
|
|
30
|
+
return;
|
|
31
|
+
const consent = consentOf(deps.env, runtime.dir);
|
|
32
|
+
if (!consent.enabled)
|
|
33
|
+
return;
|
|
34
|
+
if (!consent.noticeShown && !isHook(argv, observed) && markNoticeShown(runtime.dir, deps.now))
|
|
35
|
+
deps.io.stderr(`${NOTICE}\n`);
|
|
36
|
+
const errorClass = errorClassOf(exit);
|
|
37
|
+
recordEvent({ ...runtime, version: deps.version, now: deps.now }, 'command', {
|
|
38
|
+
path: observed.path,
|
|
39
|
+
flags: observed.flags,
|
|
40
|
+
durationMs: Math.max(0, deps.now() - startedAt),
|
|
41
|
+
exitCode: exit,
|
|
42
|
+
...(errorClass === undefined ? {} : { errorClass }),
|
|
43
|
+
});
|
|
44
|
+
// Never from a hook: a hook has a time budget, and the send is for the nightly task and the commands a person runs.
|
|
45
|
+
if (!isHook(argv, observed))
|
|
46
|
+
startDailySend(runtime, deps.env, deps.now());
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
// Telemetry is never allowed to change what a command does.
|
|
50
|
+
}
|
|
51
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// The spool's events read back as typed values, by type: for the nightly figures that sum earlier events and for the tests that check what was written.
|
|
2
|
+
// A line that is not an event is left out.
|
|
3
|
+
import { readSpool } from './spool.js';
|
|
4
|
+
const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
5
|
+
const eventOf = (line) => {
|
|
6
|
+
try {
|
|
7
|
+
const parsed = JSON.parse(line);
|
|
8
|
+
if (isRecord(parsed) && typeof parsed.type === 'string' && typeof parsed.at === 'string' && isRecord(parsed.data))
|
|
9
|
+
return { type: parsed.type, at: parsed.at, data: parsed.data };
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
// Not JSON: not an event.
|
|
13
|
+
}
|
|
14
|
+
return undefined;
|
|
15
|
+
};
|
|
16
|
+
/** Every spooled event, oldest first, or only those of `type`. */
|
|
17
|
+
export const spoolEvents = (dir, type) => readSpool(dir).flatMap((line) => eventOf(line) ?? []).filter((event) => type === undefined || event.type === type);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Where telemetry keeps its folder; absent from a run, nothing is recorded. */
|
|
2
|
+
export interface TelemetryRuntime {
|
|
3
|
+
/** `<config>/oars/telemetry`. */
|
|
4
|
+
dir: string;
|
|
5
|
+
platform: NodeJS.Platform;
|
|
6
|
+
/** Starts the detached send (`spawnDetachedSend`); only `main.ts` supplies it, so a run without one never sends. */
|
|
7
|
+
spawnSend?: (dir: string) => void;
|
|
8
|
+
}
|
|
9
|
+
export interface RecordContext extends TelemetryRuntime {
|
|
10
|
+
/** The CLI's version. */
|
|
11
|
+
version: string;
|
|
12
|
+
/** Epoch milliseconds. */
|
|
13
|
+
now: () => number;
|
|
14
|
+
/** The spool's byte cap per file; absent means `SPOOL_CAP_BYTES`. */
|
|
15
|
+
capBytes?: number;
|
|
16
|
+
}
|
|
17
|
+
export declare const recordEvent: (context: RecordContext, type: string, fields: object) => void;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// Records one event: closed schema, then the spool. It never throws and returns nothing, so a command's exit code and output cannot depend on it.
|
|
2
|
+
import { SCHEMA_VERSION, sanitize } from './schema.js';
|
|
3
|
+
import { appendLine, installId } from './spool.js';
|
|
4
|
+
const OS = ['win32', 'darwin', 'linux'];
|
|
5
|
+
const VERSION = /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]{1,32})?$/;
|
|
6
|
+
export const recordEvent = (context, type, fields) => {
|
|
7
|
+
try {
|
|
8
|
+
const checked = sanitize(type, fields);
|
|
9
|
+
if (checked === undefined)
|
|
10
|
+
return;
|
|
11
|
+
const event = {
|
|
12
|
+
v: SCHEMA_VERSION,
|
|
13
|
+
id: installId(context.dir),
|
|
14
|
+
at: new Date(context.now()).toISOString(),
|
|
15
|
+
type,
|
|
16
|
+
cli: VERSION.test(context.version) ? context.version : 'unknown',
|
|
17
|
+
os: OS.includes(context.platform) ? context.platform : 'other',
|
|
18
|
+
...(checked.dropped > 0 ? { dropped: checked.dropped } : {}),
|
|
19
|
+
data: checked.data,
|
|
20
|
+
};
|
|
21
|
+
appendLine(context.dir, JSON.stringify(event), context.capBytes);
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
// Telemetry is never allowed to change what a command does.
|
|
25
|
+
}
|
|
26
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { type Kind, type Measure } from './measures.js';
|
|
2
|
+
/** The fleet's number for one cell: percentiles of the fleet's latency, or the median install's rate. */
|
|
3
|
+
export interface FleetNumbers {
|
|
4
|
+
p50: number;
|
|
5
|
+
p95?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface FleetCell extends FleetNumbers {
|
|
8
|
+
metric: string;
|
|
9
|
+
dimension: Measure['dimension'];
|
|
10
|
+
value: string;
|
|
11
|
+
kind: Kind;
|
|
12
|
+
}
|
|
13
|
+
export interface OwnCell {
|
|
14
|
+
metric: string;
|
|
15
|
+
dimension: Measure['dimension'];
|
|
16
|
+
value: string;
|
|
17
|
+
kind: Kind;
|
|
18
|
+
/** Observations in the window. */
|
|
19
|
+
n: number;
|
|
20
|
+
p50?: number;
|
|
21
|
+
p95?: number;
|
|
22
|
+
/** For a rate: the share of the user's observations that were a yes. */
|
|
23
|
+
rate?: number;
|
|
24
|
+
fleet?: FleetNumbers;
|
|
25
|
+
}
|
|
26
|
+
export interface StoreSnapshot {
|
|
27
|
+
at: string;
|
|
28
|
+
readFailed: boolean;
|
|
29
|
+
sessions?: number;
|
|
30
|
+
subagents?: number;
|
|
31
|
+
skeletons?: number;
|
|
32
|
+
unparseable?: number;
|
|
33
|
+
verifyChecked?: number;
|
|
34
|
+
verifyFailed?: number;
|
|
35
|
+
rawBytes?: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The user's own cells over the last `days`, ordered by metric and then value; with `asOf`, over the `days` ending with that day instead. A latency is counted in the same buckets the fleet's is,
|
|
39
|
+
* so the user's p50 and the fleet's are the same kind of number (the bucket's upper edge) and a spool identical to a fleet cell reads equal to it.
|
|
40
|
+
*/
|
|
41
|
+
export declare const ownCells: (lines: readonly string[], now: number, days: number, asOf?: string) => OwnCell[];
|
|
42
|
+
/** The newest store-health snapshot in the window: its counts as the nightly recorded them. */
|
|
43
|
+
export declare const latestStore: (lines: readonly string[], now: number, days: number) => StoreSnapshot | undefined;
|
|
44
|
+
/** The user's cells with the fleet's number put on each cell the fleet serves; the fleet is `undefined` when no read answered. */
|
|
45
|
+
export declare const besideFleet: (own: readonly OwnCell[], fleet: readonly FleetCell[] | undefined) => OwnCell[];
|
|
46
|
+
export interface FleetWindow {
|
|
47
|
+
days: number;
|
|
48
|
+
cells: FleetCell[];
|
|
49
|
+
}
|
|
50
|
+
export type FleetRead = {
|
|
51
|
+
status: 'not-configured';
|
|
52
|
+
} | {
|
|
53
|
+
status: 'unavailable';
|
|
54
|
+
server?: string;
|
|
55
|
+
reason: string;
|
|
56
|
+
} | {
|
|
57
|
+
status: 'answered';
|
|
58
|
+
server: string;
|
|
59
|
+
asOf: string;
|
|
60
|
+
floor: number;
|
|
61
|
+
windows: FleetWindow[];
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* The fleet's numbers from the pool server in `OARS_POOL_URL`, or why there are none. The request is a bare GET: it names no install and sends no credential, cookie, or event.
|
|
65
|
+
* Anything wrong, a refusal, a timeout, or an answer of the wrong shape, comes back as `unavailable`; it never fails the report.
|
|
66
|
+
*/
|
|
67
|
+
export declare const readFleet: (env: NodeJS.ProcessEnv) => Promise<FleetRead>;
|
|
68
|
+
export interface Report {
|
|
69
|
+
windows: {
|
|
70
|
+
days: number;
|
|
71
|
+
cells: OwnCell[];
|
|
72
|
+
}[];
|
|
73
|
+
store: StoreSnapshot | undefined;
|
|
74
|
+
fleet: FleetRead;
|
|
75
|
+
}
|
|
76
|
+
export declare const reportOf: (lines: readonly string[], now: number, fleet: FleetRead) => Report;
|
|
77
|
+
export declare const textOf: (report: Report, dir: string) => string;
|