@indigoai-us/hq-cloud 6.14.39 → 6.14.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/skill-telemetry.d.ts +21 -5
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +238 -31
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +235 -1
- package/dist/skill-telemetry.test.js.map +1 -1
- package/package.json +2 -2
- package/src/skill-telemetry.test.ts +286 -0
- package/src/skill-telemetry.ts +297 -31
- package/test/e2e/sync/skill-telemetry-oversized-transcript.test.ts +124 -0
package/src/skill-telemetry.ts
CHANGED
|
@@ -106,6 +106,12 @@ export interface CollectSkillTelemetryOptions {
|
|
|
106
106
|
cursorPath?: string;
|
|
107
107
|
/** Override `~/.hq/menubar.json` (the offline opt-in fallback) for tests. */
|
|
108
108
|
menubarPath?: string;
|
|
109
|
+
/**
|
|
110
|
+
* Maximum transcript bytes to inspect per source (Claude or Codex) in one
|
|
111
|
+
* collection pass. The next pass resumes from the last complete line.
|
|
112
|
+
* Override for deterministic bounded-scan tests.
|
|
113
|
+
*/
|
|
114
|
+
maxScanBytesPerSource?: number;
|
|
109
115
|
/**
|
|
110
116
|
* Override skillVersion resolution (skills-first-class US-015). Given a skill
|
|
111
117
|
* name, return its content-hash version marker (`sha256:<hex>`) or undefined.
|
|
@@ -156,14 +162,78 @@ const INCLUDE_ARGS_PREVIEW = false;
|
|
|
156
162
|
|
|
157
163
|
// ── Cursor schema (independent from the token collector's) ──────────────────────
|
|
158
164
|
|
|
165
|
+
type SkillTelemetrySource = "claude" | "codex";
|
|
166
|
+
|
|
167
|
+
interface PendingLineCursor {
|
|
168
|
+
/** Offset of the unterminated JSONL record's first byte. */
|
|
169
|
+
start: number;
|
|
170
|
+
/** Furthest byte inspected for that record; no transcript bytes are stored. */
|
|
171
|
+
scannedOffset: number;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
interface CodexCursorState {
|
|
175
|
+
/** Latest turn context, needed by function_call rows after a bounded resume. */
|
|
176
|
+
turnId?: string;
|
|
177
|
+
/** Dedupe keys from the active turn, so repeated skill reads stay collapsed. */
|
|
178
|
+
seen?: string[];
|
|
179
|
+
}
|
|
180
|
+
|
|
159
181
|
interface CursorEntry {
|
|
160
182
|
offset: number;
|
|
161
183
|
mtime: number;
|
|
184
|
+
pendingLine?: PendingLineCursor;
|
|
185
|
+
codex?: CodexCursorState;
|
|
162
186
|
}
|
|
163
187
|
|
|
164
188
|
interface SkillCursor {
|
|
165
189
|
version: string;
|
|
166
190
|
files: Record<string, CursorEntry>;
|
|
191
|
+
/** Rotate each runtime's file order so one growing backlog cannot starve peers. */
|
|
192
|
+
lastScannedBySource?: Partial<Record<SkillTelemetrySource, string>>;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function clonePendingLine(value: unknown): PendingLineCursor | undefined {
|
|
196
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
197
|
+
const pending = value as Partial<PendingLineCursor>;
|
|
198
|
+
const { start, scannedOffset } = pending;
|
|
199
|
+
if (
|
|
200
|
+
typeof start !== "number" ||
|
|
201
|
+
typeof scannedOffset !== "number" ||
|
|
202
|
+
!Number.isSafeInteger(start) ||
|
|
203
|
+
!Number.isSafeInteger(scannedOffset) ||
|
|
204
|
+
start < 0 ||
|
|
205
|
+
scannedOffset < start
|
|
206
|
+
) {
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
209
|
+
return { start, scannedOffset };
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function cloneCodexCursorState(value: unknown): CodexCursorState | undefined {
|
|
213
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
214
|
+
const state = value as Partial<CodexCursorState>;
|
|
215
|
+
const turnId = typeof state.turnId === "string" ? state.turnId : undefined;
|
|
216
|
+
const seen = Array.isArray(state.seen)
|
|
217
|
+
? state.seen.filter((entry): entry is string => typeof entry === "string")
|
|
218
|
+
: [];
|
|
219
|
+
return turnId === undefined && seen.length === 0 ? undefined : { turnId, seen };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function snapshotCodexCursorState(
|
|
223
|
+
turnId: string | undefined,
|
|
224
|
+
seen: Set<string> | undefined,
|
|
225
|
+
): CodexCursorState | undefined {
|
|
226
|
+
if (turnId === undefined && (seen === undefined || seen.size === 0)) return undefined;
|
|
227
|
+
return { turnId, ...(seen && seen.size > 0 ? { seen: [...seen] } : {}) };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function rotateFilesFromLast(
|
|
231
|
+
files: string[],
|
|
232
|
+
lastScanned: string | undefined,
|
|
233
|
+
): string[] {
|
|
234
|
+
if (!lastScanned) return files;
|
|
235
|
+
const index = files.indexOf(lastScanned);
|
|
236
|
+
return index < 0 ? files : [...files.slice(index + 1), ...files.slice(0, index + 1)];
|
|
167
237
|
}
|
|
168
238
|
|
|
169
239
|
async function loadCursor(cursorPath: string): Promise<SkillCursor> {
|
|
@@ -171,7 +241,17 @@ async function loadCursor(cursorPath: string): Promise<SkillCursor> {
|
|
|
171
241
|
const raw = await fs.readFile(cursorPath, "utf-8");
|
|
172
242
|
const parsed = JSON.parse(raw) as Partial<SkillCursor>;
|
|
173
243
|
if (parsed && typeof parsed === "object" && parsed.files && typeof parsed.files === "object") {
|
|
174
|
-
return {
|
|
244
|
+
return {
|
|
245
|
+
version: parsed.version ?? "1",
|
|
246
|
+
files: parsed.files as Record<string, CursorEntry>,
|
|
247
|
+
...(parsed.lastScannedBySource && typeof parsed.lastScannedBySource === "object"
|
|
248
|
+
? {
|
|
249
|
+
lastScannedBySource: parsed.lastScannedBySource as Partial<
|
|
250
|
+
Record<SkillTelemetrySource, string>
|
|
251
|
+
>,
|
|
252
|
+
}
|
|
253
|
+
: {}),
|
|
254
|
+
};
|
|
175
255
|
}
|
|
176
256
|
} catch {
|
|
177
257
|
// Missing / unparseable — start fresh.
|
|
@@ -718,7 +798,7 @@ async function listJsonlFiles(root: string): Promise<string[]> {
|
|
|
718
798
|
}
|
|
719
799
|
}
|
|
720
800
|
await walk(root);
|
|
721
|
-
return out;
|
|
801
|
+
return out.sort((a, b) => a.localeCompare(b));
|
|
722
802
|
}
|
|
723
803
|
|
|
724
804
|
// Codex's `session_meta` is always the first, small line of a rollout. Read a
|
|
@@ -737,6 +817,18 @@ const CODEX_META_PREFIX_BYTES = 64 * 1024;
|
|
|
737
817
|
*/
|
|
738
818
|
const MAX_FS_READ_CHUNK_BYTES = 256 * 1024 * 1024;
|
|
739
819
|
|
|
820
|
+
/**
|
|
821
|
+
* A sync pass must never turn an unbounded transcript tail into one V8 string.
|
|
822
|
+
* Node aborts the process once Buffer#toString receives a length >= 2**31; this
|
|
823
|
+
* conservative cap is also below the smaller practical string ceilings across
|
|
824
|
+
* supported Node versions. Keep this assertion close to the call site so a
|
|
825
|
+
* future increase cannot reintroduce the native SIGTRAP failure.
|
|
826
|
+
*/
|
|
827
|
+
export const MAX_DECODE_BYTES = 256 * 1024 * 1024;
|
|
828
|
+
if (MAX_DECODE_BYTES >= 2 ** 31) {
|
|
829
|
+
throw new Error("MAX_DECODE_BYTES must remain below the V8 string length limit");
|
|
830
|
+
}
|
|
831
|
+
|
|
740
832
|
/** Minimal structural view of the `FileHandle.read` we depend on. */
|
|
741
833
|
type ReadableFileHandle = {
|
|
742
834
|
read(
|
|
@@ -796,6 +888,7 @@ async function readCodexSessionContext(
|
|
|
796
888
|
|
|
797
889
|
const MAX_BATCH_EVENTS = 100;
|
|
798
890
|
const MAX_BATCH_BYTES = 240 * 1024;
|
|
891
|
+
const MAX_SCAN_BYTES_PER_SOURCE = 4 * 1024 * 1024;
|
|
799
892
|
const ROW_TRUNCATION_SUFFIX = "...[truncated]";
|
|
800
893
|
|
|
801
894
|
function jsonBytes(value: unknown): number {
|
|
@@ -847,11 +940,13 @@ function boundRowForPost(
|
|
|
847
940
|
* Scan, extract, and POST any new skill-invocation events.
|
|
848
941
|
*
|
|
849
942
|
* Cursor model (per-batch commit, matching the token collector for robustness):
|
|
850
|
-
* each file is scanned from its stored byte offset
|
|
851
|
-
* carry the byte offset of the line they came from. Events are
|
|
852
|
-
* server-sized batches, and the cursor advances **per successful
|
|
853
|
-
* batch in a large (e.g. first-run backfill) fails, the
|
|
854
|
-
* succeeded stay committed and only the rest re-send next
|
|
943
|
+
* each file is scanned from its stored byte offset through a bounded region;
|
|
944
|
+
* extracted events carry the byte offset of the line they came from. Events are
|
|
945
|
+
* flushed in server-sized batches, and the cursor advances **per successful
|
|
946
|
+
* batch** — so if one batch in a large (e.g. first-run backfill) fails, the
|
|
947
|
+
* batches that already succeeded stay committed and only the rest re-send next
|
|
948
|
+
* sync. A bounded region settles only at a complete-line boundary, preventing
|
|
949
|
+
* a cursor from skipping an unread tail or resuming mid-line.
|
|
855
950
|
*
|
|
856
951
|
* Per-file commit rule:
|
|
857
952
|
* - All of a file's events sent OK (including zero-event files) → commit EOF,
|
|
@@ -873,6 +968,12 @@ export async function collectAndSendSkillTelemetry(
|
|
|
873
968
|
opts.cursorPath ?? path.join(home, ".hq", "skill-telemetry-cursor.json");
|
|
874
969
|
const menubarPath = opts.menubarPath ?? path.join(home, ".hq", "menubar.json");
|
|
875
970
|
const log = opts.log ?? (() => {});
|
|
971
|
+
// Match the usage collector's per-runtime budget, with an independent hard
|
|
972
|
+
// decode ceiling as defence in depth for future callers/configuration.
|
|
973
|
+
const maxScanBytesPerSource = Math.min(
|
|
974
|
+
opts.maxScanBytesPerSource ?? MAX_SCAN_BYTES_PER_SOURCE,
|
|
975
|
+
MAX_DECODE_BYTES,
|
|
976
|
+
);
|
|
876
977
|
|
|
877
978
|
// Normalize the scope path once (drop a single trailing slash, keeping "/").
|
|
878
979
|
const normalizePath = (p: string): string => (p.length > 1 ? p.replace(/\/+$/, "") : p);
|
|
@@ -928,9 +1029,15 @@ export async function collectAndSendSkillTelemetry(
|
|
|
928
1029
|
const cursor = await loadCursor(cursorPath);
|
|
929
1030
|
const claudeFiles = await listJsonlFiles(claudeProjectsRoot);
|
|
930
1031
|
const codexFiles = await listJsonlFiles(codexSessionsRoot);
|
|
931
|
-
const files: { filePath: string; kind:
|
|
932
|
-
...
|
|
933
|
-
|
|
1032
|
+
const files: { filePath: string; kind: SkillTelemetrySource }[] = [
|
|
1033
|
+
...rotateFilesFromLast(
|
|
1034
|
+
claudeFiles,
|
|
1035
|
+
cursor.lastScannedBySource?.claude,
|
|
1036
|
+
).map((f) => ({ filePath: f, kind: "claude" as const })),
|
|
1037
|
+
...rotateFilesFromLast(
|
|
1038
|
+
codexFiles,
|
|
1039
|
+
cursor.lastScannedBySource?.codex,
|
|
1040
|
+
).map((f) => ({ filePath: f, kind: "codex" as const })),
|
|
934
1041
|
];
|
|
935
1042
|
|
|
936
1043
|
// 3. Scan every file from its stored offset, collecting events tagged with
|
|
@@ -939,15 +1046,22 @@ export async function collectAndSendSkillTelemetry(
|
|
|
939
1046
|
eof: number;
|
|
940
1047
|
mtime: number;
|
|
941
1048
|
eventCount: number; // events extracted from this file this run
|
|
1049
|
+
pendingLine?: PendingLineCursor;
|
|
1050
|
+
codex?: CodexCursorState;
|
|
942
1051
|
}
|
|
943
1052
|
interface Sourced {
|
|
944
1053
|
row: Record<string, unknown>;
|
|
945
1054
|
filePath: string;
|
|
946
1055
|
endOffset: number; // absolute byte offset at the end of the source line
|
|
1056
|
+
codex?: CodexCursorState;
|
|
947
1057
|
}
|
|
948
1058
|
|
|
949
1059
|
const fileScans: Record<string, FileScan> = {};
|
|
950
1060
|
const rotationResets: Record<string, CursorEntry> = {};
|
|
1061
|
+
const scannedBytes: Record<SkillTelemetrySource, number> = { claude: 0, codex: 0 };
|
|
1062
|
+
const lastScannedBySource: Partial<Record<SkillTelemetrySource, string>> = {
|
|
1063
|
+
...cursor.lastScannedBySource,
|
|
1064
|
+
};
|
|
951
1065
|
const sourced: Sourced[] = [];
|
|
952
1066
|
const envelopeBytes = Buffer.byteLength(
|
|
953
1067
|
JSON.stringify({ machineId: opts.machineId, installerVersion: opts.installerVersion, events: [] }),
|
|
@@ -956,6 +1070,9 @@ export async function collectAndSendSkillTelemetry(
|
|
|
956
1070
|
const maxRowBytes = MAX_BATCH_BYTES - envelopeBytes;
|
|
957
1071
|
|
|
958
1072
|
for (const { filePath, kind } of files) {
|
|
1073
|
+
const sourceBudgetRemaining = maxScanBytesPerSource - scannedBytes[kind];
|
|
1074
|
+
if (sourceBudgetRemaining <= 0) continue;
|
|
1075
|
+
|
|
959
1076
|
let stat;
|
|
960
1077
|
try {
|
|
961
1078
|
stat = await fs.stat(filePath);
|
|
@@ -967,32 +1084,57 @@ export async function collectAndSendSkillTelemetry(
|
|
|
967
1084
|
|
|
968
1085
|
const stored = cursor.files[filePath] ?? { offset: 0, mtime: 0 };
|
|
969
1086
|
let offset = stored.offset;
|
|
1087
|
+
let pendingLine = clonePendingLine(stored.pendingLine);
|
|
1088
|
+
let codexCursor = kind === "codex" ? cloneCodexCursorState(stored.codex) : undefined;
|
|
970
1089
|
|
|
971
1090
|
// Rotation / truncation → re-read from the top.
|
|
972
1091
|
const rotated =
|
|
973
1092
|
currentSize < offset || (stored.mtime > 0 && currentMtime < stored.mtime);
|
|
974
1093
|
if (rotated) {
|
|
975
1094
|
offset = 0;
|
|
1095
|
+
pendingLine = undefined;
|
|
1096
|
+
codexCursor = undefined;
|
|
976
1097
|
rotationResets[filePath] = { offset: 0, mtime: currentMtime };
|
|
977
1098
|
}
|
|
1099
|
+
// A malformed/stale pending continuation is never allowed to move the
|
|
1100
|
+
// durable complete-line cursor. Start scanning fresh from that cursor.
|
|
1101
|
+
if (
|
|
1102
|
+
pendingLine &&
|
|
1103
|
+
(pendingLine.start !== offset || pendingLine.scannedOffset > currentSize)
|
|
1104
|
+
) {
|
|
1105
|
+
pendingLine = undefined;
|
|
1106
|
+
}
|
|
1107
|
+
const scanOffset = pendingLine?.scannedOffset ?? offset;
|
|
978
1108
|
|
|
979
1109
|
// Record the scan even when there are no new bytes — a fully-drained file
|
|
980
1110
|
// (eventCount 0, offset already at EOF) should still settle at EOF below.
|
|
981
|
-
fileScans[filePath] = {
|
|
1111
|
+
fileScans[filePath] = {
|
|
1112
|
+
eof: currentSize,
|
|
1113
|
+
mtime: currentMtime,
|
|
1114
|
+
eventCount: 0,
|
|
1115
|
+
...(pendingLine ? { pendingLine } : {}),
|
|
1116
|
+
...(codexCursor ? { codex: codexCursor } : {}),
|
|
1117
|
+
};
|
|
982
1118
|
|
|
983
1119
|
if (offset >= currentSize && !rotated) continue;
|
|
984
1120
|
|
|
985
1121
|
let content: string;
|
|
1122
|
+
let bytesRead = 0;
|
|
1123
|
+
let readBuffer = Buffer.alloc(0);
|
|
986
1124
|
try {
|
|
987
1125
|
const fh = await fs.open(filePath, "r");
|
|
988
1126
|
try {
|
|
989
|
-
const length = Math.max(
|
|
990
|
-
|
|
1127
|
+
const length = Math.max(
|
|
1128
|
+
0,
|
|
1129
|
+
Math.min(currentSize - scanOffset, sourceBudgetRemaining, MAX_DECODE_BYTES),
|
|
1130
|
+
);
|
|
1131
|
+
readBuffer = Buffer.alloc(length);
|
|
991
1132
|
// Chunked: a single fh.read() with length > Int32 aborts the process
|
|
992
|
-
// with SIGABRT (HQ-SYNC-WEB-15). `
|
|
993
|
-
//
|
|
994
|
-
|
|
995
|
-
|
|
1133
|
+
// with SIGABRT (HQ-SYNC-WEB-15). `MAX_DECODE_BYTES` separately keeps
|
|
1134
|
+
// the following toString below V8's fatal >= 2**31 boundary; an
|
|
1135
|
+
// oversized configured budget cannot bypass this guard.
|
|
1136
|
+
bytesRead = await readFileRegion(fh, readBuffer, scanOffset, length);
|
|
1137
|
+
content = readBuffer.toString("utf-8", 0, Math.min(bytesRead, MAX_DECODE_BYTES));
|
|
996
1138
|
} finally {
|
|
997
1139
|
await fh.close();
|
|
998
1140
|
}
|
|
@@ -1001,6 +1143,70 @@ export async function collectAndSendSkillTelemetry(
|
|
|
1001
1143
|
delete fileScans[filePath];
|
|
1002
1144
|
continue;
|
|
1003
1145
|
}
|
|
1146
|
+
scannedBytes[kind] += bytesRead;
|
|
1147
|
+
if (bytesRead > 0) lastScannedBySource[kind] = filePath;
|
|
1148
|
+
|
|
1149
|
+
let reachedPhysicalEof = scanOffset + bytesRead >= currentSize;
|
|
1150
|
+
if (!reachedPhysicalEof) {
|
|
1151
|
+
log(`[skill-telemetry] scan budget reached; deferring unread transcript bytes (${filePath})`);
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
let completedPendingLine = false;
|
|
1155
|
+
if (pendingLine) {
|
|
1156
|
+
const newlineIndex = readBuffer.subarray(0, bytesRead).indexOf(0x0a);
|
|
1157
|
+
if (newlineIndex < 0 && !reachedPhysicalEof) {
|
|
1158
|
+
// The record itself is larger than the normal per-pass budget. Persist
|
|
1159
|
+
// only byte positions and continue searching from this new suffix next
|
|
1160
|
+
// pass — never the transcript's contents, and never the same prefix.
|
|
1161
|
+
fileScans[filePath].eof = offset;
|
|
1162
|
+
fileScans[filePath].pendingLine = {
|
|
1163
|
+
start: offset,
|
|
1164
|
+
scannedOffset: scanOffset + bytesRead,
|
|
1165
|
+
};
|
|
1166
|
+
log(`[skill-telemetry] continuing oversized JSONL record on a later pass (${filePath})`);
|
|
1167
|
+
continue;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
const lineEnd =
|
|
1171
|
+
newlineIndex >= 0 ? scanOffset + newlineIndex + 1 : scanOffset + bytesRead;
|
|
1172
|
+
const lineLength = lineEnd - offset;
|
|
1173
|
+
if (lineLength > MAX_DECODE_BYTES) {
|
|
1174
|
+
// This guard still makes the failure recoverable rather than allowing a
|
|
1175
|
+
// native V8 abort. Keep the byte-only continuation so a later file
|
|
1176
|
+
// rotation/truncation can reset it safely; diagnostics reveal the
|
|
1177
|
+
// exceptional transcript without leaking its contents.
|
|
1178
|
+
fileScans[filePath].eof = offset;
|
|
1179
|
+
fileScans[filePath].pendingLine = { start: offset, scannedOffset: lineEnd };
|
|
1180
|
+
log(`[skill-telemetry] JSONL record exceeds safe decode limit (${filePath})`);
|
|
1181
|
+
continue;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
try {
|
|
1185
|
+
const fh = await fs.open(filePath, "r");
|
|
1186
|
+
try {
|
|
1187
|
+
readBuffer = Buffer.alloc(lineLength);
|
|
1188
|
+
bytesRead = await readFileRegion(fh, readBuffer, offset, lineLength);
|
|
1189
|
+
if (bytesRead !== lineLength) {
|
|
1190
|
+
fileScans[filePath].eof = offset;
|
|
1191
|
+
fileScans[filePath].pendingLine = {
|
|
1192
|
+
start: offset,
|
|
1193
|
+
scannedOffset: offset + bytesRead,
|
|
1194
|
+
};
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
content = readBuffer.toString("utf-8", 0, bytesRead);
|
|
1198
|
+
} finally {
|
|
1199
|
+
await fh.close();
|
|
1200
|
+
}
|
|
1201
|
+
} catch {
|
|
1202
|
+
delete fileScans[filePath];
|
|
1203
|
+
continue;
|
|
1204
|
+
}
|
|
1205
|
+
reachedPhysicalEof = lineEnd >= currentSize;
|
|
1206
|
+
completedPendingLine = true;
|
|
1207
|
+
pendingLine = undefined;
|
|
1208
|
+
delete fileScans[filePath].pendingLine;
|
|
1209
|
+
}
|
|
1004
1210
|
|
|
1005
1211
|
// Codex events lack per-row cwd/sessionId — they live in the file's leading
|
|
1006
1212
|
// `session_meta` line, which we read from the top regardless of the cursor.
|
|
@@ -1009,19 +1215,33 @@ export async function collectAndSendSkillTelemetry(
|
|
|
1009
1215
|
// Per-file dedup of model-driven Codex skill loads: a single skill use
|
|
1010
1216
|
// re-reads SKILL.md several times within one turn, so collapse them to one
|
|
1011
1217
|
// event per (session, turn, skill). Scoped per file = per Codex session.
|
|
1012
|
-
const codexSeen =
|
|
1218
|
+
const codexSeen =
|
|
1219
|
+
kind === "codex" ? new Set(codexCursor?.seen ?? []) : undefined;
|
|
1013
1220
|
// Running turn id for Codex: the `function_call` exec shape carries no
|
|
1014
1221
|
// turn_id of its own, so we track the latest one seen (from `turn_context`,
|
|
1015
1222
|
// which precedes a turn's execs) and attribute those execs to it.
|
|
1016
|
-
let codexTurnId: string | undefined;
|
|
1223
|
+
let codexTurnId: string | undefined = codexCursor?.turnId;
|
|
1017
1224
|
|
|
1018
|
-
//
|
|
1225
|
+
// A region capped before EOF can end part-way through a JSONL row (and even
|
|
1226
|
+
// part-way through a multibyte UTF-8 character). Only process its newline-
|
|
1227
|
+
// terminated segments; the next pass resumes at the last settled boundary.
|
|
1228
|
+
// At physical EOF retain today's behaviour of handling a final newline-less
|
|
1229
|
+
// record in the same pass.
|
|
1019
1230
|
const segments = content.split("\n");
|
|
1231
|
+
const completeSegmentCount = reachedPhysicalEof
|
|
1232
|
+
? segments.length
|
|
1233
|
+
: Math.max(0, segments.length - 1);
|
|
1020
1234
|
let cumulative = offset;
|
|
1021
|
-
|
|
1235
|
+
let settledOffset = offset;
|
|
1236
|
+
for (let i = 0; i < completeSegmentCount; i++) {
|
|
1022
1237
|
cumulative += Buffer.byteLength(segments[i], "utf-8");
|
|
1023
1238
|
if (i < segments.length - 1) cumulative += 1; // the split newline byte
|
|
1024
|
-
|
|
1239
|
+
// For the final physical-EOF segment, use the bytes actually returned by
|
|
1240
|
+
// fs rather than re-encoding it. This preserves byte-exact cursor
|
|
1241
|
+
// offsets even if a writer left an invalid/incomplete UTF-8 tail.
|
|
1242
|
+
const endOffset =
|
|
1243
|
+
i === segments.length - 1 ? offset + bytesRead : cumulative;
|
|
1244
|
+
settledOffset = endOffset;
|
|
1025
1245
|
|
|
1026
1246
|
const trimmed = segments[i].trim();
|
|
1027
1247
|
if (trimmed.length === 0) continue;
|
|
@@ -1033,7 +1253,10 @@ export async function collectAndSendSkillTelemetry(
|
|
|
1033
1253
|
}
|
|
1034
1254
|
if (kind === "codex") {
|
|
1035
1255
|
const t = codexRowTurnId(parsed);
|
|
1036
|
-
if (t !== undefined)
|
|
1256
|
+
if (t !== undefined) {
|
|
1257
|
+
if (t !== codexTurnId) codexSeen?.clear();
|
|
1258
|
+
codexTurnId = t;
|
|
1259
|
+
}
|
|
1037
1260
|
}
|
|
1038
1261
|
const events =
|
|
1039
1262
|
kind === "codex"
|
|
@@ -1083,10 +1306,34 @@ export async function collectAndSendSkillTelemetry(
|
|
|
1083
1306
|
`[skill-telemetry] oversized row truncated before send (${filePath}:${i + 1})`,
|
|
1084
1307
|
);
|
|
1085
1308
|
}
|
|
1086
|
-
sourced.push({
|
|
1309
|
+
sourced.push({
|
|
1310
|
+
row: bounded,
|
|
1311
|
+
filePath,
|
|
1312
|
+
endOffset,
|
|
1313
|
+
...(kind === "codex"
|
|
1314
|
+
? { codex: snapshotCodexCursorState(codexTurnId, codexSeen) }
|
|
1315
|
+
: {}),
|
|
1316
|
+
});
|
|
1087
1317
|
fileScans[filePath].eventCount++;
|
|
1088
1318
|
}
|
|
1089
1319
|
}
|
|
1320
|
+
// Commit only bytes that reached a complete record boundary. In particular,
|
|
1321
|
+
// never claim currentSize merely because the first budgeted region was read.
|
|
1322
|
+
fileScans[filePath].eof = settledOffset;
|
|
1323
|
+
if (!completedPendingLine && !reachedPhysicalEof) {
|
|
1324
|
+
const lastNewline = readBuffer.subarray(0, bytesRead).lastIndexOf(0x0a);
|
|
1325
|
+
const partialStart = offset + lastNewline + 1;
|
|
1326
|
+
const inspectedThrough = offset + bytesRead;
|
|
1327
|
+
if (partialStart < inspectedThrough) {
|
|
1328
|
+
fileScans[filePath].pendingLine = {
|
|
1329
|
+
start: partialStart,
|
|
1330
|
+
scannedOffset: inspectedThrough,
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
if (kind === "codex") {
|
|
1335
|
+
fileScans[filePath].codex = snapshotCodexCursorState(codexTurnId, codexSeen);
|
|
1336
|
+
}
|
|
1090
1337
|
}
|
|
1091
1338
|
|
|
1092
1339
|
// 4. Flush in server-sized batches, advancing per-file progress on each 2xx.
|
|
@@ -1095,7 +1342,7 @@ export async function collectAndSendSkillTelemetry(
|
|
|
1095
1342
|
|
|
1096
1343
|
// Per file: count of events successfully sent + max committed byte offset.
|
|
1097
1344
|
const sentCount: Record<string, number> = {};
|
|
1098
|
-
const
|
|
1345
|
+
const committed: Record<string, { offset: number; codex?: CodexCursorState }> = {};
|
|
1099
1346
|
|
|
1100
1347
|
let batch: Sourced[] = [];
|
|
1101
1348
|
let batchBytes = envelopeBytes;
|
|
@@ -1116,8 +1363,13 @@ export async function collectAndSendSkillTelemetry(
|
|
|
1116
1363
|
// Advance per-file progress for the events in this (successful) batch.
|
|
1117
1364
|
for (const s of toSend) {
|
|
1118
1365
|
sentCount[s.filePath] = (sentCount[s.filePath] ?? 0) + 1;
|
|
1119
|
-
const prev =
|
|
1120
|
-
if (s.endOffset > prev)
|
|
1366
|
+
const prev = committed[s.filePath];
|
|
1367
|
+
if (!prev || s.endOffset > prev.offset) {
|
|
1368
|
+
committed[s.filePath] = {
|
|
1369
|
+
offset: s.endOffset,
|
|
1370
|
+
...(s.codex ? { codex: s.codex } : {}),
|
|
1371
|
+
};
|
|
1372
|
+
}
|
|
1121
1373
|
}
|
|
1122
1374
|
return true;
|
|
1123
1375
|
} catch (err) {
|
|
@@ -1155,13 +1407,27 @@ export async function collectAndSendSkillTelemetry(
|
|
|
1155
1407
|
for (const [fp, entry] of Object.entries(rotationResets)) finalFiles[fp] = entry;
|
|
1156
1408
|
for (const [fp, scan] of Object.entries(fileScans)) {
|
|
1157
1409
|
if ((sentCount[fp] ?? 0) >= scan.eventCount) {
|
|
1158
|
-
finalFiles[fp] = {
|
|
1159
|
-
|
|
1160
|
-
|
|
1410
|
+
finalFiles[fp] = {
|
|
1411
|
+
offset: scan.eof,
|
|
1412
|
+
mtime: scan.mtime,
|
|
1413
|
+
...(scan.pendingLine ? { pendingLine: scan.pendingLine } : {}),
|
|
1414
|
+
...(scan.codex ? { codex: scan.codex } : {}),
|
|
1415
|
+
};
|
|
1416
|
+
} else if (fp in committed) {
|
|
1417
|
+
const entry = committed[fp];
|
|
1418
|
+
finalFiles[fp] = {
|
|
1419
|
+
offset: entry.offset,
|
|
1420
|
+
mtime: scan.mtime,
|
|
1421
|
+
...(entry.codex ? { codex: entry.codex } : {}),
|
|
1422
|
+
};
|
|
1161
1423
|
}
|
|
1162
1424
|
// else: no progress for this file — leave loaded/rotation-reset offset.
|
|
1163
1425
|
}
|
|
1164
|
-
await saveCursor(cursorPath, {
|
|
1426
|
+
await saveCursor(cursorPath, {
|
|
1427
|
+
version: "1",
|
|
1428
|
+
files: finalFiles,
|
|
1429
|
+
...(Object.keys(lastScannedBySource).length > 0 ? { lastScannedBySource } : {}),
|
|
1430
|
+
});
|
|
1165
1431
|
|
|
1166
1432
|
return {
|
|
1167
1433
|
enabled: true,
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { spawn } from "node:child_process";
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
|
|
8
|
+
type ChildResult = {
|
|
9
|
+
code: number | null;
|
|
10
|
+
signal: NodeJS.Signals | null;
|
|
11
|
+
stdout: string;
|
|
12
|
+
stderr: string;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
function runNode(scriptPath: string, env: NodeJS.ProcessEnv): Promise<ChildResult> {
|
|
16
|
+
return new Promise((resolve, reject) => {
|
|
17
|
+
const child = spawn(process.execPath, [scriptPath], {
|
|
18
|
+
env,
|
|
19
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
20
|
+
});
|
|
21
|
+
let stdout = "";
|
|
22
|
+
let stderr = "";
|
|
23
|
+
child.stdout.setEncoding("utf-8");
|
|
24
|
+
child.stderr.setEncoding("utf-8");
|
|
25
|
+
child.stdout.on("data", (chunk: string) => {
|
|
26
|
+
stdout += chunk;
|
|
27
|
+
});
|
|
28
|
+
child.stderr.on("data", (chunk: string) => {
|
|
29
|
+
stderr += chunk;
|
|
30
|
+
});
|
|
31
|
+
child.once("error", reject);
|
|
32
|
+
child.once("close", (code, signal) => {
|
|
33
|
+
resolve({ code, signal, stdout, stderr });
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
describe("built sync runner — oversized skill transcript", () => {
|
|
39
|
+
it("drains a multi-budget transcript in a child process without a fatal abort", async () => {
|
|
40
|
+
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "hqcloud-skill-e2e-"));
|
|
41
|
+
try {
|
|
42
|
+
const home = path.join(tmp, "home");
|
|
43
|
+
const hqRoot = path.join(home, "hq");
|
|
44
|
+
const sessionDir = path.join(home, ".claude", "projects", "-hq");
|
|
45
|
+
await fs.mkdir(sessionDir, { recursive: true });
|
|
46
|
+
await fs.mkdir(hqRoot, { recursive: true });
|
|
47
|
+
|
|
48
|
+
// Five 1.5 MiB JSONL records exceed the runner's default 4 MiB source
|
|
49
|
+
// budget. The normal runner entrypoint must therefore advance over three
|
|
50
|
+
// bounded passes instead of allocating/decoding the whole transcript.
|
|
51
|
+
const filler = "x".repeat(1_500_000);
|
|
52
|
+
const expectedSkills = Array.from({ length: 5 }, (_, i) => `deploy-${i}`);
|
|
53
|
+
const transcript = expectedSkills
|
|
54
|
+
.map((skill, i) =>
|
|
55
|
+
JSON.stringify({
|
|
56
|
+
type: "user",
|
|
57
|
+
sessionId: "oversized-session",
|
|
58
|
+
timestamp: `2026-08-02T12:0${i}:00.000Z`,
|
|
59
|
+
cwd: hqRoot,
|
|
60
|
+
uuid: `oversized-${i}`,
|
|
61
|
+
message: {
|
|
62
|
+
role: "user",
|
|
63
|
+
content: `<command-name>/${skill}</command-name><command-args>${filler}</command-args>`,
|
|
64
|
+
},
|
|
65
|
+
}),
|
|
66
|
+
)
|
|
67
|
+
.join("\n") + "\n";
|
|
68
|
+
const transcriptPath = path.join(sessionDir, "oversized.jsonl");
|
|
69
|
+
await fs.writeFile(transcriptPath, transcript, "utf-8");
|
|
70
|
+
|
|
71
|
+
const runnerUrl = pathToFileURL(
|
|
72
|
+
path.join(process.cwd(), "dist", "bin", "sync-runner.js"),
|
|
73
|
+
).href;
|
|
74
|
+
const wrapperPath = path.join(tmp, "run-built-telemetry.mjs");
|
|
75
|
+
await fs.writeFile(
|
|
76
|
+
wrapperPath,
|
|
77
|
+
`import { promises as fs } from "node:fs";
|
|
78
|
+
import path from "node:path";
|
|
79
|
+
import { defaultCollectTelemetry } from ${JSON.stringify(runnerUrl)};
|
|
80
|
+
const hqRoot = process.env.TEST_HQ_ROOT;
|
|
81
|
+
const cursorPath = path.join(process.env.HOME, ".hq", "skill-telemetry-cursor.json");
|
|
82
|
+
const postedSkills = [];
|
|
83
|
+
const client = {
|
|
84
|
+
entity: { listByType: async () => [{ uid: "prs_skill_telemetry_e2e", type: "person", slug: "skill-telemetry-e2e", status: "active", bucketName: "hq-vault-prs_skill_telemetry_e2e", createdAt: "2026-08-02T00:00:00Z" }] },
|
|
85
|
+
getTelemetryOptIn: async () => ({ enabled: true, updatedAt: null }),
|
|
86
|
+
postUsage: async () => ({ ok: true, written: 0, skipped: [] }),
|
|
87
|
+
postSkillInvocations: async (batch) => { postedSkills.push(...batch.events.map((event) => event.skill)); return { ok: true, written: batch.events.length, skipped: [] }; },
|
|
88
|
+
postOutcomeEvents: async () => ({ ok: true, written: 0, skipped: [] }),
|
|
89
|
+
};
|
|
90
|
+
const offsets = [];
|
|
91
|
+
for (let pass = 0; pass < 3; pass += 1) {
|
|
92
|
+
await defaultCollectTelemetry(client, false, hqRoot);
|
|
93
|
+
const cursor = JSON.parse(await fs.readFile(cursorPath, "utf-8"));
|
|
94
|
+
offsets.push(cursor.files[process.env.TEST_TRANSCRIPT_PATH]?.offset ?? 0);
|
|
95
|
+
}
|
|
96
|
+
process.stdout.write(JSON.stringify({ offsets, postedSkills }));\n`,
|
|
97
|
+
"utf-8",
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
const result = await runNode(wrapperPath, {
|
|
101
|
+
...process.env,
|
|
102
|
+
HOME: home,
|
|
103
|
+
TEST_HQ_ROOT: hqRoot,
|
|
104
|
+
TEST_TRANSCRIPT_PATH: transcriptPath,
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
expect(result.signal).toBeNull();
|
|
108
|
+
expect(result.code).toBe(0);
|
|
109
|
+
expect(result.stderr).not.toContain("Fatal error");
|
|
110
|
+
expect(result.stderr).not.toContain("Check failed: i::kMaxInt >= len");
|
|
111
|
+
const output = JSON.parse(result.stdout) as {
|
|
112
|
+
offsets: number[];
|
|
113
|
+
postedSkills: string[];
|
|
114
|
+
};
|
|
115
|
+
expect(output.offsets[0]).toBeGreaterThan(0);
|
|
116
|
+
expect(output.offsets[0]).toBeLessThan(output.offsets[1]);
|
|
117
|
+
expect(output.offsets[1]).toBeLessThan(output.offsets[2]);
|
|
118
|
+
expect(output.offsets[2]).toBe(Buffer.byteLength(transcript, "utf-8"));
|
|
119
|
+
expect(output.postedSkills).toEqual(expectedSkills);
|
|
120
|
+
} finally {
|
|
121
|
+
await fs.rm(tmp, { recursive: true, force: true });
|
|
122
|
+
}
|
|
123
|
+
}, 30_000);
|
|
124
|
+
});
|