@indigoai-us/hq-cloud 6.14.29 → 6.14.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/telemetry.d.ts +9 -4
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +338 -53
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +230 -0
- package/dist/telemetry.test.js.map +1 -1
- package/package.json +1 -1
- package/src/telemetry.test.ts +286 -0
- package/src/telemetry.ts +411 -59
package/src/telemetry.ts
CHANGED
|
@@ -7,9 +7,8 @@
|
|
|
7
7
|
* (`hq-sync-runner`, `hq-cli`, mobile wrappers) emits telemetry uniformly.
|
|
8
8
|
*
|
|
9
9
|
* What it does: after each successful sync (`all-complete` arm of
|
|
10
|
-
* `bin/sync-runner.ts`), walks
|
|
11
|
-
* file against
|
|
12
|
-
* sanitizes new rows through a tight allowlist that matches the server's
|
|
10
|
+
* `bin/sync-runner.ts`), walks Claude session logs plus live and archived Codex
|
|
11
|
+
* rollouts, then diffs each file against the persisted byte-offset cursor at `~/.hq/telemetry-cursor.json`, sanitizes new rows through a tight allowlist that matches the server's
|
|
13
12
|
* KEEP_FIELDS set in `apps/hq-pro/src/vault-service/handlers/usage.ts`,
|
|
14
13
|
* batches into server-sized POST bodies, and ships them to `/v1/usage`.
|
|
15
14
|
*
|
|
@@ -23,6 +22,7 @@
|
|
|
23
22
|
* transient outage retries automatically on the next sync.
|
|
24
23
|
*/
|
|
25
24
|
|
|
25
|
+
import { createHash } from "node:crypto";
|
|
26
26
|
import { promises as fs } from "node:fs";
|
|
27
27
|
import * as os from "node:os";
|
|
28
28
|
import * as path from "node:path";
|
|
@@ -75,6 +75,12 @@ export interface CollectTelemetryOptions {
|
|
|
75
75
|
hqRoot?: string;
|
|
76
76
|
/** Override `~/.claude/projects` for tests. */
|
|
77
77
|
claudeProjectsRoot?: string;
|
|
78
|
+
/** Override `~/.codex` for tests. Both live and archived rollouts are scanned. */
|
|
79
|
+
codexRoot?: string;
|
|
80
|
+
/** Override the per-runtime scan budget for deterministic tests. */
|
|
81
|
+
maxScanBytesPerSource?: number;
|
|
82
|
+
/** Override the upload batch cap for deterministic tests. */
|
|
83
|
+
maxBatchesPerRun?: number;
|
|
78
84
|
/** Override `~/.hq/telemetry-cursor.json` for tests. */
|
|
79
85
|
cursorPath?: string;
|
|
80
86
|
/** Override `~/.hq/menubar.json` (the offline opt-in fallback) for tests. */
|
|
@@ -98,14 +104,28 @@ export interface CollectTelemetryResult {
|
|
|
98
104
|
|
|
99
105
|
// ── Cursor schema ─────────────────────────────────────────────────────────────
|
|
100
106
|
|
|
107
|
+
interface CodexUsageContext {
|
|
108
|
+
session_id?: string;
|
|
109
|
+
cwd?: string;
|
|
110
|
+
git_branch?: string;
|
|
111
|
+
session_model?: string;
|
|
112
|
+
collaboration_model?: string;
|
|
113
|
+
turn_model?: string;
|
|
114
|
+
discarding_partial_line?: boolean;
|
|
115
|
+
}
|
|
116
|
+
|
|
101
117
|
interface CursorEntry {
|
|
102
118
|
offset: number;
|
|
103
119
|
mtime: number;
|
|
120
|
+
/** Runtime context needed to resume a Codex rollout from a byte offset. */
|
|
121
|
+
context?: CodexUsageContext;
|
|
104
122
|
}
|
|
105
123
|
|
|
106
124
|
interface TelemetryCursor {
|
|
107
125
|
version: string;
|
|
108
126
|
files: Record<string, CursorEntry>;
|
|
127
|
+
/** Preserved for cursor compatibility with the desktop collector. */
|
|
128
|
+
codex_next_rollout?: string;
|
|
109
129
|
}
|
|
110
130
|
|
|
111
131
|
function emptyCursor(): TelemetryCursor {
|
|
@@ -117,7 +137,13 @@ async function loadCursor(cursorPath: string): Promise<TelemetryCursor> {
|
|
|
117
137
|
const raw = await fs.readFile(cursorPath, "utf-8");
|
|
118
138
|
const parsed = JSON.parse(raw) as Partial<TelemetryCursor>;
|
|
119
139
|
if (parsed && typeof parsed === "object" && parsed.files && typeof parsed.files === "object") {
|
|
120
|
-
return {
|
|
140
|
+
return {
|
|
141
|
+
version: parsed.version ?? "1",
|
|
142
|
+
files: parsed.files as Record<string, CursorEntry>,
|
|
143
|
+
...(typeof parsed.codex_next_rollout === "string"
|
|
144
|
+
? { codex_next_rollout: parsed.codex_next_rollout }
|
|
145
|
+
: {}),
|
|
146
|
+
};
|
|
121
147
|
}
|
|
122
148
|
} catch {
|
|
123
149
|
// Missing / unparseable — start fresh.
|
|
@@ -377,16 +403,196 @@ async function listJsonlFiles(root: string): Promise<string[]> {
|
|
|
377
403
|
return out;
|
|
378
404
|
}
|
|
379
405
|
|
|
406
|
+
// ── Codex rollout adapter ────────────────────────────────────────────────────
|
|
407
|
+
|
|
408
|
+
const MAX_ID_BYTES = 256;
|
|
409
|
+
const MAX_TIMESTAMP_BYTES = 128;
|
|
410
|
+
const MAX_PATH_BYTES = 4 * 1024;
|
|
411
|
+
const MAX_MODEL_BYTES = 256;
|
|
412
|
+
|
|
413
|
+
function boundedString(value: unknown, maxBytes: number): string | undefined {
|
|
414
|
+
return typeof value === "string" &&
|
|
415
|
+
value.length > 0 &&
|
|
416
|
+
Buffer.byteLength(value, "utf-8") <= maxBytes
|
|
417
|
+
? value
|
|
418
|
+
: undefined;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
function tokenCount(value: unknown): number {
|
|
422
|
+
return typeof value === "number" &&
|
|
423
|
+
Number.isSafeInteger(value) &&
|
|
424
|
+
value >= 0
|
|
425
|
+
? value
|
|
426
|
+
: 0;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function cloneCodexContext(context: CodexUsageContext): CodexUsageContext {
|
|
430
|
+
return { ...context };
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function codexModel(context: CodexUsageContext): string | undefined {
|
|
434
|
+
return context.turn_model ?? context.collaboration_model ?? context.session_model;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function codexGitBranch(payload: Record<string, unknown>): string | undefined {
|
|
438
|
+
const direct =
|
|
439
|
+
boundedString(payload.gitBranch, MAX_PATH_BYTES) ??
|
|
440
|
+
boundedString(payload.git_branch, MAX_PATH_BYTES);
|
|
441
|
+
if (direct) return direct;
|
|
442
|
+
const git = payload.git;
|
|
443
|
+
return git && typeof git === "object" && !Array.isArray(git)
|
|
444
|
+
? boundedString((git as Record<string, unknown>).branch, MAX_PATH_BYTES)
|
|
445
|
+
: undefined;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function collaborationModel(
|
|
449
|
+
payload: Record<string, unknown>,
|
|
450
|
+
): string | undefined {
|
|
451
|
+
const settings = payload.settings;
|
|
452
|
+
if (!settings || typeof settings !== "object" || Array.isArray(settings)) {
|
|
453
|
+
return undefined;
|
|
454
|
+
}
|
|
455
|
+
return boundedString((settings as Record<string, unknown>).model, MAX_MODEL_BYTES);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
function stableCodexEventId(
|
|
459
|
+
rolloutIdentity: string,
|
|
460
|
+
startOffset: number,
|
|
461
|
+
endOffset: number,
|
|
462
|
+
): string {
|
|
463
|
+
const offsets = Buffer.alloc(16);
|
|
464
|
+
offsets.writeBigUInt64BE(BigInt(startOffset), 0);
|
|
465
|
+
offsets.writeBigUInt64BE(BigInt(endOffset), 8);
|
|
466
|
+
return "codex-" + createHash("sha256")
|
|
467
|
+
.update(rolloutIdentity)
|
|
468
|
+
.update(Buffer.from([0]))
|
|
469
|
+
.update(offsets)
|
|
470
|
+
.digest("hex");
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Adapt a Codex rollout record to the Claude-shaped sanitizer input. */
|
|
474
|
+
function codexUsageRow(
|
|
475
|
+
row: unknown,
|
|
476
|
+
context: CodexUsageContext,
|
|
477
|
+
rolloutIdentity: string,
|
|
478
|
+
startOffset: number,
|
|
479
|
+
endOffset: number,
|
|
480
|
+
): Record<string, unknown> | null {
|
|
481
|
+
if (!row || typeof row !== "object" || Array.isArray(row)) return null;
|
|
482
|
+
const obj = row as Record<string, unknown>;
|
|
483
|
+
const kind = obj.type;
|
|
484
|
+
const rawPayload = obj.payload;
|
|
485
|
+
const payload =
|
|
486
|
+
rawPayload && typeof rawPayload === "object" && !Array.isArray(rawPayload)
|
|
487
|
+
? (rawPayload as Record<string, unknown>)
|
|
488
|
+
: undefined;
|
|
489
|
+
|
|
490
|
+
if (kind === "session_meta") {
|
|
491
|
+
if (!payload) return null;
|
|
492
|
+
context.session_id = boundedString(payload.id, MAX_ID_BYTES) ?? context.session_id;
|
|
493
|
+
context.cwd = boundedString(payload.cwd, MAX_PATH_BYTES);
|
|
494
|
+
context.git_branch = codexGitBranch(payload);
|
|
495
|
+
context.session_model = boundedString(payload.model, MAX_MODEL_BYTES);
|
|
496
|
+
return null;
|
|
497
|
+
}
|
|
498
|
+
if (kind === "turn_context") {
|
|
499
|
+
context.turn_model = payload
|
|
500
|
+
? boundedString(payload.model, MAX_MODEL_BYTES)
|
|
501
|
+
: undefined;
|
|
502
|
+
const mode = payload?.collaboration_mode;
|
|
503
|
+
if (mode && typeof mode === "object" && !Array.isArray(mode)) {
|
|
504
|
+
const model = collaborationModel(mode as Record<string, unknown>);
|
|
505
|
+
if (model) context.collaboration_model = model;
|
|
506
|
+
}
|
|
507
|
+
return null;
|
|
508
|
+
}
|
|
509
|
+
if (kind === "collaboration_mode") {
|
|
510
|
+
context.collaboration_model = payload
|
|
511
|
+
? collaborationModel(payload)
|
|
512
|
+
: undefined;
|
|
513
|
+
return null;
|
|
514
|
+
}
|
|
515
|
+
if (kind !== "event_msg" || !payload) return null;
|
|
516
|
+
if (payload.type === "collaboration_mode") {
|
|
517
|
+
context.collaboration_model = collaborationModel(payload);
|
|
518
|
+
return null;
|
|
519
|
+
}
|
|
520
|
+
if (payload.type !== "token_count") return null;
|
|
521
|
+
|
|
522
|
+
const info = payload.info;
|
|
523
|
+
if (!info || typeof info !== "object" || Array.isArray(info)) return null;
|
|
524
|
+
const last = (info as Record<string, unknown>).last_token_usage;
|
|
525
|
+
if (!last || typeof last !== "object" || Array.isArray(last)) return null;
|
|
526
|
+
const usage = last as Record<string, unknown>;
|
|
527
|
+
const outputTokens =
|
|
528
|
+
tokenCount(usage.output_tokens) + tokenCount(usage.reasoning_output_tokens);
|
|
529
|
+
const model = codexModel(context);
|
|
530
|
+
const normalized: Record<string, unknown> = {
|
|
531
|
+
uuid:
|
|
532
|
+
boundedString(obj.uuid, MAX_ID_BYTES) ??
|
|
533
|
+
stableCodexEventId(rolloutIdentity, startOffset, endOffset),
|
|
534
|
+
message: {
|
|
535
|
+
usage: {
|
|
536
|
+
input_tokens: tokenCount(usage.input_tokens),
|
|
537
|
+
output_tokens: outputTokens,
|
|
538
|
+
...(typeof usage.cached_input_tokens === "number"
|
|
539
|
+
? { cache_read_input_tokens: tokenCount(usage.cached_input_tokens) }
|
|
540
|
+
: {}),
|
|
541
|
+
},
|
|
542
|
+
...(model ? { model } : {}),
|
|
543
|
+
},
|
|
544
|
+
};
|
|
545
|
+
if (context.session_id) normalized.sessionId = context.session_id;
|
|
546
|
+
const timestamp = boundedString(obj.timestamp, MAX_TIMESTAMP_BYTES);
|
|
547
|
+
if (timestamp) normalized.timestamp = timestamp;
|
|
548
|
+
if (context.cwd) normalized.cwd = context.cwd;
|
|
549
|
+
if (context.git_branch) normalized.gitBranch = context.git_branch;
|
|
550
|
+
return normalized;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
async function sortFilesFreshestFirst(files: string[]): Promise<string[]> {
|
|
554
|
+
const ranked = await Promise.all(
|
|
555
|
+
files.map(async (filePath) => {
|
|
556
|
+
try {
|
|
557
|
+
return { filePath, mtime: (await fs.stat(filePath)).mtimeMs };
|
|
558
|
+
} catch {
|
|
559
|
+
return { filePath, mtime: 0 };
|
|
560
|
+
}
|
|
561
|
+
}),
|
|
562
|
+
);
|
|
563
|
+
ranked.sort(
|
|
564
|
+
(left, right) =>
|
|
565
|
+
right.mtime - left.mtime || left.filePath.localeCompare(right.filePath),
|
|
566
|
+
);
|
|
567
|
+
return ranked.map(({ filePath }) => filePath);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
async function listCodexRolloutFiles(codexRoot: string): Promise<string[]> {
|
|
571
|
+
const [live, archived] = await Promise.all([
|
|
572
|
+
listJsonlFiles(path.join(codexRoot, "sessions")),
|
|
573
|
+
listJsonlFiles(path.join(codexRoot, "archived_sessions")),
|
|
574
|
+
]);
|
|
575
|
+
return sortFilesFreshestFirst(
|
|
576
|
+
[...live, ...archived].filter((file) =>
|
|
577
|
+
path.basename(file).startsWith("rollout-"),
|
|
578
|
+
),
|
|
579
|
+
);
|
|
580
|
+
}
|
|
581
|
+
|
|
380
582
|
// ── Batching primitives ───────────────────────────────────────────────────────
|
|
381
583
|
|
|
382
584
|
const MAX_BATCH_EVENTS = 100;
|
|
383
585
|
const MAX_BATCH_BYTES = 240 * 1024;
|
|
586
|
+
const MAX_SCAN_BYTES_PER_SOURCE = 4 * 1024 * 1024;
|
|
587
|
+
const MAX_BATCHES_PER_RUN = 4;
|
|
588
|
+
const MAX_PARTIAL_LINE_BYTES = 64 * 1024;
|
|
384
589
|
const ROW_TRUNCATION_SUFFIX = "...[truncated]";
|
|
385
590
|
|
|
386
591
|
interface RowSource {
|
|
387
592
|
filePath: string;
|
|
388
593
|
endOffset: number;
|
|
389
594
|
mtime: number;
|
|
595
|
+
context?: CodexUsageContext;
|
|
390
596
|
}
|
|
391
597
|
|
|
392
598
|
/**
|
|
@@ -455,7 +661,7 @@ function boundRowForPost(
|
|
|
455
661
|
// ── Main entry point ──────────────────────────────────────────────────────────
|
|
456
662
|
|
|
457
663
|
/**
|
|
458
|
-
* Scan, sanitize, and POST
|
|
664
|
+
* Scan, sanitize, and POST new Claude Code and Codex usage rows.
|
|
459
665
|
*
|
|
460
666
|
* Fire-and-forget from the caller's perspective: errors are caught internally
|
|
461
667
|
* and surfaced only via `log`. The returned summary lets observers (e.g.
|
|
@@ -467,9 +673,13 @@ export async function collectAndSendTelemetry(
|
|
|
467
673
|
): Promise<CollectTelemetryResult> {
|
|
468
674
|
const home = os.homedir();
|
|
469
675
|
const claudeProjectsRoot = opts.claudeProjectsRoot ?? path.join(home, ".claude", "projects");
|
|
676
|
+
const codexRoot = opts.codexRoot ?? path.join(home, ".codex");
|
|
470
677
|
const cursorPath = opts.cursorPath ?? path.join(home, ".hq", "telemetry-cursor.json");
|
|
471
678
|
const menubarPath = opts.menubarPath ?? path.join(home, ".hq", "menubar.json");
|
|
472
679
|
const log = opts.log ?? (() => {});
|
|
680
|
+
const maxScanBytesPerSource =
|
|
681
|
+
opts.maxScanBytesPerSource ?? MAX_SCAN_BYTES_PER_SOURCE;
|
|
682
|
+
const maxBatchesPerRun = opts.maxBatchesPerRun ?? MAX_BATCHES_PER_RUN;
|
|
473
683
|
|
|
474
684
|
// Company attribution (US-002): parse the manifest ONCE per run and reuse the
|
|
475
685
|
// repo-path→companyUid map for every event below. No per-event manifest read.
|
|
@@ -565,7 +775,17 @@ export async function collectAndSendTelemetry(
|
|
|
565
775
|
const rotationResets: Record<string, CursorEntry> = {};
|
|
566
776
|
const newlyCommitted: Record<string, CursorEntry> = {};
|
|
567
777
|
|
|
568
|
-
const
|
|
778
|
+
const [claudeFilesUnsorted, codexFiles] = await Promise.all([
|
|
779
|
+
listJsonlFiles(claudeProjectsRoot),
|
|
780
|
+
listCodexRolloutFiles(codexRoot),
|
|
781
|
+
]);
|
|
782
|
+
const claudeFiles = await sortFilesFreshestFirst(claudeFilesUnsorted);
|
|
783
|
+
// Codex goes first so a fresh Linux install cannot spend its entire bounded
|
|
784
|
+
// pass replaying an older Claude backlog before reporting current Codex use.
|
|
785
|
+
const files: Array<{ filePath: string; kind: "claude" | "codex" }> = [
|
|
786
|
+
...codexFiles.map((filePath) => ({ filePath, kind: "codex" as const })),
|
|
787
|
+
...claudeFiles.map((filePath) => ({ filePath, kind: "claude" as const })),
|
|
788
|
+
];
|
|
569
789
|
|
|
570
790
|
// 3. Walk each file, sanitize new rows, batch, flush at the server contract.
|
|
571
791
|
//
|
|
@@ -581,15 +801,45 @@ export async function collectAndSendTelemetry(
|
|
|
581
801
|
let batchBytes = ENVELOPE_BYTES;
|
|
582
802
|
let eventsSent = 0;
|
|
583
803
|
let batchesSent = 0;
|
|
804
|
+
let uploadFailed = false;
|
|
805
|
+
let batchLimitReached = false;
|
|
806
|
+
const scannedBytes = { claude: 0, codex: 0 };
|
|
584
807
|
|
|
585
|
-
const
|
|
586
|
-
|
|
808
|
+
const commitSources = (sources: RowSource[]): void => {
|
|
809
|
+
const maxPerFile = new Map<
|
|
810
|
+
string,
|
|
811
|
+
{ mtime: number; offset: number; context?: CodexUsageContext }
|
|
812
|
+
>();
|
|
813
|
+
for (const source of sources) {
|
|
814
|
+
const current = maxPerFile.get(source.filePath);
|
|
815
|
+
if (!current || source.endOffset > current.offset) {
|
|
816
|
+
maxPerFile.set(source.filePath, {
|
|
817
|
+
mtime: source.mtime,
|
|
818
|
+
offset: source.endOffset,
|
|
819
|
+
...(source.context
|
|
820
|
+
? { context: cloneCodexContext(source.context) }
|
|
821
|
+
: {}),
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
for (const [filePath, entry] of maxPerFile) {
|
|
826
|
+
const current = newlyCommitted[filePath];
|
|
827
|
+
if (!current || entry.offset > current.offset) {
|
|
828
|
+
newlyCommitted[filePath] = entry;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
};
|
|
832
|
+
|
|
833
|
+
const flush = async (): Promise<boolean> => {
|
|
587
834
|
const events = batchEvents;
|
|
588
835
|
const sources = batchSources;
|
|
589
836
|
batchEvents = [];
|
|
590
837
|
batchSources = [];
|
|
591
838
|
batchBytes = ENVELOPE_BYTES;
|
|
592
|
-
|
|
839
|
+
if (events.length === 0) {
|
|
840
|
+
commitSources(sources);
|
|
841
|
+
return true;
|
|
842
|
+
}
|
|
593
843
|
try {
|
|
594
844
|
await opts.client.postUsage({
|
|
595
845
|
machineId: opts.machineId,
|
|
@@ -598,24 +848,26 @@ export async function collectAndSendTelemetry(
|
|
|
598
848
|
});
|
|
599
849
|
batchesSent++;
|
|
600
850
|
eventsSent += events.length;
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
for (const src of sources) {
|
|
604
|
-
const cur = maxPerFile.get(src.filePath);
|
|
605
|
-
if (!cur || src.endOffset > cur.offset) {
|
|
606
|
-
maxPerFile.set(src.filePath, { mtime: src.mtime, offset: src.endOffset });
|
|
607
|
-
}
|
|
608
|
-
}
|
|
609
|
-
for (const [fp, entry] of maxPerFile) {
|
|
610
|
-
newlyCommitted[fp] = { offset: entry.offset, mtime: entry.mtime };
|
|
611
|
-
}
|
|
851
|
+
commitSources(sources);
|
|
852
|
+
return true;
|
|
612
853
|
} catch (err) {
|
|
613
|
-
log(
|
|
614
|
-
|
|
854
|
+
log(
|
|
855
|
+
"[telemetry] postUsage failed (" +
|
|
856
|
+
((err as Error).message ?? err) +
|
|
857
|
+
") — cursor not advanced for " +
|
|
858
|
+
sources.length +
|
|
859
|
+
" rows",
|
|
860
|
+
);
|
|
861
|
+
return false;
|
|
615
862
|
}
|
|
616
863
|
};
|
|
617
864
|
|
|
618
|
-
for (const filePath of files) {
|
|
865
|
+
filesLoop: for (const { filePath, kind } of files) {
|
|
866
|
+
if (uploadFailed || batchLimitReached) break;
|
|
867
|
+
const sourceBudgetRemaining =
|
|
868
|
+
maxScanBytesPerSource - scannedBytes[kind];
|
|
869
|
+
if (sourceBudgetRemaining <= 0) continue;
|
|
870
|
+
|
|
619
871
|
let stat;
|
|
620
872
|
try {
|
|
621
873
|
stat = await fs.stat(filePath);
|
|
@@ -624,116 +876,216 @@ export async function collectAndSendTelemetry(
|
|
|
624
876
|
}
|
|
625
877
|
const currentSize = stat.size;
|
|
626
878
|
const currentMtime = Math.floor(stat.mtimeMs / 1000);
|
|
627
|
-
|
|
628
879
|
const stored = cursor.files[filePath] ?? { offset: 0, mtime: 0 };
|
|
629
880
|
let offset = stored.offset;
|
|
630
|
-
|
|
881
|
+
let codexContext: CodexUsageContext | undefined =
|
|
882
|
+
kind === "codex" ? cloneCodexContext(stored.context ?? {}) : undefined;
|
|
631
883
|
const rotated =
|
|
632
884
|
currentSize < offset || (stored.mtime > 0 && currentMtime < stored.mtime);
|
|
633
885
|
if (rotated) {
|
|
634
886
|
offset = 0;
|
|
887
|
+
if (kind === "codex") codexContext = {};
|
|
635
888
|
rotationResets[filePath] = { offset: 0, mtime: currentMtime };
|
|
636
889
|
}
|
|
637
|
-
|
|
638
890
|
if (offset >= currentSize && !rotated) continue;
|
|
639
891
|
|
|
640
892
|
let content: string;
|
|
893
|
+
let bytesRead = 0;
|
|
894
|
+
let readBuffer = Buffer.alloc(0);
|
|
641
895
|
try {
|
|
642
|
-
const
|
|
896
|
+
const fileHandle = await fs.open(filePath, "r");
|
|
643
897
|
try {
|
|
644
|
-
const length = Math.max(
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
898
|
+
const length = Math.max(
|
|
899
|
+
0,
|
|
900
|
+
Math.min(currentSize - offset, sourceBudgetRemaining),
|
|
901
|
+
);
|
|
902
|
+
readBuffer = Buffer.alloc(length);
|
|
903
|
+
({ bytesRead } = await fileHandle.read(
|
|
904
|
+
readBuffer,
|
|
905
|
+
0,
|
|
906
|
+
length,
|
|
907
|
+
offset,
|
|
908
|
+
));
|
|
909
|
+
content = readBuffer.toString("utf-8", 0, bytesRead);
|
|
648
910
|
} finally {
|
|
649
|
-
await
|
|
911
|
+
await fileHandle.close();
|
|
650
912
|
}
|
|
651
913
|
} catch {
|
|
652
914
|
continue;
|
|
653
915
|
}
|
|
654
|
-
if (
|
|
916
|
+
if (bytesRead === 0) continue;
|
|
917
|
+
|
|
918
|
+
// Resume a previously discarded oversized Codex row using raw byte
|
|
919
|
+
// positions. Decoding from the middle of a multibyte UTF-8 codepoint would
|
|
920
|
+
// otherwise skew byte offsets and could skip or replay later rows.
|
|
921
|
+
if (codexContext?.discarding_partial_line) {
|
|
922
|
+
const newlineIndex = readBuffer.subarray(0, bytesRead).indexOf(0x0a);
|
|
923
|
+
const discardedBytes = newlineIndex >= 0 ? newlineIndex + 1 : bytesRead;
|
|
924
|
+
codexContext.discarding_partial_line = newlineIndex < 0;
|
|
925
|
+
scannedBytes[kind] += discardedBytes;
|
|
926
|
+
batchSources.push({
|
|
927
|
+
filePath,
|
|
928
|
+
endOffset: offset + discardedBytes,
|
|
929
|
+
mtime: currentMtime,
|
|
930
|
+
context: cloneCodexContext(codexContext),
|
|
931
|
+
});
|
|
932
|
+
if (newlineIndex < 0) continue;
|
|
933
|
+
offset += discardedBytes;
|
|
934
|
+
bytesRead -= discardedBytes;
|
|
935
|
+
readBuffer = readBuffer.subarray(discardedBytes);
|
|
936
|
+
content = readBuffer.toString("utf-8", 0, bytesRead);
|
|
937
|
+
if (bytesRead === 0) continue;
|
|
938
|
+
}
|
|
655
939
|
|
|
940
|
+
const reachedPhysicalEof = offset + bytesRead >= currentSize;
|
|
656
941
|
const segments = content.split("\n");
|
|
657
942
|
const lineEndOffsets: number[] = [];
|
|
658
943
|
let cumulative = 0;
|
|
659
944
|
for (let i = 0; i < segments.length; i++) {
|
|
660
945
|
cumulative += Buffer.byteLength(segments[i], "utf-8");
|
|
661
|
-
if (i < segments.length - 1) cumulative += 1;
|
|
946
|
+
if (i < segments.length - 1) cumulative += 1;
|
|
662
947
|
lineEndOffsets.push(offset + cumulative);
|
|
663
948
|
}
|
|
664
949
|
|
|
665
950
|
for (let i = 0; i < segments.length; i++) {
|
|
951
|
+
const startOffset = i === 0 ? offset : lineEndOffsets[i - 1];
|
|
952
|
+
const hasNewline = i < segments.length - 1;
|
|
953
|
+
const endOffset =
|
|
954
|
+
!hasNewline && !reachedPhysicalEof
|
|
955
|
+
? offset + bytesRead
|
|
956
|
+
: lineEndOffsets[i];
|
|
957
|
+
const lineBytes = endOffset - startOffset;
|
|
958
|
+
if (!hasNewline && !reachedPhysicalEof) {
|
|
959
|
+
const discarding = codexContext?.discarding_partial_line === true;
|
|
960
|
+
if (discarding || lineBytes >= MAX_PARTIAL_LINE_BYTES) {
|
|
961
|
+
if (codexContext) codexContext.discarding_partial_line = true;
|
|
962
|
+
scannedBytes[kind] += lineBytes;
|
|
963
|
+
batchSources.push({
|
|
964
|
+
filePath,
|
|
965
|
+
endOffset,
|
|
966
|
+
mtime: currentMtime,
|
|
967
|
+
...(codexContext
|
|
968
|
+
? { context: cloneCodexContext(codexContext) }
|
|
969
|
+
: {}),
|
|
970
|
+
});
|
|
971
|
+
}
|
|
972
|
+
break;
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
scannedBytes[kind] += lineBytes;
|
|
976
|
+
const source = (): RowSource => ({
|
|
977
|
+
filePath,
|
|
978
|
+
endOffset,
|
|
979
|
+
mtime: currentMtime,
|
|
980
|
+
...(codexContext
|
|
981
|
+
? { context: cloneCodexContext(codexContext) }
|
|
982
|
+
: {}),
|
|
983
|
+
});
|
|
984
|
+
if (codexContext?.discarding_partial_line) {
|
|
985
|
+
codexContext.discarding_partial_line = false;
|
|
986
|
+
batchSources.push(source());
|
|
987
|
+
continue;
|
|
988
|
+
}
|
|
666
989
|
const trimmed = segments[i].trim();
|
|
667
|
-
if (trimmed.length === 0)
|
|
990
|
+
if (trimmed.length === 0) {
|
|
991
|
+
batchSources.push(source());
|
|
992
|
+
continue;
|
|
993
|
+
}
|
|
668
994
|
let parsed: unknown;
|
|
669
995
|
try {
|
|
670
996
|
parsed = JSON.parse(trimmed);
|
|
671
997
|
} catch {
|
|
998
|
+
batchSources.push(source());
|
|
672
999
|
continue;
|
|
673
1000
|
}
|
|
674
|
-
|
|
675
|
-
|
|
1001
|
+
const sourceRow =
|
|
1002
|
+
kind === "codex"
|
|
1003
|
+
? codexUsageRow(
|
|
1004
|
+
parsed,
|
|
1005
|
+
codexContext!,
|
|
1006
|
+
filePath,
|
|
1007
|
+
startOffset,
|
|
1008
|
+
endOffset,
|
|
1009
|
+
)
|
|
1010
|
+
: parsed;
|
|
676
1011
|
const rowCwd =
|
|
677
|
-
|
|
678
|
-
? (
|
|
1012
|
+
sourceRow && typeof sourceRow === "object" && !Array.isArray(sourceRow)
|
|
1013
|
+
? (sourceRow as Record<string, unknown>).cwd
|
|
679
1014
|
: undefined;
|
|
680
1015
|
const companyUid = resolveCompanyForCwd(
|
|
681
1016
|
typeof rowCwd === "string" ? rowCwd : undefined,
|
|
682
1017
|
repoCompanyMap,
|
|
683
1018
|
);
|
|
684
|
-
const sanitized = sanitizeRow(
|
|
685
|
-
if (!sanitized)
|
|
1019
|
+
const sanitized = sanitizeRow(sourceRow, companyUid);
|
|
1020
|
+
if (!sanitized) {
|
|
1021
|
+
batchSources.push(source());
|
|
1022
|
+
continue;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
686
1025
|
const maxRowBytes = MAX_BATCH_BYTES - ENVELOPE_BYTES;
|
|
687
1026
|
const wasOversized = jsonBytes(sanitized) > maxRowBytes;
|
|
688
1027
|
const bounded = boundRowForPost(sanitized, maxRowBytes);
|
|
689
1028
|
if (!bounded) {
|
|
690
1029
|
log(
|
|
691
|
-
|
|
1030
|
+
"[telemetry] oversized row dropped before send (" +
|
|
1031
|
+
filePath +
|
|
1032
|
+
":" +
|
|
1033
|
+
(i + 1) +
|
|
1034
|
+
")",
|
|
692
1035
|
);
|
|
1036
|
+
batchSources.push(source());
|
|
693
1037
|
continue;
|
|
694
1038
|
}
|
|
695
1039
|
if (wasOversized) {
|
|
696
1040
|
log(
|
|
697
|
-
|
|
1041
|
+
"[telemetry] oversized row truncated before send (" +
|
|
1042
|
+
filePath +
|
|
1043
|
+
":" +
|
|
1044
|
+
(i + 1) +
|
|
1045
|
+
")",
|
|
698
1046
|
);
|
|
699
1047
|
}
|
|
700
1048
|
|
|
701
|
-
|
|
702
|
-
// length plus 1 byte for the leading comma when there's already at
|
|
703
|
-
// least one row. (No comma when the batch is empty — the row sits
|
|
704
|
-
// alone inside the events array.)
|
|
705
|
-
const rowJsonBytes = Buffer.byteLength(JSON.stringify(bounded), "utf-8");
|
|
1049
|
+
const rowJsonBytes = jsonBytes(bounded);
|
|
706
1050
|
const addCost = rowJsonBytes + (batchEvents.length > 0 ? 1 : 0);
|
|
707
|
-
|
|
708
1051
|
if (
|
|
709
1052
|
batchEvents.length > 0 &&
|
|
710
1053
|
(batchEvents.length >= MAX_BATCH_EVENTS ||
|
|
711
1054
|
batchBytes + addCost > MAX_BATCH_BYTES)
|
|
712
1055
|
) {
|
|
713
|
-
await flush()
|
|
714
|
-
|
|
1056
|
+
if (!(await flush())) {
|
|
1057
|
+
uploadFailed = true;
|
|
1058
|
+
break filesLoop;
|
|
1059
|
+
}
|
|
1060
|
+
if (batchesSent >= maxBatchesPerRun) {
|
|
1061
|
+
batchLimitReached = true;
|
|
1062
|
+
break filesLoop;
|
|
1063
|
+
}
|
|
715
1064
|
batchBytes = ENVELOPE_BYTES + rowJsonBytes;
|
|
716
1065
|
} else {
|
|
717
1066
|
batchBytes += addCost;
|
|
718
1067
|
}
|
|
719
|
-
|
|
720
1068
|
batchEvents.push(bounded);
|
|
721
|
-
batchSources.push(
|
|
722
|
-
filePath,
|
|
723
|
-
endOffset: lineEndOffsets[i],
|
|
724
|
-
mtime: currentMtime,
|
|
725
|
-
});
|
|
1069
|
+
batchSources.push(source());
|
|
726
1070
|
}
|
|
727
1071
|
}
|
|
728
1072
|
|
|
729
|
-
|
|
1073
|
+
if (!uploadFailed && !batchLimitReached) {
|
|
1074
|
+
uploadFailed = !(await flush());
|
|
1075
|
+
}
|
|
730
1076
|
|
|
731
1077
|
// 4. Persist cursor: loaded < rotation_resets < newly_committed.
|
|
732
1078
|
const finalFiles: Record<string, CursorEntry> = { ...loadedFiles };
|
|
733
1079
|
for (const [fp, entry] of Object.entries(rotationResets)) finalFiles[fp] = entry;
|
|
734
1080
|
for (const [fp, entry] of Object.entries(newlyCommitted)) finalFiles[fp] = entry;
|
|
735
1081
|
|
|
736
|
-
await saveCursor(cursorPath, {
|
|
1082
|
+
await saveCursor(cursorPath, {
|
|
1083
|
+
version: "1",
|
|
1084
|
+
files: finalFiles,
|
|
1085
|
+
...(cursor.codex_next_rollout
|
|
1086
|
+
? { codex_next_rollout: cursor.codex_next_rollout }
|
|
1087
|
+
: {}),
|
|
1088
|
+
});
|
|
737
1089
|
|
|
738
1090
|
return {
|
|
739
1091
|
enabled: true,
|