@indigoai-us/hq-cloud 6.15.18 → 6.15.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +1 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +6 -3
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/conflict.d.ts +25 -2
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +29 -4
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +95 -45
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +343 -66
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +23 -29
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +97 -4
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/telemetry.d.ts +23 -5
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +264 -75
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +391 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +4 -13
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/telemetry.d.ts
CHANGED
|
@@ -7,10 +7,12 @@
|
|
|
7
7
|
* (`hq-sync-runner`, `hq-cli`, mobile wrappers) emits telemetry uniformly.
|
|
8
8
|
*
|
|
9
9
|
* What it does: after each successful sync (`all-complete` arm of
|
|
10
|
-
* `bin/sync-runner.ts`), walks
|
|
11
|
-
* rollouts, then diffs each file against the
|
|
12
|
-
*
|
|
13
|
-
*
|
|
10
|
+
* `bin/sync-runner.ts`), walks Grok `updates.jsonl` sessions, live and archived
|
|
11
|
+
* Codex rollouts, and Claude session logs, then diffs each file against the
|
|
12
|
+
* persisted byte-offset cursor at `~/.hq/telemetry-cursor.json`, sanitizes new
|
|
13
|
+
* rows through a tight allowlist that matches the server's KEEP_FIELDS set in
|
|
14
|
+
* `apps/hq-pro/src/vault-service/handlers/usage.ts`, batches into server-sized
|
|
15
|
+
* POST bodies, and ships them to `/v1/usage`.
|
|
14
16
|
*
|
|
15
17
|
* Trust model: the caller's `personUid` is resolved on the server from the
|
|
16
18
|
* Cognito JWT — never from the body. `sanitizeRow` strips prompt bodies,
|
|
@@ -65,6 +67,8 @@ export interface CollectTelemetryOptions {
|
|
|
65
67
|
claudeProjectsRoot?: string;
|
|
66
68
|
/** Override `~/.codex` for tests. Both live and archived rollouts are scanned. */
|
|
67
69
|
codexRoot?: string;
|
|
70
|
+
/** Override `~/.grok` for tests. Sessions under `{grokRoot}/sessions/.../updates.jsonl`. */
|
|
71
|
+
grokRoot?: string;
|
|
68
72
|
/** Override the per-runtime scan budget for deterministic tests. */
|
|
69
73
|
maxScanBytesPerSource?: number;
|
|
70
74
|
/** Override the upload batch cap for deterministic tests. */
|
|
@@ -98,6 +102,12 @@ export interface CollectTelemetryResult {
|
|
|
98
102
|
/** Number of `POST /v1/usage` requests made. */
|
|
99
103
|
batchesSent: number;
|
|
100
104
|
}
|
|
105
|
+
/** Per-file Grok session metadata loaded once from sibling `summary.json`. */
|
|
106
|
+
export interface GrokSessionMeta {
|
|
107
|
+
cwd?: string;
|
|
108
|
+
gitBranch?: string;
|
|
109
|
+
fallback_model?: string;
|
|
110
|
+
}
|
|
101
111
|
/**
|
|
102
112
|
* SECURITY-CRITICAL (owner-telemetry-v2 US-007). Extract the MCP servers /
|
|
103
113
|
* connectors / integrations a session row touched as a NAMES-AND-COUNTS-ONLY map
|
|
@@ -145,7 +155,15 @@ export declare function extractServices(message: unknown): Record<string, number
|
|
|
145
155
|
*/
|
|
146
156
|
export declare function sanitizeRow(row: unknown, companyUid?: string): Record<string, unknown> | null;
|
|
147
157
|
/**
|
|
148
|
-
*
|
|
158
|
+
* Adapt a Grok `updates.jsonl` record to Claude-shaped sanitizer input(s).
|
|
159
|
+
* Emits only on `turn_completed` with `usage`. When `usage.modelUsage` is
|
|
160
|
+
* present, one row per model key (chart stacks by model id).
|
|
161
|
+
*/
|
|
162
|
+
export declare function grokUsageRows(row: unknown, sessionMeta: GrokSessionMeta): Array<Record<string, unknown>>;
|
|
163
|
+
/** Load cwd / branch / fallback model from sibling `summary.json` once per file. */
|
|
164
|
+
export declare function loadGrokSessionMeta(updatesPath: string): Promise<GrokSessionMeta>;
|
|
165
|
+
/**
|
|
166
|
+
* Scan, sanitize, and POST new Grok, Codex, and Claude Code usage rows.
|
|
149
167
|
*
|
|
150
168
|
* Fire-and-forget from the caller's perspective: errors are caught internally
|
|
151
169
|
* and surfaced only via `log`. The returned summary lets observers (e.g.
|
package/dist/telemetry.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAYH,OAAO,KAAK,EACV,sBAAsB,EACtB,UAAU,EACV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AAI3B;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,iBAAiB,IAAI,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACrD,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACzD;;;OAGG;IACH,iBAAiB,CAAC,CAChB,OAAO,EAAE,OAAO,EAChB,IAAI,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,GAC/B,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,sBAAsB,CAAC;IAC/B,oJAAoJ;IACpJ,SAAS,EAAE,MAAM,CAAC;IAClB,+HAA+H;IAC/H,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+CAA+C;IAC/C,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,sBAAsB;IACrC,4KAA4K;IAC5K,OAAO,EAAE,OAAO,CAAC;IACjB;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,wFAAwF;IACxF,WAAW,EAAE,QAAQ,GAAG,kBAAkB,GAAG,oBAAoB,GAAG,SAAS,CAAC;IAC9E,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAC;IACrB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;CACrB;AAeD,8EAA8E;AAC9E,MAAM,WAAW,eAAe;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAiJD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,GACf,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAyBpC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,WAAW,CACzB,GAAG,EAAE,OAAO,EACZ,UAAU,CAAC,EAAE,MAAM,GAClB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CA4ChC;AAgQD;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,GAAG,EAAE,OAAO,EACZ,WAAW,EAAE,eAAe,GAC3B,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CA+FhC;AAUD,oFAAoF;AACpF,wBAAsB,mBAAmB,CACvC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC,CA8B1B;AA0FD;;;;;;;GAOG;AACH,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,uBAAuB,GAC5B,OAAO,CAAC,sBAAsB,CAAC,CA4djC"}
|
package/dist/telemetry.js
CHANGED
|
@@ -7,10 +7,12 @@
|
|
|
7
7
|
* (`hq-sync-runner`, `hq-cli`, mobile wrappers) emits telemetry uniformly.
|
|
8
8
|
*
|
|
9
9
|
* What it does: after each successful sync (`all-complete` arm of
|
|
10
|
-
* `bin/sync-runner.ts`), walks
|
|
11
|
-
* rollouts, then diffs each file against the
|
|
12
|
-
*
|
|
13
|
-
*
|
|
10
|
+
* `bin/sync-runner.ts`), walks Grok `updates.jsonl` sessions, live and archived
|
|
11
|
+
* Codex rollouts, and Claude session logs, then diffs each file against the
|
|
12
|
+
* persisted byte-offset cursor at `~/.hq/telemetry-cursor.json`, sanitizes new
|
|
13
|
+
* rows through a tight allowlist that matches the server's KEEP_FIELDS set in
|
|
14
|
+
* `apps/hq-pro/src/vault-service/handlers/usage.ts`, batches into server-sized
|
|
15
|
+
* POST bodies, and ships them to `/v1/usage`.
|
|
14
16
|
*
|
|
15
17
|
* Trust model: the caller's `personUid` is resolved on the server from the
|
|
16
18
|
* Cognito JWT — never from the body. `sanitizeRow` strips prompt bodies,
|
|
@@ -430,6 +432,167 @@ async function listCodexRolloutFiles(codexRoot) {
|
|
|
430
432
|
]);
|
|
431
433
|
return sortFilesFreshestFirst([...live, ...archived].filter((file) => path.basename(file).startsWith("rollout-")));
|
|
432
434
|
}
|
|
435
|
+
// ── Grok session adapter ─────────────────────────────────────────────────────
|
|
436
|
+
function grokTimestamp(value) {
|
|
437
|
+
if (typeof value === "string") {
|
|
438
|
+
return boundedString(value, MAX_TIMESTAMP_BYTES);
|
|
439
|
+
}
|
|
440
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
441
|
+
const ms = value > 1e12 ? value : value * 1000;
|
|
442
|
+
try {
|
|
443
|
+
return new Date(ms).toISOString();
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
return undefined;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
return undefined;
|
|
450
|
+
}
|
|
451
|
+
function stableGrokEventId(sessionId, promptId, model) {
|
|
452
|
+
return ("grok-" +
|
|
453
|
+
createHash("sha256")
|
|
454
|
+
.update(sessionId)
|
|
455
|
+
.update(Buffer.from([0]))
|
|
456
|
+
.update(promptId)
|
|
457
|
+
.update(Buffer.from([0]))
|
|
458
|
+
.update(model)
|
|
459
|
+
.digest("hex"));
|
|
460
|
+
}
|
|
461
|
+
function grokUsageMessage(usage, model) {
|
|
462
|
+
const message = {
|
|
463
|
+
usage: {
|
|
464
|
+
input_tokens: tokenCount(usage.inputTokens),
|
|
465
|
+
output_tokens: tokenCount(usage.outputTokens) + tokenCount(usage.reasoningTokens),
|
|
466
|
+
cache_read_input_tokens: tokenCount(usage.cachedReadTokens),
|
|
467
|
+
cache_creation_input_tokens: tokenCount(usage.cacheCreationTokens),
|
|
468
|
+
},
|
|
469
|
+
};
|
|
470
|
+
if (model)
|
|
471
|
+
message.model = model;
|
|
472
|
+
return message;
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Adapt a Grok `updates.jsonl` record to Claude-shaped sanitizer input(s).
|
|
476
|
+
* Emits only on `turn_completed` with `usage`. When `usage.modelUsage` is
|
|
477
|
+
* present, one row per model key (chart stacks by model id).
|
|
478
|
+
*/
|
|
479
|
+
export function grokUsageRows(row, sessionMeta) {
|
|
480
|
+
if (!row || typeof row !== "object" || Array.isArray(row))
|
|
481
|
+
return [];
|
|
482
|
+
const obj = row;
|
|
483
|
+
const method = obj.method;
|
|
484
|
+
if (method !== "session/update" && method !== "_x.ai/session/update") {
|
|
485
|
+
return [];
|
|
486
|
+
}
|
|
487
|
+
const rawParams = obj.params;
|
|
488
|
+
if (!rawParams || typeof rawParams !== "object" || Array.isArray(rawParams)) {
|
|
489
|
+
return [];
|
|
490
|
+
}
|
|
491
|
+
const params = rawParams;
|
|
492
|
+
const rawUpdate = params.update;
|
|
493
|
+
if (!rawUpdate || typeof rawUpdate !== "object" || Array.isArray(rawUpdate)) {
|
|
494
|
+
return [];
|
|
495
|
+
}
|
|
496
|
+
const update = rawUpdate;
|
|
497
|
+
if (update.sessionUpdate !== "turn_completed")
|
|
498
|
+
return [];
|
|
499
|
+
const rawUsage = update.usage;
|
|
500
|
+
if (!rawUsage || typeof rawUsage !== "object" || Array.isArray(rawUsage)) {
|
|
501
|
+
return [];
|
|
502
|
+
}
|
|
503
|
+
const usage = rawUsage;
|
|
504
|
+
const sessionId = boundedString(params.sessionId, MAX_ID_BYTES);
|
|
505
|
+
const promptId = typeof update.prompt_id === "string" ? update.prompt_id : "";
|
|
506
|
+
const timestamp = grokTimestamp(obj.timestamp);
|
|
507
|
+
const updateMeta = update._meta;
|
|
508
|
+
const paramsMeta = params._meta;
|
|
509
|
+
const eventId = (updateMeta &&
|
|
510
|
+
typeof updateMeta === "object" &&
|
|
511
|
+
!Array.isArray(updateMeta)
|
|
512
|
+
? boundedString(updateMeta.eventId, MAX_ID_BYTES)
|
|
513
|
+
: undefined) ??
|
|
514
|
+
(paramsMeta &&
|
|
515
|
+
typeof paramsMeta === "object" &&
|
|
516
|
+
!Array.isArray(paramsMeta)
|
|
517
|
+
? boundedString(paramsMeta.eventId, MAX_ID_BYTES)
|
|
518
|
+
: undefined);
|
|
519
|
+
const buildRow = (model, modelUsage, preferEventId) => {
|
|
520
|
+
const normalized = {
|
|
521
|
+
uuid: preferEventId && eventId
|
|
522
|
+
? eventId
|
|
523
|
+
: stableGrokEventId(sessionId ?? "", promptId, model ?? ""),
|
|
524
|
+
message: grokUsageMessage(modelUsage, model),
|
|
525
|
+
};
|
|
526
|
+
if (sessionId)
|
|
527
|
+
normalized.sessionId = sessionId;
|
|
528
|
+
if (timestamp)
|
|
529
|
+
normalized.timestamp = timestamp;
|
|
530
|
+
if (sessionMeta.cwd)
|
|
531
|
+
normalized.cwd = sessionMeta.cwd;
|
|
532
|
+
if (sessionMeta.gitBranch)
|
|
533
|
+
normalized.gitBranch = sessionMeta.gitBranch;
|
|
534
|
+
return normalized;
|
|
535
|
+
};
|
|
536
|
+
const modelUsage = usage.modelUsage;
|
|
537
|
+
if (modelUsage &&
|
|
538
|
+
typeof modelUsage === "object" &&
|
|
539
|
+
!Array.isArray(modelUsage)) {
|
|
540
|
+
const entries = Object.entries(modelUsage).filter(([, entry]) => !!entry && typeof entry === "object" && !Array.isArray(entry));
|
|
541
|
+
const single = entries.length === 1;
|
|
542
|
+
const rows = [];
|
|
543
|
+
for (const [modelKey, entry] of entries) {
|
|
544
|
+
const model = boundedString(modelKey, MAX_MODEL_BYTES);
|
|
545
|
+
if (!model)
|
|
546
|
+
continue;
|
|
547
|
+
rows.push(buildRow(model, entry, single));
|
|
548
|
+
}
|
|
549
|
+
return rows;
|
|
550
|
+
}
|
|
551
|
+
return [
|
|
552
|
+
buildRow(sessionMeta.fallback_model, usage, true),
|
|
553
|
+
];
|
|
554
|
+
}
|
|
555
|
+
function decodeGrokCwdFolder(encoded) {
|
|
556
|
+
try {
|
|
557
|
+
return boundedString(decodeURIComponent(encoded), MAX_PATH_BYTES);
|
|
558
|
+
}
|
|
559
|
+
catch {
|
|
560
|
+
return boundedString(encoded, MAX_PATH_BYTES);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
/** Load cwd / branch / fallback model from sibling `summary.json` once per file. */
|
|
564
|
+
export async function loadGrokSessionMeta(updatesPath) {
|
|
565
|
+
const sessionDir = path.dirname(updatesPath);
|
|
566
|
+
const encodedCwdDir = path.basename(path.dirname(sessionDir));
|
|
567
|
+
const meta = {};
|
|
568
|
+
try {
|
|
569
|
+
const raw = await fs.readFile(path.join(sessionDir, "summary.json"), "utf-8");
|
|
570
|
+
const parsed = JSON.parse(raw);
|
|
571
|
+
const info = parsed.info && typeof parsed.info === "object" && !Array.isArray(parsed.info)
|
|
572
|
+
? parsed.info
|
|
573
|
+
: undefined;
|
|
574
|
+
meta.cwd =
|
|
575
|
+
boundedString(info?.cwd, MAX_PATH_BYTES) ??
|
|
576
|
+
boundedString(parsed.cwd, MAX_PATH_BYTES);
|
|
577
|
+
meta.gitBranch =
|
|
578
|
+
boundedString(parsed.head_branch, MAX_PATH_BYTES) ??
|
|
579
|
+
boundedString(info?.head_branch, MAX_PATH_BYTES);
|
|
580
|
+
meta.fallback_model =
|
|
581
|
+
boundedString(parsed.current_model_id, MAX_MODEL_BYTES) ??
|
|
582
|
+
boundedString(info?.current_model_id, MAX_MODEL_BYTES);
|
|
583
|
+
}
|
|
584
|
+
catch {
|
|
585
|
+
// Missing / unreadable summary — fall through to folder-name decode.
|
|
586
|
+
}
|
|
587
|
+
if (!meta.cwd) {
|
|
588
|
+
meta.cwd = decodeGrokCwdFolder(encodedCwdDir);
|
|
589
|
+
}
|
|
590
|
+
return meta;
|
|
591
|
+
}
|
|
592
|
+
async function listGrokUpdateFiles(grokRoot) {
|
|
593
|
+
const all = await listJsonlFiles(path.join(grokRoot, "sessions"));
|
|
594
|
+
return sortFilesFreshestFirst(all.filter((file) => path.basename(file) === "updates.jsonl"));
|
|
595
|
+
}
|
|
433
596
|
// ── Batching primitives ───────────────────────────────────────────────────────
|
|
434
597
|
const MAX_BATCH_EVENTS = 100;
|
|
435
598
|
const MAX_BATCH_BYTES = 240 * 1024;
|
|
@@ -494,7 +657,7 @@ function boundRowForPost(row, maxRowBytes) {
|
|
|
494
657
|
}
|
|
495
658
|
// ── Main entry point ──────────────────────────────────────────────────────────
|
|
496
659
|
/**
|
|
497
|
-
* Scan, sanitize, and POST new
|
|
660
|
+
* Scan, sanitize, and POST new Grok, Codex, and Claude Code usage rows.
|
|
498
661
|
*
|
|
499
662
|
* Fire-and-forget from the caller's perspective: errors are caught internally
|
|
500
663
|
* and surfaced only via `log`. The returned summary lets observers (e.g.
|
|
@@ -505,6 +668,7 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
505
668
|
const home = os.homedir();
|
|
506
669
|
const claudeProjectsRoot = opts.claudeProjectsRoot ?? path.join(home, ".claude", "projects");
|
|
507
670
|
const codexRoot = opts.codexRoot ?? path.join(home, ".codex");
|
|
671
|
+
const grokRoot = opts.grokRoot ?? path.join(home, ".grok");
|
|
508
672
|
const cursorPath = opts.cursorPath ?? path.join(home, ".hq", "telemetry-cursor.json");
|
|
509
673
|
const menubarPath = opts.menubarPath ?? path.join(home, ".hq", "menubar.json");
|
|
510
674
|
const log = opts.log ?? (() => { });
|
|
@@ -608,14 +772,16 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
608
772
|
const loadedFiles = { ...cursor.files };
|
|
609
773
|
const rotationResets = {};
|
|
610
774
|
const newlyCommitted = {};
|
|
611
|
-
const [claudeFilesUnsorted, codexFiles] = await Promise.all([
|
|
775
|
+
const [claudeFilesUnsorted, codexFiles, grokFiles] = await Promise.all([
|
|
612
776
|
listJsonlFiles(claudeProjectsRoot),
|
|
613
777
|
listCodexRolloutFiles(codexRoot),
|
|
778
|
+
listGrokUpdateFiles(grokRoot),
|
|
614
779
|
]);
|
|
615
780
|
const claudeFiles = await sortFilesFreshestFirst(claudeFilesUnsorted);
|
|
616
|
-
//
|
|
617
|
-
//
|
|
781
|
+
// Newest runtimes first so a bounded first pass reports current Grok/Codex
|
|
782
|
+
// use before replaying an older Claude backlog.
|
|
618
783
|
const files = [
|
|
784
|
+
...grokFiles.map((filePath) => ({ filePath, kind: "grok" })),
|
|
619
785
|
...codexFiles.map((filePath) => ({ filePath, kind: "codex" })),
|
|
620
786
|
...claudeFiles.map((filePath) => ({ filePath, kind: "claude" })),
|
|
621
787
|
];
|
|
@@ -635,7 +801,7 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
635
801
|
let batchesSent = 0;
|
|
636
802
|
let uploadFailed = false;
|
|
637
803
|
let batchLimitReached = false;
|
|
638
|
-
const scannedBytes = { claude: 0, codex: 0 };
|
|
804
|
+
const scannedBytes = { claude: 0, codex: 0, grok: 0 };
|
|
639
805
|
const commitSources = (sources) => {
|
|
640
806
|
const maxPerFile = new Map();
|
|
641
807
|
for (const source of sources) {
|
|
@@ -704,16 +870,21 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
704
870
|
const currentMtime = Math.floor(stat.mtimeMs / 1000);
|
|
705
871
|
const stored = cursor.files[filePath] ?? { offset: 0, mtime: 0 };
|
|
706
872
|
let offset = stored.offset;
|
|
707
|
-
|
|
873
|
+
const needsScanContext = kind === "codex" || kind === "grok";
|
|
874
|
+
let scanContext = needsScanContext
|
|
875
|
+
? cloneCodexContext(stored.context ?? {})
|
|
876
|
+
: undefined;
|
|
708
877
|
const rotated = currentSize < offset || (stored.mtime > 0 && currentMtime < stored.mtime);
|
|
709
878
|
if (rotated) {
|
|
710
879
|
offset = 0;
|
|
711
|
-
if (
|
|
712
|
-
|
|
880
|
+
if (needsScanContext)
|
|
881
|
+
scanContext = {};
|
|
713
882
|
rotationResets[filePath] = { offset: 0, mtime: currentMtime };
|
|
714
883
|
}
|
|
715
884
|
if (offset >= currentSize && !rotated)
|
|
716
885
|
continue;
|
|
886
|
+
// Grok session meta is loaded once per updates.jsonl (not persisted in cursor).
|
|
887
|
+
const grokSessionMeta = kind === "grok" ? await loadGrokSessionMeta(filePath) : undefined;
|
|
717
888
|
let content;
|
|
718
889
|
let bytesRead = 0;
|
|
719
890
|
let readBuffer = Buffer.alloc(0);
|
|
@@ -734,19 +905,19 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
734
905
|
}
|
|
735
906
|
if (bytesRead === 0)
|
|
736
907
|
continue;
|
|
737
|
-
// Resume a previously discarded oversized Codex row using raw byte
|
|
908
|
+
// Resume a previously discarded oversized Codex/Grok row using raw byte
|
|
738
909
|
// positions. Decoding from the middle of a multibyte UTF-8 codepoint would
|
|
739
910
|
// otherwise skew byte offsets and could skip or replay later rows.
|
|
740
|
-
if (
|
|
911
|
+
if (scanContext?.discarding_partial_line) {
|
|
741
912
|
const newlineIndex = readBuffer.subarray(0, bytesRead).indexOf(0x0a);
|
|
742
913
|
const discardedBytes = newlineIndex >= 0 ? newlineIndex + 1 : bytesRead;
|
|
743
|
-
|
|
914
|
+
scanContext.discarding_partial_line = newlineIndex < 0;
|
|
744
915
|
scannedBytes[kind] += discardedBytes;
|
|
745
916
|
batchSources.push({
|
|
746
917
|
filePath,
|
|
747
918
|
endOffset: offset + discardedBytes,
|
|
748
919
|
mtime: currentMtime,
|
|
749
|
-
context: cloneCodexContext(
|
|
920
|
+
context: cloneCodexContext(scanContext),
|
|
750
921
|
});
|
|
751
922
|
if (newlineIndex < 0)
|
|
752
923
|
continue;
|
|
@@ -775,17 +946,17 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
775
946
|
: lineEndOffsets[i];
|
|
776
947
|
const lineBytes = endOffset - startOffset;
|
|
777
948
|
if (!hasNewline && !reachedPhysicalEof) {
|
|
778
|
-
const discarding =
|
|
949
|
+
const discarding = scanContext?.discarding_partial_line === true;
|
|
779
950
|
if (discarding || lineBytes >= MAX_PARTIAL_LINE_BYTES) {
|
|
780
|
-
if (
|
|
781
|
-
|
|
951
|
+
if (scanContext)
|
|
952
|
+
scanContext.discarding_partial_line = true;
|
|
782
953
|
scannedBytes[kind] += lineBytes;
|
|
783
954
|
batchSources.push({
|
|
784
955
|
filePath,
|
|
785
956
|
endOffset,
|
|
786
957
|
mtime: currentMtime,
|
|
787
|
-
...(
|
|
788
|
-
? { context: cloneCodexContext(
|
|
958
|
+
...(scanContext
|
|
959
|
+
? { context: cloneCodexContext(scanContext) }
|
|
789
960
|
: {}),
|
|
790
961
|
});
|
|
791
962
|
}
|
|
@@ -796,12 +967,12 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
796
967
|
filePath,
|
|
797
968
|
endOffset,
|
|
798
969
|
mtime: currentMtime,
|
|
799
|
-
...(
|
|
800
|
-
? { context: cloneCodexContext(
|
|
970
|
+
...(scanContext
|
|
971
|
+
? { context: cloneCodexContext(scanContext) }
|
|
801
972
|
: {}),
|
|
802
973
|
});
|
|
803
|
-
if (
|
|
804
|
-
|
|
974
|
+
if (scanContext?.discarding_partial_line) {
|
|
975
|
+
scanContext.discarding_partial_line = false;
|
|
805
976
|
batchSources.push(source());
|
|
806
977
|
continue;
|
|
807
978
|
}
|
|
@@ -818,65 +989,83 @@ export async function collectAndSendTelemetry(opts) {
|
|
|
818
989
|
batchSources.push(source());
|
|
819
990
|
continue;
|
|
820
991
|
}
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
? sourceRow.cwd
|
|
826
|
-
: undefined;
|
|
827
|
-
const companyUid = resolveCompanyForCwd(typeof rowCwd === "string" ? rowCwd : undefined, repoCompanyMap) ?? fallbackCompanyUid;
|
|
828
|
-
// Company-scope-only mode: a row that resolves to NO company is personal
|
|
829
|
-
// work by an opted-out person — skipped, but still cursor-advanced (via
|
|
830
|
-
// batchSources) exactly like a sanitize-rejected row, so it is not
|
|
831
|
-
// re-read every run.
|
|
832
|
-
if (companyScopeOnly && !companyUid) {
|
|
833
|
-
batchSources.push(source());
|
|
834
|
-
continue;
|
|
992
|
+
let adaptedRows;
|
|
993
|
+
if (kind === "codex") {
|
|
994
|
+
const adapted = codexUsageRow(parsed, scanContext, filePath, startOffset, endOffset);
|
|
995
|
+
adaptedRows = adapted ? [adapted] : [];
|
|
835
996
|
}
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
997
|
+
else if (kind === "grok") {
|
|
998
|
+
adaptedRows = grokUsageRows(parsed, grokSessionMeta ?? {});
|
|
999
|
+
}
|
|
1000
|
+
else {
|
|
1001
|
+
adaptedRows = [parsed];
|
|
840
1002
|
}
|
|
841
|
-
|
|
842
|
-
const wasOversized = jsonBytes(sanitized) > maxRowBytes;
|
|
843
|
-
const bounded = boundRowForPost(sanitized, maxRowBytes);
|
|
844
|
-
if (!bounded) {
|
|
845
|
-
log("[telemetry] oversized row dropped before send (" +
|
|
846
|
-
filePath +
|
|
847
|
-
":" +
|
|
848
|
-
(i + 1) +
|
|
849
|
-
")");
|
|
1003
|
+
if (adaptedRows.length === 0) {
|
|
850
1004
|
batchSources.push(source());
|
|
851
1005
|
continue;
|
|
852
1006
|
}
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
(
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
1007
|
+
let emittedFromLine = 0;
|
|
1008
|
+
for (const sourceRow of adaptedRows) {
|
|
1009
|
+
const rowCwd = sourceRow && typeof sourceRow === "object" && !Array.isArray(sourceRow)
|
|
1010
|
+
? sourceRow.cwd
|
|
1011
|
+
: undefined;
|
|
1012
|
+
const companyUid = resolveCompanyForCwd(typeof rowCwd === "string" ? rowCwd : undefined, repoCompanyMap) ?? fallbackCompanyUid;
|
|
1013
|
+
// Company-scope-only mode: a row that resolves to NO company is personal
|
|
1014
|
+
// work by an opted-out person — skipped, but still cursor-advanced (via
|
|
1015
|
+
// batchSources) exactly like a sanitize-rejected row, so it is not
|
|
1016
|
+
// re-read every run.
|
|
1017
|
+
if (companyScopeOnly && !companyUid) {
|
|
1018
|
+
continue;
|
|
1019
|
+
}
|
|
1020
|
+
const sanitized = sanitizeRow(sourceRow, companyUid);
|
|
1021
|
+
if (!sanitized) {
|
|
1022
|
+
continue;
|
|
1023
|
+
}
|
|
1024
|
+
const maxRowBytes = MAX_BATCH_BYTES - ENVELOPE_BYTES;
|
|
1025
|
+
const wasOversized = jsonBytes(sanitized) > maxRowBytes;
|
|
1026
|
+
const bounded = boundRowForPost(sanitized, maxRowBytes);
|
|
1027
|
+
if (!bounded) {
|
|
1028
|
+
log("[telemetry] oversized row dropped before send (" +
|
|
1029
|
+
filePath +
|
|
1030
|
+
":" +
|
|
1031
|
+
(i + 1) +
|
|
1032
|
+
")");
|
|
1033
|
+
continue;
|
|
868
1034
|
}
|
|
869
|
-
if (
|
|
870
|
-
|
|
871
|
-
|
|
1035
|
+
if (wasOversized) {
|
|
1036
|
+
log("[telemetry] oversized row truncated before send (" +
|
|
1037
|
+
filePath +
|
|
1038
|
+
":" +
|
|
1039
|
+
(i + 1) +
|
|
1040
|
+
")");
|
|
872
1041
|
}
|
|
873
|
-
|
|
1042
|
+
const rowJsonBytes = jsonBytes(bounded);
|
|
1043
|
+
const addCost = rowJsonBytes + (batchEvents.length > 0 ? 1 : 0);
|
|
1044
|
+
if (batchEvents.length > 0 &&
|
|
1045
|
+
(batchEvents.length >= MAX_BATCH_EVENTS ||
|
|
1046
|
+
batchBytes + addCost > MAX_BATCH_BYTES)) {
|
|
1047
|
+
if (!(await flush())) {
|
|
1048
|
+
uploadFailed = true;
|
|
1049
|
+
break filesLoop;
|
|
1050
|
+
}
|
|
1051
|
+
if (batchesSent >= maxBatchesPerRun) {
|
|
1052
|
+
batchLimitReached = true;
|
|
1053
|
+
break filesLoop;
|
|
1054
|
+
}
|
|
1055
|
+
batchBytes = ENVELOPE_BYTES + rowJsonBytes;
|
|
1056
|
+
}
|
|
1057
|
+
else {
|
|
1058
|
+
batchBytes += addCost;
|
|
1059
|
+
}
|
|
1060
|
+
batchEvents.push(bounded);
|
|
1061
|
+
// Same endOffset for every model row from this JSONL line; commitSources
|
|
1062
|
+
// keeps the max, so the cursor advances once per line.
|
|
1063
|
+
batchSources.push(source());
|
|
1064
|
+
emittedFromLine++;
|
|
874
1065
|
}
|
|
875
|
-
|
|
876
|
-
|
|
1066
|
+
if (emittedFromLine === 0) {
|
|
1067
|
+
batchSources.push(source());
|
|
877
1068
|
}
|
|
878
|
-
batchEvents.push(bounded);
|
|
879
|
-
batchSources.push(source());
|
|
880
1069
|
}
|
|
881
1070
|
}
|
|
882
1071
|
if (!uploadFailed && !batchLimitReached) {
|