@iislee/opencodex 2.11.0

package/src/grok/sync.ts

@@ -0,0 +1,103 @@
+/**
+ * Shared Grok Build config sync: gather the visible model catalog and (re)inject the
+ * managed block into ~/.grok/config.toml. Used by `ocx start` (server process) and by
+ * `ocx ensure` / `ocx restart` (parent process, after live discovery or child readiness)
+ * so the fence exists deterministically once the proxy reports healthy.
+ *
+ * Deps are injectable (mirrors src/codex/sync.ts) so tests can run without a live proxy.
+ */
+import { visibleNativeSlugs, filterCatalogVisibleModels, nativeOpenAiContextWindow, type CatalogModel } from "../codex/catalog";
+import type { OcxConfig } from "../types";
+import { injectGrokConfig, type GrokInjectModel, type GrokInjectResult } from "./inject";
+import { installGrokUsageHooks, type GrokUsageHookResult } from "./usage-hook";
+
+export interface GrokSyncDeps {
+  fetchAllModels: (config: OcxConfig) => Promise<CatalogModel[]>;
+  injectGrokConfig: typeof injectGrokConfig;
+  installGrokUsageHooks?: (opts?: { grokHome?: string }) => GrokUsageHookResult;
+}
+
+async function defaultFetchAllModels(config: OcxConfig): Promise<CatalogModel[]> {
+  const { fetchAllModels } = await import("../server/management-api");
+  return fetchAllModels(config);
+}
+
+/**
+ * Build the model list and inject the fenced block. `hostname` should be the hostname the
+ * RUNNING proxy actually bound (live.hostname from proxy-liveness for ensure's live branch;
+ * config.hostname for a freshly spawned start) — a stale config.hostname could otherwise
+ * name a host the process never bound.
+ */
+export async function syncGrokConfig(
+  port: number,
+  config: OcxConfig,
+  opts: { hostname?: string; grokHome?: string } = {},
+  deps: GrokSyncDeps = { fetchAllModels: defaultFetchAllModels, injectGrokConfig },
+): Promise<GrokInjectResult> {
+  let models: GrokInjectModel[];
+  try {
+    const routed = filterCatalogVisibleModels(await deps.fetchAllModels(config), config);
+    models = [
+      // Native slugs carry their context window too. Without it Grok falls back to its own
+      // default (200k) and understates models like gpt-5.6-sol, which is 372k. This is the same
+      // accessor the dashboard's native rows use, so the two cannot disagree.
+      ...visibleNativeSlugs(config).map(id => {
+        const contextWindow = nativeOpenAiContextWindow(id);
+        return { id, ...(contextWindow !== undefined ? { contextWindow } : {}) };
+      }),
+      ...routed.map(m => ({
+        id: m.alias ?? `${m.provider}/${m.id}`,
+        ...(m.contextWindow !== undefined ? { contextWindow: m.contextWindow } : {}),
+      })),
+    ];
+  } catch (err) {
+    // Still try to install the usage hook — native-session reporting does not need the catalog.
+    const hookResult = runInstallHooks(deps, opts);
+    return {
+      ok: false,
+      changed: hookResult.changed,
+      message: [
+        `Grok config sync skipped: model catalog unavailable (${err instanceof Error ? err.message : String(err)})`,
+        hookResult.message,
+      ].filter(Boolean).join(" "),
+    };
+  }
+  // Pass the FULL list plus the exclusion set: the writer allocates aliases over
+  // everything and emits only what is switched on, so a model's alias never depends on
+  // its neighbours' switches. Absent/empty selection keeps today's behaviour exactly.
+  const injectResult = deps.injectGrokConfig(port, models, {
+    ...(opts.hostname !== undefined ? { hostname: opts.hostname } : {}),
+    ...(opts.grokHome !== undefined ? { grokHome: opts.grokHome } : {}),
+    excluded: new Set(config.grokExcludedModels ?? []),
+  });
+
+  // Always (re)install the native-usage hook when a Grok home exists — independent of the
+  // model fence (which is skipped on non-loopback binds). Best-effort; never fails the fence.
+  const hookResult = runInstallHooks(deps, opts);
+
+  if (!hookResult.changed && hookResult.ok) return injectResult;
+  const parts = [injectResult.message, hookResult.message].filter(Boolean);
+  return {
+    ok: injectResult.ok && hookResult.ok,
+    changed: injectResult.changed || hookResult.changed,
+    message: parts.join(" "),
+    ...(injectResult.skippedReason ? { skippedReason: injectResult.skippedReason } : {}),
+  };
+}
+
+function runInstallHooks(
+  deps: GrokSyncDeps,
+  opts: { grokHome?: string },
+): GrokUsageHookResult {
+  const installHooks = deps.installGrokUsageHooks ?? installGrokUsageHooks;
+  try {
+    return installHooks(opts.grokHome !== undefined ? { grokHome: opts.grokHome } : {});
+  } catch (err) {
+    return {
+      ok: false,
+      changed: false,
+      message: `Grok usage hook install failed: ${err instanceof Error ? err.message : String(err)}`,
+      homes: [],
+    };
+  }
+}

package/src/grok/usage-hook/report.mjs

@@ -0,0 +1,348 @@
+#!/usr/bin/env node
+/**
+ * Grok Build hook → opencodex usage + live Logs
+ *
+ * On Stop / SessionEnd, reads Grok's on-disk turn_completed usage (real token
+ * counts from the native session) and reports them to opencodex:
+ *
+ *   1. POST /api/usage/ingest  → usage.jsonl + live Logs ring (preferred)
+ *   2. Fallback: append usage.jsonl only (if ocx is down or old)
+ *
+ * Skips ocx-routed / non-native models so proxy traffic is not double-counted.
+ * Fail-open: any error exits 0 so Grok is never blocked.
+ */
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  writeFileSync,
+} from "node:fs";
+import { createHash } from "node:crypto";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+const OPENCODEX_DIR = process.env.OPENCODEX_HOME || join(homedir(), ".opencodex");
+const USAGE_LOG = join(OPENCODEX_DIR, "usage.jsonl");
+const STATE_PATH = join(OPENCODEX_DIR, "grok-native-usage-state.json");
+/** Prefer env, but always also search the canonical ~/.grok (sessions often live there). */
+function grokHomes() {
+  const homes = [];
+  if (process.env.GROK_HOME) homes.push(process.env.GROK_HOME);
+  const canonical = join(homedir(), ".grok");
+  if (!homes.includes(canonical)) homes.push(canonical);
+  return homes;
+}
+
+function failOpen(err) {
+  try {
+    const logDir = join(OPENCODEX_DIR, "logs");
+    mkdirSync(logDir, { recursive: true });
+    appendFileSync(
+      join(logDir, "grok-usage-hook.log"),
+      `${new Date().toISOString()} ${err?.stack || err?.message || String(err)}\n`,
+    );
+  } catch {
+    /* ignore */
+  }
+  process.exit(0);
+}
+
+async function readStdinJson() {
+  const chunks = [];
+  for await (const chunk of process.stdin) chunks.push(chunk);
+  const raw = Buffer.concat(chunks).toString("utf8").trim();
+  if (!raw) return null;
+  return JSON.parse(raw);
+}
+
+function loadState() {
+  try {
+    if (!existsSync(STATE_PATH)) return { seen: {} };
+    const parsed = JSON.parse(readFileSync(STATE_PATH, "utf8"));
+    if (!parsed || typeof parsed !== "object") return { seen: {} };
+    if (!parsed.seen || typeof parsed.seen !== "object") return { seen: {} };
+    return parsed;
+  } catch {
+    return { seen: {} };
+  }
+}
+
+function saveState(state) {
+  mkdirSync(OPENCODEX_DIR, { recursive: true, mode: 0o700 });
+  writeFileSync(STATE_PATH, `${JSON.stringify(state)}\n`, { encoding: "utf8", mode: 0o600 });
+}
+
+/** Native xAI Grok models only — anything with a provider slash is routed elsewhere. */
+function isNativeGrokModel(modelId) {
+  if (typeof modelId !== "string" || !modelId) return false;
+  if (modelId.includes("/")) return false;
+  if (modelId.startsWith("ocx-")) return false;
+  return /^(grok\b|grok[-_])/i.test(modelId);
+}
+
+function encodeSessionCwd(cwd) {
+  // Grok stores sessions under encodeURIComponent(cwd) (slashes → %2F).
+  return encodeURIComponent(cwd);
+}
+
+function findSessionDir(sessionId, cwd, workspaceRoot) {
+  if (!sessionId) return null;
+  const candidates = [];
+  for (const home of grokHomes()) {
+    const sessionsRoot = join(home, "sessions");
+    for (const base of [cwd, workspaceRoot]) {
+      if (typeof base === "string" && base) {
+        candidates.push(join(sessionsRoot, encodeSessionCwd(base), sessionId));
+      }
+    }
+  }
+  for (const dir of candidates) {
+    if (existsSync(join(dir, "updates.jsonl"))) return dir;
+  }
+  // Fallback: scan session groups (encoded cwd dirs) for this session id.
+  for (const home of grokHomes()) {
+    const sessionsRoot = join(home, "sessions");
+    try {
+      for (const group of readdirSync(sessionsRoot, { withFileTypes: true })) {
+        if (!group.isDirectory()) continue;
+        const dir = join(sessionsRoot, group.name, sessionId);
+        if (existsSync(join(dir, "updates.jsonl"))) return dir;
+      }
+    } catch {
+      /* ignore */
+    }
+  }
+  return null;
+}
+
+function readSummaryModel(sessionDir) {
+  try {
+    const summary = JSON.parse(readFileSync(join(sessionDir, "summary.json"), "utf8"));
+    return summary?.current_model_id || summary?.info?.model || null;
+  } catch {
+    return null;
+  }
+}
+
+function parseTurnCompleted(line) {
+  let obj;
+  try {
+    obj = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  const update = obj?.params?.update ?? obj?.update ?? obj;
+  if (!update || update.sessionUpdate !== "turn_completed") return null;
+  const usage = update.usage;
+  if (!usage || typeof usage !== "object") return null;
+  const promptId = update.prompt_id || update.promptId || obj?.params?.prompt_id;
+  if (!promptId) return null;
+  return {
+    promptId: String(promptId),
+    stopReason: update.stop_reason || update.stopReason || "",
+    usage,
+    timestampMs: typeof obj.timestamp === "number"
+      ? (obj.timestamp > 1e12 ? obj.timestamp : obj.timestamp * 1000)
+      : Date.now(),
+  };
+}
+
+function primaryModelFromUsage(usage, fallback) {
+  const modelUsage = usage?.modelUsage;
+  if (modelUsage && typeof modelUsage === "object") {
+    const keys = Object.keys(modelUsage);
+    if (keys.length === 1) return keys[0];
+    // Prefer a native grok key when multiple are present.
+    const native = keys.find(isNativeGrokModel);
+    if (native) return native;
+    if (keys[0]) return keys[0];
+  }
+  return fallback || "grok-build";
+}
+
+function toOcxUsage(usage) {
+  const inputTokens = Number(usage.inputTokens) || 0;
+  const outputTokens = Number(usage.outputTokens) || 0;
+  const totalTokens = Number(usage.totalTokens) || inputTokens + outputTokens;
+  const cached = Number(usage.cachedReadTokens ?? usage.cachedInputTokens) || 0;
+  const reasoning = Number(usage.reasoningTokens ?? usage.reasoningOutputTokens) || 0;
+  return {
+    inputTokens,
+    outputTokens,
+    totalTokens,
+    cachedInputTokens: cached,
+    cacheReadInputTokens: cached,
+    cacheCreationInputTokens: 0,
+    reasoningOutputTokens: reasoning,
+  };
+}
+
+function conversationId(sessionId) {
+  // Match opencodex's non-PII correlation style: short hash, not the raw id if sensitive.
+  return createHash("sha256").update(String(sessionId)).digest("hex").slice(0, 32);
+}
+
+function appendUsageEntry(entry) {
+  mkdirSync(OPENCODEX_DIR, { recursive: true, mode: 0o700 });
+  appendFileSync(USAGE_LOG, `${JSON.stringify(entry)}\n`, { encoding: "utf8", mode: 0o600 });
+}
+
+function resolveOpencodexBaseUrl() {
+  if (process.env.OPENCODEX_URL?.trim()) {
+    return process.env.OPENCODEX_URL.trim().replace(/\/$/, "");
+  }
+  let port = 10100;
+  try {
+    const runtime = JSON.parse(readFileSync(join(OPENCODEX_DIR, "runtime-port.json"), "utf8"));
+    if (typeof runtime?.port === "number" && runtime.port > 0) port = runtime.port;
+  } catch {
+    /* default */
+  }
+  return `http://127.0.0.1:${port}`;
+}
+
+function readAdminToken() {
+  if (process.env.OPENCODEX_ADMIN_TOKEN?.trim()) return process.env.OPENCODEX_ADMIN_TOKEN.trim();
+  try {
+    return readFileSync(join(OPENCODEX_DIR, "admin-api-token"), "utf8").trim();
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Push entries into the running opencodex process (usage.jsonl + live Logs).
+ * Returns true on success (including all-skipped duplicates), false to fall back.
+ */
+async function ingestViaApi(entries) {
+  const token = readAdminToken();
+  if (!token) return false;
+  const base = resolveOpencodexBaseUrl();
+  const url = `${base}/api/usage/ingest`;
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "x-opencodex-api-key": token,
+      },
+      body: JSON.stringify({ entries }),
+      signal: AbortSignal.timeout(8000),
+    });
+    // 404 = old ocx without ingest; connection errors fall to catch.
+    if (res.status === 404) return false;
+    if (!res.ok && res.status !== 207) return false;
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function toIngestEntry(turn, conv) {
+  return {
+    requestId: `grok-native-${turn.promptId}`,
+    timestamp: turn.timestamp,
+    provider: "xai-native",
+    model: turn.model,
+    surface: "grok",
+    conversationId: conv,
+    requestedModel: turn.model,
+    resolvedModel: turn.model,
+    status: 200,
+    durationMs: turn.durationMs,
+    usageStatus: "reported",
+    usage: turn.usage,
+    totalTokens: turn.usage.totalTokens,
+  };
+}
+
+function collectNewTurns(sessionDir, sessionId, state) {
+  const updatesPath = join(sessionDir, "updates.jsonl");
+  if (!existsSync(updatesPath)) return [];
+
+  const seenKey = sessionId;
+  const seen = new Set(
+    Array.isArray(state.seen[seenKey]) ? state.seen[seenKey] : [],
+  );
+  const summaryModel = readSummaryModel(sessionDir);
+  const fresh = [];
+
+  const text = readFileSync(updatesPath, "utf8");
+  for (const line of text.split("\n")) {
+    if (!line.includes("turn_completed")) continue;
+    const turn = parseTurnCompleted(line);
+    if (!turn) continue;
+    if (seen.has(turn.promptId)) continue;
+
+    const model = primaryModelFromUsage(turn.usage, summaryModel);
+    if (!isNativeGrokModel(model)) {
+      // Still mark seen so we don't re-scan forever on ocx turns.
+      seen.add(turn.promptId);
+      continue;
+    }
+
+    const ocxUsage = toOcxUsage(turn.usage);
+    if (ocxUsage.totalTokens <= 0 && ocxUsage.inputTokens <= 0 && ocxUsage.outputTokens <= 0) {
+      seen.add(turn.promptId);
+      continue;
+    }
+
+    fresh.push({
+      promptId: turn.promptId,
+      model,
+      timestamp: turn.timestampMs,
+      durationMs: Number(turn.usage.apiDurationMs) || 0,
+      usage: ocxUsage,
+    });
+    seen.add(turn.promptId);
+  }
+
+  // Cap seen set growth per session.
+  const seenList = [...seen];
+  state.seen[seenKey] = seenList.length > 500 ? seenList.slice(-500) : seenList;
+  return fresh;
+}
+
+async function main() {
+  const input = await readStdinJson();
+  if (!input || typeof input !== "object") process.exit(0);
+
+  const event = String(input.hookEventName || input.hook_event_name || "").toLowerCase();
+  // Accept stop / session_end (and camelCase variants already lowercased).
+  if (event && event !== "stop" && event !== "sessionend" && event !== "session_end") {
+    process.exit(0);
+  }
+
+  const sessionId = input.sessionId || input.session_id;
+  if (!sessionId) process.exit(0);
+
+  const sessionDir = findSessionDir(
+    sessionId,
+    input.cwd,
+    input.workspaceRoot || input.workspace_root,
+  );
+  if (!sessionDir) process.exit(0);
+
+  const state = loadState();
+  const turns = collectNewTurns(sessionDir, sessionId, state);
+  if (turns.length === 0) {
+    saveState(state);
+    process.exit(0);
+  }
+
+  const conv = conversationId(sessionId);
+  const entries = turns.map(turn => toIngestEntry(turn, conv));
+
+  // Prefer live ingest so Dashboard Logs updates immediately; fall back to file-only.
+  const viaApi = await ingestViaApi(entries);
+  if (!viaApi) {
+    for (const entry of entries) appendUsageEntry(entry);
+  }
+
+  saveState(state);
+  process.exit(0);
+}
+
+main().catch(failOpen);