agentel 0.2.6 → 0.3.0

package/src/importers/grok.js

@@ -0,0 +1,367 @@
+"use strict";
+
+const fs = require("fs");
+
+const PROVIDER = "grok";
+
+function parseGrokUpdatesFile(file, options = {}) {
+  let text;
+  try {
+    text = fs.readFileSync(file, "utf8");
+  } catch {
+    return null;
+  }
+  return parseGrokUpdatesText(text, options);
+}
+
+// Grok Build session dirs hold updates.jsonl: ACP (Agent Client Protocol)
+// JSON-RPC session/update notifications. Message content streams in as
+// chunked entries (user_message_chunk / agent_message_chunk /
+// agent_thought_chunk) that must be concatenated; the only token telemetry is
+// a cumulative _meta.totalTokens counter that can rewind during streaming, so
+// the session total is the maximum observed value.
+function parseGrokUpdatesText(text, options = {}) {
+  const lines = String(text || "").split(/\r?\n/);
+  const fallbackTime = toIso(options.fallbackTime) || new Date().toISOString();
+  const messages = [];
+  const models = new Set();
+  let sessionId = "";
+  let maxTotalTokens = 0;
+  let currentModel = "";
+  let pending = null;
+  let index = 0;
+
+  const flush = () => {
+    if (!pending) return;
+    const content = pending.texts.join("").trim();
+    const thinking = pending.thinking.join("").trim();
+    if (!content && !thinking && !pending.toolCalls.length) {
+      pending = null;
+      return;
+    }
+    messages.push({
+      role: pending.role,
+      content,
+      timestamp: pending.timestamp,
+      metadata: compactObject({
+        provider: PROVIDER,
+        model: pending.role === "assistant" ? pending.model || undefined : undefined,
+        thinking: thinking || undefined,
+        toolCalls: pending.toolCalls.length ? pending.toolCalls : undefined
+      })
+    });
+    pending = null;
+  };
+
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    const record = parseJson(line);
+    if (!record || typeof record !== "object") continue;
+    const params = record.params && typeof record.params === "object" ? record.params : record;
+    const update = params.update && typeof params.update === "object" ? params.update : null;
+    if (!update) continue;
+    const meta = { ...(params._meta && typeof params._meta === "object" ? params._meta : {}), ...(update._meta && typeof update._meta === "object" ? update._meta : {}) };
+    sessionId ||= firstString(params.sessionId, params.session_id);
+    const timestamp =
+      toIso(meta.agentTimestampMs) || toIso(params.timestamp) || toIso(record.timestamp) || toIso(record.ts) || offsetTimestamp(fallbackTime, index);
+    index++;
+    const totalTokens = Number(meta.totalTokens);
+    if (Number.isFinite(totalTokens) && totalTokens > maxTotalTokens) maxTotalTokens = totalTokens;
+    const model = firstString(meta.modelId, params.modelId, params.model_id, params.model);
+    if (model) {
+      models.add(model);
+      currentModel = model;
+    }
+    const kind = String(update.sessionUpdate || update.session_update || "");
+    if (kind === "user_message_chunk") {
+      if (!pending || pending.role !== "user") {
+        flush();
+        pending = { role: "user", texts: [], thinking: [], toolCalls: [], timestamp, model: "" };
+      }
+      pending.texts.push(grokContentText(update));
+      continue;
+    }
+    if (kind === "agent_message_chunk" || kind === "agent_thought_chunk") {
+      if (!pending || pending.role !== "assistant") {
+        flush();
+        pending = { role: "assistant", texts: [], thinking: [], toolCalls: [], timestamp, model: currentModel };
+      }
+      pending.model = currentModel || pending.model;
+      (kind === "agent_thought_chunk" ? pending.thinking : pending.texts).push(grokContentText(update));
+      continue;
+    }
+    if (kind === "tool_call") {
+      if (!pending || pending.role !== "assistant") {
+        flush();
+        pending = { role: "assistant", texts: [], thinking: [], toolCalls: [], timestamp, model: currentModel };
+      }
+      const call = grokToolCall(update);
+      if (call) pending.toolCalls.push(call);
+      continue;
+    }
+    if (kind === "tool_call_update") {
+      const status = firstString(update.status);
+      const output = grokToolOutput(update);
+      if ((status === "completed" || status === "failed") || output) {
+        flush();
+        const toolResult = grokToolResult(update, output);
+        if (toolResult) {
+          messages.push({ role: "tool", content: toolResult.output, timestamp, metadata: { provider: PROVIDER, toolResult } });
+        }
+      }
+      continue;
+    }
+    if (kind === "plan") {
+      flush();
+      const planText = grokPlanText(update);
+      if (planText) {
+        messages.push({
+          role: "system",
+          content: planText,
+          timestamp,
+          metadata: { provider: PROVIDER, providerGenerated: true, contextKind: "plan", eventType: "grok-plan-update" }
+        });
+      }
+      continue;
+    }
+    if (kind === "current_mode_update") {
+      flush();
+      const mode = firstString(update.currentModeId, update.current_mode_id, update.modeId, update.mode);
+      if (mode) {
+        messages.push({
+          role: "system",
+          content: `Mode changed to ${mode}`,
+          timestamp,
+          metadata: { provider: PROVIDER, providerGenerated: true, contextKind: "mode_change", eventType: "grok-mode-update" }
+        });
+      }
+      continue;
+    }
+    // available_commands_update, hook_execution, subagent_spawned/finished
+    // (linkage comes from subagents/<id>/meta.json instead), and unknown
+    // kinds: ignore (product is young and the vocabulary is still growing).
+  }
+  flush();
+  if (!sessionId && !messages.length) return null;
+  return {
+    sessionId,
+    messages,
+    models: [...models],
+    maxTotalTokens,
+    startedAt: messages[0]?.timestamp || fallbackTime,
+    endedAt: messages[messages.length - 1]?.timestamp || fallbackTime
+  };
+}
+
+// summary.json + signals.json sidecars carry the per-session rollup.
+function grokSessionSummary({ summary, signals, models, maxTotalTokens }) {
+  const totalTokens = positiveNumber(maxTotalTokens);
+  const signalModels = Array.isArray(signals?.modelsUsed) ? signals.modelsUsed.filter(Boolean) : [];
+  const allModels = [...new Set([...(models || []), ...signalModels, firstString(summary?.current_model_id, summary?.model_id, signals?.primaryModelId)].filter(Boolean))];
+  return compactObject({
+    usage: totalTokens ? { totalTokens, authoritativeTotalTokens: true, source: "grok-updates-total-tokens" } : undefined,
+    modelUsage: allModels.length ? allModels.map((model) => ({ model, source: "grok-session" })) : undefined,
+    grokBuild: compactObject({
+      turnCount: positiveNumber(signals?.turnCount),
+      contextTokensUsed: positiveNumber(signals?.contextTokensUsed),
+      contextWindowTokens: positiveNumber(signals?.contextWindowTokens),
+      totalTokensBeforeCompaction: positiveNumber(signals?.totalTokensBeforeCompaction),
+      sessionDurationSeconds: positiveNumber(signals?.sessionDurationSeconds),
+      primaryModelId: firstString(signals?.primaryModelId) || undefined,
+      createdAt: toIso(summary?.created_at) || undefined,
+      updatedAt: toIso(summary?.updated_at) || undefined
+    })
+  });
+}
+
+// Session dirs are grouped under percent-encoded absolute cwd paths,
+// e.g. ~/.grok/sessions/%2FUsers%2Fme%2Fproject/<session-id>/.
+function decodeGrokCwd(name) {
+  try {
+    const decoded = decodeURIComponent(String(name || ""));
+    return decoded.startsWith("/") || /^[A-Za-z]:[\\/]/.test(decoded) ? decoded : "";
+  } catch {
+    return "";
+  }
+}
+
+function grokContentText(update) {
+  const content = update.content;
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) return content.map(grokBlockText).filter(Boolean).join("");
+  if (content && typeof content === "object") return grokBlockText(content);
+  return firstString(update.text, update.chunk);
+}
+
+function grokBlockText(block) {
+  if (typeof block === "string") return block;
+  if (!block || typeof block !== "object") return "";
+  if (typeof block.text === "string") return block.text;
+  if (block.content) return grokContentText({ content: block.content });
+  return "";
+}
+
+function grokToolCall(update) {
+  const id = firstString(update.toolCallId, update.tool_call_id);
+  const title = firstString(update.title, update.name);
+  if (!id && !title) return null;
+  const args = update.rawInput && typeof update.rawInput === "object" ? update.rawInput : undefined;
+  const summary = firstString(title, summarizeToolArguments(args));
+  return compactObject({
+    id: id || undefined,
+    name: firstString(update.name, update.kind, title, "tool"),
+    displayName: title || toolDisplayName(firstString(update.kind, "tool")),
+    rawCategory: "tool_call",
+    category: grokToolCategory(firstString(update.kind, title)),
+    title: title || undefined,
+    status: firstString(update.status, "tool_call"),
+    argument: summary,
+    rawInputSummary: summary,
+    inputPreview: summary,
+    target: grokToolLocation(update) || undefined,
+    arguments: args,
+    provider: PROVIDER
+  });
+}
+
+function grokToolResult(update, output) {
+  const id = firstString(update.toolCallId, update.tool_call_id);
+  if (!id && !output) return null;
+  const isError = firstString(update.status) === "failed";
+  const lineCount = output ? output.split(/\r?\n/).length : 0;
+  return compactObject({
+    provider: PROVIDER,
+    id: id || undefined,
+    kind: "Tool result",
+    title: isError ? "Tool error" : "Tool result",
+    rawCategory: "tool_call_update",
+    category: grokToolCategory(firstString(update.kind, update.title)),
+    summary: firstLine(output),
+    output: output || "",
+    lineCount: lineCount || undefined,
+    collapsed: lineCount > 18 || undefined,
+    status: isError ? "error" : "completed"
+  });
+}
+
+function grokToolOutput(update) {
+  const fromContent = grokContentText(update);
+  if (fromContent) return fromContent;
+  const raw = update.rawOutput ?? update.raw_output;
+  if (typeof raw === "string") return raw.trim();
+  if (raw && typeof raw === "object") return JSON.stringify(raw);
+  return "";
+}
+
+function grokPlanText(update) {
+  const entries = Array.isArray(update.entries) ? update.entries : [];
+  if (!entries.length) return grokContentText(update);
+  return entries
+    .map((entry) => {
+      const status = firstString(entry?.status);
+      const content = firstString(entry?.content, entry?.title);
+      if (!content) return "";
+      return status ? `[${status}] ${content}` : content;
+    })
+    .filter(Boolean)
+    .join("\n");
+}
+
+function grokToolLocation(update) {
+  const locations = Array.isArray(update.locations) ? update.locations : [];
+  return firstString(locations[0]?.path, locations[0]?.uri);
+}
+
+function grokToolCategory(kind) {
+  const text = String(kind || "");
+  if (/think/i.test(text)) return "status";
+  if (/execute|shell|bash|command/i.test(text)) return "shell";
+  if (/fetch|web|url|search(?!_files)/i.test(text)) return "web";
+  if (/edit|delete|move|write|patch/i.test(text)) return "edit";
+  if (/read|view/i.test(text)) return "read";
+  if (/glob|grep|find/i.test(text)) return "search";
+  if (/agent|task|subagent/i.test(text)) return "task";
+  return "tool";
+}
+
+function summarizeToolArguments(value) {
+  if (value == null) return "";
+  if (typeof value === "string") return value.slice(0, 240);
+  if (typeof value !== "object") return String(value).slice(0, 240);
+  for (const key of ["command", "prompt", "query", "pattern", "path", "file_path", "url"]) {
+    if (value[key]) return String(value[key]).slice(0, 240);
+  }
+  return Object.entries(value)
+    .slice(0, 3)
+    .map(([key, item]) => `${key}: ${typeof item === "string" ? item : JSON.stringify(item)}`)
+    .join(", ")
+    .slice(0, 240);
+}
+
+function toolDisplayName(value) {
+  const text = String(value || "tool").trim();
+  return text
+    .split(/_+|(?=[A-Z])/g)
+    .filter(Boolean)
+    .map((part) => part.replace(/(^|[-\s])([a-z])/g, (_, prefix, char) => `${prefix}${char.toUpperCase()}`))
+    .join(" ");
+}
+
+function firstLine(value) {
+  return String(value || "").split(/\r?\n/).find((line) => line.trim())?.trim() || "";
+}
+
+function offsetTimestamp(base, index) {
+  const date = new Date(base);
+  if (Number.isNaN(date.valueOf())) return base;
+  return new Date(date.valueOf() + index * 1000).toISOString();
+}
+
+function toIso(value) {
+  if (!value && value !== 0) return "";
+  if (typeof value === "number") {
+    const ms = value > 1e12 ? value : value * 1000;
+    const date = new Date(ms);
+    return Number.isNaN(date.valueOf()) ? "" : date.toISOString();
+  }
+  const date = new Date(value);
+  return Number.isNaN(date.valueOf()) ? "" : date.toISOString();
+}
+
+function parseJson(text) {
+  if (typeof text !== "string" || !text.trim()) return null;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}
+
+function firstString(...values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return "";
+}
+
+function positiveNumber(value) {
+  const number = Number(value);
+  return Number.isFinite(number) && number > 0 ? number : undefined;
+}
+
+function compactObject(value) {
+  if (!value || typeof value !== "object") return value;
+  const result = {};
+  for (const [key, item] of Object.entries(value)) {
+    if (item === undefined || item === null || item === "") continue;
+    result[key] = item;
+  }
+  return Object.keys(result).length ? result : undefined;
+}
+
+module.exports = {
+  decodeGrokCwd,
+  grokSessionSummary,
+  parseGrokUpdatesFile,
+  parseGrokUpdatesText
+};