agentel 0.2.0

package/src/importers/aider.js

@@ -0,0 +1,553 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { spawnSync } = require("child_process");
+
+const DEFAULT_COMMIT_WINDOW_MS = 15 * 60 * 1000;
+
+function parseAiderHistoryFile(file, options = {}) {
+  const stat = safeStat(file);
+  const fallbackTime = toIso(options.fallbackTime || stat?.mtimeMs || Date.now());
+  const text = fs.readFileSync(file, "utf8");
+  const cwd = options.cwd || findGitRoot(path.dirname(file)) || path.dirname(file);
+  const sourceFiles = aiderSourceFiles(file, options);
+  const llmHistoryFile = sourceFiles.find((candidate) => path.basename(candidate) === ".aider.llm.history");
+  const llmEntries = llmHistoryFile ? parseAiderLlmHistoryFile(llmHistoryFile) : [];
+  let messages = parseAiderChatHistoryText(text, { fallbackTime });
+  messages = applyAiderLlmHistory(messages, llmEntries);
+  if (options.includeGitDiffs !== false) {
+    messages = correlateAiderMessagesWithGit(messages, cwd, {
+      maxTimeDeltaMs: options.maxTimeDeltaMs,
+      maxCommits: options.maxCommits
+    });
+  }
+  return { cwd, messages, sourceFiles, llmEntries };
+}
+
+function parseAiderChatHistoryText(text, options = {}) {
+  const fallbackTime = toIso(options.fallbackTime || Date.now());
+  const lines = String(text || "").split(/\r?\n/);
+  const messages = [];
+  let pendingUser = "";
+  let assistant = [];
+  const flush = () => {
+    if (pendingUser.trim()) {
+      messages.push({
+        role: "user",
+        content: pendingUser.trim(),
+        timestamp: offsetTimestamp(fallbackTime, messages.length),
+        metadata: { provider: "aider", source: "chat_history" }
+      });
+    }
+    const assistantText = assistant.join("\n").trim();
+    if (assistantText) {
+      messages.push({
+        role: "assistant",
+        content: assistantText,
+        timestamp: offsetTimestamp(fallbackTime, messages.length),
+        metadata: { provider: "aider", source: "chat_history" }
+      });
+    }
+    pendingUser = "";
+    assistant = [];
+  };
+
+  for (const line of lines) {
+    const heading = line.match(/^####\s+(.+?)\s*$/);
+    if (heading) {
+      flush();
+      pendingUser = heading[1];
+      continue;
+    }
+    if (pendingUser) assistant.push(line);
+  }
+  flush();
+  return dedupeAdjacentMessages(messages);
+}
+
+function parseAiderLlmHistoryFile(file) {
+  if (!safeStat(file)) return [];
+  return parseAiderLlmHistoryText(fs.readFileSync(file, "utf8"));
+}
+
+function parseAiderLlmHistoryText(text) {
+  const entries = [];
+  const lines = String(text || "").split(/\r?\n/);
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const parsed = tryParseJson(trimmed);
+    if (!parsed) continue;
+    const extracted = extractAiderLlmEntry(parsed);
+    if (extracted) entries.push(extracted);
+  }
+  if (entries.length) return entries;
+  return parseRoleDelimitedLlmHistory(text);
+}
+
+function applyAiderLlmHistory(messages, entries) {
+  if (!Array.isArray(entries) || !entries.length) return messages;
+  const nextMessages = messages.map((message) => ({ ...message, metadata: { ...(message.metadata || {}) } }));
+  const assistantIndexes = nextMessages.map((message, index) => (message.role === "assistant" ? index : -1)).filter((index) => index >= 0);
+  const used = new Set();
+  for (const entry of entries) {
+    const index = bestAssistantIndexForLlmEntry(nextMessages, entry, used, assistantIndexes);
+    if (index < 0) continue;
+    used.add(index);
+    const metadata = nextMessages[index].metadata || {};
+    nextMessages[index].metadata = {
+      ...metadata,
+      ...(entry.model && !metadata.model ? { model: entry.model } : {}),
+      ...(entry.usage && !metadata.usage ? { usage: entry.usage } : {}),
+      llmHistory: compactObject({
+        timestamp: entry.timestamp,
+        requestId: entry.requestId,
+        model: entry.model,
+        usage: entry.usage
+      })
+    };
+  }
+  return nextMessages;
+}
+
+function correlateAiderMessagesWithGit(messages, cwd, options = {}) {
+  if (!cwd || !safeStat(path.join(cwd, ".git"))) return messages;
+  const commits = Array.isArray(options.commits) ? options.commits : readAiderGitCommits(cwd, options);
+  if (!commits.length) return messages;
+
+  const nextMessages = messages.map((message) => ({
+    ...message,
+    metadata: { ...(message.metadata || {}) }
+  }));
+  const turns = aiderTurns(nextMessages);
+
+  for (const commit of commits) {
+    const match = bestTurnForCommit(commit, turns, options);
+    if (!match) continue;
+    const diff = commit.diff || readCommitDiff(cwd, commit.hash);
+    if (!diff.trim()) continue;
+    const assistant = nextMessages[match.assistantIndex];
+    const existingToolCalls = Array.isArray(assistant.metadata.toolCalls) ? assistant.metadata.toolCalls : [];
+    const commitMetadata = {
+      hash: commit.hash,
+      subject: commit.subject,
+      timestamp: commit.timestamp,
+      source: "aider"
+    };
+    const existingCommits = Array.isArray(assistant.metadata.gitCommits) ? assistant.metadata.gitCommits : [];
+    assistant.metadata = {
+      ...assistant.metadata,
+      toolCalls: [
+        ...existingToolCalls,
+        aiderDiffToolCall(commit, diff, match.prompt)
+      ],
+      gitCommit: assistant.metadata.gitCommit || commitMetadata,
+      gitCommits: [...existingCommits, commitMetadata]
+    };
+  }
+  return nextMessages;
+}
+
+function readAiderGitCommits(cwd, options = {}) {
+  const args = [
+    "-C",
+    cwd,
+    "log",
+    `--max-count=${Number(options.maxCommits || 100)}`,
+    "--date=iso-strict",
+    "--format=%x1e%H%x00%aI%x00%an%x00%ae%x00%s%x00%b"
+  ];
+  const result = spawnSync("git", args, { encoding: "utf8", maxBuffer: 10 * 1024 * 1024 });
+  if (result.error || result.status !== 0) return [];
+  return result.stdout
+    .split("\x1e")
+    .map((record) => parseGitLogRecord(record))
+    .filter(Boolean);
+}
+
+function aiderSourceFiles(file, options = {}) {
+  const dir = path.dirname(file);
+  const candidates = [
+    file,
+    options.llmHistoryFile,
+    path.join(dir, ".aider.llm.history"),
+    options.inputHistoryFile,
+    path.join(dir, ".aider.input.history")
+  ];
+  const files = [];
+  for (const candidate of candidates) {
+    if (candidate && safeStat(candidate) && !files.includes(candidate)) files.push(candidate);
+  }
+  return files;
+}
+
+function extractAiderLlmEntry(value) {
+  if (!value || typeof value !== "object") return null;
+  const prompt = firstNonEmptyString(
+    value.prompt,
+    value.input,
+    value.user,
+    value.userPrompt,
+    lastMessageContent(value.messages, "user"),
+    lastMessageContent(value.request?.messages, "user"),
+    lastMessageContent(value.kwargs?.messages, "user")
+  );
+  const response = firstNonEmptyString(
+    value.response,
+    value.output,
+    value.assistant,
+    value.completion,
+    contentFromMessage(value.message),
+    contentFromMessage(value.response?.message),
+    contentFromChoice(value.choices),
+    contentFromChoice(value.response?.choices),
+    lastMessageContent(value.messages, "assistant")
+  );
+  const model = firstNonEmptyString(value.model, value.response?.model, value.request?.model, value.kwargs?.model, value.params?.model);
+  const usage = normalizeUsage(value.usage || value.response?.usage || value.token_usage || value.tokenUsage || value.metrics?.usage);
+  const timestamp = firstTimestamp(value.timestamp, value.created_at, value.createdAt, value.time, value.ts);
+  const requestId = firstNonEmptyString(value.request_id, value.requestId, value.id, value.response?.id);
+  if (!prompt && !response && !model && !usage) return null;
+  return compactObject({ prompt, response, model, usage, timestamp, requestId, raw: value });
+}
+
+function parseRoleDelimitedLlmHistory(text) {
+  const entries = [];
+  let current = null;
+  let role = null;
+  for (const rawLine of String(text || "").split(/\r?\n/)) {
+    const line = rawLine.trimEnd();
+    const marker = line.match(/^(?:#{1,6}\s*)?(USER|ASSISTANT|MODEL|RESPONSE|SYSTEM)\s*:?\s*$/i);
+    if (marker) {
+      if (!current) current = {};
+      role = marker[1].toLowerCase();
+      continue;
+    }
+    const inline = line.match(/^(USER|ASSISTANT|MODEL|RESPONSE):\s+(.+)$/i);
+    if (inline) {
+      if (!current) current = {};
+      role = inline[1].toLowerCase();
+      appendRoleText(current, role, inline[2]);
+      continue;
+    }
+    if (line.match(/^-{3,}$/) && current) {
+      const entry = extractAiderLlmEntry(current);
+      if (entry) entries.push(entry);
+      current = null;
+      role = null;
+      continue;
+    }
+    if (current && role) appendRoleText(current, role, line);
+  }
+  if (current) {
+    const entry = extractAiderLlmEntry(current);
+    if (entry) entries.push(entry);
+  }
+  return entries;
+}
+
+function appendRoleText(target, role, line) {
+  const key = role === "assistant" || role === "model" || role === "response" ? "response" : "prompt";
+  target[key] = [target[key], line].filter(Boolean).join("\n").trim();
+}
+
+function bestAssistantIndexForLlmEntry(messages, entry, used, assistantIndexes) {
+  let best = { index: -1, score: 0 };
+  for (const index of assistantIndexes) {
+    if (used.has(index)) continue;
+    const previousUser = previousUserMessage(messages, index);
+    let score = 0;
+    if (entry.prompt && previousUser) score += textSimilarity(entry.prompt, previousUser.content) * 6;
+    if (entry.response && messages[index].content) score += textSimilarity(entry.response, messages[index].content) * 3;
+    if (entry.timestamp && messages[index].timestamp) {
+      const delta = Math.abs(Date.parse(entry.timestamp) - Date.parse(messages[index].timestamp));
+      if (Number.isFinite(delta) && delta <= DEFAULT_COMMIT_WINDOW_MS) score += 2;
+    }
+    if (!entry.prompt && !entry.response && !used.size) score += 1;
+    if (score > best.score) best = { index, score };
+  }
+  return best.score >= 1.5 ? best.index : -1;
+}
+
+function aiderTurns(messages) {
+  const turns = [];
+  for (let index = 0; index < messages.length; index++) {
+    if (messages[index].role !== "assistant") continue;
+    const userIndex = previousUserIndex(messages, index);
+    if (userIndex < 0) continue;
+    turns.push({
+      index: turns.length,
+      userIndex,
+      assistantIndex: index,
+      prompt: messages[userIndex].content || "",
+      timestamp: messages[index].timestamp || messages[userIndex].timestamp
+    });
+  }
+  return turns;
+}
+
+function bestTurnForCommit(commit, turns, options = {}) {
+  let best = null;
+  for (const turn of turns) {
+    const score = aiderCommitMatchScore(commit, turn, options);
+    if (!best || score > best.score) best = { ...turn, score };
+  }
+  return best && best.score >= 5 ? best : null;
+}
+
+function aiderCommitMatchScore(commit, turn, options = {}) {
+  const maxTimeDeltaMs = Number(options.maxTimeDeltaMs || DEFAULT_COMMIT_WINDOW_MS);
+  const message = [commit.subject, commit.body].filter(Boolean).join("\n");
+  let score = 0;
+  if (isLikelyAiderCommit(commit)) score += 3;
+  const similarity = textSimilarity(turn.prompt, message);
+  if (similarity >= 0.8) score += 4;
+  else if (similarity >= 0.35) score += 2;
+  if (commit.timestamp && turn.timestamp) {
+    const delta = Math.abs(Date.parse(commit.timestamp) - Date.parse(turn.timestamp));
+    if (Number.isFinite(delta) && delta <= maxTimeDeltaMs) score += 2;
+  }
+  return score;
+}
+
+function isLikelyAiderCommit(commit) {
+  const text = [commit.subject, commit.body, commit.authorName, commit.authorEmail].filter(Boolean).join("\n");
+  return /\baider\b/i.test(text) || /^\s*(AI|LLM)\s*:/i.test(commit.subject || "");
+}
+
+function aiderDiffToolCall(commit, diff, prompt) {
+  const shortHash = commit.hash ? commit.hash.slice(0, 12) : "";
+  const title = shortHash ? `Aider commit ${shortHash}` : "Aider commit diff";
+  const summary = firstNonEmptyString(commit.subject, prompt, "Aider code changes");
+  return {
+    name: "aider_git_diff",
+    displayName: "Aider Git Diff",
+    category: "edit",
+    title,
+    status: "completed",
+    argument: summary,
+    rawInputSummary: summary,
+    inputPreview: summary,
+    arguments: compactObject({
+      diff,
+      commit: commit.hash,
+      subject: commit.subject,
+      timestamp: commit.timestamp
+    }),
+    provider: "aider"
+  };
+}
+
+function parseGitLogRecord(record) {
+  const trimmed = record.replace(/^\n+|\n+$/g, "");
+  if (!trimmed) return null;
+  const parts = trimmed.split("\x00");
+  if (parts.length < 5) return null;
+  const [hash, timestamp, authorName, authorEmail, subject, ...bodyParts] = parts;
+  return {
+    hash,
+    timestamp,
+    authorName,
+    authorEmail,
+    subject: subject || "",
+    body: bodyParts.join("\x00").trim()
+  };
+}
+
+function readCommitDiff(cwd, hash) {
+  if (!hash) return "";
+  const result = spawnSync(
+    "git",
+    ["-C", cwd, "show", "--format=", "--no-ext-diff", "--patch", "--find-renames", "--find-copies", "--unified=3", hash],
+    { encoding: "utf8", maxBuffer: 20 * 1024 * 1024 }
+  );
+  if (result.error || result.status !== 0) return "";
+  return result.stdout.trim();
+}
+
+function previousUserMessage(messages, index) {
+  const userIndex = previousUserIndex(messages, index);
+  return userIndex >= 0 ? messages[userIndex] : null;
+}
+
+function previousUserIndex(messages, index) {
+  for (let cursor = index - 1; cursor >= 0; cursor--) {
+    if (messages[cursor].role === "user") return cursor;
+  }
+  return -1;
+}
+
+function contentFromChoice(choices) {
+  if (!Array.isArray(choices)) return "";
+  for (const choice of choices) {
+    const content = contentFromMessage(choice?.message) || contentFromMessage(choice?.delta) || firstNonEmptyString(choice?.text, choice?.content);
+    if (content) return content;
+  }
+  return "";
+}
+
+function lastMessageContent(messages, role) {
+  if (!Array.isArray(messages)) return "";
+  for (let index = messages.length - 1; index >= 0; index--) {
+    const message = messages[index];
+    if (!role || message?.role === role) {
+      const content = contentFromMessage(message);
+      if (content) return content;
+    }
+  }
+  return "";
+}
+
+function contentFromMessage(message) {
+  if (!message) return "";
+  if (typeof message === "string") return message.trim();
+  if (Array.isArray(message)) return message.map(contentFromMessage).filter(Boolean).join("\n").trim();
+  if (typeof message !== "object") return String(message).trim();
+  if (typeof message.content === "string") return message.content.trim();
+  if (Array.isArray(message.content)) return message.content.map(contentFromPart).filter(Boolean).join("\n").trim();
+  return firstNonEmptyString(message.text, message.message, message.output);
+}
+
+function contentFromPart(part) {
+  if (!part) return "";
+  if (typeof part === "string") return part.trim();
+  if (typeof part !== "object") return String(part).trim();
+  return firstNonEmptyString(part.text, part.content, part.input?.text);
+}
+
+function normalizeUsage(usage) {
+  if (!usage || typeof usage !== "object") return null;
+  return compactObject({
+    inputTokens: firstNumber(usage.input_tokens, usage.prompt_tokens, usage.inputTokens, usage.promptTokens),
+    outputTokens: firstNumber(usage.output_tokens, usage.completion_tokens, usage.outputTokens, usage.completionTokens),
+    totalTokens: firstNumber(usage.total_tokens, usage.totalTokens),
+    cacheCreationInputTokens: firstNumber(usage.cache_creation_input_tokens, usage.cacheCreationInputTokens),
+    cacheReadInputTokens: firstNumber(usage.cache_read_input_tokens, usage.cacheReadInputTokens),
+    costUsd: firstNumber(usage.cost, usage.cost_usd, usage.costUsd)
+  });
+}
+
+function textSimilarity(a, b) {
+  const left = tokenSet(a);
+  const right = tokenSet(b);
+  if (!left.size || !right.size) return 0;
+  let overlap = 0;
+  for (const token of left) {
+    if (right.has(token)) overlap++;
+  }
+  return overlap / Math.min(left.size, right.size);
+}
+
+function tokenSet(value) {
+  return new Set(
+    String(value || "")
+      .toLowerCase()
+      .replace(/[`"'’]/g, "")
+      .split(/[^a-z0-9_.-]+/)
+      .filter((token) => token.length >= 2)
+  );
+}
+
+function dedupeAdjacentMessages(messages) {
+  const deduped = [];
+  for (const message of messages) {
+    const previous = deduped[deduped.length - 1];
+    if (previous && previous.role === message.role && previous.content === message.content) continue;
+    deduped.push(message);
+  }
+  return deduped;
+}
+
+function findGitRoot(start) {
+  let dir = path.resolve(start);
+  while (dir && dir !== path.dirname(dir)) {
+    if (safeStat(path.join(dir, ".git"))) return dir;
+    dir = path.dirname(dir);
+  }
+  return null;
+}
+
+function firstTimestamp(...values) {
+  for (const value of values) {
+    const iso = toIso(value);
+    if (iso) return iso;
+  }
+  return "";
+}
+
+function toIso(value) {
+  if (value === undefined || value === null || value === "") return "";
+  if (value instanceof Date) return Number.isNaN(value.getTime()) ? "" : value.toISOString();
+  if (typeof value === "number") {
+    const date = new Date(value < 10000000000 ? value * 1000 : value);
+    return Number.isNaN(date.getTime()) ? "" : date.toISOString();
+  }
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? "" : date.toISOString();
+}
+
+function offsetTimestamp(base, offset) {
+  const time = Date.parse(base);
+  if (!Number.isFinite(time)) return new Date(Date.now() + offset).toISOString();
+  return new Date(time + offset).toISOString();
+}
+
+function safeStat(file) {
+  if (!file) return null;
+  try {
+    return fs.statSync(file);
+  } catch {
+    return null;
+  }
+}
+
+function tryParseJson(value) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return null;
+  }
+}
+
+function firstNonEmptyString(...values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) return value.trim();
+    if (value !== undefined && value !== null && typeof value !== "object") {
+      const text = String(value).trim();
+      if (text) return text;
+    }
+  }
+  return "";
+}
+
+function firstNumber(...values) {
+  for (const value of values) {
+    const number = Number(value);
+    if (Number.isFinite(number)) return number;
+  }
+  return undefined;
+}
+
+function compactObject(value) {
+  const out = {};
+  for (const [key, item] of Object.entries(value || {})) {
+    if (item === undefined || item === null || item === "") continue;
+    if (typeof item === "object" && !Array.isArray(item) && !Object.keys(item).length) continue;
+    out[key] = item;
+  }
+  return out;
+}
+
+module.exports = {
+  aiderCommitMatchScore,
+  aiderSourceFiles,
+  applyAiderLlmHistory,
+  correlateAiderMessagesWithGit,
+  isLikelyAiderCommit,
+  parseAiderChatHistoryText,
+  parseAiderHistoryFile,
+  parseAiderLlmHistoryFile,
+  parseAiderLlmHistoryText,
+  readAiderGitCommits
+};