llm-cost-attribution 0.2.0 → 0.3.0

package/src/git-diff-source.mjs

@@ -0,0 +1,278 @@
+/**
+ * Local git diff-size adapter for cost-driver feature extraction.
+ *
+ * Limits: this reads only history available in the local checkout, and it can
+ * key commits only when their subjects contain issue identifiers.
+ */
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+
+export const DEFAULT_KEY_PATTERN = /[A-Z][A-Z0-9]+-\d+/;
+export const GIT_LOG_FORMAT = '%x1e%H%x1f%s';
+
+const COMMIT_SEPARATOR = '\x1e';
+const FIELD_SEPARATOR = '\x1f';
+
+/**
+ * Local-git implementation of the DiffSource port.
+ */
+export class LocalGitDiffSource {
+  /**
+   * @param {string} [repoPath]
+   * @param {object} [options]
+   * @returns {AsyncGenerator<object, object, void>}
+   */
+  async *read(repoPath = process.cwd(), options = {}) {
+    const summary = await this.readResult(repoPath, options);
+    try {
+      for (const record of summary.records) yield record;
+      return summary;
+    } finally {
+      if (typeof options.onSummary === 'function') options.onSummary(summary);
+    }
+  }
+
+  /**
+   * @param {string} [repoPath]
+   * @param {object} [options]
+   * @returns {Promise<{ records: object[], unmatched: object, error: object | null }>}
+   */
+  async readResult(repoPath = process.cwd(), options = {}) {
+    if (typeof repoPath !== 'string' || repoPath === '') {
+      return emptyResult('repoPath must be a non-empty string');
+    }
+
+    if (typeof options.gitLogText === 'string') {
+      return withNoError(parseGitNumstatLog(options.gitLogText, options));
+    }
+
+    let stdout;
+    try {
+      ({ stdout } = await execFileAsync('git', gitLogArgs(repoPath, options), {
+        encoding: 'utf8',
+        maxBuffer: options.maxBuffer ?? 1024 * 1024 * 64,
+      }));
+    } catch (err) {
+      return emptyResult(gitErrorMessage(err, repoPath));
+    }
+
+    return withNoError(parseGitNumstatLog(stdout, options));
+  }
+}
+
+/**
+ * Read git diff statistics from a local repository and yield one aggregated
+ * diff record per issue key found in commit subjects.
+ *
+ * The async generator never throws for git failures. On completion, its return
+ * value is a summary object `{ records, unmatched, error }`; callers using
+ * `for await` can also pass `onSummary(summary)` to receive it.
+ *
+ * @param {string} [repoPath]
+ *   Local git repository path. Defaults to `process.cwd()`.
+ * @param {object} [options]
+ * @param {RegExp | string} [options.keyPattern]
+ *   Pattern used to extract issue keys from commit subjects.
+ * @param {string | string[]} [options.revRange]
+ *   Optional rev range or list of git rev arguments.
+ * @param {string} [options.gitLogText]
+ *   Recorded `git log --numstat` text. Intended for fixtures/tests.
+ * @param {(summary: object) => void} [options.onSummary]
+ *   Receives `{ records, unmatched, error }` after parsing or git failure.
+ * @returns {AsyncGenerator<object, object, void>}
+ */
+export async function *readGitDiffs(repoPath = process.cwd(), options = {}) {
+  return yield* new LocalGitDiffSource().read(repoPath, options);
+}
+
+/**
+ * Read and collect local-git diff records with their unmatched/error summary.
+ *
+ * @param {string} [repoPath]
+ * @param {object} [options]
+ * @returns {Promise<{ records: object[], unmatched: object, error: object | null }>}
+ */
+export async function readGitDiffResult(repoPath = process.cwd(), options = {}) {
+  return new LocalGitDiffSource().readResult(repoPath, options);
+}
+
+/**
+ * Parse recorded `git log --numstat --format=%x1e%H%x1f%s` output.
+ *
+ * @param {string} logText
+ * @param {object} [options]
+ * @param {RegExp | string} [options.keyPattern]
+ * @returns {{ records: object[], unmatched: object }}
+ */
+export function parseGitNumstatLog(logText, options = {}) {
+  const keyPattern = globalKeyPattern(options.keyPattern ?? DEFAULT_KEY_PATTERN);
+  const aggregates = new Map();
+  const unmatched = { count: 0, shas: [] };
+  const stats = {
+    commits: 0,
+    matchedCommits: 0,
+    unmatchedCommits: 0,
+    skippedEmptyCommits: 0,
+  };
+
+  let current = null;
+  for (const rawLine of String(logText).split(/\r?\n/)) {
+    if (rawLine.startsWith(COMMIT_SEPARATOR)) {
+      consumeCommit(current, keyPattern, aggregates, unmatched, stats);
+      current = parseCommitHeader(rawLine);
+      continue;
+    }
+
+    if (current === null || rawLine === '') continue;
+    const numstat = parseNumstatLine(rawLine);
+    if (numstat === null) continue;
+    current.additions += numstat.additions;
+    current.deletions += numstat.deletions;
+    current.changedFiles += 1;
+  }
+  consumeCommit(current, keyPattern, aggregates, unmatched, stats);
+
+  return {
+    records: Array.from(aggregates.values()),
+    unmatched: { ...unmatched, ...stats },
+  };
+}
+
+function gitLogArgs(repoPath, options) {
+  const args = [
+    '-C',
+    repoPath,
+    'log',
+    '--numstat',
+    `--format=${GIT_LOG_FORMAT}`,
+  ];
+  if (options.revRange !== undefined) args.push(...revRangeArgs(options.revRange));
+  return args;
+}
+
+function revRangeArgs(revRange) {
+  if (Array.isArray(revRange)) return revRange.map(String).filter((arg) => arg !== '');
+  if (typeof revRange === 'string' && revRange !== '') return [revRange];
+  return [];
+}
+
+function parseCommitHeader(line) {
+  const header = line.slice(COMMIT_SEPARATOR.length);
+  const fieldIndex = header.indexOf(FIELD_SEPARATOR);
+  const sha = fieldIndex === -1 ? header : header.slice(0, fieldIndex);
+  const subject = fieldIndex === -1 ? '' : header.slice(fieldIndex + FIELD_SEPARATOR.length);
+  return {
+    sha,
+    subject,
+    additions: 0,
+    deletions: 0,
+    changedFiles: 0,
+  };
+}
+
+function parseNumstatLine(line) {
+  const fields = line.split('\t');
+  if (fields.length < 3) return null;
+  return {
+    additions: parseNumstatCount(fields[0]),
+    deletions: parseNumstatCount(fields[1]),
+  };
+}
+
+function parseNumstatCount(value) {
+  if (value === '-') return 0;
+  const count = Number(value);
+  return Number.isFinite(count) && count >= 0 ? count : 0;
+}
+
+function consumeCommit(commit, keyPattern, aggregates, unmatched, stats) {
+  if (commit === null) return;
+  stats.commits += 1;
+
+  if (commit.changedFiles === 0) {
+    stats.skippedEmptyCommits += 1;
+    return;
+  }
+
+  const keys = uniqueMatches(commit.subject, keyPattern);
+  if (keys.length === 0) {
+    unmatched.count += 1;
+    unmatched.shas.push(commit.sha);
+    stats.unmatchedCommits += 1;
+    return;
+  }
+
+  stats.matchedCommits += 1;
+  for (const key of keys) {
+    let record = aggregates.get(key);
+    if (record === undefined) {
+      record = {
+        key,
+        additions: 0,
+        deletions: 0,
+        changedFiles: 0,
+        shas: [],
+      };
+      aggregates.set(key, record);
+    }
+    record.additions += commit.additions;
+    record.deletions += commit.deletions;
+    record.changedFiles += commit.changedFiles;
+    record.shas.push(commit.sha);
+  }
+}
+
+function uniqueMatches(subject, keyPattern) {
+  keyPattern.lastIndex = 0;
+  const keys = [];
+  const seen = new Set();
+  let match;
+  while ((match = keyPattern.exec(subject)) !== null) {
+    const key = match[0];
+    if (!seen.has(key)) {
+      seen.add(key);
+      keys.push(key);
+    }
+    if (match[0] === '') keyPattern.lastIndex += 1;
+  }
+  return keys;
+}
+
+function globalKeyPattern(pattern) {
+  if (typeof pattern === 'string') return new RegExp(pattern, 'g');
+  if (!(pattern instanceof RegExp)) return globalKeyPattern(DEFAULT_KEY_PATTERN);
+  const flags = pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`;
+  return new RegExp(pattern.source, flags);
+}
+
+function withNoError(result) {
+  return {
+    ...result,
+    error: null,
+  };
+}
+
+function emptyResult(message) {
+  return {
+    records: [],
+    unmatched: {
+      count: 0,
+      shas: [],
+      commits: 0,
+      matchedCommits: 0,
+      unmatchedCommits: 0,
+      skippedEmptyCommits: 0,
+    },
+    error: {
+      message,
+    },
+  };
+}
+
+function gitErrorMessage(err, repoPath) {
+  const stderr = typeof err?.stderr === 'string' ? err.stderr.trim() : '';
+  const detail = stderr || (err instanceof Error ? err.message : String(err));
+  return `git log --numstat failed for ${repoPath}: ${detail}`;
+}

package/src/index.mjs

@@ -73,6 +73,15 @@ export {
   normalizeModelName,
   ratesForModel,
 } from './pricing.mjs';
+export { correlateCostWithFeature } from './correlate.mjs';
+export {
+  DEFAULT_KEY_PATTERN,
+  LocalGitDiffSource,
+  parseGitNumstatLog,
+  readGitDiffResult,
+  readGitDiffs,
+} from './git-diff-source.mjs';
+export { joinCostWithFeature, BUILTIN_JOIN_STRATEGIES } from './cost-feature-join.mjs';
 
 /**
  * Default `PricingTable` adapter for `forecastIssueCost`. Delegates to
@@ -231,6 +240,74 @@ export async function computeIssueCostFromUsage(issueIdentifier, usageSource) {
   return rollupUsageRecords(issueIdentifier, records);
 }
 
+/**
+ * Walk every Claude session + every Codex rollout and yield spec-compliant
+ * usage.jsonl records one at a time. Sessions whose cwd doesn't match the
+ * configured pattern are skipped.
+ *
+ * Use this when you want to stream records to a downstream consumer (such as
+ * `dump-usage` or an in-process correlator) instead of writing them to a file.
+ * `backfillUsageFromTranscripts` is built on top of this generator.
+ *
+ * The generator's return value (from `gen.next().value` once `done` is true)
+ * is `{ recordsYielded, sessionsProcessed, sessionsSkipped }`.
+ *
+ * @param {object} [options]
+ * @param {RegExp} [options.cwdPattern]
+ * @param {string} [options.claudeProjectsDir]
+ * @param {string} [options.codexSessionsDir]
+ * @param {(progress: { phase: string, file?: string, processed: number, total: number, recordsYielded: number }) => void} [options.onProgress]
+ */
+export async function *iterateUsageFromTranscripts(options = {}) {
+  const cwdPattern = options.cwdPattern ?? DEFAULT_CWD_PATTERN;
+  const claudeRootDir = options.claudeProjectsDir ?? join(homedir(), '.claude', 'projects');
+  const codexRootDir = options.codexSessionsDir ?? join(homedir(), '.codex', 'sessions');
+  const onProgress = options.onProgress ?? (() => undefined);
+
+  let recordsYielded = 0;
+  let sessionsProcessed = 0;
+  let sessionsSkipped = 0;
+
+  // Phase 1: walk Claude project dirs and yield records for matching sessions.
+  const claudeDirs = await findClaudeProjectDirs(claudeRootDir, (encoded) => issueFromClaudeProjectDirName(encoded, cwdPattern) !== null);
+  for (let i = 0; i < claudeDirs.length; i++) {
+    const dir = claudeDirs[i];
+    const encoded = dir.split('/').pop() ?? '';
+    const issueIdentifier = issueFromClaudeProjectDirName(encoded, cwdPattern);
+    if (issueIdentifier === null) continue;
+    for (const file of await listJsonlsRecursively(dir)) {
+      const session = await parseClaudeSession(file);
+      if (session === null) { sessionsSkipped += 1; continue; }
+      const records = sessionToUsageRecords(session, issueIdentifier);
+      if (records.length === 0) { sessionsSkipped += 1; continue; }
+      for (const rec of records) yield rec;
+      recordsYielded += records.length;
+      sessionsProcessed += 1;
+    }
+    onProgress({ phase: 'claude', file: dir, processed: i + 1, total: claudeDirs.length, recordsYielded });
+  }
+
+  // Phase 2: walk Codex rollouts.
+  const codexFiles = await listCodexRollouts(codexRootDir);
+  for (let i = 0; i < codexFiles.length; i++) {
+    const file = codexFiles[i];
+    const session = await parseCodexSession(file);
+    if (session === null) { sessionsSkipped += 1; continue; }
+    const issueIdentifier = issueFromCwd(session.cwd, cwdPattern);
+    if (issueIdentifier === null) { sessionsSkipped += 1; continue; }
+    const records = sessionToUsageRecords(session, issueIdentifier);
+    if (records.length === 0) { sessionsSkipped += 1; continue; }
+    for (const rec of records) yield rec;
+    recordsYielded += records.length;
+    sessionsProcessed += 1;
+    if ((i + 1) % 100 === 0 || i + 1 === codexFiles.length) {
+      onProgress({ phase: 'codex', file, processed: i + 1, total: codexFiles.length, recordsYielded });
+    }
+  }
+
+  return { recordsYielded, sessionsProcessed, sessionsSkipped };
+}
+
 /**
  * Walk every Claude session + every Codex rollout, derive spec-compliant
  * usage.jsonl records for each, and append them to a single output file.