copilot-metrics 0.1.3 → 0.1.5

package/src/ingest.js

@@ -2,13 +2,23 @@
 
 const crypto = require('node:crypto');
 const fs = require('node:fs');
+const os = require('node:os');
 const path = require('node:path');
 const { readJsonl } = require('./jsonl');
 const { normalizePayload, normalizeHookEvent, normalizeCopilotSessionEvents } = require('./otel');
 const { estimateCost, PRICING_VERSION } = require('./pricing');
-const { existingRawFingerprints, insertImport } = require('./sqlite-store');
+const {
+  attachVscodeChatLabelEvidence,
+  existingRawFingerprints,
+  importedLineHighWater,
+  insertImport,
+  queryRows,
+  updateUsageCostEstimates,
+  updateVscodeUsageResponseIds,
+  vscodeRawRecordsNeedingResponseBackfill,
+} = require('./sqlite-store');
 const { attachUsageLabelEvidence, attachHookLabelEvidence } = require('./labels');
-const { loadConfiguredExtractors } = require('./label-extractors');
+const { loadConfiguredExtractors, runLabelExtractors } = require('./label-extractors');
 
 function enrichCosts(records) {
   return records.map((record) => {
@@ -42,11 +52,241 @@ function isCopilotSessionUsageRecord(record) {
   return record.value && record.value.type === 'session.shutdown';
 }
 
+function pushText(values, value) {
+  if (typeof value === 'string' && value.trim()) values.push(value);
+}
+
+function pushPromptCandidates(values, value) {
+  if (!value || typeof value !== 'object') return;
+  if (Array.isArray(value)) {
+    for (const item of value) pushPromptCandidates(values, item);
+    return;
+  }
+  pushText(values, value.text);
+  pushText(values, value.value);
+  pushText(values, value.message);
+  pushText(values, value.prompt);
+  pushText(values, value.promptText);
+  pushText(values, value.renderedUserMessage);
+  pushText(values, value.userMessage);
+  if (value.renderedUserMessage && typeof value.renderedUserMessage === 'object') {
+    pushPromptCandidates(values, value.renderedUserMessage);
+  }
+  if (value.message && typeof value.message === 'object') pushPromptCandidates(values, value.message);
+  if (value.result && typeof value.result === 'object') pushPromptCandidates(values, value.result);
+  if (value.metadata && typeof value.metadata === 'object') pushPromptCandidates(values, value.metadata);
+}
+
+function responseId(value) {
+  if (!value || typeof value !== 'object') return null;
+  return value.responseId
+    || value.metadata?.responseId
+    || value.result?.responseId
+    || value.result?.metadata?.responseId
+    || value.modelMessageId
+    || value.metadata?.modelMessageId
+    || null;
+}
+
+function chatSessionId(value) {
+  if (!value || typeof value !== 'object') return null;
+  return value.sessionId
+    || value.sessionID
+    || value.metadata?.sessionId
+    || value.result?.sessionId
+    || value.result?.metadata?.sessionId
+    || null;
+}
+
+function chatRequestIndex(record) {
+  const key = Array.isArray(record.k) ? record.k : Array.isArray(record.key) ? record.key : [];
+  if (key[0] !== 'requests') return null;
+  const index = Number(key[1]);
+  return Number.isInteger(index) ? index : null;
+}
+
+function normalizeVscodeChatSession(records, extractors = []) {
+  const requests = new Map();
+  let defaultSessionId = null;
+
+  function entry(index) {
+    const key = String(index);
+    if (!requests.has(key)) requests.set(key, { texts: [] });
+    return requests.get(key);
+  }
+
+  function mergeRequest(index, request, sessionId) {
+    if (!request || typeof request !== 'object') return;
+    const current = entry(index);
+    current.sessionId = chatSessionId(request) || sessionId || current.sessionId;
+    current.responseId = responseId(request) || current.responseId;
+    pushPromptCandidates(current.texts, request);
+  }
+
+  for (const record of records) {
+    const value = record.value;
+    if (!value || typeof value !== 'object') continue;
+    const root = value.v && typeof value.v === 'object' ? value.v : value;
+    defaultSessionId = root.sessionId || root.sessionID || defaultSessionId;
+
+    if (Array.isArray(root.requests)) {
+      root.requests.forEach((request, index) => mergeRequest(index, request, defaultSessionId));
+    }
+
+    const key = Array.isArray(value.k) ? value.k : Array.isArray(value.key) ? value.key : [];
+    if (key.length === 1 && key[0] === 'requests' && Array.isArray(value.v)) {
+      const startIndex = requests.size;
+      value.v.forEach((request, offset) => mergeRequest(startIndex + offset, request, defaultSessionId));
+    }
+
+    const index = chatRequestIndex(value);
+    if (index !== null) {
+      const current = entry(index);
+      const patch = value.v;
+      if (patch && typeof patch === 'object') {
+        current.sessionId = chatSessionId(patch) || defaultSessionId || current.sessionId;
+        current.responseId = responseId(patch) || current.responseId;
+        pushPromptCandidates(current.texts, patch);
+      } else {
+        pushText(current.texts, patch);
+      }
+    }
+  }
+
+  return Array.from(requests.values())
+    .filter((request) => request.responseId)
+    .map((request) => {
+      const labelEvidence = runLabelExtractors('usage', { prompt: request.texts }, extractors)
+        .map((evidence) => ({
+          ...evidence,
+          source_type: 'usage',
+          source_field: 'vscode_chat_response',
+          source_value: request.responseId,
+          confidence: Math.max(Number(evidence.confidence || 0), 0.95),
+        }));
+      return {
+        responseId: request.responseId,
+        sessionId: request.sessionId || defaultSessionId || null,
+        label_evidence: labelEvidence,
+      };
+    })
+    .filter((request) => request.label_evidence.length > 0);
+}
+
+async function ingestVscodeChatSessionFile(options) {
+  const { dbPath, file } = options;
+  const sourceFile = path.resolve(file);
+  const parsed = readJsonl(sourceFile);
+  const mappings = normalizeVscodeChatSession(parsed.records, options.extractors || []);
+  const attached = await attachVscodeChatLabelEvidence(dbPath, mappings);
+  return {
+    source: 'vscode-chat',
+    file,
+    dbPath,
+    raw_records: 0,
+    new_raw_records: 0,
+    skipped_existing_records: 0,
+    usage_records: attached.matched_usage_records,
+    hook_events: 0,
+    label_evidence: attached.label_evidence,
+    warnings: parsed.warnings,
+    estimate_label: `estimate:${PRICING_VERSION}`,
+  };
+}
+
+async function backfillVscodeUsageResponseIds(dbPath, sourceFile) {
+  const rows = await vscodeRawRecordsNeedingResponseBackfill(dbPath, sourceFile);
+  const updates = [];
+  for (const row of rows) {
+    let payload;
+    try {
+      payload = JSON.parse(row.payload_json);
+    } catch {
+      continue;
+    }
+    for (const usage of normalizePayload(payload, 'vscode', row.line)) {
+      if (!usage.span_id) continue;
+      updates.push({
+        raw_line: usage.raw_line,
+        span_id: usage.span_id,
+        session_id: usage.session_id,
+        timestamp: usage.timestamp,
+        requested_model: usage.requested_model,
+        resolved_model: usage.resolved_model,
+        input_tokens: usage.input_tokens,
+        output_tokens: usage.output_tokens,
+        cache_read_tokens: usage.cache_read_tokens,
+        cache_creation_tokens: usage.cache_creation_tokens,
+        reasoning_tokens: usage.reasoning_tokens,
+      });
+    }
+  }
+  return updateVscodeUsageResponseIds(dbPath, updates);
+}
+
+function parseWarningsJson(value) {
+  try {
+    const parsed = JSON.parse(value || '[]');
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+
+async function repairUsageCostEstimates(dbPath) {
+  const rows = await queryRows(dbPath, `
+    SELECT id, requested_model, resolved_model, input_tokens, output_tokens,
+      cache_read_tokens, cache_creation_tokens, reasoning_tokens, warnings_json
+    FROM usage_records
+    WHERE estimated_ai_credits IS NULL
+      OR estimated_ai_credits = 0
+      OR warnings_json LIKE '%unknown_model:%'
+      OR warnings_json LIKE '%missing_model%'
+  `);
+  const updates = [];
+  for (const row of rows) {
+    const estimate = estimateCost(row);
+    if (estimate.warning) continue;
+    const warnings = parseWarningsJson(row.warnings_json)
+      .filter((warning) => !String(warning).startsWith('unknown_model:') && warning !== 'missing_model');
+    updates.push({
+      id: row.id,
+      estimated_usd: estimate.estimated_usd,
+      estimated_ai_credits: estimate.estimated_ai_credits,
+      warnings,
+    });
+  }
+  return updateUsageCostEstimates(dbPath, updates);
+}
+
 async function ingestFile(options) {
   const { dbPath, file, source } = options;
-  const parsed = readJsonl(file);
-  const warnings = [...parsed.warnings];
+  if (source === 'vscode-chat') return ingestVscodeChatSessionFile(options);
+
   const sourceFile = path.resolve(file);
+  const backfilledUsageRecords = source === 'vscode'
+    ? await backfillVscodeUsageResponseIds(dbPath, sourceFile)
+    : 0;
+  const highWaterLine = await importedLineHighWater(dbPath, source, sourceFile);
+  if (source === 'copilot-session' && highWaterLine > 0) {
+    return {
+      source,
+      file,
+      dbPath,
+      raw_records: 0,
+      new_raw_records: 0,
+      skipped_existing_records: highWaterLine,
+      usage_records: 0,
+      hook_events: 0,
+      label_evidence: 0,
+      backfilled_usage_records: backfilledUsageRecords,
+      warnings: [],
+      estimate_label: `estimate:${PRICING_VERSION}`,
+    };
+  }
+  const needsSessionContext = source === 'copilot-session' && highWaterLine === 0;
+  const parsed = readJsonl(file, { afterLine: needsSessionContext ? 0 : highWaterLine });
+  const warnings = [...parsed.warnings];
   const parsedRecords = parsed.records.map((record) => ({
     ...record,
     raw_fingerprint: rawFingerprint(source, sourceFile, record),
@@ -91,6 +331,7 @@ async function ingestFile(options) {
   }
 
   await insertImport(dbPath, source, sourceFile, newRecords, enrichedUsage, enrichedHooks, warnings);
+  const repairedCostRecords = await repairUsageCostEstimates(dbPath);
 
   return {
     source,
@@ -98,9 +339,11 @@ async function ingestFile(options) {
     dbPath,
     raw_records: importableRecords.length,
     new_raw_records: newRecords.length,
-    skipped_existing_records: importableRecords.length - newRecords.length,
+    skipped_existing_records: highWaterLine,
     usage_records: enrichedUsage.length,
     hook_events: enrichedHooks.length,
+    backfilled_usage_records: backfilledUsageRecords,
+    repaired_cost_records: repairedCostRecords,
     label_evidence: enrichedUsage.reduce((sum, usage) => sum + (usage.label_evidence || []).length, 0)
       + enrichedHooks.reduce((sum, event) => sum + (event.label_evidence || []).length, 0),
     warnings,
@@ -113,6 +356,7 @@ function configuredSourceFiles(paths, config = {}) {
   const telemetryConfig = config.telemetry || {};
   const files = [
     { source: 'vscode', file: sourceConfig.vscode?.telemetry || telemetryConfig.vscode || paths.vscodeOtelJsonl },
+    ...discoverVscodeChatSessionFiles(sourceConfig.vscode?.chatSessions),
     { source: 'hooks', file: sourceConfig.vscode?.hooks || paths.hookEventsJsonl },
     { source: 'copilot-cli', file: sourceConfig.copilotCli?.telemetry || telemetryConfig.copilotCli || paths.copilotCliOtelJsonl },
     { source: 'hooks', file: sourceConfig.copilotCli?.hooks || paths.hookEventsJsonl },
@@ -130,6 +374,40 @@ function configuredSourceFiles(paths, config = {}) {
     });
 }
 
+function listJsonlFiles(dir) {
+  if (!dir || !fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir, { withFileTypes: true })
+    .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
+    .map((entry) => path.join(dir, entry.name));
+}
+
+function discoverWorkspaceChatSessions(workspaceStorageDir) {
+  if (!workspaceStorageDir || !fs.existsSync(workspaceStorageDir)) return [];
+  return fs.readdirSync(workspaceStorageDir, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .flatMap((entry) => listJsonlFiles(path.join(workspaceStorageDir, entry.name, 'chatSessions')));
+}
+
+function discoverVscodeChatSessionFiles(configured) {
+  const configuredEntries = Array.isArray(configured) ? configured : configured ? [configured] : [];
+  const files = configuredEntries.length > 0
+    ? configuredEntries.flatMap((entry) => {
+      const resolved = path.resolve(entry);
+      if (!fs.existsSync(resolved)) return [];
+      const stat = fs.statSync(resolved);
+      if (stat.isFile()) return [resolved];
+      return listJsonlFiles(resolved).concat(discoverWorkspaceChatSessions(resolved));
+    })
+    : [
+      path.join(os.homedir(), '.config', 'Code', 'User', 'workspaceStorage'),
+      path.join(os.homedir(), '.config', 'Code - Insiders', 'User', 'workspaceStorage'),
+    ].flatMap(discoverWorkspaceChatSessions);
+
+  return files
+    .sort()
+    .map((file) => ({ source: 'vscode-chat', file }));
+}
+
 function discoverCopilotSessionFiles(sessionStateDir) {
   if (!sessionStateDir || !fs.existsSync(sessionStateDir)) return [];
   return fs.readdirSync(sessionStateDir, { withFileTypes: true })
@@ -167,5 +445,9 @@ module.exports = {
   autoImportConfiguredSources,
   configuredSourceFiles,
   discoverCopilotSessionFiles,
+  discoverVscodeChatSessionFiles,
+  backfillVscodeUsageResponseIds,
   ingestFile,
+  normalizeVscodeChatSession,
+  repairUsageCostEstimates,
 };

package/src/jsonl.js

@@ -2,19 +2,21 @@
 
 const fs = require('node:fs');
 
-function readJsonl(file) {
+function readJsonl(file, options = {}) {
   const text = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '';
   const records = [];
   const warnings = [];
+  const afterLine = Number(options.afterLine || 0);
 
   text.split(/\r?\n/).forEach((line, index) => {
-    if (!line.trim()) return;
+    const lineNumber = index + 1;
+    if (lineNumber <= afterLine || !line.trim()) return;
     try {
-      records.push({ line: index + 1, value: JSON.parse(line) });
+      records.push({ line: lineNumber, value: JSON.parse(line) });
     } catch (error) {
       warnings.push({
         code: 'malformed_jsonl',
-        line: index + 1,
+        line: lineNumber,
         message: error.message,
       });
     }

package/src/otel.js

@@ -2,9 +2,16 @@
 
 function attrsToObject(attrs) {
   if (!attrs) return {};
+  if (attrs && typeof attrs === 'object' && Array.isArray(attrs._rawAttributes)) {
+    return Object.fromEntries(attrs._rawAttributes);
+  }
   if (!Array.isArray(attrs)) return attrs;
   const out = {};
   for (const attr of attrs) {
+    if (Array.isArray(attr) && attr.length >= 2) {
+      out[attr[0]] = attr[1];
+      continue;
+    }
     const value = attr.value;
     if (value && typeof value === 'object') {
       out[attr.key] = value.stringValue ?? value.intValue ?? value.doubleValue ?? value.boolValue ?? value.arrayValue;
@@ -61,8 +68,28 @@ function flattenSpans(payload) {
   return spans;
 }
 
+function timestampValue(value) {
+  if (!value) return null;
+  if (Array.isArray(value) && value.length >= 2) {
+    const millis = (Number(value[0]) * 1000) + (Number(value[1]) / 1e6);
+    return Number.isFinite(millis) ? new Date(millis).toISOString() : null;
+  }
+  if (typeof value === 'string' && /^\d+$/.test(value)) {
+    const numeric = Number(value);
+    if (!Number.isFinite(numeric)) return null;
+    const millis = numeric > 1e15 ? numeric / 1e6 : numeric;
+    return new Date(millis).toISOString();
+  }
+  if (typeof value === 'number') {
+    const millis = value > 1e15 ? value / 1e6 : value;
+    return new Date(millis).toISOString();
+  }
+  return value;
+}
+
 function classifySpan(span) {
   const attrs = attrsToObject(span.attributes);
+  const eventName = String(pick(attrs, ['event.name']) || '').toLowerCase();
   const operation = String(pick(attrs, ['gen_ai.operation.name', 'llm.operation']) || '').toLowerCase();
   const name = String(span.name || '').toLowerCase();
   const hasTokens = number(attrs, [
@@ -72,7 +99,14 @@ function classifySpan(span) {
     'llm.usage.completion_tokens',
   ]) > 0;
 
-  if (operation.includes('agent') || operation.includes('tool') || name.includes('agent') || name.includes('tool')) {
+  if (
+    eventName.includes('agent')
+    || eventName.includes('tool')
+    || operation.includes('agent')
+    || operation.includes('tool')
+    || name.includes('agent')
+    || name.includes('tool')
+  ) {
     return 'non_billable';
   }
   if (hasTokens || operation.includes('chat') || operation.includes('completion') || operation.includes('generate')) {
@@ -83,19 +117,19 @@ function classifySpan(span) {
 
 function normalizeSpan(span, source, rawLine) {
   const attrs = attrsToObject(span.attributes);
-  const resourceAttrs = attrsToObject(span.resourceAttributes);
+  const resourceAttrs = attrsToObject(span.resourceAttributes || span.resource);
   const type = classifySpan(span);
   if (type !== 'llm') return null;
 
   return {
     raw_line: rawLine,
-    span_id: span.spanId || span.span_id || null,
+    span_id: span.spanId || span.span_id || pick(attrs, ['gen_ai.response.id']) || null,
     trace_id: span.traceId || span.trace_id || null,
     parent_span_id: span.parentSpanId || span.parent_span_id || null,
-    timestamp: span.startTimeUnixNano || span.start_time || attrs['timestamp'] || null,
+    timestamp: timestampValue(span.startTimeUnixNano || span.start_time || span.hrTime || attrs.timestamp),
     surface: source,
     conversation_id: pick(attrs, ['gen_ai.conversation.id', 'conversation.id', 'copilot.conversation.id']),
-    session_id: pick(attrs, ['session.id', 'copilot.session.id']),
+    session_id: pick(attrs, ['session.id', 'copilot.session.id']) || pick(resourceAttrs, ['session.id', 'copilot.session.id']),
     requested_model: pick(attrs, ['gen_ai.request.model', 'llm.request.model', 'llm.model_name']),
     resolved_model: pick(attrs, ['gen_ai.response.model', 'llm.response.model', 'model']),
     repo: pick(attrs, ['vcs.repository.name', 'git.repository', 'repo']) || pick(resourceAttrs, ['vcs.repository.name', 'service.name']),
@@ -194,6 +228,7 @@ function normalizeHookEvent(payload, source, rawLine) {
   return {
     raw_line: rawLine,
     event: payload.event || null,
+    timestamp: payload.captured_at || payload.timestamp || null,
     session_id: payload.session_id || payload.sessionId || null,
     cwd: payload.cwd || null,
     repo: payload.repo || payload.repository || null,

package/src/pricing.js

@@ -2,7 +2,7 @@
 
 const PRICING_VERSION = 'github-copilot-2026-06-01';
 
-// USD per 1M tokens. Source: GitHub Copilot models and pricing docs, checked 2026-05-30.
+// USD per 1M tokens. Source: GitHub Copilot models and pricing docs, checked 2026-05-31.
 const MODEL_PRICES = {
   'gpt-4.1': { input: 2.00, cacheRead: 0.50, cacheWrite: 0, output: 8.00 },
   'gpt-5 mini': { input: 0.25, cacheRead: 0.025, cacheWrite: 0, output: 2.00 },
@@ -32,11 +32,19 @@ const MODEL_PRICES = {
 };
 
 function normalizeModelName(model) {
-  return String(model || '').trim().toLowerCase();
+  return String(model || '').trim().toLowerCase().replace(/^copilot\//, '');
+}
+
+function modelPriceKey(model) {
+  const normalized = normalizeModelName(model);
+  if (MODEL_PRICES[normalized]) return normalized;
+  const withoutDate = normalized.replace(/-\d{4}-\d{2}-\d{2}$/, '');
+  if (MODEL_PRICES[withoutDate]) return withoutDate;
+  return normalized;
 }
 
 function estimateCost(record) {
-  const model = normalizeModelName(record.resolved_model || record.requested_model);
+  const model = modelPriceKey(record.resolved_model || record.requested_model);
   const price = MODEL_PRICES[model];
   if (!model) {
     return { estimated_usd: null, estimated_ai_credits: null, warning: 'missing_model' };
@@ -63,4 +71,5 @@ module.exports = {
   PRICING_VERSION,
   MODEL_PRICES,
   estimateCost,
+  modelPriceKey,
 };