copilot-metrics 0.1.0

package/src/cli.js

@@ -0,0 +1,288 @@
+'use strict';
+
+const { resolvePaths } = require('./paths');
+const {
+  ensureDataDirs,
+  vscodeSettings,
+  copilotCliEnvironment,
+  shellExports,
+  hookConfig,
+  installHook,
+  setupSnapshot,
+} = require('./setup');
+const { appendHookEvent, readJsonFromStream } = require('./hook-logger');
+const { initStore } = require('./sqlite-store');
+const { ingestFile } = require('./ingest');
+const { MODEL_PRICES, PRICING_VERSION } = require('./pricing');
+const {
+  labelOverview,
+  labelSummary,
+  labelDetails,
+  modelReport,
+  repoReport,
+  unattributedReport,
+  formatLabels,
+  formatLabelSummary,
+  formatModels,
+  formatRepos,
+  formatUnattributed,
+} = require('./reports');
+const { loadConfiguredExtractors } = require('./label-extractors');
+
+function parseFlags(args) {
+  const flags = {};
+  const rest = [];
+  for (let i = 0; i < args.length; i += 1) {
+    const arg = args[i];
+    if (!arg.startsWith('--')) {
+      rest.push(arg);
+      continue;
+    }
+    const [rawKey, inlineValue] = arg.slice(2).split('=', 2);
+    const key = rawKey.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+    if (inlineValue !== undefined) {
+      flags[key] = inlineValue;
+    } else if (args[i + 1] && !args[i + 1].startsWith('--')) {
+      flags[key] = args[i + 1];
+      i += 1;
+    } else {
+      flags[key] = true;
+    }
+  }
+  return { flags, rest };
+}
+
+function writeOutput(stdout, value, asJson = false) {
+  if (asJson) {
+    stdout.write(`${JSON.stringify(value, null, 2)}\n`);
+    return;
+  }
+  stdout.write(`${value}\n`);
+}
+
+function helpText() {
+  return `copilot-metrics
+
+Usage:
+  copilot-metrics init [--json]
+  copilot-metrics paths [--json]
+  copilot-metrics setup vscode [--json]
+  copilot-metrics setup copilot-cli [--json]
+  copilot-metrics hooks preview [--scope local|global] [--surface both|vscode|copilot-cli] [--json]
+  copilot-metrics hooks install [--scope local|global] [--surface both|vscode|copilot-cli] [--json]
+  copilot-metrics hook-log --event <name>
+  copilot-metrics store init [--json]
+  copilot-metrics import --source vscode|copilot-cli|hooks --file <path> [--json]
+  copilot-metrics report labels [--json]
+  copilot-metrics report label <id> [--detail] [--json]
+  copilot-metrics report models [--json]
+  copilot-metrics report repos [--json]
+  copilot-metrics report unattributed [--json]
+  copilot-metrics pricing list [--json]
+
+Environment:
+  COPILOT_METRICS_HOME  Override the central data directory.
+`;
+}
+
+function formatPaths(paths) {
+  return [
+    `home: ${paths.home}`,
+    `telemetry: ${paths.telemetryDir}`,
+    `hooks: ${paths.hooksDir}`,
+    `store: ${paths.storeDir}`,
+    `VS Code OTel JSONL: ${paths.vscodeOtelJsonl}`,
+    `Copilot CLI OTel JSONL: ${paths.copilotCliOtelJsonl}`,
+    `Hook events JSONL: ${paths.hookEventsJsonl}`,
+    `SQLite store: ${paths.usageDb}`,
+  ].join('\n');
+}
+
+function formatVscode(settings) {
+  return [
+    'Add these settings to VS Code Insiders settings.json:',
+    JSON.stringify(settings, null, 2),
+    '',
+    'Content capture is disabled by default.',
+  ].join('\n');
+}
+
+function formatCopilotCli(env) {
+  return [
+    'Export these variables before running Copilot CLI:',
+    shellExports(env),
+    '',
+    'Content capture is disabled by default.',
+  ].join('\n');
+}
+
+async function main(args, io) {
+  const { flags, rest } = parseFlags(args);
+  const json = flags.json === true;
+  const paths = resolvePaths({ env: io.env, cwd: io.cwd, home: flags.home });
+  const [command, subcommand] = rest;
+
+  if (!command || command === 'help' || command === '--help' || command === '-h') {
+    writeOutput(io.stdout, helpText(), false);
+    return;
+  }
+
+  if (command === 'init') {
+    ensureDataDirs(paths);
+    writeOutput(io.stdout, json ? paths : `Initialized Copilot Metrics data directory:\n${formatPaths(paths)}`, json);
+    return;
+  }
+
+  if (command === 'paths') {
+    writeOutput(io.stdout, json ? paths : formatPaths(paths), json);
+    return;
+  }
+
+  if (command === 'setup') {
+    if (subcommand === 'vscode') {
+      const settings = vscodeSettings(paths);
+      writeOutput(io.stdout, json ? settings : formatVscode(settings), json);
+      return;
+    }
+    if (subcommand === 'copilot-cli') {
+      const env = copilotCliEnvironment(paths);
+      writeOutput(io.stdout, json ? env : formatCopilotCli(env), json);
+      return;
+    }
+    if (!subcommand || subcommand === 'all') {
+      const snapshot = setupSnapshot({ env: io.env, cwd: io.cwd, home: flags.home, command: io.commandPath });
+      writeOutput(io.stdout, json ? snapshot : [
+        formatPaths(snapshot.paths),
+        '',
+        formatVscode(snapshot.vscode),
+        '',
+        formatCopilotCli(snapshot.copilotCli),
+      ].join('\n'), json);
+      return;
+    }
+    throw new Error(`Unknown setup target "${subcommand}". Use vscode or copilot-cli.`);
+  }
+
+  if (command === 'hooks') {
+    const scope = flags.scope || 'local';
+    const surface = flags.surface || 'both';
+    if (subcommand === 'preview') {
+      const config = hookConfig(paths, { cwd: io.cwd, scope, surface, command: io.commandPath });
+      writeOutput(io.stdout, json ? config : JSON.stringify(config, null, 2), json);
+      return;
+    }
+    if (subcommand === 'install') {
+      ensureDataDirs(paths);
+      const result = installHook(paths, { cwd: io.cwd, scope, surface, command: io.commandPath });
+      writeOutput(io.stdout, json ? result : `Installed ${scope} ${surface} hook config: ${result.target}`, json);
+      return;
+    }
+    throw new Error(`Unknown hooks action "${subcommand}". Use preview or install.`);
+  }
+
+  if (command === 'hook-log') {
+    const payload = await readJsonFromStream(io.stdin);
+    const result = appendHookEvent(payload, {
+      env: io.env,
+      cwd: io.cwd,
+      home: flags.home,
+      event: flags.event,
+      includePromptPreview: flags.includePromptPreview === true,
+    });
+    if (flags.quiet === true) return;
+    writeOutput(io.stdout, json ? result : `Logged hook event: ${result.path}`, json);
+    return;
+  }
+
+  if (command === 'store') {
+    if (subcommand !== 'init') {
+      throw new Error(`Unknown store action "${subcommand}". Use init.`);
+    }
+    ensureDataDirs(paths);
+    await initStore(paths.usageDb);
+    writeOutput(io.stdout, json ? { dbPath: paths.usageDb } : `Initialized SQLite store: ${paths.usageDb}`, json);
+    return;
+  }
+
+  if (command === 'import') {
+    const source = flags.source;
+    const file = flags.file || (source === 'vscode'
+      ? paths.vscodeOtelJsonl
+      : source === 'copilot-cli'
+        ? paths.copilotCliOtelJsonl
+        : source === 'hooks'
+          ? paths.hookEventsJsonl
+          : null);
+    if (!['vscode', 'copilot-cli', 'hooks'].includes(source)) {
+      throw new Error('import requires --source vscode|copilot-cli|hooks');
+    }
+    if (!file) throw new Error('import requires --file <path>');
+    ensureDataDirs(paths);
+    const result = await ingestFile({
+      dbPath: paths.usageDb,
+      file,
+      source,
+      extractors: loadConfiguredExtractors(paths.configJson, io.cwd),
+    });
+    writeOutput(io.stdout, json ? result : [
+      `Imported ${result.raw_records} raw ${source} records into ${result.dbPath}`,
+      `Normalized usage records: ${result.usage_records}`,
+      `Hook events: ${result.hook_events}`,
+      `Label evidence: ${result.label_evidence}`,
+      `Warnings: ${result.warnings.length}`,
+      `Costs are ${result.estimate_label}`,
+    ].join('\n'), json);
+    return;
+  }
+
+  if (command === 'report') {
+    if (subcommand === 'labels') {
+      const rows = await labelOverview(paths.usageDb);
+      writeOutput(io.stdout, json ? { labels: rows } : formatLabels(rows), json);
+      return;
+    }
+    if (subcommand === 'label') {
+      const label = rest[2];
+      if (!label) throw new Error('report label requires <id>');
+      const summary = await labelSummary(paths.usageDb, label);
+      if (flags.detail === true) {
+        const details = await labelDetails(paths.usageDb, label);
+        writeOutput(io.stdout, json ? { label: summary, details } : formatLabelSummary(summary, details), json);
+        return;
+      }
+      writeOutput(io.stdout, json ? { label: summary } : formatLabelSummary(summary), json);
+      return;
+    }
+    if (subcommand === 'models') {
+      const rows = await modelReport(paths.usageDb);
+      writeOutput(io.stdout, json ? { models: rows } : formatModels(rows), json);
+      return;
+    }
+    if (subcommand === 'repos') {
+      const rows = await repoReport(paths.usageDb);
+      writeOutput(io.stdout, json ? { repos: rows } : formatRepos(rows), json);
+      return;
+    }
+    if (subcommand === 'unattributed') {
+      const rows = await unattributedReport(paths.usageDb);
+      writeOutput(io.stdout, json ? { unattributed: rows } : formatUnattributed(rows), json);
+      return;
+    }
+    throw new Error(`Unknown report "${subcommand}". Use labels, label, models, repos, or unattributed.`);
+  }
+
+  if (command === 'pricing') {
+    if (subcommand !== 'list') throw new Error(`Unknown pricing action "${subcommand}". Use list.`);
+    const payload = { version: PRICING_VERSION, unit: 'USD per 1M tokens', models: MODEL_PRICES };
+    writeOutput(io.stdout, json ? payload : JSON.stringify(payload, null, 2), json);
+    return;
+  }
+
+  throw new Error(`Unknown command "${command}". Run copilot-metrics --help.`);
+}
+
+module.exports = {
+  main,
+  parseFlags,
+  helpText,
+};

package/src/hook-logger.js

@@ -0,0 +1,79 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { resolvePaths } = require('./paths');
+
+const LABEL_RE = /\b[A-Z][A-Z0-9]+-\d+\b/g;
+
+function extractLabelsFromValue(value, labels = new Set()) {
+  if (typeof value === 'string') {
+    for (const match of value.matchAll(LABEL_RE)) labels.add(match[0]);
+    return labels;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) extractLabelsFromValue(item, labels);
+    return labels;
+  }
+  if (value && typeof value === 'object') {
+    for (const item of Object.values(value)) extractLabelsFromValue(item, labels);
+  }
+  return labels;
+}
+
+function firstString(payload, keys) {
+  for (const key of keys) {
+    const value = payload && payload[key];
+    if (typeof value === 'string' && value.trim()) return value;
+  }
+  return null;
+}
+
+function redactHookPayload(payload, options = {}) {
+  if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
+    payload = {};
+  }
+  const includePromptPreview = options.includePromptPreview === true;
+  const labels = Array.from(extractLabelsFromValue(payload)).sort();
+  const prompt = firstString(payload, ['prompt', 'userPrompt', 'message', 'input']);
+
+  return {
+    captured_at: new Date().toISOString(),
+    event: options.event || firstString(payload, ['event', 'hookEventName', 'hook_event_name']) || null,
+    session_id: firstString(payload, ['session_id', 'sessionId', 'conversationId']),
+    cwd: firstString(payload, ['cwd', 'workingDirectory']),
+    repo: firstString(payload, ['repo', 'repository']),
+    branch: firstString(payload, ['branch', 'gitBranch']),
+    transcript_path: firstString(payload, ['transcript_path', 'transcriptPath']),
+    task_hint: firstString(payload, ['task_hint', 'taskHint', 'title']),
+    tool_name: firstString(payload, ['tool_name', 'toolName', 'name']),
+    surface: firstString(payload, ['surface', 'agentSurface']),
+    labels,
+    prompt_preview: includePromptPreview && prompt ? prompt.slice(0, 160) : undefined,
+    raw_prompt_stored: false,
+  };
+}
+
+function appendHookEvent(payload, options = {}) {
+  const paths = resolvePaths(options);
+  const record = redactHookPayload(payload, options);
+  fs.mkdirSync(path.dirname(paths.hookEventsJsonl), { recursive: true, mode: 0o700 });
+  fs.appendFileSync(paths.hookEventsJsonl, `${JSON.stringify(record)}\n`, { mode: 0o600 });
+  return { path: paths.hookEventsJsonl, record };
+}
+
+async function readJsonFromStream(stream) {
+  const chunks = [];
+  for await (const chunk of stream) chunks.push(Buffer.from(chunk));
+  const text = Buffer.concat(chunks).toString('utf8').trim();
+  if (!text) return {};
+  return JSON.parse(text);
+}
+
+module.exports = {
+  LABEL_RE,
+  extractLabelsFromValue,
+  redactHookPayload,
+  appendHookEvent,
+  readJsonFromStream,
+};

package/src/ingest.js

@@ -0,0 +1,71 @@
+'use strict';
+
+const { readJsonl } = require('./jsonl');
+const { normalizePayload, normalizeHookEvent } = require('./otel');
+const { estimateCost, PRICING_VERSION } = require('./pricing');
+const { insertImport } = require('./sqlite-store');
+const { attachUsageLabelEvidence, attachHookLabelEvidence } = require('./labels');
+
+function enrichCosts(records) {
+  return records.map((record) => {
+    const estimate = estimateCost(record);
+    const warnings = [...record.warnings];
+    if (estimate.warning) warnings.push(estimate.warning);
+    return {
+      ...record,
+      estimated_usd: estimate.estimated_usd,
+      estimated_ai_credits: estimate.estimated_ai_credits,
+      estimate_label: `estimate:${PRICING_VERSION}`,
+      warnings,
+    };
+  });
+}
+
+async function ingestFile(options) {
+  const { dbPath, file, source } = options;
+  const parsed = readJsonl(file);
+  const warnings = [...parsed.warnings];
+  const usageRecords = [];
+  const hookEvents = [];
+
+  for (const record of parsed.records) {
+    if (source === 'hooks') {
+      const event = normalizeHookEvent(record.value, source, record.line);
+      if (event) hookEvents.push(event);
+      continue;
+    }
+    usageRecords.push(...normalizePayload(record.value, source, record.line));
+  }
+
+  const extractorOptions = { extractors: options.extractors || [] };
+  const enrichedUsage = attachUsageLabelEvidence(enrichCosts(usageRecords), extractorOptions);
+  const enrichedHooks = attachHookLabelEvidence(hookEvents, extractorOptions);
+  for (const usage of enrichedUsage) {
+    for (const warning of usage.warnings) {
+      warnings.push({
+        code: warning.split(':')[0],
+        line: usage.raw_line,
+        message: warning,
+      });
+    }
+  }
+
+  await insertImport(dbPath, source, parsed.records, enrichedUsage, enrichedHooks, warnings);
+
+  return {
+    source,
+    file,
+    dbPath,
+    raw_records: parsed.records.length,
+    usage_records: enrichedUsage.length,
+    hook_events: enrichedHooks.length,
+    label_evidence: enrichedUsage.reduce((sum, usage) => sum + (usage.label_evidence || []).length, 0)
+      + enrichedHooks.reduce((sum, event) => sum + (event.label_evidence || []).length, 0),
+    warnings,
+    estimate_label: `estimate:${PRICING_VERSION}`,
+  };
+}
+
+module.exports = {
+  ingestFile,
+};

package/src/jsonl.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const fs = require('node:fs');
+
+function readJsonl(file) {
+  const text = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '';
+  const records = [];
+  const warnings = [];
+
+  text.split(/\r?\n/).forEach((line, index) => {
+    if (!line.trim()) return;
+    try {
+      records.push({ line: index + 1, value: JSON.parse(line) });
+    } catch (error) {
+      warnings.push({
+        code: 'malformed_jsonl',
+        line: index + 1,
+        message: error.message,
+      });
+    }
+  });
+
+  return { records, warnings };
+}
+
+module.exports = {
+  readJsonl,
+};

package/src/label-extractors.js

@@ -0,0 +1,120 @@
+'use strict';
+
+const path = require('node:path');
+
+const JIRA_LABEL_RE = /\b([A-Z][A-Z0-9]+-\d+)\b/gi;
+
+function canonicalLabel(label) {
+  return String(label || '').trim().toUpperCase();
+}
+
+function fieldConfidence(field) {
+  if (field === 'labels') return 1;
+  if (field === 'branch') return 0.85;
+  if (field === 'cwd') return 0.75;
+  if (field === 'task_hint') return 0.7;
+  if (field === 'repo') return 0.35;
+  return 0.5;
+}
+
+function sourceValue(field, value, label) {
+  if (['prompt', 'message', 'input', 'prompt_preview', 'task_hint'].includes(field)) return label;
+  if (typeof value === 'string') return value.slice(0, 240);
+  if (Array.isArray(value)) return value.map((item) => String(item).slice(0, 120)).join(',');
+  return label;
+}
+
+function extractJiraLabels(sourceType, sourceData = {}) {
+  const evidence = [];
+  const fields = {
+    labels: sourceData.labels,
+    branch: sourceData.branch,
+    cwd: sourceData.cwd,
+    repo: sourceData.repo,
+    task_hint: sourceData.task_hint,
+    prompt: sourceData.prompt,
+    prompt_preview: sourceData.prompt_preview,
+    message: sourceData.message,
+    input: sourceData.input,
+  };
+
+  for (const [field, value] of Object.entries(fields)) {
+    const values = Array.isArray(value) ? value : [value];
+    for (const item of values) {
+      if (item === undefined || item === null) continue;
+      const text = String(item);
+      for (const match of text.matchAll(JIRA_LABEL_RE)) {
+        const label = canonicalLabel(match[1]);
+        evidence.push({
+          label,
+          source_type: sourceType,
+          source_field: field,
+          source_value: sourceValue(field, item, label),
+          confidence: fieldConfidence(field),
+        });
+      }
+    }
+  }
+
+  return evidence;
+}
+
+function normalizeExtractorResult(result, sourceType) {
+  const items = Array.isArray(result) ? result : [];
+  return items
+    .map((item) => (typeof item === 'string' ? { label: item } : item))
+    .filter((item) => item && item.label)
+    .map((item) => ({
+      label: canonicalLabel(item.label),
+      source_type: item.source_type || sourceType,
+      source_field: item.source_field || item.field || 'custom',
+      source_value: item.source_value || item.value || canonicalLabel(item.label),
+      confidence: Number.isFinite(Number(item.confidence)) ? Number(item.confidence) : 0.5,
+    }));
+}
+
+function runLabelExtractors(sourceType, sourceData, customExtractors = []) {
+  const extractors = [extractJiraLabels, ...customExtractors];
+  const seen = new Set();
+  const evidence = [];
+
+  for (const extractor of extractors) {
+    const results = normalizeExtractorResult(extractor(sourceType, sourceData), sourceType);
+    for (const item of results) {
+      const key = `${item.label}\0${item.source_type}\0${item.source_field}\0${item.source_value}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      evidence.push(item);
+    }
+  }
+
+  return evidence;
+}
+
+function loadConfiguredExtractors(configPath, cwd = process.cwd()) {
+  let config;
+  try {
+    config = require(configPath);
+  } catch {
+    return [];
+  }
+
+  const configured = Array.isArray(config.labelExtractors) ? config.labelExtractors : [];
+  return configured.map((entry) => {
+    const modulePath = typeof entry === 'string' ? entry : entry && entry.path;
+    if (!modulePath) return null;
+    const resolved = path.isAbsolute(modulePath) ? modulePath : path.resolve(cwd, modulePath);
+    const mod = require(resolved);
+    if (typeof mod === 'function') return mod;
+    if (typeof mod.extractLabels === 'function') return mod.extractLabels;
+    throw new Error(`Configured label extractor does not export a function: ${modulePath}`);
+  }).filter(Boolean);
+}
+
+module.exports = {
+  JIRA_LABEL_RE,
+  canonicalLabel,
+  extractJiraLabels,
+  loadConfiguredExtractors,
+  runLabelExtractors,
+};

package/src/labels.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const { runLabelExtractors } = require('./label-extractors');
+
+function usageSourceData(usage) {
+  return {
+    labels: usage.labels || [],
+    repo: usage.repo,
+    branch: usage.branch,
+    cwd: usage.cwd,
+    session_id: usage.session_id,
+    conversation_id: usage.conversation_id,
+    task_hint: usage.task_hint,
+    prompt: usage.prompt,
+    prompt_preview: usage.prompt_preview,
+    message: usage.message,
+    input: usage.input,
+  };
+}
+
+function hookSourceData(event) {
+  const payload = event.payload || {};
+  return {
+    labels: event.labels || payload.labels || [],
+    repo: event.repo || payload.repo,
+    branch: event.branch || payload.branch,
+    cwd: event.cwd || payload.cwd,
+    session_id: event.session_id || payload.session_id,
+    task_hint: event.task_hint || payload.task_hint || payload.taskHint,
+    prompt: payload.prompt,
+    prompt_preview: payload.prompt_preview,
+    message: payload.message,
+    input: payload.input,
+  };
+}
+
+function attachUsageLabelEvidence(usageRecords, options = {}) {
+  return usageRecords.map((usage) => ({
+    ...usage,
+    label_evidence: runLabelExtractors('usage', usageSourceData(usage), options.extractors),
+  }));
+}
+
+function attachHookLabelEvidence(hookEvents, options = {}) {
+  return hookEvents.map((event) => ({
+    ...event,
+    label_evidence: runLabelExtractors('hook', hookSourceData(event), options.extractors),
+  }));
+}
+
+module.exports = {
+  usageSourceData,
+  hookSourceData,
+  attachUsageLabelEvidence,
+  attachHookLabelEvidence,
+};