incremnt 0.4.0 → 0.5.0

package/src/remote.js

@@ -40,7 +40,8 @@ const remoteCommandHandlers = {
   'training-load': executeRemoteRead,
   'ask-history': executeRemoteRead,
   'ask-show': executeRemoteRead,
-  'program-share-fetch': executeRemoteRead
+  'program-share-fetch': executeRemoteRead,
+  'increment-score-history': executeRemoteRead
 };
 
 async function executeRemoteRead(options, sessionState, normalizedCommand) {
@@ -161,6 +162,13 @@ function endpointForCommand(baseUrl, normalizedCommand, options) {
       return resolveServiceUrl(baseUrl, `/cli/ask/history/${options.id}`);
     case 'program-share-fetch':
       return resolveServiceUrl(baseUrl, `/program-share/${options.token}`);
+    case 'increment-score-history': {
+      const url = resolveServiceUrl(baseUrl, '/mobile/score-snapshots');
+      if (options.from) url.searchParams.set('from', options.from);
+      if (options.to) url.searchParams.set('to', options.to);
+      if (options.limit) url.searchParams.set('limit', options.limit);
+      return url;
+    }
     default:
       return resolveServiceUrl(baseUrl, '/');
   }
@@ -346,16 +354,54 @@ const remoteWriteCommandHandlers = {
     return response.json();
   },
 
+  'increment-score-upload': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options.file) {
+      const error = new Error('--file is required for increment-score upload.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const raw = await fs.readFile(options.file, 'utf8');
+    const body = JSON.parse(raw);
+    if (!body || !Array.isArray(body.snapshots)) {
+      const error = new Error('Invalid file: expected an object with a snapshots array.');
+      error.code = 'INVALID_PAYLOAD';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, '/mobile/score-snapshots');
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      },
+      body: JSON.stringify(body)
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
+
+    return response.json();
+  },
+
   'program-share-revoke': async (options, sessionState) => {
     const baseUrl = sessionState.session?.transport?.baseUrl;
     if (!baseUrl) throw notImplementedError();
-    if (!options.token) {
-      const error = new Error('--token is required for programs share revoke.');
+    if (!options['share-id']) {
+      const error = new Error('--share-id is required for programs share revoke.');
       error.code = 'MISSING_OPTION';
       throw error;
     }
 
-    const endpoint = resolveServiceUrl(baseUrl, `/cli/program-share/${options.token}/revoke`);
+    const endpoint = resolveServiceUrl(baseUrl, `/cli/program-share/${options['share-id']}/revoke`);
     const response = await fetch(endpoint, {
       method: 'POST',
       headers: {
@@ -365,7 +411,7 @@ const remoteWriteCommandHandlers = {
 
     if (response.status === 401 || response.status === 403) throw authenticationFailedError();
     if (response.status === 404) {
-      const error = new Error(`Program share not found: ${options.token}`);
+      const error = new Error(`Program share not found: ${options['share-id']}`);
       error.code = 'REMOTE_NOT_FOUND';
       throw error;
     }

package/src/state.js

@@ -4,12 +4,19 @@ import path from 'node:path';
 
 export const sessionSchemaVersion = 1;
 
+// Prefer HOME env over os.homedir() so test fixtures and explicit overrides
+// work consistently across platforms. On Linux, os.homedir() reads from
+// /etc/passwd via getpwuid and ignores HOME, breaking tests that override HOME.
+export function userHomeDir() {
+  return process.env.HOME || os.homedir();
+}
+
 function fallbackConfigRoot() {
   if (process.platform === 'darwin') {
-    return path.join(os.homedir(), 'Library', 'Application Support');
+    return path.join(userHomeDir(), 'Library', 'Application Support');
   }
 
-  return path.join(os.homedir(), '.config');
+  return path.join(userHomeDir(), '.config');
 }
 
 export function resolveConfigDir() {

package/src/stored-summary-eval-report.js

@@ -1,3 +1,10 @@
+function withPassRate(entry) {
+  return {
+    ...entry,
+    passRate: entry.total > 0 ? entry.passed / entry.total : 0
+  };
+}
+
 export function summarizeResults(results) {
   const counts = {
     total: results.length,
@@ -8,75 +15,85 @@ export function summarizeResults(results) {
   return counts;
 }
 
-export function summarizeBySurface(results) {
+function summarizeByKey(results, keyFn) {
   const grouped = new Map();
   for (const result of results) {
-    const entry = grouped.get(result.surface) ?? { total: 0, passed: 0, failed: 0 };
+    const key = keyFn(result);
+    const entry = grouped.get(key) ?? { total: 0, passed: 0, failed: 0 };
     entry.total += 1;
     if (result.passed) entry.passed += 1;
     else entry.failed += 1;
-    grouped.set(result.surface, entry);
+    grouped.set(key, entry);
   }
 
   return Object.fromEntries(
-    [...grouped.entries()].map(([surface, entry]) => [
-      surface,
-      {
-        ...entry,
-        passRate: entry.total > 0 ? entry.passed / entry.total : 0
-      }
-    ])
+    [...grouped.entries()]
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([key, entry]) => [key, withPassRate(entry)])
   );
 }
 
-export function buildStoredSummaryReport(snapshotPath, results) {
+export function summarizeBySurface(results) {
+  return summarizeByKey(results, (result) => result.surface);
+}
+
+function normalizeGeneratedDate(generatedAt) {
+  if (typeof generatedAt !== 'string' || generatedAt.trim().length === 0) {
+    return 'legacy';
+  }
+  return generatedAt.slice(0, 10);
+}
+
+function metadataValue(value) {
+  return typeof value === 'string' && value.trim().length > 0 ? value : 'legacy';
+}
+
+function summarizeMetadata(results) {
+  return {
+    byPromptVersion: summarizeByKey(results, (result) => metadataValue(result.metadata?.promptVersion)),
+    byModel: summarizeByKey(results, (result) => metadataValue(result.metadata?.model)),
+    byGeneratedDate: summarizeByKey(results, (result) => normalizeGeneratedDate(result.metadata?.generatedAt)),
+    byGitSha: summarizeByKey(results, (result) => metadataValue(result.metadata?.gitSha)),
+    byCohort: summarizeByKey(results, (result) => {
+      const promptVersion = metadataValue(result.metadata?.promptVersion);
+      const model = metadataValue(result.metadata?.model);
+      const generatedDate = normalizeGeneratedDate(result.metadata?.generatedAt);
+      return `${result.surface} / ${promptVersion} / ${model} / ${generatedDate}`;
+    })
+  };
+}
+
+function summarizeStoredResults(results) {
   return {
-    snapshotPath,
     summary: summarizeResults(results),
     bySurface: summarizeBySurface(results),
+    metadata: summarizeMetadata(results)
+  };
+}
+
+export function buildStoredSummaryReport(snapshotPath, results) {
+  const summary = summarizeStoredResults(results);
+  return {
+    snapshotPath,
+    summary: summary.summary,
+    bySurface: summary.bySurface,
+    metadata: summary.metadata,
     results: results.map((result) => ({
       id: result.id,
       surface: result.surface,
       passed: result.passed,
       output: result.output,
+      metadata: result.metadata ?? null,
       failedChecks: result.checks.filter((check) => !check.passed)
     }))
   };
 }
 
 export function summarizeBatchReports(reports) {
-  const bySurface = new Map();
-  let total = 0;
-  let passed = 0;
-
-  for (const report of reports) {
-    total += report.summary.total;
-    passed += report.summary.passed;
-
-    for (const [surface, entry] of Object.entries(report.bySurface ?? {})) {
-      const current = bySurface.get(surface) ?? { total: 0, passed: 0, failed: 0 };
-      current.total += entry.total;
-      current.passed += entry.passed;
-      current.failed += entry.failed;
-      bySurface.set(surface, current);
-    }
-  }
-
+  const results = reports.flatMap((report) => report.results ?? []);
   return {
     snapshotCount: reports.length,
-    total,
-    passed,
-    failed: total - passed,
-    passRate: total > 0 ? passed / total : 0,
-    bySurface: Object.fromEntries(
-      [...bySurface.entries()].map(([surface, entry]) => [
-        surface,
-        {
-          ...entry,
-          passRate: entry.total > 0 ? entry.passed / entry.total : 0
-        }
-      ])
-    )
+    ...summarizeStoredResults(results)
   };
 }
 
@@ -90,8 +107,8 @@ export function evaluateBatchThresholds(summary, {
 } = {}) {
   const failures = [];
 
-  if (typeof minPassRate === 'number' && summary.passRate < minPassRate) {
-    failures.push(`Overall pass rate ${percentage(summary.passRate)} is below required ${percentage(minPassRate)}.`);
+  if (typeof minPassRate === 'number' && summary.summary.passRate < minPassRate) {
+    failures.push(`Overall pass rate ${percentage(summary.summary.passRate)} is below required ${percentage(minPassRate)}.`);
   }
 
   for (const [surface, minimum] of Object.entries(minSurfacePassRates)) {
@@ -105,22 +122,38 @@ export function evaluateBatchThresholds(summary, {
   return failures;
 }
 
+function formatSummaryLines(entries, { limit = null } = {}) {
+  const lines = Object.entries(entries).map(
+    ([label, entry]) => `- ${label}: ${entry.passed}/${entry.total} passed (${(entry.passRate * 100).toFixed(1)}%)`
+  );
+  return limit == null ? lines : lines.slice(0, limit);
+}
+
 export function formatBatchSummaryMarkdown(summary, reports, failures = []) {
   const lines = [
     '# Stored Summary Eval Report',
     '',
     `- Snapshots: ${summary.snapshotCount}`,
-    `- Total summaries: ${summary.total}`,
-    `- Passed: ${summary.passed}`,
-    `- Failed: ${summary.failed}`,
-    `- Pass rate: ${(summary.passRate * 100).toFixed(1)}%`,
+    `- Total summaries: ${summary.summary.total}`,
+    `- Passed: ${summary.summary.passed}`,
+    `- Failed: ${summary.summary.failed}`,
+    `- Pass rate: ${(summary.summary.passRate * 100).toFixed(1)}%`,
     '',
-    '## By Surface'
+    '## By Surface',
+    ...formatSummaryLines(summary.bySurface)
   ];
 
-  for (const [surface, entry] of Object.entries(summary.bySurface)) {
-    lines.push(`- ${surface}: ${entry.passed}/${entry.total} passed (${(entry.passRate * 100).toFixed(1)}%)`);
-  }
+  lines.push('', '## By Prompt Version');
+  lines.push(...formatSummaryLines(summary.metadata.byPromptVersion));
+
+  lines.push('', '## By Model');
+  lines.push(...formatSummaryLines(summary.metadata.byModel));
+
+  lines.push('', '## By Generated Date');
+  lines.push(...formatSummaryLines(summary.metadata.byGeneratedDate));
+
+  lines.push('', '## Versioned Cohorts');
+  lines.push(...formatSummaryLines(summary.metadata.byCohort, { limit: 20 }));
 
   lines.push('', '## Snapshots');
   for (const report of reports) {