incremnt 0.3.0 → 0.5.0

package/src/remote.js

@@ -39,7 +39,9 @@ const remoteCommandHandlers = {
   'health-ai': executeRemoteRead,
   'training-load': executeRemoteRead,
   'ask-history': executeRemoteRead,
-  'ask-show': executeRemoteRead
+  'ask-show': executeRemoteRead,
+  'program-share-fetch': executeRemoteRead,
+  'increment-score-history': executeRemoteRead
 };
 
 async function executeRemoteRead(options, sessionState, normalizedCommand) {
@@ -158,6 +160,15 @@ function endpointForCommand(baseUrl, normalizedCommand, options) {
     }
     case 'ask-show':
       return resolveServiceUrl(baseUrl, `/cli/ask/history/${options.id}`);
+    case 'program-share-fetch':
+      return resolveServiceUrl(baseUrl, `/program-share/${options.token}`);
+    case 'increment-score-history': {
+      const url = resolveServiceUrl(baseUrl, '/mobile/score-snapshots');
+      if (options.from) url.searchParams.set('from', options.from);
+      if (options.to) url.searchParams.set('to', options.to);
+      if (options.limit) url.searchParams.set('limit', options.limit);
+      return url;
+    }
     default:
       return resolveServiceUrl(baseUrl, '/');
   }
@@ -180,6 +191,10 @@ function resourceNotFoundMessage(normalizedCommand, options) {
     return `Conversation not found: ${options.id}`;
   }
 
+  if (normalizedCommand === 'program-share-fetch') {
+    return `Program share not found: ${options.token}`;
+  }
+
   return 'Requested resource was not found.';
 }
 
@@ -278,6 +293,134 @@ const remoteWriteCommandHandlers = {
       throw error;
     }
 
+    return response.json();
+  },
+
+  'program-share-create': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options['program-id']) {
+      const error = new Error('--program-id is required for programs share create.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, `/cli/programs/${options['program-id']}/share`);
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      }
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (response.status === 404) {
+      const error = new Error(`Program not found: ${options['program-id']}`);
+      error.code = 'REMOTE_NOT_FOUND';
+      throw error;
+    }
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
+    return response.json();
+  },
+
+  'program-share-list': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options['program-id']) {
+      const error = new Error('--program-id is required for programs share list.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, `/cli/programs/${options['program-id']}/shares`);
+    const response = await fetch(endpoint, {
+      headers: {
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      }
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
+    return response.json();
+  },
+
+  'increment-score-upload': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options.file) {
+      const error = new Error('--file is required for increment-score upload.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const raw = await fs.readFile(options.file, 'utf8');
+    const body = JSON.parse(raw);
+    if (!body || !Array.isArray(body.snapshots)) {
+      const error = new Error('Invalid file: expected an object with a snapshots array.');
+      error.code = 'INVALID_PAYLOAD';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, '/mobile/score-snapshots');
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      },
+      body: JSON.stringify(body)
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
+
+    return response.json();
+  },
+
+  'program-share-revoke': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options['share-id']) {
+      const error = new Error('--share-id is required for programs share revoke.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, `/cli/program-share/${options['share-id']}/revoke`);
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      }
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (response.status === 404) {
+      const error = new Error(`Program share not found: ${options['share-id']}`);
+      error.code = 'REMOTE_NOT_FOUND';
+      throw error;
+    }
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
     return response.json();
   }
 };

package/src/state.js

@@ -4,12 +4,19 @@ import path from 'node:path';
 
 export const sessionSchemaVersion = 1;
 
+// Prefer HOME env over os.homedir() so test fixtures and explicit overrides
+// work consistently across platforms. On Linux, os.homedir() reads from
+// /etc/passwd via getpwuid and ignores HOME, breaking tests that override HOME.
+export function userHomeDir() {
+  return process.env.HOME || os.homedir();
+}
+
 function fallbackConfigRoot() {
   if (process.platform === 'darwin') {
-    return path.join(os.homedir(), 'Library', 'Application Support');
+    return path.join(userHomeDir(), 'Library', 'Application Support');
   }
 
-  return path.join(os.homedir(), '.config');
+  return path.join(userHomeDir(), '.config');
 }
 
 export function resolveConfigDir() {

package/src/stored-summary-eval-report.js

@@ -0,0 +1,171 @@
+function withPassRate(entry) {
+  return {
+    ...entry,
+    passRate: entry.total > 0 ? entry.passed / entry.total : 0
+  };
+}
+
+export function summarizeResults(results) {
+  const counts = {
+    total: results.length,
+    passed: results.filter((result) => result.passed).length
+  };
+  counts.failed = counts.total - counts.passed;
+  counts.passRate = counts.total > 0 ? counts.passed / counts.total : 0;
+  return counts;
+}
+
+function summarizeByKey(results, keyFn) {
+  const grouped = new Map();
+  for (const result of results) {
+    const key = keyFn(result);
+    const entry = grouped.get(key) ?? { total: 0, passed: 0, failed: 0 };
+    entry.total += 1;
+    if (result.passed) entry.passed += 1;
+    else entry.failed += 1;
+    grouped.set(key, entry);
+  }
+
+  return Object.fromEntries(
+    [...grouped.entries()]
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([key, entry]) => [key, withPassRate(entry)])
+  );
+}
+
+export function summarizeBySurface(results) {
+  return summarizeByKey(results, (result) => result.surface);
+}
+
+function normalizeGeneratedDate(generatedAt) {
+  if (typeof generatedAt !== 'string' || generatedAt.trim().length === 0) {
+    return 'legacy';
+  }
+  return generatedAt.slice(0, 10);
+}
+
+function metadataValue(value) {
+  return typeof value === 'string' && value.trim().length > 0 ? value : 'legacy';
+}
+
+function summarizeMetadata(results) {
+  return {
+    byPromptVersion: summarizeByKey(results, (result) => metadataValue(result.metadata?.promptVersion)),
+    byModel: summarizeByKey(results, (result) => metadataValue(result.metadata?.model)),
+    byGeneratedDate: summarizeByKey(results, (result) => normalizeGeneratedDate(result.metadata?.generatedAt)),
+    byGitSha: summarizeByKey(results, (result) => metadataValue(result.metadata?.gitSha)),
+    byCohort: summarizeByKey(results, (result) => {
+      const promptVersion = metadataValue(result.metadata?.promptVersion);
+      const model = metadataValue(result.metadata?.model);
+      const generatedDate = normalizeGeneratedDate(result.metadata?.generatedAt);
+      return `${result.surface} / ${promptVersion} / ${model} / ${generatedDate}`;
+    })
+  };
+}
+
+function summarizeStoredResults(results) {
+  return {
+    summary: summarizeResults(results),
+    bySurface: summarizeBySurface(results),
+    metadata: summarizeMetadata(results)
+  };
+}
+
+export function buildStoredSummaryReport(snapshotPath, results) {
+  const summary = summarizeStoredResults(results);
+  return {
+    snapshotPath,
+    summary: summary.summary,
+    bySurface: summary.bySurface,
+    metadata: summary.metadata,
+    results: results.map((result) => ({
+      id: result.id,
+      surface: result.surface,
+      passed: result.passed,
+      output: result.output,
+      metadata: result.metadata ?? null,
+      failedChecks: result.checks.filter((check) => !check.passed)
+    }))
+  };
+}
+
+export function summarizeBatchReports(reports) {
+  const results = reports.flatMap((report) => report.results ?? []);
+  return {
+    snapshotCount: reports.length,
+    ...summarizeStoredResults(results)
+  };
+}
+
+function percentage(value) {
+  return `${(value * 100).toFixed(1)}%`;
+}
+
+export function evaluateBatchThresholds(summary, {
+  minPassRate = null,
+  minSurfacePassRates = {}
+} = {}) {
+  const failures = [];
+
+  if (typeof minPassRate === 'number' && summary.summary.passRate < minPassRate) {
+    failures.push(`Overall pass rate ${percentage(summary.summary.passRate)} is below required ${percentage(minPassRate)}.`);
+  }
+
+  for (const [surface, minimum] of Object.entries(minSurfacePassRates)) {
+    const surfaceSummary = summary.bySurface?.[surface];
+    if (!surfaceSummary || surfaceSummary.total === 0) continue;
+    if (surfaceSummary.passRate < minimum) {
+      failures.push(`Surface ${surface} pass rate ${percentage(surfaceSummary.passRate)} is below required ${percentage(minimum)}.`);
+    }
+  }
+
+  return failures;
+}
+
+function formatSummaryLines(entries, { limit = null } = {}) {
+  const lines = Object.entries(entries).map(
+    ([label, entry]) => `- ${label}: ${entry.passed}/${entry.total} passed (${(entry.passRate * 100).toFixed(1)}%)`
+  );
+  return limit == null ? lines : lines.slice(0, limit);
+}
+
+export function formatBatchSummaryMarkdown(summary, reports, failures = []) {
+  const lines = [
+    '# Stored Summary Eval Report',
+    '',
+    `- Snapshots: ${summary.snapshotCount}`,
+    `- Total summaries: ${summary.summary.total}`,
+    `- Passed: ${summary.summary.passed}`,
+    `- Failed: ${summary.summary.failed}`,
+    `- Pass rate: ${(summary.summary.passRate * 100).toFixed(1)}%`,
+    '',
+    '## By Surface',
+    ...formatSummaryLines(summary.bySurface)
+  ];
+
+  lines.push('', '## By Prompt Version');
+  lines.push(...formatSummaryLines(summary.metadata.byPromptVersion));
+
+  lines.push('', '## By Model');
+  lines.push(...formatSummaryLines(summary.metadata.byModel));
+
+  lines.push('', '## By Generated Date');
+  lines.push(...formatSummaryLines(summary.metadata.byGeneratedDate));
+
+  lines.push('', '## Versioned Cohorts');
+  lines.push(...formatSummaryLines(summary.metadata.byCohort, { limit: 20 }));
+
+  lines.push('', '## Snapshots');
+  for (const report of reports) {
+    lines.push(`- ${report.snapshotLabel}: ${report.summary.passed}/${report.summary.total} passed`);
+  }
+
+  if (failures.length > 0) {
+    lines.push('', '## Threshold Failures');
+    for (const failure of failures) {
+      lines.push(`- ${failure}`);
+    }
+  }
+
+  return `${lines.join('\n')}\n`;
+}