incremnt 0.2.0 → 0.4.0

package/src/remote.js

@@ -39,7 +39,8 @@ const remoteCommandHandlers = {
   'health-ai': executeRemoteRead,
   'training-load': executeRemoteRead,
   'ask-history': executeRemoteRead,
-  'ask-show': executeRemoteRead
+  'ask-show': executeRemoteRead,
+  'program-share-fetch': executeRemoteRead
 };
 
 async function executeRemoteRead(options, sessionState, normalizedCommand) {
@@ -158,6 +159,8 @@ function endpointForCommand(baseUrl, normalizedCommand, options) {
     }
     case 'ask-show':
       return resolveServiceUrl(baseUrl, `/cli/ask/history/${options.id}`);
+    case 'program-share-fetch':
+      return resolveServiceUrl(baseUrl, `/program-share/${options.token}`);
     default:
       return resolveServiceUrl(baseUrl, '/');
   }
@@ -180,6 +183,10 @@ function resourceNotFoundMessage(normalizedCommand, options) {
     return `Conversation not found: ${options.id}`;
   }
 
+  if (normalizedCommand === 'program-share-fetch') {
+    return `Program share not found: ${options.token}`;
+  }
+
   return 'Requested resource was not found.';
 }
 
@@ -278,6 +285,96 @@ const remoteWriteCommandHandlers = {
       throw error;
     }
 
+    return response.json();
+  },
+
+  'program-share-create': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options['program-id']) {
+      const error = new Error('--program-id is required for programs share create.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, `/cli/programs/${options['program-id']}/share`);
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      }
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (response.status === 404) {
+      const error = new Error(`Program not found: ${options['program-id']}`);
+      error.code = 'REMOTE_NOT_FOUND';
+      throw error;
+    }
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
+    return response.json();
+  },
+
+  'program-share-list': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options['program-id']) {
+      const error = new Error('--program-id is required for programs share list.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, `/cli/programs/${options['program-id']}/shares`);
+    const response = await fetch(endpoint, {
+      headers: {
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      }
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
+    return response.json();
+  },
+
+  'program-share-revoke': async (options, sessionState) => {
+    const baseUrl = sessionState.session?.transport?.baseUrl;
+    if (!baseUrl) throw notImplementedError();
+    if (!options.token) {
+      const error = new Error('--token is required for programs share revoke.');
+      error.code = 'MISSING_OPTION';
+      throw error;
+    }
+
+    const endpoint = resolveServiceUrl(baseUrl, `/cli/program-share/${options.token}/revoke`);
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${sessionState.session?.auth?.accessToken ?? ''}`
+      }
+    });
+
+    if (response.status === 401 || response.status === 403) throw authenticationFailedError();
+    if (response.status === 404) {
+      const error = new Error(`Program share not found: ${options.token}`);
+      error.code = 'REMOTE_NOT_FOUND';
+      throw error;
+    }
+    if (!response.ok) {
+      const payload = await response.json().catch(() => null);
+      const error = new Error(payload?.error ?? `Unexpected error (HTTP ${response.status}).`);
+      error.code = 'REMOTE_HTTP_ERROR';
+      throw error;
+    }
     return response.json();
   }
 };

package/src/stored-summary-eval-report.js

@@ -0,0 +1,138 @@
+export function summarizeResults(results) {
+  const counts = {
+    total: results.length,
+    passed: results.filter((result) => result.passed).length
+  };
+  counts.failed = counts.total - counts.passed;
+  counts.passRate = counts.total > 0 ? counts.passed / counts.total : 0;
+  return counts;
+}
+
+export function summarizeBySurface(results) {
+  const grouped = new Map();
+  for (const result of results) {
+    const entry = grouped.get(result.surface) ?? { total: 0, passed: 0, failed: 0 };
+    entry.total += 1;
+    if (result.passed) entry.passed += 1;
+    else entry.failed += 1;
+    grouped.set(result.surface, entry);
+  }
+
+  return Object.fromEntries(
+    [...grouped.entries()].map(([surface, entry]) => [
+      surface,
+      {
+        ...entry,
+        passRate: entry.total > 0 ? entry.passed / entry.total : 0
+      }
+    ])
+  );
+}
+
+export function buildStoredSummaryReport(snapshotPath, results) {
+  return {
+    snapshotPath,
+    summary: summarizeResults(results),
+    bySurface: summarizeBySurface(results),
+    results: results.map((result) => ({
+      id: result.id,
+      surface: result.surface,
+      passed: result.passed,
+      output: result.output,
+      failedChecks: result.checks.filter((check) => !check.passed)
+    }))
+  };
+}
+
+export function summarizeBatchReports(reports) {
+  const bySurface = new Map();
+  let total = 0;
+  let passed = 0;
+
+  for (const report of reports) {
+    total += report.summary.total;
+    passed += report.summary.passed;
+
+    for (const [surface, entry] of Object.entries(report.bySurface ?? {})) {
+      const current = bySurface.get(surface) ?? { total: 0, passed: 0, failed: 0 };
+      current.total += entry.total;
+      current.passed += entry.passed;
+      current.failed += entry.failed;
+      bySurface.set(surface, current);
+    }
+  }
+
+  return {
+    snapshotCount: reports.length,
+    total,
+    passed,
+    failed: total - passed,
+    passRate: total > 0 ? passed / total : 0,
+    bySurface: Object.fromEntries(
+      [...bySurface.entries()].map(([surface, entry]) => [
+        surface,
+        {
+          ...entry,
+          passRate: entry.total > 0 ? entry.passed / entry.total : 0
+        }
+      ])
+    )
+  };
+}
+
+function percentage(value) {
+  return `${(value * 100).toFixed(1)}%`;
+}
+
+export function evaluateBatchThresholds(summary, {
+  minPassRate = null,
+  minSurfacePassRates = {}
+} = {}) {
+  const failures = [];
+
+  if (typeof minPassRate === 'number' && summary.passRate < minPassRate) {
+    failures.push(`Overall pass rate ${percentage(summary.passRate)} is below required ${percentage(minPassRate)}.`);
+  }
+
+  for (const [surface, minimum] of Object.entries(minSurfacePassRates)) {
+    const surfaceSummary = summary.bySurface?.[surface];
+    if (!surfaceSummary || surfaceSummary.total === 0) continue;
+    if (surfaceSummary.passRate < minimum) {
+      failures.push(`Surface ${surface} pass rate ${percentage(surfaceSummary.passRate)} is below required ${percentage(minimum)}.`);
+    }
+  }
+
+  return failures;
+}
+
+export function formatBatchSummaryMarkdown(summary, reports, failures = []) {
+  const lines = [
+    '# Stored Summary Eval Report',
+    '',
+    `- Snapshots: ${summary.snapshotCount}`,
+    `- Total summaries: ${summary.total}`,
+    `- Passed: ${summary.passed}`,
+    `- Failed: ${summary.failed}`,
+    `- Pass rate: ${(summary.passRate * 100).toFixed(1)}%`,
+    '',
+    '## By Surface'
+  ];
+
+  for (const [surface, entry] of Object.entries(summary.bySurface)) {
+    lines.push(`- ${surface}: ${entry.passed}/${entry.total} passed (${(entry.passRate * 100).toFixed(1)}%)`);
+  }
+
+  lines.push('', '## Snapshots');
+  for (const report of reports) {
+    lines.push(`- ${report.snapshotLabel}: ${report.summary.passed}/${report.summary.total} passed`);
+  }
+
+  if (failures.length > 0) {
+    lines.push('', '## Threshold Failures');
+    for (const failure of failures) {
+      lines.push(`- ${failure}`);
+    }
+  }
+
+  return `${lines.join('\n')}\n`;
+}