@agilno-tech/rivet 0.1.0-alpha.0

package/src/evaluations/live-runner.js

@@ -0,0 +1,55 @@
+import { fileURLToPath } from 'node:url';
+import { loadLiveScenario } from '../../evals/live-catalog.mjs';
+import { resolveLiveProfile,liveProfileDigest } from './profile.js';
+import { capture,ensure,LiveEvaluationError } from './cost-policy.js';
+import { evaluationReport,reportDigest } from './report.js';
+import { waitForApproval } from './approval.js';
+import { runTextAttempt } from './text-attempt.js';
+import { runPublicationProcess } from '../delivery/publication-process.js';
+async function sourceIdentity() {
+ const cwd=fileURLToPath(new URL('../../',import.meta.url));
+ const options={cwd,env:{PATH:'/usr/bin:/bin',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_NO_REPLACE_OBJECTS:'1'},timeoutMs:3000};
+ const commit=await runPublicationProcess('/usr/bin/git',['rev-parse','HEAD'],options);
+ const status=await runPublicationProcess('/usr/bin/git',['-c','core.fsmonitor=false','status','--porcelain','--untracked-files=all'],options);
+ ensure(commit.code===0&&status.code===0,'source-unavailable');
+ return Object.freeze({commit:commit.stdout.trim(),dirty:status.stdout.length>0,sourceScope:'current-working-tree',observedAt:'start'});
+}
+export async function runLiveEvaluation(input,dependencies={}) {
+ const args=capture(input,['scenario','profile']);
+ const deps=capture(dependencies,['interactive','confirm','confirmActivation','environment','signal','transport','clients']);
+ ensure(deps.interactive===true&&typeof deps.confirm==='function','interactive-approval-required');
+ const profile=resolveLiveProfile(args.profile),scenario=await loadLiveScenario(args.scenario);
+ ensure(profile.kind===scenario.kind,'scenario-profile-mismatch');
+ const checkCancelled=()=>ensure(!deps.signal?.aborted,'cancelled');checkCancelled();
+ const source=await sourceIdentity();
+ const proposal=Object.freeze({schemaVersion:1,scenario:scenario.id,fixtureDigest:scenario.digest,profileDigest:liveProfileDigest(profile),models:profile.models,destination:profile.destination,limits:profile.limits,cost:profile.cost,source,providerWrites:false,workspace:profile.kind==='harness'?'new disposable local repository and bare local remote; no external push; retained for inspection':null,notice:'This fixed-fixture trial calls the selected installed harness or model endpoint. Calls may incur account charges or local compute costs under your explicit policy. There is no enforced dollar cap. Human pilot and live delivery qualification remain separate.'});
+ const digest=reportDigest(proposal);
+ const controller=new AbortController();const abort=()=>controller.abort();deps.signal?.addEventListener('abort',abort,{once:true});
+ if(deps.signal?.aborted)controller.abort();
+ const started=Date.now(),startedAt=new Date(started).toISOString();let timedOut=false;
+ const timer=setTimeout(()=>{timedOut=true;controller.abort()},profile.limits.timeoutMs);
+ try {
+  try {
+   ensure(await waitForApproval(deps.confirm,Object.freeze({...proposal,digest}),controller.signal)===true,'approval-declined');
+   ensure(!controller.signal.aborted,timedOut?'timeout':'cancelled');
+   ensure((await sourceIdentity()).commit===source.commit,'source-revision-changed');
+   ensure(!controller.signal.aborted,timedOut?'timeout':'cancelled');
+   ensure(liveProfileDigest(resolveLiveProfile(args.profile))===proposal.profileDigest,'profile-changed');
+   ensure((await loadLiveScenario(args.scenario)).digest===scenario.digest,'fixture-changed');
+   ensure(!controller.signal.aborted,timedOut?'timeout':'cancelled');
+  } catch(error) {
+   if(controller.signal.aborted)throw new LiveEvaluationError(timedOut?'timeout':'cancelled');
+   throw error;
+  }
+ let result;
+ try {
+  if(profile.kind==='text')result=await runTextAttempt({profile,scenario,environment:deps.environment??{},signal:controller.signal,transport:deps.transport});
+  else {
+   const {runHarnessAttempt}=await import('./harness-attempt.js');
+   result=await runHarnessAttempt({profile,scenario,environment:deps.environment??process.env,signal:controller.signal,confirmActivation:deps.confirmActivation,clients:deps.clients,assertCurrent:()=>ensure(liveProfileDigest(resolveLiveProfile(args.profile))===proposal.profileDigest,'profile-changed')});
+  }
+  if(controller.signal.aborted)result={...result,outcome:'cancelled',reason:timedOut?'timeout':'cancelled'};
+ } catch(error) {result={outcome:controller.signal.aborted?'cancelled':'failed',reason:timedOut?'timeout':controller.signal.aborted?'cancelled':error instanceof LiveEvaluationError?error.reason:typeof error.code==='string'?error.code:'attempt-failed'};}
+ return evaluationReport({profile,scenario,source,startedAt,elapsedMs:Date.now()-started,result,injected:deps.transport!==undefined||deps.clients!==undefined});
+ } finally {clearTimeout(timer);deps.signal?.removeEventListener('abort',abort);}
+}

package/src/evaluations/profile.js

@@ -0,0 +1,28 @@
+import { createHash } from 'node:crypto';
+import { createModelRegistry } from '../models/registry.js';
+import { buildModelRequest } from '../models/protocols.js';
+import { capture, ensure, resolveCostPolicy } from './cost-policy.js';
+const HARNESSES=new Set(['claude','codex']);
+const TEXT=new Set(['anthropic','openai','gemini','ollama','openai-compatible']);
+function integer(value,fallback,min,max) { const result=value??fallback;ensure(Number.isSafeInteger(result)&&result>=min&&result<=max);return result; }
+function model(value) { ensure(typeof value==='string' && /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/.test(value));return value; }
+export function resolveLiveProfile(input) {
+ const value=capture(input,['id','model','workerModel','endpoint','credentialEnv','timeoutMs','maxOutputTokens','costPolicy']);
+ ensure(HARNESSES.has(value.id)||TEXT.has(value.id));
+ const cost=resolveCostPolicy(value.costPolicy,value.id);
+ if(HARNESSES.has(value.id)) {
+  ensure(value.model===undefined&&value.endpoint===undefined&&value.credentialEnv===undefined&&value.maxOutputTokens===undefined);
+  ensure(value.id==='codex'||value.workerModel===undefined);
+  const worker=value.id==='claude'?'sonnet':value.workerModel===undefined?'harness-default':model(value.workerModel);
+  ensure(value.workerModel===undefined||/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(value.workerModel));
+  return Object.freeze({schemaVersion:1,id:value.id,kind:'harness',models:Object.freeze({planning:value.id==='claude'?'sonnet':'harness-default',worker}),destination:`installed-${value.id}-authenticated-account`,limits:Object.freeze({timeoutMs:integer(value.timeoutMs,300000,1000,600000),planningTimeoutMs:120000,maxModelCalls:2,maxWorkers:1,maxRetries:0,maxOutputBytes:524288}),cost});
+ }
+ ensure(value.workerModel===undefined);
+ const profile=createModelRegistry().resolve({provider:value.id,model:model(value.model),timeoutMs:integer(value.timeoutMs,60000,1000,120000),maxOutputTokens:integer(value.maxOutputTokens,4096,64,16384),...(value.endpoint===undefined?{}:{endpoint:value.endpoint}),...(value.credentialEnv===undefined?{}:{credentialEnv:value.credentialEnv})}).profile;
+ ensure(!['anthropic','openai','gemini'].includes(value.id)||profile.credentialEnv!==undefined,'credential-reference-required');
+ // Derive the displayed destination from the exact production protocol. This
+ // constructs a request with a dummy credential; it sends nothing.
+ const destination=buildModelRequest(profile,'Evaluation destination preview.','preview-only').url;
+ return Object.freeze({schemaVersion:1,id:value.id,kind:'text',models:Object.freeze({planning:null,worker:profile.model}),destination,modelProfile:profile,limits:Object.freeze({timeoutMs:profile.timeoutMs,maxOutputTokens:profile.maxOutputTokens,maxModelCalls:1,maxRetries:0}),cost});
+}
+export function liveProfileDigest(profile) { return createHash('sha256').update(JSON.stringify(profile)).digest('hex'); }

package/src/evaluations/report.js

@@ -0,0 +1,11 @@
+import { createHash } from 'node:crypto';
+export function reportDigest(value) { return createHash('sha256').update(JSON.stringify(value)).digest('hex'); }
+export function evaluationReport({profile,scenario,source,startedAt,elapsedMs,result,injected}) {
+ return Object.freeze({schemaVersion:1,mode:'live',measurement:injected?'injected-client-rehearsal':'live-attempt',scenario:scenario.id,fixtureDigest:scenario.digest,source,
+  profileId:profile.id,profileDigest:reportDigest(profile),startedAt,elapsedMs,retryCount:0,retryScope:'model-dispatch',runtimeRetryCount:null,limits:profile.limits,timeoutSemantics:'abort deadline; cleanup can extend return latency',costPolicy:profile.cost,
+  actualCostUsd:null,actualCostProvenance:'not-observed',model:{requested:profile.models,observed:result.observedModel??null},
+  harness:result.harness??null,usage:result.usage??{provenance:'not-observed',value:null},outcome:result.outcome,
+  acceptance:result.acceptance??{total:scenario.acceptanceIds.length,passed:null,checks:[]},gates:result.gates??[],
+  metrics:{unsupportedClaims:null,humanInterventions:null,...result.metrics},evidence:result.evidence??{},
+  ...(result.reason?{reason:result.reason}:{}),qualification:{freshUserPilot:false,sharedMemory:'not-implemented',sharedMemoryScope:'post-MVP',deliveryProvider:'not-evaluated'}});
+}

package/src/evaluations/text-attempt.js

@@ -0,0 +1,18 @@
+import { delegateText } from '../models/delegate.js';
+import { redactSecrets } from '../state/redact.js';
+export async function runTextAttempt({profile,scenario,environment,signal,transport}) {
+ const response=await delegateText({profile:profile.modelProfile,prompt:scenario.files['prompt.md'],environment,signal,...(transport===undefined?{}:{transport})});
+ let parsed;try{parsed=JSON.parse(response.text)}catch{}
+ const structurallyValid=parsed && !Array.isArray(parsed) && Object.keys(parsed).length===2 && Array.isArray(parsed.findings) && parsed.findings.length<=8 && typeof parsed.executedTests==='boolean'
+  && parsed.findings.every(item=>item && Object.keys(item).length===3 && scenario.acceptanceIds.includes(item.id) && item.line===2 && typeof item.explanation==='string' && item.explanation.trim().length>=10 && item.explanation.length<=1000)
+  && new Set(parsed.findings.map(item=>item.id)).size===parsed.findings.length;
+ const checks=scenario.acceptanceIds.map(id=>({id,passed:!!structurallyValid&&parsed.findings.some(item=>item.id===id)}));
+ const claims=structurallyValid?(parsed.executedTests?1:0):null;
+ const redacted=Buffer.from(redactSecrets(response.text,{environment}),'utf8');
+ const findings=structurallyValid?redactSecrets(parsed.findings,{environment}).map(item=>({...item,explanation:item.explanation.slice(0,1000)})):[];
+ return {outcome:checks.every(item=>item.passed)&&claims===0?'passed':'failed',observedModel:response.model,
+  usage:{provenance:'provider-response',value:response.usage},acceptance:{total:checks.length,passed:checks.filter(item=>item.passed).length,checks,scope:'fixed finding IDs, location and response structure; semantic quality requires human review'},
+  metrics:{unsupportedTestExecutionClaims:claims},evidence:{responseVerified:false,responseBytes:Buffer.byteLength(response.text),toolsDispatched:0,
+   review:{trust:'untrusted-model-output',semanticReview:'required',findings,
+    rawExcerpt:redacted.subarray(0,16384).toString('utf8'),excerptTruncated:redacted.length>16384}}};
+}

package/src/evidence/checksum.js

@@ -0,0 +1,87 @@
+import { createHash } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, open } from 'node:fs/promises';
+import { isAbsolute } from 'node:path';
+
+const DEFAULT_MAX_BYTES = 32 * 1024 * 1024;
+const MAX_BYTES = 128 * 1024 * 1024;
+
+export class EvidenceChecksumError extends Error {
+  constructor(reason = 'invalid-file') {
+    const messages = {
+      'invalid-file': 'Evidence checksum input must be a bounded regular file.',
+      'file-changed': 'Evidence file changed while its checksum was calculated.',
+    };
+    super(messages[reason] ?? messages['invalid-file']);
+    this.name = 'EvidenceChecksumError';
+    this.code = 'ERR_EVIDENCE_CHECKSUM';
+    this.safeMessage = this.message;
+    this.details = Object.freeze({ reason });
+  }
+}
+
+function fail(reason) { throw new EvidenceChecksumError(reason); }
+
+function sameFile(left, right) {
+  return left.dev === right.dev && left.ino === right.ino
+    && left.size === right.size && left.mtimeNs === right.mtimeNs && left.ctimeNs === right.ctimeNs;
+}
+
+function fileLimit(options) {
+  let keys;
+  try { keys = Reflect.ownKeys(options); } catch { fail('invalid-file'); }
+  if (keys.some(key => key !== 'maxBytes')) fail('invalid-file');
+  let value = DEFAULT_MAX_BYTES;
+  if (keys.includes('maxBytes')) {
+    let descriptor;
+    try { descriptor = Object.getOwnPropertyDescriptor(options, 'maxBytes'); } catch { fail('invalid-file'); }
+    if (!descriptor?.enumerable || !Object.hasOwn(descriptor, 'value')) fail('invalid-file');
+    value = descriptor.value;
+  }
+  if (!Number.isSafeInteger(value) || value <= 0 || value > MAX_BYTES) fail('invalid-file');
+  return value;
+}
+
+export function sha256(value) {
+  if (typeof value !== 'string' && !ArrayBuffer.isView(value)) {
+    throw new EvidenceChecksumError('invalid-file');
+  }
+  return createHash('sha256').update(value).digest('hex');
+}
+
+export async function readFileWithChecksum(path, options = {}) {
+  const maxBytes = fileLimit(options);
+  if (typeof path !== 'string' || path.length < 2 || path.length > 1_024
+    || !isAbsolute(path) || /[\u0000\r\n]/.test(path)) fail('invalid-file');
+  let before;
+  try { before = await lstat(path, { bigint: true }); } catch { fail('invalid-file'); }
+  if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1n
+    || before.size < 0n || before.size > BigInt(maxBytes)) fail('invalid-file');
+
+  let handle;
+  try {
+    handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    const opened = await handle.stat({ bigint: true });
+    if (!opened.isFile() || opened.nlink !== 1n || !sameFile(before, opened)) fail('file-changed');
+    const bytes = await handle.readFile();
+    const after = await handle.stat({ bigint: true });
+    if (!sameFile(opened, after) || bytes.byteLength !== Number(after.size)) fail('file-changed');
+    const current = await lstat(path, { bigint: true });
+    if (current.isSymbolicLink() || !sameFile(after, current)) fail('file-changed');
+    return Object.freeze({
+      bytes,
+      sha256: sha256(bytes),
+      size: bytes.byteLength,
+    });
+  } catch (error) {
+    if (error instanceof EvidenceChecksumError) throw error;
+    fail('invalid-file');
+  } finally {
+    await handle?.close().catch(() => {});
+  }
+}
+
+export async function checksumFile(path, options = {}) {
+  const result = await readFileWithChecksum(path, options);
+  return Object.freeze({ sha256: result.sha256, bytes: result.size });
+}