@gestaltrun/dsh-agent-eval 0.0.0-stage → 0.1.1

package/lib/types/server-script-source.d.ts

@@ -0,0 +1,65 @@
+import type { MerchantAccount } from '@gestaltrun/dsh-merchant-account';
+import { type SalesCaseFeature, type SalesCoverageResult, type SalesScriptVersion } from '@gestaltrun/dsh-cs-hosting/sales-case-contract';
+import { z } from 'zod';
+import { type EvaluationCaseRecord } from './record.ts';
+import type { EvaluationCaseId } from './record.ts';
+import type { EvaluationReplayRequirement, EvaluationScript, EvaluationScriptSource } from './types.ts';
+type StoredScript = z.infer<typeof SalesScriptVersion>;
+type StoredFeature = z.infer<typeof SalesCaseFeature>;
+/** The runner script plus the server checks that produced it. */
+export interface ServerEvaluationScript extends EvaluationScript {
+    /** Human-readable script subject for administrative inspection. */
+    readonly displayTitle?: string;
+    readonly scriptVersionId: string;
+    /** Demand-line revision the script was written from; selects the matching locally collected replay. */
+    readonly caseRevision: number;
+    /** Reads that supply the cited replay records. */
+    readonly requiredReads: readonly EvaluationReplayRequirement[];
+    readonly checks: StoredScript['script']['checks'];
+    /** Judge inputs derived only from frozen checks and verified replay records. */
+    readonly metrics: Pick<EvaluationCaseRecord, 'checks' | 'requiredFacts' | 'orderFactsAvailable'>;
+}
+/** Tenant-authenticated reads used by the source; a caller can substitute another transport. */
+export interface ServerScriptReader {
+    /** @param scriptVersionId - Concrete script version UUID. @param signal - Case lifetime. @returns Stored version. */
+    loadScript(scriptVersionId: string, signal: AbortSignal): Promise<z.infer<typeof SalesScriptVersion>>;
+    /** @param setId - Coverage set UUID. @param signal - Case lifetime. @returns Coverage result with case material. */
+    loadCoverage(setId: string, signal: AbortSignal): Promise<z.infer<typeof SalesCoverageResult>>;
+}
+/** Merchant-session reader used by the source; only request is needed. */
+export type ServerScriptAccount = Pick<MerchantAccount, 'request'>;
+/** Options for the server-backed source. */
+export interface ServerScriptSourceOptions {
+    /** Operation label recorded in generated replay envelopes. */
+    readonly operationId?: string;
+}
+/** Loads a frozen script by its concrete server version UUID. */
+export declare class MerchantServerScriptSource implements EvaluationScriptSource {
+    private readonly reader;
+    private readonly options;
+    /** @param reader - Tenant-authenticated script and coverage reads. @param options - Stable replay operation label. */
+    constructor(reader: ServerScriptReader, options?: ServerScriptSourceOptions);
+    /**
+     * @param id - Concrete `SalesScriptVersion.id` UUID, branded as a case id by the runner.
+     * @param signal - Case lifetime.
+     * @returns Script, replay entries and server checks.
+     * @throws When the reader rejects the read or the script has no matching case material.
+     */
+    load(id: EvaluationCaseId, signal: AbortSignal): Promise<ServerEvaluationScript>;
+}
+/**
+ * Build the production reader from a signed-in merchant account.
+ * @param account - Merchant account whose tenant routes are requested.
+ * @returns Reader using the synced script and coverage routes.
+ */
+export declare function merchantScriptReader(account: Pick<MerchantAccount, 'request'>): ServerScriptReader;
+/**
+ * Convert one stored script and its case material into the runner form.
+ * @param stored - Immutable server script version.
+ * @param feature - Coverage feature containing original messages.
+ * @param destinationName - Display name of the script destination.
+ * @param operationId - Replay envelope operation label.
+ * @returns Runner script with messages, replay entries and checks.
+ */
+export declare function scriptOf(stored: StoredScript, feature: StoredFeature, destinationName: string, operationId?: string): ServerEvaluationScript;
+export {};

package/lib/types/simulated-buyer-role.d.ts

@@ -0,0 +1,41 @@
+import type { Context } from '@deepseek-ai/cordis';
+import { type BuyerJudge, type SimulatedBuyerThresholds } from '@gestaltrun/dsh-cs-hosting';
+import type { ImEvaluationBinding } from './im-target.ts';
+import type { EvaluationAction, EvaluationRole, EvaluationScript } from './types.ts';
+/** Options of the simulated-buyer role. */
+export interface SimulatedBuyerRoleOptions {
+    /** Host with the buyer Session, IM runtime, Session persistence and tool registry. */
+    readonly ctx: Context;
+    /** Recorded or live System One decision client. */
+    readonly judge: BuyerJudge;
+    /** Deployment-resolved probability thresholds; the customer-service Host exposes them through its service. */
+    readonly thresholds: SimulatedBuyerThresholds;
+    /** Real milliseconds allowed for one buyer model turn; defaults to 120000. */
+    readonly stepTimeoutMs?: number;
+}
+/**
+ * Buyer role backed by the simulated-buyer Workspace Agent.
+ *
+ * The buyer Agent is the dual-Session simulated user created by `ImEvaluationTarget`; this role
+ * prepares each Host step, admits native IM replies and relays only approved buyer messages.
+ */
+export declare class SimulatedBuyerRole implements EvaluationRole<ImEvaluationBinding> {
+    private readonly options;
+    private buyer;
+    /** @param options - Context, decision client and step bounds. */
+    constructor(options: SimulatedBuyerRoleOptions);
+    /** @returns Whether the buyer mindset reached the ordering state during this case. */
+    outcome(): {
+        readonly simulatedDeal: boolean;
+    };
+    /**
+     * @param script - Frozen script, knowledge card and optional buying psychology.
+     * @param signal - Case lifetime.
+     * @param binding - Running pair and its buyer Agent.
+     * @returns Host-approved messages in order, ending with the script completion or an invalid marker.
+     */
+    actions(script: EvaluationScript, signal: AbortSignal, binding: ImEvaluationBinding): AsyncIterable<EvaluationAction>;
+    private historyCursor;
+    private readMindsetEvidence;
+    private rows;
+}

package/lib/types/snapshot-replay.d.ts

@@ -0,0 +1,15 @@
+import type { CliResult, SimulationReadRequest } from '@gestaltrun/dsh-cs-hosting/merchant-cli';
+import type { EvaluationReplayEntry } from './types.ts';
+type Selection = {
+    readonly entry?: EvaluationReplayEntry;
+    readonly result?: CliResult;
+    readonly reason: string;
+};
+/**
+ * Resolve recorded first/second screens and case-local search; third/sku have no collected baseline.
+ * @param request - Validated simulated seller read and its case clock.
+ * @param entries - Case-local CLI results; collection records require their actual observedAt.
+ * @returns A replay result or explicit missing facts; undefined delegates unrelated commands.
+ */
+export declare function snapshotReplay(request: SimulationReadRequest, entries: readonly EvaluationReplayEntry[]): Selection | undefined;
+export {};

package/lib/types/step-failure.d.ts

@@ -0,0 +1,25 @@
+/** Partial, settled target evidence retained when a model step cannot finish normally. */
+import type { EvaluationTurn } from './types.ts';
+import type { EvaluationCaseRecord } from './record.ts';
+/** Host-only failure; its cause is not serialized into the case record. */
+export declare class EvaluationStepFailure extends Error {
+    readonly partial: EvaluationTurn;
+    /** @param partial - Messages, tools and usage already observed after target cleanup. @param cause - Original failure for Host diagnostics. */
+    constructor(partial: EvaluationTurn, cause: unknown);
+}
+/** Settled case evidence retained when setup or cleanup prevents the runner from returning. */
+export declare class EvaluationCaseFailure extends Error {
+    readonly record: EvaluationCaseRecord;
+    /** @param record - Observed case data, including an empty transcript before admission. @param cause - Original Host failure, kept out of wire records. */
+    constructor(record: EvaluationCaseRecord, cause: unknown);
+}
+/** Provider-neutral failure from the tested Agent's persisted terminal event. */
+export declare class EvaluationModelFailure extends Error {
+    readonly code: string;
+    /** @param code - Machine code already recorded by the model adapter. */
+    constructor(code: string);
+}
+/** Host reply-wait deadline; it does not cancel the tested Agent or the evaluation. */
+export declare class EvaluationReplyTimeout extends Error {
+    constructor();
+}

package/lib/types/types.d.ts

@@ -0,0 +1,174 @@
+/** Host extension points for evaluating an Agent without embedding domain rules in the runner. */
+import type { Branded } from '@deepseek-ai/dsh-brand';
+import type { CliResult } from '@gestaltrun/dsh-cs-hosting/merchant-cli';
+import type { FirstReplyTiming } from '@gestaltrun/dsh-cs-hosting/types';
+import type { EvaluationCaseRecord } from './record.ts';
+import type { EvaluationHumanAction } from './human-actions.ts';
+import type { EvaluationInputId } from './purchase-intent.ts';
+/** Ephemeral tool identity used to join a domain world's read observations. */
+export type EvaluationCallId = Branded<'EvaluationCallId'>;
+/** One production CLI result retained for a script; the world returns it at or after `at`. */
+export interface EvaluationReplayEntry {
+    readonly command: string;
+    readonly args: Readonly<Record<string, unknown>>;
+    readonly at: number;
+    /** Inclusive last equal sample, when retained by collection; diagnostic only, not an expiry. */
+    readonly validThrough?: number | undefined;
+    /** Actual collection time of detail data; omission never substitutes the historical anchor at. */
+    readonly observedAt?: number | undefined;
+    /** Collection data supplies only explicitly classified slow fields; omission retains command-specific interpretation. */
+    readonly usage?: 'historical' | 'collection' | undefined;
+    readonly result: CliResult;
+}
+/** A replay read that supplies a fact the reference conclusion or a check cites; only misses on these reads leave the environment incomplete. */
+export interface EvaluationReplayRequirement {
+    readonly command: `seller.${string}` | `platform.${string}`;
+    /** Product `itemId`, order `orderId` or POI `poiId` of the cited record. */
+    readonly entityId: string;
+    /** Detail screen of a cited collection record; absent for snapshot and order reads. */
+    readonly screen?: string;
+}
+/** Buying-psychology card the Host uses to judge concern resolution; never model-generated. */
+export interface EvaluationBuyingPsychology {
+    readonly intent: string;
+    readonly drivers: readonly string[];
+    readonly concerns: readonly {
+        readonly id: string;
+        readonly text: string;
+        readonly resolution: string;
+    }[];
+    readonly bottomLine: string;
+    readonly decisionMode: string;
+    /** Condition under which the buyer places the order. */
+    readonly purchaseCondition?: string;
+}
+/** Input shared by deterministic and Agent-backed buyer roles. */
+export interface EvaluationScript extends Pick<EvaluationCaseRecord, 'caseId' | 'scriptId' | 'scriptVersion' | 'originalMessages' | 'buyerKnowledge' | 'referenceConclusion' | 'facts'> {
+    readonly segments: readonly {
+        readonly at: number;
+        readonly messages: readonly string[];
+    }[];
+    readonly completion: 'goal_reached' | 'buyer_abandoned';
+    /** Frozen script card; absent for scripts that do not evaluate a model-driven buyer. */
+    readonly buyingPsychology?: EvaluationBuyingPsychology;
+    /** Script-authored candidates for Host-side human collaboration sampling. */
+    readonly humanActions?: readonly EvaluationHumanAction[];
+    /** Script-local replay results; a server script source supplies these. */
+    readonly replay?: readonly EvaluationReplayEntry[];
+    /** Sales phase; present only for scripts scored by the cs-sales-v1 metric pack. */
+    readonly sales?: {
+        readonly phase: 'presales' | 'other';
+    };
+}
+/** A domain-independent role action; advancement is between completed target steps. */
+export type EvaluationAction = {
+    readonly kind: 'advance';
+    readonly at: number;
+} | {
+    readonly kind: 'message' | 'staff';
+    readonly text: string;
+} | {
+    readonly kind: 'resume';
+} | {
+    readonly kind: 'finish';
+    readonly reason: EvaluationScript['completion'];
+} | {
+    readonly kind: 'invalid';
+    readonly reason: EvaluationInvalidReason;
+};
+/** Host-observed role invalidity; the case must not be scored. */
+export type EvaluationInvalidReason = NonNullable<EvaluationCaseRecord['buyerInvalidReason']>;
+/** Script selection may read a local fixture or a versioned server source. */
+export interface EvaluationScriptSource {
+    /** @param id - Requested case. @param signal - Caller lifetime. @returns The frozen script and source facts. */
+    load(id: EvaluationCaseRecord['caseId'], signal: AbortSignal): Promise<EvaluationScript>;
+}
+/** Buyer-side generation; implementations must stop when the signal aborts. */
+export interface EvaluationRole<Binding = undefined> {
+    /** @param script - Frozen knowledge and segments. @param signal - Caller lifetime. @param binding - Target-owned references for Agent-backed roles; opaque to the runner. @returns Ordered role actions. */
+    actions(script: EvaluationScript, signal: AbortSignal, binding: Binding): AsyncIterable<EvaluationAction>;
+    /** @returns Host-observed sales outcome after the run; omitted by roles that do not model purchase. */
+    outcome?(): {
+        readonly simulatedDeal: boolean;
+    };
+}
+/** One switchable run condition; paired sales runs apply exactly an annotation-on and annotation-off variant. */
+export interface EvaluationVariant {
+    readonly id: string;
+    /** Whether this group receives buying-psychology annotations; absent means off. */
+    readonly annotation?: boolean;
+    /** @param signal - Run lifetime. @returns Disposer restoring the deployment condition. */
+    apply(signal: AbortSignal): Promise<() => void | Promise<void>>;
+}
+/** Observations through target settlement or the reply-wait deadline. */
+export interface EvaluationTurn {
+    /** Transport inputs whose receipt completed; observers additionally require their persisted Session admission. */
+    readonly inputIds?: readonly EvaluationInputId[];
+    /** Simulated sending instant from the accepted buyer receipt; absent when no buyer input was accepted. */
+    readonly inputAt?: number;
+    readonly messages: readonly {
+        readonly role: 'assistant' | 'operator';
+        readonly text: string;
+        readonly at: number;
+    }[];
+    readonly tools: readonly {
+        readonly at: number;
+        readonly id: EvaluationCallId;
+        readonly command: string;
+        readonly arguments: EvaluationCaseRecord['tools'][number]['arguments'];
+        readonly result: EvaluationCaseRecord['tools'][number]['result'];
+        readonly status: 'ok' | 'error' | 'timeout';
+    }[];
+    /** Present only after a buyer input was durably accepted; excludes target cleanup. */
+    readonly replyTiming?: FirstReplyTiming;
+    /** Reply waiting expired; the tested Agent remains running and the case can continue. */
+    readonly replyTimedOut?: true;
+    readonly usage: EvaluationCaseRecord['usage'];
+    readonly end?: 'handoff' | 'paused';
+}
+/** Domain bindings stay opaque to the runner and are shared only with its world adapter. */
+export interface EvaluationConversation<Binding> {
+    readonly binding: Binding;
+    /** @returns Current simulated Unix milliseconds, including real elapsed time. */
+    currentTime(): number;
+    /** @param at - Next original segment time; earlier instants leave the running clock unchanged. */
+    advance(at: number): void;
+    /** @param text - Buyer message. @param signal - Step deadline and caller lifetime. @param human - Optional action admitted after buyer acceptance; omission sends only the buyer. @returns Observations through settlement or reply timeout; caller cancellation rejects. */
+    send(text: string, signal: AbortSignal, human?: EvaluationHumanInput): Promise<EvaluationTurn>;
+    /** @param text - Simulated shop staff reply during a confirmed handoff. @param signal - Step lifetime. @returns Durable operator record without awaiting an AI reply. */
+    staff?(text: string, signal: AbortSignal): Promise<EvaluationTurn>;
+    /** @param signal - Step lifetime. @returns Observations after restoring automatic handling; failure rejects without discarding earlier records. */
+    resume?(signal: AbortSignal): Promise<EvaluationTurn>;
+    /** @param action - Selected script plan, never generated by the buyer role. @param signal - Step lifetime. @param accepted - Called once after durable input acceptance, before awaiting a response. @returns Settled observations; a human conversation step may produce no customer reply. */
+    intervene?(action: EvaluationHumanAction, signal: AbortSignal, accepted: () => void): Promise<EvaluationTurn>;
+    /** Cancel outstanding work and await all resources owned by this conversation. */
+    close(): Promise<void>;
+}
+/** One human input admitted immediately after a buyer message, before collecting the reply. */
+export interface EvaluationHumanInput {
+    readonly action: EvaluationHumanAction;
+    /** Acknowledge durable operator input exactly once. */
+    readonly accepted: () => void;
+}
+/** Creates the actual target, including its production preset and configuration. */
+export interface EvaluationTarget<Binding> {
+    /** @param script - Case input. @param signal - Caller lifetime. @returns One isolated target conversation. */
+    open(script: EvaluationScript, signal: AbortSignal): Promise<EvaluationConversation<Binding>>;
+}
+/** World evidence joined to the target's logged tool calls. */
+export interface EvaluationWorldHandle {
+    /** @param id - Target tool identity. @returns Replay classification, or not_applicable for unrelated tools. */
+    classify(id: EvaluationCallId): Pick<EvaluationCaseRecord['tools'][number], 'replay' | 'affectsConclusion'>;
+    /** Detach all per-case registrations after the target is closed. */
+    dispose(): Promise<void>;
+}
+/** Installs domain-specific replay and live-world behavior. */
+export interface EvaluationWorld<Binding> {
+    /** @param binding - Target-owned references. @param script - Frozen case input. @param signal - Caller lifetime. @returns Per-case world evidence and cleanup. */
+    attach(binding: Binding, script: EvaluationScript, signal: AbortSignal): Promise<EvaluationWorldHandle>;
+}
+/** Metric packs declare questions and required facts; the runner performs no semantic scoring. */
+export interface EvaluationMetricPack {
+    /** @param script - Frozen case input. @param messages - Completed simulated timeline; excludes unexecuted turns. @returns Applicable checks and required facts for the server judge. */
+    prepare(script: EvaluationScript, messages: EvaluationCaseRecord['messages']): Pick<EvaluationCaseRecord, 'checks' | 'requiredFacts' | 'orderFactsAvailable'>;
+}

package/locale/en.json

@@ -0,0 +1,6 @@
+{
+  "meta": {
+    "title": "Agent evaluation",
+    "description": "Evaluate AI customer service with shop conversation history and inspect scope, usage and results"
+  }
+}

package/locale/zh.json

@@ -0,0 +1,6 @@
+{
+  "meta": {
+    "title": "Agent 评测",
+    "description": "使用店铺历史对话预先评测 AI 客服，并查看评测范围、用量和运行结果"
+  }
+}

package/package.json

@@ -1,6 +1,109 @@
 {
   "name": "@gestaltrun/dsh-agent-eval",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.1",
+  "description": "Product agent evaluation runner and customer-service replay adapters",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/types/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    "./cordis.patch.yml": "./cordis.patch.yml",
+    "./locale/*.json": "./locale/*.json",
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./cs-hosting": {
+      "default": "./lib/cs-hosting.js"
+    }
+  },
+  "files": [
+    "cordis.patch.yml",
+    "locale/*.json",
+    "lib",
+    "README.md",
+    "README.zh.md",
+    "LICENSE"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gestaltrun/deepseek-harness.git",
+    "directory": "product/agent-eval"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/",
+    "tag": "candidate"
+  },
+  "dsh": {
+    "engines": {
+      "dsh": ">=0.2.1-rc.2"
+    },
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    }
+  },
+  "dependencies": {
+    "zod": "4.4.3"
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "4.0.6-alpha.1",
+    "@deepseek-ai/dsh-agent": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-brand": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-session": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-session-persistence": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-llm": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-workspace": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-storage-domain": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-agent-default-model": "0.2.2-alpha.2",
+    "@gestaltrun/dsh-workspace-bundle": ">=0.1.0",
+    "@gestaltrun/dsh-im-runtime": ">=0.1.5 <0.2.0",
+    "@gestaltrun/dsh-cs-hosting": ">=0.1.7 <0.2.0",
+    "@deepseek-ai/dsh-agent-preset-registry": "0.2.2-alpha.2",
+    "@gestaltrun/dsh-merchant-account": ">=0.1.0-gestaltrun.0",
+    "@deepseek-ai/dsh-tool-executor-ptc": "0.2.2-alpha.2"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "4.0.6-alpha.1",
+    "@deepseek-ai/dsh-agent": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-brand": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-session": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-session-persistence": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-llm": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-workspace": "0.2.2-alpha.2",
+    "@gestaltrun/dsh-im-runtime": "^0.1.6",
+    "@gestaltrun/dsh-cs-hosting": "^0.1.7",
+    "typescript": "6.0.3",
+    "vitest": "4.1.8",
+    "tsdown": "0.22.2",
+    "@types/node": "22.20.0",
+    "@deepseek-ai/dsh-agent-preset-registry": "0.2.2-alpha.2",
+    "@deepseek-ai/cordis-plugin-include": "1.0.11-alpha.1",
+    "@deepseek-ai/cordis-plugin-loader": "1.0.7-alpha.1",
+    "@deepseek-ai/dsh-agent-loop": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-agent-loop-testkit": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-agent-default-model": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-credentials-local": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-storage": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-storage-domain": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-storage-json": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-session-persistence-jsonl": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-skill": "0.2.2-alpha.2",
+    "@gestaltrun/dsh-workspace-bundle": "^0.1.1",
+    "js-yaml": "4.2.0",
+    "@deepseek-ai/dsh-subprocess-local": "0.2.2-alpha.2",
+    "@gestaltrun/dsh-merchant-account": "^0.1.5",
+    "@deepseek-ai/cordis-plugin-group": "1.0.6-alpha.1",
+    "@deepseek-ai/dsh-ptc-runtime-node": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-sandbox-local": "0.2.2-alpha.2",
+    "@deepseek-ai/dsh-tool-executor-ptc": "0.2.2-alpha.2"
+  },
+  "scripts": {
+    "build": "node -e \"require('node:fs').rmSync('lib', { recursive: true, force: true })\" && tsc -p tsconfig.build.json && tsdown",
+    "typecheck": "tsc --noEmit && tsc -p tsconfig.test.json --noEmit",
+    "test": "pnpm run test:source && pnpm run test:built",
+    "test:built": "vitest run --config vitest.built.config.ts",
+    "test:source": "node --test tests/bundle.test.mjs ../cs-hosting/tests/fixture-start.test.mjs && vitest run"
+  }
 }