@gestaltrun/dsh-agent-eval 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.i18n.yaml +58 -0
- package/README.md +194 -2
- package/README.zh.md +195 -0
- package/cordis.patch.yml +13 -0
- package/lib/cs-hosting-plugin-B9bjLaoE.js +6980 -0
- package/lib/cs-hosting.js +2 -0
- package/lib/index.js +2 -0
- package/lib/types/agent-runs.d.ts +56 -0
- package/lib/types/cs-hosting-evaluation.d.ts +235 -0
- package/lib/types/cs-hosting-plugin.d.ts +10 -0
- package/lib/types/handoff.d.ts +4 -0
- package/lib/types/human-actions.d.ts +36 -0
- package/lib/types/im-purchase-intent.d.ts +12 -0
- package/lib/types/im-target.d.ts +45 -0
- package/lib/types/index.d.ts +22 -0
- package/lib/types/input-admission.d.ts +10 -0
- package/lib/types/practice-failure.d.ts +17 -0
- package/lib/types/product-field-policy.d.ts +129 -0
- package/lib/types/production-budget.d.ts +21 -0
- package/lib/types/production-buyers.d.ts +20 -0
- package/lib/types/production-workspace.d.ts +64 -0
- package/lib/types/purchase-intent.d.ts +66 -0
- package/lib/types/record.d.ts +169 -0
- package/lib/types/replay-world.d.ts +25 -0
- package/lib/types/reply-latency.d.ts +21 -0
- package/lib/types/runner.d.ts +97 -0
- package/lib/types/sales.d.ts +33 -0
- package/lib/types/script-production.d.ts +955 -0
- package/lib/types/server-script-source.d.ts +65 -0
- package/lib/types/simulated-buyer-role.d.ts +41 -0
- package/lib/types/snapshot-replay.d.ts +15 -0
- package/lib/types/step-failure.d.ts +25 -0
- package/lib/types/types.d.ts +174 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +106 -3
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { type CaseDataTask, type CaseDataReplayEntry } from '@gestaltrun/dsh-cs-hosting/case-data';
|
|
4
|
+
import { type SalesCase, type SalesTimelineMessage } from '@gestaltrun/dsh-cs-hosting/sales-case-contract';
|
|
5
|
+
import { type ProductionOperations } from './script-production.ts';
|
|
6
|
+
import type { AgentRuns } from './agent-runs.ts';
|
|
7
|
+
declare module '@deepseek-ai/dsh-llm' {
|
|
8
|
+
interface MessageSourceMap {
|
|
9
|
+
/** Interrupted evaluation turn which received a recovery follow-up. */
|
|
10
|
+
'evaluation-restart': {
|
|
11
|
+
readonly kind: 'evaluation-restart';
|
|
12
|
+
readonly turnStartSeq: import('@deepseek-ai/dsh-session').SessionSeq;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** Task timing choices, resolved by the evaluation plugin at load. */
|
|
17
|
+
export interface ProductionWorkspaceConfig {
|
|
18
|
+
readonly taskTimeoutMs: number;
|
|
19
|
+
readonly scriptTaskTimeoutMs: number;
|
|
20
|
+
readonly dealWindowDays: number;
|
|
21
|
+
readonly orderLookbackDays: number;
|
|
22
|
+
}
|
|
23
|
+
/** Opens the built-in workspace only; every task owns and disposes its own Agent Session. */
|
|
24
|
+
export declare class ProductionWorkspace {
|
|
25
|
+
private readonly ctx;
|
|
26
|
+
private readonly tenantId;
|
|
27
|
+
private readonly config;
|
|
28
|
+
private readonly agentRuns?;
|
|
29
|
+
private readonly fromCollection;
|
|
30
|
+
private readonly runId?;
|
|
31
|
+
private readonly replay;
|
|
32
|
+
private path;
|
|
33
|
+
private loaded;
|
|
34
|
+
/** @param ctx - Production task and merchant capabilities. @param tenantId - Captured shop identity. @param config - Resolved timing choices. @param agentRuns - Optional administrator task observer. @param fromCollection - Uses a separate fresh-collection checkpoint. @param runId - Durable production identity for reopening interrupted Sessions. */
|
|
35
|
+
constructor(ctx: Context, tenantId: string, config: ProductionWorkspaceConfig, agentRuns?: AgentRuns | undefined, fromCollection?: boolean, runId?: string | undefined);
|
|
36
|
+
private taskReference;
|
|
37
|
+
/** @param buyers - Authoritative server buyer pagination. @param budget - Live quota admission. @returns Pipeline task and checkpoint operations. */
|
|
38
|
+
operations(buyers: ProductionOperations['buyers'], budget: ProductionOperations['budget']): ProductionOperations;
|
|
39
|
+
/**
|
|
40
|
+
* @param caseId - Demand line the script was written from.
|
|
41
|
+
* @param revision - Case revision recorded on the script.
|
|
42
|
+
* @param signal - Checkpoint read lifetime.
|
|
43
|
+
* @returns Replay collected for that revision on this Host, or empty when this Host retains none.
|
|
44
|
+
*/
|
|
45
|
+
replayOf(caseId: string, revision: number, signal: AbortSignal): Promise<readonly CaseDataReplayEntry[]>;
|
|
46
|
+
private location;
|
|
47
|
+
private load;
|
|
48
|
+
private save;
|
|
49
|
+
private timeline;
|
|
50
|
+
private collect;
|
|
51
|
+
private author;
|
|
52
|
+
private session;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Build one current-revision task from formal messages; only explicit linked entity identifiers enter read scopes.
|
|
56
|
+
* Messages carry readable bodies; the Host renders them as a dated transcript with short references.
|
|
57
|
+
* @param row - Current sales case.
|
|
58
|
+
* @param timeline - Complete buyer timeline.
|
|
59
|
+
* @param destinations - Server dictionary.
|
|
60
|
+
* @param captureTime - Host capture instant.
|
|
61
|
+
* @param config - Configured order windows.
|
|
62
|
+
* @returns One buyer-bound case task.
|
|
63
|
+
*/
|
|
64
|
+
export declare function caseDataInput(row: z.infer<typeof SalesCase>, timeline: readonly z.infer<typeof SalesTimelineMessage>[], destinations: CaseDataTask['destinations'], captureTime: number, config: Pick<ProductionWorkspaceConfig, 'dealWindowDays' | 'orderLookbackDays'>): CaseDataTask;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/** Product-owned capture of the original buying-analysis results for persisted buyer inputs. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import type { Branded } from '@deepseek-ai/dsh-brand';
|
|
4
|
+
import type { BuyingAdmissionObservations } from '@gestaltrun/dsh-cs-hosting';
|
|
5
|
+
import type { EvaluationScript } from './types.ts';
|
|
6
|
+
/** Formal optional v1 schema; legacy records omit it. */
|
|
7
|
+
export { PurchaseIntent as EvaluationPurchaseIntent } from '@gestaltrun/dsh-cs-hosting/sales-case-contract';
|
|
8
|
+
import { PurchaseIntent as EvaluationPurchaseIntent } from '@gestaltrun/dsh-cs-hosting/sales-case-contract';
|
|
9
|
+
/** Failure categories retained only in operator records. */
|
|
10
|
+
export declare const PurchaseIntentUnavailableReason: z.ZodEnum<{
|
|
11
|
+
failed: "failed";
|
|
12
|
+
timeout: "timeout";
|
|
13
|
+
disabled: "disabled";
|
|
14
|
+
"not-collected": "not-collected";
|
|
15
|
+
reconstructed: "reconstructed";
|
|
16
|
+
cancelled: "cancelled";
|
|
17
|
+
}>;
|
|
18
|
+
export type PurchaseIntentUnavailableReason = z.infer<typeof PurchaseIntentUnavailableReason>;
|
|
19
|
+
/** Opaque input identity supplied by a target transport. */
|
|
20
|
+
export type EvaluationInputId = Branded<'EvaluationInputId'>;
|
|
21
|
+
/** Opaque persisted admission identity supplied by a target. */
|
|
22
|
+
export type EvaluationAdmissionId = Branded<'EvaluationAdmissionId'>;
|
|
23
|
+
/** A persisted external-buyer batch, projected from the target Session source. */
|
|
24
|
+
export interface PurchaseIntentAdmission {
|
|
25
|
+
readonly id: EvaluationAdmissionId;
|
|
26
|
+
readonly messageIds: readonly EvaluationInputId[];
|
|
27
|
+
}
|
|
28
|
+
/** Runner-owned turn and message identities joined to transport input identities. */
|
|
29
|
+
export interface PurchaseIntentStep {
|
|
30
|
+
readonly turn: number;
|
|
31
|
+
readonly at: number;
|
|
32
|
+
readonly inputs: readonly {
|
|
33
|
+
readonly messageId: string;
|
|
34
|
+
readonly sourceMessageIds: readonly EvaluationInputId[];
|
|
35
|
+
}[];
|
|
36
|
+
readonly failure?: PurchaseIntentUnavailableReason;
|
|
37
|
+
}
|
|
38
|
+
/** Capture lifetime for one isolated case. */
|
|
39
|
+
export interface PurchaseIntentObserver {
|
|
40
|
+
/** @param step - Persisted target inputs and the runner's canonical identities. */
|
|
41
|
+
record(step: PurchaseIntentStep): void;
|
|
42
|
+
/** @returns Original observations retained during this case; no cache rebuild or model request occurs. */
|
|
43
|
+
snapshot(): EvaluationPurchaseIntent;
|
|
44
|
+
/** Release the underlying observation reader. */
|
|
45
|
+
dispose(): void;
|
|
46
|
+
}
|
|
47
|
+
/** Product adapter that observes one isolated target without requesting model work. */
|
|
48
|
+
export interface PurchaseIntentObserverFactory<Binding> {
|
|
49
|
+
/** @param script - Frozen case. @param binding - Target-owned resources. @returns One case-scoped capture. */
|
|
50
|
+
open(script: EvaluationScript, binding: Binding): PurchaseIntentObserver;
|
|
51
|
+
}
|
|
52
|
+
/** Joins only exact persisted source ids; equal text and equal clocks do not establish identity. */
|
|
53
|
+
export declare class PurchaseIntentCapture implements PurchaseIntentObserver {
|
|
54
|
+
private readonly reader;
|
|
55
|
+
private readonly admissions;
|
|
56
|
+
private readonly inputs;
|
|
57
|
+
private readonly points;
|
|
58
|
+
/** @param reader - Case-scoped raw observations. @param admissions - Current persisted admission sources for this case. */
|
|
59
|
+
constructor(reader: BuyingAdmissionObservations, admissions: () => readonly PurchaseIntentAdmission[]);
|
|
60
|
+
/** @inheritdoc */
|
|
61
|
+
record(step: PurchaseIntentStep): void;
|
|
62
|
+
/** @inheritdoc */
|
|
63
|
+
snapshot(): EvaluationPurchaseIntent;
|
|
64
|
+
/** @inheritdoc */
|
|
65
|
+
dispose(): void;
|
|
66
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/** Formal server record validation with local opaque identities. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { type Branded } from '@deepseek-ai/dsh-brand';
|
|
4
|
+
/** Opaque case identity from the evaluation set. */
|
|
5
|
+
export type EvaluationCaseId = Branded<'EvaluationCaseId'>;
|
|
6
|
+
/** Opaque script identity from the versioned script source. */
|
|
7
|
+
export type EvaluationScriptId = Branded<'EvaluationScriptId'>;
|
|
8
|
+
/** Server contract package version consumed by this runner. */
|
|
9
|
+
export declare const EVALUATION_CONTRACT_REVISION = "0.2.32-pre.3";
|
|
10
|
+
/** Formal record timestamp, message and check schemas. */
|
|
11
|
+
export { EvaluationTime, EvaluationMessage, EvaluationCheck } from '@gestaltrun/dsh-cs-hosting/sales-case-contract';
|
|
12
|
+
/** Branded identities serialize unchanged; all server fields and reference checks remain active. */
|
|
13
|
+
export declare const EvaluationCaseRecord: z.ZodObject<{
|
|
14
|
+
buyerInvalidReason: z.ZodOptional<z.ZodEnum<{
|
|
15
|
+
"boundary-rewrites-exhausted": "boundary-rewrites-exhausted";
|
|
16
|
+
"boundary-check-failed": "boundary-check-failed";
|
|
17
|
+
}>>;
|
|
18
|
+
sales: z.ZodOptional<z.ZodObject<{
|
|
19
|
+
phase: z.ZodEnum<{
|
|
20
|
+
other: "other";
|
|
21
|
+
presales: "presales";
|
|
22
|
+
}>;
|
|
23
|
+
simulatedDeal: z.ZodBoolean;
|
|
24
|
+
purchaseIntent: z.ZodOptional<z.ZodObject<{
|
|
25
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
26
|
+
source: z.ZodLiteral<"buying-insights">;
|
|
27
|
+
questionVersion: z.ZodString;
|
|
28
|
+
samplePhase: z.ZodLiteral<"buyer-admitted-before-assistant">;
|
|
29
|
+
points: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
30
|
+
observationId: z.ZodString;
|
|
31
|
+
turn: z.ZodNumber;
|
|
32
|
+
buyerMessageIds: z.ZodArray<z.ZodString>;
|
|
33
|
+
at: z.ZodNumber;
|
|
34
|
+
status: z.ZodLiteral<"observed">;
|
|
35
|
+
score: z.ZodNumber;
|
|
36
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
37
|
+
observationId: z.ZodString;
|
|
38
|
+
turn: z.ZodNumber;
|
|
39
|
+
buyerMessageIds: z.ZodArray<z.ZodString>;
|
|
40
|
+
at: z.ZodNumber;
|
|
41
|
+
status: z.ZodLiteral<"unavailable">;
|
|
42
|
+
reason: z.ZodEnum<{
|
|
43
|
+
failed: "failed";
|
|
44
|
+
timeout: "timeout";
|
|
45
|
+
disabled: "disabled";
|
|
46
|
+
"not-collected": "not-collected";
|
|
47
|
+
reconstructed: "reconstructed";
|
|
48
|
+
cancelled: "cancelled";
|
|
49
|
+
}>;
|
|
50
|
+
}, z.core.$strict>], "status">>;
|
|
51
|
+
}, z.core.$strict>>;
|
|
52
|
+
humanComparisons: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
53
|
+
turn: z.ZodNumber;
|
|
54
|
+
buyerMessageId: z.ZodString;
|
|
55
|
+
originalBuyerMessageId: z.ZodString;
|
|
56
|
+
originalHumanMessageId: z.ZodString;
|
|
57
|
+
}, z.core.$strict>>>;
|
|
58
|
+
}, z.core.$strict>>;
|
|
59
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
60
|
+
scriptVersion: z.ZodString;
|
|
61
|
+
messages: z.ZodArray<z.ZodObject<{
|
|
62
|
+
id: z.ZodString;
|
|
63
|
+
turn: z.ZodNumber;
|
|
64
|
+
role: z.ZodEnum<{
|
|
65
|
+
operator: "operator";
|
|
66
|
+
buyer: "buyer";
|
|
67
|
+
assistant: "assistant";
|
|
68
|
+
}>;
|
|
69
|
+
at: z.ZodNumber;
|
|
70
|
+
clockAt: z.ZodNumber;
|
|
71
|
+
text: z.ZodString;
|
|
72
|
+
}, z.core.$strict>>;
|
|
73
|
+
originalMessages: z.ZodArray<z.ZodObject<{
|
|
74
|
+
id: z.ZodString;
|
|
75
|
+
turn: z.ZodNumber;
|
|
76
|
+
role: z.ZodEnum<{
|
|
77
|
+
operator: "operator";
|
|
78
|
+
buyer: "buyer";
|
|
79
|
+
assistant: "assistant";
|
|
80
|
+
}>;
|
|
81
|
+
at: z.ZodNumber;
|
|
82
|
+
clockAt: z.ZodNumber;
|
|
83
|
+
text: z.ZodString;
|
|
84
|
+
}, z.core.$strict>>;
|
|
85
|
+
tools: z.ZodArray<z.ZodObject<{
|
|
86
|
+
id: z.ZodString;
|
|
87
|
+
turn: z.ZodNumber;
|
|
88
|
+
at: z.ZodNumber;
|
|
89
|
+
command: z.ZodString;
|
|
90
|
+
arguments: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull, z.ZodArray<z.ZodUnknown>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>;
|
|
91
|
+
result: z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull, z.ZodArray<z.ZodUnknown>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>;
|
|
92
|
+
status: z.ZodEnum<{
|
|
93
|
+
error: "error";
|
|
94
|
+
timeout: "timeout";
|
|
95
|
+
ok: "ok";
|
|
96
|
+
}>;
|
|
97
|
+
replay: z.ZodEnum<{
|
|
98
|
+
hit: "hit";
|
|
99
|
+
miss: "miss";
|
|
100
|
+
passthrough: "passthrough";
|
|
101
|
+
not_applicable: "not_applicable";
|
|
102
|
+
}>;
|
|
103
|
+
affectsConclusion: z.ZodBoolean;
|
|
104
|
+
}, z.core.$strict>>;
|
|
105
|
+
requiredFacts: z.ZodArray<z.ZodObject<{
|
|
106
|
+
kind: z.ZodString;
|
|
107
|
+
turn: z.ZodNumber;
|
|
108
|
+
command: z.ZodString;
|
|
109
|
+
available: z.ZodBoolean;
|
|
110
|
+
}, z.core.$strict>>;
|
|
111
|
+
orderFactsAvailable: z.ZodBoolean;
|
|
112
|
+
facts: z.ZodArray<z.ZodObject<{
|
|
113
|
+
turn: z.ZodNumber;
|
|
114
|
+
at: z.ZodNumber;
|
|
115
|
+
text: z.ZodString;
|
|
116
|
+
}, z.core.$strict>>;
|
|
117
|
+
buyerKnowledge: z.ZodString;
|
|
118
|
+
referenceConclusion: z.ZodString;
|
|
119
|
+
checks: z.ZodArray<z.ZodObject<{
|
|
120
|
+
id: z.ZodString;
|
|
121
|
+
kind: z.ZodEnum<{
|
|
122
|
+
required: "required";
|
|
123
|
+
recommended: "recommended";
|
|
124
|
+
forbidden: "forbidden";
|
|
125
|
+
}>;
|
|
126
|
+
text: z.ZodString;
|
|
127
|
+
turns: z.ZodArray<z.ZodNumber>;
|
|
128
|
+
destination: z.ZodOptional<z.ZodString>;
|
|
129
|
+
dependsOnOrderFacts: z.ZodBoolean;
|
|
130
|
+
}, z.core.$strict>>;
|
|
131
|
+
humanActions: z.ZodArray<z.ZodObject<{
|
|
132
|
+
id: z.ZodString;
|
|
133
|
+
turn: z.ZodNumber;
|
|
134
|
+
at: z.ZodNumber;
|
|
135
|
+
kind: z.ZodEnum<{
|
|
136
|
+
answer: "answer";
|
|
137
|
+
correction: "correction";
|
|
138
|
+
stop: "stop";
|
|
139
|
+
proposal: "proposal";
|
|
140
|
+
violation: "violation";
|
|
141
|
+
}>;
|
|
142
|
+
channel: z.ZodEnum<{
|
|
143
|
+
conversation: "conversation";
|
|
144
|
+
private: "private";
|
|
145
|
+
}>;
|
|
146
|
+
text: z.ZodString;
|
|
147
|
+
}, z.core.$strict>>;
|
|
148
|
+
startedAt: z.ZodNumber;
|
|
149
|
+
endedAt: z.ZodNumber;
|
|
150
|
+
endReason: z.ZodEnum<{
|
|
151
|
+
error: "error";
|
|
152
|
+
timeout: "timeout";
|
|
153
|
+
paused: "paused";
|
|
154
|
+
handoff: "handoff";
|
|
155
|
+
cancelled: "cancelled";
|
|
156
|
+
goal_reached: "goal_reached";
|
|
157
|
+
buyer_abandoned: "buyer_abandoned";
|
|
158
|
+
message_limit: "message_limit";
|
|
159
|
+
}>;
|
|
160
|
+
usage: z.ZodObject<{
|
|
161
|
+
inputTokens: z.ZodNumber;
|
|
162
|
+
outputTokens: z.ZodNumber;
|
|
163
|
+
cost: z.ZodNumber;
|
|
164
|
+
currency: z.ZodString;
|
|
165
|
+
}, z.core.$strict>;
|
|
166
|
+
caseId: z.ZodPipe<z.ZodString, z.ZodTransform<EvaluationCaseId, string>>;
|
|
167
|
+
scriptId: z.ZodPipe<z.ZodString, z.ZodTransform<EvaluationScriptId, string>>;
|
|
168
|
+
}, z.core.$strict>;
|
|
169
|
+
export type EvaluationCaseRecord = z.infer<typeof EvaluationCaseRecord>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { MerchantPlatformCommands } from '@gestaltrun/dsh-cs-hosting/merchant-cli';
|
|
2
|
+
import type { ImAccountId } from '@gestaltrun/dsh-im-runtime';
|
|
3
|
+
import type { EvaluationCaseRecord } from './record.ts';
|
|
4
|
+
import type { ImEvaluationBinding } from './im-target.ts';
|
|
5
|
+
import type { EvaluationReplayEntry, EvaluationReplayRequirement, EvaluationScript, EvaluationWorld, EvaluationWorldHandle } from './types.ts';
|
|
6
|
+
/** One already-validated historical command result. */
|
|
7
|
+
export type ReplayEntry = EvaluationReplayEntry;
|
|
8
|
+
/** World inputs are case-local and must belong to the evaluated shop. */
|
|
9
|
+
export interface ReplayWorldOptions {
|
|
10
|
+
readonly accountId: ImAccountId;
|
|
11
|
+
readonly caseId: EvaluationCaseRecord['caseId'];
|
|
12
|
+
readonly entries: readonly ReplayEntry[];
|
|
13
|
+
/** Reads whose misses affect scoring; omission makes every seller-read miss affect scoring; platform misses require an explicit entry. */
|
|
14
|
+
readonly requiredReads?: readonly EvaluationReplayRequirement[];
|
|
15
|
+
}
|
|
16
|
+
/** Each attachment owns its trace; facts must precede both the query time and case clock, with exact non-time arguments except the explicit snapshot product views and recorded snapshot `include*` sections, which answer any requested subset. */
|
|
17
|
+
export declare class CustomerServiceReplayWorld implements EvaluationWorld<ImEvaluationBinding> {
|
|
18
|
+
private readonly commands;
|
|
19
|
+
private readonly options;
|
|
20
|
+
/** @param commands - Production merchant CLI Host. @param options - Validated historical results for one shop. */
|
|
21
|
+
constructor(commands: Pick<MerchantPlatformCommands, 'registerSimulationRead'>, options: ReplayWorldOptions);
|
|
22
|
+
/** @param binding - Running target pair. @param script - Frozen role input. @param signal - Case lifetime. @returns Instance-local replay evidence and disposer. */
|
|
23
|
+
attach(binding: ImEvaluationBinding, script: EvaluationScript, signal: AbortSignal): Promise<EvaluationWorldHandle>;
|
|
24
|
+
private find;
|
|
25
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** First durable AI reply to one accepted simulation input, independent of the script clock. */
|
|
2
|
+
import type { ToolCallId } from '@deepseek-ai/dsh-llm';
|
|
3
|
+
import type { Session } from '@deepseek-ai/dsh-session';
|
|
4
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
5
|
+
import { type ImDeliveryScope, type ImMessageId, type ImRuntimeService } from '@gestaltrun/dsh-im-runtime';
|
|
6
|
+
import type { FirstReplyTiming } from '@gestaltrun/dsh-cs-hosting/types';
|
|
7
|
+
/** One round's observation, frozen before cancellation cleanup can deliver late results. */
|
|
8
|
+
export interface ReplyLatencyObserver {
|
|
9
|
+
/** @param messageId - Exact buyer receipt; paused receipts have no reply observation. */
|
|
10
|
+
acceptInput(messageId: ImMessageId): void;
|
|
11
|
+
/** @param callId - Send-tool call identity. @param value - Structured send result from the tool pipeline; unrecognized receipts are ignored. */
|
|
12
|
+
acceptReply(callId: ToolCallId, value: unknown): void;
|
|
13
|
+
/** Preserve the observation endpoint; late receipts can still identify sends completed before it. Repeated calls keep the same endpoint. */
|
|
14
|
+
freeze(): void;
|
|
15
|
+
/** @returns Frozen timing, or undefined when no matching buyer input was observed. */
|
|
16
|
+
snapshot(): FirstReplyTiming | undefined;
|
|
17
|
+
/** Freeze and release the delivery listener. */
|
|
18
|
+
dispose(): void;
|
|
19
|
+
}
|
|
20
|
+
/** @param ctx - Event owner for this active round. @param runtime - Durable outbound and handling-state reader. @param scope - Exact evaluated conversation. @param now - Monotonic Host milliseconds. @param session - Tested Session containing consumed inputs and send-tool receipts. @returns Observer installed before buyer injection; the caller disposes it after the round. */
|
|
21
|
+
export declare function observeReplyLatency(ctx: Context, runtime: Pick<ImRuntimeService, 'getOutbound' | 'queryOutbound' | 'getHandlingControl'>, scope: ImDeliveryScope, now: () => number, session: Session): ReplyLatencyObserver;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/** Domain-independent execution of one versioned evaluation case. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { EvaluationCaseRecord } from './record.ts';
|
|
4
|
+
import { type EvaluationHumanAction, type EvaluationRunRecord } from './human-actions.ts';
|
|
5
|
+
import { EvaluationSalesOptions } from './sales.ts';
|
|
6
|
+
import type { PurchaseIntentObserverFactory } from './purchase-intent.ts';
|
|
7
|
+
import type { FirstReplyTiming } from '@gestaltrun/dsh-cs-hosting/types';
|
|
8
|
+
import type { EvaluationMetricPack, EvaluationRole, EvaluationScript, EvaluationScriptSource, EvaluationTarget, EvaluationVariant, EvaluationWorld } from './types.ts';
|
|
9
|
+
/** Host-controlled execution limits. */
|
|
10
|
+
export interface Config {
|
|
11
|
+
/** Buyer message ceiling, including the last answered message; defaults to 150. */
|
|
12
|
+
readonly maxBuyerMessages?: number;
|
|
13
|
+
/** Real elapsed milliseconds allowed for one reply; defaults to 120000. */
|
|
14
|
+
readonly responseTimeoutMs?: number;
|
|
15
|
+
/** Currency used by this run's usage observations; defaults to CNY. */
|
|
16
|
+
readonly currency?: string;
|
|
17
|
+
/** Number of eligible cases receiving one planned human action; defaults to 3. */
|
|
18
|
+
readonly humanActionSampleSize?: number;
|
|
19
|
+
}
|
|
20
|
+
/** Concrete execution limits. */
|
|
21
|
+
export declare const Config: z.ZodObject<{
|
|
22
|
+
maxBuyerMessages: z.ZodDefault<z.ZodNumber>;
|
|
23
|
+
responseTimeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
24
|
+
currency: z.ZodDefault<z.ZodString>;
|
|
25
|
+
humanActionSampleSize: z.ZodDefault<z.ZodNumber>;
|
|
26
|
+
}, z.core.$strip>;
|
|
27
|
+
/** One buyer turn's monotonic first-reply observation. */
|
|
28
|
+
export interface EvaluationReplyObservation extends FirstReplyTiming {
|
|
29
|
+
readonly turn: number;
|
|
30
|
+
}
|
|
31
|
+
/** All domain choices needed for one run. */
|
|
32
|
+
export interface EvaluationRequest<Binding> {
|
|
33
|
+
readonly script: EvaluationScript;
|
|
34
|
+
readonly target: EvaluationTarget<Binding>;
|
|
35
|
+
readonly role: EvaluationRole<Binding>;
|
|
36
|
+
readonly world: EvaluationWorld<Binding>;
|
|
37
|
+
readonly metrics: EvaluationMetricPack;
|
|
38
|
+
/** Explicit Host-side plan for a controlled single-case run. */
|
|
39
|
+
readonly humanAction?: EvaluationHumanAction;
|
|
40
|
+
/** Optional product-owned capture; it does not change model requests or annotation injection. */
|
|
41
|
+
readonly purchaseIntent?: PurchaseIntentObserverFactory<Binding>;
|
|
42
|
+
/** Observe a contained step failure; absence leaves only the terminal record. */
|
|
43
|
+
readonly onFailure?: (error: unknown) => void;
|
|
44
|
+
/** Receives first-reply observations without extending the server case record. */
|
|
45
|
+
readonly onReplyTiming?: (value: EvaluationReplyObservation) => void;
|
|
46
|
+
}
|
|
47
|
+
/** One seeded run; roles never choose or emit its human actions. */
|
|
48
|
+
export interface EvaluationRunRequest<Binding> {
|
|
49
|
+
readonly cases: readonly Omit<EvaluationRequest<Binding>, 'humanAction'>[];
|
|
50
|
+
/** Replay seed; omission creates one that is retained in the returned record. */
|
|
51
|
+
readonly seed?: string;
|
|
52
|
+
/** Server sales-run options; every case script must declare its sales phase. */
|
|
53
|
+
readonly sales?: EvaluationSalesOptions;
|
|
54
|
+
}
|
|
55
|
+
/** Two annotation groups over the same scripts, seed and human-action plan. */
|
|
56
|
+
export interface EvaluationPairedRequest<Binding> {
|
|
57
|
+
readonly cases: readonly Omit<EvaluationRequest<Binding>, 'humanAction'>[];
|
|
58
|
+
/** Same-store category and optional calibration; pair and comparison ids are filled by the runner. */
|
|
59
|
+
readonly sales: Pick<EvaluationSalesOptions, 'category'> & Pick<Partial<EvaluationSalesOptions>, 'calibrationId'>;
|
|
60
|
+
readonly variants: readonly EvaluationVariant[];
|
|
61
|
+
/** Comparison seed; omission creates one shared by both groups. */
|
|
62
|
+
readonly seed?: string;
|
|
63
|
+
/** Annotation pair identity; omission creates one shared by both groups. */
|
|
64
|
+
readonly pairId?: string;
|
|
65
|
+
}
|
|
66
|
+
/** Both completed groups of one paired sales run. */
|
|
67
|
+
export interface EvaluationPairedResult {
|
|
68
|
+
readonly pairId: string;
|
|
69
|
+
readonly seed: string;
|
|
70
|
+
readonly groups: readonly EvaluationRunRecord[];
|
|
71
|
+
}
|
|
72
|
+
/** Execute the fixed buyer messages without generating or inferring new buyer knowledge. */
|
|
73
|
+
export declare const deterministicRole: EvaluationRole;
|
|
74
|
+
/** A runner owns limits; target and world instances own all external resources. */
|
|
75
|
+
export declare class EvaluationRunner {
|
|
76
|
+
private readonly config;
|
|
77
|
+
/** @param config - Validated deployment limits; omitted fields use the documented defaults. */
|
|
78
|
+
constructor(config?: Config);
|
|
79
|
+
/** @param request - Cases with optional script-authored human plans and an optional replay seed. @param signal - Run lifetime. @returns The seed, selected plans and completed case records; setup/cleanup errors reject. */
|
|
80
|
+
runCases<Binding>(request: EvaluationRunRequest<Binding>, signal?: AbortSignal): Promise<EvaluationRunRecord>;
|
|
81
|
+
/**
|
|
82
|
+
* Run the same scripts once under each annotation variant, sharing seed, pair id and human-action plan.
|
|
83
|
+
* @param request - Cases, sales category and exactly one annotation-on and one annotation-off variant.
|
|
84
|
+
* @param signal - Run lifetime.
|
|
85
|
+
* @returns Both group records whose `sales.annotation` marks the group.
|
|
86
|
+
*/
|
|
87
|
+
runPaired<Binding>(request: EvaluationPairedRequest<Binding>, signal?: AbortSignal): Promise<EvaluationPairedResult>;
|
|
88
|
+
/** @param source - Versioned script source. @param caseId - Requested case. @param adapters - Target, world, role and metric pack. @param signal - Caller lifetime. @returns The server-compatible version-1 record. */
|
|
89
|
+
runFrom<Binding>(source: EvaluationScriptSource, caseId: EvaluationCaseRecord['caseId'], adapters: Omit<EvaluationRequest<Binding>, 'script'>, signal?: AbortSignal): Promise<EvaluationCaseRecord>;
|
|
90
|
+
/**
|
|
91
|
+
* Run one case and join target work before removing its world.
|
|
92
|
+
* @param request - Frozen script and domain adapters.
|
|
93
|
+
* @param signal - Caller cancellation; omitted means no caller deadline.
|
|
94
|
+
* @returns A record with Host-derived termination and usage. Setup/cleanup failures reject; step failures produce error records.
|
|
95
|
+
*/
|
|
96
|
+
run<Binding>(request: EvaluationRequest<Binding>, signal?: AbortSignal): Promise<EvaluationCaseRecord>;
|
|
97
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Sales-run options and per-case results shared with server#164 (agent-evaluation-sales.ts). */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import type { EvaluationCaseRecord } from './record.ts';
|
|
4
|
+
import { EvaluationSalesOptions, EvaluationSalesRecord } from '@gestaltrun/dsh-cs-hosting/sales-case-contract';
|
|
5
|
+
/** Formal sales schemas; ordinary runs omit sales options and non-presales records omit intent. */
|
|
6
|
+
export { EvaluationSalesOptions, EvaluationSalesRecord } from '@gestaltrun/dsh-cs-hosting/sales-case-contract';
|
|
7
|
+
export type EvaluationSalesOptions = z.infer<typeof EvaluationSalesOptions>;
|
|
8
|
+
export type EvaluationSalesRecord = z.infer<typeof EvaluationSalesRecord>;
|
|
9
|
+
/** One simulated buyer turn paired with the recorded buyer wording and human reply. */
|
|
10
|
+
export declare const EvaluationSalesHumanComparison: z.ZodObject<{
|
|
11
|
+
turn: z.ZodNumber;
|
|
12
|
+
buyerMessageId: z.ZodString;
|
|
13
|
+
originalBuyerMessageId: z.ZodString;
|
|
14
|
+
originalHumanMessageId: z.ZodString;
|
|
15
|
+
}, z.core.$strict>;
|
|
16
|
+
export type EvaluationSalesHumanComparison = z.infer<typeof EvaluationSalesHumanComparison>;
|
|
17
|
+
/**
|
|
18
|
+
* Pair each answered simulated buyer message with the next original buyer message of identical wording
|
|
19
|
+
* that has an operator reply before the next original buyer message.
|
|
20
|
+
* @param messages - Simulated timeline.
|
|
21
|
+
* @param originalMessages - Historical human conversation.
|
|
22
|
+
* @returns One reference per simulated turn, in message order.
|
|
23
|
+
*/
|
|
24
|
+
export declare function humanComparisonsFor(messages: EvaluationCaseRecord['messages'], originalMessages: EvaluationCaseRecord['originalMessages']): EvaluationSalesHumanComparison[];
|
|
25
|
+
/**
|
|
26
|
+
* Build the case-level sales extension.
|
|
27
|
+
* @param phase - Script sales phase; non-presales cases never report a deal or comparisons.
|
|
28
|
+
* @param simulatedDeal - Whether the buyer mindset reached the ordering state.
|
|
29
|
+
* @param messages - Simulated timeline.
|
|
30
|
+
* @param originalMessages - Historical human conversation.
|
|
31
|
+
* @returns Validated sales record.
|
|
32
|
+
*/
|
|
33
|
+
export declare function salesRecordFor(phase: 'presales' | 'other', simulatedDeal: boolean, messages: EvaluationCaseRecord['messages'], originalMessages: EvaluationCaseRecord['originalMessages']): EvaluationSalesRecord;
|