@deepseek-ai/dsh-llm-replay 0.0.1-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.i18n.yaml +6 -0
- package/README.md +80 -0
- package/README.zh.md +80 -0
- package/lib/index.js +536 -0
- package/lib/invariant.js +23 -0
- package/lib/types/index.d.ts +259 -0
- package/lib/types/invariant.d.ts +16 -0
- package/package.json +48 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, DeepSeek
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
package/README.i18n.yaml
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each
|
|
2
|
+
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
|
3
|
+
# after editing either side, bring the other along and re-record with:
|
|
4
|
+
# pnpm run verify-translation-pairing --write packages/support/llm-replay/README.md
|
|
5
|
+
README.md: ae7cb5414a28bcf33f9878f6f02220861df9a0d5
|
|
6
|
+
README.zh.md: 2acc0ff8e8c011126452ba3458aaeb0800d9d8f3
|
package/README.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# @deepseek-ai/dsh-llm-replay
|
|
2
|
+
|
|
3
|
+
English | [中文](README.zh.md)
|
|
4
|
+
|
|
5
|
+
A replay LLM plugin for keyless snapshot tests. It yields model streams reconstructed from a recorded **session JSONL** fixture, so a test can boot the real agent against a fixed model transcript with no API key. With `providers` configured it registers a replay-only adapter whose catalog is available to scenarios that exercise model discovery; without `providers` it installs the catch-all `llm/stream` waterfall used by tests that do not need discovery.
|
|
6
|
+
|
|
7
|
+
Its consumers are the ACP and headless `stream-json` snapshot suites plus the Web browser e2e lane. Loader-driven suites mount this plugin in place of a real LLM adapter; the Web lane installs it directly to retain the teardown consumption handle.
|
|
8
|
+
|
|
9
|
+
## How the fixture works
|
|
10
|
+
|
|
11
|
+
The fixture IS the persisted session log (`<scenario>/session.jsonl`). Its `assistant/chunk` events carry every `StreamChunk`, so grouping them by `(turn, step)` reconstructs each agent-loop `stream()` call's chunk sequence. A successful compaction summarizer is logged differently: when `compact/summary` carries `llmStreamCall: true` and its complete `rawOutput`, replay reconstructs a canonical successful stream at that event's position using one `block-start`/`block-end` pair per block, the recorded usage when present, and a terminal `stop`. Exact provider delta partitioning is not part of the durable compaction result. `rawOutput` without the marker does not imply a local LLM call because template and remote summarizers may retain complete output without using this context's adapter.
|
|
12
|
+
|
|
13
|
+
Recording is therefore "run the real agent once and harvest the `.jsonl`", done by the snapshot harness — this plugin does not record. A fixture may carry its `request/header` content tokenized to `{{system}}`/`{{tools}}` (the harness pins that content in one scenario and scrubs the rest); replay is indifferent — derivation reads only `assistant/chunk` and `compact/summary` events plus the line-0 session header.
|
|
14
|
+
|
|
15
|
+
Two failure modes are not reconstructable from `assistant/chunk` alone — a pure throw before any chunk (e.g. an HTTP 401, where the log holds only a `turn/end {error}` and no chunks) and a cancel/hang (timing, not chunk content). A scenario that needs those supplies an optional sidecar (`<scenario>/replay.override.json`) that either replaces the derived script (a bare `ReplayEntry[]`) or augments it (`{ patches: [{ at, entry }] }`: keep every JSONL-derived call and swap the named 0-based call indexes; `at` equal to the derived length appends the retry attempt after an injected transient throw). Patch indexes must be unique. The override document, each patch and entry, and every chunk discriminant are validated when the file loads. A `hang` entry may name `readyFile`; replay writes that empty marker after its prefix chunks reach the loop and before it waits for cancellation, so an external driver can cancel deterministically without observing a presentation update.
|
|
16
|
+
|
|
17
|
+
A scripted string may embed `{{fromRequest:<regex>}}` to fill a value no static sidecar can know — for example a randomly minted goal id the model must echo back into `update_goal`. At stream time every placeholder resolves against the live request: the corpus is every string leaf of the request messages joined by newlines, the pattern's LAST corpus match wins, and its first capture group (or the whole match without one) substitutes in place. A pattern that matches nothing, an invalid pattern, and an unterminated placeholder each fail loud. The last two braces of a consecutive `}` run terminate the placeholder, so a pattern may end with a brace quantifier (`[0-9a-f]{4}`) but cannot contain `}}` followed by further pattern content. Resolution applies to every scripted entry, including ones derived from the recorded JSONL — a recorded fixture whose text legitimately contains the literal marker must be expressed through a sidecar without it.
|
|
18
|
+
|
|
19
|
+
## Nested agents: per-session keying
|
|
20
|
+
|
|
21
|
+
A scenario where a parent agent delegates to in-process subagents records more than one log: the parent (`session.jsonl`) plus one per child (`session.1.jsonl`, …). Each agent runs as its own `Session` on the same context, so replay must serve each one its own script.
|
|
22
|
+
|
|
23
|
+
Replay keys every call by its calling session id (`GenerateOptions.sessionId`, stamped by the agent loop). Live session ids are freshly random each run and never equal the recorded ones, so a live session binds to a recorded script by **first-call order**: scripts are ordered by header `createdAt` (parent first — it streams before it can delegate), and the first live session to make any call claims the first script, the next new session the next, and so on. Each session then advances its own cursor. A call with no `sessionId` is one anonymous session bound to the primary script. More distinct live sessions than recorded scripts fails loud.
|
|
24
|
+
|
|
25
|
+
## Config
|
|
26
|
+
|
|
27
|
+
| Key | Type | Default | Notes |
|
|
28
|
+
|---|---|---|---|
|
|
29
|
+
| `file` | string | `$DSH_SNAPSHOT_FILE` | Path to the primary (parent) `session.jsonl` fixture. Required (config or env). |
|
|
30
|
+
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | Optional `ReplayOverrideDoc` sidecar for the primary session: a bare `ReplayEntry[]` replaces its derived script, while `{ patches }` augments it by call index. |
|
|
31
|
+
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES` (path-delimited) | Recorded subagent child-session logs for a nested scenario; empty for a single-session scenario. |
|
|
32
|
+
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Each provider may set `retryPolicy`, and each model may publish `contextWindow`; configured routes dispatch through the replay adapter and never perform provider I/O. |
|
|
33
|
+
| `paceMs` | number | — (burst) | Optional per-chunk delay in ms so downstream transports (e.g. the web SSE mux observed by a real browser) see genuinely incremental delivery. A realism knob only — tests must not depend on it for correctness. Non-negative integer; abort during a pace wait cancels the stream promptly. |
|
|
34
|
+
|
|
35
|
+
```yaml
|
|
36
|
+
- id: llm-replay
|
|
37
|
+
name: '@deepseek-ai/dsh-llm-replay'
|
|
38
|
+
config:
|
|
39
|
+
providers:
|
|
40
|
+
- id: deepseek-official
|
|
41
|
+
name: DeepSeek
|
|
42
|
+
retryPolicy:
|
|
43
|
+
mode: normal
|
|
44
|
+
backoff:
|
|
45
|
+
initialDelayMs: 1
|
|
46
|
+
maxDelayMs: 1
|
|
47
|
+
jitterRatio: 0
|
|
48
|
+
models:
|
|
49
|
+
- id: deepseek-v4-flash
|
|
50
|
+
contextWindow: 128000
|
|
51
|
+
- id: deepseek-v4-pro
|
|
52
|
+
# file/overrideFile/childFiles default to $DSH_SNAPSHOT_FILE /
|
|
53
|
+
# $DSH_SNAPSHOT_OVERRIDE / $DSH_SNAPSHOT_CHILD_FILES, set by the snapshot
|
|
54
|
+
# harness per scenario.
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Exports
|
|
58
|
+
|
|
59
|
+
- `installLlmReplay(ctx, config)` — install the configured replay adapter or catch-all `llm/stream` listener; returns a `ReplayHandle` (`dispose()` for HMR safety plus `assertConsumed()`, the teardown check that every recorded script bound to a live session and every bound cursor drained — turning a scenario that silently drove fewer model calls than recorded into a crisp diagnostic). Use this in tests to drive replay without the Loader or env vars.
|
|
60
|
+
- `loadSessionScripts(config)` — resolve the ordered `SessionScript[]` (primary + children) for a scenario, ready to bind to live sessions in first-call order.
|
|
61
|
+
- `loadReplayScript(config)` — resolve the `ReplayEntry[]` for the primary session only (validated sidecar replacement/patches if present, else derived from the JSONL; fail-loud if the fixture is missing).
|
|
62
|
+
- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` / `resolveScriptedEntry(entry, messages)` — the pure helpers that turn ordinary loop chunks and explicitly marked local compaction outputs in a recorded session log into a script, read its header `id`/`createdAt`, and resolve `{{fromRequest:...}}` placeholders against one live request. A derived assistant group must end in a `finish` chunk; a group without one is the fingerprint of a thrown `stream()` and must instead be expressed via an override sidecar.
|
|
63
|
+
- Types `ReplayEntry` / `ReplayOverrideDoc` / `ReplayOverridePatch` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `ReplayHandle` / `Config`.
|
|
64
|
+
|
|
65
|
+
## Plugin export shape
|
|
66
|
+
|
|
67
|
+
Named `name` / `inject` / `Config` / `apply`, with **no default export**: the cordis Loader's `unwrapExports` does `exports.default ?? exports`, so a stray default would collapse the module to the bare function and drop the `inject` namespace (see [docs/postmortem/0001](../../../docs/postmortem/0001-acp-default-export-drops-inject.md)).
|
|
68
|
+
|
|
69
|
+
## Model Experience
|
|
70
|
+
|
|
71
|
+
None, as this keyless test adapter sends no request to a provider model; it only replays recorded assistant chunks into the test loop.
|
|
72
|
+
|
|
73
|
+
#### KV Cache effect
|
|
74
|
+
|
|
75
|
+
None; this package neither assembles nor sends a provider request.
|
|
76
|
+
|
|
77
|
+
## Known Limitations and Deferred Work
|
|
78
|
+
|
|
79
|
+
- **First-call-order script binding assumes sequential delegation** — a cut that runs sibling subagents concurrently would bind live sessions to recorded scripts non-deterministically; a stronger keying is deferred until such a scenario exists (`XXX(concurrent-subagents)`).
|
|
80
|
+
- **Only ordinary loop chunks and marked local compaction outputs are derivable** — a pure pre-chunk throw, a cancel/hang, or an unmarked external summarizer call needs the `replay.override.json` sidecar. Replacement and patch forms affect only the primary session; child scripts still derive from their logs.
|
package/README.zh.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# @deepseek-ai/dsh-llm-replay
|
|
2
|
+
|
|
3
|
+
[English](README.md) | 中文
|
|
4
|
+
|
|
5
|
+
用于无密钥快照测试的 LLM(大语言模型)回放插件。它根据已记录的**会话 JSONL** fixture(测试前置数据)重建模型流,使测试无需 API 密钥即可针对固定的模型 transcript(文本记录)启动真实 agent(智能体)。配置 `providers` 后,它会注册仅用于回放的适配器,其模型目录可供测试模型发现功能的场景使用;未配置 `providers` 时,它会安装无需模型发现功能的测试所用 catch-all `llm/stream` waterfall(瀑布式事件)。
|
|
6
|
+
|
|
7
|
+
其消费方包括 ACP(Agent Client Protocol)与 headless `stream-json` 快照套件,以及 Web 浏览器 e2e 流水线。Loader 驱动的套件使用此插件替代真实 LLM 适配器;Web 流水线直接安装它,以保留清理阶段的消费检查句柄。
|
|
8
|
+
|
|
9
|
+
## fixture 的工作方式
|
|
10
|
+
|
|
11
|
+
fixture 就是持久化的会话日志(`<scenario>/session.jsonl`)。其 `assistant/chunk` 事件包含每个 `StreamChunk`,因此按 `(turn, step)` 分组即可重建每次 agent loop(智能体循环)的 `stream()` 调用的分片序列。压缩(compaction)摘要器成功时,日志记录方式有所不同:当 `compact/summary` 携带 `llmStreamCall: true` 和完整的 `rawOutput` 时,回放会在该事件的位置重建一条规范成功流,其中每个块各使用一对 `block-start`/`block-end`,带上已记录的用量(如有),并以 `stop` 终止。提供方增量的精确切分不属于持久压缩结果。不带该标记的 `rawOutput` 并不意味着发生了本地 LLM 调用,因为模板摘要器和远程摘要器即使未使用此上下文的适配器,也可能保留完整输出。
|
|
12
|
+
|
|
13
|
+
因此,录制就是「运行一次真实 agent 并收集 `.jsonl`」,由快照 harness 完成;该插件本身不录制。fixture 的 `request/header` 内容可能被标记化为 `{{system}}`/`{{tools}}`(harness 会在一个场景中固定该内容,并清除其余场景中的内容);回放不受影响,因为派生过程只读取 `assistant/chunk` 和 `compact/summary` 事件以及第 0 行的会话 header。
|
|
14
|
+
|
|
15
|
+
有两种失败模式无法仅根据 `assistant/chunk` 重建:在产生任何分片前直接抛出异常(例如 HTTP 401,此时日志只有 `turn/end {error}` 而没有分片),以及取消或挂起(差异在时序,而非分片内容)。需要这些行为的场景可提供伴随文件(`<scenario>/replay.override.json`):它可以替换派生脚本(裸 `ReplayEntry[]`),也可以增补派生脚本(`{ patches: [{ at, entry }] }`:保留所有从 JSONL 派生的调用,只替换指定的从 0 开始计数的调用索引;当 `at` 等于派生长度时,则在注入瞬态异常后的重试位置追加一次调用)。补丁索引不得重复。文件加载时会校验覆写文档、每个补丁和条目,以及每个分片的判别标签。`hang` 条目可以指定 `readyFile`;当前缀分片到达循环后、开始等待取消前,回放会写入这个空标记,使外部驱动程序无需观察展示层更新即可确定性地取消。
|
|
16
|
+
|
|
17
|
+
脚本字符串可以内嵌 `{{fromRequest:<regex>}}`,用来填入静态伴随文件不可能预知的值——例如模型必须原样回填到 `update_goal` 的随机生成 goal id。回放时每个占位符针对实时请求解析:语料是请求消息的所有字符串叶子按换行拼接的结果,取该模式在语料中的最后一次匹配,用其第一个捕获组(无捕获组时用整个匹配)原位替换。模式匹配不到内容、模式非法、占位符未闭合都会明确报错。连续右花括号串的最后两个花括号才是占位符结束符,因此模式可以以花括号量词收尾(如 `[0-9a-f]{4}`),但不能在 `}}` 之后还有后续模式内容。解析作用于所有脚本条目,包括从已记录 JSONL 派生的条目——若录制文本本身合法地含有该字面量标记,需改用不含标记的伴随文件表达。
|
|
18
|
+
|
|
19
|
+
## 嵌套 agent:每会话键控
|
|
20
|
+
|
|
21
|
+
父 agent 委托给进程内 subagent 的场景会记录多个日志:父会话使用 `session.jsonl`,每个子会话各使用一个日志(`session.1.jsonl` 等)。每个 agent 都在同一上下文中作为独立的 `Session` 运行,因此回放必须为每个 agent 提供各自的脚本。
|
|
22
|
+
|
|
23
|
+
回放根据发起调用的会话 id 为每次调用建立键(`GenerateOptions.sessionId` 由 agent loop 写入)。实时会话 id 每次运行时都会重新随机生成,绝不会等于记录中的 id,因此实时会话按**首次调用顺序**绑定到已记录脚本:脚本按 header 中的 `createdAt` 排序(父会话在前,因为它必须先开始流式输出才能委托);第一个发起调用的实时会话取得第一个脚本,下一个新会话取得下一个脚本,以此类推。此后每个会话分别推进自己的游标。没有 `sessionId` 的调用视为一个绑定主脚本的匿名会话。不同实时会话的数量超过已记录脚本数时会明确报错。
|
|
24
|
+
|
|
25
|
+
## 配置
|
|
26
|
+
|
|
27
|
+
| 键 | 类型 | 默认值 | 说明 |
|
|
28
|
+
|---|---|---|---|
|
|
29
|
+
| `file` | string | `$DSH_SNAPSHOT_FILE` | 主(父)`session.jsonl` fixture 的路径。必需(配置或 env)。 |
|
|
30
|
+
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | 主会话的可选 `ReplayOverrideDoc` 伴随文件:裸 `ReplayEntry[]` 替换其派生脚本,`{ patches }` 则按调用索引增补该脚本。 |
|
|
31
|
+
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES`(以路径分隔符分隔) | 嵌套场景中已记录的 subagent 子会话日志;单会话场景为空。 |
|
|
32
|
+
| `providers` | `ReplayProviderConfig[]` | 无 | 可选的仅回放提供方和模型目录。每个提供方可以设置 `retryPolicy`,每个模型可以发布 `contextWindow`;已配置路由通过回放适配器分派,绝不执行提供方 I/O。 |
|
|
33
|
+
| `paceMs` | number | 无(突发) | 可选的每分片毫秒延迟,使下游传输(例如真实浏览器观察到的 Web SSE(Server-Sent Events)多路复用器)看到真正的增量传递。它只是仿真开关,测试不得依赖它保证正确性。值必须是非负整数;pace 等待期间中止会迅速取消流。 |
|
|
34
|
+
|
|
35
|
+
```yaml
|
|
36
|
+
- id: llm-replay
|
|
37
|
+
name: '@deepseek-ai/dsh-llm-replay'
|
|
38
|
+
config:
|
|
39
|
+
providers:
|
|
40
|
+
- id: deepseek-official
|
|
41
|
+
name: DeepSeek
|
|
42
|
+
retryPolicy:
|
|
43
|
+
mode: normal
|
|
44
|
+
backoff:
|
|
45
|
+
initialDelayMs: 1
|
|
46
|
+
maxDelayMs: 1
|
|
47
|
+
jitterRatio: 0
|
|
48
|
+
models:
|
|
49
|
+
- id: deepseek-v4-flash
|
|
50
|
+
contextWindow: 128000
|
|
51
|
+
- id: deepseek-v4-pro
|
|
52
|
+
# file/overrideFile/childFiles default to $DSH_SNAPSHOT_FILE /
|
|
53
|
+
# $DSH_SNAPSHOT_OVERRIDE / $DSH_SNAPSHOT_CHILD_FILES, set by the snapshot
|
|
54
|
+
# harness per scenario.
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## 导出项
|
|
58
|
+
|
|
59
|
+
- `installLlmReplay(ctx, config)`:安装已配置回放适配器或 catch-all `llm/stream` 监听器;返回 `ReplayHandle`(包含用于保证 HMR(热模块替换)安全的 `dispose()`,以及清理阶段执行的 `assertConsumed()` 检查;后者确保每个已记录脚本都绑定到实时会话,且每个已绑定游标都已耗尽,从而将场景静默驱动的模型调用少于记录数转换为明确诊断)。在测试中使用它,可以不通过 Loader 或 env var 驱动回放。
|
|
60
|
+
- `loadSessionScripts(config)`:解析场景中有序的 `SessionScript[]`(主会话 + 子会话),准备按首次调用顺序绑定到实时会话。
|
|
61
|
+
- `loadReplayScript(config)`:只解析主会话的 `ReplayEntry[]`(如果伴随文件存在,则使用经校验的替换或补丁;否则从 JSONL 派生;fixture 缺失时明确报错)。
|
|
62
|
+
- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` / `resolveScriptedEntry(entry, messages)`:将已记录会话日志中的普通 loop 分片和显式标记的本地压缩输出转换为脚本、读取其 header `id`/`createdAt`、并针对单次实时请求解析 `{{fromRequest:...}}` 占位符的纯辅助工具。派生的 assistant 分组必须以 `finish` 分片结束;没有该分片的分组是 `stream()` 抛出异常的指纹,必须改用 override 伴随文件表达。
|
|
63
|
+
- 类型 `ReplayEntry` / `ReplayOverrideDoc` / `ReplayOverridePatch` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `ReplayHandle` / `Config`。
|
|
64
|
+
|
|
65
|
+
## 插件导出形态
|
|
66
|
+
|
|
67
|
+
命名导出 `name` / `inject` / `Config` / `apply`,且**没有默认导出**:Cordis Loader 的 `unwrapExports` 执行 `exports.default ?? exports`,因此意外的默认导出会将模块折叠为纯函数,并丢弃 `inject` 命名空间(见 [docs/postmortem/0001](../../../docs/postmortem/0001-acp-default-export-drops-inject.md))。
|
|
68
|
+
|
|
69
|
+
## 模型体验
|
|
70
|
+
|
|
71
|
+
无。该无密钥测试适配器不向提供方模型发送请求,只将已记录 assistant 分片回放到测试 loop 中。
|
|
72
|
+
|
|
73
|
+
#### KV Cache 影响
|
|
74
|
+
|
|
75
|
+
无;该包既不组装也不发送提供方请求。
|
|
76
|
+
|
|
77
|
+
## 已知限制与暂缓事项
|
|
78
|
+
|
|
79
|
+
- **首次调用顺序脚本绑定假设串行委托**:并发运行同级 subagent 的 cut 会非确定性地将实时会话绑定到已记录脚本;在这种场景出现前暂不实现更强的键控(`XXX(concurrent-subagents)`)。
|
|
80
|
+
- **只有普通 loop 分片和带标记的本地压缩输出才能派生**:在产生分片前直接抛出异常、取消/挂起,或未标记的外部摘要器调用场景需要 `replay.override.json` 伴随文件。替换和补丁两种形式都只影响主会话;子会话脚本仍从各自日志派生。
|
package/lib/index.js
ADDED
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { delimiter } from "node:path";
|
|
3
|
+
import { decodeStorageRecord } from "@deepseek-ai/dsh-session";
|
|
4
|
+
import { LlmAdapter, LlmError, ReasoningEffortId, assertNever, resolveRetryPolicy } from "@deepseek-ai/dsh-llm";
|
|
5
|
+
//#region lib/types/index.js
|
|
6
|
+
/**
|
|
7
|
+
* Keyless snapshot-test LLM replay. It derives one model-call script per
|
|
8
|
+
* recorded session from `assistant/chunk` events and explicitly marked local
|
|
9
|
+
* compaction calls, then binds fresh live sessions to parent/child scripts by
|
|
10
|
+
* first-call order. Throw and hang cases require an explicit override because
|
|
11
|
+
* a session log cannot reconstruct them alone.
|
|
12
|
+
* @module @deepseek-ai/dsh-llm-replay
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Parse a session `.jsonl` buffer into its event list. Line 0 is the session
|
|
16
|
+
* header (a `{type:'session',…}` record), every subsequent non-empty line is a
|
|
17
|
+
* {@link SessionEvent} or a packed chunk row (expanded back into its events, so
|
|
18
|
+
* a fixture recorded with `packChunks` on derives the same script). The header
|
|
19
|
+
* is skipped; malformed lines fail loud.
|
|
20
|
+
* @param text - the raw `.jsonl` file contents.
|
|
21
|
+
* @returns every event after the header, in log order.
|
|
22
|
+
*/
|
|
23
|
+
function parseSessionLog(text) {
|
|
24
|
+
const lines = text.split("\n").filter((line) => line.trim().length > 0);
|
|
25
|
+
const events = [];
|
|
26
|
+
for (let i = 1; i < lines.length; i++) events.push(...decodeStorageRecord(JSON.parse(lines[i])));
|
|
27
|
+
return events;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Read replay identity, ordering, and fork-seed facts from the JSONL header.
|
|
31
|
+
*
|
|
32
|
+
* @param text - the raw `.jsonl` file contents (only the header line is read).
|
|
33
|
+
* @returns the header's `id`, `createdAt`, and `seedLength`, defaulted when absent.
|
|
34
|
+
*/
|
|
35
|
+
function parseSessionHeader(text) {
|
|
36
|
+
const firstLine = text.split("\n").find((line) => line.trim().length > 0) ?? "{}";
|
|
37
|
+
const parsed = JSON.parse(firstLine);
|
|
38
|
+
return {
|
|
39
|
+
id: typeof parsed.id === "string" ? parsed.id : "",
|
|
40
|
+
createdAt: typeof parsed.createdAt === "number" ? parsed.createdAt : 0,
|
|
41
|
+
seedLength: typeof parsed.seedLength === "number" ? parsed.seedLength : 0
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Reconstruct the per-`stream()` replay script from a recorded session log.
|
|
46
|
+
*
|
|
47
|
+
* Splits `assistant/chunk` events at every `finish`, using turn and step changes
|
|
48
|
+
* to detect an unterminated prior call. A `compact/summary` explicitly marked
|
|
49
|
+
* as one local LLM-stream call becomes a canonical successful stream from its
|
|
50
|
+
* complete `rawOutput` at the summary's log position. A
|
|
51
|
+
* missing assistant terminator means the live stream threw, so derivation
|
|
52
|
+
* rejects and the scenario must provide an explicit override. Multiple calls
|
|
53
|
+
* may share one turn and step when the loop retries.
|
|
54
|
+
* @param events - the recorded session's events.
|
|
55
|
+
* @returns one `chunks` entry per recorded model call, in call order.
|
|
56
|
+
*/
|
|
57
|
+
function deriveReplayScript(events) {
|
|
58
|
+
const script = [];
|
|
59
|
+
let currentKey;
|
|
60
|
+
let current = [];
|
|
61
|
+
const close = (key, chunks) => {
|
|
62
|
+
if (chunks.length === 0) return;
|
|
63
|
+
if (chunks[chunks.length - 1]?.type !== "finish") throw new Error(`llm-replay: model call ${key} ended without a finish chunk (a thrown stream); this scenario needs a replay.override.json sidecar`);
|
|
64
|
+
script.push({
|
|
65
|
+
kind: "chunks",
|
|
66
|
+
chunks
|
|
67
|
+
});
|
|
68
|
+
};
|
|
69
|
+
for (const event of events) {
|
|
70
|
+
if (event.type === "compact/summary") {
|
|
71
|
+
close(currentKey, current);
|
|
72
|
+
currentKey = void 0;
|
|
73
|
+
current = [];
|
|
74
|
+
const persisted = event.data;
|
|
75
|
+
if (persisted.llmStreamCall === true) {
|
|
76
|
+
if (persisted.rawOutput === void 0) throw new Error("llm-replay: compact/summary marks an LLM stream call without rawOutput");
|
|
77
|
+
const chunks = [];
|
|
78
|
+
for (const [index, block] of persisted.rawOutput.entries()) {
|
|
79
|
+
chunks.push({
|
|
80
|
+
type: "block-start",
|
|
81
|
+
index,
|
|
82
|
+
blockType: block.type
|
|
83
|
+
});
|
|
84
|
+
chunks.push({
|
|
85
|
+
type: "block-end",
|
|
86
|
+
index,
|
|
87
|
+
block
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
if (persisted.usage !== void 0) chunks.push({
|
|
91
|
+
type: "usage",
|
|
92
|
+
usage: persisted.usage
|
|
93
|
+
});
|
|
94
|
+
chunks.push({
|
|
95
|
+
type: "finish",
|
|
96
|
+
reason: { kind: "stop" }
|
|
97
|
+
});
|
|
98
|
+
script.push({
|
|
99
|
+
kind: "chunks",
|
|
100
|
+
chunks
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (event.type !== "assistant/chunk") continue;
|
|
106
|
+
const { turn, step, chunk } = event.data;
|
|
107
|
+
const key = `${turn}/${step}`;
|
|
108
|
+
if (current.length > 0 && key !== currentKey) close(currentKey, current);
|
|
109
|
+
if (current.length === 0) currentKey = key;
|
|
110
|
+
current.push(chunk);
|
|
111
|
+
if (chunk.type === "finish") {
|
|
112
|
+
close(currentKey, current);
|
|
113
|
+
currentKey = void 0;
|
|
114
|
+
current = [];
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
close(currentKey, current);
|
|
118
|
+
return script;
|
|
119
|
+
}
|
|
120
|
+
const REPLAY_CHUNK_TYPES = new Set([
|
|
121
|
+
"block-start",
|
|
122
|
+
"text-delta",
|
|
123
|
+
"reasoning-delta",
|
|
124
|
+
"tool-call-delta",
|
|
125
|
+
"block-end",
|
|
126
|
+
"usage",
|
|
127
|
+
"finish"
|
|
128
|
+
]);
|
|
129
|
+
const FROM_REQUEST_OPEN = "{{fromRequest:";
|
|
130
|
+
const FROM_REQUEST_CLOSE = "}}";
|
|
131
|
+
/** Collect every string leaf of one JSON-compatible value, in traversal order. */
|
|
132
|
+
function collectStrings(value, out) {
|
|
133
|
+
if (typeof value === "string") {
|
|
134
|
+
out.push(value);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (Array.isArray(value)) {
|
|
138
|
+
for (const item of value) collectStrings(item, out);
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
if (value !== null && typeof value === "object") for (const item of Object.values(value)) collectStrings(item, out);
|
|
142
|
+
}
|
|
143
|
+
/** Resolve one placeholder pattern against the request corpus; the LAST match wins. */
|
|
144
|
+
function resolveFromRequest(pattern, corpus) {
|
|
145
|
+
let regex;
|
|
146
|
+
try {
|
|
147
|
+
regex = new RegExp(pattern, "g");
|
|
148
|
+
} catch (error) {
|
|
149
|
+
throw new Error(`llm-replay: fromRequest has an invalid pattern ${JSON.stringify(pattern)}: ${String(error)}`);
|
|
150
|
+
}
|
|
151
|
+
let last;
|
|
152
|
+
for (const match of corpus.matchAll(regex)) last = match;
|
|
153
|
+
if (last === void 0) throw new Error(`llm-replay: fromRequest pattern ${JSON.stringify(pattern)} matched nothing in the request`);
|
|
154
|
+
return last[1] ?? last[0];
|
|
155
|
+
}
|
|
156
|
+
/** Replace every `{{fromRequest:<pattern>}}` occurrence in one scripted string. */
|
|
157
|
+
function substituteString(text, corpus) {
|
|
158
|
+
let result = "";
|
|
159
|
+
let cursor = 0;
|
|
160
|
+
while (true) {
|
|
161
|
+
const open = text.indexOf(FROM_REQUEST_OPEN, cursor);
|
|
162
|
+
if (open === -1) return result + text.slice(cursor);
|
|
163
|
+
let close = text.indexOf(FROM_REQUEST_CLOSE, open + 14);
|
|
164
|
+
if (close === -1) throw new Error(`llm-replay: fromRequest placeholder is unterminated in ${JSON.stringify(text)}`);
|
|
165
|
+
while (text[close + 2] === "}") close += 1;
|
|
166
|
+
const pattern = text.slice(open + 14, close);
|
|
167
|
+
result += text.slice(cursor, open) + resolveFromRequest(pattern, corpus);
|
|
168
|
+
cursor = close + 2;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/** Deep-copy one JSON-compatible value with scripted placeholders resolved. */
|
|
172
|
+
function substituteValue(value, corpus) {
|
|
173
|
+
if (typeof value === "string") return value.includes(FROM_REQUEST_OPEN) ? substituteString(value, corpus) : value;
|
|
174
|
+
if (Array.isArray(value)) return value.map((item) => substituteValue(item, corpus));
|
|
175
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, substituteValue(item, corpus)]));
|
|
176
|
+
return value;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Resolve every `{{fromRequest:<regex>}}` placeholder in one scripted entry
|
|
180
|
+
* against the live request. The corpus is every string leaf of the request
|
|
181
|
+
* messages joined by newlines; the pattern's LAST corpus match wins and its
|
|
182
|
+
* first capture group (or, without one, the whole match) substitutes in place.
|
|
183
|
+
* Scenario sidecars use this to script arguments no static file can know,
|
|
184
|
+
* such as a randomly minted goal id the model must echo back. A pattern that
|
|
185
|
+
* matches nothing, an invalid pattern, and an unterminated placeholder each
|
|
186
|
+
* fail loud. The last two braces of a consecutive `}` run terminate the
|
|
187
|
+
* placeholder, so a pattern may end with a brace quantifier but cannot
|
|
188
|
+
* contain `}}` followed by further pattern content. Derived entries pass
|
|
189
|
+
* through the same resolution as sidecar entries.
|
|
190
|
+
* @param entry - the scripted entry about to replay.
|
|
191
|
+
* @param messages - the live request messages searched by the placeholders.
|
|
192
|
+
* @returns the entry itself when no placeholder appears, else a resolved deep copy.
|
|
193
|
+
*/
|
|
194
|
+
function resolveScriptedEntry(entry, messages) {
|
|
195
|
+
if (!JSON.stringify(entry).includes(FROM_REQUEST_OPEN)) return entry;
|
|
196
|
+
const leaves = [];
|
|
197
|
+
collectStrings(messages, leaves);
|
|
198
|
+
return substituteValue(entry, leaves.join("\n"));
|
|
199
|
+
}
|
|
200
|
+
function isRecord(value) {
|
|
201
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
202
|
+
}
|
|
203
|
+
function hasExactKeys(value, keys) {
|
|
204
|
+
return Object.keys(value).length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
|
205
|
+
}
|
|
206
|
+
function invalidOverride(file, location, detail) {
|
|
207
|
+
throw new Error(`llm-replay: invalid override ${file}: ${location} ${detail}`);
|
|
208
|
+
}
|
|
209
|
+
function readChunks(value, file, location) {
|
|
210
|
+
if (!Array.isArray(value)) invalidOverride(file, location, "chunks must be an array");
|
|
211
|
+
for (const [index, chunk] of value.entries()) if (!isRecord(chunk) || typeof chunk["type"] !== "string" || !REPLAY_CHUNK_TYPES.has(chunk["type"])) invalidOverride(file, `${location}.chunks[${index}]`, "must have a known StreamChunk type");
|
|
212
|
+
return value;
|
|
213
|
+
}
|
|
214
|
+
function readReplayEntry(value, file, location) {
|
|
215
|
+
if (!isRecord(value)) invalidOverride(file, location, "must be an object");
|
|
216
|
+
switch (value["kind"]) {
|
|
217
|
+
case "chunks":
|
|
218
|
+
if (!hasExactKeys(value, ["kind", "chunks"])) invalidOverride(file, location, "has invalid chunks-entry fields");
|
|
219
|
+
return {
|
|
220
|
+
kind: "chunks",
|
|
221
|
+
chunks: readChunks(value["chunks"], file, location)
|
|
222
|
+
};
|
|
223
|
+
case "throw":
|
|
224
|
+
if (!hasExactKeys(value, [
|
|
225
|
+
"kind",
|
|
226
|
+
"chunks",
|
|
227
|
+
"message",
|
|
228
|
+
"code"
|
|
229
|
+
])) invalidOverride(file, location, "has invalid throw-entry fields");
|
|
230
|
+
if (typeof value["message"] !== "string" || value["message"].length === 0) invalidOverride(file, location, "message must be a non-empty string");
|
|
231
|
+
if (typeof value["code"] !== "string" || value["code"].length === 0) invalidOverride(file, location, "code must be a non-empty string");
|
|
232
|
+
return {
|
|
233
|
+
kind: "throw",
|
|
234
|
+
chunks: readChunks(value["chunks"], file, location),
|
|
235
|
+
message: value["message"],
|
|
236
|
+
code: value["code"]
|
|
237
|
+
};
|
|
238
|
+
case "hang": {
|
|
239
|
+
const readyFile = value["readyFile"];
|
|
240
|
+
if (!hasExactKeys(value, readyFile === void 0 ? ["kind"] : ["kind", "readyFile"])) invalidOverride(file, location, "has invalid hang-entry fields");
|
|
241
|
+
if (readyFile !== void 0 && (typeof readyFile !== "string" || readyFile.length === 0)) invalidOverride(file, location, "readyFile must be a non-empty string");
|
|
242
|
+
return {
|
|
243
|
+
kind: "hang",
|
|
244
|
+
...readyFile === void 0 ? {} : { readyFile }
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
default: return invalidOverride(file, location, `has unknown kind ${JSON.stringify(value["kind"])}`);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
function readOverrideDoc(value, file) {
|
|
251
|
+
if (Array.isArray(value)) return value.map((entry, index) => readReplayEntry(entry, file, `entry ${index}`));
|
|
252
|
+
if (!isRecord(value) || !hasExactKeys(value, ["patches"]) || !Array.isArray(value["patches"])) return invalidOverride(file, "document", "must be a ReplayEntry[] or { patches: [...] }");
|
|
253
|
+
return { patches: value["patches"].map((value, index) => {
|
|
254
|
+
const location = `patch ${index}`;
|
|
255
|
+
if (!isRecord(value) || !hasExactKeys(value, ["at", "entry"])) return invalidOverride(file, location, "must contain exactly at and entry");
|
|
256
|
+
const at = value["at"];
|
|
257
|
+
if (typeof at !== "number" || !Number.isSafeInteger(at) || at < 0) return invalidOverride(file, location, "at must be a non-negative safe integer");
|
|
258
|
+
return {
|
|
259
|
+
at,
|
|
260
|
+
entry: readReplayEntry(value["entry"], file, `${location}.entry`)
|
|
261
|
+
};
|
|
262
|
+
}) };
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Load the PRIMARY session's replay script: the sidecar override when present
|
|
266
|
+
* (whole-script replacement or `{ patches }` augmentation over the derived
|
|
267
|
+
* script), else the script derived from the session JSONL (fail-loud when the
|
|
268
|
+
* fixture is missing).
|
|
269
|
+
* @param config - the fixture paths; only `file` and `overrideFile` are consulted.
|
|
270
|
+
* @returns the resolved primary-session script.
|
|
271
|
+
*/
|
|
272
|
+
function loadReplayScript(config) {
|
|
273
|
+
if (config.overrideFile !== void 0 && existsSync(config.overrideFile)) {
|
|
274
|
+
const doc = readOverrideDoc(JSON.parse(readFileSync(config.overrideFile, "utf8")), config.overrideFile);
|
|
275
|
+
if (Array.isArray(doc)) return doc;
|
|
276
|
+
const script = deriveScriptFromFile(config.file);
|
|
277
|
+
const derivedLength = script.length;
|
|
278
|
+
const seenIndexes = /* @__PURE__ */ new Set();
|
|
279
|
+
for (const patch of doc.patches) {
|
|
280
|
+
if (patch.at > derivedLength) throw new Error(`llm-replay: override patch index ${String(patch.at)} out of range (derived script has ${derivedLength} call(s); == length appends): ${config.overrideFile}`);
|
|
281
|
+
if (seenIndexes.has(patch.at)) throw new Error(`llm-replay: duplicate override patch index ${patch.at}: ${config.overrideFile}`);
|
|
282
|
+
seenIndexes.add(patch.at);
|
|
283
|
+
script[patch.at] = patch.entry;
|
|
284
|
+
}
|
|
285
|
+
return script;
|
|
286
|
+
}
|
|
287
|
+
return deriveScriptFromFile(config.file);
|
|
288
|
+
}
|
|
289
|
+
/** Derive the primary script from the session JSONL, failing loud on a missing fixture. */
|
|
290
|
+
function deriveScriptFromFile(file) {
|
|
291
|
+
if (!existsSync(file)) throw new Error(`llm-replay: fixture not found: ${file} — run \`pnpm run test:snapshot:record\` first`);
|
|
292
|
+
return deriveReplayScript(parseSessionLog(readFileSync(file, "utf8")));
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Load the primary and child scripts in bind order. Child derivation begins at
|
|
296
|
+
* `seedLength` so inherited parent chunks are never replayed as child calls.
|
|
297
|
+
*
|
|
298
|
+
* @param config - the fixture paths: the primary log plus any recorded child logs.
|
|
299
|
+
* @returns the primary script first, then the child scripts in bind order.
|
|
300
|
+
*/
|
|
301
|
+
function loadSessionScripts(config) {
|
|
302
|
+
const primaryEntries = loadReplayScript(config);
|
|
303
|
+
const primaryHeader = existsSync(config.file) ? parseSessionHeader(readFileSync(config.file, "utf8")) : {
|
|
304
|
+
id: "",
|
|
305
|
+
createdAt: 0
|
|
306
|
+
};
|
|
307
|
+
const primary = {
|
|
308
|
+
recordedId: primaryHeader.id,
|
|
309
|
+
createdAt: primaryHeader.createdAt,
|
|
310
|
+
entries: primaryEntries,
|
|
311
|
+
primary: true
|
|
312
|
+
};
|
|
313
|
+
const children = [];
|
|
314
|
+
for (const childFile of config.childFiles ?? []) {
|
|
315
|
+
if (!existsSync(childFile)) throw new Error(`llm-replay: child fixture not found: ${childFile} — re-record the scenario`);
|
|
316
|
+
const text = readFileSync(childFile, "utf8");
|
|
317
|
+
const header = parseSessionHeader(text);
|
|
318
|
+
const ownEvents = parseSessionLog(text).slice(header.seedLength);
|
|
319
|
+
children.push({
|
|
320
|
+
recordedId: header.id,
|
|
321
|
+
createdAt: header.createdAt,
|
|
322
|
+
entries: deriveReplayScript(ownEvents),
|
|
323
|
+
primary: false
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
children.sort((a, b) => a.createdAt - b.createdAt || a.recordedId.localeCompare(b.recordedId));
|
|
327
|
+
return [primary, ...children];
|
|
328
|
+
}
|
|
329
|
+
/** Replay adapter that makes a configured provider catalog discoverable without provider I/O. */
|
|
330
|
+
var ReplayAdapter = class extends LlmAdapter {
|
|
331
|
+
replay;
|
|
332
|
+
providers;
|
|
333
|
+
constructor(providers, replay) {
|
|
334
|
+
super();
|
|
335
|
+
this.replay = replay;
|
|
336
|
+
this.providers = new Map(providers.map((provider) => [provider.id, provider]));
|
|
337
|
+
}
|
|
338
|
+
providerInfo(provider) {
|
|
339
|
+
const configured = this.providers.get(provider);
|
|
340
|
+
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
|
341
|
+
if (configured === void 0) return super.providerInfo(provider);
|
|
342
|
+
return {
|
|
343
|
+
id: provider,
|
|
344
|
+
name: configured.name ?? provider
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
providerRetryPolicy(provider) {
|
|
348
|
+
const configured = this.providers.get(provider);
|
|
349
|
+
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
|
350
|
+
if (configured === void 0) return super.providerRetryPolicy(provider);
|
|
351
|
+
return configured.retryPolicy === void 0 ? void 0 : resolveRetryPolicy(configured.retryPolicy, `llm-replay: provider "${provider}" retryPolicy`);
|
|
352
|
+
}
|
|
353
|
+
listModels(provider) {
|
|
354
|
+
const configured = this.providers.get(provider);
|
|
355
|
+
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
|
356
|
+
if (configured === void 0) return Promise.resolve([]);
|
|
357
|
+
return Promise.resolve((configured.models ?? []).map((model) => ({
|
|
358
|
+
provider,
|
|
359
|
+
id: model.id,
|
|
360
|
+
name: model.name ?? model.id,
|
|
361
|
+
...model.description === void 0 ? {} : { description: model.description }
|
|
362
|
+
})));
|
|
363
|
+
}
|
|
364
|
+
resolveModel(provider, model) {
|
|
365
|
+
const configured = this.providers.get(provider);
|
|
366
|
+
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
|
367
|
+
if (configured === void 0) return Promise.resolve({
|
|
368
|
+
provider,
|
|
369
|
+
id: model,
|
|
370
|
+
name: model
|
|
371
|
+
});
|
|
372
|
+
const configuredModel = configured.models?.find((candidate) => candidate.id === model);
|
|
373
|
+
return Promise.resolve({
|
|
374
|
+
provider,
|
|
375
|
+
id: model,
|
|
376
|
+
name: configuredModel?.name ?? model,
|
|
377
|
+
...configuredModel?.description === void 0 ? {} : { description: configuredModel.description },
|
|
378
|
+
...configuredModel?.contextWindow === void 0 ? {} : { context: { contextWindow: configuredModel.contextWindow } },
|
|
379
|
+
...configuredModel?.defaultMaxTokens === void 0 ? {} : { defaultMaxTokens: configuredModel.defaultMaxTokens },
|
|
380
|
+
...configuredModel?.reasoningEfforts === void 0 ? {} : { reasoning: {
|
|
381
|
+
efforts: configuredModel.reasoningEfforts.map((id) => ({
|
|
382
|
+
id: ReasoningEffortId(id),
|
|
383
|
+
name: id
|
|
384
|
+
})),
|
|
385
|
+
...configuredModel.defaultReasoningEffort === void 0 ? {} : { defaultEffort: ReasoningEffortId(configuredModel.defaultReasoningEffort) }
|
|
386
|
+
} }
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
stream(options) {
|
|
390
|
+
return this.replay(options);
|
|
391
|
+
}
|
|
392
|
+
};
|
|
393
|
+
/**
|
|
394
|
+
* Wait `paceMs` between chunk yields, aborting the wait (and the stream) the
|
|
395
|
+
* moment the signal fires — a paced replay must cancel as promptly as a burst
|
|
396
|
+
* one.
|
|
397
|
+
*/
|
|
398
|
+
function paceDelay(paceMs, signal) {
|
|
399
|
+
return new Promise((resolve, reject) => {
|
|
400
|
+
const timer = setTimeout(() => {
|
|
401
|
+
signal?.removeEventListener("abort", onAbort);
|
|
402
|
+
resolve();
|
|
403
|
+
}, paceMs);
|
|
404
|
+
const onAbort = () => {
|
|
405
|
+
clearTimeout(timer);
|
|
406
|
+
reject(/* @__PURE__ */ new Error("aborted"));
|
|
407
|
+
};
|
|
408
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
/** Yield a recorded stream back, honoring abort like a real adapter. */
|
|
412
|
+
async function* replayEntry(entry, signal, paceMs) {
|
|
413
|
+
switch (entry.kind) {
|
|
414
|
+
case "chunks":
|
|
415
|
+
for (const chunk of entry.chunks) {
|
|
416
|
+
if (signal?.aborted) throw new Error("aborted");
|
|
417
|
+
if (paceMs > 0) await paceDelay(paceMs, signal);
|
|
418
|
+
yield chunk;
|
|
419
|
+
}
|
|
420
|
+
return;
|
|
421
|
+
case "throw":
|
|
422
|
+
for (const chunk of entry.chunks) {
|
|
423
|
+
if (signal?.aborted) throw new Error("aborted");
|
|
424
|
+
if (paceMs > 0) await paceDelay(paceMs, signal);
|
|
425
|
+
yield chunk;
|
|
426
|
+
}
|
|
427
|
+
throw new LlmError(entry.message, entry.code);
|
|
428
|
+
case "hang":
|
|
429
|
+
yield {
|
|
430
|
+
type: "block-start",
|
|
431
|
+
index: 0,
|
|
432
|
+
blockType: "text"
|
|
433
|
+
};
|
|
434
|
+
yield {
|
|
435
|
+
type: "text-delta",
|
|
436
|
+
index: 0,
|
|
437
|
+
text: "partial"
|
|
438
|
+
};
|
|
439
|
+
if (entry.readyFile !== void 0) writeFileSync(entry.readyFile, "");
|
|
440
|
+
await new Promise((_resolve, reject) => {
|
|
441
|
+
if (signal?.aborted) {
|
|
442
|
+
reject(/* @__PURE__ */ new Error("aborted"));
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
445
|
+
signal?.addEventListener("abort", () => {
|
|
446
|
+
reject(/* @__PURE__ */ new Error("aborted"));
|
|
447
|
+
}, { once: true });
|
|
448
|
+
});
|
|
449
|
+
/* v8 ignore next -- unreachable: the hang promise only ever rejects (on abort), never resolves; control never reaches here */
|
|
450
|
+
return;
|
|
451
|
+
/* v8 ignore next -- sidecar entries are validated before they reach the closed local union. */
|
|
452
|
+
default: return assertNever(entry, "llm-replay replay entry");
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Install per-session positional replay. A newly seen live session takes the
|
|
457
|
+
* next ordered recorded script, then advances its own cursor synchronously at
|
|
458
|
+
* invocation time; calls without `sessionId` share one anonymous session. A
|
|
459
|
+
* non-empty provider catalog registers a routed replay adapter; otherwise a
|
|
460
|
+
* catch-all waterfall intercepts requests.
|
|
461
|
+
*
|
|
462
|
+
* @param ctx - the context whose LLM service receives the replay route or waterfall.
|
|
463
|
+
* @param config - the resolved fixture paths (env-var defaulting is `apply`'s job).
|
|
464
|
+
* @returns the {@link ReplayHandle} carrying the disposer and the teardown consumption check.
|
|
465
|
+
*/
|
|
466
|
+
function installLlmReplay(ctx, config) {
|
|
467
|
+
const paceMs = config.paceMs ?? 0;
|
|
468
|
+
if (!Number.isInteger(paceMs) || paceMs < 0) throw new Error(`llm-replay: paceMs must be a non-negative integer, got ${String(config.paceMs)}`);
|
|
469
|
+
const scripts = loadSessionScripts(config);
|
|
470
|
+
const bound = /* @__PURE__ */ new Map();
|
|
471
|
+
let nextScript = 0;
|
|
472
|
+
const ANON = "\0anon\0";
|
|
473
|
+
const replay = (options) => {
|
|
474
|
+
const key = options.sessionId ?? ANON;
|
|
475
|
+
let state = bound.get(key);
|
|
476
|
+
let unrecorded = false;
|
|
477
|
+
if (state === void 0) {
|
|
478
|
+
const script = scripts[nextScript];
|
|
479
|
+
if (script === void 0) {
|
|
480
|
+
unrecorded = true;
|
|
481
|
+
state = {
|
|
482
|
+
entries: [],
|
|
483
|
+
cursor: 0
|
|
484
|
+
};
|
|
485
|
+
} else {
|
|
486
|
+
nextScript++;
|
|
487
|
+
state = {
|
|
488
|
+
entries: script.entries,
|
|
489
|
+
cursor: 0
|
|
490
|
+
};
|
|
491
|
+
bound.set(key, state);
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
const boundState = state;
|
|
495
|
+
const seenSessions = nextScript;
|
|
496
|
+
const totalScripts = scripts.length;
|
|
497
|
+
const index = boundState.cursor++;
|
|
498
|
+
const entry = boundState.entries[index];
|
|
499
|
+
return (async function* () {
|
|
500
|
+
if (unrecorded) throw new Error(`llm-replay: a model call arrived from an unrecorded session (#${seenSessions + 1}); the scenario recorded only ${totalScripts} session(s) — re-record it`);
|
|
501
|
+
if (entry === void 0) throw new Error(`llm-replay: script exhausted — session requested model call #${index + 1} but its script has only ${boundState.entries.length}; re-record the scenario`);
|
|
502
|
+
yield* replayEntry(resolveScriptedEntry(entry, options.messages), options.signal, paceMs);
|
|
503
|
+
})();
|
|
504
|
+
};
|
|
505
|
+
const providers = config.providers ?? [];
|
|
506
|
+
return {
|
|
507
|
+
dispose: providers.length > 0 ? ctx.llm.registerAdapter(providers.map((provider) => provider.id), new ReplayAdapter(providers, replay)) : ctx.on("llm/stream", (options, _next) => replay(options)),
|
|
508
|
+
assertConsumed() {
|
|
509
|
+
const problems = [];
|
|
510
|
+
if (nextScript < scripts.length) problems.push(`${scripts.length - nextScript} recorded script(s) never bound to a live session`);
|
|
511
|
+
for (const [key, state] of bound) if (state.cursor < state.entries.length) {
|
|
512
|
+
const who = key === ANON ? "the anonymous session" : `session ${key}`;
|
|
513
|
+
problems.push(`${who} consumed ${state.cursor}/${state.entries.length} recorded call(s)`);
|
|
514
|
+
}
|
|
515
|
+
if (problems.length > 0) throw new Error(`llm-replay: fixture not fully consumed — ${problems.join("; ")}; the scenario drove fewer model calls than recorded`);
|
|
516
|
+
}
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
const name = "llm-replay";
|
|
520
|
+
const inject = ["llm"];
|
|
521
|
+
function apply(ctx, config = {}) {
|
|
522
|
+
const file = config.file ?? process.env.DSH_SNAPSHOT_FILE;
|
|
523
|
+
if (file === void 0 || file.length === 0) throw new Error("llm-replay: a fixture path is required (Config.file or $DSH_SNAPSHOT_FILE)");
|
|
524
|
+
const overrideFile = config.overrideFile ?? process.env.DSH_SNAPSHOT_OVERRIDE;
|
|
525
|
+
const childEnv = process.env.DSH_SNAPSHOT_CHILD_FILES;
|
|
526
|
+
const childFiles = config.childFiles ?? (childEnv !== void 0 && childEnv.length > 0 ? childEnv.split(delimiter) : []);
|
|
527
|
+
installLlmReplay(ctx, {
|
|
528
|
+
file,
|
|
529
|
+
...overrideFile !== void 0 && overrideFile.length > 0 ? { overrideFile } : {},
|
|
530
|
+
...childFiles.length > 0 ? { childFiles } : {},
|
|
531
|
+
...config.providers !== void 0 ? { providers: config.providers } : {},
|
|
532
|
+
...config.paceMs !== void 0 ? { paceMs: config.paceMs } : {}
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
//#endregion
|
|
536
|
+
export { apply, deriveReplayScript, inject, installLlmReplay, loadReplayScript, loadSessionScripts, name, parseSessionHeader, parseSessionLog, resolveScriptedEntry };
|
package/lib/invariant.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
//#region lib/types/invariant.js
|
|
2
|
+
/**
|
|
3
|
+
* Package-owned invariant companion for `@deepseek-ai/dsh-llm-replay`.
|
|
4
|
+
* @module @deepseek-ai/dsh-llm-replay/invariant
|
|
5
|
+
*/
|
|
6
|
+
const PACKAGE_NAME = "@deepseek-ai/dsh-llm-replay";
|
|
7
|
+
/** Cordis companion plugin name. */
|
|
8
|
+
const name = "llm-replay-invariant";
|
|
9
|
+
/** Service required before the companion can reserve package ownership. */
|
|
10
|
+
const inject = ["invariants"];
|
|
11
|
+
/**
|
|
12
|
+
* No runtime invariant: this test-only adapter consumes a fixed replay script; its stream grammar
|
|
13
|
+
* is checked by the LLM companion and fixture derivation tests.
|
|
14
|
+
*/
|
|
15
|
+
const install = () => {};
|
|
16
|
+
/**
|
|
17
|
+
* Register this package's invariant companion.
|
|
18
|
+
* @param ctx - Cordis context carrying the invariant service.
|
|
19
|
+
* @returns the installed registration's disposer after setup succeeds.
|
|
20
|
+
*/
|
|
21
|
+
const apply = (ctx) => Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install));
|
|
22
|
+
//#endregion
|
|
23
|
+
export { apply, inject, name };
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keyless snapshot-test LLM replay. It derives one model-call script per
|
|
3
|
+
* recorded session from `assistant/chunk` events and explicitly marked local
|
|
4
|
+
* compaction calls, then binds fresh live sessions to parent/child scripts by
|
|
5
|
+
* first-call order. Throw and hang cases require an explicit override because
|
|
6
|
+
* a session log cannot reconstruct them alone.
|
|
7
|
+
* @module @deepseek-ai/dsh-llm-replay
|
|
8
|
+
*/
|
|
9
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
10
|
+
import type { SessionEvent } from '@deepseek-ai/dsh-session';
|
|
11
|
+
import type { GenerateOptions, RetryPolicyConfig, StreamChunk } from '@deepseek-ai/dsh-llm';
|
|
12
|
+
/**
|
|
13
|
+
* One recorded model call. `throw` may replay prefix chunks before failing;
|
|
14
|
+
* `hang` models cancellation. Derived chunk entries come from ordinary model
|
|
15
|
+
* streams and complete outputs of explicitly marked local compaction calls;
|
|
16
|
+
* an override sidecar can supply any variant.
|
|
17
|
+
*/
|
|
18
|
+
export type ReplayEntry = {
|
|
19
|
+
kind: 'chunks';
|
|
20
|
+
chunks: StreamChunk[];
|
|
21
|
+
} | {
|
|
22
|
+
kind: 'throw';
|
|
23
|
+
chunks: StreamChunk[];
|
|
24
|
+
message: string;
|
|
25
|
+
code: string;
|
|
26
|
+
} | {
|
|
27
|
+
kind: 'hang';
|
|
28
|
+
/** Optional marker written after the prefix chunks are consumed and before the stream waits for cancellation. */
|
|
29
|
+
readyFile?: string;
|
|
30
|
+
};
|
|
31
|
+
/** One model exposed by a replay-only provider catalog. */
|
|
32
|
+
export interface ReplayModelConfig {
|
|
33
|
+
/** Model id used for replay requests. */
|
|
34
|
+
id: string;
|
|
35
|
+
/** Selector label; defaults to {@link id}. */
|
|
36
|
+
name?: string;
|
|
37
|
+
/** Optional selector description. */
|
|
38
|
+
description?: string;
|
|
39
|
+
/** Optional positive integer context capacity published by the replay adapter. */
|
|
40
|
+
contextWindow?: number;
|
|
41
|
+
/**
|
|
42
|
+
* Optional per-request output cap the replay route materializes when callers
|
|
43
|
+
* omit one, so replay reconstructs the request header a live catalog produced.
|
|
44
|
+
*/
|
|
45
|
+
defaultMaxTokens?: number;
|
|
46
|
+
/** Optional reasoning-effort ids the replay route accepts, in display order. */
|
|
47
|
+
reasoningEfforts?: string[];
|
|
48
|
+
/**
|
|
49
|
+
* Optional effort materialized when callers omit one; must appear in
|
|
50
|
+
* {@link reasoningEfforts} or call resolution rejects the route.
|
|
51
|
+
*/
|
|
52
|
+
defaultReasoningEffort?: string;
|
|
53
|
+
}
|
|
54
|
+
/** One provider route exposed by the replay adapter. */
|
|
55
|
+
export interface ReplayProviderConfig {
|
|
56
|
+
/** Provider route used for replay requests. */
|
|
57
|
+
id: string;
|
|
58
|
+
/** Selector label; defaults to {@link id}. */
|
|
59
|
+
name?: string;
|
|
60
|
+
/** Advisory models exposed to replay scenarios that exercise discovery. */
|
|
61
|
+
models?: ReplayModelConfig[];
|
|
62
|
+
/** Optional provider-owned retry policy used by assembled recovery snapshots. */
|
|
63
|
+
retryPolicy?: RetryPolicyConfig;
|
|
64
|
+
}
|
|
65
|
+
/** Resolved plugin configuration. */
|
|
66
|
+
export interface ReplayConfig {
|
|
67
|
+
/**
|
|
68
|
+
* Path to the PRIMARY (parent) `session.jsonl` fixture. For a single-session
|
|
69
|
+
* scenario this is the only log; for a nested-agent scenario it is the parent,
|
|
70
|
+
* and the child logs ride in {@link childFiles}.
|
|
71
|
+
*/
|
|
72
|
+
file: string;
|
|
73
|
+
/**
|
|
74
|
+
* Optional sidecar for the PRIMARY session: a bare `ReplayEntry[]` replaces
|
|
75
|
+
* the derived script; `{ patches }` keeps it and swaps the named call
|
|
76
|
+
* indexes ({@link ReplayOverrideDoc}). Used by single-session scenarios not
|
|
77
|
+
* expressible as `assistant/chunk` (throw-before-chunk, cancel/hang,
|
|
78
|
+
* injected transient failures). Absent for normal and nested scenarios.
|
|
79
|
+
*/
|
|
80
|
+
overrideFile?: string;
|
|
81
|
+
/**
|
|
82
|
+
* Additional recorded child-session logs (a nested-agent scenario's subagent
|
|
83
|
+
* sessions). Each is derived independently; the full set is ordered by
|
|
84
|
+
* `createdAt` so the parent (earliest) binds to the first live session. Empty
|
|
85
|
+
* for a single-session scenario.
|
|
86
|
+
*/
|
|
87
|
+
childFiles?: string[];
|
|
88
|
+
/**
|
|
89
|
+
* Optional provider catalog. When non-empty, replay registers an adapter for
|
|
90
|
+
* these routes; when absent or empty, it retains the catch-all waterfall used
|
|
91
|
+
* by tests that do not need discovery.
|
|
92
|
+
*/
|
|
93
|
+
providers?: ReplayProviderConfig[];
|
|
94
|
+
/**
|
|
95
|
+
* Optional per-chunk pacing delay in milliseconds: each replayed chunk waits
|
|
96
|
+
* this long before yielding, so a downstream transport (e.g. the web SSE
|
|
97
|
+
* mux observed by a browser) sees genuinely incremental delivery. A realism
|
|
98
|
+
* knob only — correctness must never depend on it. Absent or `0` keeps
|
|
99
|
+
* today's synchronous burst yield. Must be a non-negative finite integer;
|
|
100
|
+
* aborting mid-wait cancels the stream like any other abort.
|
|
101
|
+
*/
|
|
102
|
+
paceMs?: number;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Handle returned by {@link installLlmReplay}: removal plus the end-of-run
|
|
106
|
+
* consumption check that turns silent fixture underruns (a scenario that
|
|
107
|
+
* issued fewer calls than recorded, or never bound a recorded child script)
|
|
108
|
+
* into a crisp diagnostic at teardown.
|
|
109
|
+
*/
|
|
110
|
+
export interface ReplayHandle {
|
|
111
|
+
/** Remove the registered adapter or waterfall listener (HMR safety). Freestanding closure — safe to destructure. */
|
|
112
|
+
dispose(this: void): void;
|
|
113
|
+
/**
|
|
114
|
+
* Throw unless every recorded script was bound to a live session and every
|
|
115
|
+
* bound cursor consumed its full entry list. Call at scenario teardown.
|
|
116
|
+
* Freestanding closure — safe to destructure.
|
|
117
|
+
*/
|
|
118
|
+
assertConsumed(this: void): void;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Recorded calls plus header facts used to order parent and child scripts.
|
|
122
|
+
* Recorded ids are diagnostic; fresh live ids bind by ordered first use.
|
|
123
|
+
*/
|
|
124
|
+
export interface SessionScript {
|
|
125
|
+
/** The recorded session id (diagnostics only — the live id differs). */
|
|
126
|
+
recordedId: string;
|
|
127
|
+
/** Session creation time; the deterministic ordering key (parent < child). */
|
|
128
|
+
createdAt: number;
|
|
129
|
+
/** The per-`stream()`-call replay entries, in recorded call order. */
|
|
130
|
+
entries: ReplayEntry[];
|
|
131
|
+
/**
|
|
132
|
+
* Whether this is the PRIMARY (parent) session. Breaks a `createdAt` tie in
|
|
133
|
+
* favor of the parent, which always issues the first model call.
|
|
134
|
+
*/
|
|
135
|
+
primary: boolean;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Parse a session `.jsonl` buffer into its event list. Line 0 is the session
|
|
139
|
+
* header (a `{type:'session',…}` record), every subsequent non-empty line is a
|
|
140
|
+
* {@link SessionEvent} or a packed chunk row (expanded back into its events, so
|
|
141
|
+
* a fixture recorded with `packChunks` on derives the same script). The header
|
|
142
|
+
* is skipped; malformed lines fail loud.
|
|
143
|
+
* @param text - the raw `.jsonl` file contents.
|
|
144
|
+
* @returns every event after the header, in log order.
|
|
145
|
+
*/
|
|
146
|
+
export declare function parseSessionLog(text: string): SessionEvent[];
|
|
147
|
+
/**
|
|
148
|
+
* Read replay identity, ordering, and fork-seed facts from the JSONL header.
|
|
149
|
+
*
|
|
150
|
+
* @param text - the raw `.jsonl` file contents (only the header line is read).
|
|
151
|
+
* @returns the header's `id`, `createdAt`, and `seedLength`, defaulted when absent.
|
|
152
|
+
*/
|
|
153
|
+
export declare function parseSessionHeader(text: string): {
|
|
154
|
+
id: string;
|
|
155
|
+
createdAt: number;
|
|
156
|
+
seedLength: number;
|
|
157
|
+
};
|
|
158
|
+
/**
|
|
159
|
+
* Reconstruct the per-`stream()` replay script from a recorded session log.
|
|
160
|
+
*
|
|
161
|
+
* Splits `assistant/chunk` events at every `finish`, using turn and step changes
|
|
162
|
+
* to detect an unterminated prior call. A `compact/summary` explicitly marked
|
|
163
|
+
* as one local LLM-stream call becomes a canonical successful stream from its
|
|
164
|
+
* complete `rawOutput` at the summary's log position. A
|
|
165
|
+
* missing assistant terminator means the live stream threw, so derivation
|
|
166
|
+
* rejects and the scenario must provide an explicit override. Multiple calls
|
|
167
|
+
* may share one turn and step when the loop retries.
|
|
168
|
+
* @param events - the recorded session's events.
|
|
169
|
+
* @returns one `chunks` entry per recorded model call, in call order.
|
|
170
|
+
*/
|
|
171
|
+
export declare function deriveReplayScript(events: SessionEvent[]): ReplayEntry[];
|
|
172
|
+
/**
|
|
173
|
+
* One positional patch in an augmentation sidecar: replaces the derived
|
|
174
|
+
* entry at call index `at` (0-based) with `entry`, or appends when `at`
|
|
175
|
+
* equals the derived length (an extra recorded-after-the-fact call, e.g. the
|
|
176
|
+
* retry attempt following an injected transient throw).
|
|
177
|
+
*/
|
|
178
|
+
export interface ReplayOverridePatch {
|
|
179
|
+
/** 0-based call index into the derived script; == length appends. */
|
|
180
|
+
at: number;
|
|
181
|
+
/** The replacement (or appended) entry at that call position. */
|
|
182
|
+
entry: ReplayEntry;
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Override sidecar document: either a whole-script replacement (a
|
|
186
|
+
* bare `ReplayEntry[]`) or the augmentation form `{ patches }`, which keeps
|
|
187
|
+
* the JSONL-derived script and swaps only the named call indexes — the shape
|
|
188
|
+
* for "turn N errors, everything else replays as recorded".
|
|
189
|
+
*/
|
|
190
|
+
export type ReplayOverrideDoc = ReplayEntry[] | {
|
|
191
|
+
patches: ReplayOverridePatch[];
|
|
192
|
+
};
|
|
193
|
+
/**
|
|
194
|
+
* Resolve every `{{fromRequest:<regex>}}` placeholder in one scripted entry
|
|
195
|
+
* against the live request. The corpus is every string leaf of the request
|
|
196
|
+
* messages joined by newlines; the pattern's LAST corpus match wins and its
|
|
197
|
+
* first capture group (or, without one, the whole match) substitutes in place.
|
|
198
|
+
* Scenario sidecars use this to script arguments no static file can know,
|
|
199
|
+
* such as a randomly minted goal id the model must echo back. A pattern that
|
|
200
|
+
* matches nothing, an invalid pattern, and an unterminated placeholder each
|
|
201
|
+
* fail loud. The last two braces of a consecutive `}` run terminate the
|
|
202
|
+
* placeholder, so a pattern may end with a brace quantifier but cannot
|
|
203
|
+
* contain `}}` followed by further pattern content. Derived entries pass
|
|
204
|
+
* through the same resolution as sidecar entries.
|
|
205
|
+
* @param entry - the scripted entry about to replay.
|
|
206
|
+
* @param messages - the live request messages searched by the placeholders.
|
|
207
|
+
* @returns the entry itself when no placeholder appears, else a resolved deep copy.
|
|
208
|
+
*/
|
|
209
|
+
export declare function resolveScriptedEntry(entry: ReplayEntry, messages: GenerateOptions['messages']): ReplayEntry;
|
|
210
|
+
/**
|
|
211
|
+
* Load the PRIMARY session's replay script: the sidecar override when present
|
|
212
|
+
* (whole-script replacement or `{ patches }` augmentation over the derived
|
|
213
|
+
* script), else the script derived from the session JSONL (fail-loud when the
|
|
214
|
+
* fixture is missing).
|
|
215
|
+
* @param config - the fixture paths; only `file` and `overrideFile` are consulted.
|
|
216
|
+
* @returns the resolved primary-session script.
|
|
217
|
+
*/
|
|
218
|
+
export declare function loadReplayScript(config: ReplayConfig): ReplayEntry[];
|
|
219
|
+
/**
|
|
220
|
+
* Load the primary and child scripts in bind order. Child derivation begins at
|
|
221
|
+
* `seedLength` so inherited parent chunks are never replayed as child calls.
|
|
222
|
+
*
|
|
223
|
+
* @param config - the fixture paths: the primary log plus any recorded child logs.
|
|
224
|
+
* @returns the primary script first, then the child scripts in bind order.
|
|
225
|
+
*/
|
|
226
|
+
export declare function loadSessionScripts(config: ReplayConfig): SessionScript[];
|
|
227
|
+
/**
|
|
228
|
+
* Install per-session positional replay. A newly seen live session takes the
|
|
229
|
+
* next ordered recorded script, then advances its own cursor synchronously at
|
|
230
|
+
* invocation time; calls without `sessionId` share one anonymous session. A
|
|
231
|
+
* non-empty provider catalog registers a routed replay adapter; otherwise a
|
|
232
|
+
* catch-all waterfall intercepts requests.
|
|
233
|
+
*
|
|
234
|
+
* @param ctx - the context whose LLM service receives the replay route or waterfall.
|
|
235
|
+
* @param config - the resolved fixture paths (env-var defaulting is `apply`'s job).
|
|
236
|
+
* @returns the {@link ReplayHandle} carrying the disposer and the teardown consumption check.
|
|
237
|
+
*/
|
|
238
|
+
export declare function installLlmReplay(ctx: Context, config: ReplayConfig): ReplayHandle;
|
|
239
|
+
export declare const name = "llm-replay";
|
|
240
|
+
export declare const inject: string[];
|
|
241
|
+
/** Plugin config: the {@link ReplayConfig} inputs, each defaulting to its `DSH_SNAPSHOT_*` env var in `apply`. */
|
|
242
|
+
export interface Config {
|
|
243
|
+
/** Override the fixture path; defaults to `$DSH_SNAPSHOT_FILE`. */
|
|
244
|
+
file?: string;
|
|
245
|
+
/** Override the sidecar path; defaults to `$DSH_SNAPSHOT_OVERRIDE`. */
|
|
246
|
+
overrideFile?: string;
|
|
247
|
+
/**
|
|
248
|
+
* Override the child-log paths; defaults to `$DSH_SNAPSHOT_CHILD_FILES` (a
|
|
249
|
+
* path-separator-delimited list). Each is a recorded subagent session log for
|
|
250
|
+
* a nested-agent scenario; absent/empty for a single-session scenario.
|
|
251
|
+
*/
|
|
252
|
+
childFiles?: string[];
|
|
253
|
+
/** Optional replay-only provider catalog; absent or empty selects catch-all waterfall replay. */
|
|
254
|
+
providers?: ReplayProviderConfig[];
|
|
255
|
+
/** Optional per-chunk pacing delay in ms (see {@link ReplayConfig.paceMs}); absent keeps burst yield. */
|
|
256
|
+
paceMs?: number;
|
|
257
|
+
}
|
|
258
|
+
export declare function apply(ctx: Context, config?: Config): void;
|
|
259
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Package-owned invariant companion for `@deepseek-ai/dsh-llm-replay`.
|
|
3
|
+
* @module @deepseek-ai/dsh-llm-replay/invariant
|
|
4
|
+
*/
|
|
5
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
6
|
+
/** Cordis companion plugin name. */
|
|
7
|
+
export declare const name = "llm-replay-invariant";
|
|
8
|
+
/** Service required before the companion can reserve package ownership. */
|
|
9
|
+
export declare const inject: string[];
|
|
10
|
+
/**
|
|
11
|
+
* Register this package's invariant companion.
|
|
12
|
+
* @param ctx - Cordis context carrying the invariant service.
|
|
13
|
+
* @returns the installed registration's disposer after setup succeeds.
|
|
14
|
+
*/
|
|
15
|
+
export declare const apply: (ctx: Context) => Promise<() => void>;
|
|
16
|
+
//# sourceMappingURL=invariant.d.ts.map
|
package/package.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@deepseek-ai/dsh-llm-replay",
|
|
3
|
+
"description": "Replay LLM plugin: short-circuits llm/stream with model chunks reconstructed from a recorded session JSONL (keyless snapshot tests)",
|
|
4
|
+
"version": "0.0.1-rc.1",
|
|
5
|
+
"publishConfig": {
|
|
6
|
+
"access": "restricted"
|
|
7
|
+
},
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
|
|
11
|
+
"directory": "packages/support/llm-replay"
|
|
12
|
+
},
|
|
13
|
+
"type": "module",
|
|
14
|
+
"main": "lib/index.js",
|
|
15
|
+
"types": "lib/types/index.d.ts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"types": "./lib/types/index.d.ts",
|
|
19
|
+
"default": "./lib/index.js"
|
|
20
|
+
},
|
|
21
|
+
"./invariant": {
|
|
22
|
+
"types": "./lib/types/invariant.d.ts",
|
|
23
|
+
"default": "./lib/invariant.js"
|
|
24
|
+
},
|
|
25
|
+
"./src/*": "./src/*",
|
|
26
|
+
"./package.json": "./package.json"
|
|
27
|
+
},
|
|
28
|
+
"files": [
|
|
29
|
+
"lib/index.js",
|
|
30
|
+
"lib/invariant.js",
|
|
31
|
+
"lib/types/**/*.d.ts"
|
|
32
|
+
],
|
|
33
|
+
"license": "BSD-3-Clause",
|
|
34
|
+
"peerDependencies": {
|
|
35
|
+
"@deepseek-ai/dsh-compact": "^0.0.1-rc.1",
|
|
36
|
+
"@deepseek-ai/dsh-invariants": "^0.0.1-rc.1",
|
|
37
|
+
"@deepseek-ai/dsh-llm": "^0.0.1-rc.1",
|
|
38
|
+
"@deepseek-ai/dsh-session": "^0.0.1-rc.1",
|
|
39
|
+
"@deepseek-ai/cordis": "^4.0.1-rc.1"
|
|
40
|
+
},
|
|
41
|
+
"devDependencies": {
|
|
42
|
+
"@deepseek-ai/dsh-compact": "^0.0.1-rc.1",
|
|
43
|
+
"@deepseek-ai/dsh-invariants": "^0.0.1-rc.1",
|
|
44
|
+
"@deepseek-ai/dsh-llm": "^0.0.1-rc.1",
|
|
45
|
+
"@deepseek-ai/dsh-session": "^0.0.1-rc.1",
|
|
46
|
+
"@deepseek-ai/cordis": "^4.0.1-rc.1"
|
|
47
|
+
}
|
|
48
|
+
}
|