dsh-data-domain-knowledge 0.1.1-alpha.2 → 0.1.1-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -16
- package/lib/client.js +1042 -2097
- package/lib/contract-refs.d.ts +0 -1
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +2489 -1060
- package/lib/types/agent-behavior-rules.d.ts.map +1 -1
- package/lib/types/agent-behavior-rules.js +3 -2
- package/lib/types/agent-behavior-rules.js.map +1 -1
- package/lib/types/axis-contract.js +1 -1
- package/lib/types/axis-contract.js.map +1 -1
- package/lib/types/card-organize.d.ts +2 -1
- package/lib/types/card-organize.d.ts.map +1 -1
- package/lib/types/card-organize.js +5 -3
- package/lib/types/card-organize.js.map +1 -1
- package/lib/types/config-schema.d.ts +1 -1
- package/lib/types/config-schema.js +1 -1
- package/lib/types/convert/host.d.ts +2 -3
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +11 -2
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +14 -20
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +8 -5
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/diagnosis/store.d.ts +2 -2
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +10 -2
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +1 -1
- package/lib/types/diagnosis/types.js +1 -1
- package/lib/types/extract-model.d.ts +3 -0
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/images.js +1 -1
- package/lib/types/index.d.ts +0 -4
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +99 -91
- package/lib/types/index.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts +3 -0
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +22 -2
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge-contract.d.ts +6 -0
- package/lib/types/knowledge-contract.d.ts.map +1 -1
- package/lib/types/knowledge-contract.js +6 -2
- package/lib/types/knowledge-contract.js.map +1 -1
- package/lib/types/layout.d.ts +9 -26
- package/lib/types/layout.d.ts.map +1 -1
- package/lib/types/layout.js +10 -28
- package/lib/types/layout.js.map +1 -1
- package/lib/types/materialize-snapshot.d.ts +5 -0
- package/lib/types/materialize-snapshot.d.ts.map +1 -1
- package/lib/types/migrate.d.ts +3 -1
- package/lib/types/migrate.d.ts.map +1 -1
- package/lib/types/migrate.js +3 -1
- package/lib/types/migrate.js.map +1 -1
- package/lib/types/model-text.d.ts +7 -0
- package/lib/types/model-text.d.ts.map +1 -1
- package/lib/types/model-text.js +31 -9
- package/lib/types/model-text.js.map +1 -1
- package/lib/types/organize-contract.d.ts.map +1 -1
- package/lib/types/organize-contract.js +2 -1
- package/lib/types/organize-contract.js.map +1 -1
- package/lib/types/pipeline/engine.d.ts +28 -4
- package/lib/types/pipeline/engine.d.ts.map +1 -1
- package/lib/types/pipeline/engine.js +68 -15
- package/lib/types/pipeline/engine.js.map +1 -1
- package/lib/types/pipeline/progress.d.ts +12 -0
- package/lib/types/pipeline/progress.d.ts.map +1 -0
- package/lib/types/pipeline/progress.js +37 -0
- package/lib/types/pipeline/progress.js.map +1 -0
- package/lib/types/pipeline/resumable-extract.d.ts +45 -0
- package/lib/types/pipeline/resumable-extract.d.ts.map +1 -0
- package/lib/types/pipeline/resumable-extract.js +262 -0
- package/lib/types/pipeline/resumable-extract.js.map +1 -0
- package/lib/types/pipeline/store.d.ts +8 -2
- package/lib/types/pipeline/store.d.ts.map +1 -1
- package/lib/types/pipeline/store.js +27 -5
- package/lib/types/pipeline/store.js.map +1 -1
- package/lib/types/pipeline/types.d.ts +56 -0
- package/lib/types/pipeline/types.d.ts.map +1 -1
- package/lib/types/pipeline/types.js +18 -1
- package/lib/types/pipeline/types.js.map +1 -1
- package/lib/types/present.d.ts +1 -1
- package/lib/types/present.js +1 -1
- package/lib/types/preset-ids.d.ts +0 -1
- package/lib/types/preset-ids.d.ts.map +1 -1
- package/lib/types/preset-ids.js +0 -3
- package/lib/types/preset-ids.js.map +1 -1
- package/lib/types/query-analysis.d.ts +0 -5
- package/lib/types/query-analysis.d.ts.map +1 -1
- package/lib/types/query-analysis.js +0 -16
- package/lib/types/query-analysis.js.map +1 -1
- package/lib/types/recall-delivery.d.ts +36 -0
- package/lib/types/recall-delivery.d.ts.map +1 -0
- package/lib/types/recall-delivery.js +16 -0
- package/lib/types/recall-delivery.js.map +1 -0
- package/lib/types/retrieval/scope.d.ts +17 -0
- package/lib/types/retrieval/scope.d.ts.map +1 -0
- package/lib/types/retrieval/scope.js +11 -0
- package/lib/types/retrieval/scope.js.map +1 -0
- package/lib/types/runtime.d.ts +75 -14
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +454 -198
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/settings-volatile.d.ts +0 -7
- package/lib/types/settings-volatile.d.ts.map +1 -1
- package/lib/types/settings-volatile.js +0 -7
- package/lib/types/settings-volatile.js.map +1 -1
- package/lib/types/source-identity.d.ts +34 -0
- package/lib/types/source-identity.d.ts.map +1 -0
- package/lib/types/source-identity.js +85 -0
- package/lib/types/source-identity.js.map +1 -0
- package/lib/types/source-read.d.ts +18 -0
- package/lib/types/source-read.d.ts.map +1 -0
- package/lib/types/source-read.js +49 -0
- package/lib/types/source-read.js.map +1 -0
- package/lib/types/store/axis-references.d.ts.map +1 -1
- package/lib/types/store/axis-references.js +5 -3
- package/lib/types/store/axis-references.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +5 -29
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +57 -2
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/source-identity.d.ts +8 -0
- package/lib/types/store/source-identity.d.ts.map +1 -0
- package/lib/types/store/source-identity.js +56 -0
- package/lib/types/store/source-identity.js.map +1 -0
- package/lib/types/types.d.ts +46 -5
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js +6 -3
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-availability.d.ts +11 -0
- package/lib/types/wiki-availability.d.ts.map +1 -0
- package/lib/types/wiki-availability.js +39 -0
- package/lib/types/wiki-availability.js.map +1 -0
- package/lib/types/wiki-finalize.d.ts +4 -2
- package/lib/types/wiki-finalize.d.ts.map +1 -1
- package/lib/types/wiki-finalize.js +9 -27
- package/lib/types/wiki-finalize.js.map +1 -1
- package/lib/types/wiki-links.d.ts +35 -0
- package/lib/types/wiki-links.d.ts.map +1 -0
- package/lib/types/wiki-links.js +129 -0
- package/lib/types/wiki-links.js.map +1 -0
- package/lib/types/wiki-trial-types.d.ts +3 -0
- package/lib/types/wiki-trial-types.d.ts.map +1 -1
- package/lib/types/wiki-trial.d.ts +3 -3
- package/lib/types/wiki-trial.d.ts.map +1 -1
- package/lib/types/wiki-trial.js +14 -3
- package/lib/types/wiki-trial.js.map +1 -1
- package/package.json +16 -16
- package/scripts/doctor.mjs +0 -7
- package/scripts/setup-embedding.sh +0 -0
- package/src/agent-behavior-rules.ts +3 -2
- package/src/axis-contract.ts +1 -1
- package/src/card-organize.ts +5 -3
- package/src/client/AttributionEventGuide.tsx +3 -3
- package/src/client/AxisMatrix.tsx +86 -19
- package/src/client/AxisRegistrySection.tsx +1 -1
- package/src/client/CompileSettingsView.tsx +473 -0
- package/src/client/DirectoryPicker.tsx +0 -2
- package/src/client/ExtractionTasksPage.tsx +557 -423
- package/src/client/GuideArchitectureCanvas.tsx +5 -12
- package/src/client/ImportFilesDialog.tsx +1 -0
- package/src/client/InfoHint.tsx +1 -1
- package/src/client/KnowledgeBaseTab.tsx +141 -148
- package/src/client/KnowledgePanel.tsx +16 -9
- package/src/client/OperationLogTab.tsx +1 -1
- package/src/client/PipelineRunView.tsx +173 -0
- package/src/client/RecallPanel.tsx +5 -16
- package/src/client/TabGuide.tsx +3 -7
- package/src/client/WikiFlowPanel.tsx +10 -11
- package/src/client/arch/NodeLogicPanel.tsx +3 -3
- package/src/client/arch/model.ts +5 -113
- package/src/client/architecture.css +1 -1
- package/src/client/architecture.tsx +13 -27
- package/src/client/attribution/AttributionTab.tsx +141 -10
- package/src/client/pages.css +305 -180
- package/src/client/pages.tsx +7 -12
- package/src/client/styles.css +15 -96
- package/src/config-schema.ts +1 -1
- package/src/convert/host.ts +13 -5
- package/src/convert/ingest.ts +20 -7
- package/src/diagnosis/store.ts +7 -2
- package/src/diagnosis/types.ts +1 -1
- package/src/extract-model.ts +3 -0
- package/src/images.ts +1 -1
- package/src/index.ts +92 -97
- package/src/knowledge-bases.ts +23 -3
- package/src/knowledge-contract.ts +7 -2
- package/src/layout.ts +10 -32
- package/src/materialize-snapshot.ts +1 -0
- package/src/migrate.ts +3 -1
- package/src/model-text.ts +33 -11
- package/src/organize-contract.ts +2 -1
- package/src/pipeline/engine.ts +85 -22
- package/src/pipeline/progress.ts +40 -0
- package/src/pipeline/resumable-extract.ts +233 -0
- package/src/pipeline/store.ts +28 -6
- package/src/pipeline/types.ts +47 -2
- package/src/present.ts +1 -1
- package/src/preset-ids.ts +0 -4
- package/src/query-analysis.ts +0 -20
- package/src/recall-delivery.ts +19 -0
- package/src/retrieval/scope.ts +10 -0
- package/src/runtime.ts +412 -206
- package/src/settings-volatile.ts +0 -7
- package/src/source-identity.ts +85 -0
- package/src/source-read.ts +47 -0
- package/src/store/axis-references.ts +5 -3
- package/src/store/knowledge-db.ts +54 -2
- package/src/store/source-identity.ts +42 -0
- package/src/types.ts +18 -5
- package/src/wiki-availability.ts +37 -0
- package/src/wiki-finalize.ts +9 -25
- package/src/wiki-links.ts +120 -0
- package/src/wiki-trial-types.ts +3 -2
- package/src/wiki-trial.ts +17 -6
- package/lib/types/intent/engine.d.ts +0 -122
- package/lib/types/intent/engine.d.ts.map +0 -1
- package/lib/types/intent/engine.js +0 -488
- package/lib/types/intent/engine.js.map +0 -1
- package/lib/types/intent/host.d.ts +0 -68
- package/lib/types/intent/host.d.ts.map +0 -1
- package/lib/types/intent/host.js +0 -249
- package/lib/types/intent/host.js.map +0 -1
- package/lib/types/intent/scope.d.ts +0 -25
- package/lib/types/intent/scope.d.ts.map +0 -1
- package/lib/types/intent/scope.js +0 -116
- package/lib/types/intent/scope.js.map +0 -1
- package/lib/types/intent/store.d.ts +0 -33
- package/lib/types/intent/store.d.ts.map +0 -1
- package/lib/types/intent/store.js +0 -239
- package/lib/types/intent/store.js.map +0 -1
- package/lib/types/intent/tools.d.ts +0 -62
- package/lib/types/intent/tools.d.ts.map +0 -1
- package/lib/types/intent/tools.js +0 -130
- package/lib/types/intent/tools.js.map +0 -1
- package/lib/types/intent/types.d.ts +0 -170
- package/lib/types/intent/types.d.ts.map +0 -1
- package/lib/types/intent/types.js +0 -56
- package/lib/types/intent/types.js.map +0 -1
- package/src/client/IntentLogic.tsx +0 -192
- package/src/client/IntentTab.tsx +0 -343
- package/src/client/TypeDefinitionDrawer.tsx +0 -195
- package/src/intent/README.md +0 -42
- package/src/intent/engine.ts +0 -515
- package/src/intent/host.ts +0 -315
- package/src/intent/scope.ts +0 -125
- package/src/intent/store.ts +0 -218
- package/src/intent/tools.ts +0 -154
- package/src/intent/types.ts +0 -229
|
@@ -18,8 +18,13 @@ export interface KnowledgeContent {
|
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
export function wikiProjection(kind: KnowledgeKind, coordinate: { objectId: string; actionId: string; sceneId: string }): { objectId: string; actionId: string; sceneId: string } | undefined {
|
|
21
|
-
if (kind === 'case' || !coordinate.objectId
|
|
22
|
-
return {
|
|
21
|
+
if (kind === 'case' || !coordinate.objectId) return undefined
|
|
22
|
+
return { ...coordinate }
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Wiki identity preserves the full coordinate and separates Fact from Way. Empty axes are exact values. */
|
|
26
|
+
export function wikiGroupKey(kind: WikiKind, coordinate: { objectId: string; actionId: string; sceneId: string }): string {
|
|
27
|
+
return `wiki:${kind}:${coordinate.objectId}\u0001${coordinate.actionId}\u0001${coordinate.sceneId}`
|
|
23
28
|
}
|
|
24
29
|
|
|
25
30
|
export function procedureText(value: ProcedureContent): string {
|
package/src/layout.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Tita Agent 在磁盘上的布局(
|
|
2
|
+
* Tita Agent 在磁盘上的布局(Agent 根路径固定四段,知识库内部不再使用段目录)。
|
|
3
3
|
*
|
|
4
4
|
* ## 为什么要有这个文件
|
|
5
5
|
*
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* 都可能指向不同的卷,于是「这个 Agent 的东西在哪」这个问题没有唯一答案 ——
|
|
9
9
|
* 备份要记得两个路径,换机器要改两处,用户也没法一眼看出磁盘上哪块是它的。
|
|
10
10
|
*
|
|
11
|
-
* 现在**只有一个设置**:Tita Agent
|
|
11
|
+
* 现在**只有一个设置**:Tita Agent 根路径。根路径下固定四段,职责互不重叠:
|
|
12
12
|
*
|
|
13
13
|
* ```
|
|
14
14
|
* <agentRoot>/
|
|
@@ -24,7 +24,6 @@
|
|
|
24
24
|
* extraction-runs.sqlite 抽取轨迹
|
|
25
25
|
* access-audit.ndjson 访问审计
|
|
26
26
|
* feedback.ndjson 反馈
|
|
27
|
-
* intent/ 意图:intent.db(会话审计留痕)
|
|
28
27
|
* tmp/ 运行期临时文件:转换工作区、任务状态
|
|
29
28
|
* <YYYY-MM-DD>/ 二级按日期分桶
|
|
30
29
|
* resource/ Agent 跑起来所需要的家当(不是它的产出)
|
|
@@ -100,11 +99,7 @@
|
|
|
100
99
|
* 不可重建的命名决定。按原理 5 的判据(「要不要跟着知识一起备份、迁移、删除」)它们
|
|
101
100
|
* 必须跟着知识走,所以 0062 把它们搬进了 `knowledge.db`。今天 `memory/` 里**没有一样
|
|
102
101
|
* 东西是不可重建的知识**,这才是这条原理想要的终态。
|
|
103
|
-
* 6.
|
|
104
|
-
* 在哪个知识库下说,理解方式不该变。它的两份东西 —— 分类/act 注册表(配置)
|
|
105
|
-
* 与每轮记录(留痕)—— 原来一个住在宿主的 settings 文档里、一个只在内存里,
|
|
106
|
-
* 都不是「这个 Agent 自己的文件」;落进根路径之后,它们和知识、记忆一样可以
|
|
107
|
-
* 被备份、被 diff、被用户直接读。
|
|
102
|
+
* 6. 会话目标由当前请求和坐标直接提供,不在 Agent 根路径下复制分类词表或会话记录库。
|
|
108
103
|
*
|
|
109
104
|
* 这个文件是布局的**唯一真源**:运行时、知识库注册表、转换内核、设置面板读到的
|
|
110
105
|
* 都是这里算出来的路径,不在别处再拼一次字符串。
|
|
@@ -121,8 +116,8 @@ import { basename, dirname, join, resolve } from 'node:path'
|
|
|
121
116
|
*
|
|
122
117
|
* 分两组看,判据不同:
|
|
123
118
|
*
|
|
124
|
-
* - **Agent 自己的东西**:`domain_knowledge`(知识)、`memory
|
|
125
|
-
*
|
|
119
|
+
* - **Agent 自己的东西**:`domain_knowledge`(知识)、`memory`(审计与版本记录)、
|
|
120
|
+
* `tmp`(运行期临时态)。判据是「能不能重建」。
|
|
126
121
|
* - **跑起来所需的依赖**:`resource`(模型、Python 环境、注册表)。判据是「属于谁」。
|
|
127
122
|
*
|
|
128
123
|
* **`output` 不在这里。** 交付物是**某个工作项目的产物**,不是这个 Agent 的固有状态:
|
|
@@ -130,7 +125,7 @@ import { basename, dirname, join, resolve } from 'node:path'
|
|
|
130
125
|
* 所以它落在**当前工作区**的 `output/<日期>/` 下,而不是跟着 Agent 的家走。
|
|
131
126
|
* 约定写在知识插件追加到标准模式系统提示词的行为规则里。
|
|
132
127
|
*/
|
|
133
|
-
export const AGENT_DIRS = ['domain_knowledge', 'memory', '
|
|
128
|
+
export const AGENT_DIRS = ['domain_knowledge', 'memory', 'tmp', 'resource'] as const
|
|
134
129
|
export type AgentDirName = typeof AGENT_DIRS[number]
|
|
135
130
|
|
|
136
131
|
/** 知识树那一段的目录名。 */
|
|
@@ -162,19 +157,10 @@ export const RESOURCE_DIR = 'resource'
|
|
|
162
157
|
* 反馈与访问审计。它从知识树里**搬出来**了,理由见文件头第 5 条原理。
|
|
163
158
|
*/
|
|
164
159
|
export const MEMORY_DIR = 'memory'
|
|
165
|
-
/**
|
|
166
|
-
* 意图那一段的目录名(0032)。
|
|
167
|
-
*
|
|
168
|
-
* 里面装的是意图的**会话审计留痕**:`intent.db`(当前记录 + 追加历史)。旧的
|
|
169
|
-
* `records.ndjson` 只在首次打开时迁移,不再继续写入。
|
|
170
|
-
* 分类与 act 是固定契约,不属于 Agent 的可编辑配置。
|
|
171
|
-
*/
|
|
172
|
-
export const INTENT_DIR = 'intent'
|
|
173
|
-
|
|
174
160
|
/**
|
|
175
161
|
* 默认根路径:`~/.dsh/tita-agent`。
|
|
176
162
|
*
|
|
177
|
-
* `.dsh` 是 DSH 的用户级数据根;`tita-agent`
|
|
163
|
+
* `.dsh` 是 DSH 的用户级数据根;`tita-agent` 是本插件的专属目录。知识、记忆、
|
|
178
164
|
* 临时态与运行所需家当都收在这里,用户不需要再管理一条插件自己的路径。
|
|
179
165
|
*/
|
|
180
166
|
export function defaultAgentRoot(): string {
|
|
@@ -223,7 +209,7 @@ export function knowledgeBaseRoot(agentRoot: string, kbId: string): string {
|
|
|
223
209
|
* 交付物的落点:`<工作区>/output`。
|
|
224
210
|
*
|
|
225
211
|
* 刻意**不**由 `agentRoot` 推出来。交付物是**某个工作项目的产物** —— 同一个 Agent
|
|
226
|
-
*
|
|
212
|
+
* 换一个项目打开,不该再看见上一个项目产出的图和数据。知识与记忆是 Agent
|
|
227
213
|
* 自己的、跨项目累积的东西,所以留在 `agentRoot`;交付物属于项目,所以跟着项目走。
|
|
228
214
|
*
|
|
229
215
|
* 二级目录按日期分桶(`output/<YYYY-MM-DD>/`),由转换内核的 `bucketByDay` 打开;
|
|
@@ -297,11 +283,9 @@ export interface AgentLayout {
|
|
|
297
283
|
resource: string
|
|
298
284
|
/** 记忆那一段(版本快照 / 反馈 / 访问审计),二级按知识库 id 分叉(0032)。 */
|
|
299
285
|
memory: string
|
|
300
|
-
/** 意图那一段(intent.db)(0032)。 */
|
|
301
|
-
intent: string
|
|
302
286
|
}
|
|
303
287
|
|
|
304
|
-
/**
|
|
288
|
+
/** 由根路径算出四段布局。纯函数,测试直接断言。 */
|
|
305
289
|
export function agentLayout(agentRoot: string): AgentLayout {
|
|
306
290
|
const root = resolve(agentRoot)
|
|
307
291
|
return {
|
|
@@ -310,7 +294,6 @@ export function agentLayout(agentRoot: string): AgentLayout {
|
|
|
310
294
|
tmp: join(root, TMP_DIR),
|
|
311
295
|
resource: join(root, RESOURCE_DIR),
|
|
312
296
|
memory: join(root, MEMORY_DIR),
|
|
313
|
-
intent: join(root, INTENT_DIR),
|
|
314
297
|
}
|
|
315
298
|
}
|
|
316
299
|
|
|
@@ -326,11 +309,6 @@ export function memoryRoot(agentRoot: string, kbId: string): string {
|
|
|
326
309
|
return join(resolve(agentRoot), MEMORY_DIR, kbId)
|
|
327
310
|
}
|
|
328
311
|
|
|
329
|
-
/** `<agentRoot>/intent`:会话意图审计留痕(0032)。意图不分库。 */
|
|
330
|
-
export function intentRoot(agentRoot: string): string {
|
|
331
|
-
return join(resolve(agentRoot), INTENT_DIR)
|
|
332
|
-
}
|
|
333
|
-
|
|
334
312
|
/**
|
|
335
313
|
* 一个知识库根目录里**固定存在的三样东西**(PRD v2 §0.2)。
|
|
336
314
|
*
|
|
@@ -396,7 +374,7 @@ export function looksLikeLegacyKnowledgeRoot(path: string): boolean {
|
|
|
396
374
|
*/
|
|
397
375
|
export function ensureAgentLayout(agentRoot: string): AgentLayout {
|
|
398
376
|
const layout = agentLayout(agentRoot)
|
|
399
|
-
for (const dir of [layout.domainKnowledge, layout.tmp, layout.resource, layout.memory
|
|
377
|
+
for (const dir of [layout.domainKnowledge, layout.tmp, layout.resource, layout.memory]) {
|
|
400
378
|
try {
|
|
401
379
|
mkdirSync(dir, { recursive: true })
|
|
402
380
|
} catch {
|
|
@@ -33,6 +33,7 @@ export interface MaterializationSnapshot {
|
|
|
33
33
|
blockedAmbiguous: number
|
|
34
34
|
/** 来源已变化 / 新抽取尚未归一的 Knowledge 条数。 */
|
|
35
35
|
blockedAwaitingNormalize: number
|
|
36
|
+
blockedItems: Array<{ knowledgeId: string; sourceIds: string[]; groupKey?: string }>
|
|
36
37
|
/** key → 这一组参与成卡的知识。 */
|
|
37
38
|
knowledge: Map<string, Knowledge[]>
|
|
38
39
|
/** key → 这一组的 Event。 */
|
package/src/migrate.ts
CHANGED
|
@@ -87,7 +87,9 @@ function skipped(reason: string, from?: string, to?: string): MigrationOutcome {
|
|
|
87
87
|
* 把旧版完整的 Agent 根目录合并到新的固定根目录。
|
|
88
88
|
*
|
|
89
89
|
* 这和 `migrateLegacyKnowledgeTree` 不同:后者只认识一棵知识库树,而这里要把
|
|
90
|
-
* `domain_knowledge`、`memory`、`
|
|
90
|
+
* `domain_knowledge`、`memory`、`resource` 与 `tmp` 一起搬走。历史 `intent` 目录不再由插件
|
|
91
|
+
* 读取或迁移;如果它仍在旧根路径中,会阻止旧根目录被清空,保留原始数据供用户自行处理。
|
|
92
|
+
* 目标里已经有
|
|
91
93
|
* 同名目录时逐层合并,任何同名文件都不覆盖,并把冲突留下来交给界面报告。
|
|
92
94
|
*/
|
|
93
95
|
export function migrateLegacyAgentRoot(input: { agentRoot: string; legacyRoot: string }): MigrationOutcome {
|
package/src/model-text.ts
CHANGED
|
@@ -32,7 +32,7 @@ interface StreamChunkLike {
|
|
|
32
32
|
type?: string
|
|
33
33
|
text?: string
|
|
34
34
|
usage?: TrialTokenUsage
|
|
35
|
-
reason?: { kind?: string; failure?: { code?: string; message?: string } }
|
|
35
|
+
reason?: { kind?: string; failure?: { code?: string; message?: string; status?: number; requestId?: string; retryAfterMs?: number } }
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
/**
|
|
@@ -69,21 +69,24 @@ export function buildModelRequest(input: {
|
|
|
69
69
|
system?: string
|
|
70
70
|
prompt: string
|
|
71
71
|
reasoningEffort?: string
|
|
72
|
+
maxTokens?: number
|
|
72
73
|
/** 路由与缓存亲和的会话身份。知识编译用 `knowledge:<库 id>`。 */
|
|
73
74
|
sessionId: string
|
|
74
75
|
}): {
|
|
75
76
|
provider: string
|
|
76
77
|
model: string
|
|
77
78
|
reasoningEffort?: string
|
|
79
|
+
maxTokens?: number
|
|
78
80
|
system?: string
|
|
79
81
|
sessionId: string
|
|
80
82
|
messages: Array<{ role: 'user'; content: Array<{ type: 'text'; text: string }> }>
|
|
81
83
|
} {
|
|
82
|
-
const { provider, model, system, prompt, reasoningEffort, sessionId } = input
|
|
84
|
+
const { provider, model, system, prompt, reasoningEffort, maxTokens, sessionId } = input
|
|
83
85
|
return {
|
|
84
86
|
provider,
|
|
85
87
|
model,
|
|
86
88
|
...(reasoningEffort ? { reasoningEffort } : {}),
|
|
89
|
+
...(maxTokens !== undefined ? { maxTokens } : {}),
|
|
87
90
|
...(system ? { system } : {}),
|
|
88
91
|
sessionId,
|
|
89
92
|
messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
|
|
@@ -106,7 +109,7 @@ export async function collectModelText(
|
|
|
106
109
|
let text = ''
|
|
107
110
|
/** 流怎么结束的。用来区分「正常但空」「失败」「中止」三种情况。 */
|
|
108
111
|
let finish: string | undefined
|
|
109
|
-
let failure:
|
|
112
|
+
let failure: NonNullable<StreamChunkLike['reason']>['failure'] | undefined
|
|
110
113
|
|
|
111
114
|
for await (const chunk of stream) {
|
|
112
115
|
if (!chunk || typeof chunk !== 'object') continue
|
|
@@ -125,8 +128,9 @@ export async function collectModelText(
|
|
|
125
128
|
if (failure || finish === 'error' || finish === 'aborted' || finish === 'max-tokens') {
|
|
126
129
|
const code = failure?.code?.trim() || (finish === 'max-tokens' ? 'OUTPUT_TRUNCATED' : '未知错误码')
|
|
127
130
|
const message = failure?.message?.trim() || (finish === 'max-tokens' ? '输出达到长度上限,部分返回已保留,请调整模型限制后重试' : '模型服务没有给出说明')
|
|
128
|
-
throw new Error(`模型调用失败(${code}):${message}`)
|
|
131
|
+
throw Object.assign(new Error(`模型调用失败(${code}):${message}`), { code, status: failure?.status, requestId: failure?.requestId, retryAfterMs: failure?.retryAfterMs })
|
|
129
132
|
}
|
|
133
|
+
if (!finish && text.length > 0) throw new Error('Stream ended without finish_reason;部分返回已保留')
|
|
130
134
|
if (text.length === 0) {
|
|
131
135
|
// 走到这里说明流正常结束(或者压根没有 finish 块)却没有一个字符。
|
|
132
136
|
throw new Error(`模型没有返回任何文本(结束原因:${finish ?? '没有收到 finish 块'})`)
|
|
@@ -156,7 +160,7 @@ export async function collectModelText(
|
|
|
156
160
|
*/
|
|
157
161
|
export function isRetryableModelFailure(error: unknown): boolean {
|
|
158
162
|
const message = error instanceof Error ? error.message : String(error)
|
|
159
|
-
const code = /模型调用失败(([^)]*))/.exec(message)?.[1]?.trim().toUpperCase() ?? ''
|
|
163
|
+
const code = (typeof (error as { code?: unknown })?.code === 'string' ? String((error as { code: string }).code).toUpperCase() : '') || (/模型调用失败(([^)]*))/.exec(message)?.[1]?.trim().toUpperCase() ?? '')
|
|
160
164
|
|
|
161
165
|
// 明确不可重试:配置问题与用户取消。宁可让人去改设置,也不要在错的地方重试。
|
|
162
166
|
if (['ABORTED', 'INVALID_CREDENTIAL', 'AUTH', 'UNAUTHORIZED', 'NOT_FOUND', 'INVALID_REQUEST', 'CONTEXT_LENGTH', 'OUTPUT_TRUNCATED'].includes(code)) return false
|
|
@@ -166,9 +170,10 @@ export function isRetryableModelFailure(error: unknown): boolean {
|
|
|
166
170
|
|
|
167
171
|
// 没有 code 的情况(旧适配器、或错误被包了一层):按可辨认的措辞判断。
|
|
168
172
|
if (/没有返回任何文本|没有收到 finish 块|Stream ended without finish_reason/i.test(message)) return true
|
|
169
|
-
if (/ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|socket hang up|premature close|other side closed|fetch failed|network error|HTTP 5\d\d|HTTP 429/i.test(message)) return true
|
|
173
|
+
if (/request\s+timed?\s*out|request_timeout|timeout_error|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|socket hang up|premature close|other side closed|fetch failed|network error|HTTP 5\d\d|HTTP 429/i.test(message)) return true
|
|
170
174
|
|
|
171
|
-
|
|
175
|
+
const status = Number((error as { status?: number })?.status)
|
|
176
|
+
return status === 408 || status === 429 || (status >= 500 && status < 600)
|
|
172
177
|
}
|
|
173
178
|
|
|
174
179
|
/** 重试策略:最多几次、每次之前等多久。 */
|
|
@@ -183,6 +188,8 @@ export interface ModelRetryOptions {
|
|
|
183
188
|
notice?: ((info: { attempt: number; attempts: number; error: unknown }) => string | null) | null
|
|
184
189
|
/** 注入 sleep,测试里不必真的等。 */
|
|
185
190
|
sleep?: (ms: number) => Promise<void>
|
|
191
|
+
signal?: AbortSignal
|
|
192
|
+
jitter?: boolean
|
|
186
193
|
}
|
|
187
194
|
|
|
188
195
|
/**
|
|
@@ -200,7 +207,13 @@ export async function callModelWithRetry(
|
|
|
200
207
|
): Promise<string> {
|
|
201
208
|
const attempts = Math.max(1, Math.floor(options.attempts ?? 3))
|
|
202
209
|
const delays = options.delaysMs ?? [2000, 6000]
|
|
203
|
-
const sleep = options.sleep ?? ((ms: number) => new Promise<void>(
|
|
210
|
+
const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve, reject) => {
|
|
211
|
+
const done = (): void => { options.signal?.removeEventListener('abort', abort); resolve() }
|
|
212
|
+
const timer = setTimeout(done, ms)
|
|
213
|
+
const abort = (): void => { clearTimeout(timer); reject(options.signal?.reason) }
|
|
214
|
+
options.signal?.addEventListener('abort', abort, { once: true })
|
|
215
|
+
if (options.signal?.aborted) abort()
|
|
216
|
+
}))
|
|
204
217
|
const notice = options.notice === undefined
|
|
205
218
|
? (info: { attempt: number; attempts: number; error: unknown }) =>
|
|
206
219
|
`\n\n[模型流中断(${info.error instanceof Error ? info.error.message : String(info.error)}),正在重试 ${info.attempt}/${info.attempts - 1}]\n\n`
|
|
@@ -208,16 +221,21 @@ export async function callModelWithRetry(
|
|
|
208
221
|
|
|
209
222
|
let lastError: unknown
|
|
210
223
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
224
|
+
options.signal?.throwIfAborted()
|
|
211
225
|
try {
|
|
212
226
|
return await send(text => options.onDelta?.(text))
|
|
213
227
|
} catch (error) {
|
|
214
228
|
lastError = error
|
|
215
|
-
if (
|
|
216
|
-
|
|
229
|
+
if (!isRetryableModelFailure(error)) throw error
|
|
230
|
+
if (attempt >= attempts) break
|
|
231
|
+
const retryAfter = Number((error as { retryAfterMs?: number })?.retryAfterMs)
|
|
232
|
+
const baseDelay = Number.isFinite(retryAfter) && retryAfter >= 0 ? Math.min(60_000, retryAfter) : delays[Math.min(attempt - 1, delays.length - 1)] ?? 0
|
|
233
|
+
const delayMs = Math.round(baseDelay * (options.jitter === false ? 1 : 0.7 + Math.random() * 0.6))
|
|
217
234
|
options.onRetry?.({ attempt, attempts, error, delayMs })
|
|
218
235
|
const line = notice?.({ attempt, attempts, error }) ?? ''
|
|
219
236
|
if (line) options.onDelta?.(line)
|
|
220
237
|
await sleep(delayMs)
|
|
238
|
+
options.signal?.throwIfAborted()
|
|
221
239
|
}
|
|
222
240
|
}
|
|
223
241
|
/**
|
|
@@ -227,5 +245,9 @@ export async function callModelWithRetry(
|
|
|
227
245
|
* 后者查额度、代理和上游状态。不写这一句,新加的重试在界面上等于不存在。
|
|
228
246
|
*/
|
|
229
247
|
const message = lastError instanceof Error ? lastError.message : String(lastError)
|
|
230
|
-
|
|
248
|
+
if (attempts === 1) throw lastError
|
|
249
|
+
const details = lastError as { code?: string; status?: number; requestId?: string; retryAfterMs?: number } | undefined
|
|
250
|
+
throw Object.assign(new Error(`${message}(已自动重试 ${attempts - 1} 次仍失败;可继续未完成部分)`, { cause: lastError }), {
|
|
251
|
+
code: details?.code, status: details?.status, requestId: details?.requestId, retryAfterMs: details?.retryAfterMs,
|
|
252
|
+
})
|
|
231
253
|
}
|
package/src/organize-contract.ts
CHANGED
|
@@ -19,7 +19,7 @@ export function buildOrganizeSystem(wikiKind: 'fact' | 'way' = 'fact'): string {
|
|
|
19
19
|
return [
|
|
20
20
|
'你在把一批已经抽好的知识组织成一张 Wiki 卡片。',
|
|
21
21
|
'',
|
|
22
|
-
`文件类型:${wikiKind === 'way' ? 'Wiki Way
|
|
22
|
+
`文件类型:${wikiKind === 'way' ? 'Wiki Way;同一 Object × Action × Scene 坐标下的 Procedure' : 'Wiki Fact;同一 Object × Action × Scene 坐标下的 Fact'}。不得把不同坐标的内容合并。空坐标分量不表示通用,保留每条的条件、状态与限制。`,
|
|
23
23
|
'这些条目是 Knowledge 真源。你不许新增、',
|
|
24
24
|
'删除或改动它们说的事实。你能做的是:分小节、定顺序、把它们改写成读得下去的',
|
|
25
25
|
'叙述、给卡片起一个自然的标题、写一句概述。',
|
|
@@ -33,6 +33,7 @@ export function buildOrganizeSystem(wikiKind: 'fact' | 'way' = 'fact'): string {
|
|
|
33
33
|
'3. **每句都要挂引用。** 陈述事实的句子末尾挂 `[^知识id]`,可以挂多个。',
|
|
34
34
|
' 只有很短的导语(「这一节讲取数流程」)可以不挂。长句子不挂引用 = 没有来源的',
|
|
35
35
|
' 断言,整份方案会被打回。',
|
|
36
|
+
'涉及其他 Object 的定义或方法时,使用给定目标的双链,不把另一份 Wiki 的全文复制进来。无法唯一定位时保留对象原名,由程序补充导航,不编造链接或新坐标。',
|
|
36
37
|
'4. **标题只做归纳,不要机械拼接。** 标题要能让人一眼知道卡片讲什么;不要照抄候选卡名,',
|
|
37
38
|
' 不要写成「对象的动作」「动作的动作」这类重复结构,不要加入知识里没有的新事实。',
|
|
38
39
|
' 保留区分这张卡所需的业务对象、字段名和动作词。知识条目附带的原文标题路径是定位线索,',
|
package/src/pipeline/engine.ts
CHANGED
|
@@ -3,17 +3,30 @@ import type { ModelCall } from '../extract-model.js'
|
|
|
3
3
|
import type { SourceRaw, WikiTypeDef } from '../types.js'
|
|
4
4
|
import type { TrialReport } from '../wiki-trial-types.js'
|
|
5
5
|
import { PipelineStore, type ExtractionCheckpoint } from './store.js'
|
|
6
|
-
import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_LABEL, type MaterializationBatch, type MaterializationGroup, type PipelineAction, type PipelineModel, type PipelineSelection, type PipelineStage, type PipelineTask } from './types.js'
|
|
6
|
+
import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_LABEL, normalizeExtractionTimeoutMinutes, type ExtractionTimeoutMinutes, type MaterializationBatch, type MaterializationGroup, type PipelineAction, type PipelineActivityUpdate, type PipelineItemState, type PipelineModel, type PipelineSelection, type PipelineStage, type PipelineTask, type PipelineWorkStage } from './types.js'
|
|
7
7
|
|
|
8
8
|
export interface PipelineSource extends SourceRaw { version: number; sourceHash: string }
|
|
9
9
|
export interface PipelineContext {
|
|
10
10
|
model: PipelineModel & { call?: ModelCall; why?: string }
|
|
11
11
|
note: string
|
|
12
|
+
extractionTimeoutMinutes?: ExtractionTimeoutMinutes
|
|
12
13
|
profile?: WikiTypeDef
|
|
13
14
|
organize: boolean
|
|
14
15
|
format: 'json' | 'table'
|
|
15
16
|
}
|
|
16
17
|
export interface PipelineBuildResult { summary: string; skipped?: boolean }
|
|
18
|
+
export interface PipelineDeriveProgress {
|
|
19
|
+
done: number; total: number; reused: number; failed?: number
|
|
20
|
+
phase: PipelineActivityUpdate['phase']; summary: string
|
|
21
|
+
item?: { id: string; title: string; state: PipelineItemState['state'] }
|
|
22
|
+
}
|
|
23
|
+
export interface ExtractionExecution {
|
|
24
|
+
fingerprint: string
|
|
25
|
+
force: boolean
|
|
26
|
+
signal: AbortSignal
|
|
27
|
+
checkpoint: () => Promise<void>
|
|
28
|
+
progress: (text: string, activity?: PipelineActivityUpdate) => void
|
|
29
|
+
}
|
|
17
30
|
export interface PipelineHost {
|
|
18
31
|
capture(): PipelineContext
|
|
19
32
|
serialize<T>(work: () => Promise<T>): Promise<T>
|
|
@@ -21,23 +34,29 @@ export interface PipelineHost {
|
|
|
21
34
|
/** One source by id. Same shape as an element of `sources()`. */
|
|
22
35
|
source(sourceId: string): Promise<PipelineSource | undefined>
|
|
23
36
|
fingerprint(source: PipelineSource, context: PipelineContext): Promise<string>
|
|
24
|
-
extract(source: PipelineSource, context: PipelineContext, output: (text: string) => void): Promise<TrialReport>
|
|
37
|
+
extract(source: PipelineSource, context: PipelineContext, output: (text: string) => void, execution: ExtractionExecution): Promise<TrialReport>
|
|
25
38
|
normalize(source: PipelineSource, checkpoint: ExtractionCheckpoint, context: PipelineContext, force: boolean, output: (text: string) => void): Promise<{ reused: boolean; warnings: string[] }>
|
|
26
|
-
derive?(sourceIds: string[] | undefined, context: PipelineContext, force: boolean, output: (text: string) => void, checkpoint: () => Promise<void
|
|
39
|
+
derive?(sourceIds: string[] | undefined, context: PipelineContext, force: boolean, output: (text: string) => void, checkpoint: () => Promise<void>, progress?: (value: PipelineDeriveProgress) => void): Promise<{ done: number; total: number; reused: number; summary: string }>
|
|
27
40
|
groups(context: PipelineContext): Promise<MaterializationBatch>
|
|
28
41
|
materialize(group: MaterializationGroup, context: PipelineContext, output: (text: string) => void, snapshot: unknown): Promise<void>
|
|
29
|
-
|
|
42
|
+
link?(): Promise<string>
|
|
43
|
+
build(taskId: string, checkpoint: () => Promise<void>, progress?: (text: string) => void): Promise<PipelineBuildResult>
|
|
30
44
|
record(checkpoint: ExtractionCheckpoint): Promise<void>
|
|
31
45
|
}
|
|
32
46
|
|
|
33
47
|
class TaskCancelled extends Error {}
|
|
34
|
-
interface Control { pause: boolean; cancel: boolean; resume?: () => void }
|
|
48
|
+
interface Control { abort: AbortController; pause: boolean; cancel: boolean; resume?: () => void }
|
|
35
49
|
|
|
36
50
|
/** One writer queue per base, shared with the legacy index builder. */
|
|
37
51
|
export class KnowledgePipeline {
|
|
38
52
|
private closed = false
|
|
39
53
|
private controls = new Map<string, Control>()
|
|
40
|
-
constructor(readonly store: PipelineStore, private readonly host: PipelineHost) {
|
|
54
|
+
constructor(readonly store: PipelineStore, private readonly host: PipelineHost) {
|
|
55
|
+
for (const previous of store.tasks(10000).tasks.filter(task => task.recoverableOnRestart && !task.recoveryTaskId)) {
|
|
56
|
+
const resumed = this.start(previous.stage, { ...previous.selection, force: false })
|
|
57
|
+
this.patch(previous.id, { recoverableOnRestart: false, recoveryTaskId: resumed.task.id, summary: '服务重启,已自动创建续跑任务;复用已完成分片。' })
|
|
58
|
+
}
|
|
59
|
+
}
|
|
41
60
|
|
|
42
61
|
start(stage: PipelineStage, selection: PipelineSelection = {}): { task: PipelineTask; completion: Promise<PipelineTask> } {
|
|
43
62
|
if (this.closed) throw new Error('知识处理已关闭')
|
|
@@ -45,15 +64,16 @@ export class KnowledgePipeline {
|
|
|
45
64
|
for (const value of [selection.sourceIds, selection.groupKeys]) {
|
|
46
65
|
if (value !== undefined && (!Array.isArray(value) || value.length === 0 || value.some(id => typeof id !== 'string' || !id.trim()))) throw new Error('选择范围不能为空;不传范围表示处理当前阶段的全部待办')
|
|
47
66
|
}
|
|
48
|
-
const
|
|
67
|
+
const captured = this.host.capture()
|
|
68
|
+
const context = { ...captured, extractionTimeoutMinutes: normalizeExtractionTimeoutMinutes(captured.extractionTimeoutMinutes) }
|
|
49
69
|
const { call: _call, why: _why, ...model } = context.model
|
|
50
70
|
const task: PipelineTask = {
|
|
51
|
-
id: `pipeline_${randomUUID()}`, stage, state: 'queued', selection: structuredClone(selection), model,
|
|
71
|
+
id: `pipeline_${randomUUID()}`, stage, state: 'queued', selection: structuredClone(selection), model, extractionTimeoutMinutes: context.extractionTimeoutMinutes,
|
|
52
72
|
queuedAt: new Date().toISOString(), done: 0, total: 0, reused: 0, failed: 0,
|
|
53
73
|
summary: '等待运行', warnings: [],
|
|
54
74
|
}
|
|
55
75
|
this.store.putTask(task)
|
|
56
|
-
this.controls.set(task.id, { pause: false, cancel: false })
|
|
76
|
+
this.controls.set(task.id, { abort: new AbortController(), pause: false, cancel: false })
|
|
57
77
|
const completion = this.host.serialize(() => this.execute(task.id, context))
|
|
58
78
|
// Starting a background task must not create an unhandled rejection.
|
|
59
79
|
void completion.catch(() => undefined)
|
|
@@ -62,7 +82,7 @@ export class KnowledgePipeline {
|
|
|
62
82
|
|
|
63
83
|
close(): void {
|
|
64
84
|
this.closed = true
|
|
65
|
-
for (const control of this.controls.values()) { control.cancel = true; control.pause = false; control.resume?.() }
|
|
85
|
+
for (const control of this.controls.values()) { control.abort.abort(new TaskCancelled('知识库已关闭,已完成结果保留')); control.cancel = true; control.pause = false; control.resume?.() }
|
|
66
86
|
}
|
|
67
87
|
|
|
68
88
|
control(id: string, action: PipelineAction): PipelineTask {
|
|
@@ -71,17 +91,18 @@ export class KnowledgePipeline {
|
|
|
71
91
|
if (!task || !control || !PIPELINE_ACTIVE_STATES.includes(task.state)) throw new Error('任务已结束;请重新运行所需阶段')
|
|
72
92
|
if (action === 'pause') {
|
|
73
93
|
control.pause = true
|
|
74
|
-
this.patch(id, { state: task.state === 'queued' ? 'paused' : 'pausing', summary: '
|
|
94
|
+
this.patch(id, { state: task.state === 'queued' ? 'paused' : 'pausing', summary: '已请求暂停,当前步骤到达安全点后生效;已保存结果保留' })
|
|
75
95
|
} else if (action === 'resume') {
|
|
76
96
|
if (!control.pause) throw new Error('任务没有暂停')
|
|
77
97
|
control.pause = false
|
|
78
|
-
this.patch(id, { state: task.startedAt ? 'running' : 'queued' })
|
|
98
|
+
this.patch(id, { state: task.startedAt ? 'running' : 'queued', summary: task.startedAt ? '已继续运行' : '等待运行' })
|
|
79
99
|
control.resume?.()
|
|
80
100
|
control.resume = undefined
|
|
81
101
|
} else {
|
|
102
|
+
control.abort.abort(new TaskCancelled('任务已停止,已完成分片保留'))
|
|
82
103
|
control.cancel = true
|
|
83
104
|
control.pause = false
|
|
84
|
-
this.patch(id, { state: 'cancelling', summary: '
|
|
105
|
+
this.patch(id, { state: 'cancelling', summary: '已请求停止,正在结束当前步骤;已保存结果保留' })
|
|
85
106
|
control.resume?.()
|
|
86
107
|
}
|
|
87
108
|
return this.store.task(id)!
|
|
@@ -92,7 +113,16 @@ export class KnowledgePipeline {
|
|
|
92
113
|
this.store.putTask(task)
|
|
93
114
|
return task
|
|
94
115
|
}
|
|
95
|
-
private
|
|
116
|
+
private observe(id: string, activity: PipelineActivityUpdate, text?: string): void {
|
|
117
|
+
const observed = { ...activity, at: new Date().toISOString() }
|
|
118
|
+
const task = this.patch(id, { activity: observed, ...(text ? { summary: text } : {}) })
|
|
119
|
+
if (text) this.store.event(id, 'progress', text, task.currentStage === 'extract' || task.currentStage === 'normalize' ? activity.objectId : undefined, task.currentStage, observed)
|
|
120
|
+
}
|
|
121
|
+
private item(id: string, stage: PipelineWorkStage, key: string, value: PipelineItemState): void {
|
|
122
|
+
const task = this.store.task(id)!
|
|
123
|
+
this.patch(id, { items: { ...task.items, [stage]: { ...task.items?.[stage], [key]: value } } })
|
|
124
|
+
}
|
|
125
|
+
private finishStage(id: string, outcome?: Pick<PipelineBuildResult, 'summary' | 'skipped'> & { blocked?: number }): void {
|
|
96
126
|
const task = this.store.task(id)!
|
|
97
127
|
if (task.currentStage) this.patch(id, { results: { ...task.results, [task.currentStage]: { done: task.done, total: task.total, reused: task.reused, failed: task.failed, ...outcome } } })
|
|
98
128
|
}
|
|
@@ -113,6 +143,9 @@ export class KnowledgePipeline {
|
|
|
113
143
|
const flush = (): void => {
|
|
114
144
|
if (!buffer) return
|
|
115
145
|
this.store.event(id, 'output', buffer, sourceId, stage)
|
|
146
|
+
const activity = this.store.task(id)?.activity
|
|
147
|
+
// Persist liveness at most once a second; raw output remains in the event stream.
|
|
148
|
+
if (activity && Date.now() - Date.parse(activity.at) >= 1000) this.observe(id, { ...activity, phase: 'receiving' })
|
|
116
149
|
buffer = ''
|
|
117
150
|
last = Date.now()
|
|
118
151
|
}
|
|
@@ -151,20 +184,27 @@ export class KnowledgePipeline {
|
|
|
151
184
|
for (const stage of stages) {
|
|
152
185
|
await this.checkpoint(id)
|
|
153
186
|
this.patch(id, { currentStage: stage, done: 0, total: 0, reused: 0, failed: 0, summary: PIPELINE_STAGE_LABEL[stage] })
|
|
154
|
-
this.
|
|
187
|
+
this.observe(id, { phase: 'preparing' }, `开始${PIPELINE_STAGE_LABEL[stage]}`)
|
|
155
188
|
if (stage === 'derive') {
|
|
156
189
|
if (!this.host.derive) throw new Error('当前运行时不支持 Case 提炼')
|
|
157
190
|
const output = this.output(id, stage)
|
|
191
|
+
this.observe(id, { phase: 'processing' }, '正在检查 Case 并提炼可复用方法')
|
|
158
192
|
try {
|
|
159
|
-
const result = await this.host.derive(task.selection.sourceIds, context, Boolean(task.selection.force), output.append, () => this.checkpoint(id)
|
|
193
|
+
const result = await this.host.derive(task.selection.sourceIds, context, Boolean(task.selection.force), output.append, () => this.checkpoint(id), value => {
|
|
194
|
+
this.patch(id, { done: value.done, total: value.total, reused: value.reused, failed: value.failed ?? 0 })
|
|
195
|
+
if (value.item) this.item(id, stage, value.item.id, { title: value.item.title, state: value.item.state })
|
|
196
|
+
this.observe(id, { phase: value.phase, objectId: value.item?.id, objectTitle: value.item?.title }, value.summary)
|
|
197
|
+
})
|
|
160
198
|
this.patch(id, result)
|
|
161
199
|
this.finishStage(id, result)
|
|
162
200
|
} finally { output.flush() }
|
|
163
201
|
continue
|
|
164
202
|
}
|
|
165
203
|
if (stage === 'build') {
|
|
166
|
-
|
|
204
|
+
this.observe(id, { phase: 'processing' }, '正在构建并发布检索索引')
|
|
205
|
+
const result = await this.host.build(id, () => this.checkpoint(id), text => this.observe(id, { phase: 'processing' }, text))
|
|
167
206
|
this.patch(id, { done: 1, total: 1, summary: result.summary })
|
|
207
|
+
this.observe(id, { phase: 'completed' }, result.summary)
|
|
168
208
|
this.finishStage(id, result)
|
|
169
209
|
continue
|
|
170
210
|
}
|
|
@@ -186,10 +226,13 @@ export class KnowledgePipeline {
|
|
|
186
226
|
* 「坐标有歧义」与「来源变了、抽取结果还没归一」是两件事,报同一句话会让读的人
|
|
187
227
|
* 去查错的地方。更要紧的是范围:按 `sourceIds` / `groupKeys` 缩小的那一趟,
|
|
188
228
|
* 如果拿全库的数字报 `partial`,调用方会去重试一件它明确排除的事 —— 而它其实
|
|
189
|
-
*
|
|
190
|
-
*
|
|
229
|
+
* 全都做完了。被阻断的 Knowledge 保留来源归属和可确定的分组键,因此也按
|
|
230
|
+
* 同一选择范围计数;尚无坐标的行只参与来源范围与全库统计。
|
|
191
231
|
*/
|
|
192
|
-
const ambiguous = scoped
|
|
232
|
+
const ambiguous = scoped
|
|
233
|
+
? (batch.blockedItems ?? []).filter(item => (!affectedSources || item.sourceIds.some(id => affectedSources.has(id)))
|
|
234
|
+
&& (!selected || (item.groupKey !== undefined && selected.has(item.groupKey)))).length
|
|
235
|
+
: batch.blockedAmbiguous
|
|
193
236
|
const awaiting = scoped
|
|
194
237
|
? groups.filter(group => group.state === 'blocked' && inScope(group)).reduce((sum, group) => sum + group.knowledgeCount, 0)
|
|
195
238
|
: batch.blockedAwaitingNormalize
|
|
@@ -207,19 +250,27 @@ export class KnowledgePipeline {
|
|
|
207
250
|
this.patch(id, { total: pending.length })
|
|
208
251
|
for (const group of pending) {
|
|
209
252
|
await this.checkpoint(id)
|
|
253
|
+
this.item(id, stage, group.key, { state: 'processing', title: group.title })
|
|
254
|
+
this.observe(id, { phase: 'processing', objectId: group.key, objectTitle: group.title }, `正在物化:${group.title}`)
|
|
210
255
|
const output = this.output(id, stage)
|
|
256
|
+
let itemError: string | undefined
|
|
211
257
|
try { await this.host.materialize(group, context, output.append, batch.snapshot) }
|
|
212
258
|
catch (cause) {
|
|
213
259
|
if (cause instanceof TaskCancelled) throw cause
|
|
214
260
|
const message = `${group.title}:${cause instanceof Error ? cause.message : String(cause)}`
|
|
261
|
+
itemError = message
|
|
215
262
|
const current = this.store.task(id)!
|
|
216
263
|
this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
|
|
217
264
|
this.store.event(id, 'warning', message, undefined, stage)
|
|
218
265
|
}
|
|
219
266
|
finally { output.flush() }
|
|
267
|
+
this.item(id, stage, group.key, { state: itemError ? 'failed' : 'succeeded', title: group.title, summary: itemError })
|
|
268
|
+
this.observe(id, { phase: itemError ? 'failed' : 'completed', objectId: group.key, objectTitle: group.title }, itemError ?? `已保存卡片:${group.title}`)
|
|
220
269
|
const current = this.store.task(id)!
|
|
221
270
|
this.patch(id, { done: current.done + 1, summary: `已物化 ${current.done + 1}/${pending.length} 张:${group.title}` })
|
|
222
271
|
}
|
|
272
|
+
await this.checkpoint(id)
|
|
273
|
+
if (this.host.link) this.store.event(id, 'output', await this.host.link(), undefined, stage)
|
|
223
274
|
const current = this.store.task(id)!
|
|
224
275
|
/**
|
|
225
276
|
* 阶段结论要给 `summary`:任务列表的四段步轨读的是 `results[stage]`,
|
|
@@ -227,6 +278,7 @@ export class KnowledgePipeline {
|
|
|
227
278
|
* 同一件事在半个屏幕上两种说法。
|
|
228
279
|
*/
|
|
229
280
|
this.finishStage(id, {
|
|
281
|
+
blocked: deferred,
|
|
230
282
|
summary: `已物化 ${current.done - current.failed}/${current.total} 张${deferred > 0 ? `;${deferred} 条待归一` : ''}${affectedSources !== undefined ? `(按来源范围缩小:${affectedSources.size} 份来源)` : ''}`,
|
|
231
283
|
})
|
|
232
284
|
/**
|
|
@@ -242,16 +294,23 @@ export class KnowledgePipeline {
|
|
|
242
294
|
this.patch(id, { total: sources.length })
|
|
243
295
|
for (const source of sources) {
|
|
244
296
|
await this.checkpoint(id)
|
|
245
|
-
this.
|
|
297
|
+
this.item(id, stage, source.sourceId, { state: 'processing', title: source.title })
|
|
298
|
+
this.observe(id, { phase: 'processing', objectId: source.sourceId, objectTitle: source.title }, `${PIPELINE_STAGE_LABEL[stage]}:${source.title}`)
|
|
246
299
|
const output = this.output(id, stage, source.sourceId)
|
|
247
300
|
let reused = false
|
|
301
|
+
let itemError: string | undefined
|
|
248
302
|
try {
|
|
249
303
|
if (stage === 'extract') {
|
|
250
304
|
const fingerprint = await this.host.fingerprint(source, context)
|
|
251
305
|
const cached = task.selection.force ? undefined : this.store.cached(source.sourceId, source.version, fingerprint)
|
|
252
306
|
if (cached) { this.store.selectExtraction(source.sourceId, cached.id); reused = true }
|
|
253
307
|
else {
|
|
254
|
-
const report = await this.host.extract(source, context, output.append
|
|
308
|
+
const report = await this.host.extract(source, context, output.append, {
|
|
309
|
+
fingerprint, force: task.selection.force === true, signal: this.controls.get(id)!.abort.signal,
|
|
310
|
+
checkpoint: () => this.checkpoint(id), progress: (text, activity) => {
|
|
311
|
+
this.observe(id, activity ?? { phase: 'processing', objectId: source.sourceId, objectTitle: source.title }, text)
|
|
312
|
+
},
|
|
313
|
+
})
|
|
255
314
|
if (this.closed) throw new TaskCancelled('知识库已关闭,已完成结果保留')
|
|
256
315
|
const checkpoint: ExtractionCheckpoint = { id: `extract_${randomUUID()}`, sourceId: source.sourceId, sourceVersion: source.version, sourceHash: source.sourceHash, fingerprint, report, createdAt: new Date().toISOString() }
|
|
257
316
|
// The host checks the captured source version again before activation.
|
|
@@ -280,6 +339,7 @@ export class KnowledgePipeline {
|
|
|
280
339
|
const detail = report.exceptions.slice(0, 3).map(item => `${item.step}:${item.message}`).join(';')
|
|
281
340
|
throw new Error(`${report.model.why ?? '抽取返回未通过校验'}${detail === '' ? '' : `(${detail})`}`)
|
|
282
341
|
}
|
|
342
|
+
this.observe(id, { ...this.store.task(id)!.activity!, phase: 'saving' }, '正在提交文档抽取结果')
|
|
283
343
|
this.store.saveExtraction(checkpoint)
|
|
284
344
|
}
|
|
285
345
|
} else {
|
|
@@ -294,10 +354,13 @@ export class KnowledgePipeline {
|
|
|
294
354
|
} catch (cause) {
|
|
295
355
|
if (cause instanceof TaskCancelled) throw cause
|
|
296
356
|
const message = `${source.title}:${cause instanceof Error ? cause.message : String(cause)}`
|
|
357
|
+
itemError = message
|
|
297
358
|
const current = this.store.task(id)!
|
|
298
359
|
this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
|
|
299
360
|
this.store.event(id, 'warning', message, source.sourceId, stage)
|
|
300
361
|
} finally { output.flush() }
|
|
362
|
+
this.item(id, stage, source.sourceId, { state: itemError ? 'failed' : reused ? 'reused' : 'succeeded', title: source.title, summary: itemError })
|
|
363
|
+
this.observe(id, { ...this.store.task(id)!.activity!, phase: itemError ? 'failed' : reused ? 'reused' : 'completed' }, itemError ?? `${source.title}:${reused ? '复用已有结果' : '处理完成'}`)
|
|
301
364
|
const current = this.store.task(id)!
|
|
302
365
|
this.patch(id, { done: current.done + 1, reused: current.reused + Number(reused), summary: `${PIPELINE_STAGE_LABEL[stage]} ${current.done + 1}/${sources.length}${reused ? ' · 复用已有结果' : ''}` })
|
|
303
366
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_ORDER, type PipelineState, type PipelineTask, type PipelineWorkStage } from './types.js'
|
|
2
|
+
|
|
3
|
+
export const PIPELINE_STATE_LABEL: Record<PipelineState, string> = {
|
|
4
|
+
queued: '排队中', running: '进行中', pausing: '等待暂停', paused: '已暂停', cancelling: '正在停止',
|
|
5
|
+
succeeded: '已完成', partial: '部分完成', failed: '失败', cancelled: '已停止', interrupted: '已中断',
|
|
6
|
+
}
|
|
7
|
+
export interface TaskStageObservation {
|
|
8
|
+
label: string
|
|
9
|
+
tone: 'neutral' | 'success' | 'warning' | 'blue'
|
|
10
|
+
detail: string
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** A stage observation always belongs to one execution, never a mixture of past runs. */
|
|
14
|
+
export function observeTaskStage(task: PipelineTask, stage: PipelineWorkStage): TaskStageObservation {
|
|
15
|
+
if (task.stage !== 'all' && task.stage !== stage) return { label: '本次未运行', tone: 'neutral', detail: '此阶段不在本次任务范围内。' }
|
|
16
|
+
const result = task.results?.[stage]
|
|
17
|
+
if (result) {
|
|
18
|
+
const counts = `成功 ${Math.max(0, result.done - result.failed)}/${result.total}${result.reused ? ` · 复用 ${result.reused}` : ''}${result.failed ? ` · 失败 ${result.failed}` : ''}`
|
|
19
|
+
if (result.failed || result.blocked) return { label: result.failed ? '部分失败' : '有暂缓项', tone: 'warning', detail: `${result.summary ?? counts}${result.blocked ? ` · 暂缓 ${result.blocked} 条` : ''}` }
|
|
20
|
+
if (result.skipped || result.total === 0) return { label: '已跳过', tone: 'neutral', detail: result.summary ?? '本次没有需要处理的内容。' }
|
|
21
|
+
return { label: result.reused === result.total && stage !== 'build' ? '全部复用' : '已完成', tone: 'success', detail: result.summary ?? counts }
|
|
22
|
+
}
|
|
23
|
+
const current = task.currentStage ?? (task.stage === 'all' ? 'extract' : task.stage)
|
|
24
|
+
if (current === stage) return {
|
|
25
|
+
label: PIPELINE_STATE_LABEL[task.state],
|
|
26
|
+
tone: ['failed', 'partial', 'interrupted'].includes(task.state) ? 'warning' : task.state === 'paused' || task.state === 'cancelled' ? 'neutral' : task.state === 'succeeded' ? 'success' : 'blue',
|
|
27
|
+
detail: task.total > 0 ? `已处理 ${task.done}/${task.total} · 失败 ${task.failed} · 复用 ${task.reused}` : task.summary,
|
|
28
|
+
}
|
|
29
|
+
if (PIPELINE_STAGE_ORDER.indexOf(stage) < PIPELINE_STAGE_ORDER.indexOf(current)) return { label: '已跳过', tone: 'neutral', detail: '本次运行时未执行此阶段。' }
|
|
30
|
+
return PIPELINE_ACTIVE_STATES.includes(task.state)
|
|
31
|
+
? { label: '等待执行', tone: 'neutral', detail: '本次任务到达此阶段后自动运行。' }
|
|
32
|
+
: { label: '未执行', tone: 'neutral', detail: '任务在到达此阶段前已结束。' }
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Running work wins over newer queued work; queue order remains FIFO. */
|
|
36
|
+
export function orderActiveTasks(tasks: PipelineTask[]): PipelineTask[] {
|
|
37
|
+
return tasks.filter(task => PIPELINE_ACTIVE_STATES.includes(task.state)).sort((left, right) =>
|
|
38
|
+
Number(left.state === 'queued' || !left.startedAt) - Number(right.state === 'queued' || !right.startedAt)
|
|
39
|
+
|| left.queuedAt.localeCompare(right.queuedAt))
|
|
40
|
+
}
|