dsh-data-domain-knowledge 0.1.1-alpha.3 → 0.1.1-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/README.md +11 -1
  2. package/lib/client.js +736 -1546
  3. package/lib/doc-page.js +7 -7
  4. package/lib/pages.js +3042 -1396
  5. package/lib/types/agent-behavior-rules.d.ts.map +1 -1
  6. package/lib/types/agent-behavior-rules.js +2 -0
  7. package/lib/types/agent-behavior-rules.js.map +1 -1
  8. package/lib/types/convert/host.d.ts.map +1 -1
  9. package/lib/types/convert/host.js +7 -1
  10. package/lib/types/convert/host.js.map +1 -1
  11. package/lib/types/convert/ingest.d.ts +12 -20
  12. package/lib/types/convert/ingest.d.ts.map +1 -1
  13. package/lib/types/convert/ingest.js +7 -5
  14. package/lib/types/convert/ingest.js.map +1 -1
  15. package/lib/types/diagnosis/store.d.ts +2 -2
  16. package/lib/types/diagnosis/store.d.ts.map +1 -1
  17. package/lib/types/diagnosis/store.js +10 -2
  18. package/lib/types/diagnosis/store.js.map +1 -1
  19. package/lib/types/extract-model.d.ts +3 -0
  20. package/lib/types/extract-model.d.ts.map +1 -1
  21. package/lib/types/extract-model.js.map +1 -1
  22. package/lib/types/index.d.ts.map +1 -1
  23. package/lib/types/index.js +75 -56
  24. package/lib/types/index.js.map +1 -1
  25. package/lib/types/knowledge-bases.d.ts +3 -0
  26. package/lib/types/knowledge-bases.d.ts.map +1 -1
  27. package/lib/types/knowledge-bases.js +22 -2
  28. package/lib/types/knowledge-bases.js.map +1 -1
  29. package/lib/types/model-text.d.ts +7 -0
  30. package/lib/types/model-text.d.ts.map +1 -1
  31. package/lib/types/model-text.js +31 -9
  32. package/lib/types/model-text.js.map +1 -1
  33. package/lib/types/pipeline/engine.d.ts +27 -4
  34. package/lib/types/pipeline/engine.d.ts.map +1 -1
  35. package/lib/types/pipeline/engine.js +59 -12
  36. package/lib/types/pipeline/engine.js.map +1 -1
  37. package/lib/types/pipeline/progress.d.ts +12 -0
  38. package/lib/types/pipeline/progress.d.ts.map +1 -0
  39. package/lib/types/pipeline/progress.js +37 -0
  40. package/lib/types/pipeline/progress.js.map +1 -0
  41. package/lib/types/pipeline/resumable-extract.d.ts +45 -0
  42. package/lib/types/pipeline/resumable-extract.d.ts.map +1 -0
  43. package/lib/types/pipeline/resumable-extract.js +262 -0
  44. package/lib/types/pipeline/resumable-extract.js.map +1 -0
  45. package/lib/types/pipeline/store.d.ts +8 -2
  46. package/lib/types/pipeline/store.d.ts.map +1 -1
  47. package/lib/types/pipeline/store.js +27 -5
  48. package/lib/types/pipeline/store.js.map +1 -1
  49. package/lib/types/pipeline/types.d.ts +50 -0
  50. package/lib/types/pipeline/types.d.ts.map +1 -1
  51. package/lib/types/pipeline/types.js +18 -1
  52. package/lib/types/pipeline/types.js.map +1 -1
  53. package/lib/types/retrieval/scope.d.ts +17 -0
  54. package/lib/types/retrieval/scope.d.ts.map +1 -0
  55. package/lib/types/retrieval/scope.js +11 -0
  56. package/lib/types/retrieval/scope.js.map +1 -0
  57. package/lib/types/runtime.d.ts +55 -8
  58. package/lib/types/runtime.d.ts.map +1 -1
  59. package/lib/types/runtime.js +252 -121
  60. package/lib/types/runtime.js.map +1 -1
  61. package/lib/types/source-identity.d.ts +31 -0
  62. package/lib/types/source-identity.d.ts.map +1 -1
  63. package/lib/types/source-identity.js +80 -0
  64. package/lib/types/source-identity.js.map +1 -1
  65. package/lib/types/source-read.d.ts +18 -0
  66. package/lib/types/source-read.d.ts.map +1 -0
  67. package/lib/types/source-read.js +49 -0
  68. package/lib/types/source-read.js.map +1 -0
  69. package/lib/types/store/knowledge-db.d.ts +5 -29
  70. package/lib/types/store/knowledge-db.d.ts.map +1 -1
  71. package/lib/types/store/knowledge-db.js +42 -1
  72. package/lib/types/store/knowledge-db.js.map +1 -1
  73. package/lib/types/store/source-identity.d.ts +8 -0
  74. package/lib/types/store/source-identity.d.ts.map +1 -0
  75. package/lib/types/store/source-identity.js +56 -0
  76. package/lib/types/store/source-identity.js.map +1 -0
  77. package/lib/types/types.d.ts +41 -1
  78. package/lib/types/types.d.ts.map +1 -1
  79. package/lib/types/types.js.map +1 -1
  80. package/lib/types/wiki-trial-types.d.ts +3 -0
  81. package/lib/types/wiki-trial-types.d.ts.map +1 -1
  82. package/lib/types/wiki-trial.d.ts +3 -3
  83. package/lib/types/wiki-trial.d.ts.map +1 -1
  84. package/lib/types/wiki-trial.js +14 -3
  85. package/lib/types/wiki-trial.js.map +1 -1
  86. package/package.json +1 -1
  87. package/src/agent-behavior-rules.ts +2 -0
  88. package/src/client/AxisRegistrySection.tsx +1 -1
  89. package/src/client/CompileSettingsView.tsx +473 -0
  90. package/src/client/ExtractionTasksPage.tsx +557 -423
  91. package/src/client/KnowledgeBaseTab.tsx +28 -109
  92. package/src/client/PipelineRunView.tsx +173 -0
  93. package/src/client/pages.css +303 -178
  94. package/src/client/pages.tsx +3 -1
  95. package/src/convert/host.ts +7 -1
  96. package/src/convert/ingest.ts +17 -7
  97. package/src/diagnosis/store.ts +7 -2
  98. package/src/extract-model.ts +3 -0
  99. package/src/index.ts +73 -56
  100. package/src/knowledge-bases.ts +23 -3
  101. package/src/model-text.ts +33 -11
  102. package/src/pipeline/engine.ts +76 -19
  103. package/src/pipeline/progress.ts +40 -0
  104. package/src/pipeline/resumable-extract.ts +233 -0
  105. package/src/pipeline/store.ts +28 -6
  106. package/src/pipeline/types.ts +45 -2
  107. package/src/retrieval/scope.ts +10 -0
  108. package/src/runtime.ts +222 -134
  109. package/src/source-identity.ts +81 -0
  110. package/src/source-read.ts +47 -0
  111. package/src/store/knowledge-db.ts +41 -1
  112. package/src/store/source-identity.ts +42 -0
  113. package/src/types.ts +12 -1
  114. package/src/wiki-trial-types.ts +3 -2
  115. package/src/wiki-trial.ts +17 -6
  116. package/src/client/TypeDefinitionDrawer.tsx +0 -195
@@ -31,6 +31,7 @@ import {
31
31
  import { defaultKnowledgeRoot, failure, KnowledgeRuntime, success, type KnowledgeRuntimeOptions } from './runtime.js'
32
32
  import { defaultAgentRoot, knowledgeBaseRoot, memoryRoot, resourceRoot } from './layout.js'
33
33
  import { writeTextAtomic } from './fs-atomic.js'
34
+ import { DEFAULT_EXTRACTION_TIMEOUT_MINUTES, isExtractionTimeoutMinutes, normalizeExtractionTimeoutMinutes, type ExtractionTimeoutMinutes } from './pipeline/types.js'
34
35
 
35
36
  /** 持久化形态:配置 + 计算字段缓存分区。 */
36
37
  interface KnowledgeBaseRecord {
@@ -49,6 +50,7 @@ interface KnowledgeBaseRecord {
49
50
  extractionNote: string
50
51
  /** 编译模型路由;它是注册表配置,不属于知识正文。 */
51
52
  compileModel: CompileModelSelection
53
+ extractionTimeoutMinutes: ExtractionTimeoutMinutes
52
54
  createdAt: string
53
55
  updatedAt: string
54
56
  }
@@ -120,12 +122,13 @@ function textOr(input: unknown, fallback: string): string {
120
122
  function compileModelOf(input: unknown): CompileModelSelection {
121
123
  const value = record(input)
122
124
  const effort = typeof value.reasoningEffort === 'string' && value.reasoningEffort.trim() ? { reasoningEffort: value.reasoningEffort.trim() } : {}
123
- if (value.mode !== 'fixed') return { ...DEFAULT_COMPILE_MODEL, ...effort }
125
+ const output = typeof value.maxTokens === 'number' && Number.isSafeInteger(value.maxTokens) && value.maxTokens > 0 ? { maxTokens: value.maxTokens } : {}
126
+ if (value.mode !== 'fixed') return { ...DEFAULT_COMPILE_MODEL, ...effort, ...output }
124
127
  const provider = typeof value.provider === 'string' ? value.provider.trim() : ''
125
128
  const model = typeof value.model === 'string' ? value.model.trim() : ''
126
129
  // 旧版本没有这个字段,或留下了半截配置时,安全地回到 DSH 默认模型。
127
130
  if (!provider || !model) return { ...DEFAULT_COMPILE_MODEL }
128
- return { mode: 'fixed', provider, model, ...effort }
131
+ return { mode: 'fixed', provider, model, ...effort, ...output }
129
132
  }
130
133
 
131
134
  function normalizeExt(value: unknown, fallback: string): string {
@@ -140,6 +143,7 @@ function emptyBase(id: string): KnowledgeBase {
140
143
  id, name: id, description: '', rootDir: '', isDefault: false,
141
144
  extractionNote: '',
142
145
  compileModel: { ...DEFAULT_COMPILE_MODEL },
146
+ extractionTimeoutMinutes: DEFAULT_EXTRACTION_TIMEOUT_MINUTES,
143
147
  createdAt: '', updatedAt: '', sourceCount: 0, cardCount: 0, noteCount: 0, buildVersion: 0,
144
148
  }
145
149
  }
@@ -243,6 +247,7 @@ export class KnowledgeBases {
243
247
  // 卡片模板是系统固定契约,不按知识库维护注册表。
244
248
  // `KnowledgeRuntime` 自己提供默认模板。
245
249
  extractionNote: () => this.extractionNoteOf(base.id),
250
+ extractionTimeoutMinutes: () => this.extractionTimeoutMinutesOf(base.id),
246
251
  // 0032:记忆段跟着**这个库**走,二级用 id。`agentRoot` 没给时(只测注册表
247
252
  // 逻辑的单元测试)留空,runtime 会退回知识树里的 `04_memory`。
248
253
  ...(this.options.agentRoot ? { memoryRoot: memoryRoot(this.options.agentRoot(), base.id) } : {}),
@@ -338,6 +343,7 @@ export class KnowledgeBases {
338
343
  isDefault: false,
339
344
  extractionNote: '',
340
345
  compileModel: { ...DEFAULT_COMPILE_MODEL },
346
+ extractionTimeoutMinutes: DEFAULT_EXTRACTION_TIMEOUT_MINUTES,
341
347
  createdAt: now,
342
348
  updatedAt: now,
343
349
  }
@@ -351,14 +357,20 @@ export class KnowledgeBases {
351
357
  * 更新名称 / 描述 / 坐标提取备注。
352
358
  *
353
359
  */
354
- async update(id: string, patch: { name?: string; description?: string; extractionNote?: string; compileModel?: CompileModelSelection }): Promise<Envelope<KnowledgeBase>> {
360
+ async update(id: string, patch: { name?: string; description?: string; extractionNote?: string; compileModel?: CompileModelSelection; extractionTimeoutMinutes?: ExtractionTimeoutMinutes }): Promise<Envelope<KnowledgeBase>> {
355
361
  this.ensureLoaded()
356
362
  const base = this.records.find(item => item.id === id)
357
363
  if (!base) return failure(emptyBase(id), 'BASE_NOT_FOUND', `知识库不存在: ${id}`, true)
358
364
  // 守卫必须基于精确的实时计数,不能用可能过期的缓存。
359
365
  await this.refreshOne(base)
360
366
  let nextCompileModel = base.compileModel
367
+ if (patch.extractionTimeoutMinutes !== undefined && !isExtractionTimeoutMinutes(patch.extractionTimeoutMinutes)) {
368
+ return failure(this.view(base), 'EXTRACTION_TIMEOUT_INVALID', '文档抽取时限只能选择 30、60 或 100 分钟,最大 100 分钟。', false)
369
+ }
361
370
  if (patch.compileModel !== undefined) {
371
+ if (patch.compileModel.maxTokens !== undefined && (!Number.isSafeInteger(patch.compileModel.maxTokens) || patch.compileModel.maxTokens <= 0)) {
372
+ return failure(this.view(base), 'COMPILE_MODEL_INVALID', '单次最大输出必须是正整数 token;留空表示跟随模型默认。', false)
373
+ }
362
374
  if (patch.compileModel.mode !== 'default' && patch.compileModel.mode !== 'fixed') {
363
375
  return failure(this.view(base), 'COMPILE_MODEL_INVALID', '编译模型 mode 只能是 default 或 fixed', false, '从编译模型选择项中选择“跟随 DSH 默认模型”或一个固定模型。')
364
376
  }
@@ -377,6 +389,7 @@ export class KnowledgeBases {
377
389
  // 长度上限不是配额,是防呆:这条备注逐字进每一次抽取的提示词,几万字会把文档挤出去。
378
390
  if (patch.extractionNote !== undefined) base.extractionNote = String(patch.extractionNote).slice(0, 4000)
379
391
  base.compileModel = nextCompileModel
392
+ if (patch.extractionTimeoutMinutes !== undefined) base.extractionTimeoutMinutes = patch.extractionTimeoutMinutes
380
393
  base.updatedAt = new Date().toISOString()
381
394
  await this.refreshOne(base)
382
395
  await this.persist()
@@ -391,6 +404,10 @@ export class KnowledgeBases {
391
404
  this.ensureLoaded()
392
405
  return this.records.find(item => item.id === id)?.extractionNote ?? ''
393
406
  }
407
+ extractionTimeoutMinutesOf(id: string): ExtractionTimeoutMinutes {
408
+ this.ensureLoaded()
409
+ return normalizeExtractionTimeoutMinutes(this.records.find(item => item.id === id)?.extractionTimeoutMinutes)
410
+ }
394
411
 
395
412
  /** 当前知识库的编译模型路由。不存在的库回到默认路由,避免读路径抛错。 */
396
413
  compileModelOf(id: string): CompileModelSelection {
@@ -443,6 +460,7 @@ export class KnowledgeBases {
443
460
  isDefault: base.isDefault,
444
461
  extractionNote: base.extractionNote,
445
462
  compileModel: { ...base.compileModel },
463
+ extractionTimeoutMinutes: base.extractionTimeoutMinutes,
446
464
  createdAt: base.createdAt,
447
465
  updatedAt: base.updatedAt,
448
466
  sourceCount: stats?.sourceCount ?? 0,
@@ -462,6 +480,7 @@ export class KnowledgeBases {
462
480
  isDefault: true,
463
481
  extractionNote: '',
464
482
  compileModel: { ...DEFAULT_COMPILE_MODEL },
483
+ extractionTimeoutMinutes: DEFAULT_EXTRACTION_TIMEOUT_MINUTES,
465
484
  createdAt: now,
466
485
  updatedAt: now,
467
486
  }
@@ -524,6 +543,7 @@ export class KnowledgeBases {
524
543
  // 0068 之前存下来的注册表里没有这个键:算空串,不报错,也不影响别的字段。
525
544
  extractionNote: typeof value.extractionNote === 'string' ? value.extractionNote : '',
526
545
  compileModel: compileModelOf(value.compileModel),
546
+ extractionTimeoutMinutes: normalizeExtractionTimeoutMinutes(value.extractionTimeoutMinutes),
527
547
  createdAt: typeof value.createdAt === 'string' && value.createdAt ? value.createdAt : now,
528
548
  updatedAt: typeof value.updatedAt === 'string' && value.updatedAt ? value.updatedAt : now,
529
549
  }
package/src/model-text.ts CHANGED
@@ -32,7 +32,7 @@ interface StreamChunkLike {
32
32
  type?: string
33
33
  text?: string
34
34
  usage?: TrialTokenUsage
35
- reason?: { kind?: string; failure?: { code?: string; message?: string } }
35
+ reason?: { kind?: string; failure?: { code?: string; message?: string; status?: number; requestId?: string; retryAfterMs?: number } }
36
36
  }
37
37
 
38
38
  /**
@@ -69,21 +69,24 @@ export function buildModelRequest(input: {
69
69
  system?: string
70
70
  prompt: string
71
71
  reasoningEffort?: string
72
+ maxTokens?: number
72
73
  /** 路由与缓存亲和的会话身份。知识编译用 `knowledge:<库 id>`。 */
73
74
  sessionId: string
74
75
  }): {
75
76
  provider: string
76
77
  model: string
77
78
  reasoningEffort?: string
79
+ maxTokens?: number
78
80
  system?: string
79
81
  sessionId: string
80
82
  messages: Array<{ role: 'user'; content: Array<{ type: 'text'; text: string }> }>
81
83
  } {
82
- const { provider, model, system, prompt, reasoningEffort, sessionId } = input
84
+ const { provider, model, system, prompt, reasoningEffort, maxTokens, sessionId } = input
83
85
  return {
84
86
  provider,
85
87
  model,
86
88
  ...(reasoningEffort ? { reasoningEffort } : {}),
89
+ ...(maxTokens !== undefined ? { maxTokens } : {}),
87
90
  ...(system ? { system } : {}),
88
91
  sessionId,
89
92
  messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
@@ -106,7 +109,7 @@ export async function collectModelText(
106
109
  let text = ''
107
110
  /** 流怎么结束的。用来区分「正常但空」「失败」「中止」三种情况。 */
108
111
  let finish: string | undefined
109
- let failure: { code?: string; message?: string } | undefined
112
+ let failure: NonNullable<StreamChunkLike['reason']>['failure'] | undefined
110
113
 
111
114
  for await (const chunk of stream) {
112
115
  if (!chunk || typeof chunk !== 'object') continue
@@ -125,8 +128,9 @@ export async function collectModelText(
125
128
  if (failure || finish === 'error' || finish === 'aborted' || finish === 'max-tokens') {
126
129
  const code = failure?.code?.trim() || (finish === 'max-tokens' ? 'OUTPUT_TRUNCATED' : '未知错误码')
127
130
  const message = failure?.message?.trim() || (finish === 'max-tokens' ? '输出达到长度上限,部分返回已保留,请调整模型限制后重试' : '模型服务没有给出说明')
128
- throw new Error(`模型调用失败(${code}):${message}`)
131
+ throw Object.assign(new Error(`模型调用失败(${code}):${message}`), { code, status: failure?.status, requestId: failure?.requestId, retryAfterMs: failure?.retryAfterMs })
129
132
  }
133
+ if (!finish && text.length > 0) throw new Error('Stream ended without finish_reason;部分返回已保留')
130
134
  if (text.length === 0) {
131
135
  // 走到这里说明流正常结束(或者压根没有 finish 块)却没有一个字符。
132
136
  throw new Error(`模型没有返回任何文本(结束原因:${finish ?? '没有收到 finish 块'})`)
@@ -156,7 +160,7 @@ export async function collectModelText(
156
160
  */
157
161
  export function isRetryableModelFailure(error: unknown): boolean {
158
162
  const message = error instanceof Error ? error.message : String(error)
159
- const code = /模型调用失败(([^)]*))/.exec(message)?.[1]?.trim().toUpperCase() ?? ''
163
+ const code = (typeof (error as { code?: unknown })?.code === 'string' ? String((error as { code: string }).code).toUpperCase() : '') || (/模型调用失败(([^)]*))/.exec(message)?.[1]?.trim().toUpperCase() ?? '')
160
164
 
161
165
  // 明确不可重试:配置问题与用户取消。宁可让人去改设置,也不要在错的地方重试。
162
166
  if (['ABORTED', 'INVALID_CREDENTIAL', 'AUTH', 'UNAUTHORIZED', 'NOT_FOUND', 'INVALID_REQUEST', 'CONTEXT_LENGTH', 'OUTPUT_TRUNCATED'].includes(code)) return false
@@ -166,9 +170,10 @@ export function isRetryableModelFailure(error: unknown): boolean {
166
170
 
167
171
  // 没有 code 的情况(旧适配器、或错误被包了一层):按可辨认的措辞判断。
168
172
  if (/没有返回任何文本|没有收到 finish 块|Stream ended without finish_reason/i.test(message)) return true
169
- if (/ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|socket hang up|premature close|other side closed|fetch failed|network error|HTTP 5\d\d|HTTP 429/i.test(message)) return true
173
+ if (/request\s+timed?\s*out|request_timeout|timeout_error|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|socket hang up|premature close|other side closed|fetch failed|network error|HTTP 5\d\d|HTTP 429/i.test(message)) return true
170
174
 
171
- return false
175
+ const status = Number((error as { status?: number })?.status)
176
+ return status === 408 || status === 429 || (status >= 500 && status < 600)
172
177
  }
173
178
 
174
179
  /** 重试策略:最多几次、每次之前等多久。 */
@@ -183,6 +188,8 @@ export interface ModelRetryOptions {
183
188
  notice?: ((info: { attempt: number; attempts: number; error: unknown }) => string | null) | null
184
189
  /** 注入 sleep,测试里不必真的等。 */
185
190
  sleep?: (ms: number) => Promise<void>
191
+ signal?: AbortSignal
192
+ jitter?: boolean
186
193
  }
187
194
 
188
195
  /**
@@ -200,7 +207,13 @@ export async function callModelWithRetry(
200
207
  ): Promise<string> {
201
208
  const attempts = Math.max(1, Math.floor(options.attempts ?? 3))
202
209
  const delays = options.delaysMs ?? [2000, 6000]
203
- const sleep = options.sleep ?? ((ms: number) => new Promise<void>(resolve => { setTimeout(resolve, ms) }))
210
+ const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve, reject) => {
211
+ const done = (): void => { options.signal?.removeEventListener('abort', abort); resolve() }
212
+ const timer = setTimeout(done, ms)
213
+ const abort = (): void => { clearTimeout(timer); reject(options.signal?.reason) }
214
+ options.signal?.addEventListener('abort', abort, { once: true })
215
+ if (options.signal?.aborted) abort()
216
+ }))
204
217
  const notice = options.notice === undefined
205
218
  ? (info: { attempt: number; attempts: number; error: unknown }) =>
206
219
  `\n\n[模型流中断(${info.error instanceof Error ? info.error.message : String(info.error)}),正在重试 ${info.attempt}/${info.attempts - 1}]\n\n`
@@ -208,16 +221,21 @@ export async function callModelWithRetry(
208
221
 
209
222
  let lastError: unknown
210
223
  for (let attempt = 1; attempt <= attempts; attempt++) {
224
+ options.signal?.throwIfAborted()
211
225
  try {
212
226
  return await send(text => options.onDelta?.(text))
213
227
  } catch (error) {
214
228
  lastError = error
215
- if (attempt >= attempts || !isRetryableModelFailure(error)) throw error
216
- const delayMs = delays[Math.min(attempt - 1, delays.length - 1)] ?? 0
229
+ if (!isRetryableModelFailure(error)) throw error
230
+ if (attempt >= attempts) break
231
+ const retryAfter = Number((error as { retryAfterMs?: number })?.retryAfterMs)
232
+ const baseDelay = Number.isFinite(retryAfter) && retryAfter >= 0 ? Math.min(60_000, retryAfter) : delays[Math.min(attempt - 1, delays.length - 1)] ?? 0
233
+ const delayMs = Math.round(baseDelay * (options.jitter === false ? 1 : 0.7 + Math.random() * 0.6))
217
234
  options.onRetry?.({ attempt, attempts, error, delayMs })
218
235
  const line = notice?.({ attempt, attempts, error }) ?? ''
219
236
  if (line) options.onDelta?.(line)
220
237
  await sleep(delayMs)
238
+ options.signal?.throwIfAborted()
221
239
  }
222
240
  }
223
241
  /**
@@ -227,5 +245,9 @@ export async function callModelWithRetry(
227
245
  * 后者查额度、代理和上游状态。不写这一句,新加的重试在界面上等于不存在。
228
246
  */
229
247
  const message = lastError instanceof Error ? lastError.message : String(lastError)
230
- throw new Error(`${message}(已自动重试 ${attempts - 1} 次仍失败;可稍后重试此阶段,或检查账号额度与网络代理)`)
248
+ if (attempts === 1) throw lastError
249
+ const details = lastError as { code?: string; status?: number; requestId?: string; retryAfterMs?: number } | undefined
250
+ throw Object.assign(new Error(`${message}(已自动重试 ${attempts - 1} 次仍失败;可继续未完成部分)`, { cause: lastError }), {
251
+ code: details?.code, status: details?.status, requestId: details?.requestId, retryAfterMs: details?.retryAfterMs,
252
+ })
231
253
  }
@@ -3,17 +3,30 @@ import type { ModelCall } from '../extract-model.js'
3
3
  import type { SourceRaw, WikiTypeDef } from '../types.js'
4
4
  import type { TrialReport } from '../wiki-trial-types.js'
5
5
  import { PipelineStore, type ExtractionCheckpoint } from './store.js'
6
- import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_LABEL, type MaterializationBatch, type MaterializationGroup, type PipelineAction, type PipelineModel, type PipelineSelection, type PipelineStage, type PipelineTask } from './types.js'
6
+ import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_LABEL, normalizeExtractionTimeoutMinutes, type ExtractionTimeoutMinutes, type MaterializationBatch, type MaterializationGroup, type PipelineAction, type PipelineActivityUpdate, type PipelineItemState, type PipelineModel, type PipelineSelection, type PipelineStage, type PipelineTask, type PipelineWorkStage } from './types.js'
7
7
 
8
8
  export interface PipelineSource extends SourceRaw { version: number; sourceHash: string }
9
9
  export interface PipelineContext {
10
10
  model: PipelineModel & { call?: ModelCall; why?: string }
11
11
  note: string
12
+ extractionTimeoutMinutes?: ExtractionTimeoutMinutes
12
13
  profile?: WikiTypeDef
13
14
  organize: boolean
14
15
  format: 'json' | 'table'
15
16
  }
16
17
  export interface PipelineBuildResult { summary: string; skipped?: boolean }
18
+ export interface PipelineDeriveProgress {
19
+ done: number; total: number; reused: number; failed?: number
20
+ phase: PipelineActivityUpdate['phase']; summary: string
21
+ item?: { id: string; title: string; state: PipelineItemState['state'] }
22
+ }
23
+ export interface ExtractionExecution {
24
+ fingerprint: string
25
+ force: boolean
26
+ signal: AbortSignal
27
+ checkpoint: () => Promise<void>
28
+ progress: (text: string, activity?: PipelineActivityUpdate) => void
29
+ }
17
30
  export interface PipelineHost {
18
31
  capture(): PipelineContext
19
32
  serialize<T>(work: () => Promise<T>): Promise<T>
@@ -21,24 +34,29 @@ export interface PipelineHost {
21
34
  /** One source by id. Same shape as an element of `sources()`. */
22
35
  source(sourceId: string): Promise<PipelineSource | undefined>
23
36
  fingerprint(source: PipelineSource, context: PipelineContext): Promise<string>
24
- extract(source: PipelineSource, context: PipelineContext, output: (text: string) => void): Promise<TrialReport>
37
+ extract(source: PipelineSource, context: PipelineContext, output: (text: string) => void, execution: ExtractionExecution): Promise<TrialReport>
25
38
  normalize(source: PipelineSource, checkpoint: ExtractionCheckpoint, context: PipelineContext, force: boolean, output: (text: string) => void): Promise<{ reused: boolean; warnings: string[] }>
26
- derive?(sourceIds: string[] | undefined, context: PipelineContext, force: boolean, output: (text: string) => void, checkpoint: () => Promise<void>): Promise<{ done: number; total: number; reused: number; summary: string }>
39
+ derive?(sourceIds: string[] | undefined, context: PipelineContext, force: boolean, output: (text: string) => void, checkpoint: () => Promise<void>, progress?: (value: PipelineDeriveProgress) => void): Promise<{ done: number; total: number; reused: number; summary: string }>
27
40
  groups(context: PipelineContext): Promise<MaterializationBatch>
28
41
  materialize(group: MaterializationGroup, context: PipelineContext, output: (text: string) => void, snapshot: unknown): Promise<void>
29
42
  link?(): Promise<string>
30
- build(taskId: string, checkpoint: () => Promise<void>): Promise<PipelineBuildResult>
43
+ build(taskId: string, checkpoint: () => Promise<void>, progress?: (text: string) => void): Promise<PipelineBuildResult>
31
44
  record(checkpoint: ExtractionCheckpoint): Promise<void>
32
45
  }
33
46
 
34
47
  class TaskCancelled extends Error {}
35
- interface Control { pause: boolean; cancel: boolean; resume?: () => void }
48
+ interface Control { abort: AbortController; pause: boolean; cancel: boolean; resume?: () => void }
36
49
 
37
50
  /** One writer queue per base, shared with the legacy index builder. */
38
51
  export class KnowledgePipeline {
39
52
  private closed = false
40
53
  private controls = new Map<string, Control>()
41
- constructor(readonly store: PipelineStore, private readonly host: PipelineHost) {}
54
+ constructor(readonly store: PipelineStore, private readonly host: PipelineHost) {
55
+ for (const previous of store.tasks(10000).tasks.filter(task => task.recoverableOnRestart && !task.recoveryTaskId)) {
56
+ const resumed = this.start(previous.stage, { ...previous.selection, force: false })
57
+ this.patch(previous.id, { recoverableOnRestart: false, recoveryTaskId: resumed.task.id, summary: '服务重启,已自动创建续跑任务;复用已完成分片。' })
58
+ }
59
+ }
42
60
 
43
61
  start(stage: PipelineStage, selection: PipelineSelection = {}): { task: PipelineTask; completion: Promise<PipelineTask> } {
44
62
  if (this.closed) throw new Error('知识处理已关闭')
@@ -46,15 +64,16 @@ export class KnowledgePipeline {
46
64
  for (const value of [selection.sourceIds, selection.groupKeys]) {
47
65
  if (value !== undefined && (!Array.isArray(value) || value.length === 0 || value.some(id => typeof id !== 'string' || !id.trim()))) throw new Error('选择范围不能为空;不传范围表示处理当前阶段的全部待办')
48
66
  }
49
- const context = this.host.capture()
67
+ const captured = this.host.capture()
68
+ const context = { ...captured, extractionTimeoutMinutes: normalizeExtractionTimeoutMinutes(captured.extractionTimeoutMinutes) }
50
69
  const { call: _call, why: _why, ...model } = context.model
51
70
  const task: PipelineTask = {
52
- id: `pipeline_${randomUUID()}`, stage, state: 'queued', selection: structuredClone(selection), model,
71
+ id: `pipeline_${randomUUID()}`, stage, state: 'queued', selection: structuredClone(selection), model, extractionTimeoutMinutes: context.extractionTimeoutMinutes,
53
72
  queuedAt: new Date().toISOString(), done: 0, total: 0, reused: 0, failed: 0,
54
73
  summary: '等待运行', warnings: [],
55
74
  }
56
75
  this.store.putTask(task)
57
- this.controls.set(task.id, { pause: false, cancel: false })
76
+ this.controls.set(task.id, { abort: new AbortController(), pause: false, cancel: false })
58
77
  const completion = this.host.serialize(() => this.execute(task.id, context))
59
78
  // Starting a background task must not create an unhandled rejection.
60
79
  void completion.catch(() => undefined)
@@ -63,7 +82,7 @@ export class KnowledgePipeline {
63
82
 
64
83
  close(): void {
65
84
  this.closed = true
66
- for (const control of this.controls.values()) { control.cancel = true; control.pause = false; control.resume?.() }
85
+ for (const control of this.controls.values()) { control.abort.abort(new TaskCancelled('知识库已关闭,已完成结果保留')); control.cancel = true; control.pause = false; control.resume?.() }
67
86
  }
68
87
 
69
88
  control(id: string, action: PipelineAction): PipelineTask {
@@ -72,17 +91,18 @@ export class KnowledgePipeline {
72
91
  if (!task || !control || !PIPELINE_ACTIVE_STATES.includes(task.state)) throw new Error('任务已结束;请重新运行所需阶段')
73
92
  if (action === 'pause') {
74
93
  control.pause = true
75
- this.patch(id, { state: task.state === 'queued' ? 'paused' : 'pausing', summary: '当前文档或卡片处理完成后暂停' })
94
+ this.patch(id, { state: task.state === 'queued' ? 'paused' : 'pausing', summary: '已请求暂停,当前步骤到达安全点后生效;已保存结果保留' })
76
95
  } else if (action === 'resume') {
77
96
  if (!control.pause) throw new Error('任务没有暂停')
78
97
  control.pause = false
79
- this.patch(id, { state: task.startedAt ? 'running' : 'queued' })
98
+ this.patch(id, { state: task.startedAt ? 'running' : 'queued', summary: task.startedAt ? '已继续运行' : '等待运行' })
80
99
  control.resume?.()
81
100
  control.resume = undefined
82
101
  } else {
102
+ control.abort.abort(new TaskCancelled('任务已停止,已完成分片保留'))
83
103
  control.cancel = true
84
104
  control.pause = false
85
- this.patch(id, { state: 'cancelling', summary: '当前文档或卡片处理完成后停止' })
105
+ this.patch(id, { state: 'cancelling', summary: '已请求停止,正在结束当前步骤;已保存结果保留' })
86
106
  control.resume?.()
87
107
  }
88
108
  return this.store.task(id)!
@@ -93,7 +113,16 @@ export class KnowledgePipeline {
93
113
  this.store.putTask(task)
94
114
  return task
95
115
  }
96
- private finishStage(id: string, outcome?: Pick<PipelineBuildResult, 'summary' | 'skipped'>): void {
116
+ private observe(id: string, activity: PipelineActivityUpdate, text?: string): void {
117
+ const observed = { ...activity, at: new Date().toISOString() }
118
+ const task = this.patch(id, { activity: observed, ...(text ? { summary: text } : {}) })
119
+ if (text) this.store.event(id, 'progress', text, task.currentStage === 'extract' || task.currentStage === 'normalize' ? activity.objectId : undefined, task.currentStage, observed)
120
+ }
121
+ private item(id: string, stage: PipelineWorkStage, key: string, value: PipelineItemState): void {
122
+ const task = this.store.task(id)!
123
+ this.patch(id, { items: { ...task.items, [stage]: { ...task.items?.[stage], [key]: value } } })
124
+ }
125
+ private finishStage(id: string, outcome?: Pick<PipelineBuildResult, 'summary' | 'skipped'> & { blocked?: number }): void {
97
126
  const task = this.store.task(id)!
98
127
  if (task.currentStage) this.patch(id, { results: { ...task.results, [task.currentStage]: { done: task.done, total: task.total, reused: task.reused, failed: task.failed, ...outcome } } })
99
128
  }
@@ -114,6 +143,9 @@ export class KnowledgePipeline {
114
143
  const flush = (): void => {
115
144
  if (!buffer) return
116
145
  this.store.event(id, 'output', buffer, sourceId, stage)
146
+ const activity = this.store.task(id)?.activity
147
+ // Persist liveness at most once a second; raw output remains in the event stream.
148
+ if (activity && Date.now() - Date.parse(activity.at) >= 1000) this.observe(id, { ...activity, phase: 'receiving' })
117
149
  buffer = ''
118
150
  last = Date.now()
119
151
  }
@@ -152,20 +184,27 @@ export class KnowledgePipeline {
152
184
  for (const stage of stages) {
153
185
  await this.checkpoint(id)
154
186
  this.patch(id, { currentStage: stage, done: 0, total: 0, reused: 0, failed: 0, summary: PIPELINE_STAGE_LABEL[stage] })
155
- this.store.event(id, 'progress', PIPELINE_STAGE_LABEL[stage], undefined, stage)
187
+ this.observe(id, { phase: 'preparing' }, `开始${PIPELINE_STAGE_LABEL[stage]}`)
156
188
  if (stage === 'derive') {
157
189
  if (!this.host.derive) throw new Error('当前运行时不支持 Case 提炼')
158
190
  const output = this.output(id, stage)
191
+ this.observe(id, { phase: 'processing' }, '正在检查 Case 并提炼可复用方法')
159
192
  try {
160
- const result = await this.host.derive(task.selection.sourceIds, context, Boolean(task.selection.force), output.append, () => this.checkpoint(id))
193
+ const result = await this.host.derive(task.selection.sourceIds, context, Boolean(task.selection.force), output.append, () => this.checkpoint(id), value => {
194
+ this.patch(id, { done: value.done, total: value.total, reused: value.reused, failed: value.failed ?? 0 })
195
+ if (value.item) this.item(id, stage, value.item.id, { title: value.item.title, state: value.item.state })
196
+ this.observe(id, { phase: value.phase, objectId: value.item?.id, objectTitle: value.item?.title }, value.summary)
197
+ })
161
198
  this.patch(id, result)
162
199
  this.finishStage(id, result)
163
200
  } finally { output.flush() }
164
201
  continue
165
202
  }
166
203
  if (stage === 'build') {
167
- const result = await this.host.build(id, () => this.checkpoint(id))
204
+ this.observe(id, { phase: 'processing' }, '正在构建并发布检索索引')
205
+ const result = await this.host.build(id, () => this.checkpoint(id), text => this.observe(id, { phase: 'processing' }, text))
168
206
  this.patch(id, { done: 1, total: 1, summary: result.summary })
207
+ this.observe(id, { phase: 'completed' }, result.summary)
169
208
  this.finishStage(id, result)
170
209
  continue
171
210
  }
@@ -211,16 +250,22 @@ export class KnowledgePipeline {
211
250
  this.patch(id, { total: pending.length })
212
251
  for (const group of pending) {
213
252
  await this.checkpoint(id)
253
+ this.item(id, stage, group.key, { state: 'processing', title: group.title })
254
+ this.observe(id, { phase: 'processing', objectId: group.key, objectTitle: group.title }, `正在物化:${group.title}`)
214
255
  const output = this.output(id, stage)
256
+ let itemError: string | undefined
215
257
  try { await this.host.materialize(group, context, output.append, batch.snapshot) }
216
258
  catch (cause) {
217
259
  if (cause instanceof TaskCancelled) throw cause
218
260
  const message = `${group.title}:${cause instanceof Error ? cause.message : String(cause)}`
261
+ itemError = message
219
262
  const current = this.store.task(id)!
220
263
  this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
221
264
  this.store.event(id, 'warning', message, undefined, stage)
222
265
  }
223
266
  finally { output.flush() }
267
+ this.item(id, stage, group.key, { state: itemError ? 'failed' : 'succeeded', title: group.title, summary: itemError })
268
+ this.observe(id, { phase: itemError ? 'failed' : 'completed', objectId: group.key, objectTitle: group.title }, itemError ?? `已保存卡片:${group.title}`)
224
269
  const current = this.store.task(id)!
225
270
  this.patch(id, { done: current.done + 1, summary: `已物化 ${current.done + 1}/${pending.length} 张:${group.title}` })
226
271
  }
@@ -233,6 +278,7 @@ export class KnowledgePipeline {
233
278
  * 同一件事在半个屏幕上两种说法。
234
279
  */
235
280
  this.finishStage(id, {
281
+ blocked: deferred,
236
282
  summary: `已物化 ${current.done - current.failed}/${current.total} 张${deferred > 0 ? `;${deferred} 条待归一` : ''}${affectedSources !== undefined ? `(按来源范围缩小:${affectedSources.size} 份来源)` : ''}`,
237
283
  })
238
284
  /**
@@ -248,16 +294,23 @@ export class KnowledgePipeline {
248
294
  this.patch(id, { total: sources.length })
249
295
  for (const source of sources) {
250
296
  await this.checkpoint(id)
251
- this.store.event(id, 'progress', `${PIPELINE_STAGE_LABEL[stage]}:${source.title}`, source.sourceId, stage)
297
+ this.item(id, stage, source.sourceId, { state: 'processing', title: source.title })
298
+ this.observe(id, { phase: 'processing', objectId: source.sourceId, objectTitle: source.title }, `${PIPELINE_STAGE_LABEL[stage]}:${source.title}`)
252
299
  const output = this.output(id, stage, source.sourceId)
253
300
  let reused = false
301
+ let itemError: string | undefined
254
302
  try {
255
303
  if (stage === 'extract') {
256
304
  const fingerprint = await this.host.fingerprint(source, context)
257
305
  const cached = task.selection.force ? undefined : this.store.cached(source.sourceId, source.version, fingerprint)
258
306
  if (cached) { this.store.selectExtraction(source.sourceId, cached.id); reused = true }
259
307
  else {
260
- const report = await this.host.extract(source, context, output.append)
308
+ const report = await this.host.extract(source, context, output.append, {
309
+ fingerprint, force: task.selection.force === true, signal: this.controls.get(id)!.abort.signal,
310
+ checkpoint: () => this.checkpoint(id), progress: (text, activity) => {
311
+ this.observe(id, activity ?? { phase: 'processing', objectId: source.sourceId, objectTitle: source.title }, text)
312
+ },
313
+ })
261
314
  if (this.closed) throw new TaskCancelled('知识库已关闭,已完成结果保留')
262
315
  const checkpoint: ExtractionCheckpoint = { id: `extract_${randomUUID()}`, sourceId: source.sourceId, sourceVersion: source.version, sourceHash: source.sourceHash, fingerprint, report, createdAt: new Date().toISOString() }
263
316
  // The host checks the captured source version again before activation.
@@ -286,6 +339,7 @@ export class KnowledgePipeline {
286
339
  const detail = report.exceptions.slice(0, 3).map(item => `${item.step}:${item.message}`).join(';')
287
340
  throw new Error(`${report.model.why ?? '抽取返回未通过校验'}${detail === '' ? '' : `(${detail})`}`)
288
341
  }
342
+ this.observe(id, { ...this.store.task(id)!.activity!, phase: 'saving' }, '正在提交文档抽取结果')
289
343
  this.store.saveExtraction(checkpoint)
290
344
  }
291
345
  } else {
@@ -300,10 +354,13 @@ export class KnowledgePipeline {
300
354
  } catch (cause) {
301
355
  if (cause instanceof TaskCancelled) throw cause
302
356
  const message = `${source.title}:${cause instanceof Error ? cause.message : String(cause)}`
357
+ itemError = message
303
358
  const current = this.store.task(id)!
304
359
  this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
305
360
  this.store.event(id, 'warning', message, source.sourceId, stage)
306
361
  } finally { output.flush() }
362
+ this.item(id, stage, source.sourceId, { state: itemError ? 'failed' : reused ? 'reused' : 'succeeded', title: source.title, summary: itemError })
363
+ this.observe(id, { ...this.store.task(id)!.activity!, phase: itemError ? 'failed' : reused ? 'reused' : 'completed' }, itemError ?? `${source.title}:${reused ? '复用已有结果' : '处理完成'}`)
307
364
  const current = this.store.task(id)!
308
365
  this.patch(id, { done: current.done + 1, reused: current.reused + Number(reused), summary: `${PIPELINE_STAGE_LABEL[stage]} ${current.done + 1}/${sources.length}${reused ? ' · 复用已有结果' : ''}` })
309
366
  }
@@ -0,0 +1,40 @@
1
+ import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_ORDER, type PipelineState, type PipelineTask, type PipelineWorkStage } from './types.js'
2
+
3
+ export const PIPELINE_STATE_LABEL: Record<PipelineState, string> = {
4
+ queued: '排队中', running: '进行中', pausing: '等待暂停', paused: '已暂停', cancelling: '正在停止',
5
+ succeeded: '已完成', partial: '部分完成', failed: '失败', cancelled: '已停止', interrupted: '已中断',
6
+ }
7
+ export interface TaskStageObservation {
8
+ label: string
9
+ tone: 'neutral' | 'success' | 'warning' | 'blue'
10
+ detail: string
11
+ }
12
+
13
+ /** A stage observation always belongs to one execution, never a mixture of past runs. */
14
+ export function observeTaskStage(task: PipelineTask, stage: PipelineWorkStage): TaskStageObservation {
15
+ if (task.stage !== 'all' && task.stage !== stage) return { label: '本次未运行', tone: 'neutral', detail: '此阶段不在本次任务范围内。' }
16
+ const result = task.results?.[stage]
17
+ if (result) {
18
+ const counts = `成功 ${Math.max(0, result.done - result.failed)}/${result.total}${result.reused ? ` · 复用 ${result.reused}` : ''}${result.failed ? ` · 失败 ${result.failed}` : ''}`
19
+ if (result.failed || result.blocked) return { label: result.failed ? '部分失败' : '有暂缓项', tone: 'warning', detail: `${result.summary ?? counts}${result.blocked ? ` · 暂缓 ${result.blocked} 条` : ''}` }
20
+ if (result.skipped || result.total === 0) return { label: '已跳过', tone: 'neutral', detail: result.summary ?? '本次没有需要处理的内容。' }
21
+ return { label: result.reused === result.total && stage !== 'build' ? '全部复用' : '已完成', tone: 'success', detail: result.summary ?? counts }
22
+ }
23
+ const current = task.currentStage ?? (task.stage === 'all' ? 'extract' : task.stage)
24
+ if (current === stage) return {
25
+ label: PIPELINE_STATE_LABEL[task.state],
26
+ tone: ['failed', 'partial', 'interrupted'].includes(task.state) ? 'warning' : task.state === 'paused' || task.state === 'cancelled' ? 'neutral' : task.state === 'succeeded' ? 'success' : 'blue',
27
+ detail: task.total > 0 ? `已处理 ${task.done}/${task.total} · 失败 ${task.failed} · 复用 ${task.reused}` : task.summary,
28
+ }
29
+ if (PIPELINE_STAGE_ORDER.indexOf(stage) < PIPELINE_STAGE_ORDER.indexOf(current)) return { label: '已跳过', tone: 'neutral', detail: '本次运行时未执行此阶段。' }
30
+ return PIPELINE_ACTIVE_STATES.includes(task.state)
31
+ ? { label: '等待执行', tone: 'neutral', detail: '本次任务到达此阶段后自动运行。' }
32
+ : { label: '未执行', tone: 'neutral', detail: '任务在到达此阶段前已结束。' }
33
+ }
34
+
35
+ /** Running work wins over newer queued work; queue order remains FIFO. */
36
+ export function orderActiveTasks(tasks: PipelineTask[]): PipelineTask[] {
37
+ return tasks.filter(task => PIPELINE_ACTIVE_STATES.includes(task.state)).sort((left, right) =>
38
+ Number(left.state === 'queued' || !left.startedAt) - Number(right.state === 'queued' || !right.startedAt)
39
+ || left.queuedAt.localeCompare(right.queuedAt))
40
+ }