dsh-convert-core 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +149 -0
- package/bin/media-to-md.mjs +168 -0
- package/bin/tita-convert.mjs +85 -0
- package/docs/media-to-md.md +143 -0
- package/lib/adapters/anydoc.d.ts +73 -0
- package/lib/adapters/anydoc.d.ts.map +1 -0
- package/lib/adapters/anydoc.js +295 -0
- package/lib/adapters/anydoc.js.map +1 -0
- package/lib/adapters/iddoc.d.ts +28 -0
- package/lib/adapters/iddoc.d.ts.map +1 -0
- package/lib/adapters/iddoc.js +153 -0
- package/lib/adapters/iddoc.js.map +1 -0
- package/lib/adapters/index.d.ts +41 -0
- package/lib/adapters/index.d.ts.map +1 -0
- package/lib/adapters/index.js +115 -0
- package/lib/adapters/index.js.map +1 -0
- package/lib/adapters/scanned.d.ts +28 -0
- package/lib/adapters/scanned.d.ts.map +1 -0
- package/lib/adapters/scanned.js +121 -0
- package/lib/adapters/scanned.js.map +1 -0
- package/lib/adapters/text.d.ts +29 -0
- package/lib/adapters/text.d.ts.map +1 -0
- package/lib/adapters/text.js +135 -0
- package/lib/adapters/text.js.map +1 -0
- package/lib/adapters/types.d.ts +65 -0
- package/lib/adapters/types.d.ts.map +1 -0
- package/lib/adapters/types.js +11 -0
- package/lib/adapters/types.js.map +1 -0
- package/lib/contract.d.ts +185 -0
- package/lib/contract.d.ts.map +1 -0
- package/lib/contract.js +59 -0
- package/lib/contract.js.map +1 -0
- package/lib/delivery.d.ts +36 -0
- package/lib/delivery.d.ts.map +1 -0
- package/lib/delivery.js +97 -0
- package/lib/delivery.js.map +1 -0
- package/lib/http.d.ts +34 -0
- package/lib/http.d.ts.map +1 -0
- package/lib/http.js +372 -0
- package/lib/http.js.map +1 -0
- package/lib/markdown.d.ts +32 -0
- package/lib/markdown.d.ts.map +1 -0
- package/lib/markdown.js +145 -0
- package/lib/markdown.js.map +1 -0
- package/lib/media/binaries.d.ts +26 -0
- package/lib/media/binaries.d.ts.map +1 -0
- package/lib/media/binaries.js +60 -0
- package/lib/media/binaries.js.map +1 -0
- package/lib/media/convert.d.ts +19 -0
- package/lib/media/convert.d.ts.map +1 -0
- package/lib/media/convert.js +210 -0
- package/lib/media/convert.js.map +1 -0
- package/lib/media/index.d.ts +27 -0
- package/lib/media/index.d.ts.map +1 -0
- package/lib/media/index.js +27 -0
- package/lib/media/index.js.map +1 -0
- package/lib/media/markdown.d.ts +40 -0
- package/lib/media/markdown.d.ts.map +1 -0
- package/lib/media/markdown.js +89 -0
- package/lib/media/markdown.js.map +1 -0
- package/lib/media/media.d.ts +45 -0
- package/lib/media/media.d.ts.map +1 -0
- package/lib/media/media.js +167 -0
- package/lib/media/media.js.map +1 -0
- package/lib/media/models.d.ts +31 -0
- package/lib/media/models.d.ts.map +1 -0
- package/lib/media/models.js +87 -0
- package/lib/media/models.js.map +1 -0
- package/lib/media/scanned.d.ts +18 -0
- package/lib/media/scanned.d.ts.map +1 -0
- package/lib/media/scanned.js +179 -0
- package/lib/media/scanned.js.map +1 -0
- package/lib/media/types.d.ts +154 -0
- package/lib/media/types.d.ts.map +1 -0
- package/lib/media/types.js +23 -0
- package/lib/media/types.js.map +1 -0
- package/lib/server.d.ts +35 -0
- package/lib/server.d.ts.map +1 -0
- package/lib/server.js +64 -0
- package/lib/server.js.map +1 -0
- package/lib/service.d.ts +170 -0
- package/lib/service.d.ts.map +1 -0
- package/lib/service.js +562 -0
- package/lib/service.js.map +1 -0
- package/package.json +59 -0
- package/scripts/asr.py +55 -0
- package/scripts/ocr.py +40 -0
- package/scripts/pdf_pages.py +49 -0
- package/scripts/vendor.mjs +76 -0
- package/src/adapters/anydoc.ts +356 -0
- package/src/adapters/iddoc.ts +171 -0
- package/src/adapters/index.ts +147 -0
- package/src/adapters/scanned.ts +139 -0
- package/src/adapters/text.ts +146 -0
- package/src/adapters/types.ts +76 -0
- package/src/contract.ts +230 -0
- package/src/delivery.ts +124 -0
- package/src/http.ts +394 -0
- package/src/markdown.ts +141 -0
- package/src/media/binaries.ts +67 -0
- package/src/media/convert.ts +232 -0
- package/src/media/index.ts +43 -0
- package/src/media/markdown.ts +105 -0
- package/src/media/media.ts +204 -0
- package/src/media/models.ts +124 -0
- package/src/media/scanned.ts +200 -0
- package/src/media/types.ts +171 -0
- package/src/server.ts +85 -0
- package/src/service.ts +639 -0
- package/web/app.js +382 -0
- package/web/index.html +217 -0
- package/web/styles.css +263 -0
- package/web/vendor/icons.js +25 -0
- package/web/vendor/vue.global.prod.js +14 -0
package/src/service.ts
ADDED
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The conversion service: queue, lifecycle, delivery, observability.
|
|
3
|
+
*
|
|
4
|
+
* It owns no transport. `http.ts` mounts it on `node:http`, the DSH plugin
|
|
5
|
+
* mounts the same handler on the host web server, and `bin/tita-convert.mjs`
|
|
6
|
+
* runs it standalone — the "kernel first, wrapper second" split.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
|
|
10
|
+
import { basename, dirname, extname, join, normalize, resolve, sep } from 'node:path'
|
|
11
|
+
import { homedir } from 'node:os'
|
|
12
|
+
import { capabilityReport, resolveAdapter, type CapabilityReport } from './adapters/index.js'
|
|
13
|
+
import { AdapterError } from './adapters/types.js'
|
|
14
|
+
import type { ConvertAsset, ConvertRequest, ConvertTask, ConvertWarning, IngestPayload, SourceFormat, TaskState } from './contract.js'
|
|
15
|
+
import { writeDelivery } from './delivery.js'
|
|
16
|
+
import { sha1, sha256, shortHash, slug } from './markdown.js'
|
|
17
|
+
|
|
18
|
+
export interface ConvertServiceOptions {
|
|
19
|
+
/** Root for task state (`tasks.json`). Defaults to `~/.tita-data-agent`. */
|
|
20
|
+
rootDir?: string
|
|
21
|
+
/**
|
|
22
|
+
* Root the **deliveries** are written under. Defaults to `<rootDir>/convert-out`.
|
|
23
|
+
*
|
|
24
|
+
* 与 `bucketByDay` 配合决定形状:不分桶是 `<outRoot>/<taskId>/`,分桶是
|
|
25
|
+
* `<outRoot>/<YYYY-MM-DD>/<taskId>/`。任务目录始终是原子的交付单位。
|
|
26
|
+
*/
|
|
27
|
+
outRoot?: string
|
|
28
|
+
/**
|
|
29
|
+
* Root the per-task **working** directories live under. Defaults to
|
|
30
|
+
* `<rootDir>/work`. 形状与 `outRoot` 一致。
|
|
31
|
+
*/
|
|
32
|
+
workRoot?: string
|
|
33
|
+
/**
|
|
34
|
+
* 交付与工作区是否按日期分桶(0029)。
|
|
35
|
+
*
|
|
36
|
+
* **默认 false,是刻意的**:这是内核,独立运行(`bin/tita-convert.mjs`)与既有
|
|
37
|
+
* 调用方都不该因为插件侧的目录结构调整而换位置。TitaAgent 插件显式传 `true`,
|
|
38
|
+
* 因为它的根路径契约里 `output/` 与 `tmp/` 的二级就是日期。
|
|
39
|
+
*
|
|
40
|
+
* 分桶的理由:这两处是「按时间堆积」的东西,不分桶的话一个月后就是一个几百条
|
|
41
|
+
* 平铺的目录,`ls` 出来读不出「上周那份报表在哪」。
|
|
42
|
+
*/
|
|
43
|
+
bucketByDay?: boolean
|
|
44
|
+
maxConcurrent?: number
|
|
45
|
+
maxInputBytes?: number
|
|
46
|
+
/** Keep at most this many finished tasks in the store. */
|
|
47
|
+
maxTasks?: number
|
|
48
|
+
/** Delete deliveries older than this on start. 0 disables cleanup. */
|
|
49
|
+
retentionDays?: number
|
|
50
|
+
/**
|
|
51
|
+
* How long a capability report stays valid. Defaults to
|
|
52
|
+
* {@link DEFAULT_CAPABILITY_TTL_MS}; 0 disables caching.
|
|
53
|
+
*
|
|
54
|
+
* See {@link ConvertService.capabilities} for why this exists: building the
|
|
55
|
+
* report spawns external binaries, and the console polls it on a timer.
|
|
56
|
+
*/
|
|
57
|
+
capabilityTtlMs?: number
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Default lifetime of a capability report: 30 seconds.
|
|
62
|
+
*
|
|
63
|
+
* Long enough that a polling panel costs one probe per half minute instead of one
|
|
64
|
+
* per poll; short enough that installing ffmpeg and reopening the panel shows it
|
|
65
|
+
* without restarting anything.
|
|
66
|
+
*/
|
|
67
|
+
export const DEFAULT_CAPABILITY_TTL_MS = 30_000
|
|
68
|
+
|
|
69
|
+
export interface SubmitInput {
|
|
70
|
+
filename?: string
|
|
71
|
+
/** Raw bytes for file / video uploads. */
|
|
72
|
+
data?: Buffer
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export type ConvertListener = (task: ConvertTask) => void
|
|
76
|
+
|
|
77
|
+
export const DEFAULT_ROOT = join(homedir(), '.tita-data-agent')
|
|
78
|
+
|
|
79
|
+
export function defaultConvertRoot(): string {
|
|
80
|
+
if (process.env.DSH_CONVERT_ROOT) return process.env.DSH_CONVERT_ROOT
|
|
81
|
+
return join(DEFAULT_ROOT, 'convert')
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
interface QueueEntry {
|
|
85
|
+
taskId: string
|
|
86
|
+
controller: AbortController
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const STATES: TaskState[] = ['queued', 'running', 'succeeded', 'failed', 'cancelled']
|
|
90
|
+
|
|
91
|
+
function newTaskId(): string {
|
|
92
|
+
return `conv_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* 本地日期的 `YYYY-MM-DD`。
|
|
97
|
+
*
|
|
98
|
+
* 与 `data-domain-knowledge/src/layout.ts` 的 `dayKey` 同一条规则(本地时间、
|
|
99
|
+
* 零填充)。内核不能 import 插件,所以这里重列一次 —— 它必须是**同一个**日期,
|
|
100
|
+
* 否则「今天的交付」会跨零点落进两个不同的桶。
|
|
101
|
+
*/
|
|
102
|
+
function dayKey(date: Date): string {
|
|
103
|
+
const pad = (value: number): string => String(value).padStart(2, '0')
|
|
104
|
+
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** 任务创建当天的日期桶。用 `createdAt` 而不是「现在」:重试不该换桶。 */
|
|
108
|
+
function dayOf(createdAt: string | undefined): string {
|
|
109
|
+
const parsed = createdAt ? new Date(createdAt) : new Date()
|
|
110
|
+
return dayKey(Number.isNaN(parsed.getTime()) ? new Date() : parsed)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export class ConvertService {
|
|
114
|
+
readonly rootDir: string
|
|
115
|
+
/** 交付物根。每个任务的具体交付目录见 `deliveryDirOf()`。 */
|
|
116
|
+
readonly outRoot: string
|
|
117
|
+
/** 工作区根。每个任务的具体工作目录见 `workDirOf()`。 */
|
|
118
|
+
readonly workRoot: string
|
|
119
|
+
/** 交付与工作区是否按日期分桶。见 `ConvertServiceOptions.bucketByDay`。 */
|
|
120
|
+
readonly bucketByDay: boolean
|
|
121
|
+
private readonly options: ConvertServiceOptions
|
|
122
|
+
private readonly tasks = new Map<string, ConvertTask>()
|
|
123
|
+
private readonly controllers = new Map<string, AbortController>()
|
|
124
|
+
private readonly listeners = new Set<ConvertListener>()
|
|
125
|
+
private readonly queue: string[] = []
|
|
126
|
+
private running = 0
|
|
127
|
+
private closed = false
|
|
128
|
+
private persistTimer: NodeJS.Timeout | undefined
|
|
129
|
+
private persistTail: Promise<void> = Promise.resolve()
|
|
130
|
+
/** Last capability report and when it was built. See {@link capabilities}. */
|
|
131
|
+
private capabilityCache: { at: number; report: CapabilityReport } | undefined
|
|
132
|
+
|
|
133
|
+
constructor(options: ConvertServiceOptions = {}) {
|
|
134
|
+
this.options = options
|
|
135
|
+
this.rootDir = options.rootDir ?? defaultConvertRoot()
|
|
136
|
+
this.outRoot = options.outRoot ?? join(this.rootDir, 'convert-out')
|
|
137
|
+
this.workRoot = options.workRoot ?? join(this.rootDir, 'work')
|
|
138
|
+
this.bucketByDay = options.bucketByDay ?? false
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* @deprecated 0029:`outDir` 现在是 `outRoot` 的别名。保留这个名字只为兼容
|
|
143
|
+
* 既有的能力报告(`host.ts` 会回显它);新代码用 `outRoot` + `deliveryDirOf()`。
|
|
144
|
+
*/
|
|
145
|
+
get outDir(): string { return this.outRoot }
|
|
146
|
+
/** @deprecated 0029:同上,用 `workRoot` + `workDirOf()`。 */
|
|
147
|
+
get workDir(): string { return this.workRoot }
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* 一个任务的交付目录。
|
|
151
|
+
*
|
|
152
|
+
* 分桶时 `<outRoot>/<日期>/<taskId>`,否则 `<outRoot>/<taskId>`。**按任务的
|
|
153
|
+
* `createdAt` 算而不是按「现在」**:重试一个昨天的任务不该把它换到今天来,
|
|
154
|
+
* 否则同一个交付会因为一次重试而换位置。
|
|
155
|
+
*/
|
|
156
|
+
deliveryDirOf(task: Pick<ConvertTask, 'id' | 'createdAt'>): string {
|
|
157
|
+
return this.taskDir(this.outRoot, task)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** 一个任务的工作目录。形状与交付目录一致。 */
|
|
161
|
+
workDirOf(task: Pick<ConvertTask, 'id' | 'createdAt'>): string {
|
|
162
|
+
return this.taskDir(this.workRoot, task)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private taskDir(root: string, task: Pick<ConvertTask, 'id' | 'createdAt'>): string {
|
|
166
|
+
return this.bucketByDay ? join(root, dayOf(task.createdAt), task.id) : join(root, task.id)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async start(): Promise<void> {
|
|
170
|
+
await mkdir(this.outRoot, { recursive: true })
|
|
171
|
+
await mkdir(this.workRoot, { recursive: true })
|
|
172
|
+
await this.load()
|
|
173
|
+
await this.cleanup()
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async close(): Promise<void> {
|
|
177
|
+
this.closed = true
|
|
178
|
+
for (const controller of this.controllers.values()) controller.abort()
|
|
179
|
+
if (this.persistTimer) clearTimeout(this.persistTimer)
|
|
180
|
+
await this.persist()
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
subscribe(listener: ConvertListener): () => void {
|
|
184
|
+
this.listeners.add(listener)
|
|
185
|
+
return () => this.listeners.delete(listener)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private emit(task: ConvertTask): void {
|
|
189
|
+
for (const listener of this.listeners) {
|
|
190
|
+
try {
|
|
191
|
+
listener(task)
|
|
192
|
+
} catch {
|
|
193
|
+
/* a broken listener must not break conversion */
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
get(id: string): ConvertTask | undefined {
|
|
199
|
+
const task = this.tasks.get(id)
|
|
200
|
+
return task ? structuredClone(task) : undefined
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
list(): ConvertTask[] {
|
|
204
|
+
return [...this.tasks.values()]
|
|
205
|
+
.sort((a, b) => (a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0))
|
|
206
|
+
.map(task => structuredClone(task))
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Report which adapters can run and which external binaries exist.
|
|
211
|
+
*
|
|
212
|
+
* **Cached, because building this is not free.** `capabilityReport()` probes
|
|
213
|
+
* every adapter, and the media adapter's probe runs `ffprobe -version`,
|
|
214
|
+
* `ffmpeg -version` and `uvx --version` — three child processes per call, which
|
|
215
|
+
* measured ~60ms of server time. The conversion console polls this endpoint
|
|
216
|
+
* every four seconds while its tab is visible, so an uncached report cost about
|
|
217
|
+
* 45 processes and a second of CPU per minute. That lands on the same
|
|
218
|
+
* single-threaded process that serves the GUI's HTTP and event stream, which is
|
|
219
|
+
* how a status panel becomes everyone's latency.
|
|
220
|
+
*
|
|
221
|
+
* Nothing it reports changes second to second: an installed binary stays
|
|
222
|
+
* installed. {@link ConvertServiceOptions.capabilityTtlMs} bounds the staleness.
|
|
223
|
+
*
|
|
224
|
+
* @param options - `force` bypasses the cache, for a caller that just changed
|
|
225
|
+
* the environment (or a test that must not read another test's report).
|
|
226
|
+
* @returns the capability report.
|
|
227
|
+
*/
|
|
228
|
+
async capabilities(options: { force?: boolean } = {}): Promise<CapabilityReport> {
|
|
229
|
+
const ttl = this.options.capabilityTtlMs ?? DEFAULT_CAPABILITY_TTL_MS
|
|
230
|
+
const now = Date.now()
|
|
231
|
+
const cached = this.capabilityCache
|
|
232
|
+
if (options.force !== true && cached !== undefined && ttl > 0 && now - cached.at < ttl) return cached.report
|
|
233
|
+
const report = await capabilityReport()
|
|
234
|
+
this.capabilityCache = { at: now, report }
|
|
235
|
+
return report
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
async submit(request: ConvertRequest, input: SubmitInput = {}): Promise<ConvertTask> {
|
|
239
|
+
if (this.closed) throw new AdapterError('SERVICE_CLOSED', '转换服务已关闭。')
|
|
240
|
+
validateRequest(request)
|
|
241
|
+
|
|
242
|
+
const id = newTaskId()
|
|
243
|
+
const filename = input.filename ?? request.filename ?? defaultFilename(request)
|
|
244
|
+
const createdAt = new Date().toISOString()
|
|
245
|
+
|
|
246
|
+
const task: ConvertTask = {
|
|
247
|
+
id,
|
|
248
|
+
state: 'queued',
|
|
249
|
+
inputKind: request.kind,
|
|
250
|
+
label: request.title?.trim() || filename || 'inline text',
|
|
251
|
+
sourceFormat: 'text',
|
|
252
|
+
createdAt,
|
|
253
|
+
updatedAt: createdAt,
|
|
254
|
+
progress: 0,
|
|
255
|
+
stage: 'queued',
|
|
256
|
+
message: '已排队',
|
|
257
|
+
warnings: [],
|
|
258
|
+
attempts: 0,
|
|
259
|
+
request: { ...request },
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// A local path is read here rather than by every caller, so the HTTP API,
|
|
263
|
+
// the agent tool and the console share exactly one staging behaviour.
|
|
264
|
+
let payload = input.data
|
|
265
|
+
let payloadName = filename
|
|
266
|
+
if (!payload && request.localPath && request.kind === 'file') {
|
|
267
|
+
const absolute = resolve(request.localPath)
|
|
268
|
+
const info = await stat(absolute).catch(() => undefined)
|
|
269
|
+
if (!info?.isFile()) throw new AdapterError('PATH_NOT_A_FILE', `路径不是文件:${absolute}`)
|
|
270
|
+
const maxBytes = this.options.maxInputBytes ?? 256 * 1024 * 1024
|
|
271
|
+
if (info.size > maxBytes) {
|
|
272
|
+
throw new AdapterError('INPUT_TOO_LARGE', `输入文件 ${info.size} 字节,超过上限 ${maxBytes} 字节。`, '调大插件设置里的 maxInputBytes,或先压缩/拆分文件。')
|
|
273
|
+
}
|
|
274
|
+
payload = await readFile(absolute)
|
|
275
|
+
payloadName = request.filename ?? basename(absolute)
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
if (payload) {
|
|
279
|
+
const maxBytes = this.options.maxInputBytes ?? 256 * 1024 * 1024
|
|
280
|
+
if (payload.length > maxBytes) {
|
|
281
|
+
throw new AdapterError('INPUT_TOO_LARGE', `输入文件 ${payload.length} 字节,超过上限 ${maxBytes} 字节。`, '调大插件设置里的 maxInputBytes,或先压缩/拆分文件。')
|
|
282
|
+
}
|
|
283
|
+
const taskWorkDir = this.workDirOf(task)
|
|
284
|
+
await mkdir(taskWorkDir, { recursive: true })
|
|
285
|
+
const stagePath = join(taskWorkDir, safeFilename(payloadName))
|
|
286
|
+
await writeFile(stagePath, payload)
|
|
287
|
+
task.inputPath = stagePath
|
|
288
|
+
task.request = { ...task.request, filename: payloadName }
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
this.tasks.set(id, task)
|
|
292
|
+
this.queue.push(id)
|
|
293
|
+
await this.persist()
|
|
294
|
+
this.emit(this.get(id)!)
|
|
295
|
+
this.pump()
|
|
296
|
+
return this.get(id)!
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
cancel(id: string): ConvertTask {
|
|
300
|
+
const task = this.tasks.get(id)
|
|
301
|
+
if (!task) throw new AdapterError('TASK_NOT_FOUND', `任务不存在:${id}`)
|
|
302
|
+
if (task.state === 'succeeded' || task.state === 'failed' || task.state === 'cancelled') return this.get(id)!
|
|
303
|
+
const index = this.queue.indexOf(id)
|
|
304
|
+
if (index >= 0) this.queue.splice(index, 1)
|
|
305
|
+
const controller = this.controllers.get(id)
|
|
306
|
+
if (controller) controller.abort()
|
|
307
|
+
this.patch(id, { state: 'cancelled', stage: 'cancelled', message: '已取消', finishedAt: new Date().toISOString(), progress: null })
|
|
308
|
+
// Deliveries of cancelled tasks are removed: half a bundle is not ingestible.
|
|
309
|
+
// 用任务自己的交付目录:它可能落在**过去某一天**的日期桶里,不能按今天算。
|
|
310
|
+
void rm(task.deliveryDir ?? this.deliveryDirOf(task), { recursive: true, force: true })
|
|
311
|
+
return this.get(id)!
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async retry(id: string): Promise<ConvertTask> {
|
|
315
|
+
const task = this.tasks.get(id)
|
|
316
|
+
if (!task) throw new AdapterError('TASK_NOT_FOUND', `任务不存在:${id}`)
|
|
317
|
+
if (task.state === 'running' || task.state === 'queued') {
|
|
318
|
+
throw new AdapterError('TASK_NOT_RETRYABLE', `任务当前状态为 ${task.state},不能重试。`)
|
|
319
|
+
}
|
|
320
|
+
// Reuse the staged input and work dir: "retry" must not re-download.
|
|
321
|
+
this.patch(id, {
|
|
322
|
+
state: 'queued',
|
|
323
|
+
stage: 'queued',
|
|
324
|
+
message: '重试排队中',
|
|
325
|
+
progress: 0,
|
|
326
|
+
error: undefined,
|
|
327
|
+
finishedAt: undefined,
|
|
328
|
+
warnings: [],
|
|
329
|
+
})
|
|
330
|
+
this.queue.push(id)
|
|
331
|
+
await this.persist()
|
|
332
|
+
this.pump()
|
|
333
|
+
return this.get(id)!
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
private patch(id: string, changes: Partial<ConvertTask>): ConvertTask {
|
|
337
|
+
const current = this.tasks.get(id)
|
|
338
|
+
if (!current) throw new AdapterError('TASK_NOT_FOUND', `任务不存在:${id}`)
|
|
339
|
+
const next: ConvertTask = { ...current, ...changes, updatedAt: new Date().toISOString() }
|
|
340
|
+
this.tasks.set(id, next)
|
|
341
|
+
this.schedulePersist()
|
|
342
|
+
this.emit(structuredClone(next))
|
|
343
|
+
return next
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
private pump(): void {
|
|
347
|
+
const limit = Math.max(1, this.options.maxConcurrent ?? 2)
|
|
348
|
+
while (this.running < limit && this.queue.length > 0) {
|
|
349
|
+
const id = this.queue.shift()!
|
|
350
|
+
const task = this.tasks.get(id)
|
|
351
|
+
if (!task || task.state === 'cancelled') continue
|
|
352
|
+
const controller = new AbortController()
|
|
353
|
+
this.controllers.set(id, controller)
|
|
354
|
+
this.running += 1
|
|
355
|
+
void this.execute(id, controller)
|
|
356
|
+
.catch(() => undefined)
|
|
357
|
+
.finally(() => {
|
|
358
|
+
this.controllers.delete(id)
|
|
359
|
+
this.running -= 1
|
|
360
|
+
this.pump()
|
|
361
|
+
})
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
private async execute(id: string, controller: AbortController): Promise<void> {
|
|
366
|
+
const task = this.tasks.get(id)
|
|
367
|
+
if (!task) return
|
|
368
|
+
const startedAt = new Date().toISOString()
|
|
369
|
+
this.patch(id, { state: 'running', stage: 'starting', message: '开始转换', startedAt, progress: 0, attempts: task.attempts + 1 })
|
|
370
|
+
|
|
371
|
+
const taskWorkDir = this.workDirOf(task)
|
|
372
|
+
const deliveryDir = this.deliveryDirOf(task)
|
|
373
|
+
try {
|
|
374
|
+
await mkdir(taskWorkDir, { recursive: true })
|
|
375
|
+
const request = task.request
|
|
376
|
+
const filename = request.filename ?? (task.inputPath ? basename(task.inputPath) : defaultFilename(request))
|
|
377
|
+
const adapter = await resolveAdapter(request, filename, task.inputPath)
|
|
378
|
+
this.patch(id, { stage: 'resolving', message: `使用 ${adapter.id} 适配器` })
|
|
379
|
+
|
|
380
|
+
const output = await adapter.run({
|
|
381
|
+
request,
|
|
382
|
+
...(task.inputPath ? { inputPath: task.inputPath } : {}),
|
|
383
|
+
...(request.kind === 'text' ? { text: request.text ?? '' } : {}),
|
|
384
|
+
filename,
|
|
385
|
+
signal: controller.signal,
|
|
386
|
+
report: (progress, stage, message) => {
|
|
387
|
+
const current = this.tasks.get(id)
|
|
388
|
+
if (!current || current.state !== 'running') return
|
|
389
|
+
this.patch(id, { progress, stage, ...(message ? { message } : {}) })
|
|
390
|
+
},
|
|
391
|
+
})
|
|
392
|
+
|
|
393
|
+
if (controller.signal.aborted) throw new AdapterError('TASK_CANCELLED', '任务已取消')
|
|
394
|
+
|
|
395
|
+
const warnings: ConvertWarning[] = [...output.warnings]
|
|
396
|
+
|
|
397
|
+
// Assets arrive as bytes from the engine, and the service owns the
|
|
398
|
+
// delivery directory, so this is the only place that writes them.
|
|
399
|
+
const allAssets: ConvertAsset[] = []
|
|
400
|
+
for (const prepared of output.assets ?? []) {
|
|
401
|
+
const target = this.resolveDeliveryFile(id, prepared.rel)
|
|
402
|
+
if (!target) {
|
|
403
|
+
warnings.push({ code: 'ASSET_PATH_REJECTED', message: `资产路径越出交付目录,已跳过:${prepared.rel}` })
|
|
404
|
+
continue
|
|
405
|
+
}
|
|
406
|
+
await mkdir(dirname(target), { recursive: true })
|
|
407
|
+
await writeFile(target, prepared.data)
|
|
408
|
+
allAssets.push({
|
|
409
|
+
rel: prepared.rel,
|
|
410
|
+
path: target,
|
|
411
|
+
bytes: prepared.data.length,
|
|
412
|
+
sha1: sha1(prepared.data),
|
|
413
|
+
origin: 'inline',
|
|
414
|
+
...(prepared.contentType ? { contentType: prepared.contentType } : {}),
|
|
415
|
+
})
|
|
416
|
+
}
|
|
417
|
+
const now = new Date().toISOString()
|
|
418
|
+
const manifest = await writeDelivery(deliveryDir, {
|
|
419
|
+
taskId: id,
|
|
420
|
+
title: output.title,
|
|
421
|
+
request,
|
|
422
|
+
markdown: output.markdown,
|
|
423
|
+
sourceFormat: output.sourceFormat as SourceFormat,
|
|
424
|
+
parser: output.parser,
|
|
425
|
+
...(output.sourcePath ? { sourcePath: output.sourcePath } : {}),
|
|
426
|
+
assets: allAssets,
|
|
427
|
+
warnings,
|
|
428
|
+
dependencyVersions: output.dependencyVersions,
|
|
429
|
+
externalId: request.externalId ?? output.externalId ?? `sha256:${sha256(output.markdown)}`,
|
|
430
|
+
createdAt: task.createdAt,
|
|
431
|
+
updatedAt: now,
|
|
432
|
+
})
|
|
433
|
+
|
|
434
|
+
this.patch(id, {
|
|
435
|
+
state: 'succeeded',
|
|
436
|
+
stage: 'delivered',
|
|
437
|
+
message: `已交付 ${allAssets.length} 个图片资产`,
|
|
438
|
+
progress: 1,
|
|
439
|
+
finishedAt: now,
|
|
440
|
+
deliveryDir,
|
|
441
|
+
manifest,
|
|
442
|
+
sourceFormat: output.sourceFormat,
|
|
443
|
+
warnings,
|
|
444
|
+
error: undefined,
|
|
445
|
+
})
|
|
446
|
+
await rm(taskWorkDir, { recursive: true, force: true })
|
|
447
|
+
} catch (cause) {
|
|
448
|
+
if (controller.signal.aborted || (cause instanceof AdapterError && cause.code === 'TASK_CANCELLED')) {
|
|
449
|
+
this.patch(id, { state: 'cancelled', stage: 'cancelled', message: '已取消', finishedAt: new Date().toISOString(), progress: null })
|
|
450
|
+
await rm(deliveryDir, { recursive: true, force: true })
|
|
451
|
+
return
|
|
452
|
+
}
|
|
453
|
+
const error = cause instanceof AdapterError
|
|
454
|
+
? { code: cause.code, message: cause.message, ...(cause.suggestion ? { suggestion: cause.suggestion } : {}) }
|
|
455
|
+
: {
|
|
456
|
+
code: (cause as Error & { code?: string }).code ?? 'CONVERT_FAILED',
|
|
457
|
+
message: cause instanceof Error ? cause.message : String(cause),
|
|
458
|
+
...((cause as Error & { suggestion?: string }).suggestion ? { suggestion: (cause as Error & { suggestion?: string }).suggestion! } : {}),
|
|
459
|
+
}
|
|
460
|
+
await rm(deliveryDir, { recursive: true, force: true })
|
|
461
|
+
this.patch(id, {
|
|
462
|
+
state: 'failed',
|
|
463
|
+
stage: 'failed',
|
|
464
|
+
message: error.message,
|
|
465
|
+
finishedAt: new Date().toISOString(),
|
|
466
|
+
progress: null,
|
|
467
|
+
error,
|
|
468
|
+
})
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** Resolve a path inside a delivery directory, refusing traversal. */
|
|
473
|
+
resolveDeliveryFile(id: string, relative: string): string | undefined {
|
|
474
|
+
// 交付目录跟着任务走(日期桶 + 任务 id),所以先问任务自己。任务已经不在
|
|
475
|
+
// store 里(例如被 maxTasks 挤掉)时退回「按今天算」——这是唯一的近似,
|
|
476
|
+
// 也是唯一还能给出的答案。
|
|
477
|
+
const task = this.tasks.get(id)
|
|
478
|
+
const base = task?.deliveryDir ? resolve(task.deliveryDir) : resolve(this.deliveryDirOf({ id, createdAt: new Date().toISOString() }))
|
|
479
|
+
const target = resolve(base, normalize(relative).replace(/^([/\\])+/, ''))
|
|
480
|
+
if (target !== base && !target.startsWith(base + sep)) return undefined
|
|
481
|
+
return target
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
async readDeliveryFile(id: string, relative: string): Promise<{ data: Buffer; contentType: string } | undefined> {
|
|
485
|
+
const path = this.resolveDeliveryFile(id, relative)
|
|
486
|
+
if (!path) return undefined
|
|
487
|
+
try {
|
|
488
|
+
const info = await stat(path)
|
|
489
|
+
if (!info.isFile()) return undefined
|
|
490
|
+
return { data: await readFile(path), contentType: contentTypeOf(path) }
|
|
491
|
+
} catch {
|
|
492
|
+
return undefined
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
async ingestPlan(id: string): Promise<IngestPayload> {
|
|
497
|
+
const task = this.tasks.get(id)
|
|
498
|
+
if (!task) throw new AdapterError('TASK_NOT_FOUND', `任务不存在:${id}`)
|
|
499
|
+
if (task.state !== 'succeeded' || !task.manifest) {
|
|
500
|
+
throw new AdapterError('TASK_NOT_READY', `任务未成功交付(当前 ${task.state}),没有可录入的产物。`)
|
|
501
|
+
}
|
|
502
|
+
return task.manifest.ingest
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
listDeliveries(): Array<{ taskId: string; title: string; deliveryDir: string; state: TaskState; assets: number }> {
|
|
506
|
+
return this.list()
|
|
507
|
+
.filter(task => task.state === 'succeeded' && task.deliveryDir)
|
|
508
|
+
.map(task => ({
|
|
509
|
+
taskId: task.id,
|
|
510
|
+
title: task.manifest?.title ?? task.label,
|
|
511
|
+
deliveryDir: task.deliveryDir!,
|
|
512
|
+
state: task.state,
|
|
513
|
+
assets: task.manifest?.assets.length ?? 0,
|
|
514
|
+
}))
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
private schedulePersist(): void {
|
|
518
|
+
if (this.persistTimer) return
|
|
519
|
+
this.persistTimer = setTimeout(() => {
|
|
520
|
+
this.persistTimer = undefined
|
|
521
|
+
void this.persist()
|
|
522
|
+
}, 150)
|
|
523
|
+
this.persistTimer.unref?.()
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
private async persist(): Promise<void> {
|
|
527
|
+
const snapshot = {
|
|
528
|
+
version: 1,
|
|
529
|
+
updatedAt: new Date().toISOString(),
|
|
530
|
+
tasks: [...this.tasks.values()].slice(-(this.options.maxTasks ?? 200)),
|
|
531
|
+
}
|
|
532
|
+
const path = join(this.rootDir, 'tasks.json')
|
|
533
|
+
this.persistTail = this.persistTail
|
|
534
|
+
.then(async () => {
|
|
535
|
+
await mkdir(this.rootDir, { recursive: true })
|
|
536
|
+
const temporary = `${path}.${process.pid}.tmp`
|
|
537
|
+
await writeFile(temporary, `${JSON.stringify(snapshot, null, 2)}\n`, 'utf8')
|
|
538
|
+
const { rename } = await import('node:fs/promises')
|
|
539
|
+
await rename(temporary, path)
|
|
540
|
+
})
|
|
541
|
+
.catch(() => undefined)
|
|
542
|
+
await this.persistTail
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
private async load(): Promise<void> {
|
|
546
|
+
try {
|
|
547
|
+
const raw = await readFile(join(this.rootDir, 'tasks.json'), 'utf8')
|
|
548
|
+
const parsed = JSON.parse(raw) as { tasks?: unknown }
|
|
549
|
+
if (!Array.isArray(parsed.tasks)) return
|
|
550
|
+
for (const entry of parsed.tasks) {
|
|
551
|
+
const task = entry as ConvertTask
|
|
552
|
+
if (!task || typeof task.id !== 'string' || !STATES.includes(task.state)) continue
|
|
553
|
+
// A task that was running when the process died cannot be resumed safely.
|
|
554
|
+
this.tasks.set(
|
|
555
|
+
task.id,
|
|
556
|
+
task.state === 'running' || task.state === 'queued'
|
|
557
|
+
? {
|
|
558
|
+
...task,
|
|
559
|
+
state: 'failed',
|
|
560
|
+
stage: 'interrupted',
|
|
561
|
+
progress: null,
|
|
562
|
+
finishedAt: task.finishedAt ?? new Date().toISOString(),
|
|
563
|
+
error: {
|
|
564
|
+
code: 'INTERRUPTED_BY_RESTART',
|
|
565
|
+
message: '转换进程重启,该任务被中断。',
|
|
566
|
+
suggestion: '点击重试:已下载的输入与中间产物会复用。',
|
|
567
|
+
},
|
|
568
|
+
}
|
|
569
|
+
: task,
|
|
570
|
+
)
|
|
571
|
+
}
|
|
572
|
+
} catch {
|
|
573
|
+
/* first run */
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
/**
|
|
578
|
+
* 按保留期清理交付物。
|
|
579
|
+
*
|
|
580
|
+
* `outRoot` 下面那一层**要么是日期桶、要么是任务目录**,取决于 `bucketByDay`;
|
|
581
|
+
* 两种情况下它都是「一个可整体丢弃的单位」,所以清理规则只有一条:这一层的
|
|
582
|
+
* mtime 早于截止就整块删。分桶时不必再往下一层逐个判 —— 桶内任务通常是同一天
|
|
583
|
+
* 建的,逐个判既白判又会留下一个空壳日期目录。
|
|
584
|
+
*/
|
|
585
|
+
private async cleanup(): Promise<void> {
|
|
586
|
+
const days = this.options.retentionDays ?? 0
|
|
587
|
+
if (days <= 0) return
|
|
588
|
+
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000
|
|
589
|
+
const entries = await readdir(this.outRoot, { withFileTypes: true }).catch(() => [])
|
|
590
|
+
for (const entry of entries) {
|
|
591
|
+
if (!entry.isDirectory()) continue
|
|
592
|
+
const path = join(this.outRoot, entry.name)
|
|
593
|
+
const info = await stat(path).catch(() => undefined)
|
|
594
|
+
if (info && info.mtimeMs < cutoff) await rm(path, { recursive: true, force: true }).catch(() => undefined)
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
function validateRequest(request: ConvertRequest): void {
|
|
600
|
+
if (!['file', 'url', 'video', 'text'].includes(request.kind)) {
|
|
601
|
+
throw new AdapterError('REQUEST_INVALID', `未知的 kind:${String(request.kind)}`, '可用值:file | url | video | text')
|
|
602
|
+
}
|
|
603
|
+
if (request.kind === 'text' && !request.text?.trim()) {
|
|
604
|
+
throw new AdapterError('CONTENT_REQUIRED', 'kind=text 时 text 不能为空。')
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
function defaultFilename(request: ConvertRequest): string {
|
|
609
|
+
if (request.kind === 'text') return 'inline.md'
|
|
610
|
+
if (request.localPath) return basename(request.localPath)
|
|
611
|
+
return 'input'
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
export function safeFilename(value: string): string {
|
|
615
|
+
const base = basename(value).replace(/[^\w.\-\u4e00-\u9fff]+/g, '_')
|
|
616
|
+
return base.slice(0, 160) || `input-${Date.now().toString(36)}`
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/** Slug used when the DSH layer files a delivery as a knowledge source. */
|
|
620
|
+
export function knowledgeSlugFor(taskId: string, title: string): string {
|
|
621
|
+
return slug(`${title}-${shortHash(taskId, 6)}`)
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
export function contentTypeOf(path: string): string {
|
|
625
|
+
switch (extname(path).toLowerCase()) {
|
|
626
|
+
case '.md': return 'text/markdown; charset=utf-8'
|
|
627
|
+
case '.json': return 'application/json; charset=utf-8'
|
|
628
|
+
case '.png': return 'image/png'
|
|
629
|
+
case '.jpg':
|
|
630
|
+
case '.jpeg': return 'image/jpeg'
|
|
631
|
+
case '.gif': return 'image/gif'
|
|
632
|
+
case '.webp': return 'image/webp'
|
|
633
|
+
case '.avif': return 'image/avif'
|
|
634
|
+
case '.svg': return 'image/svg+xml'
|
|
635
|
+
case '.tiff': return 'image/tiff'
|
|
636
|
+
case '.bmp': return 'image/bmp'
|
|
637
|
+
default: return 'application/octet-stream'
|
|
638
|
+
}
|
|
639
|
+
}
|