@biaoo/tiangong-wiki 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/db.js +4 -0
- package/dist/core/indexer.js +9 -9
- package/dist/core/vault-processing.js +237 -23
- package/dist/core/vault.js +12 -0
- package/dist/operations/dashboard.js +4 -0
- package/package.json +1 -1
package/dist/core/db.js
CHANGED
|
@@ -97,6 +97,8 @@ function ensureBaseTables(db, embeddingDimensions) {
|
|
|
97
97
|
queued_at TEXT NOT NULL,
|
|
98
98
|
claimed_at TEXT,
|
|
99
99
|
started_at TEXT,
|
|
100
|
+
heartbeat_at TEXT,
|
|
101
|
+
processing_owner_id TEXT,
|
|
100
102
|
processed_at TEXT,
|
|
101
103
|
result_page_id TEXT,
|
|
102
104
|
error_message TEXT,
|
|
@@ -133,6 +135,8 @@ function ensureBaseTables(db, embeddingDimensions) {
|
|
|
133
135
|
ensureTableColumns(db, "vault_processing_queue", {
|
|
134
136
|
claimed_at: "TEXT",
|
|
135
137
|
started_at: "TEXT",
|
|
138
|
+
heartbeat_at: "TEXT",
|
|
139
|
+
processing_owner_id: "TEXT",
|
|
136
140
|
thread_id: "TEXT",
|
|
137
141
|
workflow_version: "TEXT",
|
|
138
142
|
decision: "TEXT",
|
package/dist/core/indexer.js
CHANGED
|
@@ -184,6 +184,15 @@ export function applyChanges(db, changes, wikiPath, config, ftsExtensionVersion)
|
|
|
184
184
|
const summaryChangedIds = [];
|
|
185
185
|
const hasContentChanges = parsedEntries.length > 0 || changes.deleted.length > 0;
|
|
186
186
|
const transaction = db.transaction(() => {
|
|
187
|
+
for (const page of changes.deleted) {
|
|
188
|
+
const existing = selectPageRowid.get(page.id);
|
|
189
|
+
deleteEdgesBySourcePage.run(page.id);
|
|
190
|
+
if (existing) {
|
|
191
|
+
deleteVecRow.run(BigInt(existing.rowid));
|
|
192
|
+
}
|
|
193
|
+
deletePage.run(page.id);
|
|
194
|
+
deleted.push(page.id);
|
|
195
|
+
}
|
|
187
196
|
for (const { entry, parsed } of parsedEntries) {
|
|
188
197
|
const existing = selectExistingPage.get(entry.id);
|
|
189
198
|
const isInsert = !existing;
|
|
@@ -217,15 +226,6 @@ export function applyChanges(db, changes, wikiPath, config, ftsExtensionVersion)
|
|
|
217
226
|
});
|
|
218
227
|
}
|
|
219
228
|
}
|
|
220
|
-
for (const page of changes.deleted) {
|
|
221
|
-
const existing = selectPageRowid.get(page.id);
|
|
222
|
-
deleteEdgesBySourcePage.run(page.id);
|
|
223
|
-
if (existing) {
|
|
224
|
-
deleteVecRow.run(BigInt(existing.rowid));
|
|
225
|
-
}
|
|
226
|
-
deletePage.run(page.id);
|
|
227
|
-
deleted.push(page.id);
|
|
228
|
-
}
|
|
229
229
|
if (hasContentChanges) {
|
|
230
230
|
rebuildFts(db, config, ftsExtensionVersion);
|
|
231
231
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import os from "node:os";
|
|
1
3
|
import path from "node:path";
|
|
2
4
|
import { CODEX_WORKFLOW_VERSION, createDefaultWorkflowRunner, } from "./codex-workflow.js";
|
|
3
5
|
import { loadConfig } from "./config.js";
|
|
@@ -14,6 +16,8 @@ const INLINE_WORKFLOW_ATTEMPTS = 2;
|
|
|
14
16
|
const MAX_QUEUE_ERROR_RETRIES = 3;
|
|
15
17
|
const QUEUE_FULL_RETRY_DELAY_SECONDS = 300;
|
|
16
18
|
const WORKFLOW_TIMEOUT_RETRY_DELAY_SECONDS = 120;
|
|
19
|
+
const PROCESSING_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
20
|
+
const PROCESSING_STALE_THRESHOLD_SECONDS = 600;
|
|
17
21
|
const NON_RETRYABLE_QUEUE_ERROR_CODES = new Set(["config_error", "invalid_request"]);
|
|
18
22
|
function buildFileIdFilterClause(filterFileIds) {
|
|
19
23
|
if (!filterFileIds || filterFileIds.length === 0) {
|
|
@@ -66,6 +70,8 @@ function mapQueueRow(row) {
|
|
|
66
70
|
queuedAt: String(row.queuedAt),
|
|
67
71
|
claimedAt: typeof row.claimedAt === "string" ? row.claimedAt : null,
|
|
68
72
|
startedAt: typeof row.startedAt === "string" ? row.startedAt : null,
|
|
73
|
+
heartbeatAt: typeof row.heartbeatAt === "string" ? row.heartbeatAt : null,
|
|
74
|
+
processingOwnerId: typeof row.processingOwnerId === "string" ? row.processingOwnerId : null,
|
|
69
75
|
processedAt: typeof row.processedAt === "string" ? row.processedAt : null,
|
|
70
76
|
resultPageId: typeof row.resultPageId === "string" ? row.resultPageId : null,
|
|
71
77
|
errorMessage: typeof row.errorMessage === "string" ? row.errorMessage : null,
|
|
@@ -90,7 +96,7 @@ function mapQueueRow(row) {
|
|
|
90
96
|
filePath: typeof row.filePath === "string" ? row.filePath : undefined,
|
|
91
97
|
};
|
|
92
98
|
}
|
|
93
|
-
function claimQueueItems(db, limit, options
|
|
99
|
+
function claimQueueItems(db, limit, options) {
|
|
94
100
|
const filter = buildFileIdFilterClause(options.filterFileIds);
|
|
95
101
|
const exclude = buildExcludedFileIdClause(options.excludeFileIds);
|
|
96
102
|
const manualClaim = Boolean(options.filterFileIds && options.filterFileIds.length > 0);
|
|
@@ -112,6 +118,8 @@ function claimQueueItems(db, limit, options = {}) {
|
|
|
112
118
|
queued_at AS queuedAt,
|
|
113
119
|
claimed_at AS claimedAt,
|
|
114
120
|
started_at AS startedAt,
|
|
121
|
+
heartbeat_at AS heartbeatAt,
|
|
122
|
+
processing_owner_id AS processingOwnerId,
|
|
115
123
|
processed_at AS processedAt,
|
|
116
124
|
result_page_id AS resultPageId,
|
|
117
125
|
error_message AS errorMessage,
|
|
@@ -150,8 +158,20 @@ function claimQueueItems(db, limit, options = {}) {
|
|
|
150
158
|
status = 'processing',
|
|
151
159
|
claimed_at = @claimed_at,
|
|
152
160
|
started_at = @started_at,
|
|
161
|
+
heartbeat_at = @heartbeat_at,
|
|
162
|
+
processing_owner_id = @processing_owner_id,
|
|
163
|
+
processed_at = NULL,
|
|
164
|
+
result_page_id = NULL,
|
|
153
165
|
error_message = NULL,
|
|
154
|
-
|
|
166
|
+
decision = NULL,
|
|
167
|
+
last_error_at = NULL,
|
|
168
|
+
last_error_code = NULL,
|
|
169
|
+
retry_after = NULL,
|
|
170
|
+
created_page_ids = NULL,
|
|
171
|
+
updated_page_ids = NULL,
|
|
172
|
+
applied_type_names = NULL,
|
|
173
|
+
proposed_type_names = NULL,
|
|
174
|
+
skills_used = NULL
|
|
155
175
|
WHERE file_id = @file_id AND status IN ('pending', 'error')
|
|
156
176
|
`);
|
|
157
177
|
return db.transaction((claimLimit, claimFilterParams) => {
|
|
@@ -165,12 +185,16 @@ function claimQueueItems(db, limit, options = {}) {
|
|
|
165
185
|
file_id: item.fileId,
|
|
166
186
|
claimed_at: startedAt,
|
|
167
187
|
started_at: startedAt,
|
|
188
|
+
heartbeat_at: startedAt,
|
|
189
|
+
processing_owner_id: options.processingOwnerId,
|
|
168
190
|
});
|
|
169
191
|
}
|
|
170
192
|
return items.map((item) => ({
|
|
171
193
|
...item,
|
|
172
194
|
claimedAt: startedAt,
|
|
173
195
|
startedAt,
|
|
196
|
+
heartbeatAt: startedAt,
|
|
197
|
+
processingOwnerId: options.processingOwnerId,
|
|
174
198
|
}));
|
|
175
199
|
})(limit, [...filter.params, ...exclude.params]);
|
|
176
200
|
}
|
|
@@ -183,6 +207,8 @@ function fetchQueueItemsByStatus(db, status) {
|
|
|
183
207
|
queued_at AS queuedAt,
|
|
184
208
|
claimed_at AS claimedAt,
|
|
185
209
|
started_at AS startedAt,
|
|
210
|
+
heartbeat_at AS heartbeatAt,
|
|
211
|
+
processing_owner_id AS processingOwnerId,
|
|
186
212
|
processed_at AS processedAt,
|
|
187
213
|
result_page_id AS resultPageId,
|
|
188
214
|
error_message AS errorMessage,
|
|
@@ -220,6 +246,8 @@ function fetchQueueItemByFileId(db, fileId) {
|
|
|
220
246
|
queued_at AS queuedAt,
|
|
221
247
|
claimed_at AS claimedAt,
|
|
222
248
|
started_at AS startedAt,
|
|
249
|
+
heartbeat_at AS heartbeatAt,
|
|
250
|
+
processing_owner_id AS processingOwnerId,
|
|
223
251
|
processed_at AS processedAt,
|
|
224
252
|
result_page_id AS resultPageId,
|
|
225
253
|
error_message AS errorMessage,
|
|
@@ -264,12 +292,129 @@ function fetchVaultFile(db, fileId) {
|
|
|
264
292
|
`).get(fileId);
|
|
265
293
|
return row ?? null;
|
|
266
294
|
}
|
|
295
|
+
function buildProcessingOwnerId() {
|
|
296
|
+
return `${os.hostname()}:${process.pid}:${Date.now()}:${randomUUID().slice(0, 8)}`;
|
|
297
|
+
}
|
|
298
|
+
function updateQueueProcessingHeartbeat(db, fileId, processingOwnerId, heartbeatAt = toOffsetIso()) {
|
|
299
|
+
const result = db.prepare(`
|
|
300
|
+
UPDATE vault_processing_queue
|
|
301
|
+
SET heartbeat_at = @heartbeat_at
|
|
302
|
+
WHERE file_id = @file_id
|
|
303
|
+
AND status = 'processing'
|
|
304
|
+
AND processing_owner_id = @processing_owner_id
|
|
305
|
+
`).run({
|
|
306
|
+
file_id: fileId,
|
|
307
|
+
heartbeat_at: heartbeatAt,
|
|
308
|
+
processing_owner_id: processingOwnerId,
|
|
309
|
+
});
|
|
310
|
+
return result.changes > 0;
|
|
311
|
+
}
|
|
312
|
+
function startProcessingHeartbeat(input) {
|
|
313
|
+
const processingOwnerId = input.processingOwnerId?.trim() ?? "";
|
|
314
|
+
if (!processingOwnerId) {
|
|
315
|
+
return {
|
|
316
|
+
refresh: () => { },
|
|
317
|
+
stop: () => { },
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
const refresh = () => {
|
|
321
|
+
updateQueueProcessingHeartbeat(input.db, input.fileId, processingOwnerId);
|
|
322
|
+
};
|
|
323
|
+
const timer = setInterval(refresh, PROCESSING_HEARTBEAT_INTERVAL_MS);
|
|
324
|
+
timer.unref?.();
|
|
325
|
+
return {
|
|
326
|
+
refresh,
|
|
327
|
+
stop: () => clearInterval(timer),
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
function fetchStaleProcessingQueueItems(db, processingOwnerId) {
|
|
331
|
+
const cutoff = toOffsetIso(addSeconds(new Date(), -PROCESSING_STALE_THRESHOLD_SECONDS));
|
|
332
|
+
const rows = db.prepare(`
|
|
333
|
+
SELECT
|
|
334
|
+
file_id AS fileId,
|
|
335
|
+
status,
|
|
336
|
+
priority,
|
|
337
|
+
queued_at AS queuedAt,
|
|
338
|
+
claimed_at AS claimedAt,
|
|
339
|
+
started_at AS startedAt,
|
|
340
|
+
heartbeat_at AS heartbeatAt,
|
|
341
|
+
processing_owner_id AS processingOwnerId,
|
|
342
|
+
processed_at AS processedAt,
|
|
343
|
+
result_page_id AS resultPageId,
|
|
344
|
+
error_message AS errorMessage,
|
|
345
|
+
attempts,
|
|
346
|
+
thread_id AS threadId,
|
|
347
|
+
workflow_version AS workflowVersion,
|
|
348
|
+
decision,
|
|
349
|
+
result_manifest_path AS resultManifestPath,
|
|
350
|
+
last_error_at AS lastErrorAt,
|
|
351
|
+
last_error_code AS lastErrorCode,
|
|
352
|
+
retry_after AS retryAfter,
|
|
353
|
+
created_page_ids AS createdPageIds,
|
|
354
|
+
updated_page_ids AS updatedPageIds,
|
|
355
|
+
applied_type_names AS appliedTypeNames,
|
|
356
|
+
proposed_type_names AS proposedTypeNames,
|
|
357
|
+
skills_used AS skillsUsed,
|
|
358
|
+
vault_files.file_name AS fileName,
|
|
359
|
+
vault_files.file_ext AS fileExt,
|
|
360
|
+
vault_files.source_type AS sourceType,
|
|
361
|
+
vault_files.file_size AS fileSize,
|
|
362
|
+
vault_files.file_path AS filePath
|
|
363
|
+
FROM vault_processing_queue
|
|
364
|
+
LEFT JOIN vault_files ON vault_files.id = vault_processing_queue.file_id
|
|
365
|
+
WHERE status = 'processing'
|
|
366
|
+
AND COALESCE(processing_owner_id, '') != ?
|
|
367
|
+
AND (
|
|
368
|
+
(heartbeat_at IS NOT NULL AND julianday(heartbeat_at) <= julianday(?))
|
|
369
|
+
OR (
|
|
370
|
+
heartbeat_at IS NULL
|
|
371
|
+
AND claimed_at IS NOT NULL
|
|
372
|
+
AND julianday(claimed_at) <= julianday(?)
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
ORDER BY COALESCE(heartbeat_at, claimed_at) ASC, priority DESC, queued_at ASC
|
|
376
|
+
`).all(processingOwnerId, cutoff, cutoff);
|
|
377
|
+
return rows.map(mapQueueRow);
|
|
378
|
+
}
|
|
379
|
+
function requeueRecoveredProcessingItem(db, fileId) {
|
|
380
|
+
db.prepare(`
|
|
381
|
+
UPDATE vault_processing_queue
|
|
382
|
+
SET
|
|
383
|
+
status = 'pending',
|
|
384
|
+
queued_at = @queued_at,
|
|
385
|
+
claimed_at = NULL,
|
|
386
|
+
started_at = NULL,
|
|
387
|
+
heartbeat_at = NULL,
|
|
388
|
+
processing_owner_id = NULL,
|
|
389
|
+
processed_at = NULL,
|
|
390
|
+
result_page_id = NULL,
|
|
391
|
+
error_message = NULL,
|
|
392
|
+
thread_id = NULL,
|
|
393
|
+
workflow_version = NULL,
|
|
394
|
+
decision = NULL,
|
|
395
|
+
result_manifest_path = NULL,
|
|
396
|
+
last_error_at = NULL,
|
|
397
|
+
last_error_code = NULL,
|
|
398
|
+
retry_after = NULL,
|
|
399
|
+
created_page_ids = NULL,
|
|
400
|
+
updated_page_ids = NULL,
|
|
401
|
+
applied_type_names = NULL,
|
|
402
|
+
proposed_type_names = NULL,
|
|
403
|
+
skills_used = NULL
|
|
404
|
+
WHERE file_id = @file_id
|
|
405
|
+
`).run({
|
|
406
|
+
file_id: fileId,
|
|
407
|
+
queued_at: toOffsetIso(),
|
|
408
|
+
});
|
|
409
|
+
}
|
|
267
410
|
function updateQueueStatus(db, fileId, payload) {
|
|
268
411
|
db.prepare(`
|
|
269
412
|
UPDATE vault_processing_queue
|
|
270
413
|
SET
|
|
271
414
|
status = @status,
|
|
272
415
|
processed_at = @processed_at,
|
|
416
|
+
heartbeat_at = NULL,
|
|
417
|
+
processing_owner_id = NULL,
|
|
273
418
|
result_page_id = COALESCE(@result_page_id, result_page_id),
|
|
274
419
|
error_message = @error_message,
|
|
275
420
|
attempts = CASE WHEN @increment_attempts = 1 THEN attempts + 1 ELSE attempts END
|
|
@@ -289,13 +434,17 @@ function updateQueueWorkflowTracking(db, fileId, payload) {
|
|
|
289
434
|
SET
|
|
290
435
|
thread_id = @thread_id,
|
|
291
436
|
workflow_version = @workflow_version,
|
|
292
|
-
result_manifest_path = @result_manifest_path
|
|
437
|
+
result_manifest_path = @result_manifest_path,
|
|
438
|
+
heartbeat_at = COALESCE(@heartbeat_at, heartbeat_at)
|
|
293
439
|
WHERE file_id = @file_id
|
|
294
440
|
`).run({
|
|
295
441
|
file_id: fileId,
|
|
296
442
|
thread_id: payload.threadId,
|
|
297
443
|
workflow_version: CODEX_WORKFLOW_VERSION,
|
|
298
444
|
result_manifest_path: payload.resultManifestPath,
|
|
445
|
+
heartbeat_at: payload.processingOwnerId && payload.processingOwnerId.trim()
|
|
446
|
+
? toOffsetIso()
|
|
447
|
+
: null,
|
|
299
448
|
});
|
|
300
449
|
}
|
|
301
450
|
function serializeArray(value) {
|
|
@@ -376,6 +525,8 @@ function applyWorkflowManifest(db, fileId, manifest, resultManifestPath, current
|
|
|
376
525
|
SET
|
|
377
526
|
status = 'error',
|
|
378
527
|
processed_at = @processed_at,
|
|
528
|
+
heartbeat_at = NULL,
|
|
529
|
+
processing_owner_id = NULL,
|
|
379
530
|
result_page_id = @result_page_id,
|
|
380
531
|
error_message = @error_message,
|
|
381
532
|
attempts = attempts + 1,
|
|
@@ -415,6 +566,8 @@ function applyWorkflowManifest(db, fileId, manifest, resultManifestPath, current
|
|
|
415
566
|
SET
|
|
416
567
|
status = @status,
|
|
417
568
|
processed_at = @processed_at,
|
|
569
|
+
heartbeat_at = NULL,
|
|
570
|
+
processing_owner_id = NULL,
|
|
418
571
|
result_page_id = @result_page_id,
|
|
419
572
|
error_message = NULL,
|
|
420
573
|
workflow_version = @workflow_version,
|
|
@@ -513,13 +666,16 @@ async function runWithWorkflowTimeout(phase, timeoutMs, controller, run) {
|
|
|
513
666
|
}
|
|
514
667
|
// A runner failure can happen after the agent has already written a final result.json.
|
|
515
668
|
// Recover that manifest instead of blindly re-injecting the same task.
|
|
516
|
-
function readRecoverableWorkflowResult(resultPath, expectedThreadId) {
|
|
669
|
+
function readRecoverableWorkflowResult(resultPath, expectedThreadId, options = {}) {
|
|
517
670
|
if (!resultPath || !expectedThreadId) {
|
|
518
671
|
return null;
|
|
519
672
|
}
|
|
520
673
|
try {
|
|
521
674
|
const manifest = readWorkflowResult(resultPath);
|
|
522
|
-
if (manifest.threadId !== expectedThreadId
|
|
675
|
+
if (manifest.threadId !== expectedThreadId) {
|
|
676
|
+
return null;
|
|
677
|
+
}
|
|
678
|
+
if (manifest.status === "error" && options.allowError !== true) {
|
|
523
679
|
return null;
|
|
524
680
|
}
|
|
525
681
|
return manifest;
|
|
@@ -528,6 +684,37 @@ function readRecoverableWorkflowResult(resultPath, expectedThreadId) {
|
|
|
528
684
|
return null;
|
|
529
685
|
}
|
|
530
686
|
}
|
|
687
|
+
function recoverStaleProcessingQueueItems(input) {
|
|
688
|
+
const staleItems = fetchStaleProcessingQueueItems(input.db, input.processingOwnerId);
|
|
689
|
+
const recovered = [];
|
|
690
|
+
for (const item of staleItems) {
|
|
691
|
+
const recoveredManifest = readRecoverableWorkflowResult(item.resultManifestPath, item.threadId ?? null, {
|
|
692
|
+
allowError: true,
|
|
693
|
+
});
|
|
694
|
+
if (recoveredManifest && item.resultManifestPath) {
|
|
695
|
+
assertTemplateEvolutionAllowed(recoveredManifest, input.templateEvolution);
|
|
696
|
+
const outcome = applyWorkflowManifest(input.db, item.fileId, recoveredManifest, item.resultManifestPath, item.attempts);
|
|
697
|
+
input.log?.(`${item.fileId}: recovered stale processing with persisted result status=${outcome.status} thread=${recoveredManifest.threadId} ${formatManifestLogFields(recoveredManifest)} result=${item.resultManifestPath}`);
|
|
698
|
+
recovered.push({
|
|
699
|
+
fileId: item.fileId,
|
|
700
|
+
status: outcome.status,
|
|
701
|
+
pageId: outcome.pageId,
|
|
702
|
+
reason: recoveredManifest.reason,
|
|
703
|
+
threadId: recoveredManifest.threadId,
|
|
704
|
+
decision: recoveredManifest.decision,
|
|
705
|
+
skillsUsed: recoveredManifest.skillsUsed,
|
|
706
|
+
createdPageIds: recoveredManifest.createdPageIds,
|
|
707
|
+
updatedPageIds: recoveredManifest.updatedPageIds,
|
|
708
|
+
proposedTypeNames: recoveredManifest.proposedTypes.map((entry) => entry.name),
|
|
709
|
+
resultManifestPath: item.resultManifestPath,
|
|
710
|
+
});
|
|
711
|
+
continue;
|
|
712
|
+
}
|
|
713
|
+
requeueRecoveredProcessingItem(input.db, item.fileId);
|
|
714
|
+
input.log?.(`${item.fileId}: requeued stale processing claim=${item.claimedAt ?? "-"} heartbeat=${item.heartbeatAt ?? "-"} owner=${item.processingOwnerId ?? "-"} result=${item.resultManifestPath ?? "-"}`);
|
|
715
|
+
}
|
|
716
|
+
return recovered;
|
|
717
|
+
}
|
|
531
718
|
function shouldAttemptManifestRecovery(error) {
|
|
532
719
|
if (error instanceof AppError && error.type === "config") {
|
|
533
720
|
return false;
|
|
@@ -564,6 +751,8 @@ function updateQueueWorkflowError(db, fileId, payload) {
|
|
|
564
751
|
SET
|
|
565
752
|
status = 'error',
|
|
566
753
|
processed_at = @processed_at,
|
|
754
|
+
heartbeat_at = NULL,
|
|
755
|
+
processing_owner_id = NULL,
|
|
567
756
|
error_message = @error_message,
|
|
568
757
|
attempts = attempts + 1,
|
|
569
758
|
started_at = COALESCE(started_at, @processed_at),
|
|
@@ -636,28 +825,34 @@ function prepareCodexWorkflowInput(paths, item, file, localFilePath, env, allowT
|
|
|
636
825
|
async function processClaimedQueueItem(input) {
|
|
637
826
|
const { db, env, paths, item, workflowRunner, templateEvolution, maxWorkflowAttempts, workflowTimeoutMs } = input;
|
|
638
827
|
input.log?.(`${item.fileId}: start processing attempt=${item.attempts + 1} queuedAt=${item.queuedAt} thread=${item.threadId ?? "-"}`);
|
|
639
|
-
const
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
errorMessage: `Vault file missing from index: ${item.fileId}`,
|
|
645
|
-
incrementAttempts: true,
|
|
646
|
-
});
|
|
647
|
-
input.log?.(`${item.fileId}: error thread=- result=- message=Vault file missing from index`);
|
|
648
|
-
return {
|
|
649
|
-
status: "error",
|
|
650
|
-
item: {
|
|
651
|
-
fileId: item.fileId,
|
|
652
|
-
status: "error",
|
|
653
|
-
reason: "Vault file missing from index",
|
|
654
|
-
},
|
|
655
|
-
};
|
|
656
|
-
}
|
|
828
|
+
const processingHeartbeat = startProcessingHeartbeat({
|
|
829
|
+
db,
|
|
830
|
+
fileId: item.fileId,
|
|
831
|
+
processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
|
|
832
|
+
});
|
|
657
833
|
let threadId = item.threadId ?? null;
|
|
658
834
|
let resultManifestPath = null;
|
|
659
835
|
try {
|
|
836
|
+
const file = fetchVaultFile(db, item.fileId);
|
|
837
|
+
if (!file) {
|
|
838
|
+
updateQueueStatus(db, item.fileId, {
|
|
839
|
+
status: "error",
|
|
840
|
+
processedAt: toOffsetIso(),
|
|
841
|
+
errorMessage: `Vault file missing from index: ${item.fileId}`,
|
|
842
|
+
incrementAttempts: true,
|
|
843
|
+
});
|
|
844
|
+
input.log?.(`${item.fileId}: error thread=- result=- message=Vault file missing from index`);
|
|
845
|
+
return {
|
|
846
|
+
status: "error",
|
|
847
|
+
item: {
|
|
848
|
+
fileId: item.fileId,
|
|
849
|
+
status: "error",
|
|
850
|
+
reason: "Vault file missing from index",
|
|
851
|
+
},
|
|
852
|
+
};
|
|
853
|
+
}
|
|
660
854
|
const localFilePath = await ensureLocalVaultFile(file, paths.vaultPath, env);
|
|
855
|
+
processingHeartbeat.refresh();
|
|
661
856
|
const { artifacts, input: workflowInput } = prepareCodexWorkflowInput(paths, item, file, localFilePath, env, templateEvolution.canApply);
|
|
662
857
|
resultManifestPath = artifacts.resultPath;
|
|
663
858
|
let finalOutcome = null;
|
|
@@ -679,6 +874,7 @@ async function processClaimedQueueItem(input) {
|
|
|
679
874
|
updateQueueWorkflowTracking(db, item.fileId, {
|
|
680
875
|
threadId: startedThreadId,
|
|
681
876
|
resultManifestPath: artifacts.resultPath,
|
|
877
|
+
processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
|
|
682
878
|
});
|
|
683
879
|
input.log?.(`${item.fileId}: workflow started mode=${mode} attempt=${attempt}/${maxWorkflowAttempts} thread=${startedThreadId} result=${artifacts.resultPath}`);
|
|
684
880
|
},
|
|
@@ -695,7 +891,9 @@ async function processClaimedQueueItem(input) {
|
|
|
695
891
|
updateQueueWorkflowTracking(db, item.fileId, {
|
|
696
892
|
threadId: handle.threadId,
|
|
697
893
|
resultManifestPath: artifacts.resultPath,
|
|
894
|
+
processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
|
|
698
895
|
});
|
|
896
|
+
processingHeartbeat.refresh();
|
|
699
897
|
input.log?.(`${item.fileId}: waiting for workflow result thread=${handle.threadId} attempt=${attempt}/${maxWorkflowAttempts} result=${artifacts.resultPath}`);
|
|
700
898
|
const collectController = new AbortController();
|
|
701
899
|
const manifest = await runWithWorkflowTimeout("collectResult", workflowTimeoutMs, collectController, () => workflowRunner.collectResult(handle, {
|
|
@@ -717,6 +915,7 @@ async function processClaimedQueueItem(input) {
|
|
|
717
915
|
updateQueueWorkflowTracking(db, item.fileId, {
|
|
718
916
|
threadId,
|
|
719
917
|
resultManifestPath: artifacts.resultPath,
|
|
918
|
+
processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
|
|
720
919
|
});
|
|
721
920
|
}
|
|
722
921
|
const recoveredManifest = shouldAttemptManifestRecovery(error)
|
|
@@ -811,6 +1010,9 @@ async function processClaimedQueueItem(input) {
|
|
|
811
1010
|
},
|
|
812
1011
|
};
|
|
813
1012
|
}
|
|
1013
|
+
finally {
|
|
1014
|
+
processingHeartbeat.stop();
|
|
1015
|
+
}
|
|
814
1016
|
}
|
|
815
1017
|
export function getVaultQueueSnapshot(env = process.env, status) {
|
|
816
1018
|
const paths = resolveRuntimePaths(env);
|
|
@@ -880,6 +1082,7 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
|
|
|
880
1082
|
const maxWorkflowAttempts = isInlineRetryCapable(workflowRunner) ? INLINE_WORKFLOW_ATTEMPTS : 1;
|
|
881
1083
|
const workflowTimeoutMs = agentSettings.workflowTimeoutSeconds * 1000;
|
|
882
1084
|
const workerSlots = Math.max(0, options.maxItems ?? agentSettings.batchSize);
|
|
1085
|
+
const processingOwnerId = buildProcessingOwnerId();
|
|
883
1086
|
const attemptedFileIds = new Set();
|
|
884
1087
|
const orderedItems = [];
|
|
885
1088
|
let nextSequence = 0;
|
|
@@ -895,6 +1098,15 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
|
|
|
895
1098
|
}
|
|
896
1099
|
result.processed += 1;
|
|
897
1100
|
};
|
|
1101
|
+
for (const recoveredItem of recoverStaleProcessingQueueItems({
|
|
1102
|
+
db,
|
|
1103
|
+
processingOwnerId,
|
|
1104
|
+
log: options.log,
|
|
1105
|
+
templateEvolution,
|
|
1106
|
+
})) {
|
|
1107
|
+
countOutcome(recoveredItem.status);
|
|
1108
|
+
orderedItems.push({ sequence: nextSequence++, item: recoveredItem });
|
|
1109
|
+
}
|
|
898
1110
|
const claimNextQueueItem = () => {
|
|
899
1111
|
if (options.shouldStop?.() === true) {
|
|
900
1112
|
return null;
|
|
@@ -906,6 +1118,7 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
|
|
|
906
1118
|
const item = claimQueueItems(db, 1, {
|
|
907
1119
|
filterFileIds: remainingFilterFileIds,
|
|
908
1120
|
excludeFileIds: attemptedFileIds,
|
|
1121
|
+
processingOwnerId,
|
|
909
1122
|
})[0];
|
|
910
1123
|
if (!item) {
|
|
911
1124
|
return null;
|
|
@@ -931,6 +1144,7 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
|
|
|
931
1144
|
env,
|
|
932
1145
|
paths,
|
|
933
1146
|
item: claimed.item,
|
|
1147
|
+
processingOwnerId,
|
|
934
1148
|
log: options.log,
|
|
935
1149
|
workflowRunner,
|
|
936
1150
|
templateEvolution,
|
package/dist/core/vault.js
CHANGED
|
@@ -432,6 +432,8 @@ export function syncVaultIndex(db, currentFiles, syncId) {
|
|
|
432
432
|
queued_at,
|
|
433
433
|
claimed_at,
|
|
434
434
|
started_at,
|
|
435
|
+
heartbeat_at,
|
|
436
|
+
processing_owner_id,
|
|
435
437
|
processed_at,
|
|
436
438
|
result_page_id,
|
|
437
439
|
error_message,
|
|
@@ -458,6 +460,8 @@ export function syncVaultIndex(db, currentFiles, syncId) {
|
|
|
458
460
|
NULL,
|
|
459
461
|
NULL,
|
|
460
462
|
NULL,
|
|
463
|
+
NULL,
|
|
464
|
+
NULL,
|
|
461
465
|
0,
|
|
462
466
|
NULL,
|
|
463
467
|
NULL,
|
|
@@ -487,6 +491,14 @@ export function syncVaultIndex(db, currentFiles, syncId) {
|
|
|
487
491
|
WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.started_at
|
|
488
492
|
ELSE NULL
|
|
489
493
|
END,
|
|
494
|
+
heartbeat_at = CASE
|
|
495
|
+
WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.heartbeat_at
|
|
496
|
+
ELSE NULL
|
|
497
|
+
END,
|
|
498
|
+
processing_owner_id = CASE
|
|
499
|
+
WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.processing_owner_id
|
|
500
|
+
ELSE NULL
|
|
501
|
+
END,
|
|
490
502
|
processed_at = CASE
|
|
491
503
|
WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.processed_at
|
|
492
504
|
ELSE NULL
|
|
@@ -252,6 +252,7 @@ function buildQueueTiming(item) {
|
|
|
252
252
|
queuedAt: item.queuedAt,
|
|
253
253
|
claimedAt: item.claimedAt ?? null,
|
|
254
254
|
startedAt: item.startedAt ?? null,
|
|
255
|
+
heartbeatAt: item.heartbeatAt ?? null,
|
|
255
256
|
processedAt: item.processedAt,
|
|
256
257
|
lastErrorAt: item.lastErrorAt ?? null,
|
|
257
258
|
lastErrorCode: item.lastErrorCode ?? null,
|
|
@@ -276,6 +277,7 @@ function buildQueueListItem(item) {
|
|
|
276
277
|
resultPageId: item.resultPageId,
|
|
277
278
|
errorMessage: item.errorMessage,
|
|
278
279
|
lastErrorCode: item.lastErrorCode ?? null,
|
|
280
|
+
processingOwnerId: item.processingOwnerId ?? null,
|
|
279
281
|
threadId: item.threadId ?? null,
|
|
280
282
|
decision: item.decision ?? null,
|
|
281
283
|
workflowVersion: item.workflowVersion ?? null,
|
|
@@ -790,6 +792,8 @@ export function retryDashboardQueueItem(env = process.env, fileId) {
|
|
|
790
792
|
queued_at = @queued_at,
|
|
791
793
|
claimed_at = NULL,
|
|
792
794
|
started_at = NULL,
|
|
795
|
+
heartbeat_at = NULL,
|
|
796
|
+
processing_owner_id = NULL,
|
|
793
797
|
processed_at = NULL,
|
|
794
798
|
result_page_id = NULL,
|
|
795
799
|
error_message = NULL,
|