@biaoo/tiangong-wiki 0.3.5 → 0.3.6

package/dist/core/db.js

@@ -97,6 +97,8 @@ function ensureBaseTables(db, embeddingDimensions) {
       queued_at TEXT NOT NULL,
       claimed_at TEXT,
       started_at TEXT,
+      heartbeat_at TEXT,
+      processing_owner_id TEXT,
       processed_at TEXT,
       result_page_id TEXT,
       error_message TEXT,
@@ -133,6 +135,8 @@ function ensureBaseTables(db, embeddingDimensions) {
     ensureTableColumns(db, "vault_processing_queue", {
         claimed_at: "TEXT",
         started_at: "TEXT",
+        heartbeat_at: "TEXT",
+        processing_owner_id: "TEXT",
         thread_id: "TEXT",
         workflow_version: "TEXT",
         decision: "TEXT",

package/dist/core/vault-processing.js

@@ -1,3 +1,5 @@
+import { randomUUID } from "node:crypto";
+import os from "node:os";
 import path from "node:path";
 import { CODEX_WORKFLOW_VERSION, createDefaultWorkflowRunner, } from "./codex-workflow.js";
 import { loadConfig } from "./config.js";
@@ -14,6 +16,8 @@ const INLINE_WORKFLOW_ATTEMPTS = 2;
 const MAX_QUEUE_ERROR_RETRIES = 3;
 const QUEUE_FULL_RETRY_DELAY_SECONDS = 300;
 const WORKFLOW_TIMEOUT_RETRY_DELAY_SECONDS = 120;
+const PROCESSING_HEARTBEAT_INTERVAL_MS = 15_000;
+const PROCESSING_STALE_THRESHOLD_SECONDS = 600;
 const NON_RETRYABLE_QUEUE_ERROR_CODES = new Set(["config_error", "invalid_request"]);
 function buildFileIdFilterClause(filterFileIds) {
     if (!filterFileIds || filterFileIds.length === 0) {
@@ -66,6 +70,8 @@ function mapQueueRow(row) {
         queuedAt: String(row.queuedAt),
         claimedAt: typeof row.claimedAt === "string" ? row.claimedAt : null,
         startedAt: typeof row.startedAt === "string" ? row.startedAt : null,
+        heartbeatAt: typeof row.heartbeatAt === "string" ? row.heartbeatAt : null,
+        processingOwnerId: typeof row.processingOwnerId === "string" ? row.processingOwnerId : null,
         processedAt: typeof row.processedAt === "string" ? row.processedAt : null,
         resultPageId: typeof row.resultPageId === "string" ? row.resultPageId : null,
         errorMessage: typeof row.errorMessage === "string" ? row.errorMessage : null,
@@ -90,7 +96,7 @@ function mapQueueRow(row) {
         filePath: typeof row.filePath === "string" ? row.filePath : undefined,
     };
 }
-function claimQueueItems(db, limit, options = {}) {
+function claimQueueItems(db, limit, options) {
     const filter = buildFileIdFilterClause(options.filterFileIds);
     const exclude = buildExcludedFileIdClause(options.excludeFileIds);
     const manualClaim = Boolean(options.filterFileIds && options.filterFileIds.length > 0);
@@ -112,6 +118,8 @@ function claimQueueItems(db, limit, options = {}) {
         queued_at AS queuedAt,
         claimed_at AS claimedAt,
         started_at AS startedAt,
+        heartbeat_at AS heartbeatAt,
+        processing_owner_id AS processingOwnerId,
         processed_at AS processedAt,
         result_page_id AS resultPageId,
         error_message AS errorMessage,
@@ -150,8 +158,20 @@ function claimQueueItems(db, limit, options = {}) {
         status = 'processing',
         claimed_at = @claimed_at,
         started_at = @started_at,
+        heartbeat_at = @heartbeat_at,
+        processing_owner_id = @processing_owner_id,
+        processed_at = NULL,
+        result_page_id = NULL,
         error_message = NULL,
-        retry_after = NULL
+        decision = NULL,
+        last_error_at = NULL,
+        last_error_code = NULL,
+        retry_after = NULL,
+        created_page_ids = NULL,
+        updated_page_ids = NULL,
+        applied_type_names = NULL,
+        proposed_type_names = NULL,
+        skills_used = NULL
       WHERE file_id = @file_id AND status IN ('pending', 'error')
     `);
     return db.transaction((claimLimit, claimFilterParams) => {
@@ -165,12 +185,16 @@ function claimQueueItems(db, limit, options = {}) {
                 file_id: item.fileId,
                 claimed_at: startedAt,
                 started_at: startedAt,
+                heartbeat_at: startedAt,
+                processing_owner_id: options.processingOwnerId,
             });
         }
         return items.map((item) => ({
             ...item,
             claimedAt: startedAt,
             startedAt,
+            heartbeatAt: startedAt,
+            processingOwnerId: options.processingOwnerId,
         }));
     })(limit, [...filter.params, ...exclude.params]);
 }
@@ -183,6 +207,8 @@ function fetchQueueItemsByStatus(db, status) {
         queued_at AS queuedAt,
         claimed_at AS claimedAt,
         started_at AS startedAt,
+        heartbeat_at AS heartbeatAt,
+        processing_owner_id AS processingOwnerId,
         processed_at AS processedAt,
         result_page_id AS resultPageId,
         error_message AS errorMessage,
@@ -220,6 +246,8 @@ function fetchQueueItemByFileId(db, fileId) {
         queued_at AS queuedAt,
         claimed_at AS claimedAt,
         started_at AS startedAt,
+        heartbeat_at AS heartbeatAt,
+        processing_owner_id AS processingOwnerId,
         processed_at AS processedAt,
         result_page_id AS resultPageId,
         error_message AS errorMessage,
@@ -264,12 +292,129 @@ function fetchVaultFile(db, fileId) {
     `).get(fileId);
     return row ?? null;
 }
+function buildProcessingOwnerId() {
+    return `${os.hostname()}:${process.pid}:${Date.now()}:${randomUUID().slice(0, 8)}`;
+}
+function updateQueueProcessingHeartbeat(db, fileId, processingOwnerId, heartbeatAt = toOffsetIso()) {
+    const result = db.prepare(`
+      UPDATE vault_processing_queue
+      SET heartbeat_at = @heartbeat_at
+      WHERE file_id = @file_id
+        AND status = 'processing'
+        AND processing_owner_id = @processing_owner_id
+    `).run({
+        file_id: fileId,
+        heartbeat_at: heartbeatAt,
+        processing_owner_id: processingOwnerId,
+    });
+    return result.changes > 0;
+}
+function startProcessingHeartbeat(input) {
+    const processingOwnerId = input.processingOwnerId?.trim() ?? "";
+    if (!processingOwnerId) {
+        return {
+            refresh: () => { },
+            stop: () => { },
+        };
+    }
+    const refresh = () => {
+        updateQueueProcessingHeartbeat(input.db, input.fileId, processingOwnerId);
+    };
+    const timer = setInterval(refresh, PROCESSING_HEARTBEAT_INTERVAL_MS);
+    timer.unref?.();
+    return {
+        refresh,
+        stop: () => clearInterval(timer),
+    };
+}
+function fetchStaleProcessingQueueItems(db, processingOwnerId) {
+    const cutoff = toOffsetIso(addSeconds(new Date(), -PROCESSING_STALE_THRESHOLD_SECONDS));
+    const rows = db.prepare(`
+      SELECT
+        file_id AS fileId,
+        status,
+        priority,
+        queued_at AS queuedAt,
+        claimed_at AS claimedAt,
+        started_at AS startedAt,
+        heartbeat_at AS heartbeatAt,
+        processing_owner_id AS processingOwnerId,
+        processed_at AS processedAt,
+        result_page_id AS resultPageId,
+        error_message AS errorMessage,
+        attempts,
+        thread_id AS threadId,
+        workflow_version AS workflowVersion,
+        decision,
+        result_manifest_path AS resultManifestPath,
+        last_error_at AS lastErrorAt,
+        last_error_code AS lastErrorCode,
+        retry_after AS retryAfter,
+        created_page_ids AS createdPageIds,
+        updated_page_ids AS updatedPageIds,
+        applied_type_names AS appliedTypeNames,
+        proposed_type_names AS proposedTypeNames,
+        skills_used AS skillsUsed,
+        vault_files.file_name AS fileName,
+        vault_files.file_ext AS fileExt,
+        vault_files.source_type AS sourceType,
+        vault_files.file_size AS fileSize,
+        vault_files.file_path AS filePath
+      FROM vault_processing_queue
+      LEFT JOIN vault_files ON vault_files.id = vault_processing_queue.file_id
+      WHERE status = 'processing'
+        AND COALESCE(processing_owner_id, '') != ?
+        AND (
+          (heartbeat_at IS NOT NULL AND julianday(heartbeat_at) <= julianday(?))
+          OR (
+            heartbeat_at IS NULL
+            AND claimed_at IS NOT NULL
+            AND julianday(claimed_at) <= julianday(?)
+          )
+        )
+      ORDER BY COALESCE(heartbeat_at, claimed_at) ASC, priority DESC, queued_at ASC
+    `).all(processingOwnerId, cutoff, cutoff);
+    return rows.map(mapQueueRow);
+}
+function requeueRecoveredProcessingItem(db, fileId) {
+    db.prepare(`
+      UPDATE vault_processing_queue
+      SET
+        status = 'pending',
+        queued_at = @queued_at,
+        claimed_at = NULL,
+        started_at = NULL,
+        heartbeat_at = NULL,
+        processing_owner_id = NULL,
+        processed_at = NULL,
+        result_page_id = NULL,
+        error_message = NULL,
+        thread_id = NULL,
+        workflow_version = NULL,
+        decision = NULL,
+        result_manifest_path = NULL,
+        last_error_at = NULL,
+        last_error_code = NULL,
+        retry_after = NULL,
+        created_page_ids = NULL,
+        updated_page_ids = NULL,
+        applied_type_names = NULL,
+        proposed_type_names = NULL,
+        skills_used = NULL
+      WHERE file_id = @file_id
+    `).run({
+        file_id: fileId,
+        queued_at: toOffsetIso(),
+    });
+}
 function updateQueueStatus(db, fileId, payload) {
     db.prepare(`
       UPDATE vault_processing_queue
       SET
         status = @status,
         processed_at = @processed_at,
+        heartbeat_at = NULL,
+        processing_owner_id = NULL,
         result_page_id = COALESCE(@result_page_id, result_page_id),
         error_message = @error_message,
         attempts = CASE WHEN @increment_attempts = 1 THEN attempts + 1 ELSE attempts END
@@ -289,13 +434,17 @@ function updateQueueWorkflowTracking(db, fileId, payload) {
       SET
         thread_id = @thread_id,
         workflow_version = @workflow_version,
-        result_manifest_path = @result_manifest_path
+        result_manifest_path = @result_manifest_path,
+        heartbeat_at = COALESCE(@heartbeat_at, heartbeat_at)
       WHERE file_id = @file_id
     `).run({
         file_id: fileId,
         thread_id: payload.threadId,
         workflow_version: CODEX_WORKFLOW_VERSION,
         result_manifest_path: payload.resultManifestPath,
+        heartbeat_at: payload.processingOwnerId && payload.processingOwnerId.trim()
+            ? toOffsetIso()
+            : null,
     });
 }
 function serializeArray(value) {
@@ -376,6 +525,8 @@ function applyWorkflowManifest(db, fileId, manifest, resultManifestPath, current
         SET
           status = 'error',
           processed_at = @processed_at,
+          heartbeat_at = NULL,
+          processing_owner_id = NULL,
           result_page_id = @result_page_id,
           error_message = @error_message,
           attempts = attempts + 1,
@@ -415,6 +566,8 @@ function applyWorkflowManifest(db, fileId, manifest, resultManifestPath, current
       SET
         status = @status,
         processed_at = @processed_at,
+        heartbeat_at = NULL,
+        processing_owner_id = NULL,
         result_page_id = @result_page_id,
         error_message = NULL,
         workflow_version = @workflow_version,
@@ -513,13 +666,16 @@ async function runWithWorkflowTimeout(phase, timeoutMs, controller, run) {
 }
 // A runner failure can happen after the agent has already written a final result.json.
 // Recover that manifest instead of blindly re-injecting the same task.
-function readRecoverableWorkflowResult(resultPath, expectedThreadId) {
+function readRecoverableWorkflowResult(resultPath, expectedThreadId, options = {}) {
     if (!resultPath || !expectedThreadId) {
         return null;
     }
     try {
         const manifest = readWorkflowResult(resultPath);
-        if (manifest.threadId !== expectedThreadId || manifest.status === "error") {
+        if (manifest.threadId !== expectedThreadId) {
+            return null;
+        }
+        if (manifest.status === "error" && options.allowError !== true) {
             return null;
         }
         return manifest;
@@ -528,6 +684,37 @@ function readRecoverableWorkflowResult(resultPath, expectedThreadId) {
         return null;
     }
 }
+function recoverStaleProcessingQueueItems(input) {
+    const staleItems = fetchStaleProcessingQueueItems(input.db, input.processingOwnerId);
+    const recovered = [];
+    for (const item of staleItems) {
+        const recoveredManifest = readRecoverableWorkflowResult(item.resultManifestPath, item.threadId ?? null, {
+            allowError: true,
+        });
+        if (recoveredManifest && item.resultManifestPath) {
+            assertTemplateEvolutionAllowed(recoveredManifest, input.templateEvolution);
+            const outcome = applyWorkflowManifest(input.db, item.fileId, recoveredManifest, item.resultManifestPath, item.attempts);
+            input.log?.(`${item.fileId}: recovered stale processing with persisted result status=${outcome.status} thread=${recoveredManifest.threadId} ${formatManifestLogFields(recoveredManifest)} result=${item.resultManifestPath}`);
+            recovered.push({
+                fileId: item.fileId,
+                status: outcome.status,
+                pageId: outcome.pageId,
+                reason: recoveredManifest.reason,
+                threadId: recoveredManifest.threadId,
+                decision: recoveredManifest.decision,
+                skillsUsed: recoveredManifest.skillsUsed,
+                createdPageIds: recoveredManifest.createdPageIds,
+                updatedPageIds: recoveredManifest.updatedPageIds,
+                proposedTypeNames: recoveredManifest.proposedTypes.map((entry) => entry.name),
+                resultManifestPath: item.resultManifestPath,
+            });
+            continue;
+        }
+        requeueRecoveredProcessingItem(input.db, item.fileId);
+        input.log?.(`${item.fileId}: requeued stale processing claim=${item.claimedAt ?? "-"} heartbeat=${item.heartbeatAt ?? "-"} owner=${item.processingOwnerId ?? "-"} result=${item.resultManifestPath ?? "-"}`);
+    }
+    return recovered;
+}
 function shouldAttemptManifestRecovery(error) {
     if (error instanceof AppError && error.type === "config") {
         return false;
@@ -564,6 +751,8 @@ function updateQueueWorkflowError(db, fileId, payload) {
       SET
         status = 'error',
         processed_at = @processed_at,
+        heartbeat_at = NULL,
+        processing_owner_id = NULL,
         error_message = @error_message,
         attempts = attempts + 1,
         started_at = COALESCE(started_at, @processed_at),
@@ -636,28 +825,34 @@ function prepareCodexWorkflowInput(paths, item, file, localFilePath, env, allowT
 async function processClaimedQueueItem(input) {
     const { db, env, paths, item, workflowRunner, templateEvolution, maxWorkflowAttempts, workflowTimeoutMs } = input;
     input.log?.(`${item.fileId}: start processing attempt=${item.attempts + 1} queuedAt=${item.queuedAt} thread=${item.threadId ?? "-"}`);
-    const file = fetchVaultFile(db, item.fileId);
-    if (!file) {
-        updateQueueStatus(db, item.fileId, {
-            status: "error",
-            processedAt: toOffsetIso(),
-            errorMessage: `Vault file missing from index: ${item.fileId}`,
-            incrementAttempts: true,
-        });
-        input.log?.(`${item.fileId}: error thread=- result=- message=Vault file missing from index`);
-        return {
-            status: "error",
-            item: {
-                fileId: item.fileId,
-                status: "error",
-                reason: "Vault file missing from index",
-            },
-        };
-    }
+    const processingHeartbeat = startProcessingHeartbeat({
+        db,
+        fileId: item.fileId,
+        processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
+    });
     let threadId = item.threadId ?? null;
     let resultManifestPath = null;
     try {
+        const file = fetchVaultFile(db, item.fileId);
+        if (!file) {
+            updateQueueStatus(db, item.fileId, {
+                status: "error",
+                processedAt: toOffsetIso(),
+                errorMessage: `Vault file missing from index: ${item.fileId}`,
+                incrementAttempts: true,
+            });
+            input.log?.(`${item.fileId}: error thread=- result=- message=Vault file missing from index`);
+            return {
+                status: "error",
+                item: {
+                    fileId: item.fileId,
+                    status: "error",
+                    reason: "Vault file missing from index",
+                },
+            };
+        }
         const localFilePath = await ensureLocalVaultFile(file, paths.vaultPath, env);
+        processingHeartbeat.refresh();
         const { artifacts, input: workflowInput } = prepareCodexWorkflowInput(paths, item, file, localFilePath, env, templateEvolution.canApply);
         resultManifestPath = artifacts.resultPath;
         let finalOutcome = null;
@@ -679,6 +874,7 @@ async function processClaimedQueueItem(input) {
                         updateQueueWorkflowTracking(db, item.fileId, {
                             threadId: startedThreadId,
                             resultManifestPath: artifacts.resultPath,
+                            processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
                         });
                         input.log?.(`${item.fileId}: workflow started mode=${mode} attempt=${attempt}/${maxWorkflowAttempts} thread=${startedThreadId} result=${artifacts.resultPath}`);
                     },
@@ -695,7 +891,9 @@ async function processClaimedQueueItem(input) {
                 updateQueueWorkflowTracking(db, item.fileId, {
                     threadId: handle.threadId,
                     resultManifestPath: artifacts.resultPath,
+                    processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
                 });
+                processingHeartbeat.refresh();
                 input.log?.(`${item.fileId}: waiting for workflow result thread=${handle.threadId} attempt=${attempt}/${maxWorkflowAttempts} result=${artifacts.resultPath}`);
                 const collectController = new AbortController();
                 const manifest = await runWithWorkflowTimeout("collectResult", workflowTimeoutMs, collectController, () => workflowRunner.collectResult(handle, {
@@ -717,6 +915,7 @@ async function processClaimedQueueItem(input) {
                     updateQueueWorkflowTracking(db, item.fileId, {
                         threadId,
                         resultManifestPath: artifacts.resultPath,
+                        processingOwnerId: item.processingOwnerId ?? input.processingOwnerId,
                     });
                 }
                 const recoveredManifest = shouldAttemptManifestRecovery(error)
@@ -811,6 +1010,9 @@ async function processClaimedQueueItem(input) {
             },
         };
     }
+    finally {
+        processingHeartbeat.stop();
+    }
 }
 export function getVaultQueueSnapshot(env = process.env, status) {
     const paths = resolveRuntimePaths(env);
@@ -880,6 +1082,7 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
         const maxWorkflowAttempts = isInlineRetryCapable(workflowRunner) ? INLINE_WORKFLOW_ATTEMPTS : 1;
         const workflowTimeoutMs = agentSettings.workflowTimeoutSeconds * 1000;
         const workerSlots = Math.max(0, options.maxItems ?? agentSettings.batchSize);
+        const processingOwnerId = buildProcessingOwnerId();
         const attemptedFileIds = new Set();
         const orderedItems = [];
         let nextSequence = 0;
@@ -895,6 +1098,15 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
             }
             result.processed += 1;
         };
+        for (const recoveredItem of recoverStaleProcessingQueueItems({
+            db,
+            processingOwnerId,
+            log: options.log,
+            templateEvolution,
+        })) {
+            countOutcome(recoveredItem.status);
+            orderedItems.push({ sequence: nextSequence++, item: recoveredItem });
+        }
         const claimNextQueueItem = () => {
             if (options.shouldStop?.() === true) {
                 return null;
@@ -906,6 +1118,7 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
             const item = claimQueueItems(db, 1, {
                 filterFileIds: remainingFilterFileIds,
                 excludeFileIds: attemptedFileIds,
+                processingOwnerId,
             })[0];
             if (!item) {
                 return null;
@@ -931,6 +1144,7 @@ export async function processVaultQueueBatch(env = process.env, options = {}) {
                     env,
                     paths,
                     item: claimed.item,
+                    processingOwnerId,
                     log: options.log,
                     workflowRunner,
                     templateEvolution,

package/dist/core/vault.js

@@ -432,6 +432,8 @@ export function syncVaultIndex(db, currentFiles, syncId) {
         queued_at,
         claimed_at,
         started_at,
+        heartbeat_at,
+        processing_owner_id,
         processed_at,
         result_page_id,
         error_message,
@@ -458,6 +460,8 @@ export function syncVaultIndex(db, currentFiles, syncId) {
         NULL,
         NULL,
         NULL,
+        NULL,
+        NULL,
         0,
         NULL,
         NULL,
@@ -487,6 +491,14 @@ export function syncVaultIndex(db, currentFiles, syncId) {
           WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.started_at
           ELSE NULL
         END,
+        heartbeat_at = CASE
+          WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.heartbeat_at
+          ELSE NULL
+        END,
+        processing_owner_id = CASE
+          WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.processing_owner_id
+          ELSE NULL
+        END,
         processed_at = CASE
           WHEN vault_processing_queue.status = 'processing' THEN vault_processing_queue.processed_at
           ELSE NULL

package/dist/operations/dashboard.js

@@ -252,6 +252,7 @@ function buildQueueTiming(item) {
         queuedAt: item.queuedAt,
         claimedAt: item.claimedAt ?? null,
         startedAt: item.startedAt ?? null,
+        heartbeatAt: item.heartbeatAt ?? null,
         processedAt: item.processedAt,
         lastErrorAt: item.lastErrorAt ?? null,
         lastErrorCode: item.lastErrorCode ?? null,
@@ -276,6 +277,7 @@ function buildQueueListItem(item) {
         resultPageId: item.resultPageId,
         errorMessage: item.errorMessage,
         lastErrorCode: item.lastErrorCode ?? null,
+        processingOwnerId: item.processingOwnerId ?? null,
         threadId: item.threadId ?? null,
         decision: item.decision ?? null,
         workflowVersion: item.workflowVersion ?? null,
@@ -790,6 +792,8 @@ export function retryDashboardQueueItem(env = process.env, fileId) {
           queued_at = @queued_at,
           claimed_at = NULL,
           started_at = NULL,
+          heartbeat_at = NULL,
+          processing_owner_id = NULL,
           processed_at = NULL,
           result_page_id = NULL,
           error_message = NULL,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@biaoo/tiangong-wiki",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Local-first wiki index and query engine for Markdown knowledge pages (Tiangong Wiki).",
   "type": "module",
   "publishConfig": {