@geotechcli/core 0.4.25 → 0.4.27

package/dist/ingest/job-worker.js

@@ -1,11 +1,11 @@
 import { basename, join } from 'node:path';
 import { homedir } from 'node:os';
 import { buildLLMConfig } from '../config/index.js';
-import { readDocumentPdfPageInputs } from './document-inputs.js';
-import { inspectPdfDocument } from './pdf.js';
+import { countDocumentPdfPages, readDocumentPdfPageInputs } from './document-inputs.js';
+import { inferPdfDocumentPageCountFallback, inspectPdfDocument, } from './pdf.js';
 import { ingestBoreholeLogDocument, summarizeBoreholeIngestInspection, } from './geotech-extract.js';
 import { buildPreflightLowYieldInsight, ingestGeotechDocument, inferPreflightLowYieldPageRole, } from './geotech-document.js';
-import { extractGeotechDocumentFactsFromText, interpretGeotechDocumentPage, } from '../vision/geotech-document.js';
+import { extractGeotechDocumentDeterministicFactsFromText, extractGeotechDocumentFactsFromText, interpretGeotechDocumentPage, } from '../vision/geotech-document.js';
 import { interpretBoreholeLogWithContext, transcribeDocumentImageText, } from '../vision/index.js';
 import { recoverDocumentTextHint } from '../vision/ocr.js';
 import { persistBoreholeIngestReview } from './review-store.js';
@@ -13,6 +13,8 @@ import { buildPersistedIngestJobSegments, createPersistedIngestJob, loadPersiste
 import { HOSTED_BETA_EFFECTIVE_PAGE_LIMIT, slicePdfInspectionToRange, writePdfPageSubset, } from './segmentation.js';
 const SLOW_VISUAL_ERROR_PATTERNS = [
     /timeout/i,
+    /\b524\b/i,
+    /upstream(?: request)? (?:timed out|timeout|failed)/i,
     /provider is busy/i,
     /returned no content/i,
     /did not contain assistant text/i,
@@ -22,6 +24,15 @@ const SLOW_VISUAL_ERROR_PATTERNS = [
     /\b503\b/i,
     /\b504\b/i,
 ];
+const RETRYABLE_UPSTREAM_ERROR_PATTERNS = [
+    /timeout/i,
+    /\b524\b/i,
+    /upstream(?: request)? (?:timed out|timeout|failed)/i,
+    /provider is busy/i,
+    /temporarily unavailable/i,
+    /\b503\b/i,
+    /\b504\b/i,
+];
 const FATAL_PROVIDER_STOP_PATTERNS = [
     /daily limit reached/i,
     /remaining today:\s*0/i,
@@ -30,6 +41,8 @@ const FATAL_PROVIDER_STOP_PATTERNS = [
     /rate limit/i,
     /\b429\b/i,
 ];
+const VISUAL_TAIL_RUN_MIN_PAGES = 3;
+const VISUAL_TAIL_RUN_SLOW_FAILURE_THRESHOLD = 2;
 function nowIso(now) {
     return (now ?? (() => new Date()))().toISOString();
 }
@@ -458,12 +471,123 @@ function advanceBoreholeProcessingState(state, result, pageTextHint, overrideBor
     return next;
 }
 function isSlowVisualPageError(message, classification, sourceKind) {
-    const looksVisual = sourceKind === 'raster-image' || classification === 'image-only' || classification === 'text-unreadable';
+    const looksVisual = sourceKind === 'raster-image' || isVisualRunClassification(classification);
     return looksVisual && SLOW_VISUAL_ERROR_PATTERNS.some((pattern) => pattern.test(message));
 }
+function isVisualRunClassification(classification) {
+    return classification === 'image-only' || classification === 'text-unreadable' || classification === 'graphics-only';
+}
+function isVisualRunCheckpoint(page) {
+    return page.sourceKind === 'raster-image' || isVisualRunClassification(page.classification);
+}
+function getInspectionLeadText(inspectionPage) {
+    if (!inspectionPage) {
+        return '';
+    }
+    return uniqueStrings([
+        ...(inspectionPage.normalizedArtifact?.headingHints ?? []),
+        inspectionPage.normalizedArtifact?.nativeText,
+        inspectionPage.normalizedText,
+    ])
+        .join('\n')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .toLowerCase();
+}
+function hasBoreholeAppendixCue(text) {
+    return /\b(record of boreholes?|borehole logs?|borehole records?|test pits?|cpt|cone penetration|standard penetration test|spt)\b/i.test(text);
+}
+function hasVisualAppendixCue(text) {
+    return (/\bappendix\b/i.test(text)
+        && /\b(figures?|plates?|photos?|photographs?|drawings?|plans?|sketches?|site layout|certificates?|laboratory)\b/i.test(text)
+        && !hasBoreholeAppendixCue(text));
+}
+function findConsecutiveVisualRun(job, pageNumber) {
+    const pages = [...job.checkpoints.pages].sort((left, right) => left.pageNumber - right.pageNumber);
+    const centerIndex = pages.findIndex((page) => page.pageNumber === pageNumber);
+    if (centerIndex < 0 || !isVisualRunCheckpoint(pages[centerIndex])) {
+        return [];
+    }
+    let startIndex = centerIndex;
+    while (startIndex > 0 && isVisualRunCheckpoint(pages[startIndex - 1])) {
+        startIndex -= 1;
+    }
+    let endIndex = centerIndex;
+    while (endIndex + 1 < pages.length && isVisualRunCheckpoint(pages[endIndex + 1])) {
+        endIndex += 1;
+    }
+    return pages.slice(startIndex, endIndex + 1);
+}
+function hasNearbyVisualAppendixCue(job, runStartPage) {
+    if (!job.inspection) {
+        return false;
+    }
+    const firstCandidatePage = Math.max(1, runStartPage - 3);
+    for (let pageNumber = firstCandidatePage; pageNumber <= runStartPage; pageNumber += 1) {
+        const inspectionPage = job.inspection.pages.find((page) => page.pageNumber === pageNumber);
+        if (hasVisualAppendixCue(getInspectionLeadText(inspectionPage))) {
+            return true;
+        }
+    }
+    return false;
+}
+function isTailOrAppendixVisualRun(job, run) {
+    if (run.length < VISUAL_TAIL_RUN_MIN_PAGES) {
+        return false;
+    }
+    const startPage = run[0].pageNumber;
+    const endPage = run[run.length - 1].pageNumber;
+    const totalPages = Math.max(job.source.totalPages, job.inspection?.totalPages ?? 0);
+    const touchesTail = totalPages > 0
+        && endPage >= totalPages
+        && startPage >= Math.max(2, Math.ceil(totalPages * 0.55));
+    return touchesTail || hasNearbyVisualAppendixCue(job, startPage);
+}
+function applyConsecutiveVisualRunDowngrades(job, triggerPageNumber, now) {
+    const run = findConsecutiveVisualRun(job, triggerPageNumber);
+    if (!isTailOrAppendixVisualRun(job, run)) {
+        return job;
+    }
+    const slowFailureCount = run.filter((page) => page.status === 'failed'
+        && page.downgraded
+        && isSlowVisualPageError(page.error ?? '', page.classification, page.sourceKind)).length;
+    if (slowFailureCount < VISUAL_TAIL_RUN_SLOW_FAILURE_THRESHOLD) {
+        return job;
+    }
+    const pendingPageNumbers = new Set(run
+        .filter((page) => page.status === 'pending' && page.pageNumber > triggerPageNumber)
+        .map((page) => page.pageNumber));
+    if (pendingPageNumbers.size === 0) {
+        return job;
+    }
+    const timestamp = nowIso(now);
+    return {
+        ...job,
+        updatedAt: timestamp,
+        checkpoints: {
+            pages: job.checkpoints.pages.map((page) => pendingPageNumbers.has(page.pageNumber)
+                ? {
+                    ...page,
+                    status: 'failed',
+                    updatedAt: timestamp,
+                    error: `Skipped page ${page.pageNumber} after ${slowFailureCount} slow visual failures in a consecutive image-only tail/appendix run.`,
+                    downgraded: true,
+                }
+                : page),
+        },
+    };
+}
+function isRetryableUpstreamPageError(message) {
+    return RETRYABLE_UPSTREAM_ERROR_PATTERNS.some((pattern) => pattern.test(message));
+}
 function isFatalProviderStopError(message) {
     return FATAL_PROVIDER_STOP_PATTERNS.some((pattern) => pattern.test(message));
 }
+function isTextExtractionTimeoutError(message) {
+    return (/text extraction timed out/i.test(message)
+        || (/hosted beta request timed out|timed out after \d+s/i.test(message)
+            && !/\b524\b|upstream request failed/i.test(message)));
+}
 function normalizeCheckpointErrorMessage(message) {
     let normalized = message.trim();
     for (let iteration = 0; iteration < 4; iteration += 1) {
@@ -475,6 +599,13 @@ function normalizeCheckpointErrorMessage(message) {
     }
     return normalized;
 }
+async function waitForCheckpointRetryBackoff(attempt) {
+    const delayMs = Math.min(1000, 100 * Math.max(1, 2 ** Math.max(0, attempt - 1)));
+    await new Promise((resolvePromise) => {
+        const timer = setTimeout(resolvePromise, delayMs);
+        timer.unref?.();
+    });
+}
 async function withWorkerPageTimeout(promise, timeoutMs, errorMessage) {
     if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
         return promise;
@@ -489,14 +620,17 @@ async function withWorkerPageTimeout(promise, timeoutMs, errorMessage) {
         }),
     ]);
 }
-function resolveWorkerPhaseTimeoutMs(config, input) {
+function resolveWorkerPhaseTimeoutMs(config, input, options) {
     const baseTimeoutMs = Math.min(Math.max(config.timeout ?? 120000, 60000), 120000);
     const isHeavyVisualPage = input.sourceKind === 'raster-image'
         || input.classification === 'image-only'
         || input.classification === 'text-unreadable';
-    return isHeavyVisualPage
+    const phaseTimeoutMs = isHeavyVisualPage
         ? Math.min(Math.max(baseTimeoutMs, 180000), 180000)
         : baseTimeoutMs;
+    return options?.cheapRetry
+        ? Math.min(phaseTimeoutMs, isHeavyVisualPage ? 60000 : 90000)
+        : phaseTimeoutMs;
 }
 function resolveWorkerTextExtractionTimeoutMs(baseTimeoutMs, textHint) {
     if (!textHint) {
@@ -555,6 +689,59 @@ function buildInspectionSummary(inspection) {
         ocrRecoveredPageCount: 0,
     };
 }
+function countCheckpointOcrRecoveredPages(checkpoints) {
+    return checkpoints.filter((page) => page.ocrSource === 'local-ocr' || page.ocrSource === 'vision-ocr').length;
+}
+function applyCheckpointOcrRecoveredSummary(result, checkpoints) {
+    const recoveredPageCount = countCheckpointOcrRecoveredPages(checkpoints);
+    if (recoveredPageCount === 0 || !result.inspectionSummary) {
+        return result;
+    }
+    return {
+        ...result,
+        inspectionSummary: {
+            ...result.inspectionSummary,
+            ocrRecoveredPageCount: Math.max(result.inspectionSummary.ocrRecoveredPageCount, recoveredPageCount),
+        },
+    };
+}
+function buildFallbackWorkerCheckpoints(job, pageCount, timestamp) {
+    if (job.checkpoints.pages.length > 0) {
+        return job.checkpoints.pages;
+    }
+    const startPage = job.source.pageRange?.[0] ?? job.segmentation?.pageRange?.[0] ?? 1;
+    return Array.from({ length: pageCount }, (_, index) => ({
+        pageNumber: startPage + index,
+        classification: null,
+        sourceKind: 'pdf-page',
+        weight: 1,
+        status: 'pending',
+        attempts: 0,
+        updatedAt: timestamp,
+    }));
+}
+async function inferWorkerPdfPageCount(job) {
+    if (job.source.totalPages > 0) {
+        return job.source.totalPages;
+    }
+    try {
+        const fullPageCount = await countDocumentPdfPages(job.source.filePath);
+        return job.source.pageRange
+            ? Math.max(0, Math.min(fullPageCount, job.source.pageRange[1]) - job.source.pageRange[0] + 1)
+            : fullPageCount;
+    }
+    catch {
+        try {
+            const fullPageCount = inferPdfDocumentPageCountFallback(job.source.filePath);
+            return job.source.pageRange
+                ? Math.max(0, Math.min(fullPageCount, job.source.pageRange[1]) - job.source.pageRange[0] + 1)
+                : fullPageCount;
+        }
+        catch {
+            return 0;
+        }
+    }
+}
 function summarizeReviewReasons(findings) {
     return uniqueStrings(findings
         .filter((finding) => finding.severity !== 'advisory')
@@ -595,7 +782,7 @@ function buildSyntheticBoreholeResult(job, inspection, now) {
             message: `${normalFailedPages.length} page(s) failed during ingest and should be reviewed.`,
         });
     }
-    return {
+    return applyCheckpointOcrRecoveredSummary({
         kind: 'geotech-ingest-result',
         schemaVersion: 1,
         documentType: 'borehole-log',
@@ -625,7 +812,7 @@ function buildSyntheticBoreholeResult(job, inspection, now) {
         reviewRequired: reviewFindings.some((finding) => finding.severity !== 'advisory'),
         confidence: 0,
         canAutoProceed: false,
-    };
+    }, job.checkpoints.pages);
 }
 function buildSyntheticGeotechDocumentResult(job, inspection, now) {
     const pageFailures = job.checkpoints.pages
@@ -662,7 +849,7 @@ function buildSyntheticGeotechDocumentResult(job, inspection, now) {
             message: `${normalFailedPages.length} page(s) failed during ingest and should be reviewed.`,
         });
     }
-    return {
+    return applyCheckpointOcrRecoveredSummary({
         kind: 'geotech-ingest-result',
         schemaVersion: 1,
         documentType: 'geotech-document',
@@ -700,7 +887,7 @@ function buildSyntheticGeotechDocumentResult(job, inspection, now) {
         confidence: 0,
         reviewRequired: reviewFindings.some((finding) => finding.severity !== 'advisory'),
         canAutoProceed: false,
-    };
+    }, job.checkpoints.pages);
 }
 function dedupeReviewFindings(reviewFindings) {
     return [
@@ -819,11 +1006,13 @@ function findCheckpoint(job, pageNumber) {
     }
     return checkpoint;
 }
-async function processGeotechDocumentPage(job, pageInput, config, dependencies) {
+async function processGeotechDocumentPage(job, pageInput, config, dependencies, options = {}) {
     const inspect = dependencies.inspectPdfDocument ?? inspectPdfDocument;
     const recoverTextHint = dependencies.recoverDocumentTextHint ?? recoverDocumentTextHint;
     const interpretation = dependencies.interpretGeotechDocumentPage ?? interpretGeotechDocumentPage;
     const extractTextFacts = dependencies.extractGeotechDocumentFactsFromText ?? extractGeotechDocumentFactsFromText;
+    const extractDeterministicFacts = dependencies.extractGeotechDocumentDeterministicFactsFromText
+        ?? extractGeotechDocumentDeterministicFactsFromText;
     const transcribe = dependencies.transcribeDocumentImageText ?? transcribeDocumentImageText;
     const inspectionPage = job.inspection?.pages[pageInput.pageNumber - 1] ?? inspect(job.source.filePath).pages[pageInput.pageNumber - 1];
     const lowYieldRole = inferPreflightLowYieldPageRole({
@@ -854,6 +1043,8 @@ async function processGeotechDocumentPage(job, pageInput, config, dependencies)
     const phaseTimeoutMs = resolveWorkerPhaseTimeoutMs(config, {
         classification: inspectionPage?.classification,
         sourceKind: pageInput.sourceKind,
+    }, {
+        cheapRetry: options.cheapRetry,
     });
     const phaseConfig = {
         ...config,
@@ -862,7 +1053,9 @@ async function processGeotechDocumentPage(job, pageInput, config, dependencies)
     let pageTextHint;
     let ocrSource = 'none';
     let ocrWarnings = [];
+    let textRecoveryAttempted = false;
     try {
+        textRecoveryAttempted = true;
         const recovery = await withWorkerPageTimeout(recoverTextHint({
             existingTextHint: inspectionPage?.normalizedArtifact?.nativeText ?? inspectionPage?.normalizedText,
             existingTextAccepted: inspectionPage?.normalizedArtifact?.textQuality.accepted ?? true,
@@ -871,6 +1064,7 @@ async function processGeotechDocumentPage(job, pageInput, config, dependencies)
             config: phaseConfig,
             pdfFilePath: job.source.filePath,
             pdfPageNumber: pageInput.pageNumber,
+            allowVisionOcr: !(options.cheapRetry && pageInput.sourceKind === 'raster-image'),
             visionTranscribe: transcribe,
         }), phaseTimeoutMs, `Page ${pageInput.pageNumber}: OCR/text recovery timed out after ${Math.round(phaseTimeoutMs / 1000)}s`);
         pageTextHint = normalizeTextHint(recovery.textHint);
@@ -888,15 +1082,35 @@ async function processGeotechDocumentPage(job, pageInput, config, dependencies)
         totalPages: pageInput.totalPages,
         pageClassification: inspectionPage?.classification,
         pageTextHint,
+        textRecoveryAttempted: !pageTextHint && textRecoveryAttempted,
     };
     const extractionTimeoutMs = resolveWorkerTextExtractionTimeoutMs(phaseTimeoutMs, pageTextHint);
     const extractionConfig = {
         ...config,
         timeout: extractionTimeoutMs,
     };
-    const result = pageTextHint
-        ? await withWorkerPageTimeout(extractTextFacts(pageTextHint, extractionConfig, context), extractionTimeoutMs, `Page ${pageInput.pageNumber}: text extraction timed out after ${Math.round(extractionTimeoutMs / 1000)}s`)
-        : await withWorkerPageTimeout(interpretation(pageInput.base64, pageInput.mimeType, phaseConfig, context), phaseTimeoutMs, `Page ${pageInput.pageNumber}: visual page interpretation timed out after ${Math.round(phaseTimeoutMs / 1000)}s`);
+    let result;
+    if (pageTextHint) {
+        try {
+            result = await withWorkerPageTimeout(extractTextFacts(pageTextHint, extractionConfig, context), extractionTimeoutMs, `Page ${pageInput.pageNumber}: text extraction timed out after ${Math.round(extractionTimeoutMs / 1000)}s`);
+        }
+        catch (error) {
+            const message = normalizeCheckpointErrorMessage(error instanceof Error ? error.message : String(error));
+            const deterministic = isTextExtractionTimeoutError(message)
+                ? extractDeterministicFacts(pageTextHint, context, {
+                    forcePartial: true,
+                    warning: `Text extraction timed out (${message}); used deterministic partial extraction instead.`,
+                })
+                : null;
+            if (!deterministic) {
+                throw error;
+            }
+            result = deterministic;
+        }
+    }
+    else {
+        result = await withWorkerPageTimeout(interpretation(pageInput.base64, pageInput.mimeType, phaseConfig, context), phaseTimeoutMs, `Page ${pageInput.pageNumber}: visual page interpretation timed out after ${Math.round(phaseTimeoutMs / 1000)}s`);
+    }
     return {
         result,
         ocrTextHint: pageTextHint,
@@ -904,7 +1118,7 @@ async function processGeotechDocumentPage(job, pageInput, config, dependencies)
         ocrWarnings,
     };
 }
-async function processBoreholePage(job, pageInput, config, state, dependencies) {
+async function processBoreholePage(job, pageInput, config, state, dependencies, options = {}) {
     const recoverTextHint = dependencies.recoverDocumentTextHint ?? recoverDocumentTextHint;
     const transcribe = dependencies.transcribeDocumentImageText ?? transcribeDocumentImageText;
     const interpret = dependencies.interpretBoreholeLogWithContext ?? interpretBoreholeLogWithContext;
@@ -912,6 +1126,8 @@ async function processBoreholePage(job, pageInput, config, state, dependencies)
     const phaseTimeoutMs = resolveWorkerPhaseTimeoutMs(config, {
         classification: inspectionPage?.classification,
         sourceKind: pageInput.sourceKind,
+    }, {
+        cheapRetry: options.cheapRetry,
     });
     const phaseConfig = {
         ...config,
@@ -926,6 +1142,7 @@ async function processBoreholePage(job, pageInput, config, state, dependencies)
         config: phaseConfig,
         pdfFilePath: job.source.filePath,
         pdfPageNumber: pageInput.pageNumber,
+        allowVisionOcr: !(options.cheapRetry && pageInput.sourceKind === 'raster-image'),
         visionTranscribe: transcribe,
     }), phaseTimeoutMs, `Page ${pageInput.pageNumber}: OCR/text recovery timed out after ${Math.round(phaseTimeoutMs / 1000)}s`);
     if (recovery.textHint) {
@@ -990,7 +1207,7 @@ async function finalizeJobResult(job, pageInputs, config, dependencies) {
                 },
                 now: dependencies.now,
             });
-            return applyBoreholeFailureDowngrades(result, job.checkpoints.pages);
+            return applyCheckpointOcrRecoveredSummary(applyBoreholeFailureDowngrades(result, job.checkpoints.pages), job.checkpoints.pages);
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
@@ -1043,7 +1260,7 @@ async function finalizeJobResult(job, pageInputs, config, dependencies) {
             },
             now: dependencies.now,
         });
-        return applyGeotechFailureDowngrades(result, job.checkpoints.pages);
+        return applyCheckpointOcrRecoveredSummary(applyGeotechFailureDowngrades(result, job.checkpoints.pages), job.checkpoints.pages);
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -1096,30 +1313,56 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
             ? currentJob.inspection
             : inspect(currentJob.source.filePath);
         if (!currentJob.inspection || currentJob.inspection.totalPages === 0) {
-            await mutateJob((job) => ({
-                ...job,
-                inspection,
-                source: {
-                    ...job.source,
-                    totalPages: inspection.totalPages,
-                    weightedPageCost: inspection.pages.reduce((sum, page) => sum + (page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1), 0),
-                },
-                processing: {
-                    ...job.processing,
-                    chunkExtractionConcurrency: resolvePersistedIngestJobExtractionConcurrency(job.config, inspection),
-                },
-                checkpoints: {
-                    pages: inspection.pages.map((page) => job.checkpoints.pages.find((existing) => existing.pageNumber === page.pageNumber) ?? ({
-                        pageNumber: page.pageNumber,
-                        classification: page.classification,
-                        sourceKind: mapPageSourceKind(page.classification),
-                        weight: page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1,
-                        status: 'pending',
-                        attempts: 0,
-                        updatedAt: nowIso(dependencies.now),
-                    })),
-                },
-            }));
+            if (inspection.totalPages > 0) {
+                await mutateJob((job) => ({
+                    ...job,
+                    inspection,
+                    source: {
+                        ...job.source,
+                        totalPages: inspection.totalPages,
+                        weightedPageCost: inspection.pages.reduce((sum, page) => sum + (page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1), 0),
+                    },
+                    processing: {
+                        ...job.processing,
+                        chunkExtractionConcurrency: resolvePersistedIngestJobExtractionConcurrency(job.config, inspection, job.segmentation),
+                    },
+                    checkpoints: {
+                        pages: inspection.pages.map((page) => job.checkpoints.pages.find((existing) => existing.pageNumber === page.pageNumber) ?? ({
+                            pageNumber: page.pageNumber,
+                            classification: page.classification,
+                            sourceKind: mapPageSourceKind(page.classification),
+                            weight: page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1,
+                            status: 'pending',
+                            attempts: 0,
+                            updatedAt: nowIso(dependencies.now),
+                        })),
+                    },
+                }));
+            }
+            else {
+                const inferredPageCount = await inferWorkerPdfPageCount(currentJob);
+                if (inferredPageCount > 0) {
+                    await mutateJob((job) => {
+                        const timestamp = nowIso(dependencies.now);
+                        return {
+                            ...job,
+                            inspection: null,
+                            source: {
+                                ...job.source,
+                                totalPages: Math.max(job.source.totalPages, inferredPageCount),
+                                weightedPageCost: Math.max(job.source.weightedPageCost, inferredPageCount),
+                            },
+                            processing: {
+                                ...job.processing,
+                                chunkExtractionConcurrency: resolvePersistedIngestJobExtractionConcurrency(job.config, null, job.segmentation),
+                            },
+                            checkpoints: {
+                                pages: buildFallbackWorkerCheckpoints(job, inferredPageCount, timestamp),
+                            },
+                        };
+                    });
+                }
+            }
         }
         currentJob = loadPersistedIngestJob(jobId) ?? currentJob;
         const config = buildJobConfig(currentJob, dependencies);
@@ -1138,6 +1381,9 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                 if (isCancelled(jobId)) {
                     return;
                 }
+                if (findCheckpoint(currentJob, page.pageNumber).status !== 'pending') {
+                    return;
+                }
                 if (fatalProviderStopMessage) {
                     await mutateJob((job) => ({
                         ...job,
@@ -1177,8 +1423,47 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                             : checkpoint),
                     },
                 }));
-                try {
-                    const processed = await processGeotechDocumentPage(currentJob, page, config, dependencies);
+                let processed = null;
+                let finalErrorMessage = '';
+                for (let attemptIndex = 0; attemptIndex < 2; attemptIndex += 1) {
+                    try {
+                        processed = await processGeotechDocumentPage(currentJob, page, config, dependencies, {
+                            cheapRetry: attemptIndex > 0,
+                        });
+                        break;
+                    }
+                    catch (error) {
+                        const message = error instanceof Error ? error.message : String(error);
+                        const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                        if (attemptIndex === 0
+                            && isRetryableUpstreamPageError(normalizedMessage)
+                            && !isFatalProviderStopError(normalizedMessage)) {
+                            await waitForCheckpointRetryBackoff(findCheckpoint(currentJob, page.pageNumber).attempts);
+                            await mutateJob((job) => ({
+                                ...job,
+                                updatedAt: nowIso(dependencies.now),
+                                execution: {
+                                    ...job.execution,
+                                    lastHeartbeatAt: nowIso(dependencies.now),
+                                },
+                                checkpoints: {
+                                    pages: job.checkpoints.pages.map((checkpoint) => checkpoint.pageNumber === page.pageNumber
+                                        ? {
+                                            ...checkpoint,
+                                            attempts: checkpoint.attempts + 1,
+                                            updatedAt: nowIso(dependencies.now),
+                                            error: `retrying after upstream timeout: ${normalizedMessage}`,
+                                        }
+                                        : checkpoint),
+                                },
+                            }));
+                            continue;
+                        }
+                        finalErrorMessage = normalizedMessage;
+                        break;
+                    }
+                }
+                if (processed) {
                     processedNewPages += 1;
                     await mutateJob((job) => ({
                         ...job,
@@ -1205,32 +1490,35 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                         },
                     }));
                 }
-                catch (error) {
-                    const message = error instanceof Error ? error.message : String(error);
-                    const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                else {
+                    const normalizedMessage = finalErrorMessage || `Page ${page.pageNumber} failed during async ingest.`;
                     const checkpoint = findCheckpoint(currentJob, page.pageNumber);
                     if (!fatalProviderStopMessage && isFatalProviderStopError(normalizedMessage)) {
                         fatalProviderStopMessage = normalizedMessage;
                     }
-                    await mutateJob((job) => ({
-                        ...job,
-                        updatedAt: nowIso(dependencies.now),
-                        execution: {
-                            ...job.execution,
-                            lastHeartbeatAt: nowIso(dependencies.now),
-                        },
-                        checkpoints: {
-                            pages: job.checkpoints.pages.map((pageCheckpoint) => pageCheckpoint.pageNumber === page.pageNumber
-                                ? {
-                                    ...pageCheckpoint,
-                                    status: 'failed',
-                                    updatedAt: nowIso(dependencies.now),
-                                    error: normalizedMessage,
-                                    downgraded: isSlowVisualPageError(normalizedMessage, checkpoint.classification, checkpoint.sourceKind),
-                                }
-                                : pageCheckpoint),
-                        },
-                    }));
+                    await mutateJob((job) => {
+                        const timestamp = nowIso(dependencies.now);
+                        const failedJob = {
+                            ...job,
+                            updatedAt: timestamp,
+                            execution: {
+                                ...job.execution,
+                                lastHeartbeatAt: timestamp,
+                            },
+                            checkpoints: {
+                                pages: job.checkpoints.pages.map((pageCheckpoint) => pageCheckpoint.pageNumber === page.pageNumber
+                                    ? {
+                                        ...pageCheckpoint,
+                                        status: 'failed',
+                                        updatedAt: timestamp,
+                                        error: normalizedMessage,
+                                        downgraded: isSlowVisualPageError(normalizedMessage, checkpoint.classification, checkpoint.sourceKind),
+                                    }
+                                    : pageCheckpoint),
+                            },
+                        };
+                        return applyConsecutiveVisualRunDowngrades(failedJob, page.pageNumber, dependencies.now);
+                    });
                 }
             });
             if (fatalProviderStopMessage) {
@@ -1292,8 +1580,47 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                             : pageCheckpoint),
                     },
                 }));
-                try {
-                    const processed = await processBoreholePage(currentJob, page, config, state, dependencies);
+                let processed = null;
+                let finalErrorMessage = '';
+                for (let attemptIndex = 0; attemptIndex < 2; attemptIndex += 1) {
+                    try {
+                        processed = await processBoreholePage(currentJob, page, config, state, dependencies, {
+                            cheapRetry: attemptIndex > 0,
+                        });
+                        break;
+                    }
+                    catch (error) {
+                        const message = error instanceof Error ? error.message : String(error);
+                        const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                        if (attemptIndex === 0
+                            && isRetryableUpstreamPageError(normalizedMessage)
+                            && !isFatalProviderStopError(normalizedMessage)) {
+                            await waitForCheckpointRetryBackoff(findCheckpoint(currentJob, page.pageNumber).attempts);
+                            await mutateJob((job) => ({
+                                ...job,
+                                updatedAt: nowIso(dependencies.now),
+                                execution: {
+                                    ...job.execution,
+                                    lastHeartbeatAt: nowIso(dependencies.now),
+                                },
+                                checkpoints: {
+                                    pages: job.checkpoints.pages.map((pageCheckpoint) => pageCheckpoint.pageNumber === page.pageNumber
+                                        ? {
+                                            ...pageCheckpoint,
+                                            attempts: pageCheckpoint.attempts + 1,
+                                            updatedAt: nowIso(dependencies.now),
+                                            error: `retrying after upstream timeout: ${normalizedMessage}`,
+                                        }
+                                        : pageCheckpoint),
+                                },
+                            }));
+                            continue;
+                        }
+                        finalErrorMessage = normalizedMessage;
+                        break;
+                    }
+                }
+                if (processed) {
                     processedNewPages += 1;
                     state = processed.nextState;
                     await mutateJob((job) => ({
@@ -1321,9 +1648,8 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                         },
                     }));
                 }
-                catch (error) {
-                    const message = error instanceof Error ? error.message : String(error);
-                    const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                else {
+                    const normalizedMessage = finalErrorMessage || `Page ${page.pageNumber} failed during async ingest.`;
                     await mutateJob((job) => ({
                         ...job,
                         updatedAt: nowIso(dependencies.now),
@@ -1357,7 +1683,7 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                                         ...pageCheckpoint,
                                         status: 'failed',
                                         updatedAt: nowIso(dependencies.now),
-                                        error: `skipped after upstream provider stop. ${normalizeCheckpointErrorMessage(message)}`,
+                                        error: `skipped after upstream provider stop. ${normalizedMessage}`,
                                         downgraded: false,
                                     }
                                     : pageCheckpoint),