@geotechcli/core 0.4.24 → 0.4.26

package/dist/ingest/job-worker.js

@@ -1,16 +1,20 @@
-import { basename } from 'node:path';
+import { basename, join } from 'node:path';
+import { homedir } from 'node:os';
 import { buildLLMConfig } from '../config/index.js';
-import { readDocumentPdfPageInputs } from './document-inputs.js';
-import { inspectPdfDocument } from './pdf.js';
+import { countDocumentPdfPages, readDocumentPdfPageInputs } from './document-inputs.js';
+import { inferPdfDocumentPageCountFallback, inspectPdfDocument, } from './pdf.js';
 import { ingestBoreholeLogDocument, summarizeBoreholeIngestInspection, } from './geotech-extract.js';
 import { buildPreflightLowYieldInsight, ingestGeotechDocument, inferPreflightLowYieldPageRole, } from './geotech-document.js';
 import { extractGeotechDocumentFactsFromText, interpretGeotechDocumentPage, } from '../vision/geotech-document.js';
 import { interpretBoreholeLogWithContext, transcribeDocumentImageText, } from '../vision/index.js';
 import { recoverDocumentTextHint } from '../vision/ocr.js';
 import { persistBoreholeIngestReview } from './review-store.js';
-import { loadPersistedIngestJob, savePersistedIngestJob, } from './job-store.js';
+import { buildPersistedIngestJobSegments, createPersistedIngestJob, loadPersistedIngestJob, resolvePersistedIngestJobExtractionConcurrency, savePersistedIngestJob, } from './job-store.js';
+import { HOSTED_BETA_EFFECTIVE_PAGE_LIMIT, slicePdfInspectionToRange, writePdfPageSubset, } from './segmentation.js';
 const SLOW_VISUAL_ERROR_PATTERNS = [
     /timeout/i,
+    /\b524\b/i,
+    /upstream(?: request)? (?:timed out|timeout|failed)/i,
     /provider is busy/i,
     /returned no content/i,
     /did not contain assistant text/i,
@@ -20,6 +24,15 @@ const SLOW_VISUAL_ERROR_PATTERNS = [
     /\b503\b/i,
     /\b504\b/i,
 ];
+const RETRYABLE_UPSTREAM_ERROR_PATTERNS = [
+    /timeout/i,
+    /\b524\b/i,
+    /upstream(?: request)? (?:timed out|timeout|failed)/i,
+    /provider is busy/i,
+    /temporarily unavailable/i,
+    /\b503\b/i,
+    /\b504\b/i,
+];
 const FATAL_PROVIDER_STOP_PATTERNS = [
     /daily limit reached/i,
     /remaining today:\s*0/i,
@@ -34,6 +47,310 @@ function nowIso(now) {
 function uniqueStrings(values) {
     return [...new Set(values.filter((value) => typeof value === 'string' && value.trim().length > 0))];
 }
+function getJobSourceDisplayPath(job) {
+    return job.source.originalFilePath ?? job.source.filePath;
+}
+function getJobSourceDisplayName(job) {
+    return job.source.originalFileName ?? basename(getJobSourceDisplayPath(job));
+}
+function cloneSegmentationSummary(segmentation) {
+    if (!segmentation) {
+        return undefined;
+    }
+    return {
+        ...segmentation,
+        pageRange: [...segmentation.pageRange],
+        segments: segmentation.segments?.map((segment) => ({ ...segment })),
+    };
+}
+function buildJobResultSource(job, counts) {
+    return {
+        filePath: getJobSourceDisplayPath(job),
+        fileName: getJobSourceDisplayName(job),
+        inputKind: 'pdf',
+        totalPages: job.source.totalPages,
+        successfulPages: counts.successfulPages,
+        failedPages: counts.failedPages,
+        pageRange: job.source.pageRange,
+        segmentation: cloneSegmentationSummary(job.segmentation),
+    };
+}
+function remapPageReferenceText(message, localPageNumber, originalPageNumber) {
+    if (!message) {
+        return undefined;
+    }
+    return message
+        .replace(new RegExp(`\\bPage ${localPageNumber}\\b`, 'g'), `Page ${originalPageNumber}`)
+        .replace(new RegExp(`\\bpage ${localPageNumber}\\b`, 'g'), `page ${originalPageNumber}`);
+}
+function remapChildGeotechInsightToOriginalPage(insight, localPageNumber, originalPageNumber, parentTotalPages) {
+    return {
+        ...insight,
+        pageNumber: originalPageNumber,
+        totalPages: parentTotalPages,
+        warnings: uniqueStrings(insight.warnings.map((warning) => remapPageReferenceText(warning, localPageNumber, originalPageNumber))),
+    };
+}
+function updateSegmentSummaryStatus(segmentation, segmentIndex, patch) {
+    if (!segmentation?.segments) {
+        return segmentation;
+    }
+    return {
+        ...segmentation,
+        segments: segmentation.segments.map((segment) => segment.segmentIndex === segmentIndex
+            ? {
+                ...segment,
+                ...patch,
+            }
+            : segment),
+    };
+}
+function getIngestJobArtifactsRoot() {
+    return process.env.GEOTECHCLI_CONFIG_DIR ?? join(homedir(), '.geotechcli');
+}
+function getSegmentArtifactsDir(parentJobId) {
+    return join(getIngestJobArtifactsRoot(), 'ingest-jobs', parentJobId, 'segments');
+}
+function getJobScopedPageRange(job) {
+    const [startPage, endPage] = job.source.pageRange ?? job.segmentation?.pageRange ?? [1, job.source.totalPages];
+    return {
+        startPage,
+        endPage,
+    };
+}
+function getJobOriginalTotalPages(job) {
+    return Math.max(job.source.totalPages, job.source.pageRange?.[1] ?? 0, job.segmentation?.pageRange?.[1] ?? 0);
+}
+function filterPageInputsToSelectedPages(pageInputs, selectedPageNumbers) {
+    const selected = new Set(selectedPageNumbers);
+    return pageInputs.filter((pageInput) => selected.has(pageInput.pageNumber));
+}
+function countCheckpointStatuses(pages, range) {
+    let completedPages = 0;
+    let failedPages = 0;
+    for (const page of pages) {
+        if (page.pageNumber < range.startPage || page.pageNumber > range.endPage) {
+            continue;
+        }
+        if (page.status === 'completed') {
+            completedPages += 1;
+        }
+        else if (page.status === 'failed') {
+            failedPages += 1;
+        }
+    }
+    return { completedPages, failedPages };
+}
+function isSegmentResolved(job, range) {
+    return job.checkpoints.pages
+        .filter((page) => page.pageNumber >= range.startPage && page.pageNumber <= range.endPage)
+        .every((page) => page.status !== 'pending');
+}
+function remapSegmentCheckpointError(message, localPageNumber, originalPageNumber) {
+    const normalized = normalizeCheckpointErrorMessage(message ?? '');
+    return remapPageReferenceText(normalized, localPageNumber, originalPageNumber);
+}
+function mergeSegmentChildJobIntoParentJob(parentJob, childJob, range, now) {
+    const timestamp = nowIso(now);
+    const originalTotalPages = getJobOriginalTotalPages(parentJob);
+    return {
+        ...parentJob,
+        updatedAt: timestamp,
+        execution: {
+            ...parentJob.execution,
+            lastHeartbeatAt: timestamp,
+        },
+        checkpoints: {
+            pages: parentJob.checkpoints.pages.map((pageCheckpoint) => {
+                if (pageCheckpoint.pageNumber < range.startPage || pageCheckpoint.pageNumber > range.endPage) {
+                    return pageCheckpoint;
+                }
+                const localPageNumber = pageCheckpoint.pageNumber - range.startPage + 1;
+                const childCheckpoint = childJob.checkpoints.pages.find((page) => page.pageNumber === localPageNumber);
+                if (!childCheckpoint || childCheckpoint.status === 'pending') {
+                    return {
+                        ...pageCheckpoint,
+                        status: 'failed',
+                        updatedAt: timestamp,
+                        error: `Page ${pageCheckpoint.pageNumber} did not complete within segment ${range.startPage}-${range.endPage}.`,
+                        downgraded: false,
+                    };
+                }
+                const remappedResult = parentJob.documentType === 'geotech-document'
+                    && childCheckpoint.status === 'completed'
+                    && childCheckpoint.result
+                    ? remapChildGeotechInsightToOriginalPage(childCheckpoint.result, localPageNumber, pageCheckpoint.pageNumber, originalTotalPages)
+                    : childCheckpoint.result;
+                return {
+                    ...pageCheckpoint,
+                    status: childCheckpoint.status,
+                    attempts: Math.max(pageCheckpoint.attempts, childCheckpoint.attempts),
+                    updatedAt: timestamp,
+                    completedAt: childCheckpoint.completedAt,
+                    error: remapSegmentCheckpointError(childCheckpoint.error, localPageNumber, pageCheckpoint.pageNumber),
+                    downgraded: childCheckpoint.downgraded,
+                    ocrTextHint: childCheckpoint.ocrTextHint,
+                    ocrSource: childCheckpoint.ocrSource,
+                    ocrWarnings: childCheckpoint.ocrWarnings,
+                    result: remappedResult,
+                };
+            }),
+        },
+    };
+}
+async function runSegmentedParentGeotechJob(jobId, currentJob, mutateJob, config, dependencies) {
+    const effectivePageLimit = currentJob.segmentation?.effectivePageLimit
+        ?? HOSTED_BETA_EFFECTIVE_PAGE_LIMIT;
+    const scopedRange = getJobScopedPageRange(currentJob);
+    const baseInspection = currentJob.inspection;
+    if (!baseInspection) {
+        throw new Error('Segmented parent ingest requires PDF inspection metadata.');
+    }
+    const baseSegments = currentJob.segmentation?.segments?.length
+        ? currentJob.segmentation.segments
+        : buildPersistedIngestJobSegments(baseInspection, {
+            pageRange: scopedRange,
+            effectivePageLimit,
+        }).map((segment, index, allSegments) => ({
+            ...segment,
+            segmentIndex: index + 1,
+            segmentCount: allSegments.length,
+            status: 'queued',
+        }));
+    if (!currentJob.segmentation?.segments?.length) {
+        await mutateJob((job) => ({
+            ...job,
+            updatedAt: nowIso(dependencies.now),
+            segmentation: {
+                mode: 'segmented-parent',
+                pageRange: [scopedRange.startPage, scopedRange.endPage],
+                effectivePageLimit,
+                segmentCount: baseSegments.length,
+                segments: baseSegments,
+            },
+        }));
+        currentJob = loadPersistedIngestJob(jobId) ?? currentJob;
+    }
+    const segmentArtifactsDir = getSegmentArtifactsDir(jobId);
+    for (const segment of currentJob.segmentation?.segments ?? baseSegments) {
+        if (isCancelled(jobId)) {
+            break;
+        }
+        currentJob = loadPersistedIngestJob(jobId) ?? currentJob;
+        const latestSegment = currentJob.segmentation?.segments?.find((entry) => entry.segmentIndex === segment.segmentIndex) ?? segment;
+        const range = {
+            startPage: latestSegment.startPage,
+            endPage: latestSegment.endPage,
+        };
+        if (isSegmentResolved(currentJob, range) && latestSegment.status === 'completed') {
+            continue;
+        }
+        const packetPath = join(segmentArtifactsDir, `segment-${String(latestSegment.segmentIndex).padStart(2, '0')}-pages-${range.startPage}-${range.endPage}.pdf`);
+        await writePdfPageSubset(getJobSourceDisplayPath(currentJob), range, packetPath);
+        const childInspection = slicePdfInspectionToRange(baseInspection, range, { rebasePageNumbers: true });
+        if (!childInspection) {
+            throw new Error(`Unable to create a scoped inspection for segment ${latestSegment.segmentIndex}.`);
+        }
+        let childJob = latestSegment.childJobId ? loadPersistedIngestJob(latestSegment.childJobId) : null;
+        if (!childJob || childJob.status === 'failed' || childJob.status === 'canceled') {
+            childJob = createPersistedIngestJob({
+                documentType: currentJob.documentType,
+                filePath: packetPath,
+                inspection: childInspection,
+                config: currentJob.config,
+                overrideBoreholeId: currentJob.request.overrideBoreholeId,
+                originalFilePath: getJobSourceDisplayPath(currentJob),
+                originalFileName: getJobSourceDisplayName(currentJob),
+                segmentation: {
+                    mode: 'segment-child',
+                    pageRange: [range.startPage, range.endPage],
+                    effectivePageLimit,
+                    segmentCount: currentJob.segmentation?.segmentCount ?? baseSegments.length,
+                    segmentIndex: latestSegment.segmentIndex,
+                    parentJobId: currentJob.jobId,
+                },
+                now: dependencies.now,
+            });
+        }
+        const startedAt = Date.now();
+        await mutateJob((job) => ({
+            ...job,
+            updatedAt: nowIso(dependencies.now),
+            execution: {
+                ...job.execution,
+                lastHeartbeatAt: nowIso(dependencies.now),
+            },
+            segmentation: updateSegmentSummaryStatus(job.segmentation, latestSegment.segmentIndex, {
+                childJobId: childJob.jobId,
+                status: 'running',
+            }),
+        }));
+        const completedChild = await runPersistedIngestJobWorker(childJob.jobId, dependencies);
+        const durationMs = Date.now() - startedAt;
+        await mutateJob((job) => {
+            const mergedJob = mergeSegmentChildJobIntoParentJob(job, completedChild, range, dependencies.now);
+            const counts = countCheckpointStatuses(mergedJob.checkpoints.pages, range);
+            const childStatus = completedChild.status === 'completed'
+                ? 'completed'
+                : completedChild.status === 'canceled'
+                    ? 'canceled'
+                    : 'failed';
+            return {
+                ...mergedJob,
+                segmentation: updateSegmentSummaryStatus(mergedJob.segmentation, latestSegment.segmentIndex, {
+                    childJobId: completedChild.jobId,
+                    status: childStatus,
+                    completedPages: counts.completedPages,
+                    failedPages: counts.failedPages,
+                    durationMs,
+                }),
+            };
+        });
+    }
+    currentJob = loadPersistedIngestJob(jobId) ?? currentJob;
+    if (currentJob.execution.cancelRequested) {
+        await mutateJob((job) => ({
+            ...job,
+            status: 'canceled',
+            updatedAt: nowIso(dependencies.now),
+            canceledAt: nowIso(dependencies.now),
+            execution: {
+                ...job.execution,
+                pid: undefined,
+            },
+        }));
+        return loadPersistedIngestJob(jobId) ?? currentJob;
+    }
+    const replayPageInputs = filterPageInputsToSelectedPages(await preparePdfPageInputs(currentJob.source.filePath, null, currentJob.processing.pagePreprocessingConcurrency, dependencies), currentJob.checkpoints.pages.map((page) => page.pageNumber));
+    const finalResult = await finalizeJobResult(currentJob, replayPageInputs, config, dependencies);
+    const persistedReview = currentJob.request.projectId
+        ? (dependencies.persistReview ?? persistBoreholeIngestReview)(currentJob.request.projectId, finalResult, {
+            title: currentJob.request.reviewTitle,
+        })
+        : null;
+    await mutateJob((job) => ({
+        ...job,
+        status: 'completed',
+        updatedAt: nowIso(dependencies.now),
+        completedAt: nowIso(dependencies.now),
+        execution: {
+            ...job.execution,
+            pid: undefined,
+            lastHeartbeatAt: nowIso(dependencies.now),
+        },
+        result: {
+            ingestResult: finalResult,
+            persistedReview: persistedReview
+                ? {
+                    datasetName: persistedReview.datasetName,
+                    reviewId: persistedReview.reviewId,
+                    createdAt: persistedReview.createdAt,
+                }
+                : undefined,
+        },
+    }));
+    return loadPersistedIngestJob(jobId) ?? currentJob;
+}
 function isBoreholeResult(result) {
     return result.documentType === 'borehole-log';
 }
@@ -155,6 +472,9 @@ function isSlowVisualPageError(message, classification, sourceKind) {
     const looksVisual = sourceKind === 'raster-image' || classification === 'image-only' || classification === 'text-unreadable';
     return looksVisual && SLOW_VISUAL_ERROR_PATTERNS.some((pattern) => pattern.test(message));
 }
+function isRetryableUpstreamPageError(message) {
+    return RETRYABLE_UPSTREAM_ERROR_PATTERNS.some((pattern) => pattern.test(message));
+}
 function isFatalProviderStopError(message) {
     return FATAL_PROVIDER_STOP_PATTERNS.some((pattern) => pattern.test(message));
 }
@@ -169,6 +489,13 @@ function normalizeCheckpointErrorMessage(message) {
     }
     return normalized;
 }
+async function waitForCheckpointRetryBackoff(attempt) {
+    const delayMs = Math.min(1000, 100 * Math.max(1, 2 ** Math.max(0, attempt - 1)));
+    await new Promise((resolvePromise) => {
+        const timer = setTimeout(resolvePromise, delayMs);
+        timer.unref?.();
+    });
+}
 async function withWorkerPageTimeout(promise, timeoutMs, errorMessage) {
     if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
         return promise;
@@ -249,6 +576,59 @@ function buildInspectionSummary(inspection) {
         ocrRecoveredPageCount: 0,
     };
 }
+function countCheckpointOcrRecoveredPages(checkpoints) {
+    return checkpoints.filter((page) => page.ocrSource === 'local-ocr' || page.ocrSource === 'vision-ocr').length;
+}
+function applyCheckpointOcrRecoveredSummary(result, checkpoints) {
+    const recoveredPageCount = countCheckpointOcrRecoveredPages(checkpoints);
+    if (recoveredPageCount === 0 || !result.inspectionSummary) {
+        return result;
+    }
+    return {
+        ...result,
+        inspectionSummary: {
+            ...result.inspectionSummary,
+            ocrRecoveredPageCount: Math.max(result.inspectionSummary.ocrRecoveredPageCount, recoveredPageCount),
+        },
+    };
+}
+function buildFallbackWorkerCheckpoints(job, pageCount, timestamp) {
+    if (job.checkpoints.pages.length > 0) {
+        return job.checkpoints.pages;
+    }
+    const startPage = job.source.pageRange?.[0] ?? job.segmentation?.pageRange?.[0] ?? 1;
+    return Array.from({ length: pageCount }, (_, index) => ({
+        pageNumber: startPage + index,
+        classification: null,
+        sourceKind: 'pdf-page',
+        weight: 1,
+        status: 'pending',
+        attempts: 0,
+        updatedAt: timestamp,
+    }));
+}
+async function inferWorkerPdfPageCount(job) {
+    if (job.source.totalPages > 0) {
+        return job.source.totalPages;
+    }
+    try {
+        const fullPageCount = await countDocumentPdfPages(job.source.filePath);
+        return job.source.pageRange
+            ? Math.max(0, Math.min(fullPageCount, job.source.pageRange[1]) - job.source.pageRange[0] + 1)
+            : fullPageCount;
+    }
+    catch {
+        try {
+            const fullPageCount = inferPdfDocumentPageCountFallback(job.source.filePath);
+            return job.source.pageRange
+                ? Math.max(0, Math.min(fullPageCount, job.source.pageRange[1]) - job.source.pageRange[0] + 1)
+                : fullPageCount;
+        }
+        catch {
+            return 0;
+        }
+    }
+}
 function summarizeReviewReasons(findings) {
     return uniqueStrings(findings
         .filter((finding) => finding.severity !== 'advisory')
@@ -289,19 +669,15 @@ function buildSyntheticBoreholeResult(job, inspection, now) {
             message: `${normalFailedPages.length} page(s) failed during ingest and should be reviewed.`,
         });
     }
-    return {
+    return applyCheckpointOcrRecoveredSummary({
         kind: 'geotech-ingest-result',
         schemaVersion: 1,
         documentType: 'borehole-log',
         generatedAt: nowIso(now),
-        source: {
-            filePath: job.source.filePath,
-            fileName: basename(job.source.filePath),
-            inputKind: 'pdf',
-            totalPages: job.source.totalPages,
+        source: buildJobResultSource(job, {
             successfulPages: 0,
             failedPages: pageFailures.length,
-        },
+        }),
         inspection,
         inspectionSummary: summarizeBoreholeIngestInspection(inspection),
         boreholes: [],
@@ -323,7 +699,7 @@ function buildSyntheticBoreholeResult(job, inspection, now) {
         reviewRequired: reviewFindings.some((finding) => finding.severity !== 'advisory'),
         confidence: 0,
         canAutoProceed: false,
-    };
+    }, job.checkpoints.pages);
 }
 function buildSyntheticGeotechDocumentResult(job, inspection, now) {
     const pageFailures = job.checkpoints.pages
@@ -360,19 +736,15 @@ function buildSyntheticGeotechDocumentResult(job, inspection, now) {
             message: `${normalFailedPages.length} page(s) failed during ingest and should be reviewed.`,
         });
     }
-    return {
+    return applyCheckpointOcrRecoveredSummary({
         kind: 'geotech-ingest-result',
         schemaVersion: 1,
         documentType: 'geotech-document',
         generatedAt: nowIso(now),
-        source: {
-            filePath: job.source.filePath,
-            fileName: basename(job.source.filePath),
-            inputKind: 'pdf',
-            totalPages: job.source.totalPages,
+        source: buildJobResultSource(job, {
             successfulPages: 0,
             failedPages: pageFailures.length,
-        },
+        }),
         inspection,
         inspectionSummary: buildInspectionSummary(inspection),
         documentClass: null,
@@ -402,7 +774,7 @@ function buildSyntheticGeotechDocumentResult(job, inspection, now) {
         confidence: 0,
         reviewRequired: reviewFindings.some((finding) => finding.severity !== 'advisory'),
         canAutoProceed: false,
-    };
+    }, job.checkpoints.pages);
 }
 function dedupeReviewFindings(reviewFindings) {
     return [
@@ -531,6 +903,7 @@ async function processGeotechDocumentPage(job, pageInput, config, dependencies)
     const lowYieldRole = inferPreflightLowYieldPageRole({
         inspectionPage,
         previousInspectionPage: job.inspection?.pages[pageInput.pageNumber - 2],
+        nextInspectionPage: job.inspection?.pages[pageInput.pageNumber],
         pageNumber: pageInput.pageNumber,
         totalPages: pageInput.totalPages,
         sourceKind: pageInput.sourceKind,
@@ -662,11 +1035,7 @@ async function finalizeJobResult(job, pageInputs, config, dependencies) {
         try {
             const result = await ingestBoreholeLogDocument({
                 config,
-                source: {
-                    filePath: job.source.filePath,
-                    fileName: basename(job.source.filePath),
-                    inputKind: 'pdf',
-                },
+                source: buildJobResultSource(job, { successfulPages: 0, failedPages: 0 }),
                 overrideBoreholeId: job.request.overrideBoreholeId,
                 inspection: job.inspection,
                 pages: job.checkpoints.pages
@@ -695,7 +1064,7 @@ async function finalizeJobResult(job, pageInputs, config, dependencies) {
                 },
                 now: dependencies.now,
             });
-            return applyBoreholeFailureDowngrades(result, job.checkpoints.pages);
+            return applyCheckpointOcrRecoveredSummary(applyBoreholeFailureDowngrades(result, job.checkpoints.pages), job.checkpoints.pages);
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
@@ -709,11 +1078,7 @@ async function finalizeJobResult(job, pageInputs, config, dependencies) {
     try {
         const result = await ingestGeotechDocument({
             config,
-            source: {
-                filePath: job.source.filePath,
-                fileName: basename(job.source.filePath),
-                inputKind: 'pdf',
-            },
+            source: buildJobResultSource(job, { successfulPages: 0, failedPages: 0 }),
             inspection: job.inspection,
             pages: job.checkpoints.pages
                 .map((checkpoint) => geotechPageInputMap.get(checkpoint.pageNumber))
@@ -752,7 +1117,7 @@ async function finalizeJobResult(job, pageInputs, config, dependencies) {
             },
             now: dependencies.now,
         });
-        return applyGeotechFailureDowngrades(result, job.checkpoints.pages);
+        return applyCheckpointOcrRecoveredSummary(applyGeotechFailureDowngrades(result, job.checkpoints.pages), job.checkpoints.pages);
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -805,28 +1170,62 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
             ? currentJob.inspection
             : inspect(currentJob.source.filePath);
         if (!currentJob.inspection || currentJob.inspection.totalPages === 0) {
-            await mutateJob((job) => ({
-                ...job,
-                inspection,
-                source: {
-                    ...job.source,
-                    totalPages: inspection.totalPages,
-                    weightedPageCost: inspection.pages.reduce((sum, page) => sum + (page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1), 0),
-                },
-                checkpoints: {
-                    pages: inspection.pages.map((page) => job.checkpoints.pages.find((existing) => existing.pageNumber === page.pageNumber) ?? ({
-                        pageNumber: page.pageNumber,
-                        classification: page.classification,
-                        sourceKind: mapPageSourceKind(page.classification),
-                        weight: page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1,
-                        status: 'pending',
-                        attempts: 0,
-                        updatedAt: nowIso(dependencies.now),
-                    })),
-                },
-            }));
+            if (inspection.totalPages > 0) {
+                await mutateJob((job) => ({
+                    ...job,
+                    inspection,
+                    source: {
+                        ...job.source,
+                        totalPages: inspection.totalPages,
+                        weightedPageCost: inspection.pages.reduce((sum, page) => sum + (page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1), 0),
+                    },
+                    processing: {
+                        ...job.processing,
+                        chunkExtractionConcurrency: resolvePersistedIngestJobExtractionConcurrency(job.config, inspection),
+                    },
+                    checkpoints: {
+                        pages: inspection.pages.map((page) => job.checkpoints.pages.find((existing) => existing.pageNumber === page.pageNumber) ?? ({
+                            pageNumber: page.pageNumber,
+                            classification: page.classification,
+                            sourceKind: mapPageSourceKind(page.classification),
+                            weight: page.classification === 'image-only' || page.classification === 'text-unreadable' ? 2 : 1,
+                            status: 'pending',
+                            attempts: 0,
+                            updatedAt: nowIso(dependencies.now),
+                        })),
+                    },
+                }));
+            }
+            else {
+                const inferredPageCount = await inferWorkerPdfPageCount(currentJob);
+                if (inferredPageCount > 0) {
+                    await mutateJob((job) => {
+                        const timestamp = nowIso(dependencies.now);
+                        return {
+                            ...job,
+                            inspection: null,
+                            source: {
+                                ...job.source,
+                                totalPages: Math.max(job.source.totalPages, inferredPageCount),
+                                weightedPageCost: Math.max(job.source.weightedPageCost, inferredPageCount),
+                            },
+                            processing: {
+                                ...job.processing,
+                                chunkExtractionConcurrency: resolvePersistedIngestJobExtractionConcurrency(job.config, null),
+                            },
+                            checkpoints: {
+                                pages: buildFallbackWorkerCheckpoints(job, inferredPageCount, timestamp),
+                            },
+                        };
+                    });
+                }
+            }
         }
+        currentJob = loadPersistedIngestJob(jobId) ?? currentJob;
         const config = buildJobConfig(currentJob, dependencies);
+        if (currentJob.documentType === 'geotech-document' && currentJob.segmentation?.mode === 'segmented-parent') {
+            return await runSegmentedParentGeotechJob(jobId, currentJob, mutateJob, config, dependencies);
+        }
         const pageInputs = await preparePdfPageInputs(currentJob.source.filePath, currentJob.inspection, currentJob.processing.pagePreprocessingConcurrency, dependencies);
         let processedNewPages = 0;
         if (currentJob.documentType === 'geotech-document') {
@@ -878,8 +1277,45 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                             : checkpoint),
                     },
                 }));
-                try {
-                    const processed = await processGeotechDocumentPage(currentJob, page, config, dependencies);
+                let processed = null;
+                let finalErrorMessage = '';
+                for (let attemptIndex = 0; attemptIndex < 2; attemptIndex += 1) {
+                    try {
+                        processed = await processGeotechDocumentPage(currentJob, page, config, dependencies);
+                        break;
+                    }
+                    catch (error) {
+                        const message = error instanceof Error ? error.message : String(error);
+                        const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                        if (attemptIndex === 0
+                            && isRetryableUpstreamPageError(normalizedMessage)
+                            && !isFatalProviderStopError(normalizedMessage)) {
+                            await waitForCheckpointRetryBackoff(findCheckpoint(currentJob, page.pageNumber).attempts);
+                            await mutateJob((job) => ({
+                                ...job,
+                                updatedAt: nowIso(dependencies.now),
+                                execution: {
+                                    ...job.execution,
+                                    lastHeartbeatAt: nowIso(dependencies.now),
+                                },
+                                checkpoints: {
+                                    pages: job.checkpoints.pages.map((checkpoint) => checkpoint.pageNumber === page.pageNumber
+                                        ? {
+                                            ...checkpoint,
+                                            attempts: checkpoint.attempts + 1,
+                                            updatedAt: nowIso(dependencies.now),
+                                            error: `retrying after upstream timeout: ${normalizedMessage}`,
+                                        }
+                                        : checkpoint),
+                                },
+                            }));
+                            continue;
+                        }
+                        finalErrorMessage = normalizedMessage;
+                        break;
+                    }
+                }
+                if (processed) {
                     processedNewPages += 1;
                     await mutateJob((job) => ({
                         ...job,
@@ -906,9 +1342,8 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                         },
                     }));
                 }
-                catch (error) {
-                    const message = error instanceof Error ? error.message : String(error);
-                    const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                else {
+                    const normalizedMessage = finalErrorMessage || `Page ${page.pageNumber} failed during async ingest.`;
                     const checkpoint = findCheckpoint(currentJob, page.pageNumber);
                     if (!fatalProviderStopMessage && isFatalProviderStopError(normalizedMessage)) {
                         fatalProviderStopMessage = normalizedMessage;
@@ -993,8 +1428,45 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                             : pageCheckpoint),
                     },
                 }));
-                try {
-                    const processed = await processBoreholePage(currentJob, page, config, state, dependencies);
+                let processed = null;
+                let finalErrorMessage = '';
+                for (let attemptIndex = 0; attemptIndex < 2; attemptIndex += 1) {
+                    try {
+                        processed = await processBoreholePage(currentJob, page, config, state, dependencies);
+                        break;
+                    }
+                    catch (error) {
+                        const message = error instanceof Error ? error.message : String(error);
+                        const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                        if (attemptIndex === 0
+                            && isRetryableUpstreamPageError(normalizedMessage)
+                            && !isFatalProviderStopError(normalizedMessage)) {
+                            await waitForCheckpointRetryBackoff(findCheckpoint(currentJob, page.pageNumber).attempts);
+                            await mutateJob((job) => ({
+                                ...job,
+                                updatedAt: nowIso(dependencies.now),
+                                execution: {
+                                    ...job.execution,
+                                    lastHeartbeatAt: nowIso(dependencies.now),
+                                },
+                                checkpoints: {
+                                    pages: job.checkpoints.pages.map((pageCheckpoint) => pageCheckpoint.pageNumber === page.pageNumber
+                                        ? {
+                                            ...pageCheckpoint,
+                                            attempts: pageCheckpoint.attempts + 1,
+                                            updatedAt: nowIso(dependencies.now),
+                                            error: `retrying after upstream timeout: ${normalizedMessage}`,
+                                        }
+                                        : pageCheckpoint),
+                                },
+                            }));
+                            continue;
+                        }
+                        finalErrorMessage = normalizedMessage;
+                        break;
+                    }
+                }
+                if (processed) {
                     processedNewPages += 1;
                     state = processed.nextState;
                     await mutateJob((job) => ({
@@ -1022,9 +1494,8 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                         },
                     }));
                 }
-                catch (error) {
-                    const message = error instanceof Error ? error.message : String(error);
-                    const normalizedMessage = normalizeCheckpointErrorMessage(message);
+                else {
+                    const normalizedMessage = finalErrorMessage || `Page ${page.pageNumber} failed during async ingest.`;
                     await mutateJob((job) => ({
                         ...job,
                         updatedAt: nowIso(dependencies.now),
@@ -1058,7 +1529,7 @@ export async function runPersistedIngestJobWorker(jobId, dependencies = {}) {
                                         ...pageCheckpoint,
                                         status: 'failed',
                                         updatedAt: nowIso(dependencies.now),
-                                        error: `skipped after upstream provider stop. ${normalizeCheckpointErrorMessage(message)}`,
+                                        error: `skipped after upstream provider stop. ${normalizedMessage}`,
                                         downgraded: false,
                                     }
                                     : pageCheckpoint),