@heripo/pdf-parser 0.1.33 → 0.1.35

package/dist/index.d.cts

@@ -54,6 +54,12 @@ interface PDFCorrectionOptions {
      * Enabled by the engine demo; defaults to false everywhere else.
      */
     forceAutoApply?: boolean;
+    /**
+     * When false, the review-assistance stage (the per-page eligibility gate plus
+     * the structural review-assistance runner) is skipped entirely and only the
+     * text-correction stage runs. Defaults to true.
+     */
+    reviewAssistanceEnabled?: boolean;
     temperature?: number;
 }
 interface NormalizedPDFCorrectionOptions {
@@ -67,6 +73,7 @@ interface NormalizedPDFCorrectionOptions {
     autoApplyThreshold: number;
     proposalThreshold: number;
     forceAutoApply: boolean;
+    reviewAssistanceEnabled: boolean;
     temperature: number;
 }
 declare const PDF_CORRECTION_DEFAULTS: Omit<NormalizedPDFCorrectionOptions, 'models'>;

package/dist/index.d.ts

@@ -54,6 +54,12 @@ interface PDFCorrectionOptions {
      * Enabled by the engine demo; defaults to false everywhere else.
      */
     forceAutoApply?: boolean;
+    /**
+     * When false, the review-assistance stage (the per-page eligibility gate plus
+     * the structural review-assistance runner) is skipped entirely and only the
+     * text-correction stage runs. Defaults to true.
+     */
+    reviewAssistanceEnabled?: boolean;
     temperature?: number;
 }
 interface NormalizedPDFCorrectionOptions {
@@ -67,6 +73,7 @@ interface NormalizedPDFCorrectionOptions {
     autoApplyThreshold: number;
     proposalThreshold: number;
     forceAutoApply: boolean;
+    reviewAssistanceEnabled: boolean;
     temperature: number;
 }
 declare const PDF_CORRECTION_DEFAULTS: Omit<NormalizedPDFCorrectionOptions, 'models'>;

package/dist/index.js

@@ -343,6 +343,29 @@ var LLMCaller = class {
 function formatTokens(usage) {
   return `${usage.inputTokens} input, ${usage.outputTokens} output, ${usage.totalTokens} total`;
 }
+function emptyUsage() {
+  return { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+}
+function addInto(target, src) {
+  target.inputTokens += src.inputTokens;
+  target.outputTokens += src.outputTokens;
+  target.totalTokens += src.totalTokens;
+}
+function sumUsage(a, b) {
+  return {
+    inputTokens: (a?.inputTokens ?? 0) + (b?.inputTokens ?? 0),
+    outputTokens: (a?.outputTokens ?? 0) + (b?.outputTokens ?? 0),
+    totalTokens: (a?.totalTokens ?? 0) + (b?.totalTokens ?? 0)
+  };
+}
+function accumulateModel(bucket, modelName, usage) {
+  let entry = bucket.get(modelName);
+  if (!entry) {
+    entry = emptyUsage();
+    bucket.set(modelName, entry);
+  }
+  addInto(entry, usage);
+}
 var LLMTokenUsageAggregator = class {
   usage = {};
   /**
@@ -355,21 +378,15 @@ var LLMTokenUsageAggregator = class {
       this.usage[usage.component] = {
         component: usage.component,
         phases: {},
-        total: {
-          inputTokens: 0,
-          outputTokens: 0,
-          totalTokens: 0
-        }
+        total: emptyUsage()
       };
     }
     const component = this.usage[usage.component];
     if (!component.phases[usage.phase]) {
       component.phases[usage.phase] = {
-        total: {
-          inputTokens: 0,
-          outputTokens: 0,
-          totalTokens: 0
-        },
+        primary: /* @__PURE__ */ new Map(),
+        fallback: /* @__PURE__ */ new Map(),
+        total: emptyUsage(),
         metadata: []
       };
     }
@@ -378,41 +395,17 @@ var LLMTokenUsageAggregator = class {
       phase.metadata.push(usage.metadata);
     }
     if (usage.model === "primary") {
-      if (!phase.primary) {
-        phase.primary = {
-          modelName: usage.modelName,
-          inputTokens: 0,
-          outputTokens: 0,
-          totalTokens: 0
-        };
-      }
-      phase.primary.inputTokens += usage.inputTokens;
-      phase.primary.outputTokens += usage.outputTokens;
-      phase.primary.totalTokens += usage.totalTokens;
+      accumulateModel(phase.primary, usage.modelName, usage);
     } else if (usage.model === "fallback") {
-      if (!phase.fallback) {
-        phase.fallback = {
-          modelName: usage.modelName,
-          inputTokens: 0,
-          outputTokens: 0,
-          totalTokens: 0
-        };
-      }
-      phase.fallback.inputTokens += usage.inputTokens;
-      phase.fallback.outputTokens += usage.outputTokens;
-      phase.fallback.totalTokens += usage.totalTokens;
+      accumulateModel(phase.fallback, usage.modelName, usage);
     }
-    phase.total.inputTokens += usage.inputTokens;
-    phase.total.outputTokens += usage.outputTokens;
-    phase.total.totalTokens += usage.totalTokens;
-    component.total.inputTokens += usage.inputTokens;
-    component.total.outputTokens += usage.outputTokens;
-    component.total.totalTokens += usage.totalTokens;
+    addInto(phase.total, usage);
+    addInto(component.total, usage);
   }
   /**
    * Get aggregated usage grouped by component
    *
-   * @returns Array of component aggregates with phase breakdown
+   * @returns Array of component aggregates with per-model phase breakdown
    */
   getByComponent() {
     return Object.values(this.usage);
@@ -421,8 +414,16 @@ var LLMTokenUsageAggregator = class {
    * Get token usage report in structured JSON format
    *
    * Converts internal usage data to external TokenUsageReport format suitable
-   * for serialization and reporting. The report includes component breakdown,
-   * phase-level details, and both primary and fallback model usage.
+   * for serialization and reporting.
+   *
+   * Each (phase, model) pair becomes its own PhaseUsageReport entry. A phase
+   * that used a single primary (and/or a single fallback) model yields exactly
+   * one entry — identical to the legacy shape — while a phase that mixed models
+   * yields one entry per model, all sharing the same `phase` name. Every
+   * (phase, tier, modelName) combination appears in exactly one entry, so
+   * consumers that key by `${component}|${phase}|${tier}|${modelName}`
+   * (heripo-web's token-usage emitter / ledger recorder) attribute each model's
+   * tokens — and cost — correctly without double counting.
    *
    * @returns Structured token usage report with components and total
    */
@@ -431,34 +432,41 @@ var LLMTokenUsageAggregator = class {
     for (const component of Object.values(this.usage)) {
       const phases = [];
       for (const [phaseName, phaseData] of Object.entries(component.phases)) {
-        const phaseReport = {
-          phase: phaseName,
-          total: {
-            inputTokens: phaseData.total.inputTokens,
-            outputTokens: phaseData.total.outputTokens,
-            totalTokens: phaseData.total.totalTokens
-          }
-        };
-        if (phaseData.metadata.length > 0) {
-          phaseReport.metadata = [...phaseData.metadata];
-        }
-        if (phaseData.primary) {
-          phaseReport.primary = {
-            modelName: phaseData.primary.modelName,
-            inputTokens: phaseData.primary.inputTokens,
-            outputTokens: phaseData.primary.outputTokens,
-            totalTokens: phaseData.primary.totalTokens
+        const primaryEntries = [...phaseData.primary.entries()];
+        const fallbackEntries = [...phaseData.fallback.entries()];
+        const entryCount = Math.max(
+          primaryEntries.length,
+          fallbackEntries.length
+        );
+        if (entryCount === 0) {
+          const phaseReport = {
+            phase: phaseName,
+            total: { ...phaseData.total }
           };
+          if (phaseData.metadata.length > 0) {
+            phaseReport.metadata = [...phaseData.metadata];
+          }
+          phases.push(phaseReport);
+          continue;
         }
-        if (phaseData.fallback) {
-          phaseReport.fallback = {
-            modelName: phaseData.fallback.modelName,
-            inputTokens: phaseData.fallback.inputTokens,
-            outputTokens: phaseData.fallback.outputTokens,
-            totalTokens: phaseData.fallback.totalTokens
+        for (let i = 0; i < entryCount; i++) {
+          const primary = primaryEntries[i];
+          const fallback = fallbackEntries[i];
+          const phaseReport = {
+            phase: phaseName,
+            total: sumUsage(primary?.[1], fallback?.[1])
           };
+          if (primary) {
+            phaseReport.primary = { modelName: primary[0], ...primary[1] };
+          }
+          if (fallback) {
+            phaseReport.fallback = { modelName: fallback[0], ...fallback[1] };
+          }
+          if (i === 0 && phaseData.metadata.length > 0) {
+            phaseReport.metadata = [...phaseData.metadata];
+          }
+          phases.push(phaseReport);
         }
-        phases.push(phaseReport);
       }
       components.push({
         component: component.component,
@@ -503,8 +511,8 @@ var LLMTokenUsageAggregator = class {
   /**
    * Log comprehensive token usage summary
    *
-   * Outputs usage grouped by component, with phase and model breakdown.
-   * Shows primary and fallback token usage separately for each phase.
+   * Outputs usage grouped by component, with phase and per-model breakdown.
+   * Shows each primary and fallback model on its own line for each phase.
    * Call this once at the end of document processing.
    *
    * @param logger - Logger instance for output
@@ -520,31 +528,23 @@ var LLMTokenUsageAggregator = class {
     let grandInputTokens = 0;
     let grandOutputTokens = 0;
     let grandTotalTokens = 0;
-    let grandPrimaryInputTokens = 0;
-    let grandPrimaryOutputTokens = 0;
-    let grandPrimaryTotalTokens = 0;
-    let grandFallbackInputTokens = 0;
-    let grandFallbackOutputTokens = 0;
-    let grandFallbackTotalTokens = 0;
+    const grandPrimary = emptyUsage();
+    const grandFallback = emptyUsage();
     for (const component of components) {
       logger.info(`${component.component}:`);
-      for (const [phase, phaseData] of Object.entries(component.phases)) {
-        logger.info(`  - ${phase}:`);
-        if (phaseData.primary) {
+      for (const [phaseName, phaseData] of Object.entries(component.phases)) {
+        logger.info(`  - ${phaseName}:`);
+        for (const [modelName, modelUsage] of phaseData.primary) {
           logger.info(
-            `      primary (${phaseData.primary.modelName}): ${formatTokens(phaseData.primary)}`
+            `      primary (${modelName}): ${formatTokens(modelUsage)}`
           );
-          grandPrimaryInputTokens += phaseData.primary.inputTokens;
-          grandPrimaryOutputTokens += phaseData.primary.outputTokens;
-          grandPrimaryTotalTokens += phaseData.primary.totalTokens;
+          addInto(grandPrimary, modelUsage);
         }
-        if (phaseData.fallback) {
+        for (const [modelName, modelUsage] of phaseData.fallback) {
           logger.info(
-            `      fallback (${phaseData.fallback.modelName}): ${formatTokens(phaseData.fallback)}`
+            `      fallback (${modelName}): ${formatTokens(modelUsage)}`
           );
-          grandFallbackInputTokens += phaseData.fallback.inputTokens;
-          grandFallbackOutputTokens += phaseData.fallback.outputTokens;
-          grandFallbackTotalTokens += phaseData.fallback.totalTokens;
+          addInto(grandFallback, modelUsage);
         }
         logger.info(`      subtotal: ${formatTokens(phaseData.total)}`);
       }
@@ -557,23 +557,11 @@ var LLMTokenUsageAggregator = class {
       grandTotalTokens += component.total.totalTokens;
     }
     logger.info("--- Summary ---");
-    if (grandPrimaryTotalTokens > 0) {
-      logger.info(
-        `Primary total: ${formatTokens({
-          inputTokens: grandPrimaryInputTokens,
-          outputTokens: grandPrimaryOutputTokens,
-          totalTokens: grandPrimaryTotalTokens
-        })}`
-      );
+    if (grandPrimary.totalTokens > 0) {
+      logger.info(`Primary total: ${formatTokens(grandPrimary)}`);
     }
-    if (grandFallbackTotalTokens > 0) {
-      logger.info(
-        `Fallback total: ${formatTokens({
-          inputTokens: grandFallbackInputTokens,
-          outputTokens: grandFallbackOutputTokens,
-          totalTokens: grandFallbackTotalTokens
-        })}`
-      );
+    if (grandFallback.totalTokens > 0) {
+      logger.info(`Fallback total: ${formatTokens(grandFallback)}`);
     }
     logger.info(
       `Grand total: ${formatTokens({
@@ -10449,6 +10437,7 @@ var PDF_CORRECTION_DEFAULTS = {
   autoApplyThreshold: 0.85,
   proposalThreshold: 0.5,
   forceAutoApply: false,
+  reviewAssistanceEnabled: true,
   temperature: 0
 };
 function normalizeNumber(value, fallback) {
@@ -10548,6 +10537,7 @@ function normalizePDFCorrectionOptions(value) {
     autoApplyThreshold,
     proposalThreshold,
     forceAutoApply: value.forceAutoApply ?? PDF_CORRECTION_DEFAULTS.forceAutoApply,
+    reviewAssistanceEnabled: value.reviewAssistanceEnabled ?? PDF_CORRECTION_DEFAULTS.reviewAssistanceEnabled,
     temperature: normalizeTemperature(
       value.temperature,
       PDF_CORRECTION_DEFAULTS.temperature
@@ -10581,44 +10571,49 @@ var PostDoclingCorrectionPipeline = class {
           onTokenUsage: options.onTokenUsage,
           documentLanguages: options.ocr_lang,
           pageTexts,
-          reviewAssistanceGate: {
+          // When review assistance is disabled, omit the gate entirely so the
+          // per-page eligibility checks (pageGate LLM calls + sidecar report)
+          // never run — only text correction executes.
+          reviewAssistanceGate: correction.reviewAssistanceEnabled ? {
             model: correction.models.pageGate,
             fallback: correction.models.pageGateFallback,
             maxRetries: correction.maxRetries.pageGate,
             temperature: correction.temperature,
             outputLanguage: correction.outputLanguage
-          }
-        }
-      );
-      const runner = new ReviewAssistanceRunner(this.logger);
-      await runner.analyzeAndSave(
-        outputDir,
-        reportId,
-        this.buildReviewAssistanceModelResolver(correction),
-        {
-          pdfPath,
-          pageTexts,
-          pageGateModel: correction.models.pageGate,
-          pageGateFallback: correction.models.pageGateFallback,
-          pageGateMaxRetries: correction.maxRetries.pageGate,
-          pageGateTemperature: correction.temperature,
-          pageConcurrency: correction.concurrency.pages,
-          taskConcurrency: correction.concurrency.reviewTasks,
-          modelConcurrency: correction.modelConcurrency,
-          workItemTimeoutMs: correction.workItemTimeoutMs,
-          autoApplyThreshold: correction.autoApplyThreshold,
-          proposalThreshold: correction.proposalThreshold,
-          forceAutoApply: correction.forceAutoApply,
-          maxRetries: correction.maxRetries.reviewAssistance,
-          tableMaxRetries: correction.maxRetries.tableCorrection,
-          temperature: correction.temperature,
-          outputLanguage: correction.outputLanguage,
-          aggregator: options.aggregator,
-          abortSignal,
-          onTokenUsage: options.onTokenUsage,
-          onProgress: options.onReviewAssistanceProgress
+          } : void 0
         }
       );
+      if (correction.reviewAssistanceEnabled) {
+        const runner = new ReviewAssistanceRunner(this.logger);
+        await runner.analyzeAndSave(
+          outputDir,
+          reportId,
+          this.buildReviewAssistanceModelResolver(correction),
+          {
+            pdfPath,
+            pageTexts,
+            pageGateModel: correction.models.pageGate,
+            pageGateFallback: correction.models.pageGateFallback,
+            pageGateMaxRetries: correction.maxRetries.pageGate,
+            pageGateTemperature: correction.temperature,
+            pageConcurrency: correction.concurrency.pages,
+            taskConcurrency: correction.concurrency.reviewTasks,
+            modelConcurrency: correction.modelConcurrency,
+            workItemTimeoutMs: correction.workItemTimeoutMs,
+            autoApplyThreshold: correction.autoApplyThreshold,
+            proposalThreshold: correction.proposalThreshold,
+            forceAutoApply: correction.forceAutoApply,
+            maxRetries: correction.maxRetries.reviewAssistance,
+            tableMaxRetries: correction.maxRetries.tableCorrection,
+            temperature: correction.temperature,
+            outputLanguage: correction.outputLanguage,
+            aggregator: options.aggregator,
+            abortSignal,
+            onTokenUsage: options.onTokenUsage,
+            onProgress: options.onReviewAssistanceProgress
+          }
+        );
+      }
       await originalCallback(outputDir);
     };
   }