@clazic/kordoc 2.4.19 → 2.5.1

package/dist/index.d.cts

@@ -421,6 +421,8 @@ interface UnifiedOcrProgressEvent {
     code?: UnifiedOcrErrorCode;
     message?: string;
     model?: string;
+    pageNumber?: number;
+    workerCount?: number;
 }
 interface UnifiedOcrOptions {
     workspaceDir?: string;
@@ -436,6 +438,7 @@ interface UnifiedOcrOptions {
     probeConcurrency?: number;
     logger?: Logger;
     runId?: string;
+    concurrencyPerKey?: number;
 }
 interface UnifiedOcrResult {
     outputPath: string;

package/dist/index.d.ts

@@ -421,6 +421,8 @@ interface UnifiedOcrProgressEvent {
     code?: UnifiedOcrErrorCode;
     message?: string;
     model?: string;
+    pageNumber?: number;
+    workerCount?: number;
 }
 interface UnifiedOcrOptions {
     workspaceDir?: string;
@@ -436,6 +438,7 @@ interface UnifiedOcrOptions {
     probeConcurrency?: number;
     logger?: Logger;
     runId?: string;
+    concurrencyPerKey?: number;
 }
 interface UnifiedOcrResult {
     outputPath: string;

package/dist/index.js

@@ -3035,7 +3035,7 @@ import JSZip2 from "jszip";
 import { DOMParser } from "@xmldom/xmldom";
 
 // src/utils.ts
-var VERSION = true ? "2.4.19" : "0.0.0-dev";
+var VERSION = true ? "2.5.0" : "0.0.0-dev";
 function toArrayBuffer(buf) {
   if (buf.byteOffset === 0 && buf.byteLength === buf.buffer.byteLength) {
     return buf.buffer;
@@ -11231,6 +11231,66 @@ import { spawn as spawn2 } from "child_process";
 import { performance } from "perf_hooks";
 import libre from "libreoffice-convert";
 init_logger();
+
+// src/pipeline/bounded-queue.ts
+var QUEUE_DONE = /* @__PURE__ */ Symbol("QUEUE_DONE");
+var BoundedQueue = class {
+  buffer = [];
+  capacity;
+  closed = false;
+  producerWaiters = [];
+  consumerWaiters = [];
+  constructor(capacity) {
+    if (capacity < 1) throw new RangeError("BoundedQueue capacity must be >= 1");
+    this.capacity = capacity;
+  }
+  async enqueue(item) {
+    if (this.closed) throw new Error("BoundedQueue: cannot enqueue after close()");
+    if (this.consumerWaiters.length > 0) {
+      const resolve4 = this.consumerWaiters.shift();
+      resolve4(item);
+      return;
+    }
+    while (this.buffer.length >= this.capacity) {
+      await new Promise((resolve4) => this.producerWaiters.push(resolve4));
+      if (this.closed) throw new Error("BoundedQueue: closed while waiting to enqueue");
+    }
+    this.buffer.push(item);
+  }
+  async dequeue() {
+    if (this.buffer.length > 0) {
+      const item = this.buffer.shift();
+      this._wakeProducer();
+      return item;
+    }
+    if (this.closed) return QUEUE_DONE;
+    return new Promise((resolve4) => {
+      this.consumerWaiters.push(resolve4);
+    });
+  }
+  close() {
+    if (this.closed) return;
+    this.closed = true;
+    for (const resolve4 of this.consumerWaiters) {
+      resolve4(QUEUE_DONE);
+    }
+    this.consumerWaiters = [];
+    for (const wake of this.producerWaiters) {
+      wake();
+    }
+    this.producerWaiters = [];
+  }
+  get size() {
+    return this.buffer.length;
+  }
+  _wakeProducer() {
+    if (this.producerWaiters.length > 0) {
+      this.producerWaiters.shift()();
+    }
+  }
+};
+
+// src/pipeline/unified-ocr.ts
 var libreConvert = libre.convert;
 var UnifiedOcrError = class extends Error {
   code;
@@ -11262,11 +11322,11 @@ var DEFAULT_MODEL_MAX_TOKENS = {
 };
 var DEFAULT_STAGE_WEIGHTS = {
   convert: 15,
-  render: 20,
+  render: 15,
   probe: 5,
-  ocr: 45,
+  ocr: 55,
   proofread: 0,
-  merge: 5
+  merge: 10
 };
 var OCR_PROMPT2 = [
   "\uC774 PDF \uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0\uC5D0\uC11C \uD14D\uC2A4\uD2B8\uC640 \uD45C\uB97C \uCD94\uCD9C\uD558\uC5EC Markdown\uC73C\uB85C \uBCC0\uD658\uD558\uACE0, OCR \uC624\uC778\uC2DD \uC624\uB958\uB97C \uC989\uC2DC \uAD50\uC815\uD558\uC5EC \uCD5C\uC885 \uACB0\uACFC\uBB3C\uC744 \uCD9C\uB825\uD558\uC138\uC694.",
@@ -11355,39 +11415,19 @@ async function runUnifiedOcrPipeline(inputPath, options = {}) {
     logStage("info", "convert", "done", "PDF \uBCC0\uD658 \uC644\uB8CC", { elapsedMs: timingsMs.convert });
     const renderStart = performance.now();
     currentStage = "render";
+    const totalPages = await getPdfPageCount(workingPdfPath).catch(() => 0);
+    if (totalPages === 0) throw new UnifiedOcrError("RENDER_FAILED", "render", "\uD398\uC774\uC9C0 \uC218\uB97C \uD655\uC778\uD560 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4.");
     markStageStart("render", "PDF \uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC911");
-    logStage("info", "render", "start", "PDF \uD398\uC774\uC9C0 \uB80C\uB354\uB9C1 \uC2DC\uC791", { pdf: workingPdfPath, dpi });
-    const renderWithProgress = await renderPdfToPngWithProgress(
-      workingPdfPath,
-      join4(imagesDir, "page"),
-      dpi,
-      (current, total) => {
-        markStageProgress(
-          "render",
-          Math.round(current / total * 100),
-          current,
-          total,
-          `\uD398\uC774\uC9C0 ${current}/${total} \uB80C\uB354\uB9C1`
-        );
-      }
-    );
-    const images = await listPageImages(imagesDir);
-    if (images.length === 0) throw new UnifiedOcrError("RENDER_FAILED", "render", "\uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC2E4\uD328: \uACB0\uACFC \uC774\uBBF8\uC9C0\uAC00 \uC5C6\uC2B5\uB2C8\uB2E4.");
-    if (!renderWithProgress.emittedPerPageProgress) {
-      markStageProgress("render", 100, images.length, images.length, `\uD398\uC774\uC9C0 ${images.length}\uC7A5 \uC0DD\uC131`);
-    }
-    timingsMs.render = elapsedMs(renderStart);
-    markStageDone("render", "\uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC");
-    logStage("info", "render", "done", "\uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC", {
-      pages: images.length,
-      elapsedMs: timingsMs.render,
-      pageCountSource: renderWithProgress.pageCountSource
-    });
+    logStage("info", "render", "start", "PDF \uD398\uC774\uC9C0 \uB80C\uB354\uB9C1 \uC2DC\uC791", { pdf: workingPdfPath, dpi, totalPages });
+    await runCommand("pdftoppm", ["-png", "-r", String(dpi), "-f", "1", "-l", "1", workingPdfPath, join4(imagesDir, "page")]);
+    const firstFiles = (await readdir(imagesDir)).filter((f) => f.endsWith(".png")).sort((a, b) => naturalPageSort(a, b));
+    if (firstFiles.length === 0) throw new UnifiedOcrError("RENDER_FAILED", "render", "\uCCAB \uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC2E4\uD328");
+    const probeImage = join4(imagesDir, firstFiles[0]);
+    markStageProgress("render", Math.round(1 / totalPages * 100), 1, totalPages, `\uD398\uC774\uC9C0 1/${totalPages} \uB80C\uB354\uB9C1`);
     const probeStart = performance.now();
     currentStage = "probe";
     markStageStart("probe", "\uBAA8\uB378 \uC18D\uB3C4 \uD504\uB85C\uBE0C \uC218\uD589 \uC911");
     logStage("info", "probe", "start", "\uBAA8\uB378 \uC18D\uB3C4 \uD504\uB85C\uBE0C \uC2DC\uC791", { models, probeConcurrency });
-    const probeImage = await pickRepresentativeImage(images);
     let probeDone = 0;
     const probeRuns = startParallelProbeRuns({
       models,
@@ -11414,33 +11454,74 @@ async function runUnifiedOcrPipeline(inputPath, options = {}) {
       await updateModelCache(modelCachePath, results);
       return results;
     });
+    const concurrencyPerKey = Math.max(1, options.concurrencyPerKey ?? 2);
+    const keyCount = keyPool.snapshot().length;
+    const workerCount = Math.max(1, keyCount * concurrencyPerKey);
+    const queueCapacity = workerCount * 2;
+    const queue = new BoundedQueue(queueCapacity);
     const ocrStart = performance.now();
     currentStage = "ocr";
-    markStageStart("ocr", `OCR \uC9C4\uD589 \uC911 (${selectedModel})`);
-    logStage("info", "ocr", "start", "\uD398\uC774\uC9C0 OCR \uC2DC\uC791", { selectedModel, pageCount: images.length });
-    const rawPagePaths = [];
-    for (let i = 0; i < images.length; i++) {
-      const imagePath = images[i];
-      const markdown = await ocrImageWithFallback({
-        imagePath,
-        prompt: OCR_PROMPT2,
-        models: fallbackModelOrder,
-        modelMaxTokens,
-        baseUrl,
-        keyPool,
-        timeoutMs,
-        maxRetriesPerPage,
+    markStageStart("ocr", `OCR \uC9C4\uD589 \uC911 (\uC6CC\uCEE4 ${workerCount}\uAC1C)`);
+    logStage("info", "ocr", "start", "\uD398\uC774\uC9C0 OCR \uC2DC\uC791", { workerCount, keyCount, pageCount: totalPages });
+    let renderDone = 1;
+    const renderProducer = (async () => {
+      try {
+        await queue.enqueue({ pageNumber: 1, imagePath: probeImage });
+        if (totalPages > 1) {
+          for await (const item of renderPdfToPngStream(workingPdfPath, join4(imagesDir, "page"), dpi, totalPages, 2)) {
+            await queue.enqueue(item);
+            renderDone++;
+            markStageProgress("render", Math.round(renderDone / totalPages * 100), renderDone, totalPages, `\uD398\uC774\uC9C0 ${renderDone}/${totalPages} \uB80C\uB354\uB9C1`);
+            logStage("debug", "render", "progress", "\uD398\uC774\uC9C0 \uB80C\uB354 \uC644\uB8CC", { page: item.pageNumber });
+          }
+        }
+      } finally {
+        queue.close();
+        timingsMs.render = elapsedMs(renderStart);
+        markStageDone("render", "\uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC");
+        logStage("info", "render", "done", "\uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC", { pages: renderDone, elapsedMs: timingsMs.render });
+      }
+    })();
+    const [, pageResultsMap] = await Promise.all([
+      renderProducer,
+      ocrWorkerPool({
+        queue,
+        workerCount,
+        totalPages,
+        ocrInput: {
+          prompt: OCR_PROMPT2,
+          models: fallbackModelOrder,
+          modelMaxTokens,
+          baseUrl,
+          keyPool,
+          timeoutMs,
+          maxRetriesPerPage,
+          logger
+        },
+        onPageDone: (pageNumber, completedCount, model) => {
+          markStageProgress(
+            "ocr",
+            Math.round(completedCount / totalPages * 100),
+            completedCount,
+            totalPages,
+            `OCR ${completedCount}/${totalPages}`,
+            model || void 0
+          );
+          logStage("debug", "ocr", "progress", "\uD398\uC774\uC9C0 OCR \uC644\uB8CC", { page: pageNumber, total: totalPages, model });
+        },
         logger
-      });
-      const pagePath = join4(rawDir, `page_${String(i + 1).padStart(4, "0")}.md`);
-      await writeFile(pagePath, markdown, "utf-8");
-      rawPagePaths.push(pagePath);
-      markStageProgress("ocr", Math.round((i + 1) / images.length * 100), i + 1, images.length, `OCR ${i + 1}/${images.length}`, selectedModel);
-      logStage("debug", "ocr", "progress", "\uD398\uC774\uC9C0 OCR \uC644\uB8CC", { page: i + 1, total: images.length });
-    }
+      })
+    ]);
     timingsMs.ocr = elapsedMs(ocrStart);
     markStageDone("ocr", "OCR \uC644\uB8CC");
     logStage("info", "ocr", "done", "\uD398\uC774\uC9C0 OCR \uC644\uB8CC", { elapsedMs: timingsMs.ocr });
+    const sortedEntries = Array.from(pageResultsMap.entries()).sort((a, b) => a[0] - b[0]);
+    const rawPagePaths = [];
+    for (const [pageNum, markdown] of sortedEntries) {
+      const pagePath = join4(rawDir, `page_${String(pageNum).padStart(4, "0")}.md`);
+      await writeFile(pagePath, markdown, "utf-8");
+      rawPagePaths.push(pagePath);
+    }
     const mergeStart = performance.now();
     currentStage = "merge";
     markStageStart("merge", "\uCD5C\uC885 Markdown \uBCD1\uD569 \uC911");
@@ -11457,7 +11538,7 @@ async function runUnifiedOcrPipeline(inputPath, options = {}) {
       selectedModel,
       probeImage,
       probeResults: await probeResultsPromise,
-      pageCount: images.length,
+      pageCount: totalPages,
       keyHealth: keyPool.snapshot(),
       timingsMs,
       modelCachePath
@@ -11535,13 +11616,6 @@ async function convertWithLibreOffice(buffer, ext) {
     });
   });
 }
-async function renderPdfToPng(pdfPath, prefixPath, dpi) {
-  try {
-    await runCommand("pdftoppm", ["-png", "-r", String(dpi), pdfPath, prefixPath]);
-  } catch (err) {
-    throw new UnifiedOcrError("RENDER_FAILED", "render", err instanceof Error ? err.message : String(err));
-  }
-}
 async function getPdfPageCount(pdfPath) {
   const stdout = await runCommandWithStdout("pdfinfo", [pdfPath]);
   const m = stdout.match(/^\s*Pages:\s*(\d+)\s*$/mi);
@@ -11554,36 +11628,33 @@ async function getPdfPageCount(pdfPath) {
   }
   return n;
 }
-async function renderPdfToPngWithProgress(pdfPath, prefixPath, dpi, onPageDone) {
-  let totalPages = 0;
-  try {
-    totalPages = await getPdfPageCount(pdfPath);
-  } catch {
-    totalPages = 0;
-  }
-  if (totalPages > 0) {
+async function* renderPdfToPngStream(pdfPath, prefixPath, dpi, totalPages, startPage = 1) {
+  const imagesDir = dirname3(prefixPath);
+  for (let page = startPage; page <= totalPages; page++) {
     try {
-      for (let page = 1; page <= totalPages; page++) {
-        await runCommand("pdftoppm", [
-          "-png",
-          "-r",
-          String(dpi),
-          "-f",
-          String(page),
-          "-l",
-          String(page),
-          pdfPath,
-          prefixPath
-        ]);
-        onPageDone(page, totalPages);
-      }
-      return { emittedPerPageProgress: true, pageCountSource: "pdfinfo" };
+      await runCommand("pdftoppm", [
+        "-png",
+        "-r",
+        String(dpi),
+        "-f",
+        String(page),
+        "-l",
+        String(page),
+        pdfPath,
+        prefixPath
+      ]);
+      const files = await readdir(imagesDir);
+      const pageFiles = files.filter((f) => f.endsWith(".png")).sort((a, b) => naturalPageSort(a, b));
+      const imagePath = join4(imagesDir, pageFiles[pageFiles.length - 1]);
+      yield { pageNumber: page, imagePath };
     } catch (err) {
-      throw new UnifiedOcrError("RENDER_FAILED", "render", err instanceof Error ? err.message : String(err));
+      yield {
+        pageNumber: page,
+        imagePath: null,
+        error: err instanceof Error ? err : new Error(String(err))
+      };
     }
   }
-  await renderPdfToPng(pdfPath, prefixPath, dpi);
-  return { emittedPerPageProgress: false, pageCountSource: "fallback" };
 }
 async function runCommand(cmd, args) {
   await new Promise((resolvePromise, reject) => {
@@ -11624,26 +11695,11 @@ async function assertSofficeAvailable() {
     throw new UnifiedOcrError("SOFFICE_NOT_FOUND", "convert", "soffice\uB97C \uCC3E\uC744 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4. LibreOffice\uB97C \uC124\uCE58\uD574 \uC8FC\uC138\uC694.");
   }
 }
-async function listPageImages(imagesDir) {
-  const files = await readdir(imagesDir);
-  return files.filter((f) => f.endsWith(".png")).sort((a, b) => naturalPageSort(a, b)).map((f) => join4(imagesDir, f));
-}
 function naturalPageSort(a, b) {
   const na = Number((a.match(/\d+/g) || ["0"]).at(-1) || 0);
   const nb = Number((b.match(/\d+/g) || ["0"]).at(-1) || 0);
   return na - nb;
 }
-async function pickRepresentativeImage(images) {
-  const sample = images.slice(0, Math.min(images.length, 8));
-  const weighted = [];
-  for (const p of sample) {
-    const st = await stat(p);
-    if (st.size > 8 * 1024) weighted.push({ path: p, size: st.size });
-  }
-  const use = weighted.length > 0 ? weighted : await Promise.all(sample.map(async (p) => ({ path: p, size: (await stat(p)).size })));
-  use.sort((a, b) => a.size - b.size);
-  return use[Math.floor(use.length / 2)].path;
-}
 async function mapWithConcurrency(items, concurrency, mapper) {
   const results = new Array(items.length);
   let nextIndex = 0;
@@ -11745,11 +11801,43 @@ async function updateModelCache(path, probes) {
   current.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
   await writeFile(path, JSON.stringify(current, null, 2), "utf-8");
 }
+async function ocrWorkerPool(input) {
+  const { queue, workerCount, ocrInput, onPageDone } = input;
+  const results = /* @__PURE__ */ new Map();
+  let completedCount = 0;
+  async function worker() {
+    while (true) {
+      const item = await queue.dequeue();
+      if (item === QUEUE_DONE) break;
+      const { pageNumber, imagePath, error } = item;
+      if (imagePath === null) {
+        input.logger?.log({
+          level: "warn",
+          stage: "ocr",
+          event: "message",
+          message: `\uD398\uC774\uC9C0 ${pageNumber} \uB80C\uB354 \uC2E4\uD328 \u2014 \uBE48 \uD398\uC774\uC9C0\uB85C \uCC98\uB9AC`,
+          meta: { error: String(error) }
+        });
+        results.set(pageNumber, "");
+        completedCount++;
+        onPageDone(pageNumber, completedCount, "");
+        continue;
+      }
+      const { markdown, model } = await ocrImageWithFallback({ ...ocrInput, imagePath });
+      results.set(pageNumber, markdown);
+      completedCount++;
+      onPageDone(pageNumber, completedCount, model);
+    }
+  }
+  const workers = Array.from({ length: workerCount }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
 async function ocrImageWithFallback(input) {
   let lastErr = "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958";
   for (const model of input.models) {
     try {
-      return await ocrImageViaNim({
+      const markdown = await ocrImageViaNim({
         imagePath: input.imagePath,
         prompt: input.prompt,
         model,
@@ -11761,6 +11849,7 @@ async function ocrImageWithFallback(input) {
         logger: input.logger,
         stage: "ocr"
       });
+      return { markdown, model };
     } catch (err) {
       lastErr = err instanceof Error ? err.message : String(err);
     }