@clazic/kordoc 2.4.19 → 2.5.1

package/dist/cli.js

@@ -4,11 +4,11 @@ import {
   markdownToHwpx,
   markdownToXlsx,
   parse
-} from "./chunk-463YQ2WL.js";
+} from "./chunk-QG6BYZMR.js";
 import {
   VERSION,
   toArrayBuffer
-} from "./chunk-MZN7PLTZ.js";
+} from "./chunk-IJGNPAK2.js";
 import "./chunk-MOL7MDBG.js";
 import "./chunk-Y4WFKJ5P.js";
 import "./chunk-YW5G6BCJ.js";
@@ -173,7 +173,7 @@ async function runParse(files, opts) {
         saveImages(absPath);
       }
     } catch (err) {
-      const { sanitizeError } = await import("./utils-YUAT7LFD.js");
+      const { sanitizeError } = await import("./utils-RBXHHCLI.js");
       process.stderr.write(`
 [kordoc] ERROR: ${fileName} \u2014 ${sanitizeError(err)}
 `);
@@ -255,7 +255,7 @@ program.command("convert <input>").description("\uB9C8\uD06C\uB2E4\uC6B4 \uD30C\
 `));
     }
   } catch (err) {
-    const { sanitizeError } = await import("./utils-YUAT7LFD.js");
+    const { sanitizeError } = await import("./utils-RBXHHCLI.js");
     process.stderr.write(` FAIL
 `);
     process.stderr.write(`  \u2192 ${sanitizeError(err)}
@@ -287,7 +287,7 @@ program.command("init-env").description("kordoc\uC6A9 .env \uD15C\uD50C\uB9BF \u
   }
 });
 program.command("watch <dir>").description("\uB514\uB809\uD1A0\uB9AC \uAC10\uC2DC \u2014 \uC0C8 \uBB38\uC11C \uC790\uB3D9 \uBCC0\uD658").option("--webhook <url>", "\uACB0\uACFC \uC804\uC1A1 \uC6F9\uD6C5 URL").option("-d, --out-dir <dir>", "\uBCC0\uD658 \uACB0\uACFC \uCD9C\uB825 \uB514\uB809\uD1A0\uB9AC").option("-p, --pages <range>", "\uD398\uC774\uC9C0/\uC139\uC158 \uBC94\uC704").option("--format <type>", "\uCD9C\uB825 \uD615\uC2DD: markdown \uB610\uB294 json", "markdown").option("--silent", "\uC9C4\uD589 \uBA54\uC2DC\uC9C0 \uC228\uAE30\uAE30").action(async (dir, opts) => {
-  const { watchDirectory } = await import("./watch-WEOFVVDO.js");
+  const { watchDirectory } = await import("./watch-5CCMTZ7F.js");
   await watchDirectory({
     dir,
     outDir: opts.outDir,

package/dist/index.cjs

@@ -3057,7 +3057,7 @@ var import_jszip2 = __toESM(require("jszip"), 1);
 var import_xmldom = require("@xmldom/xmldom");
 
 // src/utils.ts
-var VERSION = true ? "2.4.19" : "0.0.0-dev";
+var VERSION = true ? "2.5.0" : "0.0.0-dev";
 function toArrayBuffer(buf) {
   if (buf.byteOffset === 0 && buf.byteLength === buf.buffer.byteLength) {
     return buf.buffer;
@@ -11253,6 +11253,66 @@ var import_child_process4 = require("child_process");
 var import_node_perf_hooks = require("perf_hooks");
 var import_libreoffice_convert = __toESM(require("libreoffice-convert"), 1);
 init_logger();
+
+// src/pipeline/bounded-queue.ts
+var QUEUE_DONE = /* @__PURE__ */ Symbol("QUEUE_DONE");
+var BoundedQueue = class {
+  buffer = [];
+  capacity;
+  closed = false;
+  producerWaiters = [];
+  consumerWaiters = [];
+  constructor(capacity) {
+    if (capacity < 1) throw new RangeError("BoundedQueue capacity must be >= 1");
+    this.capacity = capacity;
+  }
+  async enqueue(item) {
+    if (this.closed) throw new Error("BoundedQueue: cannot enqueue after close()");
+    if (this.consumerWaiters.length > 0) {
+      const resolve4 = this.consumerWaiters.shift();
+      resolve4(item);
+      return;
+    }
+    while (this.buffer.length >= this.capacity) {
+      await new Promise((resolve4) => this.producerWaiters.push(resolve4));
+      if (this.closed) throw new Error("BoundedQueue: closed while waiting to enqueue");
+    }
+    this.buffer.push(item);
+  }
+  async dequeue() {
+    if (this.buffer.length > 0) {
+      const item = this.buffer.shift();
+      this._wakeProducer();
+      return item;
+    }
+    if (this.closed) return QUEUE_DONE;
+    return new Promise((resolve4) => {
+      this.consumerWaiters.push(resolve4);
+    });
+  }
+  close() {
+    if (this.closed) return;
+    this.closed = true;
+    for (const resolve4 of this.consumerWaiters) {
+      resolve4(QUEUE_DONE);
+    }
+    this.consumerWaiters = [];
+    for (const wake of this.producerWaiters) {
+      wake();
+    }
+    this.producerWaiters = [];
+  }
+  get size() {
+    return this.buffer.length;
+  }
+  _wakeProducer() {
+    if (this.producerWaiters.length > 0) {
+      this.producerWaiters.shift()();
+    }
+  }
+};
+
+// src/pipeline/unified-ocr.ts
 var libreConvert = import_libreoffice_convert.default.convert;
 var UnifiedOcrError = class extends Error {
   code;
@@ -11284,11 +11344,11 @@ var DEFAULT_MODEL_MAX_TOKENS = {
 };
 var DEFAULT_STAGE_WEIGHTS = {
   convert: 15,
-  render: 20,
+  render: 15,
   probe: 5,
-  ocr: 45,
+  ocr: 55,
   proofread: 0,
-  merge: 5
+  merge: 10
 };
 var OCR_PROMPT2 = [
   "\uC774 PDF \uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0\uC5D0\uC11C \uD14D\uC2A4\uD2B8\uC640 \uD45C\uB97C \uCD94\uCD9C\uD558\uC5EC Markdown\uC73C\uB85C \uBCC0\uD658\uD558\uACE0, OCR \uC624\uC778\uC2DD \uC624\uB958\uB97C \uC989\uC2DC \uAD50\uC815\uD558\uC5EC \uCD5C\uC885 \uACB0\uACFC\uBB3C\uC744 \uCD9C\uB825\uD558\uC138\uC694.",
@@ -11377,39 +11437,19 @@ async function runUnifiedOcrPipeline(inputPath, options = {}) {
     logStage("info", "convert", "done", "PDF \uBCC0\uD658 \uC644\uB8CC", { elapsedMs: timingsMs.convert });
     const renderStart = import_node_perf_hooks.performance.now();
     currentStage = "render";
+    const totalPages = await getPdfPageCount(workingPdfPath).catch(() => 0);
+    if (totalPages === 0) throw new UnifiedOcrError("RENDER_FAILED", "render", "\uD398\uC774\uC9C0 \uC218\uB97C \uD655\uC778\uD560 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4.");
     markStageStart("render", "PDF \uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC911");
-    logStage("info", "render", "start", "PDF \uD398\uC774\uC9C0 \uB80C\uB354\uB9C1 \uC2DC\uC791", { pdf: workingPdfPath, dpi });
-    const renderWithProgress = await renderPdfToPngWithProgress(
-      workingPdfPath,
-      (0, import_path5.join)(imagesDir, "page"),
-      dpi,
-      (current, total) => {
-        markStageProgress(
-          "render",
-          Math.round(current / total * 100),
-          current,
-          total,
-          `\uD398\uC774\uC9C0 ${current}/${total} \uB80C\uB354\uB9C1`
-        );
-      }
-    );
-    const images = await listPageImages(imagesDir);
-    if (images.length === 0) throw new UnifiedOcrError("RENDER_FAILED", "render", "\uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC2E4\uD328: \uACB0\uACFC \uC774\uBBF8\uC9C0\uAC00 \uC5C6\uC2B5\uB2C8\uB2E4.");
-    if (!renderWithProgress.emittedPerPageProgress) {
-      markStageProgress("render", 100, images.length, images.length, `\uD398\uC774\uC9C0 ${images.length}\uC7A5 \uC0DD\uC131`);
-    }
-    timingsMs.render = elapsedMs(renderStart);
-    markStageDone("render", "\uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC");
-    logStage("info", "render", "done", "\uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC", {
-      pages: images.length,
-      elapsedMs: timingsMs.render,
-      pageCountSource: renderWithProgress.pageCountSource
-    });
+    logStage("info", "render", "start", "PDF \uD398\uC774\uC9C0 \uB80C\uB354\uB9C1 \uC2DC\uC791", { pdf: workingPdfPath, dpi, totalPages });
+    await runCommand("pdftoppm", ["-png", "-r", String(dpi), "-f", "1", "-l", "1", workingPdfPath, (0, import_path5.join)(imagesDir, "page")]);
+    const firstFiles = (await (0, import_promises2.readdir)(imagesDir)).filter((f) => f.endsWith(".png")).sort((a, b) => naturalPageSort(a, b));
+    if (firstFiles.length === 0) throw new UnifiedOcrError("RENDER_FAILED", "render", "\uCCAB \uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC2E4\uD328");
+    const probeImage = (0, import_path5.join)(imagesDir, firstFiles[0]);
+    markStageProgress("render", Math.round(1 / totalPages * 100), 1, totalPages, `\uD398\uC774\uC9C0 1/${totalPages} \uB80C\uB354\uB9C1`);
     const probeStart = import_node_perf_hooks.performance.now();
     currentStage = "probe";
     markStageStart("probe", "\uBAA8\uB378 \uC18D\uB3C4 \uD504\uB85C\uBE0C \uC218\uD589 \uC911");
     logStage("info", "probe", "start", "\uBAA8\uB378 \uC18D\uB3C4 \uD504\uB85C\uBE0C \uC2DC\uC791", { models, probeConcurrency });
-    const probeImage = await pickRepresentativeImage(images);
     let probeDone = 0;
     const probeRuns = startParallelProbeRuns({
       models,
@@ -11436,33 +11476,74 @@ async function runUnifiedOcrPipeline(inputPath, options = {}) {
       await updateModelCache(modelCachePath, results);
       return results;
     });
+    const concurrencyPerKey = Math.max(1, options.concurrencyPerKey ?? 2);
+    const keyCount = keyPool.snapshot().length;
+    const workerCount = Math.max(1, keyCount * concurrencyPerKey);
+    const queueCapacity = workerCount * 2;
+    const queue = new BoundedQueue(queueCapacity);
     const ocrStart = import_node_perf_hooks.performance.now();
     currentStage = "ocr";
-    markStageStart("ocr", `OCR \uC9C4\uD589 \uC911 (${selectedModel})`);
-    logStage("info", "ocr", "start", "\uD398\uC774\uC9C0 OCR \uC2DC\uC791", { selectedModel, pageCount: images.length });
-    const rawPagePaths = [];
-    for (let i = 0; i < images.length; i++) {
-      const imagePath = images[i];
-      const markdown = await ocrImageWithFallback({
-        imagePath,
-        prompt: OCR_PROMPT2,
-        models: fallbackModelOrder,
-        modelMaxTokens,
-        baseUrl,
-        keyPool,
-        timeoutMs,
-        maxRetriesPerPage,
+    markStageStart("ocr", `OCR \uC9C4\uD589 \uC911 (\uC6CC\uCEE4 ${workerCount}\uAC1C)`);
+    logStage("info", "ocr", "start", "\uD398\uC774\uC9C0 OCR \uC2DC\uC791", { workerCount, keyCount, pageCount: totalPages });
+    let renderDone = 1;
+    const renderProducer = (async () => {
+      try {
+        await queue.enqueue({ pageNumber: 1, imagePath: probeImage });
+        if (totalPages > 1) {
+          for await (const item of renderPdfToPngStream(workingPdfPath, (0, import_path5.join)(imagesDir, "page"), dpi, totalPages, 2)) {
+            await queue.enqueue(item);
+            renderDone++;
+            markStageProgress("render", Math.round(renderDone / totalPages * 100), renderDone, totalPages, `\uD398\uC774\uC9C0 ${renderDone}/${totalPages} \uB80C\uB354\uB9C1`);
+            logStage("debug", "render", "progress", "\uD398\uC774\uC9C0 \uB80C\uB354 \uC644\uB8CC", { page: item.pageNumber });
+          }
+        }
+      } finally {
+        queue.close();
+        timingsMs.render = elapsedMs(renderStart);
+        markStageDone("render", "\uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC");
+        logStage("info", "render", "done", "\uD398\uC774\uC9C0 \uC774\uBBF8\uC9C0 \uC0DD\uC131 \uC644\uB8CC", { pages: renderDone, elapsedMs: timingsMs.render });
+      }
+    })();
+    const [, pageResultsMap] = await Promise.all([
+      renderProducer,
+      ocrWorkerPool({
+        queue,
+        workerCount,
+        totalPages,
+        ocrInput: {
+          prompt: OCR_PROMPT2,
+          models: fallbackModelOrder,
+          modelMaxTokens,
+          baseUrl,
+          keyPool,
+          timeoutMs,
+          maxRetriesPerPage,
+          logger
+        },
+        onPageDone: (pageNumber, completedCount, model) => {
+          markStageProgress(
+            "ocr",
+            Math.round(completedCount / totalPages * 100),
+            completedCount,
+            totalPages,
+            `OCR ${completedCount}/${totalPages}`,
+            model || void 0
+          );
+          logStage("debug", "ocr", "progress", "\uD398\uC774\uC9C0 OCR \uC644\uB8CC", { page: pageNumber, total: totalPages, model });
+        },
         logger
-      });
-      const pagePath = (0, import_path5.join)(rawDir, `page_${String(i + 1).padStart(4, "0")}.md`);
-      await (0, import_promises2.writeFile)(pagePath, markdown, "utf-8");
-      rawPagePaths.push(pagePath);
-      markStageProgress("ocr", Math.round((i + 1) / images.length * 100), i + 1, images.length, `OCR ${i + 1}/${images.length}`, selectedModel);
-      logStage("debug", "ocr", "progress", "\uD398\uC774\uC9C0 OCR \uC644\uB8CC", { page: i + 1, total: images.length });
-    }
+      })
+    ]);
     timingsMs.ocr = elapsedMs(ocrStart);
     markStageDone("ocr", "OCR \uC644\uB8CC");
     logStage("info", "ocr", "done", "\uD398\uC774\uC9C0 OCR \uC644\uB8CC", { elapsedMs: timingsMs.ocr });
+    const sortedEntries = Array.from(pageResultsMap.entries()).sort((a, b) => a[0] - b[0]);
+    const rawPagePaths = [];
+    for (const [pageNum, markdown] of sortedEntries) {
+      const pagePath = (0, import_path5.join)(rawDir, `page_${String(pageNum).padStart(4, "0")}.md`);
+      await (0, import_promises2.writeFile)(pagePath, markdown, "utf-8");
+      rawPagePaths.push(pagePath);
+    }
     const mergeStart = import_node_perf_hooks.performance.now();
     currentStage = "merge";
     markStageStart("merge", "\uCD5C\uC885 Markdown \uBCD1\uD569 \uC911");
@@ -11479,7 +11560,7 @@ async function runUnifiedOcrPipeline(inputPath, options = {}) {
       selectedModel,
       probeImage,
       probeResults: await probeResultsPromise,
-      pageCount: images.length,
+      pageCount: totalPages,
       keyHealth: keyPool.snapshot(),
       timingsMs,
       modelCachePath
@@ -11557,13 +11638,6 @@ async function convertWithLibreOffice(buffer, ext) {
     });
   });
 }
-async function renderPdfToPng(pdfPath, prefixPath, dpi) {
-  try {
-    await runCommand("pdftoppm", ["-png", "-r", String(dpi), pdfPath, prefixPath]);
-  } catch (err) {
-    throw new UnifiedOcrError("RENDER_FAILED", "render", err instanceof Error ? err.message : String(err));
-  }
-}
 async function getPdfPageCount(pdfPath) {
   const stdout = await runCommandWithStdout("pdfinfo", [pdfPath]);
   const m = stdout.match(/^\s*Pages:\s*(\d+)\s*$/mi);
@@ -11576,36 +11650,33 @@ async function getPdfPageCount(pdfPath) {
   }
   return n;
 }
-async function renderPdfToPngWithProgress(pdfPath, prefixPath, dpi, onPageDone) {
-  let totalPages = 0;
-  try {
-    totalPages = await getPdfPageCount(pdfPath);
-  } catch {
-    totalPages = 0;
-  }
-  if (totalPages > 0) {
+async function* renderPdfToPngStream(pdfPath, prefixPath, dpi, totalPages, startPage = 1) {
+  const imagesDir = (0, import_path5.dirname)(prefixPath);
+  for (let page = startPage; page <= totalPages; page++) {
     try {
-      for (let page = 1; page <= totalPages; page++) {
-        await runCommand("pdftoppm", [
-          "-png",
-          "-r",
-          String(dpi),
-          "-f",
-          String(page),
-          "-l",
-          String(page),
-          pdfPath,
-          prefixPath
-        ]);
-        onPageDone(page, totalPages);
-      }
-      return { emittedPerPageProgress: true, pageCountSource: "pdfinfo" };
+      await runCommand("pdftoppm", [
+        "-png",
+        "-r",
+        String(dpi),
+        "-f",
+        String(page),
+        "-l",
+        String(page),
+        pdfPath,
+        prefixPath
+      ]);
+      const files = await (0, import_promises2.readdir)(imagesDir);
+      const pageFiles = files.filter((f) => f.endsWith(".png")).sort((a, b) => naturalPageSort(a, b));
+      const imagePath = (0, import_path5.join)(imagesDir, pageFiles[pageFiles.length - 1]);
+      yield { pageNumber: page, imagePath };
     } catch (err) {
-      throw new UnifiedOcrError("RENDER_FAILED", "render", err instanceof Error ? err.message : String(err));
+      yield {
+        pageNumber: page,
+        imagePath: null,
+        error: err instanceof Error ? err : new Error(String(err))
+      };
     }
   }
-  await renderPdfToPng(pdfPath, prefixPath, dpi);
-  return { emittedPerPageProgress: false, pageCountSource: "fallback" };
 }
 async function runCommand(cmd, args) {
   await new Promise((resolvePromise, reject) => {
@@ -11646,26 +11717,11 @@ async function assertSofficeAvailable() {
     throw new UnifiedOcrError("SOFFICE_NOT_FOUND", "convert", "soffice\uB97C \uCC3E\uC744 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4. LibreOffice\uB97C \uC124\uCE58\uD574 \uC8FC\uC138\uC694.");
   }
 }
-async function listPageImages(imagesDir) {
-  const files = await (0, import_promises2.readdir)(imagesDir);
-  return files.filter((f) => f.endsWith(".png")).sort((a, b) => naturalPageSort(a, b)).map((f) => (0, import_path5.join)(imagesDir, f));
-}
 function naturalPageSort(a, b) {
   const na = Number((a.match(/\d+/g) || ["0"]).at(-1) || 0);
   const nb = Number((b.match(/\d+/g) || ["0"]).at(-1) || 0);
   return na - nb;
 }
-async function pickRepresentativeImage(images) {
-  const sample = images.slice(0, Math.min(images.length, 8));
-  const weighted = [];
-  for (const p of sample) {
-    const st = await (0, import_promises2.stat)(p);
-    if (st.size > 8 * 1024) weighted.push({ path: p, size: st.size });
-  }
-  const use = weighted.length > 0 ? weighted : await Promise.all(sample.map(async (p) => ({ path: p, size: (await (0, import_promises2.stat)(p)).size })));
-  use.sort((a, b) => a.size - b.size);
-  return use[Math.floor(use.length / 2)].path;
-}
 async function mapWithConcurrency(items, concurrency, mapper) {
   const results = new Array(items.length);
   let nextIndex = 0;
@@ -11767,11 +11823,43 @@ async function updateModelCache(path, probes) {
   current.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
   await (0, import_promises2.writeFile)(path, JSON.stringify(current, null, 2), "utf-8");
 }
+async function ocrWorkerPool(input) {
+  const { queue, workerCount, ocrInput, onPageDone } = input;
+  const results = /* @__PURE__ */ new Map();
+  let completedCount = 0;
+  async function worker() {
+    while (true) {
+      const item = await queue.dequeue();
+      if (item === QUEUE_DONE) break;
+      const { pageNumber, imagePath, error } = item;
+      if (imagePath === null) {
+        input.logger?.log({
+          level: "warn",
+          stage: "ocr",
+          event: "message",
+          message: `\uD398\uC774\uC9C0 ${pageNumber} \uB80C\uB354 \uC2E4\uD328 \u2014 \uBE48 \uD398\uC774\uC9C0\uB85C \uCC98\uB9AC`,
+          meta: { error: String(error) }
+        });
+        results.set(pageNumber, "");
+        completedCount++;
+        onPageDone(pageNumber, completedCount, "");
+        continue;
+      }
+      const { markdown, model } = await ocrImageWithFallback({ ...ocrInput, imagePath });
+      results.set(pageNumber, markdown);
+      completedCount++;
+      onPageDone(pageNumber, completedCount, model);
+    }
+  }
+  const workers = Array.from({ length: workerCount }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
 async function ocrImageWithFallback(input) {
   let lastErr = "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958";
   for (const model of input.models) {
     try {
-      return await ocrImageViaNim({
+      const markdown = await ocrImageViaNim({
         imagePath: input.imagePath,
         prompt: input.prompt,
         model,
@@ -11783,6 +11871,7 @@ async function ocrImageWithFallback(input) {
         logger: input.logger,
         stage: "ocr"
       });
+      return { markdown, model };
     } catch (err) {
       lastErr = err instanceof Error ? err.message : String(err);
     }