@stream-mdx/worker 0.2.0 → 0.4.0

package/dist/node/index.mjs

@@ -1,13 +1,45 @@
 // src/node/index.ts
+import { createHash } from "crypto";
+import { existsSync } from "fs";
+import { promises as fs } from "fs";
+import { createRequire } from "module";
 import path from "path";
-import { pathToFileURL } from "url";
+import { fileURLToPath, pathToFileURL } from "url";
 import { Worker } from "worker_threads";
+import { PATCH_ROOT_ID, applyPatchBatch, createInitialSnapshot } from "@stream-mdx/core";
 function getHostedWorkerBundleUrl() {
-  return new URL("../hosted/markdown-worker.js", getModuleUrl());
+  const packageRoot = resolvePackageRootFromRequire();
+  if (packageRoot) {
+    const resolved = firstExistingPath([
+      path.join(packageRoot, "dist/hosted/markdown-worker.js"),
+      path.join(packageRoot, "dist/worker.mjs"),
+      path.join(packageRoot, "dist/worker.js"),
+      path.join(packageRoot, "src/worker.ts")
+    ]);
+    if (resolved) {
+      return pathToFileURL(resolved);
+    }
+  }
+  const moduleUrl = getModuleUrl();
+  const candidates = [
+    new URL("../hosted/markdown-worker.js", moduleUrl),
+    new URL("../../dist/hosted/markdown-worker.js", moduleUrl),
+    new URL("../../../public/workers/markdown-worker.js", moduleUrl),
+    new URL("../../../../public/workers/markdown-worker.js", moduleUrl),
+    new URL("../../dist/worker.mjs", moduleUrl),
+    new URL("../../dist/worker.js", moduleUrl),
+    new URL("../worker.mjs", moduleUrl),
+    new URL("../worker.js", moduleUrl),
+    new URL("../worker.ts", moduleUrl)
+  ];
+  for (const candidate of candidates) {
+    if (urlExists(candidate)) return candidate;
+  }
+  return candidates[0];
 }
 function createWorkerThread(options = {}) {
   const { workerBundle, workerData, ...workerOptions } = options;
-  const runnerUrl = new URL("./worker-thread-entry.mjs", getModuleUrl());
+  const runnerUrl = resolveWorkerThreadEntryUrl();
   const bundleUrl = normalizeWorkerBundleUrl(workerBundle) ?? getHostedWorkerBundleUrl();
   return new Worker(runnerUrl, {
     ...workerOptions,
@@ -17,6 +49,37 @@ function createWorkerThread(options = {}) {
     }
   });
 }
+function resolveWorkerThreadEntryUrl() {
+  const packageRoot = resolvePackageRootFromRequire();
+  if (packageRoot) {
+    const resolved = firstExistingPath([
+      path.join(packageRoot, "dist/node/worker-thread-entry.mjs"),
+      path.join(packageRoot, "dist/node/worker-thread-entry.cjs"),
+      path.join(packageRoot, "src/node/worker-thread-entry.ts")
+    ]);
+    if (resolved) {
+      return pathToFileURL(resolved);
+    }
+  }
+  const moduleUrl = getModuleUrl();
+  const distEntry = new URL("./worker-thread-entry.mjs", moduleUrl);
+  const sourceEntry = new URL("./worker-thread-entry.ts", moduleUrl);
+  if (urlExists(distEntry)) {
+    return distEntry;
+  }
+  if (urlExists(sourceEntry)) {
+    return sourceEntry;
+  }
+  return distEntry;
+}
+function urlExists(url) {
+  if (url.protocol !== "file:") return false;
+  try {
+    return existsSync(path.normalize(fileURLToPath(url)));
+  } catch {
+    return false;
+  }
+}
 function normalizeWorkerBundleUrl(value) {
   if (!value) return void 0;
   if (value instanceof URL) return value;
@@ -26,13 +89,327 @@ function normalizeWorkerBundleUrl(value) {
     return pathToFileURL(path.resolve(value));
   }
 }
+function firstExistingPath(candidates) {
+  for (const candidate of candidates) {
+    try {
+      if (existsSync(candidate)) return candidate;
+    } catch {
+    }
+  }
+  return void 0;
+}
+function resolvePackageRootFromRequire() {
+  const bases = [
+    path.join(process.cwd(), "package.json"),
+    typeof __filename === "string" && __filename.length > 0 ? __filename : path.join(process.cwd(), "__stream-mdx-node-resolver__.cjs")
+  ];
+  for (const base of bases) {
+    try {
+      const req = createRequire(base);
+      const nodeEntry = req.resolve("@stream-mdx/worker/node");
+      return path.resolve(path.dirname(nodeEntry), "..", "..");
+    } catch {
+    }
+  }
+  return void 0;
+}
 function getModuleUrl() {
+  const fromImportMeta = getImportMetaUrl();
+  if (fromImportMeta) {
+    return fromImportMeta;
+  }
   if (typeof __filename === "string" && __filename.length > 0) {
     return pathToFileURL(__filename).href;
   }
-  return import.meta.url;
+  const fromStack = getModuleUrlFromStack();
+  if (fromStack) {
+    return fromStack;
+  }
+  throw new Error("[stream-mdx] Unable to resolve module URL.");
+}
+function getImportMetaUrl() {
+  try {
+    const candidate = (0, eval)("import.meta.url");
+    return typeof candidate === "string" && candidate.length > 0 ? candidate : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function getModuleUrlFromStack() {
+  const previous = Error.prepareStackTrace;
+  try {
+    Error.prepareStackTrace = (_error, stackTrace2) => stackTrace2;
+    const stackTrace = new Error().stack;
+    if (!Array.isArray(stackTrace)) return void 0;
+    const files = stackTrace.map((frame) => frame.getFileName()).filter((fileName) => Boolean(fileName));
+    const preferred = files.find(
+      (fileName) => fileName.includes("/markdown-v2-worker/src/node/index.") || fileName.includes("/markdown-v2-worker/dist/node/index.") || fileName.includes("\\markdown-v2-worker\\src\\node\\index.") || fileName.includes("\\markdown-v2-worker\\dist\\node\\index.")
+    );
+    if (preferred) {
+      return preferred.startsWith("file://") ? preferred : pathToFileURL(preferred).href;
+    }
+    const firstAbsolute = files.find((fileName) => fileName.startsWith("file://") || path.isAbsolute(fileName));
+    if (!firstAbsolute) return void 0;
+    return firstAbsolute.startsWith("file://") ? firstAbsolute : pathToFileURL(firstAbsolute).href;
+  } catch {
+    return void 0;
+  } finally {
+    Error.prepareStackTrace = previous;
+  }
+}
+function computeSnapshotHash(text, init, hashSalt) {
+  return hashCompileInput(text, init, hashSalt);
+}
+async function compileMarkdownSnapshot(options) {
+  const {
+    text,
+    init,
+    hashSalt,
+    worker: providedWorker,
+    workerOptions,
+    timeoutMs = 3e4,
+    settleMs = 50,
+    cache,
+    finalize = true
+  } = options;
+  const hash = hashCompileInput(text, init, hashSalt);
+  const cacheKey = cache?.key ?? hash;
+  const cachePath = cache ? path.join(cache.dir, `${sanitizeCacheKey(cacheKey)}.json`) : null;
+  if (cachePath) {
+    const cached = await readSnapshotCache(cachePath, hash);
+    if (cached) {
+      const snapshot = createInitialSnapshot(cached.blocks);
+      return {
+        blocks: snapshot.blocks,
+        snapshot,
+        artifact: cached,
+        fromCache: true
+      };
+    }
+  }
+  const worker = providedWorker ?? createWorkerThread(workerOptions);
+  const ownedWorker = !providedWorker;
+  return await new Promise((resolve, reject) => {
+    let snapshot = null;
+    let initialized = false;
+    let settled = false;
+    let idleTimer = null;
+    let timeoutTimer = null;
+    const finalizeNow = () => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      const finalSnapshot = snapshot ?? createInitialSnapshot();
+      const blocks = finalSnapshot.blocks;
+      const contentHash = createHash("sha256").update(text).digest("hex");
+      const configHash = createHash("sha256").update(
+        stableStringify({
+          init: init ?? null,
+          salt: hashSalt ?? null
+        })
+      ).digest("hex");
+      const artifact = {
+        version: 1,
+        schemaId: "streammdx.snapshot.v1",
+        createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+        hash,
+        contentHash,
+        configHash,
+        hashSalt: hashSalt ?? void 0,
+        blocks,
+        tocHeadings: (() => {
+          const root = finalSnapshot.nodes.get(PATCH_ROOT_ID);
+          const maybe = root?.props?.tocHeadings;
+          return Array.isArray(maybe) ? maybe : void 0;
+        })(),
+        init: init ? {
+          docPlugins: init.docPlugins,
+          mdx: init.mdx,
+          prewarmLangs: init.prewarmLangs
+        } : void 0
+      };
+      if (cachePath && !cache?.readOnly) {
+        void writeSnapshotCache(cachePath, artifact);
+      }
+      resolve({
+        blocks,
+        snapshot: finalSnapshot,
+        artifact,
+        fromCache: false
+      });
+    };
+    const fail = (error) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(error);
+    };
+    const scheduleIdleFinalize = () => {
+      if (finalize || !initialized) return;
+      if (idleTimer) {
+        clearTimeout(idleTimer);
+      }
+      idleTimer = setTimeout(() => finalizeNow(), settleMs);
+    };
+    const handleMessage = (message) => {
+      switch (message.type) {
+        case "INITIALIZED":
+          snapshot = createInitialSnapshot(message.blocks);
+          initialized = true;
+          scheduleIdleFinalize();
+          break;
+        case "PATCH":
+          if (snapshot) {
+            snapshot.blocks = applyPatchBatch(snapshot, message.patches);
+          }
+          scheduleIdleFinalize();
+          break;
+        case "FINALIZED":
+          if (finalize) {
+            finalizeNow();
+          }
+          break;
+        case "RESET":
+          fail(new Error(`Worker reset during compile: ${message.reason}`));
+          break;
+        case "ERROR":
+          fail(new Error(`Worker error (${message.phase}): ${message.error.message}`));
+          break;
+        default:
+          break;
+      }
+    };
+    const cleanup = () => {
+      if (idleTimer) clearTimeout(idleTimer);
+      if (timeoutTimer) clearTimeout(timeoutTimer);
+      worker.off("message", handleMessage);
+      worker.off("error", fail);
+      if (ownedWorker) {
+        try {
+          worker.terminate();
+        } catch {
+        }
+      }
+    };
+    worker.on("message", handleMessage);
+    worker.on("error", fail);
+    timeoutTimer = setTimeout(() => {
+      fail(new Error("Worker compile timed out."));
+    }, timeoutMs);
+    const initMessage = {
+      type: "INIT",
+      initialContent: text,
+      prewarmLangs: init?.prewarmLangs,
+      docPlugins: init?.docPlugins,
+      mdx: init?.mdx
+    };
+    worker.postMessage(initMessage);
+    if (finalize) {
+      worker.postMessage({ type: "FINALIZE" });
+    }
+  });
+}
+function createCompileMarkdownSnapshotPool(options = {}) {
+  const size = Math.max(1, Math.min(8, Math.floor(options.size ?? 2)));
+  const workers = new Array(size).fill(null).map(() => createWorkerThread(options.workerOptions));
+  const queues = new Array(size).fill(Promise.resolve());
+  let rr = 0;
+  const maxMemoryEntries = Math.max(0, Math.floor(options.maxMemoryEntries ?? 64));
+  const memory = maxMemoryEntries > 0 ? /* @__PURE__ */ new Map() : null;
+  const enqueue = (index, fn) => {
+    let resolveOuter;
+    let rejectOuter;
+    const outer = new Promise((resolve, reject) => {
+      resolveOuter = resolve;
+      rejectOuter = reject;
+    });
+    queues[index] = queues[index].catch(() => void 0).then(async () => {
+      try {
+        const value = await fn();
+        resolveOuter(value);
+      } catch (err) {
+        rejectOuter(err);
+      }
+    }).then(() => void 0);
+    return outer;
+  };
+  const compile = async (compileOptions) => {
+    const hash = computeSnapshotHash(compileOptions.text, compileOptions.init, compileOptions.hashSalt);
+    if (memory && memory.has(hash)) {
+      return memory.get(hash);
+    }
+    const index = rr++ % workers.length;
+    const result = await enqueue(index, async () => {
+      return await compileMarkdownSnapshot({
+        ...compileOptions,
+        worker: workers[index]
+      });
+    });
+    if (memory) {
+      memory.set(hash, result);
+      if (memory.size > maxMemoryEntries) {
+        const firstKey = memory.keys().next().value;
+        if (firstKey) memory.delete(firstKey);
+      }
+    }
+    return result;
+  };
+  const close = async () => {
+    for (const worker of workers) {
+      try {
+        await worker.terminate();
+      } catch {
+      }
+    }
+  };
+  return { compile, close };
+}
+async function readSnapshotCache(cachePath, hash) {
+  try {
+    const raw = await fs.readFile(cachePath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (!parsed || parsed.version !== 1 || parsed.hash !== hash || !Array.isArray(parsed.blocks)) {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+async function writeSnapshotCache(cachePath, artifact) {
+  try {
+    await fs.mkdir(path.dirname(cachePath), { recursive: true });
+    await fs.writeFile(cachePath, JSON.stringify(artifact, null, 2), "utf8");
+  } catch {
+  }
+}
+function sanitizeCacheKey(value) {
+  return value.replace(/[^a-z0-9._-]+/gi, "_");
+}
+function hashCompileInput(text, init, hashSalt) {
+  const payload = stableStringify({
+    text,
+    init: init ?? null,
+    salt: hashSalt ?? null
+  });
+  return createHash("sha256").update(payload).digest("hex");
+}
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") {
+    const primitive = JSON.stringify(value);
+    return primitive === void 0 ? "null" : primitive;
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map((item) => stableStringify(item)).join(",")}]`;
+  }
+  const entries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b));
+  const body = entries.map(([key, val]) => `${JSON.stringify(key)}:${stableStringify(val)}`).join(",");
+  return `{${body}}`;
 }
 export {
+  compileMarkdownSnapshot,
+  computeSnapshotHash,
+  createCompileMarkdownSnapshotPool,
   createWorkerThread,
   getHostedWorkerBundleUrl
 };

package/dist/node/worker-thread-entry.cjs

@@ -1,9 +1,33 @@
 "use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 
 // src/node/worker-thread-entry.ts
+var import_node_path = __toESM(require("path"), 1);
+var import_node_fs = require("fs");
+var import_node_module = require("module");
 var import_node_url = require("url");
 var import_node_worker_threads = require("worker_threads");
-var import_meta = {};
 var port = import_node_worker_threads.parentPort;
 if (!port) {
   throw new Error("[stream-mdx] worker thread bootstrap missing parentPort.");
@@ -54,7 +78,7 @@ port.on("message", (data) => {
   dispatchMessage(data);
 });
 var bootstrap = import_node_worker_threads.workerData ?? {};
-var bundleUrl = typeof bootstrap.bundleUrl === "string" && bootstrap.bundleUrl.length > 0 ? bootstrap.bundleUrl : new URL("../hosted/markdown-worker.js", getModuleUrl()).href;
+var bundleUrl = typeof bootstrap.bundleUrl === "string" && bootstrap.bundleUrl.length > 0 ? bootstrap.bundleUrl : resolveDefaultBundleUrl().href;
 void (async () => {
   await import(bundleUrl);
   ready = true;
@@ -68,8 +92,108 @@ void (async () => {
   throw error;
 });
 function getModuleUrl() {
+  const fromImportMeta = getImportMetaUrl();
+  if (fromImportMeta) {
+    return fromImportMeta;
+  }
   if (typeof __filename === "string" && __filename.length > 0) {
     return (0, import_node_url.pathToFileURL)(__filename).href;
   }
-  return import_meta.url;
+  const fromStack = getModuleUrlFromStack();
+  if (fromStack) {
+    return fromStack;
+  }
+  throw new Error("[stream-mdx] Unable to resolve worker-thread module URL.");
+}
+function getImportMetaUrl() {
+  try {
+    const candidate = (0, eval)("import.meta.url");
+    return typeof candidate === "string" && candidate.length > 0 ? candidate : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function resolveDefaultBundleUrl() {
+  const packageRoot = resolvePackageRootFromRequire();
+  if (packageRoot) {
+    const resolved = firstExistingPath([
+      import_node_path.default.join(packageRoot, "dist/hosted/markdown-worker.js"),
+      import_node_path.default.join(packageRoot, "dist/worker.mjs"),
+      import_node_path.default.join(packageRoot, "dist/worker.js"),
+      import_node_path.default.join(packageRoot, "src/worker.ts")
+    ]);
+    if (resolved) {
+      return (0, import_node_url.pathToFileURL)(resolved);
+    }
+  }
+  const moduleUrl = getModuleUrl();
+  const candidates = [
+    new URL("../hosted/markdown-worker.js", moduleUrl),
+    new URL("../../dist/hosted/markdown-worker.js", moduleUrl),
+    new URL("../../../public/workers/markdown-worker.js", moduleUrl),
+    new URL("../../../../public/workers/markdown-worker.js", moduleUrl),
+    new URL("../../dist/worker.mjs", moduleUrl),
+    new URL("../../dist/worker.js", moduleUrl),
+    new URL("../worker.mjs", moduleUrl),
+    new URL("../worker.js", moduleUrl),
+    new URL("../worker.ts", moduleUrl)
+  ];
+  for (const candidate of candidates) {
+    if (urlExists(candidate)) return candidate;
+  }
+  return candidates[0];
+}
+function urlExists(url) {
+  if (url.protocol !== "file:") return false;
+  try {
+    return (0, import_node_fs.existsSync)(import_node_path.default.normalize((0, import_node_url.fileURLToPath)(url)));
+  } catch {
+    return false;
+  }
+}
+function firstExistingPath(candidates) {
+  for (const candidate of candidates) {
+    try {
+      if ((0, import_node_fs.existsSync)(candidate)) return candidate;
+    } catch {
+    }
+  }
+  return void 0;
+}
+function resolvePackageRootFromRequire() {
+  const bases = [
+    import_node_path.default.join(process.cwd(), "package.json"),
+    typeof __filename === "string" && __filename.length > 0 ? __filename : import_node_path.default.join(process.cwd(), "__stream-mdx-worker-thread-resolver__.cjs")
+  ];
+  for (const base of bases) {
+    try {
+      const req = (0, import_node_module.createRequire)(base);
+      const nodeEntry = req.resolve("@stream-mdx/worker/node");
+      return import_node_path.default.resolve(import_node_path.default.dirname(nodeEntry), "..", "..");
+    } catch {
+    }
+  }
+  return void 0;
+}
+function getModuleUrlFromStack() {
+  const previous = Error.prepareStackTrace;
+  try {
+    Error.prepareStackTrace = (_error, stackTrace2) => stackTrace2;
+    const stackTrace = new Error().stack;
+    if (!Array.isArray(stackTrace)) return void 0;
+    const files = stackTrace.map((frame) => frame.getFileName()).filter((fileName) => Boolean(fileName));
+    const preferred = files.find(
+      (fileName) => fileName.includes("/markdown-v2-worker/src/node/worker-thread-entry.") || fileName.includes("/markdown-v2-worker/dist/node/worker-thread-entry.") || fileName.includes("\\markdown-v2-worker\\src\\node\\worker-thread-entry.") || fileName.includes("\\markdown-v2-worker\\dist\\node\\worker-thread-entry.")
+    );
+    if (preferred) {
+      return preferred.startsWith("file://") ? preferred : (0, import_node_url.pathToFileURL)(preferred).href;
+    }
+    const firstAbsolute = files.find((fileName) => fileName.startsWith("file://") || import_node_path.default.isAbsolute(fileName));
+    if (!firstAbsolute) return void 0;
+    return firstAbsolute.startsWith("file://") ? firstAbsolute : (0, import_node_url.pathToFileURL)(firstAbsolute).href;
+  } catch {
+    return void 0;
+  } finally {
+    Error.prepareStackTrace = previous;
+  }
 }

package/dist/node/worker-thread-entry.d.cts

@@ -1 +1,2 @@
-import '@stream-mdx/core/streaming/custom-matcher';
+
+export {  }

package/dist/node/worker-thread-entry.d.ts

@@ -1 +1,2 @@
-import '@stream-mdx/core/streaming/custom-matcher';
+
+export {  }