@stream-mdx/worker 0.3.0 → 0.4.0

package/dist/node/index.cjs

@@ -30,20 +30,54 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/node/index.ts
 var node_exports = {};
 __export(node_exports, {
+  compileMarkdownSnapshot: () => compileMarkdownSnapshot,
+  computeSnapshotHash: () => computeSnapshotHash,
+  createCompileMarkdownSnapshotPool: () => createCompileMarkdownSnapshotPool,
   createWorkerThread: () => createWorkerThread,
   getHostedWorkerBundleUrl: () => getHostedWorkerBundleUrl
 });
 module.exports = __toCommonJS(node_exports);
+var import_node_crypto = require("crypto");
+var import_node_fs = require("fs");
+var import_node_fs2 = require("fs");
+var import_node_module = require("module");
 var import_node_path = __toESM(require("path"), 1);
 var import_node_url = require("url");
 var import_node_worker_threads = require("worker_threads");
-var import_meta = {};
+var import_core = require("@stream-mdx/core");
 function getHostedWorkerBundleUrl() {
-  return new URL("../hosted/markdown-worker.js", getModuleUrl());
+  const packageRoot = resolvePackageRootFromRequire();
+  if (packageRoot) {
+    const resolved = firstExistingPath([
+      import_node_path.default.join(packageRoot, "dist/hosted/markdown-worker.js"),
+      import_node_path.default.join(packageRoot, "dist/worker.mjs"),
+      import_node_path.default.join(packageRoot, "dist/worker.js"),
+      import_node_path.default.join(packageRoot, "src/worker.ts")
+    ]);
+    if (resolved) {
+      return (0, import_node_url.pathToFileURL)(resolved);
+    }
+  }
+  const moduleUrl = getModuleUrl();
+  const candidates = [
+    new URL("../hosted/markdown-worker.js", moduleUrl),
+    new URL("../../dist/hosted/markdown-worker.js", moduleUrl),
+    new URL("../../../public/workers/markdown-worker.js", moduleUrl),
+    new URL("../../../../public/workers/markdown-worker.js", moduleUrl),
+    new URL("../../dist/worker.mjs", moduleUrl),
+    new URL("../../dist/worker.js", moduleUrl),
+    new URL("../worker.mjs", moduleUrl),
+    new URL("../worker.js", moduleUrl),
+    new URL("../worker.ts", moduleUrl)
+  ];
+  for (const candidate of candidates) {
+    if (urlExists(candidate)) return candidate;
+  }
+  return candidates[0];
 }
 function createWorkerThread(options = {}) {
   const { workerBundle, workerData, ...workerOptions } = options;
-  const runnerUrl = new URL("./worker-thread-entry.mjs", getModuleUrl());
+  const runnerUrl = resolveWorkerThreadEntryUrl();
   const bundleUrl = normalizeWorkerBundleUrl(workerBundle) ?? getHostedWorkerBundleUrl();
   return new import_node_worker_threads.Worker(runnerUrl, {
     ...workerOptions,
@@ -53,6 +87,37 @@ function createWorkerThread(options = {}) {
     }
   });
 }
+function resolveWorkerThreadEntryUrl() {
+  const packageRoot = resolvePackageRootFromRequire();
+  if (packageRoot) {
+    const resolved = firstExistingPath([
+      import_node_path.default.join(packageRoot, "dist/node/worker-thread-entry.mjs"),
+      import_node_path.default.join(packageRoot, "dist/node/worker-thread-entry.cjs"),
+      import_node_path.default.join(packageRoot, "src/node/worker-thread-entry.ts")
+    ]);
+    if (resolved) {
+      return (0, import_node_url.pathToFileURL)(resolved);
+    }
+  }
+  const moduleUrl = getModuleUrl();
+  const distEntry = new URL("./worker-thread-entry.mjs", moduleUrl);
+  const sourceEntry = new URL("./worker-thread-entry.ts", moduleUrl);
+  if (urlExists(distEntry)) {
+    return distEntry;
+  }
+  if (urlExists(sourceEntry)) {
+    return sourceEntry;
+  }
+  return distEntry;
+}
+function urlExists(url) {
+  if (url.protocol !== "file:") return false;
+  try {
+    return (0, import_node_fs.existsSync)(import_node_path.default.normalize((0, import_node_url.fileURLToPath)(url)));
+  } catch {
+    return false;
+  }
+}
 function normalizeWorkerBundleUrl(value) {
   if (!value) return void 0;
   if (value instanceof URL) return value;
@@ -62,14 +127,328 @@ function normalizeWorkerBundleUrl(value) {
     return (0, import_node_url.pathToFileURL)(import_node_path.default.resolve(value));
   }
 }
+function firstExistingPath(candidates) {
+  for (const candidate of candidates) {
+    try {
+      if ((0, import_node_fs.existsSync)(candidate)) return candidate;
+    } catch {
+    }
+  }
+  return void 0;
+}
+function resolvePackageRootFromRequire() {
+  const bases = [
+    import_node_path.default.join(process.cwd(), "package.json"),
+    typeof __filename === "string" && __filename.length > 0 ? __filename : import_node_path.default.join(process.cwd(), "__stream-mdx-node-resolver__.cjs")
+  ];
+  for (const base of bases) {
+    try {
+      const req = (0, import_node_module.createRequire)(base);
+      const nodeEntry = req.resolve("@stream-mdx/worker/node");
+      return import_node_path.default.resolve(import_node_path.default.dirname(nodeEntry), "..", "..");
+    } catch {
+    }
+  }
+  return void 0;
+}
 function getModuleUrl() {
+  const fromImportMeta = getImportMetaUrl();
+  if (fromImportMeta) {
+    return fromImportMeta;
+  }
   if (typeof __filename === "string" && __filename.length > 0) {
     return (0, import_node_url.pathToFileURL)(__filename).href;
   }
-  return import_meta.url;
+  const fromStack = getModuleUrlFromStack();
+  if (fromStack) {
+    return fromStack;
+  }
+  throw new Error("[stream-mdx] Unable to resolve module URL.");
+}
+function getImportMetaUrl() {
+  try {
+    const candidate = (0, eval)("import.meta.url");
+    return typeof candidate === "string" && candidate.length > 0 ? candidate : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function getModuleUrlFromStack() {
+  const previous = Error.prepareStackTrace;
+  try {
+    Error.prepareStackTrace = (_error, stackTrace2) => stackTrace2;
+    const stackTrace = new Error().stack;
+    if (!Array.isArray(stackTrace)) return void 0;
+    const files = stackTrace.map((frame) => frame.getFileName()).filter((fileName) => Boolean(fileName));
+    const preferred = files.find(
+      (fileName) => fileName.includes("/markdown-v2-worker/src/node/index.") || fileName.includes("/markdown-v2-worker/dist/node/index.") || fileName.includes("\\markdown-v2-worker\\src\\node\\index.") || fileName.includes("\\markdown-v2-worker\\dist\\node\\index.")
+    );
+    if (preferred) {
+      return preferred.startsWith("file://") ? preferred : (0, import_node_url.pathToFileURL)(preferred).href;
+    }
+    const firstAbsolute = files.find((fileName) => fileName.startsWith("file://") || import_node_path.default.isAbsolute(fileName));
+    if (!firstAbsolute) return void 0;
+    return firstAbsolute.startsWith("file://") ? firstAbsolute : (0, import_node_url.pathToFileURL)(firstAbsolute).href;
+  } catch {
+    return void 0;
+  } finally {
+    Error.prepareStackTrace = previous;
+  }
+}
+function computeSnapshotHash(text, init, hashSalt) {
+  return hashCompileInput(text, init, hashSalt);
+}
+async function compileMarkdownSnapshot(options) {
+  const {
+    text,
+    init,
+    hashSalt,
+    worker: providedWorker,
+    workerOptions,
+    timeoutMs = 3e4,
+    settleMs = 50,
+    cache,
+    finalize = true
+  } = options;
+  const hash = hashCompileInput(text, init, hashSalt);
+  const cacheKey = cache?.key ?? hash;
+  const cachePath = cache ? import_node_path.default.join(cache.dir, `${sanitizeCacheKey(cacheKey)}.json`) : null;
+  if (cachePath) {
+    const cached = await readSnapshotCache(cachePath, hash);
+    if (cached) {
+      const snapshot = (0, import_core.createInitialSnapshot)(cached.blocks);
+      return {
+        blocks: snapshot.blocks,
+        snapshot,
+        artifact: cached,
+        fromCache: true
+      };
+    }
+  }
+  const worker = providedWorker ?? createWorkerThread(workerOptions);
+  const ownedWorker = !providedWorker;
+  return await new Promise((resolve, reject) => {
+    let snapshot = null;
+    let initialized = false;
+    let settled = false;
+    let idleTimer = null;
+    let timeoutTimer = null;
+    const finalizeNow = () => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      const finalSnapshot = snapshot ?? (0, import_core.createInitialSnapshot)();
+      const blocks = finalSnapshot.blocks;
+      const contentHash = (0, import_node_crypto.createHash)("sha256").update(text).digest("hex");
+      const configHash = (0, import_node_crypto.createHash)("sha256").update(
+        stableStringify({
+          init: init ?? null,
+          salt: hashSalt ?? null
+        })
+      ).digest("hex");
+      const artifact = {
+        version: 1,
+        schemaId: "streammdx.snapshot.v1",
+        createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+        hash,
+        contentHash,
+        configHash,
+        hashSalt: hashSalt ?? void 0,
+        blocks,
+        tocHeadings: (() => {
+          const root = finalSnapshot.nodes.get(import_core.PATCH_ROOT_ID);
+          const maybe = root?.props?.tocHeadings;
+          return Array.isArray(maybe) ? maybe : void 0;
+        })(),
+        init: init ? {
+          docPlugins: init.docPlugins,
+          mdx: init.mdx,
+          prewarmLangs: init.prewarmLangs
+        } : void 0
+      };
+      if (cachePath && !cache?.readOnly) {
+        void writeSnapshotCache(cachePath, artifact);
+      }
+      resolve({
+        blocks,
+        snapshot: finalSnapshot,
+        artifact,
+        fromCache: false
+      });
+    };
+    const fail = (error) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(error);
+    };
+    const scheduleIdleFinalize = () => {
+      if (finalize || !initialized) return;
+      if (idleTimer) {
+        clearTimeout(idleTimer);
+      }
+      idleTimer = setTimeout(() => finalizeNow(), settleMs);
+    };
+    const handleMessage = (message) => {
+      switch (message.type) {
+        case "INITIALIZED":
+          snapshot = (0, import_core.createInitialSnapshot)(message.blocks);
+          initialized = true;
+          scheduleIdleFinalize();
+          break;
+        case "PATCH":
+          if (snapshot) {
+            snapshot.blocks = (0, import_core.applyPatchBatch)(snapshot, message.patches);
+          }
+          scheduleIdleFinalize();
+          break;
+        case "FINALIZED":
+          if (finalize) {
+            finalizeNow();
+          }
+          break;
+        case "RESET":
+          fail(new Error(`Worker reset during compile: ${message.reason}`));
+          break;
+        case "ERROR":
+          fail(new Error(`Worker error (${message.phase}): ${message.error.message}`));
+          break;
+        default:
+          break;
+      }
+    };
+    const cleanup = () => {
+      if (idleTimer) clearTimeout(idleTimer);
+      if (timeoutTimer) clearTimeout(timeoutTimer);
+      worker.off("message", handleMessage);
+      worker.off("error", fail);
+      if (ownedWorker) {
+        try {
+          worker.terminate();
+        } catch {
+        }
+      }
+    };
+    worker.on("message", handleMessage);
+    worker.on("error", fail);
+    timeoutTimer = setTimeout(() => {
+      fail(new Error("Worker compile timed out."));
+    }, timeoutMs);
+    const initMessage = {
+      type: "INIT",
+      initialContent: text,
+      prewarmLangs: init?.prewarmLangs,
+      docPlugins: init?.docPlugins,
+      mdx: init?.mdx
+    };
+    worker.postMessage(initMessage);
+    if (finalize) {
+      worker.postMessage({ type: "FINALIZE" });
+    }
+  });
+}
+function createCompileMarkdownSnapshotPool(options = {}) {
+  const size = Math.max(1, Math.min(8, Math.floor(options.size ?? 2)));
+  const workers = new Array(size).fill(null).map(() => createWorkerThread(options.workerOptions));
+  const queues = new Array(size).fill(Promise.resolve());
+  let rr = 0;
+  const maxMemoryEntries = Math.max(0, Math.floor(options.maxMemoryEntries ?? 64));
+  const memory = maxMemoryEntries > 0 ? /* @__PURE__ */ new Map() : null;
+  const enqueue = (index, fn) => {
+    let resolveOuter;
+    let rejectOuter;
+    const outer = new Promise((resolve, reject) => {
+      resolveOuter = resolve;
+      rejectOuter = reject;
+    });
+    queues[index] = queues[index].catch(() => void 0).then(async () => {
+      try {
+        const value = await fn();
+        resolveOuter(value);
+      } catch (err) {
+        rejectOuter(err);
+      }
+    }).then(() => void 0);
+    return outer;
+  };
+  const compile = async (compileOptions) => {
+    const hash = computeSnapshotHash(compileOptions.text, compileOptions.init, compileOptions.hashSalt);
+    if (memory && memory.has(hash)) {
+      return memory.get(hash);
+    }
+    const index = rr++ % workers.length;
+    const result = await enqueue(index, async () => {
+      return await compileMarkdownSnapshot({
+        ...compileOptions,
+        worker: workers[index]
+      });
+    });
+    if (memory) {
+      memory.set(hash, result);
+      if (memory.size > maxMemoryEntries) {
+        const firstKey = memory.keys().next().value;
+        if (firstKey) memory.delete(firstKey);
+      }
+    }
+    return result;
+  };
+  const close = async () => {
+    for (const worker of workers) {
+      try {
+        await worker.terminate();
+      } catch {
+      }
+    }
+  };
+  return { compile, close };
+}
+async function readSnapshotCache(cachePath, hash) {
+  try {
+    const raw = await import_node_fs2.promises.readFile(cachePath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (!parsed || parsed.version !== 1 || parsed.hash !== hash || !Array.isArray(parsed.blocks)) {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+async function writeSnapshotCache(cachePath, artifact) {
+  try {
+    await import_node_fs2.promises.mkdir(import_node_path.default.dirname(cachePath), { recursive: true });
+    await import_node_fs2.promises.writeFile(cachePath, JSON.stringify(artifact, null, 2), "utf8");
+  } catch {
+  }
+}
+function sanitizeCacheKey(value) {
+  return value.replace(/[^a-z0-9._-]+/gi, "_");
+}
+function hashCompileInput(text, init, hashSalt) {
+  const payload = stableStringify({
+    text,
+    init: init ?? null,
+    salt: hashSalt ?? null
+  });
+  return (0, import_node_crypto.createHash)("sha256").update(payload).digest("hex");
+}
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") {
+    const primitive = JSON.stringify(value);
+    return primitive === void 0 ? "null" : primitive;
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map((item) => stableStringify(item)).join(",")}]`;
+  }
+  const entries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b));
+  const body = entries.map(([key, val]) => `${JSON.stringify(key)}:${stableStringify(val)}`).join(",");
+  return `{${body}}`;
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  compileMarkdownSnapshot,
+  computeSnapshotHash,
+  createCompileMarkdownSnapshotPool,
   createWorkerThread,
   getHostedWorkerBundleUrl
 });

package/dist/node/index.d.cts

@@ -1,4 +1,5 @@
 import { WorkerOptions, Worker } from 'node:worker_threads';
+import { Block, TocHeading, WorkerIn, DocumentSnapshot } from '@stream-mdx/core';
 
 interface CreateWorkerThreadOptions extends Omit<WorkerOptions, "type" | "workerData"> {
     /**
@@ -25,5 +26,71 @@ declare function getHostedWorkerBundleUrl(): URL;
  * so the same hosted bundle used in browsers can run under Node.
  */
 declare function createWorkerThread(options?: CreateWorkerThreadOptions): Worker;
+type WorkerInitMessage = Extract<WorkerIn, {
+    type: "INIT";
+}>;
+interface SnapshotArtifactV1 {
+    version: 1;
+    schemaId: "streammdx.snapshot.v1";
+    createdAt: string;
+    hash: string;
+    contentHash: string;
+    configHash: string;
+    hashSalt?: string;
+    blocks: Block[];
+    tocHeadings?: TocHeading[];
+    init?: {
+        docPlugins?: WorkerInitMessage["docPlugins"];
+        mdx?: WorkerInitMessage["mdx"];
+        prewarmLangs?: string[];
+    };
+}
+interface CompileMarkdownSnapshotOptions {
+    text: string;
+    init?: Omit<WorkerInitMessage, "type" | "initialContent">;
+    /**
+     * Optional salt mixed into the snapshot hash and cache key.
+     *
+     * This exists so callers can invalidate on renderer/compiler changes (not just
+     * input text + init), while keeping deterministic outputs for a given salt.
+     */
+    hashSalt?: string;
+    worker?: Worker;
+    workerOptions?: CreateWorkerThreadOptions;
+    timeoutMs?: number;
+    settleMs?: number;
+    cache?: {
+        dir: string;
+        key?: string;
+        readOnly?: boolean;
+    };
+    finalize?: boolean;
+}
+interface CompileMarkdownSnapshotResult {
+    blocks: Block[];
+    snapshot: DocumentSnapshot;
+    artifact: SnapshotArtifactV1;
+    fromCache: boolean;
+}
+declare function computeSnapshotHash(text: string, init?: Omit<WorkerInitMessage, "type" | "initialContent">, hashSalt?: string): string;
+declare function compileMarkdownSnapshot(options: CompileMarkdownSnapshotOptions): Promise<CompileMarkdownSnapshotResult>;
+interface CompileMarkdownSnapshotPoolOptions {
+    /**
+     * Number of worker_threads to keep warm.
+     * Each worker is single-flight (requests are queued per worker).
+     */
+    size?: number;
+    workerOptions?: CreateWorkerThreadOptions;
+    /**
+     * In-memory cache entries keyed by the snapshot hash.
+     * Set to 0 to disable.
+     */
+    maxMemoryEntries?: number;
+}
+interface CompileMarkdownSnapshotPool {
+    compile(options: Omit<CompileMarkdownSnapshotOptions, "worker">): Promise<CompileMarkdownSnapshotResult>;
+    close(): Promise<void>;
+}
+declare function createCompileMarkdownSnapshotPool(options?: CompileMarkdownSnapshotPoolOptions): CompileMarkdownSnapshotPool;
 
-export { type CreateWorkerThreadOptions, createWorkerThread, getHostedWorkerBundleUrl };
+export { type CompileMarkdownSnapshotOptions, type CompileMarkdownSnapshotPool, type CompileMarkdownSnapshotPoolOptions, type CompileMarkdownSnapshotResult, type CreateWorkerThreadOptions, type SnapshotArtifactV1, compileMarkdownSnapshot, computeSnapshotHash, createCompileMarkdownSnapshotPool, createWorkerThread, getHostedWorkerBundleUrl };

package/dist/node/index.d.ts

@@ -1,4 +1,5 @@
 import { WorkerOptions, Worker } from 'node:worker_threads';
+import { Block, TocHeading, WorkerIn, DocumentSnapshot } from '@stream-mdx/core';
 
 interface CreateWorkerThreadOptions extends Omit<WorkerOptions, "type" | "workerData"> {
     /**
@@ -25,5 +26,71 @@ declare function getHostedWorkerBundleUrl(): URL;
  * so the same hosted bundle used in browsers can run under Node.
  */
 declare function createWorkerThread(options?: CreateWorkerThreadOptions): Worker;
+type WorkerInitMessage = Extract<WorkerIn, {
+    type: "INIT";
+}>;
+interface SnapshotArtifactV1 {
+    version: 1;
+    schemaId: "streammdx.snapshot.v1";
+    createdAt: string;
+    hash: string;
+    contentHash: string;
+    configHash: string;
+    hashSalt?: string;
+    blocks: Block[];
+    tocHeadings?: TocHeading[];
+    init?: {
+        docPlugins?: WorkerInitMessage["docPlugins"];
+        mdx?: WorkerInitMessage["mdx"];
+        prewarmLangs?: string[];
+    };
+}
+interface CompileMarkdownSnapshotOptions {
+    text: string;
+    init?: Omit<WorkerInitMessage, "type" | "initialContent">;
+    /**
+     * Optional salt mixed into the snapshot hash and cache key.
+     *
+     * This exists so callers can invalidate on renderer/compiler changes (not just
+     * input text + init), while keeping deterministic outputs for a given salt.
+     */
+    hashSalt?: string;
+    worker?: Worker;
+    workerOptions?: CreateWorkerThreadOptions;
+    timeoutMs?: number;
+    settleMs?: number;
+    cache?: {
+        dir: string;
+        key?: string;
+        readOnly?: boolean;
+    };
+    finalize?: boolean;
+}
+interface CompileMarkdownSnapshotResult {
+    blocks: Block[];
+    snapshot: DocumentSnapshot;
+    artifact: SnapshotArtifactV1;
+    fromCache: boolean;
+}
+declare function computeSnapshotHash(text: string, init?: Omit<WorkerInitMessage, "type" | "initialContent">, hashSalt?: string): string;
+declare function compileMarkdownSnapshot(options: CompileMarkdownSnapshotOptions): Promise<CompileMarkdownSnapshotResult>;
+interface CompileMarkdownSnapshotPoolOptions {
+    /**
+     * Number of worker_threads to keep warm.
+     * Each worker is single-flight (requests are queued per worker).
+     */
+    size?: number;
+    workerOptions?: CreateWorkerThreadOptions;
+    /**
+     * In-memory cache entries keyed by the snapshot hash.
+     * Set to 0 to disable.
+     */
+    maxMemoryEntries?: number;
+}
+interface CompileMarkdownSnapshotPool {
+    compile(options: Omit<CompileMarkdownSnapshotOptions, "worker">): Promise<CompileMarkdownSnapshotResult>;
+    close(): Promise<void>;
+}
+declare function createCompileMarkdownSnapshotPool(options?: CompileMarkdownSnapshotPoolOptions): CompileMarkdownSnapshotPool;
 
-export { type CreateWorkerThreadOptions, createWorkerThread, getHostedWorkerBundleUrl };
+export { type CompileMarkdownSnapshotOptions, type CompileMarkdownSnapshotPool, type CompileMarkdownSnapshotPoolOptions, type CompileMarkdownSnapshotResult, type CreateWorkerThreadOptions, type SnapshotArtifactV1, compileMarkdownSnapshot, computeSnapshotHash, createCompileMarkdownSnapshotPool, createWorkerThread, getHostedWorkerBundleUrl };