@stream-mdx/core 0.0.3 → 0.1.1

package/dist/streaming/inline-streaming.mjs

@@ -1,63 +1,127 @@
 // src/streaming/inline-streaming.ts
+var DEFAULT_FORMAT_ANTICIPATION = {
+  inline: false,
+  mathInline: false,
+  mathBlock: false,
+  html: false,
+  mdx: false,
+  regex: false
+};
+function normalizeFormatAnticipation(input) {
+  if (input === true) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
+  }
+  if (!input) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION };
+  }
+  return {
+    inline: input.inline ?? false,
+    mathInline: input.mathInline ?? false,
+    mathBlock: input.mathBlock ?? false,
+    html: input.html ?? false,
+    mdx: input.mdx ?? false,
+    regex: input.regex ?? false
+  };
+}
 function prepareInlineStreamingContent(content, options) {
-  const enableAnticipation = Boolean(options?.formatAnticipation);
   const enableMath = options?.math !== false;
-  let dollarCount = 0;
-  let backtickCount = 0;
-  let starCount = 0;
-  let doubleStarCount = 0;
-  let tildePairCount = 0;
+  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
+  const enableInlineAnticipation = anticipation.inline;
+  const enableMathInlineAnticipation = anticipation.mathInline;
+  const enableMathBlockAnticipation = anticipation.mathBlock;
+  const stack = [];
+  const toggleToken = (token) => {
+    const last = stack[stack.length - 1];
+    if (last === token) {
+      stack.pop();
+    } else {
+      stack.push(token);
+    }
+  };
+  let mathDisplayOpen = false;
+  let mathDisplayCrossedNewline = false;
   for (let i = 0; i < content.length; i++) {
     const code = content.charCodeAt(i);
-    if (code === 36) {
-      dollarCount += 1;
+    if (code === 10 || code === 13) {
+      if (mathDisplayOpen) {
+        mathDisplayCrossedNewline = true;
+      }
       continue;
     }
     if (code === 96) {
-      backtickCount += 1;
+      toggleToken("code");
+      continue;
+    }
+    if (code === 126 && i + 1 < content.length && content.charCodeAt(i + 1) === 126) {
+      toggleToken("strike");
+      i += 1;
       continue;
     }
     if (code === 42) {
       if (i + 1 < content.length && content.charCodeAt(i + 1) === 42) {
-        doubleStarCount += 1;
-        starCount += 2;
+        toggleToken("strong");
         i += 1;
       } else {
-        starCount += 1;
+        toggleToken("em");
       }
       continue;
     }
-    if (code === 126) {
-      if (i + 1 < content.length && content.charCodeAt(i + 1) === 126) {
-        tildePairCount += 1;
+    if (enableMath && code === 36) {
+      if (i + 1 < content.length && content.charCodeAt(i + 1) === 36) {
+        toggleToken("math-display");
+        if (mathDisplayOpen) {
+          mathDisplayOpen = false;
+          mathDisplayCrossedNewline = false;
+        } else {
+          mathDisplayOpen = true;
+          mathDisplayCrossedNewline = false;
+        }
         i += 1;
+      } else {
+        toggleToken("math-inline");
       }
     }
   }
-  const hasIncompleteMath = enableMath && dollarCount % 2 !== 0;
-  if (hasIncompleteMath) {
-    return { kind: "raw", status: "raw", reason: "incomplete-math" };
-  }
-  const hasIncompleteCode = backtickCount % 2 !== 0;
-  const hasIncompleteStrong = doubleStarCount % 2 !== 0;
-  const singleStarCount = starCount - doubleStarCount * 2;
-  const hasIncompleteEmphasis = singleStarCount % 2 !== 0;
-  const hasIncompleteStrike = tildePairCount % 2 !== 0;
-  const hasAnyIncomplete = hasIncompleteCode || hasIncompleteStrong || hasIncompleteEmphasis || hasIncompleteStrike;
-  if (!hasAnyIncomplete) {
-    return { kind: "parse", status: "complete", content, appended: "" };
+  const hasIncompleteFormatting = stack.some((token) => token === "code" || token === "strike" || token === "strong" || token === "em");
+  const hasIncompleteMathInline = stack.includes("math-inline");
+  const hasIncompleteMathDisplay = stack.includes("math-display");
+  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
+  if (enableMath && hasIncompleteMath) {
+    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
   }
-  if (!enableAnticipation) {
+  if (hasIncompleteFormatting && !enableInlineAnticipation) {
     return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
   }
-  let appended = "";
-  if (hasIncompleteCode) appended += "`";
-  if (hasIncompleteStrike) appended += "~~";
-  if (hasIncompleteStrong && hasIncompleteEmphasis) appended += "***";
-  else if (hasIncompleteStrong) appended += "**";
-  else if (hasIncompleteEmphasis) appended += "*";
+  if (!hasIncompleteFormatting && !hasIncompleteMath) {
+    return { kind: "parse", status: "complete", content, appended: "" };
+  }
+  const appendForToken = (token) => {
+    switch (token) {
+      case "code":
+        return "`";
+      case "strike":
+        return "~~";
+      case "strong":
+        return "**";
+      case "em":
+        return "*";
+      case "math-inline":
+        return "$";
+      case "math-display":
+        return "$$";
+      default:
+        return "";
+    }
+  };
+  const appended = stack.slice().reverse().map((token) => appendForToken(token)).join("");
   return { kind: "parse", status: "anticipated", content: content + appended, appended };
 }
 export {
+  normalizeFormatAnticipation,
   prepareInlineStreamingContent
 };

package/dist/types.d.cts

@@ -39,6 +39,14 @@ interface MixedContentSegment {
     status?: "pending" | "compiled" | "error";
     error?: string;
 }
+type FormatAnticipationConfig = boolean | {
+    inline?: boolean;
+    mathInline?: boolean;
+    mathBlock?: boolean;
+    html?: boolean;
+    mdx?: boolean;
+    regex?: boolean;
+};
 interface InlineHtmlDescriptor {
     tagName: string;
     attributes: Record<string, string>;
@@ -115,8 +123,9 @@ type WorkerIn = {
         tables?: boolean;
         callouts?: boolean;
         math?: boolean;
-        formatAnticipation?: boolean;
+        formatAnticipation?: FormatAnticipationConfig;
         liveCodeHighlighting?: boolean;
+        mdxComponentNames?: string[];
     };
     mdx?: {
         compileMode?: "server" | "worker";
@@ -126,6 +135,8 @@ type WorkerIn = {
     text: string;
 } | {
     type: "FINALIZE";
+} | {
+    type: "DEBUG_STATE";
 } | {
     type: "MDX_COMPILED";
     blockId: string;
@@ -158,6 +169,23 @@ type WorkerOut = {
 } | {
     type: "METRICS";
     metrics: PerformanceMetrics;
+} | {
+    type: "DEBUG_STATE";
+    state: {
+        contentLength: number;
+        contentTail: string;
+        blockCount: number;
+        blockTypeCounts: Record<string, number>;
+        lastBlockType?: string;
+        lastBlockRange?: {
+            from: number;
+            to: number;
+        };
+        lastBlockRawTail?: string;
+        hasInlineCodeHeading: boolean;
+        hasCodeBlocksHeading: boolean;
+        hasMediaHeading: boolean;
+    };
 } | {
     type: "ERROR";
     phase: WorkerPhase;
@@ -182,6 +210,17 @@ interface RegexInlinePlugin extends InlinePlugin {
      * the regex is skipped for that node.
      */
     fastCheck?: (text: string) => boolean;
+    /**
+     * Optional streaming anticipation config. Only used when formatAnticipation.regex is enabled.
+     */
+    anticipation?: RegexAnticipationPattern;
+}
+interface RegexAnticipationPattern {
+    start: RegExp;
+    end: RegExp;
+    full?: RegExp;
+    append: string | ((match: RegExpExecArray, content: string) => string);
+    maxScanChars?: number;
 }
 interface ASTInlinePlugin extends InlinePlugin {
     visit: (node: InlineNode, ctx: {
@@ -314,4 +353,4 @@ interface CoalescingMetrics {
     insertChildCoalesced: number;
 }
 
-export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };
+export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type FormatAnticipationConfig, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexAnticipationPattern, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };

package/dist/types.d.ts

@@ -39,6 +39,14 @@ interface MixedContentSegment {
     status?: "pending" | "compiled" | "error";
     error?: string;
 }
+type FormatAnticipationConfig = boolean | {
+    inline?: boolean;
+    mathInline?: boolean;
+    mathBlock?: boolean;
+    html?: boolean;
+    mdx?: boolean;
+    regex?: boolean;
+};
 interface InlineHtmlDescriptor {
     tagName: string;
     attributes: Record<string, string>;
@@ -115,8 +123,9 @@ type WorkerIn = {
         tables?: boolean;
         callouts?: boolean;
         math?: boolean;
-        formatAnticipation?: boolean;
+        formatAnticipation?: FormatAnticipationConfig;
         liveCodeHighlighting?: boolean;
+        mdxComponentNames?: string[];
     };
     mdx?: {
         compileMode?: "server" | "worker";
@@ -126,6 +135,8 @@ type WorkerIn = {
     text: string;
 } | {
     type: "FINALIZE";
+} | {
+    type: "DEBUG_STATE";
 } | {
     type: "MDX_COMPILED";
     blockId: string;
@@ -158,6 +169,23 @@ type WorkerOut = {
 } | {
     type: "METRICS";
     metrics: PerformanceMetrics;
+} | {
+    type: "DEBUG_STATE";
+    state: {
+        contentLength: number;
+        contentTail: string;
+        blockCount: number;
+        blockTypeCounts: Record<string, number>;
+        lastBlockType?: string;
+        lastBlockRange?: {
+            from: number;
+            to: number;
+        };
+        lastBlockRawTail?: string;
+        hasInlineCodeHeading: boolean;
+        hasCodeBlocksHeading: boolean;
+        hasMediaHeading: boolean;
+    };
 } | {
     type: "ERROR";
     phase: WorkerPhase;
@@ -182,6 +210,17 @@ interface RegexInlinePlugin extends InlinePlugin {
      * the regex is skipped for that node.
      */
     fastCheck?: (text: string) => boolean;
+    /**
+     * Optional streaming anticipation config. Only used when formatAnticipation.regex is enabled.
+     */
+    anticipation?: RegexAnticipationPattern;
+}
+interface RegexAnticipationPattern {
+    start: RegExp;
+    end: RegExp;
+    full?: RegExp;
+    append: string | ((match: RegExpExecArray, content: string) => string);
+    maxScanChars?: number;
 }
 interface ASTInlinePlugin extends InlinePlugin {
     visit: (node: InlineNode, ctx: {
@@ -314,4 +353,4 @@ interface CoalescingMetrics {
     insertChildCoalesced: number;
 }
 
-export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };
+export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type FormatAnticipationConfig, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexAnticipationPattern, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };

package/dist/worker-html-sanitizer.cjs

@@ -37,9 +37,23 @@ var rehypeParse = __toESM(require("rehype-parse"), 1);
 var rehypeSanitize = __toESM(require("rehype-sanitize"), 1);
 var rehypeStringify = __toESM(require("rehype-stringify"), 1);
 var import_unified = require("unified");
-var { defaultSchema } = rehypeSanitize;
+var rehypeSanitizeModule = rehypeSanitize;
+var defaultSchema = rehypeSanitizeModule.defaultSchema;
+var resolvePlugin = (mod) => {
+  if (typeof mod === "function") return mod;
+  if (mod && typeof mod.default === "function") {
+    return mod.default;
+  }
+  if (mod && typeof mod.default?.default === "function") {
+    return mod.default?.default;
+  }
+  return mod;
+};
+var rehypeParsePlugin = resolvePlugin(rehypeParse);
+var rehypeSanitizePlugin = resolvePlugin(rehypeSanitizeModule);
+var rehypeStringifyPlugin = resolvePlugin(rehypeStringify);
 var SANITIZED_SCHEMA = createSchema();
-var sanitizeProcessor = (0, import_unified.unified)().use(rehypeParse.default, { fragment: true }).use(rehypeSanitize.default, SANITIZED_SCHEMA).use(rehypeStringify.default).freeze();
+var sanitizeProcessor = (0, import_unified.unified)().use(rehypeParsePlugin, { fragment: true }).use(rehypeSanitizePlugin, SANITIZED_SCHEMA).use(rehypeStringifyPlugin).freeze();
 function sanitizeHtmlInWorker(html) {
   if (!html) return "";
   try {

package/dist/worker-html-sanitizer.mjs

@@ -3,9 +3,23 @@ import * as rehypeParse from "rehype-parse";
 import * as rehypeSanitize from "rehype-sanitize";
 import * as rehypeStringify from "rehype-stringify";
 import { unified } from "unified";
-var { defaultSchema } = rehypeSanitize;
+var rehypeSanitizeModule = rehypeSanitize;
+var defaultSchema = rehypeSanitizeModule.defaultSchema;
+var resolvePlugin = (mod) => {
+  if (typeof mod === "function") return mod;
+  if (mod && typeof mod.default === "function") {
+    return mod.default;
+  }
+  if (mod && typeof mod.default?.default === "function") {
+    return mod.default?.default;
+  }
+  return mod;
+};
+var rehypeParsePlugin = resolvePlugin(rehypeParse);
+var rehypeSanitizePlugin = resolvePlugin(rehypeSanitizeModule);
+var rehypeStringifyPlugin = resolvePlugin(rehypeStringify);
 var SANITIZED_SCHEMA = createSchema();
-var sanitizeProcessor = unified().use(rehypeParse.default, { fragment: true }).use(rehypeSanitize.default, SANITIZED_SCHEMA).use(rehypeStringify.default).freeze();
+var sanitizeProcessor = unified().use(rehypeParsePlugin, { fragment: true }).use(rehypeSanitizePlugin, SANITIZED_SCHEMA).use(rehypeStringifyPlugin).freeze();
 function sanitizeHtmlInWorker(html) {
   if (!html) return "";
   try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-mdx/core",
-  "version": "0.0.3",
+  "version": "0.1.1",
   "description": "Core types, snapshot utilities, and perf helpers for the Streaming Markdown V2 stack",
   "license": "MIT",
   "repository": {
@@ -90,7 +90,7 @@
   "sideEffects": false,
   "scripts": {
     "build": "tsup",
-    "test": "tsx __tests__/*.test.ts",
+    "test": "tsx ../../scripts/run-tests.ts __tests__",
     "clean": "rm -rf dist",
     "prepack": "npm run build"
   },