@stream-mdx/core 0.0.2 → 0.1.0

package/dist/streaming/inline-streaming.cjs

@@ -0,0 +1,153 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/streaming/inline-streaming.ts
+var inline_streaming_exports = {};
+__export(inline_streaming_exports, {
+  normalizeFormatAnticipation: () => normalizeFormatAnticipation,
+  prepareInlineStreamingContent: () => prepareInlineStreamingContent
+});
+module.exports = __toCommonJS(inline_streaming_exports);
+var DEFAULT_FORMAT_ANTICIPATION = {
+  inline: false,
+  mathInline: false,
+  mathBlock: false,
+  html: false,
+  mdx: false,
+  regex: false
+};
+function normalizeFormatAnticipation(input) {
+  if (input === true) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
+  }
+  if (!input) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION };
+  }
+  return {
+    inline: input.inline ?? false,
+    mathInline: input.mathInline ?? false,
+    mathBlock: input.mathBlock ?? false,
+    html: input.html ?? false,
+    mdx: input.mdx ?? false,
+    regex: input.regex ?? false
+  };
+}
+function prepareInlineStreamingContent(content, options) {
+  const enableMath = options?.math !== false;
+  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
+  const enableInlineAnticipation = anticipation.inline;
+  const enableMathInlineAnticipation = anticipation.mathInline;
+  const enableMathBlockAnticipation = anticipation.mathBlock;
+  const stack = [];
+  const toggleToken = (token) => {
+    const last = stack[stack.length - 1];
+    if (last === token) {
+      stack.pop();
+    } else {
+      stack.push(token);
+    }
+  };
+  let mathDisplayOpen = false;
+  let mathDisplayCrossedNewline = false;
+  for (let i = 0; i < content.length; i++) {
+    const code = content.charCodeAt(i);
+    if (code === 10 || code === 13) {
+      if (mathDisplayOpen) {
+        mathDisplayCrossedNewline = true;
+      }
+      continue;
+    }
+    if (code === 96) {
+      toggleToken("code");
+      continue;
+    }
+    if (code === 126 && i + 1 < content.length && content.charCodeAt(i + 1) === 126) {
+      toggleToken("strike");
+      i += 1;
+      continue;
+    }
+    if (code === 42) {
+      if (i + 1 < content.length && content.charCodeAt(i + 1) === 42) {
+        toggleToken("strong");
+        i += 1;
+      } else {
+        toggleToken("em");
+      }
+      continue;
+    }
+    if (enableMath && code === 36) {
+      if (i + 1 < content.length && content.charCodeAt(i + 1) === 36) {
+        toggleToken("math-display");
+        if (mathDisplayOpen) {
+          mathDisplayOpen = false;
+          mathDisplayCrossedNewline = false;
+        } else {
+          mathDisplayOpen = true;
+          mathDisplayCrossedNewline = false;
+        }
+        i += 1;
+      } else {
+        toggleToken("math-inline");
+      }
+    }
+  }
+  const hasIncompleteFormatting = stack.some((token) => token === "code" || token === "strike" || token === "strong" || token === "em");
+  const hasIncompleteMathInline = stack.includes("math-inline");
+  const hasIncompleteMathDisplay = stack.includes("math-display");
+  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
+  if (enableMath && hasIncompleteMath) {
+    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+  }
+  if (hasIncompleteFormatting && !enableInlineAnticipation) {
+    return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
+  }
+  if (!hasIncompleteFormatting && !hasIncompleteMath) {
+    return { kind: "parse", status: "complete", content, appended: "" };
+  }
+  const appendForToken = (token) => {
+    switch (token) {
+      case "code":
+        return "`";
+      case "strike":
+        return "~~";
+      case "strong":
+        return "**";
+      case "em":
+        return "*";
+      case "math-inline":
+        return "$";
+      case "math-display":
+        return "$$";
+      default:
+        return "";
+    }
+  };
+  const appended = stack.slice().reverse().map((token) => appendForToken(token)).join("");
+  return { kind: "parse", status: "anticipated", content: content + appended, appended };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  normalizeFormatAnticipation,
+  prepareInlineStreamingContent
+});

package/dist/streaming/inline-streaming.d.cts

@@ -0,0 +1,28 @@
+import { FormatAnticipationConfig } from '../types.cjs';
+
+type InlineStreamingInlineStatus = "complete" | "anticipated" | "raw";
+type NormalizedFormatAnticipation = {
+    inline: boolean;
+    mathInline: boolean;
+    mathBlock: boolean;
+    html: boolean;
+    mdx: boolean;
+    regex: boolean;
+};
+declare function normalizeFormatAnticipation(input?: FormatAnticipationConfig): NormalizedFormatAnticipation;
+type InlineStreamingPrepareResult = {
+    kind: "raw";
+    status: "raw";
+    reason: "incomplete-math" | "incomplete-formatting";
+} | {
+    kind: "parse";
+    status: Exclude<InlineStreamingInlineStatus, "raw">;
+    content: string;
+    appended: string;
+};
+declare function prepareInlineStreamingContent(content: string, options?: {
+    formatAnticipation?: FormatAnticipationConfig;
+    math?: boolean;
+}): InlineStreamingPrepareResult;
+
+export { type InlineStreamingInlineStatus, type InlineStreamingPrepareResult, type NormalizedFormatAnticipation, normalizeFormatAnticipation, prepareInlineStreamingContent };

package/dist/streaming/inline-streaming.d.ts

@@ -0,0 +1,28 @@
+import { FormatAnticipationConfig } from '../types.js';
+
+type InlineStreamingInlineStatus = "complete" | "anticipated" | "raw";
+type NormalizedFormatAnticipation = {
+    inline: boolean;
+    mathInline: boolean;
+    mathBlock: boolean;
+    html: boolean;
+    mdx: boolean;
+    regex: boolean;
+};
+declare function normalizeFormatAnticipation(input?: FormatAnticipationConfig): NormalizedFormatAnticipation;
+type InlineStreamingPrepareResult = {
+    kind: "raw";
+    status: "raw";
+    reason: "incomplete-math" | "incomplete-formatting";
+} | {
+    kind: "parse";
+    status: Exclude<InlineStreamingInlineStatus, "raw">;
+    content: string;
+    appended: string;
+};
+declare function prepareInlineStreamingContent(content: string, options?: {
+    formatAnticipation?: FormatAnticipationConfig;
+    math?: boolean;
+}): InlineStreamingPrepareResult;
+
+export { type InlineStreamingInlineStatus, type InlineStreamingPrepareResult, type NormalizedFormatAnticipation, normalizeFormatAnticipation, prepareInlineStreamingContent };

package/dist/streaming/inline-streaming.mjs

@@ -0,0 +1,127 @@
+// src/streaming/inline-streaming.ts
+var DEFAULT_FORMAT_ANTICIPATION = {
+  inline: false,
+  mathInline: false,
+  mathBlock: false,
+  html: false,
+  mdx: false,
+  regex: false
+};
+function normalizeFormatAnticipation(input) {
+  if (input === true) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
+  }
+  if (!input) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION };
+  }
+  return {
+    inline: input.inline ?? false,
+    mathInline: input.mathInline ?? false,
+    mathBlock: input.mathBlock ?? false,
+    html: input.html ?? false,
+    mdx: input.mdx ?? false,
+    regex: input.regex ?? false
+  };
+}
+function prepareInlineStreamingContent(content, options) {
+  const enableMath = options?.math !== false;
+  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
+  const enableInlineAnticipation = anticipation.inline;
+  const enableMathInlineAnticipation = anticipation.mathInline;
+  const enableMathBlockAnticipation = anticipation.mathBlock;
+  const stack = [];
+  const toggleToken = (token) => {
+    const last = stack[stack.length - 1];
+    if (last === token) {
+      stack.pop();
+    } else {
+      stack.push(token);
+    }
+  };
+  let mathDisplayOpen = false;
+  let mathDisplayCrossedNewline = false;
+  for (let i = 0; i < content.length; i++) {
+    const code = content.charCodeAt(i);
+    if (code === 10 || code === 13) {
+      if (mathDisplayOpen) {
+        mathDisplayCrossedNewline = true;
+      }
+      continue;
+    }
+    if (code === 96) {
+      toggleToken("code");
+      continue;
+    }
+    if (code === 126 && i + 1 < content.length && content.charCodeAt(i + 1) === 126) {
+      toggleToken("strike");
+      i += 1;
+      continue;
+    }
+    if (code === 42) {
+      if (i + 1 < content.length && content.charCodeAt(i + 1) === 42) {
+        toggleToken("strong");
+        i += 1;
+      } else {
+        toggleToken("em");
+      }
+      continue;
+    }
+    if (enableMath && code === 36) {
+      if (i + 1 < content.length && content.charCodeAt(i + 1) === 36) {
+        toggleToken("math-display");
+        if (mathDisplayOpen) {
+          mathDisplayOpen = false;
+          mathDisplayCrossedNewline = false;
+        } else {
+          mathDisplayOpen = true;
+          mathDisplayCrossedNewline = false;
+        }
+        i += 1;
+      } else {
+        toggleToken("math-inline");
+      }
+    }
+  }
+  const hasIncompleteFormatting = stack.some((token) => token === "code" || token === "strike" || token === "strong" || token === "em");
+  const hasIncompleteMathInline = stack.includes("math-inline");
+  const hasIncompleteMathDisplay = stack.includes("math-display");
+  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
+  if (enableMath && hasIncompleteMath) {
+    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+  }
+  if (hasIncompleteFormatting && !enableInlineAnticipation) {
+    return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
+  }
+  if (!hasIncompleteFormatting && !hasIncompleteMath) {
+    return { kind: "parse", status: "complete", content, appended: "" };
+  }
+  const appendForToken = (token) => {
+    switch (token) {
+      case "code":
+        return "`";
+      case "strike":
+        return "~~";
+      case "strong":
+        return "**";
+      case "em":
+        return "*";
+      case "math-inline":
+        return "$";
+      case "math-display":
+        return "$$";
+      default:
+        return "";
+    }
+  };
+  const appended = stack.slice().reverse().map((token) => appendForToken(token)).join("");
+  return { kind: "parse", status: "anticipated", content: content + appended, appended };
+}
+export {
+  normalizeFormatAnticipation,
+  prepareInlineStreamingContent
+};

package/dist/types.d.cts

@@ -39,6 +39,14 @@ interface MixedContentSegment {
     status?: "pending" | "compiled" | "error";
     error?: string;
 }
+type FormatAnticipationConfig = boolean | {
+    inline?: boolean;
+    mathInline?: boolean;
+    mathBlock?: boolean;
+    html?: boolean;
+    mdx?: boolean;
+    regex?: boolean;
+};
 interface InlineHtmlDescriptor {
     tagName: string;
     attributes: Record<string, string>;
@@ -114,6 +122,10 @@ type WorkerIn = {
         mdx?: boolean;
         tables?: boolean;
         callouts?: boolean;
+        math?: boolean;
+        formatAnticipation?: FormatAnticipationConfig;
+        liveCodeHighlighting?: boolean;
+        mdxComponentNames?: string[];
     };
     mdx?: {
         compileMode?: "server" | "worker";
@@ -179,6 +191,17 @@ interface RegexInlinePlugin extends InlinePlugin {
      * the regex is skipped for that node.
      */
     fastCheck?: (text: string) => boolean;
+    /**
+     * Optional streaming anticipation config. Only used when formatAnticipation.regex is enabled.
+     */
+    anticipation?: RegexAnticipationPattern;
+}
+interface RegexAnticipationPattern {
+    start: RegExp;
+    end: RegExp;
+    full?: RegExp;
+    append: string | ((match: RegExpExecArray, content: string) => string);
+    maxScanChars?: number;
 }
 interface ASTInlinePlugin extends InlinePlugin {
     visit: (node: InlineNode, ctx: {
@@ -311,4 +334,4 @@ interface CoalescingMetrics {
     insertChildCoalesced: number;
 }
 
-export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };
+export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type FormatAnticipationConfig, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexAnticipationPattern, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };

package/dist/types.d.ts

@@ -39,6 +39,14 @@ interface MixedContentSegment {
     status?: "pending" | "compiled" | "error";
     error?: string;
 }
+type FormatAnticipationConfig = boolean | {
+    inline?: boolean;
+    mathInline?: boolean;
+    mathBlock?: boolean;
+    html?: boolean;
+    mdx?: boolean;
+    regex?: boolean;
+};
 interface InlineHtmlDescriptor {
     tagName: string;
     attributes: Record<string, string>;
@@ -114,6 +122,10 @@ type WorkerIn = {
         mdx?: boolean;
         tables?: boolean;
         callouts?: boolean;
+        math?: boolean;
+        formatAnticipation?: FormatAnticipationConfig;
+        liveCodeHighlighting?: boolean;
+        mdxComponentNames?: string[];
     };
     mdx?: {
         compileMode?: "server" | "worker";
@@ -179,6 +191,17 @@ interface RegexInlinePlugin extends InlinePlugin {
      * the regex is skipped for that node.
      */
     fastCheck?: (text: string) => boolean;
+    /**
+     * Optional streaming anticipation config. Only used when formatAnticipation.regex is enabled.
+     */
+    anticipation?: RegexAnticipationPattern;
+}
+interface RegexAnticipationPattern {
+    start: RegExp;
+    end: RegExp;
+    full?: RegExp;
+    append: string | ((match: RegExpExecArray, content: string) => string);
+    maxScanChars?: number;
 }
 interface ASTInlinePlugin extends InlinePlugin {
     visit: (node: InlineNode, ctx: {
@@ -311,4 +334,4 @@ interface CoalescingMetrics {
     insertChildCoalesced: number;
 }
 
-export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };
+export { type ASTInlinePlugin, type Block, type CoalescingMetrics, type CompiledMdxModule, type FormatAnticipationConfig, type InlineHtmlDescriptor, type InlineNode, type InlinePlugin, LANGUAGE_ALIASES, type MixedContentSegment, type NodePath, type NodeSnapshot, PATCH_ROOT_ID, type Patch, type PatchMetrics, type PerformanceMetrics, type ProtectedRange, type ProtectedRangeKind, type RegexAnticipationPattern, type RegexInlinePlugin, type SetPropsBatchEntry, type WorkerErrorPayload, type WorkerIn, type WorkerOut, type WorkerPhase };

package/dist/worker-html-sanitizer.cjs

@@ -37,9 +37,23 @@ var rehypeParse = __toESM(require("rehype-parse"), 1);
 var rehypeSanitize = __toESM(require("rehype-sanitize"), 1);
 var rehypeStringify = __toESM(require("rehype-stringify"), 1);
 var import_unified = require("unified");
-var { defaultSchema } = rehypeSanitize;
+var rehypeSanitizeModule = rehypeSanitize;
+var defaultSchema = rehypeSanitizeModule.defaultSchema;
+var resolvePlugin = (mod) => {
+  if (typeof mod === "function") return mod;
+  if (mod && typeof mod.default === "function") {
+    return mod.default;
+  }
+  if (mod && typeof mod.default?.default === "function") {
+    return mod.default?.default;
+  }
+  return mod;
+};
+var rehypeParsePlugin = resolvePlugin(rehypeParse);
+var rehypeSanitizePlugin = resolvePlugin(rehypeSanitizeModule);
+var rehypeStringifyPlugin = resolvePlugin(rehypeStringify);
 var SANITIZED_SCHEMA = createSchema();
-var sanitizeProcessor = (0, import_unified.unified)().use(rehypeParse.default, { fragment: true }).use(rehypeSanitize.default, SANITIZED_SCHEMA).use(rehypeStringify.default).freeze();
+var sanitizeProcessor = (0, import_unified.unified)().use(rehypeParsePlugin, { fragment: true }).use(rehypeSanitizePlugin, SANITIZED_SCHEMA).use(rehypeStringifyPlugin).freeze();
 function sanitizeHtmlInWorker(html) {
   if (!html) return "";
   try {

package/dist/worker-html-sanitizer.mjs

@@ -3,9 +3,23 @@ import * as rehypeParse from "rehype-parse";
 import * as rehypeSanitize from "rehype-sanitize";
 import * as rehypeStringify from "rehype-stringify";
 import { unified } from "unified";
-var { defaultSchema } = rehypeSanitize;
+var rehypeSanitizeModule = rehypeSanitize;
+var defaultSchema = rehypeSanitizeModule.defaultSchema;
+var resolvePlugin = (mod) => {
+  if (typeof mod === "function") return mod;
+  if (mod && typeof mod.default === "function") {
+    return mod.default;
+  }
+  if (mod && typeof mod.default?.default === "function") {
+    return mod.default?.default;
+  }
+  return mod;
+};
+var rehypeParsePlugin = resolvePlugin(rehypeParse);
+var rehypeSanitizePlugin = resolvePlugin(rehypeSanitizeModule);
+var rehypeStringifyPlugin = resolvePlugin(rehypeStringify);
 var SANITIZED_SCHEMA = createSchema();
-var sanitizeProcessor = unified().use(rehypeParse.default, { fragment: true }).use(rehypeSanitize.default, SANITIZED_SCHEMA).use(rehypeStringify.default).freeze();
+var sanitizeProcessor = unified().use(rehypeParsePlugin, { fragment: true }).use(rehypeSanitizePlugin, SANITIZED_SCHEMA).use(rehypeStringifyPlugin).freeze();
 function sanitizeHtmlInWorker(html) {
   if (!html) return "";
   try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-mdx/core",
-  "version": "0.0.2",
+  "version": "0.1.0",
   "description": "Core types, snapshot utilities, and perf helpers for the Streaming Markdown V2 stack",
   "license": "MIT",
   "repository": {
@@ -75,6 +75,11 @@
       "types": "./dist/streaming/custom-matcher.d.ts",
       "import": "./dist/streaming/custom-matcher.mjs",
       "require": "./dist/streaming/custom-matcher.cjs"
+    },
+    "./streaming/inline-streaming": {
+      "types": "./dist/streaming/inline-streaming.d.ts",
+      "import": "./dist/streaming/inline-streaming.mjs",
+      "require": "./dist/streaming/inline-streaming.cjs"
     }
   },
   "files": [
@@ -85,7 +90,7 @@
   "sideEffects": false,
   "scripts": {
     "build": "tsup",
-    "test": "tsx __tests__/*.test.ts",
+    "test": "tsx ../../scripts/run-tests.ts __tests__",
     "clean": "rm -rf dist",
     "prepack": "npm run build"
   },