@dxos/react-ui-markdown 0.8.4-main.9be5663bfe → 0.8.4-main.abd8ff62ef

package/src/MarkdownStream/stream.ts

@@ -0,0 +1,229 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Stream from 'effect/Stream';
+
+import { Obj } from '@dxos/echo';
+
+export const renderObjectLink = (obj: Obj.Unknown, block?: boolean) =>
+  `${block ? '!' : ''}[${Obj.getLabel(obj)}](${Obj.getDXN(obj).toString()})`;
+
+export type StreamerOptions = {
+  /**
+   * How to subdivide plain-text spans before emitting them downstream.
+   *  - `'span'` (default) — keep entire spans intact; one CM dispatch per chunk that arrived at the source.
+   *  - `'word'` — split text spans at whitespace boundaries (whitespace runs become their own tokens).
+   *  - `'character'` — split text spans into individual characters.
+   * XML/HTML fragments (`<tag>…</tag>`, self-closing tags, hyphenated custom elements) are always
+   * emitted as one token regardless of `chunkSize`, otherwise widget mounting would see partial markup.
+   */
+  chunkSize?: 'character' | 'word' | 'span';
+  /**
+   * Inter-token delay in ms. Default `0`. Useful for slowing down rapid bursts so the
+   * downstream renderer (CodeMirror) can show a visible streaming cadence even when the
+   * source emits large chunks at once.
+   */
+  delayMs?: number;
+};
+
+/**
+ * Streams tokens to the consumer, keeping XML/HTML fragments intact.
+ *
+ * The cadence is controlled by `options.chunkSize`. `'span'` (default) preserves the
+ * one-token-per-source-chunk behaviour the function had originally. Use `'word'` or
+ * `'character'` to decouple the visible cadence from the source's chunk size — useful when
+ * the AI service emits large partial blocks but you want a smoother typewriter effect.
+ */
+export const createStreamer = (
+  source: Stream.Stream<string>,
+  { chunkSize = 'span', delayMs = 0 }: StreamerOptions = {},
+) => {
+  const subdivide =
+    chunkSize === 'span'
+      ? (token: string) => [token]
+      : (token: string) => (isXmlFragment(token) ? [token] : splitTextSpan(token, chunkSize));
+
+  let stream: Stream.Stream<string> = source.pipe(
+    Stream.flatMap((chunk) => Stream.fromIterable(splitFragments(chunk).flatMap(subdivide))),
+  );
+  if (delayMs > 0) {
+    stream = stream.pipe(Stream.tap(() => Effect.sleep(`${delayMs} millis`)));
+  }
+  return stream;
+};
+
+/** A token starts with `<` if and only if it is an XML/HTML fragment produced by `splitFragments`. */
+const isXmlFragment = (token: string): boolean => token.startsWith('<');
+
+/**
+ * Subdivide a non-XML text span into smaller tokens. `'word'` splits into runs of non-whitespace
+ * and runs of whitespace as separate tokens (so the renderer can display whitespace cadence too).
+ */
+const splitTextSpan = (span: string, chunkSize: 'character' | 'word'): string[] => {
+  if (chunkSize === 'character') {
+    return [...span];
+  }
+  return span.match(/\s+|\S+/g) ?? [span];
+};
+
+/** Matches opening tag names, including custom elements with hyphens (e.g. dom-widget). */
+const OPENING_TAG_NAME = /^<([a-zA-Z][\w-]*)(?:\s[^>]*)?>/;
+
+/**
+ * Splits text into chunks, preserving XML/HTML fragments.
+ */
+export const splitFragments = (text: string): string[] => {
+  // First tokenize with tags to get tags as complete tokens.
+  const initialTokens = splitSpans(text);
+  const tokens: string[] = [];
+
+  let i = 0;
+  while (i < initialTokens.length) {
+    const token = initialTokens[i];
+
+    // Check if this is an opening tag.
+    if (token.startsWith('<') && !token.startsWith('</') && !token.endsWith('/>')) {
+      const tagMatch = token.match(OPENING_TAG_NAME);
+      if (tagMatch) {
+        const tagName = tagMatch[1];
+        const closingTag = `</${tagName}>`;
+
+        // Collect tokens until we find the closing tag.
+        let fragment = token;
+        let foundClosing = false;
+        let j = i + 1;
+        while (j < initialTokens.length) {
+          fragment += initialTokens[j];
+          if (initialTokens[j] === closingTag) {
+            foundClosing = true;
+            break;
+          }
+          j++;
+        }
+
+        if (foundClosing) {
+          // Return the complete element as one token.
+          tokens.push(fragment);
+          i = j + 1;
+        } else {
+          // No closing tag found, just add the opening tag.
+          tokens.push(token);
+          i++;
+        }
+      } else {
+        // Not a valid opening tag.
+        tokens.push(token);
+        i++;
+      }
+    } else {
+      // Not an opening tag (could be closing tag, self-closing, or regular character).
+      tokens.push(token);
+      i++;
+    }
+  }
+
+  return tokens;
+};
+
+// TODO(burdon): Split into paragraphs and tags.
+// export const tokenizeWithTags = (text: string): string[] => {
+//   const tokens: string[] = [];
+//   let i = 0;
+//   while (i < text.length) {
+//     if (text[i] === '<') {
+//       // Find the closing bracket.
+//       const closeIndex = text.indexOf('>', i);
+//       if (closeIndex !== -1) {
+//         // Include the complete tag.
+//         tokens.push(text.slice(i, closeIndex + 1));
+//         i = closeIndex + 1;
+//       } else {
+//         // No closing bracket found, return the entire remaining fragment.
+//         tokens.push(text.slice(i));
+//         break;
+//       }
+//     } else {
+//       // Regular character.
+//       tokens.push(text[i]);
+//       i++;
+//     }
+//   }
+//   return tokens;
+// };
+
+export const splitSpans = (text: string): string[] => {
+  const spans: string[] = [];
+  let currentText = '';
+
+  let i = 0;
+  while (i < text.length) {
+    if (text[i] === '<') {
+      // If we have accumulated text, split it into sentences and push them.
+      if (currentText) {
+        // const sentences = splitSentences(currentText);
+        // spans.push(...sentences);
+        spans.push(currentText);
+        currentText = '';
+      }
+
+      // Find the closing bracket.
+      const closeIndex = text.indexOf('>', i);
+      if (closeIndex !== -1) {
+        // Include the complete tag.
+        spans.push(text.slice(i, closeIndex + 1));
+        i = closeIndex + 1;
+      } else {
+        // No closing bracket found, treat the rest as text.
+        currentText = text.slice(i);
+        break;
+      }
+    } else {
+      // Accumulate regular text.
+      currentText += text[i];
+      i++;
+    }
+  }
+
+  // Push any remaining text split into sentences.
+  if (currentText) {
+    spans.push(currentText);
+    // const sentences = splitSentences(currentText);
+    // spans.push(...sentences);
+  }
+
+  return spans;
+};
+
+/**
+ * Split text into sentences, preserving the sentence-ending punctuation.
+ */
+export const splitSentences = (text: string): string[] => {
+  // Match sentences ending with ., !, or ? followed by space or end of string.
+  // This regex captures the sentence including its ending punctuation.
+  const sentenceRegex = /[^.!?]*[.!?]+(?:\s+|$)/g;
+  const sentences: string[] = [];
+  let lastIndex = 0;
+  let match;
+
+  while ((match = sentenceRegex.exec(text)) !== null) {
+    sentences.push(match[0]);
+    lastIndex = match.index + match[0].length;
+  }
+
+  // If there's remaining text that doesn't end with punctuation, include it.
+  if (lastIndex < text.length) {
+    const remaining = text.slice(lastIndex);
+    if (remaining) {
+      sentences.push(remaining);
+    }
+  }
+
+  // If no sentences were found, return the original text.
+  if (sentences.length === 0 && text) {
+    sentences.push(text);
+  }
+
+  return sentences;
+};

package/src/MarkdownStream/testing/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './testing';

package/src/MarkdownStream/testing/testing.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Options for the streaming text generator.
+ */
+export type TextStreamOptions = {
+  /** Delay between chunks in ms. */
+  chunkDelay?: number;
+  /** Variance in timing (0-1). */
+  variance?: number;
+  /** Number of words per chunk. */
+  wordsPerChunk?: number;
+};
+
+/**
+ * Simulates word-by-word streaming (more natural for LLMs).
+ */
+export async function* textStream(
+  text: string,
+  options: TextStreamOptions = {},
+): AsyncGenerator<string, void, unknown> {
+  const { chunkDelay = 100, variance = 0.3, wordsPerChunk = 3 } = options;
+
+  // Split into words while preserving whitespace.
+  const words = text.match(/\S+|\s+/g) || [];
+
+  let i = 0;
+  while (i < words.length) {
+    // Collect multiple words for this chunk.
+    const chunkWords: string[] = [];
+
+    // Add words up to wordsPerChunk (counting only non-whitespace as words).
+    let wordCount = 0;
+    while (i < words.length && wordCount < wordsPerChunk) {
+      const word = words[i];
+      chunkWords.push(word);
+
+      // Only count non-whitespace as words.
+      if (word.trim()) {
+        wordCount++;
+      }
+      i++;
+    }
+
+    // Yield the chunk.
+    const chunk = chunkWords.join('');
+    yield chunk;
+
+    // Calculate delay based on chunk length.
+    const varianceMultiplier = 1 + (Math.random() - 0.5) * variance * 2;
+    const delay = chunkDelay * varianceMultiplier;
+    await new Promise((resolve) => setTimeout(resolve, delay));
+  }
+}

package/src/MarkdownStream/testing/text.md

@@ -0,0 +1,67 @@
+## Markdown
+
+Markdown is a lightweight markup language used to format plain text in a simple and readable way. It allows you to create structured documents using conventions for headings, lists, emphasis (bold/italic), links, images, code, blockquotes, tables, and horizontal rules.
+
+It’s widely used in:
+
+- documentation
+- note-taking
+- online writing
+
+There are task lists also:
+
+- [ ] Not done
+- [x] Done
+
+## Benefits
+
+Markdown is designed to be human-readable, meaning that even without rendering, the text remains understandable. It’s highly portable and supported across many platforms like GitHub, documentation tools, blogging systems, and note-taking apps.
+
+JSON fenced code:
+
+```json
+{
+  "hello": "world",
+  "items": [1, 2, 3, 4, 5]
+}
+```
+
+We can also have <xml>content</xml>.
+
+## Structured data
+
+Table:
+
+| Column 1 | Column 2 |
+| -------- | -------- |
+| Item 1   | Item 2   |
+| Item 3   | Item 4   |
+| Item 5   | Item 6   |
+
+There are also extended flavors of Markdown (like GitHub Flavored Markdown) that add features such as checkboxes, footnotes, and task lists, expanding its capabilities for more complex documents.
+
+Markdown’s simplicity makes it ideal for writing structured content quickly while keeping the source clean and readable.
+
+If you want, I can also break down how Markdown parsing actually works behind the scenes, which explains how these plain-text symbols get converted to formatted output. Do you want me to do that?
+
+## Summary
+
+Markdown is a quiet kind of craft,
+Hashes, dashes, lines that draft.
+Asterisks to tilt a phrase,
+Backticks cage a block of blaze.
+
+A link is just a name and route,
+No lacquered gloss to dress it out.
+It travels clean from eye to page,
+A small instruction on the stage.
+
+Lists march down in measured rows,
+Indented thoughts in steady prose.
+Blockquotes lean and speak aside,
+A softer voice, but not denied.
+
+No ornate fonts, no tricks of light,
+Just structure drawn in plainest white.
+Yet from that spare, unmoving art,
+A document can find its heart.

package/src/index.ts

@@ -2,4 +2,5 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './MarkdownViewer';
+export * from './MarkdownBlock';
+export * from './MarkdownStream';

package/dist/types/src/MarkdownViewer/MarkdownViewer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"MarkdownViewer.d.ts","sourceRoot":"","sources":["../../../../src/MarkdownViewer/MarkdownViewer.tsx"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,EAAE,KAAK,iBAAiB,EAAE,MAAM,OAAO,CAAC;AACtD,OAAsB,EAAE,KAAK,OAAO,IAAI,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAGrF,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAItD,MAAM,MAAM,mBAAmB,GAAG,eAAe,CAC/C,iBAAiB,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;CACjD,CAAC,CACH,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,cAAc,GAAI,+CAAoD,mBAAmB,sBASrG,CAAC"}

package/dist/types/src/MarkdownViewer/MarkdownViewer.stories.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"MarkdownViewer.stories.d.ts","sourceRoot":"","sources":["../../../../src/MarkdownViewer/MarkdownViewer.stories.tsx"],"names":[],"mappings":"AAIA,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAMjE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIlD,QAAA,MAAM,IAAI;;;;CAI6B,CAAC;AAExC,eAAe,IAAI,CAAC;AAEpB,KAAK,KAAK,GAAG,QAAQ,CAAC,OAAO,cAAc,CAAC,CAAC;AA+C7C,eAAO,MAAM,OAAO,EAAE,KAKrB,CAAC"}

package/dist/types/src/MarkdownViewer/index.d.ts

@@ -1,2 +0,0 @@
-export * from './MarkdownViewer';
-//# sourceMappingURL=index.d.ts.map

package/dist/types/src/MarkdownViewer/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/MarkdownViewer/index.ts"],"names":[],"mappings":"AAIA,cAAc,kBAAkB,CAAC"}