@byteatatime/mdstream 0.0.1

package/README.md

@@ -0,0 +1,15 @@
+# mdstream
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run src/index.ts
+```
+
+This project was created using `bun init` in bun v1.3.4. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.

package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@byteatatime/mdstream",
+  "version": "0.0.1",
+  "module": "src/index.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest",
+    "@types/mdast": "^4.0.4"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "dependencies": {
+    "mdast": "^3.0.0",
+    "remark": "^15.0.1"
+  }
+}

package/src/index.ts

@@ -0,0 +1,138 @@
+import { unified } from "unified";
+import remarkParse from "remark-parse";
+import type { Node, Root } from "mdast";
+import type { PluggableList } from "unified";
+
+type Block = {
+  readonly ast: Node;
+  readonly start: number;
+  readonly end: number;
+  readonly source: string;
+};
+
+type ParserState = {
+  buffer: string;
+  finalizedBlocks: Block[];
+  totalOffset: number;
+};
+
+type ParserOptions = {
+  plugins?: PluggableList;
+};
+
+const createStreamingParser = (options: ParserOptions = {}) => {
+  const { plugins = [] } = options;
+  const state: ParserState = {
+    buffer: "",
+    finalizedBlocks: [],
+    totalOffset: 0,
+  };
+
+  let processor = unified().use(remarkParse);
+  if (plugins.length > 0) {
+    processor = processor.use(plugins);
+  }
+
+  const processBuffer = (): void => {
+    if (!state.buffer) return;
+
+    const ast = processor.parse(state.buffer) as Root;
+    const tree = processor.runSync(ast) as Root;
+    const children = tree.children;
+
+    if (children.length < 2) {
+      return;
+    }
+
+    const lastNode = children[children.length - 1];
+    let finalizableNodes = children.slice(0, -1);
+
+    const lastNodeSource = lastNode?.position
+      ? state.buffer.slice(
+          lastNode.position.start.offset!,
+          lastNode.position.end.offset!,
+        )
+      : "";
+    const isSetextUnderline = /^(=+|-+)\s*$/.test(lastNodeSource);
+
+    if (isSetextUnderline && finalizableNodes.length > 0) {
+      const secondToLastNode = finalizableNodes[finalizableNodes.length - 1];
+      const secondToLastPosition = secondToLastNode?.position;
+      const lastPosition = lastNode?.position;
+
+      if (secondToLastNode?.type === "paragraph" && 
+          secondToLastPosition && 
+          lastPosition &&
+          secondToLastPosition.end.offset === lastPosition.start.offset) {
+        finalizableNodes = finalizableNodes.slice(0, -1);
+      }
+    }
+
+    for (const node of finalizableNodes) {
+      const position = node?.position;
+      if (node && position) {
+        const start = state.totalOffset + position.start.offset!;
+        const end = state.totalOffset + position.end.offset!;
+        state.finalizedBlocks.push({
+          ast: node,
+          start,
+          end,
+          source: state.buffer.slice(
+            position.start.offset!,
+            position.end.offset!,
+          ),
+        });
+      }
+    }
+
+    const lastPosition = lastNode?.position;
+    if (lastPosition) {
+      const newBufferStart = lastPosition.start.offset!;
+      state.buffer = state.buffer.slice(newBufferStart);
+      state.totalOffset += newBufferStart;
+    }
+  };
+
+  const append = (chunk: string): void => {
+    state.buffer += chunk;
+    processBuffer();
+  };
+
+  const getPendingBlock = (): Block | undefined => {
+    if (!state.buffer) return undefined;
+
+    const ast = processor.parse(state.buffer) as Root;
+    const tree = processor.runSync(ast) as Root;
+    const children = tree.children;
+
+    if (children.length === 0) return undefined;
+
+    const firstNode = children[0];
+    const position = firstNode?.position;
+
+    if (!firstNode || !position) return undefined;
+
+    const start = state.totalOffset + position.start.offset!;
+    const end = state.totalOffset + position.end.offset!;
+
+    return {
+      ast: firstNode,
+      start,
+      end,
+      source: state.buffer.slice(position.start.offset!, position.end.offset!),
+    };
+  };
+
+  return {
+    append,
+    get blocks(): readonly Block[] {
+      const pending = getPendingBlock();
+      return pending
+        ? [...state.finalizedBlocks, pending]
+        : state.finalizedBlocks;
+    },
+  };
+};
+
+export { createStreamingParser };
+export type { Block, ParserOptions };

package/test/finalization.test.ts

@@ -0,0 +1,115 @@
+import { test, expect, describe } from 'bun:test';
+import { createStreamingParser } from '../src/index.ts';
+
+describe('Block Finalization', () => {
+  test('finalizes blocks when 2+ top-level nodes exist', () => {
+    const parser = createStreamingParser();
+    parser.append('# Hello\n\nThis is a paragraph.\n\n');
+    expect(parser.blocks.length).toBe(2);
+    expect(parser.blocks.map(b => b.ast.type)).toEqual(['heading', 'paragraph']);
+
+    const headingRef = parser.blocks[0];
+    parser.append('Another paragraph');
+
+    expect(parser.blocks.length).toBe(3);
+    expect(parser.blocks[0]).toBe(headingRef);
+    expect(parser.blocks.map(b => b.ast.type)).toEqual(['heading', 'paragraph', 'paragraph']);
+  });
+});
+
+describe('Block Stability', () => {
+  test('finalized blocks maintain stable object references across multiple appends', () => {
+    const parser = createStreamingParser();
+
+    parser.append('# First\n\nPara 1\n\nPara 2\n\n');
+    const block0 = parser.blocks[0];
+    const block1 = parser.blocks[1];
+
+    expect(parser.blocks.length).toBe(3);
+
+    parser.append('Para 3');
+    const newBlock2 = parser.blocks[2]!;
+
+    expect(parser.blocks.length).toBe(4);
+    expect(parser.blocks[0]).toBe(block0);
+    expect(parser.blocks[1]).toBe(block1);
+    expect(newBlock2.ast.type).toBe('paragraph');
+
+    parser.append('\n\nPara 4');
+    expect(parser.blocks.length).toBe(5);
+    expect(parser.blocks[0]).toBe(block0);
+    expect(parser.blocks[1]).toBe(block1);
+    expect(parser.blocks[2]).toBe(newBlock2);
+  });
+
+  test('pending block gets new reference each cycle while finalized blocks stay stable', () => {
+    const parser = createStreamingParser();
+
+    parser.append('# Heading\n\nPara 1');
+    const block0 = parser.blocks[0]!;
+    const pending1 = parser.blocks[1]!;
+
+    expect(parser.blocks.length).toBe(2);
+
+    parser.append(' continues');
+    const pending2 = parser.blocks[1]!;
+
+    expect(parser.blocks.length).toBe(2);
+    expect(parser.blocks[0]).toBe(block0);
+    expect(pending1).not.toBe(pending2);
+    expect(pending1.source).toBe('Para 1');
+    expect(pending2.source).toBe('Para 1 continues');
+  });
+
+  test('blocks array can be recreated but finalized block references remain stable', () => {
+    const parser = createStreamingParser();
+
+    parser.append('# First\n\nSecond\n\nThird\n\n');
+    const firstRef = parser.blocks[0];
+    const secondRef = parser.blocks[1];
+
+    const firstArray = parser.blocks;
+    parser.append('Fourth');
+
+    const secondArray = parser.blocks;
+
+    expect(firstArray).not.toBe(secondArray);
+    expect(secondArray[0]).toBe(firstRef);
+    expect(secondArray[1]).toBe(secondRef);
+    expect(secondArray.length).toBe(4);
+  });
+
+  test('all finalized blocks keep references when many blocks accumulate', () => {
+    const parser = createStreamingParser();
+    const refs: any[] = [];
+
+    parser.append('# 1\n\n2\n\n3\n\n4\n\n5\n\n6\n\n');
+    for (let i = 0; i < 5; i++) {
+      refs.push(parser.blocks[i]);
+    }
+
+    parser.append('7\n\n8\n\n9\n\n');
+
+    for (let i = 0; i < 5; i++) {
+      expect(parser.blocks[i]).toBe(refs[i]);
+    }
+
+    expect(parser.blocks.length).toBe(9);
+  });
+
+  test('block stability with different markdown types', () => {
+    const parser = createStreamingParser();
+
+    parser.append('# Heading\n\n- List item\n\n```\ncode\n```\n\nParagraph\n\n');
+    const heading = parser.blocks[0];
+    const list = parser.blocks[1];
+    const code = parser.blocks[2];
+
+    parser.append('More text');
+
+    expect(parser.blocks[0]).toBe(heading);
+    expect(parser.blocks[1]).toBe(list);
+    expect(parser.blocks[2]).toBe(code);
+    expect(parser.blocks[3]!.ast.type).toBe('paragraph');
+  });
+});

package/test/index.test.ts

@@ -0,0 +1,48 @@
+import { test, expect, describe } from 'bun:test';
+import { createStreamingParser } from '../src/index.ts';
+
+describe('Initialization', () => {
+  test('creates empty blocks initially', () => {
+    const parser = createStreamingParser();
+    expect(parser.blocks).toEqual([]);
+  });
+});
+
+describe('Basic Parsing', () => {
+  test('parses heading', () => {
+    const parser = createStreamingParser();
+    parser.append('# Hello');
+
+    const block = parser.blocks[0];
+    expect(block?.ast.type).toBe('heading');
+    expect(block?.source).toBe('# Hello');
+  });
+
+  test('parses multiple blocks', () => {
+    const parser = createStreamingParser();
+    parser.append('# Hello\n\nParagraph one\n\nParagraph two');
+
+    const types = parser.blocks.map((b) => b.ast.type);
+    expect(types).toEqual(['heading', 'paragraph', 'paragraph']);
+  });
+
+  test('calculates correct block offsets', () => {
+    const parser = createStreamingParser();
+    parser.append('# Hello\n\nWorld');
+
+    const [heading, paragraph] = parser.blocks;
+    expect(heading?.start).toBe(0);
+    expect(heading?.end).toBe(7);
+    expect(paragraph?.start).toBe(9);
+    expect(paragraph?.end).toBe(14);
+  });
+
+  test('captures heading depth', () => {
+    const parser = createStreamingParser();
+    parser.append('# Heading');
+
+    const heading = parser.blocks[0]?.ast as any;
+    expect(heading?.type).toBe('heading');
+    expect(heading?.depth).toBe(1);
+  });
+});

package/test/plugins.test.ts

@@ -0,0 +1,46 @@
+import { test, expect, describe } from "bun:test";
+import { createStreamingParser } from "../src/index.ts";
+
+describe("Custom Plugin Support", () => {
+  test("accepts plugins in options", () => {
+    const customPlugin = () => (tree: any) => {
+      for (const child of tree.children) {
+        (child as any).custom = true;
+      }
+    };
+
+    const parser = createStreamingParser({
+      plugins: [customPlugin],
+    });
+
+    parser.append("# Hello");
+
+    const block = parser.blocks[0];
+    expect(block?.ast.type).toBe("heading");
+    expect((block?.ast as any).custom).toBe(true);
+  });
+
+  test("accepts multiple plugins", () => {
+    const plugin1 = () => (tree: any) => {
+      for (const child of tree.children) {
+        (child as any).plugin1Applied = true;
+      }
+    };
+
+    const plugin2 = () => (tree: any) => {
+      for (const child of tree.children) {
+        (child as any).plugin2Applied = true;
+      }
+    };
+
+    const parser = createStreamingParser({
+      plugins: [plugin1, plugin2],
+    });
+
+    parser.append("# Hello");
+
+    const block = parser.blocks[0];
+    expect((block?.ast as any).plugin1Applied).toBe(true);
+    expect((block?.ast as any).plugin2Applied).toBe(true);
+  });
+});

package/test/setext.test.ts

@@ -0,0 +1,103 @@
+import { test, expect, describe } from 'bun:test';
+import { createStreamingParser } from '../src/index.ts';
+
+describe('Setext Headings', () => {
+  test('setext heading with === creates h1', () => {
+    const parser = createStreamingParser();
+    parser.append('Hello World\n===');
+
+    const block = parser.blocks[0];
+    expect(block?.ast.type).toBe('heading');
+    const heading = block?.ast as any;
+    expect(heading.depth).toBe(1);
+  });
+
+  test('setext heading with --- creates h2', () => {
+    const parser = createStreamingParser();
+    parser.append('Hello World\n---');
+
+    const block = parser.blocks[0];
+    expect(block?.ast.type).toBe('heading');
+    const heading = block?.ast as any;
+    expect(heading.depth).toBe(2);
+  });
+
+  test('setext heading does not prematurely finalize paragraph when underline arrives', () => {
+    const parser = createStreamingParser();
+    parser.append('Hello World\n');
+
+    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
+
+    parser.append('===');
+
+    const block = parser.blocks[0];
+    expect(block?.ast.type).toBe('heading');
+    const heading = block?.ast as any;
+    expect(heading.depth).toBe(1);
+  });
+
+  test('setext heading with preceding content finalizes correctly', () => {
+    const parser = createStreamingParser();
+    parser.append('# First\n\nHello World\n');
+
+    expect(parser.blocks[0]?.ast.type).toBe('heading');
+    expect(parser.blocks[1]?.ast.type).toBe('paragraph');
+
+    parser.append('===\n\nSecond paragraph');
+
+    expect(parser.blocks.length).toBe(3);
+    expect(parser.blocks[0]?.ast.type).toBe('heading');
+    expect(parser.blocks[1]?.ast.type).toBe('heading');
+    expect(parser.blocks[2]?.ast.type).toBe('paragraph');
+  });
+
+  test('setext underline only affects preceding paragraph', () => {
+    const parser = createStreamingParser();
+    parser.append('First paragraph\n\nSecond paragraph\n===\n\nThird paragraph');
+
+    expect(parser.blocks.length).toBe(3);
+    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
+    expect(parser.blocks[1]?.ast.type).toBe('heading');
+    expect(parser.blocks[2]?.ast.type).toBe('paragraph');
+  });
+
+  test('setext underline after blank line is separate paragraph', () => {
+    const parser = createStreamingParser();
+    parser.append('Paragraph 1\n\n');
+
+    expect(parser.blocks.length).toBe(1);
+    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
+    expect(parser.blocks[0]?.source).toBe('Paragraph 1');
+
+    parser.append('===');
+
+    expect(parser.blocks.length).toBe(2);
+    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
+    expect(parser.blocks[0]?.source).toBe('Paragraph 1');
+    expect(parser.blocks[1]?.ast.type).toBe('paragraph');
+    expect(parser.blocks[1]?.source).toBe('===');
+  });
+
+  test('setext underline after blank line does not cause data loss', () => {
+    const parser = createStreamingParser();
+    parser.append('First paragraph\n\n');
+    parser.append('===');
+
+    const blocks = parser.blocks;
+    expect(blocks.length).toBe(2);
+    
+    const hasFirstPara = blocks.some((b) => b.source.includes('First paragraph'));
+    expect(hasFirstPara).toBe(true);
+  });
+
+  test('setext underline with multiple blank lines', () => {
+    const parser = createStreamingParser();
+    parser.append('First\n\n\n===');
+
+    expect(parser.blocks.length).toBe(2);
+    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
+    expect(parser.blocks[0]?.source).toBe('First');
+    expect(parser.blocks[1]?.ast.type).toBe('paragraph');
+    expect(parser.blocks[1]?.source).toBe('===');
+  });
+});

package/tsconfig.json

@@ -0,0 +1,29 @@
+{
+  "compilerOptions": {
+    // Environment setup & latest features
+    "lib": ["ESNext"],
+    "target": "ESNext",
+    "module": "Preserve",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitOverride": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}