@byteatatime/mdstream 0.0.1 → 0.0.3

package/package.json

@@ -1,8 +1,25 @@
 {
   "name": "@byteatatime/mdstream",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "module": "src/index.ts",
+  "types": "src/index.ts",
   "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "import": "./src/index.ts"
+    }
+  },
+  "files": [
+    "src",
+    "package.json",
+    "README.md"
+  ],
+  "author": "ByteAtATime",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ByteAtATime/mdstream"
+  },
   "devDependencies": {
     "@types/bun": "latest",
     "@types/mdast": "^4.0.4"

package/src/index.ts

@@ -3,6 +3,28 @@ import remarkParse from "remark-parse";
 import type { Node, Root } from "mdast";
 import type { PluggableList } from "unified";
 
+/**
+ * A function that fixes up incomplete markdown syntax so it renders correctly.
+ * Applied only to the pending (active) block — finalized blocks are never mended.
+ *
+ * The `remend` package is the recommended choice:
+ *
+ * @example
+ * ```ts
+ * import remend from "remend";
+ * const parser = createStreamingParser({ mend: remend });
+ * ```
+ *
+ * @example With options:
+ * ```ts
+ * import remend from "remend";
+ * const parser = createStreamingParser({
+ *   mend: (source) => remend(source, { links: false }),
+ * });
+ * ```
+ */
+type MendFunction = (source: string) => string;
+
 type Block = {
   readonly ast: Node;
   readonly start: number;
@@ -18,10 +40,21 @@ type ParserState = {
 
 type ParserOptions = {
   plugins?: PluggableList;
+  /**
+   * A mending function that fixes incomplete markdown syntax in the active block.
+   * Only applied to the pending block — finalized blocks are never touched.
+   *
+   * The `remend` package is the recommended mending function:
+   * ```ts
+   * import remend from "remend";
+   * const parser = createStreamingParser({ mend: remend });
+   * ```
+   */
+  mend?: MendFunction;
 };
 
 const createStreamingParser = (options: ParserOptions = {}) => {
-  const { plugins = [] } = options;
+  const { plugins = [], mend } = options;
   const state: ParserState = {
     buffer: "",
     finalizedBlocks: [],
@@ -60,10 +93,12 @@ const createStreamingParser = (options: ParserOptions = {}) => {
       const secondToLastPosition = secondToLastNode?.position;
       const lastPosition = lastNode?.position;
 
-      if (secondToLastNode?.type === "paragraph" && 
-          secondToLastPosition && 
-          lastPosition &&
-          secondToLastPosition.end.offset === lastPosition.start.offset) {
+      if (
+        secondToLastNode?.type === "paragraph" &&
+        secondToLastPosition &&
+        lastPosition &&
+        secondToLastPosition.end.offset === lastPosition.start.offset
+      ) {
         finalizableNodes = finalizableNodes.slice(0, -1);
       }
     }
@@ -101,7 +136,9 @@ const createStreamingParser = (options: ParserOptions = {}) => {
   const getPendingBlock = (): Block | undefined => {
     if (!state.buffer) return undefined;
 
-    const ast = processor.parse(state.buffer) as Root;
+    const mendedBuffer = mend ? mend(state.buffer) : state.buffer;
+
+    const ast = processor.parse(mendedBuffer) as Root;
     const tree = processor.runSync(ast) as Root;
     const children = tree.children;
 
@@ -113,13 +150,14 @@ const createStreamingParser = (options: ParserOptions = {}) => {
     if (!firstNode || !position) return undefined;
 
     const start = state.totalOffset + position.start.offset!;
-    const end = state.totalOffset + position.end.offset!;
+    // end offset is relative to the original (unmended) buffer length
+    const end = state.totalOffset + state.buffer.length;
 
     return {
       ast: firstNode,
       start,
       end,
-      source: state.buffer.slice(position.start.offset!, position.end.offset!),
+      source: mendedBuffer.slice(position.start.offset!, position.end.offset!),
     };
   };
 
@@ -135,4 +173,4 @@ const createStreamingParser = (options: ParserOptions = {}) => {
 };
 
 export { createStreamingParser };
-export type { Block, ParserOptions };
+export type { Block, ParserOptions, MendFunction };

package/test/finalization.test.ts

@@ -1,115 +0,0 @@
-import { test, expect, describe } from 'bun:test';
-import { createStreamingParser } from '../src/index.ts';
-
-describe('Block Finalization', () => {
-  test('finalizes blocks when 2+ top-level nodes exist', () => {
-    const parser = createStreamingParser();
-    parser.append('# Hello\n\nThis is a paragraph.\n\n');
-    expect(parser.blocks.length).toBe(2);
-    expect(parser.blocks.map(b => b.ast.type)).toEqual(['heading', 'paragraph']);
-
-    const headingRef = parser.blocks[0];
-    parser.append('Another paragraph');
-
-    expect(parser.blocks.length).toBe(3);
-    expect(parser.blocks[0]).toBe(headingRef);
-    expect(parser.blocks.map(b => b.ast.type)).toEqual(['heading', 'paragraph', 'paragraph']);
-  });
-});
-
-describe('Block Stability', () => {
-  test('finalized blocks maintain stable object references across multiple appends', () => {
-    const parser = createStreamingParser();
-
-    parser.append('# First\n\nPara 1\n\nPara 2\n\n');
-    const block0 = parser.blocks[0];
-    const block1 = parser.blocks[1];
-
-    expect(parser.blocks.length).toBe(3);
-
-    parser.append('Para 3');
-    const newBlock2 = parser.blocks[2]!;
-
-    expect(parser.blocks.length).toBe(4);
-    expect(parser.blocks[0]).toBe(block0);
-    expect(parser.blocks[1]).toBe(block1);
-    expect(newBlock2.ast.type).toBe('paragraph');
-
-    parser.append('\n\nPara 4');
-    expect(parser.blocks.length).toBe(5);
-    expect(parser.blocks[0]).toBe(block0);
-    expect(parser.blocks[1]).toBe(block1);
-    expect(parser.blocks[2]).toBe(newBlock2);
-  });
-
-  test('pending block gets new reference each cycle while finalized blocks stay stable', () => {
-    const parser = createStreamingParser();
-
-    parser.append('# Heading\n\nPara 1');
-    const block0 = parser.blocks[0]!;
-    const pending1 = parser.blocks[1]!;
-
-    expect(parser.blocks.length).toBe(2);
-
-    parser.append(' continues');
-    const pending2 = parser.blocks[1]!;
-
-    expect(parser.blocks.length).toBe(2);
-    expect(parser.blocks[0]).toBe(block0);
-    expect(pending1).not.toBe(pending2);
-    expect(pending1.source).toBe('Para 1');
-    expect(pending2.source).toBe('Para 1 continues');
-  });
-
-  test('blocks array can be recreated but finalized block references remain stable', () => {
-    const parser = createStreamingParser();
-
-    parser.append('# First\n\nSecond\n\nThird\n\n');
-    const firstRef = parser.blocks[0];
-    const secondRef = parser.blocks[1];
-
-    const firstArray = parser.blocks;
-    parser.append('Fourth');
-
-    const secondArray = parser.blocks;
-
-    expect(firstArray).not.toBe(secondArray);
-    expect(secondArray[0]).toBe(firstRef);
-    expect(secondArray[1]).toBe(secondRef);
-    expect(secondArray.length).toBe(4);
-  });
-
-  test('all finalized blocks keep references when many blocks accumulate', () => {
-    const parser = createStreamingParser();
-    const refs: any[] = [];
-
-    parser.append('# 1\n\n2\n\n3\n\n4\n\n5\n\n6\n\n');
-    for (let i = 0; i < 5; i++) {
-      refs.push(parser.blocks[i]);
-    }
-
-    parser.append('7\n\n8\n\n9\n\n');
-
-    for (let i = 0; i < 5; i++) {
-      expect(parser.blocks[i]).toBe(refs[i]);
-    }
-
-    expect(parser.blocks.length).toBe(9);
-  });
-
-  test('block stability with different markdown types', () => {
-    const parser = createStreamingParser();
-
-    parser.append('# Heading\n\n- List item\n\n```\ncode\n```\n\nParagraph\n\n');
-    const heading = parser.blocks[0];
-    const list = parser.blocks[1];
-    const code = parser.blocks[2];
-
-    parser.append('More text');
-
-    expect(parser.blocks[0]).toBe(heading);
-    expect(parser.blocks[1]).toBe(list);
-    expect(parser.blocks[2]).toBe(code);
-    expect(parser.blocks[3]!.ast.type).toBe('paragraph');
-  });
-});

package/test/index.test.ts

@@ -1,48 +0,0 @@
-import { test, expect, describe } from 'bun:test';
-import { createStreamingParser } from '../src/index.ts';
-
-describe('Initialization', () => {
-  test('creates empty blocks initially', () => {
-    const parser = createStreamingParser();
-    expect(parser.blocks).toEqual([]);
-  });
-});
-
-describe('Basic Parsing', () => {
-  test('parses heading', () => {
-    const parser = createStreamingParser();
-    parser.append('# Hello');
-
-    const block = parser.blocks[0];
-    expect(block?.ast.type).toBe('heading');
-    expect(block?.source).toBe('# Hello');
-  });
-
-  test('parses multiple blocks', () => {
-    const parser = createStreamingParser();
-    parser.append('# Hello\n\nParagraph one\n\nParagraph two');
-
-    const types = parser.blocks.map((b) => b.ast.type);
-    expect(types).toEqual(['heading', 'paragraph', 'paragraph']);
-  });
-
-  test('calculates correct block offsets', () => {
-    const parser = createStreamingParser();
-    parser.append('# Hello\n\nWorld');
-
-    const [heading, paragraph] = parser.blocks;
-    expect(heading?.start).toBe(0);
-    expect(heading?.end).toBe(7);
-    expect(paragraph?.start).toBe(9);
-    expect(paragraph?.end).toBe(14);
-  });
-
-  test('captures heading depth', () => {
-    const parser = createStreamingParser();
-    parser.append('# Heading');
-
-    const heading = parser.blocks[0]?.ast as any;
-    expect(heading?.type).toBe('heading');
-    expect(heading?.depth).toBe(1);
-  });
-});

package/test/plugins.test.ts

@@ -1,46 +0,0 @@
-import { test, expect, describe } from "bun:test";
-import { createStreamingParser } from "../src/index.ts";
-
-describe("Custom Plugin Support", () => {
-  test("accepts plugins in options", () => {
-    const customPlugin = () => (tree: any) => {
-      for (const child of tree.children) {
-        (child as any).custom = true;
-      }
-    };
-
-    const parser = createStreamingParser({
-      plugins: [customPlugin],
-    });
-
-    parser.append("# Hello");
-
-    const block = parser.blocks[0];
-    expect(block?.ast.type).toBe("heading");
-    expect((block?.ast as any).custom).toBe(true);
-  });
-
-  test("accepts multiple plugins", () => {
-    const plugin1 = () => (tree: any) => {
-      for (const child of tree.children) {
-        (child as any).plugin1Applied = true;
-      }
-    };
-
-    const plugin2 = () => (tree: any) => {
-      for (const child of tree.children) {
-        (child as any).plugin2Applied = true;
-      }
-    };
-
-    const parser = createStreamingParser({
-      plugins: [plugin1, plugin2],
-    });
-
-    parser.append("# Hello");
-
-    const block = parser.blocks[0];
-    expect((block?.ast as any).plugin1Applied).toBe(true);
-    expect((block?.ast as any).plugin2Applied).toBe(true);
-  });
-});

package/test/setext.test.ts

@@ -1,103 +0,0 @@
-import { test, expect, describe } from 'bun:test';
-import { createStreamingParser } from '../src/index.ts';
-
-describe('Setext Headings', () => {
-  test('setext heading with === creates h1', () => {
-    const parser = createStreamingParser();
-    parser.append('Hello World\n===');
-
-    const block = parser.blocks[0];
-    expect(block?.ast.type).toBe('heading');
-    const heading = block?.ast as any;
-    expect(heading.depth).toBe(1);
-  });
-
-  test('setext heading with --- creates h2', () => {
-    const parser = createStreamingParser();
-    parser.append('Hello World\n---');
-
-    const block = parser.blocks[0];
-    expect(block?.ast.type).toBe('heading');
-    const heading = block?.ast as any;
-    expect(heading.depth).toBe(2);
-  });
-
-  test('setext heading does not prematurely finalize paragraph when underline arrives', () => {
-    const parser = createStreamingParser();
-    parser.append('Hello World\n');
-
-    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
-
-    parser.append('===');
-
-    const block = parser.blocks[0];
-    expect(block?.ast.type).toBe('heading');
-    const heading = block?.ast as any;
-    expect(heading.depth).toBe(1);
-  });
-
-  test('setext heading with preceding content finalizes correctly', () => {
-    const parser = createStreamingParser();
-    parser.append('# First\n\nHello World\n');
-
-    expect(parser.blocks[0]?.ast.type).toBe('heading');
-    expect(parser.blocks[1]?.ast.type).toBe('paragraph');
-
-    parser.append('===\n\nSecond paragraph');
-
-    expect(parser.blocks.length).toBe(3);
-    expect(parser.blocks[0]?.ast.type).toBe('heading');
-    expect(parser.blocks[1]?.ast.type).toBe('heading');
-    expect(parser.blocks[2]?.ast.type).toBe('paragraph');
-  });
-
-  test('setext underline only affects preceding paragraph', () => {
-    const parser = createStreamingParser();
-    parser.append('First paragraph\n\nSecond paragraph\n===\n\nThird paragraph');
-
-    expect(parser.blocks.length).toBe(3);
-    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
-    expect(parser.blocks[1]?.ast.type).toBe('heading');
-    expect(parser.blocks[2]?.ast.type).toBe('paragraph');
-  });
-
-  test('setext underline after blank line is separate paragraph', () => {
-    const parser = createStreamingParser();
-    parser.append('Paragraph 1\n\n');
-
-    expect(parser.blocks.length).toBe(1);
-    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
-    expect(parser.blocks[0]?.source).toBe('Paragraph 1');
-
-    parser.append('===');
-
-    expect(parser.blocks.length).toBe(2);
-    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
-    expect(parser.blocks[0]?.source).toBe('Paragraph 1');
-    expect(parser.blocks[1]?.ast.type).toBe('paragraph');
-    expect(parser.blocks[1]?.source).toBe('===');
-  });
-
-  test('setext underline after blank line does not cause data loss', () => {
-    const parser = createStreamingParser();
-    parser.append('First paragraph\n\n');
-    parser.append('===');
-
-    const blocks = parser.blocks;
-    expect(blocks.length).toBe(2);
-    
-    const hasFirstPara = blocks.some((b) => b.source.includes('First paragraph'));
-    expect(hasFirstPara).toBe(true);
-  });
-
-  test('setext underline with multiple blank lines', () => {
-    const parser = createStreamingParser();
-    parser.append('First\n\n\n===');
-
-    expect(parser.blocks.length).toBe(2);
-    expect(parser.blocks[0]?.ast.type).toBe('paragraph');
-    expect(parser.blocks[0]?.source).toBe('First');
-    expect(parser.blocks[1]?.ast.type).toBe('paragraph');
-    expect(parser.blocks[1]?.source).toBe('===');
-  });
-});

package/tsconfig.json

@@ -1,29 +0,0 @@
-{
-  "compilerOptions": {
-    // Environment setup & latest features
-    "lib": ["ESNext"],
-    "target": "ESNext",
-    "module": "Preserve",
-    "moduleDetection": "force",
-    "jsx": "react-jsx",
-    "allowJs": true,
-
-    // Bundler mode
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "verbatimModuleSyntax": true,
-    "noEmit": true,
-
-    // Best practices
-    "strict": true,
-    "skipLibCheck": true,
-    "noFallthroughCasesInSwitch": true,
-    "noUncheckedIndexedAccess": true,
-    "noImplicitOverride": true,
-
-    // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noPropertyAccessFromIndexSignature": false
-  }
-}