@crewhaus/chunker 0.1.3 → 0.1.5

package/dist/index.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Catalog R12 `chunker` — split documents into retrievable chunks.
+ *
+ * Three strategies:
+ *
+ *   "fixed"    — split into fixed-character windows with optional overlap
+ *   "semantic" — `Intl.Segmenter` sentence boundaries, packed into windows
+ *   "markdown" — header-bounded chunks (split at `^#+ ` boundaries)
+ *
+ * Every strategy preserves the source bytes — `chunk(doc).map(c => c.text).join("")`
+ * (mod overlap) reconstructs the doc, modulo any whitespace that
+ * sat at chunk boundaries. The `id` field is `<docId>:<index>:<startOffset>`
+ * for stable reference back to the source.
+ *
+ * Layer R12. Pairs with `embedder`, `vector-store`, `pipeline-engine`.
+ */
+import { CrewhausError } from "@crewhaus/errors";
+export type ChunkStrategy = "fixed" | "semantic" | "markdown";
+export type ChunkOptions = {
+    readonly strategy: ChunkStrategy;
+    /** Window size: chars for "fixed", sentences for "semantic", chars for "markdown" sub-chunking. */
+    readonly size: number;
+    /** Overlap (in the same units as size). Defaults to 0. */
+    readonly overlap?: number;
+    /** Locale for `Intl.Segmenter` ("semantic" only). Defaults to "en". */
+    readonly locale?: string;
+};
+export type Document = {
+    /** Stable identifier (URL, path, hash). Used to derive chunk ids. */
+    readonly id: string;
+    readonly text: string;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+};
+export type Chunk = {
+    readonly id: string;
+    readonly docId: string;
+    readonly index: number;
+    /** Inclusive byte offset in `doc.text` where the chunk begins. */
+    readonly startOffset: number;
+    /** Exclusive byte offset where the chunk ends. */
+    readonly endOffset: number;
+    readonly text: string;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+};
+export declare class ChunkerError extends CrewhausError {
+    readonly name = "ChunkerError";
+    constructor(message: string, cause?: unknown);
+}
+export declare function chunk(doc: Document, opts: ChunkOptions): Chunk[];

package/dist/index.js

@@ -0,0 +1,175 @@
+/**
+ * Catalog R12 `chunker` — split documents into retrievable chunks.
+ *
+ * Three strategies:
+ *
+ *   "fixed"    — split into fixed-character windows with optional overlap
+ *   "semantic" — `Intl.Segmenter` sentence boundaries, packed into windows
+ *   "markdown" — header-bounded chunks (split at `^#+ ` boundaries)
+ *
+ * Every strategy preserves the source bytes — `chunk(doc).map(c => c.text).join("")`
+ * (mod overlap) reconstructs the doc, modulo any whitespace that
+ * sat at chunk boundaries. The `id` field is `<docId>:<index>:<startOffset>`
+ * for stable reference back to the source.
+ *
+ * Layer R12. Pairs with `embedder`, `vector-store`, `pipeline-engine`.
+ */
+import { CrewhausError } from "@crewhaus/errors";
+export class ChunkerError extends CrewhausError {
+    name = "ChunkerError";
+    constructor(message, cause) {
+        super("config", message, cause);
+    }
+}
+function makeChunkId(docId, index, start) {
+    return `${docId}:${index}:${start}`;
+}
+function chunkFixed(doc, opts) {
+    if (opts.size <= 0)
+        throw new ChunkerError("size must be positive");
+    const overlap = opts.overlap ?? 0;
+    if (overlap < 0 || overlap >= opts.size) {
+        throw new ChunkerError(`overlap (${overlap}) must be in [0, size) (size=${opts.size})`);
+    }
+    const out = [];
+    const stride = opts.size - overlap;
+    let start = 0;
+    let index = 0;
+    while (start < doc.text.length) {
+        const end = Math.min(start + opts.size, doc.text.length);
+        out.push({
+            id: makeChunkId(doc.id, index, start),
+            docId: doc.id,
+            index,
+            startOffset: start,
+            endOffset: end,
+            text: doc.text.slice(start, end),
+            ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
+        });
+        if (end >= doc.text.length)
+            break;
+        start += stride;
+        index += 1;
+    }
+    return out;
+}
+function chunkSemantic(doc, opts) {
+    if (opts.size <= 0)
+        throw new ChunkerError("size must be positive");
+    const overlap = opts.overlap ?? 0;
+    // Intl.Segmenter is widely available in modern Node + Bun. An invalid
+    // `locale` makes the constructor throw a raw `RangeError`; wrap it so callers
+    // see the package's typed `ChunkerError` contract (with the original as cause).
+    let segmenter;
+    try {
+        segmenter = new Intl.Segmenter(opts.locale ?? "en", { granularity: "sentence" });
+    }
+    catch (cause) {
+        throw new ChunkerError(`invalid locale ${JSON.stringify(opts.locale)}`, cause);
+    }
+    const sentences = [];
+    for (const seg of segmenter.segment(doc.text)) {
+        if (seg.segment.trim().length === 0)
+            continue;
+        const start = seg.index;
+        const end = start + seg.segment.length;
+        sentences.push({ text: seg.segment, start, end });
+    }
+    if (sentences.length === 0)
+        return [];
+    const out = [];
+    let i = 0;
+    let index = 0;
+    while (i < sentences.length) {
+        const window = sentences.slice(i, Math.min(i + opts.size, sentences.length));
+        if (window.length === 0)
+            break;
+        const startOffset = window[0]?.start ?? 0;
+        const endOffset = window[window.length - 1]?.end ?? doc.text.length;
+        const text = doc.text.slice(startOffset, endOffset);
+        out.push({
+            id: makeChunkId(doc.id, index, startOffset),
+            docId: doc.id,
+            index,
+            startOffset,
+            endOffset,
+            text,
+            ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
+        });
+        index += 1;
+        if (i + opts.size >= sentences.length)
+            break;
+        i += Math.max(1, opts.size - overlap);
+    }
+    return out;
+}
+function chunkMarkdown(doc, opts) {
+    // Split the document into header-bounded sections, then sub-chunk each
+    // section with the "fixed" strategy if it's longer than `opts.size`.
+    const headerRe = /(^|\n)(#{1,6}\s)/g;
+    const sectionRanges = [];
+    const matches = [0];
+    let m;
+    // biome-ignore lint/suspicious/noAssignInExpressions: classical regex iter
+    while ((m = headerRe.exec(doc.text)) !== null) {
+        const headerStart = m[1] === "\n" ? m.index + 1 : m.index;
+        if (headerStart !== 0 && !matches.includes(headerStart))
+            matches.push(headerStart);
+    }
+    matches.push(doc.text.length);
+    for (let i = 0; i < matches.length - 1; i += 1) {
+        const start = matches[i];
+        const end = matches[i + 1];
+        if (end > start)
+            sectionRanges.push({ start, end });
+    }
+    const out = [];
+    let chunkIndex = 0;
+    for (const range of sectionRanges) {
+        const sliceLen = range.end - range.start;
+        if (sliceLen <= opts.size) {
+            out.push({
+                id: makeChunkId(doc.id, chunkIndex, range.start),
+                docId: doc.id,
+                index: chunkIndex,
+                startOffset: range.start,
+                endOffset: range.end,
+                text: doc.text.slice(range.start, range.end),
+                ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
+            });
+            chunkIndex += 1;
+            continue;
+        }
+        // Sub-chunk via fixed strategy on this section.
+        const sub = chunkFixed({
+            id: doc.id,
+            text: doc.text.slice(range.start, range.end),
+            ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
+        }, opts);
+        for (const c of sub) {
+            out.push({
+                ...c,
+                id: makeChunkId(doc.id, chunkIndex, range.start + c.startOffset),
+                index: chunkIndex,
+                startOffset: range.start + c.startOffset,
+                endOffset: range.start + c.endOffset,
+            });
+            chunkIndex += 1;
+        }
+    }
+    return out;
+}
+export function chunk(doc, opts) {
+    switch (opts.strategy) {
+        case "fixed":
+            return chunkFixed(doc, opts);
+        case "semantic":
+            return chunkSemantic(doc, opts);
+        case "markdown":
+            return chunkMarkdown(doc, opts);
+        default: {
+            const exhaustive = opts.strategy;
+            throw new ChunkerError(`unknown strategy "${exhaustive}"`);
+        }
+    }
+}

package/package.json

@@ -1,18 +1,21 @@
 {
   "name": "@crewhaus/chunker",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "type": "module",
   "description": "Document chunking — fixed / semantic (Intl.Segmenter) / markdown strategies",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "exports": {
-    ".": "./src/index.ts"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
   },
   "scripts": {
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/errors": "0.1.3"
+    "@crewhaus/errors": "0.1.5"
   },
   "license": "Apache-2.0",
   "author": {
@@ -32,5 +35,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "files": ["src", "README.md", "LICENSE", "NOTICE"]
+  "files": ["dist", "README.md", "LICENSE", "NOTICE"]
 }

package/src/index.test.ts

@@ -1,235 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import { ChunkerError, chunk } from "./index";
-
-describe("fixed strategy", () => {
-  test("splits a long doc into fixed-size windows", () => {
-    const text = "0123456789".repeat(20);
-    const out = chunk({ id: "doc1", text }, { strategy: "fixed", size: 50 });
-    expect(out.length).toBe(4);
-    expect(out[0]?.text.length).toBe(50);
-    expect(out[3]?.text.length).toBe(50);
-  });
-
-  test("respects overlap when stride < size", () => {
-    const text = "0123456789".repeat(10); // 100 chars
-    const out = chunk({ id: "doc", text }, { strategy: "fixed", size: 30, overlap: 10 });
-    // stride = 20, so chunks start at 0, 20, 40, 60, 80
-    expect(out.length).toBe(5);
-    expect(out[0]?.startOffset).toBe(0);
-    expect(out[1]?.startOffset).toBe(20);
-    expect(out[4]?.startOffset).toBe(80);
-  });
-
-  test("rejects overlap >= size", () => {
-    expect(() =>
-      chunk({ id: "x", text: "abc" }, { strategy: "fixed", size: 5, overlap: 5 }),
-    ).toThrow(ChunkerError);
-  });
-
-  test("preserves source bytes (T9 reconstruction with overlap=0)", () => {
-    const text = "abcdefghijklmnopqrstuvwxyz";
-    const out = chunk({ id: "x", text }, { strategy: "fixed", size: 5 });
-    expect(out.map((c) => c.text).join("")).toBe(text);
-  });
-
-  test("ids carry docId + index + startOffset", () => {
-    const out = chunk({ id: "doc99", text: "hello" }, { strategy: "fixed", size: 3 });
-    expect(out[0]?.id).toBe("doc99:0:0");
-    expect(out[1]?.id).toBe("doc99:1:3");
-  });
-
-  test("metadata flows through", () => {
-    const out = chunk(
-      { id: "x", text: "abc", metadata: { kind: "doc" } },
-      { strategy: "fixed", size: 2 },
-    );
-    expect(out[0]?.metadata).toEqual({ kind: "doc" });
-  });
-});
-
-describe("semantic strategy", () => {
-  test("groups sentences into windows of N sentences", () => {
-    const text = "Alpha. Bravo. Charlie. Delta. Echo. Foxtrot.";
-    const out = chunk({ id: "doc", text }, { strategy: "semantic", size: 2 });
-    // 6 sentences, window 2 → 3 chunks
-    expect(out.length).toBe(3);
-  });
-
-  test("empty text yields no chunks", () => {
-    const out = chunk({ id: "doc", text: "" }, { strategy: "semantic", size: 3 });
-    expect(out).toEqual([]);
-  });
-
-  test("whitespace-only text yields no chunks (segments trimmed away)", () => {
-    const out = chunk({ id: "doc", text: "   \n\t  " }, { strategy: "semantic", size: 2 });
-    expect(out).toEqual([]);
-  });
-
-  test("overlap produces overlapping sentence windows", () => {
-    const text = "Alpha. Bravo. Charlie. Delta. Echo. Foxtrot.";
-    const out = chunk({ id: "doc", text }, { strategy: "semantic", size: 2, overlap: 1 });
-    // stride = max(1, 2 - 1) = 1, so a size-2 window starts at sentences
-    // 0,1,2,3,4 (the loop breaks once i + size reaches the 6-sentence end): 5 chunks.
-    expect(out.length).toBe(5);
-    // Consecutive windows share a sentence (overlap), so each later window
-    // starts strictly before the previous window's end.
-    expect(out[1]?.startOffset).toBeLessThan(out[0]?.endOffset ?? -1);
-  });
-
-  test("overlap >= size still terminates (stride clamped to 1)", () => {
-    const text = "Alpha. Bravo. Charlie.";
-    const out = chunk({ id: "doc", text }, { strategy: "semantic", size: 2, overlap: 5 });
-    // No infinite loop; stride floored at 1 yields one window per sentence
-    // until the final window reaches the end.
-    expect(out.length).toBe(2);
-  });
-
-  test("rejects non-positive size", () => {
-    expect(() => chunk({ id: "x", text: "A. B." }, { strategy: "semantic", size: 0 })).toThrow(
-      ChunkerError,
-    );
-  });
-
-  test("metadata flows through semantic chunks", () => {
-    const out = chunk(
-      { id: "doc", text: "Alpha. Bravo.", metadata: { lang: "en" } },
-      { strategy: "semantic", size: 1 },
-    );
-    expect(out.length).toBe(2);
-    expect(out[0]?.metadata).toEqual({ lang: "en" });
-    expect(out[1]?.metadata).toEqual({ lang: "en" });
-  });
-
-  test("a valid non-default locale is accepted", () => {
-    const out = chunk(
-      { id: "doc", text: "Hallo. Welt." },
-      { strategy: "semantic", size: 1, locale: "de" },
-    );
-    expect(out.length).toBe(2);
-  });
-
-  test("an invalid locale throws ChunkerError (not a raw RangeError)", () => {
-    let caught: unknown;
-    try {
-      chunk(
-        { id: "doc", text: "Hello. World." },
-        { strategy: "semantic", size: 2, locale: "not a valid tag!!!" },
-      );
-    } catch (e) {
-      caught = e;
-    }
-    expect(caught).toBeInstanceOf(ChunkerError);
-    // The original RangeError is preserved as the cause for debugging.
-    expect((caught as Error).cause).toBeInstanceOf(RangeError);
-    expect((caught as Error).message).toContain("invalid locale");
-  });
-
-  test("preserves source bytes across a single full window", () => {
-    const text = "Alpha. Bravo. Charlie.";
-    const out = chunk({ id: "doc", text }, { strategy: "semantic", size: 10 });
-    expect(out.length).toBe(1);
-    const only = out[0];
-    expect(only).toBeDefined();
-    // The single window spans from the first sentence start to the last end.
-    expect(text.slice(only?.startOffset, only?.endOffset)).toBe(only?.text ?? "");
-  });
-});
-
-describe("markdown strategy", () => {
-  test("splits at headers", () => {
-    const md = `# A
-para
-## B
-para
-### C
-para`;
-    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 100 });
-    expect(out.length).toBe(3);
-    expect(out[0]?.text.startsWith("# A")).toBe(true);
-    expect(out[1]?.text.startsWith("## B")).toBe(true);
-    expect(out[2]?.text.startsWith("### C")).toBe(true);
-  });
-
-  test("sub-chunks long sections via the fixed strategy", () => {
-    const long = "x".repeat(120);
-    const md = `# Big\n${long}\n## Small\nshort`;
-    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 50 });
-    // Section 1 is ~125 chars → sub-chunked into 3 pieces (50/50/25-ish).
-    // Section 2 is short → 1 piece.
-    expect(out.length).toBeGreaterThan(2);
-  });
-
-  test("sub-chunk ids and offsets are remapped to absolute document positions", () => {
-    const long = "y".repeat(120);
-    const md = `# Big\n${long}`; // single section, > size → sub-chunked
-    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 50 });
-    expect(out.length).toBeGreaterThan(1);
-    // Indices are contiguous and ids encode absolute startOffset.
-    out.forEach((c, i) => {
-      expect(c.index).toBe(i);
-      expect(c.id).toBe(`doc:${i}:${c.startOffset}`);
-      // Each chunk's text is exactly the document slice at its absolute offsets.
-      expect(md.slice(c.startOffset, c.endOffset)).toBe(c.text);
-    });
-    expect(out[0]?.startOffset).toBe(0);
-  });
-
-  test("text before the first header becomes its own leading section", () => {
-    const md = "preamble line\n# Header\nbody";
-    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 100 });
-    expect(out.length).toBe(2);
-    expect(out[0]?.text.startsWith("preamble")).toBe(true);
-    expect(out[1]?.text.startsWith("# Header")).toBe(true);
-  });
-
-  test("a document with no headers yields a single chunk", () => {
-    const md = "just plain text, no headers here";
-    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 100 });
-    expect(out.length).toBe(1);
-    expect(out[0]?.text).toBe(md);
-  });
-
-  test("an empty document yields no chunks", () => {
-    const out = chunk({ id: "doc", text: "" }, { strategy: "markdown", size: 100 });
-    expect(out).toEqual([]);
-  });
-
-  test("metadata flows through markdown chunks (both short and sub-chunked paths)", () => {
-    const long = "z".repeat(120);
-    const md = `# Big\n${long}\n## Small\nshort`;
-    const out = chunk(
-      { id: "doc", text: md, metadata: { src: "wiki" } },
-      { strategy: "markdown", size: 50 },
-    );
-    expect(out.length).toBeGreaterThan(2);
-    for (const c of out) {
-      expect(c.metadata).toEqual({ src: "wiki" });
-    }
-  });
-
-  test("reconstructs the source bytes by concatenating chunk text (overlap=0)", () => {
-    const md = "# A\npara one\n## B\npara two\n### C\npara three";
-    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 100 });
-    expect(out.map((c) => c.text).join("")).toBe(md);
-  });
-});
-
-describe("strategy dispatch", () => {
-  test("an unknown strategy throws ChunkerError", () => {
-    expect(() =>
-      // @ts-expect-error — deliberately bypass the typed strategy union
-      chunk({ id: "x", text: "abc" }, { strategy: "bogus", size: 5 }),
-    ).toThrow(ChunkerError);
-  });
-
-  test("the unknown-strategy message names the offending value", () => {
-    let caught: unknown;
-    try {
-      // @ts-expect-error — deliberately bypass the typed strategy union
-      chunk({ id: "x", text: "abc" }, { strategy: "nope", size: 5 });
-    } catch (e) {
-      caught = e;
-    }
-    expect((caught as Error).message).toContain("nope");
-  });
-});

package/src/index.ts

@@ -1,207 +0,0 @@
-/**
- * Catalog R12 `chunker` — split documents into retrievable chunks.
- *
- * Three strategies:
- *
- *   "fixed"    — split into fixed-character windows with optional overlap
- *   "semantic" — `Intl.Segmenter` sentence boundaries, packed into windows
- *   "markdown" — header-bounded chunks (split at `^#+ ` boundaries)
- *
- * Every strategy preserves the source bytes — `chunk(doc).map(c => c.text).join("")`
- * (mod overlap) reconstructs the doc, modulo any whitespace that
- * sat at chunk boundaries. The `id` field is `<docId>:<index>:<startOffset>`
- * for stable reference back to the source.
- *
- * Layer R12. Pairs with `embedder`, `vector-store`, `pipeline-engine`.
- */
-
-import { CrewhausError } from "@crewhaus/errors";
-
-export type ChunkStrategy = "fixed" | "semantic" | "markdown";
-
-export type ChunkOptions = {
-  readonly strategy: ChunkStrategy;
-  /** Window size: chars for "fixed", sentences for "semantic", chars for "markdown" sub-chunking. */
-  readonly size: number;
-  /** Overlap (in the same units as size). Defaults to 0. */
-  readonly overlap?: number;
-  /** Locale for `Intl.Segmenter` ("semantic" only). Defaults to "en". */
-  readonly locale?: string;
-};
-
-export type Document = {
-  /** Stable identifier (URL, path, hash). Used to derive chunk ids. */
-  readonly id: string;
-  readonly text: string;
-  readonly metadata?: Readonly<Record<string, unknown>>;
-};
-
-export type Chunk = {
-  readonly id: string;
-  readonly docId: string;
-  readonly index: number;
-  /** Inclusive byte offset in `doc.text` where the chunk begins. */
-  readonly startOffset: number;
-  /** Exclusive byte offset where the chunk ends. */
-  readonly endOffset: number;
-  readonly text: string;
-  readonly metadata?: Readonly<Record<string, unknown>>;
-};
-
-export class ChunkerError extends CrewhausError {
-  override readonly name = "ChunkerError";
-  constructor(message: string, cause?: unknown) {
-    super("config", message, cause);
-  }
-}
-
-function makeChunkId(docId: string, index: number, start: number): string {
-  return `${docId}:${index}:${start}`;
-}
-
-function chunkFixed(doc: Document, opts: ChunkOptions): Chunk[] {
-  if (opts.size <= 0) throw new ChunkerError("size must be positive");
-  const overlap = opts.overlap ?? 0;
-  if (overlap < 0 || overlap >= opts.size) {
-    throw new ChunkerError(`overlap (${overlap}) must be in [0, size) (size=${opts.size})`);
-  }
-  const out: Chunk[] = [];
-  const stride = opts.size - overlap;
-  let start = 0;
-  let index = 0;
-  while (start < doc.text.length) {
-    const end = Math.min(start + opts.size, doc.text.length);
-    out.push({
-      id: makeChunkId(doc.id, index, start),
-      docId: doc.id,
-      index,
-      startOffset: start,
-      endOffset: end,
-      text: doc.text.slice(start, end),
-      ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
-    });
-    if (end >= doc.text.length) break;
-    start += stride;
-    index += 1;
-  }
-  return out;
-}
-
-function chunkSemantic(doc: Document, opts: ChunkOptions): Chunk[] {
-  if (opts.size <= 0) throw new ChunkerError("size must be positive");
-  const overlap = opts.overlap ?? 0;
-  // Intl.Segmenter is widely available in modern Node + Bun. An invalid
-  // `locale` makes the constructor throw a raw `RangeError`; wrap it so callers
-  // see the package's typed `ChunkerError` contract (with the original as cause).
-  let segmenter: Intl.Segmenter;
-  try {
-    segmenter = new Intl.Segmenter(opts.locale ?? "en", { granularity: "sentence" });
-  } catch (cause) {
-    throw new ChunkerError(`invalid locale ${JSON.stringify(opts.locale)}`, cause);
-  }
-  type Sentence = { readonly text: string; readonly start: number; readonly end: number };
-  const sentences: Sentence[] = [];
-  for (const seg of segmenter.segment(doc.text)) {
-    if (seg.segment.trim().length === 0) continue;
-    const start = seg.index;
-    const end = start + seg.segment.length;
-    sentences.push({ text: seg.segment, start, end });
-  }
-  if (sentences.length === 0) return [];
-  const out: Chunk[] = [];
-  let i = 0;
-  let index = 0;
-  while (i < sentences.length) {
-    const window = sentences.slice(i, Math.min(i + opts.size, sentences.length));
-    if (window.length === 0) break;
-    const startOffset = window[0]?.start ?? 0;
-    const endOffset = window[window.length - 1]?.end ?? doc.text.length;
-    const text = doc.text.slice(startOffset, endOffset);
-    out.push({
-      id: makeChunkId(doc.id, index, startOffset),
-      docId: doc.id,
-      index,
-      startOffset,
-      endOffset,
-      text,
-      ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
-    });
-    index += 1;
-    if (i + opts.size >= sentences.length) break;
-    i += Math.max(1, opts.size - overlap);
-  }
-  return out;
-}
-
-function chunkMarkdown(doc: Document, opts: ChunkOptions): Chunk[] {
-  // Split the document into header-bounded sections, then sub-chunk each
-  // section with the "fixed" strategy if it's longer than `opts.size`.
-  const headerRe = /(^|\n)(#{1,6}\s)/g;
-  const sectionRanges: Array<{ start: number; end: number }> = [];
-  const matches: number[] = [0];
-  let m: RegExpExecArray | null;
-  // biome-ignore lint/suspicious/noAssignInExpressions: classical regex iter
-  while ((m = headerRe.exec(doc.text)) !== null) {
-    const headerStart = m[1] === "\n" ? m.index + 1 : m.index;
-    if (headerStart !== 0 && !matches.includes(headerStart)) matches.push(headerStart);
-  }
-  matches.push(doc.text.length);
-  for (let i = 0; i < matches.length - 1; i += 1) {
-    const start = matches[i] as number;
-    const end = matches[i + 1] as number;
-    if (end > start) sectionRanges.push({ start, end });
-  }
-  const out: Chunk[] = [];
-  let chunkIndex = 0;
-  for (const range of sectionRanges) {
-    const sliceLen = range.end - range.start;
-    if (sliceLen <= opts.size) {
-      out.push({
-        id: makeChunkId(doc.id, chunkIndex, range.start),
-        docId: doc.id,
-        index: chunkIndex,
-        startOffset: range.start,
-        endOffset: range.end,
-        text: doc.text.slice(range.start, range.end),
-        ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
-      });
-      chunkIndex += 1;
-      continue;
-    }
-    // Sub-chunk via fixed strategy on this section.
-    const sub = chunkFixed(
-      {
-        id: doc.id,
-        text: doc.text.slice(range.start, range.end),
-        ...(doc.metadata !== undefined ? { metadata: doc.metadata } : {}),
-      },
-      opts,
-    );
-    for (const c of sub) {
-      out.push({
-        ...c,
-        id: makeChunkId(doc.id, chunkIndex, range.start + c.startOffset),
-        index: chunkIndex,
-        startOffset: range.start + c.startOffset,
-        endOffset: range.start + c.endOffset,
-      });
-      chunkIndex += 1;
-    }
-  }
-  return out;
-}
-
-export function chunk(doc: Document, opts: ChunkOptions): Chunk[] {
-  switch (opts.strategy) {
-    case "fixed":
-      return chunkFixed(doc, opts);
-    case "semantic":
-      return chunkSemantic(doc, opts);
-    case "markdown":
-      return chunkMarkdown(doc, opts);
-    default: {
-      const exhaustive: never = opts.strategy;
-      throw new ChunkerError(`unknown strategy "${exhaustive}"`);
-    }
-  }
-}