@crewhaus/chunker 0.1.0 → 0.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/chunker",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "description": "Document chunking — fixed / semantic (Intl.Segmenter) / markdown strategies",
   "main": "src/index.ts",
@@ -12,13 +12,13 @@
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/errors": "0.0.0"
+    "@crewhaus/errors": "0.1.2"
   },
   "license": "Apache-2.0",
   "author": {
     "name": "Max Meier",
-    "email": "max@studiomax.io",
-    "url": "https://studiomax.io"
+    "email": "max@crewhaus.ai",
+    "url": "https://crewhaus.ai"
   },
   "repository": {
     "type": "git",
@@ -30,12 +30,7 @@
     "url": "https://github.com/crewhaus/factory/issues"
   },
   "publishConfig": {
-    "access": "restricted"
+    "access": "public"
   },
-  "files": [
-    "src",
-    "README.md",
-    "LICENSE",
-    "NOTICE"
-  ]
+  "files": ["src", "README.md", "LICENSE", "NOTICE"]
 }

package/src/index.test.ts

@@ -59,6 +59,80 @@ describe("semantic strategy", () => {
     const out = chunk({ id: "doc", text: "" }, { strategy: "semantic", size: 3 });
     expect(out).toEqual([]);
   });
+
+  test("whitespace-only text yields no chunks (segments trimmed away)", () => {
+    const out = chunk({ id: "doc", text: "   \n\t  " }, { strategy: "semantic", size: 2 });
+    expect(out).toEqual([]);
+  });
+
+  test("overlap produces overlapping sentence windows", () => {
+    const text = "Alpha. Bravo. Charlie. Delta. Echo. Foxtrot.";
+    const out = chunk({ id: "doc", text }, { strategy: "semantic", size: 2, overlap: 1 });
+    // stride = max(1, 2 - 1) = 1, so a size-2 window starts at sentences
+    // 0,1,2,3,4 (the loop breaks once i + size reaches the 6-sentence end): 5 chunks.
+    expect(out.length).toBe(5);
+    // Consecutive windows share a sentence (overlap), so each later window
+    // starts strictly before the previous window's end.
+    expect(out[1]?.startOffset).toBeLessThan(out[0]?.endOffset ?? -1);
+  });
+
+  test("overlap >= size still terminates (stride clamped to 1)", () => {
+    const text = "Alpha. Bravo. Charlie.";
+    const out = chunk({ id: "doc", text }, { strategy: "semantic", size: 2, overlap: 5 });
+    // No infinite loop; stride floored at 1 yields one window per sentence
+    // until the final window reaches the end.
+    expect(out.length).toBe(2);
+  });
+
+  test("rejects non-positive size", () => {
+    expect(() => chunk({ id: "x", text: "A. B." }, { strategy: "semantic", size: 0 })).toThrow(
+      ChunkerError,
+    );
+  });
+
+  test("metadata flows through semantic chunks", () => {
+    const out = chunk(
+      { id: "doc", text: "Alpha. Bravo.", metadata: { lang: "en" } },
+      { strategy: "semantic", size: 1 },
+    );
+    expect(out.length).toBe(2);
+    expect(out[0]?.metadata).toEqual({ lang: "en" });
+    expect(out[1]?.metadata).toEqual({ lang: "en" });
+  });
+
+  test("a valid non-default locale is accepted", () => {
+    const out = chunk(
+      { id: "doc", text: "Hallo. Welt." },
+      { strategy: "semantic", size: 1, locale: "de" },
+    );
+    expect(out.length).toBe(2);
+  });
+
+  test("an invalid locale throws ChunkerError (not a raw RangeError)", () => {
+    let caught: unknown;
+    try {
+      chunk(
+        { id: "doc", text: "Hello. World." },
+        { strategy: "semantic", size: 2, locale: "not a valid tag!!!" },
+      );
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(ChunkerError);
+    // The original RangeError is preserved as the cause for debugging.
+    expect((caught as Error).cause).toBeInstanceOf(RangeError);
+    expect((caught as Error).message).toContain("invalid locale");
+  });
+
+  test("preserves source bytes across a single full window", () => {
+    const text = "Alpha. Bravo. Charlie.";
+    const out = chunk({ id: "doc", text }, { strategy: "semantic", size: 10 });
+    expect(out.length).toBe(1);
+    const only = out[0];
+    expect(only).toBeDefined();
+    // The single window spans from the first sentence start to the last end.
+    expect(text.slice(only?.startOffset, only?.endOffset)).toBe(only?.text ?? "");
+  });
 });
 
 describe("markdown strategy", () => {
@@ -84,4 +158,78 @@ para`;
     // Section 2 is short → 1 piece.
     expect(out.length).toBeGreaterThan(2);
   });
+
+  test("sub-chunk ids and offsets are remapped to absolute document positions", () => {
+    const long = "y".repeat(120);
+    const md = `# Big\n${long}`; // single section, > size → sub-chunked
+    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 50 });
+    expect(out.length).toBeGreaterThan(1);
+    // Indices are contiguous and ids encode absolute startOffset.
+    out.forEach((c, i) => {
+      expect(c.index).toBe(i);
+      expect(c.id).toBe(`doc:${i}:${c.startOffset}`);
+      // Each chunk's text is exactly the document slice at its absolute offsets.
+      expect(md.slice(c.startOffset, c.endOffset)).toBe(c.text);
+    });
+    expect(out[0]?.startOffset).toBe(0);
+  });
+
+  test("text before the first header becomes its own leading section", () => {
+    const md = "preamble line\n# Header\nbody";
+    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 100 });
+    expect(out.length).toBe(2);
+    expect(out[0]?.text.startsWith("preamble")).toBe(true);
+    expect(out[1]?.text.startsWith("# Header")).toBe(true);
+  });
+
+  test("a document with no headers yields a single chunk", () => {
+    const md = "just plain text, no headers here";
+    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 100 });
+    expect(out.length).toBe(1);
+    expect(out[0]?.text).toBe(md);
+  });
+
+  test("an empty document yields no chunks", () => {
+    const out = chunk({ id: "doc", text: "" }, { strategy: "markdown", size: 100 });
+    expect(out).toEqual([]);
+  });
+
+  test("metadata flows through markdown chunks (both short and sub-chunked paths)", () => {
+    const long = "z".repeat(120);
+    const md = `# Big\n${long}\n## Small\nshort`;
+    const out = chunk(
+      { id: "doc", text: md, metadata: { src: "wiki" } },
+      { strategy: "markdown", size: 50 },
+    );
+    expect(out.length).toBeGreaterThan(2);
+    for (const c of out) {
+      expect(c.metadata).toEqual({ src: "wiki" });
+    }
+  });
+
+  test("reconstructs the source bytes by concatenating chunk text (overlap=0)", () => {
+    const md = "# A\npara one\n## B\npara two\n### C\npara three";
+    const out = chunk({ id: "doc", text: md }, { strategy: "markdown", size: 100 });
+    expect(out.map((c) => c.text).join("")).toBe(md);
+  });
+});
+
+describe("strategy dispatch", () => {
+  test("an unknown strategy throws ChunkerError", () => {
+    expect(() =>
+      // @ts-expect-error — deliberately bypass the typed strategy union
+      chunk({ id: "x", text: "abc" }, { strategy: "bogus", size: 5 }),
+    ).toThrow(ChunkerError);
+  });
+
+  test("the unknown-strategy message names the offending value", () => {
+    let caught: unknown;
+    try {
+      // @ts-expect-error — deliberately bypass the typed strategy union
+      chunk({ id: "x", text: "abc" }, { strategy: "nope", size: 5 });
+    } catch (e) {
+      caught = e;
+    }
+    expect((caught as Error).message).toContain("nope");
+  });
 });

package/src/index.ts

@@ -90,8 +90,15 @@ function chunkFixed(doc: Document, opts: ChunkOptions): Chunk[] {
 function chunkSemantic(doc: Document, opts: ChunkOptions): Chunk[] {
   if (opts.size <= 0) throw new ChunkerError("size must be positive");
   const overlap = opts.overlap ?? 0;
-  // Intl.Segmenter is widely available in modern Node + Bun.
-  const segmenter = new Intl.Segmenter(opts.locale ?? "en", { granularity: "sentence" });
+  // Intl.Segmenter is widely available in modern Node + Bun. An invalid
+  // `locale` makes the constructor throw a raw `RangeError`; wrap it so callers
+  // see the package's typed `ChunkerError` contract (with the original as cause).
+  let segmenter: Intl.Segmenter;
+  try {
+    segmenter = new Intl.Segmenter(opts.locale ?? "en", { granularity: "sentence" });
+  } catch (cause) {
+    throw new ChunkerError(`invalid locale ${JSON.stringify(opts.locale)}`, cause);
+  }
   type Sentence = { readonly text: string; readonly start: number; readonly end: number };
   const sentences: Sentence[] = [];
   for (const seg of segmenter.segment(doc.text)) {