dravoice 0.1.1 → 0.1.3

package/src/v2/document-model.js

@@ -1,260 +1,351 @@
-import fs from "node:fs";
-import path from "node:path";
-import { normalizeText, splitSentences, tokenizeWords } from "./text-utils.js";
-
-export const VOICE_EXTENSIONS = new Set([".md", ".mdx", ".txt"]);
-const DEFAULT_MAX_FILES = 500;
-const DEFAULT_MAX_FILE_BYTES = 1024 * 1024;
-const DEFAULT_MAX_TOTAL_BYTES = 20 * 1024 * 1024;
-
-export function loadDocuments({
-  examplesDir,
-  maxFiles = DEFAULT_MAX_FILES,
-  maxFileBytes = DEFAULT_MAX_FILE_BYTES,
-  maxTotalBytes = DEFAULT_MAX_TOTAL_BYTES,
-}) {
-  const root = path.resolve(examplesDir);
-  const files = walkVoiceFiles(root);
-  if (files.length === 0) {
-    throw new Error(`No Markdown, MDX, or text examples found at ${examplesDir}`);
-  }
-  if (files.length > maxFiles) {
-    throw new Error(`Corpus contains ${files.length} voice files, more than ${maxFiles} voice file(s) allowed.`);
-  }
-
-  let totalBytes = 0;
-  return files.map((filePath) => {
-    const stats = fs.statSync(filePath);
-    if (stats.size > maxFileBytes) {
-      throw new Error(`Voice file ${displayPath(filePath, root)} exceeds the ${maxFileBytes} byte limit.`);
-    }
-    totalBytes += stats.size;
-    if (totalBytes > maxTotalBytes) {
-      throw new Error(`Voice corpus exceeds the ${maxTotalBytes} byte total limit.`);
-    }
-    const contents = fs.readFileSync(filePath, "utf8");
-    if (looksBinary(contents)) {
-      throw new Error(`Voice file ${displayPath(filePath, root)} looks like binary-looking text and cannot be analyzed.`);
-    }
-    return parseDocument({
-      filePath,
-      rootDir: root,
-      contents,
-    });
-  });
-}
-
-export function parseDocument({ filePath, rootDir = process.cwd(), contents }) {
-  const relative = displayPath(filePath, rootDir);
-  const lines = String(contents ?? "").split(/\r?\n/);
-  const headings = [];
-  const blocks = [];
-  let currentHeading = null;
-  let currentParagraph = null;
-  let inFence = false;
-  let inFrontmatter = lines[0]?.trim() === "---";
-  let inHtmlComment = false;
-
-  const flushParagraph = () => {
-    if (currentParagraph?.lines.length) {
-      blocks.push(currentParagraph);
-    }
-    currentParagraph = null;
-  };
-
-  lines.forEach((line, index) => {
-    const lineNumber = index + 1;
-    const trimmed = line.trim();
-
-    if (inHtmlComment) {
-      if (trimmed.includes("-->")) {
-        inHtmlComment = false;
-      }
-      return;
-    }
-
-    if (inFrontmatter) {
-      if (index > 0 && trimmed === "---") {
-        inFrontmatter = false;
-      }
-      return;
-    }
-
-    if (/^(```|~~~)/.test(trimmed)) {
-      flushParagraph();
-      inFence = !inFence;
-      return;
-    }
-
-    if (trimmed.startsWith("<!--")) {
-      flushParagraph();
-      if (!trimmed.includes("-->")) {
-        inHtmlComment = true;
-      }
-      return;
-    }
-
-    if (inFence || !trimmed || isMdxScaffold(trimmed) || isTableLine(trimmed) || isJsxLike(trimmed)) {
-      if (!trimmed) {
-        flushParagraph();
-      }
-      return;
-    }
-
-    const headingMatch = /^(#{1,6})\s+(.+)$/.exec(trimmed);
-    if (headingMatch) {
-      flushParagraph();
-      currentHeading = {
-        id: `h${headings.length + 1}`,
-        depth: headingMatch[1].length,
-        text: stripMarkdown(headingMatch[2]).trim(),
-        line: lineNumber,
-      };
-      headings.push(currentHeading);
-      return;
-    }
-
-    const blockType = blockTypeFor(trimmed);
-    const text = stripMarkdown(trimmed.replace(/^[-*]\s+/, "").replace(/^\d+[.)]\s+/, "").replace(/^>\s+/, ""));
-    if (blockType === "paragraph") {
-      if (!currentParagraph) {
-        currentParagraph = makeBlock({ type: "paragraph", line: lineNumber, heading: currentHeading });
-      }
-      currentParagraph.lines.push(text);
-      return;
-    }
-
-    flushParagraph();
-    blocks.push({
-      ...makeBlock({ type: blockType, line: lineNumber, heading: currentHeading }),
-      lines: [text],
-    });
-  });
-  flushParagraph();
-
-  const paragraphs = blocks.map(blockToParagraph).filter((paragraph) => paragraph.text);
-  const sentences = paragraphs.flatMap(paragraphToSentences);
-  return {
-    file: relative,
-    path: filePath,
-    headings,
-    sections: buildSections(headings, blocks),
-    blocks,
-    paragraphs,
-    sentences,
-    wordCount: sentences.reduce((sum, sentence) => sum + sentence.tokens.length, 0),
-    text: paragraphs.map((paragraph) => paragraph.text).join("\n\n"),
-  };
-}
-
-function makeBlock({ type, line, heading }) {
-  return {
-    type,
-    line,
-    heading: heading?.text ?? null,
-    headingId: heading?.id ?? null,
-    headingDepth: heading?.depth ?? 0,
-    lines: [],
-  };
-}
-
-function blockTypeFor(trimmed) {
-  if (/^>\s+/.test(trimmed)) {
-    return "quote";
-  }
-  if (/^[-*]\s+/.test(trimmed)) {
-    return "list";
-  }
-  if (/^\d+[.)]\s+/.test(trimmed)) {
-    return "list";
-  }
-  return "paragraph";
-}
-
-function buildSections(headings, blocks) {
-  if (headings.length === 0) {
-    return [{ heading: null, blocks }];
-  }
-  return headings.map((heading, index) => {
-    const nextBoundary = headings.slice(index + 1).find((candidate) => candidate.depth <= heading.depth);
-    return {
-      heading,
-      blocks: blocks.filter((block) =>
-        block.line > heading.line &&
-        (!nextBoundary || block.line < nextBoundary.line)
-      ),
-    };
-  });
-}
-
-function blockToParagraph(block) {
-  return {
-    type: block.type,
-    line: block.line,
-    heading: block.heading,
-    headingId: block.headingId,
-    text: block.lines.join(" ").replace(/\s+/g, " ").trim(),
-  };
-}
-
-function paragraphToSentences(paragraph) {
-  return splitSentences(paragraph.text).map((text, offset) => ({
-    text,
-    normalized: normalizeText(text),
-    line: paragraph.line + offset,
-    blockType: paragraph.type,
-    heading: paragraph.heading,
-    headingId: paragraph.headingId,
-    tokens: tokenizeWords(text),
-  }));
-}
-
-function stripMarkdown(text) {
-  return text
-    .replace(/\*\*/g, "")
-    .replace(/__+/g, "")
-    .replace(/`([^`]+)`/g, "$1")
-    .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
-}
-
-function isJsxLike(trimmed) {
-  return /^<\/?[A-Za-z][^>]*>/.test(trimmed) || /^<[A-Za-z][^>]*\/>$/.test(trimmed);
-}
-
-function isMdxScaffold(trimmed) {
-  return /^(?:import|export)\s/.test(trimmed);
-}
-
-function isTableLine(trimmed) {
-  return /^\|.*\|$/.test(trimmed);
-}
-
-function looksBinary(contents) {
-  return String(contents ?? "").includes("\0");
-}
-
-function walkVoiceFiles(rootDir) {
-  const result = [];
-  if (!fs.existsSync(rootDir)) {
-    return result;
-  }
-  for (const entry of fs.readdirSync(rootDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
-    const fullPath = path.join(rootDir, entry.name);
-    if (entry.isDirectory()) {
-      if (![".git", "node_modules", "dist", "build", "__pycache__", "prompts", "voice-pack", "dravoice-voice"].includes(entry.name)) {
-        result.push(...walkVoiceFiles(fullPath));
-      }
-    } else if (VOICE_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) {
-      result.push(fullPath);
-    }
-  }
-  return result;
-}
-
-function displayPath(filePath, rootDir) {
-  const relative = path.relative(rootDir, filePath);
-  if (relative && !relative.startsWith("..") && !path.isAbsolute(relative)) {
-    return relative.split(path.sep).join("/");
-  }
-  return filePath.split(path.sep).join("/");
-}
+import fs from "node:fs";
+import path from "node:path";
+import { normalizeText, splitSentences, tokenizeWords } from "./text-utils.js";
+
+export const VOICE_EXTENSIONS = new Set([".md", ".mdx", ".txt"]);
+const DEFAULT_MAX_FILES = 500;
+const DEFAULT_MAX_FILE_BYTES = 1024 * 1024;
+const DEFAULT_MAX_TOTAL_BYTES = 20 * 1024 * 1024;
+
+export function loadDocuments({
+  examplesDir,
+  maxFiles = DEFAULT_MAX_FILES,
+  maxFileBytes = DEFAULT_MAX_FILE_BYTES,
+  maxTotalBytes = DEFAULT_MAX_TOTAL_BYTES,
+  excludePaths = [],
+}) {
+  const root = path.resolve(examplesDir);
+  const files = walkVoiceFiles(root, {
+    maxFiles,
+    excludePaths: excludePaths.map((item) => path.resolve(item)),
+  });
+  if (files.length === 0) {
+    throw new Error(`No Markdown, MDX, or text examples found at ${examplesDir}`);
+  }
+
+  let totalBytes = 0;
+  return files.map((filePath) => {
+    const stats = fs.statSync(filePath);
+    if (stats.size > maxFileBytes) {
+      throw new Error(`Voice file ${displayPath(filePath, root)} exceeds the ${maxFileBytes} byte limit.`);
+    }
+    totalBytes += stats.size;
+    if (totalBytes > maxTotalBytes) {
+      throw new Error(`Voice corpus exceeds the ${maxTotalBytes} byte total limit.`);
+    }
+    const contents = fs.readFileSync(filePath, "utf8");
+    if (looksBinary(contents)) {
+      throw new Error(`Voice file ${displayPath(filePath, root)} looks like binary-looking text and cannot be analyzed.`);
+    }
+    return parseDocument({
+      filePath,
+      rootDir: root,
+      contents,
+    });
+  });
+}
+
+export function parseDocument({ filePath, rootDir = process.cwd(), contents }) {
+  const relative = displayPath(filePath, rootDir);
+  const lines = String(contents ?? "").split(/\r?\n/);
+  const headings = [];
+  const blocks = [];
+  let currentHeading = null;
+  let currentParagraph = null;
+  let inFence = false;
+  let inFrontmatter = lines[0]?.trim() === "---";
+  let inHtmlComment = false;
+  let jsxBlockTag = null;
+
+  const flushParagraph = () => {
+    if (currentParagraph?.lines.length) {
+      blocks.push(currentParagraph);
+    }
+    currentParagraph = null;
+  };
+
+  lines.forEach((line, index) => {
+    const lineNumber = index + 1;
+    const trimmed = line.trim();
+
+    if (inFrontmatter) {
+      if (index > 0 && trimmed === "---") {
+        inFrontmatter = false;
+      }
+      return;
+    }
+
+    if (/^(```|~~~)/.test(trimmed)) {
+      flushParagraph();
+      inFence = !inFence;
+      return;
+    }
+
+    if (inFence) {
+      return;
+    }
+
+    if (inHtmlComment) {
+      if (trimmed.includes("-->")) {
+        inHtmlComment = false;
+      }
+      return;
+    }
+
+    if (jsxBlockTag) {
+      if (new RegExp(`</${escapeRegExp(jsxBlockTag)}\\s*>`).test(trimmed)) {
+        jsxBlockTag = null;
+      }
+      return;
+    }
+
+    if (trimmed.startsWith("<!--")) {
+      flushParagraph();
+      if (!trimmed.includes("-->")) {
+        inHtmlComment = true;
+      }
+      return;
+    }
+
+    const jsxBlock = jsxBlockStart(trimmed);
+    if (jsxBlock) {
+      flushParagraph();
+      jsxBlockTag = jsxBlock;
+      return;
+    }
+
+    if (!trimmed || isMdxScaffold(trimmed) || isTableLine(trimmed) || isJsxLike(trimmed)) {
+      if (!trimmed) {
+        flushParagraph();
+      }
+      return;
+    }
+
+    const headingMatch = /^(#{1,6})\s+(.+)$/.exec(trimmed);
+    if (headingMatch) {
+      flushParagraph();
+      currentHeading = {
+        id: `h${headings.length + 1}`,
+        depth: headingMatch[1].length,
+        text: stripMarkdown(headingMatch[2]).trim(),
+        line: lineNumber,
+      };
+      headings.push(currentHeading);
+      return;
+    }
+
+    const blockType = blockTypeFor(trimmed);
+    const text = stripMarkdown(trimmed.replace(/^[-*]\s+/, "").replace(/^\d+[.)]\s+/, "").replace(/^>\s+/, ""));
+    if (blockType === "paragraph") {
+      if (!currentParagraph) {
+        currentParagraph = makeBlock({ type: "paragraph", line: lineNumber, heading: currentHeading });
+      }
+      currentParagraph.lines.push(text);
+      currentParagraph.lineNumbers.push(lineNumber);
+      return;
+    }
+
+    flushParagraph();
+    blocks.push({
+      ...makeBlock({ type: blockType, line: lineNumber, heading: currentHeading }),
+      lines: [text],
+      lineNumbers: [lineNumber],
+    });
+  });
+  flushParagraph();
+
+  const paragraphs = blocks.map(blockToParagraph).filter((paragraph) => paragraph.text);
+  const sentences = paragraphs.flatMap(paragraphToSentences);
+  return {
+    file: relative,
+    path: filePath,
+    headings,
+    sections: buildSections(headings, blocks),
+    blocks,
+    paragraphs,
+    sentences,
+    wordCount: sentences.reduce((sum, sentence) => sum + sentence.tokens.length, 0),
+    text: paragraphs.map((paragraph) => paragraph.text).join("\n\n"),
+  };
+}
+
+function makeBlock({ type, line, heading }) {
+  return {
+    type,
+    line,
+    heading: heading?.text ?? null,
+    headingId: heading?.id ?? null,
+    headingDepth: heading?.depth ?? 0,
+    lines: [],
+    lineNumbers: [],
+  };
+}
+
+function blockTypeFor(trimmed) {
+  if (/^>\s+/.test(trimmed)) {
+    return "quote";
+  }
+  if (/^[-*]\s+/.test(trimmed)) {
+    return "list";
+  }
+  if (/^\d+[.)]\s+/.test(trimmed)) {
+    return "list";
+  }
+  return "paragraph";
+}
+
+function buildSections(headings, blocks) {
+  if (headings.length === 0) {
+    return [{ heading: null, blocks }];
+  }
+  return headings.map((heading, index) => {
+    const nextBoundary = headings.slice(index + 1).find((candidate) => candidate.depth <= heading.depth);
+    return {
+      heading,
+      blocks: blocks.filter((block) =>
+        block.line > heading.line &&
+        (!nextBoundary || block.line < nextBoundary.line)
+      ),
+    };
+  });
+}
+
+function blockToParagraph(block) {
+  return {
+    type: block.type,
+    line: block.line,
+    heading: block.heading,
+    headingId: block.headingId,
+    lines: block.lines,
+    lineNumbers: block.lineNumbers?.length ? block.lineNumbers : [block.line],
+    text: block.lines.join(" ").replace(/\s+/g, " ").trim(),
+  };
+}
+
+function paragraphToSentences(paragraph) {
+  const lineStarts = [];
+  let offset = 0;
+  for (let index = 0; index < paragraph.lines.length; index += 1) {
+    const text = String(paragraph.lines[index] ?? "").replace(/\s+/g, " ").trim();
+    if (!text) {
+      continue;
+    }
+    lineStarts.push({ offset, line: paragraph.lineNumbers[index] ?? paragraph.line });
+    offset += text.length + 1;
+  }
+
+  let searchFrom = 0;
+  return splitSentences(paragraph.text).map((text) => {
+    const sentenceOffset = paragraph.text.indexOf(text, searchFrom);
+    if (sentenceOffset >= 0) {
+      searchFrom = sentenceOffset + text.length;
+    }
+    const line = lineForOffset(lineStarts, sentenceOffset >= 0 ? sentenceOffset : searchFrom);
+    return {
+      text,
+      normalized: normalizeText(text),
+      line,
+      blockType: paragraph.type,
+      heading: paragraph.heading,
+      headingId: paragraph.headingId,
+      tokens: tokenizeWords(text),
+    };
+  });
+}
+
+function stripMarkdown(text) {
+  return text
+    .replace(/\*\*/g, "")
+    .replace(/__+/g, "")
+    .replace(/`([^`]+)`/g, "$1")
+    .replace(/\[([^\]]+)\]\(([^)]+)\)/g, "$1 $2");
+}
+
+function isJsxLike(trimmed) {
+  return /^<\/?[A-Za-z][^>]*>/.test(trimmed) || /^<[A-Za-z][^>]*\/>$/.test(trimmed);
+}
+
+function jsxBlockStart(trimmed) {
+  const match = /^<([A-Za-z][\w.-]*)(?:\s[^>]*)?>\s*$/.exec(trimmed);
+  if (!match || /\/>\s*$/.test(trimmed) || new RegExp(`</${escapeRegExp(match[1])}\\s*>`).test(trimmed)) {
+    return null;
+  }
+  return match[1];
+}
+
+function isMdxScaffold(trimmed) {
+  return /^(?:import|export)\s/.test(trimmed);
+}
+
+function isTableLine(trimmed) {
+  return /^\|.*\|$/.test(trimmed);
+}
+
+function looksBinary(contents) {
+  return String(contents ?? "").includes("\0");
+}
+
+function walkVoiceFiles(rootDir, { maxFiles, excludePaths }) {
+  const result = [];
+  if (!fs.existsSync(rootDir)) {
+    return result;
+  }
+  const stack = [rootDir];
+  const skipDirs = new Set([".git", "node_modules", "dist", "build", "__pycache__", "prompts", "voice-pack", "dravoice-voice"]);
+  while (stack.length) {
+    const current = stack.pop();
+    if (isExcludedPath(current, excludePaths)) {
+      continue;
+    }
+    const entries = fs.readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
+    const directories = [];
+    for (const entry of entries) {
+      const fullPath = path.join(current, entry.name);
+      if (isExcludedPath(fullPath, excludePaths)) {
+        continue;
+      }
+      if (entry.isDirectory()) {
+        if (!skipDirs.has(entry.name)) {
+          directories.push(fullPath);
+        }
+      } else if (entry.isFile() && VOICE_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) {
+        result.push(fullPath);
+        if (result.length > maxFiles) {
+          throw new Error(`Corpus contains more than ${maxFiles} voice file(s) allowed.`);
+        }
+      }
+    }
+    stack.push(...directories.reverse());
+  }
+  return result;
+}
+
+function lineForOffset(lineStarts, offset) {
+  let line = lineStarts[0]?.line ?? 1;
+  for (const item of lineStarts) {
+    if (item.offset > offset) {
+      break;
+    }
+    line = item.line;
+  }
+  return line;
+}
+
+function isExcludedPath(filePath, excludePaths) {
+  return excludePaths.some((excluded) => {
+    const relative = path.relative(excluded, filePath);
+    return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+  });
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function displayPath(filePath, rootDir) {
+  const relative = path.relative(rootDir, filePath);
+  if (relative && !relative.startsWith("..") && !path.isAbsolute(relative)) {
+    return relative.split(path.sep).join("/");
+  }
+  return filePath.split(path.sep).join("/");
+}