forge-server 0.1.0

package/src/ingestion/markdown-chunker.ts

@@ -0,0 +1,337 @@
+/**
+ * Section-aware markdown chunker.
+ * Replaces chunkFixed for .md/.mdx files during indexing.
+ *
+ * Strategy:
+ *   1. Parse markdown into a section tree by headings (H1 → H2 → H3)
+ *   2. Each section becomes a chunk with heading breadcrumb as context header
+ *   3. Code blocks stay attached to their parent section
+ *   4. Sections exceeding 1024 tokens split at sub-heading or paragraph boundaries
+ *   5. Sections under ~30 tokens merge with the next sibling
+ *   6. Frontmatter (YAML between ---) becomes its own chunk
+ */
+
+import type { CodeChunk } from '../util/types.js';
+import { estimateTokens } from '../util/token-counter.js';
+import { makeChunk } from './chunker.js';
+
+const SECTION_MAX_TOKENS = 1024;
+const SECTION_MIN_TOKENS = 30;
+
+// ---------------------------------------------------------------------------
+// Section tree types
+// ---------------------------------------------------------------------------
+
+interface MarkdownSection {
+  heading: string;       // The heading text (e.g. "## Authentication")
+  level: number;         // Heading level (1-6), 0 for root/frontmatter
+  breadcrumb: string;    // Full path: "Authentication > JWT Flow"
+  lines: string[];       // Body lines (excluding heading line itself)
+  startLine: number;     // 1-based line number of heading
+  endLine: number;       // 1-based line number of last body line
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Chunk a markdown file into section-aware chunks.
+ * Each section gets a dk-forge context header with heading breadcrumb.
+ */
+export function chunkMarkdown(
+  content: string,
+  filePath: string,
+  repoId: string,
+  indexedAt: number,
+): CodeChunk[] {
+  const allLines = content.split('\n');
+  const chunks: CodeChunk[] = [];
+
+  // Extract frontmatter first
+  const { frontmatter, bodyStartLine } = extractFrontmatter(allLines);
+  if (frontmatter) {
+    const fmContent = `// [dk-forge] file: ${filePath} | section: "frontmatter" | lines: 1-${bodyStartLine - 1}\n` +
+      frontmatter;
+    chunks.push(makeChunk(
+      fmContent, filePath, repoId, 'markdown',
+      1, bodyStartLine - 1, 'markdown', indexedAt, 'frontmatter',
+    ));
+  }
+
+  // Parse body into sections
+  const bodyLines = allLines.slice(bodyStartLine - 1);
+  const sections = parseSections(bodyLines, bodyStartLine);
+
+  // Merge tiny sections with their next sibling
+  const merged = mergeTinySections(sections);
+
+  // Emit chunks, splitting oversized sections
+  for (const section of merged) {
+    const sectionChunks = sectionToChunks(section, filePath, repoId, indexedAt);
+    chunks.push(...sectionChunks);
+  }
+
+  return chunks;
+}
+
+// ---------------------------------------------------------------------------
+// Frontmatter extraction
+// ---------------------------------------------------------------------------
+
+function extractFrontmatter(lines: string[]): { frontmatter: string | null; bodyStartLine: number } {
+  if (lines.length === 0 || lines[0]!.trim() !== '---') {
+    return { frontmatter: null, bodyStartLine: 1 };
+  }
+
+  // Find closing ---
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i]!.trim() === '---') {
+      const fmLines = lines.slice(0, i + 1);
+      return {
+        frontmatter: fmLines.join('\n'),
+        bodyStartLine: i + 2, // 1-based, line after closing ---
+      };
+    }
+  }
+
+  // No closing --- found, treat entire file as body
+  return { frontmatter: null, bodyStartLine: 1 };
+}
+
+// ---------------------------------------------------------------------------
+// Section parsing
+// ---------------------------------------------------------------------------
+
+const HEADING_RE = /^(#{1,6})\s+(.+)$/;
+
+function parseSections(lines: string[], lineOffset: number): MarkdownSection[] {
+  const sections: MarkdownSection[] = [];
+  const headingStack: Array<{ level: number; text: string }> = [];
+
+  let currentSection: MarkdownSection | null = null;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]!;
+    const match = HEADING_RE.exec(line);
+
+    if (match) {
+      // Flush current section
+      if (currentSection) {
+        currentSection.endLine = lineOffset + i - 1;
+        sections.push(currentSection);
+      }
+
+      const level = match[1]!.length;
+      const headingText = match[2]!.trim();
+
+      // Update heading stack for breadcrumb
+      while (headingStack.length > 0 && headingStack[headingStack.length - 1]!.level >= level) {
+        headingStack.pop();
+      }
+      headingStack.push({ level, text: headingText });
+
+      const breadcrumb = headingStack.map(h => h.text).join(' > ');
+
+      currentSection = {
+        heading: headingText,
+        level,
+        breadcrumb,
+        lines: [],
+        startLine: lineOffset + i,
+        endLine: lineOffset + i, // Will be updated
+      };
+    } else {
+      if (currentSection) {
+        currentSection.lines.push(line);
+      } else {
+        // Content before first heading — create a root section
+        if (line.trim()) {
+          currentSection = {
+            heading: '',
+            level: 0,
+            breadcrumb: '',
+            lines: [line],
+            startLine: lineOffset + i,
+            endLine: lineOffset + i,
+          };
+        }
+      }
+    }
+  }
+
+  // Flush final section
+  if (currentSection) {
+    currentSection.endLine = lineOffset + lines.length - 1;
+    sections.push(currentSection);
+  }
+
+  return sections;
+}
+
+// ---------------------------------------------------------------------------
+// Merge tiny sections
+// ---------------------------------------------------------------------------
+
+function mergeTinySections(sections: MarkdownSection[]): MarkdownSection[] {
+  if (sections.length <= 1) return sections;
+
+  const result: MarkdownSection[] = [];
+  let i = 0;
+
+  while (i < sections.length) {
+    const section = sections[i]!;
+    const bodyText = section.lines.join('\n');
+    const tokens = estimateTokens(bodyText);
+
+    if (tokens < SECTION_MIN_TOKENS && i + 1 < sections.length) {
+      // Merge with next section
+      const next = sections[i + 1]!;
+      const merged: MarkdownSection = {
+        heading: section.heading || next.heading,
+        level: Math.min(section.level, next.level) || section.level || next.level,
+        breadcrumb: section.breadcrumb || next.breadcrumb,
+        lines: [
+          ...(section.heading ? [`# ${section.heading}`, ''] : []),
+          ...section.lines,
+          '',
+          ...(next.heading ? [`${'#'.repeat(next.level)} ${next.heading}`, ''] : []),
+          ...next.lines,
+        ],
+        startLine: section.startLine,
+        endLine: next.endLine,
+      };
+      // Replace next with merged, re-check merged against following
+      sections[i + 1] = merged;
+      i++;
+    } else {
+      result.push(section);
+      i++;
+    }
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Section to chunks (with splitting for oversized)
+// ---------------------------------------------------------------------------
+
+function sectionToChunks(
+  section: MarkdownSection,
+  filePath: string,
+  repoId: string,
+  indexedAt: number,
+): CodeChunk[] {
+  const bodyText = section.lines.join('\n');
+  const headerComment = buildSectionHeader(filePath, section.breadcrumb, section.startLine, section.endLine);
+  const fullContent = headerComment + (section.heading ? `${'#'.repeat(section.level || 1)} ${section.heading}\n\n` : '') + bodyText;
+
+  const tokens = estimateTokens(fullContent);
+
+  if (tokens <= SECTION_MAX_TOKENS) {
+    return [makeChunk(
+      fullContent, filePath, repoId, 'markdown',
+      section.startLine, section.endLine,
+      'markdown', indexedAt,
+      section.heading || undefined,
+    )];
+  }
+
+  // Split oversized section at paragraph boundaries
+  return splitOversizedSection(section, filePath, repoId, indexedAt);
+}
+
+function splitOversizedSection(
+  section: MarkdownSection,
+  filePath: string,
+  repoId: string,
+  indexedAt: number,
+): CodeChunk[] {
+  const chunks: CodeChunk[] = [];
+  const paragraphs = splitIntoParagraphs(section.lines);
+
+  let currentLines: string[] = [];
+  let currentTokens = 0;
+  let chunkStartLine = section.startLine;
+  let partIndex = 0;
+
+  const flush = (endLine: number) => {
+    if (currentLines.length === 0) return;
+    const headerComment = buildSectionHeader(filePath, section.breadcrumb, chunkStartLine, endLine);
+    const content = headerComment + currentLines.join('\n');
+    const entityName = section.heading
+      ? `${section.heading}_part${partIndex++}`
+      : `section_part${partIndex++}`;
+    chunks.push(makeChunk(
+      content, filePath, repoId, 'markdown',
+      chunkStartLine, endLine,
+      'markdown', indexedAt, entityName,
+    ));
+    currentLines = [];
+    currentTokens = 0;
+    chunkStartLine = endLine + 1;
+  };
+
+  for (const para of paragraphs) {
+    const paraTokens = estimateTokens(para.text);
+
+    if (currentTokens + paraTokens > SECTION_MAX_TOKENS && currentLines.length > 0) {
+      flush(chunkStartLine + currentLines.length - 1);
+    }
+
+    currentLines.push(...para.lines);
+    currentTokens += paraTokens;
+  }
+
+  if (currentLines.length > 0) {
+    flush(section.endLine);
+  }
+
+  return chunks;
+}
+
+interface Paragraph {
+  lines: string[];
+  text: string;
+}
+
+function splitIntoParagraphs(lines: string[]): Paragraph[] {
+  const paragraphs: Paragraph[] = [];
+  let currentLines: string[] = [];
+
+  for (const line of lines) {
+    if (line.trim() === '' && currentLines.length > 0) {
+      paragraphs.push({
+        lines: [...currentLines, ''],
+        text: currentLines.join('\n'),
+      });
+      currentLines = [];
+    } else {
+      currentLines.push(line);
+    }
+  }
+
+  if (currentLines.length > 0) {
+    paragraphs.push({
+      lines: currentLines,
+      text: currentLines.join('\n'),
+    });
+  }
+
+  return paragraphs;
+}
+
+// ---------------------------------------------------------------------------
+// Header builder
+// ---------------------------------------------------------------------------
+
+function buildSectionHeader(
+  filePath: string,
+  breadcrumb: string,
+  startLine: number,
+  endLine: number,
+): string {
+  const sectionStr = breadcrumb ? ` | section: "${breadcrumb}"` : '';
+  return `// [dk-forge] file: ${filePath}${sectionStr} | lines: ${startLine}-${endLine}\n`;
+}

package/src/ingestion/markdown-knowledge.ts

@@ -0,0 +1,175 @@
+/**
+ * Markdown knowledge extractor.
+ * Scans markdown document headings for keywords that signal knowledge items
+ * (gotchas, patterns, decisions, conventions) and extracts matching sections.
+ *
+ * Used during indexing to automatically populate the knowledge collection
+ * from documentation files without manual YAML curation.
+ */
+
+import { createHash } from 'crypto';
+import type { KnowledgeItem, KnowledgeCategory } from '../util/types.js';
+
+const MAX_CONTENT_LENGTH = 2000;
+
+// ---------------------------------------------------------------------------
+// Heading keyword patterns
+// ---------------------------------------------------------------------------
+
+interface CategoryMatcher {
+  category: KnowledgeCategory;
+  keywords: string[];
+}
+
+const CATEGORY_MATCHERS: CategoryMatcher[] = [
+  {
+    category: 'gotcha',
+    keywords: ['gotcha', 'warning', 'caveat', 'pitfall', 'watch out', 'known issue'],
+  },
+  {
+    category: 'pattern',
+    keywords: ['pattern', 'best practice', 'convention', 'approach'],
+  },
+  {
+    category: 'decision',
+    keywords: ['decision', 'adr', 'why we', 'trade-off', 'tradeoff', 'we chose'],
+  },
+  {
+    category: 'convention',
+    keywords: ['convention', 'rule', 'standard', 'naming'],
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract knowledge items from a markdown file's content.
+ * Scans headings for category keywords and returns matching sections
+ * as KnowledgeItem objects with deterministic IDs.
+ *
+ * @param content - Raw markdown file content
+ * @param filePath - Absolute path to the file (used for ID generation)
+ * @param repoId - Repository ID (not used in item directly but available for caller context)
+ * @param stackTags - Stack tags from repo manifest to attach to items
+ */
+export function extractKnowledge(
+  content: string,
+  filePath: string,
+  _repoId: string,
+  stackTags: string[] = [],
+): KnowledgeItem[] {
+  const items: KnowledgeItem[] = [];
+  const sections = parseIntoSections(content);
+  const now = Date.now();
+
+  for (const section of sections) {
+    const category = matchCategory(section.heading);
+    if (!category) continue;
+
+    const id = generateDeterministicId(filePath, section.heading, category);
+    const body = section.body.slice(0, MAX_CONTENT_LENGTH).trim();
+
+    if (!body) continue;
+
+    items.push({
+      id,
+      title: section.heading,
+      content: body,
+      stack_tags: stackTags,
+      confidence: 0.6,
+      source: 'agent',
+      source_phase: null,
+      source_agent: 'indexer',
+      created_at: now,
+      updated_at: now,
+    });
+  }
+
+  return items;
+}
+
+// ---------------------------------------------------------------------------
+// Section parsing
+// ---------------------------------------------------------------------------
+
+interface HeadingSection {
+  heading: string;
+  level: number;
+  body: string;
+}
+
+const HEADING_RE = /^(#{1,6})\s+(.+)$/;
+
+function parseIntoSections(content: string): HeadingSection[] {
+  const lines = content.split('\n');
+  const sections: HeadingSection[] = [];
+  let current: HeadingSection | null = null;
+  let bodyLines: string[] = [];
+
+  const flush = () => {
+    if (current) {
+      current.body = bodyLines.join('\n');
+      sections.push(current);
+      bodyLines = [];
+    }
+  };
+
+  for (const line of lines) {
+    const match = HEADING_RE.exec(line);
+    if (match) {
+      flush();
+      current = {
+        heading: match[2]!.trim(),
+        level: match[1]!.length,
+        body: '',
+      };
+    } else if (current) {
+      bodyLines.push(line);
+    }
+  }
+
+  flush();
+  return sections;
+}
+
+// ---------------------------------------------------------------------------
+// Category matching
+// ---------------------------------------------------------------------------
+
+function matchCategory(heading: string): KnowledgeCategory | null {
+  const lower = heading.toLowerCase();
+
+  for (const matcher of CATEGORY_MATCHERS) {
+    for (const keyword of matcher.keywords) {
+      if (lower.includes(keyword)) {
+        return matcher.category;
+      }
+    }
+  }
+
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Deterministic ID generation
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate a deterministic ID from filePath + heading.
+ * Format: `<category>-<hash-first-12>` to match the convention used by
+ * KnowledgeHydrator._categoryFromId().
+ *
+ * Idempotent: re-indexing the same file produces the same IDs, so Qdrant
+ * upserts overwrite rather than duplicate.
+ */
+function generateDeterministicId(
+  filePath: string,
+  heading: string,
+  category: KnowledgeCategory,
+): string {
+  const input = `${filePath}::${heading}`;
+  const hash = createHash('sha256').update(input).digest('hex').slice(0, 12);
+  return `${category}-auto-${hash}`;
+}