mdcontext 0.0.1 → 0.2.0

package/src/parser/parser.test.ts

@@ -0,0 +1,291 @@
+import { Effect } from 'effect'
+import { describe, expect, it } from 'vitest'
+import { parse } from './parser.js'
+
+describe('markdown parser', () => {
+  describe('basic parsing', () => {
+    it('parses a simple markdown document', async () => {
+      const content = `# Hello World
+
+This is a paragraph.
+
+## Section One
+
+Content for section one.
+
+## Section Two
+
+Content for section two.
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.title).toBe('Hello World')
+      expect(result.sections).toHaveLength(1) // One H1 as root
+      expect(result.sections[0]?.heading).toBe('Hello World')
+      expect(result.sections[0]?.children).toHaveLength(2) // Two H2 children
+    })
+
+    it('extracts frontmatter', async () => {
+      const content = `---
+title: Custom Title
+author: Test Author
+tags:
+  - markdown
+  - parser
+---
+
+# Heading
+
+Content here.
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.frontmatter).toEqual({
+        title: 'Custom Title',
+        author: 'Test Author',
+        tags: ['markdown', 'parser'],
+      })
+      expect(result.title).toBe('Heading') // H1 takes precedence
+    })
+
+    it('uses frontmatter title when no H1 present', async () => {
+      const content = `---
+title: Frontmatter Title
+---
+
+## Only an H2
+
+Some content.
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.title).toBe('Frontmatter Title')
+    })
+
+    it('handles malformed YAML frontmatter gracefully', async () => {
+      const content = `---
+title: Valid Start
+But this is not valid YAML:
+  - missing colon here
+  invalid: [unclosed bracket
+---
+
+# Actual Content
+
+This should still parse.
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      // Should not throw, should parse with empty frontmatter
+      expect(result.frontmatter).toEqual({})
+      expect(result.title).toBe('Actual Content')
+    })
+  })
+
+  describe('section hierarchy', () => {
+    it('builds proper section hierarchy', async () => {
+      const content = `# Root
+
+## Level 2 A
+
+### Level 3 A1
+
+Content
+
+### Level 3 A2
+
+Content
+
+## Level 2 B
+
+Content
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.sections).toHaveLength(1)
+
+      const root = result.sections[0]!
+      expect(root.heading).toBe('Root')
+      expect(root.children).toHaveLength(2)
+
+      const level2A = root.children[0]!
+      expect(level2A.heading).toBe('Level 2 A')
+      expect(level2A.children).toHaveLength(2)
+
+      const level3A1 = level2A.children[0]!
+      expect(level3A1.heading).toBe('Level 3 A1')
+      expect(level3A1.children).toHaveLength(0)
+    })
+  })
+
+  describe('links', () => {
+    it('extracts internal links', async () => {
+      const content = `# Links
+
+Check out [other doc](./other.md).
+
+And [section link](#section).
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.links).toHaveLength(2)
+      expect(result.links[0]?.type).toBe('internal')
+      expect(result.links[0]?.href).toBe('./other.md')
+      expect(result.links[1]?.type).toBe('internal')
+      expect(result.links[1]?.href).toBe('#section')
+    })
+
+    it('extracts external links', async () => {
+      const content = `# External Links
+
+Visit [Google](https://google.com).
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.links).toHaveLength(1)
+      expect(result.links[0]?.type).toBe('external')
+      expect(result.links[0]?.href).toBe('https://google.com')
+    })
+
+    it('extracts image links', async () => {
+      const content = `# Images
+
+![Alt text](./image.png)
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.links).toHaveLength(1)
+      expect(result.links[0]?.type).toBe('image')
+      expect(result.links[0]?.text).toBe('Alt text')
+    })
+  })
+
+  describe('code blocks', () => {
+    it('extracts code blocks with language', async () => {
+      const content = `# Code
+
+\`\`\`typescript
+const x = 1;
+\`\`\`
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.codeBlocks).toHaveLength(1)
+      expect(result.codeBlocks[0]?.language).toBe('typescript')
+      expect(result.codeBlocks[0]?.content).toBe('const x = 1;')
+    })
+
+    it('extracts code blocks without language', async () => {
+      const content = `# Code
+
+\`\`\`
+plain text
+\`\`\`
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.codeBlocks).toHaveLength(1)
+      expect(result.codeBlocks[0]?.language).toBeNull()
+    })
+  })
+
+  describe('GFM features', () => {
+    it('detects tables in sections', async () => {
+      const content = `# Tables
+
+| Header 1 | Header 2 |
+| -------- | -------- |
+| Cell 1   | Cell 2   |
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.sections[0]?.metadata.hasTable).toBe(true)
+    })
+
+    it('detects lists in sections', async () => {
+      const content = `# Lists
+
+- Item 1
+- Item 2
+- Item 3
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.sections[0]?.metadata.hasList).toBe(true)
+    })
+
+    it('detects task lists', async () => {
+      const content = `# Tasks
+
+- [ ] Todo item
+- [x] Completed item
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.sections[0]?.metadata.hasList).toBe(true)
+    })
+  })
+
+  describe('metadata', () => {
+    it('counts tokens and words', async () => {
+      const content = `# Document
+
+This is some text content for testing token counting.
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.metadata.tokenCount).toBeGreaterThan(0)
+      expect(result.metadata.wordCount).toBeGreaterThan(0)
+    })
+
+    it('counts links and code blocks', async () => {
+      const content = `# Test
+
+[Link 1](./a.md)
+[Link 2](./b.md)
+
+\`\`\`js
+code
+\`\`\`
+
+\`\`\`py
+code
+\`\`\`
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.metadata.linkCount).toBe(2)
+      expect(result.metadata.codeBlockCount).toBe(2)
+    })
+
+    it('counts headings', async () => {
+      const content = `# H1
+
+## H2
+
+### H3
+
+## Another H2
+`
+
+      const result = await Effect.runPromise(parse(content))
+
+      expect(result.metadata.headingCount).toBe(4)
+    })
+  })
+})

package/src/parser/parser.ts

@@ -0,0 +1,394 @@
+/**
+ * Markdown parser using remark/unified
+ * Handles GFM (tables, task lists) and YAML frontmatter
+ */
+
+import * as crypto from 'node:crypto'
+import { Effect } from 'effect'
+import matter from 'gray-matter'
+import type { Code, Heading, Image, Link, Parent, Root, Text } from 'mdast'
+import remarkGfm from 'remark-gfm'
+import remarkParse from 'remark-parse'
+import { unified } from 'unified'
+import { visit } from 'unist-util-visit'
+
+import type {
+  DocumentMetadata,
+  HeadingLevel,
+  MdCodeBlock,
+  MdDocument,
+  MdLink,
+  MdSection,
+  ParseError,
+} from '../core/types.js'
+import { FileReadError } from '../errors/index.js'
+import { countTokensApprox, countWords } from '../utils/tokens.js'
+
+// ============================================================================
+// Parser Configuration
+// ============================================================================
+
+const processor = unified().use(remarkParse).use(remarkGfm)
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+const generateId = (input: string): string => {
+  return crypto.createHash('md5').update(input).digest('hex').slice(0, 12)
+}
+
+const slugify = (text: string): string => {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, '')
+    .replace(/\s+/g, '-')
+    .replace(/-+/g, '-')
+    .trim()
+}
+
+const isInternalLink = (href: string): boolean => {
+  if (href.startsWith('http://') || href.startsWith('https://')) return false
+  if (href.startsWith('mailto:')) return false
+  if (href.startsWith('#')) return true
+  if (href.endsWith('.md') || href.includes('.md#')) return true
+  return !href.includes('://')
+}
+
+const extractPlainText = (node: Parent | Root): string => {
+  const texts: string[] = []
+  visit(node, 'text', (textNode: Text) => {
+    texts.push(textNode.value)
+  })
+  return texts.join(' ')
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const getNodeEndLine = (node: any): number => {
+  return node?.position?.end?.line ?? 0
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const getNodeStartLine = (node: any): number => {
+  return node?.position?.start?.line ?? 0
+}
+
+// ============================================================================
+// Section Extraction
+// ============================================================================
+
+interface RawSection {
+  heading: string
+  level: HeadingLevel
+  startLine: number
+  endLine: number
+  contentStartLine: number
+  contentNodes: unknown[]
+}
+
+const extractRawSections = (tree: Root): RawSection[] => {
+  const sections: RawSection[] = []
+  const headings: {
+    heading: string
+    level: HeadingLevel
+    line: number
+    index: number
+  }[] = []
+
+  // First pass: collect all headings with their positions
+  tree.children.forEach((node, index) => {
+    if (node.type === 'heading') {
+      const heading = node as Heading
+      headings.push({
+        heading: extractPlainText(heading),
+        level: heading.depth as HeadingLevel,
+        line: getNodeStartLine(node),
+        index,
+      })
+    }
+  })
+
+  // Second pass: create sections from headings
+  headings.forEach((h, i) => {
+    const nextHeading = headings[i + 1]
+    const endIndex = nextHeading ? nextHeading.index : tree.children.length
+
+    // Get content nodes between this heading and the next
+    const contentNodes = tree.children.slice(h.index + 1, endIndex)
+    const lastContentNode = contentNodes[contentNodes.length - 1]
+    const endLine = lastContentNode ? getNodeEndLine(lastContentNode) : h.line
+
+    sections.push({
+      heading: h.heading,
+      level: h.level,
+      startLine: h.line,
+      endLine,
+      contentStartLine: h.line + 1,
+      contentNodes,
+    })
+  })
+
+  return sections
+}
+
+const buildSectionHierarchy = (
+  rawSections: RawSection[],
+  docId: string,
+  lines: string[],
+): MdSection[] => {
+  const result: MdSection[] = []
+  const stack: { section: MdSection; level: number }[] = []
+
+  for (const raw of rawSections) {
+    const contentLines = lines.slice(raw.startLine - 1, raw.endLine)
+    const content = contentLines.join('\n')
+    const plainText = extractSectionPlainText(raw.contentNodes as Parent[])
+
+    const hasCode = (raw.contentNodes as { type: string }[]).some(
+      (n) => n.type === 'code',
+    )
+    const hasList = (raw.contentNodes as { type: string }[]).some(
+      (n) => n.type === 'list',
+    )
+    const hasTable = (raw.contentNodes as { type: string }[]).some(
+      (n) => n.type === 'table',
+    )
+
+    const section: MdSection = {
+      id: `${docId}-${slugify(raw.heading)}`,
+      heading: raw.heading,
+      level: raw.level,
+      content,
+      plainText,
+      startLine: raw.startLine,
+      endLine: raw.endLine,
+      children: [],
+      metadata: {
+        wordCount: countWords(plainText),
+        tokenCount: countTokensApprox(content),
+        hasCode,
+        hasList,
+        hasTable,
+      },
+    }
+
+    // Build hierarchy: find parent for this section
+    while (stack.length > 0 && stack[stack.length - 1]!.level >= raw.level) {
+      stack.pop()
+    }
+
+    if (stack.length === 0) {
+      result.push(section)
+    } else {
+      const parent = stack[stack.length - 1]!
+      ;(parent.section.children as MdSection[]).push(section)
+    }
+
+    stack.push({ section, level: raw.level })
+  }
+
+  return result
+}
+
+const extractSectionPlainText = (nodes: Parent[]): string => {
+  const texts: string[] = []
+  for (const node of nodes) {
+    if ('value' in node && typeof node.value === 'string') {
+      texts.push(node.value)
+    } else if ('children' in node) {
+      texts.push(extractPlainText(node))
+    }
+  }
+  return texts.join(' ')
+}
+
+const countAllSections = (sections: MdSection[]): number => {
+  let count = 0
+  for (const section of sections) {
+    count += 1
+    count += countAllSections(section.children as MdSection[])
+  }
+  return count
+}
+
+// ============================================================================
+// Link Extraction
+// ============================================================================
+
+const extractLinks = (tree: Root, docId: string): MdLink[] => {
+  const links: MdLink[] = []
+  let currentSectionId = docId
+
+  visit(tree, (node) => {
+    if (node.type === 'heading') {
+      currentSectionId = `${docId}-${slugify(extractPlainText(node as Heading))}`
+    }
+
+    if (node.type === 'link') {
+      const link = node as Link
+      const internal = isInternalLink(link.url)
+      links.push({
+        type: internal ? 'internal' : 'external',
+        href: link.url,
+        text: extractPlainText(link),
+        sectionId: currentSectionId,
+        line: getNodeStartLine(node),
+      })
+    }
+
+    if (node.type === 'image') {
+      const img = node as Image
+      links.push({
+        type: 'image',
+        href: img.url,
+        text: img.alt ?? '',
+        sectionId: currentSectionId,
+        line: getNodeStartLine(node),
+      })
+    }
+  })
+
+  return links
+}
+
+// ============================================================================
+// Code Block Extraction
+// ============================================================================
+
+const extractCodeBlocks = (tree: Root, docId: string): MdCodeBlock[] => {
+  const codeBlocks: MdCodeBlock[] = []
+  let currentSectionId = docId
+
+  visit(tree, (node) => {
+    if (node.type === 'heading') {
+      currentSectionId = `${docId}-${slugify(extractPlainText(node as Heading))}`
+    }
+
+    if (node.type === 'code') {
+      const code = node as Code
+      codeBlocks.push({
+        language: code.lang ?? null,
+        content: code.value,
+        sectionId: currentSectionId,
+        startLine: getNodeStartLine(node),
+        endLine: getNodeEndLine(node),
+      })
+    }
+  })
+
+  return codeBlocks
+}
+
+// ============================================================================
+// Main Parser Function
+// ============================================================================
+
+export interface ParseOptions {
+  readonly path?: string
+  readonly lastModified?: Date
+}
+
+export const parse = (
+  content: string,
+  options: ParseOptions = {},
+): Effect.Effect<MdDocument, ParseError> =>
+  Effect.gen(function* () {
+    const path = options.path ?? 'unknown'
+    const docId = generateId(path)
+    const now = new Date()
+
+    // Extract frontmatter (graceful handling for malformed YAML)
+    let frontmatter: Record<string, unknown> = {}
+    let markdownContent: string = content
+
+    try {
+      const parsed = matter(content)
+      frontmatter = parsed.data
+      markdownContent = parsed.content
+    } catch (error) {
+      // Malformed frontmatter - treat entire content as markdown
+      const msg = error instanceof Error ? error.message : String(error)
+      console.warn(
+        `Warning: Malformed frontmatter in ${path}, skipping: ${msg.split('\n')[0]}`,
+      )
+    }
+
+    // Parse markdown to AST
+    const tree = processor.parse(markdownContent) as Root
+
+    // Split content into lines for reference
+    const lines = markdownContent.split('\n')
+
+    // Extract sections
+    const rawSections = extractRawSections(tree)
+    const sections = buildSectionHierarchy(rawSections, docId, lines)
+
+    // Extract links and code blocks
+    const links = extractLinks(tree, docId)
+    const codeBlocks = extractCodeBlocks(tree, docId)
+
+    // Determine title (first H1 or filename)
+    const firstH1 = sections.find((s) => s.level === 1)
+    const title =
+      firstH1?.heading ??
+      (typeof frontmatter.title === 'string' ? frontmatter.title : null) ??
+      path.split('/').pop()?.replace(/\.md$/, '') ??
+      'Untitled'
+
+    // Calculate metadata
+    const totalContent = sections.map((s) => s.content).join('\n')
+    const metadata: DocumentMetadata = {
+      wordCount: countWords(totalContent),
+      tokenCount: countTokensApprox(content),
+      headingCount: countAllSections(sections),
+      linkCount: links.length,
+      codeBlockCount: codeBlocks.length,
+      lastModified: options.lastModified ?? now,
+      indexedAt: now,
+    }
+
+    const document: MdDocument = {
+      id: docId,
+      path,
+      title,
+      frontmatter,
+      sections,
+      links,
+      codeBlocks,
+      metadata,
+    }
+
+    return document
+  })
+
+/**
+ * Parse a markdown file from the filesystem
+ *
+ * @throws ParseError - File content cannot be parsed
+ * @throws FileReadError - File cannot be read from filesystem
+ */
+export const parseFile = (
+  filePath: string,
+): Effect.Effect<MdDocument, ParseError | FileReadError> =>
+  Effect.gen(function* () {
+    const fs = yield* Effect.promise(() => import('node:fs/promises'))
+
+    const [content, stats] = yield* Effect.tryPromise({
+      try: () =>
+        Promise.all([
+          fs.readFile(filePath, 'utf-8'),
+          fs.stat(filePath),
+        ] as const),
+      catch: (error) =>
+        new FileReadError({
+          path: filePath,
+          message: error instanceof Error ? error.message : 'Unknown error',
+          cause: error,
+        }),
+    })
+
+    return yield* parse(content, {
+      path: filePath,
+      lastModified: stats.mtime,
+    })
+  })