@incremark/core 0.1.2 → 0.2.0

package/src/parser/IncremarkParser.ts

@@ -11,6 +11,7 @@
 import { fromMarkdown } from 'mdast-util-from-markdown'
 import { gfmFromMarkdown } from 'mdast-util-gfm'
 import { gfm } from 'micromark-extension-gfm'
+import { gfmFootnoteFromMarkdown } from 'mdast-util-gfm-footnote'
 import type { Extension as MicromarkExtension } from 'micromark-util-types'
 import type { Extension as MdastExtension } from 'mdast-util-from-markdown'
 
@@ -22,9 +23,16 @@ import type {
   ParserOptions,
   BlockStatus,
   BlockContext,
-  ContainerConfig
+  ContainerConfig,
+  ParserState
 } from '../types'
 
+import { transformHtmlNodes, type HtmlTreeExtensionOptions } from '../extensions/html-extension'
+import { micromarkReferenceExtension } from '../extensions/micromark-reference-extension'
+import { gfmFootnoteIncremental } from '../extensions/micromark-gfm-footnote-incremental'
+import type { HTML, Paragraph, Text, Parent as MdastParent, Definition, FootnoteDefinition } from 'mdast'
+import type { DefinitionMap, FootnoteDefinitionMap } from '../types'
+
 import {
   createInitialContext,
   updateContext,
@@ -34,8 +42,11 @@ import {
   isThematicBreak,
   isBlockquoteStart,
   isListItemStart,
-  detectContainer
+  detectContainer,
+  isFootnoteDefinitionStart,
+  isFootnoteContinuation
 } from '../detector'
+import { isDefinitionNode, isFootnoteDefinitionNode } from '../utils'
 
 // ============ 解析器类 ============
 
@@ -51,8 +62,16 @@ export class IncremarkParser {
   private options: ParserOptions
   /** 缓存的容器配置，避免重复计算 */
   private readonly containerConfig: ContainerConfig | undefined
+  /** 缓存的 HTML 树配置，避免重复计算 */
+  private readonly htmlTreeConfig: HtmlTreeExtensionOptions | undefined
   /** 上次 append 返回的 pending blocks，用于 getAst 复用 */
   private lastPendingBlocks: ParsedBlock[] = []
+  /** Definition 映射表（用于引用式图片和链接） */
+  private definitionMap: DefinitionMap = {}
+  /** Footnote Definition 映射表 */
+  private footnoteDefinitionMap: FootnoteDefinitionMap = {}
+  /** Footnote Reference 出现顺序（按引用在文档中的顺序） */
+  private footnoteReferenceOrder: string[] = []
 
   constructor(options: ParserOptions = {}) {
     this.options = {
@@ -62,6 +81,8 @@ export class IncremarkParser {
     this.context = createInitialContext()
     // 初始化容器配置（构造时计算一次）
     this.containerConfig = this.computeContainerConfig()
+    // 初始化 HTML 树配置
+    this.htmlTreeConfig = this.computeHtmlTreeConfig()
   }
 
   private generateBlockId(): string {
@@ -74,13 +95,103 @@ export class IncremarkParser {
     return containers === true ? {} : containers
   }
 
+  private computeHtmlTreeConfig(): HtmlTreeExtensionOptions | undefined {
+    const htmlTree = this.options.htmlTree
+    if (!htmlTree) return undefined
+    return htmlTree === true ? {} : htmlTree
+  }
+
+  /**
+   * 将 HTML 节点转换为纯文本
+   * 递归处理 AST 中所有 html 类型的节点
+   * - 块级 HTML 节点 → 转换为 paragraph 包含 text
+   * - 内联 HTML 节点（在段落内部）→ 转换为 text 节点
+   */
+  private convertHtmlToText(ast: Root): Root {
+    // 处理内联节点（段落内部的 children）
+    const processInlineChildren = (children: unknown[]): unknown[] => {
+      return children.map(node => {
+        const n = node as RootContent
+        // 内联 html 节点转换为纯文本节点
+        if (n.type === 'html') {
+          const htmlNode = n as HTML
+          const textNode: Text = {
+            type: 'text',
+            value: htmlNode.value,
+            position: htmlNode.position
+          }
+          return textNode
+        }
+        
+        // 递归处理有 children 的内联节点（如 strong, emphasis 等）
+        if ('children' in n && Array.isArray(n.children)) {
+          const parent = n as MdastParent
+          return {
+            ...parent,
+            children: processInlineChildren(parent.children)
+          }
+        }
+        
+        return n
+      })
+    }
+
+    // 处理块级节点
+    const processBlockChildren = (children: RootContent[]): RootContent[] => {
+      return children.map(node => {
+        // 块级 html 节点转换为段落包含纯文本
+        if (node.type === 'html') {
+          const htmlNode = node as HTML
+          const textNode: Text = {
+            type: 'text',
+            value: htmlNode.value
+          }
+          const paragraphNode: Paragraph = {
+            type: 'paragraph',
+            children: [textNode],
+            position: htmlNode.position
+          }
+          return paragraphNode as RootContent
+        }
+        
+        // 递归处理有 children 的块级节点
+        if ('children' in node && Array.isArray(node.children)) {
+          const parent = node as MdastParent
+          // 对于段落等内联容器，使用 processInlineChildren
+          if (node.type === 'paragraph' || node.type === 'heading' || 
+              node.type === 'tableCell' || node.type === 'delete' ||
+              node.type === 'emphasis' || node.type === 'strong' ||
+              node.type === 'link' || node.type === 'linkReference') {
+            return {
+              ...parent,
+              children: processInlineChildren(parent.children)
+            } as RootContent
+          }
+          // 对于其他块级容器，递归处理
+          return {
+            ...parent,
+            children: processBlockChildren(parent.children as RootContent[])
+          } as RootContent
+        }
+        
+        return node
+      })
+    }
+    
+    return {
+      ...ast,
+      children: processBlockChildren(ast.children)
+    }
+  }
+
   private parse(text: string): Root {
     const extensions: MicromarkExtension[] = []
     const mdastExtensions: MdastExtension[] = []
 
+    // 先添加 GFM（包含原始的脚注扩展）
     if (this.options.gfm) {
       extensions.push(gfm())
-      mdastExtensions.push(...gfmFromMarkdown())
+      mdastExtensions.push(...gfmFromMarkdown(), gfmFootnoteFromMarkdown())
     }
 
     // 如果用户传入了自定义扩展，添加它们
@@ -91,7 +202,110 @@ export class IncremarkParser {
       mdastExtensions.push(...this.options.mdastExtensions)
     }
 
-    return fromMarkdown(text, { extensions, mdastExtensions })
+    // 添加增量脚注扩展，覆盖 GFM 脚注的定义检查
+    // ⚠️ 必须在 micromarkReferenceExtension 之前添加
+    // 因为 micromarkReferenceExtension 会拦截 `]`，并将 `[^1]` 交给脚注扩展处理
+    if (this.options.gfm) {
+      extensions.push(gfmFootnoteIncremental())
+    }
+    
+    // 添加 reference 扩展（支持增量解析），覆盖 commonmark 的 labelEnd
+    // ⚠️ 必须最后添加，确保它能拦截 `]` 并正确处理脚注
+    extensions.push(micromarkReferenceExtension())
+
+    // 生成 AST
+    let ast = fromMarkdown(text, { extensions, mdastExtensions })
+    
+    // 如果启用了 HTML 树转换，应用转换
+    if (this.htmlTreeConfig) {
+      ast = transformHtmlNodes(ast, this.htmlTreeConfig)
+    } else {
+      // 如果未启用 HTML 树，将 HTML 节点转换为纯文本
+      ast = this.convertHtmlToText(ast)
+    }
+    
+    return ast
+  }
+
+  private updateDefinationsFromComplatedBlocks(blocks: ParsedBlock[]): void{
+    for (const block of blocks) {
+      this.definitionMap = {
+        ...this.definitionMap,
+        ...this.findDefinition(block)
+      }
+
+      this.footnoteDefinitionMap = {
+        ...this.footnoteDefinitionMap,
+        ...this.findFootnoteDefinition(block)
+      }
+    }
+  }
+
+  private findDefinition(block: ParsedBlock): DefinitionMap {
+    const definitions: Definition[] = [];
+
+    function findDefination(node: RootContent) {
+      if (isDefinitionNode(node)) {
+        definitions.push(node as Definition);
+      }
+      
+      if ('children' in node && Array.isArray(node.children)) {
+        for (const child of node.children) {
+          findDefination(child as RootContent);
+        }
+      }
+    }
+
+    findDefination(block.node);
+  
+    return definitions.reduce<DefinitionMap>((acc, node) => {
+      acc[node.identifier] = node;
+      return acc;
+    }, {});
+
+  }
+
+  private findFootnoteDefinition(block: ParsedBlock): FootnoteDefinitionMap {
+    const footnoteDefinitions: FootnoteDefinition[] = [];
+
+    function findFootnoteDefinition(node: RootContent) {
+      if (isFootnoteDefinitionNode(node)) {
+        footnoteDefinitions.push(node as FootnoteDefinition);
+      }
+    }
+
+    findFootnoteDefinition(block.node);
+
+    return footnoteDefinitions.reduce<FootnoteDefinitionMap>((acc, node) => {
+      acc[node.identifier] = node;
+      return acc;
+    }, {});
+  }
+
+  /**
+   * 收集 AST 中的脚注引用（按出现顺序）
+   * 用于确定脚注的显示顺序
+   */
+  private collectFootnoteReferences(nodes: RootContent[]): void {
+    const visitNode = (node: any): void => {
+      if (!node) return
+
+      // 检查是否是脚注引用
+      if (node.type === 'footnoteReference') {
+        const identifier = node.identifier
+        // 去重：只记录第一次出现的位置
+        if (!this.footnoteReferenceOrder.includes(identifier)) {
+          this.footnoteReferenceOrder.push(identifier)
+        }
+      }
+
+      // 递归遍历子节点
+      if (node.children && Array.isArray(node.children)) {
+        node.children.forEach(visitNode)
+      }
+    }
+
+    nodes.forEach(visitNode)
   }
 
   /**
@@ -206,8 +420,48 @@ export class IncremarkParser {
       return -1
     }
 
+    // ============ 脚注定义的特殊处理 ============
+    
+    // 情况 1: 前一行是脚注定义开始
+    if (isFootnoteDefinitionStart(prevLine)) {
+      // 当前行是空行或缩进行，脚注可能继续（不稳定）
+      if (isEmptyLine(line) || isFootnoteContinuation(line)) {
+        return -1
+      }
+      // 当前行是新脚注定义，前一个脚注完成
+      if (isFootnoteDefinitionStart(line)) {
+        return lineIndex - 1
+      }
+      // 当前行是非缩进的新块，前一个脚注完成
+      // 这种情况会在后续的判断中处理
+    }
+    
+    // 情况 2: 前一行是缩进行，可能是脚注延续
+    if (!isEmptyLine(prevLine) && isFootnoteContinuation(prevLine)) {
+      // 向上查找最近的脚注定义
+      const footnoteStartLine = this.findFootnoteStart(lineIndex - 1)
+      if (footnoteStartLine >= 0) {
+        // 确认属于脚注定义
+        // 当前行仍然是缩进或空行，脚注继续（不稳定）
+        if (isEmptyLine(line) || isFootnoteContinuation(line)) {
+          return -1
+        }
+        // 当前行是新脚注定义，前一个脚注完成
+        if (isFootnoteDefinitionStart(line)) {
+          return lineIndex - 1
+        }
+        // 当前行是非缩进的新块，前一个脚注完成
+        return lineIndex - 1
+      }
+    }
+
     // 前一行非空时，如果当前行是新块开始，则前一块已完成
     if (!isEmptyLine(prevLine)) {
+      // 新脚注定义开始（排除连续脚注定义）
+      if (isFootnoteDefinitionStart(line) && !isFootnoteDefinitionStart(prevLine)) {
+        return lineIndex - 1
+      }
+
       // 新标题开始
       if (isHeading(line)) {
         return lineIndex - 1
@@ -248,6 +502,47 @@ export class IncremarkParser {
     return -1
   }
 
+  /**
+   * 从指定行向上查找脚注定义的起始行
+   * 
+   * @param fromLine 开始查找的行索引
+   * @returns 脚注起始行索引，如果不属于脚注返回 -1
+   * 
+   * @example
+   * // 假设 lines 为:
+   * // 0: "[^1]: 第一行"
+   * // 1: "    第二行"
+   * // 2: "    第三行"
+   * findFootnoteStart(2) // 返回 0
+   * findFootnoteStart(1) // 返回 0
+   */
+  private findFootnoteStart(fromLine: number): number {
+    // 限制向上查找的最大行数，避免性能问题
+    const maxLookback = 20
+    const startLine = Math.max(0, fromLine - maxLookback)
+    
+    for (let i = fromLine; i >= startLine; i--) {
+      const line = this.lines[i]
+      
+      // 遇到脚注定义起始行
+      if (isFootnoteDefinitionStart(line)) {
+        return i
+      }
+      
+      // 遇到空行，继续向上查找（可能是脚注内部的段落分隔）
+      if (isEmptyLine(line)) {
+        continue
+      }
+      
+      // 遇到非缩进的普通行，说明不属于脚注
+      if (!isFootnoteContinuation(line)) {
+        return -1
+      }
+    }
+    
+    return -1
+  }
+
   private nodesToBlocks(
     nodes: RootContent[],
     startOffset: number,
@@ -290,7 +585,10 @@ export class IncremarkParser {
       completed: [],
       updated: [],
       pending: [],
-      ast: { type: 'root', children: [] }
+      ast: { type: 'root', children: [] },
+      definitions: {},
+      footnoteDefinitions: {},
+      footnoteReferenceOrder: []
     }
 
     if (stableBoundary >= this.pendingStartLine && stableBoundary >= 0) {
@@ -303,6 +601,9 @@ export class IncremarkParser {
       this.completedBlocks.push(...newBlocks)
       update.completed = newBlocks
 
+      // 更新 definitions 从新完成的 blocks
+      this.updateDefinationsFromComplatedBlocks(newBlocks)
+
       // 直接使用 findStableBoundary 计算好的上下文，避免重复遍历
       this.context = contextAtLine
       this.pendingStartLine = stableBoundary + 1
@@ -327,6 +628,14 @@ export class IncremarkParser {
       children: [...this.completedBlocks.map((b) => b.node), ...update.pending.map((b) => b.node)]
     }
 
+    // 收集脚注引用顺序
+    this.collectFootnoteReferences(update.ast.children)
+
+    // 填充 definitions 和 footnote 相关数据
+    update.definitions = this.getDefinitionMap()
+    update.footnoteDefinitions = this.getFootnoteDefinitionMap()
+    update.footnoteReferenceOrder = this.getFootnoteReferenceOrder()
+
     // 触发状态变化回调
     this.emitChange(update.pending)
 
@@ -338,7 +647,7 @@ export class IncremarkParser {
    */
   private emitChange(pendingBlocks: ParsedBlock[] = []): void {
     if (this.options.onChange) {
-      this.options.onChange({
+      const state: ParserState = {
         completedBlocks: this.completedBlocks,
         pendingBlocks,
         markdown: this.buffer,
@@ -348,8 +657,11 @@ export class IncremarkParser {
             ...this.completedBlocks.map((b) => b.node),
             ...pendingBlocks.map((b) => b.node)
           ]
-        }
-      })
+        },
+        definitions: { ...this.definitionMap },
+        footnoteDefinitions: { ...this.footnoteDefinitionMap }
+      }
+      this.options.onChange(state)
     }
   }
 
@@ -362,7 +674,10 @@ export class IncremarkParser {
       completed: [],
       updated: [],
       pending: [],
-      ast: { type: 'root', children: [] }
+      ast: { type: 'root', children: [] },
+      definitions: {},
+      footnoteDefinitions: {},
+      footnoteReferenceOrder: []
     }
 
     if (this.pendingStartLine < this.lines.length) {
@@ -381,6 +696,9 @@ export class IncremarkParser {
 
         this.completedBlocks.push(...finalBlocks)
         update.completed = finalBlocks
+
+        // 更新 definitions 从最终完成的 blocks
+        this.updateDefinationsFromComplatedBlocks(finalBlocks)
       }
     }
 
@@ -393,6 +711,14 @@ export class IncremarkParser {
       children: this.completedBlocks.map((b) => b.node)
     }
 
+    // 收集脚注引用顺序
+    this.collectFootnoteReferences(update.ast.children)
+
+    // 填充 definitions 和 footnote 相关数据
+    update.definitions = this.getDefinitionMap()
+    update.footnoteDefinitions = this.getFootnoteDefinitionMap()
+    update.footnoteReferenceOrder = this.getFootnoteReferenceOrder()
+
     // 触发状态变化回调
     this.emitChange([])
 
@@ -412,12 +738,17 @@ export class IncremarkParser {
    * 复用上次 append 的 pending 结果，避免重复解析
    */
   getAst(): Root {
+    const children = [
+      ...this.completedBlocks.map((b) => b.node),
+      ...this.lastPendingBlocks.map((b) => b.node)
+    ]
+
+    // 收集脚注引用顺序
+    this.collectFootnoteReferences(children)
+
     return {
       type: 'root',
-      children: [
-        ...this.completedBlocks.map((b) => b.node),
-        ...this.lastPendingBlocks.map((b) => b.node)
-      ]
+      children
     }
   }
 
@@ -435,11 +766,36 @@ export class IncremarkParser {
     return this.buffer
   }
 
+  /**
+   * 获取 Definition 映射表（用于引用式图片和链接）
+   */
+  getDefinitionMap(): DefinitionMap {
+    return { ...this.definitionMap }
+  }
+
+  /**
+   * 获取 Footnote Definition 映射表
+   */
+  getFootnoteDefinitionMap(): FootnoteDefinitionMap {
+    return { ...this.footnoteDefinitionMap }
+  }
+
+  /**
+   * 获取脚注引用的出现顺序
+   */
+  getFootnoteReferenceOrder(): string[] {
+    return [...this.footnoteReferenceOrder]
+  }
+
   /**
    * 设置状态变化回调（用于 DevTools 等）
    */
   setOnChange(callback: ((state: import('../types').ParserState) => void) | undefined): void {
-    this.options.onChange = callback
+    const originalOnChange = this.options.onChange;
+    this.options.onChange = (state: ParserState) => {
+      originalOnChange?.(state);
+      callback?.(state);
+    }
   }
 
   /**
@@ -454,6 +810,10 @@ export class IncremarkParser {
     this.blockIdCounter = 0
     this.context = createInitialContext()
     this.lastPendingBlocks = []
+    // 清空 definition 映射
+    this.definitionMap = {}
+    this.footnoteDefinitionMap = {}
+    this.footnoteReferenceOrder = []
 
     // 触发状态变化回调
     this.emitChange([])

package/src/types/index.ts

@@ -1,6 +1,19 @@
 import type { Root, RootContent } from 'mdast'
 import type { Extension as MicromarkExtension } from 'micromark-util-types'
 import type { Extension as MdastExtension } from 'mdast-util-from-markdown'
+import type { HtmlTreeExtensionOptions } from '../extensions/html-extension'
+import type { Definition, FootnoteDefinition } from 'mdast'
+
+/**
+ * Definition 映射类型
+ */
+export interface DefinitionMap {
+  [identifier: string]: Definition
+}
+
+export interface FootnoteDefinitionMap {
+  [identifier: string]: FootnoteDefinition
+}
 
 /**
  * 解析块的状态
@@ -51,6 +64,12 @@ export interface IncrementalUpdate {
   pending: ParsedBlock[]
   /** 完整的 AST（包含所有已解析的内容） */
   ast: Root
+  /** Definition 映射表（用于引用式图片和链接） */
+  definitions: DefinitionMap
+  /** Footnote Definition 映射表 */
+  footnoteDefinitions: FootnoteDefinitionMap
+  /** 脚注引用的出现顺序（用于渲染时排序） */
+  footnoteReferenceOrder: string[]
 }
 
 /**
@@ -76,7 +95,9 @@ export interface ParserState {
   /** 完整的 Markdown 内容 */
   markdown: string
   /** 完整的 AST */
-  ast: Root
+  ast: Root,
+  definitions: DefinitionMap,
+  footnoteDefinitions: FootnoteDefinitionMap
 }
 
 /**
@@ -92,6 +113,13 @@ export interface ParserOptions {
    * - ContainerConfig: 使用自定义配置启用
    */
   containers?: boolean | ContainerConfig
+  /**
+   * 启用 HTML 树转换
+   * - false/undefined: 禁用（默认），HTML 节点保持原始 type: 'html' 格式
+   * - true: 使用默认配置启用，将 HTML 节点转换为结构化的 htmlElement 节点
+   * - HtmlTreeExtensionOptions: 使用自定义配置启用（可配置黑名单等）
+   */
+  htmlTree?: boolean | HtmlTreeExtensionOptions
   /** 自定义块边界检测函数 */
   blockBoundaryDetector?: (content: string, position: number) => boolean
   /** 自定义 micromark 扩展（如 directive） */

package/src/utils/index.ts

@@ -2,6 +2,8 @@
  * 工具函数
  */
 
+import type { Definition, FootnoteDefinition, RootContent } from "mdast"
+
 /**
  * 生成唯一 ID
  */
@@ -42,3 +44,10 @@ export function joinLines(lines: string[], start: number, end: number): string {
   return lines.slice(start, end + 1).join('\n')
 }
 
+export function isDefinitionNode(node: RootContent): node is Definition {
+  return node.type === 'definition'
+}
+
+export function isFootnoteDefinitionNode(node: RootContent): node is FootnoteDefinition {
+  return node.type === 'footnoteDefinition'
+}