@weavelogic/knowledge-graph-agent 0.2.0 → 0.4.0

package/dist/chunking/index.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Document Chunking Module
+ *
+ * Provides intelligent document chunking for RAG and vector search.
+ */
+/**
+ * Chunking strategy
+ */
+export type ChunkStrategy = 'fixed' | 'sentence' | 'paragraph' | 'semantic' | 'markdown' | 'code';
+/**
+ * Chunking options
+ */
+export interface ChunkOptions {
+    /** Chunking strategy */
+    strategy: ChunkStrategy;
+    /** Target chunk size in characters */
+    chunkSize?: number;
+    /** Overlap between chunks in characters */
+    overlap?: number;
+    /** Minimum chunk size */
+    minChunkSize?: number;
+    /** Maximum chunk size */
+    maxChunkSize?: number;
+    /** Include metadata in chunks */
+    includeMetadata?: boolean;
+    /** Preserve code blocks */
+    preserveCodeBlocks?: boolean;
+}
+/**
+ * Chunk metadata
+ */
+export interface ChunkMetadata {
+    /** Source document path */
+    source: string;
+    /** Chunk index in document */
+    index: number;
+    /** Total chunks in document */
+    total: number;
+    /** Start position in original document */
+    startPosition: number;
+    /** End position in original document */
+    endPosition: number;
+    /** Heading hierarchy */
+    headings?: string[];
+    /** Code language (for code chunks) */
+    language?: string;
+}
+/**
+ * A single chunk
+ */
+export interface Chunk {
+    /** Unique chunk ID */
+    id: string;
+    /** Chunk content */
+    content: string;
+    /** Chunk metadata */
+    metadata: ChunkMetadata;
+    /** Token count estimate */
+    tokenCount?: number;
+}
+/**
+ * Chunking result
+ */
+export interface ChunkResult {
+    /** Generated chunks */
+    chunks: Chunk[];
+    /** Original document length */
+    originalLength: number;
+    /** Strategy used */
+    strategy: ChunkStrategy;
+    /** Processing time in milliseconds */
+    processingTime: number;
+}
+/**
+ * Document Chunker for splitting documents into chunks
+ */
+export declare class Chunker {
+    private defaultOptions;
+    constructor(options?: Partial<ChunkOptions>);
+    /**
+     * Chunk a document
+     */
+    chunk(content: string, source: string, options?: Partial<ChunkOptions>): ChunkResult;
+    /**
+     * Fixed-size chunking
+     */
+    private chunkFixed;
+    /**
+     * Sentence-based chunking
+     */
+    private chunkBySentence;
+    /**
+     * Paragraph-based chunking
+     */
+    private chunkByParagraph;
+    /**
+     * Markdown-aware chunking
+     */
+    private chunkByMarkdown;
+    /**
+     * Code-aware chunking
+     */
+    private chunkByCode;
+    /**
+     * Semantic chunking (placeholder for more advanced implementation)
+     */
+    private chunkSemantic;
+    /**
+     * Aggregate small chunks into larger ones
+     */
+    private aggregateChunks;
+    /**
+     * Estimate token count
+     */
+    private estimateTokens;
+}
+/**
+ * Create a new Chunker instance
+ */
+export declare function createChunker(options?: Partial<ChunkOptions>): Chunker;
+/**
+ * Convenience function to chunk a document
+ */
+export declare function chunkDocument(content: string, source: string, options?: Partial<ChunkOptions>): ChunkResult;
+//# sourceMappingURL=index.d.ts.map

package/dist/chunking/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunking/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,OAAO,GACP,UAAU,GACV,WAAW,GACX,UAAU,GACV,UAAU,GACV,MAAM,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,wBAAwB;IACxB,QAAQ,EAAE,aAAa,CAAC;IACxB,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yBAAyB;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,2BAA2B;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,wBAAwB;IACxB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,sCAAsC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,qBAAqB;IACrB,QAAQ,EAAE,aAAa,CAAC;IACxB,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,+BAA+B;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB;IACpB,QAAQ,EAAE,aAAa,CAAC;IACxB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;CACxB;AAMD;;GAEG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,cAAc,CAQpB;gBAEU,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC;IAM3C;;OAEG;IACH,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,WAAW;IAoCpF;;OAEG;IACH,OAAO,CAAC,UAAU;IAqClB;;OAEG;IACH,OAAO,CAAC,eAAe;IAKvB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAKxB;;OAEG;IACH,OAAO,CAAC,eAAe;IAgDvB;;OAEG;IACH,OAAO,CAAC,WAAW;IA4EnB;;OAEG;IACH,OAAO,CAAC,aAAa;IAMrB;;OAEG;IACH,OAAO,CAAC,eAAe;IAwDvB;;OAEG;IACH,OAAO,CAAC,cAAc;CAIvB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,OAAO,CAEtE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAC9B,WAAW,CAGb"}

package/dist/chunking/index.js

@@ -0,0 +1,283 @@
+class Chunker {
+  defaultOptions = {
+    strategy: "paragraph",
+    chunkSize: 1e3,
+    overlap: 100,
+    minChunkSize: 100,
+    maxChunkSize: 2e3,
+    includeMetadata: true,
+    preserveCodeBlocks: true
+  };
+  constructor(options) {
+    if (options) {
+      this.defaultOptions = { ...this.defaultOptions, ...options };
+    }
+  }
+  /**
+   * Chunk a document
+   */
+  chunk(content, source, options) {
+    const startTime = Date.now();
+    const opts = { ...this.defaultOptions, ...options };
+    let chunks;
+    switch (opts.strategy) {
+      case "fixed":
+        chunks = this.chunkFixed(content, source, opts);
+        break;
+      case "sentence":
+        chunks = this.chunkBySentence(content, source, opts);
+        break;
+      case "paragraph":
+        chunks = this.chunkByParagraph(content, source, opts);
+        break;
+      case "markdown":
+        chunks = this.chunkByMarkdown(content, source, opts);
+        break;
+      case "code":
+        chunks = this.chunkByCode(content, source, opts);
+        break;
+      case "semantic":
+        chunks = this.chunkSemantic(content, source, opts);
+        break;
+      default:
+        chunks = this.chunkByParagraph(content, source, opts);
+    }
+    return {
+      chunks,
+      originalLength: content.length,
+      strategy: opts.strategy,
+      processingTime: Date.now() - startTime
+    };
+  }
+  /**
+   * Fixed-size chunking
+   */
+  chunkFixed(content, source, opts) {
+    const chunks = [];
+    const chunkSize = opts.chunkSize || 1e3;
+    const overlap = opts.overlap || 0;
+    let position = 0;
+    let index = 0;
+    while (position < content.length) {
+      const end = Math.min(position + chunkSize, content.length);
+      const chunkContent = content.slice(position, end);
+      chunks.push({
+        id: `${source}-${index}`,
+        content: chunkContent,
+        metadata: {
+          source,
+          index,
+          total: 0,
+          // Will be updated after
+          startPosition: position,
+          endPosition: end
+        },
+        tokenCount: this.estimateTokens(chunkContent)
+      });
+      position = end - overlap;
+      index++;
+    }
+    chunks.forEach((chunk) => {
+      chunk.metadata.total = chunks.length;
+    });
+    return chunks;
+  }
+  /**
+   * Sentence-based chunking
+   */
+  chunkBySentence(content, source, opts) {
+    const sentences = content.split(/(?<=[.!?])\s+/);
+    return this.aggregateChunks(sentences, source, opts);
+  }
+  /**
+   * Paragraph-based chunking
+   */
+  chunkByParagraph(content, source, opts) {
+    const paragraphs = content.split(/\n\n+/);
+    return this.aggregateChunks(paragraphs, source, opts);
+  }
+  /**
+   * Markdown-aware chunking
+   */
+  chunkByMarkdown(content, source, opts) {
+    const sections = [];
+    const headings = [];
+    const headerRegex = /^(#{1,6})\s+(.+)$/gm;
+    let lastIndex = 0;
+    let currentHeadings = [];
+    let match;
+    while ((match = headerRegex.exec(content)) !== null) {
+      if (lastIndex < match.index) {
+        sections.push(content.slice(lastIndex, match.index).trim());
+        headings.push([...currentHeadings]);
+      }
+      const level = match[1].length;
+      const heading = match[2];
+      currentHeadings = currentHeadings.slice(0, level - 1);
+      currentHeadings[level - 1] = heading;
+      lastIndex = match.index;
+    }
+    if (lastIndex < content.length) {
+      sections.push(content.slice(lastIndex).trim());
+      headings.push([...currentHeadings]);
+    }
+    return sections.filter((s) => s.length > 0).map((section, index) => ({
+      id: `${source}-${index}`,
+      content: section,
+      metadata: {
+        source,
+        index,
+        total: sections.length,
+        startPosition: 0,
+        endPosition: section.length,
+        headings: headings[index]
+      },
+      tokenCount: this.estimateTokens(section)
+    }));
+  }
+  /**
+   * Code-aware chunking
+   */
+  chunkByCode(content, source, opts) {
+    const chunks = [];
+    const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+    let lastIndex = 0;
+    let index = 0;
+    let match;
+    while ((match = codeBlockRegex.exec(content)) !== null) {
+      if (lastIndex < match.index) {
+        const textContent = content.slice(lastIndex, match.index).trim();
+        if (textContent) {
+          chunks.push({
+            id: `${source}-${index}`,
+            content: textContent,
+            metadata: {
+              source,
+              index,
+              total: 0,
+              startPosition: lastIndex,
+              endPosition: match.index
+            },
+            tokenCount: this.estimateTokens(textContent)
+          });
+          index++;
+        }
+      }
+      const language = match[1] || "unknown";
+      const codeContent = match[2].trim();
+      chunks.push({
+        id: `${source}-${index}`,
+        content: codeContent,
+        metadata: {
+          source,
+          index,
+          total: 0,
+          startPosition: match.index,
+          endPosition: match.index + match[0].length,
+          language
+        },
+        tokenCount: this.estimateTokens(codeContent)
+      });
+      index++;
+      lastIndex = match.index + match[0].length;
+    }
+    if (lastIndex < content.length) {
+      const textContent = content.slice(lastIndex).trim();
+      if (textContent) {
+        chunks.push({
+          id: `${source}-${index}`,
+          content: textContent,
+          metadata: {
+            source,
+            index,
+            total: 0,
+            startPosition: lastIndex,
+            endPosition: content.length
+          },
+          tokenCount: this.estimateTokens(textContent)
+        });
+      }
+    }
+    chunks.forEach((chunk) => {
+      chunk.metadata.total = chunks.length;
+    });
+    return chunks;
+  }
+  /**
+   * Semantic chunking (placeholder for more advanced implementation)
+   */
+  chunkSemantic(content, source, opts) {
+    return this.chunkByParagraph(content, source, opts);
+  }
+  /**
+   * Aggregate small chunks into larger ones
+   */
+  aggregateChunks(parts, source, opts) {
+    const chunks = [];
+    const chunkSize = opts.chunkSize || 1e3;
+    const minSize = opts.minChunkSize || 100;
+    let currentContent = "";
+    let index = 0;
+    let startPos = 0;
+    for (const part of parts) {
+      const trimmedPart = part.trim();
+      if (!trimmedPart) continue;
+      if (currentContent.length + trimmedPart.length > chunkSize && currentContent.length >= minSize) {
+        chunks.push({
+          id: `${source}-${index}`,
+          content: currentContent.trim(),
+          metadata: {
+            source,
+            index,
+            total: 0,
+            startPosition: startPos,
+            endPosition: startPos + currentContent.length
+          },
+          tokenCount: this.estimateTokens(currentContent)
+        });
+        index++;
+        startPos += currentContent.length;
+        currentContent = trimmedPart;
+      } else {
+        currentContent += (currentContent ? "\n\n" : "") + trimmedPart;
+      }
+    }
+    if (currentContent.trim()) {
+      chunks.push({
+        id: `${source}-${index}`,
+        content: currentContent.trim(),
+        metadata: {
+          source,
+          index,
+          total: 0,
+          startPosition: startPos,
+          endPosition: startPos + currentContent.length
+        },
+        tokenCount: this.estimateTokens(currentContent)
+      });
+    }
+    chunks.forEach((chunk) => {
+      chunk.metadata.total = chunks.length;
+    });
+    return chunks;
+  }
+  /**
+   * Estimate token count
+   */
+  estimateTokens(content) {
+    return Math.ceil(content.length / 4);
+  }
+}
+function createChunker(options) {
+  return new Chunker(options);
+}
+function chunkDocument(content, source, options) {
+  const chunker = new Chunker(options);
+  return chunker.chunk(content, source, options);
+}
+export {
+  Chunker,
+  chunkDocument,
+  createChunker
+};
+//# sourceMappingURL=index.js.map

package/dist/chunking/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sources":["../../src/chunking/index.ts"],"sourcesContent":["/**\n * Document Chunking Module\n *\n * Provides intelligent document chunking for RAG and vector search.\n */\n\n// ============================================================================\n// Types\n// ============================================================================\n\n/**\n * Chunking strategy\n */\nexport type ChunkStrategy =\n  | 'fixed'      // Fixed size chunks\n  | 'sentence'   // Sentence-based chunks\n  | 'paragraph'  // Paragraph-based chunks\n  | 'semantic'   // Semantic-based chunks\n  | 'markdown'   // Markdown structure-aware chunks\n  | 'code';      // Code-aware chunks\n\n/**\n * Chunking options\n */\nexport interface ChunkOptions {\n  /** Chunking strategy */\n  strategy: ChunkStrategy;\n  /** Target chunk size in characters */\n  chunkSize?: number;\n  /** Overlap between chunks in characters */\n  overlap?: number;\n  /** Minimum chunk size */\n  minChunkSize?: number;\n  /** Maximum chunk size */\n  maxChunkSize?: number;\n  /** Include metadata in chunks */\n  includeMetadata?: boolean;\n  /** Preserve code blocks */\n  preserveCodeBlocks?: boolean;\n}\n\n/**\n * Chunk metadata\n */\nexport interface ChunkMetadata {\n  /** Source document path */\n  source: string;\n  /** Chunk index in document */\n  index: number;\n  /** Total chunks in document */\n  total: number;\n  /** Start position in original document */\n  startPosition: number;\n  /** End position in original document */\n  endPosition: number;\n  /** Heading hierarchy */\n  headings?: string[];\n  /** Code language (for code chunks) */\n  language?: string;\n}\n\n/**\n * A single chunk\n */\nexport interface Chunk {\n  /** Unique chunk ID */\n  id: string;\n  /** Chunk content */\n  content: string;\n  /** Chunk metadata */\n  metadata: ChunkMetadata;\n  /** Token count estimate */\n  tokenCount?: number;\n}\n\n/**\n * Chunking result\n */\nexport interface ChunkResult {\n  /** Generated chunks */\n  chunks: Chunk[];\n  /** Original document length */\n  originalLength: number;\n  /** Strategy used */\n  strategy: ChunkStrategy;\n  /** Processing time in milliseconds */\n  processingTime: number;\n}\n\n// ============================================================================\n// Chunker Implementation\n// ============================================================================\n\n/**\n * Document Chunker for splitting documents into chunks\n */\nexport class Chunker {\n  private defaultOptions: ChunkOptions = {\n    strategy: 'paragraph',\n    chunkSize: 1000,\n    overlap: 100,\n    minChunkSize: 100,\n    maxChunkSize: 2000,\n    includeMetadata: true,\n    preserveCodeBlocks: true,\n  };\n\n  constructor(options?: Partial<ChunkOptions>) {\n    if (options) {\n      this.defaultOptions = { ...this.defaultOptions, ...options };\n    }\n  }\n\n  /**\n   * Chunk a document\n   */\n  chunk(content: string, source: string, options?: Partial<ChunkOptions>): ChunkResult {\n    const startTime = Date.now();\n    const opts = { ...this.defaultOptions, ...options };\n\n    let chunks: Chunk[];\n    switch (opts.strategy) {\n      case 'fixed':\n        chunks = this.chunkFixed(content, source, opts);\n        break;\n      case 'sentence':\n        chunks = this.chunkBySentence(content, source, opts);\n        break;\n      case 'paragraph':\n        chunks = this.chunkByParagraph(content, source, opts);\n        break;\n      case 'markdown':\n        chunks = this.chunkByMarkdown(content, source, opts);\n        break;\n      case 'code':\n        chunks = this.chunkByCode(content, source, opts);\n        break;\n      case 'semantic':\n        chunks = this.chunkSemantic(content, source, opts);\n        break;\n      default:\n        chunks = this.chunkByParagraph(content, source, opts);\n    }\n\n    return {\n      chunks,\n      originalLength: content.length,\n      strategy: opts.strategy,\n      processingTime: Date.now() - startTime,\n    };\n  }\n\n  /**\n   * Fixed-size chunking\n   */\n  private chunkFixed(content: string, source: string, opts: ChunkOptions): Chunk[] {\n    const chunks: Chunk[] = [];\n    const chunkSize = opts.chunkSize || 1000;\n    const overlap = opts.overlap || 0;\n\n    let position = 0;\n    let index = 0;\n\n    while (position < content.length) {\n      const end = Math.min(position + chunkSize, content.length);\n      const chunkContent = content.slice(position, end);\n\n      chunks.push({\n        id: `${source}-${index}`,\n        content: chunkContent,\n        metadata: {\n          source,\n          index,\n          total: 0, // Will be updated after\n          startPosition: position,\n          endPosition: end,\n        },\n        tokenCount: this.estimateTokens(chunkContent),\n      });\n\n      position = end - overlap;\n      index++;\n    }\n\n    // Update total count\n    chunks.forEach((chunk) => {\n      chunk.metadata.total = chunks.length;\n    });\n\n    return chunks;\n  }\n\n  /**\n   * Sentence-based chunking\n   */\n  private chunkBySentence(content: string, source: string, opts: ChunkOptions): Chunk[] {\n    const sentences = content.split(/(?<=[.!?])\\s+/);\n    return this.aggregateChunks(sentences, source, opts);\n  }\n\n  /**\n   * Paragraph-based chunking\n   */\n  private chunkByParagraph(content: string, source: string, opts: ChunkOptions): Chunk[] {\n    const paragraphs = content.split(/\\n\\n+/);\n    return this.aggregateChunks(paragraphs, source, opts);\n  }\n\n  /**\n   * Markdown-aware chunking\n   */\n  private chunkByMarkdown(content: string, source: string, opts: ChunkOptions): Chunk[] {\n    const sections: string[] = [];\n    const headings: string[][] = [];\n\n    // Split by headers\n    const headerRegex = /^(#{1,6})\\s+(.+)$/gm;\n    let lastIndex = 0;\n    let currentHeadings: string[] = [];\n    let match;\n\n    while ((match = headerRegex.exec(content)) !== null) {\n      if (lastIndex < match.index) {\n        sections.push(content.slice(lastIndex, match.index).trim());\n        headings.push([...currentHeadings]);\n      }\n\n      const level = match[1].length;\n      const heading = match[2];\n\n      // Update heading hierarchy\n      currentHeadings = currentHeadings.slice(0, level - 1);\n      currentHeadings[level - 1] = heading;\n\n      lastIndex = match.index;\n    }\n\n    if (lastIndex < content.length) {\n      sections.push(content.slice(lastIndex).trim());\n      headings.push([...currentHeadings]);\n    }\n\n    return sections\n      .filter((s) => s.length > 0)\n      .map((section, index) => ({\n        id: `${source}-${index}`,\n        content: section,\n        metadata: {\n          source,\n          index,\n          total: sections.length,\n          startPosition: 0,\n          endPosition: section.length,\n          headings: headings[index],\n        },\n        tokenCount: this.estimateTokens(section),\n      }));\n  }\n\n  /**\n   * Code-aware chunking\n   */\n  private chunkByCode(content: string, source: string, opts: ChunkOptions): Chunk[] {\n    const chunks: Chunk[] = [];\n    const codeBlockRegex = /```(\\w+)?\\n([\\s\\S]*?)```/g;\n\n    let lastIndex = 0;\n    let index = 0;\n    let match;\n\n    while ((match = codeBlockRegex.exec(content)) !== null) {\n      // Add text before code block\n      if (lastIndex < match.index) {\n        const textContent = content.slice(lastIndex, match.index).trim();\n        if (textContent) {\n          chunks.push({\n            id: `${source}-${index}`,\n            content: textContent,\n            metadata: {\n              source,\n              index,\n              total: 0,\n              startPosition: lastIndex,\n              endPosition: match.index,\n            },\n            tokenCount: this.estimateTokens(textContent),\n          });\n          index++;\n        }\n      }\n\n      // Add code block\n      const language = match[1] || 'unknown';\n      const codeContent = match[2].trim();\n      chunks.push({\n        id: `${source}-${index}`,\n        content: codeContent,\n        metadata: {\n          source,\n          index,\n          total: 0,\n          startPosition: match.index,\n          endPosition: match.index + match[0].length,\n          language,\n        },\n        tokenCount: this.estimateTokens(codeContent),\n      });\n      index++;\n      lastIndex = match.index + match[0].length;\n    }\n\n    // Add remaining text\n    if (lastIndex < content.length) {\n      const textContent = content.slice(lastIndex).trim();\n      if (textContent) {\n        chunks.push({\n          id: `${source}-${index}`,\n          content: textContent,\n          metadata: {\n            source,\n            index,\n            total: 0,\n            startPosition: lastIndex,\n            endPosition: content.length,\n          },\n          tokenCount: this.estimateTokens(textContent),\n        });\n      }\n    }\n\n    // Update totals\n    chunks.forEach((chunk) => {\n      chunk.metadata.total = chunks.length;\n    });\n\n    return chunks;\n  }\n\n  /**\n   * Semantic chunking (placeholder for more advanced implementation)\n   */\n  private chunkSemantic(content: string, source: string, opts: ChunkOptions): Chunk[] {\n    // For now, fall back to paragraph chunking\n    // Future: Use embeddings to find semantic boundaries\n    return this.chunkByParagraph(content, source, opts);\n  }\n\n  /**\n   * Aggregate small chunks into larger ones\n   */\n  private aggregateChunks(parts: string[], source: string, opts: ChunkOptions): Chunk[] {\n    const chunks: Chunk[] = [];\n    const chunkSize = opts.chunkSize || 1000;\n    const minSize = opts.minChunkSize || 100;\n\n    let currentContent = '';\n    let index = 0;\n    let startPos = 0;\n\n    for (const part of parts) {\n      const trimmedPart = part.trim();\n      if (!trimmedPart) continue;\n\n      if (currentContent.length + trimmedPart.length > chunkSize && currentContent.length >= minSize) {\n        chunks.push({\n          id: `${source}-${index}`,\n          content: currentContent.trim(),\n          metadata: {\n            source,\n            index,\n            total: 0,\n            startPosition: startPos,\n            endPosition: startPos + currentContent.length,\n          },\n          tokenCount: this.estimateTokens(currentContent),\n        });\n        index++;\n        startPos += currentContent.length;\n        currentContent = trimmedPart;\n      } else {\n        currentContent += (currentContent ? '\\n\\n' : '') + trimmedPart;\n      }\n    }\n\n    if (currentContent.trim()) {\n      chunks.push({\n        id: `${source}-${index}`,\n        content: currentContent.trim(),\n        metadata: {\n          source,\n          index,\n          total: 0,\n          startPosition: startPos,\n          endPosition: startPos + currentContent.length,\n        },\n        tokenCount: this.estimateTokens(currentContent),\n      });\n    }\n\n    chunks.forEach((chunk) => {\n      chunk.metadata.total = chunks.length;\n    });\n\n    return chunks;\n  }\n\n  /**\n   * Estimate token count\n   */\n  private estimateTokens(content: string): number {\n    // Rough estimation: ~4 characters per token for English\n    return Math.ceil(content.length / 4);\n  }\n}\n\n/**\n * Create a new Chunker instance\n */\nexport function createChunker(options?: Partial<ChunkOptions>): Chunker {\n  return new Chunker(options);\n}\n\n/**\n * Convenience function to chunk a document\n */\nexport function chunkDocument(\n  content: string,\n  source: string,\n  options?: Partial<ChunkOptions>\n): ChunkResult {\n  const chunker = new Chunker(options);\n  return chunker.chunk(content, source, options);\n}\n"],"names":[],"mappings":"AAgGO,MAAM,QAAQ;AAAA,EACX,iBAA+B;AAAA,IACrC,UAAU;AAAA,IACV,WAAW;AAAA,IACX,SAAS;AAAA,IACT,cAAc;AAAA,IACd,cAAc;AAAA,IACd,iBAAiB;AAAA,IACjB,oBAAoB;AAAA,EAAA;AAAA,EAGtB,YAAY,SAAiC;AAC3C,QAAI,SAAS;AACX,WAAK,iBAAiB,EAAE,GAAG,KAAK,gBAAgB,GAAG,QAAA;AAAA,IACrD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,SAAiB,QAAgB,SAA8C;AACnF,UAAM,YAAY,KAAK,IAAA;AACvB,UAAM,OAAO,EAAE,GAAG,KAAK,gBAAgB,GAAG,QAAA;AAE1C,QAAI;AACJ,YAAQ,KAAK,UAAA;AAAA,MACX,KAAK;AACH,iBAAS,KAAK,WAAW,SAAS,QAAQ,IAAI;AAC9C;AAAA,MACF,KAAK;AACH,iBAAS,KAAK,gBAAgB,SAAS,QAAQ,IAAI;AACnD;AAAA,MACF,KAAK;AACH,iBAAS,KAAK,iBAAiB,SAAS,QAAQ,IAAI;AACpD;AAAA,MACF,KAAK;AACH,iBAAS,KAAK,gBAAgB,SAAS,QAAQ,IAAI;AACnD;AAAA,MACF,KAAK;AACH,iBAAS,KAAK,YAAY,SAAS,QAAQ,IAAI;AAC/C;AAAA,MACF,KAAK;AACH,iBAAS,KAAK,cAAc,SAAS,QAAQ,IAAI;AACjD;AAAA,MACF;AACE,iBAAS,KAAK,iBAAiB,SAAS,QAAQ,IAAI;AAAA,IAAA;AAGxD,WAAO;AAAA,MACL;AAAA,MACA,gBAAgB,QAAQ;AAAA,MACxB,UAAU,KAAK;AAAA,MACf,gBAAgB,KAAK,QAAQ;AAAA,IAAA;AAAA,EAEjC;AAAA;AAAA;AAAA;AAAA,EAKQ,WAAW,SAAiB,QAAgB,MAA6B;AAC/E,UAAM,SAAkB,CAAA;AACxB,UAAM,YAAY,KAAK,aAAa;AACpC,UAAM,UAAU,KAAK,WAAW;AAEhC,QAAI,WAAW;AACf,QAAI,QAAQ;AAEZ,WAAO,WAAW,QAAQ,QAAQ;AAChC,YAAM,MAAM,KAAK,IAAI,WAAW,WAAW,QAAQ,MAAM;AACzD,YAAM,eAAe,QAAQ,MAAM,UAAU,GAAG;AAEhD,aAAO,KAAK;AAAA,QACV,IAAI,GAAG,MAAM,IAAI,KAAK;AAAA,QACtB,SAAS;AAAA,QACT,UAAU;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO;AAAA;AAAA,UACP,eAAe;AAAA,UACf,aAAa;AAAA,QAAA;AAAA,QAEf,YAAY,KAAK,eAAe,YAAY;AAAA,MAAA,CAC7C;AAED,iBAAW,MAAM;AACjB;AAAA,IACF;AAGA,WAAO,QAAQ,CAAC,UAAU;AACxB,YAAM,SAAS,QAAQ,OAAO;AAAA,IAChC,CAAC;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,SAAiB,QAAgB,MAA6B;AACpF,UAAM,YAAY,QAAQ,MAAM,eAAe;AAC/C,WAAO,KAAK,gBAAgB,WAAW,QAAQ,IAAI;AAAA,EACrD;AAAA;AAAA;AAAA;AAAA,EAKQ,iBAAiB,SAAiB,QAAgB,MAA6B;AACrF,UAAM,aAAa,QAAQ,MAAM,OAAO;AACxC,WAAO,KAAK,gBAAgB,YAAY,QAAQ,IAAI;AAAA,EACtD;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,SAAiB,QAAgB,MAA6B;AACpF,UAAM,WAAqB,CAAA;AAC3B,UAAM,WAAuB,CAAA;AAG7B,UAAM,cAAc;AACpB,QAAI,YAAY;AAChB,QAAI,kBAA4B,CAAA;AAChC,QAAI;AAEJ,YAAQ,QAAQ,YAAY,KAAK,OAAO,OAAO,MAAM;AACnD,UAAI,YAAY,MAAM,OAAO;AAC3B,iBAAS,KAAK,QAAQ,MAAM,WAAW,MAAM,KAAK,EAAE,MAAM;AAC1D,iBAAS,KAAK,CAAC,GAAG,eAAe,CAAC;AAAA,MACpC;AAEA,YAAM,QAAQ,MAAM,CAAC,EAAE;AACvB,YAAM,UAAU,MAAM,CAAC;AAGvB,wBAAkB,gBAAgB,MAAM,GAAG,QAAQ,CAAC;AACpD,sBAAgB,QAAQ,CAAC,IAAI;AAE7B,kBAAY,MAAM;AAAA,IACpB;AAEA,QAAI,YAAY,QAAQ,QAAQ;AAC9B,eAAS,KAAK,QAAQ,MAAM,SAAS,EAAE,MAAM;AAC7C,eAAS,KAAK,CAAC,GAAG,eAAe,CAAC;AAAA,IACpC;AAEA,WAAO,SACJ,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAC1B,IAAI,CAAC,SAAS,WAAW;AAAA,MACxB,IAAI,GAAG,MAAM,IAAI,KAAK;AAAA,MACtB,SAAS;AAAA,MACT,UAAU;AAAA,QACR;AAAA,QACA;AAAA,QACA,OAAO,SAAS;AAAA,QAChB,eAAe;AAAA,QACf,aAAa,QAAQ;AAAA,QACrB,UAAU,SAAS,KAAK;AAAA,MAAA;AAAA,MAE1B,YAAY,KAAK,eAAe,OAAO;AAAA,IAAA,EACvC;AAAA,EACN;AAAA;AAAA;AAAA;AAAA,EAKQ,YAAY,SAAiB,QAAgB,MAA6B;AAChF,UAAM,SAAkB,CAAA;AACxB,UAAM,iBAAiB;AAEvB,QAAI,YAAY;AAChB,QAAI,QAAQ;AACZ,QAAI;AAEJ,YAAQ,QAAQ,eAAe,KAAK,OAAO,OAAO,MAAM;AAEtD,UAAI,YAAY,MAAM,OAAO;AAC3B,cAAM,cAAc,QAAQ,MAAM,WAAW,MAAM,KAAK,EAAE,KAAA;AAC1D,YAAI,aAAa;AACf,iBAAO,KAAK;AAAA,YACV,IAAI,GAAG,MAAM,IAAI,KAAK;AAAA,YACtB,SAAS;AAAA,YACT,UAAU;AAAA,cACR;AAAA,cACA;AAAA,cACA,OAAO;AAAA,cACP,eAAe;AAAA,cACf,aAAa,MAAM;AAAA,YAAA;AAAA,YAErB,YAAY,KAAK,eAAe,WAAW;AAAA,UAAA,CAC5C;AACD;AAAA,QACF;AAAA,MACF;AAGA,YAAM,WAAW,MAAM,CAAC,KAAK;AAC7B,YAAM,cAAc,MAAM,CAAC,EAAE,KAAA;AAC7B,aAAO,KAAK;AAAA,QACV,IAAI,GAAG,MAAM,IAAI,KAAK;AAAA,QACtB,SAAS;AAAA,QACT,UAAU;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP,eAAe,MAAM;AAAA,UACrB,aAAa,MAAM,QAAQ,MAAM,CAAC,EAAE;AAAA,UACpC;AAAA,QAAA;AAAA,QAEF,YAAY,KAAK,eAAe,WAAW;AAAA,MAAA,CAC5C;AACD;AACA,kBAAY,MAAM,QAAQ,MAAM,CAAC,EAAE;AAAA,IACrC;AAGA,QAAI,YAAY,QAAQ,QAAQ;AAC9B,YAAM,cAAc,QAAQ,MAAM,SAAS,EAAE,KAAA;AAC7C,UAAI,aAAa;AACf,eAAO,KAAK;AAAA,UACV,IAAI,GAAG,MAAM,IAAI,KAAK;AAAA,UACtB,SAAS;AAAA,UACT,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA,OAAO;AAAA,YACP,eAAe;AAAA,YACf,aAAa,QAAQ;AAAA,UAAA;AAAA,UAEvB,YAAY,KAAK,eAAe,WAAW;AAAA,QAAA,CAC5C;AAAA,MACH;AAAA,IACF;AAGA,WAAO,QAAQ,CAAC,UAAU;AACxB,YAAM,SAAS,QAAQ,OAAO;AAAA,IAChC,CAAC;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,cAAc,SAAiB,QAAgB,MAA6B;AAGlF,WAAO,KAAK,iBAAiB,SAAS,QAAQ,IAAI;AAAA,EACpD;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,OAAiB,QAAgB,MAA6B;AACpF,UAAM,SAAkB,CAAA;AACxB,UAAM,YAAY,KAAK,aAAa;AACpC,UAAM,UAAU,KAAK,gBAAgB;AAErC,QAAI,iBAAiB;AACrB,QAAI,QAAQ;AACZ,QAAI,WAAW;AAEf,eAAW,QAAQ,OAAO;AACxB,YAAM,cAAc,KAAK,KAAA;AACzB,UAAI,CAAC,YAAa;AAElB,UAAI,eAAe,SAAS,YAAY,SAAS,aAAa,eAAe,UAAU,SAAS;AAC9F,eAAO,KAAK;AAAA,UACV,IAAI,GAAG,MAAM,IAAI,KAAK;AAAA,UACtB,SAAS,eAAe,KAAA;AAAA,UACxB,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA,OAAO;AAAA,YACP,eAAe;AAAA,YACf,aAAa,WAAW,eAAe;AAAA,UAAA;AAAA,UAEzC,YAAY,KAAK,eAAe,cAAc;AAAA,QAAA,CAC/C;AACD;AACA,oBAAY,eAAe;AAC3B,yBAAiB;AAAA,MACnB,OAAO;AACL,2BAAmB,iBAAiB,SAAS,MAAM;AAAA,MACrD;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ;AACzB,aAAO,KAAK;AAAA,QACV,IAAI,GAAG,MAAM,IAAI,KAAK;AAAA,QACtB,SAAS,eAAe,KAAA;AAAA,QACxB,UAAU;AAAA,UACR;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP,eAAe;AAAA,UACf,aAAa,WAAW,eAAe;AAAA,QAAA;AAAA,QAEzC,YAAY,KAAK,eAAe,cAAc;AAAA,MAAA,CAC/C;AAAA,IACH;AAEA,WAAO,QAAQ,CAAC,UAAU;AACxB,YAAM,SAAS,QAAQ,OAAO;AAAA,IAChC,CAAC;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,eAAe,SAAyB;AAE9C,WAAO,KAAK,KAAK,QAAQ,SAAS,CAAC;AAAA,EACrC;AACF;AAKO,SAAS,cAAc,SAA0C;AACtE,SAAO,IAAI,QAAQ,OAAO;AAC5B;AAKO,SAAS,cACd,SACA,QACA,SACa;AACb,QAAM,UAAU,IAAI,QAAQ,OAAO;AACnC,SAAO,QAAQ,MAAM,SAAS,QAAQ,OAAO;AAC/C;"}

package/dist/chunking/types.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Chunking System Types
+ *
+ * Type definitions for the document chunking system.
+ * Supports multiple chunking strategies for different content types.
+ *
+ * @module chunking/types
+ */
+/**
+ * Available chunking strategies
+ *
+ * - fixed: Fixed-size chunks with optional overlap
+ * - semantic: Paragraph-based semantic boundaries
+ * - markdown: Respects markdown header structure
+ * - code: Preserves code blocks and structure
+ * - adaptive: Auto-selects best strategy based on content
+ */
+export type ChunkStrategy = 'fixed' | 'semantic' | 'markdown' | 'code' | 'adaptive';
+/**
+ * Configuration options for chunking
+ */
+export interface ChunkOptions {
+    /** Chunking strategy to use */
+    strategy: ChunkStrategy;
+    /** Maximum chunk size in characters */
+    maxSize: number;
+    /** Minimum chunk size in characters (chunks smaller than this may be merged) */
+    minSize?: number;
+    /** Number of characters to overlap between chunks */
+    overlap?: number;
+    /** Whether to preserve document structure (headers, code blocks, etc.) */
+    preserveStructure?: boolean;
+}
+/**
+ * A single chunk of content
+ */
+export interface Chunk {
+    /** Unique identifier for this chunk (hash-based) */
+    id: string;
+    /** The content of the chunk */
+    content: string;
+    /** Start offset in the original document */
+    startOffset: number;
+    /** End offset in the original document */
+    endOffset: number;
+    /** Metadata about the chunk */
+    metadata: ChunkMetadata;
+}
+/**
+ * Metadata about a chunk
+ */
+export interface ChunkMetadata {
+    /** Type of content in the chunk */
+    type: 'header' | 'paragraph' | 'code' | 'list' | 'mixed';
+    /** Header level (1-6) if type is 'header' */
+    level?: number;
+    /** Programming language if type is 'code' */
+    language?: string;
+    /** Word count in the chunk */
+    wordCount: number;
+    /** Line count in the chunk */
+    lineCount: number;
+    /** Whether the chunk contains links (wiki-style or markdown) */
+    hasLinks: boolean;
+    /** Parent chunk ID if this is a sub-chunk */
+    parentChunkId?: string;
+}
+/**
+ * Result of a chunking operation
+ */
+export interface ChunkResult {
+    /** Array of generated chunks */
+    chunks: Chunk[];
+    /** Total size of the original content */
+    totalSize: number;
+    /** Strategy that was used */
+    strategy: ChunkStrategy;
+    /** Time taken to process in milliseconds */
+    processingTime: number;
+}
+/**
+ * Chunker configuration
+ */
+export interface ChunkerConfig {
+    /** Default strategy to use when none specified */
+    defaultStrategy: ChunkStrategy;
+    /** Default maximum chunk size */
+    defaultMaxSize: number;
+    /** Default overlap between chunks */
+    defaultOverlap: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/chunking/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/chunking/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;;;;;;;GAQG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC;AAEpF;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,QAAQ,EAAE,aAAa,CAAC;IAExB,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAEhB,gFAAgF;IAChF,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,qDAAqD;IACrD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,oDAAoD;IACpD,EAAE,EAAE,MAAM,CAAC;IAEX,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAEhB,4CAA4C;IAC5C,WAAW,EAAE,MAAM,CAAC;IAEpB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAElB,+BAA+B;IAC/B,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAEzD,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAElB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAElB,gEAAgE;IAChE,QAAQ,EAAE,OAAO,CAAC;IAElB,6CAA6C;IAC7C,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,MAAM,EAAE,KAAK,EAAE,CAAC;IAEhB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAElB,6BAA6B;IAC7B,QAAQ,EAAE,aAAa,CAAC;IAExB,4CAA4C;IAC5C,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kDAAkD;IAClD,eAAe,EAAE,aAAa,CAAC;IAE/B,iCAAiC;IACjC,cAAc,EAAE,MAAM,CAAC;IAEvB,qCAAqC;IACrC,cAAc,EAAE,MAAM,CAAC;CACxB"}

package/dist/cli/commands/audit.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Audit Command
+ *
+ * CLI commands for querying and managing the exochain audit log.
+ * Provides access to:
+ * - Query audit events with filtering
+ * - Create checkpoints for chain finality
+ * - Verify audit chain integrity
+ * - Check and manage sync status with peers
+ * - Export audit log data
+ *
+ * @module cli/commands/audit
+ */
+import { Command } from 'commander';
+/**
+ * Create audit command with subcommands
+ *
+ * @returns Configured audit command
+ */
+export declare function createAuditCommand(): Command;
+//# sourceMappingURL=audit.d.ts.map

package/dist/cli/commands/audit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA2QpC;;;;GAIG;AACH,wBAAgB,kBAAkB,IAAI,OAAO,CAyqB5C"}