@fragments-sdk/context 0.1.3 → 0.2.0

package/dist/chunk-KQIRG24U.js

@@ -0,0 +1,260 @@
+import {
+  AST_SUPPORTED_LANGUAGES,
+  getTreeSitterGrammar
+} from "./chunk-JFV27WLV.js";
+
+// src/chunking/line-chunker.ts
+function chunkByLines(content, filePath, language, options) {
+  const chunkSize = options?.maxChunkLines ?? 50;
+  const overlap = options?.overlapLines ?? 10;
+  const lines = content.split("\n");
+  if (lines.length === 0 || lines.length === 1 && lines[0].trim() === "") {
+    return [];
+  }
+  const header = `// File: ${filePath}
+`;
+  const chunks = [];
+  if (lines.length <= chunkSize) {
+    chunks.push({
+      content: header + lines.join("\n"),
+      filePath,
+      startLine: 1,
+      endLine: lines.length,
+      language,
+      chunkType: "module"
+    });
+    return chunks;
+  }
+  let start = 0;
+  while (start < lines.length) {
+    const end = Math.min(start + chunkSize, lines.length);
+    chunks.push({
+      content: header + lines.slice(start, end).join("\n"),
+      filePath,
+      startLine: start + 1,
+      endLine: end,
+      language,
+      chunkType: "block"
+    });
+    if (end >= lines.length) break;
+    start += chunkSize - overlap;
+  }
+  return chunks;
+}
+
+// src/chunking/ast-chunker.ts
+var SYMBOL_NODE_TYPES = /* @__PURE__ */ new Set([
+  // TypeScript / JavaScript
+  "function_declaration",
+  "method_definition",
+  "arrow_function",
+  "class_declaration",
+  "interface_declaration",
+  "type_alias_declaration",
+  "enum_declaration",
+  "export_statement",
+  "lexical_declaration",
+  "variable_declaration",
+  // Python
+  "function_definition",
+  "class_definition",
+  "decorated_definition",
+  // Go
+  "function_declaration",
+  "method_declaration",
+  "type_declaration",
+  // Rust
+  "function_item",
+  "impl_item",
+  "struct_item",
+  "enum_item",
+  "trait_item",
+  // Java
+  "method_declaration",
+  "class_declaration",
+  "interface_declaration",
+  "enum_declaration",
+  "constructor_declaration"
+]);
+var NAME_FIELD_NODES = /* @__PURE__ */ new Set([
+  "function_declaration",
+  "method_definition",
+  "class_declaration",
+  "interface_declaration",
+  "type_alias_declaration",
+  "enum_declaration",
+  "function_definition",
+  "class_definition",
+  "function_item",
+  "impl_item",
+  "struct_item",
+  "enum_item",
+  "trait_item",
+  "method_declaration",
+  "constructor_declaration"
+]);
+var tsModule = null;
+var initialized = false;
+async function getTreeSitter() {
+  if (!tsModule) {
+    const mod = await import("web-tree-sitter");
+    tsModule = mod;
+  }
+  if (!initialized) {
+    await tsModule.Parser.init();
+    initialized = true;
+  }
+  return tsModule;
+}
+var GRAMMAR_PACKAGES = {
+  typescript: "tree-sitter-typescript/tree-sitter-typescript.wasm",
+  tsx: "tree-sitter-typescript/tree-sitter-tsx.wasm",
+  javascript: "tree-sitter-javascript/tree-sitter-javascript.wasm",
+  python: "tree-sitter-python/tree-sitter-python.wasm",
+  go: "tree-sitter-go/tree-sitter-go.wasm",
+  rust: "tree-sitter-rust/tree-sitter-rust.wasm",
+  java: "tree-sitter-java/tree-sitter-java.wasm"
+};
+var languageCache = /* @__PURE__ */ new Map();
+async function loadLanguage(grammarName, wasmBasePath) {
+  const cached = languageCache.get(grammarName);
+  if (cached) return cached;
+  const { Language } = await getTreeSitter();
+  const grammarFile = GRAMMAR_PACKAGES[grammarName];
+  if (!grammarFile) {
+    throw new Error(`No grammar available for: ${grammarName}`);
+  }
+  let wasmPath;
+  if (wasmBasePath) {
+    wasmPath = `${wasmBasePath}/${grammarFile}`;
+  } else {
+    const { createRequire } = await import("module");
+    const require2 = createRequire(import.meta.url);
+    wasmPath = require2.resolve(grammarFile);
+  }
+  const lang = await Language.load(wasmPath);
+  languageCache.set(grammarName, lang);
+  return lang;
+}
+function getNodeName(node) {
+  if (!NAME_FIELD_NODES.has(node.type)) return void 0;
+  const nameNode = node.childForFieldName("name");
+  return nameNode?.text;
+}
+function getChunkType(nodeType) {
+  if (nodeType.includes("function") || nodeType.includes("method") || nodeType === "arrow_function" || nodeType === "constructor_declaration") {
+    return "function";
+  }
+  if (nodeType.includes("class") || nodeType.includes("impl") || nodeType.includes("struct") || nodeType.includes("trait") || nodeType.includes("interface") || nodeType.includes("enum")) {
+    return "class";
+  }
+  return "block";
+}
+function extractChunks(node, parentScope, maxChars, minChars) {
+  const text = node.text;
+  const charCount = text.length;
+  if (charCount <= maxChars && SYMBOL_NODE_TYPES.has(node.type)) {
+    const name2 = getNodeName(node);
+    const scopeChain = parentScope ? name2 ? `${parentScope}.${name2}` : parentScope : name2;
+    return [{
+      text,
+      startLine: node.startPosition.row + 1,
+      endLine: node.endPosition.row + 1,
+      symbolName: name2,
+      scopeChain,
+      chunkType: getChunkType(node.type)
+    }];
+  }
+  if (charCount <= maxChars && node.childCount === 0) {
+    return [{
+      text,
+      startLine: node.startPosition.row + 1,
+      endLine: node.endPosition.row + 1,
+      chunkType: "block"
+    }];
+  }
+  const name = getNodeName(node);
+  const currentScope = parentScope ? name ? `${parentScope}.${name}` : parentScope : name ?? "";
+  const childChunks = [];
+  for (const child of node.namedChildren) {
+    const chunks = extractChunks(child, currentScope, maxChars, minChars);
+    childChunks.push(...chunks);
+  }
+  return mergeSmallChunks(childChunks, maxChars, minChars);
+}
+function mergeSmallChunks(chunks, maxChars, minChars) {
+  if (chunks.length === 0) return chunks;
+  const merged = [];
+  let current = chunks[0];
+  for (let i = 1; i < chunks.length; i++) {
+    const next = chunks[i];
+    const combinedLength = current.text.length + next.text.length + 1;
+    if (combinedLength <= maxChars && current.text.length < minChars) {
+      current = {
+        text: current.text + "\n" + next.text,
+        startLine: current.startLine,
+        endLine: next.endLine,
+        symbolName: current.symbolName ?? next.symbolName,
+        scopeChain: current.scopeChain ?? next.scopeChain,
+        chunkType: current.symbolName ? current.chunkType : next.symbolName ? next.chunkType : "mixed"
+      };
+    } else {
+      merged.push(current);
+      current = next;
+    }
+  }
+  merged.push(current);
+  return merged;
+}
+async function chunkByAST(content, filePath, language, grammarName, options) {
+  const maxChars = options?.maxChunkChars ?? 1500;
+  const minChars = options?.minChunkChars ?? 100;
+  const lang = await loadLanguage(grammarName, options?.wasmBasePath);
+  const { Parser } = await getTreeSitter();
+  const parser = new Parser();
+  parser.setLanguage(lang);
+  const tree = parser.parse(content);
+  const rawChunks = extractChunks(tree.rootNode, "", maxChars, minChars);
+  if (rawChunks.length === 0) {
+    return [{
+      content: `// File: ${filePath}
+${content}`,
+      filePath,
+      startLine: 1,
+      endLine: content.split("\n").length,
+      language,
+      chunkType: "module"
+    }];
+  }
+  return rawChunks.map((chunk) => ({
+    content: `// File: ${filePath}
+${chunk.scopeChain ? `// Scope: ${chunk.scopeChain}
+` : ""}${chunk.text}`,
+    filePath,
+    startLine: chunk.startLine,
+    endLine: chunk.endLine,
+    language,
+    symbolName: chunk.symbolName,
+    scopeChain: chunk.scopeChain,
+    chunkType: chunk.chunkType
+  }));
+}
+
+// src/chunking/index.ts
+async function chunkFile(content, filePath, language, options) {
+  const grammarName = getTreeSitterGrammar(filePath);
+  if (grammarName && AST_SUPPORTED_LANGUAGES.has(grammarName)) {
+    try {
+      return await chunkByAST(content, filePath, language, grammarName, options);
+    } catch (err) {
+      console.warn(`AST chunking failed for ${filePath}, falling back to line-based:`, err);
+    }
+  }
+  return chunkByLines(content, filePath, language, options);
+}
+
+export {
+  chunkByLines,
+  chunkByAST,
+  chunkFile
+};

package/dist/chunk-ZMBYQK43.js

@@ -0,0 +1,91 @@
+// src/search/fusion.ts
+function reciprocalRankFusion(resultSets, k = 60) {
+  const scoreMap = /* @__PURE__ */ new Map();
+  for (const { label, results } of resultSets) {
+    for (let rank = 0; rank < results.length; rank++) {
+      const result = results[rank];
+      const rrfScore = 1 / (k + rank + 1);
+      const existing = scoreMap.get(result.id);
+      if (existing) {
+        existing.score += rrfScore;
+        existing.sources.push(label);
+      } else {
+        scoreMap.set(result.id, {
+          score: rrfScore,
+          sources: [label]
+        });
+      }
+    }
+  }
+  const fused = [];
+  for (const [id, { score, sources }] of scoreMap) {
+    fused.push({ id, fusedScore: score, sources });
+  }
+  fused.sort((a, b) => b.fusedScore - a.fusedScore);
+  return fused;
+}
+
+// src/search/deduplicator.ts
+function deduplicateChunks(chunks) {
+  if (chunks.length <= 1) return chunks;
+  const byFile = /* @__PURE__ */ new Map();
+  for (const chunk of chunks) {
+    const group = byFile.get(chunk.filePath);
+    if (group) {
+      group.push(chunk);
+    } else {
+      byFile.set(chunk.filePath, [chunk]);
+    }
+  }
+  const result = [];
+  for (const [, fileChunks] of byFile) {
+    if (fileChunks.length === 1) {
+      result.push(fileChunks[0]);
+      continue;
+    }
+    fileChunks.sort((a, b) => a.startLine - b.startLine);
+    const merged = mergeOverlapping(fileChunks);
+    result.push(...merged);
+  }
+  result.sort((a, b) => b.score - a.score);
+  return result;
+}
+function mergeOverlapping(chunks) {
+  const merged = [];
+  let current = { ...chunks[0] };
+  for (let i = 1; i < chunks.length; i++) {
+    const next = chunks[i];
+    if (next.startLine <= current.endLine + 1) {
+      const currentLines = current.content.split("\n");
+      const nextLines = next.content.split("\n");
+      const overlapStart = next.startLine - current.startLine;
+      const newLines = nextLines.slice(current.endLine - next.startLine + 1);
+      if (newLines.length > 0) {
+        current = {
+          ...current,
+          content: currentLines.join("\n") + "\n" + newLines.join("\n"),
+          endLine: Math.max(current.endLine, next.endLine),
+          score: Math.max(current.score, next.score),
+          // Keep symbolName from higher-scored chunk
+          symbolName: current.score >= next.score ? current.symbolName : next.symbolName
+        };
+      } else {
+        current = {
+          ...current,
+          endLine: Math.max(current.endLine, next.endLine),
+          score: Math.max(current.score, next.score)
+        };
+      }
+    } else {
+      merged.push(current);
+      current = { ...next };
+    }
+  }
+  merged.push(current);
+  return merged;
+}
+
+export {
+  reciprocalRankFusion,
+  deduplicateChunks
+};

package/dist/chunking/index.d.ts

@@ -0,0 +1,53 @@
+interface CodeChunk {
+    content: string;
+    filePath: string;
+    startLine: number;
+    endLine: number;
+    language: string;
+    /** Name of the top-level symbol (function, class, etc.) this chunk represents. */
+    symbolName?: string;
+    /** Dot-separated scope chain, e.g. "UserService.login". */
+    scopeChain?: string;
+    /** What kind of AST node this chunk represents. */
+    chunkType?: "function" | "class" | "module" | "block" | "mixed";
+}
+interface ChunkOptions {
+    /** Max characters per chunk (default: 1500). */
+    maxChunkChars?: number;
+    /** Min characters to avoid tiny chunks (default: 100). */
+    minChunkChars?: number;
+    /** Number of overlap lines for line-based fallback (default: 10). */
+    overlapLines?: number;
+    /** Max lines per chunk for line-based fallback (default: 50). */
+    maxChunkLines?: number;
+}
+
+interface ASTChunkerOptions extends ChunkOptions {
+    /** Base path for WASM grammar files. If not set, resolves from node_modules. */
+    wasmBasePath?: string;
+}
+/**
+ * AST-aware chunker using tree-sitter.
+ *
+ * Algorithm: parse AST → if node fits budget, keep as one chunk →
+ * if too large, recurse into children → greedily merge adjacent small siblings.
+ *
+ * Each chunk is enriched with file path, scope chain, and symbol name.
+ */
+declare function chunkByAST(content: string, filePath: string, language: string, grammarName: string, options?: ASTChunkerOptions): Promise<CodeChunk[]>;
+
+/**
+ * Line-based chunker — splits file content into fixed-size chunks with overlap.
+ * Used as fallback when AST parsing is unavailable for a language.
+ */
+declare function chunkByLines(content: string, filePath: string, language: string, options?: ChunkOptions): CodeChunk[];
+
+/**
+ * Main entry point for chunking a file.
+ *
+ * Tries AST-aware chunking if tree-sitter supports the language,
+ * falls back to line-based chunking otherwise.
+ */
+declare function chunkFile(content: string, filePath: string, language: string, options?: ASTChunkerOptions): Promise<CodeChunk[]>;
+
+export { type ASTChunkerOptions, type ChunkOptions, type CodeChunk, chunkByAST, chunkByLines, chunkFile };

package/dist/chunking/index.js

@@ -0,0 +1,11 @@
+import {
+  chunkByAST,
+  chunkByLines,
+  chunkFile
+} from "../chunk-KQIRG24U.js";
+import "../chunk-JFV27WLV.js";
+export {
+  chunkByAST,
+  chunkByLines,
+  chunkFile
+};

package/dist/citations/index.d.ts

@@ -0,0 +1,94 @@
+import { a as ScoredChunk } from '../types-B7duBj6U.js';
+
+interface CitationDocumentBlock {
+    type: "document";
+    source: {
+        type: "content";
+        content: Array<{
+            type: "text";
+            text: string;
+        }>;
+    } | {
+        type: "text";
+        media_type: "text/plain";
+        data: string;
+    };
+    title?: string | null;
+    context?: string | null;
+    citations: {
+        enabled: true;
+    };
+}
+interface ChunkMapping {
+    /** Position in the content array of the document block. */
+    blockIndex: number;
+    startLine: number;
+    endLine: number;
+    language: string;
+    symbolName?: string;
+    score: number;
+}
+interface DocumentMapping {
+    /** Index of this document in the documents array sent to Anthropic. */
+    documentIndex: number;
+    filePath: string;
+    sourceType: "code" | "plaintext";
+    chunks: ChunkMapping[];
+}
+interface ResolvedCitation {
+    /** 1-indexed display number. */
+    index: number;
+    citedText: string;
+    documentTitle: string | null;
+    filePath: string;
+    sourceType: "code" | "plaintext";
+    startLine?: number;
+    endLine?: number;
+    language?: string;
+    symbolName?: string;
+    /** Character position in the response text where this citation appears. */
+    textOffset: number;
+}
+interface CitationDocumentOptions {
+    /** Extra plain-text documents to include (e.g. spec content, commit history). */
+    additionalDocuments?: Array<{
+        title: string;
+        content: string;
+    }>;
+}
+interface RawCitation {
+    type: "content_block_location" | "char_location";
+    cited_text: string;
+    document_index: number;
+    document_title: string | null;
+    start_block_index?: number;
+    end_block_index?: number;
+    start_char_index?: number;
+    end_char_index?: number;
+}
+
+interface BuildResult {
+    documents: CitationDocumentBlock[];
+    documentMap: DocumentMapping[];
+}
+/**
+ * Converts deduplicated ScoredChunk[] into Anthropic-compatible citation
+ * document blocks with a mapping table for resolving raw citations back
+ * to source file/line metadata.
+ *
+ * Pipeline position: after deduplicateChunks() → before Anthropic API call.
+ */
+declare function buildCitationDocuments(chunks: ScoredChunk[], options?: CitationDocumentOptions): BuildResult;
+
+/**
+ * Resolves a single raw Anthropic citation against the document map,
+ * producing a fully enriched ResolvedCitation with file/line metadata.
+ */
+declare function resolveCitation(raw: RawCitation, documentMap: DocumentMapping[], citationIndex: number, textOffset: number): ResolvedCitation;
+/**
+ * Batch-resolves an array of raw citations. Auto-increments citation index
+ * starting from 1.
+ */
+declare function resolveCitations(raws: RawCitation[], documentMap: DocumentMapping[], textOffsets: number[]): ResolvedCitation[];
+
+export { type ChunkMapping, type CitationDocumentBlock, type CitationDocumentOptions, type DocumentMapping, type RawCitation, type ResolvedCitation, buildCitationDocuments, resolveCitation, resolveCitations };

package/dist/citations/index.js

@@ -0,0 +1,10 @@
+import {
+  buildCitationDocuments,
+  resolveCitation,
+  resolveCitations
+} from "../chunk-3FEHRHFQ.js";
+export {
+  buildCitationDocuments,
+  resolveCitation,
+  resolveCitations
+};

package/dist/embeddings/voyage.d.ts

@@ -0,0 +1,44 @@
+interface EmbeddingOptions {
+    /** API key for the embedding provider. */
+    apiKey: string;
+    /** Whether the input is a document (for indexing) or a query (for search). */
+    inputType: "document" | "query";
+    /** Number of dimensions for the embedding vector (default: 1024). */
+    dimensions?: number;
+    /** Max characters per text before truncation (default: 32000). */
+    maxChars?: number;
+    /** Number of texts to embed per API call (default: 128). */
+    batchSize?: number;
+}
+interface RerankOptions {
+    /** API key for the reranking provider. */
+    apiKey: string;
+    /** Number of top results to return (default: 15). */
+    topK?: number;
+    /** Model to use for reranking (default: "rerank-2.5"). */
+    model?: string;
+}
+interface EmbeddingResult {
+    embedding: number[];
+    index: number;
+}
+interface RerankResult {
+    index: number;
+    relevanceScore: number;
+}
+
+/**
+ * Generate embeddings using Voyage AI's code-optimized model.
+ *
+ * Supports asymmetric embedding: use inputType "document" for indexing
+ * and "query" for search queries.
+ */
+declare function generateEmbeddings(texts: string[], options: EmbeddingOptions): Promise<number[][]>;
+/**
+ * Rerank documents using Voyage AI's cross-encoder model.
+ *
+ * Takes a query and candidate documents, returns them reranked by relevance.
+ */
+declare function rerankResults(query: string, documents: string[], options: RerankOptions): Promise<RerankResult[]>;
+
+export { type EmbeddingOptions, type EmbeddingResult, type RerankOptions, type RerankResult, generateEmbeddings, rerankResults };

package/dist/embeddings/voyage.js

@@ -0,0 +1,8 @@
+import {
+  generateEmbeddings,
+  rerankResults
+} from "../chunk-3VPR67FN.js";
+export {
+  generateEmbeddings,
+  rerankResults
+};

package/dist/generate/index.d.ts

@@ -0,0 +1,43 @@
+import { CompiledSegment, CompiledBlock } from '../types/index.js';
+
+/**
+ * Context generation for AI agents.
+ *
+ * Generates AI-ready context documents from compiled fragment data.
+ */
+
+/**
+ * Placeholder patterns to filter out from usage text.
+ */
+declare const PLACEHOLDER_PATTERNS: RegExp[];
+/**
+ * Filter out placeholder text from usage arrays
+ */
+declare function filterPlaceholders(items: string[] | undefined): string[];
+/**
+ * Options for context generation
+ */
+interface ContextOptions {
+    format?: "markdown" | "json";
+    include?: {
+        props?: boolean;
+        variants?: boolean;
+        usage?: boolean;
+        relations?: boolean;
+        code?: boolean;
+    };
+    compact?: boolean;
+}
+/**
+ * Result of context generation
+ */
+interface ContextResult {
+    content: string;
+    tokenEstimate: number;
+}
+/**
+ * Generate AI-ready context from compiled segments and optional blocks
+ */
+declare function generateContext(segments: CompiledSegment[], options?: ContextOptions, blocks?: CompiledBlock[]): ContextResult;
+
+export { type ContextOptions, type ContextResult, PLACEHOLDER_PATTERNS, filterPlaceholders, generateContext };

package/dist/generate/index.js

@@ -0,0 +1,10 @@
+import {
+  PLACEHOLDER_PATTERNS,
+  filterPlaceholders,
+  generateContext
+} from "../chunk-KKABP4K4.js";
+export {
+  PLACEHOLDER_PATTERNS,
+  filterPlaceholders,
+  generateContext
+};

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export { AIMetadata, CompiledBlock, CompiledSegment, CompiledSegmentsFile, CompiledTokenData, CompiledTokenEntry, ComponentRelation, PropDefinition, SegmentContract, SegmentGenerated, SegmentMeta, SegmentUsage, Theme, VerifyResult } from './types/index.js';
+export { ContextOptions, ContextResult, PLACEHOLDER_PATTERNS, filterPlaceholders, generateContext } from './generate/index.js';
+export { ASTChunkerOptions, ChunkOptions, CodeChunk, chunkByAST, chunkByLines, chunkFile } from './chunking/index.js';
+export { EmbeddingOptions, EmbeddingResult, RerankOptions, RerankResult, generateEmbeddings, rerankResults } from './embeddings/voyage.js';
+export { deduplicateChunks, reciprocalRankFusion } from './search/index.js';
+export { F as FusedResult, R as RankedResult, a as ScoredChunk, S as SearchResult } from './types-B7duBj6U.js';
+export { AST_SUPPORTED_LANGUAGES, ChangedFiles, FileEntry, GrammarMapping, INDEXABLE_EXTENSIONS, detectLanguage, getTreeSitterGrammar, hashContent, resolveChanges, shouldIndexFile } from './indexing/index.js';
+export { ChunkMapping, CitationDocumentBlock, CitationDocumentOptions, DocumentMapping, RawCitation, ResolvedCitation, buildCitationDocuments, resolveCitation, resolveCitations } from './citations/index.js';

package/dist/index.js

@@ -0,0 +1,56 @@
+import {
+  chunkByAST,
+  chunkByLines,
+  chunkFile
+} from "./chunk-KQIRG24U.js";
+import {
+  generateEmbeddings,
+  rerankResults
+} from "./chunk-3VPR67FN.js";
+import {
+  deduplicateChunks,
+  reciprocalRankFusion
+} from "./chunk-ZMBYQK43.js";
+import {
+  hashContent,
+  resolveChanges
+} from "./chunk-HINI3FCI.js";
+import {
+  AST_SUPPORTED_LANGUAGES,
+  INDEXABLE_EXTENSIONS,
+  detectLanguage,
+  getTreeSitterGrammar,
+  shouldIndexFile
+} from "./chunk-JFV27WLV.js";
+import {
+  PLACEHOLDER_PATTERNS,
+  filterPlaceholders,
+  generateContext
+} from "./chunk-KKABP4K4.js";
+import {
+  buildCitationDocuments,
+  resolveCitation,
+  resolveCitations
+} from "./chunk-3FEHRHFQ.js";
+export {
+  AST_SUPPORTED_LANGUAGES,
+  INDEXABLE_EXTENSIONS,
+  PLACEHOLDER_PATTERNS,
+  buildCitationDocuments,
+  chunkByAST,
+  chunkByLines,
+  chunkFile,
+  deduplicateChunks,
+  detectLanguage,
+  filterPlaceholders,
+  generateContext,
+  generateEmbeddings,
+  getTreeSitterGrammar,
+  hashContent,
+  reciprocalRankFusion,
+  rerankResults,
+  resolveChanges,
+  resolveCitation,
+  resolveCitations,
+  shouldIndexFile
+};