gsd-pi 2.38.0-dev.96dc7fb → 2.38.0-dev.98b44dc

package/src/resources/extensions/gsd/semantic-chunker.ts

@@ -1,336 +0,0 @@
-// GSD Extension — Semantic Chunker with TF-IDF Relevance Scoring
-// Splits code/text into semantic chunks and selects the most relevant ones for a given task.
-// Pure TypeScript — no external dependencies.
-
-// ─── Types ──────────────────────────────────────────────────────────────────
-
-export interface Chunk {
-	content: string;
-	startLine: number;
-	endLine: number;
-	score: number;
-}
-
-export interface ChunkResult {
-	chunks: Chunk[];
-	totalChunks: number;
-	omittedChunks: number;
-	savingsPercent: number;
-}
-
-interface ChunkOptions {
-	minLines?: number;
-	maxLines?: number;
-}
-
-interface RelevanceOptions {
-	maxChunks?: number;
-	minChunkLines?: number;
-	maxChunkLines?: number;
-	minScore?: number;
-}
-
-// ─── Constants ──────────────────────────────────────────────────────────────
-
-const CODE_BOUNDARY_RE = /^(export\s+)?(async\s+)?(function|class|interface|type|const|enum)\s/;
-
-const MARKDOWN_HEADING_RE = /^#{1,6}\s/;
-
-const STOP_WORDS = new Set([
-	"the", "a", "an", "is", "are", "was", "were", "be", "to", "of", "in",
-	"for", "on", "with", "at", "by", "from", "this", "that", "it", "as",
-	"or", "and", "not", "but", "if", "do", "no", "so", "up", "its", "has",
-	"had", "get", "set", "can", "may", "all", "use", "new", "one", "two",
-	"also", "each", "than", "been", "into", "most", "only", "over", "such",
-	"how", "some", "any", "our", "his", "her", "out", "did", "let", "say", "she",
-]);
-
-const DEFAULT_MIN_LINES = 3;
-const DEFAULT_MAX_LINES = 80;
-const DEFAULT_MAX_CHUNKS = 5;
-const DEFAULT_MIN_SCORE = 0.1;
-
-// ─── Content Type Detection ─────────────────────────────────────────────────
-
-type ContentType = "code" | "markdown" | "text";
-
-function detectContentType(lines: string[]): ContentType {
-	let codeSignals = 0;
-	let mdSignals = 0;
-	const sampleSize = Math.min(lines.length, 50);
-
-	for (let i = 0; i < sampleSize; i++) {
-		const line = lines[i];
-		if (CODE_BOUNDARY_RE.test(line) || /^\s*import\s/.test(line)) {
-			codeSignals++;
-		}
-		if (MARKDOWN_HEADING_RE.test(line)) {
-			mdSignals++;
-		}
-	}
-
-	if (mdSignals >= 2 && mdSignals > codeSignals) return "markdown";
-	if (codeSignals >= 2) return "code";
-	return "text";
-}
-
-// ─── Tokenizer ──────────────────────────────────────────────────────────────
-
-function tokenize(text: string): string[] {
-	return text
-		.toLowerCase()
-		.split(/[\s\W]+/)
-		.filter((w) => w.length >= 2 && !STOP_WORDS.has(w));
-}
-
-// ─── splitIntoChunks ────────────────────────────────────────────────────────
-
-export function splitIntoChunks(
-	content: string,
-	options?: ChunkOptions,
-): Chunk[] {
-	if (!content || content.trim().length === 0) return [];
-
-	const minLines = options?.minLines ?? DEFAULT_MIN_LINES;
-	const maxLines = options?.maxLines ?? DEFAULT_MAX_LINES;
-	const lines = content.split("\n");
-
-	if (lines.length === 0) return [];
-
-	const contentType = detectContentType(lines);
-	let boundaries: number[];
-
-	switch (contentType) {
-		case "code":
-			boundaries = findCodeBoundaries(lines);
-			break;
-		case "markdown":
-			boundaries = findMarkdownBoundaries(lines);
-			break;
-		default:
-			boundaries = findTextBoundaries(lines);
-			break;
-	}
-
-	// Always include 0 as first boundary
-	if (boundaries.length === 0 || boundaries[0] !== 0) {
-		boundaries.unshift(0);
-	}
-
-	// Build raw chunks from boundaries
-	const rawChunks: Chunk[] = [];
-	for (let i = 0; i < boundaries.length; i++) {
-		const start = boundaries[i];
-		const end = i + 1 < boundaries.length ? boundaries[i + 1] - 1 : lines.length - 1;
-		const chunkLines = lines.slice(start, end + 1);
-		rawChunks.push({
-			content: chunkLines.join("\n"),
-			startLine: start + 1,  // 1-based
-			endLine: end + 1,      // 1-based
-			score: 0,
-		});
-	}
-
-	// Split oversized chunks at maxLines
-	const splitChunks: Chunk[] = [];
-	for (const chunk of rawChunks) {
-		const chunkLineCount = chunk.endLine - chunk.startLine + 1;
-		if (chunkLineCount <= maxLines) {
-			splitChunks.push(chunk);
-		} else {
-			const chunkLines = chunk.content.split("\n");
-			for (let offset = 0; offset < chunkLines.length; offset += maxLines) {
-				const slice = chunkLines.slice(offset, offset + maxLines);
-				splitChunks.push({
-					content: slice.join("\n"),
-					startLine: chunk.startLine + offset,
-					endLine: chunk.startLine + offset + slice.length - 1,
-					score: 0,
-				});
-			}
-		}
-	}
-
-	// Merge tiny chunks into predecessor
-	const merged: Chunk[] = [];
-	for (const chunk of splitChunks) {
-		const chunkLineCount = chunk.endLine - chunk.startLine + 1;
-		if (chunkLineCount < minLines && merged.length > 0) {
-			const prev = merged[merged.length - 1];
-			prev.content += "\n" + chunk.content;
-			prev.endLine = chunk.endLine;
-		} else {
-			merged.push({ ...chunk });
-		}
-	}
-
-	return merged;
-}
-
-function findCodeBoundaries(lines: string[]): number[] {
-	const boundaries: number[] = [];
-	for (let i = 0; i < lines.length; i++) {
-		if (CODE_BOUNDARY_RE.test(lines[i])) {
-			// Also consider a blank line before a boundary marker
-			if (i > 0 && lines[i - 1].trim() === "" && !boundaries.includes(i)) {
-				boundaries.push(i);
-			} else if (!boundaries.includes(i)) {
-				boundaries.push(i);
-			}
-		}
-	}
-	return boundaries;
-}
-
-function findMarkdownBoundaries(lines: string[]): number[] {
-	const boundaries: number[] = [];
-	for (let i = 0; i < lines.length; i++) {
-		if (MARKDOWN_HEADING_RE.test(lines[i])) {
-			boundaries.push(i);
-		}
-	}
-	return boundaries;
-}
-
-function findTextBoundaries(lines: string[]): number[] {
-	const boundaries: number[] = [0];
-	for (let i = 1; i < lines.length; i++) {
-		if (lines[i - 1].trim() === "" && lines[i].trim() !== "") {
-			boundaries.push(i);
-		}
-	}
-	return boundaries;
-}
-
-// ─── scoreChunks ────────────────────────────────────────────────────────────
-
-export function scoreChunks(chunks: Chunk[], query: string): Chunk[] {
-	if (chunks.length === 0) return [];
-
-	const queryTerms = tokenize(query);
-	if (queryTerms.length === 0) {
-		return chunks.map((c) => ({ ...c, score: 0 }));
-	}
-
-	const totalChunks = chunks.length;
-
-	// Pre-compute IDF for each query term
-	const termChunkCounts = new Map<string, number>();
-	const chunkTokenSets: Set<string>[] = [];
-
-	for (const chunk of chunks) {
-		const tokens = new Set(tokenize(chunk.content));
-		chunkTokenSets.push(tokens);
-		for (const term of queryTerms) {
-			if (tokens.has(term)) {
-				termChunkCounts.set(term, (termChunkCounts.get(term) ?? 0) + 1);
-			}
-		}
-	}
-
-	const idf = new Map<string, number>();
-	for (const term of queryTerms) {
-		const df = termChunkCounts.get(term) ?? 0;
-		idf.set(term, Math.log(1 + totalChunks / (1 + df)));
-	}
-
-	// Score each chunk
-	const scored = chunks.map((chunk, idx) => {
-		const chunkTokens = tokenize(chunk.content);
-		const totalTerms = chunkTokens.length;
-		if (totalTerms === 0) return { ...chunk, score: 0 };
-
-		// Count term frequencies
-		const termFreq = new Map<string, number>();
-		for (const token of chunkTokens) {
-			termFreq.set(token, (termFreq.get(token) ?? 0) + 1);
-		}
-
-		let score = 0;
-		for (const term of queryTerms) {
-			const tf = (termFreq.get(term) ?? 0) / totalTerms;
-			const termIdf = idf.get(term) ?? 0;
-			score += tf * termIdf;
-		}
-
-		return { ...chunk, score };
-	});
-
-	// Normalize to 0-1
-	const maxScore = Math.max(...scored.map((c) => c.score));
-	if (maxScore > 0) {
-		for (const chunk of scored) {
-			chunk.score = chunk.score / maxScore;
-		}
-	}
-
-	return scored;
-}
-
-// ─── chunkByRelevance ───────────────────────────────────────────────────────
-
-export function chunkByRelevance(
-	content: string,
-	query: string,
-	options?: RelevanceOptions,
-): ChunkResult {
-	const maxChunks = options?.maxChunks ?? DEFAULT_MAX_CHUNKS;
-	const minScore = options?.minScore ?? DEFAULT_MIN_SCORE;
-	const minLines = options?.minChunkLines ?? DEFAULT_MIN_LINES;
-	const maxLines = options?.maxChunkLines ?? DEFAULT_MAX_LINES;
-
-	const rawChunks = splitIntoChunks(content, { minLines, maxLines });
-	if (rawChunks.length === 0) {
-		return { chunks: [], totalChunks: 0, omittedChunks: 0, savingsPercent: 0 };
-	}
-
-	const scored = scoreChunks(rawChunks, query);
-
-	// Filter by minScore and take top maxChunks by score
-	const qualifying = scored
-		.filter((c) => c.score >= minScore)
-		.sort((a, b) => b.score - a.score)
-		.slice(0, maxChunks);
-
-	// Return in original document order (by startLine)
-	const selected = qualifying.sort((a, b) => a.startLine - b.startLine);
-
-	const totalChars = content.length;
-	const selectedChars = selected.reduce((sum, c) => sum + c.content.length, 0);
-	const savingsPercent = totalChars > 0
-		? Math.round(((totalChars - selectedChars) / totalChars) * 100)
-		: 0;
-
-	return {
-		chunks: selected,
-		totalChunks: rawChunks.length,
-		omittedChunks: rawChunks.length - selected.length,
-		savingsPercent: Math.max(0, savingsPercent),
-	};
-}
-
-// ─── formatChunks ───────────────────────────────────────────────────────────
-
-export function formatChunks(result: ChunkResult, filePath: string): string {
-	if (result.chunks.length === 0) {
-		return `[${filePath}: empty or no relevant chunks]`;
-	}
-
-	const parts: string[] = [];
-	let lastEndLine = 0;
-
-	for (const chunk of result.chunks) {
-		// Show omission gap
-		if (lastEndLine > 0 && chunk.startLine > lastEndLine + 1) {
-			const gapLines = chunk.startLine - lastEndLine - 1;
-			parts.push(`[...${gapLines} lines omitted...]`);
-		}
-
-		parts.push(`[Lines ${chunk.startLine}-${chunk.endLine}]`);
-		parts.push(chunk.content);
-
-		lastEndLine = chunk.endLine;
-	}
-
-	return parts.join("\n");
-}

package/src/resources/extensions/gsd/summary-distiller.ts

@@ -1,258 +0,0 @@
-/**
- * Summary distiller — extracts essential structured data from SUMMARY.md files,
- * dropping verbose prose to save context budget.
- */
-
-export interface DistillationResult {
-  content: string;
-  summaryCount: number;
-  savingsPercent: number;
-  originalChars: number;
-  distilledChars: number;
-}
-
-interface ParsedFrontmatter {
-  id: string;
-  provides: string[];
-  requires: string[];
-  key_files: string[];
-  key_decisions: string[];
-  patterns_established: string[];
-}
-
-interface DistilledEntry {
-  id: string;
-  oneLiner: string;
-  provides: string[];
-  requires: string[];
-  key_files: string[];
-  key_decisions: string[];
-  patterns: string[];
-}
-
-// ─── Frontmatter parsing ─────────────────────────────────────────────────────
-
-function parseFrontmatter(raw: string): ParsedFrontmatter {
-  const result: ParsedFrontmatter = {
-    id: "",
-    provides: [],
-    requires: [],
-    key_files: [],
-    key_decisions: [],
-    patterns_established: [],
-  };
-
-  // Extract frontmatter block between --- markers
-  const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
-  if (!fmMatch) return result;
-
-  const fmBlock = fmMatch[1];
-  const lines = fmBlock.split(/\r?\n/);
-
-  let currentKey: string | null = null;
-
-  for (const line of lines) {
-    // Scalar value: key: value
-    const scalarMatch = line.match(/^(\w[\w_]*):\s*(.+)$/);
-    if (scalarMatch) {
-      const [, key, value] = scalarMatch;
-      currentKey = key;
-      setScalar(result, key, value.trim());
-      continue;
-    }
-
-    // Array-start key with empty value: key:\n  or key: []\n
-    const arrayStartMatch = line.match(/^(\w[\w_]*):\s*(\[\])?\s*$/);
-    if (arrayStartMatch) {
-      currentKey = arrayStartMatch[1];
-      continue;
-    }
-
-    // Array item:   - value
-    const itemMatch = line.match(/^\s+-\s+(.+)$/);
-    if (itemMatch && currentKey) {
-      pushItem(result, currentKey, itemMatch[1].trim());
-      continue;
-    }
-  }
-
-  return result;
-}
-
-function setScalar(fm: ParsedFrontmatter, key: string, value: string): void {
-  if (key === "id") fm.id = value;
-}
-
-function pushItem(fm: ParsedFrontmatter, key: string, value: string): void {
-  switch (key) {
-    case "provides": fm.provides.push(value); break;
-    case "requires": fm.requires.push(value); break;
-    case "key_files": fm.key_files.push(value); break;
-    case "key_decisions": fm.key_decisions.push(value); break;
-    case "patterns_established": fm.patterns_established.push(value); break;
-  }
-}
-
-// ─── Body parsing ────────────────────────────────────────────────────────────
-
-function extractTitleAndOneLiner(body: string): { id: string; oneLiner: string } {
-  const lines = body.split(/\r?\n/);
-  let titleId = "";
-  let oneLiner = "";
-  let foundTitle = false;
-
-  for (const line of lines) {
-    const titleMatch = line.match(/^#\s+(\S+):\s*(.*)$/);
-    if (titleMatch && !foundTitle) {
-      titleId = titleMatch[1];
-      // If the title line itself has text after "S01: ", use that as a fallback
-      if (titleMatch[2].trim()) {
-        oneLiner = titleMatch[2].trim();
-      }
-      foundTitle = true;
-      continue;
-    }
-
-    // First non-empty line after the title is the one-liner
-    if (foundTitle && !oneLiner && line.trim() && !line.startsWith("#")) {
-      oneLiner = line.trim();
-      break;
-    }
-  }
-
-  return { id: titleId, oneLiner };
-}
-
-function getBodyAfterFrontmatter(raw: string): string {
-  const fmMatch = raw.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/);
-  if (fmMatch) {
-    return raw.slice(fmMatch[0].length);
-  }
-  return raw;
-}
-
-// ─── Public API ──────────────────────────────────────────────────────────────
-
-/**
- * Distill a single SUMMARY.md content string into a compact structured block.
- */
-export function distillSingle(summary: string): string {
-  const fm = parseFrontmatter(summary);
-  const body = getBodyAfterFrontmatter(summary);
-  const { id: titleId, oneLiner } = extractTitleAndOneLiner(body);
-
-  const id = fm.id || titleId || "???";
-
-  return formatEntry({
-    id,
-    oneLiner,
-    provides: fm.provides,
-    requires: fm.requires,
-    key_files: fm.key_files,
-    key_decisions: fm.key_decisions,
-    patterns: fm.patterns_established,
-  });
-}
-
-function formatEntry(entry: DistilledEntry): string {
-  return formatEntryWithDropLevel(entry, 0);
-}
-
-/**
- * Format an entry, progressively dropping fields based on dropLevel:
- *   0 = full output
- *   1 = drop patterns
- *   2 = drop patterns + key_decisions
- *   3 = drop patterns + key_decisions + key_files
- */
-function formatEntryWithDropLevel(entry: DistilledEntry, dropLevel: number): string {
-  const lines: string[] = [];
-  lines.push(`## ${entry.id}: ${entry.oneLiner}`);
-
-  if (entry.provides.length > 0) {
-    lines.push(`provides: ${entry.provides.join(", ")}`);
-  }
-  if (entry.requires.length > 0) {
-    lines.push(`requires: ${entry.requires.join(", ")}`);
-  }
-  if (dropLevel < 3 && entry.key_files.length > 0) {
-    lines.push(`key_files: ${entry.key_files.join(", ")}`);
-  }
-  if (dropLevel < 2 && entry.key_decisions.length > 0) {
-    lines.push(`key_decisions: ${entry.key_decisions.join(", ")}`);
-  }
-  if (dropLevel < 1 && entry.patterns.length > 0) {
-    lines.push(`patterns: ${entry.patterns.join(", ")}`);
-  }
-
-  return lines.join("\n");
-}
-
-/**
- * Distill multiple SUMMARY.md contents into a budget-constrained output.
- */
-export function distillSummaries(summaries: string[], budgetChars: number): DistillationResult {
-  const originalChars = summaries.reduce((sum, s) => sum + s.length, 0);
-
-  if (summaries.length === 0) {
-    return {
-      content: "",
-      summaryCount: 0,
-      savingsPercent: 0,
-      originalChars: 0,
-      distilledChars: 0,
-    };
-  }
-
-  // Parse all entries up front
-  const entries: DistilledEntry[] = summaries.map((summary) => {
-    const fm = parseFrontmatter(summary);
-    const body = getBodyAfterFrontmatter(summary);
-    const { id: titleId, oneLiner } = extractTitleAndOneLiner(body);
-    return {
-      id: fm.id || titleId || "???",
-      oneLiner,
-      provides: fm.provides,
-      requires: fm.requires,
-      key_files: fm.key_files,
-      key_decisions: fm.key_decisions,
-      patterns: fm.patterns_established,
-    };
-  });
-
-  // Try progressively more aggressive dropping until it fits
-  for (let dropLevel = 0; dropLevel <= 3; dropLevel++) {
-    const blocks = entries.map((e) => formatEntryWithDropLevel(e, dropLevel));
-    const content = blocks.join("\n\n");
-    if (content.length <= budgetChars) {
-      const distilledChars = content.length;
-      return {
-        content,
-        summaryCount: summaries.length,
-        savingsPercent: originalChars > 0
-          ? Math.round((1 - distilledChars / originalChars) * 100)
-          : 0,
-        originalChars,
-        distilledChars,
-      };
-    }
-  }
-
-  // Even at max drop level it doesn't fit — truncate
-  const blocks = entries.map((e) => formatEntryWithDropLevel(e, 3));
-  let content = blocks.join("\n\n");
-  if (content.length > budgetChars) {
-    content = content.slice(0, Math.max(0, budgetChars - 15)) + "\n[...truncated]";
-  }
-
-  const distilledChars = content.length;
-  return {
-    content,
-    summaryCount: summaries.length,
-    savingsPercent: originalChars > 0
-      ? Math.round((1 - distilledChars / originalChars) * 100)
-      : 0,
-    originalChars,
-    distilledChars,
-  };
-}