gsd-pi 2.38.0-dev.add4f78 → 2.38.0-dev.d533afb

package/dist/resources/extensions/gsd/semantic-chunker.js

@@ -1,254 +0,0 @@
-// GSD Extension — Semantic Chunker with TF-IDF Relevance Scoring
-// Splits code/text into semantic chunks and selects the most relevant ones for a given task.
-// Pure TypeScript — no external dependencies.
-// ─── Constants ──────────────────────────────────────────────────────────────
-const CODE_BOUNDARY_RE = /^(export\s+)?(async\s+)?(function|class|interface|type|const|enum)\s/;
-const MARKDOWN_HEADING_RE = /^#{1,6}\s/;
-const STOP_WORDS = new Set([
-    "the", "a", "an", "is", "are", "was", "were", "be", "to", "of", "in",
-    "for", "on", "with", "at", "by", "from", "this", "that", "it", "as",
-    "or", "and", "not", "but", "if", "do", "no", "so", "up", "its", "has",
-    "had", "get", "set", "can", "may", "all", "use", "new", "one", "two",
-    "also", "each", "than", "been", "into", "most", "only", "over", "such",
-    "how", "some", "any", "our", "his", "her", "out", "did", "let", "say", "she",
-]);
-const DEFAULT_MIN_LINES = 3;
-const DEFAULT_MAX_LINES = 80;
-const DEFAULT_MAX_CHUNKS = 5;
-const DEFAULT_MIN_SCORE = 0.1;
-function detectContentType(lines) {
-    let codeSignals = 0;
-    let mdSignals = 0;
-    const sampleSize = Math.min(lines.length, 50);
-    for (let i = 0; i < sampleSize; i++) {
-        const line = lines[i];
-        if (CODE_BOUNDARY_RE.test(line) || /^\s*import\s/.test(line)) {
-            codeSignals++;
-        }
-        if (MARKDOWN_HEADING_RE.test(line)) {
-            mdSignals++;
-        }
-    }
-    if (mdSignals >= 2 && mdSignals > codeSignals)
-        return "markdown";
-    if (codeSignals >= 2)
-        return "code";
-    return "text";
-}
-// ─── Tokenizer ──────────────────────────────────────────────────────────────
-function tokenize(text) {
-    return text
-        .toLowerCase()
-        .split(/[\s\W]+/)
-        .filter((w) => w.length >= 2 && !STOP_WORDS.has(w));
-}
-// ─── splitIntoChunks ────────────────────────────────────────────────────────
-export function splitIntoChunks(content, options) {
-    if (!content || content.trim().length === 0)
-        return [];
-    const minLines = options?.minLines ?? DEFAULT_MIN_LINES;
-    const maxLines = options?.maxLines ?? DEFAULT_MAX_LINES;
-    const lines = content.split("\n");
-    if (lines.length === 0)
-        return [];
-    const contentType = detectContentType(lines);
-    let boundaries;
-    switch (contentType) {
-        case "code":
-            boundaries = findCodeBoundaries(lines);
-            break;
-        case "markdown":
-            boundaries = findMarkdownBoundaries(lines);
-            break;
-        default:
-            boundaries = findTextBoundaries(lines);
-            break;
-    }
-    // Always include 0 as first boundary
-    if (boundaries.length === 0 || boundaries[0] !== 0) {
-        boundaries.unshift(0);
-    }
-    // Build raw chunks from boundaries
-    const rawChunks = [];
-    for (let i = 0; i < boundaries.length; i++) {
-        const start = boundaries[i];
-        const end = i + 1 < boundaries.length ? boundaries[i + 1] - 1 : lines.length - 1;
-        const chunkLines = lines.slice(start, end + 1);
-        rawChunks.push({
-            content: chunkLines.join("\n"),
-            startLine: start + 1, // 1-based
-            endLine: end + 1, // 1-based
-            score: 0,
-        });
-    }
-    // Split oversized chunks at maxLines
-    const splitChunks = [];
-    for (const chunk of rawChunks) {
-        const chunkLineCount = chunk.endLine - chunk.startLine + 1;
-        if (chunkLineCount <= maxLines) {
-            splitChunks.push(chunk);
-        }
-        else {
-            const chunkLines = chunk.content.split("\n");
-            for (let offset = 0; offset < chunkLines.length; offset += maxLines) {
-                const slice = chunkLines.slice(offset, offset + maxLines);
-                splitChunks.push({
-                    content: slice.join("\n"),
-                    startLine: chunk.startLine + offset,
-                    endLine: chunk.startLine + offset + slice.length - 1,
-                    score: 0,
-                });
-            }
-        }
-    }
-    // Merge tiny chunks into predecessor
-    const merged = [];
-    for (const chunk of splitChunks) {
-        const chunkLineCount = chunk.endLine - chunk.startLine + 1;
-        if (chunkLineCount < minLines && merged.length > 0) {
-            const prev = merged[merged.length - 1];
-            prev.content += "\n" + chunk.content;
-            prev.endLine = chunk.endLine;
-        }
-        else {
-            merged.push({ ...chunk });
-        }
-    }
-    return merged;
-}
-function findCodeBoundaries(lines) {
-    const boundaries = [];
-    for (let i = 0; i < lines.length; i++) {
-        if (CODE_BOUNDARY_RE.test(lines[i])) {
-            // Also consider a blank line before a boundary marker
-            if (i > 0 && lines[i - 1].trim() === "" && !boundaries.includes(i)) {
-                boundaries.push(i);
-            }
-            else if (!boundaries.includes(i)) {
-                boundaries.push(i);
-            }
-        }
-    }
-    return boundaries;
-}
-function findMarkdownBoundaries(lines) {
-    const boundaries = [];
-    for (let i = 0; i < lines.length; i++) {
-        if (MARKDOWN_HEADING_RE.test(lines[i])) {
-            boundaries.push(i);
-        }
-    }
-    return boundaries;
-}
-function findTextBoundaries(lines) {
-    const boundaries = [0];
-    for (let i = 1; i < lines.length; i++) {
-        if (lines[i - 1].trim() === "" && lines[i].trim() !== "") {
-            boundaries.push(i);
-        }
-    }
-    return boundaries;
-}
-// ─── scoreChunks ────────────────────────────────────────────────────────────
-export function scoreChunks(chunks, query) {
-    if (chunks.length === 0)
-        return [];
-    const queryTerms = tokenize(query);
-    if (queryTerms.length === 0) {
-        return chunks.map((c) => ({ ...c, score: 0 }));
-    }
-    const totalChunks = chunks.length;
-    // Pre-compute IDF for each query term
-    const termChunkCounts = new Map();
-    const chunkTokenSets = [];
-    for (const chunk of chunks) {
-        const tokens = new Set(tokenize(chunk.content));
-        chunkTokenSets.push(tokens);
-        for (const term of queryTerms) {
-            if (tokens.has(term)) {
-                termChunkCounts.set(term, (termChunkCounts.get(term) ?? 0) + 1);
-            }
-        }
-    }
-    const idf = new Map();
-    for (const term of queryTerms) {
-        const df = termChunkCounts.get(term) ?? 0;
-        idf.set(term, Math.log(1 + totalChunks / (1 + df)));
-    }
-    // Score each chunk
-    const scored = chunks.map((chunk, idx) => {
-        const chunkTokens = tokenize(chunk.content);
-        const totalTerms = chunkTokens.length;
-        if (totalTerms === 0)
-            return { ...chunk, score: 0 };
-        // Count term frequencies
-        const termFreq = new Map();
-        for (const token of chunkTokens) {
-            termFreq.set(token, (termFreq.get(token) ?? 0) + 1);
-        }
-        let score = 0;
-        for (const term of queryTerms) {
-            const tf = (termFreq.get(term) ?? 0) / totalTerms;
-            const termIdf = idf.get(term) ?? 0;
-            score += tf * termIdf;
-        }
-        return { ...chunk, score };
-    });
-    // Normalize to 0-1
-    const maxScore = Math.max(...scored.map((c) => c.score));
-    if (maxScore > 0) {
-        for (const chunk of scored) {
-            chunk.score = chunk.score / maxScore;
-        }
-    }
-    return scored;
-}
-// ─── chunkByRelevance ───────────────────────────────────────────────────────
-export function chunkByRelevance(content, query, options) {
-    const maxChunks = options?.maxChunks ?? DEFAULT_MAX_CHUNKS;
-    const minScore = options?.minScore ?? DEFAULT_MIN_SCORE;
-    const minLines = options?.minChunkLines ?? DEFAULT_MIN_LINES;
-    const maxLines = options?.maxChunkLines ?? DEFAULT_MAX_LINES;
-    const rawChunks = splitIntoChunks(content, { minLines, maxLines });
-    if (rawChunks.length === 0) {
-        return { chunks: [], totalChunks: 0, omittedChunks: 0, savingsPercent: 0 };
-    }
-    const scored = scoreChunks(rawChunks, query);
-    // Filter by minScore and take top maxChunks by score
-    const qualifying = scored
-        .filter((c) => c.score >= minScore)
-        .sort((a, b) => b.score - a.score)
-        .slice(0, maxChunks);
-    // Return in original document order (by startLine)
-    const selected = qualifying.sort((a, b) => a.startLine - b.startLine);
-    const totalChars = content.length;
-    const selectedChars = selected.reduce((sum, c) => sum + c.content.length, 0);
-    const savingsPercent = totalChars > 0
-        ? Math.round(((totalChars - selectedChars) / totalChars) * 100)
-        : 0;
-    return {
-        chunks: selected,
-        totalChunks: rawChunks.length,
-        omittedChunks: rawChunks.length - selected.length,
-        savingsPercent: Math.max(0, savingsPercent),
-    };
-}
-// ─── formatChunks ───────────────────────────────────────────────────────────
-export function formatChunks(result, filePath) {
-    if (result.chunks.length === 0) {
-        return `[${filePath}: empty or no relevant chunks]`;
-    }
-    const parts = [];
-    let lastEndLine = 0;
-    for (const chunk of result.chunks) {
-        // Show omission gap
-        if (lastEndLine > 0 && chunk.startLine > lastEndLine + 1) {
-            const gapLines = chunk.startLine - lastEndLine - 1;
-            parts.push(`[...${gapLines} lines omitted...]`);
-        }
-        parts.push(`[Lines ${chunk.startLine}-${chunk.endLine}]`);
-        parts.push(chunk.content);
-        lastEndLine = chunk.endLine;
-    }
-    return parts.join("\n");
-}

package/dist/resources/extensions/gsd/summary-distiller.js

@@ -1,212 +0,0 @@
-/**
- * Summary distiller — extracts essential structured data from SUMMARY.md files,
- * dropping verbose prose to save context budget.
- */
-// ─── Frontmatter parsing ─────────────────────────────────────────────────────
-function parseFrontmatter(raw) {
-    const result = {
-        id: "",
-        provides: [],
-        requires: [],
-        key_files: [],
-        key_decisions: [],
-        patterns_established: [],
-    };
-    // Extract frontmatter block between --- markers
-    const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
-    if (!fmMatch)
-        return result;
-    const fmBlock = fmMatch[1];
-    const lines = fmBlock.split(/\r?\n/);
-    let currentKey = null;
-    for (const line of lines) {
-        // Scalar value: key: value
-        const scalarMatch = line.match(/^(\w[\w_]*):\s*(.+)$/);
-        if (scalarMatch) {
-            const [, key, value] = scalarMatch;
-            currentKey = key;
-            setScalar(result, key, value.trim());
-            continue;
-        }
-        // Array-start key with empty value: key:\n  or key: []\n
-        const arrayStartMatch = line.match(/^(\w[\w_]*):\s*(\[\])?\s*$/);
-        if (arrayStartMatch) {
-            currentKey = arrayStartMatch[1];
-            continue;
-        }
-        // Array item:   - value
-        const itemMatch = line.match(/^\s+-\s+(.+)$/);
-        if (itemMatch && currentKey) {
-            pushItem(result, currentKey, itemMatch[1].trim());
-            continue;
-        }
-    }
-    return result;
-}
-function setScalar(fm, key, value) {
-    if (key === "id")
-        fm.id = value;
-}
-function pushItem(fm, key, value) {
-    switch (key) {
-        case "provides":
-            fm.provides.push(value);
-            break;
-        case "requires":
-            fm.requires.push(value);
-            break;
-        case "key_files":
-            fm.key_files.push(value);
-            break;
-        case "key_decisions":
-            fm.key_decisions.push(value);
-            break;
-        case "patterns_established":
-            fm.patterns_established.push(value);
-            break;
-    }
-}
-// ─── Body parsing ────────────────────────────────────────────────────────────
-function extractTitleAndOneLiner(body) {
-    const lines = body.split(/\r?\n/);
-    let titleId = "";
-    let oneLiner = "";
-    let foundTitle = false;
-    for (const line of lines) {
-        const titleMatch = line.match(/^#\s+(\S+):\s*(.*)$/);
-        if (titleMatch && !foundTitle) {
-            titleId = titleMatch[1];
-            // If the title line itself has text after "S01: ", use that as a fallback
-            if (titleMatch[2].trim()) {
-                oneLiner = titleMatch[2].trim();
-            }
-            foundTitle = true;
-            continue;
-        }
-        // First non-empty line after the title is the one-liner
-        if (foundTitle && !oneLiner && line.trim() && !line.startsWith("#")) {
-            oneLiner = line.trim();
-            break;
-        }
-    }
-    return { id: titleId, oneLiner };
-}
-function getBodyAfterFrontmatter(raw) {
-    const fmMatch = raw.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/);
-    if (fmMatch) {
-        return raw.slice(fmMatch[0].length);
-    }
-    return raw;
-}
-// ─── Public API ──────────────────────────────────────────────────────────────
-/**
- * Distill a single SUMMARY.md content string into a compact structured block.
- */
-export function distillSingle(summary) {
-    const fm = parseFrontmatter(summary);
-    const body = getBodyAfterFrontmatter(summary);
-    const { id: titleId, oneLiner } = extractTitleAndOneLiner(body);
-    const id = fm.id || titleId || "???";
-    return formatEntry({
-        id,
-        oneLiner,
-        provides: fm.provides,
-        requires: fm.requires,
-        key_files: fm.key_files,
-        key_decisions: fm.key_decisions,
-        patterns: fm.patterns_established,
-    });
-}
-function formatEntry(entry) {
-    return formatEntryWithDropLevel(entry, 0);
-}
-/**
- * Format an entry, progressively dropping fields based on dropLevel:
- *   0 = full output
- *   1 = drop patterns
- *   2 = drop patterns + key_decisions
- *   3 = drop patterns + key_decisions + key_files
- */
-function formatEntryWithDropLevel(entry, dropLevel) {
-    const lines = [];
-    lines.push(`## ${entry.id}: ${entry.oneLiner}`);
-    if (entry.provides.length > 0) {
-        lines.push(`provides: ${entry.provides.join(", ")}`);
-    }
-    if (entry.requires.length > 0) {
-        lines.push(`requires: ${entry.requires.join(", ")}`);
-    }
-    if (dropLevel < 3 && entry.key_files.length > 0) {
-        lines.push(`key_files: ${entry.key_files.join(", ")}`);
-    }
-    if (dropLevel < 2 && entry.key_decisions.length > 0) {
-        lines.push(`key_decisions: ${entry.key_decisions.join(", ")}`);
-    }
-    if (dropLevel < 1 && entry.patterns.length > 0) {
-        lines.push(`patterns: ${entry.patterns.join(", ")}`);
-    }
-    return lines.join("\n");
-}
-/**
- * Distill multiple SUMMARY.md contents into a budget-constrained output.
- */
-export function distillSummaries(summaries, budgetChars) {
-    const originalChars = summaries.reduce((sum, s) => sum + s.length, 0);
-    if (summaries.length === 0) {
-        return {
-            content: "",
-            summaryCount: 0,
-            savingsPercent: 0,
-            originalChars: 0,
-            distilledChars: 0,
-        };
-    }
-    // Parse all entries up front
-    const entries = summaries.map((summary) => {
-        const fm = parseFrontmatter(summary);
-        const body = getBodyAfterFrontmatter(summary);
-        const { id: titleId, oneLiner } = extractTitleAndOneLiner(body);
-        return {
-            id: fm.id || titleId || "???",
-            oneLiner,
-            provides: fm.provides,
-            requires: fm.requires,
-            key_files: fm.key_files,
-            key_decisions: fm.key_decisions,
-            patterns: fm.patterns_established,
-        };
-    });
-    // Try progressively more aggressive dropping until it fits
-    for (let dropLevel = 0; dropLevel <= 3; dropLevel++) {
-        const blocks = entries.map((e) => formatEntryWithDropLevel(e, dropLevel));
-        const content = blocks.join("\n\n");
-        if (content.length <= budgetChars) {
-            const distilledChars = content.length;
-            return {
-                content,
-                summaryCount: summaries.length,
-                savingsPercent: originalChars > 0
-                    ? Math.round((1 - distilledChars / originalChars) * 100)
-                    : 0,
-                originalChars,
-                distilledChars,
-            };
-        }
-    }
-    // Even at max drop level it doesn't fit — truncate
-    const blocks = entries.map((e) => formatEntryWithDropLevel(e, 3));
-    let content = blocks.join("\n\n");
-    if (content.length > budgetChars) {
-        content = content.slice(0, Math.max(0, budgetChars - 15)) + "\n[...truncated]";
-    }
-    const distilledChars = content.length;
-    return {
-        content,
-        summaryCount: summaries.length,
-        savingsPercent: originalChars > 0
-            ? Math.round((1 - distilledChars / originalChars) * 100)
-            : 0,
-        originalChars,
-        distilledChars,
-    };
-}