gsd-pi 2.38.0-dev.96dc7fb → 2.38.0-dev.98b44dc

package/dist/resources/extensions/gsd/prompt-compressor.js

@@ -1,393 +0,0 @@
-/**
- * Prompt Compressor — deterministic text compression for context reduction.
- *
- * Applies a series of lossless and near-lossless transformations to reduce
- * token count while preserving semantic meaning. No LLM calls, no external
- * dependencies. Sub-millisecond for typical prompt sizes.
- *
- * Compression techniques (applied in order):
- * 1. Redundant whitespace normalization
- * 2. Markdown formatting reduction (collapse verbose tables, lists)
- * 3. Common phrase abbreviation
- * 4. Repeated pattern deduplication
- * 5. Low-information content removal (empty sections, boilerplate)
- */
-// ─── Phrase Abbreviation Map ────────────────────────────────────────────────
-/**
- * Build a regex that matches a verbose phrase even when split across lines.
- * Whitespace between words is matched with \s+ to handle line wrapping.
- */
-function phraseRegex(phrase) {
-    const words = phrase.split(/\s+/);
-    const pattern = `\\b${words.join("\\s+")}\\b`;
-    return new RegExp(pattern, "gi");
-}
-const VERBOSE_PHRASES = [
-    [phraseRegex("In order to"), "To"],
-    [phraseRegex("It is important to note that"), "Note:"],
-    [phraseRegex("As mentioned previously"), "(see above)"],
-    [phraseRegex("The following"), "These"],
-    [phraseRegex("In addition to"), "Also,"],
-    [phraseRegex("Due to the fact that"), "Because"],
-    [phraseRegex("At this point in time"), "Now"],
-    [phraseRegex("For the purpose of"), "For"],
-    [phraseRegex("In the event that"), "If"],
-    [phraseRegex("With regard to"), "Re:"],
-    [phraseRegex("Prior to"), "Before"],
-    [phraseRegex("Subsequent to"), "After"],
-    [phraseRegex("In accordance with"), "Per"],
-    [phraseRegex("A number of"), "Several"],
-    [phraseRegex("In the case of"), "For"],
-    [phraseRegex("On the basis of"), "Based on"],
-];
-function extractCodeBlocks(content) {
-    const blocks = new Map();
-    let counter = 0;
-    const text = content.replace(/```[\s\S]*?```/g, (match) => {
-        const placeholder = `\x00CODEBLOCK_${counter++}\x00`;
-        blocks.set(placeholder, match);
-        return placeholder;
-    });
-    return { text, blocks };
-}
-function restoreCodeBlocks(text, blocks) {
-    let result = text;
-    for (const [placeholder, block] of blocks) {
-        result = result.replace(placeholder, block);
-    }
-    return result;
-}
-// ─── Light Transformations ──────────────────────────────────────────────────
-function normalizeWhitespace(content) {
-    // Collapse 3+ consecutive blank lines to 2
-    let result = content.replace(/(\n\s*){3,}\n/g, "\n\n");
-    // Trim trailing whitespace on every line
-    result = result.replace(/[ \t]+$/gm, "");
-    return result;
-}
-function removeMarkdownComments(content) {
-    return content.replace(/<!--[\s\S]*?-->/g, "");
-}
-function removeHorizontalRules(content) {
-    // Remove horizontal rules (---, ***, ___) that stand alone on a line
-    return content.replace(/^\s*[-*_]{3,}\s*$/gm, "");
-}
-function collapseEmptyListItems(content) {
-    // Collapse repeated empty list items (- \n- \n- \n) into one
-    return content.replace(/(^[ \t]*[-*+]\s*$\n){2,}/gm, "$1");
-}
-function applyLightTransformations(content) {
-    let count = 0;
-    let result = content;
-    const after1 = normalizeWhitespace(result);
-    if (after1 !== result)
-        count++;
-    result = after1;
-    const after2 = removeMarkdownComments(result);
-    if (after2 !== result)
-        count++;
-    result = after2;
-    const after3 = removeHorizontalRules(result);
-    if (after3 !== result)
-        count++;
-    result = after3;
-    const after4 = collapseEmptyListItems(result);
-    if (after4 !== result)
-        count++;
-    result = after4;
-    return { content: result, count };
-}
-// ─── Moderate Transformations ───────────────────────────────────────────────
-function abbreviateVerbosePhrases(content) {
-    let count = 0;
-    let result = content;
-    for (const [pattern, replacement] of VERBOSE_PHRASES) {
-        const after = result.replace(pattern, replacement);
-        if (after !== result)
-            count++;
-        result = after;
-    }
-    return { content: result, count };
-}
-function removeBoilerplateLines(content) {
-    const lines = content.split("\n");
-    const filtered = lines.filter((line) => {
-        const trimmed = line.trim();
-        // Remove lines that are just N/A, (none), (empty), (not applicable)
-        if (/^(?:N\/A|\(none\)|\(empty\)|\(not applicable\))$/i.test(trimmed)) {
-            return false;
-        }
-        return true;
-    });
-    return filtered.join("\n");
-}
-function deduplicateConsecutiveLines(content) {
-    const lines = content.split("\n");
-    const result = [];
-    for (let i = 0; i < lines.length; i++) {
-        if (i === 0 || lines[i] !== lines[i - 1] || lines[i].trim() === "") {
-            result.push(lines[i]);
-        }
-    }
-    return result.join("\n");
-}
-function collapseTableFormatting(content) {
-    // Remove excessive padding in markdown table cells
-    // Matches table rows like |  cell  |  cell  | and collapses to | cell | cell |
-    return content.replace(/\|[ \t]{2,}([^|\n]*?)[ \t]{2,}\|/g, (_, cellContent) => {
-        return `| ${cellContent.trim()} |`;
-    });
-}
-function applyModerateTransformations(content) {
-    let count = 0;
-    let result = content;
-    const phraseResult = abbreviateVerbosePhrases(result);
-    count += phraseResult.count;
-    result = phraseResult.content;
-    const after1 = removeBoilerplateLines(result);
-    if (after1 !== result)
-        count++;
-    result = after1;
-    const after2 = deduplicateConsecutiveLines(result);
-    if (after2 !== result)
-        count++;
-    result = after2;
-    const after3 = collapseTableFormatting(result);
-    if (after3 !== result)
-        count++;
-    result = after3;
-    return { content: result, count };
-}
-// ─── Aggressive Transformations ─────────────────────────────────────────────
-function removeMarkdownEmphasis(content) {
-    // Bold: **text** or __text__
-    let result = content.replace(/\*\*(.+?)\*\*/g, "$1");
-    result = result.replace(/__(.+?)__/g, "$1");
-    // Italic: *text* or _text_ (single, not inside words)
-    result = result.replace(/(?<!\w)\*([^*\n]+?)\*(?!\w)/g, "$1");
-    result = result.replace(/(?<!\w)_([^_\n]+?)_(?!\w)/g, "$1");
-    return result;
-}
-function removeMarkdownLinks(content) {
-    // [text](url) → text
-    return content.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
-}
-function truncateLongLines(content) {
-    const lines = content.split("\n");
-    const result = lines.map((line) => {
-        if (line.length <= 300)
-            return line;
-        // Find a sentence boundary (. ! ?) near the 300 char mark
-        const truncateZone = line.slice(0, 300);
-        const lastSentenceEnd = Math.max(truncateZone.lastIndexOf(". "), truncateZone.lastIndexOf("! "), truncateZone.lastIndexOf("? "));
-        if (lastSentenceEnd > 150) {
-            return line.slice(0, lastSentenceEnd + 1);
-        }
-        // Fallback: cut at last space before 300
-        const lastSpace = truncateZone.lastIndexOf(" ");
-        if (lastSpace > 150) {
-            return line.slice(0, lastSpace);
-        }
-        return truncateZone;
-    });
-    return result.join("\n");
-}
-function removeBulletMarkers(content) {
-    // Remove bullet markers: - , * , + , numbered (1. 2. etc)
-    return content.replace(/^[ \t]*(?:[-*+]|\d+\.)\s+/gm, "");
-}
-function removeBlockquoteMarkers(content) {
-    return content.replace(/^[ \t]*>+\s?/gm, "");
-}
-function deduplicateStructuralPatterns(content) {
-    // Deduplicate consecutive lines that match the same "Key: value" pattern
-    const lines = content.split("\n");
-    const result = [];
-    const seen = new Set();
-    let lastWasStructural = false;
-    for (const line of lines) {
-        const trimmed = line.trim();
-        // Detect structural patterns: "Key: value"
-        const structMatch = trimmed.match(/^(\w[\w\s]*?):\s+(.+)$/);
-        if (structMatch) {
-            if (seen.has(trimmed)) {
-                lastWasStructural = true;
-                continue;
-            }
-            seen.add(trimmed);
-            lastWasStructural = true;
-        }
-        else {
-            // Reset seen set when structural block ends
-            if (!lastWasStructural || trimmed === "") {
-                seen.clear();
-            }
-            lastWasStructural = false;
-        }
-        result.push(line);
-    }
-    return result.join("\n");
-}
-function applyAggressiveTransformations(content, preserveHeadings) {
-    let count = 0;
-    let result = content;
-    const after1 = removeMarkdownEmphasis(result);
-    if (after1 !== result)
-        count++;
-    result = after1;
-    const after2 = removeMarkdownLinks(result);
-    if (after2 !== result)
-        count++;
-    result = after2;
-    const after3 = truncateLongLines(result);
-    if (after3 !== result)
-        count++;
-    result = after3;
-    const after4 = removeBulletMarkers(result);
-    if (after4 !== result)
-        count++;
-    result = after4;
-    const after5 = removeBlockquoteMarkers(result);
-    if (after5 !== result)
-        count++;
-    result = after5;
-    const after6 = deduplicateStructuralPatterns(result);
-    if (after6 !== result)
-        count++;
-    result = after6;
-    return { content: result, count };
-}
-function extractHeadings(content) {
-    const headings = new Map();
-    let counter = 0;
-    const text = content.replace(/^(#{1,6}\s.+)$/gm, (match) => {
-        const placeholder = `\x00HEADING_${counter++}\x00`;
-        headings.set(placeholder, match);
-        return placeholder;
-    });
-    return { text, headings };
-}
-function restoreHeadings(text, headings) {
-    let result = text;
-    for (const [placeholder, heading] of headings) {
-        result = result.replace(placeholder, heading);
-    }
-    return result;
-}
-// ─── Public API ─────────────────────────────────────────────────────────────
-/**
- * Compress prompt content using deterministic text transformations.
- */
-export function compressPrompt(content, options) {
-    const level = options?.level ?? "moderate";
-    const preserveHeadings = options?.preserveHeadings ?? true;
-    const preserveCodeBlocks = options?.preserveCodeBlocks ?? true;
-    if (content === "") {
-        return {
-            content: "",
-            originalChars: 0,
-            compressedChars: 0,
-            savingsPercent: 0,
-            level,
-            transformationsApplied: 0,
-        };
-    }
-    const originalChars = content.length;
-    let working = content;
-    let totalTransformations = 0;
-    // Extract code blocks if preserving
-    let codeBlocks = null;
-    if (preserveCodeBlocks) {
-        const extracted = extractCodeBlocks(working);
-        working = extracted.text;
-        codeBlocks = extracted.blocks;
-    }
-    // Extract headings if preserving
-    let headings = null;
-    if (preserveHeadings) {
-        const extracted = extractHeadings(working);
-        working = extracted.text;
-        headings = extracted.headings;
-    }
-    // Apply light transformations (always)
-    const lightResult = applyLightTransformations(working);
-    working = lightResult.content;
-    totalTransformations += lightResult.count;
-    // Check target
-    if (options?.targetChars && getRestoredLength(working, codeBlocks, headings) <= options.targetChars) {
-        return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings);
-    }
-    // Apply moderate transformations
-    if (level === "moderate" || level === "aggressive") {
-        const modResult = applyModerateTransformations(working);
-        working = modResult.content;
-        totalTransformations += modResult.count;
-        if (options?.targetChars && getRestoredLength(working, codeBlocks, headings) <= options.targetChars) {
-            return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings);
-        }
-    }
-    // Apply aggressive transformations
-    if (level === "aggressive") {
-        const aggResult = applyAggressiveTransformations(working, preserveHeadings);
-        working = aggResult.content;
-        totalTransformations += aggResult.count;
-    }
-    return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings);
-}
-/**
- * Compress with a target size — applies progressively more aggressive
- * compression until the target is reached or all transformations exhausted.
- */
-export function compressToTarget(content, targetChars) {
-    if (content.length <= targetChars) {
-        return {
-            content,
-            originalChars: content.length,
-            compressedChars: content.length,
-            savingsPercent: 0,
-            level: "light",
-            transformationsApplied: 0,
-        };
-    }
-    const levels = ["light", "moderate", "aggressive"];
-    for (const level of levels) {
-        const result = compressPrompt(content, { level, targetChars });
-        if (result.compressedChars <= targetChars) {
-            return result;
-        }
-        // If aggressive and still over target, return best effort
-        if (level === "aggressive") {
-            return result;
-        }
-    }
-    // Unreachable, but satisfy TypeScript
-    return compressPrompt(content, { level: "aggressive" });
-}
-// ─── Helpers ────────────────────────────────────────────────────────────────
-function getRestoredLength(text, codeBlocks, headings) {
-    let result = text;
-    if (headings)
-        result = restoreHeadings(result, headings);
-    if (codeBlocks)
-        result = restoreCodeBlocks(result, codeBlocks);
-    return result.length;
-}
-function buildResult(working, originalChars, level, transformationsApplied, codeBlocks, headings) {
-    let content = working;
-    if (headings)
-        content = restoreHeadings(content, headings);
-    if (codeBlocks)
-        content = restoreCodeBlocks(content, codeBlocks);
-    const compressedChars = content.length;
-    const savingsPercent = originalChars > 0
-        ? Math.round(((originalChars - compressedChars) / originalChars) * 10000) / 100
-        : 0;
-    return {
-        content,
-        originalChars,
-        compressedChars,
-        savingsPercent,
-        level,
-        transformationsApplied,
-    };
-}

package/dist/resources/extensions/gsd/semantic-chunker.js

@@ -1,254 +0,0 @@
-// GSD Extension — Semantic Chunker with TF-IDF Relevance Scoring
-// Splits code/text into semantic chunks and selects the most relevant ones for a given task.
-// Pure TypeScript — no external dependencies.
-// ─── Constants ──────────────────────────────────────────────────────────────
-const CODE_BOUNDARY_RE = /^(export\s+)?(async\s+)?(function|class|interface|type|const|enum)\s/;
-const MARKDOWN_HEADING_RE = /^#{1,6}\s/;
-const STOP_WORDS = new Set([
-    "the", "a", "an", "is", "are", "was", "were", "be", "to", "of", "in",
-    "for", "on", "with", "at", "by", "from", "this", "that", "it", "as",
-    "or", "and", "not", "but", "if", "do", "no", "so", "up", "its", "has",
-    "had", "get", "set", "can", "may", "all", "use", "new", "one", "two",
-    "also", "each", "than", "been", "into", "most", "only", "over", "such",
-    "how", "some", "any", "our", "his", "her", "out", "did", "let", "say", "she",
-]);
-const DEFAULT_MIN_LINES = 3;
-const DEFAULT_MAX_LINES = 80;
-const DEFAULT_MAX_CHUNKS = 5;
-const DEFAULT_MIN_SCORE = 0.1;
-function detectContentType(lines) {
-    let codeSignals = 0;
-    let mdSignals = 0;
-    const sampleSize = Math.min(lines.length, 50);
-    for (let i = 0; i < sampleSize; i++) {
-        const line = lines[i];
-        if (CODE_BOUNDARY_RE.test(line) || /^\s*import\s/.test(line)) {
-            codeSignals++;
-        }
-        if (MARKDOWN_HEADING_RE.test(line)) {
-            mdSignals++;
-        }
-    }
-    if (mdSignals >= 2 && mdSignals > codeSignals)
-        return "markdown";
-    if (codeSignals >= 2)
-        return "code";
-    return "text";
-}
-// ─── Tokenizer ──────────────────────────────────────────────────────────────
-function tokenize(text) {
-    return text
-        .toLowerCase()
-        .split(/[\s\W]+/)
-        .filter((w) => w.length >= 2 && !STOP_WORDS.has(w));
-}
-// ─── splitIntoChunks ────────────────────────────────────────────────────────
-export function splitIntoChunks(content, options) {
-    if (!content || content.trim().length === 0)
-        return [];
-    const minLines = options?.minLines ?? DEFAULT_MIN_LINES;
-    const maxLines = options?.maxLines ?? DEFAULT_MAX_LINES;
-    const lines = content.split("\n");
-    if (lines.length === 0)
-        return [];
-    const contentType = detectContentType(lines);
-    let boundaries;
-    switch (contentType) {
-        case "code":
-            boundaries = findCodeBoundaries(lines);
-            break;
-        case "markdown":
-            boundaries = findMarkdownBoundaries(lines);
-            break;
-        default:
-            boundaries = findTextBoundaries(lines);
-            break;
-    }
-    // Always include 0 as first boundary
-    if (boundaries.length === 0 || boundaries[0] !== 0) {
-        boundaries.unshift(0);
-    }
-    // Build raw chunks from boundaries
-    const rawChunks = [];
-    for (let i = 0; i < boundaries.length; i++) {
-        const start = boundaries[i];
-        const end = i + 1 < boundaries.length ? boundaries[i + 1] - 1 : lines.length - 1;
-        const chunkLines = lines.slice(start, end + 1);
-        rawChunks.push({
-            content: chunkLines.join("\n"),
-            startLine: start + 1, // 1-based
-            endLine: end + 1, // 1-based
-            score: 0,
-        });
-    }
-    // Split oversized chunks at maxLines
-    const splitChunks = [];
-    for (const chunk of rawChunks) {
-        const chunkLineCount = chunk.endLine - chunk.startLine + 1;
-        if (chunkLineCount <= maxLines) {
-            splitChunks.push(chunk);
-        }
-        else {
-            const chunkLines = chunk.content.split("\n");
-            for (let offset = 0; offset < chunkLines.length; offset += maxLines) {
-                const slice = chunkLines.slice(offset, offset + maxLines);
-                splitChunks.push({
-                    content: slice.join("\n"),
-                    startLine: chunk.startLine + offset,
-                    endLine: chunk.startLine + offset + slice.length - 1,
-                    score: 0,
-                });
-            }
-        }
-    }
-    // Merge tiny chunks into predecessor
-    const merged = [];
-    for (const chunk of splitChunks) {
-        const chunkLineCount = chunk.endLine - chunk.startLine + 1;
-        if (chunkLineCount < minLines && merged.length > 0) {
-            const prev = merged[merged.length - 1];
-            prev.content += "\n" + chunk.content;
-            prev.endLine = chunk.endLine;
-        }
-        else {
-            merged.push({ ...chunk });
-        }
-    }
-    return merged;
-}
-function findCodeBoundaries(lines) {
-    const boundaries = [];
-    for (let i = 0; i < lines.length; i++) {
-        if (CODE_BOUNDARY_RE.test(lines[i])) {
-            // Also consider a blank line before a boundary marker
-            if (i > 0 && lines[i - 1].trim() === "" && !boundaries.includes(i)) {
-                boundaries.push(i);
-            }
-            else if (!boundaries.includes(i)) {
-                boundaries.push(i);
-            }
-        }
-    }
-    return boundaries;
-}
-function findMarkdownBoundaries(lines) {
-    const boundaries = [];
-    for (let i = 0; i < lines.length; i++) {
-        if (MARKDOWN_HEADING_RE.test(lines[i])) {
-            boundaries.push(i);
-        }
-    }
-    return boundaries;
-}
-function findTextBoundaries(lines) {
-    const boundaries = [0];
-    for (let i = 1; i < lines.length; i++) {
-        if (lines[i - 1].trim() === "" && lines[i].trim() !== "") {
-            boundaries.push(i);
-        }
-    }
-    return boundaries;
-}
-// ─── scoreChunks ────────────────────────────────────────────────────────────
-export function scoreChunks(chunks, query) {
-    if (chunks.length === 0)
-        return [];
-    const queryTerms = tokenize(query);
-    if (queryTerms.length === 0) {
-        return chunks.map((c) => ({ ...c, score: 0 }));
-    }
-    const totalChunks = chunks.length;
-    // Pre-compute IDF for each query term
-    const termChunkCounts = new Map();
-    const chunkTokenSets = [];
-    for (const chunk of chunks) {
-        const tokens = new Set(tokenize(chunk.content));
-        chunkTokenSets.push(tokens);
-        for (const term of queryTerms) {
-            if (tokens.has(term)) {
-                termChunkCounts.set(term, (termChunkCounts.get(term) ?? 0) + 1);
-            }
-        }
-    }
-    const idf = new Map();
-    for (const term of queryTerms) {
-        const df = termChunkCounts.get(term) ?? 0;
-        idf.set(term, Math.log(1 + totalChunks / (1 + df)));
-    }
-    // Score each chunk
-    const scored = chunks.map((chunk, idx) => {
-        const chunkTokens = tokenize(chunk.content);
-        const totalTerms = chunkTokens.length;
-        if (totalTerms === 0)
-            return { ...chunk, score: 0 };
-        // Count term frequencies
-        const termFreq = new Map();
-        for (const token of chunkTokens) {
-            termFreq.set(token, (termFreq.get(token) ?? 0) + 1);
-        }
-        let score = 0;
-        for (const term of queryTerms) {
-            const tf = (termFreq.get(term) ?? 0) / totalTerms;
-            const termIdf = idf.get(term) ?? 0;
-            score += tf * termIdf;
-        }
-        return { ...chunk, score };
-    });
-    // Normalize to 0-1
-    const maxScore = Math.max(...scored.map((c) => c.score));
-    if (maxScore > 0) {
-        for (const chunk of scored) {
-            chunk.score = chunk.score / maxScore;
-        }
-    }
-    return scored;
-}
-// ─── chunkByRelevance ───────────────────────────────────────────────────────
-export function chunkByRelevance(content, query, options) {
-    const maxChunks = options?.maxChunks ?? DEFAULT_MAX_CHUNKS;
-    const minScore = options?.minScore ?? DEFAULT_MIN_SCORE;
-    const minLines = options?.minChunkLines ?? DEFAULT_MIN_LINES;
-    const maxLines = options?.maxChunkLines ?? DEFAULT_MAX_LINES;
-    const rawChunks = splitIntoChunks(content, { minLines, maxLines });
-    if (rawChunks.length === 0) {
-        return { chunks: [], totalChunks: 0, omittedChunks: 0, savingsPercent: 0 };
-    }
-    const scored = scoreChunks(rawChunks, query);
-    // Filter by minScore and take top maxChunks by score
-    const qualifying = scored
-        .filter((c) => c.score >= minScore)
-        .sort((a, b) => b.score - a.score)
-        .slice(0, maxChunks);
-    // Return in original document order (by startLine)
-    const selected = qualifying.sort((a, b) => a.startLine - b.startLine);
-    const totalChars = content.length;
-    const selectedChars = selected.reduce((sum, c) => sum + c.content.length, 0);
-    const savingsPercent = totalChars > 0
-        ? Math.round(((totalChars - selectedChars) / totalChars) * 100)
-        : 0;
-    return {
-        chunks: selected,
-        totalChunks: rawChunks.length,
-        omittedChunks: rawChunks.length - selected.length,
-        savingsPercent: Math.max(0, savingsPercent),
-    };
-}
-// ─── formatChunks ───────────────────────────────────────────────────────────
-export function formatChunks(result, filePath) {
-    if (result.chunks.length === 0) {
-        return `[${filePath}: empty or no relevant chunks]`;
-    }
-    const parts = [];
-    let lastEndLine = 0;
-    for (const chunk of result.chunks) {
-        // Show omission gap
-        if (lastEndLine > 0 && chunk.startLine > lastEndLine + 1) {
-            const gapLines = chunk.startLine - lastEndLine - 1;
-            parts.push(`[...${gapLines} lines omitted...]`);
-        }
-        parts.push(`[Lines ${chunk.startLine}-${chunk.endLine}]`);
-        parts.push(chunk.content);
-        lastEndLine = chunk.endLine;
-    }
-    return parts.join("\n");
-}