@noetic-tools/context 1.0.0

package/dist/context/layers/file-reference.js

@@ -0,0 +1,729 @@
+import { createMessage, estimateTokens, Slot } from '@noetic-tools/types';
+/** Default max file size: 1MB */
+const DEFAULT_MAX_FILE_SIZE = 1024 * 1024;
+/** Default allowed extensions */
+const DEFAULT_ALLOWED_EXTENSIONS = [
+    '.ts',
+    '.tsx',
+    '.js',
+    '.jsx',
+    '.mjs',
+    '.cjs',
+    '.json',
+    '.yaml',
+    '.yml',
+    '.toml',
+    '.md',
+    '.mdx',
+    '.txt',
+    '.csv',
+    '.html',
+    '.css',
+    '.scss',
+    '.less',
+    '.py',
+    '.rb',
+    '.go',
+    '.rs',
+    '.java',
+    '.kt',
+    '.swift',
+    '.c',
+    '.cpp',
+    '.h',
+    '.hpp',
+    '.sh',
+    '.bash',
+    '.zsh',
+    '.sql',
+    '.graphql',
+    '.xml',
+    '.svg',
+    '.env',
+    '.env.local',
+    '.env.example',
+    '.gitignore',
+    '.dockerignore',
+    'Dockerfile',
+    'Makefile',
+    'Taskfile',
+];
+//#endregion
+//#region Helpers
+// Pattern requires file-like structure: path with extension or explicit path separator
+// Avoids matching #hashtag, #123, #region, etc.
+const FILE_REF_PATTERN = /#((?:[a-zA-Z0-9_.-]+\/)*[a-zA-Z0-9_.-]+\.[a-zA-Z0-9]+)/g;
+function sanitizeAnchor(refPath) {
+    return refPath.replace(/[^a-zA-Z0-9-]/g, '-').toLowerCase();
+}
+function transformReferencesToAnchors(text) {
+    return text.replace(FILE_REF_PATTERN, (_match, filePath) => {
+        const anchor = sanitizeAnchor(filePath);
+        return `[#${filePath}](#${anchor})`;
+    });
+}
+function extractFileReferences(text) {
+    const refs = [];
+    const pattern = new RegExp(FILE_REF_PATTERN);
+    let match = pattern.exec(text);
+    while (match !== null) {
+        refs.push(match[1]);
+        match = pattern.exec(text);
+    }
+    return [
+        ...new Set(refs),
+    ]; // Dedupe
+}
+function normalizePath(input) {
+    const raw = input.length > 0 ? input : '.';
+    const absolute = raw.startsWith('/') ? raw : `/${raw}`;
+    const parts = [];
+    for (const part of absolute.split('/')) {
+        if (part.length === 0 || part === '.') {
+            continue;
+        }
+        if (part === '..') {
+            parts.pop();
+            continue;
+        }
+        parts.push(part);
+    }
+    return `/${parts.join('/')}`;
+}
+function resolvePath(base, ref = '') {
+    return normalizePath(ref.startsWith('/') ? ref : `${base}/${ref}`);
+}
+/** Resolves a raw reference to the absolute path used for dedup/reads. */
+function resolveRefPath(baseDir, ref) {
+    return isAbsolutePath(ref) ? ref : resolvePath(baseDir, ref);
+}
+function pathBasename(input) {
+    const normalized = normalizePath(input);
+    if (normalized === '/') {
+        return '';
+    }
+    return normalized.slice(normalized.lastIndexOf('/') + 1);
+}
+function pathExtname(input) {
+    const name = pathBasename(input);
+    const idx = name.lastIndexOf('.');
+    return idx > 0 ? name.slice(idx) : '';
+}
+function isAbsolutePath(input) {
+    return input.startsWith('/');
+}
+function currentWorkingDirectory() {
+    return typeof process !== 'undefined' ? process.cwd() : '/';
+}
+function simpleHash(content) {
+    let hash = 0;
+    for (let i = 0; i < content.length; i++) {
+        const char = content.charCodeAt(i);
+        hash = (hash << 5) - hash + char;
+        hash = hash & hash; // Convert to 32bit integer
+    }
+    // Include content length to reduce collision probability
+    return `${content.length}:${hash.toString(16)}`;
+}
+function isInputTextPart(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        'type' in value &&
+        value.type === 'input_text' &&
+        'text' in value &&
+        typeof value.text === 'string');
+}
+function isInputMessage(item) {
+    return (item.type === 'message' &&
+        'role' in item &&
+        'content' in item &&
+        Array.isArray(item.content) &&
+        item.content.every(isInputTextPart));
+}
+function extractTextFromItem(item) {
+    if (!isInputMessage(item)) {
+        return '';
+    }
+    return item.content.map((c) => c.text).join('\n');
+}
+/**
+ * lstat every path component of `target` strictly BELOW `base`; any symlink
+ * component (directory or leaf) is rejected. Lexical containment alone is not
+ * enough — `base/link → /etc` keeps the unresolved string inside `base` while
+ * the OS resolves the intermediate symlink, escaping the sandbox. `base`
+ * itself and its ancestors are deliberately never lstat'ed (macOS `/tmp` is a
+ * symlink). Uses the existing `FsAdapter.lstat` (no realpath — adding it
+ * would fan out to every adapter). The leaf's stats are returned so the
+ * caller can reuse them for the size check.
+ */
+async function walkRejectingSymlinks(base, target, fsAdapter) {
+    const relative = target === base ? '' : target.slice(base.length + 1);
+    const segments = relative.length > 0 ? relative.split('/') : [];
+    if (segments.length === 0) {
+        // Degenerate case (target === base): the base is trusted and never
+        // symlink-checked, so resolve through it with stat for the size check.
+        return {
+            leafStats: await fsAdapter.stat(target),
+        };
+    }
+    let current = base;
+    let leafStats;
+    for (const segment of segments) {
+        current = `${current}/${segment}`;
+        const stats = await fsAdapter.lstat(current);
+        if (stats.isSymbolicLink()) {
+            return {
+                error: 'SYMLINK: Symlinks not allowed',
+            };
+        }
+        leafStats = stats;
+    }
+    // `segments.length > 0` guarantees the loop assigned leafStats.
+    if (!leafStats) {
+        return {
+            error: 'SYMLINK: Symlinks not allowed',
+        };
+    }
+    return {
+        leafStats,
+    };
+}
+async function readFileContent(absolutePath, opts, fsAdapter) {
+    try {
+        // Security: Validate path stays within baseDir (prevent path traversal)
+        const normalizedBase = resolvePath(opts.baseDir);
+        const normalizedPath = resolvePath(absolutePath);
+        if (!normalizedPath.startsWith(`${normalizedBase}/`) && normalizedPath !== normalizedBase) {
+            return {
+                content: null,
+                deleted: false,
+                error: 'PATH_TRAVERSAL: File outside allowed directory',
+            };
+        }
+        // Security: Check file extension
+        const ext = pathExtname(normalizedPath).toLowerCase();
+        const basename = pathBasename(normalizedPath);
+        const isAllowedExt = opts.allowedExtensions.some((allowed) => ext === allowed.toLowerCase() || basename === allowed);
+        if (!isAllowedExt && opts.allowedExtensions.length > 0) {
+            return {
+                content: null,
+                deleted: false,
+                error: `DISALLOWED_EXTENSION: ${ext || 'no extension'}`,
+            };
+        }
+        // Security: Reject symlinked path components (directory or leaf) unless
+        // the followSymlinks opt-out is enabled. The lexical containment check
+        // above is not enough on its own — `baseDir/link → /etc` keeps the
+        // unresolved string inside baseDir while the OS resolves the intermediate
+        // symlink, escaping the sandbox. The walk's leaf stats double as the
+        // size-check stats.
+        let fileStats;
+        if (opts.followSymlinks) {
+            fileStats = await fsAdapter.stat(absolutePath);
+        }
+        else {
+            const walk = await walkRejectingSymlinks(normalizedBase, normalizedPath, fsAdapter);
+            if (walk.error !== undefined) {
+                return {
+                    content: null,
+                    deleted: false,
+                    error: walk.error,
+                };
+            }
+            fileStats = walk.leafStats;
+        }
+        // Security: Check file size
+        if (fileStats.size > opts.maxFileSize) {
+            return {
+                content: null,
+                deleted: false,
+                error: `FILE_TOO_LARGE: ${fileStats.size} bytes exceeds limit of ${opts.maxFileSize}`,
+            };
+        }
+        const content = await fsAdapter.readFileText(absolutePath);
+        return {
+            content,
+            deleted: false,
+        };
+    }
+    catch (e) {
+        if (isNodeError(e) && e.code === 'ENOENT') {
+            return {
+                content: null,
+                deleted: true,
+            };
+        }
+        if (isNodeError(e) && e.code === 'EACCES') {
+            return {
+                content: null,
+                deleted: false,
+                error: 'PERMISSION_DENIED: Cannot read file',
+            };
+        }
+        throw e;
+    }
+}
+function isNodeError(e) {
+    return e instanceof Error && 'code' in e;
+}
+function extractResponseText(items) {
+    const texts = [];
+    for (const item of items) {
+        if (item.type !== 'message') {
+            continue;
+        }
+        if (!('content' in item) || !Array.isArray(item.content)) {
+            continue;
+        }
+        for (const part of item.content) {
+            if ('text' in part && typeof part.text === 'string') {
+                texts.push(part.text);
+            }
+        }
+    }
+    return texts.join('').trim();
+}
+async function scoreFileRelevance(params) {
+    const { filePath, fileContent, userQuery, ctx, model } = params;
+    if (!ctx.callModel) {
+        const queryLower = userQuery.toLowerCase();
+        const pathLower = filePath.toLowerCase();
+        if (pathLower.includes(queryLower) || queryLower.includes(pathBasename(pathLower))) {
+            return 80;
+        }
+        return 50;
+    }
+    const scoringPrompt = `Rate the relevance of this file to the user's query.
+
+User Query: ${userQuery}
+
+File Path: ${filePath}
+File Content (first 2000 chars):
+${fileContent.slice(0, 2e3)}
+
+Respond with ONLY a number from 0-100 where:
+- 0-20: Not relevant at all
+- 21-40: Slightly relevant
+- 41-60: Moderately relevant
+- 61-80: Highly relevant
+- 81-100: Critical/essential
+
+Score:`;
+    try {
+        const response = await ctx.callModel({
+            model,
+            items: [
+                createMessage(scoringPrompt, 'user'),
+            ],
+            instructions: 'You are a relevance scorer. Respond with only a number 0-100.',
+        });
+        const responseText = extractResponseText(response.items);
+        const score = Number.parseInt(responseText, 10);
+        if (Number.isNaN(score) || score < 0 || score > 100) {
+            return 50;
+        }
+        return score;
+    }
+    catch {
+        return 50;
+    }
+}
+function createFileReferenceRuntime(opts) {
+    const baseDir = opts?.baseDir ?? currentWorkingDirectory();
+    const maxFileSize = opts?.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
+    const followSymlinks = opts?.followSymlinks ?? false;
+    const allowedExtensions = opts?.allowedExtensions ?? DEFAULT_ALLOWED_EXTENSIONS;
+    return {
+        baseDir,
+        slot: opts?.slot ?? Slot.RAG,
+        scoringModel: opts?.scoringModel ?? 'anthropic/claude-haiku-4-5-20251001',
+        readOpts: {
+            maxFileSize,
+            followSymlinks,
+            baseDir,
+            allowedExtensions,
+        },
+    };
+}
+async function initFileReferenceState(baseDir) {
+    return {
+        state: {
+            files: new Map(),
+            baseDir,
+        },
+    };
+}
+function absolutePathReference(ref) {
+    return {
+        absolutePath: ref,
+        referencePath: ref,
+        content: null,
+        priority: 0,
+        contentHash: '',
+        deleted: false,
+        tokenCount: 0,
+        error: 'ABSOLUTE_PATH: Only relative paths are allowed',
+    };
+}
+async function trackNewReference(args) {
+    if (isAbsolutePath(args.ref)) {
+        return absolutePathReference(args.ref);
+    }
+    const absolutePath = resolvePath(args.state.baseDir, args.ref);
+    const result = await readFileContent(absolutePath, args.readOpts, args.ctx.fs);
+    const priority = result.content
+        ? await scoreFileRelevance({
+            filePath: args.ref,
+            fileContent: result.content,
+            userQuery: args.userQuery,
+            ctx: args.ctx,
+            model: args.scoringModel,
+        })
+        : 0;
+    return {
+        absolutePath,
+        referencePath: args.ref,
+        content: result.content,
+        priority,
+        contentHash: result.content ? simpleHash(result.content) : '',
+        deleted: result.deleted,
+        tokenCount: result.content ? estimateTokens(result.content) : 0,
+        error: result.error,
+    };
+}
+async function addNewReferences(args) {
+    // Dedupe by RESOLVED absolute path so `#config.json` and `#./config.json`
+    // (which point at the same file) only get tracked once.
+    const seenPaths = new Set();
+    for (const tracked of args.files.values()) {
+        seenPaths.add(tracked.absolutePath);
+    }
+    const newRefs = [];
+    for (const ref of args.refs) {
+        if (args.files.has(ref)) {
+            continue;
+        }
+        const absolutePath = resolveRefPath(args.state.baseDir, ref);
+        if (seenPaths.has(absolutePath)) {
+            continue;
+        }
+        seenPaths.add(absolutePath);
+        newRefs.push(ref);
+    }
+    if (newRefs.length === 0) {
+        return false;
+    }
+    // Read + score every new reference in PARALLEL — k sequential LLM scoring
+    // calls would multiply wall time and blow the onItemAppend timeout. Results
+    // are inserted in deterministic ref order.
+    const tracked = await Promise.all(newRefs.map((ref) => trackNewReference({
+        ...args,
+        ref,
+    })));
+    for (let i = 0; i < newRefs.length; i++) {
+        args.files.set(newRefs[i], tracked[i]);
+    }
+    return true;
+}
+function transformItemReferences(item, refs) {
+    if (refs.length === 0 || !isInputMessage(item)) {
+        return item;
+    }
+    const transformedContent = item.content.map((c) => ({
+        type: 'input_text',
+        text: transformReferencesToAnchors(c.text),
+    }));
+    return {
+        ...item,
+        content: transformedContent,
+    };
+}
+async function processAppendedItems(args) {
+    const transformedItems = [];
+    let hasChanges = false;
+    let userQuery = '';
+    for (const item of args.items) {
+        const text = extractTextFromItem(item);
+        if (!text) {
+            transformedItems.push(item);
+            continue;
+        }
+        if (item.type === 'message' && 'role' in item && item.role === 'user') {
+            userQuery = text;
+        }
+        const refs = extractFileReferences(text);
+        const added = await addNewReferences({
+            refs,
+            files: args.files,
+            state: args.state,
+            readOpts: args.readOpts,
+            ctx: args.ctx,
+            userQuery,
+            scoringModel: args.scoringModel,
+        });
+        hasChanges = hasChanges || added;
+        transformedItems.push(transformItemReferences(item, refs));
+    }
+    return {
+        transformedItems,
+        hasChanges,
+        userQuery,
+    };
+}
+async function refreshTrackedFile(args) {
+    // NB: a previously-errored file is retried (no early bail) so it can recover
+    // once the underlying condition (too large / permission) is resolved.
+    const result = await readFileContent(args.tracked.absolutePath, args.readOpts, args.ctx.fs);
+    if (result.deleted && !args.tracked.deleted) {
+        return {
+            ...args.tracked,
+            content: null,
+            deleted: true,
+        };
+    }
+    if (result.error && !args.tracked.error) {
+        return {
+            ...args.tracked,
+            content: null,
+            error: result.error,
+        };
+    }
+    if (result.deleted || !result.content) {
+        return null;
+    }
+    const newHash = simpleHash(result.content);
+    if (newHash === args.tracked.contentHash) {
+        return null;
+    }
+    const priority = args.userQuery
+        ? await scoreFileRelevance({
+            filePath: args.ref,
+            fileContent: result.content,
+            userQuery: args.userQuery,
+            ctx: args.ctx,
+            model: args.scoringModel,
+        })
+        : args.tracked.priority;
+    return {
+        ...args.tracked,
+        content: result.content,
+        contentHash: newHash,
+        deleted: false,
+        tokenCount: estimateTokens(result.content),
+        priority,
+        error: undefined,
+    };
+}
+async function refreshTrackedFiles(args) {
+    // Re-read (and possibly re-score) every tracked file in PARALLEL — see
+    // addNewReferences. Applied in deterministic map-iteration order.
+    const entries = [
+        ...args.files.entries(),
+    ];
+    const refreshed = await Promise.all(entries.map(([ref, tracked]) => refreshTrackedFile({
+        ...args,
+        ref,
+        tracked,
+    })));
+    let hasChanges = false;
+    for (let i = 0; i < entries.length; i++) {
+        const next = refreshed[i];
+        if (next === null) {
+            continue;
+        }
+        args.files.set(entries[i][0], next);
+        hasChanges = true;
+    }
+    return hasChanges;
+}
+async function onFileReferenceItemAppend(args, runtime) {
+    const newFiles = new Map(args.state.files);
+    const currentReadOpts = {
+        ...runtime.readOpts,
+        baseDir: args.state.baseDir,
+    };
+    const processed = await processAppendedItems({
+        items: args.items,
+        files: newFiles,
+        state: args.state,
+        readOpts: currentReadOpts,
+        ctx: args.ctx,
+        scoringModel: runtime.scoringModel,
+    });
+    const refreshed = await refreshTrackedFiles({
+        files: newFiles,
+        readOpts: currentReadOpts,
+        ctx: args.ctx,
+        userQuery: processed.userQuery,
+        scoringModel: runtime.scoringModel,
+    });
+    return {
+        items: processed.transformedItems,
+        state: {
+            ...args.state,
+            files: newFiles,
+        },
+        rerender: processed.hasChanges || refreshed,
+        timing: 'immediate',
+        scope: 'self',
+    };
+}
+function buildDeletedOrErrorBlock(file) {
+    if (file.deleted) {
+        return `## ${file.referencePath}\n\n[FILE DELETED: ${file.absolutePath}]`;
+    }
+    if (file.error) {
+        return `## ${file.referencePath}\n\n[ERROR: ${file.error}]`;
+    }
+    return null;
+}
+function truncationMarker(omittedLines) {
+    return `\n\n... [truncated ${omittedLines} lines] ...\n\n`;
+}
+/** Number of leading lines whose joined length fits within `charBudget`. */
+function fitLeadingLines(lines, start, charBudget) {
+    let count = 0;
+    let used = 0;
+    for (let i = start; i < lines.length; i++) {
+        const next = used === 0 ? lines[i].length : lines[i].length + 1;
+        if (used + next > charBudget) {
+            break;
+        }
+        used += next;
+        count++;
+    }
+    return count;
+}
+/** Number of trailing lines (not before `floor`) whose joined length fits. */
+function fitTrailingLines(lines, floor, charBudget) {
+    let count = 0;
+    let used = 0;
+    for (let i = lines.length - 1; i >= floor; i--) {
+        const next = used === 0 ? lines[i].length : lines[i].length + 1;
+        if (used + next > charBudget) {
+            break;
+        }
+        used += next;
+        count++;
+    }
+    return count;
+}
+// Truncates content so the rendered text honors `remainingBudget` tokens
+// (estimateTokens ≈ chars / 4). The previous head(60%)+tail(30%) line split
+// ignored the budget and kept ~90% of any file, blowing the budget wide open.
+function truncateFileContent(file, remainingBudget) {
+    if (!file.content || file.tokenCount <= remainingBudget) {
+        return file.content ?? '';
+    }
+    const lines = file.content.split('\n');
+    const totalChars = Math.max(0, remainingBudget * 4);
+    // Reserve room for the marker (sized for the worst case of all lines omitted).
+    const contentChars = Math.max(0, totalChars - truncationMarker(lines.length).length);
+    const headBudget = Math.floor(contentChars * 0.6);
+    const headCount = fitLeadingLines(lines, 0, headBudget);
+    const tailBudget = contentChars - lines.slice(0, headCount).join('\n').length;
+    const tailCount = fitTrailingLines(lines, headCount, tailBudget);
+    const omitted = lines.length - headCount - tailCount;
+    const head = lines.slice(0, headCount).join('\n');
+    const tail = tailCount > 0 ? lines.slice(lines.length - tailCount).join('\n') : '';
+    if (omitted <= 0) {
+        return [
+            head,
+            tail,
+        ]
+            .filter((s) => s.length > 0)
+            .join('\n');
+    }
+    return `${head}${truncationMarker(omitted)}${tail}`;
+}
+function buildContentBlock(file, totalTokens, budget) {
+    const statusBlock = buildDeletedOrErrorBlock(file);
+    if (statusBlock !== null) {
+        return totalTokens + estimateTokens(statusBlock) <= budget ? statusBlock : null;
+    }
+    if (!file.content) {
+        return null;
+    }
+    const header = `## ${file.referencePath}`;
+    const headerTokens = estimateTokens(header);
+    if (totalTokens + headerTokens >= budget) {
+        return null;
+    }
+    const remainingBudget = budget - totalTokens - headerTokens - 10;
+    const contentToInclude = truncateFileContent(file, remainingBudget);
+    return `${header}\n\n\`\`\`\n${contentToInclude}\n\`\`\``;
+}
+async function recallFileReferences({ state, budget, }) {
+    if (state.files.size === 0) {
+        return {
+            items: [],
+            tokenCount: 0,
+        };
+    }
+    const sortedFiles = [
+        ...state.files.values(),
+    ].sort((a, b) => b.priority - a.priority);
+    const blocks = [];
+    let totalTokens = 0;
+    for (const file of sortedFiles) {
+        const block = buildContentBlock(file, totalTokens, budget);
+        if (block === null) {
+            continue;
+        }
+        const blockTokens = estimateTokens(block);
+        blocks.push(block);
+        totalTokens += blockTokens;
+    }
+    if (blocks.length === 0) {
+        return {
+            items: [],
+            tokenCount: 0,
+        };
+    }
+    const content = `# Referenced Files\n\n${blocks.join('\n\n')}`;
+    return {
+        items: [
+            createMessage(content, 'developer'),
+        ],
+        tokenCount: estimateTokens(content),
+    };
+}
+//#endregion
+//#region Layer Factory
+/**
+ * Creates a context layer that tracks file references in user messages.
+ *
+ * Syntax: `#path/to/file` in user messages
+ *
+ * Behavior:
+ * - Transforms references to markdown anchor links `[#path/to/file](#path-to-file)`
+ * - Scores file relevance using LLM when first referenced
+ * - Injects file contents into context via recall(), ordered by priority
+ * - Detects file changes on each new message, triggers immediate re-render
+ * - Shows warning for deleted files
+ *
+ * @public
+ */
+export function fileReference(opts) {
+    const runtime = createFileReferenceRuntime(opts);
+    return {
+        id: 'file-reference',
+        name: 'File Reference',
+        slot: runtime.slot,
+        scope: 'thread',
+        budget: 'auto',
+        rerenderTiming: 'immediate',
+        timeouts: {
+            // onItemAppend reads files AND runs an LLM scoring call per new
+            // reference (parallelized) — the 5s pipeline default silently drops the
+            // transform + state update on timeout, killing the layer's feature.
+            onItemAppend: 30_000,
+        },
+        hooks: {
+            init: () => initFileReferenceState(runtime.baseDir),
+            onItemAppend: (args) => onFileReferenceItemAppend(args, runtime),
+            recall: recallFileReferences,
+        },
+    };
+}
+//#endregion