@noetic-tools/core 0.2.0 → 0.3.0

package/dist/memory/layers/file-reference.js

@@ -1,594 +0,0 @@
-import { Slot } from '../../types/memory.js';
-import { createMessage, estimateTokens } from '../../util/message-helpers.js';
-/** Default max file size: 1MB */
-const DEFAULT_MAX_FILE_SIZE = 1024 * 1024;
-/** Default allowed extensions */
-const DEFAULT_ALLOWED_EXTENSIONS = [
-    '.ts',
-    '.tsx',
-    '.js',
-    '.jsx',
-    '.mjs',
-    '.cjs',
-    '.json',
-    '.yaml',
-    '.yml',
-    '.toml',
-    '.md',
-    '.mdx',
-    '.txt',
-    '.csv',
-    '.html',
-    '.css',
-    '.scss',
-    '.less',
-    '.py',
-    '.rb',
-    '.go',
-    '.rs',
-    '.java',
-    '.kt',
-    '.swift',
-    '.c',
-    '.cpp',
-    '.h',
-    '.hpp',
-    '.sh',
-    '.bash',
-    '.zsh',
-    '.sql',
-    '.graphql',
-    '.xml',
-    '.svg',
-    '.env',
-    '.env.local',
-    '.env.example',
-    '.gitignore',
-    '.dockerignore',
-    'Dockerfile',
-    'Makefile',
-    'Taskfile',
-];
-//#endregion
-//#region Helpers
-// Pattern requires file-like structure: path with extension or explicit path separator
-// Avoids matching #hashtag, #123, #region, etc.
-const FILE_REF_PATTERN = /#((?:[a-zA-Z0-9_.-]+\/)*[a-zA-Z0-9_.-]+\.[a-zA-Z0-9]+)/g;
-function sanitizeAnchor(refPath) {
-    return refPath.replace(/[^a-zA-Z0-9-]/g, '-').toLowerCase();
-}
-function transformReferencesToAnchors(text) {
-    return text.replace(FILE_REF_PATTERN, (_match, filePath) => {
-        const anchor = sanitizeAnchor(filePath);
-        return `[#${filePath}](#${anchor})`;
-    });
-}
-function extractFileReferences(text) {
-    const refs = [];
-    const pattern = new RegExp(FILE_REF_PATTERN);
-    let match = pattern.exec(text);
-    while (match !== null) {
-        refs.push(match[1]);
-        match = pattern.exec(text);
-    }
-    return [
-        ...new Set(refs),
-    ]; // Dedupe
-}
-function normalizePath(input) {
-    const raw = input.length > 0 ? input : '.';
-    const absolute = raw.startsWith('/') ? raw : `/${raw}`;
-    const parts = [];
-    for (const part of absolute.split('/')) {
-        if (part.length === 0 || part === '.') {
-            continue;
-        }
-        if (part === '..') {
-            parts.pop();
-            continue;
-        }
-        parts.push(part);
-    }
-    return `/${parts.join('/')}`;
-}
-function resolvePath(base, ref = '') {
-    return normalizePath(ref.startsWith('/') ? ref : `${base}/${ref}`);
-}
-function pathBasename(input) {
-    const normalized = normalizePath(input);
-    if (normalized === '/') {
-        return '';
-    }
-    return normalized.slice(normalized.lastIndexOf('/') + 1);
-}
-function pathExtname(input) {
-    const name = pathBasename(input);
-    const idx = name.lastIndexOf('.');
-    return idx > 0 ? name.slice(idx) : '';
-}
-function isAbsolutePath(input) {
-    return input.startsWith('/');
-}
-function currentWorkingDirectory() {
-    return typeof process !== 'undefined' ? process.cwd() : '/';
-}
-function simpleHash(content) {
-    let hash = 0;
-    for (let i = 0; i < content.length; i++) {
-        const char = content.charCodeAt(i);
-        hash = (hash << 5) - hash + char;
-        hash = hash & hash; // Convert to 32bit integer
-    }
-    // Include content length to reduce collision probability
-    return `${content.length}:${hash.toString(16)}`;
-}
-function isInputTextPart(value) {
-    return (typeof value === 'object' &&
-        value !== null &&
-        'type' in value &&
-        value.type === 'input_text' &&
-        'text' in value &&
-        typeof value.text === 'string');
-}
-function isInputMessage(item) {
-    return (item.type === 'message' &&
-        'role' in item &&
-        'content' in item &&
-        Array.isArray(item.content) &&
-        item.content.every(isInputTextPart));
-}
-function extractTextFromItem(item) {
-    if (!isInputMessage(item)) {
-        return '';
-    }
-    return item.content.map((c) => c.text).join('\n');
-}
-async function readFileContent(absolutePath, opts, fsAdapter) {
-    try {
-        // Security: Validate path stays within baseDir (prevent path traversal)
-        const normalizedBase = resolvePath(opts.baseDir);
-        const normalizedPath = resolvePath(absolutePath);
-        if (!normalizedPath.startsWith(`${normalizedBase}/`) && normalizedPath !== normalizedBase) {
-            return {
-                content: null,
-                deleted: false,
-                error: 'PATH_TRAVERSAL: File outside allowed directory',
-            };
-        }
-        // Security: Check file extension
-        const ext = pathExtname(normalizedPath).toLowerCase();
-        const basename = pathBasename(normalizedPath);
-        const isAllowedExt = opts.allowedExtensions.some((allowed) => ext === allowed.toLowerCase() || basename === allowed);
-        if (!isAllowedExt && opts.allowedExtensions.length > 0) {
-            return {
-                content: null,
-                deleted: false,
-                error: `DISALLOWED_EXTENSION: ${ext || 'no extension'}`,
-            };
-        }
-        // Security: Check for symlinks if not allowed
-        const stats = await fsAdapter.lstat(absolutePath);
-        if (stats.isSymbolicLink() && !opts.followSymlinks) {
-            return {
-                content: null,
-                deleted: false,
-                error: 'SYMLINK: Symlinks not allowed',
-            };
-        }
-        // Security: Check file size
-        const fileStats = opts.followSymlinks ? await fsAdapter.stat(absolutePath) : stats;
-        if (fileStats.size > opts.maxFileSize) {
-            return {
-                content: null,
-                deleted: false,
-                error: `FILE_TOO_LARGE: ${fileStats.size} bytes exceeds limit of ${opts.maxFileSize}`,
-            };
-        }
-        const content = await fsAdapter.readFileText(absolutePath);
-        return {
-            content,
-            deleted: false,
-        };
-    }
-    catch (e) {
-        if (isNodeError(e) && e.code === 'ENOENT') {
-            return {
-                content: null,
-                deleted: true,
-            };
-        }
-        if (isNodeError(e) && e.code === 'EACCES') {
-            return {
-                content: null,
-                deleted: false,
-                error: 'PERMISSION_DENIED: Cannot read file',
-            };
-        }
-        throw e;
-    }
-}
-function isNodeError(e) {
-    return e instanceof Error && 'code' in e;
-}
-function extractResponseText(items) {
-    const texts = [];
-    for (const item of items) {
-        if (item.type !== 'message') {
-            continue;
-        }
-        if (!('content' in item) || !Array.isArray(item.content)) {
-            continue;
-        }
-        for (const part of item.content) {
-            if ('text' in part && typeof part.text === 'string') {
-                texts.push(part.text);
-            }
-        }
-    }
-    return texts.join('').trim();
-}
-async function scoreFileRelevance(params) {
-    const { filePath, fileContent, userQuery, ctx, model } = params;
-    if (!ctx.callModel) {
-        const queryLower = userQuery.toLowerCase();
-        const pathLower = filePath.toLowerCase();
-        if (pathLower.includes(queryLower) || queryLower.includes(pathBasename(pathLower))) {
-            return 80;
-        }
-        return 50;
-    }
-    const scoringPrompt = `Rate the relevance of this file to the user's query.
-
-User Query: ${userQuery}
-
-File Path: ${filePath}
-File Content (first 2000 chars):
-${fileContent.slice(0, 2e3)}
-
-Respond with ONLY a number from 0-100 where:
-- 0-20: Not relevant at all
-- 21-40: Slightly relevant
-- 41-60: Moderately relevant
-- 61-80: Highly relevant
-- 81-100: Critical/essential
-
-Score:`;
-    try {
-        const response = await ctx.callModel({
-            model,
-            items: [
-                createMessage(scoringPrompt, 'user'),
-            ],
-            instructions: 'You are a relevance scorer. Respond with only a number 0-100.',
-        });
-        const responseText = extractResponseText(response.items);
-        const score = Number.parseInt(responseText, 10);
-        if (Number.isNaN(score) || score < 0 || score > 100) {
-            return 50;
-        }
-        return score;
-    }
-    catch {
-        return 50;
-    }
-}
-function createFileReferenceRuntime(opts) {
-    const baseDir = opts?.baseDir ?? currentWorkingDirectory();
-    const maxFileSize = opts?.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
-    const followSymlinks = opts?.followSymlinks ?? false;
-    const allowedExtensions = opts?.allowedExtensions ?? DEFAULT_ALLOWED_EXTENSIONS;
-    return {
-        baseDir,
-        slot: opts?.slot ?? Slot.RAG,
-        scoringModel: opts?.scoringModel ?? 'anthropic/claude-haiku-4-5-20251001',
-        readOpts: {
-            maxFileSize,
-            followSymlinks,
-            baseDir,
-            allowedExtensions,
-        },
-    };
-}
-async function initFileReferenceState(baseDir) {
-    return {
-        state: {
-            files: new Map(),
-            baseDir,
-        },
-    };
-}
-function absolutePathReference(ref) {
-    return {
-        absolutePath: ref,
-        referencePath: ref,
-        content: null,
-        priority: 0,
-        contentHash: '',
-        deleted: false,
-        tokenCount: 0,
-        error: 'ABSOLUTE_PATH: Only relative paths are allowed',
-    };
-}
-async function trackNewReference(args) {
-    if (isAbsolutePath(args.ref)) {
-        return absolutePathReference(args.ref);
-    }
-    const absolutePath = resolvePath(args.state.baseDir, args.ref);
-    const result = await readFileContent(absolutePath, args.readOpts, args.ctx.fs);
-    const priority = result.content
-        ? await scoreFileRelevance({
-            filePath: args.ref,
-            fileContent: result.content,
-            userQuery: args.userQuery,
-            ctx: args.ctx,
-            model: args.scoringModel,
-        })
-        : 0;
-    return {
-        absolutePath,
-        referencePath: args.ref,
-        content: result.content,
-        priority,
-        contentHash: result.content ? simpleHash(result.content) : '',
-        deleted: result.deleted,
-        tokenCount: result.content ? estimateTokens(result.content) : 0,
-        error: result.error,
-    };
-}
-async function addNewReferences(args) {
-    let hasChanges = false;
-    for (const ref of args.refs) {
-        if (args.files.has(ref)) {
-            continue;
-        }
-        args.files.set(ref, await trackNewReference({
-            ...args,
-            ref,
-        }));
-        hasChanges = true;
-    }
-    return hasChanges;
-}
-function transformItemReferences(item, refs) {
-    if (refs.length === 0 || !isInputMessage(item)) {
-        return item;
-    }
-    const transformedContent = item.content.map((c) => ({
-        type: 'input_text',
-        text: transformReferencesToAnchors(c.text),
-    }));
-    return {
-        ...item,
-        content: transformedContent,
-    };
-}
-async function processAppendedItems(args) {
-    const transformedItems = [];
-    let hasChanges = false;
-    let userQuery = '';
-    for (const item of args.items) {
-        const text = extractTextFromItem(item);
-        if (!text) {
-            transformedItems.push(item);
-            continue;
-        }
-        if (item.type === 'message' && 'role' in item && item.role === 'user') {
-            userQuery = text;
-        }
-        const refs = extractFileReferences(text);
-        const added = await addNewReferences({
-            refs,
-            files: args.files,
-            state: args.state,
-            readOpts: args.readOpts,
-            ctx: args.ctx,
-            userQuery,
-            scoringModel: args.scoringModel,
-        });
-        hasChanges = hasChanges || added;
-        transformedItems.push(transformItemReferences(item, refs));
-    }
-    return {
-        transformedItems,
-        hasChanges,
-        userQuery,
-    };
-}
-async function refreshTrackedFile(args) {
-    if (args.tracked.error) {
-        return null;
-    }
-    const result = await readFileContent(args.tracked.absolutePath, args.readOpts, args.ctx.fs);
-    if (result.deleted && !args.tracked.deleted) {
-        return {
-            ...args.tracked,
-            content: null,
-            deleted: true,
-        };
-    }
-    if (result.error && !args.tracked.error) {
-        return {
-            ...args.tracked,
-            content: null,
-            error: result.error,
-        };
-    }
-    if (result.deleted || !result.content) {
-        return null;
-    }
-    const newHash = simpleHash(result.content);
-    if (newHash === args.tracked.contentHash) {
-        return null;
-    }
-    const priority = args.userQuery
-        ? await scoreFileRelevance({
-            filePath: args.ref,
-            fileContent: result.content,
-            userQuery: args.userQuery,
-            ctx: args.ctx,
-            model: args.scoringModel,
-        })
-        : args.tracked.priority;
-    return {
-        ...args.tracked,
-        content: result.content,
-        contentHash: newHash,
-        deleted: false,
-        tokenCount: estimateTokens(result.content),
-        priority,
-        error: undefined,
-    };
-}
-async function refreshTrackedFiles(args) {
-    let hasChanges = false;
-    for (const [ref, tracked] of args.files) {
-        const next = await refreshTrackedFile({
-            ...args,
-            ref,
-            tracked,
-        });
-        if (next === null) {
-            continue;
-        }
-        args.files.set(ref, next);
-        hasChanges = true;
-    }
-    return hasChanges;
-}
-async function onFileReferenceItemAppend(args, runtime) {
-    const newFiles = new Map(args.state.files);
-    const currentReadOpts = {
-        ...runtime.readOpts,
-        baseDir: args.state.baseDir,
-    };
-    const processed = await processAppendedItems({
-        items: args.items,
-        files: newFiles,
-        state: args.state,
-        readOpts: currentReadOpts,
-        ctx: args.ctx,
-        scoringModel: runtime.scoringModel,
-    });
-    const refreshed = await refreshTrackedFiles({
-        files: newFiles,
-        readOpts: currentReadOpts,
-        ctx: args.ctx,
-        userQuery: processed.userQuery,
-        scoringModel: runtime.scoringModel,
-    });
-    return {
-        items: processed.transformedItems,
-        state: {
-            ...args.state,
-            files: newFiles,
-        },
-        rerender: processed.hasChanges || refreshed,
-        timing: 'immediate',
-        scope: 'self',
-    };
-}
-function buildDeletedOrErrorBlock(file) {
-    if (file.deleted) {
-        return `## ${file.referencePath}\n\n[FILE DELETED: ${file.absolutePath}]`;
-    }
-    if (file.error) {
-        return `## ${file.referencePath}\n\n[ERROR: ${file.error}]`;
-    }
-    return null;
-}
-function truncateFileContent(file, remainingBudget) {
-    if (!file.content || file.tokenCount <= remainingBudget) {
-        return file.content ?? '';
-    }
-    const lines = file.content.split('\n');
-    const headLines = Math.floor(lines.length * 0.6);
-    const tailLines = Math.floor(lines.length * 0.3);
-    const head = lines.slice(0, headLines).join('\n');
-    const tail = lines.slice(-tailLines).join('\n');
-    return `${head}\n\n... [truncated ${lines.length - headLines - tailLines} lines] ...\n\n${tail}`;
-}
-function buildContentBlock(file, totalTokens, budget) {
-    const statusBlock = buildDeletedOrErrorBlock(file);
-    if (statusBlock !== null) {
-        return totalTokens + estimateTokens(statusBlock) <= budget ? statusBlock : null;
-    }
-    if (!file.content) {
-        return null;
-    }
-    const header = `## ${file.referencePath}`;
-    const headerTokens = estimateTokens(header);
-    if (totalTokens + headerTokens >= budget) {
-        return null;
-    }
-    const remainingBudget = budget - totalTokens - headerTokens - 10;
-    const contentToInclude = truncateFileContent(file, remainingBudget);
-    return `${header}\n\n\`\`\`\n${contentToInclude}\n\`\`\``;
-}
-async function recallFileReferences({ state, budget, }) {
-    if (state.files.size === 0) {
-        return {
-            items: [],
-            tokenCount: 0,
-        };
-    }
-    const sortedFiles = [
-        ...state.files.values(),
-    ].sort((a, b) => b.priority - a.priority);
-    const blocks = [];
-    let totalTokens = 0;
-    for (const file of sortedFiles) {
-        const block = buildContentBlock(file, totalTokens, budget);
-        if (block === null) {
-            continue;
-        }
-        const blockTokens = estimateTokens(block);
-        blocks.push(block);
-        totalTokens += blockTokens;
-    }
-    if (blocks.length === 0) {
-        return {
-            items: [],
-            tokenCount: 0,
-        };
-    }
-    const content = `# Referenced Files\n\n${blocks.join('\n\n')}`;
-    return {
-        items: [
-            createMessage(content, 'developer'),
-        ],
-        tokenCount: estimateTokens(content),
-    };
-}
-//#endregion
-//#region Layer Factory
-/**
- * Creates a memory layer that tracks file references in user messages.
- *
- * Syntax: `#path/to/file` in user messages
- *
- * Behavior:
- * - Transforms references to markdown anchor links `[#path/to/file](#path-to-file)`
- * - Scores file relevance using LLM when first referenced
- * - Injects file contents into context via recall(), ordered by priority
- * - Detects file changes on each new message, triggers immediate re-render
- * - Shows warning for deleted files
- *
- * @public
- */
-export function fileReference(opts) {
-    const runtime = createFileReferenceRuntime(opts);
-    return {
-        id: 'file-reference',
-        name: 'File Reference',
-        slot: runtime.slot,
-        scope: 'thread',
-        budget: 'auto',
-        rerenderTiming: 'immediate',
-        hooks: {
-            init: () => initFileReferenceState(runtime.baseDir),
-            onItemAppend: (args) => onFileReferenceItemAppend(args, runtime),
-            recall: recallFileReferences,
-        },
-    };
-}
-//#endregion

package/dist/memory/layers/history-window.d.ts

@@ -1,29 +0,0 @@
-/**
- * History-window memory layer — caps the number of items projected to the LLM
- * without touching `itemLog` storage. Pure read-side projection via the
- * `projectHistory` hook.
- *
- * Algorithm per turn:
- *   1. Slice the last `maxItems`.
- *   2. If the slice lacks a user `message` or an assistant `message`, expand
- *      backward until both are present. The cap may be exceeded — for
- *      tool-burst histories the projected length can grow well beyond
- *      `maxItems` because the most recent role-pair may be far back.
- *   3. Drop any orphan `function_call` / `function_call_output` left at the
- *      slice boundary via `stripUnresolvedToolCalls`.
- */
-import type { MemoryLayer } from '../../types/memory.js';
-/** @public Configuration for the {@link historyWindow} layer. */
-export interface HistoryWindowConfig {
-    /** Cap on the number of trailing items projected to the LLM. Defaults to 40. */
-    maxItems?: number;
-}
-/**
- * Creates a history-window memory layer that caps the trailing items projected
- * to the LLM. Storage stays unbounded; only the wire payload is narrowed.
- *
- * @public
- * @param config - Optional `{ maxItems }`. Defaults to 40.
- * @returns A `MemoryLayer` that contributes only a `projectHistory` hook.
- */
-export declare function historyWindow(config?: HistoryWindowConfig): MemoryLayer<null>;