@noetic-tools/core 0.0.1

package/dist/memory/layers/file-reference.js

@@ -0,0 +1,538 @@
+import { createMessage, estimateTokens } from '../../interpreter/message-helpers';
+import { Slot } from '../../types/memory';
+/** Default max file size: 1MB */
+const DEFAULT_MAX_FILE_SIZE = 1024 * 1024;
+/** Default allowed extensions */
+const DEFAULT_ALLOWED_EXTENSIONS = [
+    '.ts',
+    '.tsx',
+    '.js',
+    '.jsx',
+    '.mjs',
+    '.cjs',
+    '.json',
+    '.yaml',
+    '.yml',
+    '.toml',
+    '.md',
+    '.mdx',
+    '.txt',
+    '.csv',
+    '.html',
+    '.css',
+    '.scss',
+    '.less',
+    '.py',
+    '.rb',
+    '.go',
+    '.rs',
+    '.java',
+    '.kt',
+    '.swift',
+    '.c',
+    '.cpp',
+    '.h',
+    '.hpp',
+    '.sh',
+    '.bash',
+    '.zsh',
+    '.sql',
+    '.graphql',
+    '.xml',
+    '.svg',
+    '.env',
+    '.env.local',
+    '.env.example',
+    '.gitignore',
+    '.dockerignore',
+    'Dockerfile',
+    'Makefile',
+    'Taskfile',
+];
+//#endregion
+//#region Helpers
+// Pattern requires file-like structure: path with extension or explicit path separator
+// Avoids matching #hashtag, #123, #region, etc.
+const FILE_REF_PATTERN = /#((?:[a-zA-Z0-9_.-]+\/)*[a-zA-Z0-9_.-]+\.[a-zA-Z0-9]+)/g;
+function sanitizeAnchor(refPath) {
+    return refPath.replace(/[^a-zA-Z0-9-]/g, '-').toLowerCase();
+}
+function transformReferencesToAnchors(text) {
+    return text.replace(FILE_REF_PATTERN, (_match, filePath) => {
+        const anchor = sanitizeAnchor(filePath);
+        return `[#${filePath}](#${anchor})`;
+    });
+}
+function extractFileReferences(text) {
+    const refs = [];
+    const pattern = new RegExp(FILE_REF_PATTERN);
+    let match = pattern.exec(text);
+    while (match !== null) {
+        refs.push(match[1]);
+        match = pattern.exec(text);
+    }
+    return [
+        ...new Set(refs),
+    ]; // Dedupe
+}
+function normalizePath(input) {
+    const raw = input.length > 0 ? input : '.';
+    const absolute = raw.startsWith('/') ? raw : `/${raw}`;
+    const parts = [];
+    for (const part of absolute.split('/')) {
+        if (part.length === 0 || part === '.') {
+            continue;
+        }
+        if (part === '..') {
+            parts.pop();
+            continue;
+        }
+        parts.push(part);
+    }
+    return `/${parts.join('/')}`;
+}
+function resolvePath(base, ref = '') {
+    return normalizePath(ref.startsWith('/') ? ref : `${base}/${ref}`);
+}
+function pathBasename(input) {
+    const normalized = normalizePath(input);
+    if (normalized === '/') {
+        return '';
+    }
+    return normalized.slice(normalized.lastIndexOf('/') + 1);
+}
+function pathExtname(input) {
+    const name = pathBasename(input);
+    const idx = name.lastIndexOf('.');
+    return idx > 0 ? name.slice(idx) : '';
+}
+function isAbsolutePath(input) {
+    return input.startsWith('/');
+}
+function currentWorkingDirectory() {
+    return typeof process !== 'undefined' ? process.cwd() : '/';
+}
+function simpleHash(content) {
+    let hash = 0;
+    for (let i = 0; i < content.length; i++) {
+        const char = content.charCodeAt(i);
+        hash = (hash << 5) - hash + char;
+        hash = hash & hash; // Convert to 32bit integer
+    }
+    // Include content length to reduce collision probability
+    return `${content.length}:${hash.toString(16)}`;
+}
+function isInputMessage(item) {
+    return (item.type === 'message' &&
+        'role' in item &&
+        'content' in item &&
+        Array.isArray(item.content) &&
+        item.content.every((c) => typeof c === 'object' && c !== null && 'type' in c && c.type === 'input_text'));
+}
+function extractTextFromItem(item) {
+    if (!isInputMessage(item)) {
+        return '';
+    }
+    return item.content.map((c) => c.text).join('\n');
+}
+async function readFileContent(absolutePath, opts, fsAdapter) {
+    try {
+        // Security: Validate path stays within baseDir (prevent path traversal)
+        const normalizedBase = resolvePath(opts.baseDir);
+        const normalizedPath = resolvePath(absolutePath);
+        if (!normalizedPath.startsWith(`${normalizedBase}/`) && normalizedPath !== normalizedBase) {
+            return {
+                content: null,
+                deleted: false,
+                error: 'PATH_TRAVERSAL: File outside allowed directory',
+            };
+        }
+        // Security: Check file extension
+        const ext = pathExtname(normalizedPath).toLowerCase();
+        const basename = pathBasename(normalizedPath);
+        const isAllowedExt = opts.allowedExtensions.some((allowed) => ext === allowed.toLowerCase() || basename === allowed);
+        if (!isAllowedExt && opts.allowedExtensions.length > 0) {
+            return {
+                content: null,
+                deleted: false,
+                error: `DISALLOWED_EXTENSION: ${ext || 'no extension'}`,
+            };
+        }
+        // Security: Check for symlinks if not allowed
+        const stats = await fsAdapter.lstat(absolutePath);
+        if (stats.isSymbolicLink() && !opts.followSymlinks) {
+            return {
+                content: null,
+                deleted: false,
+                error: 'SYMLINK: Symlinks not allowed',
+            };
+        }
+        // Security: Check file size
+        const fileStats = opts.followSymlinks ? await fsAdapter.stat(absolutePath) : stats;
+        if (fileStats.size > opts.maxFileSize) {
+            return {
+                content: null,
+                deleted: false,
+                error: `FILE_TOO_LARGE: ${fileStats.size} bytes exceeds limit of ${opts.maxFileSize}`,
+            };
+        }
+        const content = await fsAdapter.readFileText(absolutePath);
+        return {
+            content,
+            deleted: false,
+        };
+    }
+    catch (e) {
+        if (isNodeError(e) && e.code === 'ENOENT') {
+            return {
+                content: null,
+                deleted: true,
+            };
+        }
+        if (isNodeError(e) && e.code === 'EACCES') {
+            return {
+                content: null,
+                deleted: false,
+                error: 'PERMISSION_DENIED: Cannot read file',
+            };
+        }
+        throw e;
+    }
+}
+function isNodeError(e) {
+    return e instanceof Error && 'code' in e;
+}
+function extractResponseText(items) {
+    const texts = [];
+    for (const item of items) {
+        if (item.type !== 'message') {
+            continue;
+        }
+        if (!('content' in item) || !Array.isArray(item.content)) {
+            continue;
+        }
+        for (const part of item.content) {
+            if ('text' in part && typeof part.text === 'string') {
+                texts.push(part.text);
+            }
+        }
+    }
+    return texts.join('').trim();
+}
+async function scoreFileRelevance(params) {
+    const { filePath, fileContent, userQuery, ctx, model } = params;
+    if (!ctx.callModel) {
+        const queryLower = userQuery.toLowerCase();
+        const pathLower = filePath.toLowerCase();
+        if (pathLower.includes(queryLower) || queryLower.includes(pathBasename(pathLower))) {
+            return 80;
+        }
+        return 50;
+    }
+    const scoringPrompt = `Rate the relevance of this file to the user's query.
+
+User Query: ${userQuery}
+
+File Path: ${filePath}
+File Content (first 2000 chars):
+${fileContent.slice(0, 2e3)}
+
+Respond with ONLY a number from 0-100 where:
+- 0-20: Not relevant at all
+- 21-40: Slightly relevant
+- 41-60: Moderately relevant
+- 61-80: Highly relevant
+- 81-100: Critical/essential
+
+Score:`;
+    try {
+        const response = await ctx.callModel({
+            model,
+            items: [
+                createMessage(scoringPrompt, 'user'),
+            ],
+            instructions: 'You are a relevance scorer. Respond with only a number 0-100.',
+        });
+        const responseText = extractResponseText(response.items);
+        const score = Number.parseInt(responseText, 10);
+        if (Number.isNaN(score) || score < 0 || score > 100) {
+            return 50;
+        }
+        return score;
+    }
+    catch {
+        return 50;
+    }
+}
+//#endregion
+//#region Layer Factory
+/**
+ * Creates a memory layer that tracks file references in user messages.
+ *
+ * Syntax: `#path/to/file` in user messages
+ *
+ * Behavior:
+ * - Transforms references to markdown anchor links `[#path/to/file](#path-to-file)`
+ * - Scores file relevance using LLM when first referenced
+ * - Injects file contents into context via recall(), ordered by priority
+ * - Detects file changes on each new message, triggers immediate re-render
+ * - Shows warning for deleted files
+ *
+ * @public
+ */
+export function fileReference(opts) {
+    const baseDir = opts?.baseDir ?? currentWorkingDirectory();
+    const slot = opts?.slot ?? Slot.RAG;
+    const scoringModel = opts?.scoringModel ?? 'anthropic/claude-haiku-4-5-20251001';
+    const maxFileSize = opts?.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
+    const followSymlinks = opts?.followSymlinks ?? false;
+    const allowedExtensions = opts?.allowedExtensions ?? DEFAULT_ALLOWED_EXTENSIONS;
+    // Pre-compute read options for reuse
+    const readOpts = {
+        maxFileSize,
+        followSymlinks,
+        baseDir,
+        allowedExtensions,
+    };
+    return {
+        id: 'file-reference',
+        name: 'File Reference',
+        slot,
+        scope: 'thread',
+        budget: 'auto',
+        rerenderTiming: 'immediate',
+        hooks: {
+            async init() {
+                return {
+                    state: {
+                        files: new Map(),
+                        baseDir,
+                    },
+                };
+            },
+            async onItemAppend({ items, ctx, state }) {
+                const newFiles = new Map(state.files);
+                let hasChanges = false;
+                let userQuery = '';
+                // Update readOpts with current state's baseDir
+                const currentReadOpts = {
+                    ...readOpts,
+                    baseDir: state.baseDir,
+                };
+                // Process each item
+                const transformedItems = [];
+                for (const item of items) {
+                    const text = extractTextFromItem(item);
+                    if (!text) {
+                        transformedItems.push(item);
+                        continue;
+                    }
+                    // Extract user query for scoring
+                    if (item.type === 'message' && 'role' in item && item.role === 'user') {
+                        userQuery = text;
+                    }
+                    // Extract file references
+                    const refs = extractFileReferences(text);
+                    // Process new references
+                    for (const ref of refs) {
+                        if (!newFiles.has(ref)) {
+                            // Security: Reject absolute paths - all paths must be relative to baseDir
+                            if (isAbsolutePath(ref)) {
+                                newFiles.set(ref, {
+                                    absolutePath: ref,
+                                    referencePath: ref,
+                                    content: null,
+                                    priority: 0,
+                                    contentHash: '',
+                                    deleted: false,
+                                    tokenCount: 0,
+                                    error: 'ABSOLUTE_PATH: Only relative paths are allowed',
+                                });
+                                hasChanges = true;
+                                continue;
+                            }
+                            const absolutePath = resolvePath(state.baseDir, ref);
+                            const result = await readFileContent(absolutePath, currentReadOpts, ctx.fs);
+                            const priority = result.content
+                                ? await scoreFileRelevance({
+                                    filePath: ref,
+                                    fileContent: result.content,
+                                    userQuery,
+                                    ctx,
+                                    model: scoringModel,
+                                })
+                                : 0;
+                            newFiles.set(ref, {
+                                absolutePath,
+                                referencePath: ref,
+                                content: result.content,
+                                priority,
+                                contentHash: result.content ? simpleHash(result.content) : '',
+                                deleted: result.deleted,
+                                tokenCount: result.content ? estimateTokens(result.content) : 0,
+                                error: result.error,
+                            });
+                            hasChanges = true;
+                        }
+                    }
+                    // Transform references to anchor links
+                    if (refs.length > 0 && isInputMessage(item)) {
+                        const transformedContent = item.content.map((c) => {
+                            const part = {
+                                type: 'input_text',
+                                text: transformReferencesToAnchors(c.text),
+                            };
+                            return part;
+                        });
+                        const transformedItem = {
+                            ...item,
+                            content: transformedContent,
+                        };
+                        transformedItems.push(transformedItem);
+                    }
+                    else {
+                        transformedItems.push(item);
+                    }
+                }
+                // Check existing files for changes
+                for (const [ref, tracked] of newFiles) {
+                    // Skip files with errors - they won't change
+                    if (tracked.error) {
+                        continue;
+                    }
+                    const result = await readFileContent(tracked.absolutePath, currentReadOpts, ctx.fs);
+                    if (result.deleted && !tracked.deleted) {
+                        // File was deleted
+                        newFiles.set(ref, {
+                            ...tracked,
+                            content: null,
+                            deleted: true,
+                        });
+                        hasChanges = true;
+                    }
+                    else if (result.error && !tracked.error) {
+                        // File now has error (e.g., permission changed)
+                        newFiles.set(ref, {
+                            ...tracked,
+                            content: null,
+                            error: result.error,
+                        });
+                        hasChanges = true;
+                    }
+                    else if (!result.deleted && result.content) {
+                        const newHash = simpleHash(result.content);
+                        if (newHash !== tracked.contentHash) {
+                            // File content changed
+                            const priority = userQuery
+                                ? await scoreFileRelevance({
+                                    filePath: ref,
+                                    fileContent: result.content,
+                                    userQuery,
+                                    ctx,
+                                    model: scoringModel,
+                                })
+                                : tracked.priority;
+                            newFiles.set(ref, {
+                                ...tracked,
+                                content: result.content,
+                                contentHash: newHash,
+                                deleted: false,
+                                tokenCount: estimateTokens(result.content),
+                                priority,
+                                error: undefined,
+                            });
+                            hasChanges = true;
+                        }
+                    }
+                }
+                return {
+                    items: transformedItems,
+                    state: {
+                        ...state,
+                        files: newFiles,
+                    },
+                    rerender: hasChanges,
+                    timing: 'immediate',
+                    scope: 'self',
+                };
+            },
+            async recall({ state, budget }) {
+                if (state.files.size === 0) {
+                    return {
+                        items: [],
+                        tokenCount: 0,
+                    };
+                }
+                // Sort files by priority (highest first)
+                const sortedFiles = [
+                    ...state.files.values(),
+                ].sort((a, b) => b.priority - a.priority);
+                // Build content blocks within budget
+                const blocks = [];
+                let totalTokens = 0;
+                for (const file of sortedFiles) {
+                    // Handle deleted files
+                    if (file.deleted) {
+                        const block = `## ${file.referencePath}\n\n[FILE DELETED: ${file.absolutePath}]`;
+                        const tokens = estimateTokens(block);
+                        if (totalTokens + tokens <= budget) {
+                            blocks.push(block);
+                            totalTokens += tokens;
+                        }
+                        continue;
+                    }
+                    // Handle files with errors (security, size, permission)
+                    if (file.error) {
+                        const block = `## ${file.referencePath}\n\n[ERROR: ${file.error}]`;
+                        const tokens = estimateTokens(block);
+                        if (totalTokens + tokens <= budget) {
+                            blocks.push(block);
+                            totalTokens += tokens;
+                        }
+                        continue;
+                    }
+                    if (!file.content) {
+                        continue;
+                    }
+                    const header = `## ${file.referencePath}`;
+                    const headerTokens = estimateTokens(header);
+                    if (totalTokens + headerTokens >= budget) {
+                        // Can't fit even the header
+                        break;
+                    }
+                    const remainingBudget = budget - totalTokens - headerTokens - 10; // Buffer for formatting
+                    let contentToInclude = file.content;
+                    if (file.tokenCount > remainingBudget) {
+                        // Truncate content - show head + tail
+                        const lines = file.content.split('\n');
+                        const headLines = Math.floor(lines.length * 0.6);
+                        const tailLines = Math.floor(lines.length * 0.3);
+                        const head = lines.slice(0, headLines).join('\n');
+                        const tail = lines.slice(-tailLines).join('\n');
+                        contentToInclude = `${head}\n\n... [truncated ${lines.length - headLines - tailLines} lines] ...\n\n${tail}`;
+                    }
+                    const block = `${header}\n\n\`\`\`\n${contentToInclude}\n\`\`\``;
+                    const blockTokens = estimateTokens(block);
+                    if (totalTokens + blockTokens <= budget) {
+                        blocks.push(block);
+                        totalTokens += blockTokens;
+                    }
+                }
+                if (blocks.length === 0) {
+                    return {
+                        items: [],
+                        tokenCount: 0,
+                    };
+                }
+                const content = `# Referenced Files\n\n${blocks.join('\n\n')}`;
+                return {
+                    items: [
+                        createMessage(content, 'developer'),
+                    ],
+                    tokenCount: estimateTokens(content),
+                };
+            },
+        },
+    };
+}
+//#endregion

package/dist/memory/layers/history-window.d.ts

@@ -0,0 +1,29 @@
+/**
+ * History-window memory layer — caps the number of items projected to the LLM
+ * without touching `itemLog` storage. Pure read-side projection via the
+ * `projectHistory` hook.
+ *
+ * Algorithm per turn:
+ *   1. Slice the last `maxItems`.
+ *   2. If the slice lacks a user `message` or an assistant `message`, expand
+ *      backward until both are present. The cap may be exceeded — for
+ *      tool-burst histories the projected length can grow well beyond
+ *      `maxItems` because the most recent role-pair may be far back.
+ *   3. Drop any orphan `function_call` / `function_call_output` left at the
+ *      slice boundary via `stripUnresolvedToolCalls`.
+ */
+import type { MemoryLayer } from '../../types/memory';
+/** @public Configuration for the {@link historyWindow} layer. */
+export interface HistoryWindowConfig {
+    /** Cap on the number of trailing items projected to the LLM. Defaults to 40. */
+    maxItems?: number;
+}
+/**
+ * Creates a history-window memory layer that caps the trailing items projected
+ * to the LLM. Storage stays unbounded; only the wire payload is narrowed.
+ *
+ * @public
+ * @param config - Optional `{ maxItems }`. Defaults to 40.
+ * @returns A `MemoryLayer` that contributes only a `projectHistory` hook.
+ */
+export declare function historyWindow(config?: HistoryWindowConfig): MemoryLayer<null>;

package/dist/memory/layers/history-window.js

@@ -0,0 +1,114 @@
+/**
+ * History-window memory layer — caps the number of items projected to the LLM
+ * without touching `itemLog` storage. Pure read-side projection via the
+ * `projectHistory` hook.
+ *
+ * Algorithm per turn:
+ *   1. Slice the last `maxItems`.
+ *   2. If the slice lacks a user `message` or an assistant `message`, expand
+ *      backward until both are present. The cap may be exceeded — for
+ *      tool-burst histories the projected length can grow well beyond
+ *      `maxItems` because the most recent role-pair may be far back.
+ *   3. Drop any orphan `function_call` / `function_call_output` left at the
+ *      slice boundary via `stripUnresolvedToolCalls`.
+ */
+import { NoeticConfigError } from '../../errors/noetic-config-error';
+import { isAssistantMessage, isUserMessage } from '../../interpreter/typeguards';
+import { Slot } from '../../types/memory';
+import { stripUnresolvedToolCalls } from '../strip-unresolved';
+//#region Constants
+const DEFAULT_MAX_ITEMS = 40;
+const MIN_MAX_ITEMS = 2;
+const MAX_MAX_ITEMS = 1e4;
+const HISTORY_WINDOW_SLOT = Slot.PROCEDURAL + 25; // 275 — runs after recall-contributing layers
+//#endregion
+//#region Helpers
+function validateMaxItems(value) {
+    if (!Number.isInteger(value) || value < MIN_MAX_ITEMS || value > MAX_MAX_ITEMS) {
+        throw new NoeticConfigError({
+            code: 'INVALID_HISTORY_WINDOW_MAX_ITEMS',
+            message: `historyWindow: maxItems must be an integer in [${MIN_MAX_ITEMS}, ${MAX_MAX_ITEMS}], got ${value}`,
+            hint: 'Pass a sensible cap such as { maxItems: 40 }, or omit the layer to keep history uncapped.',
+        });
+    }
+}
+/**
+ * Walk backwards from the natural slice point until both a user message and
+ * an assistant message are present. Returns the index at which the projected
+ * window should start in the full `items` array.
+ */
+function expandToMinimumExchange(items, sliceStart) {
+    let start = sliceStart;
+    let needsUser = true;
+    let needsAssistant = true;
+    // Pass 1: scan the natural slice for the user/assistant pair. Early-return
+    // at the original sliceStart when both roles are already inside.
+    for (let i = items.length - 1; i >= start; i--) {
+        const item = items[i];
+        if (needsUser && isUserMessage(item)) {
+            needsUser = false;
+        }
+        if (needsAssistant && isAssistantMessage(item)) {
+            needsAssistant = false;
+        }
+        if (!needsUser && !needsAssistant) {
+            return start;
+        }
+    }
+    // Pass 2: walk backward past the slice until the missing role(s) are found.
+    for (let i = start - 1; i >= 0 && (needsUser || needsAssistant); i--) {
+        const item = items[i];
+        if (needsUser && isUserMessage(item)) {
+            needsUser = false;
+            start = i;
+            continue;
+        }
+        if (needsAssistant && isAssistantMessage(item)) {
+            needsAssistant = false;
+            start = i;
+        }
+    }
+    return start;
+}
+//#endregion
+//#region Public API
+/**
+ * Creates a history-window memory layer that caps the trailing items projected
+ * to the LLM. Storage stays unbounded; only the wire payload is narrowed.
+ *
+ * @public
+ * @param config - Optional `{ maxItems }`. Defaults to 40.
+ * @returns A `MemoryLayer` that contributes only a `projectHistory` hook.
+ */
+export function historyWindow(config) {
+    const maxItems = config?.maxItems ?? DEFAULT_MAX_ITEMS;
+    validateMaxItems(maxItems);
+    return {
+        id: 'history-window',
+        name: 'History Window',
+        slot: HISTORY_WINDOW_SLOT,
+        scope: 'execution',
+        hooks: {
+            async init() {
+                return {
+                    state: null,
+                };
+            },
+            async projectHistory({ items }) {
+                if (items.length <= maxItems) {
+                    return {
+                        items,
+                    };
+                }
+                const sliceStart = items.length - maxItems;
+                const expandedStart = expandToMinimumExchange(items, sliceStart);
+                const window = items.slice(expandedStart);
+                const cleaned = stripUnresolvedToolCalls(window);
+                return {
+                    items: cleaned,
+                };
+            },
+        },
+    };
+}
+//#endregion