@noetic-tools/context 1.0.0

package/dist/context/layers/temporal-context.js

@@ -0,0 +1,327 @@
+import { collectInputText, collectOutputText, createMessage, estimateTokens, Slot, } from '@noetic-tools/types';
+import { z } from 'zod';
+import { layerFn } from '../layer-provides.js';
+function emptyState() {
+    return {
+        facts: {},
+        buffer: [],
+        bufferTokens: 0,
+        version: 0,
+    };
+}
+const DEFAULT_BUFFER_THRESHOLD_TOKENS = 2_000;
+const DEFAULT_MAX_FACTS = 200;
+//#endregion
+//#region Date grounding
+const WEEKDAYS = [
+    'Sun',
+    'Mon',
+    'Tue',
+    'Wed',
+    'Thu',
+    'Fri',
+    'Sat',
+];
+/** Formats an instant as `YYYY/MM/DD (Ddd) HH:MM` using local fields. */
+function formatNow(date) {
+    const yyyy = date.getFullYear();
+    const mm = String(date.getMonth() + 1).padStart(2, '0');
+    const dd = String(date.getDate()).padStart(2, '0');
+    const day = WEEKDAYS[date.getDay()];
+    const hh = String(date.getHours()).padStart(2, '0');
+    const min = String(date.getMinutes()).padStart(2, '0');
+    return `${yyyy}/${mm}/${dd} (${day}) ${hh}:${min}`;
+}
+function renderDateTimeBlock(date) {
+    return [
+        '<current_datetime>',
+        `Now: ${formatNow(date)}`,
+        'Resolve relative time ("N days/weeks/months ago", "last/next <weekday>",',
+        '"before/after", "first/earliest/most recent", "how long since/until") against',
+        'Now above and compute the difference explicitly before answering. Use the',
+        'temporal/searchMemory tool to look up when something happened.',
+        '</current_datetime>',
+    ].join('\n');
+}
+//#endregion
+//#region Ledger helpers
+/** Flattens the KV ledger into a flat, chronologically sorted fact list. */
+function ledgerToFacts(state) {
+    const out = [];
+    for (const [ts, facts] of Object.entries(state.facts)) {
+        for (const fact of facts) {
+            out.push({
+                ts,
+                fact,
+            });
+        }
+    }
+    out.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0));
+    return out;
+}
+/** Merges new facts into the KV ledger, capping total facts to `maxFacts` (oldest dropped). */
+function mergeFacts(state, incoming, maxFacts) {
+    const facts = {};
+    for (const [ts, list] of Object.entries(state.facts)) {
+        facts[ts] = [
+            ...list,
+        ];
+    }
+    for (const { ts, fact } of incoming) {
+        const list = facts[ts] ?? [];
+        if (!list.includes(fact)) {
+            list.push(fact);
+        }
+        facts[ts] = list;
+    }
+    return capLedger(facts, maxFacts);
+}
+/**
+ * Caps the ledger at FACT granularity: when over `maxFacts`, drops the OLDEST
+ * facts and always retains the most recent `maxFacts`. Operating per-fact (not
+ * per-timestamp) prevents a single oversized extraction at one instant from
+ * evicting the just-added newest facts.
+ */
+function capLedger(facts, maxFacts) {
+    let total = 0;
+    for (const list of Object.values(facts)) {
+        total += list.length;
+    }
+    if (total <= maxFacts) {
+        return facts;
+    }
+    // Flatten chronologically (oldest first; stable within a timestamp), then keep
+    // only the most recent `maxFacts`.
+    const flat = [];
+    for (const [ts, list] of Object.entries(facts)) {
+        for (const fact of list) {
+            flat.push({
+                ts,
+                fact,
+            });
+        }
+    }
+    flat.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0));
+    const kept = flat.slice(flat.length - maxFacts);
+    const capped = {};
+    for (const { ts, fact } of kept) {
+        const list = capped[ts] ?? [];
+        list.push(fact);
+        capped[ts] = list;
+    }
+    return capped;
+}
+/**
+ * Renders the `<remembered_facts>` block, greedily including the most recent
+ * facts first and stopping before the block would exceed `budget` tokens.
+ * Returns `null` when there are no facts or not even one fits.
+ */
+function renderLedgerBlock(facts, budget) {
+    if (facts.length === 0) {
+        return null;
+    }
+    const open = '<remembered_facts>';
+    const close = '</remembered_facts>';
+    // Most recent first so the budget retains the freshest facts.
+    const ordered = [
+        ...facts,
+    ].reverse();
+    const selected = [];
+    for (const f of ordered) {
+        const line = `- [${f.ts}] ${f.fact}`;
+        const candidate = [
+            open,
+            ...selected,
+            line,
+            close,
+        ].join('\n');
+        if (estimateTokens(candidate) > budget) {
+            break;
+        }
+        selected.push(line);
+    }
+    if (selected.length === 0) {
+        return null;
+    }
+    return [
+        open,
+        ...selected,
+        close,
+    ].join('\n');
+}
+/**
+ * Appends `texts` into the buffer and, once the token threshold is crossed (and
+ * an extractor is configured), distills the buffer into ledger facts. Shared by
+ * `store` (assistant output) and `onItemAppend` (user/tool input).
+ */
+async function accumulate(s, texts, cfg) {
+    const newBuffer = [
+        ...s.buffer,
+        ...texts,
+    ];
+    const newTokens = texts.reduce((sum, t) => sum + estimateTokens(t), 0);
+    const totalBufferTokens = s.bufferTokens + newTokens;
+    if (cfg.extract && totalBufferTokens >= cfg.threshold && newBuffer.length > 0) {
+        const extracted = await cfg.extract({
+            transcript: newBuffer.join('\n'),
+            now: formatNow(cfg.now()),
+        });
+        return {
+            facts: mergeFacts(s, extracted, cfg.maxFacts),
+            buffer: [],
+            bufferTokens: 0,
+            version: s.version + 1,
+        };
+    }
+    return {
+        ...s,
+        buffer: newBuffer,
+        bufferTokens: totalBufferTokens,
+    };
+}
+//#endregion
+//#region Layer
+/**
+ * A non-atomic, LLM-backed context layer that extracts timestamped facts from the
+ * conversation into a key-value ledger (key = ISO timestamp) and exposes a
+ * `temporal/searchMemory` tool for relative-time recall — given a fact/query it
+ * returns matching facts and, when applicable, a resolved (possibly fuzzy) date.
+ *
+ * The layer is LLM-agnostic: the host injects `extract` and `search` callbacks
+ * (mirrors `observationalContext`'s `observer`), keeping `context/` tree-shakable.
+ * Without them, the layer still buffers text and the search tool returns the raw
+ * ledger — it never fabricates facts. A `<current_datetime>` grounding block is
+ * injected on recall by default to anchor relative-time reasoning.
+ *
+ * @public
+ * @param config - Clock, scope, injected LLM callbacks, and tuning knobs.
+ * @returns A `ContextLayer` with a fact-ledger store + `searchMemory` tool.
+ */
+export function temporalContext(config) {
+    const now = config?.now ?? (() => new Date());
+    const threshold = config?.bufferThreshold ?? DEFAULT_BUFFER_THRESHOLD_TOKENS;
+    const maxFacts = config?.maxFacts ?? DEFAULT_MAX_FACTS;
+    const groundDateTime = config?.groundDateTime ?? true;
+    const injectLedger = config?.injectLedger ?? false;
+    const extract = config?.extract;
+    const search = config?.search;
+    return {
+        id: 'temporal',
+        name: 'Temporal Context',
+        // Grounding sits near the top of the window, before reasoning content.
+        slot: Slot.REMINDER,
+        scope: config?.scope ?? 'resource',
+        budget: {
+            min: 0,
+            max: injectLedger ? 800 : 200,
+        },
+        timeouts: {
+            store: 60_000,
+            // onItemAppend runs the same LLM-backed accumulate/extract path as
+            // store — it needs the same headroom, not the 5s pipeline default.
+            onItemAppend: 60_000,
+        },
+        provides: {
+            searchMemory: layerFn({
+                description: 'Search remembered facts about the user by topic or fact, and resolve relative time. Returns matching facts and, when the query implies a time, a resolved date (which may be approximate).',
+                input: z.object({
+                    query: z.string(),
+                }),
+                output: z.object({
+                    facts: z.array(z.string()),
+                    date: z.string().optional(),
+                    fuzzy: z.boolean().optional(),
+                }),
+                execute: async (args, state) => {
+                    const facts = ledgerToFacts(state ?? emptyState());
+                    if (!search) {
+                        return {
+                            result: {
+                                facts: facts.map((f) => `[${f.ts}] ${f.fact}`),
+                            },
+                        };
+                    }
+                    const result = await search({
+                        query: args.query,
+                        facts,
+                        now: formatNow(now()),
+                    });
+                    return {
+                        result,
+                    };
+                },
+            }),
+        },
+        hooks: {
+            async init({ storage }) {
+                const saved = await storage.get('state');
+                return {
+                    state: saved ?? emptyState(),
+                };
+            },
+            async recall({ state, budget }) {
+                const blocks = [];
+                if (groundDateTime) {
+                    blocks.push(renderDateTimeBlock(now()));
+                }
+                if (injectLedger) {
+                    // Reserve whatever the grounding block (plus its `\n\n` separator)
+                    // already consumed so the rendered output stays within budget.
+                    const used = blocks.length > 0 ? estimateTokens(`${blocks.join('\n\n')}\n\n`) : 0;
+                    const ledgerBlock = renderLedgerBlock(ledgerToFacts(state ?? emptyState()), budget - used);
+                    if (ledgerBlock !== null) {
+                        blocks.push(ledgerBlock);
+                    }
+                }
+                if (blocks.length === 0) {
+                    return null;
+                }
+                const text = blocks.join('\n\n');
+                return {
+                    items: [
+                        createMessage(text, 'developer'),
+                    ],
+                    tokenCount: estimateTokens(text),
+                };
+            },
+            // Captures assistant output text.
+            async store({ newItems, state }) {
+                const s = state ?? emptyState();
+                const texts = collectOutputText(newItems);
+                return {
+                    state: await accumulate(s, texts, {
+                        threshold,
+                        maxFacts,
+                        now,
+                        extract,
+                    }),
+                };
+            },
+            // Captures user input and tool output text (pass-through; no transform).
+            async onItemAppend({ items, state }) {
+                const s = state ?? emptyState();
+                const texts = collectInputText(items);
+                if (texts.length === 0) {
+                    return {
+                        items,
+                    };
+                }
+                return {
+                    items,
+                    state: await accumulate(s, texts, {
+                        threshold,
+                        maxFacts,
+                        now,
+                        extract,
+                    }),
+                };
+            },
+            async onSpawn({ parentState }) {
+                return {
+                    childState: structuredClone(parentState),
+                };
+            },
+        },
+    };
+}
+//#endregion

package/dist/context/layers/tool-context-layer.d.ts

@@ -0,0 +1,15 @@
+import type { ContextLayer, Tool } from '@noetic-tools/types';
+interface ToolContextLayerOpts {
+    slot?: number;
+}
+/**
+ * Generates one ContextLayer per unique context id among the provided tools.
+ * Tools sharing the same `context.id` share a single layer (and thus state).
+ *
+ * @public
+ * @param tools - Array of tools to extract context declarations from.
+ * @param opts - Optional slot override for the generated layers.
+ * @returns An array of `ContextLayer` instances, one per unique tool context id.
+ */
+export declare function toolContextLayer(tools: ReadonlyArray<Tool>, opts?: ToolContextLayerOpts): ContextLayer[];
+export {};

package/dist/context/layers/tool-context-layer.js

@@ -0,0 +1,43 @@
+import { Slot } from '@noetic-tools/types';
+const EXECUTION_SCOPE = 'execution';
+/**
+ * Generates one ContextLayer per unique context id among the provided tools.
+ * Tools sharing the same `context.id` share a single layer (and thus state).
+ *
+ * @public
+ * @param tools - Array of tools to extract context declarations from.
+ * @param opts - Optional slot override for the generated layers.
+ * @returns An array of `ContextLayer` instances, one per unique tool context id.
+ */
+export function toolContextLayer(tools, opts) {
+    const seen = new Map();
+    for (const t of tools) {
+        if (!t.context) {
+            continue;
+        }
+        const layerId = t.context.id ?? t.name;
+        if (seen.has(layerId)) {
+            continue;
+        }
+        seen.set(layerId, t.context);
+    }
+    const slot = opts?.slot ?? Slot.WORKING_MEMORY + 10;
+    return [
+        ...seen.entries(),
+    ].map(([layerId, decl]) => ({
+        id: layerId,
+        name: layerId,
+        slot,
+        scope: EXECUTION_SCOPE,
+        hooks: {
+            async init() {
+                return {
+                    state: decl.init(),
+                };
+            },
+            async recall({ state }) {
+                return decl.recall(state);
+            },
+        },
+    }));
+}

package/dist/context/layers/working-memory-context.d.ts

@@ -0,0 +1,46 @@
+import type { ZodType } from 'zod';
+export type WorkingMemoryContextState = string | Record<string, unknown>;
+export interface WorkingMemoryContextConfig {
+    scope?: 'thread' | 'resource';
+    schema?: ZodType;
+    template?: string;
+    readOnly?: boolean;
+}
+/**
+ * Creates a mutable working context layer that the model can update via the `working-context/update` tool.
+ *
+ * @public
+ * @param config - Optional configuration for scope, Zod schema, template, and read-only mode.
+ * @returns A `ContextLayer` providing scratchpad state the model can read and write.
+ */
+export declare function workingMemoryContext(config?: WorkingMemoryContextConfig): {
+    id: 'working-context';
+    name: string;
+    slot: 100;
+    scope: "resource" | "thread";
+    budget: {
+        min: number;
+        max: number;
+    };
+    provides: {
+        snapshot: import("@noetic-tools/types").LayerDataDecl<WorkingMemoryContextState, WorkingMemoryContextState>;
+        update: import("@noetic-tools/types").LayerFunctionDecl<Record<string, unknown>, void, WorkingMemoryContextState>;
+    };
+    hooks: {
+        init({ storage }: import("@noetic-tools/types").InitParams): Promise<{
+            state: {};
+        } | {
+            state: WorkingMemoryContextState;
+        }>;
+        recall({ state }: import("@noetic-tools/types").RecallParams<WorkingMemoryContextState>): Promise<{
+            items: import("@noetic-tools/types").InputMessageItem[];
+            tokenCount: number;
+        } | null>;
+        store({ newItems, state }: import("@noetic-tools/types").StoreParams<WorkingMemoryContextState>): Promise<{
+            state: WorkingMemoryContextState;
+        } | undefined>;
+        onSpawn({ parentState }: import("@noetic-tools/types").SpawnParams<WorkingMemoryContextState>): Promise<{
+            childState: WorkingMemoryContextState;
+        } | null>;
+    };
+};

package/dist/context/layers/working-memory-context.js

@@ -0,0 +1,155 @@
+import { createMessage, estimateTokens, Slot } from '@noetic-tools/types';
+import { z } from 'zod';
+import { findFunctionCall } from '../function-call-utils.js';
+import { layerData, layerFn } from '../layer-provides.js';
+function isPlainObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/** Cap recursion depth so a cyclic or pathologically-nested update can't overflow the stack. */
+const MAX_MERGE_DEPTH = 32;
+/**
+ * Recursively merges `source` into `target`: object-valued keys are deep-merged,
+ * while arrays and primitives replace. Prototype-pollution keys (`__proto__`,
+ * `constructor`) are stripped at every depth. Beyond `MAX_MERGE_DEPTH` the source
+ * value replaces rather than recurses (cycle/over-nesting guard).
+ */
+function deepMerge(target, source, depth = 0) {
+    const out = {
+        ...target,
+    };
+    for (const [key, value] of Object.entries(source)) {
+        if (key === '__proto__' || key === 'constructor') {
+            continue;
+        }
+        const existing = out[key];
+        if (isPlainObject(value) && depth < MAX_MERGE_DEPTH) {
+            out[key] = isPlainObject(existing)
+                ? deepMerge(existing, value, depth + 1)
+                : deepMerge({}, value, depth + 1);
+            continue;
+        }
+        out[key] = value;
+    }
+    return out;
+}
+/**
+ * Validate a MERGED working-context state against the configured schema (a
+ * pure gate — the original value is returned, transforms are not applied).
+ * Validating the merged state (not the raw update) keeps partial updates
+ * legal under schemas with required fields. Throws on failure so the update
+ * is rejected and the prior state stays untouched.
+ */
+function applySchema(schema, merged) {
+    if (!schema) {
+        return merged;
+    }
+    const result = schema.safeParse(merged);
+    if (!result.success) {
+        throw new Error(`working-context update rejected by schema: ${result.error.message}`);
+    }
+    return merged;
+}
+function safeMerge(state, args) {
+    if (typeof state === 'object' && state !== null) {
+        return deepMerge(state, args);
+    }
+    // Freeform string state: an object update must not silently discard prior content.
+    const merged = deepMerge({}, args);
+    if (typeof state === 'string' && state.length > 0) {
+        // `_previous` last so the preserved prior content always wins over a
+        // (pathological) update key literally named `_previous`.
+        return {
+            ...merged,
+            _previous: state,
+        };
+    }
+    return merged;
+}
+/**
+ * Creates a mutable working context layer that the model can update via the `working-context/update` tool.
+ *
+ * @public
+ * @param config - Optional configuration for scope, Zod schema, template, and read-only mode.
+ * @returns A `ContextLayer` providing scratchpad state the model can read and write.
+ */
+export function workingMemoryContext(config) {
+    const scope = config?.scope ?? 'thread';
+    return {
+        id: 'working-context',
+        name: 'Working Memory',
+        slot: Slot.WORKING_MEMORY,
+        scope,
+        budget: {
+            min: 200,
+            max: 1_500,
+        },
+        provides: {
+            snapshot: layerData({
+                read: (state) => state,
+            }),
+            update: layerFn({
+                description: 'Update the agent working context with new key-value pairs.',
+                input: z.record(z.string(), z.unknown()),
+                output: z.void(),
+                // Validation failure throws → surfaces as a tool error the model can
+                // see; the prior state is untouched.
+                execute: async (args, state) => ({
+                    result: undefined,
+                    state: applySchema(config?.schema, safeMerge(state, args)),
+                }),
+            }),
+        },
+        hooks: {
+            async init({ storage }) {
+                const saved = await storage.get('state');
+                // Corrupt persisted state (fails the configured schema) falls back to
+                // the default rather than aborting the execution.
+                if (saved !== null && config?.schema && !config.schema.safeParse(saved).success) {
+                    return {
+                        state: {},
+                    };
+                }
+                const state = saved ?? (config?.schema ? {} : '');
+                return {
+                    state,
+                };
+            },
+            async recall({ state }) {
+                if (!state || (typeof state === 'object' && Object.keys(state).length === 0)) {
+                    return null;
+                }
+                const text = typeof state === 'string' ? state : JSON.stringify(state, null, 2);
+                const content = `<working_memory>\n${text}\n</working_memory>`;
+                return {
+                    items: [
+                        createMessage(content, 'developer'),
+                    ],
+                    tokenCount: estimateTokens(content),
+                };
+            },
+            async store({ newItems, state }) {
+                if (config?.readOnly) {
+                    return;
+                }
+                // Backward compat: detect implicit updateWorkingMemory calls from LLMs.
+                // Schema-violating merges throw — storeLayers catches, reports a
+                // diagnostic, and drops the update (prior state preserved).
+                const args = findFunctionCall(newItems, 'updateWorkingMemory');
+                if (!args) {
+                    return;
+                }
+                return {
+                    state: applySchema(config?.schema, safeMerge(state, args)),
+                };
+            },
+            async onSpawn({ parentState }) {
+                if (scope === 'resource') {
+                    return {
+                        childState: structuredClone(parentState),
+                    };
+                }
+                return null;
+            },
+        },
+    };
+}

package/dist/context/projector.d.ts

@@ -0,0 +1,19 @@
+import type { Item, ProjectionPolicy } from '@noetic-tools/types';
+interface AssembleViewParams {
+    systemPromptItems: Item[];
+    layerOutputItems: Item[];
+    historyItems: Item[];
+    policy?: ProjectionPolicy;
+}
+/**
+ * Assemble the model's context window from system prompt items, context-layer
+ * recall output (slot-ascending), and conversation history.
+ *
+ * Without a `policy` the inputs are concatenated as-is (optionally sliding the
+ * history window by `windowSize`). With a `policy` the assembled view is held to
+ * a hard token budget: system items are always kept; layer output is kept
+ * low-slot-first (highest-slot dropped when tight); history takes the remainder,
+ * keeping the most recent turns.
+ */
+export declare function assembleView({ systemPromptItems, layerOutputItems, historyItems, policy, }: AssembleViewParams): Item[];
+export {};

package/dist/context/projector.js

@@ -0,0 +1,89 @@
+import { estimateTokens } from '@noetic-tools/types';
+import { stripUnresolvedToolCalls } from './strip-unresolved.js';
+//#endregion
+//#region Helpers
+/** Conservative per-item token estimate (serialized form ⇒ never under-counts). */
+function itemTokens(item) {
+    return estimateTokens(JSON.stringify(item));
+}
+function totalTokens(items) {
+    let total = 0;
+    for (const item of items) {
+        total += itemTokens(item);
+    }
+    return total;
+}
+/**
+ * Keep items from a slot-ascending list within `budget`, considering items in
+ * slot order and dropping each non-fitting item INDIVIDUALLY. Layer-output
+ * items are independent contributions (no contiguity requirement, unlike
+ * history), so an oversized low-slot item must not evict later items that
+ * still fit — lower-slot output gets first claim on the budget, and
+ * higher-slot output is dropped first when space runs out.
+ */
+function keepFrontWithinBudget(items, budget) {
+    const kept = [];
+    let used = 0;
+    for (const item of items) {
+        const cost = itemTokens(item);
+        if (used + cost > budget) {
+            continue;
+        }
+        kept.push(item);
+        used += cost;
+    }
+    return kept;
+}
+/**
+ * Keep the MOST RECENT history items within `budget`, then strip any orphan
+ * tool calls/outputs left dangling at the slice boundary. An optional
+ * `windowSize` caps item count first (sliding-window overflow mode).
+ */
+function keepRecentWithinBudget(items, budget, windowSize) {
+    const windowed = windowSize ? items.slice(-windowSize) : items;
+    const kept = [];
+    let used = 0;
+    for (let i = windowed.length - 1; i >= 0; i--) {
+        const item = windowed[i];
+        const cost = itemTokens(item);
+        if (used + cost > budget) {
+            break;
+        }
+        kept.unshift(item);
+        used += cost;
+    }
+    return stripUnresolvedToolCalls(kept);
+}
+//#endregion
+//#region Public API
+/**
+ * Assemble the model's context window from system prompt items, context-layer
+ * recall output (slot-ascending), and conversation history.
+ *
+ * Without a `policy` the inputs are concatenated as-is (optionally sliding the
+ * history window by `windowSize`). With a `policy` the assembled view is held to
+ * a hard token budget: system items are always kept; layer output is kept
+ * low-slot-first (highest-slot dropped when tight); history takes the remainder,
+ * keeping the most recent turns.
+ */
+export function assembleView({ systemPromptItems, layerOutputItems, historyItems, policy, }) {
+    if (!policy) {
+        return [
+            ...systemPromptItems,
+            ...layerOutputItems,
+            ...historyItems,
+        ];
+    }
+    const budget = Math.max(0, policy.tokenBudget - policy.responseReserve);
+    // System items are never dropped — they anchor the conversation.
+    const afterSystem = Math.max(0, budget - totalTokens(systemPromptItems));
+    const keptLayers = keepFrontWithinBudget(layerOutputItems, afterSystem);
+    const afterLayers = Math.max(0, afterSystem - totalTokens(keptLayers));
+    const keptHistory = keepRecentWithinBudget(historyItems, afterLayers, policy.windowSize);
+    return [
+        ...systemPromptItems,
+        ...keptLayers,
+        ...keptHistory,
+    ];
+}
+//#endregion