@dory-agentic/dory-agentic-sdk 0.2.0

package/dist/lfs/memoryParser.js

@@ -0,0 +1,743 @@
+import { buildMsgNumberToIdMap, formatScorableIdListForPrompt, isScorableMessage, resolveMessageId, } from "../history/messageManifest";
+export { collectScorableMessageIds } from "../history/messageManifest";
+import { queryVectorCache } from "./vectorDb";
+/** All non-archived rows with IDs (for prompt listing). */
+export function collectActiveMessageIds(history) {
+    return history
+        .filter((m) => !m.isArchived && m.role !== 'system' && m.id)
+        .map((m) => m.id);
+}
+/** Map agent output keys to exact IDs from history (fixes msg_1 vs msg_01; never by list position). */
+export function canonicalizeMessageIds(weights, activeIds) {
+    const numberMap = buildMsgNumberToIdMap(activeIds);
+    const invalidIds = [];
+    const canonical = {};
+    for (const [rawId, val] of Object.entries(weights)) {
+        if (val <= 0)
+            continue;
+        const match = resolveMessageId(rawId, activeIds, numberMap);
+        if (match) {
+            canonical[match] = (canonical[match] ?? 0) + val;
+        }
+        else {
+            invalidIds.push(rawId);
+        }
+    }
+    return { canonical, invalidIds };
+}
+/**
+ * Merge agent relevance % with referenced_message_ids.
+ * Cited IDs with no score get a fair share so used context is never left at 0%.
+ */
+export function mergeRelevanceWithReferences(agentPercents, referencedIds, activeIds) {
+    const merged = { ...agentPercents };
+    const cited = referencedIds.filter((id) => activeIds.has(id));
+    if (cited.length === 0)
+        return merged;
+    const citedMissing = cited.filter((id) => !(merged[id] > 0));
+    if (citedMissing.length === 0)
+        return merged;
+    const existingSum = Object.values(merged).reduce((a, b) => a + b, 0);
+    const reserveForCited = citedMissing.length * 5;
+    const scale = existingSum > 0 && existingSum + reserveForCited > 100
+        ? (100 - reserveForCited) / existingSum
+        : 1;
+    if (scale < 1) {
+        for (const id of Object.keys(merged)) {
+            merged[id] = Number((merged[id] * scale).toFixed(2));
+        }
+    }
+    const perMissing = citedMissing.length > 0
+        ? Number(((100 - Object.values(merged).reduce((a, b) => a + b, 0)) /
+            citedMissing.length).toFixed(2))
+        : 0;
+    for (const id of citedMissing) {
+        merged[id] = Math.max(merged[id] ?? 0, perMissing);
+    }
+    return merged;
+}
+/** Shared instructions for optimized-lane relevance % (server converts to credits). */
+export function buildRelevanceScoringInstructions(params) {
+    const idList = formatScorableIdListForPrompt(params.history);
+    const hasChunks = params.history.some(m => m.chunks && m.chunks.length > 0);
+    if (hasChunks) {
+        return `Context pressure is active. Review the active memory manifest. Return an explicit evaluation dictionary inside your |||RATIO:{}||| block scoring the functional importance of historical chunk IDs to your current task on a strict scale from 0.0 (Unrelated noise) to 1.0 (Core critical asset).
+
+Scorable active chunk IDs:
+${idList}`;
+    }
+    return `--- MEMORY SCORING (after you finish "response") ---
+1. Write your full answer in "response" first.
+2. Set "referenced_message_ids" to every ID you actually used (subset of scorable IDs below).
+3. Set "credit_allocation" to relevance PERCENTAGES (0–100, sum ~100) for same-topic messages that helped produce THIS answer.
+   - Keys MUST be exact Message IDs from conversation rows (msg_01, msg_02, …) — NOT conversation # position alone.
+   - Chat history is oldest → newest; msg_01 is the first user message unless your thread started differently.
+   - Do NOT score Internal trace or Tool result rows.
+   - Percentages are NOT credits — the server maps % → ${params.budget} credits for this turn.
+
+"context_shift": "continue" | "new_topic" (unrelated → credit_allocation: {}).
+
+Scorable messages (same order as [Message ID: …] rows in history above):
+${idList}`;
+}
+/** Extract |||ALLOCATION:{...}||| from any text (response body or full JSON stream). */
+export function extractAllocation(content) {
+    if (!content)
+        return {};
+    const patterns = [
+        /\|\|\|RATIO:\s*([\s\S]*?)\s*\|\|\|/,
+        /RATIO:\s*(\{[\s\S]*?\})(?:\s*\|\|\||\s*$)/,
+        /\|\|\|ALLOCATION:\s*([\s\S]*?)\s*\|\|\|/,
+        /ALLOCATION:\s*(\{[\s\S]*?\})(?:\s*\|\|\||\s*$)/,
+    ];
+    for (const pattern of patterns) {
+        const match = content.match(pattern);
+        if (!match)
+            continue;
+        const parsed = parseAllocationJson(match[1].trim());
+        if (Object.keys(parsed).length > 0)
+            return parsed;
+    }
+    return {};
+}
+function parseAllocationJson(raw) {
+    const attempts = [raw, raw.replace(/\\"/g, '"')];
+    for (const attempt of attempts) {
+        try {
+            const parsed = JSON.parse(attempt);
+            if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
+                return normalizeAllocation(parsed);
+            }
+        }
+        catch {
+            // try next
+        }
+    }
+    // Fallback: "msg_01": 25 style without valid JSON
+    const loose = {};
+    const idMatches = raw.matchAll(/"?((?:msg|slice)_\d+)"?\s*:\s*(\d+(?:\.\d+)?)/g);
+    for (const m of idMatches) {
+        const val = Number(m[2]);
+        if (!isNaN(val) && val > 0)
+            loose[m[1]] = val;
+    }
+    return loose;
+}
+function normalizeAllocation(obj) {
+    const allocation = {};
+    for (const key in obj) {
+        const val = Number(obj[key]);
+        if (!isNaN(val) && val > 0)
+            allocation[key] = val;
+    }
+    return allocation;
+}
+export function extractReferencedIds(content) {
+    if (!content)
+        return [];
+    try {
+        const parsed = JSON.parse(content);
+        if (Array.isArray(parsed.referenced_message_ids)) {
+            return parsed.referenced_message_ids.filter((id) => typeof id === 'string');
+        }
+    }
+    catch {
+        // partial stream
+    }
+    const match = content.match(/"referenced_message_ids"\s*:\s*\[([\s\S]*?)\]/);
+    if (!match)
+        return [];
+    const ids = match[1].match(/"((?:msg|slice)_\d+)"/g);
+    return ids ? ids.map((id) => id.replace(/"/g, '')) : [];
+}
+function extractCreditAllocationFromParsed(parsed) {
+    const raw = parsed.credit_allocation ?? parsed.relevance_percent ?? parsed.relevance_percentages;
+    if (raw && typeof raw === 'object') {
+        return normalizeAllocation(raw);
+    }
+    return {};
+}
+function extractContextShift(parsed) {
+    const raw = parsed.context_shift;
+    if (raw === 'new_topic' || raw === 'continue')
+        return raw;
+    return 'continue';
+}
+/** Map normalized 0–100% shares to credits that sum to budget (2 decimal places). */
+export function percentMapToCredits(normalizedPercents, budget) {
+    const entries = Object.entries(normalizedPercents).filter(([, p]) => p > 0);
+    if (entries.length === 0 || budget <= 0)
+        return {};
+    const credits = {};
+    let allocated = 0;
+    entries.forEach(([id, pct], index) => {
+        if (index === entries.length - 1) {
+            credits[id] = Math.max(0, Number((budget - allocated).toFixed(2)));
+        }
+        else {
+            const c = Number(((pct / 100) * budget).toFixed(2));
+            credits[id] = c;
+            allocated += c;
+        }
+    });
+    return credits;
+}
+/**
+ * Agent sends relevance as percentages (0–100) per message ID.
+ * Values are normalized to sum to 100%, then converted to credits in the backend.
+ */
+export function normalizePercentagesToBudget(agentPercents, budget, activeIds) {
+    const invalidIds = [];
+    const rawPercents = {};
+    for (const id in agentPercents) {
+        if (agentPercents[id] <= 0)
+            continue;
+        if (!activeIds.has(id)) {
+            invalidIds.push(id);
+            continue;
+        }
+        rawPercents[id] = agentPercents[id];
+    }
+    const rawSum = Object.values(rawPercents).reduce((a, b) => a + b, 0);
+    if (rawSum <= 0 || budget <= 0) {
+        return {
+            rawPercents,
+            normalizedPercents: {},
+            credits: {},
+            rawSum,
+            invalidIds,
+        };
+    }
+    const normalizedPercents = {};
+    for (const [id, p] of Object.entries(rawPercents)) {
+        normalizedPercents[id] = Number(((p / rawSum) * 100).toFixed(2));
+    }
+    return {
+        rawPercents,
+        normalizedPercents,
+        credits: percentMapToCredits(normalizedPercents, budget),
+        rawSum,
+        invalidIds,
+    };
+}
+/** Distribute integer credits so they sum exactly to budget using relevance weights. */
+export function normalizeRelevanceToBudget(weights, budget, activeIds) {
+    const valid = {};
+    let totalWeight = 0;
+    for (const id in weights) {
+        if (activeIds.has(id) && weights[id] > 0) {
+            valid[id] = weights[id];
+            totalWeight += weights[id];
+        }
+    }
+    if (totalWeight <= 0 || budget <= 0)
+        return {};
+    const entries = Object.entries(valid);
+    const allocation = {};
+    let allocatedSum = 0;
+    entries.forEach(([id, weight], index) => {
+        if (index === entries.length - 1) {
+            allocation[id] = Math.max(0, budget - allocatedSum);
+        }
+        else {
+            const share = Math.round((weight / totalWeight) * budget);
+            allocation[id] = share;
+            allocatedSum += share;
+        }
+    });
+    return allocation;
+}
+export function parsePartialJson(jsonStr, isMmuActive) {
+    if (isMmuActive === false) {
+        try {
+            const parsed = JSON.parse(jsonStr);
+            const response = typeof parsed.response === 'string' ? parsed.response : (Object.entries(parsed)
+                .filter(([k]) => k !== 'thinking')
+                .map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`)
+                .join('\n'));
+            return {
+                thinking: typeof parsed.thinking === 'string' ? parsed.thinking : '',
+                response,
+                referencedIds: [],
+                creditAllocation: {},
+                contextShift: 'continue',
+            };
+        }
+        catch {
+            // partial stream
+        }
+        let thinking = '';
+        let response = '';
+        const thinkingMatch = jsonStr.match(/"thinking"\s*:\s*"(|.*?[^\\])"/);
+        if (thinkingMatch) {
+            thinking = thinkingMatch[1];
+        }
+        else {
+            const thinkingStartMatch = jsonStr.match(/"thinking"\s*:\s*"([\s\S]*)/);
+            if (thinkingStartMatch)
+                thinking = thinkingStartMatch[1];
+        }
+        const responseMatch = jsonStr.match(/"response"\s*:\s*"(|.*?[^\\])"/);
+        if (responseMatch) {
+            response = responseMatch[1];
+        }
+        else {
+            const responseStartMatch = jsonStr.match(/"response"\s*:\s*"([\s\S]*)/);
+            if (responseStartMatch)
+                response = responseStartMatch[1];
+        }
+        const unescapeStr = (str) => str
+            .replace(/\\"/g, '"')
+            .replace(/\\n/g, '\n')
+            .replace(/\\t/g, '\t')
+            .replace(/\\\\/g, '\\');
+        return {
+            thinking: unescapeStr(thinking),
+            response: unescapeStr(response),
+            referencedIds: [],
+            creditAllocation: {},
+            contextShift: 'continue',
+        };
+    }
+    try {
+        const parsed = JSON.parse(jsonStr);
+        const response = typeof parsed.response === 'string' ? parsed.response : (Object.entries(parsed)
+            .filter(([k]) => k !== 'thinking' && k !== 'credit_allocation' && k !== 'referenced_message_ids' && k !== 'context_shift')
+            .map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`)
+            .join('\n'));
+        const fromField = extractCreditAllocationFromParsed(parsed);
+        const fromResponse = extractAllocation(response);
+        const creditAllocation = Object.keys(fromField).length > 0 ? fromField : fromResponse;
+        return {
+            thinking: typeof parsed.thinking === 'string' ? parsed.thinking : '',
+            response,
+            referencedIds: Array.isArray(parsed.referenced_message_ids)
+                ? parsed.referenced_message_ids.filter((id) => typeof id === 'string')
+                : [],
+            creditAllocation,
+            contextShift: extractContextShift(parsed),
+        };
+    }
+    catch {
+        // partial stream — continue below
+    }
+    let thinking = '';
+    let response = '';
+    let referencedIds = [];
+    const thinkingMatch = jsonStr.match(/"thinking"\s*:\s*"(|.*?[^\\])"/);
+    if (thinkingMatch) {
+        thinking = thinkingMatch[1];
+    }
+    else {
+        const thinkingStartMatch = jsonStr.match(/"thinking"\s*:\s*"([\s\S]*)/);
+        if (thinkingStartMatch)
+            thinking = thinkingStartMatch[1];
+    }
+    const responseMatch = jsonStr.match(/"response"\s*:\s*"(|.*?[^\\])"/);
+    if (responseMatch) {
+        response = responseMatch[1];
+    }
+    else {
+        const responseStartMatch = jsonStr.match(/"response"\s*:\s*"([\s\S]*)/);
+        if (responseStartMatch)
+            response = responseStartMatch[1];
+    }
+    referencedIds = extractReferencedIds(jsonStr);
+    const unescapeStr = (str) => str
+        .replace(/\\"/g, '"')
+        .replace(/\\n/g, '\n')
+        .replace(/\\t/g, '\t')
+        .replace(/\\\\/g, '\\');
+    const unescapedResponse = unescapeStr(response);
+    const creditFromPartial = extractCreditAllocationFromParsed(tryParseObject(jsonStr) ?? {});
+    const partialObj = tryParseObject(jsonStr) ?? {};
+    let contextShift = 'continue';
+    if (partialObj.context_shift === 'new_topic' || partialObj.context_shift === 'continue') {
+        contextShift = partialObj.context_shift;
+    }
+    return {
+        thinking: unescapeStr(thinking),
+        response: unescapedResponse,
+        referencedIds,
+        creditAllocation: Object.keys(creditFromPartial).length > 0
+            ? creditFromPartial
+            : extractAllocation(unescapedResponse),
+        contextShift,
+    };
+}
+function tryParseObject(jsonStr) {
+    try {
+        const parsed = JSON.parse(jsonStr);
+        return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)
+            ? parsed
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Search credit_allocation JSON field, response |||ALLOCATION||| block, then stream buffer. */
+export function extractAllocationFromSources(responseText, streamBuffer, parsedCreditAllocation) {
+    if (parsedCreditAllocation && Object.keys(parsedCreditAllocation).length > 0) {
+        return parsedCreditAllocation;
+    }
+    const fromResponse = extractAllocation(responseText);
+    if (Object.keys(fromResponse).length > 0)
+        return fromResponse;
+    const parsedBuffer = tryParseObject(streamBuffer);
+    if (parsedBuffer) {
+        const fromField = extractCreditAllocationFromParsed(parsedBuffer);
+        if (Object.keys(fromField).length > 0)
+            return fromField;
+    }
+    const fromBuffer = extractAllocation(streamBuffer);
+    if (Object.keys(fromBuffer).length > 0)
+        return fromBuffer;
+    // Partial stream: credit_allocation object before closing brace
+    const partialField = streamBuffer.match(/"credit_allocation"\s*:\s*(\{[\s\S]*?\})/);
+    if (partialField) {
+        const parsed = parseAllocationJson(partialField[1]);
+        if (Object.keys(parsed).length > 0)
+            return parsed;
+    }
+    return {};
+}
+export function sanitizeContent(content) {
+    let sanitized = content.replace(/\|\|\|ALLOCATION:[\s\S]*?\|\|\|/g, '');
+    sanitized = sanitized.replace(/\|\|\|(?:ALLOCATION:[\s\S]*)?$/, '');
+    return sanitized.trim();
+}
+export function deriveMemoryStatus(id, referencedIds, creditsApplied, forgetScore, decayApplied, forceDecayingIds) {
+    if (forceDecayingIds?.has(id) && decayApplied > 0)
+        return 'DECAYING';
+    if (creditsApplied > 0)
+        return 'SHIELDED';
+    if (referencedIds.includes(id))
+        return 'CITED';
+    if (forgetScore >= 1.0)
+        return 'EVICTED';
+    if (forgetScore >= 0.75)
+        return 'DANGER';
+    if (decayApplied > 0)
+        return 'DECAYING';
+    return 'FRESH';
+}
+export function buildTurnMemorySnapshot(params) {
+    const { turnBudget, contextShift, creditsConsumed, agentAllocation, effectiveAllocation, usedFallback, scaledAllocation, scale, totalRequested, referencedIds, messages, estimateTokens, forceDecayingIds, } = params;
+    const rows = messages
+        .filter((m) => isScorableMessage(m))
+        .map((m) => {
+        const tokens = m.tokenCount ?? estimateTokens(m.content);
+        const forgetScore = m.forgetScore ?? 0;
+        const decayApplied = m.decayApplied ?? 0;
+        const agentPercentRaw = agentAllocation[m.id] ?? 0;
+        const agentPercentNorm = effectiveAllocation[m.id] ?? 0;
+        const creditsApplied = scaledAllocation[m.id] ?? 0;
+        const budgetSharePercent = agentPercentNorm;
+        return {
+            id: m.id,
+            role: m.role,
+            tokens,
+            forgetScore,
+            decayApplied,
+            creditsRequested: agentPercentRaw,
+            creditsApplied,
+            relevanceWeight: agentPercentRaw,
+            agentPercentNormalized: agentPercentNorm,
+            budgetSharePercent,
+            status: deriveMemoryStatus(m.id, referencedIds, creditsApplied, forgetScore, decayApplied, forceDecayingIds),
+        };
+    });
+    return {
+        turnBudget,
+        contextShift,
+        creditsConsumed,
+        agentAllocation,
+        effectiveAllocation,
+        usedFallback,
+        scaledAllocation,
+        totalRequested,
+        scaleApplied: scale,
+        invalidAgentIds: params.invalidAgentIds ?? [],
+        rows,
+    };
+}
+/** Turn agent relevance percentages into final credits for this round. */
+export function resolveTurnCredits(params) {
+    const { contextShift, agentWeights, referencedIds, budget, activeIds, activeTurnId } = params;
+    if (contextShift === 'new_topic' || budget <= 0) {
+        return {
+            agentAllocation: {},
+            effectiveAllocation: {},
+            scaledAllocation: {},
+            scale: 1,
+            totalRequested: 0,
+            usedFallback: false,
+            creditsConsumed: 0,
+            invalidAgentIds: [],
+        };
+    }
+    const { canonical, invalidIds: canonInvalid } = canonicalizeMessageIds(agentWeights, activeIds);
+    const numberMap = buildMsgNumberToIdMap(activeIds);
+    const citedCanonical = referencedIds
+        .map((id) => resolveMessageId(id, activeIds, numberMap))
+        .filter((id) => id != null);
+    const invalidIds = [...canonInvalid];
+    // Exclude activeTurnId and referencedIds (citedCanonical) from receiving credits
+    const creditEligibleWeights = {};
+    for (const [id, weight] of Object.entries(canonical)) {
+        if (id !== activeTurnId && !citedCanonical.includes(id)) {
+            creditEligibleWeights[id] = weight;
+        }
+    }
+    const { normalizedPercents, credits, rawSum, invalidIds: normInvalid, } = normalizePercentagesToBudget(creditEligibleWeights, budget, activeIds);
+    invalidIds.push(...normInvalid);
+    if (rawSum > 0) {
+        const consumed = Object.values(credits).reduce((a, b) => a + b, 0);
+        return {
+            agentAllocation: canonical,
+            effectiveAllocation: normalizedPercents,
+            scaledAllocation: credits,
+            scale: 1,
+            totalRequested: rawSum,
+            usedFallback: false,
+            creditsConsumed: consumed,
+            invalidAgentIds: invalidIds,
+        };
+    }
+    return {
+        agentAllocation: {},
+        effectiveAllocation: {},
+        scaledAllocation: {},
+        scale: 1,
+        totalRequested: 0,
+        usedFallback: false,
+        creditsConsumed: 0,
+        invalidAgentIds: invalidIds,
+    };
+}
+export function scaleAllocation(allocation, budget, activeIds) {
+    const validAllocation = {};
+    let totalRequested = 0;
+    for (const id in allocation) {
+        if (activeIds.has(id)) {
+            validAllocation[id] = allocation[id];
+            totalRequested += allocation[id];
+        }
+    }
+    let scale = 1;
+    if (totalRequested > budget && budget > 0) {
+        scale = budget / totalRequested;
+    }
+    const scaledAllocation = {};
+    for (const id in validAllocation) {
+        scaledAllocation[id] = Number((validAllocation[id] * scale).toFixed(2));
+    }
+    return { validAllocation, scaledAllocation, scale, totalRequested };
+}
+export function extractHydrationBeaconVariables(messages, query) {
+    let pgPool = '';
+    let redisSentinel = '';
+    let otelCollector = '';
+    let maxOldSpace = '';
+    for (const m of messages) {
+        const textSources = [];
+        if (m.content) {
+            textSources.push(m.content);
+        }
+        if (m.chunks) {
+            for (const chunk of m.chunks) {
+                if (chunk.content) {
+                    textSources.push(chunk.content);
+                }
+            }
+        }
+        for (const text of textSources) {
+            // PG_POOL_SIZE
+            const pgMatch = text.match(/PG_POOL_SIZE\s*=\s*(\d+)/i);
+            if (pgMatch && !pgPool)
+                pgPool = pgMatch[1];
+            // Redis sentinel
+            const redisMatch = text.match(/sentinel-\d+\.redis\.internal:\d+/i) || text.match(/sentinel-01\.redis\.internal/i);
+            if (redisMatch && !redisSentinel)
+                redisSentinel = 'sentinel-01.redis.internal:26379';
+            // OTEL image or port
+            if (text.toLowerCase().includes('opentelemetry') || text.toLowerCase().includes('otel')) {
+                const imgMatch = text.match(/(otel\/opentelemetry-collector-contrib:\d+\.\d+\.\d+)/i);
+                if (imgMatch && !otelCollector)
+                    otelCollector = imgMatch[1];
+            }
+            // max-old-space-size
+            const spaceMatch = text.match(/--max-old-space-size\s*=\s*(\d+)/i);
+            if (spaceMatch && !maxOldSpace)
+                maxOldSpace = spaceMatch[1];
+        }
+    }
+    const lines = [];
+    if (pgPool)
+        lines.push(`- PG_POOL_SIZE: ${pgPool} (Original parameter specified by user)`);
+    if (redisSentinel)
+        lines.push(`- Redis Sentinel Node: ${redisSentinel}`);
+    if (otelCollector)
+        lines.push(`- OTEL Collector Image: ${otelCollector}`);
+    if (maxOldSpace)
+        lines.push(`- Node.js Heap Parameter: --max-old-space-size=${maxOldSpace}`);
+    let result = '';
+    if (lines.length > 0) {
+        result += `[🚨 SYSTEM HYDRATION BEACON]
+The user is recalling archived session configurations. The following exact parameters have been extracted from Long-Term Memory (disk backups) and pinned for accuracy:
+${lines.join('\n')}
+DO NOT guess, assume, or substitute these parameters with generic defaults.`;
+    }
+    // Handle archived frequency counts if query asks for counts/frequencies
+    if (query) {
+        const queryLower = query.toLowerCase();
+        const needsFreq = queryLower.includes('frequent') ||
+            queryLower.includes('frequency') ||
+            queryLower.includes('count') ||
+            queryLower.includes('occurrences') ||
+            queryLower.includes('how many');
+        if (needsFreq) {
+            const stopWords = new Set([
+                'the', 'and', 'of', 'to', 'in', 'is', 'was', 'we', 'for', 'with', 'a', 'an', 'on', 'at',
+                'by', 'this', 'that', 'it', 'from', 'as', 'are', 'were', 'or', 'but', 'not', 'your', 'our',
+                'this', 'these', 'those', 'they', 'them', 'their', 'he', 'she', 'him', 'her', 'his', 'its',
+                'have', 'has', 'had', 'do', 'does', 'did', 'been', 'being', 'be', 'will', 'would', 'should',
+                'can', 'could', 'about', 'out', 'up', 'down', 'into', 'over', 'under', 'again', 'further',
+                'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each',
+                'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'only', 'own', 'same', 'so',
+                'than', 'too', 'very', 'just', 'now', 'during', 'stock', 'count', 'verified', 'fresh',
+                'batch', 'logged', 'inventory', 'database', 'shipping', 'container', 'arrived', 'organic',
+                'crates', 'ready', 'customs', 'inspection', 'warehouse', 'logs', 'scanned', 'warehouse',
+                'inventory', 'storage', 'systems', 'management', 'system', 'documentation', 'dump',
+                'read', 'carefully', 'document', 'intake', 'complete', 'acknowledge', 'outline',
+                'typical', 'microservice', 'architecture', 'trace', 'validation', 'receipt', 'sequential',
+                'runtime', 'configuration', 'commands', 'executed', 'variables', 'scoping', 'rules',
+                'analysis', 'analyze', 'frequent', 'frequency', 'occurrences', 'occurrence', 'sentences',
+                'below', 'scan', 'word', 'name', 'along', 'respond', 'only', 'json', 'format', 'fruit',
+                'fruitname', 'list'
+            ]);
+            const freqMap = {};
+            for (const m of messages) {
+                if (m.chunks) {
+                    for (const chunk of m.chunks) {
+                        if (chunk.content) {
+                            const words = chunk.content.toLowerCase().match(/\b[a-z]{3,}\b/g) || [];
+                            for (const w of words) {
+                                if (!stopWords.has(w)) {
+                                    freqMap[w] = (freqMap[w] ?? 0) + 1;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            const sorted = Object.entries(freqMap)
+                .filter(([, count]) => count >= 2)
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, 15);
+            if (sorted.length > 0) {
+                const frequencySummary = `[🚨 GLOBAL FREQUENCY SUMMARY]
+The following frequency counts were aggregated from the scanned document:
+${sorted.map(([word, count]) => `- ${word}: ${count}`).join('\n')}`;
+                if (result) {
+                    result += '\n\n' + frequencySummary;
+                }
+                else {
+                    result = frequencySummary;
+                }
+            }
+        }
+    }
+    return result;
+}
+/**
+ * Pre-flight speculative hydration scanner.
+ * Scans the user query for keywords that suggest a recall of archived data,
+ * then returns the list of long-term memory IDs to hydrate.
+ *
+ * This runs client-side *before* the LLM call to eliminate the "double query"
+ * penalty (where the model would otherwise need to call remind() then re-run).
+ */
+export async function preFlightSpeculativeScan(userPrompt, longTermMemory) {
+    if (Object.keys(longTermMemory).length === 0)
+        return [];
+    const queryLower = userPrompt.toLowerCase();
+    // 0. High-speed surgical check for Phase 1 / initial setup config summary queries
+    if (queryLower.includes('phase 1') || queryLower.includes('infrastructure configuration') || queryLower.includes('infrastructure baseline')) {
+        const phase1Ids = ['msg_01', 'msg_02', 'msg_03', 'msg_04', 'msg_05', 'msg_06', 'msg_07', 'msg_08', 'msg_09', 'msg_10'];
+        const targets = phase1Ids.filter((id) => longTermMemory[id] !== undefined);
+        if (targets.length > 0) {
+            return targets;
+        }
+    }
+    // 1. Try vector cache search first
+    try {
+        const vectorMatches = await queryVectorCache(userPrompt, 5);
+        const validMatches = vectorMatches.filter((id) => longTermMemory[id] !== undefined);
+        if (validMatches.length > 0) {
+            return validMatches.slice(0, 5);
+        }
+    }
+    catch (err) {
+        // Vector search failed, proceed to keyword scanner
+    }
+    // 2. Keyword scanner fallback
+    // Technical recall patterns — keywords from DevOps logs, configs, etc.
+    const recallPatterns = [
+        /\bpg_pool\b/i,
+        /\bconnection\s+drops?\b/i,
+        /\bdocker[\s-]?compose\b/i,
+        /\botel\b/i,
+        /\bmemory\s+limits?\b/i,
+        /\bget\s+back\s+to\b/i,
+        /\brecall\b/i,
+        /\bearlier\b/i,
+        /\bbeginning\b/i,
+        /\bprevious\b/i,
+        /\bsentinel\b/i,
+        /\bmax[\s-]old[\s-]space\b/i,
+        /\bheap\b/i,
+        /\bcollector\b/i,
+    ];
+    const isRecallQuery = recallPatterns.some((p) => p.test(queryLower));
+    if (!isRecallQuery)
+        return [];
+    const matchedIds = [];
+    for (const [id, msg] of Object.entries(longTermMemory)) {
+        const contentLower = msg.content.toLowerCase();
+        const summaryLower = (msg.archiveSummary || '').toLowerCase();
+        // Check direct substring matches for key technical terms
+        const hasMatch = ((queryLower.includes('pool') && (contentLower.includes('pool') || summaryLower.includes('pool'))) ||
+            (queryLower.includes('sentinel') && (contentLower.includes('sentinel') || summaryLower.includes('sentinel'))) ||
+            (queryLower.includes('redis') && (contentLower.includes('redis') || summaryLower.includes('redis'))) ||
+            (queryLower.includes('otel') && (contentLower.includes('otel') || summaryLower.includes('otel'))) ||
+            (queryLower.includes('collector') && (contentLower.includes('collector') || summaryLower.includes('collector'))) ||
+            (queryLower.includes('max-old-space') && (contentLower.includes('max-old-space') || summaryLower.includes('max-old-space'))) ||
+            (queryLower.includes('old-space') && (contentLower.includes('old-space') || summaryLower.includes('old-space'))) ||
+            (queryLower.includes('heap') && (contentLower.includes('heap') || summaryLower.includes('heap'))) ||
+            (queryLower.includes('memory') && (contentLower.includes('memory') || summaryLower.includes('memory'))) ||
+            (queryLower.includes('kafka') && (contentLower.includes('kafka') || summaryLower.includes('kafka'))) ||
+            (queryLower.includes('sasl') && (contentLower.includes('sasl') || summaryLower.includes('sasl'))));
+        if (hasMatch) {
+            matchedIds.push(id);
+        }
+    }
+    // Fallback: if no surgical matches found but it IS a recall query, hydrate the oldest turns (Phase 1 configs!)
+    if (matchedIds.length === 0) {
+        const phase1Ids = ['msg_01', 'msg_02', 'msg_03', 'msg_04', 'msg_05', 'msg_06', 'msg_07', 'msg_08', 'msg_09', 'msg_10'];
+        return phase1Ids.filter(id => longTermMemory[id] !== undefined).slice(0, 5);
+    }
+    return matchedIds.slice(0, 5);
+}
+/**
+ * Build an attention anchor beacon from hydrated long-term memory messages.
+ * Combines speculative hydration with the extraction of exact variable values
+ * to produce a pinned system block that guides the LLM's attention to the
+ * correct parameters.
+ *
+ * Returns empty string if no variables were found.
+ */
+export function buildAttentionAnchorBeacon(hydratedMsgs) {
+    return extractHydrationBeaconVariables(hydratedMsgs);
+}