@wangzhizhi/remi 0.0.1-alpha

package/node_modules/@remi/core/dist/contextBuilder.js

@@ -0,0 +1,344 @@
+import { createHeuristicTokenEstimator } from '@remi/llm';
+const defaultTokenEstimator = createHeuristicTokenEstimator();
+export function buildProviderContext(options) {
+    const tokenEstimator = options.tokenEstimator ?? defaultTokenEstimator;
+    const tokenEstimateProfile = options.tokenEstimateProfile ?? 'default';
+    const messages = [];
+    const layers = [];
+    addMessageLayer({
+        id: 'system',
+        source: 'prompt/base-system.md + response style',
+        messages,
+        layers,
+        layerMessages: [
+            {
+                role: 'system',
+                content: applyLayerBudget(options.systemPrompt, options.budgets?.system, options.tokenBudgets?.system, tokenEstimator, tokenEstimateProfile),
+            },
+        ],
+        budgetChars: options.budgets?.system,
+        budgetTokens: options.tokenBudgets?.system,
+        tokenEstimator,
+        tokenEstimateProfile,
+    });
+    if (options.toolPrompt && options.toolPrompt.trim().length > 0) {
+        addMessageLayer({
+            id: 'tool',
+            source: 'prompt/tool-use-system.md',
+            messages,
+            layers,
+            layerMessages: [
+                {
+                    role: 'system',
+                    content: applyLayerBudget(options.toolPrompt, options.budgets?.tool, options.tokenBudgets?.tool, tokenEstimator, tokenEstimateProfile),
+                },
+            ],
+            budgetChars: options.budgets?.tool,
+            budgetTokens: options.tokenBudgets?.tool,
+            tokenEstimator,
+            tokenEstimateProfile,
+        });
+    }
+    else {
+        addTrace(layers, {
+            id: 'tool',
+            source: 'tool prompt disabled for this turn',
+            included: false,
+            messageCount: 0,
+            charCount: 0,
+            tokenCount: 0,
+        }, options.budgets?.tool, options.tokenBudgets?.tool);
+    }
+    if (options.projectInstructions && options.projectInstructions.trim().length > 0) {
+        addMessageLayer({
+            id: 'project',
+            source: options.projectInstructionsSource ?? 'project instructions',
+            messages,
+            layers,
+            layerMessages: [
+                {
+                    role: 'system',
+                    content: applyLayerBudget(options.projectInstructions, options.budgets?.project, options.tokenBudgets?.project, tokenEstimator, tokenEstimateProfile),
+                },
+            ],
+            budgetChars: options.budgets?.project,
+            budgetTokens: options.tokenBudgets?.project,
+            tokenEstimator,
+            tokenEstimateProfile,
+        });
+    }
+    else {
+        addTrace(layers, {
+            id: 'project',
+            source: options.projectInstructionsSource ?? 'project instructions unavailable',
+            included: false,
+            messageCount: 0,
+            charCount: 0,
+            tokenCount: 0,
+        }, options.budgets?.project, options.tokenBudgets?.project);
+    }
+    const memoryPrompt = formatMemoryContext(options.memories ?? [], options.budgets?.memory, options.tokenBudgets?.memory, tokenEstimator, tokenEstimateProfile);
+    if (memoryPrompt) {
+        addMessageLayer({
+            id: 'memory',
+            source: '.agent/memory recall',
+            messages,
+            layers,
+            layerMessages: [{ role: 'system', content: memoryPrompt }],
+            budgetChars: options.budgets?.memory,
+            budgetTokens: options.tokenBudgets?.memory,
+            tokenEstimator,
+            tokenEstimateProfile,
+        });
+    }
+    else {
+        addTrace(layers, {
+            id: 'memory',
+            source: '.agent/memory recall',
+            included: false,
+            messageCount: 0,
+            charCount: 0,
+            tokenCount: 0,
+        }, options.budgets?.memory, options.tokenBudgets?.memory);
+    }
+    if (options.sessionMemorySummary && options.sessionMemorySummary.trim().length > 0) {
+        addMessageLayer({
+            id: 'session',
+            source: 'latest session memory summary',
+            messages,
+            layers,
+            layerMessages: [
+                {
+                    role: 'system',
+                    content: applyLayerBudget(options.sessionMemorySummary, options.budgets?.session, options.tokenBudgets?.session, tokenEstimator, tokenEstimateProfile),
+                },
+            ],
+            budgetChars: options.budgets?.session,
+            budgetTokens: options.tokenBudgets?.session,
+            tokenEstimator,
+            tokenEstimateProfile,
+        });
+    }
+    else {
+        addTrace(layers, {
+            id: 'session',
+            source: 'session memory unavailable',
+            included: false,
+            messageCount: 0,
+            charCount: 0,
+            tokenCount: 0,
+        }, options.budgets?.session, options.tokenBudgets?.session);
+    }
+    if (options.compactSummary && options.compactSummary.trim().length > 0) {
+        addMessageLayer({
+            id: 'compact',
+            source: 'latest compact summary',
+            messages,
+            layers,
+            layerMessages: [
+                {
+                    role: 'system',
+                    content: applyLayerBudget(options.compactSummary, options.budgets?.compact, options.tokenBudgets?.compact, tokenEstimator, tokenEstimateProfile),
+                },
+            ],
+            budgetChars: options.budgets?.compact,
+            budgetTokens: options.tokenBudgets?.compact,
+            tokenEstimator,
+            tokenEstimateProfile,
+        });
+    }
+    else {
+        addTrace(layers, {
+            id: 'compact',
+            source: 'compact summary unavailable',
+            included: false,
+            messageCount: 0,
+            charCount: 0,
+            tokenCount: 0,
+        }, options.budgets?.compact, options.tokenBudgets?.compact);
+    }
+    addMessageLayer({
+        id: 'recent',
+        source: '.agent/sessions recent user/assistant transcript',
+        messages,
+        layers,
+        layerMessages: applyMessagesBudget(options.recentMessages ?? [], options.budgets?.recent, options.tokenBudgets?.recent, tokenEstimator, tokenEstimateProfile),
+        budgetChars: options.budgets?.recent,
+        budgetTokens: options.tokenBudgets?.recent,
+        tokenEstimator,
+        tokenEstimateProfile,
+    });
+    addMessageLayer({
+        id: 'current',
+        source: 'current user input',
+        messages,
+        layers,
+        layerMessages: [
+            {
+                role: 'user',
+                content: applyLayerBudget(options.currentUserMessage, options.budgets?.current, options.tokenBudgets?.current, tokenEstimator, tokenEstimateProfile),
+            },
+        ],
+        budgetChars: options.budgets?.current,
+        budgetTokens: options.tokenBudgets?.current,
+        tokenEstimator,
+        tokenEstimateProfile,
+    });
+    const totalChars = messages.reduce((sum, message) => sum + message.content.length, 0);
+    const totalTokens = estimateMessagesTokensWith(messages, tokenEstimator, tokenEstimateProfile);
+    const budgetTokens = options.effectiveInputBudgetTokens ?? options.contextWindowTokens;
+    const usedPercent = budgetTokens && budgetTokens > 0
+        ? totalTokens > 0
+            ? Math.max(1, Math.min(100, Math.ceil((totalTokens / budgetTokens) * 100)))
+            : 0
+        : undefined;
+    return {
+        messages,
+        trace: {
+            layers,
+            totalMessages: messages.length,
+            totalChars,
+            totalTokens,
+            ...(options.contextWindowTokens !== undefined ? { contextWindowTokens: options.contextWindowTokens } : {}),
+            ...(options.effectiveInputBudgetTokens !== undefined ? { effectiveInputBudgetTokens: options.effectiveInputBudgetTokens } : {}),
+            ...(options.autoCompactThresholdTokens !== undefined ? { autoCompactThresholdTokens: options.autoCompactThresholdTokens } : {}),
+            tokenEstimatorKind: options.tokenEstimatorKind ?? tokenEstimator.kind,
+            tokenEstimatorRequestedKind: options.tokenEstimatorRequestedKind ?? options.tokenEstimatorKind ?? tokenEstimator.kind,
+            tokenEstimateProfile,
+            ...(options.tokenEstimatorFallback !== undefined ? { tokenEstimatorFallback: options.tokenEstimatorFallback } : {}),
+            ...(options.tokenEstimatorFallbackReason !== undefined ? { tokenEstimatorFallbackReason: options.tokenEstimatorFallbackReason } : {}),
+            ...(usedPercent !== undefined ? { usedPercent, remainingPercent: 100 - usedPercent } : {}),
+            truncated: layers.some(layer => layer.budgetChars !== undefined && layer.charCount >= layer.budgetChars) ||
+                layers.some(layer => layer.budgetTokens !== undefined && layer.tokenCount >= layer.budgetTokens),
+        },
+    };
+}
+function addMessageLayer(options) {
+    const included = options.layerMessages.length > 0;
+    options.messages.push(...options.layerMessages);
+    const content = options.layerMessages.map(message => message.content).join('\n');
+    addTrace(options.layers, {
+        id: options.id,
+        source: options.source,
+        included,
+        messageCount: options.layerMessages.length,
+        charCount: content.length,
+        tokenCount: estimateTextTokens(content, options.tokenEstimator, options.tokenEstimateProfile),
+    }, options.budgetChars, options.budgetTokens);
+}
+function addTrace(layers, trace, budgetChars, budgetTokens) {
+    layers.push({
+        ...trace,
+        ...(budgetChars === undefined ? {} : { budgetChars }),
+        ...(budgetTokens === undefined ? {} : { budgetTokens }),
+    });
+}
+function formatMemoryContext(memories, budgetChars, budgetTokens, tokenEstimator, tokenEstimateProfile) {
+    if (memories.length === 0) {
+        return undefined;
+    }
+    const body = [
+        '## Relevant Long-Term Memory',
+        '',
+        'Use these memories only when they are relevant to the current task. Explicit user instructions override memory.',
+        'Memory records are point-in-time observations. Verify file paths, function names, flags, branch state, and other current-state claims against the live repo before acting on them.',
+        '',
+        ...memories.map(memory => [
+            `### ${memory.id} (${memory.type})`,
+            `Description: ${memory.description}`,
+            `Updated: ${memory.updatedAt}`,
+            `Reason: ${memory.reason}`,
+            '',
+            memory.body.trim(),
+        ].join('\n')),
+    ].join('\n\n');
+    return applyLayerBudget(body, budgetChars, budgetTokens, tokenEstimator, tokenEstimateProfile);
+}
+function applyLayerBudget(text, budgetChars, budgetTokens, tokenEstimator, tokenEstimateProfile) {
+    let result = text;
+    if (budgetChars !== undefined && result.length > budgetChars) {
+        result = truncateText(result, budgetChars, '[Context layer truncated.]');
+    }
+    if (budgetTokens !== undefined && estimateTextTokens(result, tokenEstimator, tokenEstimateProfile) > budgetTokens) {
+        result = truncateTextToTokenBudget(result, budgetTokens, tokenEstimator, tokenEstimateProfile);
+    }
+    return result;
+}
+function applyMessagesBudget(messages, budgetChars, budgetTokens, tokenEstimator, tokenEstimateProfile) {
+    if (messages.length === 0 || (budgetChars === undefined && budgetTokens === undefined)) {
+        return messages;
+    }
+    const selected = [];
+    let chars = 0;
+    let tokens = 0;
+    for (let index = messages.length - 1; index >= 0; index -= 1) {
+        const message = messages[index];
+        if (!message) {
+            continue;
+        }
+        const messageChars = message.content.length;
+        const messageTokens = estimateMessageTokens(message, tokenEstimator, tokenEstimateProfile);
+        const wouldExceedChars = budgetChars !== undefined && chars + messageChars > budgetChars;
+        const wouldExceedTokens = budgetTokens !== undefined && tokens + messageTokens > budgetTokens;
+        if (wouldExceedChars || wouldExceedTokens) {
+            const remainingChars = budgetChars === undefined ? undefined : Math.max(0, budgetChars - chars);
+            const remainingTokens = budgetTokens === undefined ? undefined : Math.max(0, budgetTokens - tokens);
+            if (selected.length === 0 && (remainingChars === undefined || remainingChars > 64) && (remainingTokens === undefined || remainingTokens > 16)) {
+                selected.unshift({
+                    ...message,
+                    content: applyLayerBudget(message.content, remainingChars, remainingTokens, tokenEstimator, tokenEstimateProfile),
+                });
+            }
+            break;
+        }
+        selected.unshift(message);
+        chars += messageChars;
+        tokens += messageTokens;
+    }
+    return selected;
+}
+export function estimateMessagesTokens(messages) {
+    return estimateMessagesTokensWith(messages, defaultTokenEstimator, 'default');
+}
+export function estimateMessagesTokensWith(messages, tokenEstimator, profile) {
+    return messages.reduce((sum, message) => sum + estimateMessageTokens(message, tokenEstimator, profile), 0);
+}
+function estimateMessageTokens(message, tokenEstimator = defaultTokenEstimator, profile = 'default') {
+    return estimateTextTokens(`${message.role}\n${message.content}`, tokenEstimator, profile);
+}
+function estimateTextTokens(text, tokenEstimator = defaultTokenEstimator, profile = 'default') {
+    return tokenEstimator.estimateText({ text, profile }).tokens;
+}
+function truncateTextToTokenBudget(text, budgetTokens, tokenEstimator = defaultTokenEstimator, profile = 'default') {
+    if (budgetTokens <= 0) {
+        return '';
+    }
+    if (estimateTextTokens(text, tokenEstimator, profile) <= budgetTokens) {
+        return text;
+    }
+    let low = 0;
+    let high = text.length;
+    while (low < high) {
+        const mid = Math.floor((low + high + 1) / 2);
+        const candidate = truncateText(text, mid, '[Context layer truncated.]');
+        if (estimateTextTokens(candidate, tokenEstimator, profile) <= budgetTokens) {
+            low = mid;
+        }
+        else {
+            high = mid - 1;
+        }
+    }
+    return truncateText(text, low, '[Context layer truncated.]');
+}
+function truncateText(text, maxLength, marker) {
+    if (maxLength <= 0) {
+        return '';
+    }
+    if (text.length <= maxLength) {
+        return text;
+    }
+    if (maxLength <= marker.length + 1) {
+        return text.slice(0, maxLength);
+    }
+    return `${text.slice(0, maxLength - marker.length - 1)}\n${marker}`;
+}