mixdog 0.9.2 → 0.9.4

package/src/runtime/agent/orchestrator/session/manager/tool-resolution.mjs

@@ -0,0 +1,303 @@
+// Session tool schema resolution + permission narrowing, extracted verbatim
+// from manager.mjs. Behavior-preserving: same toolset ids, same ordering,
+// same dedup, same fail-closed permission intersection.
+import { getMcpTools } from '../../mcp/client.mjs';
+import { getInternalTools } from '../../internal-tools.mjs';
+import { BUILTIN_TOOLS } from '../../tools/builtin/builtin-tools.mjs';
+import { PATCH_TOOL_DEFS } from '../../tools/patch-tool-defs.mjs';
+import { CODE_GRAPH_TOOL_DEFS } from '../../tools/code-graph-tool-defs.mjs';
+import { buildSkillToolDefs } from '../../context/collect.mjs';
+import { getHiddenAgent, listHiddenAgentNames } from '../../internal-agents.mjs';
+
+// Merge externally-connected MCP tools with the plugin's in-process tools
+// (registered by agent's toolExecutor adapter). Internal tools are exposed
+// under their bare names — no mcp__ prefix, since the dispatcher in
+// server.mjs handles them directly without a transport.
+// Sorted deterministically by name — protects BP_1 hash stability from
+// listTools() ordering churn. Anthropic / OpenAI / Gemini all hash the
+// tools array verbatim, so any reorder rewrites the prefix.
+// No cache: getMcpTools() and getInternalTools() are O(n) in-memory reads;
+// the sort overhead on ~30 tools is negligible.
+function _getMcpTools() {
+    const mcp = getMcpTools() || [];
+    const internalRaw = getInternalTools() || [];
+    const internal = internalRaw.map(t => ({
+        name: t.name,
+        description: typeof t.description === 'string' ? t.description : '',
+        inputSchema: t.inputSchema || { type: 'object', properties: {} },
+        // Keep annotations so the permission filter / role invariants can
+        // tell read-only from write-capable internal tools, and so
+        // agentHidden can be read during deny filtering.
+        annotations: t.annotations || {},
+    }));
+    return [...mcp, ...internal].sort((a, b) => {
+        const an = a?.name || '';
+        const bn = b?.name || '';
+        return an < bn ? -1 : an > bn ? 1 : 0;
+    });
+}
+
+const SESSION_ROUTE_TOOL_ORDER = [
+    'code_graph',
+    'find',
+    'glob',
+    'list',
+    'grep',
+    'read',
+    'apply_patch',
+    'shell',
+    'task',
+];
+const SESSION_ROUTE_TOOL_RANK = new Map(SESSION_ROUTE_TOOL_ORDER.map((name, index) => [name, index]));
+const FILESYSTEM_TOOL_NAMES = new Set([
+    'code_graph',
+    'find',
+    'glob',
+    'list',
+    'grep',
+    'read',
+    'apply_patch',
+]);
+const READONLY_TOOL_NAMES = new Set([
+    'code_graph',
+    'find',
+    'glob',
+    'list',
+    'grep',
+    'read',
+]);
+
+const AGENT_STRING_PERMISSION_READ_ALLOW = Object.freeze([
+    'code_graph',
+    'find',
+    'glob',
+    'list',
+    'grep',
+    'read',
+    'explore',
+    'search',
+    'web_fetch',
+    'Skill',
+]);
+
+function stringToolPermissionAllowList(toolPermission) {
+    if (toolPermission === 'read') return AGENT_STRING_PERMISSION_READ_ALLOW;
+    if (toolPermission === 'read-write') return AGENT_STRING_PERMISSION_READ_WRITE_ALLOW;
+    if (toolPermission === 'none') return [];
+    return null;
+}
+
+const AGENT_STRING_PERMISSION_READ_WRITE_ALLOW = Object.freeze([
+    'code_graph',
+    'find',
+    'glob',
+    'list',
+    'grep',
+    'read',
+    'apply_patch',
+    'explore',
+    'search',
+    'web_fetch',
+    'Skill',
+]);
+
+export function applyToolPermissionNarrowing(tools, toolPermission, warnRole = null) {
+    if (toolPermission === 'none') return [];
+    const allowList = stringToolPermissionAllowList(toolPermission);
+    if (allowList) {
+        const allowSet = new Set(allowList.map((n) => String(n).toLowerCase()));
+        return tools.filter((t) => allowSet.has(String(t?.name || '').toLowerCase()));
+    }
+    if (toolPermission && typeof toolPermission === 'object') {
+        const allowSet = Array.isArray(toolPermission.allow) && toolPermission.allow.length > 0
+            ? new Set(toolPermission.allow.map(n => String(n).toLowerCase()))
+            : null;
+        const denySet = Array.isArray(toolPermission.deny) && toolPermission.deny.length > 0
+            ? new Set(toolPermission.deny.map(n => String(n).toLowerCase()))
+            : null;
+        if (allowSet || denySet) {
+            const filtered = tools.filter(t => {
+                const name = String(t?.name || '').toLowerCase();
+                if (denySet && denySet.has(name)) return false;
+                if (allowSet && !allowSet.has(name)) return false;
+                return true;
+            });
+            if (filtered.length === 0) {
+                process.stderr.write(`[session] WARN: role permission intersection produced 0 tools — failing closed (role=${warnRole || 'unknown'})\n`);
+            }
+            return filtered;
+        }
+    }
+    return tools;
+}
+
+function recursiveWrapperToolNameForPublicAgent(agent) {
+    if (!agent) return null;
+    const key = String(agent).trim();
+    if (key === 'explore') return 'explore';
+    for (const hiddenName of listHiddenAgentNames()) {
+        const def = getHiddenAgent(hiddenName);
+        const invokedBy = typeof def?.invokedBy === 'string' ? def.invokedBy.trim() : '';
+        if (hiddenName === key && invokedBy) return invokedBy;
+        if (invokedBy && invokedBy === key) return invokedBy;
+    }
+    return null;
+}
+
+export function finalizeSessionToolList(tools, {
+    schemaAllowedTools = null,
+    disallowedTools = null,
+    ownerIsAgent = false,
+    resolvedAgent = null,
+} = {}) {
+    let out = Array.isArray(tools) ? tools : [];
+    const hasCallerAllow = Array.isArray(schemaAllowedTools);
+    if (hasCallerAllow) {
+        const allowSet = new Set(schemaAllowedTools.map(n => String(n).toLowerCase()));
+        out = out.filter(t => allowSet.has(String(t?.name || '').toLowerCase()));
+    }
+    const callerDeny = Array.isArray(disallowedTools) ? disallowedTools.map(n => String(n)) : [];
+    if (callerDeny.length) {
+        const denySet = new Set(callerDeny.map(n => n.toLowerCase()));
+        out = out.filter(t => !denySet.has(String(t?.name || '').toLowerCase()));
+    }
+    const recursiveDeny = ownerIsAgent ? recursiveWrapperToolNameForPublicAgent(resolvedAgent) : null;
+    if (recursiveDeny) {
+        const deny = recursiveDeny.toLowerCase();
+        out = out.filter(t => String(t?.name || '').toLowerCase() !== deny);
+    }
+    if (ownerIsAgent) {
+        out = out.filter(t => !t?.annotations?.agentHidden);
+        out = orderSessionTools(out);
+    }
+    return out;
+}
+
+function orderSessionTools(tools) {
+    return tools.map((tool, index) => ({ tool, index }))
+        .sort((a, b) => {
+            const ar = SESSION_ROUTE_TOOL_RANK.get(a.tool?.name) ?? 10_000;
+            const br = SESSION_ROUTE_TOOL_RANK.get(b.tool?.name) ?? 10_000;
+            if (ar !== br) return ar - br;
+            return a.index - b.index;
+        })
+        .map((entry) => entry.tool);
+}
+
+const ALL_BUILTIN_SESSION_TOOLS = orderSessionTools(_dedupByName([
+    ...BUILTIN_TOOLS,
+    ...PATCH_TOOL_DEFS,
+    ...CODE_GRAPH_TOOL_DEFS,
+]));
+
+export function resolveSessionTools(toolSpec, skills, { ownerIsAgentSession = false } = {}) {
+    const mcp = _getMcpTools();
+    // Agent sessions freeze the skill meta-tool into the schema
+    // unconditionally — concrete skill resolution is cwd-scoped at tool-call
+    // time (loop.mjs), so the schema bytes stay bit-identical across roles /
+    // cwds and the provider cache shard does not fragment.
+    const skillTools = buildSkillToolDefs(skills, { ownerIsAgentSession });
+    return _computeBaseTools(toolSpec, mcp, skillTools);
+}
+
+export function previewSessionTools(toolSpec, skills = [], options = {}) {
+    return resolveSessionTools(toolSpec, skills, options);
+}
+
+// Dedup by name, first occurrence wins. BUILTIN_TOOLS is passed in ahead
+// of the MCP-registered internal tools so plugin-side definitions take
+// precedence when both surfaces declare the same name (e.g. read / grep / glob).
+// Without this merge, Anthropic rejected the request with
+// "tools: Tool names must be unique" and the orchestrator burned up to
+// 20 iterations retrying before the final answer landed.
+function _dedupByName(tools) {
+    const seen = new Map();
+    for (const t of tools) {
+        const n = t?.name;
+        if (!n || seen.has(n)) continue;
+        seen.set(n, t);
+    }
+    return [...seen.values()];
+}
+
+// Agent visibility is declared per-tool via annotations.agentHidden.
+// Tools with agentHidden:true are stripped from agent sessions at schema
+// build time (see deny filtering below). No code-level name list needed.
+
+function _computeBaseTools(toolSpec, mcp, skillTools) {
+    if (Array.isArray(toolSpec)) {
+        if (toolSpec.length === 0) {
+            // Explicit "no tools" — skill meta tools still travel so the model
+            // can at least discover and invoke skills if that is the one
+            // dynamic surface the profile retains.
+            return _dedupByName([...skillTools]);
+        }
+        if (toolSpec.includes('full')) {
+            return _dedupByName([...ALL_BUILTIN_SESSION_TOOLS, ...mcp, ...skillTools]);
+        }
+        const byName = new Map();
+        const add = (tool) => { if (tool?.name && !byName.has(tool.name)) byName.set(tool.name, tool); };
+        const addMany = (arr) => { for (const t of arr) add(t); };
+        for (const tagRaw of toolSpec) {
+            const tag = String(tagRaw || '').trim();
+            switch (tag) {
+                case 'tools:filesystem':
+                    addMany(ALL_BUILTIN_SESSION_TOOLS.filter(t => FILESYSTEM_TOOL_NAMES.has(t.name)));
+                    break;
+                case 'tools:readonly':
+                    addMany(ALL_BUILTIN_SESSION_TOOLS.filter(t => READONLY_TOOL_NAMES.has(t.name)));
+                    break;
+                case 'tools:shell':
+                case 'tools:git':
+                case 'tools:analysis':
+                    // Shell-class toolset. `tools:git` / `tools:analysis` exist so
+                    // profile authors can name the intent (git workflows / data
+                    // analysis) without inventing new toolset ids.
+                    addMany(ALL_BUILTIN_SESSION_TOOLS.filter(t => t.name === 'shell' || t.name === 'task'));
+                    break;
+                case 'tools:mcp':
+                    addMany(mcp);
+                    break;
+                case 'tools:search':
+                    // Name-pattern match: picks up `search` and any future tool
+                    // whose name contains `search`. `recall` and `explore` deliberately do NOT match
+                    // — they need `tools:mcp` (full mcp surface) or their own
+                    // toolset id if a role wants targeted retrieval. Public agent
+                    // roles never reach the wrapper bodies regardless: see the
+                    // isBlockedPublicWrapperCall guard in session/loop.mjs.
+                    addMany(mcp.filter(t => /search/i.test(t?.name || '')));
+                    break;
+                default:
+                    process.stderr.write(`[session] unknown toolset id "${tag}" (profile.tools); skipping\n`);
+            }
+        }
+        return _dedupByName([...byName.values(), ...skillTools]);
+    }
+
+    switch (toolSpec) {
+        case 'mcp':
+            return _dedupByName([...mcp, ...skillTools]);
+        case 'readonly': {
+            const readTools = ALL_BUILTIN_SESSION_TOOLS.filter(t => READONLY_TOOL_NAMES.has(t.name));
+            return _dedupByName([...readTools, ...mcp, ...skillTools]);
+        }
+        case 'full':
+        default:
+            return _dedupByName([...ALL_BUILTIN_SESSION_TOOLS, ...mcp, ...skillTools]);
+    }
+}
+
+export function permissionFromToolSpec(toolSpec) {
+    if (toolSpec === 'readonly') return 'read';
+    if (toolSpec === 'mcp') return 'mcp';
+    if (Array.isArray(toolSpec)) {
+        const tags = new Set(toolSpec.map(t => String(t || '').trim()));
+        const hasWriteOrShell = tags.has('full')
+            || tags.has('tools:filesystem')
+            || tags.has('tools:shell')
+            || tags.has('tools:git')
+            || tags.has('tools:analysis');
+        if (tags.has('tools:readonly') && !hasWriteOrShell) return 'read';
+    }
+    return null;
+}

package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs

@@ -0,0 +1,210 @@
+// Incremental + terminal usage-metrics accounting for sessions.
+// Extracted verbatim from manager.mjs (behavior-preserving). Pure helpers plus
+// per-session idempotency tracking for incremental usage persistence.
+//
+// Runtime coupling is injected: persistIterationMetrics needs the live
+// _runtimeState map (owned by manager.mjs) to read the in-memory session and
+// flag usageMetricsTurnIncremental. manager.mjs wires it via configureUsageMetricsRuntime().
+import { providerInputExcludesCache } from '../../providers/registry.mjs';
+import { loadSession, saveSessionAsync } from '../store.mjs';
+
+// Per-session idempotency tracking: sessionId → Set of seen
+// turn:epoch:iteration:source keys.
+const _metricSeenIter = new Map();
+
+// Injected accessor for manager's _runtimeState. Defaults to a no-op lookup so
+// the pure helpers remain usable (and unit-testable) before wiring.
+let _getRuntimeEntry = () => null;
+
+/** Wire the live runtime-state accessor from manager.mjs. */
+export function configureUsageMetricsRuntime({ getRuntimeEntry } = {}) {
+    if (typeof getRuntimeEntry === 'function') _getRuntimeEntry = getRuntimeEntry;
+}
+
+/** Drop the per-session metric-idempotency Set (called on session close). */
+export function dropMetricSeenState(sessionId) {
+    if (sessionId) _metricSeenIter.delete(sessionId);
+}
+
+/** Monotonic per-session ask/turn id for incremental usage idempotency. */
+export function bumpUsageMetricsTurnId(session) {
+    if (!session || typeof session !== 'object') return 0;
+    const next = (Number(session.usageMetricsTurnId) || 0) + 1;
+    session.usageMetricsTurnId = next;
+    return next;
+}
+
+export function resolveUsageMetricsTurnId(session, delta = {}) {
+    if (delta.usageMetricsTurnId != null && Number.isFinite(Number(delta.usageMetricsTurnId))) {
+        return Number(delta.usageMetricsTurnId);
+    }
+    return Number(session?.usageMetricsTurnId) || 0;
+}
+
+/** Advance loop metrics epoch when agentLoop resets its iteration counter (post-compact). */
+export function bumpUsageMetricsEpoch(session) {
+    if (!session || typeof session !== 'object') return 0;
+    const next = (Number(session.usageMetricsEpoch) || 0) + 1;
+    session.usageMetricsEpoch = next;
+    return next;
+}
+
+/**
+ * Resolve usage-metrics epoch for idempotency (exported for regression smoke).
+ * Prefers session.usageMetricsEpoch (bumped in loop on compact reset) and optional
+ * delta.usageMetricsEpoch; falls back to iteration regression when loop did not bump.
+ */
+export function resolveUsageMetricsEpoch(session, delta = {}) {
+    if (!session) return 0;
+    let epoch = Number(session.usageMetricsEpoch) || 0;
+    if (delta.usageMetricsEpoch != null && Number.isFinite(Number(delta.usageMetricsEpoch))) {
+        epoch = Math.max(epoch, Number(delta.usageMetricsEpoch));
+    }
+    const idx = Number(delta.iterationIndex);
+    const prevLastIdx = typeof session.lastIterationIndex === 'number'
+        ? session.lastIterationIndex
+        : null;
+    if (
+        (delta.usageMetricsEpoch == null || !Number.isFinite(Number(delta.usageMetricsEpoch)))
+        && prevLastIdx !== null
+        && Number.isFinite(idx)
+        && idx < prevLastIdx
+    ) {
+        epoch += 1;
+    }
+    return epoch;
+}
+
+export function usageMetricsSourceKey(delta = {}) {
+    const raw = delta.source ?? delta.usageSource;
+    if (raw == null || raw === '') return 'provider_send';
+    return String(raw);
+}
+
+/** Idempotency key for incremental usage persistence (exported for regression smoke). */
+export function usageMetricsIdempotencyKey(sessionId, session, delta = {}) {
+    const turnId = resolveUsageMetricsTurnId(session, delta);
+    const epoch = resolveUsageMetricsEpoch(session, delta);
+    const source = usageMetricsSourceKey(delta);
+    return `${sessionId}:${turnId}:${epoch}:${delta.iterationIndex}:${source}`;
+}
+
+export function uncachedInputTokensForProvider(provider, inputTokens, cachedReadTokens = 0, cacheWriteTokens = 0) {
+    const input = Number(inputTokens) || 0;
+    if (input <= 0) return 0;
+    // Anthropic-style providers report input_tokens excluding cache reads; OpenAI
+    // Responses/Gemini-style providers report input_tokens inclusive of cached
+    // prefix tokens. Keep both views so UI can show the real context footprint
+    // and the fresh/new token portion without mistaking cache hits for a cache
+    // break.
+    if (providerInputExcludesCache(provider)) return input + (Number(cacheWriteTokens) || 0);
+    return Math.max(input - (Number(cachedReadTokens) || 0) - (Number(cacheWriteTokens) || 0), 0);
+}
+
+/**
+ * Apply terminal ask usage to session totals. Skips lifetime totals when incremental
+ * per-iteration persistence already counted this turn (askSession path).
+ */
+export function applyAskTerminalUsageTotals(session, result, options = {}) {
+    if (!session || !result?.usage) return;
+    const skipTotals = options.skipTotalsIfIncremental === true;
+    if (!skipTotals) {
+        const inputTokens = result.usage.inputTokens || 0;
+        const outputTokens = result.usage.outputTokens || 0;
+        const cachedTokens = result.usage.cachedTokens || 0;
+        const cacheWriteTokens = result.usage.cacheWriteTokens || 0;
+        const uncachedInputTokens = uncachedInputTokensForProvider(session.provider, inputTokens, cachedTokens, cacheWriteTokens);
+        session.totalInputTokens = (session.totalInputTokens || 0) + inputTokens;
+        session.totalOutputTokens = (session.totalOutputTokens || 0) + outputTokens;
+        session.tokensCumulative = (session.tokensCumulative || 0)
+            + inputTokens
+            + outputTokens;
+        session.totalCachedReadTokens = (session.totalCachedReadTokens || 0) + cachedTokens;
+        session.totalCacheWriteTokens = (session.totalCacheWriteTokens || 0) + cacheWriteTokens;
+        session.totalUncachedInputTokens = (session.totalUncachedInputTokens || 0) + uncachedInputTokens;
+    }
+    const _lastTurn = result.lastTurnUsage || result.usage || {};
+    const _lastInputTokens = _lastTurn.inputTokens || 0;
+    const _lastCachedReadTokens = _lastTurn.cachedTokens || 0;
+    const _lastCacheWriteTokens = _lastTurn.cacheWriteTokens || 0;
+    session.lastInputTokens = _lastInputTokens;
+    session.lastOutputTokens = _lastTurn.outputTokens || 0;
+    session.lastCachedReadTokens = _lastCachedReadTokens;
+    session.lastCacheWriteTokens = _lastCacheWriteTokens;
+    session.lastUncachedInputTokens = uncachedInputTokensForProvider(
+        session.provider,
+        _lastInputTokens,
+        _lastCachedReadTokens,
+        _lastCacheWriteTokens,
+    );
+    const _inputExcludesCache = providerInputExcludesCache(session.provider);
+    session.lastContextTokens = _inputExcludesCache
+        ? _lastInputTokens + _lastCachedReadTokens + _lastCacheWriteTokens
+        : _lastInputTokens;
+    session.lastContextTokensUpdatedAt = Date.now();
+    session.lastContextTokensStaleAfterCompact = false;
+}
+
+/**
+ * Persist incremental usage delta immediately after each provider.send iteration.
+ * Idempotency key `sessionId:turnId:epoch:iterationIndex:source` scopes retries
+ * per ask, compaction epoch, iteration, and usage source.
+ */
+export async function persistIterationMetrics(delta) {
+    if (!delta || !delta.sessionId) return;
+    const { sessionId, iterationIndex, deltaInput, deltaOutput, deltaCachedRead, deltaCacheWrite, ts } = delta;
+    const runtimeEntry = _getRuntimeEntry(sessionId);
+    const session = runtimeEntry?.session ?? loadSession(sessionId);
+    if (!session || session.closed) return;
+    const epoch = resolveUsageMetricsEpoch(session, delta);
+    if (epoch !== (Number(session.usageMetricsEpoch) || 0)) {
+        session.usageMetricsEpoch = epoch;
+    }
+    let seen = _metricSeenIter.get(sessionId);
+    if (!seen) {
+        seen = new Set();
+        _metricSeenIter.set(sessionId, seen);
+    }
+    const ikey = usageMetricsIdempotencyKey(sessionId, session, delta);
+    const isReplay = seen.has(ikey);
+    seen.add(ikey);
+    if (!isReplay) {
+        if (runtimeEntry) runtimeEntry.usageMetricsTurnIncremental = true;
+        const deltaUncachedInput = delta.deltaUncachedInput != null
+            ? Number(delta.deltaUncachedInput) || 0
+            : uncachedInputTokensForProvider(session.provider, deltaInput, deltaCachedRead, deltaCacheWrite);
+        session.totalInputTokens = (session.totalInputTokens || 0) + (deltaInput || 0);
+        session.totalOutputTokens = (session.totalOutputTokens || 0) + (deltaOutput || 0);
+        session.tokensCumulative = (session.tokensCumulative || 0) + (deltaInput || 0) + (deltaOutput || 0);
+        // Cache totals — additive fields, default 0 on legacy sessions; both
+        // are undefined-safe so the schema migrates lazily as new iterations
+        // land. Keeps live + terminal aggregates in lock-step (loop.mjs already
+        // includes cached_read / cache_write in its terminal usage rollup).
+        session.totalCachedReadTokens = (session.totalCachedReadTokens || 0) + (deltaCachedRead || 0);
+        session.totalCacheWriteTokens = (session.totalCacheWriteTokens || 0) + (deltaCacheWrite || 0);
+        session.totalUncachedInputTokens = (session.totalUncachedInputTokens || 0) + deltaUncachedInput;
+        // Window snapshot updated per iteration so agent type=list reflects the
+        // most-recent provider-reported input size even for short dispatches
+        // that finish before askSession's terminal save lands.
+        session.lastInputTokens = deltaInput || 0;
+        session.lastOutputTokens = deltaOutput || 0;
+        session.lastCachedReadTokens = deltaCachedRead || 0;
+        session.lastCacheWriteTokens = deltaCacheWrite || 0;
+        session.lastUncachedInputTokens = deltaUncachedInput;
+        // Normalized last-call context footprint: how many prompt tokens the
+        // model actually saw on the most-recent send, comparable ACROSS
+        // providers. Anthropic reports input_tokens EXCLUDING cache (cache_read
+        // is a separate field), so the cached portion must be added back to
+        // reflect real context size; openai/grok/gemini already fold cached
+        // tokens INTO the input count, so input alone is the footprint.
+        const _inputExcludesCache = providerInputExcludesCache(session.provider);
+        session.lastContextTokens = _inputExcludesCache
+            ? (deltaInput || 0) + (deltaCachedRead || 0) + (deltaCacheWrite || 0)
+            : (deltaInput || 0);
+        session.lastContextTokensUpdatedAt = ts || Date.now();
+        session.lastContextTokensStaleAfterCompact = false;
+    }
+    session.lastIterationIndex = iterationIndex;
+    session.updatedAt = ts || Date.now();
+    await saveSessionAsync(session, { expectedGeneration: session.generation });
+}