@zenithfoundry/slm-gate 1.2.1

package/dist/llm-gate/local-first.js

@@ -0,0 +1,217 @@
+/**
+ * @fileoverview Step A: the local model answers the first request of a conversation when that is safe and
+ * it can. `attemptLocalAnswer` is the one implementation: the gate calls it through
+ * `answerFirstRequestLocally` (budget, cancellation, one attempt at a time) and the benchmark harness calls
+ * it through `processPipeline`, so the bench measures what ships.
+ *
+ * A wrong local answer is worse than a cloud call — the coding tool trusts the reply — so eligibility is
+ * narrow when the request lists tools (it nearly always does): only general questions and formatting, and
+ * never when the typed text mentions the workspace or asks about the assistant itself. The local model sees
+ * only the typed text, not the tool's system prompt or injected instructions.
+ */
+import { Ollama } from 'ollama';
+import { checkSemanticCache, setSemanticCache } from '../cache/index.js';
+import { CONFIG } from '../config.js';
+import { getDb } from '../ledger/index.js';
+import { handleSlmError } from '../models/helpers.js';
+import { classify } from '../models/reasoning.js';
+import { SLM } from '../models/slm.js';
+import { verify } from '../verifier/index.js';
+// The classifier's categories the local model may attempt; mirrors the schema in models/reasoning.ts
+// (minus 'other'). Extending it means extending the classify() prompt too.
+const ALLOWED = ['classify', 'extract', 'format', 'boolean', 'short_factual', 'trivial_edit'];
+// With tools listed, only answers that cannot depend on the workspace or need a tool.
+const ALLOWED_WITH_TOOLS = ['short_factual', 'format'];
+const WORKSPACE_WORDS = /\b(?:repos?|repository|project|codebase|code|files?|folders?|director(?:y|ies)|dir|functions?|methods?|class(?:es)?|modules?|packages?|tests?|build|branch(?:es)?|commits?|diff|errors?|bugs?|config|scripts?|dependenc(?:y|ies)|workspace|components?|variables?)\b/i;
+const FILE_NAME = /\b[\w.-]+\.(?:ts|tsx|js|jsx|mjs|cjs|json|md|py|go|rs|java|kt|rb|php|cs|cpp|c|h|swift|ya?ml|toml|lock|sh|css|scss|html|sql|env|txt|xml|gradle|ini|cfg)\b/i;
+const PATH = /(?:^|\s)[.~]?\/\S|\b[\w-]+\/[\w./-]+/;
+const IDENTIFIER = /\b[a-z]+[A-Z]\w*\b|\b[A-Za-z]+_\w+\b|\b\w+\(\)/;
+const ASKS_ABOUT_ASSISTANT = /\b(?:who|what)\s+(?:are|r)\s+(?:you|u)\b|\byour\s+(?:name|model|version|capabilit\w*|purpose|creator)\b|\bwhat\s+can\s+you\s+do\b|\b(?:what|which)\s+model\b|\byourself\b|\bare\s+you\s+(?:an?\s+)?(?:ai|bot|claude|gpt|gemini|human)\b|^\s*\/?help\b/i;
+/** True when the text refers to files, code or the project, which only the tool's own model can look at. */
+export function mentionsWorkspace(text) {
+    return text.includes('`') || WORKSPACE_WORDS.test(text) || FILE_NAME.test(text) || PATH.test(text) || IDENTIFIER.test(text);
+}
+/** True for identity and capability questions, which only the tool's own model can answer as itself. */
+export function asksAboutAssistant(text) {
+    return ASKS_ABOUT_ASSISTANT.test(text);
+}
+/**
+ * Share of recent local attempts in a category that the verifier accepted, counting only rows of the same
+ * environment so the benchmark and real traffic never train each other (no environment known: all rows).
+ * Null below `minSamples`.
+ */
+export function getCategorySuccessRate(params) {
+    try {
+        const rows = getDb().prepare(`
+      SELECT json_extract(meta, '$.local_accepted') as accepted
+      FROM events
+      WHERE layer = 'llm'
+        AND json_extract(meta, '$.category') = ?
+        AND json_extract(meta, '$.local_attempted') = 1
+        AND (? IS NULL OR environment = ?)
+      ORDER BY ts DESC
+      LIMIT ?
+    `).all(params.category, params.environment, params.environment, params.window);
+        if (rows.length < params.minSamples)
+            return null;
+        return rows.filter(row => row.accepted === 1 || row.accepted === true).length / rows.length;
+    }
+    catch {
+        return null; // fail open, as before
+    }
+}
+/**
+ * A cache hit is served only when the answer would be eligible now. Entries written here carry their
+ * category; plain strings (older entries, and the bench's cached cloud answers) have none, so they are
+ * served only when no tools are listed, as before.
+ */
+export function servableCacheHit(hit, toolsListed) {
+    if (typeof hit === 'string')
+        return toolsListed ? null : hit;
+    const entry = hit;
+    if (typeof entry?.answer !== 'string')
+        return null;
+    return !toolsListed || ALLOWED_WITH_TOOLS.includes(String(entry.category)) ? entry.answer : null;
+}
+/** A local-model client whose every request is cancelled when `signal` aborts (ollama-js takes a custom fetch). */
+function localModelClient(signal) {
+    if (!signal)
+        return new SLM();
+    const abortableFetch = ((input, init) => fetch(input, { ...init, signal }));
+    return new SLM(new Ollama({ host: CONFIG.OLLAMA_HOST, fetch: abortableFetch }));
+}
+/**
+ * Asks the local model to answer, when the request is eligible, and checks the answer.
+ *
+ * @param params.task The typed text the answer is for
+ * @param params.messages What the model is given to answer
+ * @param params.toolsListed The request offers the model tools (narrows eligibility)
+ * @param params.routePolicy 'force-local' answers regardless of eligibility and verification (bench arms)
+ * @param params.localModel Answering model; defaults to SLM_BRAIN_MODEL
+ * @param params.environment Ledger environment ROUTING_TUNE learns from
+ * @param params.signal Aborting it cancels every model call; the attempt then returns no answer and stores nothing
+ * @returns What happened; never throws
+ */
+export async function attemptLocalAnswer(params) {
+    const { task, messages, toolsListed, routePolicy = 'auto', signal } = params;
+    const localModel = params.localModel ?? CONFIG.SLM_BRAIN_MODEL;
+    const result = { answer: null, attempted: false, accepted: false, verifierFlags: [], fromCache: false, model: localModel };
+    // Checked before any model call, so a workspace or identity question costs nothing.
+    if (routePolicy !== 'force-local' && toolsListed && (mentionsWorkspace(task) || asksAboutAssistant(task)))
+        return result;
+    if (CONFIG.SEMCACHE) {
+        let hit = null;
+        try {
+            // The cache's embedding call cannot take the attempt's signal, so stop waiting for it on abort;
+            // otherwise a slow embedding would hold the one-attempt-at-a-time slot past the budget.
+            hit = await unlessAborted(checkSemanticCache(task), signal);
+        }
+        catch (err) {
+            console.error(`[llm-gate] semantic cache lookup failed, treated as a miss: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        if (signal?.aborted)
+            return result;
+        const cached = servableCacheHit(hit, toolsListed);
+        if (cached !== null)
+            return { ...result, answer: cached, fromCache: true, model: 'semcache', verifierFlags: ['cache_hit'] };
+    }
+    const slm = localModelClient(signal);
+    let category = 'other';
+    try {
+        if (task)
+            category = await classify(slm, task);
+    }
+    catch {
+        // Classification failed: treated as 'other', as before.
+    }
+    if (signal?.aborted)
+        return result;
+    result.category = category;
+    let eligible = (toolsListed ? ALLOWED_WITH_TOOLS : ALLOWED).includes(category);
+    if (CONFIG.ROUTING_TUNE && eligible && routePolicy !== 'force-local' && Math.random() >= CONFIG.ROUTING_TUNE_EXPLORE_RATE) {
+        const rate = getCategorySuccessRate({
+            category,
+            window: CONFIG.ROUTING_TUNE_WINDOW,
+            minSamples: CONFIG.ROUTING_TUNE_MIN_SAMPLES,
+            environment: params.environment ?? CONFIG.LANGFUSE_ENVIRONMENT ?? null,
+        });
+        if (rate !== null && rate < CONFIG.ROUTING_TUNE_THRESHOLD)
+            eligible = false;
+    }
+    if (routePolicy !== 'force-local' && !eligible)
+        return result;
+    result.attempted = true;
+    try {
+        let samples = [];
+        let answer;
+        if (CONFIG.HEADLINE_STRICTNESS >= 4) {
+            // Self-consistency: several samples must agree (the verifier's level 4).
+            samples = await Promise.all(Array.from({ length: CONFIG.SELF_CONSISTENCY_K }, () => slm.generateText(localModel, messages, CONFIG.SELF_CONSISTENCY_TEMP)));
+            answer = samples[0];
+        }
+        else {
+            answer = await slm.generateText(localModel, messages, CONFIG.TEMPERATURE);
+        }
+        if (signal?.aborted)
+            return result;
+        // localAccepted means the VERIFIER accepted the answer (non-empty, no hedging, samples agree) — a
+        // proxy for quality, not ground-truth correctness.
+        const verdict = verify(answer, samples, { nonEmpty: true }, CONFIG.HEADLINE_STRICTNESS);
+        result.verifierFlags = verdict.flags;
+        result.accepted = !verdict.escalate;
+        if (result.accepted || routePolicy === 'force-local')
+            result.answer = answer;
+    }
+    catch (err) {
+        if (!signal?.aborted)
+            handleSlmError(err, 'llm-gate:generate', localModel);
+    }
+    if (result.accepted && result.answer !== null) {
+        // Remembering the answer is a bonus: a failure here must not cost the answer itself.
+        await setSemanticCache(task, { answer: result.answer, category }).catch(err => console.error(`[llm-gate] semantic cache write failed: ${err instanceof Error ? err.message : String(err)}`));
+    }
+    return result;
+}
+/** Resolves like `work`, or with null as soon as `signal` aborts (the work itself keeps running). */
+function unlessAborted(work, signal) {
+    if (!signal)
+        return work;
+    if (signal.aborted)
+        return Promise.resolve(null);
+    return Promise.race([work, new Promise(resolve => signal.addEventListener('abort', () => resolve(null), { once: true }))]);
+}
+let attemptRunning = false;
+/**
+ * Step A for the gate: one attempt at a time, cancelled at LOCAL_ATTEMPT_BUDGET_MS. A first request that
+ * arrives while another attempt runs goes straight on (it never queues behind the local model).
+ *
+ * @returns The attempt, or null when it was skipped (busy) or ran out of time
+ */
+export async function answerFirstRequestLocally(params) {
+    if (attemptRunning)
+        return { attempt: null, outcome: 'busy' };
+    attemptRunning = true;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), CONFIG.LOCAL_ATTEMPT_BUDGET_MS);
+    const work = attemptLocalAnswer({
+        task: params.task,
+        messages: [{ role: 'user', content: params.task }],
+        toolsListed: params.toolsListed,
+        environment: CONFIG.LANGFUSE_ENVIRONMENT,
+        signal: controller.signal,
+    }).finally(() => {
+        clearTimeout(timer);
+        attemptRunning = false;
+    });
+    const expired = new Promise(resolve => controller.signal.addEventListener('abort', () => resolve(null), { once: true }));
+    const attempt = await Promise.race([work, expired]);
+    if (!attempt || controller.signal.aborted)
+        return { attempt: null, outcome: 'timeout' };
+    return { attempt, outcome: attempt.answer !== null ? 'answered' : 'declined' };
+}
+/** Loads the answering model in the background so the first eligible request does not pay the cold start. */
+export function warmUpAnsweringModel() {
+    new SLM().generateText(CONFIG.SLM_BRAIN_MODEL, [{ role: 'user', content: 'ok' }], 0, 1).catch(() => {
+        // Ollama down or model missing: attempts then fall back to the normal path on their own.
+    });
+}

package/dist/llm-gate/pipeline.js

@@ -0,0 +1,267 @@
+import { setSemanticCache } from '../cache/index.js';
+import { CONFIG } from '../config.js';
+import { calculateCostUsd } from '../pricing/index.js';
+import { waitWithBackoff as sharedWaitWithBackoff } from '../utils/backoff.js';
+import { compressContext } from '../utils/compression.js';
+import { isLatestInstructionFromTool } from '../utils/safety.js';
+import { buildAnthropicRequest } from './formats/anthropic.js';
+import { buildOpenAIRequest } from './formats/openai.js';
+import { attemptLocalAnswer } from './local-first.js';
+/**
+ * Determines whether an unknown thrown value is a transient, retryable network failure.
+ *
+ * @desc Inspects error name, system error code, and message signatures for network aborts/timeouts.
+ * @param err The caught unknown exception
+ * @returns True if the error is considered transient and safe to retry
+ * @example
+ * ```ts
+ * if (isRetryableNetworkError(err) && attempt < MAX_RETRIES) { ... }
+ * ```
+ */
+export function isRetryableNetworkError(err) {
+    if (err instanceof Error) {
+        const code = err.code;
+        return err.name === 'TimeoutError' || code === 'ECONNRESET' || err.message.includes('fetch failed');
+    }
+    return false;
+}
+/**
+ * Calculates exponential backoff with jitter and awaits the delay period.
+ *
+ * @desc Computes 1500ms * 2^attempt + jitter, respecting an optional `Retry-After` header value in seconds.
+ * Logs a diagnostic warning to stderr before waiting.
+ * @param attempt Current zero-indexed retry attempt
+ * @param maxRetries Total allowed retry attempts
+ * @param reason Human-readable context for why the backoff is being executed
+ * @param retryAfter Optional `Retry-After` header string from HTTP response
+ * @returns Promise that resolves once the backoff delay has completed
+ * @example
+ * ```ts
+ * await waitWithBackoff(attempt, MAX_RETRIES, 'Upstream HTTP 429', res.headers.get('retry-after'));
+ * ```
+ */
+export async function waitWithBackoff(attempt, maxRetries, reason, retryAfter) {
+    // Implementation lives in utils/backoff.ts so the ledger's Langfuse flush can share it
+    // without a circular import. This wrapper preserves the existing call signature.
+    return sharedWaitWithBackoff(attempt, maxRetries, reason, retryAfter, 'llm-gate');
+}
+// Very basic token estimator. A real implementation would use a proper tokenizer like tiktoken.
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+function countMessagesTokens(messages, system) {
+    let text = system || '';
+    for (const m of messages) {
+        text += '\n' + m.content;
+    }
+    return estimateTokens(text);
+}
+/**
+ * The core orchestration pipeline for a single LLM Gate request.
+ *
+ * Pipeline flow:
+ * 1. Derives the core task from the last user message.
+ * 2. If routePolicy allows local deferral and heuristics approve (no tool injection),
+ *    classifies the task with the SLM.
+ * 3. If the task type is simple enough, generates an answer locally.
+ * 4. Verifies the local answer against rigorous constraints.
+ * 5. If verified and safe, returns the local answer (deferring the cloud call entirely).
+ * 6. If it escalates or is forced, compresses the context and calls the upstream Cloud API.
+ *
+ * @param reqId A unique request identifier
+ * @param internalReq The internal request representation
+ * @param options Routing configurations
+ * @returns A fully constructed response payload and detailed token/cost analytics
+ */
+export async function processPipeline(reqId, internalReq, options) {
+    console.info('LLM Gate Pipeline: Started');
+    const t0 = Date.now();
+    const messages = internalReq.messages;
+    const result = {
+        body: null,
+        route: 'forward_raw',
+        isLocal: false,
+        model: '',
+        inTok: countMessagesTokens(messages, internalReq.system),
+        outTok: 0,
+        apiInTok: 0,
+        apiOutTok: 0,
+        costUsd: 0,
+        slmLatency: 0,
+        apiLatency: 0,
+        verifierFlags: [],
+        category: undefined,
+        localAttempted: false,
+        localAccepted: false,
+        promptChars: 0,
+        promptTokEst: 0,
+        hasCodeFence: false
+    };
+    const isSafeForLocal = !isLatestInstructionFromTool(messages);
+    const routePolicy = options.routePolicy;
+    let localDeferred = false;
+    let localAnswer = '';
+    let localModel = options.localModel || CONFIG.SLM_BRAIN_MODEL;
+    // Derive task (last user message)
+    const lastUserMsg = [...messages].reverse().find(m => m.role === 'user');
+    let taskText = lastUserMsg ? lastUserMsg.content : '';
+    // Normalize by stripping simple timestamps/ids from text if needed, but for now we'll just use taskText
+    const normalizedText = taskText;
+    result.promptChars = taskText.length;
+    result.promptTokEst = estimateTokens(taskText);
+    result.hasCodeFence = /```/.test(taskText);
+    // Local answer: the same implementation the model gate uses (local-first.ts), so the bench measures
+    // what ships. It includes the semantic cache, which is therefore only consulted on this path now.
+    let fromCache = false;
+    if (routePolicy === 'force-local' || (routePolicy === 'auto' && isSafeForLocal)) {
+        const attempt = await attemptLocalAnswer({
+            task: taskText,
+            messages,
+            toolsListed: (internalReq.tools?.length ?? 0) > 0,
+            routePolicy,
+            localModel,
+            environment: options.environment,
+        });
+        result.category = attempt.category;
+        result.localAttempted = attempt.attempted;
+        result.localAccepted = attempt.accepted;
+        result.verifierFlags = attempt.verifierFlags;
+        if (attempt.answer !== null) {
+            localDeferred = true;
+            localAnswer = attempt.answer;
+            localModel = attempt.model;
+            fromCache = attempt.fromCache;
+        }
+    }
+    result.slmLatency = (Date.now() - t0) / 1000;
+    if (localDeferred) {
+        result.route = 'defer_local';
+        result.isLocal = true;
+        result.model = localModel;
+        result.outTok = estimateTokens(localAnswer);
+        result.costUsd = fromCache ? 0 : calculateCostUsd(localModel, result.inTok, result.outTok);
+        // Format local answer as a standard completion in the internal format
+        // Since we stream in the server based on the return format, here we just return the full response.
+        // Streaming wrapper is handled outside if internalReq.stream is true.
+        result.body = {
+            choices: [
+                {
+                    message: {
+                        role: 'assistant',
+                        content: localAnswer
+                    }
+                }
+            ]
+        };
+        return result;
+    }
+    // Fallthrough: Escalate to Cloud
+    const t1 = Date.now();
+    let compressedReq = { ...internalReq };
+    if (routePolicy !== 'raw') {
+        compressedReq.messages = await compressContext(messages);
+        result.route = 'forward_compressed';
+    }
+    else {
+        result.route = 'forward_raw';
+    }
+    // The system prompt goes out as the client sent it: the gate adds nothing to the model's instructions
+    // and names no toolbox (toolboxes are plug-and-play behind the MCP layer).
+    // Format the request for the cloud provider
+    let fetchUrl = CONFIG.CLOUD_BASE_URL;
+    let fetchHeaders = {
+        'Content-Type': 'application/json'
+    };
+    let fetchBody;
+    if (CONFIG.CLOUD_API_STYLE === 'anthropic') {
+        if (fetchUrl && !fetchUrl.endsWith('/messages')) {
+            fetchUrl = fetchUrl.replace(/\/+$/, '') + '/messages';
+        }
+        else if (!fetchUrl) {
+            fetchUrl = 'https://api.anthropic.com/v1/messages';
+        }
+        fetchHeaders['x-api-key'] = CONFIG.CLOUD_API_KEY;
+        fetchHeaders['anthropic-version'] = '2023-06-01';
+        fetchBody = buildAnthropicRequest(compressedReq);
+    }
+    else {
+        // openai style
+        if (fetchUrl && !fetchUrl.endsWith('/chat/completions')) {
+            fetchUrl = fetchUrl.replace(/\/+$/, '') + '/chat/completions';
+        }
+        else if (!fetchUrl) {
+            fetchUrl = 'https://api.openai.com/v1/chat/completions';
+        }
+        fetchHeaders['Authorization'] = `Bearer ${CONFIG.CLOUD_API_KEY}`;
+        fetchBody = buildOpenAIRequest(compressedReq);
+    }
+    result.apiInTok = countMessagesTokens(compressedReq.messages, compressedReq.system);
+    // Perform API request with exponential backoff for 429 / 5xx and transient network issues
+    const MAX_RETRIES = 3;
+    let lastErr = null;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+        try {
+            const apiRes = await fetch(fetchUrl, {
+                method: 'POST',
+                headers: fetchHeaders,
+                body: JSON.stringify(fetchBody),
+                signal: AbortSignal.timeout(30000)
+            });
+            if (!apiRes.ok) {
+                const isRateLimit = apiRes.status === 429;
+                const isServerErr = apiRes.status >= 500 && apiRes.status < 600;
+                if ((isRateLimit || isServerErr) && attempt < MAX_RETRIES) {
+                    await waitWithBackoff(attempt, MAX_RETRIES, `Upstream HTTP ${apiRes.status} (${apiRes.statusText})`, apiRes.headers.get('retry-after'));
+                    continue;
+                }
+                throw new Error(`Cloud API Error: ${apiRes.statusText || apiRes.status}`);
+            }
+            if (internalReq.stream) {
+                // In stream mode, we return the stream to the caller
+                result.body = apiRes.body; // Pass the readable stream
+            }
+            else {
+                const data = await apiRes.json();
+                result.body = data;
+                // Token counting
+                const usage = data?.usage;
+                if (CONFIG.CLOUD_API_STYLE === 'anthropic') {
+                    result.apiOutTok = usage?.output_tokens || 0;
+                    result.apiInTok = usage?.input_tokens || result.apiInTok;
+                }
+                else {
+                    result.apiOutTok = usage?.completion_tokens || 0;
+                    result.apiInTok = usage?.prompt_tokens || result.apiInTok;
+                }
+            }
+            lastErr = null;
+            break;
+        }
+        catch (err) {
+            const errorObj = err instanceof Error ? err : new Error(String(err));
+            lastErr = errorObj;
+            if (isRetryableNetworkError(err) && attempt < MAX_RETRIES) {
+                await waitWithBackoff(attempt, MAX_RETRIES, `Cloud network error (${errorObj.message})`);
+                continue;
+            }
+            break;
+        }
+    }
+    if (lastErr) {
+        throw new Error(`Cloud request failed: ${lastErr.message}`);
+    }
+    if (isSafeForLocal && CONFIG.SEMCACHE) {
+        let responseText = '';
+        if (result.body && result.body.choices && result.body.choices[0] && result.body.choices[0].message) {
+            responseText = result.body.choices[0].message.content;
+            if (responseText) {
+                await setSemanticCache(normalizedText, responseText);
+            }
+        }
+    }
+    result.apiLatency = (Date.now() - t1) / 1000;
+    result.model = CONFIG.CLOUD_MODEL || 'unknown';
+    result.outTok = result.apiOutTok;
+    result.costUsd = calculateCostUsd(result.model, result.apiInTok, result.apiOutTok);
+    return result;
+}