@bendyline/gezel-inference 1.0.7 → 1.0.8

package/dist/knowledge/unified-search.js

@@ -0,0 +1,373 @@
+/**
+ * Explicit knowledge search across mounted catalogs: each catalog searched in
+ * its own embedding space, the lexical and vector arms fused by reciprocal
+ * rank. Platform-neutral — the daemon's KnowledgeManager and the on-device
+ * Capacitor host both run it over their own KnowledgeCatalogHost.
+ */
+import { formatKnowledgeUri, scoreResult, } from '@bendyline/gezel';
+import { spatialDocumentIdentity } from './spatial-query.js';
+/** Explicit-search shard budget (S) across every active catalog (§3.4). */
+const ROUTE_BUDGET_EXPLICIT = 6;
+/**
+ * Shards added beyond that budget for pages the query names by title (the
+ * catalog host's title-assisted routing): the centroids alone left the ABBA
+ * article unscanned for "Who were the members of ABBA?". Two, not one per
+ * catalog, so twenty mounted catalogs cannot add twenty scans.
+ */
+const TITLE_ROUTE_EXPLICIT = 2;
+const FINAL_K = 24;
+/**
+ * Explicit search fuses three arms per catalog by reciprocal rank (k = 60):
+ * the vector chunks (best chunk per document, by rerank cosine), the
+ * doc-title FTS hits, and the chunk-body FTS hits. Rank fusion instead of
+ * score comparison because the arms' scores are not commensurable — e5
+ * cosines sit in a narrow 0.7–0.9 band for everything, so a cosine-first
+ * ordering buried an exact title match under any vaguely related passage
+ * ("Who painted the Mona Lisa?" listed three other Mona Lisa copies above
+ * the article whose title the question names). Two lexical arms agreeing
+ * on a document now outrank one vector arm; a document every arm names
+ * comes first.
+ */
+const RRF_K = 60;
+const ARM_WEIGHTS = { vector: 1, docFts: 1, chunkFts: 0.5 };
+/**
+ * Media rows fuse per row, not per document: a document's own text chunk
+ * (cosine ≈ 0.9) would otherwise always stand in for its image (≈ 0.7). Each
+ * modality is ranked on its own scale, past its own measured floor.
+ */
+const MEDIA_ARM_WEIGHTS = { vector: 0.5, chunkFts: 0.25 };
+/** Media rows asked of each catalog, per modality, on an explicit search. */
+const MEDIA_K = 2;
+/** Media results one search returns at most, and per document. */
+const MEDIA_RESULT_CAP = 4;
+const MEDIA_PER_DOCUMENT = 1;
+/**
+ * Fused documents per catalog when several are in scope: keeps one catalog
+ * from monopolizing the merged list. A search scoped to one catalog has
+ * nothing to share the list with, and the cap cut the Knowledge browser's
+ * search for "blob" in an Azure-docs catalog — 348 matching titles — to six.
+ */
+const PER_CATALOG_CAP = 6;
+/** Lexical candidates fetched per arm before fusion, at the least. */
+const DOC_FTS_MIN = 6;
+const CHUNK_FTS_MIN = 8;
+/** Rank-anchored relevance for a fused knowledge result: the best hit is a strong 1.0, decaying RRF-style. */
+function fusedRankRelevance(rank) {
+    return 11 / (11 + rank);
+}
+/**
+ * The first prose paragraph of a document body, for a hit that no chunk
+ * carries (a title-only match): the model gets something to read, not just
+ * a summary line. Skips the title heading and a key-facts list.
+ */
+function leadExcerpt(markdown, max = 600) {
+    const body = markdown.replace(/^#\s[^\n]*\n+/, '');
+    const paragraph = body
+        .split(/\n\s*\n/)
+        .map((part) => part.trim())
+        .find((part) => part.length > 0 && !part.startsWith('#') && !part.startsWith('- ')) ??
+        body.trim();
+    return paragraph.length > max ? `${paragraph.slice(0, max - 1)}…` : paragraph;
+}
+export async function searchUnifiedCatalogs(deps, active, query, opts) {
+    if (active.length === 0)
+        return [];
+    const groups = [];
+    const shared = [];
+    const keyword = [];
+    const byProfile = new Map();
+    for (const key of active) {
+        const info = deps.catalog(key);
+        if (!info)
+            continue;
+        if (info.semanticSearch === 'shared' && !opts.localModelsOnly)
+            shared.push(key);
+        else if (info.semanticSearch === 'profile' || info.semanticSearch === 'shared') {
+            const group = byProfile.get(info.embedding.id) ?? { profile: info.embedding, keys: [] };
+            group.keys.push(key);
+            byProfile.set(info.embedding.id, group);
+        }
+        else
+            keyword.push(key);
+    }
+    if (shared.length > 0) {
+        groups.push({
+            keys: shared,
+            ...(opts.vector ? { vector: Float32Array.from(opts.vector) } : {}),
+        });
+    }
+    for (const group of byProfile.values()) {
+        const vector = await deps.embedForProfile(query, group.profile, opts.queryEmbedBudgetMs, opts.localModelsOnly);
+        groups.push({ keys: group.keys, ...(vector ? { vector } : {}) });
+    }
+    if (keyword.length > 0)
+        groups.push({ keys: keyword });
+    const response = {
+        chunks: [],
+        documents: [],
+    };
+    for (const group of groups) {
+        const request = {
+            query,
+            shardBudget: ROUTE_BUDGET_EXPLICIT,
+            titleRouteShards: TITLE_ROUTE_EXPLICIT,
+            finalK: FINAL_K,
+            includeChunkFts: true,
+            catalogKeys: group.keys,
+            // A pool no deeper than the page asked for: six title hits could
+            // never fill a twenty-result browse search.
+            docFtsLimit: Math.max(DOC_FTS_MIN, opts.maxResults),
+            chunkFtsLimit: Math.max(CHUNK_FTS_MIN, opts.maxResults),
+            spatial: opts.spatial,
+        };
+        let part;
+        try {
+            part = await deps.host.search(group.vector
+                ? { ...request, vector: group.vector, ...(opts.media ? { mediaK: MEDIA_K } : {}) }
+                : request);
+        }
+        catch (error) {
+            // A catalog that refuses the query vector (a width the catalog does
+            // not store, say) is an embedding failure, not a search failure: the
+            // worker forwards only the message, so any vector-arm error falls
+            // back to the keyword arms for this group.
+            if (!group.vector)
+                throw error;
+            deps.warn(`[knowledge] semantic search failed for ${group.keys.join(', ')}; keyword-only: ${error instanceof Error ? error.message : String(error)}`);
+            part = await deps.host.search(request);
+        }
+        response.chunks.push(...part.chunks);
+        response.documents.push(...part.documents);
+    }
+    // ── fuse the arms per document ──────────────────────────────────────
+    const fused = new Map();
+    const bump = (catalogKey, documentId, weight, rank, chunk) => {
+        const key = `${catalogKey}\u0000${documentId}`;
+        const entry = fused.get(key) ?? { catalogKey, documentId, score: 0 };
+        entry.score += weight / (RRF_K + rank);
+        // The document's representative chunk: its best vector chunk, else
+        // whichever chunk-FTS row named it.
+        if (chunk && (!entry.chunk || (chunk.cosine ?? -1) > (entry.chunk.cosine ?? -1))) {
+            entry.chunk = chunk;
+        }
+        fused.set(key, entry);
+    };
+    // Every text passage the arms found, per document, best vector match first,
+    // for `chunksPerDocument`. Fusion still ranks documents, not passages.
+    const passagesByDocument = new Map();
+    if ((opts.chunksPerDocument ?? 1) > 1) {
+        for (const hit of response.chunks
+            .filter((h) => !h.media)
+            .sort((a, b) => (b.cosine ?? -1) - (a.cosine ?? -1))) {
+            const key = `${hit.catalogKey}\u0000${hit.documentId}`;
+            const passages = passagesByDocument.get(key) ?? [];
+            if (!passages.some((known) => known.chunkUid === hit.chunkUid))
+                passages.push(hit);
+            passagesByDocument.set(key, passages);
+        }
+    }
+    // Vector arm: one rank per document, in rerank-cosine order. A hit under
+    // its profile's floor is not evidence at all — KNN always returns its
+    // nearest rows, and a rank-0 row fuses to relevance 1.0 however far away
+    // it is (see vector-floors.ts).
+    const textChunks = response.chunks.filter((h) => !h.media);
+    const mediaChunks = response.chunks.filter((h) => h.media);
+    const seenVector = new Set();
+    let vectorRank = 0;
+    for (const hit of textChunks
+        .filter((h) => h.cosine !== undefined)
+        .sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0))) {
+        const key = `${hit.catalogKey}\u0000${hit.documentId}`;
+        if (seenVector.has(key))
+            continue;
+        const cosine = hit.cosine ?? 0;
+        const floor = deps.vectorFloorFor(hit.catalogKey);
+        if (floor !== null && cosine < floor)
+            continue;
+        seenVector.add(key);
+        bump(hit.catalogKey, hit.documentId, ARM_WEIGHTS.vector, vectorRank++, hit);
+        const entry = fused.get(key);
+        if (entry && floor !== null && cosine > (entry.similarity ?? Number.NEGATIVE_INFINITY)) {
+            entry.similarity = cosine;
+        }
+    }
+    // Chunk-body FTS arm: BM25 order as the shards returned it, ranked per catalog.
+    const chunkFtsRank = new Map();
+    const seenChunkFts = new Set();
+    for (const hit of textChunks.filter((h) => h.cosine === undefined)) {
+        const key = `${hit.catalogKey}\u0000${hit.documentId}`;
+        if (seenChunkFts.has(key))
+            continue;
+        seenChunkFts.add(key);
+        const rank = chunkFtsRank.get(hit.catalogKey) ?? 0;
+        chunkFtsRank.set(hit.catalogKey, rank + 1);
+        bump(hit.catalogKey, hit.documentId, ARM_WEIGHTS.chunkFts, rank, hit);
+    }
+    // Doc-title FTS arm: exact-name evidence, ranked per catalog by the host.
+    for (const doc of response.documents) {
+        bump(doc.catalogKey, doc.documentId, ARM_WEIGHTS.docFts, doc.rank);
+    }
+    // Media rows: their own vector arm per modality (a modality with no
+    // measured floor gives no vector evidence — nearest-neighbour search
+    // always returns some image, and a media row has no query words to be
+    // grounded in) and a caption FTS arm.
+    const mediaFused = new Map();
+    const bumpMedia = (hit, weight, rank) => {
+        const key = `${hit.catalogKey}\u0000${hit.chunkUid}`;
+        const entry = mediaFused.get(key) ?? {
+            catalogKey: hit.catalogKey,
+            documentId: hit.documentId,
+            score: 0,
+            chunk: hit,
+        };
+        entry.score += weight / (RRF_K + rank);
+        if ((hit.cosine ?? -1) > (entry.chunk?.cosine ?? -1))
+            entry.chunk = hit;
+        mediaFused.set(key, entry);
+    };
+    const mediaVectorRank = new Map();
+    for (const hit of mediaChunks
+        .filter((h) => h.cosine !== undefined)
+        .sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0))) {
+        const modality = hit.media?.modality;
+        const floor = deps.vectorFloorFor(hit.catalogKey, modality);
+        if (floor === null || (hit.cosine ?? 0) < floor)
+            continue;
+        const rank = mediaVectorRank.get(modality) ?? 0;
+        mediaVectorRank.set(modality, rank + 1);
+        bumpMedia(hit, MEDIA_ARM_WEIGHTS.vector, rank);
+        const entry = mediaFused.get(`${hit.catalogKey}\u0000${hit.chunkUid}`);
+        if (entry)
+            entry.similarity = Math.max(entry.similarity ?? -1, hit.cosine ?? 0);
+    }
+    let mediaFtsRank = 0;
+    for (const hit of mediaChunks.filter((h) => h.cosine === undefined)) {
+        bumpMedia(hit, MEDIA_ARM_WEIGHTS.chunkFts, mediaFtsRank++);
+    }
+    const ordered = [...fused.values(), ...mediaFused.values()].sort((a, b) => b.score - a.score ||
+        (b.chunk?.cosine ?? -1) - (a.chunk?.cosine ?? -1) ||
+        a.documentId.localeCompare(b.documentId));
+    const matches = new Map([...response.documents, ...response.chunks]
+        .filter((hit) => hit.matchedLocation)
+        .map((hit) => [
+        `${hit.catalogKey}\u0000${hit.documentId}`,
+        { distanceMeters: hit.distanceMeters, matchedLocation: hit.matchedLocation },
+    ]));
+    const seenSpatial = new Set();
+    const out = [];
+    const perCatalogCount = new Map();
+    const capPerCatalog = !opts.spatial && active.length > 1;
+    const mediaPerDocument = new Map();
+    let mediaResults = 0;
+    for (const entry of ordered) {
+        const info = deps.catalog(entry.catalogKey);
+        if (!info)
+            continue;
+        if (entry.chunk?.media) {
+            const docKey = `${entry.catalogKey}\u0000${entry.documentId}`;
+            const forDocument = mediaPerDocument.get(docKey) ?? 0;
+            if (mediaResults >= MEDIA_RESULT_CAP || forDocument >= MEDIA_PER_DOCUMENT)
+                continue;
+            const evidence = entry.similarity !== undefined
+                ? { arm: 'vector', similarity: entry.similarity }
+                : { arm: 'fts' };
+            mediaPerDocument.set(docKey, forDocument + 1);
+            mediaResults++;
+            out.push(knowledgeChunkResult(info, entry.chunk, fusedRankRelevance(out.length), evidence));
+            if (out.length >= Math.max(10, opts.maxResults))
+                break;
+            continue;
+        }
+        const count = perCatalogCount.get(entry.catalogKey) ?? 0;
+        if (capPerCatalog && count >= PER_CATALOG_CAP)
+            continue;
+        const spatialIdentity = spatialDocumentIdentity(info.ref.publisherId, info.ref.catalogId, entry.documentId);
+        if (opts.spatial && seenSpatial.has(spatialIdentity))
+            continue;
+        const relevance = fusedRankRelevance(out.length);
+        // `vector` only when the hit cleared a measured floor; everything else
+        // is lexical evidence, which proactive injection asks to be grounded.
+        const evidence = entry.similarity !== undefined
+            ? { arm: 'vector', similarity: entry.similarity }
+            : { arm: 'fts' };
+        const result = entry.chunk
+            ? knowledgeChunkResult(info, entry.chunk, relevance, evidence)
+            : await knowledgeDocumentResult(deps.host, info, entry.documentId, relevance, evidence);
+        if (!result)
+            continue;
+        perCatalogCount.set(entry.catalogKey, count + 1);
+        seenSpatial.add(spatialIdentity);
+        const docKey = `${entry.catalogKey}\u0000${entry.documentId}`;
+        out.push({ ...result, ...matches.get(docKey) });
+        if (out.length >= Math.max(10, opts.maxResults))
+            break;
+        // The document's other passages, each labelled by its own evidence: a
+        // sibling is vector evidence only if it clears the floor by itself.
+        const siblings = (passagesByDocument.get(docKey) ?? [])
+            .filter((hit) => hit.chunkUid !== entry.chunk?.chunkUid)
+            .slice(0, Math.max(0, (opts.chunksPerDocument ?? 1) - 1));
+        const floor = deps.vectorFloorFor(entry.catalogKey);
+        for (const sibling of siblings) {
+            const own = floor !== null && sibling.cosine !== undefined && sibling.cosine >= floor
+                ? { arm: 'vector', similarity: sibling.cosine }
+                : { arm: 'fts' };
+            out.push(knowledgeChunkResult(info, sibling, fusedRankRelevance(out.length), own));
+            if (out.length >= Math.max(10, opts.maxResults))
+                break;
+        }
+        if (out.length >= Math.max(10, opts.maxResults))
+            break;
+    }
+    return out;
+}
+/** A document the lexical arms named but no chunk carries: cite it with its lead paragraph. */
+export async function knowledgeDocumentResult(host, info, documentId, relevance, evidence) {
+    const doc = await host.getDocument(info.key, documentId).catch(() => null);
+    if (!doc)
+        return null;
+    const snippet = leadExcerpt(doc.markdown) || doc.summary || '';
+    return {
+        kind: 'knowledge',
+        id: `knowledge:${info.ref.catalogId}:doc:${documentId}`,
+        title: doc.title,
+        subtitle: `${info.name} · ${info.topicNames.get(doc.topicId) ?? doc.topicId}`,
+        ...(snippet ? { snippet } : {}),
+        retrievalSource: 'knowledge',
+        catalogId: info.ref.catalogId,
+        catalogVersion: info.ref.version,
+        documentId,
+        topicPath: [info.topicNames.get(doc.topicId) ?? doc.topicId],
+        uri: formatKnowledgeUri({
+            publisherId: info.ref.publisherId,
+            catalogId: info.ref.catalogId,
+            documentId,
+        }),
+        ...(doc.sourceUrl ? { sourceUrl: doc.sourceUrl } : {}),
+        ...evidence,
+        ...scoreResult('knowledge', relevance),
+    };
+}
+export function knowledgeChunkResult(info, hit, relevance, evidence) {
+    return {
+        kind: 'knowledge',
+        id: `knowledge:${info.ref.catalogId}:${hit.chunkUid}`,
+        title: hit.headingPath.length > 0 ? `${hit.title} › ${hit.headingPath.join(' › ')}` : hit.title,
+        subtitle: info.name,
+        snippet: hit.text,
+        retrievalSource: 'knowledge',
+        catalogId: info.ref.catalogId,
+        catalogVersion: info.ref.version,
+        documentId: hit.documentId,
+        uri: formatKnowledgeUri({
+            publisherId: info.ref.publisherId,
+            catalogId: info.ref.catalogId,
+            documentId: hit.documentId,
+            fragment: { chunk: hit.chunkUid },
+        }),
+        line: hit.lineStart,
+        lineEnd: hit.lineEnd,
+        ...(hit.media ? { media: hit.media } : {}),
+        ...evidence,
+        ...scoreResult('knowledge', relevance),
+    };
+}

package/dist/providers/image/sd-cpp.js

@@ -273,13 +273,8 @@ export class StableDiffusionCppProvider {
         const height = input.height ?? defaultDimension;
         const resolvedLoras = await this.resolveLoras(modelId, input.loras ?? []);
         let loras = resolvedLoras.loras;
-        // A speed LoRA's recipe is what makes its few steps valid, so it wins
-        // over a style's.
-        const render = loras.find((l) => l.purpose === 'speed')?.render ?? loras.find((l) => l.render)?.render;
-        const steps = input.steps ?? render?.steps ?? (await this.defaultSampleSteps(modelId));
         const seed = input.seed ?? Math.floor(Math.random() * 2_147_483_647);
         const started = Date.now();
-        log.info(`[sd-cpp] generate model=${modelId ?? 'default'} steps=${steps} explicitSteps=${input.steps !== undefined} size=${width}x${height}${loras.length > 0 ? ` loras=${loras.map((l) => l.id).join(',')}` : ''}`);
         // Model selection: sd-server serves one model per process. When a
         // request targets a different installed model than the one currently
         // loaded, point resolveLaunch at it (via the shared launchState) and
@@ -333,6 +328,15 @@ export class StableDiffusionCppProvider {
             }
             if (signal?.aborted)
                 throw new ImageGenerationCancelledError();
+            const settingsNotes = [...resolvedLoras.notes];
+            if (loras.length > 0) {
+                loras = await this.lorasTheEngineHas(baseUrl, loras, request.signal, settingsNotes);
+            }
+            // Read only after the engine's list: a fast mode's 4 steps and CFG 1
+            // on the bare model it was dropped from render mush.
+            const render = loraRenderOverrides(loras);
+            const steps = input.steps ?? render?.steps ?? (await this.defaultSampleSteps(modelId));
+            log.info(`[sd-cpp] generate model=${modelId ?? 'default'} steps=${steps} explicitSteps=${input.steps !== undefined} size=${width}x${height}${loras.length > 0 ? ` loras=${loras.map((l) => l.id).join(',')}` : ''}`);
             // Drive sd-server's A1111-compatible endpoints (`/sdapi/v1/txt2img`
             // and `/sdapi/v1/img2img`). Unlike the OpenAI `/v1/images/*` shape
             // — which reads only prompt/n/size and takes steps/seed/cfg from the
@@ -341,10 +345,6 @@ export class StableDiffusionCppProvider {
             // actually take effect. `cfg_scale` is set per model (distilled /
             // Flux-flow models need CFG 1; others inherit the server default).
             const cfg = render?.cfg ?? (modelId ? MODEL_CFG_DEFAULTS.get(modelId) : undefined);
-            const settingsNotes = [...resolvedLoras.notes];
-            if (loras.length > 0) {
-                loras = await this.lorasTheEngineHas(baseUrl, loras, request.signal, settingsNotes);
-            }
             let inputImages = input.inputImages ?? [];
             // Capability gate: models whose architecture doesn't honor init
             // latents on the pinned sd-server fall back to txt2img instead of
@@ -996,6 +996,13 @@ export function withoutEngineDirectives(text) {
     } while (out !== previous);
     return out;
 }
+/**
+ * The render settings the applied LoRAs set. A speed LoRA's recipe is what
+ * makes its few steps valid, so it wins over a style's.
+ */
+function loraRenderOverrides(applied) {
+    return (applied.find((l) => l.purpose === 'speed')?.render ?? applied.find((l) => l.render)?.render);
+}
 /**
  * Read an installed LoRA's manifest, or nothing when it is malformed. The
  * filename must be a bare `.safetensors` name: the engine resolves it inside

package/dist/providers/mlx/async-file-handoff.d.ts

@@ -7,5 +7,13 @@
  * recipients.
  */
 export declare function isSuccessfulAsyncHandoff(toolName: string, output: string): boolean;
-export declare function asyncHandoffClosing(count: number): string;
+/** The gezel a successful handoff result names, when it names one. */
+export declare function asyncHandoffRecipient(output: string): string | undefined;
+/**
+ * The reply the person sees when the runtime ends a delegating turn, which the
+ * model also reads back as its own last message. The tool result already told
+ * the model to end its turn, so this line only says what happened. Each entry
+ * is one handoff's recipient, or undefined when its result named nobody.
+ */
+export declare function asyncHandoffClosing(recipients: readonly (string | undefined)[]): string;
 export declare function immediateFileWriteClosing(paths: string[]): string;

package/dist/providers/mlx/async-file-handoff.js

@@ -10,10 +10,38 @@ export function isSuccessfulAsyncHandoff(toolName, output) {
     return (!output.startsWith('ERROR:') &&
         (toolName === 'message_gezel' || toolName.startsWith('delegate_')));
 }
-export function asyncHandoffClosing(count) {
-    return count === 1
-        ? 'I sent the handoff and ended my turn so the recipient can use the provider queue. Their reply will arrive asynchronously.'
-        : `I sent ${count} handoffs and ended my turn so the recipients can use the provider queue. Their replies will arrive asynchronously.`;
+/**
+ * The handoff results (core's `messageGezelText`, the MCP `delegate_*` tools)
+ * name the recipient after a fixed lead-in and again as "<name>'s turn".
+ * Requiring both means a reworded result, or a name holding a period, gives
+ * no name rather than half of one.
+ */
+const HANDOFF_RECIPIENT = /(?:Accepted the (?:message|handoff) for|Dispatched the (?:message|handoff) to|An identical file handoff is already pending with) (.+?)(?= \(|[.;])[\s\S]*?\1's turn/;
+/** The gezel a successful handoff result names, when it names one. */
+export function asyncHandoffRecipient(output) {
+    return HANDOFF_RECIPIENT.exec(output.slice(0, 2_000))?.[1]?.trim() || undefined;
+}
+function joinNames(names) {
+    if (names.length <= 2)
+        return names.join(' and ');
+    return `${names.slice(0, -1).join(', ')}, and ${names[names.length - 1]}`;
+}
+/**
+ * The reply the person sees when the runtime ends a delegating turn, which the
+ * model also reads back as its own last message. The tool result already told
+ * the model to end its turn, so this line only says what happened. Each entry
+ * is one handoff's recipient, or undefined when its result named nobody.
+ */
+export function asyncHandoffClosing(recipients) {
+    const named = [...new Set(recipients.filter((name) => Boolean(name)))];
+    if (named.length > 0 && !recipients.includes(undefined)) {
+        return named.length === 1
+            ? `${named[0]} is on it. Their reply will appear here.`
+            : `${joinNames(named)} are on it. Their replies will appear here.`;
+    }
+    return recipients.length > 1
+        ? "I've passed this on. The replies will appear here."
+        : "I've passed this on. The reply will appear here.";
 }
 export function immediateFileWriteClosing(paths) {
     const unique = [...new Set(paths.filter((path) => path.trim().length > 0))];

package/dist/providers/mlx/provider.js

@@ -51,7 +51,7 @@ import { coerceToolCallArgs } from '../tool-arg-schema-coercion.js';
 import { createToolExecutor } from '../tool-executor.js';
 import { ToolFailureTracker } from '../tool-failure-tracker.js';
 import { ToolRepeatTracker } from '../tool-repeat-tracker.js';
-import { asyncHandoffClosing, immediateFileWriteClosing, isSuccessfulAsyncHandoff, } from './async-file-handoff.js';
+import { asyncHandoffClosing, asyncHandoffRecipient, immediateFileWriteClosing, isSuccessfulAsyncHandoff, } from './async-file-handoff.js';
 export { isSuccessfulAsyncHandoff };
 import { setChatTemplateKwarg, } from './chat-protocol.js';
 import { MlxEngineGate } from './engine-gate.js';
@@ -60,7 +60,7 @@ import { EngineLogRouter } from './engine-log-router.js';
 import { MlxEngineStatsSampler } from './engine-stats-sampler.js';
 import { REASONING_OPEN_HEADER, StreamingReasoningSplit, engineOpensReasoning, reopenTemplateReasoning, requestOpensReasoning, } from './reasoning-stream.js';
 import { applyMlxRequestShape } from './request-shape.js';
-import { PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, mlxHttpError, } from './runtime-diagnostics.js';
+import { MLX_IMAGE_RESERVED_CHARS, PRE_FIRST_BYTE_BASE_MS, buildMidStreamDropMessage, buildPreFirstByteAbortMessage, computePreFirstByteBudgetMs, estimatePromptTokens, formatTps, isMidStreamConnectionDrop, mlxHttpError, } from './runtime-diagnostics.js';
 import { classifyMlxFatalErrorLine, classifyMlxStartupLine, } from './stdout-parser.js';
 import { salvageTextualToolCalls } from './textual-tool-call-salvage.js';
 import { APPEND_TO_FILE_CONTINUATION_TOOL, MlxToolCallAccumulator, applyCallableToolGrammar, callableRequestToolNames, chatCompletionToolName, externalToolsAsChatCompletions, hermesRequiredArgGrammarRequested, missingTopLevelRequiredToolArgs, toChatCompletionsTools, validatorReportedMissingRequiredArgs, } from './tool-call-protocol.js';
@@ -646,7 +646,7 @@ class MlxSession extends StreamingSessionBase {
                 total += m.content.length;
             // The sidecar bounds images to 1024px. Reserve patch tokens without
             // counting base64 bytes as language tokens or treating pixels as free.
-            total += (m.images?.length ?? 0) * 8192;
+            total += (m.images?.length ?? 0) * MLX_IMAGE_RESERVED_CHARS;
             if (m.tool_calls) {
                 for (const tc of m.tool_calls)
                     total += tc.function.arguments.length + tc.function.name.length;
@@ -1427,7 +1427,8 @@ class MlxSession extends StreamingSessionBase {
                 //   - STREAMING_IDLE_MS: budget between two consecutive SSE
                 //     content events once tokens are flowing. Generous enough
                 //     to cover a heavy reasoning gap on a 26B model.
-                const approxPromptTokens = estimatePromptTokens(this.messages, body.tools);
+                // The request as sent: an image retired from it costs no prefill.
+                const approxPromptTokens = estimatePromptTokens(body.messages, body.tools);
                 const PRE_FIRST_BYTE_IDLE_MS = computePreFirstByteBudgetMs(approxPromptTokens);
                 const PRE_FIRST_BYTE_PER_PROGRESS_MS = PRE_FIRST_BYTE_IDLE_MS;
                 const STREAMING_IDLE_MS = 300_000;
@@ -2168,6 +2169,11 @@ class MlxSession extends StreamingSessionBase {
                 });
                 turnContent = salvaged.turnContent;
                 const { proseRepaired, xmlTagRepaired, claudeInvokeRepaired, glmRepaired, hermesRepaired, shellRepaired, envelopeRepaired, envelopeTruncated, unknownCall, truncatedSalvage, bareInvokeRepaired, } = salvaged;
+                // A prose-shaped call such as lookup_record({}) was ordinary TEXT
+                // to the Hermes grammar, not a constrained tool envelope. Missing
+                // arguments in that salvage cannot establish a decoding violation.
+                // Keep provenance by id through coercion and duplicate collapse.
+                const proseCallIds = new Set(proseRepaired.map((call) => call.id));
                 const rawTurnContentBeforeReasoning = turnContent;
                 // Reasoning-tag extraction applies regardless of which (if any)
                 // salvage path fired. Qwen 3.5/3.6 leak `<think>…</think>`
@@ -2602,7 +2608,7 @@ class MlxSession extends StreamingSessionBase {
                     tool_calls: toolCalls,
                 });
                 let abortDueToFailureLoop = null;
-                let asyncHandoffCount = 0;
+                const asyncHandoffRecipients = [];
                 let terminalActionClosing = null;
                 const toolImages = [];
                 const immediateFileWritePaths = [];
@@ -2701,16 +2707,21 @@ class MlxSession extends StreamingSessionBase {
                         this.deps.bridges.hasTool(call.function.name)) {
                         const missingRequired = missingTopLevelRequiredToolArgs(toolArgSchemas.get(call.function.name), args);
                         const grammarContractViolation = missingRequired.length > 0 &&
+                            !proseCallIds.has(call.id) &&
                             hermesRequiredArgGrammarRequested(body) &&
                             !this.forceSequentialToolGrammar;
-                        if (grammarContractViolation) {
-                            this.forceSequentialToolGrammar = true;
-                            log.warn(`turn#${seq}.${turn} required-argument grammar contract violation tool=${call.function.name} missing=${missingRequired.join(',')} — latching this session to sequential decode`);
+                        if (missingRequired.length > 0) {
+                            if (grammarContractViolation) {
+                                this.forceSequentialToolGrammar = true;
+                                log.warn(`turn#${seq}.${turn} required-argument grammar contract violation tool=${call.function.name} missing=${missingRequired.join(',')} — latching this session to sequential decode`);
+                            }
                             // Do not send a call that the advertised schema proves cannot
-                            // pass validation. Keeping it out of the bridge also avoids
-                            // poisoning the session-wide unresolved-failure ledger before
-                            // the sequential retry gets a chance to recover.
-                            output = `ERROR: ${call.function.name} was not executed because the constrained decoder omitted required fields: ${missingRequired.join(', ')}. The runtime switched this session to sequential decoding; retry the call once.`;
+                            // pass validation, regardless of how it was parsed. A rejected
+                            // prose salvage gets ordinary corrective feedback and retains
+                            // speculation; native grammar violations keep the circuit breaker.
+                            output = grammarContractViolation
+                                ? `ERROR: ${call.function.name} was not executed because the constrained decoder omitted required fields: ${missingRequired.join(', ')}. The runtime switched this session to sequential decoding; retry the call once.`
+                                : `ERROR: ${call.function.name} was not executed because its arguments are missing required fields: ${missingRequired.join(', ')}. Retry the call with all declared required fields.`;
                         }
                         else {
                             try {
@@ -2741,6 +2752,7 @@ class MlxSession extends StreamingSessionBase {
                             // the advertised schema. Sequential decode may still recover on
                             // the next iteration, and the log now makes the skew explicit.
                             if (validatorReportedMissingRequiredArgs(output) &&
+                                !proseCallIds.has(call.id) &&
                                 hermesRequiredArgGrammarRequested(body) &&
                                 !this.forceSequentialToolGrammar) {
                                 this.forceSequentialToolGrammar = true;
@@ -2755,7 +2767,7 @@ class MlxSession extends StreamingSessionBase {
                             askedQuestionThisTurn = true;
                         }
                         if (isSuccessfulAsyncHandoff(call.function.name, output)) {
-                            asyncHandoffCount++;
+                            asyncHandoffRecipients.push(asyncHandoffRecipient(output));
                         }
                         if ((immediateFileWriteTurn || writeContinuationActive) &&
                             (call.function.name === 'write_file' || call.function.name === 'append_to_file') &&
@@ -2930,12 +2942,12 @@ class MlxSession extends StreamingSessionBase {
                         `${writeContinuations > 0 ? ` continuations=${writeContinuations}` : ''}`);
                     return fullText;
                 }
-                if (asyncHandoffCount > 0) {
-                    const closingText = asyncHandoffClosing(asyncHandoffCount);
+                if (asyncHandoffRecipients.length > 0) {
+                    const closingText = asyncHandoffClosing(asyncHandoffRecipients);
                     this.messages.push({ role: 'assistant', content: closingText });
                     fullText = closingText;
                     log.info(`turn#${seq} END async-handoff-bail afterMs=${Date.now() - start} ` +
-                        `handoffs=${asyncHandoffCount} loopTurns=${turn + 1}`);
+                        `handoffs=${asyncHandoffRecipients.length} loopTurns=${turn + 1}`);
                     return fullText;
                 }
                 if (forceProjectMacroBail) {

package/dist/providers/mlx/runtime-diagnostics.d.ts

@@ -38,6 +38,12 @@ export declare function buildPreFirstByteAbortMessage(lastPrefill: {
 } | null): string;
 /** Build the user-facing message for an engine stream that died mid-turn. */
 export declare function buildMidStreamDropMessage(charsReceived: number, plannedStop: boolean, engineStillRunning?: boolean): string;
+/**
+ * Prompt characters one attached image stands for. The sidecar bounds images
+ * to 1024px, so their patch tokens are bounded too, and base64 bytes are never
+ * read as language tokens.
+ */
+export declare const MLX_IMAGE_RESERVED_CHARS = 8192;
 /** Estimate request tokens from message bodies and serialized tool schemas. */
 export declare function estimatePromptTokens(messages: unknown, tools: unknown): number;
 export declare const PRE_FIRST_BYTE_BASE_MS = 300000;

package/dist/providers/mlx/runtime-diagnostics.js

@@ -146,11 +146,28 @@ export function buildMidStreamDropMessage(charsReceived, plannedStop, engineStil
     }
     return `[Mac AI] the on-device engine dropped the connection mid-turn (${got}). This usually means the mlx server crashed, ran out of memory, or was restarted while the turn was streaming — this turn's work was lost. Retry the turn; if it keeps happening, restart the engine in Settings → On-device.`;
 }
+/**
+ * Prompt characters one attached image stands for. The sidecar bounds images
+ * to 1024px, so their patch tokens are bounded too, and base64 bytes are never
+ * read as language tokens.
+ */
+export const MLX_IMAGE_RESERVED_CHARS = 8192;
 /** Estimate request tokens from message bodies and serialized tool schemas. */
 export function estimatePromptTokens(messages, tools) {
     let chars = 0;
     try {
-        chars += JSON.stringify(messages)?.length ?? 0;
+        // Counted as base64, one generated picture read as ~1.1M tokens: an
+        // image gezel's six renders and reads put the estimate past 6M while the
+        // engine prefilled 14K (qwen3.8-27b, 2026-11-04).
+        let images = 0;
+        const text = JSON.stringify(messages, (key, value) => {
+            if (key === 'images' && Array.isArray(value)) {
+                images += value.length;
+                return undefined;
+            }
+            return value;
+        });
+        chars += (text?.length ?? 0) + images * MLX_IMAGE_RESERVED_CHARS;
     }
     catch {
         /* circular or unstringifiable */