@map-audio/pam-mcp-server 1.27.0 → 1.28.0

package/dist/functiongemma-grammar.js

@@ -0,0 +1,140 @@
+// GBNF from a toolpack: declared tools, keys in renderValue's dictsort order, schema-typed values.
+// "lazy" roots at the call token (llama-server trigger); "full" also admits call-free text (wllama).
+import { byKeyDictsort, renderNumber } from "./functiongemma-format.js";
+export const CALL_TRIGGER = "<start_function_call>";
+export const DEFAULT_MAX_CALLS = 12;
+const PRIMITIVES = [
+    'string ::= <escape> [^<]* <escape>',
+    'integer ::= "-"? ([0-9] | [1-9] [0-9]{1,15})',
+    'number ::= "-"? ([0-9] | [1-9] [0-9]{1,15}) ("." [0-9]{1,16})? ([eE] [-+]? [0-9]{1,3})?',
+    'boolean ::= "true" | "false"',
+    'any-value ::= string | number | boolean | "null" | any-array | any-object',
+    'any-array ::= "[" (any-value ("," any-value)*)? "]"',
+    'any-object ::= "{" (any-key ":" any-value ("," any-key ":" any-value)*)? "}"',
+    'any-key ::= [A-Za-z_] [A-Za-z0-9_]*',
+];
+export function gbnfLiteral(text) {
+    const body = [...text]
+        .map((ch) => {
+        if (ch === '"' || ch === "\\")
+            return `\\${ch}`;
+        if (ch === "\n")
+            return "\\n";
+        if (ch === "\t")
+            return "\\t";
+        if (ch === "\r")
+            return "\\r";
+        const cp = ch.codePointAt(0);
+        return cp < 0x20 ? `\\x${cp.toString(16).padStart(2, "0")}` : ch;
+    })
+        .join("");
+    return `"${body}"`;
+}
+function enumLiteral(value) {
+    if (typeof value === "string")
+        return `<escape> ${gbnfLiteral(value)} <escape>`;
+    if (typeof value === "number")
+        return gbnfLiteral(renderNumber(value));
+    if (typeof value === "boolean" || value === null)
+        return gbnfLiteral(String(value));
+    throw new Error(`unsupported enum value in FunctionGemma grammar: ${JSON.stringify(value)}`);
+}
+class RuleSet {
+    rules = [];
+    names = new Set(PRIMITIVES.map((r) => r.split(" ::= ")[0]));
+    name(base) {
+        const stem = base.replace(/[^A-Za-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "rule";
+        let name = stem;
+        for (let i = 2; this.names.has(name); i++)
+            name = `${stem}-${i}`;
+        this.names.add(name);
+        return name;
+    }
+    add(name, body) {
+        this.rules.push(`${name} ::= ${body}`);
+        return name;
+    }
+    /** The rule (or inline expression) matching one value of `schema`. */
+    value(schema, base) {
+        if (Array.isArray(schema.enum) && schema.enum.length > 0) {
+            return this.add(this.name(base), schema.enum.map(enumLiteral).join(" | "));
+        }
+        const types = Array.isArray(schema.type) ? schema.type : schema.type ? [schema.type] : [];
+        if (types.length === 0)
+            return "any-value";
+        const alts = types.map((type) => {
+            switch (type) {
+                case "string":
+                case "integer":
+                case "number":
+                case "boolean":
+                    return type;
+                case "null":
+                    return '"null"';
+                case "array": {
+                    const item = schema.items ? this.value(schema.items, `${base}-item`) : "any-value";
+                    return this.add(this.name(`${base}-array`), `"[" (${item} ("," ${item})*)? "]"`);
+                }
+                case "object":
+                    return schema.properties ? this.object(schema, base) : "any-object";
+                default:
+                    throw new Error(`unsupported schema type in FunctionGemma grammar: ${type}`);
+            }
+        });
+        return alts.length === 1 ? alts[0] : `(${alts.join(" | ")})`;
+    }
+    /** Declared keys in renderValue order; each optional key may be skipped, every required one appears. */
+    object(schema, base) {
+        const name = this.name(base);
+        const required = new Set(schema.required ?? []);
+        const keys = Object.entries(schema.properties ?? {}).sort(byKeyDictsort);
+        if (keys.length === 0)
+            return this.add(name, '"{" "}"');
+        const pairs = keys.map(([key, sub]) => `${gbnfLiteral(`${key}:`)} ${this.value(sub, `${base}-${key}`)}`);
+        // after[i]: the rest of the object once some earlier key was written.
+        const after = new Array(keys.length + 1).fill("");
+        for (let i = keys.length - 1; i >= 1; i--) {
+            const item = required.has(keys[i][0]) ? `"," ${pairs[i]}` : `("," ${pairs[i]})?`;
+            after[i] = this.add(this.name(`${base}-after-${keys[i][0]}`), `${item}${after[i + 1] ? ` ${after[i + 1]}` : ""}`);
+        }
+        const firstRequired = keys.findIndex(([key]) => required.has(key));
+        const heads = keys
+            .slice(0, firstRequired === -1 ? keys.length : firstRequired + 1)
+            .map((_, i) => `${pairs[i]}${after[i + 1] ? ` ${after[i + 1]}` : ""}`);
+        const optional = firstRequired === -1;
+        const body = heads.length === 1 && !optional ? heads[0] : `(${heads.join(" | ")})${optional ? "?" : ""}`;
+        return this.add(name, `"{" ${body} "}"`);
+    }
+}
+/** GBNF for one assistant turn's call blocks, generated from the toolpack the model is prompted with. */
+export function buildToolGrammar(tools, opts = {}) {
+    const mode = opts.mode ?? "lazy";
+    const maxCalls = opts.maxCalls ?? DEFAULT_MAX_CALLS;
+    if (!Number.isInteger(maxCalls) || maxCalls < 1)
+        throw new Error(`maxCalls must be a positive integer: ${maxCalls}`);
+    if (tools.length === 0)
+        throw new Error("FunctionGemma grammar needs at least one tool");
+    const set = new RuleSet();
+    const toolRules = tools.map((tool) => {
+        const args = set.object((tool.parameters ?? { type: "object", properties: {} }), `${tool.name}-args`);
+        return set.add(set.name(`tool-${tool.name}`), `${gbnfLiteral(tool.name)} ${args}`);
+    });
+    const calls = `call-block{1,${maxCalls}} <start_function_response>?`;
+    const header = [
+        `root ::= ${mode === "full" ? `${calls} | free-text` : calls}`,
+        "call-block ::= <start_function_call> \"call:\" call-body <end_function_call>",
+        `call-body ::= ${toolRules.join(" | ")}`,
+        ...(mode === "full" ? ["free-text ::= (!<start_function_call>)*"] : []),
+    ];
+    return [...header, ...set.rules, ...PRIMITIVES].join("\n") + "\n";
+}
+/** llama-server request fields that apply `grammar` lazily from the first call token. */
+export function lazyGrammarRequest(grammar) {
+    return {
+        grammar,
+        grammar_lazy: true,
+        grammar_triggers: [{ type: 1, value: CALL_TRIGGER }],
+        preserved_tokens: [CALL_TRIGGER],
+    };
+}
+//# sourceMappingURL=functiongemma-grammar.js.map

package/dist/functiongemma-grammar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"functiongemma-grammar.js","sourceRoot":"","sources":["../src/functiongemma-grammar.ts"],"names":[],"mappings":"AAAA,mGAAmG;AACnG,qGAAqG;AAErG,OAAO,EAAgB,aAAa,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAUtF,MAAM,CAAC,MAAM,YAAY,GAAG,uBAAuB,CAAC;AACpD,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAUpC,MAAM,UAAU,GAAG;IACjB,oCAAoC;IACpC,8CAA8C;IAC9C,yFAAyF;IACzF,8BAA8B;IAC9B,2EAA2E;IAC3E,qDAAqD;IACrD,8EAA8E;IAC9E,qCAAqC;CACtC,CAAC;AAEF,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC;SACnB,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI;YAAE,OAAO,KAAK,EAAE,EAAE,CAAC;QAChD,IAAI,EAAE,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAC9B,IAAI,EAAE,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAC9B,IAAI,EAAE,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAC9B,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;QAC9B,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,OAAO,IAAI,IAAI,GAAG,CAAC;AACrB,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,YAAY,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC;IAChF,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IACvE,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACpF,MAAM,IAAI,KAAK,CAAC,oDAAoD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,OAAO;IACF,KAAK,GAAa,EAAE,CAAC;IACb,KAAK,GAAG,IAAI,GAAG,CAAS,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErF,IAAI,CAAC,IAAY;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC;QACnF,IAAI,IAAI,GAAG,IAAI,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YAAE,IAAI,GAAG,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,GAAG,CAAC,IAAY,EAAE,IAAY;QAC5B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,IAAI,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,MAAc,EAAE,IAAY;QAChC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,WAAW,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC9B,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,QAAQ,CAAC;gBACd,KAAK,SAAS,CAAC;gBACf,KAAK,QAAQ,CAAC;gBACd,KAAK,SAAS;oBACZ,OAAO,IAAI,CAAC;gBACd,KAAK,MAAM;oBACT,OAAO,QAAQ,CAAC;gBAClB,KAAK,OAAO,CAAC,CAAC,CAAC;oBACb,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;oBACnF,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,EAAE,QAAQ,IAAI,SAAS,IAAI,UAAU,CAAC,CAAC;gBACnF,CAAC;gBACD,KAAK,QAAQ;oBACX,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;gBACtE;oBACE,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;IAC/D,CAAC;IAED,wGAAwG;IACxG,MAAM,CAAC,MAAc,EAAE,IAAY;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACzE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,WAAW,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACzG,sEAAsE;QACtE,MAAM,KAAK,GAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YACjF,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpH,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,IAAI;aACf,KAAK,CAAC,CAAC,EAAE,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC;aAChE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACzG,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,IAAI,MAAM,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,yGAAyG;AACzG,MAAM,UAAU,gBAAgB,CAAC,KAAgB,EAAE,OAAuB,EAAE;IAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,QAAQ,EAAE,CAAC,CAAC;IACrH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACzF,MAAM,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;IAC1B,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE,CAAW,EAAE,GAAG,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC;QAChH,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,gBAAgB,QAAQ,8BAA8B,CAAC;IACrE,MAAM,MAAM,GAAG;QACb,YAAY,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,cAAc,CAAC,CAAC,CAAC,KAAK,EAAE;QAC9D,8EAA8E;QAC9E,iBAAiB,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QACxC,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,yCAAyC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KACxE,CAAC;IACF,OAAO,CAAC,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,EAAE,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACpE,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,OAAO;QACL,OAAO;QACP,YAAY,EAAE,IAAI;QAClB,gBAAgB,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;QACpD,gBAAgB,EAAE,CAAC,YAAY,CAAC;KACjC,CAAC;AACJ,CAAC"}

package/dist/functiongemma-params.d.ts

@@ -0,0 +1,33 @@
+export interface ParamKind {
+    id: string;
+    label: string;
+    words: string[];
+    phrases: string[];
+    scope: "cell" | "lfo" | "looper" | "master" | "global";
+    unit?: string;
+    text: string;
+}
+export interface ParamIndex {
+    stopwords: string[];
+    synonyms: Record<string, string[]>;
+    idf: Record<string, number>;
+    kinds: ParamKind[];
+}
+export declare const PARAMS_MAX_ENTRIES = 6;
+export declare const PARAMS_MAX_CHARS = 700;
+export declare const UNIT_FILL = 3;
+export declare function queryTokens(text: string, index: ParamIndex): string[];
+export declare function rankKinds(text: string, index: ParamIndex, sel?: number | null): ParamKind[];
+export declare function unitWord(text: string): string | null;
+/** The stated unit's kinds: the previous turn's first, then the generic ones (volume / cutoff). */
+export declare function unitFill(text: string, index: ParamIndex, opts?: {
+    sel?: number | null;
+    previous?: string | null;
+}): {
+    unit: string | null;
+    fill: ParamKind[];
+};
+export declare function renderParamsLine(text: string, index: ParamIndex, opts?: {
+    sel?: number | null;
+    previous?: string | null;
+}): string;

package/dist/functiongemma-params.js

@@ -0,0 +1,128 @@
+export const PARAMS_MAX_ENTRIES = 6;
+export const PARAMS_MAX_CHARS = 700;
+const SPLIT = /[^a-z0-9]+/;
+const CELL = /\bcell\s*\d/;
+const LFO = /\blfo\s*\d/;
+const TRACK = /\btrack\s*\d/;
+// When the best name match has a different unit, include up to UNIT_FILL matches for the stated unit.
+const UNITS = [
+    [/[0-9] *db(?![a-z0-9])/i, "db", "volume"],
+    [/[0-9] *k?hz(?![a-z0-9])/i, "hz", "cutoff"],
+];
+const UNIT_STRIP = /([0-9]) *(?:db|k?hz)(?![a-z0-9])/gi;
+export const UNIT_FILL = 3;
+const stem = (w) => (w.length > 3 && w.endsWith("s") && !w.endsWith("ss") ? w.slice(0, -1) : w);
+const prepared = new WeakMap();
+function prep(index) {
+    let p = prepared.get(index);
+    if (!p) {
+        const stop = new Set(index.stopwords);
+        const words = index.kinds.map((k) => new Set(k.words));
+        const labelWords = index.kinds.map((k) => {
+            const out = [];
+            for (const raw of k.label.toLowerCase().split(SPLIT)) {
+                if (!raw || /^\d+$/.test(raw))
+                    continue;
+                const ws = stem(raw);
+                if (!stop.has(ws))
+                    out.push(ws);
+            }
+            return out;
+        });
+        p = { stop, words, labelWords };
+        prepared.set(index, p);
+    }
+    return p;
+}
+export function queryTokens(text, index) {
+    const { stop } = prep(index);
+    const out = [];
+    const camelSplit = String(text).replace(/([a-z0-9])([A-Z])/g, "$1 $2");
+    for (const raw of camelSplit.toLowerCase().split(SPLIT)) {
+        if (!raw || /^\d+$/.test(raw))
+            continue;
+        const w = stem(raw);
+        if (stop.has(w))
+            continue;
+        const synonyms = Object.hasOwn(index.synonyms, w) ? index.synonyms[w] : [];
+        for (const t of [w, ...synonyms])
+            if (!out.includes(t))
+                out.push(t);
+    }
+    return out;
+}
+function scopeOf(text, sel) {
+    const low = String(text).toLowerCase();
+    const words = new Set(low.split(SPLIT));
+    if (words.has("master"))
+        return "master";
+    if (LFO.test(low))
+        return "lfo";
+    if (words.has("looper") || TRACK.test(low))
+        return "looper";
+    if (CELL.test(low) || sel)
+        return "cell";
+    return null;
+}
+export function rankKinds(text, index, sel) {
+    const { words, labelWords } = prep(index);
+    const tokens = queryTokens(text, index);
+    const joined = ` ${tokens.join(" ")} `;
+    const scope = scopeOf(text, sel);
+    const tokenSet = new Set(tokens);
+    const scored = [];
+    index.kinds.forEach((kind, pos) => {
+        let s = 0;
+        for (const w of tokens)
+            if (words[pos].has(w))
+                s += index.idf[w];
+        for (const phrase of kind.phrases)
+            if (joined.includes(` ${phrase} `))
+                s += 1.0;
+        if (s > 0 && labelWords[pos].length && labelWords[pos].every((lw) => tokenSet.has(lw)))
+            s += 1.0;
+        if (s > 0 && kind.scope === scope)
+            s += 0.5;
+        if (s > 0)
+            scored.push([-s, pos, kind]);
+    });
+    scored.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+    return scored.slice(0, PARAMS_MAX_ENTRIES).map(([, , k]) => k);
+}
+export function unitWord(text) {
+    return UNITS.find(([re]) => re.test(String(text)))?.[2] ?? null;
+}
+/** The stated unit's kinds: the previous turn's first, then the generic ones (volume / cutoff). */
+export function unitFill(text, index, opts = {}) {
+    const stated = UNITS.find(([re]) => re.test(String(text)));
+    if (!stated)
+        return { unit: null, fill: [] };
+    const [, unit, word] = stated;
+    const strip = (t) => String(t).replace(UNIT_STRIP, "$1 ");
+    const fill = opts.previous ? rankKinds(strip(opts.previous), index, opts.sel).filter((k) => k.unit === unit) : [];
+    const bare = strip(text);
+    for (const k of rankKinds(`${bare} ${word}`, index, opts.sel))
+        if (k.unit === unit && !fill.includes(k))
+            fill.push(k);
+    return { unit, fill: fill.slice(0, UNIT_FILL) };
+}
+export function renderParamsLine(text, index, opts = {}) {
+    const { unit, fill } = unitFill(text, index, opts);
+    // The unit token is not a parameter word ("db" would rank retriggerDecayDb), so rank without it.
+    let kinds = rankKinds(unit ? String(text).replace(UNIT_STRIP, "$1 ") : text, index, opts.sel);
+    if (unit && kinds[0]?.unit !== unit) {
+        const head = kinds.slice(0, PARAMS_MAX_ENTRIES - fill.length);
+        kinds = [...head, ...fill.filter((kind) => !head.includes(kind))];
+    }
+    if (!kinds.length && opts.previous)
+        kinds = rankKinds(opts.previous, index, opts.sel);
+    const head = "[params] ";
+    const parts = [];
+    for (const kind of kinds) {
+        if ((head + [...parts, kind.text].join(" | ")).length > PARAMS_MAX_CHARS)
+            break;
+        parts.push(kind.text);
+    }
+    return head + (parts.length ? parts.join(" | ") : "none");
+}
+//# sourceMappingURL=functiongemma-params.js.map

package/dist/functiongemma-params.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"functiongemma-params.js","sourceRoot":"","sources":["../src/functiongemma-params.ts"],"names":[],"mappings":"AAmBA,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AACpC,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC,MAAM,KAAK,GAAG,YAAY,CAAC;AAC3B,MAAM,IAAI,GAAG,aAAa,CAAC;AAC3B,MAAM,GAAG,GAAG,YAAY,CAAC;AACzB,MAAM,KAAK,GAAG,cAAc,CAAC;AAC7B,sGAAsG;AACtG,MAAM,KAAK,GAA+B;IACxC,CAAC,wBAAwB,EAAE,IAAI,EAAE,QAAQ,CAAC;IAC1C,CAAC,0BAA0B,EAAE,IAAI,EAAE,QAAQ,CAAC;CAC7C,CAAC;AACF,MAAM,UAAU,GAAG,oCAAoC,CAAC;AACxD,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC;AAE3B,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAQxG,MAAM,QAAQ,GAAG,IAAI,OAAO,EAAwB,CAAC;AAErD,SAAS,IAAI,CAAC,KAAiB;IAC7B,IAAI,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC5B,IAAI,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACvC,MAAM,GAAG,GAAa,EAAE,CAAC;YACzB,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrD,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACxC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClC,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;QAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAAiB;IACzD,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC;IACvE,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QACxC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,OAAO,CAAC,IAAY,EAAE,GAAmB;IAChD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC5D,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG;QAAE,OAAO,MAAM,CAAC;IACzC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,KAAiB,EAAE,GAAmB;IAC5E,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,MAAM,GAAkC,EAAE,CAAC;IACjD,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAChC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,CAAC,IAAI,MAAM;YAAE,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO;YAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,GAAG,CAAC;gBAAE,CAAC,IAAI,GAAG,CAAC;QAChF,IAAI,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAAE,CAAC,IAAI,GAAG,CAAC;QACjG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;YAAE,CAAC,IAAI,GAAG,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,AAAD,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AAClE,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,QAAQ,CACtB,IAAY,EACZ,KAAiB,EACjB,OAA0D,EAAE;IAE5D,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAC7C,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtH,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAY,EACZ,KAAiB,EACjB,OAA0D,EAAE;IAE5D,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IACnD,iGAAiG;IACjG,IAAI,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9F,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,WAAW,CAAC;IACzB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,gBAAgB;YAAE,MAAM;QAChF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAC5D,CAAC"}

package/dist/{fgemma-runner.d.ts → functiongemma-runner.d.ts}

@@ -1,20 +1,23 @@
 #!/usr/bin/env node
 /**
  * PAM FunctionGemma runner — drives a (fine-tuned) FunctionGemma model against
- * the PAM MCP server using the model's native wire format (fgemma-format.ts),
+ * the PAM MCP server using the model's native wire format (functiongemma-format.ts),
  * NOT the OpenAI chat+tools API: raw /completions + our own renderer/parser is
  * deterministic and byte-matches training (agent.ts remains the big-model
  * OpenAI-style baseline harness).
  *
  * Modes:
- *   REPL (default)   npx tsx src/fgemma-runner.ts [--url http://localhost:1234/v1] [--model id]
- *   Offline eval     ... --eval ../fgemma-poc/out/eval.jsonl --out ../fgemma-poc/out/predictions.jsonl
- *   Live e2e         ... --live ../fgemma-poc/scenarios/live.jsonl [--gold]
+ *   REPL (default)   npx tsx src/functiongemma-runner.ts [--url http://localhost:1234/v1] [--model id]
+ *   Offline eval     ... --eval ../ml-research/out/dev.jsonl --out ../ml-research/out/predictions.jsonl
+ *   Live e2e         ... --live ../ml-research/data/seeds/live_scenarios.jsonl [--gold]
  *
  * Flags: --toolpack <path>  --bos text|none (default none; use text if the
  * serving layer does NOT add BOS itself)  --max-tokens N  --final (generate a
  * model NL summary after write-tool calls; read-tools always continue)
  * --no-ctx (skip the live [ctx] session line).
+ * --grammar [--grammar-mode lazy|full] constrains calls to the toolpack's GBNF (functiongemma-grammar.ts);
+ * --probs [K] records P(first token = <start_function_call>) per eval row from a one-token probe;
+ * --token-probs streams the main request and records each generated token's [id, logprob, text].
  *
  * Session context (dataset v3): each command's user turn is prefixed with the
  * [ctx] state line built live from the bridge (get_transport +
@@ -22,6 +25,6 @@
  * additionally threads the last completed exchanges as history — the same
  * shapes the in-plugin executor produces. Within one command the runner does
  * up to MAX_ROUNDS call→response continuations inside the same model turn
- * (the trained repair/introspect pattern).
+ * (the trained repair/state_read pattern).
  */
 export {};

package/dist/{fgemma-runner.js → functiongemma-runner.js}

@@ -1,20 +1,23 @@
 #!/usr/bin/env node
 /**
  * PAM FunctionGemma runner — drives a (fine-tuned) FunctionGemma model against
- * the PAM MCP server using the model's native wire format (fgemma-format.ts),
+ * the PAM MCP server using the model's native wire format (functiongemma-format.ts),
  * NOT the OpenAI chat+tools API: raw /completions + our own renderer/parser is
  * deterministic and byte-matches training (agent.ts remains the big-model
  * OpenAI-style baseline harness).
  *
  * Modes:
- *   REPL (default)   npx tsx src/fgemma-runner.ts [--url http://localhost:1234/v1] [--model id]
- *   Offline eval     ... --eval ../fgemma-poc/out/eval.jsonl --out ../fgemma-poc/out/predictions.jsonl
- *   Live e2e         ... --live ../fgemma-poc/scenarios/live.jsonl [--gold]
+ *   REPL (default)   npx tsx src/functiongemma-runner.ts [--url http://localhost:1234/v1] [--model id]
+ *   Offline eval     ... --eval ../ml-research/out/dev.jsonl --out ../ml-research/out/predictions.jsonl
+ *   Live e2e         ... --live ../ml-research/data/seeds/live_scenarios.jsonl [--gold]
  *
  * Flags: --toolpack <path>  --bos text|none (default none; use text if the
  * serving layer does NOT add BOS itself)  --max-tokens N  --final (generate a
  * model NL summary after write-tool calls; read-tools always continue)
  * --no-ctx (skip the live [ctx] session line).
+ * --grammar [--grammar-mode lazy|full] constrains calls to the toolpack's GBNF (functiongemma-grammar.ts);
+ * --probs [K] records P(first token = <start_function_call>) per eval row from a one-token probe;
+ * --token-probs streams the main request and records each generated token's [id, logprob, text].
  *
  * Session context (dataset v3): each command's user turn is prefixed with the
  * [ctx] state line built live from the bridge (get_transport +
@@ -22,7 +25,7 @@
  * additionally threads the last completed exchanges as history — the same
  * shapes the in-plugin executor produces. Within one command the runner does
  * up to MAX_ROUNDS call→response continuations inside the same model turn
- * (the trained repair/introspect pattern).
+ * (the trained repair/state_read pattern).
  */
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
@@ -30,9 +33,10 @@ import { createInterface } from "node:readline";
 import { readFileSync, writeFileSync } from "node:fs";
 import { resolve, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
-import { STOP_SEQUENCES, buildGenerationPrompt, renderFunctionCall, renderFunctionResponse, parseFunctionCalls, truncateAtFabricatedResponse, withContext, } from "./fgemma-format.js";
+import { STOP_SEQUENCES, buildGenerationPrompt, renderFunctionCall, renderFunctionResponse, parseFunctionCalls, truncateAtFabricatedResponse, withContext, } from "./functiongemma-format.js";
+import { CALL_TRIGGER, buildToolGrammar, lazyGrammarRequest } from "./functiongemma-grammar.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const DEFAULT_TOOLPACK = resolve(__dirname, "../../fgemma-poc/out/toolpack.json");
+const DEFAULT_TOOLPACK = resolve(__dirname, "../../ml-research/out/toolpack.json");
 const MAX_ROUNDS = 3;
 const RESULT_TRUNC = 400; // chars of tool-result JSON fed back as function response
 function parseArgs() {
@@ -53,6 +57,9 @@ function parseArgs() {
         outPath: get("--out"),
         livePath: get("--live"),
         gold: a.includes("--gold"),
+        grammar: a.includes("--grammar") ? (get("--grammar-mode") || "lazy") : null,
+        probs: a.includes("--probs") ? Number(/^\d+$/.test(get("--probs") ?? "") ? get("--probs") : 20) : 0,
+        tokenProbs: a.includes("--token-probs"),
     };
 }
 const config = parseArgs();
@@ -73,8 +80,10 @@ function resolveAlias(call, pack) {
 }
 const READ_TOOL_RE = /^(get_|list_|connection_status)/;
 // --- generation ----------------------------------------------------------
+let grammarFields = {};
 async function complete(prompt) {
     const t0 = Date.now();
+    const stream = config.tokenProbs;
     const resp = await fetch(`${config.url}/completions`, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
@@ -84,13 +93,58 @@ async function complete(prompt) {
             max_tokens: config.maxTokens,
             temperature: 0,
             stop: STOP_SEQUENCES,
-            stream: false,
+            stream,
+            ...(stream ? { n_probs: 1 } : {}),
+            ...grammarFields,
         }),
     });
+    if (!resp.ok)
+        throw new Error(`LLM API error ${resp.status}: ${await resp.text()}`);
+    if (!stream) {
+        const data = (await resp.json());
+        return { text: data.choices[0].text, ms: Date.now() - t0 };
+    }
+    // Streamed chunks carry every token's probability; the final non-streamed response trims them by the stop word.
+    let text = "";
+    let buffer = "";
+    const tokens = [];
+    const decoder = new TextDecoder();
+    for await (const chunk of resp.body) {
+        buffer += decoder.decode(chunk, { stream: true });
+        for (let nl = buffer.indexOf("\n"); nl >= 0; nl = buffer.indexOf("\n")) {
+            const line = buffer.slice(0, nl).trim();
+            buffer = buffer.slice(nl + 1);
+            if (!line.startsWith("data:") || line.endsWith("[DONE]"))
+                continue;
+            const choice = JSON.parse(line.slice(5)).choices?.[0];
+            text += choice?.text ?? "";
+            for (const t of choice?.logprobs?.content ?? []) {
+                tokens.push([t.id, Math.round(t.logprob * 1e4) / 1e4, t.top_logprobs?.[0]?.id === t.id ? t.top_logprobs[0].token : t.token]);
+            }
+        }
+    }
+    return { text, ms: Date.now() - t0, tokens };
+}
+/** The unconstrained first-token distribution; a separate request because llama-server trims
+ *  final-response probs by the stop word's text tokenization, which can eat a short call's first token. */
+async function probeFirstToken(prompt) {
+    const t0 = Date.now();
+    const resp = await fetch(`${config.url}/completions`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ model: config.model, prompt, max_tokens: 1, temperature: 0, n_probs: config.probs, stream: false }),
+    });
     if (!resp.ok)
         throw new Error(`LLM API error ${resp.status}: ${await resp.text()}`);
     const data = (await resp.json());
-    return { text: data.choices[0].text, ms: Date.now() - t0 };
+    const top = (data.choices[0].logprobs?.content?.[0]?.top_logprobs ?? []).map((t) => ({
+        token: t.token,
+        p: Math.exp(t.logprob),
+    }));
+    if (top.length === 0)
+        throw new Error("probe returned no top_logprobs (server too old, or n_probs ignored)");
+    const call = top.find((t) => t.token === CALL_TRIGGER);
+    return { p_call: call?.p ?? 0, top: top.slice(0, 5), probe_ms: Date.now() - t0 };
 }
 // --- MCP -----------------------------------------------------------------
 async function createMcpClient() {
@@ -99,7 +153,7 @@ async function createMcpClient() {
         args: ["tsx", resolve(__dirname, "server.ts")],
         stderr: "inherit",
     });
-    const client = new Client({ name: "pam-fgemma-runner", version: "1.0.0" });
+    const client = new Client({ name: "pam-functiongemma-runner", version: "1.0.0" });
     await client.connect(transport);
     return client;
 }
@@ -191,7 +245,7 @@ async function runCommandExec(userText, tools, client, log, opts = {}) {
         const { text: rawFull, ms } = await complete(prompt);
         const raw = truncateAtFabricatedResponse(rawFull);
         if (raw !== rawFull) {
-            trace.quarantineHistory = true;
+            trace.droppedHistory = true;
             log("  (dropped fabricated function-response continuation)");
         }
         const calls = parseFunctionCalls(raw);
@@ -203,7 +257,7 @@ async function runCommandExec(userText, tools, client, log, opts = {}) {
         }
         const unknown = calls.filter((c) => !declared.has(c.name));
         if (unknown.length) {
-            trace.quarantineHistory = true;
+            trace.droppedHistory = true;
             log(`  refusing undeclared tool(s): ${unknown.map((c) => c.name).join(", ")}`);
             return trace;
         }
@@ -213,7 +267,7 @@ async function runCommandExec(userText, tools, client, log, opts = {}) {
         for (const call of calls) {
             const signature = `${call.name} ${JSON.stringify(call.arguments)}`;
             if (failedCalls.has(signature)) {
-                trace.quarantineHistory = true;
+                trace.droppedHistory = true;
                 log(`  stopped — the model repeated a call that already failed (${call.name}).`);
                 return trace;
             }
@@ -221,7 +275,7 @@ async function runCommandExec(userText, tools, client, log, opts = {}) {
             const { text, isError } = await execCall(client, call, tools);
             if (isError) {
                 failedCalls.add(signature);
-                trace.quarantineHistory = true;
+                trace.droppedHistory = true;
             }
             trace.executed.push({ call, resultText: text, isError });
             log(`  → ${text.length > 200 ? text.slice(0, 200) + "…" : text || "(ok)"}`);
@@ -307,7 +361,7 @@ async function runEval(tools) {
         const utterance = String(user?.content ?? "");
         const gold = firstAssistant?.tool_calls
             ? firstAssistant.tool_calls.map((tc) => ({ name: tc.function.name, arguments: tc.function.arguments }))
-            : null; // null = abstain expected
+            : null; // null = an irrelevance row: no call expected
         // Protocol: always generate against the FULL toolpack (deployment prompt),
         // regardless of the subset recorded in the row.
         const prompt = buildGenerationPrompt(tools, utterance, {
@@ -316,25 +370,33 @@ async function runEval(tools) {
         });
         let raw = "";
         let ms = -1;
+        let probs = {};
+        let tokenLp = {};
         try {
             const r = await complete(prompt);
             raw = r.text;
             ms = r.ms;
             latencies.push(ms);
+            if (r.tokens)
+                tokenLp = { token_lp: r.tokens };
+            if (config.probs > 0)
+                probs = await probeFirstToken(prompt);
         }
         catch (e) {
             raw = `__GENERATION_ERROR__ ${e.message}`;
         }
         const calls = parseFunctionCalls(raw);
-        out.push(JSON.stringify({ idx: i, meta: row._meta, utterance, gold, raw, calls, ms }));
+        out.push(JSON.stringify({ idx: i, meta: row._meta, utterance, gold, raw, calls, ms, ...probs, ...tokenLp }));
         if ((i + 1) % 25 === 0)
             console.error(`eval ${i + 1}/${rows.length}`);
     }
     writeFileSync(outPath, out.join("\n") + "\n");
     latencies.sort((a, b) => a - b);
     const p = (q) => latencies[Math.min(latencies.length - 1, Math.floor(q * latencies.length))] ?? -1;
-    console.log(`wrote ${out.length} predictions -> ${outPath}\nlatency ms p50=${p(0.5)} p95=${p(0.95)} (n=${latencies.length})`);
-    console.log("score with: python3 packages/fgemma-poc/eval_offline.py " + outPath);
+    const total = latencies.reduce((sum, ms) => sum + ms, 0);
+    console.log(`wrote ${out.length} predictions -> ${outPath}\nlatency ms p50=${p(0.5)} p95=${p(0.95)} total=${total} (n=${latencies.length})` +
+        (config.grammar ? ` grammar=${config.grammar}` : ""));
+    console.log("score with: uv run --project packages/ml-research pam-ml scoring calls " + outPath);
 }
 function callMatches(call, tool, argsSubset) {
     if (call.name !== tool)
@@ -378,7 +440,7 @@ async function runLive(tools) {
             let this_ok = false;
             let detail = "";
             const allCalls = trace.rounds.flatMap((r) => r.calls);
-            if (ex.kind === "no_calls") {
+            if (ex.kind === "irrelevance") {
                 this_ok = allCalls.length === 0;
                 detail = this_ok ? "" : `emitted ${allCalls.map((c) => c.name).join(",")}`;
             }
@@ -453,7 +515,7 @@ async function runRepl(tools) {
                 history: history.slice(-HISTORY_MAX_EXCHANGES),
                 failedCalls,
             });
-            if (!trace.quarantineHistory && trace.modelTurn && trace.modelTurn.length <= HISTORY_TURN_MAX_CHARS) {
+            if (!trace.droppedHistory && trace.modelTurn && trace.modelTurn.length <= HISTORY_TURN_MAX_CHARS) {
                 history.push({ user: input, model: trace.modelTurn });
             }
             if (trace.finalText)
@@ -478,6 +540,10 @@ async function runRepl(tools) {
 }
 async function main() {
     const tools = loadToolpack();
+    if (config.grammar) {
+        const grammar = buildToolGrammar(tools, { mode: config.grammar });
+        grammarFields = config.grammar === "lazy" ? lazyGrammarRequest(grammar) : { grammar };
+    }
     if (config.evalPath)
         return runEval(tools);
     if (config.livePath)
@@ -488,4 +554,4 @@ main().catch((e) => {
     console.error("Fatal:", e.message);
     process.exit(1);
 });
-//# sourceMappingURL=fgemma-runner.js.map
+//# sourceMappingURL=functiongemma-runner.js.map