@plurnk/plurnk-service 1.0.3 → 1.0.5

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@plurnk/plurnk-service",
-    "version": "1.0.3",
+    "version": "1.0.5",
     "description": "Plurnk agent runtime — the server / engine / core. Implements the @plurnk/plurnk-grammar contract.",
     "type": "module",
     "license": "MIT",
@@ -59,21 +59,21 @@
         "prepack": "npm run build"
     },
     "dependencies": {
-        "@plurnk/plurnk-agui": "1.0.3",
-        "@plurnk/plurnk-execs": "1.0.3",
-        "@plurnk/plurnk-execs-common": "1.0.3",
-        "@plurnk/plurnk-execs-mcp": "1.0.3",
-        "@plurnk/plurnk-execs-search": "1.0.3",
-        "@plurnk/plurnk-grammar": "1.0.3",
-        "@plurnk/plurnk-mimetypes": "1.0.3",
-        "@plurnk/plurnk-mimetypes-text-html": "1.0.3",
-        "@plurnk/plurnk-providers": "1.0.3",
-        "@plurnk/plurnk-schemes": "1.0.3",
-        "@plurnk/plurnk-schemes-http": "1.0.3",
+        "@plurnk/plurnk-agui": "1.0.5",
+        "@plurnk/plurnk-execs": "1.0.5",
+        "@plurnk/plurnk-execs-common": "1.0.5",
+        "@plurnk/plurnk-execs-mcp": "1.0.5",
+        "@plurnk/plurnk-execs-search": "1.0.5",
+        "@plurnk/plurnk-grammar": "1.0.5",
+        "@plurnk/plurnk-mimetypes": "1.0.5",
+        "@plurnk/plurnk-mimetypes-text-html": "1.0.5",
+        "@plurnk/plurnk-providers": "1.0.5",
+        "@plurnk/plurnk-schemes": "1.0.5",
+        "@plurnk/plurnk-schemes-http": "1.0.5",
         "@possumtech/sqlrite": "5.2.0",
         "diff": "^9.0.0",
-        "@plurnk/plurnk-mimetypes-embeddings": "1.0.3",
-        "@plurnk/plurnk-meta": "1.0.3"
+        "@plurnk/plurnk-mimetypes-embeddings": "1.0.5",
+        "@plurnk/plurnk-meta": "1.0.5"
     },
     "devDependencies": {
         "@types/diff": "^7.0.2"

package/dist/core/TokenGauge.d.ts

@@ -1,16 +0,0 @@
-import type { Mimetypes } from "@plurnk/plurnk-mimetypes";
-import type { Provider } from "@plurnk/plurnk-providers";
-import type { Db } from "./Db.ts";
-import type { TelemetryEvent } from "@plurnk/plurnk-grammar";
-export interface GaugeResolution {
-    tokenizerId: string;
-    exact: boolean;
-    count: (text: string) => Promise<number>;
-}
-export default class TokenGauge {
-    #private;
-    static contentHash(content: string): string;
-    static resolve(mimetypes: Mimetypes | undefined, provider: Provider, pushTelemetry?: (e: TelemetryEvent) => void): Promise<GaugeResolution>;
-    static tokensFor(db: Db, gauge: GaugeResolution, contentHash: string, content: string): Promise<number>;
-}
-//# sourceMappingURL=TokenGauge.d.ts.map

package/dist/core/TokenGauge.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"TokenGauge.d.ts","sourceRoot":"","sources":["../../src/core/TokenGauge.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,KAAK,EAAE,EAAE,EAAc,MAAM,SAAS,CAAC;AAC9C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE7D,MAAM,WAAW,eAAe;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CAC5C;AAED,MAAM,CAAC,OAAO,OAAO,UAAU;;IAI3B,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1C;IAQD,OAAa,OAAO,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,CAAC,EAAE,CAAC,CAAC,EAAE,cAAc,KAAK,IAAI,GAAG,OAAO,CAAC,eAAe,CAAC,CAsBhJ;IAID,OAAa,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAM5G;CACJ"}

package/dist/core/TokenGauge.js

@@ -1,58 +0,0 @@
-// #312 — the token gauge: derived token counts keyed on (content_hash, tokenizer_id), the
-// deep_hash discipline applied to token arithmetic. The mimetypes Tokenizers seam (mimetypes#44)
-// supplies the identity (tokenizer.json sha — NEVER the model id: deepseek pro↔flash share a
-// vocab, so that swap recounts nothing; gemma↔deepseek recounts all) and the exact counter.
-// When the seam resolves nothing exact for the active model, the provider's chars/2 upper bound
-// is used AND SURFACED (`tokenizer_unavailable`, once per model) — an error-class signal, never
-// a silent number. Resolutions are cached per model ref; derivation rows are shared session-wide.
-import { createHash } from "node:crypto";
-export default class TokenGauge {
-    static #resolutions = new Map();
-    static #warned = new Set();
-    static contentHash(content) {
-        return createHash("sha256").update(content).digest("hex");
-    }
-    // Resolve the active model's gauge: the seam's exact tokenizer, or the provider's surfaced
-    // upper bound labeled with a heuristic identity (so its rows never masquerade as exact ones).
-    // providers#37 (delivered, 0.35.0): Provider.servedModel is the backend's self-reported
-    // identity from the boot probe — an alias-fronted local backend ('turboderp' → the gguf name)
-    // resolves its exact tokenizer from turn 1. Absent (hosted/no-probe), the alias model tries;
-    // inexact runs as a SURFACED upper bound (#312 ask 3) — never a silent number.
-    static async resolve(mimetypes, provider, pushTelemetry) {
-        const key = provider.model;
-        const cached = TokenGauge.#resolutions.get(key);
-        if (cached !== undefined)
-            return cached;
-        let resolution;
-        try {
-            if (mimetypes === undefined)
-                throw new Error("no mimetypes seam");
-            const r = await mimetypes.tokenizer(provider.servedModel ?? provider.model);
-            resolution = { tokenizerId: r.exact ? r.tokenizerId : `heuristic:${r.tokenizerId}`, exact: r.exact, count: (text) => r.countTokens(text) };
-            if (!r.exact && !TokenGauge.#warned.has(key)) {
-                TokenGauge.#warned.add(key);
-                pushTelemetry?.({ source: "engine:tokens", kind: "tokenizer_unavailable", message: `no exact tokenizer for '${provider.model}' — counts are an upper bound (chars/2 class); budget refusals run conservative`, level: "warn" });
-            }
-        }
-        catch {
-            resolution = { tokenizerId: "heuristic:provider", exact: false, count: async (text) => provider.countTokens(text) };
-            if (!TokenGauge.#warned.has(key)) {
-                TokenGauge.#warned.add(key);
-                pushTelemetry?.({ source: "engine:tokens", kind: "tokenizer_unavailable", message: `tokenizer seam resolved nothing for '${provider.model}' — provider upper bound in use`, level: "warn" });
-            }
-        }
-        TokenGauge.#resolutions.set(key, resolution);
-        return resolution;
-    }
-    // The keyed lookup: (content_hash, tokenizer_id) → tokens; miss → count + upsert. The row is
-    // shared by every entry carrying identical content, across models sharing a vocabulary.
-    static async tokensFor(db, gauge, contentHash, content) {
-        const hit = await db.token_count_get.get({ content_hash: contentHash, tokenizer_id: gauge.tokenizerId });
-        if (hit !== undefined)
-            return hit.tokens;
-        const tokens = await gauge.count(content);
-        await db.token_count_upsert.run({ content_hash: contentHash, tokenizer_id: gauge.tokenizerId, tokens });
-        return tokens;
-    }
-}
-//# sourceMappingURL=TokenGauge.js.map

package/dist/core/TokenGauge.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"TokenGauge.js","sourceRoot":"","sources":["../../src/core/TokenGauge.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAC1F,iGAAiG;AACjG,6FAA6F;AAC7F,4FAA4F;AAC5F,gGAAgG;AAChG,gGAAgG;AAChG,kGAAkG;AAElG,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAYzC,MAAM,CAAC,OAAO,OAAO,UAAU;IAC3B,MAAM,CAAC,YAAY,GAAG,IAAI,GAAG,EAA2B,CAAC;IACzD,MAAM,CAAC,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,MAAM,CAAC,WAAW,CAAC,OAAe;QAC9B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,2FAA2F;IAC3F,8FAA8F;IAC9F,wFAAwF;IACxF,8FAA8F;IAC9F,6FAA6F;IAC7F,+EAA+E;IAC/E,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,SAAgC,EAAE,QAAkB,EAAE,aAA2C;QAClH,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,IAAI,UAA2B,CAAC;QAChC,IAAI,CAAC;YACD,IAAI,SAAS,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAClE,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5E,UAAU,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3I,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3C,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,2BAA2B,QAAQ,CAAC,KAAK,iFAAiF,EAAE,KAAK,EAAE,MAAM,EAAoB,CAAC,CAAC;YACtP,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,UAAU,GAAG,EAAE,WAAW,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACpH,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/B,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,wCAAwC,QAAQ,CAAC,KAAK,iCAAiC,EAAE,KAAK,EAAE,MAAM,EAAoB,CAAC,CAAC;YACnN,CAAC;QACL,CAAC;QACD,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAC7C,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,6FAA6F;IAC7F,wFAAwF;IACxF,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAM,EAAE,KAAsB,EAAE,WAAmB,EAAE,OAAe;QACvF,MAAM,GAAG,GAAG,MAAO,EAAE,CAAC,eAA8B,CAAC,GAAG,CAAqB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7I,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAO,EAAE,CAAC,kBAAiC,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QACxH,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ"}