gheim 0.1.4 → 0.1.6

package/dist/types/detectors/calibrated.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Calibrated detector — fixes the v1 model's "single-category mode"
+ * bug at inference time by subtracting a constant `oBias` from the O
+ * class logit before argmax.
+ *
+ * Empirical justification (see eval/calibration_sweep.json): on a
+ * 237-case multi-entity pathology suite the v1 model emits literally
+ * zero non-O tokens for "Müller" / "März" when an IBAN follows them —
+ * but the per-token softmax shows person at 47% vs O at 51%. A small
+ * additive bias on O flips those marginal cases without inducing false
+ * positives on PII-free text.
+ *
+ * Sweet spot: oBias=0.5:
+ *   - pathology full-coverage 81% → 94.5% (+13.5pp)
+ *   - test_v1 strict-F1 0.872 → 0.868 (−0.4pp)
+ *   - ~0.05 extra FP per chunk on real text
+ *
+ * `defaultDetector()` returns this with oBias=0.5, so users get the
+ * better behavior automatically.
+ */
+import type { Detector, Span } from "./base.ts";
+import type { LocalDetectorLoadEvent } from "./local.ts";
+/** Constructor options for {@link CalibratedDetector}. */
+export interface CalibratedDetectorOptions {
+    /** HuggingFace model id (must be a token-classification head). */
+    model?: string;
+    /**
+     * Constant subtracted from the O-class logit before argmax. Default
+     * `0.5` is the Pareto-clean point per the full-scale bias sweep
+     * (eval/calibration_sweep.json):
+     *
+     *   - oBias=0.0  uncalibrated (matches the raw model)
+     *   - oBias=0.5  recovers ~95% of multi-entity pathology cases for
+     *               ~0.4pp test_v1 strict-F1 cost
+     *   - oBias=1.0  more aggressive: 96.6% pathology, 1.1pp test_v1 cost
+     *   - oBias=1.5+ trades too much precision; not recommended
+     */
+    oBias?: number;
+    /** Backend selector. See {@link LocalDetectorOptions.device}. */
+    device?: string;
+    /** Quantization dtype: "fp32" | "fp16" | "q8" | "q4" | "auto". */
+    dtype?: string;
+    /** Sliding-window size in model tokens (default 510). */
+    chunkTokens?: number;
+    /** Overlap between consecutive windows (default 32). */
+    chunkOverlapTokens?: number;
+    /** Strip leading/trailing whitespace from each emitted span. Default true. */
+    trimWhitespace?: boolean;
+    /** Optional progress callback during model load (mirrors LocalDetector). */
+    onProgress?: (event: LocalDetectorLoadEvent) => void;
+}
+/** CalibratedDetector — local model + O-logit bias. */
+export declare class CalibratedDetector implements Detector {
+    readonly model: string;
+    readonly oBias: number;
+    readonly device: string;
+    readonly dtype: string;
+    readonly chunkTokens: number;
+    readonly chunkOverlapTokens: number;
+    readonly trimWhitespace: boolean;
+    /** Backend actually used after `load()` resolves; `null` before. */
+    actualDevice: string | null;
+    private _model;
+    private _tokenizer;
+    private _id2label;
+    private _oId;
+    private _loadPromise;
+    private readonly _onProgress?;
+    constructor(opts?: CalibratedDetectorOptions);
+    load(): Promise<void>;
+    detect(text: string, opts?: {
+        model?: string;
+    }): Promise<Span[]>;
+    private _detectChunked;
+    private _detectWindow;
+    private _emitProgress;
+}
+//# sourceMappingURL=calibrated.d.ts.map

package/dist/types/detectors/calibrated.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibrated.d.ts","sourceRoot":"","sources":["../../../src/detectors/calibrated.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAMzD,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;;OAUG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACtD;AA4CD,uDAAuD;AACvD,qBAAa,kBAAmB,YAAW,QAAQ;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,oEAAoE;IACpE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAQ;IACnC,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,SAAS,CAA8B;IAC/C,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA0C;gBAE3D,IAAI,GAAE,yBAA8B;IAiB1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAuCrB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAYxD,cAAc;YA0Bd,aAAa;IA6F3B,OAAO,CAAC,aAAa;CAiBtB"}

package/dist/types/detectors/index.d.ts

@@ -5,7 +5,14 @@ export { LocalDetector } from "./local.ts";
 export type { LocalDetectorOptions, LocalDetectorLoadEvent } from "./local.ts";
 export { CompositeDetector } from "./composite.ts";
 export type { CompositeDetectorOptions } from "./composite.ts";
+export { CalibratedDetector } from "./calibrated.ts";
+export type { CalibratedDetectorOptions } from "./calibrated.ts";
 import type { Detector } from "./base.ts";
-/** Pick a reasonable default: remote if `GHEIM_API_KEY` is set in env, else local. */
+/** Pick a reasonable default: remote when `GHEIM_API_KEY` is set in env,
+ *  else a calibrated local model.
+ *
+ *  The calibrated default (`oBias=0.5`) recovers ~95% of the
+ *  multi-entity-letter cases that the raw v1 model fails on, with
+ *  ~0.4pp test_v1 strict-F1 cost. See eval/calibration_sweep.json. */
 export declare function defaultDetector(): Detector;
 //# sourceMappingURL=index.d.ts.map

package/dist/types/detectors/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAE/D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,sFAAsF;AACtF,wBAAgB,eAAe,IAAI,QAAQ,CAI1C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/detectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,IAAI,oBAAoB,EACrC,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,YAAY,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C;;;;;sEAKsE;AACtE,wBAAgB,eAAe,IAAI,QAAQ,CAI1C"}

package/dist/types/index.d.ts

@@ -13,11 +13,11 @@ export { LABEL_TO_TAG, SENTINEL_RE, isPossibleSentinelPrefix, labelTag, parseSen
 export type { Sentinel } from "./core/sentinels.ts";
 export { StreamDeanonymizer } from "./core/stream.ts";
 export type { Detector, Span } from "./detectors/base.ts";
-export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, CompositeDetector, defaultDetector, } from "./detectors/index.ts";
-export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, } from "./detectors/index.ts";
+export { DEFAULT_BASE_URL, REMOTE_DEFAULT_MODEL, RemoteDetector, LocalDetector, CompositeDetector, CalibratedDetector, defaultDetector, } from "./detectors/index.ts";
+export type { RemoteDetectorOptions, LocalDetectorOptions, LocalDetectorLoadEvent, CompositeDetectorOptions, CalibratedDetectorOptions, } from "./detectors/index.ts";
 export { anonymizeMessages, anonymizeText, deanonymizeStream, deanonymizeText, } from "./plain.ts";
 export type { ChatMessage } from "./plain.ts";
-export declare const VERSION = "0.1.4";
+export declare const VERSION = "0.1.6";
 export { mergeAdjacent } from "./core/session.ts";
 export type { MergedSpan } from "./core/session.ts";
 //# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAK9C,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EACL,YAAY,EACZ,WAAW,EACX,wBAAwB,EACxB,QAAQ,EACR,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,EACxB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAK9C,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gheim",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "PII round-trip for LLM APIs: anonymize before the request, de-anonymize the stream on the way back.",
   "license": "Apache-2.0",
   "author": "Joel Barmettler <joel.barmettler@gmail.com>",