@aquex/cmos-mcp 1.0.0

package/dist/intelligence/tokenizer-bootstrap.d.ts

@@ -0,0 +1,54 @@
+import type { SupportedModel } from './types';
+type GPTEncodeFn = (text: string) => number[];
+type ClaudeTokenizer = (text: string) => Promise<{
+    input_ids: {
+        data: {
+            length: number;
+        };
+    };
+}>;
+type PreloadModel = Extract<SupportedModel, 'gpt' | 'claude'>;
+interface TokenizerStatus {
+    ready: boolean;
+    attempts: number;
+    lastReadyAt?: string;
+    lastError?: string;
+    lastFailureAt?: string;
+}
+export interface TokenizerHealthSnapshot {
+    updatedAt: string;
+    models: Record<PreloadModel, TokenizerStatus>;
+    fallbacks: Record<SupportedModel, number>;
+}
+type GPTModule = {
+    encode?: unknown;
+};
+type ClaudeModule = {
+    AutoTokenizer?: {
+        from_pretrained?: (...args: unknown[]) => unknown;
+    };
+    default?: {
+        AutoTokenizer?: {
+            from_pretrained?: (...args: unknown[]) => unknown;
+        };
+    };
+};
+type ModuleLoader<T> = () => Promise<T>;
+export declare function ensureTokenizersReady(): Promise<void>;
+export declare function getGPTEncoder(): Promise<GPTEncodeFn | null>;
+export declare function getClaudeTokenizerInstance(): Promise<ClaudeTokenizer | null>;
+export declare function recordTokenizerFallback(model: SupportedModel): void;
+export declare function getTokenizerHealth(): TokenizerHealthSnapshot;
+export declare const __test__: {
+    reset(): void;
+    setModuleLoaders(loaders: {
+        gpt?: ModuleLoader<GPTModule> | null;
+        claude?: ModuleLoader<ClaudeModule> | null;
+    }): void;
+    getState(): {
+        gptLoaded: boolean;
+        claudeLoaded: boolean;
+    };
+};
+export {};
+//# sourceMappingURL=tokenizer-bootstrap.d.ts.map

package/dist/intelligence/tokenizer-bootstrap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer-bootstrap.d.ts","sourceRoot":"","sources":["../../src/intelligence/tokenizer-bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAG9C,KAAK,WAAW,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;AAC9C,KAAK,eAAe,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IAC/C,SAAS,EAAE;QAAE,IAAI,EAAE;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACzC,CAAC,CAAC;AAEH,KAAK,YAAY,GAAG,OAAO,CAAC,cAAc,EAAE,KAAK,GAAG,QAAQ,CAAC,CAAC;AAE9D,UAAU,eAAe;IACvB,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC9C,SAAS,EAAE,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;CAC3C;AAqBD,KAAK,SAAS,GAAG;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AACtC,KAAK,YAAY,GAAG;IAClB,aAAa,CAAC,EAAE;QAAE,eAAe,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAA;KAAE,CAAC;IACtE,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE;YAAE,eAAe,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAA;SAAE,CAAA;KAAE,CAAC;CACrF,CAAC;AAEF,KAAK,YAAY,CAAC,CAAC,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC;AAgIxC,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3D;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAGjE;AAED,wBAAsB,0BAA0B,IAAI,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAGlF;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,cAAc,GAAG,IAAI,CAQnE;AAED,wBAAgB,kBAAkB,IAAI,uBAAuB,CAa5D;AAED,eAAO,MAAM,QAAQ;aACV,IAAI;8BAaa;QACxB,GAAG,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;QACrC,MAAM,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;KAC5C,GAAG,IAAI;gBAYI;QACV,SAAS,EAAE,OAAO,CAAC;QACnB,YAAY,EAAE,OAAO,CAAC;KACvB;CAMF,CAAC"}

package/dist/intelligence/tokenizer-bootstrap.js

@@ -0,0 +1,238 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.__test__ = void 0;
+exports.ensureTokenizersReady = ensureTokenizersReady;
+exports.getGPTEncoder = getGPTEncoder;
+exports.getClaudeTokenizerInstance = getClaudeTokenizerInstance;
+exports.recordTokenizerFallback = recordTokenizerFallback;
+exports.getTokenizerHealth = getTokenizerHealth;
+const telemetry_1 = require("./telemetry");
+const SOURCE = 'tokenizer-bootstrap';
+const tokenizerStatus = {
+    gpt: { ready: false, attempts: 0 },
+    claude: { ready: false, attempts: 0 },
+};
+const fallbackCounts = {
+    gpt: 0,
+    claude: 0,
+    gemini: 0,
+};
+let gptEncoder = null;
+let claudeTokenizer = null;
+let gptLoadPromise = null;
+let claudeLoadPromise = null;
+let gptModuleLoader = null;
+let claudeModuleLoader = null;
+function nowIso() {
+    return new Date().toISOString();
+}
+function errorMessage(error) {
+    if (error instanceof Error) {
+        return error.message;
+    }
+    return String(error);
+}
+function updateStatus(model, patch) {
+    tokenizerStatus[model] = {
+        ...tokenizerStatus[model],
+        ...patch,
+    };
+}
+async function loadGPTEncoder() {
+    if (gptEncoder) {
+        updateStatus('gpt', { ready: true });
+        return gptEncoder;
+    }
+    if (!gptLoadPromise) {
+        gptLoadPromise = (async () => {
+            tokenizerStatus.gpt.attempts += 1;
+            try {
+                const module = await (gptModuleLoader
+                    ? gptModuleLoader()
+                    : Promise.resolve().then(() => __importStar(require('gpt-tokenizer'))));
+                const encode = module.encode;
+                if (typeof encode !== 'function') {
+                    throw new Error('gpt-tokenizer encode export missing');
+                }
+                gptEncoder = encode;
+                updateStatus('gpt', {
+                    ready: true,
+                    lastReadyAt: nowIso(),
+                    lastError: undefined,
+                    lastFailureAt: undefined,
+                });
+                (0, telemetry_1.emitTelemetryInfo)(SOURCE, 'GPT tokenizer preloaded', {
+                    attempts: tokenizerStatus.gpt.attempts,
+                });
+                return gptEncoder;
+            }
+            catch (error) {
+                gptEncoder = null;
+                const message = errorMessage(error);
+                updateStatus('gpt', {
+                    ready: false,
+                    lastError: message,
+                    lastFailureAt: nowIso(),
+                });
+                (0, telemetry_1.emitTelemetryWarning)(SOURCE, 'Failed to preload GPT tokenizer', {
+                    error: message,
+                    attempts: tokenizerStatus.gpt.attempts,
+                });
+                return null;
+            }
+        })().finally(() => {
+            gptLoadPromise = null;
+        });
+    }
+    return gptLoadPromise;
+}
+async function loadClaudeTokenizer() {
+    if (claudeTokenizer) {
+        updateStatus('claude', { ready: true });
+        return claudeTokenizer;
+    }
+    if (!claudeLoadPromise) {
+        claudeLoadPromise = (async () => {
+            tokenizerStatus.claude.attempts += 1;
+            try {
+                const module = await (claudeModuleLoader
+                    ? claudeModuleLoader()
+                    : Promise.resolve().then(() => __importStar(require('@xenova/transformers'))));
+                const autoTokenizer = module.AutoTokenizer ?? module.default?.AutoTokenizer;
+                if (!autoTokenizer || typeof autoTokenizer.from_pretrained !== 'function') {
+                    throw new Error('AutoTokenizer.from_pretrained is not available');
+                }
+                const tokenizer = (await autoTokenizer.from_pretrained('Xenova/claude-tokenizer'));
+                claudeTokenizer = tokenizer;
+                updateStatus('claude', {
+                    ready: true,
+                    lastReadyAt: nowIso(),
+                    lastError: undefined,
+                    lastFailureAt: undefined,
+                });
+                (0, telemetry_1.emitTelemetryInfo)(SOURCE, 'Claude tokenizer preloaded', {
+                    attempts: tokenizerStatus.claude.attempts,
+                });
+                return tokenizer;
+            }
+            catch (error) {
+                claudeTokenizer = null;
+                const message = errorMessage(error);
+                updateStatus('claude', {
+                    ready: false,
+                    lastError: message,
+                    lastFailureAt: nowIso(),
+                });
+                (0, telemetry_1.emitTelemetryWarning)(SOURCE, 'Failed to preload Claude tokenizer', {
+                    error: message,
+                    attempts: tokenizerStatus.claude.attempts,
+                });
+                return null;
+            }
+        })().finally(() => {
+            claudeLoadPromise = null;
+        });
+    }
+    return claudeLoadPromise;
+}
+async function ensureTokenizersReady() {
+    await Promise.all([loadGPTEncoder(), loadClaudeTokenizer()]);
+}
+async function getGPTEncoder() {
+    const encoder = await loadGPTEncoder();
+    return encoder ?? null;
+}
+async function getClaudeTokenizerInstance() {
+    const tokenizerInstance = await loadClaudeTokenizer();
+    return tokenizerInstance ?? null;
+}
+function recordTokenizerFallback(model) {
+    fallbackCounts[model] += 1;
+    if (model === 'gpt' || model === 'claude') {
+        updateStatus(model, {
+            ready: false,
+            lastFailureAt: nowIso(),
+        });
+    }
+}
+function getTokenizerHealth() {
+    return {
+        updatedAt: nowIso(),
+        models: {
+            gpt: { ...tokenizerStatus.gpt },
+            claude: { ...tokenizerStatus.claude },
+        },
+        fallbacks: {
+            gpt: fallbackCounts.gpt,
+            claude: fallbackCounts.claude,
+            gemini: fallbackCounts.gemini,
+        },
+    };
+}
+exports.__test__ = {
+    reset() {
+        tokenizerStatus.gpt = { ready: false, attempts: 0 };
+        tokenizerStatus.claude = { ready: false, attempts: 0 };
+        fallbackCounts.gpt = 0;
+        fallbackCounts.claude = 0;
+        fallbackCounts.gemini = 0;
+        gptEncoder = null;
+        claudeTokenizer = null;
+        gptLoadPromise = null;
+        claudeLoadPromise = null;
+        gptModuleLoader = null;
+        claudeModuleLoader = null;
+    },
+    setModuleLoaders(loaders) {
+        if (typeof loaders.gpt !== 'undefined') {
+            gptModuleLoader = loaders.gpt;
+            gptEncoder = null;
+            gptLoadPromise = null;
+        }
+        if (typeof loaders.claude !== 'undefined') {
+            claudeModuleLoader = loaders.claude;
+            claudeTokenizer = null;
+            claudeLoadPromise = null;
+        }
+    },
+    getState() {
+        return {
+            gptLoaded: gptEncoder !== null,
+            claudeLoaded: claudeTokenizer !== null,
+        };
+    },
+};
+//# sourceMappingURL=tokenizer-bootstrap.js.map

package/dist/intelligence/tokenizer-bootstrap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer-bootstrap.js","sourceRoot":"","sources":["../../src/intelligence/tokenizer-bootstrap.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiLA,sDAEC;AAED,sCAGC;AAED,gEAGC;AAED,0DAQC;AAED,gDAaC;AArND,2CAAsE;AAuBtE,MAAM,MAAM,GAAG,qBAAqB,CAAC;AAErC,MAAM,eAAe,GAA0C;IAC7D,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE;IAClC,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE;CACtC,CAAC;AAEF,MAAM,cAAc,GAAmC;IACrD,GAAG,EAAE,CAAC;IACN,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;CACV,CAAC;AAEF,IAAI,UAAU,GAAuB,IAAI,CAAC;AAC1C,IAAI,eAAe,GAA2B,IAAI,CAAC;AAEnD,IAAI,cAAc,GAAuC,IAAI,CAAC;AAC9D,IAAI,iBAAiB,GAA2C,IAAI,CAAC;AAUrE,IAAI,eAAe,GAAmC,IAAI,CAAC;AAC3D,IAAI,kBAAkB,GAAsC,IAAI,CAAC;AAEjE,SAAS,MAAM;IACb,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC,OAAO,CAAC;IACvB,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,YAAY,CAAC,KAAmB,EAAE,KAA+B;IACxE,eAAe,CAAC,KAAK,CAAC,GAAG;QACvB,GAAG,eAAe,CAAC,KAAK,CAAC;QACzB,GAAG,KAAK;KACT,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc;IAC3B,IAAI,UAAU,EAAE,CAAC;QACf,YAAY,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,cAAc,GAAG,CAAC,KAAK,IAAI,EAAE;YAC3B,eAAe,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe;oBACnC,CAAC,CAAC,eAAe,EAAE;oBACnB,CAAC,CAAE,kDAAO,eAAe,GAAwB,CAAC,CAAC;gBACrD,MAAM,MAAM,GAAI,MAA+B,CAAC,MAAM,CAAC;gBACvD,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;gBACzD,CAAC;gBACD,UAAU,GAAG,MAAqB,CAAC;gBACnC,YAAY,CAAC,KAAK,EAAE;oBAClB,KAAK,EAAE,IAAI;oBACX,WAAW,EAAE,MAAM,EAAE;oBACrB,SAAS,EAAE,SAAS;oBACpB,aAAa,EAAE,SAAS;iBACzB,CAAC,CAAC;gBACH,IAAA,6BAAiB,EAAC,MAAM,EAAE,yBAAyB,EAAE;oBACnD,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,QAAQ;iBACvC,CAAC,CAAC;gBACH,OAAO,UAAU,CAAC;YACpB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,GAAG,IAAI,CAAC;gBAClB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpC,YAAY,CAAC,KAAK,EAAE;oBAClB,KAAK,EAAE,KAAK;oBACZ,SAAS,EAAE,OAAO;oBAClB,aAAa,EAAE,MAAM,EAAE;iBACxB,CAAC,CAAC;gBACH,IAAA,gCAAoB,EAAC,MAAM,EAAE,iCAAiC,EAAE;oBAC9D,KAAK,EAAE,OAAO;oBACd,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,QAAQ;iBACvC,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAChB,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,IAAI,eAAe,EAAE,CAAC;QACpB,YAAY,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC9B,eAAe,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,kBAAkB;oBACtC,CAAC,CAAC,kBAAkB,EAAE;oBACtB,CAAC,CAAE,kDAAO,sBAAsB,GAA2B,CAAC,CAAC;gBAC/D,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;gBAE5E,IAAI,CAAC,aAAa,IAAI,OAAO,aAAa,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;oBAC1E,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM,SAAS,GAAG,CAAC,MAAM,aAAa,CAAC,eAAe,CACpD,yBAAyB,CAC1B,CAAoB,CAAC;gBACtB,eAAe,GAAG,SAAS,CAAC;gBAC5B,YAAY,CAAC,QAAQ,EAAE;oBACrB,KAAK,EAAE,IAAI;oBACX,WAAW,EAAE,MAAM,EAAE;oBACrB,SAAS,EAAE,SAAS;oBACpB,aAAa,EAAE,SAAS;iBACzB,CAAC,CAAC;gBACH,IAAA,6BAAiB,EAAC,MAAM,EAAE,4BAA4B,EAAE;oBACtD,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,QAAQ;iBAC1C,CAAC,CAAC;gBACH,OAAO,SAAS,CAAC;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,eAAe,GAAG,IAAI,CAAC;gBACvB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpC,YAAY,CAAC,QAAQ,EAAE;oBACrB,KAAK,EAAE,KAAK;oBACZ,SAAS,EAAE,OAAO;oBAClB,aAAa,EAAE,MAAM,EAAE;iBACxB,CAAC,CAAC;gBACH,IAAA,gCAAoB,EAAC,MAAM,EAAE,oCAAoC,EAAE;oBACjE,KAAK,EAAE,OAAO;oBACd,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,QAAQ;iBAC1C,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAChB,iBAAiB,GAAG,IAAI,CAAC;QAC3B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEM,KAAK,UAAU,qBAAqB;IACzC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE,mBAAmB,EAAE,CAAC,CAAC,CAAC;AAC/D,CAAC;AAEM,KAAK,UAAU,aAAa;IACjC,MAAM,OAAO,GAAG,MAAM,cAAc,EAAE,CAAC;IACvC,OAAO,OAAO,IAAI,IAAI,CAAC;AACzB,CAAC;AAEM,KAAK,UAAU,0BAA0B;IAC9C,MAAM,iBAAiB,GAAG,MAAM,mBAAmB,EAAE,CAAC;IACtD,OAAO,iBAAiB,IAAI,IAAI,CAAC;AACnC,CAAC;AAED,SAAgB,uBAAuB,CAAC,KAAqB;IAC3D,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC1C,YAAY,CAAC,KAAK,EAAE;YAClB,KAAK,EAAE,KAAK;YACZ,aAAa,EAAE,MAAM,EAAE;SACxB,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAgB,kBAAkB;IAChC,OAAO;QACL,SAAS,EAAE,MAAM,EAAE;QACnB,MAAM,EAAE;YACN,GAAG,EAAE,EAAE,GAAG,eAAe,CAAC,GAAG,EAAE;YAC/B,MAAM,EAAE,EAAE,GAAG,eAAe,CAAC,MAAM,EAAE;SACtC;QACD,SAAS,EAAE;YACT,GAAG,EAAE,cAAc,CAAC,GAAG;YACvB,MAAM,EAAE,cAAc,CAAC,MAAM;YAC7B,MAAM,EAAE,cAAc,CAAC,MAAM;SAC9B;KACF,CAAC;AACJ,CAAC;AAEY,QAAA,QAAQ,GAAG;IACtB,KAAK;QACH,eAAe,CAAC,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACpD,eAAe,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACvD,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC;QACvB,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1B,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1B,UAAU,GAAG,IAAI,CAAC;QAClB,eAAe,GAAG,IAAI,CAAC;QACvB,cAAc,GAAG,IAAI,CAAC;QACtB,iBAAiB,GAAG,IAAI,CAAC;QACzB,eAAe,GAAG,IAAI,CAAC;QACvB,kBAAkB,GAAG,IAAI,CAAC;IAC5B,CAAC;IACD,gBAAgB,CAAC,OAGhB;QACC,IAAI,OAAO,OAAO,CAAC,GAAG,KAAK,WAAW,EAAE,CAAC;YACvC,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC;YAC9B,UAAU,GAAG,IAAI,CAAC;YAClB,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC1C,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC;YACpC,eAAe,GAAG,IAAI,CAAC;YACvB,iBAAiB,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,QAAQ;QAIN,OAAO;YACL,SAAS,EAAE,UAAU,KAAK,IAAI;YAC9B,YAAY,EAAE,eAAe,KAAK,IAAI;SACvC,CAAC;IACJ,CAAC;CACF,CAAC"}

package/dist/intelligence/types.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Token Optimization Types
+ *
+ * Core interfaces and types for the token optimization engine.
+ */
+/**
+ * Supported AI models for optimization
+ */
+export type SupportedModel = 'claude' | 'gpt' | 'gemini';
+/**
+ * Compression level configuration
+ */
+export type CompressionLevel = 'conservative' | 'balanced' | 'aggressive';
+/**
+ * Compression pass types
+ */
+export type CompressionPassType = 'sanitization' | 'structural' | 'linguistic' | 'model-specific';
+/**
+ * Token count result
+ */
+export interface TokenCount {
+    model: SupportedModel;
+    count: number;
+    estimatedCost?: number;
+}
+/**
+ * Detailed token usage metrics for before/after comparisons
+ */
+export interface TokenUsageComparison {
+    model: SupportedModel;
+    original: TokenCount;
+    optimized: TokenCount;
+    savings: number;
+    compressionRatio: number;
+}
+/**
+ * Options for abortable intelligence operations
+ */
+export interface AbortableOptions {
+    signal?: AbortSignal;
+}
+/**
+ * Compression statistics
+ */
+export interface CompressionStats {
+    originalTokens: number;
+    compressedTokens: number;
+    reductionPercentage: number;
+    compressionRatio: number;
+    passesApplied: CompressionPassType[];
+}
+/**
+ * Optimization result
+ */
+export interface OptimizationResult {
+    original: string;
+    optimized: string;
+    stats: CompressionStats;
+    model: SupportedModel;
+    level: CompressionLevel;
+    tokenUsage: TokenUsageComparison;
+    warnings?: string[];
+}
+/**
+ * Compression rule definition
+ */
+export interface CompressionRule {
+    type: 'regex_replace' | 'convert_prose_to_list' | 'convert_passive_to_active' | 'preserve_block';
+    pattern?: string | RegExp;
+    replacement?: string;
+    targetSection?: string;
+    enabled: boolean;
+    flags?: string;
+    delimiters?: string[];
+}
+/**
+ * Ruleset configuration for compression
+ */
+export interface CompressionRuleset {
+    sanitizationRules: CompressionRule[];
+    structuralRules: CompressionRule[];
+    linguisticRules: CompressionRule[];
+    preservePatterns: RegExp[];
+}
+/**
+ * Model-specific configuration
+ */
+export interface ModelConfig {
+    model: SupportedModel;
+    templateFormat: 'xml' | 'markdown' | 'ptcf';
+    preferredDelimiters?: string[];
+    supportsXmlTags?: boolean;
+    supportsFewShot?: boolean;
+}
+/**
+ * Token counter interface
+ */
+export interface ITokenCounter {
+    count(text: string, model: SupportedModel, options?: AbortableOptions): Promise<TokenCount>;
+}
+/**
+ * Model transpiler interface
+ */
+export interface IModelTranspiler {
+    transpile(content: string, targetModel: SupportedModel): string;
+}
+/**
+ * Token optimizer configuration
+ */
+export interface TokenOptimizerConfig {
+    model: SupportedModel;
+    level: CompressionLevel;
+    ruleset?: Partial<CompressionRuleset>;
+    preserveTags?: string[];
+    dryRun?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/intelligence/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/intelligence/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,KAAK,GAAG,QAAQ,CAAC;AAEzD;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG,UAAU,GAAG,YAAY,CAAC;AAE1E;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAC;AAElG;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,cAAc,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,cAAc,CAAC;IACtB,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,EAAE,UAAU,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,mBAAmB,EAAE,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,gBAAgB,CAAC;IACxB,KAAK,EAAE,cAAc,CAAC;IACtB,KAAK,EAAE,gBAAgB,CAAC;IACxB,UAAU,EAAE,oBAAoB,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,eAAe,GAAG,uBAAuB,GAAG,2BAA2B,GAAG,gBAAgB,CAAC;IACjG,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iBAAiB,EAAE,eAAe,EAAE,CAAC;IACrC,eAAe,EAAE,eAAe,EAAE,CAAC;IACnC,eAAe,EAAE,eAAe,EAAE,CAAC;IACnC,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,cAAc,CAAC;IACtB,cAAc,EAAE,KAAK,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5C,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CAC7F;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,GAAG,MAAM,CAAC;CACjE;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,cAAc,CAAC;IACtB,KAAK,EAAE,gBAAgB,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}

package/dist/intelligence/types.js

@@ -0,0 +1,8 @@
+"use strict";
+/**
+ * Token Optimization Types
+ *
+ * Core interfaces and types for the token optimization engine.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/intelligence/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/intelligence/types.ts"],"names":[],"mappings":";AAAA;;;;GAIG"}

package/dist/intelligence/worker-manifest-loader.d.ts

@@ -0,0 +1,70 @@
+export type WorkerManifestValidationLevel = 'info' | 'warning' | 'error';
+export interface WorkerManifestValidation {
+    readonly level: WorkerManifestValidationLevel;
+    readonly code: string;
+    readonly message: string;
+    readonly workerId?: string;
+}
+export interface WorkerPatternSupport {
+    readonly mutuallyExclusive: boolean;
+    readonly allowedPatterns: readonly string[];
+    readonly fallbackPattern?: string;
+}
+export interface WorkerConstraints {
+    readonly requiredTools: readonly string[];
+    readonly maxConcurrent: number;
+    readonly timeoutSeconds?: number;
+    readonly tokenBudget?: number;
+}
+export interface WorkerTelemetry {
+    readonly emits: readonly string[];
+}
+export interface WorkerPatterns {
+    readonly supports: readonly string[];
+}
+export interface WorkerDefinition {
+    readonly workerId: string;
+    readonly name: string;
+    readonly description?: string;
+    readonly templatePath: string;
+    readonly capabilities: readonly string[];
+    readonly patterns: WorkerPatterns;
+    readonly constraints: WorkerConstraints;
+    readonly telemetry?: WorkerTelemetry;
+}
+export interface WorkerManifest {
+    readonly version: string;
+    readonly patternSupport: WorkerPatternSupport;
+    readonly workers: readonly WorkerDefinition[];
+}
+export interface WorkerManifestLoadResult {
+    readonly manifestPath: string;
+    readonly relativePath: string;
+    readonly loaded: boolean;
+    readonly manifest?: WorkerManifest;
+    readonly validations: readonly WorkerManifestValidation[];
+    readonly sizeBytes?: number;
+    readonly loadedAt: string;
+}
+export interface WorkerManifestLoaderOptions {
+    readonly manifestPath?: string;
+    readonly allowedPatterns?: readonly string[];
+    readonly maxWorkers?: number;
+}
+declare function isRecord(value: unknown): value is Record<string, unknown>;
+declare function toStringArray(value: unknown): string[];
+export declare class WorkerManifestLoader {
+    private readonly manifestPath;
+    private readonly allowedPatterns;
+    private readonly maxWorkers;
+    constructor(options?: WorkerManifestLoaderOptions);
+    load(projectRoot: string): Promise<WorkerManifestLoadResult>;
+    private normalizeManifest;
+    private safeStatSize;
+}
+export declare const __test__: {
+    toStringArray: typeof toStringArray;
+    isRecord: typeof isRecord;
+};
+export {};
+//# sourceMappingURL=worker-manifest-loader.d.ts.map

package/dist/intelligence/worker-manifest-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-manifest-loader.d.ts","sourceRoot":"","sources":["../../src/intelligence/worker-manifest-loader.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,6BAA6B,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;AAEzE,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,KAAK,EAAE,6BAA6B,CAAC;IAC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IACpC,QAAQ,CAAC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC;IACxC,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;CACtC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,oBAAoB,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,CAAC;CAC/C;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,wBAAwB,EAAE,CAAC;IAC1D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAOD,iBAAS,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAElE;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAQ/C;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAoB;IACpD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,OAAO,GAAE,2BAAgC;IAM/C,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC;IA2GlE,OAAO,CAAC,iBAAiB;YAqPX,YAAY;CAQ3B;AAED,eAAO,MAAM,QAAQ;;;CAGpB,CAAC"}