distill-mcp 0.6.0-beta

package/dist/utils/tfidf.test.js

@@ -0,0 +1,115 @@
+/**
+ * TF-IDF Utilities Tests
+ */
+import { describe, it, expect } from "vitest";
+import { calculateTFIDF, getSegmentTFIDFScore, getTopTerms } from "./tfidf.js";
+describe("TF-IDF utilities", () => {
+    describe("calculateTFIDF", () => {
+        it("should return empty map for empty segments", () => {
+            const result = calculateTFIDF([]);
+            expect(result.size).toBe(0);
+        });
+        it("should calculate TF-IDF for single segment", () => {
+            const segments = ["hello world"];
+            const result = calculateTFIDF(segments);
+            expect(result.size).toBe(1);
+            expect(result.get(0)).toBeDefined();
+        });
+        it("should calculate higher IDF for rare terms", () => {
+            const segments = [
+                "the quick brown fox",
+                "the lazy dog",
+                "the unique unicorn", // "unicorn" only appears here
+            ];
+            const result = calculateTFIDF(segments);
+            // Get scores for segment with "unicorn"
+            const unicornSegmentScores = result.get(2);
+            const unicornScore = unicornSegmentScores?.find((s) => s.term === "unicorn");
+            // Get scores for segment with "quick"
+            const quickSegmentScores = result.get(0);
+            const quickScore = quickSegmentScores?.find((s) => s.term === "quick");
+            expect(unicornScore).toBeDefined();
+            expect(quickScore).toBeDefined();
+            // Both are unique terms (appear in 1 segment), so IDF should be the same
+            // But TF might differ based on segment length
+            expect(unicornScore.idf).toBeGreaterThan(0);
+            expect(quickScore.idf).toBeGreaterThan(0);
+        });
+        it("should filter stopwords", () => {
+            const segments = ["the a an is are was"];
+            const result = calculateTFIDF(segments);
+            // All stopwords should be filtered, resulting in empty scores
+            const scores = result.get(0);
+            expect(scores).toEqual([]);
+        });
+        it("should handle segments with code terms", () => {
+            const segments = [
+                "function calculateSum returns number",
+                "const result equals function call",
+            ];
+            const result = calculateTFIDF(segments);
+            expect(result.size).toBe(2);
+            // Check that code terms are captured
+            const scores0 = result.get(0);
+            expect(scores0?.some((s) => s.term === "calculatesum")).toBe(true);
+        });
+    });
+    describe("getSegmentTFIDFScore", () => {
+        it("should return 0 for non-existent segment", () => {
+            const tfidfMap = calculateTFIDF(["hello world"]);
+            const score = getSegmentTFIDFScore(999, tfidfMap);
+            expect(score).toBe(0);
+        });
+        it("should return 0 for empty segment", () => {
+            const tfidfMap = calculateTFIDF([""]);
+            const score = getSegmentTFIDFScore(0, tfidfMap);
+            expect(score).toBe(0);
+        });
+        it("should return higher score for unique content", () => {
+            const segments = [
+                "common words common words common words",
+                "unique specialized terminology here",
+                "common words again common words",
+            ];
+            const tfidfMap = calculateTFIDF(segments);
+            const score0 = getSegmentTFIDFScore(0, tfidfMap);
+            const score1 = getSegmentTFIDFScore(1, tfidfMap);
+            const score2 = getSegmentTFIDFScore(2, tfidfMap);
+            // Segment 1 has unique terms, should have higher score
+            expect(score1).toBeGreaterThan(score0);
+            expect(score1).toBeGreaterThan(score2);
+        });
+        it("should return normalized score between 0 and 1", () => {
+            const segments = ["hello world", "foo bar baz"];
+            const tfidfMap = calculateTFIDF(segments);
+            const score0 = getSegmentTFIDFScore(0, tfidfMap);
+            const score1 = getSegmentTFIDFScore(1, tfidfMap);
+            expect(score0).toBeGreaterThanOrEqual(0);
+            expect(score0).toBeLessThanOrEqual(1);
+            expect(score1).toBeGreaterThanOrEqual(0);
+            expect(score1).toBeLessThanOrEqual(1);
+        });
+    });
+    describe("getTopTerms", () => {
+        it("should return empty array for non-existent segment", () => {
+            const tfidfMap = calculateTFIDF(["hello world"]);
+            const terms = getTopTerms(999, tfidfMap);
+            expect(terms).toEqual([]);
+        });
+        it("should return top N terms by TF-IDF score", () => {
+            const segments = ["alpha beta gamma delta epsilon zeta"];
+            const tfidfMap = calculateTFIDF(segments);
+            const top3 = getTopTerms(0, tfidfMap, 3);
+            expect(top3.length).toBeLessThanOrEqual(3);
+        });
+        it("should sort terms by TF-IDF descending", () => {
+            const segments = ["error critical failure warning info debug"];
+            const tfidfMap = calculateTFIDF(segments);
+            const terms = getTopTerms(0, tfidfMap);
+            // Verify sorting
+            for (let i = 1; i < terms.length; i++) {
+                expect(terms[i - 1].tfidf).toBeGreaterThanOrEqual(terms[i].tfidf);
+            }
+        });
+    });
+});

package/dist/utils/token-counter.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Centralized Token Counter
+ *
+ * Provides accurate token counting using js-tiktoken with a fallback
+ * approximation when the encoder is unavailable.
+ */
+/**
+ * Count tokens in a string using tiktoken
+ * Falls back to approximation if encoder unavailable
+ *
+ * @param text - The text to count tokens for
+ * @returns The number of tokens
+ */
+export declare function countTokens(text: string): number;
+/**
+ * Count tokens with explicit fallback behavior
+ *
+ * @param text - The text to count tokens for
+ * @returns Object with token count and whether approximation was used
+ */
+export declare function countTokensWithInfo(text: string): {
+    tokens: number;
+    isApproximate: boolean;
+};
+/**
+ * Estimate cost in microdollars based on token count
+ * Uses Claude Sonnet pricing as default
+ *
+ * @param inputTokens - Number of input tokens
+ * @param outputTokens - Number of output tokens
+ * @param model - Model name for pricing lookup
+ * @returns Cost in microdollars (1 microdollar = $0.000001)
+ */
+export declare function estimateCostMicros(inputTokens: number, outputTokens: number, model?: string): number;
+//# sourceMappingURL=token-counter.d.ts.map

package/dist/utils/token-counter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-counter.d.ts","sourceRoot":"","sources":["../../src/utils/token-counter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA8BH;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAMhD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,OAAO,CAAC;CACxB,CAYA;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,KAAK,GAAE,MAA0B,GAChC,MAAM,CAUR"}

package/dist/utils/token-counter.js

@@ -0,0 +1,83 @@
+/**
+ * Centralized Token Counter
+ *
+ * Provides accurate token counting using js-tiktoken with a fallback
+ * approximation when the encoder is unavailable.
+ */
+import { encodingForModel } from "js-tiktoken";
+// Singleton encoder instance
+let encoder = null;
+/**
+ * Get or create the tiktoken encoder
+ */
+function getEncoder() {
+    if (encoder === null) {
+        try {
+            encoder = encodingForModel("gpt-4");
+        }
+        catch (error) {
+            console.error("[ctxopt] Failed to initialize tiktoken encoder:", error);
+            return null;
+        }
+    }
+    return encoder;
+}
+/**
+ * Approximate token count when encoder is unavailable
+ * Uses ~4 characters per token as a rough estimate
+ */
+function approximateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Count tokens in a string using tiktoken
+ * Falls back to approximation if encoder unavailable
+ *
+ * @param text - The text to count tokens for
+ * @returns The number of tokens
+ */
+export function countTokens(text) {
+    const enc = getEncoder();
+    if (enc) {
+        return enc.encode(text).length;
+    }
+    return approximateTokens(text);
+}
+/**
+ * Count tokens with explicit fallback behavior
+ *
+ * @param text - The text to count tokens for
+ * @returns Object with token count and whether approximation was used
+ */
+export function countTokensWithInfo(text) {
+    const enc = getEncoder();
+    if (enc) {
+        return {
+            tokens: enc.encode(text).length,
+            isApproximate: false,
+        };
+    }
+    return {
+        tokens: approximateTokens(text),
+        isApproximate: true,
+    };
+}
+/**
+ * Estimate cost in microdollars based on token count
+ * Uses Claude Sonnet pricing as default
+ *
+ * @param inputTokens - Number of input tokens
+ * @param outputTokens - Number of output tokens
+ * @param model - Model name for pricing lookup
+ * @returns Cost in microdollars (1 microdollar = $0.000001)
+ */
+export function estimateCostMicros(inputTokens, outputTokens, model = "claude-sonnet-4") {
+    // Pricing in microdollars per token
+    const pricing = {
+        "claude-opus-4": { input: 15, output: 75 }, // $15/$75 per 1M tokens
+        "claude-sonnet-4": { input: 3, output: 15 }, // $3/$15 per 1M tokens
+        "claude-3-5-haiku": { input: 1, output: 5 }, // $1/$5 per 1M tokens
+    };
+    const modelPricing = pricing[model] ?? pricing["claude-sonnet-4"];
+    return inputTokens * modelPricing.input + outputTokens * modelPricing.output;
+}

package/dist/utils/toon-serializer.d.ts

@@ -0,0 +1,120 @@
+/**
+ * TOON Serializer for MCP Tools
+ *
+ * Converts MCP tool definitions to TOON (Token-Oriented Object Notation)
+ * format for ~40% token reduction when presenting tools to LLMs.
+ *
+ * TOON spec: https://toonformat.dev/
+ */
+import type { ToolDefinition } from "../tools/registry.js";
+import type { ToolCategory } from "../tools/dynamic-loader.js";
+/**
+ * Lightweight tool metadata for lazy serialization
+ */
+export interface ToolMetadataLite {
+    name: string;
+    category: ToolCategory;
+    description: string;
+    keywords?: string[];
+}
+export interface ToonSerializerOptions {
+    /** Include parameter details (default: true) */
+    includeParams?: boolean;
+    /** Include category grouping (default: true) */
+    groupByCategory?: boolean;
+    /** Category for each tool (optional) */
+    categories?: Map<string, string>;
+}
+/**
+ * Serialize tools to TOON format
+ *
+ * Output format:
+ * ```
+ * tools[N]:
+ *   tool_name(param1 param2?:type) → Description
+ *   ...
+ * ```
+ */
+export declare function serializeToolsToToon(tools: ToolDefinition[], options?: ToonSerializerOptions): string;
+/**
+ * Serialize tools to tabular TOON format (most compact)
+ *
+ * Output format:
+ * ```
+ * tools[N]{name,params,desc}:
+ *   auto_optimize,content hint?:enum agg?:bool,Auto-compress 80-95%
+ *   ...
+ * ```
+ */
+export declare function serializeToolsToToonTabular(tools: ToolDefinition[]): string;
+/**
+ * Compare token counts between JSON and TOON
+ */
+export declare function compareTokens(tools: ToolDefinition[]): {
+    json: number;
+    toon: number;
+    toonTabular: number;
+    savings: number;
+};
+/**
+ * Serialize tool metadata to lightweight TOON format (no schema loading)
+ *
+ * Used for lazy loading - shows available tools without loading full definitions.
+ * Output format:
+ * ```
+ * tools[N]:
+ *   tool_name → Description [keywords]
+ *   ...
+ * ```
+ */
+export declare function serializeMetadataToToon(metadata: ToolMetadataLite[], options?: {
+    groupByCategory?: boolean;
+}): string;
+/**
+ * Serialize tool metadata to lightweight tabular TOON format
+ */
+export declare function serializeMetadataToToonTabular(metadata: ToolMetadataLite[]): string;
+/**
+ * Result schema for TOON serialization
+ */
+export interface ResultSchema {
+    /** Name of the result type */
+    name: string;
+    /** Fields to include in output */
+    fields?: string[];
+    /** Maximum depth for nested objects */
+    maxDepth?: number;
+}
+/**
+ * Verbosity options for result serialization
+ */
+export interface ResultSerializerOptions {
+    /** Verbosity level */
+    verbosity?: "minimal" | "normal" | "detailed";
+    /** Include statistics */
+    includeStats?: boolean;
+    /** Maximum string length for values */
+    maxValueLength?: number;
+}
+/**
+ * Serialize any result to TOON format
+ *
+ * Converts objects, arrays, and primitives to compact TOON notation.
+ *
+ * Output formats:
+ * - Object: `name{field1,field2}: value1, value2`
+ * - Array: `items[N]: item1, item2, ...`
+ * - Stats: `stats: original→compressed (-X%)`
+ */
+export declare function serializeResultToToon(result: unknown, schema: ResultSchema, options?: ResultSerializerOptions): string;
+/**
+ * Serialize result to tabular TOON format (most compact)
+ */
+export declare function serializeResultToToonTabular(result: unknown, schema: ResultSchema): string;
+/**
+ * Format output based on global configuration
+ *
+ * Uses the output config singleton to determine format.
+ */
+export declare function formatOutputWithConfig(result: unknown, schema: ResultSchema): string;
+//# sourceMappingURL=toon-serializer.d.ts.map

package/dist/utils/toon-serializer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toon-serializer.d.ts","sourceRoot":"","sources":["../../src/utils/toon-serializer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAG/D;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,YAAY,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,gDAAgD;IAChD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,gDAAgD;IAChD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,wCAAwC;IACxC,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAsGD;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,cAAc,EAAE,EACvB,OAAO,GAAE,qBAA0B,GAClC,MAAM,CAgCR;AAaD;;;;;;;;;GASG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,cAAc,EAAE,GACtB,MAAM,CAkBR;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,cAAc,EAAE,GACtB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAyBtE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,OAAO,GAAE;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAO,GAC1C,MAAM,CAmCR;AAED;;GAEG;AACH,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,MAAM,CAgBR;AAMD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,sBAAsB;IACtB,SAAS,CAAC,EAAE,SAAS,GAAG,QAAQ,GAAG,UAAU,CAAC;IAC9C,yBAAyB;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,uCAAuC;IACvC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,OAAO,EACf,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,uBAA4B,GACpC,MAAM,CA6BR;AAmKD;;GAEG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,OAAO,EACf,MAAM,EAAE,YAAY,GACnB,MAAM,CAoCR;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,OAAO,EACf,MAAM,EAAE,YAAY,GACnB,MAAM,CAsBR"}