@mxf-dev/core 1.0.0

package/src/utils/ToolPaginationUtils.ts

@@ -0,0 +1,252 @@
+/**
+ * Copyright 2024 Brad Anderson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author Brad Anderson <BradA1878@pm.me>
+ * @repository https://github.com/mxf-dev/mxf
+ * @documentation https://mxf-dev.github.io/mxf/
+ */
+
+import { Logger } from './Logger.js';
+
+/**
+ * Tool Pagination Utilities
+ *
+ * Provides a reusable pagination framework for MCP tools with large result detection
+ * and LLM feedback. This helps prevent overwhelming the LLM's context window with
+ * excessively large tool results.
+ *
+ * Features:
+ * - Standard pagination input schema properties for tool definitions
+ * - Helper function to paginate arrays with metadata
+ * - Size checking with warning thresholds
+ * - Automatic pagination hints for large results
+ */
+
+/**
+ * Standard pagination input schema properties for tool definitions.
+ * Spread these into your tool's inputSchema.properties to add pagination support.
+ *
+ * @example
+ * inputSchema: {
+ *   type: 'object',
+ *   properties: {
+ *     searchTerm: { type: 'string' },
+ *     ...paginationInputSchema
+ *   }
+ * }
+ */
+export const paginationInputSchema = {
+    limit: {
+        type: 'number',
+        description: 'Maximum number of results to return (default: 50)',
+        default: 50,
+        minimum: 1,
+        maximum: 500
+    },
+    offset: {
+        type: 'number',
+        description: 'Number of results to skip for pagination (default: 0)',
+        default: 0,
+        minimum: 0
+    }
+};
+
+/**
+ * Pagination metadata included in paginated responses.
+ * Provides the LLM with information about total results and how to get more.
+ */
+export interface PaginationMetadata {
+    /** Total number of items before pagination was applied */
+    totalCount: number;
+    /** The limit that was applied (may differ from requested if clamped) */
+    limit: number;
+    /** The offset that was applied */
+    offset: number;
+    /** Whether more results exist beyond the current page */
+    hasMore: boolean;
+    /** Convenience field: the offset value to use for the next page (only present if hasMore is true) */
+    nextOffset?: number;
+}
+
+/**
+ * Result of paginating an array.
+ */
+export interface PaginatedResult<T> {
+    /** The paginated subset of items */
+    items: T[];
+    /** Pagination metadata for the response */
+    pagination: PaginationMetadata;
+}
+
+/**
+ * Pagination hint added to results that exceed size thresholds.
+ * Provides guidance to the LLM on how to use pagination to get manageable chunks.
+ */
+export interface PaginationHint {
+    /** Warning message about the large result */
+    warning: string;
+    /** Size of the result in bytes */
+    resultSize: number;
+    /** The threshold that was exceeded */
+    threshold: number;
+    /** Suggested pagination parameters */
+    suggestion: string;
+}
+
+/**
+ * Size thresholds for result checking.
+ * Results exceeding these sizes may impact LLM context window efficiency.
+ *
+ * Thresholds tuned based on demo testing with LSP code intelligence tools
+ * which typically return 9-10KB results. These thresholds ensure meaningful
+ * pagination hints are provided for typical large tool results.
+ */
+export const RESULT_SIZE_THRESHOLDS = {
+    /** Log a warning at this size (bytes) */
+    WARNING: 5000,
+    /** Add pagination hint at this size (bytes) */
+    CRITICAL: 8000
+};
+
+/**
+ * Paginate an array and return both the paginated items and metadata.
+ *
+ * @param items - The full array of items to paginate
+ * @param limit - Maximum number of items to return (default: 50, clamped to 1-500)
+ * @param offset - Number of items to skip (default: 0, clamped to >= 0)
+ * @returns Paginated items and metadata
+ *
+ * @example
+ * const allFunctions = await findAllFunctions();
+ * const { items: functions, pagination } = paginateArray(allFunctions, args.limit, args.offset);
+ * return {
+ *   success: true,
+ *   functions,
+ *   ...pagination  // totalCount, limit, offset, hasMore, nextOffset
+ * };
+ */
+export function paginateArray<T>(
+    items: T[],
+    limit: number = 50,
+    offset: number = 0
+): PaginatedResult<T> {
+    // Clamp limit to valid range
+    const clampedLimit = Math.max(1, Math.min(500, limit));
+    // Clamp offset to non-negative
+    const clampedOffset = Math.max(0, offset);
+
+    const totalCount = items.length;
+    const paginatedItems = items.slice(clampedOffset, clampedOffset + clampedLimit);
+    const hasMore = clampedOffset + clampedLimit < totalCount;
+
+    return {
+        items: paginatedItems,
+        pagination: {
+            totalCount,
+            limit: clampedLimit,
+            offset: clampedOffset,
+            hasMore,
+            ...(hasMore ? { nextOffset: clampedOffset + clampedLimit } : {})
+        }
+    };
+}
+
+/**
+ * Check the size of a tool result and add a pagination hint if it exceeds the critical threshold.
+ * Also logs warnings for large results.
+ *
+ * @param result - The tool result object to check
+ * @param toolName - Name of the tool for logging
+ * @param logger - Logger instance to use for warnings
+ * @returns The original result, possibly with a _paginationHint property added
+ *
+ * @example
+ * const result = { success: true, functions: [...], totalCount: 500 };
+ * return checkResultSize(result, 'find_functions', logger);
+ */
+export function checkResultSize<T extends object>(
+    result: T,
+    toolName: string,
+    logger: Logger
+): T & { _paginationHint?: PaginationHint } {
+    const resultString = JSON.stringify(result);
+    const resultSize = resultString.length;
+
+    if (resultSize > RESULT_SIZE_THRESHOLDS.CRITICAL) {
+        logger.warn(`LARGE TOOL RESULT: ${toolName} returned ${resultSize} bytes - may impact LLM context window. Consider using limit/offset pagination.`);
+
+        return {
+            ...result,
+            _paginationHint: {
+                warning: 'Large result detected. Use limit/offset parameters to paginate.',
+                resultSize,
+                threshold: RESULT_SIZE_THRESHOLDS.CRITICAL,
+                suggestion: 'Try adding { "limit": 25, "offset": 0 } to get first 25 results, then increment offset for more.'
+            }
+        };
+    } else if (resultSize > RESULT_SIZE_THRESHOLDS.WARNING) {
+        logger.warn(`LARGE TOOL RESULT: ${toolName} returned ${resultSize} bytes - approaching context window impact threshold.`);
+    }
+
+    return result;
+}
+
+/**
+ * Convenience function to apply pagination to multiple arrays within a result.
+ * Useful for tools that return multiple lists (e.g., imports, exports, dependents).
+ *
+ * @param arrays - Object mapping array names to arrays
+ * @param limit - Limit to apply to each array
+ * @param offset - Offset to apply to each array
+ * @returns Object with paginated arrays and combined metadata
+ *
+ * @example
+ * const {
+ *   paginatedArrays: { imports, exports, dependents },
+ *   metadata
+ * } = paginateMultipleArrays(
+ *   { imports: allImports, exports: allExports, dependents: allDependents },
+ *   args.limit,
+ *   args.offset
+ * );
+ */
+export function paginateMultipleArrays<T extends Record<string, any[]>>(
+    arrays: T,
+    limit: number = 50,
+    offset: number = 0
+): {
+    paginatedArrays: T;
+    metadata: {
+        [K in keyof T]: PaginationMetadata;
+    } & { combinedTotalCount: number; anyHasMore: boolean };
+} {
+    const paginatedArrays: any = {};
+    const metadata: any = { combinedTotalCount: 0, anyHasMore: false };
+
+    for (const [key, items] of Object.entries(arrays)) {
+        const { items: paginatedItems, pagination } = paginateArray(items, limit, offset);
+        paginatedArrays[key] = paginatedItems;
+        metadata[key] = pagination;
+        metadata.combinedTotalCount += pagination.totalCount;
+        if (pagination.hasMore) {
+            metadata.anyHasMore = true;
+        }
+    }
+
+    return {
+        paginatedArrays,
+        metadata
+    };
+}

package/src/utils/env.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright 2024 Brad Anderson
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file for details
+ * @author Brad Anderson <BradA1878@pm.me>
+ */
+/**
+ * Fail-fast environment variable access.
+ *
+ * Project rule: no silent fallbacks. Configuration that the framework cannot
+ * run safely without must throw immediately with an actionable message
+ * instead of degrading to an insecure or surprising default.
+ */
+
+/**
+ * Returns the value of a required environment variable, throwing with an
+ * actionable message when it is missing or blank.
+ */
+export const requireEnv = (name: string, hint?: string): string => {
+    const value = process.env[name];
+    if (!value || value.trim() === '') {
+        throw new Error(
+            `Missing required environment variable ${name}.${hint ? ` ${hint}` : ''}`
+        );
+    }
+    return value;
+};

package/src/utils/toon/eligibility.ts

@@ -0,0 +1,316 @@
+/**
+ * Copyright 2024 Brad Anderson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author Brad Anderson <BradA1878@pm.me>
+ * @repository https://github.com/mxf-dev/mxf
+ * @documentation https://mxf-dev.github.io/mxf/
+ */
+
+/**
+ * TOON Eligibility Evaluator
+ *
+ * This module evaluates whether a payload is suitable for TOON encoding.
+ * Eligible payloads contain arrays of uniform objects with primitive values.
+ *
+ * Eligibility criteria:
+ * - Array contains at least minArrayLength elements (default: 5)
+ * - Array elements are objects with identical keys
+ * - All values are primitives (string, number, boolean, null)
+ * - Calculated eligibility score >= minScore (default: 0.8)
+ */
+
+import { EligibilityResult, EligibilityOptions, ToonArray, ToonPrimitive } from './types.js';
+import { Logger } from '../Logger.js';
+
+const logger = new Logger('debug', 'ToonEligibility', 'server');
+
+/**
+ * Default eligibility options
+ */
+const DEFAULT_OPTIONS: Required<EligibilityOptions> = {
+  minArrayLength: 5,
+  minScore: 0.8,
+};
+
+/**
+ * Check if a value is a primitive type allowed in TOON format
+ */
+function isPrimitive(value: any): value is ToonPrimitive {
+  const type = typeof value;
+  return (
+    value === null ||
+    type === 'string' ||
+    type === 'number' ||
+    type === 'boolean'
+  );
+}
+
+/**
+ * Check if all elements in an array are objects with the same keys
+ */
+function hasUniformKeys(elements: any[]): boolean {
+  if (elements.length === 0) return false;
+  if (!elements[0] || typeof elements[0] !== 'object') return false;
+
+  // Get keys from first element as reference
+  const referenceKeys = Object.keys(elements[0]).sort();
+
+  // Check all elements have the same keys
+  for (let i = 1; i < elements.length; i++) {
+    if (!elements[i] || typeof elements[i] !== 'object') {
+      return false;
+    }
+
+    const currentKeys = Object.keys(elements[i]).sort();
+    if (currentKeys.length !== referenceKeys.length) {
+      return false;
+    }
+
+    for (let j = 0; j < referenceKeys.length; j++) {
+      if (currentKeys[j] !== referenceKeys[j]) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+/**
+ * Check if all values in array elements are primitives
+ */
+function hasOnlyPrimitiveValues(elements: any[]): boolean {
+  for (const element of elements) {
+    for (const key in element) {
+      if (!isPrimitive(element[key])) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+/**
+ * Calculate eligibility score for an array
+ *
+ * Score calculation factors:
+ * - Array length (longer arrays benefit more from TOON)
+ * - Number of fields (more fields = more key repetition savings)
+ * - Value complexity (primitives only = higher score)
+ *
+ * Score formula:
+ * score = (lengthScore * 0.3) + (fieldScore * 0.3) + (uniformityScore * 0.4)
+ *
+ * Scoring is designed to give arrays meeting minimum criteria (5+ elements, 2+ fields, uniform)
+ * a baseline score of ~0.8, with improvements for larger/more complex arrays.
+ */
+function calculateArrayScore(elements: any[]): number {
+  if (elements.length === 0) return 0;
+
+  // Length score: normalize array length with more generous curve
+  // 5 elements = 0.8, 10 = 0.9, 15+ = 1.0
+  const lengthScore = Math.min(0.6 + (elements.length / 25), 1.0);
+
+  // Field score: more fields = better TOON benefit
+  // 2 fields = 0.8, 3 = 0.87, 5+ = 1.0
+  const fieldCount = Object.keys(elements[0]).length;
+  const fieldScore = Math.min(0.7 + (fieldCount / 15), 1.0);
+
+  // Uniformity score: all elements have same structure
+  // This is critical - non-uniform arrays get 0 score
+  const uniformityScore = hasUniformKeys(elements) ? 1.0 : 0.0;
+
+  // Weighted average - uniformity is most important
+  return (lengthScore * 0.3) + (fieldScore * 0.3) + (uniformityScore * 0.4);
+}
+
+/**
+ * Find all TOON-eligible arrays in a payload recursively
+ */
+function findEligibleArrays(
+  payload: any,
+  options: Required<EligibilityOptions>,
+  path: string = '$'
+): ToonArray[] {
+  const eligibleArrays: ToonArray[] = [];
+
+  // Check if current value is an array
+  if (Array.isArray(payload)) {
+    // Check basic eligibility
+    if (
+      payload.length >= options.minArrayLength &&
+      hasUniformKeys(payload) &&
+      hasOnlyPrimitiveValues(payload)
+    ) {
+      const score = calculateArrayScore(payload);
+      if (score >= options.minScore) {
+        eligibleArrays.push({
+          path,
+          elements: payload,
+          fields: Object.keys(payload[0]).sort(),
+          length: payload.length,
+        });
+      }
+    }
+  }
+
+  // Recursively check nested objects and arrays
+  if (payload && typeof payload === 'object') {
+    for (const key in payload) {
+      if (payload.hasOwnProperty(key)) {
+        const nestedPath = Array.isArray(payload) ? `${path}[${key}]` : `${path}.${key}`;
+        const nested = findEligibleArrays(payload[key], options, nestedPath);
+        eligibleArrays.push(...nested);
+      }
+    }
+  }
+
+  return eligibleArrays;
+}
+
+/**
+ * Evaluate whether a payload is eligible for TOON encoding
+ *
+ * This function performs a comprehensive analysis of the payload structure
+ * to determine if TOON encoding would provide benefits.
+ *
+ * @param payload - The payload to evaluate
+ * @param options - Eligibility options
+ * @returns Eligibility result with score and details
+ */
+export function evaluateEligibility(
+  payload: unknown,
+  options: EligibilityOptions = {}
+): EligibilityResult {
+  const opts: Required<EligibilityOptions> = {
+    ...DEFAULT_OPTIONS,
+    ...options,
+  };
+
+  // Start timing
+  const startTime = Date.now();
+
+  try {
+    // Handle null/undefined
+    if (payload === null || payload === undefined) {
+      return {
+        eligible: false,
+        score: 0,
+        reason: 'Payload is null or undefined',
+      };
+    }
+
+    // Handle non-object types
+    if (typeof payload !== 'object') {
+      return {
+        eligible: false,
+        score: 0,
+        reason: 'Payload is not an object or array',
+      };
+    }
+
+    // Find all eligible arrays in the payload
+    const eligibleArrays = findEligibleArrays(payload, opts);
+
+    // Calculate evaluation latency
+    const latencyMs = Date.now() - startTime;
+
+    // Log if evaluation took too long (>5ms threshold)
+    if (latencyMs > 5) {
+      logger.warn(`Eligibility evaluation took ${latencyMs}ms (threshold: 5ms)`);
+    }
+
+    // No eligible arrays found
+    if (eligibleArrays.length === 0) {
+      return {
+        eligible: false,
+        score: 0,
+        reason: 'No arrays meet eligibility criteria (uniform structure, primitives only, min length)',
+      };
+    }
+
+    // Calculate overall score based on best eligible array
+    const scores = eligibleArrays.map((arr) => calculateArrayScore(arr.elements));
+    const maxScore = Math.max(...scores);
+
+    return {
+      eligible: true,
+      score: maxScore,
+      eligiblePaths: eligibleArrays.map((arr) => arr.path),
+    };
+  } catch (error) {
+    logger.error('Error evaluating eligibility', error);
+    return {
+      eligible: false,
+      score: 0,
+      reason: `Evaluation error: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}
+
+/**
+ * Extract eligible arrays from a payload
+ *
+ * This is a helper function that returns the actual array data
+ * that can be encoded in TOON format.
+ *
+ * @param payload - The payload to extract from
+ * @param options - Eligibility options
+ * @returns Array of TOON-eligible arrays with their paths
+ */
+export function extractEligibleArrays(
+  payload: unknown,
+  options: EligibilityOptions = {}
+): ToonArray[] {
+  const opts: Required<EligibilityOptions> = {
+    ...DEFAULT_OPTIONS,
+    ...options,
+  };
+
+  if (!payload || typeof payload !== 'object') {
+    return [];
+  }
+
+  return findEligibleArrays(payload, opts);
+}
+
+/**
+ * Quick check if a value looks like it might be TOON-eligible
+ *
+ * This is a fast pre-check that can be used before doing full evaluation.
+ * It doesn't guarantee eligibility but filters out obvious non-candidates.
+ *
+ * @param payload - The payload to check
+ * @returns True if payload might be eligible (requires full evaluation to confirm)
+ */
+export function mightBeEligible(payload: unknown): boolean {
+  if (!payload || typeof payload !== 'object') {
+    return false;
+  }
+
+  // Check if payload contains any arrays (with circular reference protection)
+  const hasArrays = (obj: any, visited = new WeakSet()): boolean => {
+    if (Array.isArray(obj)) return true;
+    if (typeof obj === 'object' && obj !== null) {
+      // Detect circular references
+      if (visited.has(obj)) return false;
+      visited.add(obj);
+      return Object.values(obj).some((value) => hasArrays(value, visited));
+    }
+    return false;
+  };
+
+  return hasArrays(payload);
+}