dartmind 17.0.0

package/dist/v10/context/TokenOptimizer.d.ts

@@ -0,0 +1,177 @@
+/**
+ * FlutterAI v10 - TokenOptimizer
+ *
+ * Optimizes token usage through intelligent content transformation.
+ *
+ * Features:
+ * - Content compression (whitespace, comments, redundancy)
+ * - Code minification for context
+ * - Smart truncation with boundary awareness
+ * - Deduplication of repeated content
+ * - Format conversion for efficiency
+ * - Batch optimization
+ *
+ * @module v10/context/TokenOptimizer
+ */
+import { EventEmitter } from 'events';
+export type OptimizationLevel = 'none' | 'light' | 'moderate' | 'aggressive';
+export type ContentCategory = 'code' | 'prose' | 'data' | 'mixed';
+export interface OptimizationResult {
+    original: string;
+    optimized: string;
+    originalTokens: number;
+    optimizedTokens: number;
+    savings: number;
+    savingsPercent: number;
+    techniques: string[];
+}
+export interface BatchOptimizationResult {
+    results: OptimizationResult[];
+    totalOriginalTokens: number;
+    totalOptimizedTokens: number;
+    totalSavings: number;
+    totalSavingsPercent: number;
+}
+export interface TruncationOptions {
+    maxTokens: number;
+    preserveStart?: boolean;
+    preserveEnd?: boolean;
+    ellipsis?: string;
+    boundaryAware?: boolean;
+}
+export interface TruncationResult {
+    content: string;
+    truncated: boolean;
+    originalTokens: number;
+    finalTokens: number;
+    truncatedAt?: number;
+}
+export interface CompressionRule {
+    id: string;
+    name: string;
+    pattern: RegExp;
+    replacement: string | ((match: string, ...args: string[]) => string);
+    category: ContentCategory[];
+    level: OptimizationLevel;
+}
+export interface OptimizerConfig {
+    /** Default optimization level */
+    defaultLevel?: OptimizationLevel;
+    /** Tokens per character for estimation */
+    tokensPerChar?: number;
+    /** Preserve code structure */
+    preserveCodeStructure?: boolean;
+    /** Preserve markdown formatting */
+    preserveMarkdown?: boolean;
+    /** Max compression iterations */
+    maxIterations?: number;
+    /** Minimum savings threshold to apply optimization */
+    minSavingsThreshold?: number;
+}
+export interface DeduplicationResult {
+    content: string;
+    duplicatesFound: number;
+    tokensRecovered: number;
+}
+export interface OptimizerStats {
+    totalOptimizations: number;
+    totalTokensSaved: number;
+    avgSavingsPercent: number;
+    techniqueUsage: Record<string, number>;
+}
+export type OptimizerEvent = {
+    type: 'optimization_applied';
+    result: OptimizationResult;
+} | {
+    type: 'truncation_applied';
+    result: TruncationResult;
+} | {
+    type: 'deduplication_applied';
+    result: DeduplicationResult;
+} | {
+    type: 'rule_matched';
+    rule: string;
+    savings: number;
+};
+export declare class TokenOptimizer extends EventEmitter {
+    private config;
+    private customRules;
+    private stats;
+    constructor(config?: OptimizerConfig);
+    /**
+     * Optimize content for token efficiency
+     */
+    optimize(content: string, level?: OptimizationLevel, category?: ContentCategory): OptimizationResult;
+    /**
+     * Optimize multiple content pieces
+     */
+    optimizeBatch(contents: string[], level?: OptimizationLevel, category?: ContentCategory): BatchOptimizationResult;
+    /**
+     * Truncate content to fit token limit
+     */
+    truncate(content: string, options: TruncationOptions): TruncationResult;
+    /**
+     * Smart truncate that tries to preserve meaningful boundaries
+     */
+    smartTruncate(content: string, maxTokens: number): TruncationResult;
+    /**
+     * Remove duplicate content blocks
+     */
+    deduplicate(content: string, minBlockSize?: number): DeduplicationResult;
+    /**
+     * Find and mark repeated patterns
+     */
+    findRepeatedPatterns(content: string, minLength?: number): Array<{
+        pattern: string;
+        count: number;
+        totalChars: number;
+    }>;
+    /**
+     * Optimize code content specifically
+     */
+    optimizeCode(code: string, language?: string): OptimizationResult;
+    /**
+     * Minify code for context (preserves structure but removes non-essential parts)
+     */
+    minifyForContext(code: string): string;
+    /**
+     * Add a custom compression rule
+     */
+    addRule(rule: Omit<CompressionRule, 'id'>): string;
+    /**
+     * Remove a custom rule
+     */
+    removeRule(ruleId: string): boolean;
+    /**
+     * Get all rules
+     */
+    getRules(): CompressionRule[];
+    /**
+     * Get optimizer statistics
+     */
+    getStats(): OptimizerStats;
+    /**
+     * Reset statistics
+     */
+    resetStats(): void;
+    /**
+     * Estimate tokens for content
+     */
+    estimateTokens(content: string): number;
+    /**
+     * Detect content category
+     */
+    detectCategory(content: string): ContentCategory;
+    private getRulesForLevel;
+    private applyRule;
+    private postProcess;
+    private adjustToBoundary;
+    private findBreakPoints;
+    private removeComments;
+    private collapseWhitespace;
+    private hashContent;
+    private trackTechniqueUsage;
+    private updateStats;
+}
+export default TokenOptimizer;
+//# sourceMappingURL=TokenOptimizer.d.ts.map

package/dist/v10/context/TokenOptimizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenOptimizer.d.ts","sourceRoot":"","sources":["../../../src/v10/context/TokenOptimizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAMtC,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,GAAG,YAAY,CAAC;AAC7E,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAC9B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC,CAAC;IACrE,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,KAAK,EAAE,iBAAiB,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,iCAAiC;IACjC,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC,0CAA0C;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAED,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,MAAM,EAAE,kBAAkB,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACxD;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,MAAM,EAAE,mBAAmB,CAAA;CAAE,GAC9D;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAuH5D,qBAAa,cAAe,SAAQ,YAAY;IAC9C,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,KAAK,CAKX;gBAEU,MAAM,GAAE,eAAoB;IASxC;;OAEG;IACH,QAAQ,CACN,OAAO,EAAE,MAAM,EACf,KAAK,GAAE,iBAA4C,EACnD,QAAQ,CAAC,EAAE,eAAe,GACzB,kBAAkB;IAwErB;;OAEG;IACH,aAAa,CACX,QAAQ,EAAE,MAAM,EAAE,EAClB,KAAK,CAAC,EAAE,iBAAiB,EACzB,QAAQ,CAAC,EAAE,eAAe,GACzB,uBAAuB;IAsB1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IA0DvE;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,gBAAgB;IA6CnE;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,GAAE,MAAW,GAAG,mBAAmB;IAyC5E;;OAEG;IACH,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,GAAG,KAAK,CAAC;QACnE,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IA6BF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,kBAAkB;IAsCjE;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAwBtC;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,MAAM;IAMlD;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAOnC;;OAEG;IACH,QAAQ,IAAI,eAAe,EAAE;IAQ7B;;OAEG;IACH,QAAQ,IAAI,cAAc;IAI1B;;OAEG;IACH,UAAU,IAAI,IAAI;IAalB;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAIvC;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe;IAiChD,OAAO,CAAC,gBAAgB;IAgBxB,OAAO,CAAC,SAAS;IAOjB,OAAO,CAAC,WAAW;IAkBnB,OAAO,CAAC,gBAAgB;IAoBxB,OAAO,CAAC,eAAe;IAyBvB,OAAO,CAAC,cAAc;IAgBtB,OAAO,CAAC,kBAAkB;IA+B1B,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,mBAAmB;IAK3B,OAAO,CAAC,WAAW;CAOpB;AAMD,eAAe,cAAc,CAAC"}