@small-ltsc/ml 0.1.0

package/README.md

@@ -0,0 +1,158 @@
+# @small-ltsc/ml
+
+Optional ML features for **Small LTSC** - Pattern importance scoring, quality prediction, and adaptive region detection.
+
+## Installation
+
+```bash
+npm install @small-ltsc/ml @small-ltsc/sdk
+```
+
+Note: `@small-ltsc/sdk` is a peer dependency.
+
+## Features
+
+- **Pattern Importance Scoring** - Determine which patterns are semantically important
+- **Quality Prediction** - Predict if compression will degrade model performance
+- **Region Detection** - Identify system prompts, user input, and context for adaptive compression
+
+## Pattern Importance
+
+Score patterns to preserve semantically important content:
+
+```typescript
+import { PositionalImportanceScorer, filterByImportance } from '@small-ltsc/ml';
+import { discoverPatterns } from '@small-ltsc/sdk';
+
+const scorer = new PositionalImportanceScorer({ decayRate: 2.0 });
+const patterns = await discoverPatterns(tokens);
+const scores = await scorer.scorePatterns(tokens, patterns);
+
+// Filter out high-importance patterns (preserve them)
+const safeToCompress = filterByImportance(patterns, scores, 0.8);
+```
+
+### Embedding-Based Scoring
+
+For more accurate importance scoring with an embedding model:
+
+```typescript
+import { EmbeddingImportanceScorer } from '@small-ltsc/ml';
+
+const scorer = new EmbeddingImportanceScorer(embeddingProvider, {
+  contextWindow: 5,
+});
+
+const scores = await scorer.scorePatterns(tokens, patterns);
+```
+
+## Quality Prediction
+
+Predict if compressed output will maintain quality:
+
+```typescript
+import { createQualityPredictor } from '@small-ltsc/ml';
+import { compress } from '@small-ltsc/sdk';
+
+const predictor = createQualityPredictor();
+const result = await compress(tokens);
+const prediction = await predictor.predict(result);
+
+if (!prediction.acceptable) {
+  console.log(`Recommendation: ${prediction.recommendation}`);
+  // 'accept' | 'retry_conservative' | 'skip_compression'
+}
+```
+
+### Quality Features
+
+```typescript
+console.log(prediction.features);
+// {
+//   compressionRatio: 0.65,
+//   dictionaryOverhead: 0.15,
+//   diversityReduction: 0.2,
+//   averagePatternLength: 4.5,
+//   patternCount: 12,
+// }
+```
+
+## Region Detection
+
+Detect semantic regions for adaptive compression:
+
+```typescript
+import { detectRegions, RegionType, filterPatternsByRegion } from '@small-ltsc/ml';
+
+const regions = detectRegions(tokens, {
+  systemMarkers: [[58, 71905, 60]], // [SYSTEM] tokens
+  retentionTargets: {
+    [RegionType.SYSTEM]: 0.98,  // Minimal compression
+    [RegionType.USER]: 0.85,    // Moderate
+    [RegionType.CONTEXT]: 0.6,  // Aggressive
+  },
+});
+
+// Filter patterns based on region constraints
+const filtered = filterPatternsByRegion(patterns, regions, tokens);
+```
+
+### Region Types
+
+- `SYSTEM` - System instructions (high retention)
+- `USER` - User input (moderate retention)
+- `CONTEXT` - Injected context/documents (low retention)
+- `CODE` - Code blocks (moderate retention)
+- `UNKNOWN` - Default region
+
+## Custom Embedding Provider
+
+Implement the `EmbeddingProvider` interface for your embedding model:
+
+```typescript
+import type { EmbeddingProvider } from '@small-ltsc/ml';
+
+class OpenAIEmbeddings implements EmbeddingProvider {
+  async embed(tokens: readonly number[]): Promise<Float32Array> {
+    const text = tokenizer.decode(tokens);
+    const response = await openai.embeddings.create({
+      model: 'text-embedding-3-small',
+      input: text,
+    });
+    return new Float32Array(response.data[0].embedding);
+  }
+
+  dimension(): number {
+    return 1536;
+  }
+}
+
+const scorer = new EmbeddingImportanceScorer(new OpenAIEmbeddings());
+```
+
+## API Reference
+
+### Importance Scoring
+
+- `PositionalImportanceScorer` - Score by position (early = important)
+- `EmbeddingImportanceScorer` - Score by context diversity
+- `CombinedImportanceScorer` - Combine positional and embedding scoring
+- `adjustPrioritiesByImportance(patterns, scores, threshold)` - Adjust pattern priorities
+- `filterByImportance(patterns, scores, threshold)` - Filter high-importance patterns
+
+### Quality Prediction
+
+- `HeuristicQualityPredictor` - Rule-based quality prediction
+- `EmbeddingQualityPredictor` - Enhanced with embedding similarity
+- `createQualityPredictor(provider?, config?)` - Factory function
+
+### Region Detection
+
+- `detectRegions(tokens, config?)` - Detect semantic regions
+- `detectRegionsHeuristic(tokens)` - Simple heuristic detection
+- `filterPatternsByRegion(patterns, regions, tokens)` - Filter by region
+- `getRegionCompressionSettings(regionType)` - Get settings for region
+
+## License
+
+MIT

package/dist/esm/importance.js

@@ -0,0 +1,190 @@
+/**
+ * Pattern importance scoring for ML-aware compression.
+ *
+ * Port of `small/pattern_importance.py` to TypeScript.
+ */
+const DEFAULT_IMPORTANCE_CONFIG = {
+    decayRate: 2.0,
+    contextWindow: 5,
+    positionalWeight: 0.3,
+};
+/**
+ * Positional importance scorer.
+ *
+ * Scores patterns based on their position in the sequence,
+ * with earlier positions receiving higher importance (useful for
+ * system prompts that typically appear at the start).
+ */
+export class PositionalImportanceScorer {
+    decayRate;
+    constructor(config) {
+        const cfg = { ...DEFAULT_IMPORTANCE_CONFIG, ...config };
+        this.decayRate = cfg.decayRate;
+    }
+    async scorePatterns(tokens, patterns) {
+        const n = tokens.length;
+        if (n === 0) {
+            return patterns.map(() => 0);
+        }
+        return patterns.map((pattern) => {
+            if (pattern.positions.length === 0) {
+                return 0;
+            }
+            // Compute average positional importance across all occurrences
+            let totalImportance = 0;
+            for (const pos of pattern.positions) {
+                // Exponential decay from start of sequence
+                const normalizedPos = pos / n;
+                const importance = Math.exp(-this.decayRate * normalizedPos);
+                totalImportance += importance;
+            }
+            return totalImportance / pattern.positions.length;
+        });
+    }
+}
+/**
+ * Embedding-based importance scorer.
+ *
+ * Uses an embedding model to determine if a pattern appears in
+ * diverse semantic contexts (important, should preserve) vs.
+ * similar contexts (redundant, safe to compress).
+ */
+export class EmbeddingImportanceScorer {
+    provider;
+    contextWindow;
+    constructor(provider, config) {
+        const cfg = { ...DEFAULT_IMPORTANCE_CONFIG, ...config };
+        this.provider = provider;
+        this.contextWindow = cfg.contextWindow;
+    }
+    async scorePatterns(tokens, patterns) {
+        const tokenArray = Array.isArray(tokens) ? tokens : Array.from(tokens);
+        const n = tokenArray.length;
+        if (n === 0) {
+            return patterns.map(() => 0);
+        }
+        const scores = [];
+        for (const pattern of patterns) {
+            if (pattern.positions.length <= 1) {
+                // Single occurrence - can't compute diversity
+                scores.push(0.5);
+                continue;
+            }
+            // Extract context windows around each occurrence
+            const contextEmbeddings = [];
+            for (const pos of pattern.positions) {
+                const start = Math.max(0, pos - this.contextWindow);
+                const end = Math.min(n, pos + pattern.length + this.contextWindow);
+                const context = tokenArray.slice(start, end);
+                const embedding = await this.provider.embed(context);
+                contextEmbeddings.push(embedding);
+            }
+            // Compute pairwise cosine similarities
+            const similarities = [];
+            for (let i = 0; i < contextEmbeddings.length; i++) {
+                for (let j = i + 1; j < contextEmbeddings.length; j++) {
+                    const sim = cosineSimilarity(contextEmbeddings[i], contextEmbeddings[j]);
+                    similarities.push(sim);
+                }
+            }
+            // Low average similarity = diverse contexts = high importance
+            const avgSimilarity = similarities.length > 0
+                ? similarities.reduce((a, b) => a + b, 0) / similarities.length
+                : 0;
+            // Convert to importance (invert similarity)
+            scores.push(1 - avgSimilarity);
+        }
+        return scores;
+    }
+}
+/**
+ * Combined importance scorer that uses both positional and embedding-based scoring.
+ */
+export class CombinedImportanceScorer {
+    positionalScorer;
+    embeddingScorer;
+    positionalWeight;
+    constructor(provider, config) {
+        const cfg = { ...DEFAULT_IMPORTANCE_CONFIG, ...config };
+        this.positionalScorer = new PositionalImportanceScorer(config);
+        this.embeddingScorer = provider ? new EmbeddingImportanceScorer(provider, config) : null;
+        this.positionalWeight = cfg.positionalWeight;
+    }
+    async scorePatterns(tokens, patterns) {
+        const positionalScores = await this.positionalScorer.scorePatterns(tokens, patterns);
+        if (!this.embeddingScorer) {
+            return positionalScores;
+        }
+        const embeddingScores = await this.embeddingScorer.scorePatterns(tokens, patterns);
+        // Weighted combination
+        return positionalScores.map((posScore, i) => {
+            const embScore = embeddingScores[i];
+            return this.positionalWeight * posScore + (1 - this.positionalWeight) * embScore;
+        });
+    }
+}
+/**
+ * Compute cosine similarity between two vectors.
+ */
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        throw new Error('Vectors must have same length');
+    }
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    if (normA === 0 || normB === 0) {
+        return 0;
+    }
+    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+/**
+ * Adjust pattern priorities based on importance scores.
+ *
+ * Patterns with high importance get lower priority (compressed later),
+ * patterns with low importance get higher priority (compressed first).
+ *
+ * @param patterns - Patterns to adjust
+ * @param scores - Importance scores from an ImportanceScorer
+ * @param threshold - Patterns above this importance threshold get negative priority
+ * @returns Patterns with adjusted priorities
+ */
+export function adjustPrioritiesByImportance(patterns, scores, threshold = 0.7) {
+    return patterns.map((pattern, i) => {
+        const importance = scores[i];
+        // High importance = low priority (preserve)
+        // Low importance = high priority (compress)
+        let priorityAdjustment;
+        if (importance > threshold) {
+            // Very important - negative priority to preserve
+            priorityAdjustment = -10 * importance;
+        }
+        else {
+            // Less important - positive priority to compress
+            priorityAdjustment = 10 * (1 - importance);
+        }
+        return {
+            ...pattern,
+            // Store adjusted priority (would need to extend type)
+            _importanceScore: importance,
+            _adjustedPriority: priorityAdjustment,
+        };
+    });
+}
+/**
+ * Filter patterns to only those below an importance threshold.
+ *
+ * @param patterns - Patterns to filter
+ * @param scores - Importance scores
+ * @param threshold - Maximum importance score to include
+ * @returns Filtered patterns that are safe to compress
+ */
+export function filterByImportance(patterns, scores, threshold = 0.8) {
+    return patterns.filter((_, i) => scores[i] < threshold);
+}
+//# sourceMappingURL=importance.js.map

package/dist/esm/importance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"importance.js","sourceRoot":"","sources":["../../src/importance.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA8CH,MAAM,yBAAyB,GAA+B;IAC5D,SAAS,EAAE,GAAG;IACd,aAAa,EAAE,CAAC;IAChB,gBAAgB,EAAE,GAAG;CACtB,CAAC;AAmBF;;;;;;GAMG;AACH,MAAM,OAAO,0BAA0B;IAC7B,SAAS,CAAS;IAE1B,YAAY,MAAyB;QACnC,MAAM,GAAG,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,MAAM,EAAE,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgB,EAAE,QAA6B;QACjE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,OAAO,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnC,OAAO,CAAC,CAAC;YACX,CAAC;YAED,+DAA+D;YAC/D,IAAI,eAAe,GAAG,CAAC,CAAC;YACxB,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACpC,2CAA2C;gBAC3C,MAAM,aAAa,GAAG,GAAG,GAAG,CAAC,CAAC;gBAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,CAAC;gBAC7D,eAAe,IAAI,UAAU,CAAC;YAChC,CAAC;YAED,OAAO,eAAe,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,OAAO,yBAAyB;IAC5B,QAAQ,CAAoB;IAC5B,aAAa,CAAS;IAE9B,YAAY,QAA2B,EAAE,MAAyB;QAChE,MAAM,GAAG,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,MAAM,EAAE,CAAC;QACxD,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgB,EAAE,QAA6B;QACjE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;QAE5B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAClC,8CAA8C;gBAC9C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,SAAS;YACX,CAAC;YAED,iDAAiD;YACjD,MAAM,iBAAiB,GAAmB,EAAE,CAAC;YAE7C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;gBACpD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;gBACnE,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAE7C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACrD,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACpC,CAAC;YAED,uCAAuC;YACvC,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACtD,MAAM,GAAG,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;oBACzE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,8DAA8D;YAC9D,MAAM,aAAa,GACjB,YAAY,CAAC,MAAM,GAAG,CAAC;gBACrB,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM;gBAC/D,CAAC,CAAC,CAAC,CAAC;YAER,4CAA4C;YAC5C,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,wBAAwB;IAC3B,gBAAgB,CAA6B;IAC7C,eAAe,CAAmC;IAClD,gBAAgB,CAAS;IAEjC,YAAY,QAA4B,EAAE,MAAyB;QACjE,MAAM,GAAG,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,MAAM,EAAE,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,IAAI,0BAA0B,CAAC,MAAM,CAAC,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,yBAAyB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzF,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC,gBAAgB,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgB,EAAE,QAA6B;QACjE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAErF,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,gBAAgB,CAAC;QAC1B,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEnF,uBAAuB;QACvB,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE;YAC1C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YACpC,OAAO,IAAI,CAAC,gBAAgB,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC;QACnF,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,CAAe,EAAE,CAAe;IACxD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,4BAA4B,CAC1C,QAA6B,EAC7B,MAAgB,EAChB,SAAS,GAAG,GAAG;IAEf,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;QACjC,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE7B,4CAA4C;QAC5C,4CAA4C;QAC5C,IAAI,kBAA0B,CAAC;QAE/B,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;YAC3B,iDAAiD;YACjD,kBAAkB,GAAG,CAAC,EAAE,GAAG,UAAU,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,iDAAiD;YACjD,kBAAkB,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,GAAG,OAAO;YACV,sDAAsD;YACtD,gBAAgB,EAAE,UAAU;YAC5B,iBAAiB,EAAE,kBAAkB;SACyC,CAAC;IACnF,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAA6B,EAC7B,MAAgB,EAChB,SAAS,GAAG,GAAG;IAEf,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;AAC1D,CAAC"}

package/dist/esm/index.js

@@ -0,0 +1,19 @@
+/**
+ * Small LTSC ML - Machine Learning Features
+ *
+ * Optional ML features for enhanced compression quality:
+ * - Pattern importance scoring
+ * - Quality prediction
+ * - Adaptive region detection
+ *
+ * @packageDocumentation
+ */
+// Importance scoring
+export { PositionalImportanceScorer, EmbeddingImportanceScorer, CombinedImportanceScorer, adjustPrioritiesByImportance, filterByImportance, } from './importance.js';
+// Quality prediction
+export { HeuristicQualityPredictor, EmbeddingQualityPredictor, createQualityPredictor, } from './quality.js';
+// Region detection
+export { RegionType, detectRegions, detectRegionsHeuristic, filterPatternsByRegion, getRegionCompressionSettings, } from './regions.js';
+// Version
+export const VERSION = '0.1.0';
+//# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,qBAAqB;AACrB,OAAO,EAIL,0BAA0B,EAC1B,yBAAyB,EACzB,wBAAwB,EACxB,4BAA4B,EAC5B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AAEzB,qBAAqB;AACrB,OAAO,EAKL,yBAAyB,EACzB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,cAAc,CAAC;AAEtB,mBAAmB;AACnB,OAAO,EACL,UAAU,EAGV,aAAa,EACb,sBAAsB,EACtB,sBAAsB,EACtB,4BAA4B,GAC7B,MAAM,cAAc,CAAC;AAEtB,UAAU;AACV,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}

package/dist/esm/quality.js

@@ -0,0 +1,235 @@
+/**
+ * Quality prediction for compression validation.
+ *
+ * Predicts whether a compressed sequence will maintain
+ * sufficient quality for transformer models to learn from.
+ *
+ * Port of `small/quality_predictor.py` to TypeScript.
+ */
+const DEFAULT_QUALITY_CONFIG = {
+    maxCompressionRatio: 0.5,
+    maxDictionaryOverhead: 0.3,
+    minEmbeddingSimilarity: 0.7,
+    maxDiversityReduction: 0.4,
+};
+/**
+ * Heuristic-based quality predictor.
+ *
+ * Uses a combination of handcrafted features and thresholds
+ * to predict compression quality.
+ */
+export class HeuristicQualityPredictor {
+    config;
+    constructor(config) {
+        this.config = { ...DEFAULT_QUALITY_CONFIG, ...config };
+    }
+    async predict(result) {
+        const features = this.extractFeatures(result);
+        const score = this.computeScore(features);
+        const degradationProbability = this.computeDegradationProbability(features);
+        const acceptable = score >= 0.6 && degradationProbability < 0.3;
+        let recommendation;
+        if (acceptable) {
+            recommendation = 'accept';
+        }
+        else if (degradationProbability < 0.5) {
+            recommendation = 'retry_conservative';
+        }
+        else {
+            recommendation = 'skip_compression';
+        }
+        return {
+            score,
+            acceptable,
+            degradationProbability,
+            features,
+            recommendation,
+        };
+    }
+    /**
+     * Extract quality features from compression result.
+     */
+    extractFeatures(result) {
+        // Compression ratio
+        const compressionRatio = result.compressionRatio;
+        // Dictionary overhead
+        const dictionaryOverhead = result.originalLength > 0
+            ? result.dictionaryTokens.length / result.originalLength
+            : 0;
+        // Token diversity
+        const originalDiversity = new Set(result.originalTokens).size;
+        const compressedDiversity = new Set(result.serializedTokens).size;
+        const diversityReduction = originalDiversity > 0 ? 1 - compressedDiversity / originalDiversity : 0;
+        // Pattern statistics
+        const patternCount = result.dictionaryMap.size;
+        let totalPatternLength = 0;
+        for (const [, pattern] of result.dictionaryMap) {
+            totalPatternLength += pattern.length;
+        }
+        const averagePatternLength = patternCount > 0 ? totalPatternLength / patternCount : 0;
+        return {
+            compressionRatio,
+            dictionaryOverhead,
+            diversityReduction,
+            averagePatternLength,
+            patternCount,
+        };
+    }
+    /**
+     * Compute overall quality score from features.
+     */
+    computeScore(features) {
+        let score = 1.0;
+        // Penalize extreme compression
+        if (features.compressionRatio < this.config.maxCompressionRatio) {
+            const penalty = (this.config.maxCompressionRatio - features.compressionRatio) * 0.5;
+            score -= penalty;
+        }
+        // Penalize high dictionary overhead
+        if (features.dictionaryOverhead > this.config.maxDictionaryOverhead) {
+            const penalty = (features.dictionaryOverhead - this.config.maxDictionaryOverhead) * 0.3;
+            score -= penalty;
+        }
+        // Penalize diversity loss
+        if (features.diversityReduction > this.config.maxDiversityReduction) {
+            const penalty = (features.diversityReduction - this.config.maxDiversityReduction) * 0.4;
+            score -= penalty;
+        }
+        // Bonus for reasonable pattern lengths
+        if (features.averagePatternLength >= 3 && features.averagePatternLength <= 6) {
+            score += 0.1;
+        }
+        // Use embedding similarity if available
+        if (features.embeddingSimilarity !== undefined) {
+            if (features.embeddingSimilarity < this.config.minEmbeddingSimilarity) {
+                score -= (this.config.minEmbeddingSimilarity - features.embeddingSimilarity) * 0.5;
+            }
+        }
+        return Math.max(0, Math.min(1, score));
+    }
+    /**
+     * Compute probability of quality degradation.
+     */
+    computeDegradationProbability(features) {
+        let prob = 0;
+        // Very aggressive compression increases risk
+        if (features.compressionRatio < 0.4) {
+            prob += 0.3;
+        }
+        else if (features.compressionRatio < 0.5) {
+            prob += 0.15;
+        }
+        // High dictionary overhead increases risk
+        if (features.dictionaryOverhead > 0.3) {
+            prob += 0.2;
+        }
+        // Large diversity reduction increases risk
+        if (features.diversityReduction > 0.3) {
+            prob += 0.25;
+        }
+        else if (features.diversityReduction > 0.2) {
+            prob += 0.1;
+        }
+        // Very short patterns are risky
+        if (features.averagePatternLength < 2.5) {
+            prob += 0.15;
+        }
+        // Low embedding similarity is a strong signal
+        if (features.embeddingSimilarity !== undefined && features.embeddingSimilarity < 0.7) {
+            prob += 0.3 * (0.7 - features.embeddingSimilarity);
+        }
+        return Math.max(0, Math.min(1, prob));
+    }
+}
+/**
+ * Embedding-enhanced quality predictor.
+ *
+ * Adds embedding similarity comparison to the heuristic predictor
+ * for more accurate quality assessment.
+ */
+export class EmbeddingQualityPredictor {
+    provider;
+    heuristicPredictor;
+    config;
+    constructor(provider, config) {
+        this.provider = provider;
+        this.heuristicPredictor = new HeuristicQualityPredictor(config);
+        this.config = { ...DEFAULT_QUALITY_CONFIG, ...config };
+    }
+    async predict(result) {
+        // Get base prediction
+        const basePrediction = await this.heuristicPredictor.predict(result);
+        // Compute embedding similarity
+        const originalEmbedding = await this.provider.embed(result.originalTokens);
+        const compressedEmbedding = await this.provider.embed(result.serializedTokens);
+        const similarity = this.cosineSimilarity(originalEmbedding, compressedEmbedding);
+        // Update features with embedding similarity
+        const features = {
+            ...basePrediction.features,
+            embeddingSimilarity: similarity,
+        };
+        // Recompute scores with embedding
+        let score = basePrediction.score;
+        let degradationProbability = basePrediction.degradationProbability;
+        if (similarity < this.config.minEmbeddingSimilarity) {
+            const penalty = (this.config.minEmbeddingSimilarity - similarity) * 0.3;
+            score -= penalty;
+            degradationProbability += 0.2;
+        }
+        else {
+            // High similarity is a good signal
+            score += (similarity - this.config.minEmbeddingSimilarity) * 0.1;
+        }
+        score = Math.max(0, Math.min(1, score));
+        degradationProbability = Math.max(0, Math.min(1, degradationProbability));
+        const acceptable = score >= 0.6 && degradationProbability < 0.3;
+        let recommendation;
+        if (acceptable) {
+            recommendation = 'accept';
+        }
+        else if (degradationProbability < 0.5) {
+            recommendation = 'retry_conservative';
+        }
+        else {
+            recommendation = 'skip_compression';
+        }
+        return {
+            score,
+            acceptable,
+            degradationProbability,
+            features,
+            recommendation,
+        };
+    }
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            throw new Error('Vectors must have same length');
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        if (normA === 0 || normB === 0) {
+            return 0;
+        }
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+}
+/**
+ * Create a quality predictor.
+ *
+ * @param provider - Optional embedding provider for enhanced prediction
+ * @param config - Quality configuration
+ * @returns Quality predictor instance
+ */
+export function createQualityPredictor(provider, config) {
+    if (provider) {
+        return new EmbeddingQualityPredictor(provider, config);
+    }
+    return new HeuristicQualityPredictor(config);
+}
+//# sourceMappingURL=quality.js.map

package/dist/esm/quality.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quality.js","sourceRoot":"","sources":["../../src/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAkCH,MAAM,sBAAsB,GAA4B;IACtD,mBAAmB,EAAE,GAAG;IACxB,qBAAqB,EAAE,GAAG;IAC1B,sBAAsB,EAAE,GAAG;IAC3B,qBAAqB,EAAE,GAAG;CAC3B,CAAC;AAgFF;;;;;GAKG;AACH,MAAM,OAAO,yBAAyB;IAC5B,MAAM,CAA0B;IAExC,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,sBAAsB,EAAE,GAAG,MAAM,EAAE,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAyB;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,sBAAsB,GAAG,IAAI,CAAC,6BAA6B,CAAC,QAAQ,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,KAAK,IAAI,GAAG,IAAI,sBAAsB,GAAG,GAAG,CAAC;QAEhE,IAAI,cAAmD,CAAC;QACxD,IAAI,UAAU,EAAE,CAAC;YACf,cAAc,GAAG,QAAQ,CAAC;QAC5B,CAAC;aAAM,IAAI,sBAAsB,GAAG,GAAG,EAAE,CAAC;YACxC,cAAc,GAAG,oBAAoB,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,kBAAkB,CAAC;QACtC,CAAC;QAED,OAAO;YACL,KAAK;YACL,UAAU;YACV,sBAAsB;YACtB,QAAQ;YACR,cAAc;SACf,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,MAAyB;QAC/C,oBAAoB;QACpB,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAEjD,sBAAsB;QACtB,MAAM,kBAAkB,GACtB,MAAM,CAAC,cAAc,GAAG,CAAC;YACvB,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,MAAM,CAAC,cAAc;YACxD,CAAC,CAAC,CAAC,CAAC;QAER,kBAAkB;QAClB,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC;QAC9D,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC;QAClE,MAAM,kBAAkB,GACtB,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;QAE1E,qBAAqB;QACrB,MAAM,YAAY,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC;QAC/C,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,KAAK,MAAM,CAAC,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YAC/C,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC;QACvC,CAAC;QACD,MAAM,oBAAoB,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtF,OAAO;YACL,gBAAgB;YAChB,kBAAkB;YAClB,kBAAkB;YAClB,oBAAoB;YACpB,YAAY;SACb,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,QAAyB;QAC5C,IAAI,KAAK,GAAG,GAAG,CAAC;QAEhB,+BAA+B;QAC/B,IAAI,QAAQ,CAAC,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAChE,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAmB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;YACpF,KAAK,IAAI,OAAO,CAAC;QACnB,CAAC;QAED,oCAAoC;QACpC,IAAI,QAAQ,CAAC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC;YACpE,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,GAAG,GAAG,CAAC;YACxF,KAAK,IAAI,OAAO,CAAC;QACnB,CAAC;QAED,0BAA0B;QAC1B,IAAI,QAAQ,CAAC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC;YACpE,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,GAAG,GAAG,CAAC;YACxF,KAAK,IAAI,OAAO,CAAC;QACnB,CAAC;QAED,uCAAuC;QACvC,IAAI,QAAQ,CAAC,oBAAoB,IAAI,CAAC,IAAI,QAAQ,CAAC,oBAAoB,IAAI,CAAC,EAAE,CAAC;YAC7E,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,wCAAwC;QACxC,IAAI,QAAQ,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,QAAQ,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;gBACtE,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,GAAG,QAAQ,CAAC,mBAAmB,CAAC,GAAG,GAAG,CAAC;YACrF,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,6BAA6B,CAAC,QAAyB;QAC7D,IAAI,IAAI,GAAG,CAAC,CAAC;QAEb,6CAA6C;QAC7C,IAAI,QAAQ,CAAC,gBAAgB,GAAG,GAAG,EAAE,CAAC;YACpC,IAAI,IAAI,GAAG,CAAC;QACd,CAAC;aAAM,IAAI,QAAQ,CAAC,gBAAgB,GAAG,GAAG,EAAE,CAAC;YAC3C,IAAI,IAAI,IAAI,CAAC;QACf,CAAC;QAED,0CAA0C;QAC1C,IAAI,QAAQ,CAAC,kBAAkB,GAAG,GAAG,EAAE,CAAC;YACtC,IAAI,IAAI,GAAG,CAAC;QACd,CAAC;QAED,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,kBAAkB,GAAG,GAAG,EAAE,CAAC;YACtC,IAAI,IAAI,IAAI,CAAC;QACf,CAAC;aAAM,IAAI,QAAQ,CAAC,kBAAkB,GAAG,GAAG,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,CAAC;QACd,CAAC;QAED,gCAAgC;QAChC,IAAI,QAAQ,CAAC,oBAAoB,GAAG,GAAG,EAAE,CAAC;YACxC,IAAI,IAAI,IAAI,CAAC;QACf,CAAC;QAED,8CAA8C;QAC9C,IAAI,QAAQ,CAAC,mBAAmB,KAAK,SAAS,IAAI,QAAQ,CAAC,mBAAmB,GAAG,GAAG,EAAE,CAAC;YACrF,IAAI,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QACrD,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACxC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,yBAAyB;IAC5B,QAAQ,CAAoB;IAC5B,kBAAkB,CAA4B;IAC9C,MAAM,CAA0B;IAExC,YAAY,QAA2B,EAAE,MAAsB;QAC7D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,kBAAkB,GAAG,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,sBAAsB,EAAE,GAAG,MAAM,EAAE,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAyB;QACrC,sBAAsB;QACtB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAErE,+BAA+B;QAC/B,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC;QAEjF,4CAA4C;QAC5C,MAAM,QAAQ,GAAoB;YAChC,GAAG,cAAc,CAAC,QAAQ;YAC1B,mBAAmB,EAAE,UAAU;SAChC,CAAC;QAEF,kCAAkC;QAClC,IAAI,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;QACjC,IAAI,sBAAsB,GAAG,cAAc,CAAC,sBAAsB,CAAC;QAEnE,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC;YACxE,KAAK,IAAI,OAAO,CAAC;YACjB,sBAAsB,IAAI,GAAG,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,mCAAmC;YACnC,KAAK,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,GAAG,GAAG,CAAC;QACnE,CAAC;QAED,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACxC,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC,CAAC;QAE1E,MAAM,UAAU,GAAG,KAAK,IAAI,GAAG,IAAI,sBAAsB,GAAG,GAAG,CAAC;QAEhE,IAAI,cAAmD,CAAC;QACxD,IAAI,UAAU,EAAE,CAAC;YACf,cAAc,GAAG,QAAQ,CAAC;QAC5B,CAAC;aAAM,IAAI,sBAAsB,GAAG,GAAG,EAAE,CAAC;YACxC,cAAc,GAAG,oBAAoB,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,kBAAkB,CAAC;QACtC,CAAC;QAED,OAAO;YACL,KAAK;YACL,UAAU;YACV,sBAAsB;YACtB,QAAQ;YACR,cAAc;SACf,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,CAAe,EAAE,CAAe;QACvD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QAED,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5D,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAA4B,EAC5B,MAAsB;IAEtB,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,IAAI,yBAAyB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC;AAC/C,CAAC"}

package/dist/esm/regions.js

@@ -0,0 +1,251 @@
+/**
+ * Adaptive region detection for context-aware compression.
+ *
+ * Detects semantic regions in token sequences (system prompts, user input,
+ * injected context) and applies different compression strategies.
+ *
+ * Port of `small/adaptive.py` region detection to TypeScript.
+ */
+/**
+ * Region types with different compression strategies.
+ */
+export var RegionType;
+(function (RegionType) {
+    /** System instructions - minimal compression */
+    RegionType["SYSTEM"] = "system";
+    /** User input - moderate compression */
+    RegionType["USER"] = "user";
+    /** Injected context - aggressive compression */
+    RegionType["CONTEXT"] = "context";
+    /** Code blocks - moderate compression */
+    RegionType["CODE"] = "code";
+    /** Unknown region - default compression */
+    RegionType["UNKNOWN"] = "unknown";
+})(RegionType || (RegionType = {}));
+const DEFAULT_RETENTION_TARGETS = {
+    [RegionType.SYSTEM]: 0.98,
+    [RegionType.USER]: 0.85,
+    [RegionType.CONTEXT]: 0.6,
+    [RegionType.CODE]: 0.8,
+    [RegionType.UNKNOWN]: 0.75,
+};
+/**
+ * Detect regions in a token sequence.
+ *
+ * Uses marker patterns to identify boundaries between different
+ * semantic regions in the input.
+ *
+ * @param tokens - Token sequence to analyze
+ * @param config - Region detection configuration
+ * @returns Array of detected regions
+ */
+export function detectRegions(tokens, config) {
+    const tokenArray = Array.isArray(tokens) ? tokens : Array.from(tokens);
+    const n = tokenArray.length;
+    if (n === 0) {
+        return [];
+    }
+    const retentionTargets = {
+        ...DEFAULT_RETENTION_TARGETS,
+        ...config?.retentionTargets,
+    };
+    // Find all marker positions
+    const markers = [];
+    const systemMarkers = config?.systemMarkers ?? DEFAULT_SYSTEM_MARKERS;
+    const userMarkers = config?.userMarkers ?? DEFAULT_USER_MARKERS;
+    const contextMarkers = config?.contextMarkers ?? DEFAULT_CONTEXT_MARKERS;
+    const codeMarkers = config?.codeMarkers ?? DEFAULT_CODE_MARKERS;
+    // Search for markers
+    for (const pattern of systemMarkers) {
+        for (const pos of findPattern(tokenArray, pattern)) {
+            markers.push({ pos, type: RegionType.SYSTEM });
+        }
+    }
+    for (const pattern of userMarkers) {
+        for (const pos of findPattern(tokenArray, pattern)) {
+            markers.push({ pos, type: RegionType.USER });
+        }
+    }
+    for (const pattern of contextMarkers) {
+        for (const pos of findPattern(tokenArray, pattern)) {
+            markers.push({ pos, type: RegionType.CONTEXT });
+        }
+    }
+    for (const pattern of codeMarkers) {
+        for (const pos of findPattern(tokenArray, pattern)) {
+            markers.push({ pos, type: RegionType.CODE });
+        }
+    }
+    // Sort markers by position
+    markers.sort((a, b) => a.pos - b.pos);
+    // Build regions from markers
+    const regions = [];
+    if (markers.length === 0) {
+        // No markers found - treat entire sequence as unknown
+        regions.push({
+            type: RegionType.UNKNOWN,
+            start: 0,
+            end: n,
+            retention: retentionTargets[RegionType.UNKNOWN],
+        });
+    }
+    else {
+        // Add initial region if first marker is not at start
+        if (markers[0].pos > 0) {
+            regions.push({
+                type: RegionType.UNKNOWN,
+                start: 0,
+                end: markers[0].pos,
+                retention: retentionTargets[RegionType.UNKNOWN],
+            });
+        }
+        // Add regions from markers
+        for (let i = 0; i < markers.length; i++) {
+            const marker = markers[i];
+            const nextPos = i < markers.length - 1 ? markers[i + 1].pos : n;
+            regions.push({
+                type: marker.type,
+                start: marker.pos,
+                end: nextPos,
+                retention: retentionTargets[marker.type],
+            });
+        }
+    }
+    return regions;
+}
+/**
+ * Heuristic-based region detection without explicit markers.
+ *
+ * Uses statistical features to guess region boundaries.
+ */
+export function detectRegionsHeuristic(tokens) {
+    const tokenArray = Array.isArray(tokens) ? tokens : Array.from(tokens);
+    const n = tokenArray.length;
+    if (n === 0) {
+        return [];
+    }
+    // Simple heuristic: first 10% is likely system, rest is context
+    const systemEnd = Math.floor(n * 0.1);
+    return [
+        {
+            type: RegionType.SYSTEM,
+            start: 0,
+            end: Math.max(systemEnd, 1),
+            retention: DEFAULT_RETENTION_TARGETS[RegionType.SYSTEM],
+        },
+        {
+            type: RegionType.CONTEXT,
+            start: Math.max(systemEnd, 1),
+            end: n,
+            retention: DEFAULT_RETENTION_TARGETS[RegionType.CONTEXT],
+        },
+    ];
+}
+/**
+ * Filter patterns based on region retention targets.
+ *
+ * Removes patterns that would compress high-retention regions.
+ *
+ * @param patterns - Discovered patterns
+ * @param regions - Detected regions
+ * @param tokens - Original token sequence
+ * @returns Filtered patterns respecting region constraints
+ */
+export function filterPatternsByRegion(patterns, regions, _tokens) {
+    if (regions.length === 0) {
+        return patterns;
+    }
+    return patterns.map((pattern) => {
+        // Filter positions based on region retention
+        const filteredPositions = pattern.positions.filter((pos) => {
+            const region = findRegionAtPosition(regions, pos);
+            if (!region)
+                return true;
+            // Keep position if region allows compression (low retention)
+            // High retention regions should preserve patterns
+            return region.retention < 0.9;
+        });
+        return {
+            ...pattern,
+            positions: filteredPositions,
+            count: filteredPositions.length,
+        };
+    }).filter((pattern) => pattern.positions.length >= 2);
+}
+/**
+ * Find the region containing a position.
+ */
+function findRegionAtPosition(regions, pos) {
+    for (const region of regions) {
+        if (pos >= region.start && pos < region.end) {
+            return region;
+        }
+    }
+    return null;
+}
+/**
+ * Find all occurrences of a pattern in tokens.
+ */
+function findPattern(tokens, pattern) {
+    const positions = [];
+    const n = tokens.length;
+    const m = pattern.length;
+    for (let i = 0; i <= n - m; i++) {
+        let match = true;
+        for (let j = 0; j < m; j++) {
+            if (tokens[i + j] !== pattern[j]) {
+                match = false;
+                break;
+            }
+        }
+        if (match) {
+            positions.push(i);
+        }
+    }
+    return positions;
+}
+// Default marker patterns (tiktoken cl100k_base token IDs for common markers)
+// These are approximate - actual token IDs depend on the tokenizer
+/** Default system region markers */
+const DEFAULT_SYSTEM_MARKERS = [
+    // [SYSTEM], <<SYS>>, etc.
+    [58, 71905, 60], // [SYSTEM]
+    [27, 27, 71905, 2083, 2083], // <<SYS>>
+];
+/** Default user region markers */
+const DEFAULT_USER_MARKERS = [
+    // [USER], [INST], etc.
+    [58, 35295, 60], // [USER]
+    [58, 96746, 60], // [INST]
+];
+/** Default context region markers */
+const DEFAULT_CONTEXT_MARKERS = [
+    // [CONTEXT], [DOC], etc.
+    [58, 94034, 60], // [CONTEXT]
+    [58, 44184, 60], // [DOC]
+];
+/** Default code region markers */
+const DEFAULT_CODE_MARKERS = [
+    // ```python, ```typescript, etc.
+    [74694, 12958], // ```python
+    [74694, 92459], // ```typescript
+    [74694, 13210], // ```javascript
+];
+/**
+ * Get compression settings for a region type.
+ */
+export function getRegionCompressionSettings(regionType) {
+    switch (regionType) {
+        case RegionType.SYSTEM:
+            return { maxSubsequenceLength: 4, minOccurrences: 5 };
+        case RegionType.USER:
+            return { maxSubsequenceLength: 6, minOccurrences: 3 };
+        case RegionType.CONTEXT:
+            return { maxSubsequenceLength: 10, minOccurrences: 2 };
+        case RegionType.CODE:
+            return { maxSubsequenceLength: 6, minOccurrences: 3 };
+        default:
+            return { maxSubsequenceLength: 8, minOccurrences: 3 };
+    }
+}
+//# sourceMappingURL=regions.js.map

package/dist/esm/regions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"regions.js","sourceRoot":"","sources":["../../src/regions.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH;;GAEG;AACH,MAAM,CAAN,IAAY,UAWX;AAXD,WAAY,UAAU;IACpB,gDAAgD;IAChD,+BAAiB,CAAA;IACjB,wCAAwC;IACxC,2BAAa,CAAA;IACb,gDAAgD;IAChD,iCAAmB,CAAA;IACnB,yCAAyC;IACzC,2BAAa,CAAA;IACb,2CAA2C;IAC3C,iCAAmB,CAAA;AACrB,CAAC,EAXW,UAAU,KAAV,UAAU,QAWrB;AAgCD,MAAM,yBAAyB,GAA+B;IAC5D,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,IAAI;IACzB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI;IACvB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG;IACzB,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG;IACtB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI;CAC3B,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,MAAqB;IACnE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvE,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IAE5B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,gBAAgB,GAAG;QACvB,GAAG,yBAAyB;QAC5B,GAAG,MAAM,EAAE,gBAAgB;KAC5B,CAAC;IAEF,4BAA4B;IAC5B,MAAM,OAAO,GAAwC,EAAE,CAAC;IAExD,MAAM,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,sBAAsB,CAAC;IACtE,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,oBAAoB,CAAC;IAChE,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,uBAAuB,CAAC;IACzE,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,oBAAoB,CAAC;IAEhE,qBAAqB;IACrB,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACpC,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAEtC,6BAA6B;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,sDAAsD;QACtD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,UAAU,CAAC,OAAO;YACxB,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,CAAC;YACN,SAAS,EAAE,gBAAgB,CAAC,UAAU,CAAC,OAAO,CAAC;SAChD,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,qDAAqD;QACrD,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,UAAU,CAAC,OAAO;gBACxB,KAAK,EAAE,CAAC;gBACR,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG;gBACnB,SAAS,EAAE,gBAAgB,CAAC,UAAU,CAAC,OAAO,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;QAED,2BAA2B;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAEhE,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,MAAM,CAAC,GAAG;gBACjB,GAAG,EAAE,OAAO;gBACZ,SAAS,EAAE,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC;aACzC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAgB;IACrD,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvE,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IAE5B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,gEAAgE;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IAEtC,OAAO;QACL;YACE,IAAI,EAAE,UAAU,CAAC,MAAM;YACvB,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YAC3B,SAAS,EAAE,yBAAyB,CAAC,UAAU,CAAC,MAAM,CAAC;SACxD;QACD;YACE,IAAI,EAAE,UAAU,CAAC,OAAO;YACxB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YAC7B,GAAG,EAAE,CAAC;YACN,SAAS,EAAE,yBAAyB,CAAC,UAAU,CAAC,OAAO,CAAC;SACzD;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAA6B,EAC7B,OAAiB,EACjB,OAAiB;IAEjB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,6CAA6C;QAC7C,MAAM,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YACzD,MAAM,MAAM,GAAG,oBAAoB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC;YAEzB,6DAA6D;YAC7D,kDAAkD;YAClD,OAAO,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,OAAO;YACV,SAAS,EAAE,iBAAiB;YAC5B,KAAK,EAAE,iBAAiB,CAAC,MAAM;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,OAAiB,EAAE,GAAW;IAC1D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,GAAG,IAAI,MAAM,CAAC,KAAK,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YAC5C,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,MAAgB,EAAE,OAAiB;IACtD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,IAAI,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjC,KAAK,GAAG,KAAK,CAAC;gBACd,MAAM;YACR,CAAC;QACH,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACV,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,mEAAmE;AAEnE,oCAAoC;AACpC,MAAM,sBAAsB,GAAe;IACzC,0BAA0B;IAC1B,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAM,WAAW;IAChC,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,UAAU;CACxC,CAAC;AAEF,kCAAkC;AAClC,MAAM,oBAAoB,GAAe;IACvC,uBAAuB;IACvB,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAM,SAAS;IAC9B,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAM,SAAS;CAC/B,CAAC;AAEF,qCAAqC;AACrC,MAAM,uBAAuB,GAAe;IAC1C,yBAAyB;IACzB,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAM,YAAY;IACjC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAM,QAAQ;CAC9B,CAAC;AAEF,kCAAkC;AAClC,MAAM,oBAAoB,GAAe;IACvC,iCAAiC;IACjC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAO,YAAY;IACjC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAO,gBAAgB;IACrC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAO,gBAAgB;CACtC,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,4BAA4B,CAAC,UAAsB;IAIjE,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,UAAU,CAAC,MAAM;YACpB,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACxD,KAAK,UAAU,CAAC,IAAI;YAClB,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACxD,KAAK,UAAU,CAAC,OAAO;YACrB,OAAO,EAAE,oBAAoB,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACzD,KAAK,UAAU,CAAC,IAAI;YAClB,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACxD;YACE,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IAC1D,CAAC;AACH,CAAC"}

package/dist/types/importance.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Pattern importance scoring for ML-aware compression.
+ *
+ * Port of `small/pattern_importance.py` to TypeScript.
+ */
+import type { DiscoveredPattern, TokenSeq } from '@small-ltsc/sdk';
+/**
+ * Interface for embedding providers.
+ */
+export interface EmbeddingProvider {
+    /**
+     * Get embeddings for a sequence of tokens.
+     *
+     * @param tokens - Token sequence to embed
+     * @returns Promise resolving to embedding vector (Float32Array)
+     */
+    embed(tokens: TokenSeq): Promise<Float32Array>;
+    /**
+     * Get embedding dimension.
+     */
+    dimension(): number;
+}
+/**
+ * Configuration for importance scoring.
+ */
+export interface ImportanceConfig {
+    /**
+     * Position decay rate for positional scoring.
+     * Higher values = more weight on early positions.
+     * @default 2.0
+     */
+    decayRate?: number;
+    /**
+     * Context window size for embedding-based scoring.
+     * @default 5
+     */
+    contextWindow?: number;
+    /**
+     * Weight for positional importance (vs. embedding-based).
+     * @default 0.3
+     */
+    positionalWeight?: number;
+}
+/**
+ * Interface for pattern importance scorers.
+ */
+export interface ImportanceScorer {
+    /**
+     * Score patterns by importance.
+     *
+     * Higher scores indicate more important patterns that should be
+     * preserved (less aggressively compressed).
+     *
+     * @param tokens - Original token sequence
+     * @param patterns - Discovered patterns to score
+     * @returns Promise resolving to importance scores (0-1 range)
+     */
+    scorePatterns(tokens: TokenSeq, patterns: DiscoveredPattern[]): Promise<number[]>;
+}
+/**
+ * Positional importance scorer.
+ *
+ * Scores patterns based on their position in the sequence,
+ * with earlier positions receiving higher importance (useful for
+ * system prompts that typically appear at the start).
+ */
+export declare class PositionalImportanceScorer implements ImportanceScorer {
+    private decayRate;
+    constructor(config?: ImportanceConfig);
+    scorePatterns(tokens: TokenSeq, patterns: DiscoveredPattern[]): Promise<number[]>;
+}
+/**
+ * Embedding-based importance scorer.
+ *
+ * Uses an embedding model to determine if a pattern appears in
+ * diverse semantic contexts (important, should preserve) vs.
+ * similar contexts (redundant, safe to compress).
+ */
+export declare class EmbeddingImportanceScorer implements ImportanceScorer {
+    private provider;
+    private contextWindow;
+    constructor(provider: EmbeddingProvider, config?: ImportanceConfig);
+    scorePatterns(tokens: TokenSeq, patterns: DiscoveredPattern[]): Promise<number[]>;
+}
+/**
+ * Combined importance scorer that uses both positional and embedding-based scoring.
+ */
+export declare class CombinedImportanceScorer implements ImportanceScorer {
+    private positionalScorer;
+    private embeddingScorer;
+    private positionalWeight;
+    constructor(provider?: EmbeddingProvider, config?: ImportanceConfig);
+    scorePatterns(tokens: TokenSeq, patterns: DiscoveredPattern[]): Promise<number[]>;
+}
+/**
+ * Adjust pattern priorities based on importance scores.
+ *
+ * Patterns with high importance get lower priority (compressed later),
+ * patterns with low importance get higher priority (compressed first).
+ *
+ * @param patterns - Patterns to adjust
+ * @param scores - Importance scores from an ImportanceScorer
+ * @param threshold - Patterns above this importance threshold get negative priority
+ * @returns Patterns with adjusted priorities
+ */
+export declare function adjustPrioritiesByImportance(patterns: DiscoveredPattern[], scores: number[], threshold?: number): DiscoveredPattern[];
+/**
+ * Filter patterns to only those below an importance threshold.
+ *
+ * @param patterns - Patterns to filter
+ * @param scores - Importance scores
+ * @param threshold - Maximum importance score to include
+ * @returns Filtered patterns that are safe to compress
+ */
+export declare function filterByImportance(patterns: DiscoveredPattern[], scores: number[], threshold?: number): DiscoveredPattern[];
+//# sourceMappingURL=importance.d.ts.map

package/dist/types/importance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"importance.d.ts","sourceRoot":"","sources":["../../src/importance.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;;OAKG;IACH,KAAK,CAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAE/C;;OAEG;IACH,SAAS,IAAI,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAQD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;;;OASG;IACH,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACnF;AAED;;;;;;GAMG;AACH,qBAAa,0BAA2B,YAAW,gBAAgB;IACjE,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,CAAC,EAAE,gBAAgB;IAK/B,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAuBxF;AAED;;;;;;GAMG;AACH,qBAAa,yBAA0B,YAAW,gBAAgB;IAChE,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,aAAa,CAAS;gBAElB,QAAQ,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,gBAAgB;IAM5D,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAkDxF;AAED;;GAEG;AACH,qBAAa,wBAAyB,YAAW,gBAAgB;IAC/D,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,eAAe,CAAmC;IAC1D,OAAO,CAAC,gBAAgB,CAAS;gBAErB,QAAQ,CAAC,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,gBAAgB;IAO7D,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAexF;AA2BD;;;;;;;;;;GAUG;AACH,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,iBAAiB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,SAAM,GACd,iBAAiB,EAAE,CAuBrB;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,iBAAiB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,SAAM,GACd,iBAAiB,EAAE,CAErB"}

package/dist/types/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Small LTSC ML - Machine Learning Features
+ *
+ * Optional ML features for enhanced compression quality:
+ * - Pattern importance scoring
+ * - Quality prediction
+ * - Adaptive region detection
+ *
+ * @packageDocumentation
+ */
+export { type EmbeddingProvider, type ImportanceConfig, type ImportanceScorer, PositionalImportanceScorer, EmbeddingImportanceScorer, CombinedImportanceScorer, adjustPrioritiesByImportance, filterByImportance, } from './importance.js';
+export { type QualityConfig, type QualityPrediction, type QualityFeatures, type QualityPredictor, HeuristicQualityPredictor, EmbeddingQualityPredictor, createQualityPredictor, } from './quality.js';
+export { RegionType, type Region, type RegionConfig, detectRegions, detectRegionsHeuristic, filterPatternsByRegion, getRegionCompressionSettings, } from './regions.js';
+export declare const VERSION = "0.1.0";
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,0BAA0B,EAC1B,yBAAyB,EACzB,wBAAwB,EACxB,4BAA4B,EAC5B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,cAAc,CAAC;AAGtB,OAAO,EACL,UAAU,EACV,KAAK,MAAM,EACX,KAAK,YAAY,EACjB,aAAa,EACb,sBAAsB,EACtB,sBAAsB,EACtB,4BAA4B,GAC7B,MAAM,cAAc,CAAC;AAGtB,eAAO,MAAM,OAAO,UAAU,CAAC"}

package/dist/types/quality.d.ts

@@ -0,0 +1,147 @@
+/**
+ * Quality prediction for compression validation.
+ *
+ * Predicts whether a compressed sequence will maintain
+ * sufficient quality for transformer models to learn from.
+ *
+ * Port of `small/quality_predictor.py` to TypeScript.
+ */
+import type { CompressionResult } from '@small-ltsc/sdk';
+import type { EmbeddingProvider } from './importance.js';
+/**
+ * Configuration for quality prediction.
+ */
+export interface QualityConfig {
+    /**
+     * Maximum acceptable compression ratio.
+     * @default 0.5
+     */
+    maxCompressionRatio?: number;
+    /**
+     * Maximum acceptable dictionary overhead ratio.
+     * @default 0.3
+     */
+    maxDictionaryOverhead?: number;
+    /**
+     * Minimum embedding similarity between original and compressed.
+     * @default 0.7
+     */
+    minEmbeddingSimilarity?: number;
+    /**
+     * Maximum acceptable token diversity reduction.
+     * @default 0.4
+     */
+    maxDiversityReduction?: number;
+}
+/**
+ * Result of quality prediction.
+ */
+export interface QualityPrediction {
+    /**
+     * Overall quality score (0-1, higher is better).
+     */
+    score: number;
+    /**
+     * Whether the compression passes quality threshold.
+     */
+    acceptable: boolean;
+    /**
+     * Probability of quality degradation.
+     */
+    degradationProbability: number;
+    /**
+     * Detailed feature scores.
+     */
+    features: QualityFeatures;
+    /**
+     * Recommendation for how to proceed.
+     */
+    recommendation: 'accept' | 'retry_conservative' | 'skip_compression';
+}
+/**
+ * Feature scores used in quality prediction.
+ */
+export interface QualityFeatures {
+    /**
+     * Compression ratio feature (lower is more aggressive).
+     */
+    compressionRatio: number;
+    /**
+     * Dictionary overhead ratio.
+     */
+    dictionaryOverhead: number;
+    /**
+     * Token diversity change (0-1, 0 = no change, 1 = complete loss).
+     */
+    diversityReduction: number;
+    /**
+     * Average pattern length feature.
+     */
+    averagePatternLength: number;
+    /**
+     * Pattern count feature.
+     */
+    patternCount: number;
+    /**
+     * Embedding similarity (if available).
+     */
+    embeddingSimilarity?: number;
+}
+/**
+ * Quality predictor interface.
+ */
+export interface QualityPredictor {
+    /**
+     * Predict quality of compressed output.
+     *
+     * @param result - Compression result to evaluate
+     * @returns Promise resolving to quality prediction
+     */
+    predict(result: CompressionResult): Promise<QualityPrediction>;
+}
+/**
+ * Heuristic-based quality predictor.
+ *
+ * Uses a combination of handcrafted features and thresholds
+ * to predict compression quality.
+ */
+export declare class HeuristicQualityPredictor implements QualityPredictor {
+    private config;
+    constructor(config?: QualityConfig);
+    predict(result: CompressionResult): Promise<QualityPrediction>;
+    /**
+     * Extract quality features from compression result.
+     */
+    private extractFeatures;
+    /**
+     * Compute overall quality score from features.
+     */
+    private computeScore;
+    /**
+     * Compute probability of quality degradation.
+     */
+    private computeDegradationProbability;
+}
+/**
+ * Embedding-enhanced quality predictor.
+ *
+ * Adds embedding similarity comparison to the heuristic predictor
+ * for more accurate quality assessment.
+ */
+export declare class EmbeddingQualityPredictor implements QualityPredictor {
+    private provider;
+    private heuristicPredictor;
+    private config;
+    constructor(provider: EmbeddingProvider, config?: QualityConfig);
+    predict(result: CompressionResult): Promise<QualityPrediction>;
+    private cosineSimilarity;
+}
+/**
+ * Create a quality predictor.
+ *
+ * @param provider - Optional embedding provider for enhanced prediction
+ * @param config - Quality configuration
+ * @returns Quality predictor instance
+ */
+export declare function createQualityPredictor(provider?: EmbeddingProvider, config?: QualityConfig): QualityPredictor;
+//# sourceMappingURL=quality.d.ts.map

package/dist/types/quality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AASD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,UAAU,EAAE,OAAO,CAAC;IAEpB;;OAEG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,QAAQ,EAAE,eAAe,CAAC;IAE1B;;OAEG;IACH,cAAc,EAAE,QAAQ,GAAG,oBAAoB,GAAG,kBAAkB,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,oBAAoB,EAAE,MAAM,CAAC;IAE7B;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,OAAO,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAChE;AAED;;;;;GAKG;AACH,qBAAa,yBAA0B,YAAW,gBAAgB;IAChE,OAAO,CAAC,MAAM,CAA0B;gBAE5B,MAAM,CAAC,EAAE,aAAa;IAI5B,OAAO,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAwBpE;;OAEG;IACH,OAAO,CAAC,eAAe;IAiCvB;;OAEG;IACH,OAAO,CAAC,YAAY;IAoCpB;;OAEG;IACH,OAAO,CAAC,6BAA6B;CAkCtC;AAED;;;;;GAKG;AACH,qBAAa,yBAA0B,YAAW,gBAAgB;IAChE,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,kBAAkB,CAA4B;IACtD,OAAO,CAAC,MAAM,CAA0B;gBAE5B,QAAQ,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,aAAa;IAMzD,OAAO,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmDpE,OAAO,CAAC,gBAAgB;CAqBzB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,CAAC,EAAE,iBAAiB,EAC5B,MAAM,CAAC,EAAE,aAAa,GACrB,gBAAgB,CAKlB"}

package/dist/types/regions.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Adaptive region detection for context-aware compression.
+ *
+ * Detects semantic regions in token sequences (system prompts, user input,
+ * injected context) and applies different compression strategies.
+ *
+ * Port of `small/adaptive.py` region detection to TypeScript.
+ */
+import type { TokenSeq, DiscoveredPattern } from '@small-ltsc/sdk';
+/**
+ * Region types with different compression strategies.
+ */
+export declare enum RegionType {
+    /** System instructions - minimal compression */
+    SYSTEM = "system",
+    /** User input - moderate compression */
+    USER = "user",
+    /** Injected context - aggressive compression */
+    CONTEXT = "context",
+    /** Code blocks - moderate compression */
+    CODE = "code",
+    /** Unknown region - default compression */
+    UNKNOWN = "unknown"
+}
+/**
+ * A detected region in the token sequence.
+ */
+export interface Region {
+    /** Region type */
+    type: RegionType;
+    /** Start position (inclusive) */
+    start: number;
+    /** End position (exclusive) */
+    end: number;
+    /** Compression retention target (0-1) */
+    retention: number;
+}
+/**
+ * Configuration for region detection.
+ */
+export interface RegionConfig {
+    /** Token patterns that mark system region start */
+    systemMarkers?: number[][];
+    /** Token patterns that mark user region start */
+    userMarkers?: number[][];
+    /** Token patterns that mark context region start */
+    contextMarkers?: number[][];
+    /** Token patterns that mark code region start */
+    codeMarkers?: number[][];
+    /** Retention targets for each region type */
+    retentionTargets?: Partial<Record<RegionType, number>>;
+}
+/**
+ * Detect regions in a token sequence.
+ *
+ * Uses marker patterns to identify boundaries between different
+ * semantic regions in the input.
+ *
+ * @param tokens - Token sequence to analyze
+ * @param config - Region detection configuration
+ * @returns Array of detected regions
+ */
+export declare function detectRegions(tokens: TokenSeq, config?: RegionConfig): Region[];
+/**
+ * Heuristic-based region detection without explicit markers.
+ *
+ * Uses statistical features to guess region boundaries.
+ */
+export declare function detectRegionsHeuristic(tokens: TokenSeq): Region[];
+/**
+ * Filter patterns based on region retention targets.
+ *
+ * Removes patterns that would compress high-retention regions.
+ *
+ * @param patterns - Discovered patterns
+ * @param regions - Detected regions
+ * @param tokens - Original token sequence
+ * @returns Filtered patterns respecting region constraints
+ */
+export declare function filterPatternsByRegion(patterns: DiscoveredPattern[], regions: Region[], _tokens: TokenSeq): DiscoveredPattern[];
+/**
+ * Get compression settings for a region type.
+ */
+export declare function getRegionCompressionSettings(regionType: RegionType): {
+    maxSubsequenceLength: number;
+    minOccurrences: number;
+};
+//# sourceMappingURL=regions.d.ts.map

package/dist/types/regions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regions.d.ts","sourceRoot":"","sources":["../../src/regions.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,oBAAY,UAAU;IACpB,gDAAgD;IAChD,MAAM,WAAW;IACjB,wCAAwC;IACxC,IAAI,SAAS;IACb,gDAAgD;IAChD,OAAO,YAAY;IACnB,yCAAyC;IACzC,IAAI,SAAS;IACb,2CAA2C;IAC3C,OAAO,YAAY;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB,kBAAkB;IAClB,IAAI,EAAE,UAAU,CAAC;IACjB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,mDAAmD;IACnD,aAAa,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAC3B,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IACzB,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAC5B,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IACzB,6CAA6C;IAC7C,gBAAgB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;CACxD;AAUD;;;;;;;;;GASG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,MAAM,EAAE,CAsF/E;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,CAyBjE;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,iBAAiB,EAAE,EAC7B,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,QAAQ,GAChB,iBAAiB,EAAE,CAsBrB;AAsED;;GAEG;AACH,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG;IACpE,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,MAAM,CAAC;CACxB,CAaA"}

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@small-ltsc/ml",
+  "version": "0.1.0",
+  "description": "ML features for Small LTSC - Pattern importance scoring and quality prediction",
+  "type": "module",
+  "main": "./dist/esm/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/esm/index.js",
+      "types": "./dist/types/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "sideEffects": false,
+  "scripts": {
+    "build": "npm run build:esm && npm run build:types",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "build:types": "tsc -p tsconfig.types.json",
+    "test": "vitest",
+    "lint": "eslint src --ext .ts",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "compression",
+    "llm",
+    "tokens",
+    "transformer",
+    "ml",
+    "embeddings"
+  ],
+  "author": "",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/triage-sec/small"
+  },
+  "peerDependencies": {
+    "@small-ltsc/sdk": "^0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.0",
+    "eslint": "^8.55.0",
+    "typescript": "^5.3.0",
+    "vitest": "^1.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}