dsh-context-compression-improved 0.1.1 → 0.2.0

package/packages/selector/tests/runtime/fixtures/profile-baseline.json

@@ -0,0 +1,273 @@
+{
+  "capturedAt": "pre-tokenpilot-inspired",
+  "profiles": {
+    "off": {
+      "defaults": {
+        "profile": "off",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": false,
+        "aggregateEnabled": false,
+        "historyMode": "disabled",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 9007199254740991,
+        "freshTargetTokens": 9007199254740991,
+        "aggregateTriggerTokens": 9007199254740991,
+        "aggregateTargetTokens": 9007199254740991,
+        "historyTriggerTokens": 9007199254740991,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 9007199254740991
+      },
+      "linkage": {
+        "profile": "off",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": false,
+        "aggregateEnabled": false,
+        "historyMode": "disabled",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 9007199254740991,
+        "freshTargetTokens": 9007199254740991,
+        "aggregateTriggerTokens": 9007199254740991,
+        "aggregateTargetTokens": 9007199254740991,
+        "historyTriggerTokens": 9007199254740991,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 9007199254740991
+      }
+    },
+    "native": {
+      "defaults": {
+        "profile": "native",
+        "nativeToolResultEnabled": true,
+        "freshEnabled": false,
+        "aggregateEnabled": false,
+        "historyMode": "disabled",
+        "nativeTriggerTokens": 4096,
+        "nativeTargetTokens": 2048,
+        "freshTriggerTokens": 9007199254740991,
+        "freshTargetTokens": 9007199254740991,
+        "aggregateTriggerTokens": 9007199254740991,
+        "aggregateTargetTokens": 9007199254740991,
+        "historyTriggerTokens": 9007199254740991,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 9007199254740991
+      },
+      "linkage": {
+        "profile": "native",
+        "nativeToolResultEnabled": true,
+        "freshEnabled": false,
+        "aggregateEnabled": false,
+        "historyMode": "disabled",
+        "nativeTriggerTokens": 4096,
+        "nativeTargetTokens": 2048,
+        "freshTriggerTokens": 9007199254740991,
+        "freshTargetTokens": 9007199254740991,
+        "aggregateTriggerTokens": 9007199254740991,
+        "aggregateTargetTokens": 9007199254740991,
+        "historyTriggerTokens": 9007199254740991,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 9007199254740991
+      }
+    },
+    "balanced": {
+      "defaults": {
+        "profile": "balanced",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "routine",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 500000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 96000
+      },
+      "linkage": {
+        "profile": "balanced",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "routine",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 64000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 8192,
+        "historyMinReclaimTokens": 12288,
+        "autoCompactTokens": 102400,
+        "microDeadlineTokens": 89600
+      }
+    },
+    "cache-strict": {
+      "defaults": {
+        "profile": "cache-strict",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "capacity-pressure",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 600000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 128000
+      },
+      "linkage": {
+        "profile": "cache-strict",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "capacity-pressure",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 76800,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 8192,
+        "historyMinReclaimTokens": 16384,
+        "autoCompactTokens": 102400,
+        "microDeadlineTokens": 89600
+      }
+    },
+    "savings": {
+      "defaults": {
+        "profile": "savings",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "routine",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 4096,
+        "freshTargetTokens": 1536,
+        "aggregateTriggerTokens": 16384,
+        "aggregateTargetTokens": 4096,
+        "historyTriggerTokens": 400000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 128000
+      },
+      "linkage": {
+        "profile": "savings",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "routine",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 4096,
+        "freshTargetTokens": 1536,
+        "aggregateTriggerTokens": 16384,
+        "aggregateTargetTokens": 4096,
+        "historyTriggerTokens": 51200,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 8192,
+        "historyMinReclaimTokens": 16384,
+        "autoCompactTokens": 102400,
+        "microDeadlineTokens": 89600
+      }
+    },
+    "adaptive": {
+      "defaults": {
+        "profile": "adaptive",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "adaptive",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 500000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 96000
+      },
+      "linkage": {
+        "profile": "adaptive",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "adaptive",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 64000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 8192,
+        "historyMinReclaimTokens": 12288,
+        "autoCompactTokens": 102400,
+        "microDeadlineTokens": 89600
+      }
+    },
+    "custom": {
+      "defaults": {
+        "profile": "custom",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "routine",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 500000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 96000,
+        "tailTrim": {
+          "enabled": false,
+          "triggerTokens": 700000
+        }
+      },
+      "linkage": {
+        "profile": "custom",
+        "nativeToolResultEnabled": false,
+        "freshEnabled": true,
+        "aggregateEnabled": true,
+        "historyMode": "routine",
+        "nativeTriggerTokens": 9007199254740991,
+        "nativeTargetTokens": 9007199254740991,
+        "freshTriggerTokens": 8192,
+        "freshTargetTokens": 3072,
+        "aggregateTriggerTokens": 32768,
+        "aggregateTargetTokens": 12288,
+        "historyTriggerTokens": 500000,
+        "historyKeepRecentToolCalls": 10,
+        "historyKeepRecentTokens": 64000,
+        "historyMinReclaimTokens": 96000,
+        "tailTrim": {
+          "enabled": false,
+          "triggerTokens": 700000
+        }
+      }
+    }
+  }
+}

package/packages/selector/tests/runtime/fixtures/tokenizer-golden.json

@@ -0,0 +1,106 @@
+{
+  "generator": "tokenizers 0.22.2 on Darwin",
+  "revision": "1",
+  "tokenizers": {
+    "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": "6821d6ad3681a4b137b066b76094fa82ebd0a380",
+    "deepseek-ai/DeepSeek-V4-Pro": "0e1a0e5e52aea73055f50fef6f2423db370265b6"
+  },
+  "cases": [
+    {
+      "label": "english prose",
+      "text": "The quick brown fox jumps over the lazy dog.",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 10,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 10
+      }
+    },
+    {
+      "label": "simplified chinese",
+      "text": "上下文压缩选择器在长任务中回收工具结果 token。",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 13,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 13
+      }
+    },
+    {
+      "label": "mixed zh-en",
+      "text": "DeepSeek Harness 的 tool result 压缩,Fresh/Aggregate/History 三个阶段。",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 21,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 21
+      }
+    },
+    {
+      "label": "json tool output",
+      "text": "{\"status\":\"ok\",\"items\":[{\"id\":1,\"path\":\"/tmp/a.txt\"}],\"count\":1}",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 24,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 24
+      }
+    },
+    {
+      "label": "python code",
+      "text": "def solve(width: int, height: int) -> int:\n    return width * height // 2\n",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 22,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 22
+      }
+    },
+    {
+      "label": "shell transcript",
+      "text": "$ pnpm run test\n> vitest run\n\n Test Files  12 passed (12)\n",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 19,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 19
+      }
+    },
+    {
+      "label": "unicode whitespace",
+      "text": "line1\nline2\r\nline3\tline4  double  spaces",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 16,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 16
+      }
+    },
+    {
+      "label": "deepseek chat special tokens",
+      "text": "<｜begin▁of▁sentence｜><｜User｜>hello<｜Assistant｜><think>\n",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 6,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 6
+      }
+    },
+    {
+      "label": "vision image placeholder",
+      "text": "before<｜deepseek_image｜>after",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 9,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 3
+      }
+    },
+    {
+      "label": "empty string",
+      "text": "",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 0,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 0
+      }
+    },
+    {
+      "label": "single ascii char",
+      "text": "a",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 1,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 1
+      }
+    },
+    {
+      "label": "emoji",
+      "text": "compression 🧠✅ done",
+      "counts": {
+        "deepseek-ai/DeepSeek-V4-Pro": 7,
+        "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp": 7
+      }
+    }
+  ]
+}