dsh-context-compression-improved 0.4.0-beta.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.ja.md +68 -36
  2. package/CHANGELOG.ko.md +67 -35
  3. package/CHANGELOG.md +195 -134
  4. package/CHANGELOG.zh.md +64 -36
  5. package/README.ja.md +1 -1
  6. package/README.ko.md +1 -1
  7. package/README.md +1 -1
  8. package/README.zh.md +1 -1
  9. package/docs/installation.ja.md +2 -2
  10. package/docs/installation.ko.md +2 -2
  11. package/docs/installation.md +103 -78
  12. package/docs/installation.zh.md +100 -77
  13. package/docs/repair-log.md +54 -0
  14. package/package.json +1 -1
  15. package/packages/selector/lib/{config.js → advisor-state.js} +329 -5
  16. package/packages/selector/lib/client.d.ts +7 -0
  17. package/packages/selector/lib/client.js +33 -3
  18. package/packages/selector/lib/index.d.ts +7 -0
  19. package/packages/selector/lib/index.js +112 -3
  20. package/packages/selector/lib/pruner.d.ts +128 -1
  21. package/packages/selector/lib/pruner.js +2802 -1374
  22. package/packages/selector/src/client/ReviewOverlay.tsx +1 -1
  23. package/packages/selector/src/client/index.ts +1 -1
  24. package/packages/selector/src/client/preset-options.ts +2 -0
  25. package/packages/selector/src/index.ts +129 -49
  26. package/packages/selector/src/profiles.ts +48 -0
  27. package/packages/selector/src/pruner/content.ts +18 -5
  28. package/packages/selector/src/pruner/state.ts +3 -0
  29. package/packages/selector/src/pruner/types.ts +23 -5
  30. package/packages/selector/src/pruner.ts +297 -162
  31. package/packages/selector/src/runtime/adaptive-cost.ts +23 -12
  32. package/packages/selector/src/runtime/audit.ts +40 -2
  33. package/packages/selector/src/runtime/config.ts +88 -1
  34. package/packages/selector/src/runtime/measurement.ts +31 -2
  35. package/packages/selector/src/runtime/reducers.ts +1115 -97
  36. package/packages/selector/src/runtime/tokenpilot/advisor-prompt.ts +188 -0
  37. package/packages/selector/src/runtime/tokenpilot/advisor-state.ts +133 -0
  38. package/packages/selector/src/runtime/tokenpilot/advisor.ts +419 -0
  39. package/packages/selector/src/runtime/tokenpilot/dedup.ts +1 -1
  40. package/packages/selector/src/runtime/tokenpilot/estimator.ts +8 -118
  41. package/packages/selector/src/runtime/tokenpilot/locator.ts +1 -1
  42. package/packages/selector/src/runtime/tokenpilot/proposal.ts +76 -32
  43. package/packages/selector/src/runtime/tokenpilot/read-state.ts +23 -2
  44. package/packages/selector/src/runtime/tokenpilot/review-registry.ts +117 -0
  45. package/packages/selector/src/runtime/tokenpilot/sidechannel.ts +303 -0
  46. package/packages/selector/src/runtime/toolclass.ts +103 -0
  47. package/packages/selector/src/runtime/types.ts +37 -0
  48. package/packages/selector/tests/advisor-report.host.spec.ts +223 -0
  49. package/packages/selector/tests/public/package-contract.client.spec.ts +2 -1
  50. package/packages/selector/tests/review-routes-registry.host.spec.ts +142 -0
  51. package/packages/selector/tests/runtime/adaptive-cost.spec.ts +7 -7
  52. package/packages/selector/tests/runtime/advisor-invariant.spec.ts +272 -0
  53. package/packages/selector/tests/runtime/advisor.spec.ts +226 -0
  54. package/packages/selector/tests/runtime/audit.spec.ts +88 -1
  55. package/packages/selector/tests/runtime/char-basis.spec.ts +30 -0
  56. package/packages/selector/tests/runtime/code-skeleton.spec.ts +14 -3
  57. package/packages/selector/tests/runtime/frequency-longstrings.spec.ts +74 -0
  58. package/packages/selector/tests/runtime/html-reducer.spec.ts +212 -0
  59. package/packages/selector/tests/runtime/line-mapping.spec.ts +153 -0
  60. package/packages/selector/tests/runtime/prose-reducers.spec.ts +133 -0
  61. package/packages/selector/tests/runtime/public/public-runtime.spec.ts +198 -27
  62. package/packages/selector/tests/runtime/read-input-cap.spec.ts +33 -0
  63. package/packages/selector/tests/runtime/search-reducer.spec.ts +110 -0
  64. package/packages/selector/tests/runtime/sidechannel.spec.ts +241 -0
  65. package/packages/selector/tests/runtime/toc-and-bundled.spec.ts +159 -0
  66. package/packages/selector/tests/runtime/tokenpilot/profile-baseline.spec.ts +12 -0
  67. package/packages/selector/tests/runtime/tokenpilot/proposal.spec.ts +194 -0
  68. package/packages/selector/tests/runtime/tokenpilot/pruner-review.spec.ts +70 -1
  69. package/packages/selector/tests/runtime/tokenpilot/read-state.spec.ts +24 -0
  70. package/packages/selector/tests/runtime/toolclass.spec.ts +156 -0
  71. package/scripts/toolclass-corpus-replay.mjs +281 -0
@@ -45,6 +45,7 @@ import ToolResultPruner, {
45
45
  resolvePolicy,
46
46
  } from '../../../src/pruner.ts'
47
47
  import type { CustomCompressionPolicy } from '../../../src/pruner.ts'
48
+ import { CHARS_PER_TOKEN } from '../../../src/runtime/config.ts'
48
49
  import { DEEPSEEK_V4_TOKENIZER_ARTIFACT, deepSeekV4TokenizerForModel } from '../../../src/deepseek-v4-tokenizer.ts'
49
50
  import {
50
51
  DEEPSEEK_VISION_DEFAULT_IMAGE_TOKENS,
@@ -165,6 +166,7 @@ function appendToolTurn(
165
166
  userText?: string,
166
167
  provider = 'deepseek',
167
168
  model: string = MODEL,
169
+ toolName = 'bash',
168
170
  ): { readonly assistantSeq: number; readonly resultSeq: number } {
169
171
  const callId = CallId(`call-${String(turn)}`)
170
172
  session.append('turn/start', { turn })
@@ -187,11 +189,11 @@ function appendToolTurn(
187
189
  step: 1,
188
190
  message: createMessage({
189
191
  role: 'assistant',
190
- content: [{ type: 'tool-call', id: callId, name: 'bash', arguments: '{}' }],
192
+ content: [{ type: 'tool-call', id: callId, name: toolName, arguments: '{}' }],
191
193
  source: { kind: 'model', provider, model },
192
194
  }),
193
195
  }, { surfaceOp: 'append' })
194
- session.append('tool/call', { turn, step: 1, callId, name: 'bash', arguments: '{}' })
196
+ session.append('tool/call', { turn, step: 1, callId, name: toolName, arguments: '{}' })
195
197
  const result = session.append('tool/result', {
196
198
  turn,
197
199
  step: 1,
@@ -416,6 +418,58 @@ describe('standalone runtime on published Harness APIs', () => {
416
418
  expect(record?.tokensRemoved).toBe((record?.tokensBefore ?? 0) - (record?.tokensAfter ?? 0))
417
419
  })
418
420
 
421
+ it('threads the reducer elided-line count from a fresh plan into the rewrite audit', async () => {
422
+ // The skeleton reducer counts the lines it hid behind elision markers; that
423
+ // count must survive the whole planning middle (plan options → planned
424
+ // replacement → audit record) without ever touching the replacement text.
425
+ const ctx = await runtimeContext()
426
+ await ctx.plugin(TestSettings).await()
427
+ await ctx.plugin(SelectorHost).await()
428
+ await ctx.settings.update(nsBrand(CONTEXT_COMPRESSION_SETTINGS_NAMESPACE), {
429
+ profile: 'balanced',
430
+ codeSkeleton: { enabled: true },
431
+ })
432
+ const audit = captureAudit(ctx)
433
+ await ctx.plugin(ToolResultPruner, {
434
+ profile: 'balanced',
435
+ freshTriggerTokens: 1_000,
436
+ freshTargetTokens: 800,
437
+ aggregateTriggerTokens: 500_000,
438
+ aggregateTargetTokens: 450_000,
439
+ historyTriggerTokens: 500_000,
440
+ }).await()
441
+ const session = Session.create(SessionId('public-fresh-elided-lines'))
442
+ const fixture = [
443
+ "import { readFile } from 'node:fs/promises'",
444
+ '',
445
+ ...Array.from({ length: 40 }, (_, index) => [
446
+ `export function handler${String(index)}(input: string): string {`,
447
+ ` const normalized = input.trim().toLowerCase()`,
448
+ ` if (normalized.length === 0) return 'empty-${String(index)}'`,
449
+ ` return normalized.split('-').join('+')`,
450
+ '}',
451
+ '',
452
+ ]).flat(),
453
+ ].join('\n')
454
+ appendToolTurn(session, 1, fixture, false, undefined, 'deepseek-official', MODEL, 'read')
455
+
456
+ const result = ctx.toolResultPruner.pruneSession(session, {
457
+ stage: 'fresh',
458
+ freshTurn: 1,
459
+ freshStep: 1,
460
+ })
461
+
462
+ expect(result.pruned).toHaveLength(1)
463
+ expect(result.pruned[0]?.reducer).toBe('hypa-code-skeleton')
464
+ const record = rewrites(audit.records()).find(entry => entry.component === 'fresh')
465
+ expect(record?.reducer).toBe('hypa-code-skeleton')
466
+ expect(record?.elidedLines).toEqual(expect.any(Number))
467
+ expect(record?.elidedLines).toBeGreaterThan(0)
468
+ expect(Number.isInteger(record?.elidedLines)).toBe(true)
469
+ expect(record?.tokensBefore).toBeGreaterThan(1_000)
470
+ expect(record?.tokensAfter).toBeLessThanOrEqual(800)
471
+ })
472
+
419
473
  it('proves isolated Aggregate while Fresh and History remain below their gates', async () => {
420
474
  const ctx = await runtimeContext()
421
475
  const audit = captureAudit(ctx)
@@ -460,13 +514,16 @@ describe('standalone runtime on published Harness APIs', () => {
460
514
  )
461
515
  session.append('turn/start', { turn: 2 })
462
516
  const view = measureForCompaction(ctx, session)
517
+ // Decision gates run on the character basis, so the thresholds below are
518
+ // derived from per-result character pressure and expressed in the
519
+ // token-named settings keys via CHARS_PER_TOKEN.
463
520
  const counts = batch.resultSeqs.map((seq) => {
464
- const entry = view.measuredNodes.find(node => node.seq === seq)?.count
465
- if (entry?.kind !== 'exact-tokenizer') throw new Error('working-set test needs exact result counts')
466
- return entry.tokens
521
+ const entry = view.measuredNodes.find(node => node.seq === seq)
522
+ if (entry === undefined) throw new Error('working-set test needs measured results')
523
+ return entry.characterPressure
467
524
  })
468
- const recentTailTokens = counts.slice(-3).reduce((sum, tokens) => sum + tokens, 0) - 1
469
- const totalToolTokens = counts.reduce((sum, tokens) => sum + tokens, 0)
525
+ const recentTailChars = counts.slice(-3).reduce((sum, chars) => sum + chars, 0) - 1
526
+ const totalToolChars = counts.reduce((sum, chars) => sum + chars, 0)
470
527
 
471
528
  await ctx.plugin(ToolResultPruner, {
472
529
  profile: 'balanced',
@@ -474,12 +531,15 @@ describe('standalone runtime on published Harness APIs', () => {
474
531
  freshTargetTokens: 90_000,
475
532
  aggregateTriggerTokens: 100_000,
476
533
  aggregateTargetTokens: 90_000,
477
- // The strict required-reclaim gate demands one batch pull tool tokens
534
+ // The strict required-reclaim gate demands one batch pull tool chars
478
535
  // back below the trigger; 72% leaves exactly the two oldest results as
479
536
  // the reclaimable margin above the placeholder residue.
480
- historyTriggerTokens: Math.floor(totalToolTokens * 0.72),
537
+ historyTriggerTokens: Math.floor(totalToolChars * 0.72 / CHARS_PER_TOKEN),
481
538
  historyKeepRecentToolCalls: 2,
482
- historyKeepRecentTokens: recentTailTokens,
539
+ // 4 x floor(recentTailChars / 4) sits above the two-newest char total
540
+ // and at or below the three-newest total, so the tail protects exactly
541
+ // the three newest results.
542
+ historyKeepRecentTokens: Math.floor(recentTailChars / CHARS_PER_TOKEN),
483
543
  historyMinReclaimTokens: 1,
484
544
  }).await()
485
545
 
@@ -1327,7 +1387,7 @@ describe('standalone runtime on published Harness APIs', () => {
1327
1387
  }))
1328
1388
  })
1329
1389
 
1330
- it('preserves exact-tokenizer-unavailable before Adaptive cost authority', async () => {
1390
+ it('audits adaptive-cost-rejected on the character basis when the authority lacks request telemetry', async () => {
1331
1391
  const ctx = await runtimeContext()
1332
1392
  const audit = captureAudit(ctx)
1333
1393
  await ctx.plugin(ToolResultPruner, {
@@ -1338,7 +1398,9 @@ describe('standalone runtime on published Harness APIs', () => {
1338
1398
  aggregateTargetTokens: 90_000,
1339
1399
  historyTriggerTokens: 100,
1340
1400
  historyKeepRecentToolCalls: 0,
1341
- historyKeepRecentTokens: 1,
1401
+ // A zero tail budget keeps nothing in the char-basis protection loop
1402
+ // (0 < 0 is false), so the batch can actually form.
1403
+ historyKeepRecentTokens: 0,
1342
1404
  historyMinReclaimTokens: 1,
1343
1405
  }).await()
1344
1406
  const session = Session.create(SessionId('public-history-adaptive-exact-unavailable'))
@@ -1353,18 +1415,21 @@ describe('standalone runtime on published Harness APIs', () => {
1353
1415
  )
1354
1416
  session.append('turn/start', { turn: 2 })
1355
1417
  ctx.toolResultPruner.pruneSession(session, { stage: 'pressure' })
1418
+ // Planning now succeeds on the character basis; the batch is refused by
1419
+ // the adaptive cost authority (no adjacent request telemetry), not by a
1420
+ // missing exact count.
1356
1421
  expect(audit.records()).toContainEqual(expect.objectContaining({
1357
1422
  kind: 'component-evaluation',
1358
1423
  sessionId: String(session.id),
1359
1424
  component: 'history',
1360
1425
  status: 'skipped',
1361
- reason: 'exact-tokenizer-unavailable',
1426
+ reason: 'adaptive-cost-rejected',
1362
1427
  historyMode: 'adaptive',
1363
- measurementKind: 'unavailable',
1428
+ measurementKind: 'characters',
1364
1429
  }))
1365
1430
  })
1366
1431
 
1367
- it('audits exact-tokenizer-unavailable for History when counts are not exact', async () => {
1432
+ it('plans and lands History on the character basis when counts are not exact', async () => {
1368
1433
  const ctx = await runtimeContext()
1369
1434
  const audit = captureAudit(ctx)
1370
1435
  await ctx.plugin(ToolResultPruner, {
@@ -1375,21 +1440,28 @@ describe('standalone runtime on published Harness APIs', () => {
1375
1440
  aggregateTargetTokens: 90_000,
1376
1441
  historyTriggerTokens: 100,
1377
1442
  historyKeepRecentToolCalls: 0,
1378
- historyKeepRecentTokens: 1,
1443
+ historyKeepRecentTokens: 0,
1379
1444
  historyMinReclaimTokens: 1,
1380
1445
  }).await()
1381
1446
  const session = Session.create(SessionId('public-history-exact-unavailable'))
1382
1447
  appendToolTurn(session, 1, 'unknown model history evidence '.repeat(300), true, undefined, 'deepseek', 'unsupported-public-model')
1383
1448
  session.append('turn/start', { turn: 2 })
1384
1449
  ctx.toolResultPruner.pruneSession(session, { stage: 'pressure' })
1450
+ // A committed batch forms from character pressure alone and lands, because
1451
+ // this pruner registers its own recovery tool. The sibling case below drops
1452
+ // the tool registry to exercise the recovery-tool-unavailable path.
1385
1453
  expect(audit.records()).toContainEqual(expect.objectContaining({
1386
- kind: 'component-evaluation',
1454
+ kind: 'rewrite',
1387
1455
  sessionId: String(session.id),
1388
1456
  component: 'history',
1389
- status: 'skipped',
1390
- reason: 'exact-tokenizer-unavailable',
1391
- measurementKind: 'unavailable',
1457
+ reducer: 'historical-tool-result-aging',
1458
+ measurementBasis: 'characters',
1459
+ tokenizerId: 'characters',
1460
+ tokenizerRevision: 'chars-per-token-4.0',
1392
1461
  }))
1462
+ // The character basis never claims an exact tokenizer it does not have.
1463
+ expect(audit.records().some(record =>
1464
+ record.kind === 'rewrite' && record.measurementBasis === 'exact-tokenizer')).toBe(false)
1393
1465
  })
1394
1466
 
1395
1467
  it('audits recovery-tool-unavailable when a committed batch cannot land without the recovery tool', async () => {
@@ -1763,14 +1835,15 @@ describe('standalone runtime on published Harness APIs', () => {
1763
1835
  expect(replayRecovered).toEqual(recovered)
1764
1836
  })
1765
1837
 
1766
- it('audits TailTrim threshold, tokenizer, safety-group, and min-reclaim skip paths', async () => {
1838
+ it('audits TailTrim threshold, safety-group, and min-reclaim skip paths and lands on the character basis for an unknown model', async () => {
1767
1839
  async function runScenario(options: {
1768
1840
  readonly id: string
1769
- readonly expectedReason: string
1841
+ readonly expectedReason?: string
1770
1842
  readonly trigger: number
1771
1843
  readonly minReclaim: number
1772
1844
  readonly candidateTurns: number
1773
1845
  readonly model?: string
1846
+ readonly expectLanding?: boolean
1774
1847
  }): Promise<void> {
1775
1848
  const ctx = new Context()
1776
1849
  activeContexts.push(ctx)
@@ -1813,7 +1886,17 @@ describe('standalone runtime on published Harness APIs', () => {
1813
1886
 
1814
1887
  ctx.toolResultPruner.pruneSession(session, { stage: 'pressure' })
1815
1888
 
1816
- expect(rewrites(audit.records()).filter(record => record.component === 'tail-trim')).toHaveLength(0)
1889
+ const tailTrimRewrites = rewrites(audit.records()).filter(record => record.component === 'tail-trim')
1890
+ if (options.expectLanding === true) {
1891
+ // An unknown model id no longer skips: the group lands and the record
1892
+ // must honestly report the character basis.
1893
+ expect(tailTrimRewrites).toHaveLength(1)
1894
+ expect(tailTrimRewrites[0]?.measurementBasis).toBe('characters')
1895
+ expect(tailTrimRewrites[0]?.tokenizerId).toBe('characters')
1896
+ expect(tailTrimRewrites[0]?.tokenizerRevision).toBe('chars-per-token-4.0')
1897
+ return
1898
+ }
1899
+ expect(tailTrimRewrites).toHaveLength(0)
1817
1900
  expect(audit.records()).toContainEqual(expect.objectContaining({
1818
1901
  kind: 'component-evaluation',
1819
1902
  sessionId: String(session.id),
@@ -1831,12 +1914,12 @@ describe('standalone runtime on published Harness APIs', () => {
1831
1914
  candidateTurns: 2,
1832
1915
  })
1833
1916
  await runScenario({
1834
- id: 'public-tailtrim-tokenizer-unavailable',
1835
- expectedReason: 'exact-tokenizer-unavailable',
1917
+ id: 'public-tailtrim-character-basis-landing',
1836
1918
  trigger: 1,
1837
1919
  minReclaim: 1,
1838
1920
  candidateTurns: 2,
1839
1921
  model: 'unsupported-public-model',
1922
+ expectLanding: true,
1840
1923
  })
1841
1924
  await runScenario({
1842
1925
  id: 'public-tailtrim-first-group-protected',
@@ -2276,19 +2359,25 @@ describe('standalone runtime on published Harness APIs', () => {
2276
2359
  // The original image-bearing result stays on the surface untouched.
2277
2360
  const original = sessionEvents(session)[imageResult.seq]
2278
2361
  expect(original?.type).toBe('tool/result')
2362
+ // Character basis: the text-only guard rejects a rich node, so the fresh and
2363
+ // aggregate passes report no reducible candidate and the single candidate
2364
+ // stays inside the protected History tail. The image itself is never
2365
+ // rewritten, which is the behaviour this case exists to protect.
2279
2366
  expect(audit.records()).toContainEqual(expect.objectContaining({
2280
2367
  kind: 'component-evaluation',
2281
2368
  sessionId: String(session.id),
2282
2369
  component: 'fresh',
2283
2370
  status: 'skipped',
2284
- reason: 'exact-tokenizer-unavailable',
2371
+ reason: 'no-valid-reduction',
2372
+ measurementKind: 'characters',
2285
2373
  }))
2286
2374
  expect(audit.records()).toContainEqual(expect.objectContaining({
2287
2375
  kind: 'component-evaluation',
2288
2376
  sessionId: String(session.id),
2289
2377
  component: 'history',
2290
2378
  status: 'skipped',
2291
- reason: 'exact-tokenizer-unavailable',
2379
+ reason: 'protected-working-set',
2380
+ measurementKind: 'characters',
2292
2381
  }))
2293
2382
  })
2294
2383
 
@@ -2551,3 +2640,85 @@ describe('standalone runtime on published Harness APIs', () => {
2551
2640
  || record.reason === 'exact-tokenizer-unavailable')).toBe(true)
2552
2641
  })
2553
2642
  })
2643
+
2644
+ describe('character basis / model-id independence', () => {
2645
+ // ~57,688 characters of read-type output on the decision surface, in the
2646
+ // code shape the hypa-code-skeleton reducer collapses.
2647
+ const readFixture = [
2648
+ "import { readFile } from 'node:fs/promises'",
2649
+ '',
2650
+ ...Array.from({ length: 300 }, (_, index) => [
2651
+ `export function handler${String(index)}(input: string): string {`,
2652
+ ` const normalized = input.trim().toLowerCase()`,
2653
+ ` if (normalized.length === 0) return 'empty-${String(index)}'`,
2654
+ ` return normalized.split('-').join('+')`,
2655
+ '}',
2656
+ '',
2657
+ ]).flat(),
2658
+ ].join('\n')
2659
+
2660
+ async function characterBasisContext(): Promise<Context> {
2661
+ const ctx = await runtimeContext()
2662
+ await ctx.plugin(TestSettings).await()
2663
+ await ctx.plugin(SelectorHost).await()
2664
+ await ctx.settings.update(nsBrand(CONTEXT_COMPRESSION_SETTINGS_NAMESPACE), {
2665
+ profile: 'balanced',
2666
+ codeSkeleton: { enabled: true },
2667
+ })
2668
+ await ctx.plugin(ToolResultPruner, {
2669
+ profile: 'balanced',
2670
+ freshTriggerTokens: 1_000,
2671
+ freshTargetTokens: 800,
2672
+ aggregateTriggerTokens: 500_000,
2673
+ aggregateTargetTokens: 450_000,
2674
+ historyTriggerTokens: 500_000,
2675
+ }).await()
2676
+ return ctx
2677
+ }
2678
+
2679
+ for (const routedModel of ['deepseek-nonexistent-9', 'deepseek-flash']) {
2680
+ it(`lands a fresh reduction on the character basis for routed model ${routedModel}`, async () => {
2681
+ const ctx = await characterBasisContext()
2682
+ const audit = captureAudit(ctx)
2683
+ const session = Session.create(SessionId(`public-char-basis-${routedModel}`))
2684
+ appendToolTurn(session, 1, readFixture, false, undefined, 'deepseek-official', routedModel, 'read')
2685
+
2686
+ const result = ctx.toolResultPruner.pruneSession(session, {
2687
+ stage: 'fresh',
2688
+ freshTurn: 1,
2689
+ freshStep: 1,
2690
+ })
2691
+
2692
+ expect(result.pruned.length).toBe(1)
2693
+ expect(result.pruned[0]?.reducer).toBe('hypa-code-skeleton')
2694
+ // The decision ran on characters: the record must not claim an exact
2695
+ // tokenizer it never used.
2696
+ const record = rewrites(audit.records()).find(entry => entry.component === 'fresh')
2697
+ expect(record?.measurementBasis).toBe('characters')
2698
+ expect(record?.tokenizerId).toBe('characters')
2699
+ expect(record?.tokenizerRevision).toBe('chars-per-token-4.0')
2700
+ })
2701
+ }
2702
+
2703
+ it('keeps the exact-tokenizer basis and the same reducer for a whitelisted model', async () => {
2704
+ const ctx = await characterBasisContext()
2705
+ const audit = captureAudit(ctx)
2706
+ const session = Session.create(SessionId('public-char-basis-whitelisted'))
2707
+ appendToolTurn(session, 1, readFixture, false, undefined, 'deepseek-official', MODEL, 'read')
2708
+
2709
+ const result = ctx.toolResultPruner.pruneSession(session, {
2710
+ stage: 'fresh',
2711
+ freshTurn: 1,
2712
+ freshStep: 1,
2713
+ })
2714
+
2715
+ expect(result.pruned.length).toBe(1)
2716
+ expect(result.pruned[0]?.reducer).toBe('hypa-code-skeleton')
2717
+ const record = rewrites(audit.records()).find(entry => entry.component === 'fresh')
2718
+ expect(record?.measurementBasis).toBe('exact-tokenizer')
2719
+ expect(record?.tokenizerId).toEqual(expect.stringContaining('deepseek'))
2720
+ expect(record?.tokenizerRevision).toEqual(expect.any(String))
2721
+ expect(record?.tokensBefore).toBeGreaterThan(0)
2722
+ expect(record?.tokensAfter).toBeGreaterThan(0)
2723
+ })
2724
+ })
@@ -0,0 +1,33 @@
1
+ import { describe, expect, it } from 'vitest'
2
+ import { resolveConfig, resolvePolicy } from '../../src/pruner.ts'
3
+
4
+ /** task_4 (G2): the read input cap is in CHARACTERS while the fresh trigger is
5
+ * in TOKENS. At the conservative 4.0 chars/token upper bound, a cap at or
6
+ * below `freshTriggerTokens × 4.0` truncates every read result below the
7
+ * trigger and silently silences the fresh path — so `resolvePolicy` must
8
+ * reject that combination at startup. 32k fails (32,000 < 8192×4.0 = 32,768);
9
+ * the deployed 50k cap passes; unset stays untouched. */
10
+ describe('readInputCapChars invariant', () => {
11
+ it('throws when the cap would sit below the fresh trigger (32k vs 8192×4.0)', () => {
12
+ const config = resolveConfig({ readInputCapChars: 32_000 })
13
+ expect(() => resolvePolicy(config, 'balanced')).toThrow(
14
+ 'context compression policy: read input cap would silence the fresh path',
15
+ )
16
+ })
17
+
18
+ it('accepts the deployed 50k cap', () => {
19
+ const config = resolveConfig({ readInputCapChars: 50_000 })
20
+ const policy = resolvePolicy(config, 'balanced')
21
+ expect(policy.readInputCapChars).toBe(50_000)
22
+ })
23
+
24
+ it('leaves the policy untouched when the cap is not set', () => {
25
+ const policy = resolvePolicy(resolveConfig({}), 'balanced')
26
+ expect(policy.readInputCapChars).toBeUndefined()
27
+ })
28
+
29
+ it('keeps profiles without a configured cap byte-identical (no field on the resolved object)', () => {
30
+ const policy = resolvePolicy(resolveConfig({}), 'balanced')
31
+ expect(Object.hasOwn(policy, 'readInputCapChars')).toBe(false)
32
+ })
33
+ })
@@ -0,0 +1,110 @@
1
+ import { describe, expect, it } from 'vitest'
2
+ import { reduceFreshToolResult } from '../../src/runtime/reducers.ts'
3
+
4
+ const SOURCE_REF = 'session://s1/event/7'
5
+
6
+ function searchInput(text: string, budgetChars = 16_000): Parameters<typeof reduceFreshToolResult>[0] {
7
+ return {
8
+ toolName: 'grep_search',
9
+ argumentsText: '{"pattern":"config"}',
10
+ text,
11
+ budgetChars,
12
+ sourceRef: SOURCE_REF,
13
+ isError: false,
14
+ }
15
+ }
16
+
17
+ describe('search L1 locator (R10)', () => {
18
+ it('reports the complete line-number set for a 592-hit file (count == hits)', () => {
19
+ // 实测 upper bound: one real session had a 592-hit file; the old reducer
20
+ // reported only "(592 matches)" and dropped every line number (21.2% of
21
+ // all hits across sessions were silently unlocatable).
22
+ const lines = Array.from({ length: 592 }, (_, i) =>
23
+ `src/deep/module.ts:${String(i + 1)}: export const config${String(i)} = configure(${String(i)})`)
24
+ const output = reduceFreshToolResult(searchInput(lines.join('\n')))
25
+ expect(output).not.toBeNull()
26
+ const locator = output!.text.split('\n').find(line => line.startsWith('## src/deep/module.ts'))
27
+ expect(locator).toBeDefined()
28
+ const numbers = [...locator!.matchAll(/L(\d+)/g)].map(match => Number(match[1]))
29
+ expect(numbers).toHaveLength(592)
30
+ expect(numbers[0]).toBe(1)
31
+ expect(numbers[591]).toBe(592)
32
+ })
33
+
34
+ it('keeps L1 free of content rows (pure locator)', () => {
35
+ const lines = Array.from({ length: 40 }, (_, i) => `a.ts:${String(i + 1)}: content row ${String(i)}`)
36
+ const output = reduceFreshToolResult(searchInput(lines.join('\n'), 700))
37
+ expect(output).not.toBeNull()
38
+ const locatorLine = output!.text.split('\n').find(line => line.startsWith('## a.ts'))
39
+ expect(locatorLine).toBeDefined()
40
+ expect(locatorLine).not.toContain('content row')
41
+ })
42
+
43
+ it('announces withheld locators visibly instead of silently truncating', () => {
44
+ // Budget too small for the full L1 set: the header must state how much
45
+ // was withheld rather than quietly shrinking the list.
46
+ const lines = [
47
+ ...Array.from({ length: 300 }, (_, i) => `big.ts:${String(i + 1)}: hit ${String(i)}`),
48
+ ...Array.from({ length: 5 }, (_, i) => `small.ts:${String(i + 1)}: hit ${String(i)}`),
49
+ ].join('\n')
50
+ const output = reduceFreshToolResult(searchInput(lines, 900))
51
+ expect(output).not.toBeNull()
52
+ expect(output!.text).toMatch(/withheld|more (?:files|matches)/i)
53
+ })
54
+ })
55
+
56
+ describe('search L2 water-filling (R10)', () => {
57
+ it('serves every file one round before any file takes a second row', () => {
58
+ const fat = Array.from({ length: 500 }, (_, i) => `fat.ts:${String(i + 1)}: fat file row ${String(i)}`)
59
+ const mid = Array.from({ length: 10 }, (_, i) => `mid.ts:${String(i + 1)}: mid file row ${String(i)}`)
60
+ const thin = Array.from({ length: 5 }, (_, i) => `thin.ts:${String(i + 1)}: thin file row ${String(i)}`)
61
+ // Tight content quota: only ~15 rows fit after L1. Round-robin must give
62
+ // each file a row per round — the old "fill from index 0" let the first
63
+ // file eat the whole quota.
64
+ const output = reduceFreshToolResult(searchInput([...fat, ...mid, ...thin].join('\n'), 1_600))
65
+ expect(output).not.toBeNull()
66
+ const counts = new Map<string, number>()
67
+ for (const line of output!.text.split('\n')) {
68
+ const match = /^(fat|mid|thin)\.ts:\d+:/.exec(line)
69
+ if (match !== null) counts.set(match[1]!, (counts.get(match[1]!) ?? 0) + 1)
70
+ }
71
+ expect(counts.get('fat')!).toBeGreaterThan(0)
72
+ expect(counts.get('mid')!).toBeGreaterThan(0)
73
+ expect(counts.get('thin')!).toBeGreaterThan(0)
74
+ })
75
+
76
+ it('exhausts small files before the big file takes the remaining quota', () => {
77
+ const fat = Array.from({ length: 500 }, (_, i) => `fat.ts:${String(i + 1)}: fat file row ${String(i)}`)
78
+ const mid = Array.from({ length: 10 }, (_, i) => `mid.ts:${String(i + 1)}: mid file row ${String(i)}`)
79
+ const thin = Array.from({ length: 5 }, (_, i) => `thin.ts:${String(i + 1)}: thin file row ${String(i)}`)
80
+ // Generous quota after L1: mid and thin end up FULLY covered (100% of
81
+ // their rows) while fat takes only what is left.
82
+ const output = reduceFreshToolResult(searchInput([...fat, ...mid, ...thin].join('\n'), 6_000))
83
+ const counts = new Map<string, number>()
84
+ for (const line of output!.text.split('\n')) {
85
+ const match = /^(fat|mid|thin)\.ts:\d+:/.exec(line)
86
+ if (match !== null) counts.set(match[1]!, (counts.get(match[1]!) ?? 0) + 1)
87
+ }
88
+ expect(counts.get('thin')).toBe(5)
89
+ expect(counts.get('mid')).toBe(10)
90
+ expect(counts.get('fat')!).toBeGreaterThan(0)
91
+ })
92
+
93
+ it('keeps error rows available to the quota', () => {
94
+ const rows = Array.from({ length: 40 }, (_, i) => `app.ts:${String(i + 1)}: ordinary row ${String(i)}`)
95
+ rows.push('app.ts:99: FATAL: unhandled rejection in loader')
96
+ // Tight quota: round 0 must reach the important row (offered first within
97
+ // the file) even though 40 ordinary rows outrank it by line number.
98
+ const output = reduceFreshToolResult(searchInput(rows.join('\n'), 900))
99
+ expect(output).not.toBeNull()
100
+ expect(output!.text).toContain('FATAL: unhandled rejection in loader')
101
+ })
102
+
103
+ it('falls back to salience when no path:line form exists', () => {
104
+ const output = reduceFreshToolResult(searchInput([
105
+ ...Array.from({ length: 60 }, (_, i) => `ordinary output line ${String(i)} without any locators at all`),
106
+ 'FATAL: something exploded without a path prefix',
107
+ ].join('\n'), 1_200))
108
+ expect(output?.reducer).toBe('search-salience')
109
+ })
110
+ })