@animalabs/context-manager 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/adaptive/chunker.d.ts +70 -0
- package/dist/src/adaptive/chunker.d.ts.map +1 -0
- package/dist/src/adaptive/chunker.js +233 -0
- package/dist/src/adaptive/chunker.js.map +1 -0
- package/dist/src/adaptive/folding-strategy.d.ts +118 -0
- package/dist/src/adaptive/folding-strategy.d.ts.map +1 -0
- package/dist/src/adaptive/folding-strategy.js +12 -0
- package/dist/src/adaptive/folding-strategy.js.map +1 -0
- package/dist/src/adaptive/index.d.ts +17 -0
- package/dist/src/adaptive/index.d.ts.map +1 -0
- package/dist/src/adaptive/index.js +23 -0
- package/dist/src/adaptive/index.js.map +1 -0
- package/dist/src/adaptive/picker.d.ts +103 -0
- package/dist/src/adaptive/picker.d.ts.map +1 -0
- package/dist/src/adaptive/picker.js +357 -0
- package/dist/src/adaptive/picker.js.map +1 -0
- package/dist/src/adaptive/render.d.ts +39 -0
- package/dist/src/adaptive/render.d.ts.map +1 -0
- package/dist/src/adaptive/render.js +104 -0
- package/dist/src/adaptive/render.js.map +1 -0
- package/dist/src/adaptive/strategies/flat-profile.d.ts +19 -0
- package/dist/src/adaptive/strategies/flat-profile.d.ts.map +1 -0
- package/dist/src/adaptive/strategies/flat-profile.js +86 -0
- package/dist/src/adaptive/strategies/flat-profile.js.map +1 -0
- package/dist/src/adaptive/strategies/oldest-first.d.ts +19 -0
- package/dist/src/adaptive/strategies/oldest-first.d.ts.map +1 -0
- package/dist/src/adaptive/strategies/oldest-first.js +39 -0
- package/dist/src/adaptive/strategies/oldest-first.js.map +1 -0
- package/dist/src/context-manager.d.ts +5 -0
- package/dist/src/context-manager.d.ts.map +1 -1
- package/dist/src/context-manager.js +22 -0
- package/dist/src/context-manager.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/message-store.d.ts +36 -1
- package/dist/src/message-store.d.ts.map +1 -1
- package/dist/src/message-store.js +134 -8
- package/dist/src/message-store.js.map +1 -1
- package/dist/src/strategies/autobiographical.d.ts +217 -2
- package/dist/src/strategies/autobiographical.d.ts.map +1 -1
- package/dist/src/strategies/autobiographical.js +1525 -37
- package/dist/src/strategies/autobiographical.js.map +1 -1
- package/dist/src/strategies/index.d.ts +1 -1
- package/dist/src/strategies/index.d.ts.map +1 -1
- package/dist/src/strategies/index.js.map +1 -1
- package/dist/src/types/message.d.ts +38 -0
- package/dist/src/types/message.d.ts.map +1 -1
- package/dist/src/types/strategy.d.ts +106 -10
- package/dist/src/types/strategy.d.ts.map +1 -1
- package/dist/src/types/strategy.js +7 -0
- package/dist/src/types/strategy.js.map +1 -1
- package/dist/test/adaptive/branching.test.d.ts +20 -0
- package/dist/test/adaptive/branching.test.d.ts.map +1 -0
- package/dist/test/adaptive/branching.test.js +176 -0
- package/dist/test/adaptive/branching.test.js.map +1 -0
- package/dist/test/adaptive/chunker.test.d.ts +2 -0
- package/dist/test/adaptive/chunker.test.d.ts.map +1 -0
- package/dist/test/adaptive/chunker.test.js +120 -0
- package/dist/test/adaptive/chunker.test.js.map +1 -0
- package/dist/test/adaptive/deep-levels.test.d.ts +10 -0
- package/dist/test/adaptive/deep-levels.test.d.ts.map +1 -0
- package/dist/test/adaptive/deep-levels.test.js +167 -0
- package/dist/test/adaptive/deep-levels.test.js.map +1 -0
- package/dist/test/adaptive/doc-plus-chat.test.d.ts +10 -0
- package/dist/test/adaptive/doc-plus-chat.test.d.ts.map +1 -0
- package/dist/test/adaptive/doc-plus-chat.test.js +194 -0
- package/dist/test/adaptive/doc-plus-chat.test.js.map +1 -0
- package/dist/test/adaptive/hard-fail.test.d.ts +9 -0
- package/dist/test/adaptive/hard-fail.test.d.ts.map +1 -0
- package/dist/test/adaptive/hard-fail.test.js +154 -0
- package/dist/test/adaptive/hard-fail.test.js.map +1 -0
- package/dist/test/adaptive/harness.d.ts +109 -0
- package/dist/test/adaptive/harness.d.ts.map +1 -0
- package/dist/test/adaptive/harness.js +263 -0
- package/dist/test/adaptive/harness.js.map +1 -0
- package/dist/test/adaptive/ingestion.test.d.ts +13 -0
- package/dist/test/adaptive/ingestion.test.d.ts.map +1 -0
- package/dist/test/adaptive/ingestion.test.js +306 -0
- package/dist/test/adaptive/ingestion.test.js.map +1 -0
- package/dist/test/adaptive/integration.test.d.ts +12 -0
- package/dist/test/adaptive/integration.test.d.ts.map +1 -0
- package/dist/test/adaptive/integration.test.js +247 -0
- package/dist/test/adaptive/integration.test.js.map +1 -0
- package/dist/test/adaptive/long-chronicle.test.d.ts +13 -0
- package/dist/test/adaptive/long-chronicle.test.d.ts.map +1 -0
- package/dist/test/adaptive/long-chronicle.test.js +186 -0
- package/dist/test/adaptive/long-chronicle.test.js.map +1 -0
- package/dist/test/adaptive/persistence.test.d.ts +11 -0
- package/dist/test/adaptive/persistence.test.d.ts.map +1 -0
- package/dist/test/adaptive/persistence.test.js +200 -0
- package/dist/test/adaptive/persistence.test.js.map +1 -0
- package/dist/test/adaptive/picker.test.d.ts +2 -0
- package/dist/test/adaptive/picker.test.d.ts.map +1 -0
- package/dist/test/adaptive/picker.test.js +333 -0
- package/dist/test/adaptive/picker.test.js.map +1 -0
- package/dist/test/adaptive/render.test.d.ts +2 -0
- package/dist/test/adaptive/render.test.d.ts.map +1 -0
- package/dist/test/adaptive/render.test.js +109 -0
- package/dist/test/adaptive/render.test.js.map +1 -0
- package/dist/test/adaptive/shard-immutability.test.d.ts +14 -0
- package/dist/test/adaptive/shard-immutability.test.d.ts.map +1 -0
- package/dist/test/adaptive/shard-immutability.test.js +184 -0
- package/dist/test/adaptive/shard-immutability.test.js.map +1 -0
- package/dist/test/adaptive/strategy-integration.test.d.ts +9 -0
- package/dist/test/adaptive/strategy-integration.test.d.ts.map +1 -0
- package/dist/test/adaptive/strategy-integration.test.js +293 -0
- package/dist/test/adaptive/strategy-integration.test.js.map +1 -0
- package/dist/test/message-store-sequence.test.d.ts +25 -0
- package/dist/test/message-store-sequence.test.d.ts.map +1 -0
- package/dist/test/message-store-sequence.test.js +157 -0
- package/dist/test/message-store-sequence.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/adaptive/chunker.ts +323 -0
- package/src/adaptive/folding-strategy.ts +129 -0
- package/src/adaptive/index.ts +51 -0
- package/src/adaptive/picker.ts +442 -0
- package/src/adaptive/render.ts +115 -0
- package/src/adaptive/strategies/flat-profile.ts +96 -0
- package/src/adaptive/strategies/oldest-first.ts +48 -0
- package/src/context-manager.ts +32 -0
- package/src/index.ts +1 -1
- package/src/message-store.ts +147 -8
- package/src/strategies/autobiographical.ts +1635 -38
- package/src/strategies/index.ts +1 -1
- package/src/types/message.ts +43 -0
- package/src/types/strategy.ts +122 -10
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { JsStore } from '@animalabs/chronicle';
|
|
2
|
-
import type { Membrane } from '@animalabs/membrane';
|
|
2
|
+
import type { Membrane, ContentBlock } from '@animalabs/membrane';
|
|
3
3
|
import type { StoredMessage, MessageId, Sequence } from './message.js';
|
|
4
4
|
import type { ContextEntry, TokenBudget } from './context.js';
|
|
5
5
|
/**
|
|
@@ -110,6 +110,36 @@ export interface ContextStrategy {
|
|
|
110
110
|
* This is the core method that determines what goes in the context window.
|
|
111
111
|
*/
|
|
112
112
|
select(store: MessageStoreView, log: ContextLogView, budget: TokenBudget): ContextEntry[];
|
|
113
|
+
/**
|
|
114
|
+
* Optional: at ingestion time, decide whether to split this incoming
|
|
115
|
+
* message into multiple shards (because it's too large to be a single
|
|
116
|
+
* fold unit). If returned, the framework stores each shard as a separate
|
|
117
|
+
* StoredMessage with the shared `bodyGroupId` and per-shard `shardIndex`,
|
|
118
|
+
* and the render path will reassemble them into one API message at
|
|
119
|
+
* compile time.
|
|
120
|
+
*
|
|
121
|
+
* Return null or undefined to skip chunking (message stored as one record).
|
|
122
|
+
*
|
|
123
|
+
* See `docs/adaptive-resolution-design.md` §3.6.
|
|
124
|
+
*/
|
|
125
|
+
chunkIngressMessage?(participant: string, content: ContentBlock[]): IngressChunkResult | null;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Result of a strategy's ingestion-time chunking decision.
|
|
129
|
+
*/
|
|
130
|
+
export interface IngressChunkResult {
|
|
131
|
+
/** Stable id shared by all shards of this message. */
|
|
132
|
+
bodyGroupId: string;
|
|
133
|
+
/**
|
|
134
|
+
* The shards in source order. Each becomes a separate StoredMessage.
|
|
135
|
+
* Concatenating the shards' text content must reproduce the original
|
|
136
|
+
* message body byte-for-byte.
|
|
137
|
+
*/
|
|
138
|
+
shards: Array<{
|
|
139
|
+
content: ContentBlock[];
|
|
140
|
+
/** Order within the bodyGroup, starting at 0. */
|
|
141
|
+
shardIndex: number;
|
|
142
|
+
}>;
|
|
113
143
|
}
|
|
114
144
|
/**
|
|
115
145
|
* Strategy that supports resetting the head window for topic transitions.
|
|
@@ -270,6 +300,18 @@ export interface AutobiographicalConfig {
|
|
|
270
300
|
l2BudgetTokens?: number;
|
|
271
301
|
/** Token budget for L1 summaries in select() (default: 30000) */
|
|
272
302
|
l1BudgetTokens?: number;
|
|
303
|
+
/**
|
|
304
|
+
* Cap on the total tokens of recall-pair prior-summary content
|
|
305
|
+
* included in each chunk compression request (default: 150000).
|
|
306
|
+
*
|
|
307
|
+
* Defends against the case where the unmerged frontier itself is
|
|
308
|
+
* large enough to overflow the API window. The cap walks summaries
|
|
309
|
+
* newest-first so proximate context survives; the kept set is then
|
|
310
|
+
* re-sorted chronologically. Set higher (or to Infinity) if you want
|
|
311
|
+
* to surface overflow via a 400 rather than silently drop oldest
|
|
312
|
+
* memories from the compression context.
|
|
313
|
+
*/
|
|
314
|
+
compressionRecallBudgetTokens?: number;
|
|
273
315
|
/**
|
|
274
316
|
* When true (default), each selected summary emits as its own positioned
|
|
275
317
|
* Q/A recall pair, sorted chronologically by source range. When false,
|
|
@@ -342,42 +384,96 @@ export interface AutobiographicalConfig {
|
|
|
342
384
|
* that overflow is rare, and when it does happen you want to know.
|
|
343
385
|
*/
|
|
344
386
|
enforceBudget?: boolean;
|
|
387
|
+
/**
|
|
388
|
+
* Enable picker-driven adaptive resolution. When true, `select()` uses
|
|
389
|
+
* the FoldingStrategy + Picker to choose per-message resolution under
|
|
390
|
+
* token-budget pressure rather than the threshold-driven `checkMergeThreshold`
|
|
391
|
+
* path. Default: false (existing hierarchical behavior preserved).
|
|
392
|
+
*/
|
|
393
|
+
adaptiveResolution?: boolean;
|
|
394
|
+
/**
|
|
395
|
+
* Folding strategy name when adaptiveResolution is on. One of:
|
|
396
|
+
* 'flat-profile' (default) — level-equalizing
|
|
397
|
+
* 'oldest-first' — chronological
|
|
398
|
+
* Custom strategies can be plugged in by the host application.
|
|
399
|
+
*/
|
|
400
|
+
foldingStrategy?: 'flat-profile' | 'oldest-first';
|
|
401
|
+
/**
|
|
402
|
+
* Slack ratio (hysteresis) for the picker. The picker folds until total
|
|
403
|
+
* tokens ≤ budget * (1 - slack), and stays quiet while between slack
|
|
404
|
+
* and budget. Default 0.1.
|
|
405
|
+
*/
|
|
406
|
+
compressionSlackRatio?: number;
|
|
407
|
+
/**
|
|
408
|
+
* Enable bottom-up speculative pre-production of higher-level summaries.
|
|
409
|
+
* When a new L_k summary is produced, if N siblings exist that would share
|
|
410
|
+
* an L_{k+1} parent, the L_{k+1} is enqueued for production immediately
|
|
411
|
+
* (no picker request needed). Default true when adaptiveResolution is on.
|
|
412
|
+
*/
|
|
413
|
+
speculativeProduction?: boolean;
|
|
345
414
|
}
|
|
346
415
|
/**
|
|
347
416
|
* Compression level in the hierarchical pyramid.
|
|
417
|
+
*
|
|
418
|
+
* Historically constrained to 1 | 2 | 3. As of the adaptive-resolution design
|
|
419
|
+
* (`docs/adaptive-resolution-design.md`), levels are unbounded: the picker
|
|
420
|
+
* can recursively produce L4, L5, ... as needed. The narrower literal type
|
|
421
|
+
* is kept as `LegacySummaryLevel` for code that still assumes the old shape.
|
|
422
|
+
*/
|
|
423
|
+
export type SummaryLevel = number;
|
|
424
|
+
/**
|
|
425
|
+
* The narrow level type used by pre-adaptive-resolution code paths.
|
|
426
|
+
* Prefer `SummaryLevel` for new code.
|
|
348
427
|
*/
|
|
349
|
-
export type
|
|
428
|
+
export type LegacySummaryLevel = 1 | 2 | 3;
|
|
350
429
|
/**
|
|
351
430
|
* A summary entry in the hierarchical memory pyramid.
|
|
352
431
|
* L1: compressed from raw message chunks.
|
|
353
|
-
*
|
|
354
|
-
* L3: merged from mergeThreshold L2s.
|
|
432
|
+
* L_{k>1}: merged from mergeThreshold L_{k-1}s.
|
|
355
433
|
*/
|
|
356
434
|
export interface SummaryEntry {
|
|
357
435
|
/** Unique ID (e.g., "L1-0", "L2-3") */
|
|
358
436
|
id: string;
|
|
359
|
-
/** Compression level */
|
|
437
|
+
/** Compression level (1, 2, 3, ... — unbounded in the adaptive-resolution design) */
|
|
360
438
|
level: SummaryLevel;
|
|
361
439
|
/** The summary text */
|
|
362
440
|
content: string;
|
|
363
|
-
/** Estimated token count (content.length / 4) */
|
|
441
|
+
/** Estimated token count (content.length / 4 or tokenizer-cached) */
|
|
364
442
|
tokens: number;
|
|
365
|
-
/**
|
|
366
|
-
|
|
367
|
-
|
|
443
|
+
/**
|
|
444
|
+
* Level of the sources: 0 = raw messages, k = L_k summaries.
|
|
445
|
+
* Pre-adaptive code uses 0 | 1 | 2; new code may produce higher values.
|
|
446
|
+
*/
|
|
447
|
+
sourceLevel: number;
|
|
448
|
+
/** IDs of source items (message IDs for L1, summary IDs for L_{k>1}) */
|
|
368
449
|
sourceIds: string[];
|
|
369
450
|
/** Range of original message IDs covered */
|
|
370
451
|
sourceRange: {
|
|
371
452
|
first: string;
|
|
372
453
|
last: string;
|
|
373
454
|
};
|
|
374
|
-
/**
|
|
455
|
+
/**
|
|
456
|
+
* The L_{level+1} summary this one is a source for, if produced.
|
|
457
|
+
* Pure archive metadata in the adaptive-resolution design — display
|
|
458
|
+
* decisions live on per-chunk `currentResolution`, not here.
|
|
459
|
+
*/
|
|
460
|
+
parentId?: string;
|
|
461
|
+
/**
|
|
462
|
+
* @deprecated Renamed to `parentId` in the adaptive-resolution design.
|
|
463
|
+
* Kept for read compatibility with chronicles produced by the old
|
|
464
|
+
* threshold-driven path. New writes should set `parentId` only.
|
|
465
|
+
*/
|
|
375
466
|
mergedInto?: string;
|
|
376
467
|
/** Creation timestamp */
|
|
377
468
|
created: number;
|
|
378
469
|
/** Phase type tag (used by KnowledgeStrategy for asymmetric budget) */
|
|
379
470
|
phaseType?: string;
|
|
380
471
|
}
|
|
472
|
+
/**
|
|
473
|
+
* Helper: read the parent pointer from a summary, accepting either the
|
|
474
|
+
* new `parentId` field or the deprecated `mergedInto` alias.
|
|
475
|
+
*/
|
|
476
|
+
export declare function getSummaryParentId(s: SummaryEntry): string | undefined;
|
|
381
477
|
/**
|
|
382
478
|
* A range of messages protected from compression. Pins keep a span of raw
|
|
383
479
|
* messages visible at their original position in the rendered context.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategy.d.ts","sourceRoot":"","sources":["../../../src/types/strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"strategy.d.ts","sourceRoot":"","sources":["../../../src/types/strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAe,MAAM,cAAc,CAAC;AAE3E;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uBAAuB;IACvB,MAAM,IAAI,aAAa,EAAE,CAAC;IAC1B,6BAA6B;IAC7B,GAAG,CAAC,EAAE,EAAE,SAAS,GAAG,aAAa,GAAG,IAAI,CAAC;IACzC,yCAAyC;IACzC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,EAAE,CAAC;IACxC,8BAA8B;IAC9B,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,EAAE,CAAC;IACxC,8BAA8B;IAC9B,MAAM,IAAI,MAAM,CAAC;IACjB,oCAAoC;IACpC,cAAc,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sBAAsB;IACtB,MAAM,IAAI,YAAY,EAAE,CAAC;IACzB,wCAAwC;IACxC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvC,6BAA6B;IAC7B,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvC,4BAA4B;IAC5B,MAAM,IAAI,MAAM,CAAC;IACjB,mCAAmC;IACnC,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,sCAAsC;IACtC,YAAY,EAAE,gBAAgB,CAAC;IAC/B,oCAAoC;IACpC,UAAU,EAAE,cAAc,CAAC;IAC3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,8BAA8B;IAC9B,eAAe,EAAE,QAAQ,CAAC;IAC1B;;;;;;;OAOG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;;;;OAKG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;IACf,sDAAsD;IACtD,WAAW,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,kCAAkC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,uCAAuC;IACvC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;qCAEiC;IACjC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;OAGG;IACH,UAAU,CAAC,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD;;;OAGG;IACH,IAAI,CAAC,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3C;;;OAGG;IACH,YAAY,CAAC,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3E;;;OAGG;IACH,cAAc,IAAI,cAAc,CAAC;IAEjC;;;OAGG;IACH,MAAM,CACJ,KAAK,EAAE,gBAAgB,EACvB,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,WAAW,GAClB,YAAY,EAAE,CAAC;IAElB;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,CAClB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,YAAY,EAAE,GACtB,kBAAkB,GAAG,IAAI,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,EAAE,KAAK,CAAC;QACZ,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,iDAAiD;QACjD,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,eAAe;IACzD,4DAA4D;IAC5D,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC;IACjD,8EAA8E;IAC9E,yBAAyB,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAClE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,IAAI,kBAAkB,CAEhF;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAiB,SAAQ,eAAe;IACvD,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC;IAC1F,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC;IAClE,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9B,QAAQ,IAAI,aAAa,CAAC,cAAc,CAAC,CAAC;CAC3C;AAED,+DAA+D;AAC/D,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,IAAI,gBAAgB,CAO5E;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,gEAAgE;IAChE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;IACxB,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,uCAAuC;AACvC,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,YAAY,CAAC;IACtB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,eAAe;IACzD,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE,CAAC;IACpD,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAAC;CAC7C;AAED,qDAAqD;AACrD,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,IAAI,kBAAkB,CAOhF;AAED;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,SAAS,EAAE;QACT,EAAE,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QACtC,EAAE,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QACtC,EAAE,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;KACvC,CAAC;IACF,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7C;AAED;;;;GAIG;AACH,MAAM,WAAW,0BAA2B,SAAQ,eAAe;IACjE,cAAc,CAAC,KAAK,EAAE,gBAAgB,GAAG,WAAW,CAAC;CACtD;AAED,2DAA2D;AAC3D,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,eAAe,GAAG,CAAC,IAAI,0BAA0B,CAKxF;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,sCAAsC;IACtC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kDAAkD;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;yEAEqE;IACrE,gBAAgB,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,sBAAsB,EAAE,OAAO,CAAC;IAChC,sDAAsD;IACtD,qBAAqB,EAAE,OAAO,CAAC;IAC/B;8EAC0E;IAC1E,oBAAoB,EAAE,OAAO,CAAC;IAC9B,sCAAsC;IACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gFAAgF;IAChF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uDAAuD;IACvD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,+DAA+D;IAC/D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;kFAC8E;IAC9E,gBAAgB,EAAE,MAAM,CAAC;IAGzB,0CAA0C;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wCAAwC;IACxC,eAAe,CAAC,EAAE,MAAM,CAAC;IAIzB,iGAAiG;IACjG,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,iFAAiF;IACjF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;;;;;;;OAUG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;;;;;;;;OAWG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;;;;;OASG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAErD;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;;;;;OASG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;;;;;;;;;;OAaG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IAIxB;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;;;;OAKG;IACH,eAAe,CAAC,EAAE,cAAc,GAAG,cAAc,CAAC;IAElD;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAElC;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE3C;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,qFAAqF;IACrF,KAAK,EAAE,YAAY,CAAC;IACpB,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,4CAA4C;IAC5C,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,GAAG,SAAS,CAEtE;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,2DAA2D;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,KAAK,GAAG,UAAU,CAAC;IACzB,qCAAqC;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,QAAQ,GAAG,UAAU,CAAC;AAEzE;;;GAGG;AACH,MAAM,WAAW,eAAgB,SAAQ,sBAAsB;IAC7D;uCACmC;IACnC,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC;iCAC6B;IAC7B,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC;wDACoD;IACpD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B,2EAA2E;IAC3E,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,8EAA8E;IAC9E,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,2EAA2E;IAC3E,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,sEAAsE;IACtE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B,0EAA0E;IAC1E,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;6EACyE;IACzE,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,sBAuB7C,CAAC"}
|
|
@@ -23,6 +23,13 @@ export function isRenderStatsCapable(s) {
|
|
|
23
23
|
return ('getRenderStats' in s &&
|
|
24
24
|
typeof s.getRenderStats === 'function');
|
|
25
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Helper: read the parent pointer from a summary, accepting either the
|
|
28
|
+
* new `parentId` field or the deprecated `mergedInto` alias.
|
|
29
|
+
*/
|
|
30
|
+
export function getSummaryParentId(s) {
|
|
31
|
+
return s.parentId ?? s.mergedInto;
|
|
32
|
+
}
|
|
26
33
|
/**
|
|
27
34
|
* Default configuration for Autobiographical strategy.
|
|
28
35
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategy.js","sourceRoot":"","sources":["../../../src/types/strategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"strategy.js","sourceRoot":"","sources":["../../../src/types/strategy.ts"],"names":[],"mappings":"AA+KA;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,CAAkB;IACrD,OAAO,iBAAiB,IAAI,CAAC,IAAI,OAAQ,CAAwB,CAAC,eAAe,KAAK,UAAU,CAAC;AACnG,CAAC;AAcD,+DAA+D;AAC/D,MAAM,UAAU,kBAAkB,CAAC,CAAkB;IACnD,OAAO,CACL,UAAU,IAAI,CAAC;QACf,OAAQ,CAAsB,CAAC,QAAQ,KAAK,UAAU;QACtD,OAAO,IAAI,CAAC;QACZ,OAAQ,CAAsB,CAAC,KAAK,KAAK,UAAU,CACpD,CAAC;AACJ,CAAC;AAsCD,qDAAqD;AACrD,MAAM,UAAU,oBAAoB,CAAC,CAAkB;IACrD,OAAO,CACL,iBAAiB,IAAI,CAAC;QACtB,OAAQ,CAAwB,CAAC,eAAe,KAAK,UAAU;QAC/D,YAAY,IAAI,CAAC;QACjB,OAAQ,CAAwB,CAAC,UAAU,KAAK,UAAU,CAC3D,CAAC;AACJ,CAAC;AA6BD,2DAA2D;AAC3D,MAAM,UAAU,oBAAoB,CAAC,CAAkB;IACrD,OAAO,CACL,gBAAgB,IAAI,CAAC;QACrB,OAAQ,CAAgC,CAAC,cAAc,KAAK,UAAU,CACvE,CAAC;AACJ,CAAC;AA8OD;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,CAAe;IAChD,OAAO,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC;AACpC,CAAC;AA+DD;;GAEG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAA2B;IACrE,iBAAiB,EAAE,IAAI;IACvB,kBAAkB,EAAE,KAAK;IACzB,gBAAgB,EAAE,CAAC;IACnB,sBAAsB,EAAE,IAAI;IAC5B,qBAAqB,EAAE,IAAI;IAC3B,oBAAoB,EAAE,KAAK;IAC3B,mBAAmB,EAAE,qMAAqM;IAC1N,iBAAiB,EAAE;;;;;;;;;0DASqC;IACxD,mBAAmB,EAAE,oCAAoC;IACzD,kBAAkB,EAAE,QAAQ;IAC5B,gBAAgB,EAAE,CAAC;IACnB,qBAAqB,EAAE,IAAI;IAC3B,oBAAoB,EAAE,eAAe;CACtC,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests that branching a chronicle correctly inherits and diverges
|
|
3
|
+
* adaptive-resolution state.
|
|
4
|
+
*
|
|
5
|
+
* Per the design doc §8 open question #4: "Branching semantics for on-chunk
|
|
6
|
+
* state. New chronicle branches inherit chunk state at the fork point via
|
|
7
|
+
* the existing branch-scoped slot model. An agent unfold on branch A does
|
|
8
|
+
* not leak into branch B."
|
|
9
|
+
*
|
|
10
|
+
* Adaptive state lives in two places:
|
|
11
|
+
* - On the StoredMessage itself (bodyGroupId, shardIndex, currentResolution,
|
|
12
|
+
* lockedByAgent — set at append time, immutable for that record)
|
|
13
|
+
* - In strategy state slots (autobio:resolutions, autobio:locks,
|
|
14
|
+
* autobio:summaries)
|
|
15
|
+
*
|
|
16
|
+
* The chronicle's branch model copy-on-writes the state slots, so the fork
|
|
17
|
+
* sees state-at-fork-point and can diverge without affecting the parent.
|
|
18
|
+
*/
|
|
19
|
+
export {};
|
|
20
|
+
//# sourceMappingURL=branching.test.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"branching.test.d.ts","sourceRoot":"","sources":["../../../test/adaptive/branching.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests that branching a chronicle correctly inherits and diverges
|
|
3
|
+
* adaptive-resolution state.
|
|
4
|
+
*
|
|
5
|
+
* Per the design doc §8 open question #4: "Branching semantics for on-chunk
|
|
6
|
+
* state. New chronicle branches inherit chunk state at the fork point via
|
|
7
|
+
* the existing branch-scoped slot model. An agent unfold on branch A does
|
|
8
|
+
* not leak into branch B."
|
|
9
|
+
*
|
|
10
|
+
* Adaptive state lives in two places:
|
|
11
|
+
* - On the StoredMessage itself (bodyGroupId, shardIndex, currentResolution,
|
|
12
|
+
* lockedByAgent — set at append time, immutable for that record)
|
|
13
|
+
* - In strategy state slots (autobio:resolutions, autobio:locks,
|
|
14
|
+
* autobio:summaries)
|
|
15
|
+
*
|
|
16
|
+
* The chronicle's branch model copy-on-writes the state slots, so the fork
|
|
17
|
+
* sees state-at-fork-point and can diverge without affecting the parent.
|
|
18
|
+
*/
|
|
19
|
+
import { describe, it, before, after, beforeEach } from 'node:test';
|
|
20
|
+
import assert from 'node:assert/strict';
|
|
21
|
+
import { rmSync, existsSync } from 'node:fs';
|
|
22
|
+
import { ContextManager, AutobiographicalStrategy } from '../../src/index.js';
|
|
23
|
+
const TEST_STORE_PATH = './test-adaptive-branching-store';
|
|
24
|
+
function cleanup() {
|
|
25
|
+
if (existsSync(TEST_STORE_PATH)) {
|
|
26
|
+
rmSync(TEST_STORE_PATH, { recursive: true, force: true });
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function makeMockMembrane() {
|
|
30
|
+
let callCount = 0;
|
|
31
|
+
return {
|
|
32
|
+
complete: async () => {
|
|
33
|
+
callCount++;
|
|
34
|
+
return {
|
|
35
|
+
content: [{ type: 'text', text: `[mock summary #${callCount}]` }],
|
|
36
|
+
};
|
|
37
|
+
},
|
|
38
|
+
get callCount() {
|
|
39
|
+
return callCount;
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
describe('AutobiographicalStrategy — branching semantics', () => {
|
|
44
|
+
before(() => cleanup());
|
|
45
|
+
after(() => cleanup());
|
|
46
|
+
beforeEach(() => cleanup());
|
|
47
|
+
it('fork inherits resolutions at fork point; divergence does not leak back', async () => {
|
|
48
|
+
const mock = makeMockMembrane();
|
|
49
|
+
const strategy = new AutobiographicalStrategy({
|
|
50
|
+
adaptiveResolution: true,
|
|
51
|
+
targetChunkTokens: 100,
|
|
52
|
+
recentWindowTokens: 200,
|
|
53
|
+
});
|
|
54
|
+
const manager = await ContextManager.open({
|
|
55
|
+
path: TEST_STORE_PATH,
|
|
56
|
+
strategy,
|
|
57
|
+
membrane: mock,
|
|
58
|
+
});
|
|
59
|
+
// Build up some history
|
|
60
|
+
const ids = [];
|
|
61
|
+
for (let i = 0; i < 20; i++) {
|
|
62
|
+
ids.push(manager.addMessage('User', [{ type: 'text', text: `Turn ${i}. ` + 'word '.repeat(40) }]));
|
|
63
|
+
}
|
|
64
|
+
while (!manager.isReady()) {
|
|
65
|
+
await manager.tick();
|
|
66
|
+
}
|
|
67
|
+
// Force some folds
|
|
68
|
+
await manager.compile({ maxTokens: 3000, reserveForResponse: 200 });
|
|
69
|
+
const parentResolutions = new Map(strategy.resolutions);
|
|
70
|
+
const parentBranchName = manager.store.currentBranch().name;
|
|
71
|
+
// Fork
|
|
72
|
+
const forkName = await manager.fork('test-fork');
|
|
73
|
+
const forkResolutions = new Map(strategy.resolutions);
|
|
74
|
+
// Fork should inherit parent's resolutions at fork point
|
|
75
|
+
assert.equal(forkResolutions.size, parentResolutions.size);
|
|
76
|
+
for (const [id, level] of parentResolutions) {
|
|
77
|
+
assert.equal(forkResolutions.get(id), level, `fork should inherit resolution for ${id}`);
|
|
78
|
+
}
|
|
79
|
+
// Diverge on the fork — add more turns, force deeper folding
|
|
80
|
+
for (let i = 20; i < 40; i++) {
|
|
81
|
+
manager.addMessage('User', [{ type: 'text', text: `Fork-only turn ${i}. ` + 'word '.repeat(40) }]);
|
|
82
|
+
}
|
|
83
|
+
while (!manager.isReady()) {
|
|
84
|
+
await manager.tick();
|
|
85
|
+
}
|
|
86
|
+
await manager.compile({ maxTokens: 2000, reserveForResponse: 200 });
|
|
87
|
+
const forkDivergedResolutions = new Map(strategy.resolutions);
|
|
88
|
+
const forkDivergedMessageCount = manager.getAllMessages().length;
|
|
89
|
+
// Switch back to parent
|
|
90
|
+
await manager.switchBranch(parentBranchName);
|
|
91
|
+
const parentResolutionsAfterSwitch = new Map(strategy.resolutions);
|
|
92
|
+
const parentMessageCount = manager.getAllMessages().length;
|
|
93
|
+
// Parent should still have its ORIGINAL state — the fork's divergence
|
|
94
|
+
// didn't leak back.
|
|
95
|
+
assert.equal(parentMessageCount, 20, 'parent should still have 20 messages');
|
|
96
|
+
assert.notEqual(parentMessageCount, forkDivergedMessageCount);
|
|
97
|
+
assert.equal(parentResolutionsAfterSwitch.size, parentResolutions.size);
|
|
98
|
+
for (const [id, level] of parentResolutions) {
|
|
99
|
+
assert.equal(parentResolutionsAfterSwitch.get(id), level, `parent ${id}: expected unchanged resolution L${level} after fork diverged`);
|
|
100
|
+
}
|
|
101
|
+
manager.close();
|
|
102
|
+
});
|
|
103
|
+
it('locks on fork do not affect parent', async () => {
|
|
104
|
+
const mock = makeMockMembrane();
|
|
105
|
+
const strategy = new AutobiographicalStrategy({
|
|
106
|
+
adaptiveResolution: true,
|
|
107
|
+
targetChunkTokens: 100,
|
|
108
|
+
});
|
|
109
|
+
const manager = await ContextManager.open({
|
|
110
|
+
path: TEST_STORE_PATH,
|
|
111
|
+
strategy,
|
|
112
|
+
membrane: mock,
|
|
113
|
+
});
|
|
114
|
+
const ids = [];
|
|
115
|
+
for (let i = 0; i < 10; i++) {
|
|
116
|
+
ids.push(manager.addMessage('User', [{ type: 'text', text: `M${i}` }]));
|
|
117
|
+
}
|
|
118
|
+
const parentBranch = manager.store.currentBranch().name;
|
|
119
|
+
await manager.fork('lock-fork');
|
|
120
|
+
// Lock on the fork
|
|
121
|
+
strategy.lockChunk(ids[3]);
|
|
122
|
+
strategy.lockChunk(ids[7]);
|
|
123
|
+
const forkLocks = new Set(strategy.locked);
|
|
124
|
+
assert.equal(forkLocks.size, 2);
|
|
125
|
+
// Switch back to parent — locks should be empty
|
|
126
|
+
await manager.switchBranch(parentBranch);
|
|
127
|
+
const parentLocks = strategy.locked;
|
|
128
|
+
assert.equal(parentLocks.size, 0, 'parent should not see fork-only locks');
|
|
129
|
+
// Switch back to fork — locks should still be there
|
|
130
|
+
await manager.switchBranch('lock-fork');
|
|
131
|
+
const forkLocksAfter = strategy.locked;
|
|
132
|
+
assert.equal(forkLocksAfter.size, 2);
|
|
133
|
+
assert.ok(forkLocksAfter.has(ids[3]));
|
|
134
|
+
assert.ok(forkLocksAfter.has(ids[7]));
|
|
135
|
+
manager.close();
|
|
136
|
+
});
|
|
137
|
+
it('summaries on fork do not affect parent', async () => {
|
|
138
|
+
const mock = makeMockMembrane();
|
|
139
|
+
const strategy = new AutobiographicalStrategy({
|
|
140
|
+
adaptiveResolution: true,
|
|
141
|
+
targetChunkTokens: 80,
|
|
142
|
+
mergeThreshold: 3,
|
|
143
|
+
recentWindowTokens: 50,
|
|
144
|
+
});
|
|
145
|
+
const manager = await ContextManager.open({
|
|
146
|
+
path: TEST_STORE_PATH,
|
|
147
|
+
strategy,
|
|
148
|
+
membrane: mock,
|
|
149
|
+
});
|
|
150
|
+
// Build enough for some summaries
|
|
151
|
+
for (let i = 0; i < 20; i++) {
|
|
152
|
+
manager.addMessage('User', [{ type: 'text', text: `T${i}. ` + 'word '.repeat(20) }]);
|
|
153
|
+
}
|
|
154
|
+
while (!manager.isReady()) {
|
|
155
|
+
await manager.tick();
|
|
156
|
+
}
|
|
157
|
+
const parentSummaryCount = strategy.summaries.length;
|
|
158
|
+
const parentBranch = manager.store.currentBranch().name;
|
|
159
|
+
// Fork and add more content → more summaries on fork only
|
|
160
|
+
await manager.fork('sum-fork');
|
|
161
|
+
for (let i = 20; i < 50; i++) {
|
|
162
|
+
manager.addMessage('User', [{ type: 'text', text: `T${i}. ` + 'word '.repeat(20) }]);
|
|
163
|
+
}
|
|
164
|
+
while (!manager.isReady()) {
|
|
165
|
+
await manager.tick();
|
|
166
|
+
}
|
|
167
|
+
const forkSummaryCount = strategy.summaries.length;
|
|
168
|
+
assert.ok(forkSummaryCount > parentSummaryCount, `fork should have more summaries (${forkSummaryCount}) than parent (${parentSummaryCount})`);
|
|
169
|
+
// Switch back to parent
|
|
170
|
+
await manager.switchBranch(parentBranch);
|
|
171
|
+
const parentSummaryCountAfter = strategy.summaries.length;
|
|
172
|
+
assert.equal(parentSummaryCountAfter, parentSummaryCount, `parent should still have ${parentSummaryCount} summaries after fork diverged`);
|
|
173
|
+
manager.close();
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
//# sourceMappingURL=branching.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"branching.test.js","sourceRoot":"","sources":["../../../test/adaptive/branching.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACpE,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAE9E,MAAM,eAAe,GAAG,iCAAiC,CAAC;AAE1D,SAAS,OAAO;IACd,IAAI,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,QAAQ,EAAE,KAAK,IAAI,EAAE;YACnB,SAAS,EAAE,CAAC;YACZ,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,SAAS,GAAG,EAAE,CAAC;aAClE,CAAC;QACJ,CAAC;QACD,IAAI,SAAS;YACX,OAAO,SAAS,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,gDAAgD,EAAE,GAAG,EAAE;IAC9D,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IACxB,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IACvB,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IAE5B,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;QACtF,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,wBAAwB,CAAC;YAC5C,kBAAkB,EAAE,IAAI;YACxB,iBAAiB,EAAE,GAAG;YACtB,kBAAkB,EAAE,GAAG;SACxB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC;YACxC,IAAI,EAAE,eAAe;YACrB,QAAQ;YACR,QAAQ,EAAE,IAAW;SACtB,CAAC,CAAC;QAEH,wBAAwB;QACxB,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACrG,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,mBAAmB;QACnB,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAE,QAAgB,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAI,OAAe,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC;QAErE,OAAO;QACP,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAE,QAAgB,CAAC,WAAW,CAAC,CAAC;QAE/D,yDAAyD;QACzD,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,iBAAiB,EAAE,CAAC;YAC5C,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,sCAAsC,EAAE,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,6DAA6D;QAC7D,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACrG,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAE,QAAgB,CAAC,WAAW,CAAC,CAAC;QACvE,MAAM,wBAAwB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC;QAEjE,wBAAwB;QACxB,MAAM,OAAO,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAC;QAC7C,MAAM,4BAA4B,GAAG,IAAI,GAAG,CAAE,QAAgB,CAAC,WAAW,CAAC,CAAC;QAC5E,MAAM,kBAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC;QAE3D,sEAAsE;QACtE,oBAAoB;QACpB,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,EAAE,sCAAsC,CAAC,CAAC;QAC7E,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE,wBAAwB,CAAC,CAAC;QAC9D,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACxE,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,iBAAiB,EAAE,CAAC;YAC5C,MAAM,CAAC,KAAK,CACV,4BAA4B,CAAC,GAAG,CAAC,EAAE,CAAC,EACpC,KAAK,EACL,UAAU,EAAE,oCAAoC,KAAK,sBAAsB,CAC5E,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,wBAAwB,CAAC;YAC5C,kBAAkB,EAAE,IAAI;YACxB,iBAAiB,EAAE,GAAG;SACvB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC;YACxC,IAAI,EAAE,eAAe;YACrB,QAAQ;YACR,QAAQ,EAAE,IAAW;SACtB,CAAC,CAAC;QACH,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,YAAY,GAAI,OAAe,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC;QACjE,MAAM,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAChC,mBAAmB;QACnB,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,SAAS,GAAG,IAAI,GAAG,CAAE,QAAgB,CAAC,MAAqB,CAAC,CAAC;QACnE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEhC,gDAAgD;QAChD,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QACzC,MAAM,WAAW,GAAI,QAAgB,CAAC,MAAqB,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,uCAAuC,CAAC,CAAC;QAE3E,oDAAoD;QACpD,MAAM,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,cAAc,GAAI,QAAgB,CAAC,MAAqB,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,wBAAwB,CAAC;YAC5C,kBAAkB,EAAE,IAAI;YACxB,iBAAiB,EAAE,EAAE;YACrB,cAAc,EAAE,CAAC;YACjB,kBAAkB,EAAE,EAAE;SACvB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC;YACxC,IAAI,EAAE,eAAe;YACrB,QAAQ;YACR,QAAQ,EAAE,IAAW;SACtB,CAAC,CAAC;QACH,kCAAkC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACvF,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,kBAAkB,GAAK,QAAgB,CAAC,SAA4B,CAAC,MAAM,CAAC;QAClF,MAAM,YAAY,GAAI,OAAe,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC;QAEjE,0DAA0D;QAC1D,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACvF,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,gBAAgB,GAAK,QAAgB,CAAC,SAA4B,CAAC,MAAM,CAAC;QAChF,MAAM,CAAC,EAAE,CAAC,gBAAgB,GAAG,kBAAkB,EAAE,oCAAoC,gBAAgB,kBAAkB,kBAAkB,GAAG,CAAC,CAAC;QAE9I,wBAAwB;QACxB,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QACzC,MAAM,uBAAuB,GAAK,QAAgB,CAAC,SAA4B,CAAC,MAAM,CAAC;QACvF,MAAM,CAAC,KAAK,CACV,uBAAuB,EACvB,kBAAkB,EAClB,4BAA4B,kBAAkB,gCAAgC,CAC/E,CAAC;QACF,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.test.d.ts","sourceRoot":"","sources":["../../../test/adaptive/chunker.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { chunkMessage, DEFAULT_CHUNKER_OPTIONS } from '../../src/adaptive/chunker.js';
|
|
4
|
+
// Use small thresholds for tests so we don't have to generate huge inputs.
|
|
5
|
+
const TINY = { chunkThreshold: 100, chunkSize: 40, charsPerToken: 1.0 };
|
|
6
|
+
test('chunker: under threshold passes through as single shard', () => {
|
|
7
|
+
const body = 'Hello world. This is a short message.';
|
|
8
|
+
const result = chunkMessage(body, TINY);
|
|
9
|
+
assert.equal(result.wasSharded, false);
|
|
10
|
+
assert.equal(result.shards.length, 1);
|
|
11
|
+
assert.equal(result.shards[0].content, body);
|
|
12
|
+
assert.equal(result.shards[0].range.startByte, 0);
|
|
13
|
+
assert.equal(result.shards[0].range.endByte, Buffer.byteLength(body, 'utf8'));
|
|
14
|
+
});
|
|
15
|
+
test('chunker: over threshold produces multiple shards with byte-faithful reassembly', () => {
|
|
16
|
+
// 12 lines of ~20 chars each ≈ 240 chars total, target 40 chars/shard → ~6 shards
|
|
17
|
+
const lines = [];
|
|
18
|
+
for (let i = 0; i < 12; i++)
|
|
19
|
+
lines.push(`Line number ${i.toString().padStart(2, '0')} text`);
|
|
20
|
+
const body = lines.join('\n');
|
|
21
|
+
const result = chunkMessage(body, TINY);
|
|
22
|
+
assert.equal(result.wasSharded, true);
|
|
23
|
+
assert.ok(result.shards.length >= 3, `expected ≥3 shards, got ${result.shards.length}`);
|
|
24
|
+
// Byte-faithful reassembly
|
|
25
|
+
const reassembled = result.shards.map((s) => s.content).join('');
|
|
26
|
+
assert.equal(reassembled, body, 'reassembled content must equal original byte-for-byte');
|
|
27
|
+
// Indices sequential from 0
|
|
28
|
+
for (let i = 0; i < result.shards.length; i++) {
|
|
29
|
+
assert.equal(result.shards[i].index, i);
|
|
30
|
+
}
|
|
31
|
+
// Byte ranges contiguous and cover the entire body
|
|
32
|
+
const bodyBytes = Buffer.byteLength(body, 'utf8');
|
|
33
|
+
let prevEnd = 0;
|
|
34
|
+
for (const s of result.shards) {
|
|
35
|
+
assert.equal(s.range.startByte, prevEnd, `gap or overlap at shard ${s.index}`);
|
|
36
|
+
prevEnd = s.range.endByte;
|
|
37
|
+
}
|
|
38
|
+
assert.equal(prevEnd, bodyBytes, 'final endByte must equal body length');
|
|
39
|
+
});
|
|
40
|
+
test('chunker: deterministic — same input produces identical shards', () => {
|
|
41
|
+
const body = 'a'.repeat(200) + '\n' + 'b'.repeat(200);
|
|
42
|
+
const r1 = chunkMessage(body, TINY);
|
|
43
|
+
const r2 = chunkMessage(body, TINY);
|
|
44
|
+
assert.equal(r1.bodyGroupId, r2.bodyGroupId);
|
|
45
|
+
assert.equal(r1.shards.length, r2.shards.length);
|
|
46
|
+
for (let i = 0; i < r1.shards.length; i++) {
|
|
47
|
+
assert.equal(r1.shards[i].sourceHash, r2.shards[i].sourceHash);
|
|
48
|
+
assert.equal(r1.shards[i].content, r2.shards[i].content);
|
|
49
|
+
assert.equal(r1.shards[i].range.startByte, r2.shards[i].range.startByte);
|
|
50
|
+
assert.equal(r1.shards[i].range.endByte, r2.shards[i].range.endByte);
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
test('chunker: prefers structural splits at markdown headings', () => {
|
|
54
|
+
const body = [
|
|
55
|
+
'## Section A',
|
|
56
|
+
'Some text in section A. '.repeat(10),
|
|
57
|
+
'',
|
|
58
|
+
'## Section B',
|
|
59
|
+
'Some text in section B. '.repeat(10),
|
|
60
|
+
'',
|
|
61
|
+
'## Section C',
|
|
62
|
+
'Some text in section C. '.repeat(10),
|
|
63
|
+
].join('\n');
|
|
64
|
+
// chunkSize chosen so ~one section fits per shard
|
|
65
|
+
const result = chunkMessage(body, { chunkThreshold: 50, chunkSize: 100, charsPerToken: 1.0 });
|
|
66
|
+
assert.equal(result.wasSharded, true);
|
|
67
|
+
// We expect splits at or near "## Section" boundaries. Verify at least one
|
|
68
|
+
// shard starts with "## Section".
|
|
69
|
+
const startsWithHeading = result.shards.filter((s, i) => i > 0 && s.content.startsWith('## Section'));
|
|
70
|
+
assert.ok(startsWithHeading.length >= 1, 'at least one mid-body shard should start at a heading');
|
|
71
|
+
});
|
|
72
|
+
test('chunker: handles long line without seams via hard split', () => {
|
|
73
|
+
// One very long line with no whitespace at all
|
|
74
|
+
const body = 'x'.repeat(500);
|
|
75
|
+
const result = chunkMessage(body, TINY);
|
|
76
|
+
assert.equal(result.wasSharded, true);
|
|
77
|
+
// Byte-faithful even with hard splits
|
|
78
|
+
assert.equal(result.shards.map((s) => s.content).join(''), body);
|
|
79
|
+
});
|
|
80
|
+
test('chunker: handles multi-byte UTF-8 (CJK) without corrupting characters', () => {
|
|
81
|
+
// Some Japanese text + emoji
|
|
82
|
+
const body = '日本語のテストです。' + '🌸'.repeat(50) + 'もう少しテキストを追加します。'.repeat(20);
|
|
83
|
+
const result = chunkMessage(body, { chunkThreshold: 30, chunkSize: 25, charsPerToken: 1.0 });
|
|
84
|
+
assert.equal(result.wasSharded, true);
|
|
85
|
+
// Reassembly must be byte-faithful
|
|
86
|
+
assert.equal(result.shards.map((s) => s.content).join(''), body);
|
|
87
|
+
// Each shard's content should round-trip through UTF-8 cleanly
|
|
88
|
+
for (const s of result.shards) {
|
|
89
|
+
const bytes = Buffer.from(s.content, 'utf8');
|
|
90
|
+
const decoded = bytes.toString('utf8');
|
|
91
|
+
assert.equal(decoded, s.content, `shard ${s.index} did not round-trip through UTF-8`);
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
test('chunker: sourceHash is stable across identical content in different runs', () => {
|
|
95
|
+
const fragment = 'This is some content that should hash identically.';
|
|
96
|
+
const r1 = chunkMessage(fragment + fragment + fragment + fragment + fragment, TINY);
|
|
97
|
+
const r2 = chunkMessage(fragment + fragment + fragment + fragment + fragment, TINY);
|
|
98
|
+
assert.equal(r1.shards[0].sourceHash, r2.shards[0].sourceHash);
|
|
99
|
+
});
|
|
100
|
+
test('chunker: bodyGroupId differs for different content', () => {
|
|
101
|
+
const r1 = chunkMessage('a'.repeat(500), TINY);
|
|
102
|
+
const r2 = chunkMessage('b'.repeat(500), TINY);
|
|
103
|
+
assert.notEqual(r1.bodyGroupId, r2.bodyGroupId);
|
|
104
|
+
});
|
|
105
|
+
test('chunker: large doc (synthetic 500K-token-equivalent) produces sensible shard count', () => {
|
|
106
|
+
// 500K tokens × 4 chars/token = 2M chars. Use a smaller synthetic stand-in to keep test fast.
|
|
107
|
+
// 50K chars at default chunkSize 4096 tokens × 4 chars = 16K chars/shard → ~3-4 shards.
|
|
108
|
+
// Use 200K chars to get ~12 shards.
|
|
109
|
+
const para = 'Paragraph of content. '.repeat(20); // ~440 chars per "paragraph"
|
|
110
|
+
const body = Array.from({ length: 500 }, () => para).join('\n\n'); // ~220K chars
|
|
111
|
+
const result = chunkMessage(body); // default options
|
|
112
|
+
assert.equal(result.wasSharded, true);
|
|
113
|
+
// Each shard should be ≤ chunkSize tokens (with some slack for seam-finding)
|
|
114
|
+
for (const s of result.shards) {
|
|
115
|
+
assert.ok(s.approxTokens <= DEFAULT_CHUNKER_OPTIONS.chunkSize * 1.3, `shard ${s.index} has ${s.approxTokens} tokens, exceeds chunkSize`);
|
|
116
|
+
}
|
|
117
|
+
// Byte-faithful
|
|
118
|
+
assert.equal(result.shards.map((s) => s.content).join(''), body);
|
|
119
|
+
});
|
|
120
|
+
//# sourceMappingURL=chunker.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.test.js","sourceRoot":"","sources":["../../../test/adaptive/chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,MAAM,MAAM,oBAAoB,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAEtF,2EAA2E;AAC3E,MAAM,IAAI,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAW,CAAC;AAEjF,IAAI,CAAC,yDAAyD,EAAE,GAAG,EAAE;IACnE,MAAM,IAAI,GAAG,uCAAuC,CAAC;IACrD,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACvC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAClD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;AAChF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gFAAgF,EAAE,GAAG,EAAE;IAC1F,kFAAkF;IAClF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAExC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,2BAA2B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAExF,2BAA2B;IAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,uDAAuD,CAAC,CAAC;IAEzF,4BAA4B;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,mDAAmD;IACnD,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAC/E,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;IAC5B,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,sCAAsC,CAAC,CAAC;AAC3E,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+DAA+D,EAAE,GAAG,EAAE;IACzE,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACzE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvE,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,GAAG,EAAE;IACnE,MAAM,IAAI,GAAG;QACX,cAAc;QACd,0BAA0B,CAAC,MAAM,CAAC,EAAE,CAAC;QACrC,EAAE;QACF,cAAc;QACd,0BAA0B,CAAC,MAAM,CAAC,EAAE,CAAC;QACrC,EAAE;QACF,cAAc;QACd,0BAA0B,CAAC,MAAM,CAAC,EAAE,CAAC;KACtC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,kDAAkD;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9F,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAEtC,2EAA2E;IAC3E,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;IACtG,MAAM,CAAC,EAAE,CAAC,iBAAiB,CAAC,MAAM,IAAI,CAAC,EAAE,uDAAuD,CAAC,CAAC;AACpG,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,GAAG,EAAE;IACnE,+CAA+C;IAC/C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtC,sCAAsC;IACtC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;AACnE,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uEAAuE,EAAE,GAAG,EAAE;IACjF,6BAA6B;IAC7B,MAAM,IAAI,GAAG,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC7F,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAEtC,mCAAmC;IACnC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAEjE,+DAA+D;IAC/D,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,KAAK,mCAAmC,CAAC,CAAC;IACxF,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0EAA0E,EAAE,GAAG,EAAE;IACpF,MAAM,QAAQ,GAAG,oDAAoD,CAAC;IACtE,MAAM,EAAE,GAAG,YAAY,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,EAAE,IAAI,CAAC,CAAC;IACpF,MAAM,EAAE,GAAG,YAAY,CAAC,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,EAAE,IAAI,CAAC,CAAC;IACpF,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AACjE,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,GAAG,EAAE;IAC9D,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/C,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;AAClD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oFAAoF,EAAE,GAAG,EAAE;IAC9F,8FAA8F;IAC9F,wFAAwF;IACxF,oCAAoC;IACpC,MAAM,IAAI,GAAG,wBAAwB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,6BAA6B;IAC/E,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc;IACjF,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB;IACrD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtC,6EAA6E;IAC7E,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,CAAC,EAAE,CACP,CAAC,CAAC,YAAY,IAAI,uBAAuB,CAAC,SAAS,GAAG,GAAG,EACzD,SAAS,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,YAAY,4BAA4B,CACnE,CAAC;IACJ,CAAC;IACD,gBAAgB;IAChB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;AACnE,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests that the bottom-up speculative pre-producer (config.speculativeProduction)
|
|
3
|
+
* builds the full L_n chain — not just L2/L3 like the legacy threshold path.
|
|
4
|
+
*
|
|
5
|
+
* Drives enough conversation to produce ≥ N^2 L1s, then verifies the chronicle
|
|
6
|
+
* ends up with L2, L3, L4, ... summaries via the same checkMergeThreshold
|
|
7
|
+
* mechanism (now recursive) that previously stopped at L3.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=deep-levels.test.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deep-levels.test.d.ts","sourceRoot":"","sources":["../../../test/adaptive/deep-levels.test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}
|