@noetic-tools/context 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/cache-anchoring.d.ts +103 -0
- package/dist/context/cache-anchoring.js +273 -0
- package/dist/context/layer-lifecycle.d.ts +3 -5
- package/dist/context/layer-lifecycle.js +12 -2
- package/dist/context/layer-usage.d.ts +18 -3
- package/dist/context/layer-usage.js +10 -1
- package/dist/context/layers/file-reference.js +57 -0
- package/dist/context/layers/static-content.d.ts +1 -0
- package/dist/context/layers/static-content.js +3 -0
- package/dist/context/layers/steering.d.ts +1 -0
- package/dist/context/layers/steering.js +5 -0
- package/dist/context/layers/temporal-context.js +6 -0
- package/dist/context/projector.d.ts +27 -8
- package/dist/context/projector.js +42 -13
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import type { AnchorPin, ContextCacheConfig, ContextCacheStore, ContextEpoch, ContextLayer, ExecutionContext, LayerChurn, LLMResponse, ReanchorReason } from '@noetic-tools/types';
|
|
2
|
+
export type { AnchorPin, ContextCacheConfig, ContextCacheStore, ContextEpoch, LayerChurn, ReanchorReason, } from '@noetic-tools/types';
|
|
3
|
+
/** Defaults for prompt-cache anchoring. See `ContextCacheConfig` for what each one does. */
|
|
4
|
+
export declare const DEFAULT_CONTEXT_CACHE: Required<ContextCacheConfig>;
|
|
5
|
+
/** Fills a partial config with the defaults. */
|
|
6
|
+
export declare function resolveCacheConfig(config?: ContextCacheConfig): Required<ContextCacheConfig>;
|
|
7
|
+
/**
|
|
8
|
+
* The run of assemblies that share a cacheable prompt prefix.
|
|
9
|
+
*
|
|
10
|
+
* A thread at top level, because its turns build one growing conversation. A
|
|
11
|
+
* child execution keys on itself instead: it inherits its parent's `threadId`
|
|
12
|
+
* but assembles a different view, so sharing pins would replay a prefix that was
|
|
13
|
+
* never sent on its behalf.
|
|
14
|
+
*/
|
|
15
|
+
export declare function lineageKey(ctx: ExecutionContext): string;
|
|
16
|
+
/**
|
|
17
|
+
* Where a layer's pin lives within its epoch.
|
|
18
|
+
*
|
|
19
|
+
* The layer id alone, deliberately — the epoch is already scoped to one cache
|
|
20
|
+
* lineage, and layer ids are unique within a layer set. Folding the layer's
|
|
21
|
+
* `scope` into the key (as the eventual-recall cache does, where one map spans
|
|
22
|
+
* every thread) would rotate it every turn for an `'execution'`-scoped layer,
|
|
23
|
+
* so each turn would retract the "old" pin and add a "new" one for the very
|
|
24
|
+
* same layer — republishing its content in full, forever.
|
|
25
|
+
*/
|
|
26
|
+
export declare function pinKey(layer: ContextLayer): string;
|
|
27
|
+
export declare function createContextCacheStore(): ContextCacheStore;
|
|
28
|
+
/** Drop a lineage's epoch — used when a child execution ends. */
|
|
29
|
+
export declare function dropLineage(store: ContextCacheStore, key: string): void;
|
|
30
|
+
/** Read a layer's churn record, creating an empty one on first sight. */
|
|
31
|
+
export declare function churnFor(store: ContextCacheStore, key: string): LayerChurn;
|
|
32
|
+
/** Share of watched assemblies in which a layer's output changed, 0–1. */
|
|
33
|
+
export declare function churnRate(churn: LayerChurn): number;
|
|
34
|
+
interface OpenEpochParams {
|
|
35
|
+
store: ContextCacheStore;
|
|
36
|
+
key: string;
|
|
37
|
+
reason: ReanchorReason;
|
|
38
|
+
instructionsHash: string;
|
|
39
|
+
layers: ReadonlyArray<ContextLayer>;
|
|
40
|
+
config: Required<ContextCacheConfig>;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Start a fresh epoch for a lineage, dropping every pin so the next assembly
|
|
44
|
+
* re-anchors against current recall output.
|
|
45
|
+
*
|
|
46
|
+
* This is the only point at which `'auto'` layers change band: placement is held
|
|
47
|
+
* still inside an epoch so the prefix cannot shift under the model mid-run.
|
|
48
|
+
* Churn carries across, decayed, so a layer's band is not relearnt from nothing
|
|
49
|
+
* each time.
|
|
50
|
+
*/
|
|
51
|
+
export declare function openEpoch({ store, key, reason, instructionsHash, layers, config, }: OpenEpochParams): ContextEpoch;
|
|
52
|
+
/**
|
|
53
|
+
* Record a lineage as most recently used.
|
|
54
|
+
*
|
|
55
|
+
* `Map` keeps first-insertion order, so re-setting an existing key would leave
|
|
56
|
+
* it where it was and let eviction drop the busiest lineage. Deleting first
|
|
57
|
+
* moves it to the end, which is what makes eviction least-recently-used.
|
|
58
|
+
*/
|
|
59
|
+
export declare function touchLineage(store: ContextCacheStore, key: string, epoch?: ContextEpoch): void;
|
|
60
|
+
/**
|
|
61
|
+
* Why the coming assembly must re-anchor, or `undefined` to carry the epoch on.
|
|
62
|
+
* Ordered cheapest-check first; delta pressure is decided later, once the
|
|
63
|
+
* supersedes are actually built.
|
|
64
|
+
*/
|
|
65
|
+
export declare function reanchorReason({ epoch, key, instructionsHash, store, config, }: {
|
|
66
|
+
epoch: ContextEpoch | undefined;
|
|
67
|
+
key: string;
|
|
68
|
+
instructionsHash: string;
|
|
69
|
+
store: ContextCacheStore;
|
|
70
|
+
config: Required<ContextCacheConfig>;
|
|
71
|
+
}): ReanchorReason | undefined;
|
|
72
|
+
interface NoteCacheOutcomeParams {
|
|
73
|
+
store: ContextCacheStore;
|
|
74
|
+
key: string;
|
|
75
|
+
response: LLMResponse;
|
|
76
|
+
config: Required<ContextCacheConfig>;
|
|
77
|
+
/** Tokens the prefix was expected to have cached — system plus anchor band. */
|
|
78
|
+
expectedTokens: number;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Judge, from the model's own token report, whether the prompt prefix survived,
|
|
82
|
+
* and record a re-anchor for the next assembly if it did not.
|
|
83
|
+
*
|
|
84
|
+
* Records intent only; pins are never touched here, so a response that steering
|
|
85
|
+
* later rejects cannot leave the epoch half-rebuilt.
|
|
86
|
+
*
|
|
87
|
+
* Three things stop this eating itself:
|
|
88
|
+
*
|
|
89
|
+
* - Only the first round counts. Later rounds replay the same view plus tool
|
|
90
|
+
* traffic and so hit the cache whatever the first round did — summing them
|
|
91
|
+
* hides a total miss behind a busy tool loop.
|
|
92
|
+
* - Young epochs are spared. The assembly right after a re-anchor writes the
|
|
93
|
+
* cache rather than reading it, so its near-zero read is expected, not a miss.
|
|
94
|
+
* - A provider that reports nothing, or misses persistently, is marked blind and
|
|
95
|
+
* stops being consulted. Age and delta pressure still bound the epoch.
|
|
96
|
+
*/
|
|
97
|
+
export declare function noteCacheOutcome({ store, key, response, config, expectedTokens, }: NoteCacheOutcomeParams): void;
|
|
98
|
+
/**
|
|
99
|
+
* Record a layer's rendered output as the bytes to replay for the rest of the
|
|
100
|
+
* epoch. The caller sets `epoch.anchorTokens` from the assembled band, which is
|
|
101
|
+
* measured the same way as the supersedes it gets compared against.
|
|
102
|
+
*/
|
|
103
|
+
export declare function pin(epoch: ContextEpoch, key: string, entry: AnchorPin): void;
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
//#region Config
|
|
2
|
+
/** Defaults for prompt-cache anchoring. See `ContextCacheConfig` for what each one does. */
|
|
3
|
+
export const DEFAULT_CONTEXT_CACHE = {
|
|
4
|
+
enabled: true,
|
|
5
|
+
minCachedTokens: 100,
|
|
6
|
+
minEpochAssemblies: 2,
|
|
7
|
+
maxEpochAssemblies: 50,
|
|
8
|
+
deltaBudgetFraction: 0.15,
|
|
9
|
+
autoDemoteChurn: 0.5,
|
|
10
|
+
autoPromoteChurn: 0.2,
|
|
11
|
+
minChurnSamples: 3,
|
|
12
|
+
churnDecay: 0.5,
|
|
13
|
+
};
|
|
14
|
+
/** Fills a partial config with the defaults. */
|
|
15
|
+
export function resolveCacheConfig(config) {
|
|
16
|
+
return {
|
|
17
|
+
...DEFAULT_CONTEXT_CACHE,
|
|
18
|
+
...config,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Judged assemblies that may come back a miss before the runtime stops steering
|
|
23
|
+
* on the provider's cache figures. A provider that never caches would otherwise
|
|
24
|
+
* re-anchor on every turn forever, for no gain.
|
|
25
|
+
*/
|
|
26
|
+
const MAX_CONSECUTIVE_MISSES = 3;
|
|
27
|
+
/** Lineages retained per harness before the least recently used is dropped. */
|
|
28
|
+
const MAX_LINEAGES = 256;
|
|
29
|
+
//#endregion
|
|
30
|
+
//#region Keys
|
|
31
|
+
/**
|
|
32
|
+
* The run of assemblies that share a cacheable prompt prefix.
|
|
33
|
+
*
|
|
34
|
+
* A thread at top level, because its turns build one growing conversation. A
|
|
35
|
+
* child execution keys on itself instead: it inherits its parent's `threadId`
|
|
36
|
+
* but assembles a different view, so sharing pins would replay a prefix that was
|
|
37
|
+
* never sent on its behalf.
|
|
38
|
+
*/
|
|
39
|
+
export function lineageKey(ctx) {
|
|
40
|
+
return ctx.depth === 0 ? `t:${ctx.threadId}` : `x:${ctx.executionId}`;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Where a layer's pin lives within its epoch.
|
|
44
|
+
*
|
|
45
|
+
* The layer id alone, deliberately — the epoch is already scoped to one cache
|
|
46
|
+
* lineage, and layer ids are unique within a layer set. Folding the layer's
|
|
47
|
+
* `scope` into the key (as the eventual-recall cache does, where one map spans
|
|
48
|
+
* every thread) would rotate it every turn for an `'execution'`-scoped layer,
|
|
49
|
+
* so each turn would retract the "old" pin and add a "new" one for the very
|
|
50
|
+
* same layer — republishing its content in full, forever.
|
|
51
|
+
*/
|
|
52
|
+
export function pinKey(layer) {
|
|
53
|
+
return layer.id;
|
|
54
|
+
}
|
|
55
|
+
//#endregion
|
|
56
|
+
//#region Store
|
|
57
|
+
export function createContextCacheStore() {
|
|
58
|
+
return {
|
|
59
|
+
epochs: new Map(),
|
|
60
|
+
churn: new Map(),
|
|
61
|
+
pendingReanchor: new Map(),
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/** Drop a lineage's epoch — used when a child execution ends. */
|
|
65
|
+
export function dropLineage(store, key) {
|
|
66
|
+
store.epochs.delete(key);
|
|
67
|
+
store.pendingReanchor.delete(key);
|
|
68
|
+
}
|
|
69
|
+
/** Read a layer's churn record, creating an empty one on first sight. */
|
|
70
|
+
export function churnFor(store, key) {
|
|
71
|
+
const found = store.churn.get(key);
|
|
72
|
+
if (found) {
|
|
73
|
+
return found;
|
|
74
|
+
}
|
|
75
|
+
const fresh = {
|
|
76
|
+
observed: 0,
|
|
77
|
+
changed: 0,
|
|
78
|
+
rebillTokens: 0,
|
|
79
|
+
};
|
|
80
|
+
store.churn.set(key, fresh);
|
|
81
|
+
return fresh;
|
|
82
|
+
}
|
|
83
|
+
/** Share of watched assemblies in which a layer's output changed, 0–1. */
|
|
84
|
+
export function churnRate(churn) {
|
|
85
|
+
return churn.observed === 0 ? 0 : churn.changed / churn.observed;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Start a fresh epoch for a lineage, dropping every pin so the next assembly
|
|
89
|
+
* re-anchors against current recall output.
|
|
90
|
+
*
|
|
91
|
+
* This is the only point at which `'auto'` layers change band: placement is held
|
|
92
|
+
* still inside an epoch so the prefix cannot shift under the model mid-run.
|
|
93
|
+
* Churn carries across, decayed, so a layer's band is not relearnt from nothing
|
|
94
|
+
* each time.
|
|
95
|
+
*/
|
|
96
|
+
export function openEpoch({ store, key, reason, instructionsHash, layers, config, }) {
|
|
97
|
+
const previous = store.epochs.get(key);
|
|
98
|
+
const generation = previous ? Number(previous.id.split('#')[1] ?? 0) + 1 : 0;
|
|
99
|
+
const autoBand = new Map();
|
|
100
|
+
for (const layer of layers) {
|
|
101
|
+
const k = pinKey(layer);
|
|
102
|
+
autoBand.set(k, chooseBand({
|
|
103
|
+
store,
|
|
104
|
+
key: k,
|
|
105
|
+
current: previous?.autoBand.get(k),
|
|
106
|
+
config,
|
|
107
|
+
}));
|
|
108
|
+
}
|
|
109
|
+
decayChurn({
|
|
110
|
+
store,
|
|
111
|
+
layers,
|
|
112
|
+
decay: config.churnDecay,
|
|
113
|
+
});
|
|
114
|
+
const epoch = {
|
|
115
|
+
id: `${key}#${generation}`,
|
|
116
|
+
pins: new Map(),
|
|
117
|
+
autoBand,
|
|
118
|
+
instructionsHash,
|
|
119
|
+
assemblies: 0,
|
|
120
|
+
anchorTokens: 0,
|
|
121
|
+
deltaTokens: 0,
|
|
122
|
+
// A provider that has already shown it reports nothing stays untrusted
|
|
123
|
+
// across epochs; there is no reason to expect it to start.
|
|
124
|
+
cacheBlind: previous?.cacheBlind ?? false,
|
|
125
|
+
// Misses carry across epochs. A miss is what CAUSED this re-anchor, so
|
|
126
|
+
// resetting here would make the give-up threshold unreachable and leave a
|
|
127
|
+
// never-caching provider re-anchoring on every turn forever.
|
|
128
|
+
misses: previous?.misses ?? 0,
|
|
129
|
+
lastReanchorReason: reason,
|
|
130
|
+
};
|
|
131
|
+
store.pendingReanchor.delete(key);
|
|
132
|
+
touchLineage(store, key, epoch);
|
|
133
|
+
evictOldestLineages(store);
|
|
134
|
+
return epoch;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Record a lineage as most recently used.
|
|
138
|
+
*
|
|
139
|
+
* `Map` keeps first-insertion order, so re-setting an existing key would leave
|
|
140
|
+
* it where it was and let eviction drop the busiest lineage. Deleting first
|
|
141
|
+
* moves it to the end, which is what makes eviction least-recently-used.
|
|
142
|
+
*/
|
|
143
|
+
export function touchLineage(store, key, epoch) {
|
|
144
|
+
const current = epoch ?? store.epochs.get(key);
|
|
145
|
+
if (!current) {
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
store.epochs.delete(key);
|
|
149
|
+
store.epochs.set(key, current);
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* The band an `'auto'` layer takes for the coming epoch.
|
|
153
|
+
*
|
|
154
|
+
* The promote and demote thresholds leave a gap on purpose: a layer sitting
|
|
155
|
+
* between them keeps the band it already had, so one hovering near the boundary
|
|
156
|
+
* does not flip every epoch and undo the stability the bands exist to provide.
|
|
157
|
+
*/
|
|
158
|
+
function chooseBand({ store, key, current, config, }) {
|
|
159
|
+
const churn = store.churn.get(key);
|
|
160
|
+
const band = current ?? 'anchor';
|
|
161
|
+
if (!churn || churn.observed < config.minChurnSamples) {
|
|
162
|
+
return band;
|
|
163
|
+
}
|
|
164
|
+
const rate = churnRate(churn);
|
|
165
|
+
if (rate >= config.autoDemoteChurn) {
|
|
166
|
+
return 'live';
|
|
167
|
+
}
|
|
168
|
+
if (rate <= config.autoPromoteChurn) {
|
|
169
|
+
return 'anchor';
|
|
170
|
+
}
|
|
171
|
+
return band;
|
|
172
|
+
}
|
|
173
|
+
function decayChurn({ store, layers, decay, }) {
|
|
174
|
+
for (const layer of layers) {
|
|
175
|
+
const churn = store.churn.get(pinKey(layer));
|
|
176
|
+
if (!churn) {
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
churn.observed *= decay;
|
|
180
|
+
churn.changed *= decay;
|
|
181
|
+
churn.rebillTokens *= decay;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/** Keep the lineage map bounded on a long-lived harness that spawns many children. */
|
|
185
|
+
function evictOldestLineages(store) {
|
|
186
|
+
while (store.epochs.size > MAX_LINEAGES) {
|
|
187
|
+
const oldest = store.epochs.keys().next();
|
|
188
|
+
if (oldest.done) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
dropLineage(store, oldest.value);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Why the coming assembly must re-anchor, or `undefined` to carry the epoch on.
|
|
196
|
+
* Ordered cheapest-check first; delta pressure is decided later, once the
|
|
197
|
+
* supersedes are actually built.
|
|
198
|
+
*/
|
|
199
|
+
export function reanchorReason({ epoch, key, instructionsHash, store, config, }) {
|
|
200
|
+
if (!epoch) {
|
|
201
|
+
return 'cold-start';
|
|
202
|
+
}
|
|
203
|
+
if (epoch.instructionsHash !== instructionsHash) {
|
|
204
|
+
return 'instructions-changed';
|
|
205
|
+
}
|
|
206
|
+
const pending = store.pendingReanchor.get(key);
|
|
207
|
+
if (pending) {
|
|
208
|
+
return pending;
|
|
209
|
+
}
|
|
210
|
+
if (epoch.assemblies >= config.maxEpochAssemblies) {
|
|
211
|
+
return 'max-age';
|
|
212
|
+
}
|
|
213
|
+
return undefined;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Judge, from the model's own token report, whether the prompt prefix survived,
|
|
217
|
+
* and record a re-anchor for the next assembly if it did not.
|
|
218
|
+
*
|
|
219
|
+
* Records intent only; pins are never touched here, so a response that steering
|
|
220
|
+
* later rejects cannot leave the epoch half-rebuilt.
|
|
221
|
+
*
|
|
222
|
+
* Three things stop this eating itself:
|
|
223
|
+
*
|
|
224
|
+
* - Only the first round counts. Later rounds replay the same view plus tool
|
|
225
|
+
* traffic and so hit the cache whatever the first round did — summing them
|
|
226
|
+
* hides a total miss behind a busy tool loop.
|
|
227
|
+
* - Young epochs are spared. The assembly right after a re-anchor writes the
|
|
228
|
+
* cache rather than reading it, so its near-zero read is expected, not a miss.
|
|
229
|
+
* - A provider that reports nothing, or misses persistently, is marked blind and
|
|
230
|
+
* stops being consulted. Age and delta pressure still bound the epoch.
|
|
231
|
+
*/
|
|
232
|
+
export function noteCacheOutcome({ store, key, response, config, expectedTokens, }) {
|
|
233
|
+
const epoch = store.epochs.get(key);
|
|
234
|
+
if (!epoch || epoch.cacheBlind) {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
const round = response.rounds?.[0] ?? response.usage;
|
|
238
|
+
if (!round || round.cachedTokens === undefined) {
|
|
239
|
+
epoch.cacheBlind = true;
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
// Reads of zero alongside a write mean the prefix was just stored, not missed.
|
|
243
|
+
if ((round.cacheWriteTokens ?? 0) > 0 && round.cachedTokens === 0) {
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
if (epoch.assemblies < config.minEpochAssemblies) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
// Judge against what there actually was to cache: a short prompt can never
|
|
250
|
+
// reach a fixed floor, and holding it to one would re-anchor forever.
|
|
251
|
+
const floor = Math.min(config.minCachedTokens, expectedTokens * 0.5);
|
|
252
|
+
if (round.cachedTokens >= floor) {
|
|
253
|
+
epoch.misses = 0;
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
epoch.misses += 1;
|
|
257
|
+
if (epoch.misses >= MAX_CONSECUTIVE_MISSES) {
|
|
258
|
+
epoch.cacheBlind = true;
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
store.pendingReanchor.set(key, 'cache-miss');
|
|
262
|
+
}
|
|
263
|
+
//#endregion
|
|
264
|
+
//#region Pins
|
|
265
|
+
/**
|
|
266
|
+
* Record a layer's rendered output as the bytes to replay for the rest of the
|
|
267
|
+
* epoch. The caller sets `epoch.anchorTokens` from the assembled band, which is
|
|
268
|
+
* measured the same way as the supersedes it gets compared against.
|
|
269
|
+
*/
|
|
270
|
+
export function pin(epoch, key, entry) {
|
|
271
|
+
epoch.pins.set(key, entry);
|
|
272
|
+
}
|
|
273
|
+
//#endregion
|
|
@@ -172,15 +172,13 @@ export interface RecallCache {
|
|
|
172
172
|
export declare function createRecallCache(): RecallCache;
|
|
173
173
|
export declare function mostRestrictive(decisions: SteeringDecision[]): SteeringDecision;
|
|
174
174
|
export declare function initLayers({ layers, ctx, storage, store }: InitLayersParams): Promise<void>;
|
|
175
|
-
export declare function recallLayers({ layers, query, ctx, log, budgets, store, itemSchemas, }: RecallLayersParams): Promise<
|
|
176
|
-
layerId: string;
|
|
177
|
-
items: Item[];
|
|
178
|
-
tokenCount: number;
|
|
179
|
-
}[]>;
|
|
175
|
+
export declare function recallLayers({ layers, query, ctx, log, budgets, store, itemSchemas, }: RecallLayersParams): Promise<RecallEntry[]>;
|
|
180
176
|
type RecallEntry = {
|
|
181
177
|
layerId: string;
|
|
182
178
|
items: Item[];
|
|
183
179
|
tokenCount: number;
|
|
180
|
+
/** See `RecallLayerOutput.mutatedState` — set when `recall` returned new state. */
|
|
181
|
+
mutatedState?: boolean;
|
|
184
182
|
};
|
|
185
183
|
interface RecallLayersModeParams extends RecallLayersParams {
|
|
186
184
|
/** When true, treat every layer as atomic regardless of its `recallMode`. */
|
|
@@ -278,12 +278,16 @@ export async function recallLayers({ layers, query, ctx, log, budgets, store, it
|
|
|
278
278
|
});
|
|
279
279
|
continue;
|
|
280
280
|
}
|
|
281
|
+
// Returning state means the render changed something, so it cannot be
|
|
282
|
+
// replayed from a pin — flag it so view assembly never anchors it.
|
|
283
|
+
const mutatedState = result.state !== undefined;
|
|
281
284
|
results.push({
|
|
282
285
|
layerId: layer.id,
|
|
283
286
|
items: layerSchemas.parseMany(result.items),
|
|
284
287
|
tokenCount: result.tokenCount,
|
|
288
|
+
mutatedState,
|
|
285
289
|
});
|
|
286
|
-
if (
|
|
290
|
+
if (mutatedState) {
|
|
287
291
|
store.set(ctx.executionId, layer.id, result.state);
|
|
288
292
|
}
|
|
289
293
|
}
|
|
@@ -340,7 +344,13 @@ export async function recallLayersEventual({ cache, forceAtomic, ...params }) {
|
|
|
340
344
|
cache.stale.delete(key);
|
|
341
345
|
continue;
|
|
342
346
|
}
|
|
343
|
-
|
|
347
|
+
// Serving from cache commits nothing, so the flag from the render that
|
|
348
|
+
// populated it must not ride along — it would force the layer out of the
|
|
349
|
+
// anchor band for good, on turns where its recall never even ran.
|
|
350
|
+
results.push({
|
|
351
|
+
...cached,
|
|
352
|
+
mutatedState: false,
|
|
353
|
+
});
|
|
344
354
|
}
|
|
345
355
|
return results;
|
|
346
356
|
}
|
|
@@ -1,13 +1,28 @@
|
|
|
1
|
-
import type { Context, LastLayerUsage, RecallLayerOutput, Tool } from '@noetic-tools/types';
|
|
1
|
+
import type { Context, EpochUsage, LastLayerUsage, RecallLayerOutput, Tool } from '@noetic-tools/types';
|
|
2
|
+
/** How a layer's output was banded and pinned for this assembly. */
|
|
3
|
+
export interface LayerServeInfo {
|
|
4
|
+
placement: 'anchor' | 'live';
|
|
5
|
+
served: 'fresh' | 'pinned';
|
|
6
|
+
changed: boolean;
|
|
7
|
+
churnRate: number;
|
|
8
|
+
rebillTokens: number;
|
|
9
|
+
}
|
|
2
10
|
interface ComputeLayerUsageParams {
|
|
3
11
|
ctx: Context;
|
|
4
12
|
modelId: string;
|
|
5
13
|
instructions?: string;
|
|
6
14
|
tools?: ReadonlyArray<Tool>;
|
|
7
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* Per-layer output as it was ACTUALLY served into the view — pinned replays
|
|
17
|
+
* included. Passing raw recall output here would report content the model
|
|
18
|
+
* never saw.
|
|
19
|
+
*/
|
|
8
20
|
recallResults: ReadonlyArray<RecallLayerOutput>;
|
|
21
|
+
/** Banding and pin status per layer id. Absent when context caching is off. */
|
|
22
|
+
serveInfo?: ReadonlyMap<string, LayerServeInfo>;
|
|
23
|
+
epoch?: EpochUsage;
|
|
9
24
|
}
|
|
10
|
-
export declare function computeLayerUsage({ ctx, modelId, instructions, tools, recallResults, }: ComputeLayerUsageParams): LastLayerUsage;
|
|
25
|
+
export declare function computeLayerUsage({ ctx, modelId, instructions, tools, recallResults, serveInfo, epoch, }: ComputeLayerUsageParams): LastLayerUsage;
|
|
11
26
|
/** Commit computed usage to the Context, replacing any prior snapshot. */
|
|
12
27
|
export declare function commitLayerUsage(ctx: Context, usage: LastLayerUsage): void;
|
|
13
28
|
export {};
|
|
@@ -6,6 +6,13 @@ function canWriteLayerUsage(ctx) {
|
|
|
6
6
|
}
|
|
7
7
|
return desc.writable !== false;
|
|
8
8
|
}
|
|
9
|
+
const SERVED_FRESH_ANCHOR = {
|
|
10
|
+
placement: 'anchor',
|
|
11
|
+
served: 'fresh',
|
|
12
|
+
changed: false,
|
|
13
|
+
churnRate: 0,
|
|
14
|
+
rebillTokens: 0,
|
|
15
|
+
};
|
|
9
16
|
//#endregion
|
|
10
17
|
//#region Helpers
|
|
11
18
|
function estimateJsonTokens(value) {
|
|
@@ -18,12 +25,13 @@ function estimateJsonTokens(value) {
|
|
|
18
25
|
}
|
|
19
26
|
//#endregion
|
|
20
27
|
//#region Public API
|
|
21
|
-
export function computeLayerUsage({ ctx, modelId, instructions, tools, recallResults, }) {
|
|
28
|
+
export function computeLayerUsage({ ctx, modelId, instructions, tools, recallResults, serveInfo, epoch, }) {
|
|
22
29
|
const layers = recallResults
|
|
23
30
|
.map((r) => ({
|
|
24
31
|
layerId: r.layerId,
|
|
25
32
|
tokenCount: r.tokenCount,
|
|
26
33
|
items: r.items,
|
|
34
|
+
...(serveInfo?.get(r.layerId) ?? SERVED_FRESH_ANCHOR),
|
|
27
35
|
}))
|
|
28
36
|
.sort((a, b) => a.layerId.localeCompare(b.layerId));
|
|
29
37
|
let historyTokens = 0;
|
|
@@ -42,6 +50,7 @@ export function computeLayerUsage({ ctx, modelId, instructions, tools, recallRes
|
|
|
42
50
|
toolsTokens,
|
|
43
51
|
historyTokens,
|
|
44
52
|
totalUsedTokens,
|
|
53
|
+
epoch,
|
|
45
54
|
};
|
|
46
55
|
}
|
|
47
56
|
/** Commit computed usage to the Context, replacing any prior snapshot. */
|
|
@@ -688,6 +688,59 @@ async function recallFileReferences({ state, budget, }) {
|
|
|
688
688
|
tokenCount: estimateTokens(content),
|
|
689
689
|
};
|
|
690
690
|
}
|
|
691
|
+
/**
|
|
692
|
+
* Splits a rendered `# Referenced Files` block back into one entry per file,
|
|
693
|
+
* keyed by the `## <path>` heading.
|
|
694
|
+
*/
|
|
695
|
+
function splitFileBlocks(items) {
|
|
696
|
+
const blocks = new Map();
|
|
697
|
+
for (const item of items) {
|
|
698
|
+
if (item.type !== 'message' || !('content' in item)) {
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
for (const part of item.content ?? []) {
|
|
702
|
+
if (!('text' in part) || typeof part.text !== 'string') {
|
|
703
|
+
continue;
|
|
704
|
+
}
|
|
705
|
+
for (const chunk of part.text.split(/\n(?=## )/)) {
|
|
706
|
+
const heading = chunk.match(/^## (.+)$/m);
|
|
707
|
+
if (heading?.[1]) {
|
|
708
|
+
blocks.set(heading[1].trim(), chunk.trim());
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return blocks;
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* Describes what changed about the tracked files rather than re-sending all of
|
|
717
|
+
* them. The whole set is usually large and usually stable, so a change to one
|
|
718
|
+
* file would otherwise cost a full republish.
|
|
719
|
+
*/
|
|
720
|
+
async function renderFileReferenceDelta({ prev, next, }) {
|
|
721
|
+
const before = splitFileBlocks(prev);
|
|
722
|
+
const after = splitFileBlocks(next);
|
|
723
|
+
const changed = [];
|
|
724
|
+
for (const [path, block] of after) {
|
|
725
|
+
if (before.get(path) !== block) {
|
|
726
|
+
changed.push(block);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
const dropped = [
|
|
730
|
+
...before.keys(),
|
|
731
|
+
].filter((path) => !after.has(path));
|
|
732
|
+
if (changed.length === 0 && dropped.length === 0) {
|
|
733
|
+
return null;
|
|
734
|
+
}
|
|
735
|
+
const parts = [];
|
|
736
|
+
if (changed.length > 0) {
|
|
737
|
+
parts.push(changed.join('\n\n'));
|
|
738
|
+
}
|
|
739
|
+
if (dropped.length > 0) {
|
|
740
|
+
parts.push(`No longer referenced: ${dropped.join(', ')}`);
|
|
741
|
+
}
|
|
742
|
+
return parts.join('\n\n');
|
|
743
|
+
}
|
|
691
744
|
//#endregion
|
|
692
745
|
//#region Layer Factory
|
|
693
746
|
/**
|
|
@@ -713,6 +766,9 @@ export function fileReference(opts) {
|
|
|
713
766
|
scope: 'thread',
|
|
714
767
|
budget: 'auto',
|
|
715
768
|
rerenderTiming: 'immediate',
|
|
769
|
+
// A large payload that changes rarely, and only a file at a time — the case
|
|
770
|
+
// anchoring plus a compact supersede is worth the most on.
|
|
771
|
+
placement: 'anchor',
|
|
716
772
|
timeouts: {
|
|
717
773
|
// onItemAppend reads files AND runs an LLM scoring call per new
|
|
718
774
|
// reference (parallelized) — the 5s pipeline default silently drops the
|
|
@@ -723,6 +779,7 @@ export function fileReference(opts) {
|
|
|
723
779
|
init: () => initFileReferenceState(runtime.baseDir),
|
|
724
780
|
onItemAppend: (args) => onFileReferenceItemAppend(args, runtime),
|
|
725
781
|
recall: recallFileReferences,
|
|
782
|
+
renderDelta: renderFileReferenceDelta,
|
|
726
783
|
},
|
|
727
784
|
};
|
|
728
785
|
}
|
|
@@ -12,6 +12,9 @@ export function staticContent(opts) {
|
|
|
12
12
|
id: opts.id ?? 'static-content',
|
|
13
13
|
slot: opts.slot ?? Slot.WORKING_MEMORY + 5,
|
|
14
14
|
scope: opts.scope ?? 'resource',
|
|
15
|
+
// Loaded once in `init` and never rewritten, so it can be pinned outright
|
|
16
|
+
// rather than waiting for churn telemetry to work that out.
|
|
17
|
+
placement: 'anchor',
|
|
15
18
|
hooks: {
|
|
16
19
|
async init() {
|
|
17
20
|
const raw = await opts.load();
|
|
@@ -162,6 +162,11 @@ export function steering(config) {
|
|
|
162
162
|
name: 'Steering',
|
|
163
163
|
slot: Slot.STEERING,
|
|
164
164
|
scope,
|
|
165
|
+
// `recall` drains the pending queue as it renders, so its output can never
|
|
166
|
+
// be replayed from a pin — the drain is already committed, and serving an
|
|
167
|
+
// older render would lose the feedback for good. Rendering after history
|
|
168
|
+
// also puts guidance where the model weighs it most.
|
|
169
|
+
placement: 'live',
|
|
165
170
|
budget: {
|
|
166
171
|
min: 0,
|
|
167
172
|
max: 500,
|
|
@@ -211,6 +211,12 @@ export function temporalContext(config) {
|
|
|
211
211
|
// Grounding sits near the top of the window, before reasoning content.
|
|
212
212
|
slot: Slot.REMINDER,
|
|
213
213
|
scope: config?.scope ?? 'resource',
|
|
214
|
+
// The `<current_datetime>` block changes every turn by construction, so
|
|
215
|
+
// anchoring it would re-bill the whole window each time. Churn telemetry
|
|
216
|
+
// would reach the same verdict — this just skips the learning cost. With
|
|
217
|
+
// grounding off, only the slow-moving fact ledger remains, which is worth
|
|
218
|
+
// anchoring, so leave that to `'auto'`.
|
|
219
|
+
placement: groundDateTime ? 'live' : 'auto',
|
|
214
220
|
budget: {
|
|
215
221
|
min: 0,
|
|
216
222
|
max: injectLedger ? 800 : 200,
|
|
@@ -1,19 +1,38 @@
|
|
|
1
1
|
import type { Item, ProjectionPolicy } from '@noetic-tools/types';
|
|
2
2
|
interface AssembleViewParams {
|
|
3
3
|
systemPromptItems: Item[];
|
|
4
|
+
/** Anchor band — layer output rendered before history, where a prompt cache can hold it. */
|
|
4
5
|
layerOutputItems: Item[];
|
|
5
6
|
historyItems: Item[];
|
|
7
|
+
/** Live band — layer output rendered after history, slot-ascending. */
|
|
8
|
+
liveLayerItems?: Item[];
|
|
9
|
+
/** Supersedes for anchored layers whose pinned output went stale. */
|
|
10
|
+
deltaItems?: Item[];
|
|
11
|
+
/** Items that must land last, after everything else. Never dropped. */
|
|
12
|
+
tailItems?: Item[];
|
|
6
13
|
policy?: ProjectionPolicy;
|
|
7
14
|
}
|
|
8
15
|
/**
|
|
9
|
-
* Assemble the model's context window
|
|
10
|
-
* recall output (slot-ascending), and conversation history.
|
|
16
|
+
* Assemble the model's context window in bands:
|
|
11
17
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
18
|
+
* ```
|
|
19
|
+
* system | anchor layers | history | live layers | supersedes | tail
|
|
20
|
+
* ```
|
|
21
|
+
*
|
|
22
|
+
* The split exists for the prompt cache, which matches on a prefix. Stable
|
|
23
|
+
* layer output sits ahead of history where it can be cached; volatile output
|
|
24
|
+
* sits after it, where re-rendering costs almost nothing. Both bands arrive
|
|
25
|
+
* slot-ascending.
|
|
26
|
+
*
|
|
27
|
+
* Without a `policy` the bands are concatenated as-is (optionally sliding the
|
|
28
|
+
* history window by `windowSize`). With a `policy` the view is held to a hard
|
|
29
|
+
* token budget, claimed in this order: system items, then anchor output, then
|
|
30
|
+
* live output, then supersedes, then the tail — with history taking whatever
|
|
31
|
+
* remains and keeping the most recent turns.
|
|
32
|
+
*
|
|
33
|
+
* Supersedes are claimed ahead of history, and all together or not at all: a
|
|
34
|
+
* dropped supersede would leave the model reading a pinned block the runtime
|
|
35
|
+
* knows is stale. The caller re-anchors instead — see `prepareBandedView`.
|
|
17
36
|
*/
|
|
18
|
-
export declare function assembleView({ systemPromptItems, layerOutputItems, historyItems, policy, }: AssembleViewParams): Item[];
|
|
37
|
+
export declare function assembleView({ systemPromptItems, layerOutputItems, historyItems, liveLayerItems, deltaItems, tailItems, policy, }: AssembleViewParams): Item[];
|
|
19
38
|
export {};
|
|
@@ -57,33 +57,62 @@ function keepRecentWithinBudget(items, budget, windowSize) {
|
|
|
57
57
|
//#endregion
|
|
58
58
|
//#region Public API
|
|
59
59
|
/**
|
|
60
|
-
* Assemble the model's context window
|
|
61
|
-
* recall output (slot-ascending), and conversation history.
|
|
60
|
+
* Assemble the model's context window in bands:
|
|
62
61
|
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
62
|
+
* ```
|
|
63
|
+
* system | anchor layers | history | live layers | supersedes | tail
|
|
64
|
+
* ```
|
|
65
|
+
*
|
|
66
|
+
* The split exists for the prompt cache, which matches on a prefix. Stable
|
|
67
|
+
* layer output sits ahead of history where it can be cached; volatile output
|
|
68
|
+
* sits after it, where re-rendering costs almost nothing. Both bands arrive
|
|
69
|
+
* slot-ascending.
|
|
70
|
+
*
|
|
71
|
+
* Without a `policy` the bands are concatenated as-is (optionally sliding the
|
|
72
|
+
* history window by `windowSize`). With a `policy` the view is held to a hard
|
|
73
|
+
* token budget, claimed in this order: system items, then anchor output, then
|
|
74
|
+
* live output, then supersedes, then the tail — with history taking whatever
|
|
75
|
+
* remains and keeping the most recent turns.
|
|
76
|
+
*
|
|
77
|
+
* Supersedes are claimed ahead of history, and all together or not at all: a
|
|
78
|
+
* dropped supersede would leave the model reading a pinned block the runtime
|
|
79
|
+
* knows is stale. The caller re-anchors instead — see `prepareBandedView`.
|
|
68
80
|
*/
|
|
69
|
-
export function assembleView({ systemPromptItems, layerOutputItems, historyItems, policy, }) {
|
|
81
|
+
export function assembleView({ systemPromptItems, layerOutputItems, historyItems, liveLayerItems = [], deltaItems = [], tailItems = [], policy, }) {
|
|
70
82
|
if (!policy) {
|
|
71
83
|
return [
|
|
72
84
|
...systemPromptItems,
|
|
73
85
|
...layerOutputItems,
|
|
74
86
|
...historyItems,
|
|
87
|
+
...liveLayerItems,
|
|
88
|
+
...deltaItems,
|
|
89
|
+
...tailItems,
|
|
75
90
|
];
|
|
76
91
|
}
|
|
77
92
|
const budget = Math.max(0, policy.tokenBudget - policy.responseReserve);
|
|
78
93
|
// System items are never dropped — they anchor the conversation.
|
|
79
|
-
|
|
80
|
-
const
|
|
81
|
-
|
|
82
|
-
const
|
|
94
|
+
let left = Math.max(0, budget - totalTokens(systemPromptItems));
|
|
95
|
+
const keptAnchor = keepFrontWithinBudget(layerOutputItems, left);
|
|
96
|
+
left = Math.max(0, left - totalTokens(keptAnchor));
|
|
97
|
+
const keptLive = keepFrontWithinBudget(liveLayerItems, left);
|
|
98
|
+
left = Math.max(0, left - totalTokens(keptLive));
|
|
99
|
+
// The tail carries steering guidance, which must reach the model whatever the
|
|
100
|
+
// budget says — it is the correction the retry exists to deliver.
|
|
101
|
+
left = Math.max(0, left - totalTokens(tailItems));
|
|
102
|
+
// Supersedes are never dropped either. Each one corrects a pinned block that
|
|
103
|
+
// is already in the view, so dropping one leaves the model reading content
|
|
104
|
+
// the runtime knows is stale — silent corruption, and far worse than losing
|
|
105
|
+
// a turn of history. History absorbs the cost; `deltaBudgetFraction` keeps
|
|
106
|
+
// the supersedes from growing large enough for that to hurt.
|
|
107
|
+
left = Math.max(0, left - totalTokens(deltaItems));
|
|
108
|
+
const keptHistory = keepRecentWithinBudget(historyItems, left, policy.windowSize);
|
|
83
109
|
return [
|
|
84
110
|
...systemPromptItems,
|
|
85
|
-
...
|
|
111
|
+
...keptAnchor,
|
|
86
112
|
...keptHistory,
|
|
113
|
+
...keptLive,
|
|
114
|
+
...deltaItems,
|
|
115
|
+
...tailItems,
|
|
87
116
|
];
|
|
88
117
|
}
|
|
89
118
|
//#endregion
|
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
export type { ContextScope, ExecutionContext, ExecutionOutcome, LayerCallModelRequest, } from '@noetic-tools/types';
|
|
7
7
|
export * from '@noetic-tools/types/contract';
|
|
8
8
|
export * from './context/budget.js';
|
|
9
|
+
export * from './context/cache-anchoring.js';
|
|
9
10
|
export * from './context/exec-context-factory.js';
|
|
10
11
|
export * from './context/function-call-utils.js';
|
|
11
12
|
export * from './context/layer-api.js';
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// @noetic-tools/context remains the one-stop import for context-layer authoring.
|
|
9
9
|
export * from '@noetic-tools/types/contract';
|
|
10
10
|
export * from './context/budget.js';
|
|
11
|
+
export * from './context/cache-anchoring.js';
|
|
11
12
|
export * from './context/exec-context-factory.js';
|
|
12
13
|
export * from './context/function-call-utils.js';
|
|
13
14
|
export * from './context/layer-api.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@noetic-tools/context",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Context layer system for Noetic agents: the ContextLayer contract, lifecycle, budget/projection machinery, and built-in layer implementations.",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"zod": "^4.0.0"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@noetic-tools/types": "^3.
|
|
47
|
+
"@noetic-tools/types": "^3.1.0",
|
|
48
48
|
"zod": "^4.0.0"
|
|
49
49
|
},
|
|
50
50
|
"scripts": {
|