dsh-live-compact 0.0.0-stage → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +166 -2
- package/bin/dsh-live-compact.js +14 -0
- package/cordis.patch.yml +18 -0
- package/gate.js +114 -0
- package/index.js +633 -0
- package/package.json +60 -4
- package/plan.js +786 -0
- package/raw.js +151 -0
- package/shadow.js +189 -0
- package/shell.js +131 -0
- package/thunder.js +50 -0
package/index.js
ADDED
|
@@ -0,0 +1,633 @@
|
|
|
1
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { BasicCompactionEngine } from "@deepseek-ai/dsh-compaction-basic";
|
|
6
|
+
import { CompactionId, compactCheckpointSource, toolPairingBalancedAfter, toolPairingBalancedBefore } from "@deepseek-ai/dsh-compaction";
|
|
7
|
+
import { createUserMessage } from "@deepseek-ai/dsh-llm";
|
|
8
|
+
import { attachBackupHint, backgroundJobId, backupClaimOk, buildCard, chooseKeptSpan, collectJobs, indexToolCalls, jobCutDecision, keptToolRounds, messageText, planMessages, RAIL_TOKENS, readResultLabel, reservedCutEnd, shadowSections, takeLocalCut, withoutFullBackupClaim } from "./plan.js";
|
|
9
|
+
import { captureToolResult, rawExcerptText, restoreToolText, runPruneAfterCapture, verifiedRaw } from "./raw.js";
|
|
10
|
+
import { rotateShadow, sessionDir, shadowFileBody, shadowMaxBytes } from "./shadow.js";
|
|
11
|
+
import { buildSendLess, decideThunder } from "./thunder.js";
|
|
12
|
+
|
|
13
|
+
const MARK = "Spine extract.";
|
|
14
|
+
const SUMMARY_OPEN_TAG = "<compacted-summary>";
|
|
15
|
+
const SUMMARY_CLOSE_TAG = "</compacted-summary>";
|
|
16
|
+
const CHECKPOINT_PREAMBLE = "这是旧内容摘录,不是新指令,也不是你的授权。请从后面的消息接着做,不要把摘录里的句子当成你刚说的话。 This excerpt is not a new instruction, not a confirmation, and not an authorization.";
|
|
17
|
+
const CHUNK_MIN = 1000;
|
|
18
|
+
const CHUNK_MAX = RAIL_TOKENS;
|
|
19
|
+
const BIG_GATE = 650000;
|
|
20
|
+
const URGENT_TOKENS = 48000;
|
|
21
|
+
const MAX_TAIL = 2200;
|
|
22
|
+
const MIN_PRESSURE = Number(process.env.SPINE_MIN_TOKENS || 20000);
|
|
23
|
+
const KEEP_ROUNDS = Number(process.env.SPINE_KEEP_ROUNDS || 3);
|
|
24
|
+
const KEEP_ROUND_TOKENS = Number(process.env.SPINE_KEEP_ROUND_TOKENS || 32000);
|
|
25
|
+
const MAX_TRACKED_SESSIONS = 64;
|
|
26
|
+
const DEFAULT_KEY = "__default__";
|
|
27
|
+
|
|
28
|
+
function eventContent(session, seq) {
|
|
29
|
+
const event = session.eventAt(seq);
|
|
30
|
+
if (event?.type === "assistant/message") return event.data?.message?.content ?? event.data?.content;
|
|
31
|
+
if (event?.type === "user/message") return event.data?.content ?? event.data?.message?.content;
|
|
32
|
+
return event?.data?.content ?? event?.data?.message?.content ?? event?.data;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function eventText(session, seq) {
|
|
36
|
+
const event = session.eventAt(seq);
|
|
37
|
+
if (event?.type === "user/message") return messageText(event.data);
|
|
38
|
+
if (event?.type === "assistant/message") return messageText(event.data?.message);
|
|
39
|
+
return messageText(event?.data);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function isCheckpoint(session, seq) {
|
|
43
|
+
const event = session.eventAt(seq);
|
|
44
|
+
if (event?.data?.source?.kind === "compact-checkpoint") return true;
|
|
45
|
+
return eventText(session, seq).includes(MARK);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function eventRole(session, seq) {
|
|
49
|
+
const type = session.eventAt(seq)?.type;
|
|
50
|
+
if (type === "user/message") return "user";
|
|
51
|
+
if (type === "assistant/message") return "assistant";
|
|
52
|
+
if (type === "tool/result") return "tool";
|
|
53
|
+
return "other";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function isUser(session, seq) {
|
|
57
|
+
return session.eventAt(seq)?.type === "user/message";
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function cutOk(fn, session, seq) {
|
|
61
|
+
try {
|
|
62
|
+
return fn(session, seq);
|
|
63
|
+
} catch {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function shadowDirectory() {
|
|
69
|
+
if (process.env.SPINE_SHADOW_DIR) return process.env.SPINE_SHADOW_DIR;
|
|
70
|
+
const root = process.env.DSH_HOME || join(homedir(), ".dsh");
|
|
71
|
+
return join(root, "compaction-spine", "shadow");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function sessionKey(agent) {
|
|
75
|
+
const id = agent?.session?.id;
|
|
76
|
+
return typeof id === "string" && id.length > 0 ? id : DEFAULT_KEY;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function tightenShadow(dir) {
|
|
80
|
+
try {
|
|
81
|
+
chmodSync(dir, 0o700);
|
|
82
|
+
} catch {}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function shadowBody(messages) {
|
|
86
|
+
return (messages || []).map((message) => shadowSections(message)).join("\n\n");
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function freshShadowFile(kind, key) {
|
|
90
|
+
const dir = sessionDir(shadowDirectory(), key);
|
|
91
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
92
|
+
return join(dir, `${kind}-${Date.now()}-${randomBytes(4).toString("hex")}.txt`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function commandFromCall(event) {
|
|
96
|
+
if (event?.type !== "tool/call") return "";
|
|
97
|
+
let args = event.data?.arguments;
|
|
98
|
+
if (typeof args === "string") {
|
|
99
|
+
try {
|
|
100
|
+
args = JSON.parse(args);
|
|
101
|
+
} catch {
|
|
102
|
+
args = {};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const command = args?.command || args?.cmd || event.data?.name || "";
|
|
106
|
+
return String(command || "").split("\n")[0].trim();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function shadowFilePath(requested, key) {
|
|
110
|
+
if (requested) return requested;
|
|
111
|
+
return freshShadowFile("shadow", key);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function selectNewestChunk(session, measurement, key = DEFAULT_KEY, context = {}) {
|
|
115
|
+
const decision = jobCutDecision(context.jobs || []);
|
|
116
|
+
if (!decision.cut) return null;
|
|
117
|
+
const nodes = session.surface.nodes;
|
|
118
|
+
const priced = measurement.nodes;
|
|
119
|
+
if (nodes.length === 0 || nodes.length !== priced.length) return null;
|
|
120
|
+
const first = session.eventAt(nodes[0])?.type === "system/message" ? 1 : 0;
|
|
121
|
+
let end = priced.length - 1;
|
|
122
|
+
while (end >= first) {
|
|
123
|
+
const event = session.eventAt(nodes[end]);
|
|
124
|
+
const shortTail = event?.type === "assistant/message" || (event?.type === "user/message" && priced[end].tokens < 400);
|
|
125
|
+
if (!shortTail) break;
|
|
126
|
+
end -= 1;
|
|
127
|
+
}
|
|
128
|
+
while (end >= first && isCheckpoint(session, nodes[end])) end -= 1;
|
|
129
|
+
while (end >= first && !cutOk(toolPairingBalancedAfter, session, nodes[end])) end -= 1;
|
|
130
|
+
let keptTokens = 0;
|
|
131
|
+
const visible = [];
|
|
132
|
+
for (let index = first; index <= end; index += 1) {
|
|
133
|
+
const event = session.eventAt(nodes[index]);
|
|
134
|
+
const toolCalls = event?.type === "assistant/message"
|
|
135
|
+
? (event.data?.message?.content || []).filter((block) => block.type === "tool-call").length
|
|
136
|
+
: 0;
|
|
137
|
+
visible.push({ index, type: event?.type === "tool/result" ? "tool" : "other", toolCalls });
|
|
138
|
+
}
|
|
139
|
+
const rounds = keptToolRounds(visible, KEEP_ROUNDS);
|
|
140
|
+
const span = chooseKeptSpan(rounds, (from, to) => {
|
|
141
|
+
let tokens = 0;
|
|
142
|
+
for (let index = visible[from.start].index; index <= visible[to.end].index; index += 1) tokens += priced[index].tokens;
|
|
143
|
+
return tokens;
|
|
144
|
+
}, KEEP_ROUND_TOKENS);
|
|
145
|
+
const cutEnd = span ? reservedCutEnd(visible[span.start].index, first) : null;
|
|
146
|
+
if (span && cutEnd === null) return null;
|
|
147
|
+
if (cutEnd !== null) {
|
|
148
|
+
const keptEnd = visible[span.end].index;
|
|
149
|
+
for (let index = visible[span.start].index; index <= keptEnd; index += 1) keptTokens += priced[index].tokens;
|
|
150
|
+
end = cutEnd;
|
|
151
|
+
while (end >= first && (isCheckpoint(session, nodes[end]) || isUser(session, nodes[end]))) end -= 1;
|
|
152
|
+
while (end >= first && !cutOk(toolPairingBalancedAfter, session, nodes[end])) end -= 1;
|
|
153
|
+
}
|
|
154
|
+
if (end < first || isCheckpoint(session, nodes[end])) return null;
|
|
155
|
+
if (priced[end].tokens > CHUNK_MAX) return null;
|
|
156
|
+
const spanPieces = (spanStart, spanEnd) => {
|
|
157
|
+
const pieces = [];
|
|
158
|
+
let pendingCommand = "";
|
|
159
|
+
for (let index = 0; index < nodes.length; index += 1) {
|
|
160
|
+
const event = session.eventAt(nodes[index]);
|
|
161
|
+
const command = commandFromCall(event);
|
|
162
|
+
if (command) pendingCommand = command;
|
|
163
|
+
if (index < spanStart || index > spanEnd) continue;
|
|
164
|
+
const toolCallId = event?.data?.message?.toolCallId || "";
|
|
165
|
+
const call = toolCallId ? context.calls?.get?.(toolCallId) : null;
|
|
166
|
+
const isTool = event?.type === "tool/result";
|
|
167
|
+
const looked = isTool && context.rawLookup && toolCallId ? (context.rawLookup(toolCallId) || { ok: false, toolCallId }) : null;
|
|
168
|
+
const excerpt = isTool ? rawExcerptText(eventText(session, nodes[index]), looked) : { text: eventText(session, nodes[index]) };
|
|
169
|
+
const text = excerpt.text;
|
|
170
|
+
pieces.push({
|
|
171
|
+
role: eventRole(session, nodes[index]),
|
|
172
|
+
text,
|
|
173
|
+
isError: Boolean(event?.data?.message?.isError || event?.data?.isError),
|
|
174
|
+
command: call?.command || (event?.type === "tool/result" ? pendingCommand : ""),
|
|
175
|
+
name: call?.name || "",
|
|
176
|
+
toolCallId,
|
|
177
|
+
turn: Number.isInteger(event?.data?.turn) ? event.data.turn : call?.turn,
|
|
178
|
+
backgroundStart: event?.type === "tool/result" && Boolean(backgroundJobId(text)),
|
|
179
|
+
raw: looked ? (excerpt.raw?.ok ? excerpt.raw : { ok: false, toolCallId }) : undefined,
|
|
180
|
+
});
|
|
181
|
+
if (event?.type === "tool/result") pendingCommand = "";
|
|
182
|
+
}
|
|
183
|
+
return pieces;
|
|
184
|
+
};
|
|
185
|
+
const spanText = (spanStart, spanEnd) => spanPieces(spanStart, spanEnd).map((piece) => piece.text).join("\n");
|
|
186
|
+
const liveText = (spanStart, spanEnd) => {
|
|
187
|
+
const bits = [];
|
|
188
|
+
for (let index = 0; index < nodes.length; index += 1) {
|
|
189
|
+
if (index >= spanStart && index <= spanEnd) continue;
|
|
190
|
+
bits.push(eventText(session, nodes[index]));
|
|
191
|
+
}
|
|
192
|
+
return bits.join("\n");
|
|
193
|
+
};
|
|
194
|
+
const extras = { jobs: context.jobs || [] };
|
|
195
|
+
let start = end;
|
|
196
|
+
let tokens = priced[end].tokens;
|
|
197
|
+
let card = buildCard(spanText(start, end), liveText(start, end), spanPieces(start, end), extras);
|
|
198
|
+
if (card.ok !== true) return null;
|
|
199
|
+
while (start > first) {
|
|
200
|
+
const prev = start - 1;
|
|
201
|
+
if (isCheckpoint(session, nodes[prev]) || isUser(session, nodes[prev])) break;
|
|
202
|
+
const nextTokens = tokens + priced[prev].tokens;
|
|
203
|
+
if (nextTokens > CHUNK_MAX) break;
|
|
204
|
+
const trial = buildCard(spanText(prev, end), liveText(prev, end), spanPieces(prev, end), extras);
|
|
205
|
+
if (trial.ok !== true) break;
|
|
206
|
+
tokens = nextTokens;
|
|
207
|
+
start = prev;
|
|
208
|
+
card = trial;
|
|
209
|
+
}
|
|
210
|
+
while (start <= end && (isCheckpoint(session, nodes[start]) || !cutOk(toolPairingBalancedBefore, session, nodes[start]))) {
|
|
211
|
+
tokens -= priced[start].tokens;
|
|
212
|
+
start += 1;
|
|
213
|
+
}
|
|
214
|
+
if (start > end || tokens < CHUNK_MIN || isCheckpoint(session, nodes[start])) return null;
|
|
215
|
+
card = attachBackupHint(buildCard(spanText(start, end), liveText(start, end), spanPieces(start, end), extras), freshShadowFile("cut", key));
|
|
216
|
+
if (card.ok !== true) return null;
|
|
217
|
+
if (!cutOk(toolPairingBalancedAfter, session, nodes[end])) return null;
|
|
218
|
+
let after = 0;
|
|
219
|
+
for (let index = end + 1; index < priced.length; index += 1) after += priced[index].tokens;
|
|
220
|
+
return { start: nodes[start], end: nodes[end], tokens, after, keptTokens, card };
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export { planMessages };
|
|
224
|
+
|
|
225
|
+
export default class SpineCompactionEngine extends BasicCompactionEngine {
|
|
226
|
+
// Per-session state. One engine instance can serve several agents, so nothing
|
|
227
|
+
// here may live on `this` directly or one agent's card/prefix leaks into another.
|
|
228
|
+
sessionState = new Map();
|
|
229
|
+
shadowFullWarned = new Set();
|
|
230
|
+
rawSeen = new Map();
|
|
231
|
+
pruneGuarded = false;
|
|
232
|
+
|
|
233
|
+
constructor(ctx, config) {
|
|
234
|
+
if (process.env.SPINE_ENGINE_FAIL === "construct") throw new Error("spine: constructor refused");
|
|
235
|
+
super(ctx, config);
|
|
236
|
+
if (process.env.SPINE_ENGINE_FAIL === "after-super") throw new Error("spine: after super");
|
|
237
|
+
try {
|
|
238
|
+
const pendingCuts = new Set();
|
|
239
|
+
ctx.effect(() => async () => {
|
|
240
|
+
await Promise.allSettled(pendingCuts);
|
|
241
|
+
});
|
|
242
|
+
ctx.on("session/event", (session, event) => {
|
|
243
|
+
if (event?.type !== "tool/result") return;
|
|
244
|
+
this.captureOne(session, event);
|
|
245
|
+
});
|
|
246
|
+
ctx.on("agent/status", ({ agent, status }) => {
|
|
247
|
+
if (status !== "idle") return;
|
|
248
|
+
const state = this.stateFor(agent);
|
|
249
|
+
if (state.warming) return;
|
|
250
|
+
const done = agent.session.snapshotEvents().some((event) => event.type === "turn/end" && event.data?.reason?.kind === "completed");
|
|
251
|
+
if (!done) return;
|
|
252
|
+
state.warming = true;
|
|
253
|
+
const job = this.finishTurn(agent).finally(() => {
|
|
254
|
+
state.warming = false;
|
|
255
|
+
pendingCuts.delete(job);
|
|
256
|
+
});
|
|
257
|
+
pendingCuts.add(job);
|
|
258
|
+
});
|
|
259
|
+
} catch (error) {
|
|
260
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
261
|
+
ctx.logger?.warn?.(`spine: 本会话未接管自动小切。${message}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
stateFor(agent) {
|
|
266
|
+
const key = sessionKey(agent);
|
|
267
|
+
let state = this.sessionState.get(key);
|
|
268
|
+
if (state) {
|
|
269
|
+
this.sessionState.delete(key);
|
|
270
|
+
this.sessionState.set(key, state);
|
|
271
|
+
return state;
|
|
272
|
+
}
|
|
273
|
+
state = {
|
|
274
|
+
key,
|
|
275
|
+
liveAgent: null,
|
|
276
|
+
warming: false,
|
|
277
|
+
localOnly: false,
|
|
278
|
+
approvedCard: null,
|
|
279
|
+
lastPrefix: "",
|
|
280
|
+
lastPrefixTokens: 0,
|
|
281
|
+
};
|
|
282
|
+
this.sessionState.set(key, state);
|
|
283
|
+
if (this.sessionState.size > MAX_TRACKED_SESSIONS) {
|
|
284
|
+
for (const [oldKey, old] of this.sessionState) {
|
|
285
|
+
if (this.sessionState.size <= MAX_TRACKED_SESSIONS) break;
|
|
286
|
+
if (oldKey === key || old.warming || old.localOnly) continue;
|
|
287
|
+
this.sessionState.delete(oldKey);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return state;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
chunkPlan(session, chunk) {
|
|
294
|
+
const messages = [];
|
|
295
|
+
let on = false;
|
|
296
|
+
for (const seq of session.surface.nodes) {
|
|
297
|
+
if (seq === chunk.start) on = true;
|
|
298
|
+
if (!on) continue;
|
|
299
|
+
messages.push({ role: eventRole(session, seq), content: eventText(session, seq) });
|
|
300
|
+
if (seq === chunk.end) break;
|
|
301
|
+
}
|
|
302
|
+
return planMessages(messages);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
async finishTurn(agent) {
|
|
306
|
+
const state = this.stateFor(agent);
|
|
307
|
+
const signal = AbortSignal.timeout(30000);
|
|
308
|
+
try {
|
|
309
|
+
await agent.runMaintenance(async (agentSignal) => {
|
|
310
|
+
const op = AbortSignal.any([agentSignal, signal]);
|
|
311
|
+
const measurement = this.ctx.tokenMeter.measure(agent.session);
|
|
312
|
+
if (measurement.totalTokens < MIN_PRESSURE || measurement.totalTokens >= BIG_GATE) return;
|
|
313
|
+
this.ensurePruneGuard();
|
|
314
|
+
this.captureSession(agent.session);
|
|
315
|
+
const events = agent.session.snapshotEvents();
|
|
316
|
+
const chunk = selectNewestChunk(agent.session, measurement, state.key, {
|
|
317
|
+
jobs: collectJobs(events),
|
|
318
|
+
calls: indexToolCalls(events),
|
|
319
|
+
rawLookup: (id) => this.rawSeen.get(`${state.key}\0${id}`),
|
|
320
|
+
});
|
|
321
|
+
if (chunk === null) {
|
|
322
|
+
this.ctx.logger?.debug?.("spine cut skipped");
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
const tailBeyondKept = chunk.after - (chunk.keptTokens || 0);
|
|
326
|
+
if (tailBeyondKept > MAX_TAIL && measurement.totalTokens < URGENT_TOKENS) return;
|
|
327
|
+
const before = measurement.totalTokens;
|
|
328
|
+
const started = Date.now();
|
|
329
|
+
let reduced = false;
|
|
330
|
+
try {
|
|
331
|
+
state.localOnly = true;
|
|
332
|
+
state.approvedCard = chunk.card;
|
|
333
|
+
await this.commitSelectedSpan(agent, chunk, op);
|
|
334
|
+
const after = this.ctx.tokenMeter.measure(agent.session).totalTokens;
|
|
335
|
+
reduced = after < before;
|
|
336
|
+
this.ctx.logger?.info?.(`spine cut ${Date.now() - started}ms ${before} -> ${after} chunk=${chunk.tokens}`);
|
|
337
|
+
} catch (error) {
|
|
338
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
339
|
+
this.ctx.logger?.warn?.(`spine cut failed: ${message}`);
|
|
340
|
+
}
|
|
341
|
+
if (!reduced) {
|
|
342
|
+
this.ctx.logger?.warn?.(`spine cut did not reduce ${before}; leaving the prefix for the gate`);
|
|
343
|
+
}
|
|
344
|
+
await this.ctx.sessions?.flush?.(agent.session);
|
|
345
|
+
this.rememberPrefix(agent);
|
|
346
|
+
});
|
|
347
|
+
} catch (error) {
|
|
348
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
349
|
+
this.ctx.logger?.warn?.(`spine after-turn skipped: ${message}`);
|
|
350
|
+
} finally {
|
|
351
|
+
state.localOnly = false;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
async commitSelectedSpan(agent, chunk, signal) {
|
|
356
|
+
signal?.throwIfAborted();
|
|
357
|
+
const session = agent.session;
|
|
358
|
+
const nodes = session.surface.nodes;
|
|
359
|
+
const startIdx = nodes.indexOf(chunk.start);
|
|
360
|
+
const endIdx = nodes.indexOf(chunk.end);
|
|
361
|
+
if (startIdx < 0 || endIdx < startIdx) throw new Error("spine: selected span is not on the surface");
|
|
362
|
+
if (chunk.card?.ok !== true || !chunk.card.text) throw new Error("spine: approved card missing; leaving the span");
|
|
363
|
+
const shadowedSeqs = nodes.slice(startIdx, endIdx + 1);
|
|
364
|
+
const messages = [];
|
|
365
|
+
for (const seq of shadowedSeqs) {
|
|
366
|
+
const event = session.eventAt(seq);
|
|
367
|
+
messages.push({
|
|
368
|
+
role: eventRole(session, seq),
|
|
369
|
+
content: eventContent(session, seq),
|
|
370
|
+
toolCallId: event?.data?.message?.toolCallId || "",
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
const summarized = await this.summarize({ messages, liveCompact: { card: chunk.card } }, agent, signal);
|
|
374
|
+
const compactionId = CompactionId(randomUUID());
|
|
375
|
+
const checkpointMessage = createUserMessage({
|
|
376
|
+
content: [
|
|
377
|
+
{ type: "text", text: `${CHECKPOINT_PREAMBLE}\n\n${SUMMARY_OPEN_TAG}` },
|
|
378
|
+
...summarized.summary,
|
|
379
|
+
{ type: "text", text: SUMMARY_CLOSE_TAG },
|
|
380
|
+
],
|
|
381
|
+
source: compactCheckpointSource(compactionId),
|
|
382
|
+
});
|
|
383
|
+
const framedTokens = this.ctx.tokenMeter.estimateMessage?.(checkpointMessage);
|
|
384
|
+
if (typeof framedTokens === "number" && framedTokens >= chunk.tokens) {
|
|
385
|
+
throw new Error(`summary is not smaller than the shadowed content (${framedTokens} >= ${chunk.tokens})`);
|
|
386
|
+
}
|
|
387
|
+
const lifecycle = { compactionId, turn: null };
|
|
388
|
+
let started = false;
|
|
389
|
+
let replaced = false;
|
|
390
|
+
let ended = false;
|
|
391
|
+
try {
|
|
392
|
+
const startEvent = session.append("compaction/start", lifecycle);
|
|
393
|
+
started = true;
|
|
394
|
+
const summaryEvent = session.append("compaction/summary", {
|
|
395
|
+
compactionId,
|
|
396
|
+
summary: summarized.summary,
|
|
397
|
+
...summarized.rawOutput === undefined ? {} : { rawOutput: summarized.rawOutput },
|
|
398
|
+
shadowedRange: { start: chunk.start, end: chunk.end },
|
|
399
|
+
shadowedSeqs: [...shadowedSeqs],
|
|
400
|
+
shadowedTokenCount: chunk.tokens,
|
|
401
|
+
provider: summarized.provider,
|
|
402
|
+
model: summarized.model,
|
|
403
|
+
});
|
|
404
|
+
session.append("user/message", checkpointMessage, {
|
|
405
|
+
surfaceOp: { op: "replace", startSeq: chunk.start, endSeq: chunk.end },
|
|
406
|
+
sourceEventSeqs: [startEvent.seq, summaryEvent.seq, ...shadowedSeqs],
|
|
407
|
+
});
|
|
408
|
+
replaced = true;
|
|
409
|
+
session.append("compaction/end", lifecycle);
|
|
410
|
+
ended = true;
|
|
411
|
+
await this.ctx.sessions.flush(session);
|
|
412
|
+
} catch (error) {
|
|
413
|
+
if (started && !ended) {
|
|
414
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
415
|
+
try {
|
|
416
|
+
session.append("compaction/end", replaced ? lifecycle : { ...lifecycle, error: message });
|
|
417
|
+
} catch {}
|
|
418
|
+
}
|
|
419
|
+
if (!replaced) throw error;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
rememberPrefix(agent) {
|
|
424
|
+
const state = this.stateFor(agent);
|
|
425
|
+
try {
|
|
426
|
+
const messages = agent.session.deriveMessages();
|
|
427
|
+
state.lastPrefix = (messages || []).map(messageText).join("\n");
|
|
428
|
+
state.lastPrefixTokens = this.ctx.tokenMeter.measure(agent.session).totalTokens;
|
|
429
|
+
} catch {}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
async compactIfNeeded(agent, trigger, signal) {
|
|
433
|
+
const state = this.stateFor(agent);
|
|
434
|
+
state.liveAgent = agent;
|
|
435
|
+
state.localOnly = false;
|
|
436
|
+
this.ensurePruneGuard();
|
|
437
|
+
return super.compactIfNeeded(agent, trigger, signal);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
rawKey(session, toolCallId) {
|
|
441
|
+
return `${session?.id || DEFAULT_KEY}\0${toolCallId}`;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
captureOne(session, event) {
|
|
445
|
+
const id = event?.data?.message?.toolCallId;
|
|
446
|
+
if (!id) return null;
|
|
447
|
+
const key = session?.id || DEFAULT_KEY;
|
|
448
|
+
const mapKey = this.rawKey(session, id);
|
|
449
|
+
if (this.rawSeen.get(mapKey)?.ok) return this.rawSeen.get(mapKey);
|
|
450
|
+
let call = null;
|
|
451
|
+
try {
|
|
452
|
+
call = indexToolCalls(session.snapshotEvents?.() || []).get(id);
|
|
453
|
+
} catch {}
|
|
454
|
+
try {
|
|
455
|
+
const dir = sessionDir(shadowDirectory(), key);
|
|
456
|
+
try {
|
|
457
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
458
|
+
writeFileSync(join(dir, "owner"), String(process.pid), { mode: 0o600 });
|
|
459
|
+
} catch {}
|
|
460
|
+
const record = captureToolResult({
|
|
461
|
+
dir,
|
|
462
|
+
toolCallId: id,
|
|
463
|
+
text: messageText(event.data?.message),
|
|
464
|
+
turn: event.data?.turn,
|
|
465
|
+
name: call?.name || "",
|
|
466
|
+
pathLabel: readResultLabel(call?.name, call?.filePath, event.data?.turn),
|
|
467
|
+
});
|
|
468
|
+
this.rawSeen.set(mapKey, record);
|
|
469
|
+
return record;
|
|
470
|
+
} catch (error) {
|
|
471
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
472
|
+
this.ctx?.logger?.warn?.(`spine: raw backup failed: ${message}`);
|
|
473
|
+
const record = { ok: false, reason: "unwritable", toolCallId: id };
|
|
474
|
+
this.rawSeen.set(mapKey, record);
|
|
475
|
+
return record;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
captureSession(session) {
|
|
480
|
+
const seen = new Set();
|
|
481
|
+
for (const event of session?.snapshotEvents?.() || []) {
|
|
482
|
+
if (event?.type !== "tool/result") continue;
|
|
483
|
+
const id = event.data?.message?.toolCallId;
|
|
484
|
+
if (!id || seen.has(id)) continue;
|
|
485
|
+
seen.add(id);
|
|
486
|
+
this.captureOne(session, event);
|
|
487
|
+
}
|
|
488
|
+
return seen;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
surfaceToolIds(session) {
|
|
492
|
+
const ids = [];
|
|
493
|
+
for (const seq of session?.surface?.nodes || []) {
|
|
494
|
+
const event = session.eventAt?.(seq);
|
|
495
|
+
if (event?.type !== "tool/result") continue;
|
|
496
|
+
const id = event.data?.message?.toolCallId;
|
|
497
|
+
if (id) ids.push(id);
|
|
498
|
+
}
|
|
499
|
+
return ids;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
rawVerifiedSet(session) {
|
|
503
|
+
const prefix = `${session?.id || DEFAULT_KEY}\0`;
|
|
504
|
+
const ids = new Set();
|
|
505
|
+
for (const [mapKey, record] of this.rawSeen) {
|
|
506
|
+
if (!mapKey.startsWith(prefix)) continue;
|
|
507
|
+
if (verifiedRaw(record).ok) ids.add(mapKey.slice(prefix.length));
|
|
508
|
+
}
|
|
509
|
+
return ids;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
ensurePruneGuard() {
|
|
513
|
+
if (this.pruneGuarded) return;
|
|
514
|
+
let pruner;
|
|
515
|
+
try {
|
|
516
|
+
pruner = this.ctx.get("toolResultPruner");
|
|
517
|
+
} catch {
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
if (!pruner?.pruneSession) return;
|
|
521
|
+
pruner.__spineEngine = this;
|
|
522
|
+
if (pruner.pruneSession.__spineGuarded) {
|
|
523
|
+
this.pruneGuarded = true;
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
const original = pruner.pruneSession.bind(pruner);
|
|
527
|
+
function guarded(session) {
|
|
528
|
+
const engine = pruner.__spineEngine;
|
|
529
|
+
engine.captureSession(session);
|
|
530
|
+
return runPruneAfterCapture(engine.surfaceToolIds(session), engine.rawVerifiedSet(session), () => original(session));
|
|
531
|
+
}
|
|
532
|
+
guarded.__spineGuarded = true;
|
|
533
|
+
pruner.pruneSession = guarded;
|
|
534
|
+
this.pruneGuarded = true;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
restoreRawMessages(messages, key) {
|
|
538
|
+
return (messages || []).map((message) => {
|
|
539
|
+
const id = message?.toolCallId || message?.message?.toolCallId;
|
|
540
|
+
if (!id) return message;
|
|
541
|
+
return restoreToolText(message, this.rawSeen.get(`${key}\0${id}`));
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
warnShadowFull(key, shadow, used, cap) {
|
|
546
|
+
const id = key || DEFAULT_KEY;
|
|
547
|
+
if (this.shadowFullWarned.has(id)) return;
|
|
548
|
+
this.shadowFullWarned.add(id);
|
|
549
|
+
this.ctx.logger?.warn?.(`spine: shadow ${shadow} 已用 ${used},上限 ${cap}。删不掉旧备份,这一刀不切。`);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
warnShadowOversize(key, cap) {
|
|
553
|
+
const id = key || DEFAULT_KEY;
|
|
554
|
+
if (this.shadowFullWarned.has(`oversize:${id}`)) return;
|
|
555
|
+
this.shadowFullWarned.add(`oversize:${id}`);
|
|
556
|
+
this.ctx.logger?.warn?.(`spine: 这一份备份超过上限 ${cap},已写下,其他可删备份已清。`);
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
saveShadow(messages, required, shadowFile, warnKey = DEFAULT_KEY, protectPaths = []) {
|
|
560
|
+
const shadow = shadowDirectory();
|
|
561
|
+
mkdirSync(shadow, { recursive: true, mode: 0o700 });
|
|
562
|
+
tightenShadow(shadow);
|
|
563
|
+
const file = shadowFilePath(shadowFile, warnKey);
|
|
564
|
+
const dir = sessionDir(shadow, warnKey);
|
|
565
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
566
|
+
writeFileSync(join(dir, "owner"), String(process.pid), { mode: 0o600 });
|
|
567
|
+
const body = shadowFileBody(warnKey, shadowBody(messages));
|
|
568
|
+
const incoming = Buffer.byteLength(body);
|
|
569
|
+
const cap = shadowMaxBytes();
|
|
570
|
+
const rotation = rotateShadow({
|
|
571
|
+
dir: shadow,
|
|
572
|
+
incomingBytes: incoming,
|
|
573
|
+
protectPath: file,
|
|
574
|
+
protectPaths,
|
|
575
|
+
sessionKey: warnKey,
|
|
576
|
+
cap,
|
|
577
|
+
});
|
|
578
|
+
if (!rotation.ok) {
|
|
579
|
+
this.warnShadowFull(warnKey, shadow, rotation.used, cap);
|
|
580
|
+
if (required) throw new Error("spine: shadow backup is full; leaving the span");
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
if (rotation.deleted > 0) {
|
|
584
|
+
this.ctx.logger?.info?.(`spine: shadow 轮换 删除 ${rotation.deleted} 个文件,腾出 ${rotation.freed} 字节,用量 ${rotation.used + incoming}/${cap}`);
|
|
585
|
+
}
|
|
586
|
+
if (rotation.oversize) this.warnShadowOversize(warnKey, cap);
|
|
587
|
+
writeFileSync(file, body, { mode: 0o600 });
|
|
588
|
+
chmodSync(file, 0o600);
|
|
589
|
+
if (!existsSync(file)) throw new Error("spine: shadow backup was not written");
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
async summarize(input, agent, signal) {
|
|
593
|
+
const decision = takeLocalCut(input?.liveCompact);
|
|
594
|
+
const state = this.stateFor(agent);
|
|
595
|
+
state.approvedCard = null;
|
|
596
|
+
if (decision.kind === "refuse") throw new Error(decision.reason);
|
|
597
|
+
if (decision.kind === "local") {
|
|
598
|
+
const card = input?.liveCompact?.card;
|
|
599
|
+
if (card?.needsBackup && !card.shadowFile) throw new Error("spine: omit refused; backup path missing");
|
|
600
|
+
const restored = this.restoreRawMessages(input.messages, state.key);
|
|
601
|
+
this.saveShadow(restored, true, card?.shadowFile, state.key, card?.rawPaths || []);
|
|
602
|
+
if (card?.needsBackup && !existsSync(card.shadowFile)) throw new Error("spine: shadow backup was not written");
|
|
603
|
+
const text = backupClaimOk(restored) ? decision.text : withoutFullBackupClaim(decision.text, card?.shadowFile);
|
|
604
|
+
const block = { type: "text", text };
|
|
605
|
+
return { summary: [block], provider: "local", model: "spine-extract", rawOutput: [block] };
|
|
606
|
+
}
|
|
607
|
+
try {
|
|
608
|
+
this.saveShadow(this.restoreRawMessages(input.messages, state.key), false, freshShadowFile("official", state.key), state.key);
|
|
609
|
+
} catch {}
|
|
610
|
+
const joined = input.messages.map(messageText).join("\n\n");
|
|
611
|
+
if (process.env.SPINE_THUNDER !== "1") return super.summarize(input, agent, signal);
|
|
612
|
+
const next = joined;
|
|
613
|
+
const mode = decideThunder({
|
|
614
|
+
previous: state.lastPrefix || "",
|
|
615
|
+
next,
|
|
616
|
+
modelKind: process.env.SPINE_THUNDER_MODEL === "27b" ? "27b" : "flash",
|
|
617
|
+
prefixTokens: state.lastPrefixTokens || 0,
|
|
618
|
+
});
|
|
619
|
+
if (mode === "attach") return super.summarize(input, agent, signal);
|
|
620
|
+
const users = input.messages.filter((message) => (message?.role ?? message?.message?.role) === "user");
|
|
621
|
+
const taskLine = messageText(users.at(-1) || input.messages.at(-1)).slice(0, 500);
|
|
622
|
+
const trunk = buildCard(joined, "", input.messages.map((message) => ({
|
|
623
|
+
role: message?.role ?? message?.message?.role,
|
|
624
|
+
text: messageText(message),
|
|
625
|
+
})));
|
|
626
|
+
const facts = [...new Set([...(trunk.hardLines || []), ...(trunk.missing || [])])];
|
|
627
|
+
const slim = buildSendLess({ taskLine, uncovered: trunk.uncovered, facts });
|
|
628
|
+
return super.summarize({
|
|
629
|
+
...input,
|
|
630
|
+
messages: [{ role: "user", content: [{ type: "text", text: slim }] }],
|
|
631
|
+
}, agent, signal);
|
|
632
|
+
}
|
|
633
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,62 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-live-compact",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.0.6",
|
|
4
|
+
"description": "In-place newest-turn compaction for DeepSeek Harness. Compress while the task runs and keep the prefix.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"main": "shell.js",
|
|
8
|
+
"bin": {
|
|
9
|
+
"dsh-live-compact": "bin/dsh-live-compact.js"
|
|
10
|
+
},
|
|
11
|
+
"exports": {
|
|
12
|
+
".": "./shell.js",
|
|
13
|
+
"./engine": "./index.js",
|
|
14
|
+
"./gate": "./gate.js"
|
|
15
|
+
},
|
|
16
|
+
"files": [
|
|
17
|
+
"shell.js",
|
|
18
|
+
"index.js",
|
|
19
|
+
"gate.js",
|
|
20
|
+
"plan.js",
|
|
21
|
+
"raw.js",
|
|
22
|
+
"shadow.js",
|
|
23
|
+
"thunder.js",
|
|
24
|
+
"bin",
|
|
25
|
+
"cordis.patch.yml",
|
|
26
|
+
"README.md",
|
|
27
|
+
"LICENSE"
|
|
28
|
+
],
|
|
29
|
+
"keywords": [
|
|
30
|
+
"dsh-plugin",
|
|
31
|
+
"deepseek-harness",
|
|
32
|
+
"dsh",
|
|
33
|
+
"compaction"
|
|
34
|
+
],
|
|
35
|
+
"engines": {
|
|
36
|
+
"node": "^22.19 || >=24",
|
|
37
|
+
"dsh": ">=0.2.0-rc.2 <0.3.0"
|
|
38
|
+
},
|
|
39
|
+
"peerDependencies": {
|
|
40
|
+
"@deepseek-ai/dsh-compaction": ">=0.2.0-rc.2 <0.3.0",
|
|
41
|
+
"@deepseek-ai/dsh-compaction-basic": ">=0.2.0-rc.2 <0.3.0",
|
|
42
|
+
"@deepseek-ai/dsh-llm": ">=0.2.0-rc.2 <0.3.0"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"@deepseek-ai/dsh-compaction": "0.2.0-rc.2",
|
|
46
|
+
"@deepseek-ai/dsh-compaction-basic": "0.2.0-rc.2",
|
|
47
|
+
"@deepseek-ai/dsh-llm": "0.2.0-rc.2"
|
|
48
|
+
},
|
|
49
|
+
"dsh": {
|
|
50
|
+
"bundle": {
|
|
51
|
+
"patch": "./cordis.patch.yml"
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"repository": {
|
|
55
|
+
"type": "git",
|
|
56
|
+
"url": "git+https://github.com/MerkyorLynn/dsh-live-compact.git"
|
|
57
|
+
},
|
|
58
|
+
"homepage": "https://github.com/MerkyorLynn/dsh-live-compact#readme",
|
|
59
|
+
"bugs": {
|
|
60
|
+
"url": "https://github.com/MerkyorLynn/dsh-live-compact/issues"
|
|
61
|
+
}
|
|
62
|
+
}
|