@namzu/sdk 62.8.0 → 62.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/compaction/plan.d.ts +12 -0
- package/dist/compaction/plan.d.ts.map +1 -1
- package/dist/compaction/plan.js +39 -11
- package/dist/compaction/plan.js.map +1 -1
- package/dist/compaction/salience/minhash.d.ts +9 -0
- package/dist/compaction/salience/minhash.d.ts.map +1 -1
- package/dist/compaction/salience/minhash.js +20 -0
- package/dist/compaction/salience/minhash.js.map +1 -1
- package/dist/compaction/salience/score.d.ts.map +1 -1
- package/dist/compaction/salience/score.js +87 -45
- package/dist/compaction/salience/score.js.map +1 -1
- package/dist/compaction/salience/working-set.d.ts +14 -0
- package/dist/compaction/salience/working-set.d.ts.map +1 -1
- package/dist/compaction/salience/working-set.js +84 -33
- package/dist/compaction/salience/working-set.js.map +1 -1
- package/dist/config/runtime.d.ts +2 -0
- package/dist/config/runtime.d.ts.map +1 -1
- package/dist/config/runtime.js +14 -0
- package/dist/config/runtime.js.map +1 -1
- package/dist/connector/http-operation.d.ts +1 -0
- package/dist/connector/http-operation.d.ts.map +1 -1
- package/dist/connector/http-operation.js +1 -1
- package/dist/connector/http-operation.js.map +1 -1
- package/dist/connector/mcp/adapter.d.ts.map +1 -1
- package/dist/connector/mcp/adapter.js +10 -0
- package/dist/connector/mcp/adapter.js.map +1 -1
- package/dist/connector/mcp/client.d.ts +12 -0
- package/dist/connector/mcp/client.d.ts.map +1 -1
- package/dist/connector/mcp/client.js +59 -10
- package/dist/connector/mcp/client.js.map +1 -1
- package/dist/constants/mcp/index.d.ts +15 -0
- package/dist/constants/mcp/index.d.ts.map +1 -1
- package/dist/constants/mcp/index.js +15 -0
- package/dist/constants/mcp/index.js.map +1 -1
- package/dist/manager/session/turn-recorder.d.ts +25 -0
- package/dist/manager/session/turn-recorder.d.ts.map +1 -1
- package/dist/manager/session/turn-recorder.js +193 -17
- package/dist/manager/session/turn-recorder.js.map +1 -1
- package/dist/public-runtime.d.ts +1 -1
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +1 -1
- package/dist/public-runtime.js.map +1 -1
- package/dist/runtime/query/cancelled-before-start.js +1 -1
- package/dist/runtime/query/cancelled-before-start.js.map +1 -1
- package/dist/runtime/query/checkpoint.js +2 -1
- package/dist/runtime/query/checkpoint.js.map +1 -1
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +4 -1
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/phases/compaction.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/compaction.js +15 -1
- package/dist/runtime/query/iteration/phases/compaction.js.map +1 -1
- package/dist/runtime/query/prepare-turn.js +10 -4
- package/dist/runtime/query/prepare-turn.js.map +1 -1
- package/dist/store/session-log/core.d.ts +31 -1
- package/dist/store/session-log/core.d.ts.map +1 -1
- package/dist/store/session-log/core.js +95 -9
- package/dist/store/session-log/core.js.map +1 -1
- package/dist/store/session-log/disk.d.ts +2 -0
- package/dist/store/session-log/disk.d.ts.map +1 -1
- package/dist/store/session-log/disk.js +37 -0
- package/dist/store/session-log/disk.js.map +1 -1
- package/dist/store/session-log/fold.d.ts +9 -1
- package/dist/store/session-log/fold.d.ts.map +1 -1
- package/dist/store/session-log/fold.js +18 -2
- package/dist/store/session-log/fold.js.map +1 -1
- package/dist/store/session-log/incremental.d.ts +8 -0
- package/dist/store/session-log/incremental.d.ts.map +1 -1
- package/dist/store/session-log/incremental.js +7 -2
- package/dist/store/session-log/incremental.js.map +1 -1
- package/dist/store/session-log/index.d.ts +1 -0
- package/dist/store/session-log/index.d.ts.map +1 -1
- package/dist/store/session-log/index.js +1 -0
- package/dist/store/session-log/index.js.map +1 -1
- package/dist/store/session-log/verified-prefix.d.ts +46 -0
- package/dist/store/session-log/verified-prefix.d.ts.map +1 -0
- package/dist/store/session-log/verified-prefix.js +233 -0
- package/dist/store/session-log/verified-prefix.js.map +1 -0
- package/dist/types/connector/mcp.d.ts +19 -0
- package/dist/types/connector/mcp.d.ts.map +1 -1
- package/dist/types/session/records.d.ts +80 -1
- package/dist/types/session/records.d.ts.map +1 -1
- package/dist/types/session/records.js +22 -0
- package/dist/types/session/records.js.map +1 -1
- package/package.json +1 -1
- package/src/compaction/plan.ts +53 -11
- package/src/compaction/salience/minhash.ts +20 -0
- package/src/compaction/salience/score.ts +112 -43
- package/src/compaction/salience/working-set.ts +102 -31
- package/src/config/runtime.ts +14 -0
- package/src/connector/http-operation.ts +1 -1
- package/src/connector/mcp/adapter.ts +11 -0
- package/src/connector/mcp/client.ts +75 -12
- package/src/constants/mcp/index.ts +17 -0
- package/src/manager/session/turn-recorder.ts +218 -15
- package/src/public-runtime.ts +1 -0
- package/src/runtime/query/cancelled-before-start.ts +1 -1
- package/src/runtime/query/checkpoint.ts +2 -1
- package/src/runtime/query/index.ts +4 -1
- package/src/runtime/query/iteration/phases/compaction.ts +18 -0
- package/src/runtime/query/prepare-turn.ts +10 -3
- package/src/store/session-log/core.ts +129 -10
- package/src/store/session-log/disk.ts +32 -0
- package/src/store/session-log/fold.ts +19 -2
- package/src/store/session-log/incremental.ts +15 -2
- package/src/store/session-log/index.ts +1 -0
- package/src/store/session-log/verified-prefix.ts +243 -0
- package/src/types/connector/mcp.ts +19 -0
- package/src/types/session/records.ts +25 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isDeepStrictEqual } from 'node:util'
|
|
1
2
|
import {
|
|
2
3
|
AUTO_CONTINUATION_USER_MESSAGE,
|
|
3
4
|
REASONING_CUT_USER_MESSAGE,
|
|
@@ -94,6 +95,16 @@ function stampMessageId(message: Message, id: MessageId | undefined): void {
|
|
|
94
95
|
else (message as { id?: MessageId }).id = id
|
|
95
96
|
}
|
|
96
97
|
|
|
98
|
+
/** Two messages that differ only in their id and time are the same text. */
|
|
99
|
+
function sameStoredContent(a: Message, b: Message): boolean {
|
|
100
|
+
const { id: _a, timestamp: _ta, ...restA } = a
|
|
101
|
+
const { id: _b, timestamp: _tb, ...restB } = b
|
|
102
|
+
return isDeepStrictEqual(
|
|
103
|
+
JSON.parse(JSON.stringify(restA)) as unknown,
|
|
104
|
+
JSON.parse(JSON.stringify(restB)) as unknown,
|
|
105
|
+
)
|
|
106
|
+
}
|
|
107
|
+
|
|
97
108
|
/** How {@link TurnRecorder.open} takes the session's writer lease and starts an empty log. */
|
|
98
109
|
export interface TurnRecorderOpenOptions {
|
|
99
110
|
/** A lease the caller already holds. Absent: the recorder claims one and releases it at the end. */
|
|
@@ -255,6 +266,8 @@ export class TurnRecorder {
|
|
|
255
266
|
* these needs no re-basing, so no `compaction` record is written for it.
|
|
256
267
|
*/
|
|
257
268
|
#foldIdsAtResume: readonly MessageId[] | undefined
|
|
269
|
+
/** What the log folds to for a message a replacement or an edit changed, as of the resume. */
|
|
270
|
+
#foldEditedAtResume = new Map<MessageId, Message>()
|
|
258
271
|
/** When a resumed turn began: its `turn_started` record, not this process's start. */
|
|
259
272
|
#resumedTurnStartedAt: number | undefined
|
|
260
273
|
|
|
@@ -868,21 +881,38 @@ export class TurnRecorder {
|
|
|
868
881
|
async #readResumedLog(): Promise<readonly MessageId[] | undefined> {
|
|
869
882
|
await this.flush()
|
|
870
883
|
const fold = new SessionMessageFold()
|
|
884
|
+
// Whether the latest change to a message was a compaction edit: only
|
|
885
|
+
// that one is the recorder's to put back (a guardrail's replacement is
|
|
886
|
+
// the settled answer and the restored context is not allowed to undo it).
|
|
887
|
+
const latestChange = new Map<MessageId, 'edited' | 'replaced'>()
|
|
871
888
|
for await (const entry of this.log.read()) {
|
|
872
889
|
const record = entry.record as SessionRecord
|
|
873
890
|
if (record.type === 'turn_started' && record.turnId === this.turnId) {
|
|
874
891
|
const at = Date.parse(record.ts)
|
|
875
892
|
if (Number.isFinite(at)) this.#resumedTurnStartedAt = at
|
|
876
893
|
}
|
|
894
|
+
if (record.type === 'message_edited') latestChange.set(record.targetMessageId, 'edited')
|
|
895
|
+
else if (record.type === 'message_replaced')
|
|
896
|
+
latestChange.set(record.targetMessageId, 'replaced')
|
|
877
897
|
fold.apply(record)
|
|
878
898
|
}
|
|
879
|
-
if (fold.spilledSummary) return undefined
|
|
880
899
|
const ids: MessageId[] = []
|
|
900
|
+
let allRecorded = !fold.spilledSummary
|
|
901
|
+
this.#foldEditedAtResume.clear()
|
|
902
|
+
// Filled for every fold entry with an id, whether or not the fold also
|
|
903
|
+
// holds id-less entries (a summary): a restored context older than an edit
|
|
904
|
+
// is put right either way. An entry whose text is spilled is not
|
|
905
|
+
// compared, so a message edited to a spilled text is not put back.
|
|
881
906
|
for (const entry of fold.entries()) {
|
|
882
|
-
if (!entry.messageId)
|
|
907
|
+
if (!entry.messageId) {
|
|
908
|
+
allRecorded = false
|
|
909
|
+
continue
|
|
910
|
+
}
|
|
883
911
|
ids.push(entry.messageId)
|
|
912
|
+
if (latestChange.get(entry.messageId) === 'edited' && entry.spill === undefined)
|
|
913
|
+
this.#foldEditedAtResume.set(entry.messageId, entry.message)
|
|
884
914
|
}
|
|
885
|
-
return ids
|
|
915
|
+
return allRecorded ? ids : undefined
|
|
886
916
|
}
|
|
887
917
|
|
|
888
918
|
/** The recorded context is exactly what the log folds to: a resume changed nothing. */
|
|
@@ -1093,6 +1123,13 @@ export class TurnRecorder {
|
|
|
1093
1123
|
options: {
|
|
1094
1124
|
readonly messageId?: MessageId
|
|
1095
1125
|
readonly transient?: boolean
|
|
1126
|
+
/**
|
|
1127
|
+
* The message is part of the history a resume restores, not a new
|
|
1128
|
+
* one. While the resume's re-base is pending, a restored message with no
|
|
1129
|
+
* id is held and nothing is reconciled; the first message that is not
|
|
1130
|
+
* restored (or any other reconcile) re-bases once on the whole context.
|
|
1131
|
+
*/
|
|
1132
|
+
readonly restored?: boolean
|
|
1096
1133
|
} = {},
|
|
1097
1134
|
): void {
|
|
1098
1135
|
if (options.messageId) {
|
|
@@ -1110,6 +1147,16 @@ export class TurnRecorder {
|
|
|
1110
1147
|
})
|
|
1111
1148
|
return
|
|
1112
1149
|
}
|
|
1150
|
+
if (options.restored && this.#rebasePending && this.#phase === 'active') {
|
|
1151
|
+
// A restored message with no id (a compaction summary, the working-memory
|
|
1152
|
+
// slot, a message the host added) can arrive before the recorded ones.
|
|
1153
|
+
// Reconciling now would re-base the fold on a context holding only it and
|
|
1154
|
+
// drop the recorded messages that follow; hold it as an id-less view entry
|
|
1155
|
+
// and let the first reconciliation after the whole context is in place
|
|
1156
|
+
// write one `compaction` that keeps the recorded ids after it.
|
|
1157
|
+
this.#view.push({ message })
|
|
1158
|
+
return
|
|
1159
|
+
}
|
|
1113
1160
|
this.#syncMessages()
|
|
1114
1161
|
}
|
|
1115
1162
|
|
|
@@ -1152,6 +1199,100 @@ export class TurnRecorder {
|
|
|
1152
1199
|
return id
|
|
1153
1200
|
}
|
|
1154
1201
|
|
|
1202
|
+
/**
|
|
1203
|
+
* A compaction pass edits one message by replacing its object with a copy
|
|
1204
|
+
* that has other content (`{ ...message, content }`). The copy still carries
|
|
1205
|
+
* the id the original was stamped with, but the recorder's own indexes are
|
|
1206
|
+
* keyed by object identity, so without this it would see an unknown message
|
|
1207
|
+
* and record every message after it again. A live object that is not in the
|
|
1208
|
+
* view, whose `id` is that of a recorded view entry whose own object has left
|
|
1209
|
+
* the context, is that same message edited: one `message_edited` record
|
|
1210
|
+
* carries its new content and the view entry follows the new object.
|
|
1211
|
+
*
|
|
1212
|
+
* Only an id claimed by exactly one live object is taken, so two copies of
|
|
1213
|
+
* one message are never both treated as its edit. Anything else is left to
|
|
1214
|
+
* the callers, which record it as a new message.
|
|
1215
|
+
*/
|
|
1216
|
+
#absorbEdits(): void {
|
|
1217
|
+
if (this.#phase !== 'active') return
|
|
1218
|
+
const live = this.#turn.messages
|
|
1219
|
+
const inContext = new Set<Message>(live)
|
|
1220
|
+
let orphans: Map<MessageId, ViewEntry> | undefined
|
|
1221
|
+
for (const entry of this.#view) {
|
|
1222
|
+
if (entry.id === undefined || entry.transient || inContext.has(entry.message)) continue
|
|
1223
|
+
orphans ??= new Map()
|
|
1224
|
+
orphans.set(entry.id, entry)
|
|
1225
|
+
}
|
|
1226
|
+
if (orphans === undefined) return
|
|
1227
|
+
const known = new Set<Message>(this.#view.map((entry) => entry.message))
|
|
1228
|
+
const claimed = new Map<MessageId, Message[]>()
|
|
1229
|
+
for (const message of live) {
|
|
1230
|
+
if (known.has(message) || this.#transient.has(message)) continue
|
|
1231
|
+
const id = message.id
|
|
1232
|
+
if (id === undefined || !orphans.has(id)) continue
|
|
1233
|
+
const list = claimed.get(id)
|
|
1234
|
+
if (list) list.push(message)
|
|
1235
|
+
else claimed.set(id, [message])
|
|
1236
|
+
}
|
|
1237
|
+
const turnId = this.turnId
|
|
1238
|
+
for (const [id, messages] of claimed) {
|
|
1239
|
+
const message = messages[0] as Message
|
|
1240
|
+
const entry = orphans.get(id) as ViewEntry
|
|
1241
|
+
if (messages.length !== 1 || message.role !== entry.message.role) continue
|
|
1242
|
+
// `previous` is the live object the view holds: a message mutated in place
|
|
1243
|
+
// and then replaced by a copy of itself is adopted without a record, and
|
|
1244
|
+
// the fold keeps the text it recorded.
|
|
1245
|
+
const previous = entry.message
|
|
1246
|
+
entry.message = message
|
|
1247
|
+
this.#ids.set(message, id)
|
|
1248
|
+
this.#recorded.set(message, id)
|
|
1249
|
+
// A copy with the same text (a structured clone, a re-parsed message)
|
|
1250
|
+
// is adopted as is: the fold already shows that content.
|
|
1251
|
+
if (sameStoredContent(previous, message)) continue
|
|
1252
|
+
void this.#enqueue(() =>
|
|
1253
|
+
this.#append({
|
|
1254
|
+
type: 'message_edited',
|
|
1255
|
+
turnId,
|
|
1256
|
+
targetMessageId: id,
|
|
1257
|
+
content: message,
|
|
1258
|
+
ignorable: true,
|
|
1259
|
+
reason: 'compaction',
|
|
1260
|
+
}),
|
|
1261
|
+
).catch(() => undefined)
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
/**
|
|
1266
|
+
* A resume restores the context a checkpoint holds, which can predate a
|
|
1267
|
+
* compaction edit written after it. The ids still match, so the log is not
|
|
1268
|
+
* re-based; but the fold then shows the edited text of a message the context
|
|
1269
|
+
* holds as it was. One `message_edited` per such message makes the fold show
|
|
1270
|
+
* the context the resume actually runs with.
|
|
1271
|
+
*/
|
|
1272
|
+
#recordDriftFromFold(): void {
|
|
1273
|
+
const edited = this.#foldEditedAtResume
|
|
1274
|
+
if (edited.size === 0) return
|
|
1275
|
+
const turnId = this.turnId
|
|
1276
|
+
for (const entry of this.#view) {
|
|
1277
|
+
if (entry.id === undefined || entry.transient) continue
|
|
1278
|
+
const folded = edited.get(entry.id)
|
|
1279
|
+
if (folded === undefined || sameStoredContent(folded, entry.message)) continue
|
|
1280
|
+
const message = entry.message
|
|
1281
|
+
const id = entry.id
|
|
1282
|
+
void this.#enqueue(() =>
|
|
1283
|
+
this.#append({
|
|
1284
|
+
type: 'message_edited',
|
|
1285
|
+
turnId,
|
|
1286
|
+
targetMessageId: id,
|
|
1287
|
+
content: message,
|
|
1288
|
+
ignorable: true,
|
|
1289
|
+
reason: 'resume',
|
|
1290
|
+
}),
|
|
1291
|
+
).catch(() => undefined)
|
|
1292
|
+
}
|
|
1293
|
+
edited.clear()
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1155
1296
|
/**
|
|
1156
1297
|
* Reconcile the context with the log. Nothing is written before the turn
|
|
1157
1298
|
* begins or after it pauses or settles.
|
|
@@ -1164,9 +1305,15 @@ export class TurnRecorder {
|
|
|
1164
1305
|
this.#foldIdsAtResume = undefined
|
|
1165
1306
|
if (!unchanged) {
|
|
1166
1307
|
this.#recordCompaction()
|
|
1308
|
+
// Kept ids whose restored text differs from the edited text the fold
|
|
1309
|
+
// holds get their record; entries re-recorded under fresh ids are not
|
|
1310
|
+
// in the map and are skipped.
|
|
1311
|
+
this.#recordDriftFromFold()
|
|
1167
1312
|
return
|
|
1168
1313
|
}
|
|
1314
|
+
this.#recordDriftFromFold()
|
|
1169
1315
|
}
|
|
1316
|
+
this.#absorbEdits()
|
|
1170
1317
|
const live = this.#turn.messages
|
|
1171
1318
|
const view = this.#view
|
|
1172
1319
|
let prefix = 0
|
|
@@ -1205,6 +1352,7 @@ export class TurnRecorder {
|
|
|
1205
1352
|
* log order, the whole context becomes the summary.
|
|
1206
1353
|
*/
|
|
1207
1354
|
#recordCompaction(): void {
|
|
1355
|
+
this.#absorbEdits()
|
|
1208
1356
|
const live = this.#turn.messages
|
|
1209
1357
|
const byMessage = new Map(this.#view.map((entry) => [entry.message, entry]))
|
|
1210
1358
|
const next: ViewEntry[] = live.map((message) => {
|
|
@@ -1493,6 +1641,34 @@ async function recordedMessages(
|
|
|
1493
1641
|
return out
|
|
1494
1642
|
}
|
|
1495
1643
|
|
|
1644
|
+
type RecordedContent = { readonly content: Message; readonly spill?: SpillRef }
|
|
1645
|
+
|
|
1646
|
+
function recordedContent(record: {
|
|
1647
|
+
readonly content: Message
|
|
1648
|
+
readonly spill?: SpillRef | undefined
|
|
1649
|
+
}): RecordedContent {
|
|
1650
|
+
return { content: record.content, ...(record.spill ? { spill: record.spill } : {}) }
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
/** Every message ever recorded, and the texts an edited one held before each edit. */
|
|
1654
|
+
export interface EverRecordedMessages {
|
|
1655
|
+
/** The latest text of each message, by its durable id. */
|
|
1656
|
+
readonly latest: ReadonlyMap<MessageId, Message>
|
|
1657
|
+
/**
|
|
1658
|
+
* Whether `id` held `message`'s text before a compaction edit. A
|
|
1659
|
+
* `message_replaced` for the id forgets the texts before it: a guardrail's
|
|
1660
|
+
* replacement is the settled answer. A spilled earlier text is read only
|
|
1661
|
+
* when a comparison needs it.
|
|
1662
|
+
*/
|
|
1663
|
+
heldBeforeAnEdit(id: MessageId, message: Message): Promise<boolean>
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
interface EverRecordedState {
|
|
1667
|
+
readonly latest: Map<MessageId, RecordedContent>
|
|
1668
|
+
/** Only ids that have an edit appear here. */
|
|
1669
|
+
readonly earlier: Map<MessageId, RecordedContent[]>
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1496
1672
|
/**
|
|
1497
1673
|
* Every message this session's log has ever recorded, keyed by the durable
|
|
1498
1674
|
* id its `message` record gave it — including one a compaction has since
|
|
@@ -1507,25 +1683,37 @@ async function recordedMessages(
|
|
|
1507
1683
|
export async function readEverRecordedMessages(
|
|
1508
1684
|
log: SessionLog,
|
|
1509
1685
|
): Promise<ReadonlyMap<MessageId, Message>> {
|
|
1686
|
+
return (await readEverRecordedMessageIndex(log)).latest
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
/**
|
|
1690
|
+
* {@link readEverRecordedMessages} plus what a host that cached the history
|
|
1691
|
+
* before a compaction edit still holds, answered from the same reduction with
|
|
1692
|
+
* no read of the log of its own.
|
|
1693
|
+
*/
|
|
1694
|
+
export async function readEverRecordedMessageIndex(log: SessionLog): Promise<EverRecordedMessages> {
|
|
1510
1695
|
return foldSessionLog(log, {
|
|
1511
1696
|
name: 'ever-recorded-messages',
|
|
1512
1697
|
mode: 'strict',
|
|
1513
|
-
create: () => new Map
|
|
1514
|
-
apply: (latest, entry) => {
|
|
1698
|
+
create: (): EverRecordedState => ({ latest: new Map(), earlier: new Map() }),
|
|
1699
|
+
apply: ({ latest, earlier }, entry) => {
|
|
1515
1700
|
const record = entry.record as SessionRecord
|
|
1516
1701
|
if (record.type === 'message') {
|
|
1517
|
-
latest.set(record.messageId,
|
|
1518
|
-
content: record.content,
|
|
1519
|
-
...(record.spill ? { spill: record.spill } : {}),
|
|
1520
|
-
})
|
|
1702
|
+
latest.set(record.messageId, recordedContent(record))
|
|
1521
1703
|
} else if (record.type === 'message_replaced') {
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1704
|
+
earlier.delete(record.targetMessageId)
|
|
1705
|
+
latest.set(record.targetMessageId, recordedContent(record))
|
|
1706
|
+
} else if (record.type === 'message_edited') {
|
|
1707
|
+
const before = latest.get(record.targetMessageId)
|
|
1708
|
+
if (before !== undefined) {
|
|
1709
|
+
const held = earlier.get(record.targetMessageId)
|
|
1710
|
+
if (held) held.push(before)
|
|
1711
|
+
else earlier.set(record.targetMessageId, [before])
|
|
1712
|
+
}
|
|
1713
|
+
latest.set(record.targetMessageId, recordedContent(record))
|
|
1526
1714
|
}
|
|
1527
1715
|
},
|
|
1528
|
-
result: async (latest) => {
|
|
1716
|
+
result: async ({ latest, earlier }) => {
|
|
1529
1717
|
const resolved = new Map<MessageId, Message>()
|
|
1530
1718
|
for (const [id, entry] of latest) {
|
|
1531
1719
|
// Copied: the map this is read from is shared with the next call.
|
|
@@ -1534,7 +1722,22 @@ export async function readEverRecordedMessages(
|
|
|
1534
1722
|
: structuredClone(entry.content)
|
|
1535
1723
|
resolved.set(id, withMessageId(content, id))
|
|
1536
1724
|
}
|
|
1537
|
-
|
|
1725
|
+
// A snapshot: the arrays are advanced in place by the next call. The
|
|
1726
|
+
// entries in them are never changed once made.
|
|
1727
|
+
const held = new Map<MessageId, readonly RecordedContent[]>()
|
|
1728
|
+
for (const [id, list] of earlier) held.set(id, [...list])
|
|
1729
|
+
return {
|
|
1730
|
+
latest: resolved,
|
|
1731
|
+
async heldBeforeAnEdit(id, message) {
|
|
1732
|
+
for (const entry of held.get(id) ?? []) {
|
|
1733
|
+
const content = entry.spill
|
|
1734
|
+
? (JSON.parse(await log.readSpill(entry.spill)) as Message)
|
|
1735
|
+
: entry.content
|
|
1736
|
+
if (sameStoredContent(content, message)) return true
|
|
1737
|
+
}
|
|
1738
|
+
return false
|
|
1739
|
+
},
|
|
1740
|
+
}
|
|
1538
1741
|
},
|
|
1539
1742
|
})
|
|
1540
1743
|
}
|
package/src/public-runtime.ts
CHANGED
|
@@ -86,7 +86,7 @@ export async function* settlePreStartCancellation(
|
|
|
86
86
|
|
|
87
87
|
const push = (message: Message, ids: ReadonlyMap<Message, MessageId>): void => {
|
|
88
88
|
const messageId = ids.get(message)
|
|
89
|
-
ctx.recorder.pushMessage(message, messageId ? { messageId } : {})
|
|
89
|
+
ctx.recorder.pushMessage(message, { ...(messageId ? { messageId } : {}), restored: true })
|
|
90
90
|
}
|
|
91
91
|
try {
|
|
92
92
|
if (selectedResumeState) {
|
|
@@ -822,7 +822,8 @@ async function readRecordedMessage(
|
|
|
822
822
|
}
|
|
823
823
|
if (
|
|
824
824
|
(entry.type === 'message' && entry.messageId === messageId) ||
|
|
825
|
-
(entry.type === 'message_replaced'
|
|
825
|
+
((entry.type === 'message_replaced' || entry.type === 'message_edited') &&
|
|
826
|
+
entry.targetMessageId === messageId)
|
|
826
827
|
) {
|
|
827
828
|
found = { content: entry.content, ...(entry.spill ? { spill: entry.spill } : {}) }
|
|
828
829
|
}
|
|
@@ -2248,7 +2248,10 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
|
|
|
2248
2248
|
}
|
|
2249
2249
|
const pushRestored = (message: Message, ids: ReadonlyMap<Message, MessageId>): void => {
|
|
2250
2250
|
const messageId = ids.get(message)
|
|
2251
|
-
ctx.recorder.pushMessage(message,
|
|
2251
|
+
ctx.recorder.pushMessage(message, {
|
|
2252
|
+
...(messageId ? { messageId } : {}),
|
|
2253
|
+
restored: true,
|
|
2254
|
+
})
|
|
2252
2255
|
}
|
|
2253
2256
|
|
|
2254
2257
|
if (params.resumeFromCheckpoint) {
|
|
@@ -7,6 +7,7 @@ import { replaceableSummaries, summarizeWithModel } from '../../../../compaction
|
|
|
7
7
|
import {
|
|
8
8
|
type CompactionPlan,
|
|
9
9
|
DEFAULT_SOFT_TARGET,
|
|
10
|
+
canShedSalienceWorkingSet,
|
|
10
11
|
planCompaction,
|
|
11
12
|
planSalienceWorkingSet,
|
|
12
13
|
} from '../../../../compaction/plan.js'
|
|
@@ -787,6 +788,23 @@ async function runCompactionCheckInner(
|
|
|
787
788
|
const startAt = salience ? Math.min(config.softTarget ?? DEFAULT_SOFT_TARGET, trigger) : trigger
|
|
788
789
|
if (!options?.force && usage < startAt) return
|
|
789
790
|
|
|
791
|
+
// A pass that can shed nothing and has not reached the summary trigger
|
|
792
|
+
// does nothing but announce itself: two hooks, a started event and a log
|
|
793
|
+
// line around a plan with no action. From the soft target until the
|
|
794
|
+
// trigger it ran on every iteration of every turn once every narration
|
|
795
|
+
// was cut and every large result cleared. Not asked of a forced pass, of
|
|
796
|
+
// a turn with a reducer (it owns the reduction), or at the trigger, where
|
|
797
|
+
// the summary path is what the pass is for.
|
|
798
|
+
if (
|
|
799
|
+
salience &&
|
|
800
|
+
!options?.force &&
|
|
801
|
+
!ctx.contextReducer &&
|
|
802
|
+
usage < trigger &&
|
|
803
|
+
!canShedSalienceWorkingSet(ctx.recorder.messages, config)
|
|
804
|
+
) {
|
|
805
|
+
return
|
|
806
|
+
}
|
|
807
|
+
|
|
790
808
|
// Three failed passes in a row: stop proactive compaction for the rest of
|
|
791
809
|
// the turn. A pass the provider forced is never gated.
|
|
792
810
|
const breaker = compactionBreaker(ctx)
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
import { assertSessionLogAttribution } from '../../manager/session/attribution.js'
|
|
9
9
|
import {
|
|
10
10
|
type RecordedMessage,
|
|
11
|
-
|
|
11
|
+
readEverRecordedMessageIndex,
|
|
12
12
|
readFoldedHistory,
|
|
13
13
|
} from '../../manager/session/turn-recorder.js'
|
|
14
14
|
import { resolveModelPricing } from '../../pricing/index.js'
|
|
@@ -1164,7 +1164,8 @@ async function reconcileCallerMessages(
|
|
|
1164
1164
|
continuationMode: boolean,
|
|
1165
1165
|
): Promise<Message[]> {
|
|
1166
1166
|
const hasId = messages.some((message) => message.id !== undefined)
|
|
1167
|
-
const
|
|
1167
|
+
const everRecorded = hasId ? await readEverRecordedMessageIndex(log) : undefined
|
|
1168
|
+
const recordedById = everRecorded?.latest
|
|
1168
1169
|
// Ids a caller message already claimed by direct lookup above: a fold
|
|
1169
1170
|
// entry that gave one of THESE out must not also be offered to the no-id
|
|
1170
1171
|
// alignment below, or a message that already reconciled by id could be
|
|
@@ -1182,7 +1183,13 @@ async function reconcileCallerMessages(
|
|
|
1182
1183
|
if (id !== undefined) {
|
|
1183
1184
|
const recorded = recordedById?.get(id)
|
|
1184
1185
|
if (recorded === undefined) throw staleCachedHistoryError('foreign', message)
|
|
1185
|
-
if (!sameMessageContent(recorded, message))
|
|
1186
|
+
if (!sameMessageContent(recorded, message)) {
|
|
1187
|
+
// A compaction pass edits a message under its own id; a host that
|
|
1188
|
+
// cached the history before the pass still holds the first text.
|
|
1189
|
+
// Answered from the reduction already read: no read of the log.
|
|
1190
|
+
if (!(await everRecorded?.heldBeforeAnEdit(id, message)))
|
|
1191
|
+
throw staleCachedHistoryError('edited', message)
|
|
1192
|
+
}
|
|
1186
1193
|
claimedIds.add(id)
|
|
1187
1194
|
continue // already durable: the fold supplies it
|
|
1188
1195
|
}
|
|
@@ -52,6 +52,12 @@ import {
|
|
|
52
52
|
} from './lease.js'
|
|
53
53
|
import type { SpillReadOptions, SpillRef, SpillStore } from './spill.js'
|
|
54
54
|
import { TOOL_INPUT_RECORD_TYPES, spilledInput, toolInputSlots } from './spilled-input.js'
|
|
55
|
+
import {
|
|
56
|
+
type VerifiedPrefix,
|
|
57
|
+
rehashPrefix,
|
|
58
|
+
startVerifiedPrefix,
|
|
59
|
+
verifiedPrefixOf,
|
|
60
|
+
} from './verified-prefix.js'
|
|
55
61
|
|
|
56
62
|
// ─── public types ─────────────────────────────────────────────────────────
|
|
57
63
|
|
|
@@ -91,6 +97,17 @@ export interface ReadSessionLogOptions {
|
|
|
91
97
|
readonly mode?: 'strict' | 'tolerant'
|
|
92
98
|
/** Resume after this record: its bytes are checked, then the walk continues from it. */
|
|
93
99
|
readonly after?: RecordPointer
|
|
100
|
+
/**
|
|
101
|
+
* With `after`, and in `strict` mode only: also confirm that every byte
|
|
102
|
+
* before the cursor is what this process verified. The prefix is read and
|
|
103
|
+
* hashed line by line against what was remembered (no parsing, nothing
|
|
104
|
+
* kept); when this process remembers nothing of the log, or any byte
|
|
105
|
+
* differs, it is verified in full from byte 0, which refuses a broken chain
|
|
106
|
+
* as a read from byte 0 does. Without it a resumed read trusts the records
|
|
107
|
+
* before the cursor, as a reduction that is advanced rather than recomputed
|
|
108
|
+
* does. Ignored when the medium has no `identity`.
|
|
109
|
+
*/
|
|
110
|
+
readonly reverify?: boolean
|
|
94
111
|
/**
|
|
95
112
|
* The record the log must hold at `expectHead.seq`, byte for byte. Anchors
|
|
96
113
|
* the tail, which no later record vouches for.
|
|
@@ -288,6 +305,25 @@ export interface LogMedium extends LogBytes {
|
|
|
288
305
|
truncate(size: number, expectedSize: number): Promise<void>
|
|
289
306
|
/** The bytes from `offset` to the end, in chunks. */
|
|
290
307
|
stream(offset: number): AsyncIterable<Uint8Array>
|
|
308
|
+
/**
|
|
309
|
+
* A string that names this log's bytes for as long as the same file stands
|
|
310
|
+
* (the disk medium: device, inode and path), or `undefined` when the log
|
|
311
|
+
* has none yet or the medium cannot say. A log that has one lets a later
|
|
312
|
+
* claim in the same process re-check the bytes it already verified without
|
|
313
|
+
* parsing them again; the content is still compared, so a wrong or reused
|
|
314
|
+
* identity can only cost a full verification, never skip one.
|
|
315
|
+
*/
|
|
316
|
+
identity?(): Promise<string | undefined>
|
|
317
|
+
/**
|
|
318
|
+
* Hand the bytes from `offset` to `visit`, chunk by chunk, until `visit`
|
|
319
|
+
* answers `true` or the log ends. The chunk is only valid for the call: the
|
|
320
|
+
* medium may fill the same buffer again for the next one, so a visitor keeps
|
|
321
|
+
* no slice of it. It lets a re-check of a long log read through one buffer
|
|
322
|
+
* instead of allocating a fresh native buffer for every megabyte, which
|
|
323
|
+
* otherwise makes the collector run on every few walks. Optional; `stream`
|
|
324
|
+
* is used when a medium has no `scan`.
|
|
325
|
+
*/
|
|
326
|
+
scan?(offset: number, visit: (chunk: Uint8Array) => boolean): Promise<void>
|
|
291
327
|
}
|
|
292
328
|
|
|
293
329
|
export interface SessionLogCoreOptions {
|
|
@@ -397,6 +433,8 @@ export class SessionLogCore implements SessionLog {
|
|
|
397
433
|
readonly #holding: HeldSessionLease = { release: () => this.#releaseHeld() }
|
|
398
434
|
/** Bytes of the log this instance has verified and applied. */
|
|
399
435
|
#synced = 0
|
|
436
|
+
/** What this process remembers of the log's verified prefix (see `verified-prefix.ts`). */
|
|
437
|
+
#ledgerKey: string | undefined
|
|
400
438
|
#torn = 0
|
|
401
439
|
readonly #degradedBufferBytes: number
|
|
402
440
|
/** Records accepted into the chain that the medium has not taken yet, oldest first. */
|
|
@@ -462,13 +500,48 @@ export class SessionLogCore implements SessionLog {
|
|
|
462
500
|
|
|
463
501
|
// ── state ──
|
|
464
502
|
|
|
465
|
-
/**
|
|
503
|
+
/**
|
|
504
|
+
* Catch up with the medium: verify and apply whatever was appended since the last sync.
|
|
505
|
+
*
|
|
506
|
+
* An instance that has nothing yet (a fresh one) first asks whether this
|
|
507
|
+
* process already verified a prefix of this very log: it then re-reads and
|
|
508
|
+
* hashes every byte of that prefix, and takes it as verified only when
|
|
509
|
+
* every line's length and SHA-256 equal what was verified (no parsing, no
|
|
510
|
+
* schema check). The bytes after it, and every byte when nothing matches,
|
|
511
|
+
* are verified in full.
|
|
512
|
+
*/
|
|
466
513
|
async #catchUp(): Promise<void> {
|
|
467
514
|
// Records waiting for the disk are ahead of it; the lease keeps others out.
|
|
468
515
|
if (this.#pending.length > 0) return
|
|
469
516
|
const size = await this.#medium.size()
|
|
470
517
|
if (size === this.#synced + this.#torn && size >= this.#synced) return
|
|
471
518
|
if (size < this.#synced) this.#reset()
|
|
519
|
+
let ledger: VerifiedPrefix | undefined
|
|
520
|
+
const key = await this.#medium.identity?.()
|
|
521
|
+
this.#ledgerKey = key
|
|
522
|
+
if (key !== undefined) {
|
|
523
|
+
ledger = verifiedPrefixOf(key)
|
|
524
|
+
if (this.#synced === 0 && this.#chain.head === null) {
|
|
525
|
+
const remembered = ledger
|
|
526
|
+
if (
|
|
527
|
+
remembered !== undefined &&
|
|
528
|
+
remembered.count > 0 &&
|
|
529
|
+
remembered.sessionId === this.sessionId &&
|
|
530
|
+
remembered.end <= size &&
|
|
531
|
+
(await rehashPrefix(this.#medium, remembered, remembered.count))
|
|
532
|
+
) {
|
|
533
|
+
this.#chain = new SessionLogChain({
|
|
534
|
+
head: remembered.head,
|
|
535
|
+
sessionId: remembered.sessionId,
|
|
536
|
+
gen: remembered.gen,
|
|
537
|
+
})
|
|
538
|
+
this.#turns = remembered.turns()
|
|
539
|
+
this.#synced = remembered.end
|
|
540
|
+
} else {
|
|
541
|
+
ledger = startVerifiedPrefix(key)
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
}
|
|
472
545
|
const splitter = new LineSplitter(this.#synced)
|
|
473
546
|
for await (const chunk of this.#medium.stream(this.#synced)) {
|
|
474
547
|
for (const { line, offset } of splitter.push(chunk)) {
|
|
@@ -483,6 +556,7 @@ export class SessionLogCore implements SessionLog {
|
|
|
483
556
|
}
|
|
484
557
|
this.#turns.apply(entry.record)
|
|
485
558
|
this.#synced = offset + line.byteLength
|
|
559
|
+
ledger?.append(entry)
|
|
486
560
|
}
|
|
487
561
|
}
|
|
488
562
|
this.#torn = splitter.pendingBytes
|
|
@@ -761,9 +835,21 @@ export class SessionLogCore implements SessionLog {
|
|
|
761
835
|
const entry = this.#chain.accept(bytes, this.#synced)
|
|
762
836
|
this.#turns.apply(entry.record)
|
|
763
837
|
this.#synced += bytes.byteLength
|
|
838
|
+
// On the disk and fully verified: a later claim need not parse it again.
|
|
839
|
+
if (this.#pending.length === 0) await this.#rememberWritten(entry)
|
|
764
840
|
return entry
|
|
765
841
|
}
|
|
766
842
|
|
|
843
|
+
/** Let this process remember a record this instance just wrote and verified. */
|
|
844
|
+
async #rememberWritten(entry: SessionLogEntry): Promise<void> {
|
|
845
|
+
if (this.#ledgerKey === undefined) this.#ledgerKey = await this.#medium.identity?.()
|
|
846
|
+
if (this.#ledgerKey === undefined) return
|
|
847
|
+
const ledger =
|
|
848
|
+
verifiedPrefixOf(this.#ledgerKey) ??
|
|
849
|
+
(entry.pointer.seq === 1 ? startVerifiedPrefix(this.#ledgerKey) : undefined)
|
|
850
|
+
ledger?.append(entry)
|
|
851
|
+
}
|
|
852
|
+
|
|
767
853
|
/**
|
|
768
854
|
* Hand a record's bytes to the medium. A volume that refuses writes (full,
|
|
769
855
|
* over quota, read-only) does not fail the record: it waits in a bounded
|
|
@@ -888,6 +974,7 @@ export class SessionLogCore implements SessionLog {
|
|
|
888
974
|
if (
|
|
889
975
|
type !== 'message' &&
|
|
890
976
|
type !== 'message_replaced' &&
|
|
977
|
+
type !== 'message_edited' &&
|
|
891
978
|
type !== 'compaction' &&
|
|
892
979
|
type !== 'turn_completed'
|
|
893
980
|
)
|
|
@@ -926,7 +1013,7 @@ export class SessionLogCore implements SessionLog {
|
|
|
926
1013
|
const key =
|
|
927
1014
|
content.role === 'tool' && typeof content.toolCallId === 'string'
|
|
928
1015
|
? content.toolCallId
|
|
929
|
-
: type === 'message_replaced'
|
|
1016
|
+
: type === 'message_replaced' || type === 'message_edited'
|
|
930
1017
|
? `record:${String(candidate.id)}`
|
|
931
1018
|
: `message:${String(candidate.messageId)}`
|
|
932
1019
|
const ref = await this.#spillOrLose(key, 'message', JSON.stringify(content))
|
|
@@ -1093,7 +1180,7 @@ export async function collect(
|
|
|
1093
1180
|
* both backends and by `readSessionLog`.
|
|
1094
1181
|
*/
|
|
1095
1182
|
export async function* walkSessionLog(
|
|
1096
|
-
medium: Pick<LogMedium, 'size' | 'read' | 'stream'>,
|
|
1183
|
+
medium: Pick<LogMedium, 'size' | 'read' | 'stream' | 'identity' | 'scan'>,
|
|
1097
1184
|
options: ReadSessionLogOptions & { readonly sessionId?: SessionId },
|
|
1098
1185
|
): AsyncGenerator<SessionLogEntry, SessionLogReadSummary> {
|
|
1099
1186
|
const strict = (options.mode ?? 'strict') === 'strict'
|
|
@@ -1105,6 +1192,9 @@ export async function* walkSessionLog(
|
|
|
1105
1192
|
brokeAt = error
|
|
1106
1193
|
}
|
|
1107
1194
|
|
|
1195
|
+
// A resumed read that must also vouch for the bytes before its cursor.
|
|
1196
|
+
let covered = 0
|
|
1197
|
+
let ledger: VerifiedPrefix | undefined
|
|
1108
1198
|
if (options.after !== undefined) {
|
|
1109
1199
|
const after = options.after
|
|
1110
1200
|
if (!(await verifyPointer(medium, after))) {
|
|
@@ -1117,15 +1207,42 @@ export async function* walkSessionLog(
|
|
|
1117
1207
|
fail(error)
|
|
1118
1208
|
return { intact: false, throughSeq: 0, head: null, tornBytes: 0, break: error }
|
|
1119
1209
|
}
|
|
1120
|
-
const
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1210
|
+
const key = options.reverify && strict ? await medium.identity?.() : undefined
|
|
1211
|
+
if (key !== undefined) {
|
|
1212
|
+
const remembered = verifiedPrefixOf(key)
|
|
1213
|
+
if (
|
|
1214
|
+
remembered !== undefined &&
|
|
1215
|
+
remembered.sessionId === options.sessionId &&
|
|
1216
|
+
remembered.holds(after) &&
|
|
1217
|
+
(await rehashPrefix(medium, remembered, after.seq))
|
|
1218
|
+
) {
|
|
1219
|
+
ledger = remembered
|
|
1220
|
+
} else {
|
|
1221
|
+
// Nothing remembered, or it no longer matches: verify from byte 0 and
|
|
1222
|
+
// remember it, handing the caller only what follows the cursor.
|
|
1223
|
+
ledger = startVerifiedPrefix(key)
|
|
1224
|
+
covered = after.seq
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
if (covered > 0) {
|
|
1228
|
+
chain = new SessionLogChain({ head: null, sessionId: options.sessionId })
|
|
1229
|
+
} else {
|
|
1230
|
+
const parsed = parseSessionLogLine(await medium.read(after.offset, after.length))
|
|
1231
|
+
chain = new SessionLogChain({
|
|
1232
|
+
head: after,
|
|
1233
|
+
sessionId: options.sessionId ?? parsed.record.sessionId,
|
|
1234
|
+
gen: parsed.record.gen,
|
|
1235
|
+
})
|
|
1236
|
+
start = after.offset + after.length
|
|
1237
|
+
}
|
|
1127
1238
|
} else {
|
|
1128
1239
|
chain = new SessionLogChain({ head: null, sessionId: options.sessionId })
|
|
1240
|
+
// A whole read that asked to reverify is the first of its kind: what it verifies in
|
|
1241
|
+
// full is remembered, so the resumed reads after it can re-check by hash.
|
|
1242
|
+
if (options.reverify && strict) {
|
|
1243
|
+
const key = await medium.identity?.()
|
|
1244
|
+
if (key !== undefined) ledger = verifiedPrefixOf(key) ?? startVerifiedPrefix(key)
|
|
1245
|
+
}
|
|
1129
1246
|
}
|
|
1130
1247
|
|
|
1131
1248
|
const splitter = new LineSplitter(start)
|
|
@@ -1165,6 +1282,8 @@ export async function* walkSessionLog(
|
|
|
1165
1282
|
break outer
|
|
1166
1283
|
}
|
|
1167
1284
|
}
|
|
1285
|
+
ledger?.append(entry)
|
|
1286
|
+
if (entry.pointer.seq <= covered) continue
|
|
1168
1287
|
yield entry
|
|
1169
1288
|
if (options.throughSeq !== undefined && entry.pointer.seq >= options.throughSeq) {
|
|
1170
1289
|
stopped = true
|