@namzu/sdk 62.8.0 → 62.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/dist/compaction/plan.d.ts +12 -0
  3. package/dist/compaction/plan.d.ts.map +1 -1
  4. package/dist/compaction/plan.js +39 -11
  5. package/dist/compaction/plan.js.map +1 -1
  6. package/dist/compaction/salience/minhash.d.ts +9 -0
  7. package/dist/compaction/salience/minhash.d.ts.map +1 -1
  8. package/dist/compaction/salience/minhash.js +20 -0
  9. package/dist/compaction/salience/minhash.js.map +1 -1
  10. package/dist/compaction/salience/score.d.ts.map +1 -1
  11. package/dist/compaction/salience/score.js +87 -45
  12. package/dist/compaction/salience/score.js.map +1 -1
  13. package/dist/compaction/salience/working-set.d.ts +14 -0
  14. package/dist/compaction/salience/working-set.d.ts.map +1 -1
  15. package/dist/compaction/salience/working-set.js +84 -33
  16. package/dist/compaction/salience/working-set.js.map +1 -1
  17. package/dist/config/runtime.d.ts +2 -0
  18. package/dist/config/runtime.d.ts.map +1 -1
  19. package/dist/config/runtime.js +14 -0
  20. package/dist/config/runtime.js.map +1 -1
  21. package/dist/connector/http-operation.d.ts +1 -0
  22. package/dist/connector/http-operation.d.ts.map +1 -1
  23. package/dist/connector/http-operation.js +1 -1
  24. package/dist/connector/http-operation.js.map +1 -1
  25. package/dist/connector/mcp/adapter.d.ts.map +1 -1
  26. package/dist/connector/mcp/adapter.js +10 -0
  27. package/dist/connector/mcp/adapter.js.map +1 -1
  28. package/dist/connector/mcp/client.d.ts +12 -0
  29. package/dist/connector/mcp/client.d.ts.map +1 -1
  30. package/dist/connector/mcp/client.js +59 -10
  31. package/dist/connector/mcp/client.js.map +1 -1
  32. package/dist/constants/mcp/index.d.ts +15 -0
  33. package/dist/constants/mcp/index.d.ts.map +1 -1
  34. package/dist/constants/mcp/index.js +15 -0
  35. package/dist/constants/mcp/index.js.map +1 -1
  36. package/dist/manager/session/turn-recorder.d.ts +25 -0
  37. package/dist/manager/session/turn-recorder.d.ts.map +1 -1
  38. package/dist/manager/session/turn-recorder.js +193 -17
  39. package/dist/manager/session/turn-recorder.js.map +1 -1
  40. package/dist/public-runtime.d.ts +1 -1
  41. package/dist/public-runtime.d.ts.map +1 -1
  42. package/dist/public-runtime.js +1 -1
  43. package/dist/public-runtime.js.map +1 -1
  44. package/dist/runtime/query/cancelled-before-start.js +1 -1
  45. package/dist/runtime/query/cancelled-before-start.js.map +1 -1
  46. package/dist/runtime/query/checkpoint.js +2 -1
  47. package/dist/runtime/query/checkpoint.js.map +1 -1
  48. package/dist/runtime/query/index.d.ts.map +1 -1
  49. package/dist/runtime/query/index.js +4 -1
  50. package/dist/runtime/query/index.js.map +1 -1
  51. package/dist/runtime/query/iteration/phases/compaction.d.ts.map +1 -1
  52. package/dist/runtime/query/iteration/phases/compaction.js +15 -1
  53. package/dist/runtime/query/iteration/phases/compaction.js.map +1 -1
  54. package/dist/runtime/query/prepare-turn.js +10 -4
  55. package/dist/runtime/query/prepare-turn.js.map +1 -1
  56. package/dist/store/session-log/core.d.ts +31 -1
  57. package/dist/store/session-log/core.d.ts.map +1 -1
  58. package/dist/store/session-log/core.js +95 -9
  59. package/dist/store/session-log/core.js.map +1 -1
  60. package/dist/store/session-log/disk.d.ts +2 -0
  61. package/dist/store/session-log/disk.d.ts.map +1 -1
  62. package/dist/store/session-log/disk.js +37 -0
  63. package/dist/store/session-log/disk.js.map +1 -1
  64. package/dist/store/session-log/fold.d.ts +9 -1
  65. package/dist/store/session-log/fold.d.ts.map +1 -1
  66. package/dist/store/session-log/fold.js +18 -2
  67. package/dist/store/session-log/fold.js.map +1 -1
  68. package/dist/store/session-log/incremental.d.ts +8 -0
  69. package/dist/store/session-log/incremental.d.ts.map +1 -1
  70. package/dist/store/session-log/incremental.js +7 -2
  71. package/dist/store/session-log/incremental.js.map +1 -1
  72. package/dist/store/session-log/index.d.ts +1 -0
  73. package/dist/store/session-log/index.d.ts.map +1 -1
  74. package/dist/store/session-log/index.js +1 -0
  75. package/dist/store/session-log/index.js.map +1 -1
  76. package/dist/store/session-log/verified-prefix.d.ts +46 -0
  77. package/dist/store/session-log/verified-prefix.d.ts.map +1 -0
  78. package/dist/store/session-log/verified-prefix.js +233 -0
  79. package/dist/store/session-log/verified-prefix.js.map +1 -0
  80. package/dist/types/connector/mcp.d.ts +19 -0
  81. package/dist/types/connector/mcp.d.ts.map +1 -1
  82. package/dist/types/session/records.d.ts +80 -1
  83. package/dist/types/session/records.d.ts.map +1 -1
  84. package/dist/types/session/records.js +22 -0
  85. package/dist/types/session/records.js.map +1 -1
  86. package/package.json +1 -1
  87. package/src/compaction/plan.ts +53 -11
  88. package/src/compaction/salience/minhash.ts +20 -0
  89. package/src/compaction/salience/score.ts +112 -43
  90. package/src/compaction/salience/working-set.ts +102 -31
  91. package/src/config/runtime.ts +14 -0
  92. package/src/connector/http-operation.ts +1 -1
  93. package/src/connector/mcp/adapter.ts +11 -0
  94. package/src/connector/mcp/client.ts +75 -12
  95. package/src/constants/mcp/index.ts +17 -0
  96. package/src/manager/session/turn-recorder.ts +218 -15
  97. package/src/public-runtime.ts +1 -0
  98. package/src/runtime/query/cancelled-before-start.ts +1 -1
  99. package/src/runtime/query/checkpoint.ts +2 -1
  100. package/src/runtime/query/index.ts +4 -1
  101. package/src/runtime/query/iteration/phases/compaction.ts +18 -0
  102. package/src/runtime/query/prepare-turn.ts +10 -3
  103. package/src/store/session-log/core.ts +129 -10
  104. package/src/store/session-log/disk.ts +32 -0
  105. package/src/store/session-log/fold.ts +19 -2
  106. package/src/store/session-log/incremental.ts +15 -2
  107. package/src/store/session-log/index.ts +1 -0
  108. package/src/store/session-log/verified-prefix.ts +243 -0
  109. package/src/types/connector/mcp.ts +19 -0
  110. package/src/types/session/records.ts +25 -0
@@ -1,3 +1,4 @@
1
+ import { isDeepStrictEqual } from 'node:util'
1
2
  import {
2
3
  AUTO_CONTINUATION_USER_MESSAGE,
3
4
  REASONING_CUT_USER_MESSAGE,
@@ -94,6 +95,16 @@ function stampMessageId(message: Message, id: MessageId | undefined): void {
94
95
  else (message as { id?: MessageId }).id = id
95
96
  }
96
97
 
98
+ /** Two messages that differ only in their id and time are the same text. */
99
+ function sameStoredContent(a: Message, b: Message): boolean {
100
+ const { id: _a, timestamp: _ta, ...restA } = a
101
+ const { id: _b, timestamp: _tb, ...restB } = b
102
+ return isDeepStrictEqual(
103
+ JSON.parse(JSON.stringify(restA)) as unknown,
104
+ JSON.parse(JSON.stringify(restB)) as unknown,
105
+ )
106
+ }
107
+
97
108
  /** How {@link TurnRecorder.open} takes the session's writer lease and starts an empty log. */
98
109
  export interface TurnRecorderOpenOptions {
99
110
  /** A lease the caller already holds. Absent: the recorder claims one and releases it at the end. */
@@ -255,6 +266,8 @@ export class TurnRecorder {
255
266
  * these needs no re-basing, so no `compaction` record is written for it.
256
267
  */
257
268
  #foldIdsAtResume: readonly MessageId[] | undefined
269
+ /** What the log folds to for a message a replacement or an edit changed, as of the resume. */
270
+ #foldEditedAtResume = new Map<MessageId, Message>()
258
271
  /** When a resumed turn began: its `turn_started` record, not this process's start. */
259
272
  #resumedTurnStartedAt: number | undefined
260
273
 
@@ -868,21 +881,38 @@ export class TurnRecorder {
868
881
  async #readResumedLog(): Promise<readonly MessageId[] | undefined> {
869
882
  await this.flush()
870
883
  const fold = new SessionMessageFold()
884
+ // Whether the latest change to a message was a compaction edit: only
885
+ // that one is the recorder's to put back (a guardrail's replacement is
886
+ // the settled answer and the restored context is not allowed to undo it).
887
+ const latestChange = new Map<MessageId, 'edited' | 'replaced'>()
871
888
  for await (const entry of this.log.read()) {
872
889
  const record = entry.record as SessionRecord
873
890
  if (record.type === 'turn_started' && record.turnId === this.turnId) {
874
891
  const at = Date.parse(record.ts)
875
892
  if (Number.isFinite(at)) this.#resumedTurnStartedAt = at
876
893
  }
894
+ if (record.type === 'message_edited') latestChange.set(record.targetMessageId, 'edited')
895
+ else if (record.type === 'message_replaced')
896
+ latestChange.set(record.targetMessageId, 'replaced')
877
897
  fold.apply(record)
878
898
  }
879
- if (fold.spilledSummary) return undefined
880
899
  const ids: MessageId[] = []
900
+ let allRecorded = !fold.spilledSummary
901
+ this.#foldEditedAtResume.clear()
902
+ // Filled for every fold entry with an id, whether or not the fold also
903
+ // holds id-less entries (a summary): a restored context older than an edit
904
+ // is put right either way. An entry whose text is spilled is not
905
+ // compared, so a message edited to a spilled text is not put back.
881
906
  for (const entry of fold.entries()) {
882
- if (!entry.messageId) return undefined
907
+ if (!entry.messageId) {
908
+ allRecorded = false
909
+ continue
910
+ }
883
911
  ids.push(entry.messageId)
912
+ if (latestChange.get(entry.messageId) === 'edited' && entry.spill === undefined)
913
+ this.#foldEditedAtResume.set(entry.messageId, entry.message)
884
914
  }
885
- return ids
915
+ return allRecorded ? ids : undefined
886
916
  }
887
917
 
888
918
  /** The recorded context is exactly what the log folds to: a resume changed nothing. */
@@ -1093,6 +1123,13 @@ export class TurnRecorder {
1093
1123
  options: {
1094
1124
  readonly messageId?: MessageId
1095
1125
  readonly transient?: boolean
1126
+ /**
1127
+ * The message is part of the history a resume restores, not a new
1128
+ * one. While the resume's re-base is pending, a restored message with no
1129
+ * id is held and nothing is reconciled; the first message that is not
1130
+ * restored (or any other reconcile) re-bases once on the whole context.
1131
+ */
1132
+ readonly restored?: boolean
1096
1133
  } = {},
1097
1134
  ): void {
1098
1135
  if (options.messageId) {
@@ -1110,6 +1147,16 @@ export class TurnRecorder {
1110
1147
  })
1111
1148
  return
1112
1149
  }
1150
+ if (options.restored && this.#rebasePending && this.#phase === 'active') {
1151
+ // A restored message with no id (a compaction summary, the working-memory
1152
+ // slot, a message the host added) can arrive before the recorded ones.
1153
+ // Reconciling now would re-base the fold on a context holding only it and
1154
+ // drop the recorded messages that follow; hold it as an id-less view entry
1155
+ // and let the first reconciliation after the whole context is in place
1156
+ // write one `compaction` that keeps the recorded ids after it.
1157
+ this.#view.push({ message })
1158
+ return
1159
+ }
1113
1160
  this.#syncMessages()
1114
1161
  }
1115
1162
 
@@ -1152,6 +1199,100 @@ export class TurnRecorder {
1152
1199
  return id
1153
1200
  }
1154
1201
 
1202
+ /**
1203
+ * A compaction pass edits one message by replacing its object with a copy
1204
+ * that has other content (`{ ...message, content }`). The copy still carries
1205
+ * the id the original was stamped with, but the recorder's own indexes are
1206
+ * keyed by object identity, so without this it would see an unknown message
1207
+ * and record every message after it again. A live object that is not in the
1208
+ * view, whose `id` is that of a recorded view entry whose own object has left
1209
+ * the context, is that same message edited: one `message_edited` record
1210
+ * carries its new content and the view entry follows the new object.
1211
+ *
1212
+ * Only an id claimed by exactly one live object is taken, so two copies of
1213
+ * one message are never both treated as its edit. Anything else is left to
1214
+ * the callers, which record it as a new message.
1215
+ */
1216
+ #absorbEdits(): void {
1217
+ if (this.#phase !== 'active') return
1218
+ const live = this.#turn.messages
1219
+ const inContext = new Set<Message>(live)
1220
+ let orphans: Map<MessageId, ViewEntry> | undefined
1221
+ for (const entry of this.#view) {
1222
+ if (entry.id === undefined || entry.transient || inContext.has(entry.message)) continue
1223
+ orphans ??= new Map()
1224
+ orphans.set(entry.id, entry)
1225
+ }
1226
+ if (orphans === undefined) return
1227
+ const known = new Set<Message>(this.#view.map((entry) => entry.message))
1228
+ const claimed = new Map<MessageId, Message[]>()
1229
+ for (const message of live) {
1230
+ if (known.has(message) || this.#transient.has(message)) continue
1231
+ const id = message.id
1232
+ if (id === undefined || !orphans.has(id)) continue
1233
+ const list = claimed.get(id)
1234
+ if (list) list.push(message)
1235
+ else claimed.set(id, [message])
1236
+ }
1237
+ const turnId = this.turnId
1238
+ for (const [id, messages] of claimed) {
1239
+ const message = messages[0] as Message
1240
+ const entry = orphans.get(id) as ViewEntry
1241
+ if (messages.length !== 1 || message.role !== entry.message.role) continue
1242
+ // `previous` is the live object the view holds: a message mutated in place
1243
+ // and then replaced by a copy of itself is adopted without a record, and
1244
+ // the fold keeps the text it recorded.
1245
+ const previous = entry.message
1246
+ entry.message = message
1247
+ this.#ids.set(message, id)
1248
+ this.#recorded.set(message, id)
1249
+ // A copy with the same text (a structured clone, a re-parsed message)
1250
+ // is adopted as is: the fold already shows that content.
1251
+ if (sameStoredContent(previous, message)) continue
1252
+ void this.#enqueue(() =>
1253
+ this.#append({
1254
+ type: 'message_edited',
1255
+ turnId,
1256
+ targetMessageId: id,
1257
+ content: message,
1258
+ ignorable: true,
1259
+ reason: 'compaction',
1260
+ }),
1261
+ ).catch(() => undefined)
1262
+ }
1263
+ }
1264
+
1265
+ /**
1266
+ * A resume restores the context a checkpoint holds, which can predate a
1267
+ * compaction edit written after it. The ids still match, so the log is not
1268
+ * re-based; but the fold then shows the edited text of a message the context
1269
+ * holds as it was. One `message_edited` per such message makes the fold show
1270
+ * the context the resume actually runs with.
1271
+ */
1272
+ #recordDriftFromFold(): void {
1273
+ const edited = this.#foldEditedAtResume
1274
+ if (edited.size === 0) return
1275
+ const turnId = this.turnId
1276
+ for (const entry of this.#view) {
1277
+ if (entry.id === undefined || entry.transient) continue
1278
+ const folded = edited.get(entry.id)
1279
+ if (folded === undefined || sameStoredContent(folded, entry.message)) continue
1280
+ const message = entry.message
1281
+ const id = entry.id
1282
+ void this.#enqueue(() =>
1283
+ this.#append({
1284
+ type: 'message_edited',
1285
+ turnId,
1286
+ targetMessageId: id,
1287
+ content: message,
1288
+ ignorable: true,
1289
+ reason: 'resume',
1290
+ }),
1291
+ ).catch(() => undefined)
1292
+ }
1293
+ edited.clear()
1294
+ }
1295
+
1155
1296
  /**
1156
1297
  * Reconcile the context with the log. Nothing is written before the turn
1157
1298
  * begins or after it pauses or settles.
@@ -1164,9 +1305,15 @@ export class TurnRecorder {
1164
1305
  this.#foldIdsAtResume = undefined
1165
1306
  if (!unchanged) {
1166
1307
  this.#recordCompaction()
1308
+ // Kept ids whose restored text differs from the edited text the fold
1309
+ // holds get their record; entries re-recorded under fresh ids are not
1310
+ // in the map and are skipped.
1311
+ this.#recordDriftFromFold()
1167
1312
  return
1168
1313
  }
1314
+ this.#recordDriftFromFold()
1169
1315
  }
1316
+ this.#absorbEdits()
1170
1317
  const live = this.#turn.messages
1171
1318
  const view = this.#view
1172
1319
  let prefix = 0
@@ -1205,6 +1352,7 @@ export class TurnRecorder {
1205
1352
  * log order, the whole context becomes the summary.
1206
1353
  */
1207
1354
  #recordCompaction(): void {
1355
+ this.#absorbEdits()
1208
1356
  const live = this.#turn.messages
1209
1357
  const byMessage = new Map(this.#view.map((entry) => [entry.message, entry]))
1210
1358
  const next: ViewEntry[] = live.map((message) => {
@@ -1493,6 +1641,34 @@ async function recordedMessages(
1493
1641
  return out
1494
1642
  }
1495
1643
 
1644
+ type RecordedContent = { readonly content: Message; readonly spill?: SpillRef }
1645
+
1646
+ function recordedContent(record: {
1647
+ readonly content: Message
1648
+ readonly spill?: SpillRef | undefined
1649
+ }): RecordedContent {
1650
+ return { content: record.content, ...(record.spill ? { spill: record.spill } : {}) }
1651
+ }
1652
+
1653
+ /** Every message ever recorded, and the texts an edited one held before each edit. */
1654
+ export interface EverRecordedMessages {
1655
+ /** The latest text of each message, by its durable id. */
1656
+ readonly latest: ReadonlyMap<MessageId, Message>
1657
+ /**
1658
+ * Whether `id` held `message`'s text before a compaction edit. A
1659
+ * `message_replaced` for the id forgets the texts before it: a guardrail's
1660
+ * replacement is the settled answer. A spilled earlier text is read only
1661
+ * when a comparison needs it.
1662
+ */
1663
+ heldBeforeAnEdit(id: MessageId, message: Message): Promise<boolean>
1664
+ }
1665
+
1666
+ interface EverRecordedState {
1667
+ readonly latest: Map<MessageId, RecordedContent>
1668
+ /** Only ids that have an edit appear here. */
1669
+ readonly earlier: Map<MessageId, RecordedContent[]>
1670
+ }
1671
+
1496
1672
  /**
1497
1673
  * Every message this session's log has ever recorded, keyed by the durable
1498
1674
  * id its `message` record gave it — including one a compaction has since
@@ -1507,25 +1683,37 @@ async function recordedMessages(
1507
1683
  export async function readEverRecordedMessages(
1508
1684
  log: SessionLog,
1509
1685
  ): Promise<ReadonlyMap<MessageId, Message>> {
1686
+ return (await readEverRecordedMessageIndex(log)).latest
1687
+ }
1688
+
1689
+ /**
1690
+ * {@link readEverRecordedMessages} plus what a host that cached the history
1691
+ * before a compaction edit still holds, answered from the same reduction with
1692
+ * no read of the log of its own.
1693
+ */
1694
+ export async function readEverRecordedMessageIndex(log: SessionLog): Promise<EverRecordedMessages> {
1510
1695
  return foldSessionLog(log, {
1511
1696
  name: 'ever-recorded-messages',
1512
1697
  mode: 'strict',
1513
- create: () => new Map<MessageId, { readonly content: Message; readonly spill?: SpillRef }>(),
1514
- apply: (latest, entry) => {
1698
+ create: (): EverRecordedState => ({ latest: new Map(), earlier: new Map() }),
1699
+ apply: ({ latest, earlier }, entry) => {
1515
1700
  const record = entry.record as SessionRecord
1516
1701
  if (record.type === 'message') {
1517
- latest.set(record.messageId, {
1518
- content: record.content,
1519
- ...(record.spill ? { spill: record.spill } : {}),
1520
- })
1702
+ latest.set(record.messageId, recordedContent(record))
1521
1703
  } else if (record.type === 'message_replaced') {
1522
- latest.set(record.targetMessageId, {
1523
- content: record.content,
1524
- ...(record.spill ? { spill: record.spill } : {}),
1525
- })
1704
+ earlier.delete(record.targetMessageId)
1705
+ latest.set(record.targetMessageId, recordedContent(record))
1706
+ } else if (record.type === 'message_edited') {
1707
+ const before = latest.get(record.targetMessageId)
1708
+ if (before !== undefined) {
1709
+ const held = earlier.get(record.targetMessageId)
1710
+ if (held) held.push(before)
1711
+ else earlier.set(record.targetMessageId, [before])
1712
+ }
1713
+ latest.set(record.targetMessageId, recordedContent(record))
1526
1714
  }
1527
1715
  },
1528
- result: async (latest) => {
1716
+ result: async ({ latest, earlier }) => {
1529
1717
  const resolved = new Map<MessageId, Message>()
1530
1718
  for (const [id, entry] of latest) {
1531
1719
  // Copied: the map this is read from is shared with the next call.
@@ -1534,7 +1722,22 @@ export async function readEverRecordedMessages(
1534
1722
  : structuredClone(entry.content)
1535
1723
  resolved.set(id, withMessageId(content, id))
1536
1724
  }
1537
- return resolved
1725
+ // A snapshot: the arrays are advanced in place by the next call. The
1726
+ // entries in them are never changed once made.
1727
+ const held = new Map<MessageId, readonly RecordedContent[]>()
1728
+ for (const [id, list] of earlier) held.set(id, [...list])
1729
+ return {
1730
+ latest: resolved,
1731
+ async heldBeforeAnEdit(id, message) {
1732
+ for (const entry of held.get(id) ?? []) {
1733
+ const content = entry.spill
1734
+ ? (JSON.parse(await log.readSpill(entry.spill)) as Message)
1735
+ : entry.content
1736
+ if (sameStoredContent(content, message)) return true
1737
+ }
1738
+ return false
1739
+ },
1740
+ }
1538
1741
  },
1539
1742
  })
1540
1743
  }
@@ -1775,6 +1775,7 @@ export {
1775
1775
  ExternalRefSchema,
1776
1776
  IgnorableRecordSchema,
1777
1777
  MessageRecordSchema,
1778
+ MessageEditedRecordSchema,
1778
1779
  MessageReplacedRecordSchema,
1779
1780
  OriginSchema,
1780
1781
  OutcomeProgressWrittenRecordSchema,
@@ -86,7 +86,7 @@ export async function* settlePreStartCancellation(
86
86
 
87
87
  const push = (message: Message, ids: ReadonlyMap<Message, MessageId>): void => {
88
88
  const messageId = ids.get(message)
89
- ctx.recorder.pushMessage(message, messageId ? { messageId } : {})
89
+ ctx.recorder.pushMessage(message, { ...(messageId ? { messageId } : {}), restored: true })
90
90
  }
91
91
  try {
92
92
  if (selectedResumeState) {
@@ -822,7 +822,8 @@ async function readRecordedMessage(
822
822
  }
823
823
  if (
824
824
  (entry.type === 'message' && entry.messageId === messageId) ||
825
- (entry.type === 'message_replaced' && entry.targetMessageId === messageId)
825
+ ((entry.type === 'message_replaced' || entry.type === 'message_edited') &&
826
+ entry.targetMessageId === messageId)
826
827
  ) {
827
828
  found = { content: entry.content, ...(entry.spill ? { spill: entry.spill } : {}) }
828
829
  }
@@ -2248,7 +2248,10 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
2248
2248
  }
2249
2249
  const pushRestored = (message: Message, ids: ReadonlyMap<Message, MessageId>): void => {
2250
2250
  const messageId = ids.get(message)
2251
- ctx.recorder.pushMessage(message, messageId ? { messageId } : {})
2251
+ ctx.recorder.pushMessage(message, {
2252
+ ...(messageId ? { messageId } : {}),
2253
+ restored: true,
2254
+ })
2252
2255
  }
2253
2256
 
2254
2257
  if (params.resumeFromCheckpoint) {
@@ -7,6 +7,7 @@ import { replaceableSummaries, summarizeWithModel } from '../../../../compaction
7
7
  import {
8
8
  type CompactionPlan,
9
9
  DEFAULT_SOFT_TARGET,
10
+ canShedSalienceWorkingSet,
10
11
  planCompaction,
11
12
  planSalienceWorkingSet,
12
13
  } from '../../../../compaction/plan.js'
@@ -787,6 +788,23 @@ async function runCompactionCheckInner(
787
788
  const startAt = salience ? Math.min(config.softTarget ?? DEFAULT_SOFT_TARGET, trigger) : trigger
788
789
  if (!options?.force && usage < startAt) return
789
790
 
791
+ // A pass that can shed nothing and has not reached the summary trigger
792
+ // does nothing but announce itself: two hooks, a started event and a log
793
+ // line around a plan with no action. From the soft target until the
794
+ // trigger it ran on every iteration of every turn once every narration
795
+ // was cut and every large result cleared. Not asked of a forced pass, of
796
+ // a turn with a reducer (it owns the reduction), or at the trigger, where
797
+ // the summary path is what the pass is for.
798
+ if (
799
+ salience &&
800
+ !options?.force &&
801
+ !ctx.contextReducer &&
802
+ usage < trigger &&
803
+ !canShedSalienceWorkingSet(ctx.recorder.messages, config)
804
+ ) {
805
+ return
806
+ }
807
+
790
808
  // Three failed passes in a row: stop proactive compaction for the rest of
791
809
  // the turn. A pass the provider forced is never gated.
792
810
  const breaker = compactionBreaker(ctx)
@@ -8,7 +8,7 @@ import {
8
8
  import { assertSessionLogAttribution } from '../../manager/session/attribution.js'
9
9
  import {
10
10
  type RecordedMessage,
11
- readEverRecordedMessages,
11
+ readEverRecordedMessageIndex,
12
12
  readFoldedHistory,
13
13
  } from '../../manager/session/turn-recorder.js'
14
14
  import { resolveModelPricing } from '../../pricing/index.js'
@@ -1164,7 +1164,8 @@ async function reconcileCallerMessages(
1164
1164
  continuationMode: boolean,
1165
1165
  ): Promise<Message[]> {
1166
1166
  const hasId = messages.some((message) => message.id !== undefined)
1167
- const recordedById = hasId ? await readEverRecordedMessages(log) : undefined
1167
+ const everRecorded = hasId ? await readEverRecordedMessageIndex(log) : undefined
1168
+ const recordedById = everRecorded?.latest
1168
1169
  // Ids a caller message already claimed by direct lookup above: a fold
1169
1170
  // entry that gave one of THESE out must not also be offered to the no-id
1170
1171
  // alignment below, or a message that already reconciled by id could be
@@ -1182,7 +1183,13 @@ async function reconcileCallerMessages(
1182
1183
  if (id !== undefined) {
1183
1184
  const recorded = recordedById?.get(id)
1184
1185
  if (recorded === undefined) throw staleCachedHistoryError('foreign', message)
1185
- if (!sameMessageContent(recorded, message)) throw staleCachedHistoryError('edited', message)
1186
+ if (!sameMessageContent(recorded, message)) {
1187
+ // A compaction pass edits a message under its own id; a host that
1188
+ // cached the history before the pass still holds the first text.
1189
+ // Answered from the reduction already read: no read of the log.
1190
+ if (!(await everRecorded?.heldBeforeAnEdit(id, message)))
1191
+ throw staleCachedHistoryError('edited', message)
1192
+ }
1186
1193
  claimedIds.add(id)
1187
1194
  continue // already durable: the fold supplies it
1188
1195
  }
@@ -52,6 +52,12 @@ import {
52
52
  } from './lease.js'
53
53
  import type { SpillReadOptions, SpillRef, SpillStore } from './spill.js'
54
54
  import { TOOL_INPUT_RECORD_TYPES, spilledInput, toolInputSlots } from './spilled-input.js'
55
+ import {
56
+ type VerifiedPrefix,
57
+ rehashPrefix,
58
+ startVerifiedPrefix,
59
+ verifiedPrefixOf,
60
+ } from './verified-prefix.js'
55
61
 
56
62
  // ─── public types ─────────────────────────────────────────────────────────
57
63
 
@@ -91,6 +97,17 @@ export interface ReadSessionLogOptions {
91
97
  readonly mode?: 'strict' | 'tolerant'
92
98
  /** Resume after this record: its bytes are checked, then the walk continues from it. */
93
99
  readonly after?: RecordPointer
100
+ /**
101
+ * With `after`, and in `strict` mode only: also confirm that every byte
102
+ * before the cursor is what this process verified. The prefix is read and
103
+ * hashed line by line against what was remembered (no parsing, nothing
104
+ * kept); when this process remembers nothing of the log, or any byte
105
+ * differs, it is verified in full from byte 0, which refuses a broken chain
106
+ * as a read from byte 0 does. Without it a resumed read trusts the records
107
+ * before the cursor, as a reduction that is advanced rather than recomputed
108
+ * does. Ignored when the medium has no `identity`.
109
+ */
110
+ readonly reverify?: boolean
94
111
  /**
95
112
  * The record the log must hold at `expectHead.seq`, byte for byte. Anchors
96
113
  * the tail, which no later record vouches for.
@@ -288,6 +305,25 @@ export interface LogMedium extends LogBytes {
288
305
  truncate(size: number, expectedSize: number): Promise<void>
289
306
  /** The bytes from `offset` to the end, in chunks. */
290
307
  stream(offset: number): AsyncIterable<Uint8Array>
308
+ /**
309
+ * A string that names this log's bytes for as long as the same file stands
310
+ * (the disk medium: device, inode and path), or `undefined` when the log
311
+ * has none yet or the medium cannot say. A log that has one lets a later
312
+ * claim in the same process re-check the bytes it already verified without
313
+ * parsing them again; the content is still compared, so a wrong or reused
314
+ * identity can only cost a full verification, never skip one.
315
+ */
316
+ identity?(): Promise<string | undefined>
317
+ /**
318
+ * Hand the bytes from `offset` to `visit`, chunk by chunk, until `visit`
319
+ * answers `true` or the log ends. The chunk is only valid for the call: the
320
+ * medium may fill the same buffer again for the next one, so a visitor keeps
321
+ * no slice of it. It lets a re-check of a long log read through one buffer
322
+ * instead of allocating a fresh native buffer for every megabyte, which
323
+ * otherwise makes the collector run on every few walks. Optional; `stream`
324
+ * is used when a medium has no `scan`.
325
+ */
326
+ scan?(offset: number, visit: (chunk: Uint8Array) => boolean): Promise<void>
291
327
  }
292
328
 
293
329
  export interface SessionLogCoreOptions {
@@ -397,6 +433,8 @@ export class SessionLogCore implements SessionLog {
397
433
  readonly #holding: HeldSessionLease = { release: () => this.#releaseHeld() }
398
434
  /** Bytes of the log this instance has verified and applied. */
399
435
  #synced = 0
436
+ /** What this process remembers of the log's verified prefix (see `verified-prefix.ts`). */
437
+ #ledgerKey: string | undefined
400
438
  #torn = 0
401
439
  readonly #degradedBufferBytes: number
402
440
  /** Records accepted into the chain that the medium has not taken yet, oldest first. */
@@ -462,13 +500,48 @@ export class SessionLogCore implements SessionLog {
462
500
 
463
501
  // ── state ──
464
502
 
465
- /** Catch up with the medium: verify and apply whatever was appended since the last sync. */
503
+ /**
504
+ * Catch up with the medium: verify and apply whatever was appended since the last sync.
505
+ *
506
+ * An instance that has nothing yet (a fresh one) first asks whether this
507
+ * process already verified a prefix of this very log: it then re-reads and
508
+ * hashes every byte of that prefix, and takes it as verified only when
509
+ * every line's length and SHA-256 equal what was verified (no parsing, no
510
+ * schema check). The bytes after it, and every byte when nothing matches,
511
+ * are verified in full.
512
+ */
466
513
  async #catchUp(): Promise<void> {
467
514
  // Records waiting for the disk are ahead of it; the lease keeps others out.
468
515
  if (this.#pending.length > 0) return
469
516
  const size = await this.#medium.size()
470
517
  if (size === this.#synced + this.#torn && size >= this.#synced) return
471
518
  if (size < this.#synced) this.#reset()
519
+ let ledger: VerifiedPrefix | undefined
520
+ const key = await this.#medium.identity?.()
521
+ this.#ledgerKey = key
522
+ if (key !== undefined) {
523
+ ledger = verifiedPrefixOf(key)
524
+ if (this.#synced === 0 && this.#chain.head === null) {
525
+ const remembered = ledger
526
+ if (
527
+ remembered !== undefined &&
528
+ remembered.count > 0 &&
529
+ remembered.sessionId === this.sessionId &&
530
+ remembered.end <= size &&
531
+ (await rehashPrefix(this.#medium, remembered, remembered.count))
532
+ ) {
533
+ this.#chain = new SessionLogChain({
534
+ head: remembered.head,
535
+ sessionId: remembered.sessionId,
536
+ gen: remembered.gen,
537
+ })
538
+ this.#turns = remembered.turns()
539
+ this.#synced = remembered.end
540
+ } else {
541
+ ledger = startVerifiedPrefix(key)
542
+ }
543
+ }
544
+ }
472
545
  const splitter = new LineSplitter(this.#synced)
473
546
  for await (const chunk of this.#medium.stream(this.#synced)) {
474
547
  for (const { line, offset } of splitter.push(chunk)) {
@@ -483,6 +556,7 @@ export class SessionLogCore implements SessionLog {
483
556
  }
484
557
  this.#turns.apply(entry.record)
485
558
  this.#synced = offset + line.byteLength
559
+ ledger?.append(entry)
486
560
  }
487
561
  }
488
562
  this.#torn = splitter.pendingBytes
@@ -761,9 +835,21 @@ export class SessionLogCore implements SessionLog {
761
835
  const entry = this.#chain.accept(bytes, this.#synced)
762
836
  this.#turns.apply(entry.record)
763
837
  this.#synced += bytes.byteLength
838
+ // On the disk and fully verified: a later claim need not parse it again.
839
+ if (this.#pending.length === 0) await this.#rememberWritten(entry)
764
840
  return entry
765
841
  }
766
842
 
843
+ /** Let this process remember a record this instance just wrote and verified. */
844
+ async #rememberWritten(entry: SessionLogEntry): Promise<void> {
845
+ if (this.#ledgerKey === undefined) this.#ledgerKey = await this.#medium.identity?.()
846
+ if (this.#ledgerKey === undefined) return
847
+ const ledger =
848
+ verifiedPrefixOf(this.#ledgerKey) ??
849
+ (entry.pointer.seq === 1 ? startVerifiedPrefix(this.#ledgerKey) : undefined)
850
+ ledger?.append(entry)
851
+ }
852
+
767
853
  /**
768
854
  * Hand a record's bytes to the medium. A volume that refuses writes (full,
769
855
  * over quota, read-only) does not fail the record: it waits in a bounded
@@ -888,6 +974,7 @@ export class SessionLogCore implements SessionLog {
888
974
  if (
889
975
  type !== 'message' &&
890
976
  type !== 'message_replaced' &&
977
+ type !== 'message_edited' &&
891
978
  type !== 'compaction' &&
892
979
  type !== 'turn_completed'
893
980
  )
@@ -926,7 +1013,7 @@ export class SessionLogCore implements SessionLog {
926
1013
  const key =
927
1014
  content.role === 'tool' && typeof content.toolCallId === 'string'
928
1015
  ? content.toolCallId
929
- : type === 'message_replaced'
1016
+ : type === 'message_replaced' || type === 'message_edited'
930
1017
  ? `record:${String(candidate.id)}`
931
1018
  : `message:${String(candidate.messageId)}`
932
1019
  const ref = await this.#spillOrLose(key, 'message', JSON.stringify(content))
@@ -1093,7 +1180,7 @@ export async function collect(
1093
1180
  * both backends and by `readSessionLog`.
1094
1181
  */
1095
1182
  export async function* walkSessionLog(
1096
- medium: Pick<LogMedium, 'size' | 'read' | 'stream'>,
1183
+ medium: Pick<LogMedium, 'size' | 'read' | 'stream' | 'identity' | 'scan'>,
1097
1184
  options: ReadSessionLogOptions & { readonly sessionId?: SessionId },
1098
1185
  ): AsyncGenerator<SessionLogEntry, SessionLogReadSummary> {
1099
1186
  const strict = (options.mode ?? 'strict') === 'strict'
@@ -1105,6 +1192,9 @@ export async function* walkSessionLog(
1105
1192
  brokeAt = error
1106
1193
  }
1107
1194
 
1195
+ // A resumed read that must also vouch for the bytes before its cursor.
1196
+ let covered = 0
1197
+ let ledger: VerifiedPrefix | undefined
1108
1198
  if (options.after !== undefined) {
1109
1199
  const after = options.after
1110
1200
  if (!(await verifyPointer(medium, after))) {
@@ -1117,15 +1207,42 @@ export async function* walkSessionLog(
1117
1207
  fail(error)
1118
1208
  return { intact: false, throughSeq: 0, head: null, tornBytes: 0, break: error }
1119
1209
  }
1120
- const parsed = parseSessionLogLine(await medium.read(after.offset, after.length))
1121
- chain = new SessionLogChain({
1122
- head: after,
1123
- sessionId: options.sessionId ?? parsed.record.sessionId,
1124
- gen: parsed.record.gen,
1125
- })
1126
- start = after.offset + after.length
1210
+ const key = options.reverify && strict ? await medium.identity?.() : undefined
1211
+ if (key !== undefined) {
1212
+ const remembered = verifiedPrefixOf(key)
1213
+ if (
1214
+ remembered !== undefined &&
1215
+ remembered.sessionId === options.sessionId &&
1216
+ remembered.holds(after) &&
1217
+ (await rehashPrefix(medium, remembered, after.seq))
1218
+ ) {
1219
+ ledger = remembered
1220
+ } else {
1221
+ // Nothing remembered, or it no longer matches: verify from byte 0 and
1222
+ // remember it, handing the caller only what follows the cursor.
1223
+ ledger = startVerifiedPrefix(key)
1224
+ covered = after.seq
1225
+ }
1226
+ }
1227
+ if (covered > 0) {
1228
+ chain = new SessionLogChain({ head: null, sessionId: options.sessionId })
1229
+ } else {
1230
+ const parsed = parseSessionLogLine(await medium.read(after.offset, after.length))
1231
+ chain = new SessionLogChain({
1232
+ head: after,
1233
+ sessionId: options.sessionId ?? parsed.record.sessionId,
1234
+ gen: parsed.record.gen,
1235
+ })
1236
+ start = after.offset + after.length
1237
+ }
1127
1238
  } else {
1128
1239
  chain = new SessionLogChain({ head: null, sessionId: options.sessionId })
1240
+ // A whole read that asked to reverify is the first of its kind: what it verifies in
1241
+ // full is remembered, so the resumed reads after it can re-check by hash.
1242
+ if (options.reverify && strict) {
1243
+ const key = await medium.identity?.()
1244
+ if (key !== undefined) ledger = verifiedPrefixOf(key) ?? startVerifiedPrefix(key)
1245
+ }
1129
1246
  }
1130
1247
 
1131
1248
  const splitter = new LineSplitter(start)
@@ -1165,6 +1282,8 @@ export async function* walkSessionLog(
1165
1282
  break outer
1166
1283
  }
1167
1284
  }
1285
+ ledger?.append(entry)
1286
+ if (entry.pointer.seq <= covered) continue
1168
1287
  yield entry
1169
1288
  if (options.throughSeq !== undefined && entry.pointer.seq >= options.throughSeq) {
1170
1289
  stopped = true