@namzu/sdk 48.3.1 → 49.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/dist/config/registry.d.ts +2 -0
  3. package/dist/config/registry.d.ts.map +1 -1
  4. package/dist/config/registry.js +23 -3
  5. package/dist/config/registry.js.map +1 -1
  6. package/dist/connector/mcp/discovery.d.ts +9 -5
  7. package/dist/connector/mcp/discovery.d.ts.map +1 -1
  8. package/dist/connector/mcp/discovery.js +16 -8
  9. package/dist/connector/mcp/discovery.js.map +1 -1
  10. package/dist/connector/mcp/mcp-toolset.d.ts +3 -0
  11. package/dist/connector/mcp/mcp-toolset.d.ts.map +1 -1
  12. package/dist/connector/mcp/mcp-toolset.js +20 -2
  13. package/dist/connector/mcp/mcp-toolset.js.map +1 -1
  14. package/dist/connector/mcp/reconnect.d.ts +2 -0
  15. package/dist/connector/mcp/reconnect.d.ts.map +1 -1
  16. package/dist/connector/mcp/reconnect.js +36 -5
  17. package/dist/connector/mcp/reconnect.js.map +1 -1
  18. package/dist/plugin/lifecycle.d.ts +14 -10
  19. package/dist/plugin/lifecycle.d.ts.map +1 -1
  20. package/dist/plugin/lifecycle.js +178 -41
  21. package/dist/plugin/lifecycle.js.map +1 -1
  22. package/dist/prompt/resident-learning.js +2 -2
  23. package/dist/prompt/resident-learning.js.map +1 -1
  24. package/dist/prompt/resident-step.d.ts +2 -2
  25. package/dist/prompt/resident-step.js +4 -4
  26. package/dist/prompt/resident-step.js.map +1 -1
  27. package/dist/public-runtime.d.ts +2 -2
  28. package/dist/public-runtime.d.ts.map +1 -1
  29. package/dist/public-runtime.js +2 -2
  30. package/dist/public-runtime.js.map +1 -1
  31. package/dist/registry/tool/screen.d.ts.map +1 -1
  32. package/dist/registry/tool/screen.js +33 -2
  33. package/dist/registry/tool/screen.js.map +1 -1
  34. package/dist/runtime/jobs/registry.d.ts.map +1 -1
  35. package/dist/runtime/jobs/registry.js +26 -5
  36. package/dist/runtime/jobs/registry.js.map +1 -1
  37. package/dist/runtime/query/index.d.ts +6 -0
  38. package/dist/runtime/query/index.d.ts.map +1 -1
  39. package/dist/runtime/query/index.js +20 -1
  40. package/dist/runtime/query/index.js.map +1 -1
  41. package/dist/runtime/query/iteration/index.d.ts.map +1 -1
  42. package/dist/runtime/query/iteration/index.js +32 -8
  43. package/dist/runtime/query/iteration/index.js.map +1 -1
  44. package/dist/runtime/query/iteration/outstanding-work.d.ts.map +1 -1
  45. package/dist/runtime/query/iteration/outstanding-work.js +2 -0
  46. package/dist/runtime/query/iteration/outstanding-work.js.map +1 -1
  47. package/dist/runtime/query/iteration/phases/context.d.ts +4 -0
  48. package/dist/runtime/query/iteration/phases/context.d.ts.map +1 -1
  49. package/dist/runtime/query/iteration/phases/context.js.map +1 -1
  50. package/dist/runtime/query/iteration/phases/tool-review.d.ts.map +1 -1
  51. package/dist/runtime/query/iteration/phases/tool-review.js +7 -1
  52. package/dist/runtime/query/iteration/phases/tool-review.js.map +1 -1
  53. package/dist/runtime/query/iteration/step-shaping.d.ts +2 -0
  54. package/dist/runtime/query/iteration/step-shaping.d.ts.map +1 -1
  55. package/dist/runtime/query/iteration/step-shaping.js +5 -1
  56. package/dist/runtime/query/iteration/step-shaping.js.map +1 -1
  57. package/dist/store/index.d.ts +1 -0
  58. package/dist/store/index.d.ts.map +1 -1
  59. package/dist/store/index.js +1 -0
  60. package/dist/store/index.js.map +1 -1
  61. package/dist/store/memory/disk.d.ts +7 -2
  62. package/dist/store/memory/disk.d.ts.map +1 -1
  63. package/dist/store/memory/disk.js +58 -5
  64. package/dist/store/memory/disk.js.map +1 -1
  65. package/dist/store/memory/index.d.ts +9 -0
  66. package/dist/store/memory/index.d.ts.map +1 -1
  67. package/dist/store/memory/index.js +43 -5
  68. package/dist/store/memory/index.js.map +1 -1
  69. package/dist/store/memory/markdown.d.ts +7 -2
  70. package/dist/store/memory/markdown.d.ts.map +1 -1
  71. package/dist/store/memory/markdown.js +33 -0
  72. package/dist/store/memory/markdown.js.map +1 -1
  73. package/dist/store/memory/memory.d.ts +7 -2
  74. package/dist/store/memory/memory.d.ts.map +1 -1
  75. package/dist/store/memory/memory.js +34 -0
  76. package/dist/store/memory/memory.js.map +1 -1
  77. package/dist/store/memory/revision.d.ts +16 -0
  78. package/dist/store/memory/revision.d.ts.map +1 -0
  79. package/dist/store/memory/revision.js +144 -0
  80. package/dist/store/memory/revision.js.map +1 -0
  81. package/dist/tools/builtins/wait-for-job-bounds.d.ts +6 -4
  82. package/dist/tools/builtins/wait-for-job-bounds.d.ts.map +1 -1
  83. package/dist/tools/builtins/wait-for-job-bounds.js +42 -8
  84. package/dist/tools/builtins/wait-for-job-bounds.js.map +1 -1
  85. package/dist/tools/builtins/wait-for-job.d.ts +1 -0
  86. package/dist/tools/builtins/wait-for-job.d.ts.map +1 -1
  87. package/dist/tools/builtins/wait-for-job.js +24 -10
  88. package/dist/tools/builtins/wait-for-job.js.map +1 -1
  89. package/dist/tools/memory/delete.d.ts.map +1 -1
  90. package/dist/tools/memory/delete.js +38 -3
  91. package/dist/tools/memory/delete.js.map +1 -1
  92. package/dist/tools/memory/read.d.ts.map +1 -1
  93. package/dist/tools/memory/read.js +38 -2
  94. package/dist/tools/memory/read.js.map +1 -1
  95. package/dist/tools/memory/search.d.ts.map +1 -1
  96. package/dist/tools/memory/search.js +38 -7
  97. package/dist/tools/memory/search.js.map +1 -1
  98. package/dist/tools/memory/update.d.ts.map +1 -1
  99. package/dist/tools/memory/update.js +30 -3
  100. package/dist/tools/memory/update.js.map +1 -1
  101. package/dist/toolsets/manager.d.ts +13 -0
  102. package/dist/toolsets/manager.d.ts.map +1 -1
  103. package/dist/toolsets/manager.js +60 -1
  104. package/dist/toolsets/manager.js.map +1 -1
  105. package/dist/turn/memory-recall.d.ts +2 -0
  106. package/dist/turn/memory-recall.d.ts.map +1 -1
  107. package/dist/turn/memory-recall.js +3 -1
  108. package/dist/turn/memory-recall.js.map +1 -1
  109. package/dist/types/memory/index.d.ts +29 -0
  110. package/dist/types/memory/index.d.ts.map +1 -1
  111. package/dist/types/memory/index.js +6 -0
  112. package/dist/types/memory/index.js.map +1 -1
  113. package/dist/types/plugin/index.d.ts +12 -2
  114. package/dist/types/plugin/index.d.ts.map +1 -1
  115. package/dist/types/plugin/index.js +5 -2
  116. package/dist/types/plugin/index.js.map +1 -1
  117. package/package.json +1 -1
  118. package/src/config/registry.ts +24 -3
  119. package/src/connector/mcp/discovery.ts +29 -8
  120. package/src/connector/mcp/mcp-toolset.ts +25 -4
  121. package/src/connector/mcp/reconnect.ts +34 -4
  122. package/src/plugin/lifecycle.ts +217 -46
  123. package/src/prompt/resident-learning.ts +2 -2
  124. package/src/prompt/resident-step.ts +4 -4
  125. package/src/public-runtime.ts +2 -0
  126. package/src/registry/tool/screen.ts +34 -2
  127. package/src/runtime/jobs/registry.ts +25 -8
  128. package/src/runtime/query/index.ts +24 -1
  129. package/src/runtime/query/iteration/index.ts +35 -10
  130. package/src/runtime/query/iteration/outstanding-work.ts +2 -0
  131. package/src/runtime/query/iteration/phases/context.ts +4 -0
  132. package/src/runtime/query/iteration/phases/tool-review.ts +6 -1
  133. package/src/runtime/query/iteration/step-shaping.ts +9 -1
  134. package/src/store/index.ts +1 -0
  135. package/src/store/memory/disk.ts +82 -7
  136. package/src/store/memory/index.ts +66 -7
  137. package/src/store/memory/markdown.ts +54 -2
  138. package/src/store/memory/memory.ts +52 -2
  139. package/src/store/memory/revision.ts +158 -0
  140. package/src/tools/builtins/wait-for-job-bounds.ts +42 -7
  141. package/src/tools/builtins/wait-for-job.ts +25 -11
  142. package/src/tools/memory/delete.ts +33 -3
  143. package/src/tools/memory/read.ts +35 -2
  144. package/src/tools/memory/search.ts +38 -7
  145. package/src/tools/memory/update.ts +28 -3
  146. package/src/toolsets/manager.ts +74 -1
  147. package/src/turn/memory-recall.ts +5 -1
  148. package/src/types/memory/index.ts +43 -0
  149. package/src/types/plugin/index.ts +11 -6
@@ -4,7 +4,7 @@ import { SANDBOX_KILL_GRACE_MS } from '../../constants/sandbox/index.js'
4
4
  import { killTree } from '../../process/kill-tree.js'
5
5
  import { hostShellSpawn } from '../../tools/command-shell.js'
6
6
  import { scrubInheritedEnv } from '../../tools/env-scrub.js'
7
- import { awaitWithAbort } from '../../utils/await-with-abort.js'
7
+ import { subscribeToAbort } from '../../utils/abort.js'
8
8
 
9
9
  /**
10
10
  * Work that outlives a tool call, owned by whoever started it.
@@ -343,13 +343,30 @@ export class BackgroundJobRegistry {
343
343
  const entry = this.jobs.get(id)
344
344
  if (!entry) throw new UnknownBackgroundJobError({ id })
345
345
  if (entry.record.status !== 'running') return Promise.resolve(entry.record)
346
- // `entry` is the same object the `finalize` closure in `start()`
347
- // mutates in place, so reading `entry.record` after `exit` settles
348
- // sees the final status — the same trick `kill()` already relies on.
349
- return awaitWithAbort(
350
- entry.exit.then(() => entry.record),
351
- opts.signal,
352
- )
346
+ if (!opts.signal) {
347
+ // Keep the original promise path for callers without cancellation.
348
+ return entry.exit.then(() => entry.record)
349
+ }
350
+ const signal = opts.signal
351
+ if (signal.aborted) return Promise.reject(signal.reason)
352
+
353
+ // A Promise.race against entry.exit leaves a continuation on that
354
+ // never-ending promise after cancellation. An exit listener can instead
355
+ // be removed as soon as this particular wait is abandoned.
356
+ return new Promise<BackgroundJob>((resolve, reject) => {
357
+ let disposeAbort = () => {}
358
+ const unsubscribe = this.onExit((job) => {
359
+ if (job.id !== id) return
360
+ unsubscribe()
361
+ disposeAbort()
362
+ resolve(job)
363
+ })
364
+ disposeAbort = subscribeToAbort(signal, () => {
365
+ unsubscribe()
366
+ disposeAbort()
367
+ reject(signal.reason)
368
+ })
369
+ })
353
370
  }
354
371
 
355
372
  /**
@@ -264,6 +264,12 @@ export interface QueryParams {
264
264
  * finishes.
265
265
  */
266
266
  backgroundJobOwner?: string
267
+ /**
268
+ * Called with job ids only when their exit notices enter model context:
269
+ * a tool result or a recorded job-exit message. An exit event alone does not
270
+ * mean the model saw it. A host can carry unacknowledged exits to its next turn.
271
+ */
272
+ onJobNoticeDelivered?: (jobIds: readonly string[]) => void
267
273
 
268
274
  /**
269
275
  * What else goes in this turn's prompt.
@@ -1319,6 +1325,20 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
1319
1325
  // subscription that fills it below, because the wait-intent recorder needs
1320
1326
  // to ask whether its text has been read yet.
1321
1327
  const jobNotices = params.backgroundJobs ? new SteeringBinding() : undefined
1328
+ const pendingJobNoticeIds = new Set<string>()
1329
+ const acknowledgeJobNotices = () => {
1330
+ if (pendingJobNoticeIds.size === 0) return
1331
+ const ids = [...pendingJobNoticeIds]
1332
+ pendingJobNoticeIds.clear()
1333
+ try {
1334
+ params.onJobNoticeDelivered?.(ids)
1335
+ } catch (error) {
1336
+ // A host observer cannot retract model-visible context or fail the turn.
1337
+ ctx.log.warn('A background job notice observer failed', {
1338
+ 'exception.message': error instanceof Error ? error.message : String(error),
1339
+ })
1340
+ }
1341
+ }
1322
1342
  // Jobs the model told `wait_for_job` it is waiting on, which is the only
1323
1343
  // thing that can hold this turn open for a job. Built only where there is a
1324
1344
  // registry, so a host with no background mode carries no recorder and the
@@ -1425,6 +1445,7 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
1425
1445
  jobNotices?.steer(
1426
1446
  `Background job ${job.id} (${job.command}) ${outcome}. Read its output with the job tool if you need it.`,
1427
1447
  )
1448
+ pendingJobNoticeIds.add(job.id)
1428
1449
  void eventTranslator
1429
1450
  .emitEvent({
1430
1451
  type: 'background_job_exited',
@@ -1567,6 +1588,7 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
1567
1588
  : {}),
1568
1589
  tools: toolManager,
1569
1590
  allowedTools: effectiveAllowedTools,
1591
+ showDeferredToolContext: params.contextLevel !== 'minimal',
1570
1592
  recorder: ctx.recorder,
1571
1593
  toolExecutor,
1572
1594
  guard,
@@ -1582,6 +1604,7 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
1582
1604
  promptContributions,
1583
1605
  ...(params.steering ? { steering: params.steering } : {}),
1584
1606
  ...(jobNotices ? { jobNotices } : {}),
1607
+ ...(jobNotices ? { onJobNoticeDelivered: acknowledgeJobNotices } : {}),
1585
1608
  ...(awaitedJobs ? { awaitedJobs } : {}),
1586
1609
  checkpointMgr,
1587
1610
  planManager: ctx.planManager,
@@ -1807,7 +1830,7 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
1807
1830
  if (annotations.length > 0) {
1808
1831
  promptContributions.replace({
1809
1832
  id: 'hooks:user_prompt_submit',
1810
- placement: 'dynamic',
1833
+ placement: 'context',
1811
1834
  render: () => `Context from the operator's hooks:\n\n${annotations.join('\n\n')}`,
1812
1835
  })
1813
1836
  }
@@ -3,7 +3,9 @@ import {
3
3
  extractFromAssistantMessage,
4
4
  extractFromUserMessage,
5
5
  } from '../../../compaction/extractor.js'
6
+ import { estimateMessageTokens } from '../../../compaction/token-estimate.js'
6
7
  import { AUTO_CONTINUATION_USER_MESSAGE } from '../../../constants/continuation.js'
8
+ import { CHARS_PER_TOKEN } from '../../../constants/limits.js'
7
9
  import {
8
10
  DEFAULT_STRUCTURED_OUTPUT_RETRIES,
9
11
  STRUCTURED_OUTPUT_REPROMPT,
@@ -548,13 +550,37 @@ export class IterationOrchestrator {
548
550
  // Shape this step before calling the model. `stopWhen` decides
549
551
  // whether to keep going; this decides HOW. No-op when the host
550
552
  // supplied no hook.
553
+ // Render request-only contributions once before preparation. Their
554
+ // estimated cost reduces every stage's headroom, and the same text
555
+ // reaches the provider even if a stage changes external state.
556
+ const contextSections =
557
+ this.ctx.promptContributions?.render('context', { iteration: iterationNum }) ?? []
558
+ const deferredMcpHints =
559
+ this.ctx.showDeferredToolContext !== false
560
+ ? this.ctx.tools.snapshotUntrustedDeferredContext()
561
+ : undefined
562
+ const pricedContextText = [
563
+ ...contextSections,
564
+ ...(deferredMcpHints?.all ? [deferredMcpHints.all] : []),
565
+ ].join('\n\n')
566
+ const shaping: StepShaping = {
567
+ ...this.stepShaping(),
568
+ requestContextTokens: pricedContextText
569
+ ? estimateMessageTokens(stepContextMessage(pricedContextText)) +
570
+ Math.ceil(
571
+ ((deferredMcpHints?.maxRenderedChars ?? 0) -
572
+ (deferredMcpHints?.all.length ?? 0)) /
573
+ CHARS_PER_TOKEN,
574
+ )
575
+ : 0,
576
+ }
551
577
  const contextModelBeforePreparation = this.ctx.contextModel ?? model
552
- const step = await prepareStep(this.stepShaping(), iterationNum)
578
+ const step = await prepareStep(shaping, iterationNum)
553
579
  // Preparation inference belongs to the turn, not the main-model step.
554
580
  usageBefore = { ...recorder.tokenUsage }
555
581
  costBefore = { ...recorder.costInfo }
556
582
  stepModel = step.model ?? model
557
- await selectContextModel(this.stepShaping(), stepModel)
583
+ await selectContextModel(shaping, stepModel)
558
584
  // Preserve post-compaction preparation/recall semantics. A changed
559
585
  // model needs a second check against its own window; never replay
560
586
  // host preparation effects merely to rebuild its request guidance.
@@ -563,6 +589,11 @@ export class IterationOrchestrator {
563
589
  yield* this.ctx.drainPending()
564
590
 
565
591
  const stepAllowedTools = step.allowedTools ?? this.ctx.allowedTools
592
+ const deferredMcpContext = deferredMcpHints?.forNames(stepAllowedTools) ?? ''
593
+ const contextText = [
594
+ ...contextSections,
595
+ ...(deferredMcpContext ? [deferredMcpContext] : []),
596
+ ].join('\n\n')
566
597
  const llmTools = this.ctx.tools
567
598
  .toLLMTools(stepAllowedTools)
568
599
  .filter(
@@ -643,11 +674,6 @@ export class IterationOrchestrator {
643
674
  // (compaction preserves it there) and leaves the request's
644
675
  // system run here.
645
676
  const workingMemory = splitWorkingMemoryForRequest(baseMessages)
646
- const contextSections =
647
- this.ctx.promptContributions?.render('context', {
648
- iteration: iterationNum,
649
- }) ?? []
650
-
651
677
  const stepPreamble = [step.system, stepSkills, policyNotice, ...turnSections]
652
678
  .filter(Boolean)
653
679
  .join('\n\n')
@@ -655,15 +681,14 @@ export class IterationOrchestrator {
655
681
  ? [...workingMemory.history, createSystemMessage(stepPreamble)]
656
682
  : [...workingMemory.history]
657
683
  if (workingMemory.context) requestHistory.push(workingMemory.context)
658
- if (contextSections.length > 0)
659
- requestHistory.push(stepContextMessage(contextSections.join('\n\n')))
684
+ if (contextText) requestHistory.push(stepContextMessage(contextText))
660
685
  if (step.context) requestHistory.push(stepContextMessage(step.context))
661
686
  const messages = projectRequestRichContent(
662
687
  this.projectObservations(requestHistory),
663
688
  this.ctx.turnConfig.maxRequestRichContentBytes ??
664
689
  DEFAULT_MAX_REQUEST_RICH_CONTENT_BYTES,
665
690
  )
666
- appendWorkContext(this.stepShaping(), messages, iterationNum, step)
691
+ appendWorkContext(shaping, messages, iterationNum, step)
667
692
  if (closingDirective) messages.push(closingDirective)
668
693
  await this.reportUnsupportedToolResults(messages)
669
694
  yield* this.ctx.drainPending()
@@ -260,6 +260,7 @@ export function deliverAwaitedJobExits(ctx: IterationContext): boolean {
260
260
  'namzu.runtime.jobs': delivered.exits.map((job) => job.id),
261
261
  })
262
262
  ctx.recorder.pushMessage(createRuntimeContextMessage(formatJobNote(delivered.text), 'job-exit'))
263
+ ctx.onJobNoticeDelivered?.()
263
264
  return true
264
265
  }
265
266
 
@@ -383,4 +384,5 @@ export function deliverArrivedJobExits(ctx: IterationContext): void {
383
384
  'namzu.runtime.jobs': delivered.exits.map((job) => job.id),
384
385
  })
385
386
  ctx.recorder.pushMessage(createRuntimeContextMessage(formatJobNote(delivered.text), 'job-exit'))
387
+ ctx.onJobNoticeDelivered?.()
386
388
  }
@@ -98,6 +98,8 @@ export interface IterationContext {
98
98
  readonly structuredOutput?: StructuredOutputConfig
99
99
  readonly tools: ToolManager
100
100
  readonly allowedTools?: string[]
101
+ /** The minimal context level omits tool discovery from both system and request context. */
102
+ readonly showDeferredToolContext?: boolean
101
103
  readonly recorder: TurnRecorder
102
104
  readonly toolExecutor: ToolExecutor
103
105
  readonly guard: GuardCoordinator
@@ -141,6 +143,8 @@ export interface IterationContext {
141
143
  readonly onSteeringDelivered?: (text: string) => void
142
144
  /** Exit notices for the turn's background jobs, drained into the next tool result. */
143
145
  readonly jobNotices?: SteeringChannel
146
+ /** Acknowledge exit notices only after their text enters model context. */
147
+ readonly onJobNoticeDelivered?: () => void
144
148
  /**
145
149
  * Background jobs the model said it is waiting on, which is the only kind
146
150
  * the loop holds a finishing run open for.
@@ -186,6 +186,7 @@ export async function* runToolReview(
186
186
  // block must be answered by a `tool_result` with the same id, so a
187
187
  // user message wedged between them is rejected by the provider. Same
188
188
  // delivery a denial already uses, without the refusal.
189
+ let jobNoticeDelivered = false
189
190
  for (const msg of attachRepeatNotice(
190
191
  attachNotice(
191
192
  attachSteering(batch.messages, ctx.steering, ctx.onSteeringDelivered),
@@ -194,12 +195,16 @@ export async function* runToolReview(
194
195
  // The exits that text accounts for have now been read, so the
195
196
  // record of them stops being pending work. Left standing, it
196
197
  // buys the model a turn the next time any job queues a notice.
197
- () => ctx.awaitedJobs?.noticesDelivered(),
198
+ () => {
199
+ ctx.awaitedJobs?.noticesDelivered()
200
+ jobNoticeDelivered = true
201
+ },
198
202
  ),
199
203
  notices,
200
204
  )) {
201
205
  ctx.recorder.pushMessage(msg)
202
206
  }
207
+ if (jobNoticeDelivered) ctx.onJobNoticeDelivered?.()
203
208
  // The complete tool-result batch is already in history before host policy
204
209
  // may react, so a replacement cannot split provider-required adjacency.
205
210
  // Commit each accepted observation before entering the next one. A single
@@ -38,6 +38,8 @@ export interface StepShaping {
38
38
  readonly ctx: IterationContext
39
39
  readonly latestUserMessage: () => UserMessage | undefined
40
40
  readonly steps: () => readonly StepResult[]
41
+ /** Request-only contributions already rendered for this model step. */
42
+ readonly requestContextTokens?: number
41
43
  }
42
44
 
43
45
  export { stepContextMessage }
@@ -108,7 +110,13 @@ export function stepContext(
108
110
  windowTokens: window.tokens,
109
111
  remainingTokens: Math.max(
110
112
  0,
111
- Math.floor(window.tokens - measureContext(ctx).tokens - preparedTokens - responseReserve),
113
+ Math.floor(
114
+ window.tokens -
115
+ measureContext(ctx).tokens -
116
+ (shaping.requestContextTokens ?? 0) -
117
+ preparedTokens -
118
+ responseReserve,
119
+ ),
112
120
  ),
113
121
  },
114
122
  steps: steps(),
@@ -36,6 +36,7 @@ export {
36
36
  slugifyMemoryName,
37
37
  } from './memory/naming.js'
38
38
  export type { MemoryContentRejection } from './memory/naming.js'
39
+ export { MemoryRevisionConflictError } from './memory/revision.js'
39
40
 
40
41
  // Was that answer any good — recorded per message, durably, with
41
42
  // compare-and-set. Every consumer used to invent its own side table for the
@@ -3,14 +3,15 @@ import { dirname, join, resolve } from 'node:path'
3
3
  import { NamzuError } from '../../types/errors/index.js'
4
4
  import type { MemoryId } from '../../types/ids/index.js'
5
5
  import type {
6
+ ConditionalMemoryStore,
6
7
  CreateMemoryParams,
7
8
  MemoryContent,
8
9
  MemoryIndexEntry,
9
10
  MemoryRecord,
10
11
  MemorySearchParams,
11
12
  MemorySearchResult,
12
- MemoryStore,
13
13
  UpdateMemoryParams,
14
+ VersionedMemoryRecord,
14
15
  } from '../../types/memory/index.js'
15
16
  import { assertMemoryStatus, isMemoryType } from '../../types/memory/index.js'
16
17
  import { generateMemoryId, isEntityId } from '../../utils/id.js'
@@ -18,7 +19,7 @@ import { SCOPE_ATTRIBUTE } from '../../utils/log/types.js'
18
19
  import { type Logger, resolveLogger } from '../../utils/logger.js'
19
20
  import { DiskRecordStore } from '../kv/record-store.js'
20
21
  import { SCHEMA_VERSION_KEY, defineSchema } from '../schema.js'
21
- import { InMemoryMemoryIndex, searchMemoryEntries } from './index.js'
22
+ import { InMemoryMemoryIndex, searchMemoryEntries, selectMemorySearchCandidates } from './index.js'
22
23
  import {
23
24
  MemoryNameConflictError,
24
25
  assertOptionalMemoryFields,
@@ -31,6 +32,7 @@ import {
31
32
  acquireMemoryOperationLock,
32
33
  validateMemoryLockTimeout,
33
34
  } from './operation-lock.js'
35
+ import { MemoryRevisionConflictError, assertMemoryRevision, memoryRevision } from './revision.js'
34
36
 
35
37
  /**
36
38
  * This store's on-disk format, versioned as a unit — which is how a
@@ -181,7 +183,7 @@ export interface DiskMemoryStoreConfig {
181
183
  lockTimeoutMs?: number
182
184
  }
183
185
 
184
- export class DiskMemoryStore implements MemoryStore {
186
+ export class DiskMemoryStore implements ConditionalMemoryStore {
185
187
  private readonly baseDir: string
186
188
  private readonly log: Logger
187
189
  private readonly lockTimeoutMs: number
@@ -355,18 +357,54 @@ export class DiskMemoryStore implements MemoryStore {
355
357
  })
356
358
  }
357
359
 
360
+ async getVersionedRecord(id: MemoryId): Promise<VersionedMemoryRecord | undefined> {
361
+ return this.withAuthoritativeIndex(async (location) => {
362
+ assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' }))
363
+ const entry = this.index.getEntry(id)
364
+ if (!entry) return undefined
365
+ const record = structuredClone({ entry, content: await this.readContent(location, id) })
366
+ return { ...record, revision: memoryRevision(record) }
367
+ })
368
+ }
369
+
358
370
  async update(id: MemoryId, updates: UpdateMemoryParams): Promise<MemoryIndexEntry | undefined> {
371
+ return this.updateInternal(id, updates)
372
+ }
373
+
374
+ async updateIfRevision(
375
+ id: MemoryId,
376
+ updates: UpdateMemoryParams,
377
+ expectedRevision: string,
378
+ ): Promise<MemoryIndexEntry> {
379
+ const updated = await this.updateInternal(id, updates, expectedRevision)
380
+ if (!updated) throw new MemoryRevisionConflictError(id)
381
+ return updated
382
+ }
383
+
384
+ private async updateInternal(
385
+ id: MemoryId,
386
+ updates: UpdateMemoryParams,
387
+ expectedRevision?: string,
388
+ ): Promise<MemoryIndexEntry | undefined> {
359
389
  if (updates.status !== undefined) assertMemoryStatus(updates.status)
360
390
  assertOptionalMemoryFields(updates)
361
391
  return this.withAuthoritativeIndex(async (location) => {
362
392
  assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' }))
363
393
  const existing = this.index.getEntry(id)
394
+ const existingContent = existing ? await this.readContent(location, id) : undefined
395
+ if (expectedRevision !== undefined) {
396
+ assertMemoryRevision(
397
+ id,
398
+ existing && existingContent ? { entry: existing, content: existingContent } : undefined,
399
+ expectedRevision,
400
+ )
401
+ }
364
402
  if (!existing) return undefined
365
403
  if (updates.name !== undefined) {
366
404
  const holder = nameHolder(this.index.allEntries(), updates.name, id)
367
405
  if (holder) throw new MemoryNameConflictError(updates.name, holder.id)
368
406
  }
369
- const existingContent = await this.readContent(location, id)
407
+ if (!existingContent) invalidContent(id, 'the indexed content record is missing')
370
408
  const updated: MemoryIndexEntry = withOptionalFields(
371
409
  {
372
410
  ...existing,
@@ -403,9 +441,27 @@ export class DiskMemoryStore implements MemoryStore {
403
441
  }
404
442
 
405
443
  async delete(id: MemoryId): Promise<boolean> {
444
+ return this.deleteInternal(id)
445
+ }
446
+
447
+ async deleteIfRevision(id: MemoryId, expectedRevision: string): Promise<void> {
448
+ const deleted = await this.deleteInternal(id, expectedRevision)
449
+ if (!deleted) throw new MemoryRevisionConflictError(id)
450
+ }
451
+
452
+ private async deleteInternal(id: MemoryId, expectedRevision?: string): Promise<boolean> {
406
453
  return this.withAuthoritativeIndex(async (location) => {
407
454
  assertStorageMemoryId(id, (reason) => invalidContent(id, reason, { field: 'id' }))
408
- if (!this.index.getEntry(id)) return false
455
+ const entry = this.index.getEntry(id)
456
+ if (expectedRevision !== undefined) {
457
+ const content = entry ? await this.readContent(location, id) : undefined
458
+ assertMemoryRevision(
459
+ id,
460
+ entry && content ? { entry, content } : undefined,
461
+ expectedRevision,
462
+ )
463
+ }
464
+ if (!entry) return false
409
465
 
410
466
  try {
411
467
  await unlink(this.contentPath(location, id))
@@ -432,12 +488,31 @@ export class DiskMemoryStore implements MemoryStore {
432
488
  query: undefined,
433
489
  limit: undefined,
434
490
  requiredIdentifiers: undefined,
491
+ maxScanned: undefined,
492
+ scanOffset: undefined,
435
493
  })
494
+ const selection = selectMemorySearchCandidates(candidates.entries, params)
436
495
  const contents = new Map<MemoryId, string>()
437
- for (const entry of candidates.entries) {
496
+ for (const entry of selection.entries) {
438
497
  contents.set(entry.id, (await this.readContent(location, entry.id)).content)
439
498
  }
440
- return searchMemoryEntries(candidates.entries, params, (id) => contents.get(id) ?? '')
499
+ const matched = searchMemoryEntries(
500
+ selection.entries,
501
+ { ...params, maxScanned: undefined, scanOffset: undefined },
502
+ (id) => contents.get(id) ?? '',
503
+ )
504
+ return {
505
+ ...matched,
506
+ ...(params.maxScanned !== undefined || params.scanOffset !== undefined
507
+ ? {
508
+ truncated: selection.truncated,
509
+ scannedCount: selection.scannedCount,
510
+ ...(selection.nextScanOffset !== undefined
511
+ ? { nextScanOffset: selection.nextScanOffset }
512
+ : {}),
513
+ }
514
+ : {}),
515
+ }
441
516
  })
442
517
  }
443
518
 
@@ -26,6 +26,56 @@ export function matchesMemoryIdentifier(text: string, identifiers: ReadonlySet<s
26
26
  return false
27
27
  }
28
28
 
29
+ interface MemoryScanSelection {
30
+ readonly entries: readonly MemoryIndexEntry[]
31
+ readonly truncated: boolean
32
+ readonly scannedCount: number
33
+ readonly nextScanOffset?: number
34
+ }
35
+
36
+ /** Select a bounded, resumable snapshot of indexed candidates before reading any bodies. */
37
+ export function selectMemorySearchCandidates(
38
+ entries: readonly MemoryIndexEntry[],
39
+ params: MemorySearchParams,
40
+ ): MemoryScanSelection {
41
+ if (
42
+ params.maxScanned !== undefined &&
43
+ (!Number.isSafeInteger(params.maxScanned) || params.maxScanned < 1)
44
+ ) {
45
+ throw new RangeError('maxScanned must be a positive safe integer')
46
+ }
47
+ if (
48
+ params.scanOffset !== undefined &&
49
+ (!Number.isSafeInteger(params.scanOffset) || params.scanOffset < 0)
50
+ ) {
51
+ throw new RangeError('scanOffset must be a nonnegative safe integer')
52
+ }
53
+ const query = terms(params.query ?? '')
54
+ if (params.query?.trim() && query.size === 0)
55
+ return { entries: [], truncated: false, scannedCount: 0 }
56
+ const bodySearch =
57
+ query.size > 0 ||
58
+ Boolean(params.requiredIdentifiers?.length) ||
59
+ params.maxScanned !== undefined ||
60
+ params.scanOffset !== undefined
61
+ if (!bodySearch) return { entries, truncated: false, scannedCount: 0 }
62
+ const ordered = [...entries].sort(
63
+ (a, b) => b.updatedAt - a.updatedAt || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0),
64
+ )
65
+ const start = params.scanOffset ?? 0
66
+ const selected = ordered.slice(
67
+ start,
68
+ params.maxScanned === undefined ? undefined : start + params.maxScanned,
69
+ )
70
+ const next = start + selected.length
71
+ return {
72
+ entries: selected,
73
+ truncated: next < ordered.length,
74
+ scannedCount: selected.length,
75
+ ...(next < ordered.length ? { nextScanOffset: next } : {}),
76
+ }
77
+ }
78
+
29
79
  /** Shared lexical ranking; a caller-owned index can omit body content. */
30
80
  export function searchMemoryEntries(
31
81
  entries: readonly MemoryIndexEntry[],
@@ -36,13 +86,13 @@ export function searchMemoryEntries(
36
86
  const identifiers = new Set(
37
87
  params.requiredIdentifiers?.map((id) => id.normalize('NFKC').toLowerCase()),
38
88
  )
39
- if (params.query?.trim() && query.size === 0) return { entries: [], totalCount: 0 }
40
- const ranked = entries
41
- .filter(
42
- (entry) =>
43
- (!params.status || entry.status === params.status) &&
44
- (!params.tags?.length || params.tags.every((tag) => entry.tags.includes(tag))),
45
- )
89
+ const candidates = entries.filter(
90
+ (entry) =>
91
+ (!params.status || entry.status === params.status) &&
92
+ (!params.tags?.length || params.tags.every((tag) => entry.tags.includes(tag))),
93
+ )
94
+ const selection = selectMemorySearchCandidates(candidates, params)
95
+ const ranked = selection.entries
46
96
  .filter(
47
97
  (entry) =>
48
98
  !identifiers.size ||
@@ -79,6 +129,15 @@ export function searchMemoryEntries(
79
129
  return {
80
130
  entries: ranked.slice(0, params.limit ?? ranked.length).map(({ entry }) => entry),
81
131
  totalCount: ranked.length,
132
+ ...(params.maxScanned !== undefined || params.scanOffset !== undefined
133
+ ? {
134
+ truncated: selection.truncated,
135
+ scannedCount: selection.scannedCount,
136
+ ...(selection.nextScanOffset !== undefined
137
+ ? { nextScanOffset: selection.nextScanOffset }
138
+ : {}),
139
+ }
140
+ : {}),
82
141
  }
83
142
  }
84
143
 
@@ -5,15 +5,16 @@ import { dirname, join, resolve } from 'node:path'
5
5
  import { NamzuError } from '../../types/errors/index.js'
6
6
  import type { MemoryId } from '../../types/ids/index.js'
7
7
  import type {
8
+ ConditionalMemoryStore,
8
9
  CreateMemoryParams,
9
10
  MemoryContent,
10
11
  MemoryIndexEntry,
11
12
  MemoryRecord,
12
13
  MemorySearchParams,
13
14
  MemorySearchResult,
14
- MemoryStore,
15
15
  MemoryType,
16
16
  UpdateMemoryParams,
17
+ VersionedMemoryRecord,
17
18
  } from '../../types/memory/index.js'
18
19
  import { assertMemoryStatus, isMemoryType } from '../../types/memory/index.js'
19
20
  import { atomicWriteFile } from '../../utils/atomic-write.js'
@@ -47,6 +48,7 @@ import {
47
48
  acquireMemoryOperationLock,
48
49
  validateMemoryLockTimeout,
49
50
  } from './operation-lock.js'
51
+ import { MemoryRevisionConflictError, assertMemoryRevision, memoryRevision } from './revision.js'
50
52
 
51
53
  /**
52
54
  * Versioned like every other record this SDK persists. A memory file carries
@@ -296,7 +298,7 @@ function decodeMemoryFile(
296
298
  * holds with {@link MemoryNameConflictError}, and derives a free one from the
297
299
  * title when none is given.
298
300
  */
299
- export class MarkdownMemoryStore implements MemoryStore {
301
+ export class MarkdownMemoryStore implements ConditionalMemoryStore {
300
302
  private readonly directory: string
301
303
  private readonly log: Logger
302
304
  private readonly lockTimeoutMs: number
@@ -587,6 +589,15 @@ export class MarkdownMemoryStore implements MemoryStore {
587
589
  })
588
590
  }
589
591
 
592
+ async getVersionedRecord(id: MemoryId): Promise<VersionedMemoryRecord | undefined> {
593
+ return this.withLoaded(async (_dir, loaded) => {
594
+ const memory = loaded.get(id)
595
+ if (!memory) return undefined
596
+ const record = structuredClone({ entry: memory.entry, content: memory.content })
597
+ return { ...record, revision: memoryRevision(record) }
598
+ })
599
+ }
600
+
590
601
  /** The current record held under `name`, archived included, or `undefined`. */
591
602
  async getByName(name: string): Promise<MemoryRecord | undefined> {
592
603
  return this.withLoaded(async (_dir, loaded) => {
@@ -603,10 +614,35 @@ export class MarkdownMemoryStore implements MemoryStore {
603
614
  }
604
615
 
605
616
  async update(id: MemoryId, updates: UpdateMemoryParams): Promise<MemoryIndexEntry | undefined> {
617
+ return this.updateInternal(id, updates)
618
+ }
619
+
620
+ async updateIfRevision(
621
+ id: MemoryId,
622
+ updates: UpdateMemoryParams,
623
+ expectedRevision: string,
624
+ ): Promise<MemoryIndexEntry> {
625
+ const updated = await this.updateInternal(id, updates, expectedRevision)
626
+ if (!updated) throw new MemoryRevisionConflictError(id)
627
+ return updated
628
+ }
629
+
630
+ private async updateInternal(
631
+ id: MemoryId,
632
+ updates: UpdateMemoryParams,
633
+ expectedRevision?: string,
634
+ ): Promise<MemoryIndexEntry | undefined> {
606
635
  if (updates.status !== undefined) assertMemoryStatus(updates.status)
607
636
  assertOptionalMemoryFields(updates)
608
637
  return this.mutate(async (dir, loaded) => {
609
638
  const existing = loaded.get(id)
639
+ if (expectedRevision !== undefined) {
640
+ assertMemoryRevision(
641
+ id,
642
+ existing ? { entry: existing.entry, content: existing.content } : undefined,
643
+ expectedRevision,
644
+ )
645
+ }
610
646
  if (!existing) return undefined
611
647
  const entries = this.entries(loaded)
612
648
  if (updates.name !== undefined) {
@@ -645,8 +681,24 @@ export class MarkdownMemoryStore implements MemoryStore {
645
681
  }
646
682
 
647
683
  async delete(id: MemoryId): Promise<boolean> {
684
+ return this.deleteInternal(id)
685
+ }
686
+
687
+ async deleteIfRevision(id: MemoryId, expectedRevision: string): Promise<void> {
688
+ const deleted = await this.deleteInternal(id, expectedRevision)
689
+ if (!deleted) throw new MemoryRevisionConflictError(id)
690
+ }
691
+
692
+ private async deleteInternal(id: MemoryId, expectedRevision?: string): Promise<boolean> {
648
693
  return this.mutate(async (dir, loaded) => {
649
694
  const existing = loaded.get(id)
695
+ if (expectedRevision !== undefined) {
696
+ assertMemoryRevision(
697
+ id,
698
+ existing ? { entry: existing.entry, content: existing.content } : undefined,
699
+ expectedRevision,
700
+ )
701
+ }
650
702
  if (!existing) return false
651
703
  try {
652
704
  await unlink(existing.path)