@namzu/sdk 62.11.0 → 62.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/dist/connector/index.d.ts +1 -1
  3. package/dist/connector/index.d.ts.map +1 -1
  4. package/dist/connector/index.js +1 -1
  5. package/dist/connector/index.js.map +1 -1
  6. package/dist/connector/mcp/client.d.ts +96 -0
  7. package/dist/connector/mcp/client.d.ts.map +1 -1
  8. package/dist/connector/mcp/client.js +488 -13
  9. package/dist/connector/mcp/client.js.map +1 -1
  10. package/dist/connector/mcp/index.d.ts +1 -0
  11. package/dist/connector/mcp/index.d.ts.map +1 -1
  12. package/dist/connector/mcp/index.js +1 -0
  13. package/dist/connector/mcp/index.js.map +1 -1
  14. package/dist/connector/mcp/tasks.d.ts +40 -0
  15. package/dist/connector/mcp/tasks.d.ts.map +1 -0
  16. package/dist/connector/mcp/tasks.js +55 -0
  17. package/dist/connector/mcp/tasks.js.map +1 -0
  18. package/dist/constants/mcp/index.d.ts +1 -1
  19. package/dist/constants/mcp/index.d.ts.map +1 -1
  20. package/dist/constants/mcp/index.js +5 -0
  21. package/dist/constants/mcp/index.js.map +1 -1
  22. package/dist/eval/bootstrap.d.ts +122 -0
  23. package/dist/eval/bootstrap.d.ts.map +1 -0
  24. package/dist/eval/bootstrap.js +227 -0
  25. package/dist/eval/bootstrap.js.map +1 -0
  26. package/dist/eval/compare.d.ts +109 -0
  27. package/dist/eval/compare.d.ts.map +1 -0
  28. package/dist/eval/compare.js +324 -0
  29. package/dist/eval/compare.js.map +1 -0
  30. package/dist/eval/disclosure.d.ts +70 -0
  31. package/dist/eval/disclosure.d.ts.map +1 -0
  32. package/dist/eval/disclosure.js +64 -0
  33. package/dist/eval/disclosure.js.map +1 -0
  34. package/dist/eval/experiment.d.ts +23 -0
  35. package/dist/eval/experiment.d.ts.map +1 -1
  36. package/dist/eval/experiment.js +87 -15
  37. package/dist/eval/experiment.js.map +1 -1
  38. package/dist/eval/from-turn.d.ts.map +1 -1
  39. package/dist/eval/from-turn.js +5 -0
  40. package/dist/eval/from-turn.js.map +1 -1
  41. package/dist/eval/index.d.ts +13 -2
  42. package/dist/eval/index.d.ts.map +1 -1
  43. package/dist/eval/index.js +6 -1
  44. package/dist/eval/index.js.map +1 -1
  45. package/dist/eval/report-stats.d.ts +77 -0
  46. package/dist/eval/report-stats.d.ts.map +1 -0
  47. package/dist/eval/report-stats.js +184 -0
  48. package/dist/eval/report-stats.js.map +1 -0
  49. package/dist/eval/run-options.d.ts +30 -0
  50. package/dist/eval/run-options.d.ts.map +1 -0
  51. package/dist/eval/run-options.js +20 -0
  52. package/dist/eval/run-options.js.map +1 -0
  53. package/dist/eval/run-summary.d.ts +36 -0
  54. package/dist/eval/run-summary.d.ts.map +1 -0
  55. package/dist/eval/run-summary.js +57 -0
  56. package/dist/eval/run-summary.js.map +1 -0
  57. package/dist/eval/scorers.d.ts +32 -0
  58. package/dist/eval/scorers.d.ts.map +1 -1
  59. package/dist/eval/scorers.js +77 -0
  60. package/dist/eval/scorers.js.map +1 -1
  61. package/dist/eval/types.d.ts +74 -0
  62. package/dist/eval/types.d.ts.map +1 -1
  63. package/dist/manager/session/turn-recorder.d.ts +3 -0
  64. package/dist/manager/session/turn-recorder.d.ts.map +1 -1
  65. package/dist/manager/session/turn-recorder.js +20 -5
  66. package/dist/manager/session/turn-recorder.js.map +1 -1
  67. package/dist/public-runtime.d.ts +5 -2
  68. package/dist/public-runtime.d.ts.map +1 -1
  69. package/dist/public-runtime.js +7 -2
  70. package/dist/public-runtime.js.map +1 -1
  71. package/dist/public-types.d.ts +2 -0
  72. package/dist/public-types.d.ts.map +1 -1
  73. package/dist/registry/command/kernel-commands.d.ts +3 -0
  74. package/dist/registry/command/kernel-commands.d.ts.map +1 -1
  75. package/dist/registry/command/kernel-commands.js +76 -5
  76. package/dist/registry/command/kernel-commands.js.map +1 -1
  77. package/dist/runtime/query/abandon-turn.d.ts.map +1 -1
  78. package/dist/runtime/query/abandon-turn.js +2 -2
  79. package/dist/runtime/query/abandon-turn.js.map +1 -1
  80. package/dist/runtime/query/index.d.ts.map +1 -1
  81. package/dist/runtime/query/index.js +15 -0
  82. package/dist/runtime/query/index.js.map +1 -1
  83. package/dist/runtime/query/iteration/index.d.ts +18 -1
  84. package/dist/runtime/query/iteration/index.d.ts.map +1 -1
  85. package/dist/runtime/query/iteration/index.js +80 -1
  86. package/dist/runtime/query/iteration/index.js.map +1 -1
  87. package/dist/runtime/query/iteration/phases/tool-review.d.ts.map +1 -1
  88. package/dist/runtime/query/iteration/phases/tool-review.js +18 -1
  89. package/dist/runtime/query/iteration/phases/tool-review.js.map +1 -1
  90. package/dist/runtime/query/prelude-lease.d.ts.map +1 -1
  91. package/dist/runtime/query/prelude-lease.js +2 -2
  92. package/dist/runtime/query/prelude-lease.js.map +1 -1
  93. package/dist/runtime/query/turn/answer-claims.d.ts +52 -0
  94. package/dist/runtime/query/turn/answer-claims.d.ts.map +1 -0
  95. package/dist/runtime/query/turn/answer-claims.js +348 -0
  96. package/dist/runtime/query/turn/answer-claims.js.map +1 -0
  97. package/dist/runtime/query/turn/outcome-ledger.d.ts +7 -0
  98. package/dist/runtime/query/turn/outcome-ledger.d.ts.map +1 -1
  99. package/dist/runtime/query/turn/outcome-ledger.js +13 -0
  100. package/dist/runtime/query/turn/outcome-ledger.js.map +1 -1
  101. package/dist/store/goal/index.d.ts +3 -1
  102. package/dist/store/goal/index.d.ts.map +1 -1
  103. package/dist/store/goal/index.js +6 -0
  104. package/dist/store/goal/index.js.map +1 -1
  105. package/dist/store/session-log/index.d.ts +1 -0
  106. package/dist/store/session-log/index.d.ts.map +1 -1
  107. package/dist/store/session-log/index.js +1 -0
  108. package/dist/store/session-log/index.js.map +1 -1
  109. package/dist/store/session-log/turn-holder.d.ts +26 -0
  110. package/dist/store/session-log/turn-holder.d.ts.map +1 -0
  111. package/dist/store/session-log/turn-holder.js +79 -0
  112. package/dist/store/session-log/turn-holder.js.map +1 -0
  113. package/dist/telemetry/runtime-accessors.d.ts +14 -0
  114. package/dist/telemetry/runtime-accessors.d.ts.map +1 -1
  115. package/dist/telemetry/runtime-accessors.js +32 -0
  116. package/dist/telemetry/runtime-accessors.js.map +1 -1
  117. package/dist/tools/goal/confirmation.d.ts +154 -0
  118. package/dist/tools/goal/confirmation.d.ts.map +1 -0
  119. package/dist/tools/goal/confirmation.js +290 -0
  120. package/dist/tools/goal/confirmation.js.map +1 -0
  121. package/dist/tools/goal/index.d.ts +4 -1
  122. package/dist/tools/goal/index.d.ts.map +1 -1
  123. package/dist/tools/goal/index.js +27 -2
  124. package/dist/tools/goal/index.js.map +1 -1
  125. package/dist/tools/hand-back/index.d.ts +48 -0
  126. package/dist/tools/hand-back/index.d.ts.map +1 -0
  127. package/dist/tools/hand-back/index.js +129 -0
  128. package/dist/tools/hand-back/index.js.map +1 -0
  129. package/dist/tools/schedules/schedule-tool.d.ts.map +1 -1
  130. package/dist/tools/schedules/schedule-tool.js +15 -1
  131. package/dist/tools/schedules/schedule-tool.js.map +1 -1
  132. package/dist/tools/schedules/types.d.ts +10 -0
  133. package/dist/tools/schedules/types.d.ts.map +1 -1
  134. package/dist/types/connector/mcp.d.ts +18 -0
  135. package/dist/types/connector/mcp.d.ts.map +1 -1
  136. package/dist/types/goal/index.d.ts +12 -0
  137. package/dist/types/goal/index.d.ts.map +1 -1
  138. package/dist/types/session/config.d.ts +8 -0
  139. package/dist/types/session/config.d.ts.map +1 -1
  140. package/dist/types/session/hand-back.d.ts +29 -0
  141. package/dist/types/session/hand-back.d.ts.map +1 -0
  142. package/dist/types/session/hand-back.js +11 -0
  143. package/dist/types/session/hand-back.js.map +1 -0
  144. package/dist/types/session/outcome.d.ts +37 -0
  145. package/dist/types/session/outcome.d.ts.map +1 -1
  146. package/dist/types/session/outcome.js.map +1 -1
  147. package/dist/types/session/turn.d.ts +19 -1
  148. package/dist/types/session/turn.d.ts.map +1 -1
  149. package/dist/types/session/turn.js.map +1 -1
  150. package/package.json +1 -1
  151. package/src/connector/index.ts +1 -0
  152. package/src/connector/mcp/client.ts +531 -14
  153. package/src/connector/mcp/index.ts +1 -0
  154. package/src/connector/mcp/tasks.ts +81 -0
  155. package/src/constants/mcp/index.ts +6 -1
  156. package/src/eval/bootstrap.ts +314 -0
  157. package/src/eval/compare.ts +473 -0
  158. package/src/eval/disclosure.ts +136 -0
  159. package/src/eval/experiment.ts +127 -15
  160. package/src/eval/from-turn.ts +5 -0
  161. package/src/eval/index.ts +51 -0
  162. package/src/eval/report-stats.ts +257 -0
  163. package/src/eval/run-options.ts +49 -0
  164. package/src/eval/run-summary.ts +104 -0
  165. package/src/eval/scorers.ts +83 -0
  166. package/src/eval/types.ts +76 -0
  167. package/src/manager/session/turn-recorder.ts +25 -4
  168. package/src/public-runtime.ts +53 -0
  169. package/src/public-types.ts +9 -0
  170. package/src/registry/command/kernel-commands.ts +94 -6
  171. package/src/runtime/query/abandon-turn.ts +2 -1
  172. package/src/runtime/query/index.ts +18 -0
  173. package/src/runtime/query/iteration/index.ts +85 -1
  174. package/src/runtime/query/iteration/phases/tool-review.ts +24 -1
  175. package/src/runtime/query/prelude-lease.ts +7 -2
  176. package/src/runtime/query/turn/answer-claims.ts +413 -0
  177. package/src/runtime/query/turn/outcome-ledger.ts +12 -0
  178. package/src/store/goal/index.ts +11 -0
  179. package/src/store/session-log/index.ts +8 -0
  180. package/src/store/session-log/turn-holder.ts +95 -0
  181. package/src/telemetry/runtime-accessors.ts +29 -0
  182. package/src/tools/goal/confirmation.ts +375 -0
  183. package/src/tools/goal/index.ts +37 -1
  184. package/src/tools/hand-back/index.ts +150 -0
  185. package/src/tools/schedules/schedule-tool.ts +19 -1
  186. package/src/tools/schedules/types.ts +10 -0
  187. package/src/types/connector/mcp.ts +18 -0
  188. package/src/types/goal/index.ts +13 -0
  189. package/src/types/session/config.ts +8 -0
  190. package/src/types/session/hand-back.ts +31 -0
  191. package/src/types/session/outcome.ts +42 -0
  192. package/src/types/session/turn.ts +19 -1
@@ -3,6 +3,7 @@ import { join } from 'node:path'
3
3
  import { TenantIsolationError } from '../../session/errors.js'
4
4
  import type {
5
5
  GoalBlockReason,
6
+ GoalConfirmedBy,
6
7
  GoalPhase,
7
8
  GoalRef,
8
9
  GoalRoundAuthority,
@@ -29,6 +30,8 @@ export interface CreateSessionGoalParams {
29
30
  readonly sessionId: SessionId
30
31
  readonly objective: string
31
32
  readonly maxGoalRounds?: number
33
+ /** Set only when this goal chooses; absent leaves the choice to the host's setting. */
34
+ readonly confirmedBy?: GoalConfirmedBy
32
35
  }
33
36
 
34
37
  export interface EditSessionGoalParams {
@@ -273,6 +276,13 @@ abstract class SessionGoalStoreBase implements SessionGoalStore {
273
276
  const sessionId = await this.requireSession(params.sessionId, tenantId)
274
277
  const text = objective(params.objective)
275
278
  const maxRounds = maxGoalRounds(params.maxGoalRounds)
279
+ if (
280
+ params.confirmedBy !== undefined &&
281
+ params.confirmedBy !== 'model' &&
282
+ params.confirmedBy !== 'record'
283
+ ) {
284
+ throw new TypeError('goal confirmedBy must be "model" or "record".')
285
+ }
276
286
  return await backendFor(this).transact(sessionId, (existing) => {
277
287
  if (existing) {
278
288
  assertTenant(existing, tenantId)
@@ -290,6 +300,7 @@ abstract class SessionGoalStoreBase implements SessionGoalStore {
290
300
  phase: 'active',
291
301
  maxGoalRounds: maxRounds,
292
302
  roundsAdmitted: 0,
303
+ ...(params.confirmedBy !== undefined ? { confirmedBy: params.confirmedBy } : {}),
293
304
  createdAt: now,
294
305
  updatedAt: now,
295
306
  }
@@ -69,6 +69,14 @@ export {
69
69
  readOutcomeProgress,
70
70
  readTurnOutcome,
71
71
  } from './turn-outcome.js'
72
+ export {
73
+ type ParsedTurnHolder,
74
+ type PlaceSource,
75
+ currentPlace,
76
+ parseTurnHolder,
77
+ placeOf,
78
+ turnHolder,
79
+ } from './turn-holder.js'
72
80
  export {
73
81
  type ReleaseHeldSessionLeasesOptions,
74
82
  type ReleaseHeldSessionLeasesResult,
@@ -0,0 +1,95 @@
1
+ import { createHash } from 'node:crypto'
2
+ import { readFileSync, readlinkSync } from 'node:fs'
3
+ import { hostname } from 'node:os'
4
+
5
+ /**
6
+ * The name a turn writes into its writer lease: `namzu:<pid>:<turnId>@<place>`.
7
+ *
8
+ * A process id is valid only where it was issued. The place says where: a short token for the
9
+ * machine's host name and, on Linux, the process-id namespace and the machine's own identity. A reader may take a dead process
10
+ * id as proof that the holder is gone only when its own place is the same. A holder written
11
+ * before the place existed ends at the turn id and carries no place.
12
+ */
13
+
14
+ /** The lengths and shape of the place token: the first 12 hex characters of a SHA-256. */
15
+ const PLACE_LENGTH = 12
16
+ const PLACE_TAIL = new RegExp(`@([0-9a-f]{${PLACE_LENGTH}})$`)
17
+ const HOLDER = /^namzu:([0-9]+):([\s\S]+)$/
18
+
19
+ /** What a place is made from; a seam so a test can say two places apart. */
20
+ export interface PlaceSource {
21
+ readonly hostName: string
22
+ /** The target of `/proc/self/ns/pid`, or undefined where it cannot be read. */
23
+ readonly pidNamespace?: string | undefined
24
+ /** The machine's own identity (`/etc/machine-id`), or undefined where it cannot be read. */
25
+ readonly machineId?: string | undefined
26
+ }
27
+
28
+ /** The token for a host name and an optional process-id namespace identity. */
29
+ export function placeOf(source: PlaceSource): string {
30
+ let material = `host:${source.hostName}`
31
+ if (source.pidNamespace) material += `\npidns:${source.pidNamespace}`
32
+ if (source.machineId) material += `\nmachine:${source.machineId}`
33
+ return createHash('sha256').update(material).digest('hex').slice(0, PLACE_LENGTH)
34
+ }
35
+
36
+ function readPidNamespace(): string | undefined {
37
+ try {
38
+ return readlinkSync('/proc/self/ns/pid')
39
+ } catch {
40
+ return undefined
41
+ }
42
+ }
43
+
44
+ function readMachineId(): string | undefined {
45
+ for (const path of ['/etc/machine-id', '/var/lib/dbus/machine-id']) {
46
+ try {
47
+ const id = readFileSync(path, 'utf8').trim()
48
+ if (id !== '') return id
49
+ } catch {
50
+ // try the next, or go without
51
+ }
52
+ }
53
+ return undefined
54
+ }
55
+
56
+ let cachedPlace: string | undefined
57
+
58
+ /** Where this process's process id is valid. Computed once per process. */
59
+ export function currentPlace(): string {
60
+ cachedPlace ??= placeOf({
61
+ hostName: hostname(),
62
+ pidNamespace: readPidNamespace(),
63
+ machineId: readMachineId(),
64
+ })
65
+ return cachedPlace
66
+ }
67
+
68
+ /** A turn's holder name. `tail` is the turn id (or, for a whole-session operation, the session id). */
69
+ export function turnHolder(tail: string, options: { pid?: number; place?: string } = {}): string {
70
+ return `namzu:${options.pid ?? process.pid}:${tail}@${options.place ?? currentPlace()}`
71
+ }
72
+
73
+ export interface ParsedTurnHolder {
74
+ readonly pid: number
75
+ readonly turnId: string
76
+ /** Absent for a holder written before the place existed. */
77
+ readonly place?: string
78
+ }
79
+
80
+ /** Read a holder name; `null` when it is not a turn-shaped holder (another writer's name). */
81
+ export function parseTurnHolder(holder: string): ParsedTurnHolder | null {
82
+ const match = HOLDER.exec(holder)
83
+ if (!match) return null
84
+ const pid = Number(match[1])
85
+ if (!Number.isSafeInteger(pid)) return null
86
+ let turnId = match[2] as string
87
+ let place: string | undefined
88
+ const tail = PLACE_TAIL.exec(turnId)
89
+ if (tail) {
90
+ place = tail[1]
91
+ turnId = turnId.slice(0, turnId.length - tail[0].length)
92
+ }
93
+ if (turnId === '') return null
94
+ return place === undefined ? { pid, turnId } : { pid, turnId, place }
95
+ }
@@ -57,3 +57,32 @@ export function getActiveSpanContext(): SpanContext | undefined {
57
57
  return undefined
58
58
  }
59
59
  }
60
+
61
+ /**
62
+ * The active span as the W3C trace context an outgoing request can carry:
63
+ * `{ traceparent, tracestate? }`, or `{}` when no span is active.
64
+ *
65
+ * `{}` is what a host that never registered telemetry gets, for the same
66
+ * reason {@link getActiveSpanContext} returns `undefined` there: nothing is
67
+ * special-cased, the api's default context has no span. A span context that
68
+ * is not valid W3C (an all-zero id, a wrong length) also yields `{}`, since a
69
+ * malformed `traceparent` is worse than none.
70
+ */
71
+ export function getTraceContextHeaders(): { traceparent?: string; tracestate?: string } {
72
+ const span = getActiveSpanContext()
73
+ if (!span) return {}
74
+ const { traceId, spanId } = span
75
+ if (!/^[0-9a-f]{32}$/.test(traceId) || /^0+$/.test(traceId)) return {}
76
+ if (!/^[0-9a-f]{16}$/.test(spanId) || /^0+$/.test(spanId)) return {}
77
+ const flags = (span.traceFlags & 0xff).toString(16).padStart(2, '0')
78
+ let tracestate: string | undefined
79
+ try {
80
+ tracestate = span.traceState?.serialize() || undefined
81
+ } catch {
82
+ tracestate = undefined
83
+ }
84
+ return {
85
+ traceparent: `00-${traceId}-${spanId}-${flags}`,
86
+ ...(tracestate ? { tracestate } : {}),
87
+ }
88
+ }
@@ -0,0 +1,375 @@
1
+ import type { SessionGoalStore } from '../../store/goal/index.js'
2
+ import { StaleGoalError } from '../../store/goal/index.js'
3
+ import type { GoalConfirmedBy, GoalRoundAuthority, SessionGoal } from '../../types/goal/index.js'
4
+ import type { HandBack } from '../../types/session/hand-back.js'
5
+ import type { TurnOutcome } from '../../types/session/outcome.js'
6
+ import { showCommand } from '../coordinator/outcome-record.js'
7
+ import { handBackKindSentence, handBackReason } from '../hand-back/index.js'
8
+
9
+ /**
10
+ * Ending a goal by the record of the work.
11
+ *
12
+ * Today the model's own report ends a goal. With `confirmedBy: 'record'` the
13
+ * kernel first reads the record it keeps of the work (`Turn.outcome`, built
14
+ * from what the tool calls did and never from what the model says) and the
15
+ * goal ends only as that record allows. There is no judge model and no model
16
+ * call beyond the turns themselves: the decision is a pure function over what
17
+ * the model reported, the record and the count of refusals so far.
18
+ */
19
+
20
+ /**
21
+ * Times the record may send a goal back to the model before the goal ends as
22
+ * not confirmed. Three: a first refusal normally costs one fix and one rerun of
23
+ * the check, a second covers a fix that did not hold, a third a check that
24
+ * needed a second look; past that the model is looping, and every further
25
+ * refusal costs a model call that has stopped moving the work. The limit is
26
+ * per goal and a host can set its own.
27
+ */
28
+ export const DEFAULT_GOAL_REFUSALS = 3
29
+
30
+ /** The `reasonCode` that makes a `complete` report say that no check exists to run. */
31
+ export const NO_CHECK_REASON_CODE = 'no-check-available'
32
+
33
+ /** How a goal ended once the record had been read. */
34
+ export type GoalEnding =
35
+ /** A check passed and nothing changed after it. */
36
+ | 'confirmed'
37
+ /** The model said it was done; the record cannot confirm or contradict it. */
38
+ | 'stated-not-checked'
39
+ /** The record disagrees and the model could not or would not put it right. */
40
+ | 'not-confirmed'
41
+
42
+ /** What the model reported. */
43
+ export type GoalStatement =
44
+ /** The goal is met. */
45
+ | 'met'
46
+ /** The goal is met, and no check exists to run. */
47
+ | 'no-check'
48
+ /** The goal is not met: the model could not make the check pass. */
49
+ | 'could-not'
50
+
51
+ export type GoalDecision =
52
+ | { readonly kind: 'end'; readonly ending: GoalEnding; readonly sentence: string }
53
+ | { readonly kind: 'refuse'; readonly sentence: string; readonly note: string }
54
+
55
+ export interface GoalDecisionInput {
56
+ readonly statement: GoalStatement
57
+ /** The record of the turn the report was made in; `undefined` when none was kept. */
58
+ readonly record: TurnOutcome | undefined
59
+ /** Refusals already given to this goal. */
60
+ readonly refusals: number
61
+ /** Most refusals this goal may be given. */
62
+ readonly maxRefusals: number
63
+ /**
64
+ * An earlier round of this same goal changed files. Never makes a goal
65
+ * confirmed; it only keeps a round that changed nothing and ran nothing from
66
+ * being read as proof that nothing needed checking.
67
+ */
68
+ readonly earlierRoundsChanged?: boolean
69
+ }
70
+
71
+ /** The record in plain words, without a lead and without a final stop. */
72
+ export function recordSentence(record: TurnOutcome | undefined): string {
73
+ if (record === undefined) return 'there is no record of this work'
74
+ const named = (command: string | undefined): string =>
75
+ command === undefined ? 'a check' : (showCommand(command) ?? 'a command')
76
+ const passedCheck = [...record.checks].reverse().find((check) => check.result === 'passed')
77
+ switch (record.verdict) {
78
+ case 'checked':
79
+ return `${named(passedCheck?.command)} passed and nothing changed after it`
80
+ case 'unchanged':
81
+ return passedCheck
82
+ ? `${named(passedCheck.command)} passed and nothing changed`
83
+ : 'nothing changed and no check ran'
84
+ case 'stale':
85
+ return 'files changed after the last passing check'
86
+ case 'failing': {
87
+ const failed = record.checks.filter((check) => check.result === 'failed')
88
+ const more = failed.length > 1 ? ` (and ${failed.length - 1} more)` : ''
89
+ return `${named(failed[0]?.command)} failed${more}`
90
+ }
91
+ case 'unchecked':
92
+ return 'files changed and no check ran'
93
+ case 'inconclusive': {
94
+ const masked = record.checks.find((check) => check.result === 'masked')
95
+ if (masked) {
96
+ const how = /(?<!\|)\|(?!\|)/.test(masked.command) ? 'the pipe' : 'the command line'
97
+ return `${named(masked.command)} ran, but ${how} hides whether it passed`
98
+ }
99
+ const unsettled = record.checks.find((check) => check.result === 'unsettled')
100
+ if (unsettled) return `${named(unsettled.command)} ran, but its result was not seen`
101
+ return 'files changed and no check gave a result that could be read'
102
+ }
103
+ case 'unknown':
104
+ return 'a command ran that may have changed files, and nothing could tell whether it did'
105
+ }
106
+ }
107
+
108
+ const HOW_TO_SAY_SO =
109
+ 'To say so, call update_goal with status "blocked" and the reason if the check cannot be made to pass, or with status "complete", reasonCode "no-check-available" and the reason if no check exists here.'
110
+
111
+ function refusalNote(record: TurnOutcome | undefined, earlierChanged: boolean): string {
112
+ switch (record?.verdict) {
113
+ case 'failing':
114
+ return `The record says ${recordSentence(record)}. Fix what it reports and run it again, or say plainly that you could not make it pass. ${HOW_TO_SAY_SO}`
115
+ case 'stale':
116
+ return `The record says ${recordSentence(record)}. Run the check again, or say plainly that you could not. ${HOW_TO_SAY_SO}`
117
+ case 'unchecked':
118
+ return `The record says ${recordSentence(record)}. Run the project's check on the files as they are now, or say plainly that no check can be run here. ${HOW_TO_SAY_SO}`
119
+ case 'inconclusive':
120
+ return `The record says ${recordSentence(record)}. Run the check again on its own so that its result can be read, or say plainly that you could not. ${HOW_TO_SAY_SO}`
121
+ default:
122
+ return earlierChanged
123
+ ? `The record of this round says ${recordSentence(record)}, and an earlier round of this goal changed files. Run the project's check on the files as they are now, or say plainly that you could not. ${HOW_TO_SAY_SO}`
124
+ : `The record says ${recordSentence(record)}. Run the check, or say plainly that you could not. ${HOW_TO_SAY_SO}`
125
+ }
126
+ }
127
+
128
+ /**
129
+ * What to do with the model's report of a goal, given the record.
130
+ *
131
+ * - The model says it could not: the goal ends `not-confirmed`; it is never
132
+ * pushed again.
133
+ * - The model says no check exists: `stated-not-checked`, unless the record
134
+ * holds a failing or stale check (`not-confirmed`) or a passing one
135
+ * (`confirmed`).
136
+ * - The model says it is met: a passing check with nothing changed after it is
137
+ * `confirmed`; a failing check, files changed after the last pass, files
138
+ * changed and no check, or a check whose result could not be read are refused
139
+ * while refusals remain and `not-confirmed` once they do not; nothing
140
+ * changed, a command that may have changed files (`unknown`) or no record at
141
+ * all is `stated-not-checked`. `unknown` is not refused on purpose: the
142
+ * record keeps it for the whole turn, so no later check could clear it and a
143
+ * refusal would only cost calls until the bound.
144
+ */
145
+ export function decideGoalConfirmation(input: GoalDecisionInput): GoalDecision {
146
+ const { statement, record, refusals, maxRefusals } = input
147
+ const sentence = recordSentence(record)
148
+ if (statement === 'could-not') return { kind: 'end', ending: 'not-confirmed', sentence }
149
+
150
+ const verdict = record?.verdict
151
+ if (verdict === 'checked') return { kind: 'end', ending: 'confirmed', sentence }
152
+ if (
153
+ verdict === 'unchanged' &&
154
+ record?.checks.some((check) => check.result === 'passed' && check.fresh !== false)
155
+ ) {
156
+ return { kind: 'end', ending: 'confirmed', sentence }
157
+ }
158
+
159
+ const contradicted = verdict === 'failing' || verdict === 'stale'
160
+ if (statement === 'no-check') {
161
+ return contradicted
162
+ ? { kind: 'end', ending: 'not-confirmed', sentence }
163
+ : { kind: 'end', ending: 'stated-not-checked', sentence }
164
+ }
165
+
166
+ const earlierChanged = input.earlierRoundsChanged === true
167
+ const refusable =
168
+ contradicted ||
169
+ verdict === 'unchecked' ||
170
+ verdict === 'inconclusive' ||
171
+ (verdict === 'unchanged' && earlierChanged)
172
+ if (!refusable) return { kind: 'end', ending: 'stated-not-checked', sentence }
173
+ if (refusals >= Math.max(0, maxRefusals)) {
174
+ return { kind: 'end', ending: 'not-confirmed', sentence }
175
+ }
176
+ return { kind: 'refuse', sentence, note: refusalNote(record, earlierChanged) }
177
+ }
178
+
179
+ /** How a goal ended, for the line a person reads and for a machine reading the result. */
180
+ export interface GoalEndingReport {
181
+ readonly goalId: string
182
+ readonly ending: GoalEnding
183
+ /** The record in plain words, as {@link recordSentence} writes it. */
184
+ readonly sentence: string
185
+ /** How many times the record sent this goal back before it ended. */
186
+ readonly refusals: number
187
+ /**
188
+ * Set when the goal ended because the model handed the task back to a person
189
+ * (`hand_back_task`). The ending is `not-confirmed` and `sentence` is the
190
+ * kernel's sentence for the kind; the model's own words are in the hand-back.
191
+ */
192
+ readonly handedBack?: HandBack
193
+ }
194
+
195
+ /** The closing line of a goal: which of the three endings it was and, always, what the record held. */
196
+ export function describeGoalEnding(
197
+ report: Pick<GoalEndingReport, 'ending' | 'sentence' | 'handedBack'>,
198
+ ): string {
199
+ if (report.handedBack) {
200
+ return `Goal not confirmed: the model handed the task back to a person (${report.sentence}).`
201
+ }
202
+ switch (report.ending) {
203
+ case 'confirmed':
204
+ return `Goal confirmed by the record: ${report.sentence}.`
205
+ case 'stated-not-checked':
206
+ return `Goal stated, not checked: the record cannot confirm it (${report.sentence}).`
207
+ case 'not-confirmed':
208
+ return `Goal not confirmed: the record says ${report.sentence}.`
209
+ }
210
+ }
211
+
212
+ export interface GoalConfirmationOptions {
213
+ readonly store: SessionGoalStore
214
+ /** The same resolver the goal tools use: the authority of the turn, or none. */
215
+ readonly resolveAuthority: (turnId: string) => GoalRoundAuthority | null | undefined
216
+ /** What a goal that did not choose is confirmed by. Default `model`, the behaviour before this existed. */
217
+ readonly defaultConfirmedBy?: GoalConfirmedBy
218
+ /** Refusals a goal may be given. Default {@link DEFAULT_GOAL_REFUSALS}. */
219
+ readonly maxRefusals?: number
220
+ }
221
+
222
+ interface Claim {
223
+ readonly statement: GoalStatement
224
+ readonly reason?: string
225
+ }
226
+
227
+ /**
228
+ * Holds, for one session, what the model reported this turn, how many times a
229
+ * goal was sent back, and whether an earlier round of it changed files. All of
230
+ * it lives in memory: a restart begins the count again, which is the cautious
231
+ * side of the bound only in that the model gets its refusals again.
232
+ */
233
+ export class GoalConfirmation {
234
+ private readonly claims = new Map<string, Claim>()
235
+ private readonly refusals = new Map<string, number>()
236
+ private readonly changed = new Set<string>()
237
+ private readonly endings = new Map<string, GoalEndingReport>()
238
+
239
+ constructor(private readonly options: GoalConfirmationOptions) {}
240
+
241
+ /** What decides this goal: its own choice, else the host's setting, else the model. */
242
+ confirmedByFor(goal: Pick<SessionGoal, 'confirmedBy'>): GoalConfirmedBy {
243
+ return goal.confirmedBy ?? this.options.defaultConfirmedBy ?? 'model'
244
+ }
245
+
246
+ get maxRefusals(): number {
247
+ return this.options.maxRefusals ?? DEFAULT_GOAL_REFUSALS
248
+ }
249
+
250
+ /** Refusals given to a goal so far. */
251
+ refusalsFor(goalId: string): number {
252
+ return this.refusals.get(goalId) ?? 0
253
+ }
254
+
255
+ /**
256
+ * The model reported on the goal in the round now running; the record is read
257
+ * when it finishes its answer. Held by goal, because only one round of a goal
258
+ * runs at a time and a round that ends without an answer leaves the goal
259
+ * active: the next report replaces this one.
260
+ */
261
+ claim(goalId: string, statement: GoalStatement, reason?: string): void {
262
+ this.claims.set(goalId, { statement, ...(reason !== undefined ? { reason } : {}) })
263
+ }
264
+
265
+ /** The ending the turn's goal came to, once; `undefined` when it did not end here. */
266
+ takeEnding(turnId: string): GoalEndingReport | undefined {
267
+ const ending = this.endings.get(turnId)
268
+ this.endings.delete(turnId)
269
+ return ending
270
+ }
271
+
272
+ /**
273
+ * The model handed this round's task back to a person: the goal, when this
274
+ * turn is a round of an active one, ends as not confirmed and is left blocked
275
+ * with the reason, whatever it was confirmed by. Nothing for any other turn.
276
+ */
277
+ async handBack(turnId: string, handBack: HandBack): Promise<GoalEndingReport | undefined> {
278
+ const authority = this.options.resolveAuthority(turnId)
279
+ if (!authority) return undefined
280
+ const goal = await this.options.store.getGoal(authority.sessionId, authority.tenantId)
281
+ if (!goal || goal.id !== authority.id || goal.phase !== 'active') return undefined
282
+ const ref = { id: authority.id, revision: authority.revision }
283
+ try {
284
+ await this.options.store.blockGoal(authority.sessionId, authority.tenantId, ref, {
285
+ code: 'handed-back-to-a-person',
286
+ message: handBackReason(handBack),
287
+ })
288
+ } catch (error) {
289
+ if (error instanceof StaleGoalError) return undefined
290
+ throw error
291
+ }
292
+ const report: GoalEndingReport = {
293
+ goalId: goal.id,
294
+ ending: 'not-confirmed',
295
+ sentence: handBackKindSentence(handBack.kind),
296
+ refusals: this.refusalsFor(goal.id),
297
+ handedBack: handBack,
298
+ }
299
+ this.claims.delete(goal.id)
300
+ this.refusals.delete(goal.id)
301
+ this.changed.delete(goal.id)
302
+ return report
303
+ }
304
+
305
+ /** Drop a report that was never answered for, when its round ends without the turn finishing an answer. */
306
+ forget(goalId: string): void {
307
+ this.claims.delete(goalId)
308
+ }
309
+
310
+ /**
311
+ * For `TurnConfig.outcome.beforeEnd`: the note that sends the model back to
312
+ * the work, or `undefined` when the turn may end. A turn that is not a round
313
+ * of a goal confirmed by the record is never touched.
314
+ */
315
+ beforeEnd(
316
+ turnId: string,
317
+ ): (read: () => Promise<TurnOutcome | undefined>) => Promise<string | undefined> {
318
+ return async (read) => {
319
+ const authority = this.options.resolveAuthority(turnId)
320
+ if (!authority) return undefined
321
+ const goal = await this.options.store.getGoal(authority.sessionId, authority.tenantId)
322
+ if (!goal || goal.id !== authority.id || goal.phase !== 'active') return undefined
323
+ if (this.confirmedByFor(goal) !== 'record') return undefined
324
+
325
+ const record = await read()
326
+ const claim = this.claims.get(goal.id)
327
+ this.claims.delete(goal.id)
328
+ const changedHere = (record?.changes.total ?? 0) > 0 || record?.changes.outsideKnownTools
329
+ if (!claim) {
330
+ if (changedHere) this.changed.add(goal.id)
331
+ return undefined
332
+ }
333
+
334
+ const decision = decideGoalConfirmation({
335
+ statement: claim.statement,
336
+ record,
337
+ refusals: this.refusalsFor(goal.id),
338
+ maxRefusals: this.maxRefusals,
339
+ earlierRoundsChanged: this.changed.has(goal.id),
340
+ })
341
+ if (changedHere) this.changed.add(goal.id)
342
+ if (decision.kind === 'refuse') {
343
+ this.refusals.set(goal.id, this.refusalsFor(goal.id) + 1)
344
+ return decision.note
345
+ }
346
+
347
+ const ref = { id: authority.id, revision: authority.revision }
348
+ try {
349
+ if (decision.ending === 'not-confirmed') {
350
+ const said = claim.reason ? ` The model said: ${claim.reason}` : ''
351
+ await this.options.store.blockGoal(authority.sessionId, authority.tenantId, ref, {
352
+ code: 'not-confirmed-by-record',
353
+ message: `The record says ${decision.sentence}.${said}`,
354
+ })
355
+ } else {
356
+ await this.options.store.completeGoal(authority.sessionId, authority.tenantId, ref)
357
+ }
358
+ } catch (error) {
359
+ // The person changed the goal while the model was answering; what they
360
+ // did stands, and nothing is said about an ending that did not happen.
361
+ if (error instanceof StaleGoalError) return undefined
362
+ throw error
363
+ }
364
+ this.endings.set(turnId, {
365
+ goalId: goal.id,
366
+ ending: decision.ending,
367
+ sentence: decision.sentence,
368
+ refusals: this.refusalsFor(goal.id),
369
+ })
370
+ this.refusals.delete(goal.id)
371
+ this.changed.delete(goal.id)
372
+ return undefined
373
+ }
374
+ }
375
+ }
@@ -5,6 +5,7 @@ import type { GoalRoundAuthority, SessionGoal } from '../../types/goal/index.js'
5
5
  import type { TurnId } from '../../types/ids/index.js'
6
6
  import type { ToolDefinition } from '../../types/tool/index.js'
7
7
  import { defineTool } from '../defineTool.js'
8
+ import { type GoalConfirmation, NO_CHECK_REASON_CODE } from './confirmation.js'
8
9
 
9
10
  export const SESSION_GOAL_TOOL_NAMES = ['get_goal', 'update_goal'] as const
10
11
  export const MIN_GOAL_BLOCK_ROUND = 3
@@ -58,6 +59,7 @@ function authorityFor(
58
59
  export function buildSessionGoalTools(
59
60
  store: SessionGoalStore,
60
61
  resolveAuthority: ResolveGoalRoundAuthority,
62
+ options: { readonly confirmation?: GoalConfirmation } = {},
61
63
  ): ToolDefinition[] {
62
64
  const getGoal = defineTool({
63
65
  name: 'get_goal',
@@ -106,8 +108,42 @@ export function buildSessionGoalTools(
106
108
  concurrencySafe: false,
107
109
  async execute({ status, reasonCode, reason }, context) {
108
110
  const authority = authorityFor(resolveAuthority, context.turnId)
109
- await currentGoal(store, authority)
111
+ const current = await currentGoal(store, authority)
110
112
  const ref = { id: authority.id, revision: authority.revision }
113
+ const confirmation = options.confirmation
114
+ if (confirmation && confirmation.confirmedByFor(current) === 'record') {
115
+ // The report is held until the model finishes its answer; the kernel then
116
+ // reads the record of the work and the goal ends as the record allows.
117
+ if (status === 'complete') {
118
+ confirmation.claim(
119
+ authority.id,
120
+ reasonCode === NO_CHECK_REASON_CODE ? 'no-check' : 'met',
121
+ reason,
122
+ )
123
+ return {
124
+ success: true,
125
+ output:
126
+ 'Completion reported. This goal is confirmed by the record: when you finish your answer the kernel reads the record of the work done, and the goal ends as that record allows. Give your final answer now.',
127
+ }
128
+ }
129
+ if (
130
+ authority.round < MIN_GOAL_BLOCK_ROUND &&
131
+ confirmation.refusalsFor(authority.id) === 0
132
+ ) {
133
+ throw new Error(
134
+ `A goal cannot be marked blocked before admitted round ${MIN_GOAL_BLOCK_ROUND}, unless the record has already sent it back once; this is round ${authority.round}.`,
135
+ )
136
+ }
137
+ if (!reasonCode || !reason) {
138
+ throw new TypeError('Blocked goal updates require reasonCode and reason.')
139
+ }
140
+ confirmation.claim(authority.id, 'could-not', reason)
141
+ return {
142
+ success: true,
143
+ output:
144
+ 'Reported as not met. The goal ends as not confirmed when you finish your answer, and the record of the work is shown with your reason. Give your final answer now.',
145
+ }
146
+ }
111
147
  if (status === 'complete') {
112
148
  const goal = await store.completeGoal(authority.sessionId, authority.tenantId, ref)
113
149
  return { success: true, output: 'Goal marked complete.', data: goal }