@xingular/harness 2.1.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/NOTICE.md +22 -0
  3. package/README.md +70 -37
  4. package/dist/cli.js +6572 -6700
  5. package/dist/terminal-pi.js +51 -38
  6. package/docs/terminal-tui.md +50 -12
  7. package/package.json +2 -2
  8. package/src/accounts/accounts.ts +4 -3
  9. package/src/capabilities/api.ts +1 -1
  10. package/src/cli.ts +7 -3
  11. package/src/core/bus.test.ts +15 -1
  12. package/src/core/bus.ts +39 -0
  13. package/src/core/caller.ts +34 -0
  14. package/src/core/fs.ts +17 -1
  15. package/src/harness.ts +55 -66
  16. package/src/{memory → history}/recall.test.ts +1 -1
  17. package/src/{memory → history}/recall.ts +1 -1
  18. package/src/{memory → history}/trajectory.test.ts +4 -4
  19. package/src/{memory → history}/trajectory.ts +1 -1
  20. package/src/hub/hub.test.ts +4 -5
  21. package/src/hub/hub.ts +2 -2
  22. package/src/index.ts +28 -14
  23. package/src/mcp/connections.ts +1 -1
  24. package/src/mcp/service.ts +1 -1
  25. package/src/pi/proxy.test.ts +68 -0
  26. package/src/pi/runtime.test.ts +61 -0
  27. package/src/pi/runtime.ts +50 -0
  28. package/src/plugins/api.ts +11 -9
  29. package/src/plugins/builtin.ts +22 -40
  30. package/src/plugins/catalogue.test.ts +13 -41
  31. package/src/plugins/catalogue.ts +3 -32
  32. package/src/plugins/discover.ts +2 -1
  33. package/src/plugins/firstParty.test.ts +1 -1
  34. package/src/plugins/hooks.test.ts +21 -71
  35. package/src/plugins/hooks.ts +6 -7
  36. package/src/plugins/identity.ts +16 -2
  37. package/src/plugins/install.test.ts +1 -1
  38. package/src/plugins/install.ts +2 -1
  39. package/src/plugins/inventory.ts +19 -18
  40. package/src/plugins/plugins.ts +38 -2
  41. package/src/prompt/compile.test.ts +82 -193
  42. package/src/prompt/compile.ts +98 -363
  43. package/src/refine/cli.ts +114 -0
  44. package/src/refine/demo.ts +124 -0
  45. package/src/refine/evidence.ts +349 -0
  46. package/src/refine/host.ts +102 -0
  47. package/src/refine/ledger.ts +104 -0
  48. package/src/refine/library.ts +192 -0
  49. package/src/refine/pi.test.ts +90 -0
  50. package/src/refine/pi.ts +218 -0
  51. package/src/refine/prompts.ts +430 -0
  52. package/src/refine/recursion.ts +353 -0
  53. package/src/refine/refine.test.ts +140 -0
  54. package/src/refine/runner.ts +205 -0
  55. package/src/refine/selection.ts +291 -0
  56. package/src/refine/service.ts +194 -0
  57. package/src/refine/trajectory.ts +71 -0
  58. package/src/refine/types.ts +91 -0
  59. package/src/server/methods.ts +137 -123
  60. package/src/server/server.test.ts +37 -30
  61. package/src/sessions/contracts.ts +7 -2
  62. package/src/sessions/conversation.test.ts +2 -27
  63. package/src/sessions/planMode.test.ts +8 -1
  64. package/src/sessions/planMode.ts +17 -9
  65. package/src/sessions/policy.ts +1 -1
  66. package/src/sessions/pricing.ts +3 -1
  67. package/src/sessions/questionBatches.test.ts +15 -3
  68. package/src/sessions/registry.ts +13 -1
  69. package/src/sessions/runStore.ts +3 -1
  70. package/src/sessions/runner.ts +10 -5
  71. package/src/sessions/sessions.test.ts +60 -13
  72. package/src/sessions/sessions.ts +43 -18
  73. package/src/sessions/subagentService.test.ts +22 -1
  74. package/src/sessions/subagentService.ts +14 -1
  75. package/src/sessions/subagents.ts +2 -0
  76. package/src/terminal/cli.ts +24 -48
  77. package/src/terminal/logs.test.ts +26 -0
  78. package/src/terminal/logs.ts +67 -0
  79. package/src/terminal/piExtension.lifecycle.test.ts +1 -1
  80. package/src/terminal/piExtension.ts +3 -3
  81. package/src/terminal/pluginControl.test.ts +7 -8
  82. package/src/terminal/socket.ts +3 -3
  83. package/src/terminal/terminal.test.ts +41 -39
  84. package/src/terminal/terminals.ts +31 -13
  85. package/src/terminal/tui/application.test.ts +58 -0
  86. package/src/terminal/tui/application.ts +15 -4
  87. package/src/terminal/tui/clipboardFiles.test.ts +94 -0
  88. package/src/terminal/tui/clipboardFiles.ts +86 -0
  89. package/src/terminal/tui/commands.test.ts +13 -1
  90. package/src/terminal/tui/commands.ts +16 -0
  91. package/src/terminal/tui/logPanel.test.ts +61 -0
  92. package/src/terminal/tui/logPanel.ts +175 -0
  93. package/src/terminal/tui/piAdapter.ts +7 -4
  94. package/src/terminal/tui/questionPicker.ts +168 -0
  95. package/src/terminal/tui/questions.test.ts +86 -19
  96. package/src/terminal/tui/questions.ts +27 -22
  97. package/src/terminal/tui/runtime.ts +2 -0
  98. package/src/tools/askUser.test.ts +23 -2
  99. package/src/tools/askUser.ts +15 -5
  100. package/src/tools/definitions.ts +49 -132
  101. package/src/tools/dispatch.ts +78 -0
  102. package/src/tools/harnessTools.ts +23 -0
  103. package/src/tools/spawnSubagent.ts +3 -3
  104. package/src/tools/types.ts +3 -1
  105. package/src/tree/components.ts +221 -0
  106. package/src/tree/materialise.ts +122 -0
  107. package/src/tree/seed.ts +123 -0
  108. package/src/tree/store.ts +193 -0
  109. package/src/tree/surface.ts +135 -0
  110. package/src/tree/tree.test.ts +126 -0
  111. package/src/tree/tree.ts +346 -0
  112. package/src/tree/trust.ts +86 -0
  113. package/src/web/tools.ts +27 -5
  114. package/src/web/web.test.ts +26 -1
  115. package/src/wiki/notes.ts +313 -0
  116. package/src/memory/api.test.ts +0 -82
  117. package/src/memory/api.ts +0 -303
  118. package/src/memory/blocks.test.ts +0 -78
  119. package/src/memory/blocks.ts +0 -408
  120. package/src/memory/caller.ts +0 -69
  121. package/src/memory/curation.test.ts +0 -206
  122. package/src/memory/curation.ts +0 -64
  123. package/src/memory/dispatch.ts +0 -64
  124. package/src/memory/facts.test.ts +0 -41
  125. package/src/memory/facts.ts +0 -248
  126. package/src/memory/graph.ts +0 -103
  127. package/src/memory/health.test.ts +0 -130
  128. package/src/memory/health.ts +0 -74
  129. package/src/memory/maintainer.ts +0 -272
  130. package/src/memory/maintenance.test.ts +0 -192
  131. package/src/memory/maintenance.ts +0 -679
  132. package/src/memory/migrate.ts +0 -79
  133. package/src/memory/operations.ts +0 -150
  134. package/src/memory/proposals.test.ts +0 -168
  135. package/src/memory/proposals.ts +0 -584
  136. package/src/memory/references.ts +0 -77
  137. package/src/memory/runContext.ts +0 -93
  138. package/src/memory/store.test.ts +0 -66
  139. package/src/memory/store.ts +0 -385
  140. package/src/memory/wiki.test.ts +0 -47
  141. package/src/memory/wiki.ts +0 -423
  142. package/src/memory/wikiIndex.test.ts +0 -88
  143. package/src/memory/wikiIndex.ts +0 -117
  144. package/src/plugins/proposal.ts +0 -121
  145. package/src/skills/registry.test.ts +0 -104
  146. package/src/skills/registry.ts +0 -413
  147. package/src/tools/maintenance.ts +0 -29
  148. package/src/tools/memoryTools.ts +0 -22
  149. package/src/tools/pluginPropose.ts +0 -41
  150. package/src/tools/registryWrite.test.ts +0 -101
  151. package/src/tools/registryWrite.ts +0 -182
  152. /package/src/{memory → core}/secrets.ts +0 -0
@@ -1,679 +0,0 @@
1
- // The memory lifecycle. Imported sources and eligible settled runs become jobs; one job runs at
2
- // a time, so the Wiki has a single writer. A job asks the maintainer session for proposals,
3
- // checks every one against the evidence, applies the valid ones as one undoable operation, and
4
- // sends the rest to the Inbox. When no job arrives for a while, memory blocks that are close to
5
- // their limit are condensed.
6
- import { readFileSync } from 'node:fs'
7
- import { join } from 'node:path'
8
- import { createHash } from 'node:crypto'
9
-
10
- import type { Accounts } from '../accounts/accounts'
11
- import { log, type Bus } from '../core/bus'
12
- import type { HarnessEnv } from '../core/env'
13
- import { newId, now, readJsonOr } from '../core/fs'
14
- import { findProject } from '../core/projects'
15
- import { charCount, takeChars } from '../core/text'
16
- import type { RunEvent } from '../sessions/contracts'
17
- import { findRun, readEvents } from '../sessions/runStore'
18
- import { evidenceText, transcript } from './api'
19
- import * as blocks from './blocks'
20
- import { callerBlocks, type CallerContext } from './caller'
21
- import * as facts from './facts'
22
- import { piMaintainerFactory, runMaintenance, type ExistingBlock, type ExistingMemory, type ExistingPage, type MaintainerFactory, type MaintenanceRequest } from './maintainer'
23
- import { inbox, undo } from './operations'
24
- import { accept, apply, captureHashes, describeTrigger, parseProposalSet, writeIndex, type Evidence, type ProposalSet, type SourceRef, type Trigger } from './proposals'
25
- import { looksSecret } from './secrets'
26
- import { enqueueCuration, listCuration, curationJob, saveCuration, requireCurator, type CurationJob } from './curation'
27
- import * as registry from '../skills/registry'
28
- import * as attachments from '../plugins/attachments'
29
- import { parkPluginProposal, pluginProposalSchema } from '../plugins/proposal'
30
- import { registryProposalSchema } from './proposals'
31
- import { writeJson } from '../core/fs'
32
- import { invalid } from '../core/errors'
33
- import * as store from './store'
34
- import { namespaces, searchEntries } from './wiki'
35
- import { reindexQuietly } from './wikiIndex'
36
-
37
- export type Job = { kind: 'ingest'; namespace: string; sourceId: string } | { kind: 'run'; runId: string; namespace?: string } | { kind: 'consolidate'; blockIds: string[] } | { kind: 'review'; requestId?: string; manual?: boolean }
38
-
39
- const FULL_PAGES = 5
40
- const FULL_PAGE_LIMIT = 6000
41
- /** A block this full is condensed when nothing else is happening. */
42
- const NEAR_LIMIT_PERCENT = 90
43
- const IDLE_CHECK_MS = 600_000
44
-
45
- /** Conversations belong to no project, so only this setting can pause them. Absent means on. */
46
- export function chatUpdatesPaused(settings: unknown): boolean {
47
- return (settings as { autoMaintenanceChat?: unknown } | null)?.autoMaintenanceChat === false
48
- }
49
-
50
- /** A run that automatic maintenance leaves alone for a settled reason: a setting, a pause, or no project. Not an error. */
51
- class NotEligible extends Error {}
52
-
53
- /** Where a settled run's knowledge is filed and what the job reads of it; throws the reason it is not eligible. */
54
- function runEvidence(root: string, runId: string): { namespace: string; transcript: string } {
55
- const run = findRun(root, runId)
56
- if (!run) throw new Error('Run was not found.')
57
- const settings = readJsonOr<{ excludedSourceIds?: unknown } | null>(join(root, 'config/settings.json'), null)
58
- if (Array.isArray(settings?.excludedSourceIds) && settings.excludedSourceIds.includes(runId)) throw new NotEligible('run excluded in settings')
59
- const projectId = run.mode === 'chat' || run.mode === 'code' ? run.projectId : null
60
- let namespace: string
61
- if (projectId) {
62
- const project = findProject(root, projectId)
63
- if (!project.autoMaintenance) throw new NotEligible('automatic updates are paused for this project')
64
- namespace = project.wikiNamespace
65
- } else if (run.mode === 'chat') {
66
- if (chatUpdatesPaused(settings)) throw new NotEligible('automatic updates are paused for Chat')
67
- namespace = store.LIBRARY_NAMESPACE
68
- } else {
69
- throw new NotEligible('run has no project')
70
- }
71
- const text = transcript(run.goal ?? '', readEvents(root, runId))
72
- if (!text.includes('Assistant:')) throw new NotEligible('no recorded assistant output to use as evidence')
73
- if (looksSecret(text)) throw new NotEligible('the run contains text that looks like a credential')
74
- return { namespace, transcript: text }
75
- }
76
-
77
- /**
78
- * The blocks attached to a run, which its consolidation may rewrite: an agent's blocks for its
79
- * own conversation, or the `user` block and the project's brief for a session in a project.
80
- */
81
- function runBlocks(env: HarnessEnv, runId: string): blocks.Block[] {
82
- const run = findRun(env.root, runId)
83
- if (!run) return []
84
- const chat = run.mode === 'chat'
85
- // What the session was told it is when it started, when that was recorded.
86
- const recorded = readJsonOr<CallerContext | null>(join(env.root, 'runs', run.id, 'caller.json'), null)
87
- const context: CallerContext = recorded ?? {
88
- runId: run.id,
89
- sessionId: run.sessionId,
90
- projectId: run.projectId,
91
- // A conversation's run names its agent; older records rely on the session being the agent's own.
92
- agentId: chat ? (run.agentId ?? (env.directory.agentName(run.sessionId) !== null ? run.sessionId : null)) : null,
93
- actor: chat ? 'Chat' : 'Code'
94
- }
95
- let attached: blocks.Block[]
96
- try {
97
- attached = callerBlocks(env, context).attached
98
- } catch {
99
- // A run with no single agent has only the shared user block attached.
100
- attached = [blocks.ensureUser(store.wikiDir(env.root))]
101
- }
102
- return attached.filter((block) => !block.front.readOnly)
103
- }
104
-
105
- /**
106
- * What the job sees of the Wiki: the target namespace's pages, entries relevant to the
107
- * evidence from anywhere (the closest few in full), and the facts that apply.
108
- */
109
- function maintainerContext(root: string, namespace: string, evidence: string): { pages: ExistingPage[]; memories: ExistingMemory[] } {
110
- const wiki = store.wikiDir(root)
111
- const pages = store.loadPages(wiki)
112
- const query = evidence.split(/\s+/).filter((word) => Buffer.byteLength(word) > 4).slice(0, 12).join(' ')
113
- const ranked = searchEntries(root, query, { kinds: ['page'], limit: 30 }).map((hit) => hit.id)
114
- const inFull = new Set(ranked.slice(0, FULL_PAGES))
115
- const small = pages.filter((page) => page.location.namespace === namespace).length <= FULL_PAGES
116
- return {
117
- pages: pages
118
- .filter((page) => page.location.namespace === namespace || ranked.includes(page.frontmatter.id))
119
- .slice(0, 150)
120
- .map((page) => ({
121
- id: page.frontmatter.id,
122
- title: page.frontmatter.title,
123
- type: page.frontmatter.type,
124
- path: page.location.path,
125
- summary: store.firstParagraph(page.body),
126
- // An update replaces the whole body, so the job must see what the page says now.
127
- ...(inFull.has(page.frontmatter.id) || (page.location.namespace === namespace && small) ? { body: takeChars(page.body, FULL_PAGE_LIMIT) } : {})
128
- })),
129
- memories: facts
130
- .loadMemories(wiki)
131
- .filter((memory) => facts.appliesTo(memory.front, store.projectNamespaceOf(namespace)))
132
- .slice(0, 60)
133
- .map((memory) => ({ id: memory.front.id, title: memory.front.title, kind: memory.front.kind, namespace: memory.front.namespace, body: takeChars(memory.body, 300) }))
134
- }
135
- }
136
-
137
- /** Blocks at or over `NEAR_LIMIT_PERCENT` of their limit that have not been tried in this state. */
138
- export function fullBlocks(wiki: string, tried: Set<string>): string[] {
139
- return blocks
140
- .loadAll(wiki)
141
- .filter((block) => !block.front.readOnly && block.front.limit > 0 && charCount(block.body) * 100 >= block.front.limit * NEAR_LIMIT_PERCENT)
142
- .filter((block) => {
143
- const key = `${block.id}\u0000${store.currentRevision(wiki, block.id) ?? ''}`
144
- return !tried.has(key) && Boolean(tried.add(key))
145
- })
146
- .map((block) => block.id)
147
- }
148
-
149
- const blockValues = (found: readonly blocks.Block[]): ExistingBlock[] => found.map((block) => ({ id: block.id, label: block.front.label, description: block.front.description, limit: block.front.limit, body: block.body }))
150
-
151
- export interface MaintenanceBatchDecision {
152
- id: string
153
- decision: 'apply' | 'reject' | 'needs_user'
154
- reason: string
155
- }
156
-
157
- export interface MaintenanceOptions {
158
- env: HarnessEnv
159
- bus: Bus
160
- accounts: Accounts
161
- factory?: MaintainerFactory
162
- /** How long the queue stays empty before full blocks are condensed. */
163
- idleMs?: number
164
- external?: boolean
165
- excludedTriggers?: string[]
166
- /** False: a settled run queues no consolidation of its own. */
167
- runTrigger?: boolean
168
- }
169
-
170
- export class Maintenance {
171
- private chain: Promise<void> = Promise.resolve()
172
- private idle: ReturnType<typeof setTimeout> | null = null
173
- private readonly tried = new Set<string>()
174
- private stopped = false
175
-
176
- constructor(private readonly options: MaintenanceOptions) {}
177
-
178
- /** Starts the idle check. Jobs can be queued whether or not it runs. */
179
- start(): void {
180
- this.stopped = false
181
- if (this.options.external) {
182
- for (const job of this.jobs().filter((job) => job.status === 'running')) {
183
- job.status = 'failed'
184
- job.reason = 'The previous process stopped before this request settled. Retry it explicitly.'
185
- saveCuration(this.options.env.root, job)
186
- }
187
- }
188
- this.rest()
189
- }
190
-
191
- stop(): void {
192
- this.stopped = true
193
- if (this.idle) clearTimeout(this.idle)
194
- this.idle = null
195
- }
196
-
197
- /** Resolves when every queued job has finished. */
198
- settled(): Promise<void> {
199
- return this.chain
200
- }
201
-
202
- private rest(): void {
203
- if (this.idle) clearTimeout(this.idle)
204
- if (this.stopped) return
205
- this.idle = setTimeout(() => {
206
- try {
207
- const blockIds = fullBlocks(store.wikiDir(this.options.env.root), this.tried)
208
- if (blockIds.length) this.enqueue({ kind: 'consolidate', blockIds })
209
- else this.rest()
210
- } catch {
211
- this.rest()
212
- }
213
- }, this.options.idleMs ?? IDLE_CHECK_MS)
214
- this.idle.unref?.()
215
- }
216
-
217
- enqueue(job: Job): void {
218
- if (this.idle) clearTimeout(this.idle)
219
- if (this.options.external) {
220
- if (job.kind === 'run') {
221
- try {
222
- runEvidence(this.options.env.root, job.runId)
223
- } catch (error) {
224
- if (!(error instanceof NotEligible)) log('maintenance source skipped', error instanceof Error ? error.message : String(error))
225
- return
226
- }
227
- const run = findRun(this.options.env.root, job.runId)
228
- const namespace = run?.projectId ? findProject(this.options.env.root, run.projectId).wikiNamespace : store.LIBRARY_NAMESPACE
229
- enqueueCuration(this.options.env.root, { ...job, namespace })
230
- } else enqueueCuration(this.options.env.root, job)
231
- this.options.bus.emit({ type: 'maintenance.queued' })
232
- this.rest()
233
- return
234
- }
235
- this.chain = this.chain
236
- .then(() => this.process(job))
237
- .catch((error) => log('maintenance', error instanceof Error ? error.message : String(error)))
238
- .then(() => this.rest())
239
- }
240
-
241
- /** A settled run is evidence; whether it is eligible is decided when its job runs. */
242
- onRunEvent(event: RunEvent): void {
243
- if (this.options.runTrigger === false) return
244
- const caller = readJsonOr<CallerContext | null>(join(this.options.env.root, 'runs', event.runId, 'caller.json'), null)
245
- if (caller?.vaultAccess === 'curator' || this.options.excludedTriggers?.includes(findRun(this.options.env.root, event.runId)?.trigger ?? '')) return
246
- if (event.type === 'completed') this.enqueue({ kind: 'run', runId: event.runId })
247
- }
248
-
249
- jobs(): CurationJob[] { return listCuration(this.options.env.root) }
250
-
251
- /** Reads every request frozen into the current host-assigned review batch. */
252
- readBatch(context: CallerContext): { jobs: unknown[] } {
253
- requireCurator(context)
254
- const ids = context.maintenanceJobIds ?? (context.maintenanceJobId ? [context.maintenanceJobId] : [])
255
- return {
256
- jobs: ids.map((id) => {
257
- try {
258
- return this.readJob(context, id)
259
- } catch (error) {
260
- return { id, status: 'failed', error: error instanceof Error ? error.message : String(error) }
261
- }
262
- })
263
- }
264
- }
265
-
266
- /** Applies one evidence-checked Wiki change set for all accepted Wiki requests in a batch. */
267
- applyBatch(context: CallerContext, decisions: readonly MaintenanceBatchDecision[], replacement: unknown): { operationId: string | null; jobs: CurationJob[] } {
268
- requireCurator(context)
269
- if (!decisions.length) throw invalid('Decide every assigned request in the batch.')
270
- const ids = decisions.map((decision) => decision.id)
271
- if (new Set(ids).size !== ids.length) throw invalid('Each request can appear only once in a batch decision.')
272
- const assigned = context.maintenanceJobIds ?? (context.maintenanceJobId ? [context.maintenanceJobId] : [])
273
- const pending = assigned.map((id) => curationJob(this.options.env.root, id)).filter((job) => !['completed', 'rejected', 'needs_user'].includes(job.status))
274
- if (pending.some((job) => !ids.includes(job.id))) throw invalid('Decide every assigned request in one batch.')
275
-
276
- const entries = decisions.map((decision) => {
277
- if (!decision.reason.trim()) throw invalid('Explain every batch decision.')
278
- if (!['apply', 'reject', 'needs_user'].includes(decision.decision)) throw invalid('Choose apply, reject, or needs_user for every request.')
279
- const job = curationJob(this.options.env.root, decision.id)
280
- if (context.maintenanceJobIds !== undefined && !context.maintenanceJobIds.includes(decision.id) && !('caller' in job.input && job.input.caller.runId === context.runId)) throw invalid('This request is not assigned to this run.')
281
- if (job.runId && job.runId !== context.runId && job.status === 'running') throw invalid('Another run owns this request.')
282
- return { decision, job }
283
- })
284
- const wikiEntries = entries.filter(({ decision, job }) => decision.decision === 'apply' && !['registry', 'plugin'].includes(job.input.kind) && !['completed', 'rejected', 'needs_user'].includes(job.status))
285
- if (wikiEntries.length && replacement === undefined) throw invalid('Provide one combined, evidence-backed Wiki change set for the approved requests.')
286
- const proposals: ProposalSet | null = wikiEntries.length ? parseProposalSet(replacement) : null
287
- const operationId = wikiEntries.length ? `batch-${createHash('sha256').update(wikiEntries.map(({ job }) => job.id).sort().join('\0')).digest('hex').slice(0, 32)}` : null
288
- const results = new Map<string, CurationJob>()
289
-
290
- if (wikiEntries.length) {
291
- const prepared = wikiEntries.map(({ job }) => this.claim(context, job.id))
292
- const root = this.options.env.root
293
- const wiki = store.wikiDir(root)
294
- const evidence: Evidence = { runs: {}, sources: {}, blocks: [] }
295
- const blocksSeen = new Set<string>()
296
- const captured = new Map<string, string>()
297
- const conflicts = new Set<string>()
298
- for (const job of prepared) {
299
- const input = job.input
300
- for (const id of job.prepared?.evidence.blocks ?? (input.kind === 'proposal' ? input.proposals.blocks.map((proposal) => proposal.block) : [])) blocksSeen.add(id)
301
- for (const [id, hash] of job.hashes) {
302
- const previous = captured.get(id)
303
- if (previous !== undefined && previous !== hash) conflicts.add(id)
304
- else captured.set(id, hash)
305
- }
306
- }
307
- for (const id of conflicts) captured.set(id, `conflicting-batch-snapshot:${id}`)
308
- evidence.blocks = [...blocksSeen]
309
-
310
- const refs: SourceRef[] = [...proposals!.pages, ...proposals!.memories, ...proposals!.blocks].flatMap((proposal) => proposal.sourceRefs)
311
- const excluded = readJsonOr<{ excludedSourceIds?: unknown } | null>(join(root, 'config/settings.json'), null)?.excludedSourceIds
312
- const excludedIds = Array.isArray(excluded) ? excluded.filter((id): id is string => typeof id === 'string') : []
313
- for (const ref of refs) {
314
- if (ref.kind === 'run' && findRun(root, ref.ref)) {
315
- const caller = readJsonOr<CallerContext | null>(join(root, 'runs', ref.ref, 'caller.json'), null)
316
- if (!excludedIds.includes(ref.ref) && caller?.vaultAccess !== 'curator') {
317
- const text = evidenceText(root, ref.ref)
318
- if (!looksSecret(text)) evidence.runs![ref.ref] = takeChars(text, 60_000)
319
- }
320
- }
321
- if (ref.kind === 'source') {
322
- const source = store.listSources(wiki).find((item) => item.id === ref.ref)
323
- if (source) {
324
- try {
325
- const text = readFileSync(join(wiki, 'raw', source.id, source.name), 'utf8')
326
- if (!looksSecret(text)) evidence.sources![ref.ref] = takeChars(text, 60_000)
327
- } catch {
328
- // Missing or unreadable evidence is rejected by the proposal validator.
329
- }
330
- }
331
- }
332
- }
333
-
334
- const namespaceFor = (sourceRefs: readonly SourceRef[]): string => {
335
- for (const ref of sourceRefs) {
336
- const sourceJob = prepared.find((job) => {
337
- if (job.input.kind === 'proposal') return job.input.caller.runId === ref.ref
338
- if (job.input.kind === 'run') return job.input.runId === ref.ref
339
- if (job.input.kind === 'ingest') return job.input.sourceId === ref.ref
340
- return false
341
- })
342
- const namespace = sourceJob?.prepared?.namespace ?? (sourceJob?.input.kind === 'proposal' ? sourceJob.input.namespace : undefined)
343
- if (namespace) return namespace
344
- }
345
- return prepared[0]?.prepared?.namespace ?? (prepared[0]?.input.kind === 'proposal' ? prepared[0].input.namespace : undefined) ?? store.LIBRARY_NAMESPACE
346
- }
347
- const batchProposals: ProposalSet = {
348
- ...proposals!,
349
- pages: proposals!.pages.map((proposal) => ({ ...proposal, namespace: proposal.namespace ?? namespaceFor(proposal.sourceRefs) })),
350
- memories: proposals!.memories.map((proposal) => ({ ...proposal, namespace: proposal.namespace ?? namespaceFor(proposal.sourceRefs) }))
351
- }
352
- const trigger: Trigger = prepared.some((job) => job.input.kind === 'consolidate')
353
- ? { kind: 'consolidate' }
354
- : prepared.some((job) => job.input.kind === 'run')
355
- ? { kind: 'run', runId: (prepared.find((job) => job.input.kind === 'run')!.input as Extract<Job, { kind: 'run' }>).runId }
356
- : prepared.some((job) => job.input.kind === 'ingest')
357
- ? (prepared.find((job) => job.input.kind === 'ingest')!.prepared?.trigger ?? { kind: 'agent', runId: context.runId ?? null, actor: context.agentName ?? 'Curator' })
358
- : { kind: 'agent', runId: context.runId ?? null, actor: context.agentName ?? 'Curator' }
359
- const outcome = apply(wiki, namespaceFor(refs), trigger, batchProposals, captured, evidence, operationId!, context.agentName ?? 'Curator')
360
- if (outcome.operation) writeJson(join(wiki, 'operations', `${outcome.operation.id}.json`), { ...outcome.operation, protected: true })
361
- reindexQuietly(root)
362
- this.options.bus.emit({ type: 'wiki.changed' })
363
- let operationAssigned = false
364
- for (const { decision, job: original } of wikiEntries) {
365
- const job = curationJob(root, original.id)
366
- const jobReason = decision.reason.trim()
367
- job.status = outcome.parked ? 'needs_user' : 'completed'
368
- job.reason = jobReason
369
- job.receipt = {
370
- operationId: outcome.operation && !operationAssigned ? outcome.operation.id : null,
371
- notes: [...outcome.notes, ...(outcome.operation ? [`Combined batch operation ${outcome.operation.id}.`] : ['The combined review produced no Wiki changes.'])],
372
- parked: outcome.parked,
373
- flagged: outcome.flagged
374
- }
375
- if (outcome.operation && !operationAssigned) operationAssigned = true
376
- results.set(job.id, saveCuration(root, job))
377
- }
378
- }
379
-
380
- for (const { decision, job } of entries) {
381
- if (results.has(job.id) || ['completed', 'rejected', 'needs_user'].includes(job.status)) {
382
- results.set(job.id, curationJob(this.options.env.root, job.id))
383
- continue
384
- }
385
- if (decision.decision === 'apply') {
386
- results.set(job.id, this.applyJob(context, job.id, undefined, decision.reason))
387
- } else {
388
- results.set(job.id, this.decide(context, job.id, decision.decision, decision.reason))
389
- }
390
- }
391
- const receiptOperationId = [...results.values()].find((job) => job.receipt?.operationId)?.receipt?.operationId ?? null
392
- return { operationId: receiptOperationId, jobs: decisions.map(({ id }) => results.get(id) ?? curationJob(this.options.env.root, id)) }
393
- }
394
-
395
- claim(context: CallerContext, id: string, prepare = true): CurationJob {
396
- requireCurator(context)
397
- const root = this.options.env.root
398
- const job = curationJob(root, id)
399
- if (job.status === 'completed' || job.status === 'rejected' || job.status === 'needs_user') return job
400
- if (context.maintenanceJobIds !== undefined && !context.maintenanceJobIds.includes(id) && !('caller' in job.input && job.input.caller.runId === context.runId)) throw invalid('This request is not assigned to this run.')
401
- if (job.runId && job.runId !== context.runId && job.status === 'running') throw invalid('Another run owns this request.')
402
- job.runId = context.runId!
403
- if (prepare && job.input.kind !== 'proposal' && job.input.kind !== 'registry' && job.input.kind !== 'plugin' && !job.prepared) {
404
- let prepared: ReturnType<Maintenance['prepare']>
405
- try { prepared = this.prepare(job.input) } catch (error) { job.status = 'failed'; job.reason = error instanceof Error ? error.message : String(error); saveCuration(root, job); throw error }
406
- job.prepared = { request: prepared.request, evidence: prepared.evidence, namespace: prepared.namespace, trigger: prepared.trigger }
407
- job.hashes = [...prepared.captured]
408
- if (job.prepared.evidence.sourceText) job.prepared.evidence.sourceText = takeChars(job.prepared.evidence.sourceText, 60_000)
409
- }
410
- if (prepare && job.input.kind === 'proposal' && !job.prepared) {
411
- const input = job.input
412
- const origin = input.caller.runId
413
- const attached = input.proposals.blocks.map((proposal) => blocks.find(store.wikiDir(root), proposal.block))
414
- const evidence = origin && input.caller.vaultAccess !== 'curator' ? evidenceText(root, origin) : ''
415
- const context = maintainerContext(root, input.namespace, evidence)
416
- job.prepared = { request: { jobId: job.id, namespace: input.namespace, trigger: { kind: 'run', runId: origin ?? '', transcript: takeChars(evidence, 24_000) }, pages: context.pages, memories: context.memories, blocks: blockValues(attached), index: '' }, evidence: { runText: evidence, blocks: attached.map((block) => block.id) }, namespace: input.namespace, trigger: { kind: 'agent', runId: origin ?? null, actor: 'Curator' } }
417
- }
418
- job.status = 'running'
419
- job.runId = context.runId!
420
- return saveCuration(root, job)
421
- }
422
-
423
- readJob(context: CallerContext, id: string): unknown {
424
- const job = this.claim(context, id)
425
- return { id: job.id, status: job.status, input: job.input, context: job.prepared?.request ?? null, receipt: job.receipt }
426
- }
427
-
428
- decide(context: CallerContext, id: string, decision: 'reject' | 'needs_user', reason: string): CurationJob {
429
- if (!reason.trim()) throw invalid('Explain the decision.')
430
- const job = this.claim(context, id, false)
431
- if (job.status === 'completed' || job.status === 'rejected' || job.status === 'needs_user') return job
432
- job.status = decision === 'reject' ? 'rejected' : 'needs_user'
433
- job.reason = reason.trim()
434
- job.receipt = { operationId: null, notes: [job.reason], parked: decision === 'needs_user' ? 1 : 0, flagged: 0 }
435
- if (decision === 'needs_user') {
436
- const input = job.input
437
- if (input.kind === 'registry') {
438
- const proposal = input.proposal
439
- inbox(store.wikiDir(this.options.env.root), 'proposed', `Review ${proposal.kind}: ${proposal.name}`, reason, [], null, { kind: proposal.kind, proposal, curationJobId: id })
440
- } else if (input.kind === 'plugin') parkPluginProposal(store.wikiDir(this.options.env.root), `Curated request ${id}`, pluginProposalSchema.parse(input.proposal))
441
- else inbox(store.wikiDir(this.options.env.root), 'unresolved', 'Maintenance needs your decision', reason, [])
442
- }
443
- this.options.bus.emit({ type: 'wiki.changed' })
444
- return saveCuration(this.options.env.root, job)
445
- }
446
-
447
- applyJob(context: CallerContext, id: string, replacement?: unknown, reason = '', scope?: attachments.AttachScope): CurationJob {
448
- if (!reason.trim()) throw invalid('Explain why this change should be applied.')
449
- const root = this.options.env.root
450
- const wiki = store.wikiDir(root)
451
- const job = this.claim(context, id)
452
- if (job.status === 'completed' || job.status === 'rejected' || job.status === 'needs_user') return job
453
- const committed = readJsonOr<{ id: string } | null>(join(wiki, 'operations', `${job.id}.json`), null)
454
- if (committed) {
455
- job.status = 'completed'
456
- job.receipt = { operationId: committed.id, notes: ['This request was already committed.'], parked: 0, flagged: 0 }
457
- return saveCuration(root, job)
458
- }
459
- const input = job.input
460
- if (input.kind === 'plugin') return this.decide(context, id, 'needs_user', reason || 'Plugin files require user review before activation; executable code also needs explicit trust.')
461
- if (input.kind === 'registry') {
462
- const proposal = registryProposalSchema.parse(input.proposal)
463
- const itemId = `${proposal.kind}s/${store.slug(proposal.name)}`
464
- if (!['skill', 'prompt', 'rule'].includes(proposal.kind) || !proposal.name.trim() || !proposal.body.trim() || looksSecret(proposal.body)) throw invalid('Invalid text resource proposal.')
465
- const existing = registry.list(root).find((item) => item.id === itemId)
466
- const current = captureHashes(wiki).get(itemId)
467
- if (existing?.builtin || current !== new Map(job.hashes).get(itemId)) return this.decide(context, id, 'needs_user', 'The resource is built-in or changed since it was proposed. Review the current version.')
468
- const target = scope ?? { agentId: input.caller.agentId ?? context.agentId }
469
- if (target.agentId && target.projectId) throw invalid('Choose one attachment scope.')
470
- if (target.agentId && target.agentId !== context.agentId && target.agentId !== input.caller.agentId && !this.options.env.directory.agentName(target.agentId)) throw invalid('Unknown attachment target.')
471
- if (target.projectId) findProject(root, target.projectId)
472
- const before = existing ? store.snapshot(wiki, itemId, readFileSync(existing.path, 'utf8'), 'Curator', `Before ${id}`).id : null
473
- const beforeChoices = attachments.load(root)
474
- registry.writeItem(root, proposal.kind, proposal.name, proposal.description, proposal.body, `Curated request ${id}: ${reason || proposal.reason}`, context.agentName ?? 'Curator')
475
- const saved = registry.list(root).find((item) => item.id === itemId)!
476
- const after = store.currentRevision(wiki, itemId)!
477
- const choices = attachments.load(root)
478
- attachments.set(choices, [itemId], target, true, false)
479
- attachments.save(root, choices)
480
- writeJson(join(wiki, 'operations', `${id}.json`), { id, createdAt: now(), trigger: `Curated request ${id}`, pages: [{ pageId: itemId, path: saved.path, beforeRevisionId: before, afterRevisionId: after }], undone: false, protected: true, attachmentsBefore: beforeChoices, attachmentsAfter: choices })
481
- job.receipt = { operationId: id, notes: [`Saved ${itemId}; enabled for ${target.agentId ?? target.projectId ?? 'everyone'}. ${reason || proposal.reason}`], parked: 0, flagged: 0 }
482
- this.options.bus.emit({ type: 'plugins.changed' })
483
- } else {
484
- const prepared = job.prepared
485
- const proposalInput = input.kind === 'proposal' ? input : null
486
- const proposals = parseProposalSet(replacement ?? proposalInput?.proposals)
487
- const origin = proposalInput?.caller.runId ?? null
488
- const trigger: Trigger = prepared?.trigger ?? { kind: 'agent', runId: origin, actor: context.agentName ?? 'Curator' }
489
- const evidence: Evidence = input.kind === 'proposal' ? (prepared?.evidence ?? {}) : this.prepare(input).evidence
490
- const originEvidence: Evidence = prepared?.evidence ?? { runText: origin ? evidenceText(root, origin) : null, blocks: proposals.blocks.map((block) => block.block) }
491
- {
492
- if (input.kind === 'proposal') Object.assign(evidence, originEvidence)
493
- evidence.runs = {}
494
- evidence.sources = {}
495
- const refs = [...proposals.pages, ...proposals.memories, ...proposals.blocks].flatMap(proposal => proposal.sourceRefs).slice(0, 20)
496
- for (const ref of refs) {
497
- if (ref.kind === 'run' && findRun(root, ref.ref) && readJsonOr<CallerContext | null>(join(root, 'runs', ref.ref, 'caller.json'), null)?.vaultAccess !== 'curator') {
498
- const excluded = readJsonOr<{ excludedSourceIds?: string[] }>(join(root, 'config/settings.json'), {}).excludedSourceIds ?? []
499
- const text = evidenceText(root, ref.ref)
500
- if (!excluded.includes(ref.ref) && !looksSecret(text)) evidence.runs[ref.ref] = takeChars(text, 60_000)
501
- }
502
- if (ref.kind === 'source') {
503
- const source = store.listSources(wiki).find(source => source.id === ref.ref)
504
- if (source) {
505
- const text = readFileSync(join(wiki, 'raw', source.id, source.name), 'utf8')
506
- if (!looksSecret(text)) evidence.sources[ref.ref] = takeChars(text, 60_000)
507
- }
508
- }
509
- }
510
- }
511
- const outcome = apply(wiki, prepared?.namespace ?? proposalInput?.namespace ?? store.LIBRARY_NAMESPACE, trigger, proposals, new Map(job.hashes), evidence, id, context.agentName ?? 'Curator')
512
- if (outcome.operation) writeJson(join(wiki, 'operations', `${outcome.operation.id}.json`), { ...outcome.operation, protected: true })
513
- job.receipt = { operationId: outcome.operation?.id ?? null, notes: outcome.notes, parked: outcome.parked, flagged: outcome.flagged }
514
- }
515
- job.status = job.receipt.parked ? 'needs_user' : 'completed'
516
- job.reason = reason
517
- reindexQuietly(root)
518
- this.options.bus.emit({ type: 'wiki.changed' })
519
- return saveCuration(root, job)
520
- }
521
-
522
- finishRun(runId: string, failed: boolean): void {
523
- for (const job of this.jobs().filter((job) => job.runId === runId && job.status === 'running')) {
524
- job.status = failed ? 'failed' : 'completed'
525
- job.reason = failed ? 'The run stopped before completing this request. Retry it explicitly.' : 'The review completed without applying a change.'
526
- job.receipt ??= { operationId: null, notes: [job.reason], parked: 0, flagged: 0 }
527
- saveCuration(this.options.env.root, job)
528
- }
529
- }
530
-
531
- retry(id: string): CurationJob {
532
- const job = curationJob(this.options.env.root, id)
533
- if (job.status !== 'failed') throw invalid('Only a failed request can be retried.')
534
- return saveCuration(this.options.env.root, { ...job, status: 'queued', runId: null, reason: '', receipt: null })
535
- }
536
-
537
- prepare(job: Job): { request: MaintenanceRequest; evidence: Evidence; captured: Map<string, string>; namespace: string; trigger: Trigger } {
538
- const { env } = this.options
539
- const root = env.root
540
- const wiki = store.wikiDir(root)
541
- const evidence: Evidence = {}
542
- let attached: blocks.Block[] = []
543
- let namespace: string
544
- let trigger: Trigger
545
- let request: MaintenanceRequest['trigger']
546
- let shown = ''
547
- if (job.kind === 'ingest') {
548
- const directory = join(wiki, 'raw', job.sourceId)
549
- const name = readJsonOr<{ name?: unknown } | null>(join(directory, 'meta.json'), null)?.name
550
- const sourceName = typeof name === 'string' ? name : 'source'
551
- let text: string | null = null
552
- try {
553
- text = new TextDecoder('utf-8', { fatal: true }).decode(readFileSync(join(directory, sourceName)))
554
- } catch {
555
- text = null
556
- }
557
- if (text === null) {
558
- throw invalid('Only text sources can be synthesized automatically right now.')
559
- }
560
- shown = takeChars(text, 60_000)
561
- evidence.sourceText = text
562
- namespace = job.namespace
563
- trigger = { kind: 'ingest', sourceId: job.sourceId, sourceName }
564
- request = { kind: 'ingest', sourceId: job.sourceId, sourceName, text: shown }
565
- } else if (job.kind === 'run') {
566
- let found: ReturnType<typeof runEvidence>
567
- try {
568
- found = runEvidence(root, job.runId)
569
- } catch (error) {
570
- throw error
571
- }
572
- evidence.runText = evidenceText(root, job.runId)
573
- attached = runBlocks(env, job.runId)
574
- shown = found.transcript
575
- namespace = found.namespace
576
- trigger = { kind: 'run', runId: job.runId }
577
- request = { kind: 'run', runId: job.runId, transcript: shown }
578
- } else {
579
- attached = (job.kind === 'review' ? blocks.loadAll(wiki).filter((block) => !block.front.readOnly).sort((a, b) => charCount(b.body) / b.front.limit - charCount(a.body) / a.front.limit).slice(0, 30).map((block) => block.id) : job.blockIds).flatMap((id) => {
580
- try {
581
- const block = blocks.find(wiki, id)
582
- return block.front.readOnly ? [] : [block]
583
- } catch {
584
- return []
585
- }
586
- })
587
- if (!attached.length && job.kind !== 'review') throw invalid('No writable blocks remain in this job.')
588
- namespace = store.LIBRARY_NAMESPACE
589
- trigger = { kind: 'consolidate' }
590
- request = { kind: 'consolidate' }
591
- }
592
- evidence.blocks = attached.map((block) => block.id)
593
- const captured = captureHashes(wiki)
594
- const context = trigger.kind === 'consolidate' ? { pages: [], memories: [] } : maintainerContext(root, namespace, shown)
595
- let index = ''
596
- try {
597
- index = takeChars(readFileSync(join(wiki, 'index.md'), 'utf8'), 8000)
598
- } catch {
599
- index = ''
600
- }
601
- return { request: { jobId: '', namespace, trigger: request, pages: context.pages, memories: context.memories, blocks: blockValues(attached), namespaces: namespaces(root), index }, evidence, captured, namespace, trigger }
602
- }
603
-
604
- private async process(job: Job): Promise<void> {
605
- const { env, bus } = this.options
606
- const root = env.root
607
- const wiki = store.wikiDir(root)
608
- let prepared: ReturnType<Maintenance['prepare']>
609
- try { prepared = this.prepare(job) } catch (error) {
610
- // A run that is not eligible is an expected outcome, not a problem to report.
611
- if (error instanceof NotEligible) return
612
- const message = error instanceof Error ? error.message : String(error)
613
- if (job.kind === 'run') log(`maintenance skipped run ${job.runId}: ${message}`)
614
- else if (job.kind === 'ingest') { inbox(wiki, 'unresolved', 'Source was imported but not summarized', message, []); bus.emit({ type: 'wiki.changed' }) }
615
- return
616
- }
617
- const { namespace, trigger, evidence, captured } = prepared
618
- const described = describeTrigger(trigger)
619
- try {
620
- // The job runs on the default account.
621
- const account = this.options.accounts.context(this.options.accounts.resolve())
622
- const proposed = await runMaintenance(
623
- { ...prepared.request, jobId: newId() },
624
- { cwd: wiki, product: env.productName, account, runtime: () => this.options.accounts.runtime(account) },
625
- this.options.factory ?? piMaintainerFactory
626
- )
627
- const outcome = apply(wiki, namespace, trigger, parseProposalSet(proposed), captured, evidence)
628
- reindexQuietly(root)
629
- if (outcome.operation) {
630
- const count = outcome.operation.pages.length
631
- bus.emit({ type: 'wiki.maintenance', namespace, title: `Wiki updated ${count} entr${count === 1 ? 'y' : 'ies'}`, detail: `From ${described}. ${outcome.parked} parked, ${outcome.flagged} flagged for review.` })
632
- }
633
- } catch (error) {
634
- const message = error instanceof Error ? error.message : String(error)
635
- try {
636
- inbox(wiki, 'blocked', `Consolidation for ${described} did not finish`, message, [])
637
- store.log(wiki, `- ${now()} · Sleep-time · ${described} · failed: ${message}`)
638
- } catch {
639
- // The failure is still logged by the queue.
640
- }
641
- }
642
- bus.emit({ type: 'wiki.changed' })
643
- }
644
-
645
- // ---- What a person does from the Inbox -------------------------------------------------------
646
-
647
- /** Reverts an applied operation through new revisions. */
648
- undoOperation(operationId: string): void {
649
- const { root } = this.options.env
650
- const wiki = store.wikiDir(root)
651
- const operation = undo(wiki, operationId)
652
- writeIndex(wiki)
653
- const count = operation.pages.length
654
- inbox(wiki, 'applied', `Undid an update to ${count} entr${count === 1 ? 'y' : 'ies'}`, "Earlier content was restored as new revisions; created entries moved to the Wiki's trash folder.", operation.pages.map((page) => page.pageId))
655
- reindexQuietly(root)
656
- this.options.bus.emit({ type: 'wiki.changed' })
657
- }
658
-
659
- /** Accepts a parked proposal with the user as the authority. */
660
- acceptInboxItem(itemId: string): string {
661
- const { root } = this.options.env
662
- const message = accept(root, itemId)
663
- reindexQuietly(root)
664
- this.options.bus.emit({ type: 'wiki.changed' })
665
- return message
666
- }
667
-
668
- /** Imports a file as an immutable source and queues the job that reads it. */
669
- ingest(name: string, bytes: Uint8Array, namespace: string = store.LIBRARY_NAMESPACE): store.WikiSource {
670
- const { root } = this.options.env
671
- const wiki = store.wikiDir(root)
672
- store.namespaceDir(wiki, namespace)
673
- const source = store.importSource(wiki, name.replace(/[/\\]/g, '_'), bytes)
674
- reindexQuietly(root)
675
- this.enqueue({ kind: 'ingest', namespace, sourceId: source.id })
676
- return source
677
- }
678
- }
679
-