@xingular/harness 2.1.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/NOTICE.md +22 -0
- package/README.md +67 -36
- package/dist/cli.js +6023 -6621
- package/dist/terminal-pi.js +19 -24
- package/docs/terminal-tui.md +27 -7
- package/package.json +2 -2
- package/src/accounts/accounts.ts +4 -3
- package/src/capabilities/api.ts +1 -1
- package/src/cli.ts +7 -3
- package/src/core/caller.ts +34 -0
- package/src/core/fs.ts +17 -1
- package/src/harness.ts +55 -66
- package/src/{memory → history}/recall.test.ts +1 -1
- package/src/{memory → history}/recall.ts +1 -1
- package/src/{memory → history}/trajectory.test.ts +4 -4
- package/src/{memory → history}/trajectory.ts +1 -1
- package/src/hub/hub.test.ts +4 -5
- package/src/hub/hub.ts +2 -2
- package/src/index.ts +28 -14
- package/src/mcp/connections.ts +1 -1
- package/src/mcp/service.ts +1 -1
- package/src/pi/proxy.test.ts +68 -0
- package/src/pi/runtime.test.ts +61 -0
- package/src/pi/runtime.ts +50 -0
- package/src/plugins/api.ts +11 -9
- package/src/plugins/builtin.ts +22 -40
- package/src/plugins/catalogue.test.ts +13 -41
- package/src/plugins/catalogue.ts +3 -32
- package/src/plugins/discover.ts +2 -1
- package/src/plugins/firstParty.test.ts +1 -1
- package/src/plugins/hooks.test.ts +21 -71
- package/src/plugins/hooks.ts +6 -7
- package/src/plugins/identity.ts +16 -2
- package/src/plugins/install.test.ts +1 -1
- package/src/plugins/install.ts +2 -1
- package/src/plugins/inventory.ts +19 -18
- package/src/plugins/plugins.ts +38 -2
- package/src/prompt/compile.test.ts +82 -193
- package/src/prompt/compile.ts +98 -363
- package/src/refine/cli.ts +114 -0
- package/src/refine/demo.ts +124 -0
- package/src/refine/evidence.ts +349 -0
- package/src/refine/host.ts +102 -0
- package/src/refine/ledger.ts +104 -0
- package/src/refine/library.ts +192 -0
- package/src/refine/pi.test.ts +90 -0
- package/src/refine/pi.ts +218 -0
- package/src/refine/prompts.ts +430 -0
- package/src/refine/recursion.ts +353 -0
- package/src/refine/refine.test.ts +140 -0
- package/src/refine/runner.ts +205 -0
- package/src/refine/selection.ts +291 -0
- package/src/refine/service.ts +194 -0
- package/src/refine/trajectory.ts +71 -0
- package/src/refine/types.ts +91 -0
- package/src/server/methods.ts +137 -123
- package/src/server/server.test.ts +37 -30
- package/src/sessions/contracts.ts +6 -1
- package/src/sessions/conversation.test.ts +2 -27
- package/src/sessions/policy.ts +1 -1
- package/src/sessions/pricing.ts +3 -1
- package/src/sessions/registry.ts +5 -1
- package/src/sessions/runStore.ts +3 -1
- package/src/sessions/runner.ts +10 -5
- package/src/sessions/sessions.test.ts +45 -6
- package/src/sessions/sessions.ts +40 -17
- package/src/sessions/subagentService.test.ts +22 -1
- package/src/sessions/subagentService.ts +14 -1
- package/src/sessions/subagents.ts +2 -0
- package/src/terminal/cli.ts +24 -48
- package/src/terminal/pluginControl.test.ts +7 -8
- package/src/terminal/socket.ts +3 -3
- package/src/terminal/terminal.test.ts +21 -39
- package/src/terminal/terminals.ts +20 -13
- package/src/terminal/tui/application.test.ts +58 -0
- package/src/terminal/tui/application.ts +10 -4
- package/src/terminal/tui/clipboardFiles.test.ts +94 -0
- package/src/terminal/tui/clipboardFiles.ts +86 -0
- package/src/terminal/tui/piAdapter.ts +7 -4
- package/src/terminal/tui/runtime.ts +2 -0
- package/src/tools/definitions.ts +49 -132
- package/src/tools/dispatch.ts +78 -0
- package/src/tools/harnessTools.ts +23 -0
- package/src/tools/spawnSubagent.ts +3 -3
- package/src/tools/types.ts +1 -1
- package/src/tree/components.ts +221 -0
- package/src/tree/materialise.ts +122 -0
- package/src/tree/seed.ts +123 -0
- package/src/tree/store.ts +193 -0
- package/src/tree/surface.ts +135 -0
- package/src/tree/tree.test.ts +126 -0
- package/src/tree/tree.ts +346 -0
- package/src/tree/trust.ts +86 -0
- package/src/wiki/notes.ts +313 -0
- package/src/memory/api.test.ts +0 -82
- package/src/memory/api.ts +0 -303
- package/src/memory/blocks.test.ts +0 -78
- package/src/memory/blocks.ts +0 -408
- package/src/memory/caller.ts +0 -69
- package/src/memory/curation.test.ts +0 -206
- package/src/memory/curation.ts +0 -64
- package/src/memory/dispatch.ts +0 -64
- package/src/memory/facts.test.ts +0 -41
- package/src/memory/facts.ts +0 -248
- package/src/memory/graph.ts +0 -103
- package/src/memory/health.test.ts +0 -130
- package/src/memory/health.ts +0 -74
- package/src/memory/maintainer.ts +0 -272
- package/src/memory/maintenance.test.ts +0 -192
- package/src/memory/maintenance.ts +0 -679
- package/src/memory/migrate.ts +0 -79
- package/src/memory/operations.ts +0 -150
- package/src/memory/proposals.test.ts +0 -168
- package/src/memory/proposals.ts +0 -584
- package/src/memory/references.ts +0 -77
- package/src/memory/runContext.ts +0 -93
- package/src/memory/store.test.ts +0 -66
- package/src/memory/store.ts +0 -385
- package/src/memory/wiki.test.ts +0 -47
- package/src/memory/wiki.ts +0 -423
- package/src/memory/wikiIndex.test.ts +0 -88
- package/src/memory/wikiIndex.ts +0 -117
- package/src/plugins/proposal.ts +0 -121
- package/src/skills/registry.test.ts +0 -104
- package/src/skills/registry.ts +0 -413
- package/src/tools/maintenance.ts +0 -29
- package/src/tools/memoryTools.ts +0 -22
- package/src/tools/pluginPropose.ts +0 -41
- package/src/tools/registryWrite.test.ts +0 -101
- package/src/tools/registryWrite.ts +0 -182
- /package/src/{memory → core}/secrets.ts +0 -0
package/src/memory/wiki.ts
DELETED
|
@@ -1,423 +0,0 @@
|
|
|
1
|
-
// The global Wiki. One corpus under `<data root>/wiki/`; skills, prompts, and rules are the
|
|
2
|
-
// registry's items (`registry/`), shown and searched as Wiki objects without being copied.
|
|
3
|
-
// Projects are path namespaces (`/projects/<slug>/…`), never access boundaries: everything may
|
|
4
|
-
// be read, and the current project only makes its namespace more relevant.
|
|
5
|
-
import { readdirSync, readFileSync } from 'node:fs'
|
|
6
|
-
import { join } from 'node:path'
|
|
7
|
-
|
|
8
|
-
import { noteChange } from '../core/changes'
|
|
9
|
-
import { logLine } from '../core/bus'
|
|
10
|
-
import { HarnessError, notFound } from '../core/errors'
|
|
11
|
-
import * as frontmatter from '../core/frontmatter'
|
|
12
|
-
import { atomicWrite, now } from '../core/fs'
|
|
13
|
-
import { findProject, projectsOrNone } from '../core/projects'
|
|
14
|
-
import { compareText, lines, takeChars } from '../core/text'
|
|
15
|
-
import * as registry from '../skills/registry'
|
|
16
|
-
import * as blocks from './blocks'
|
|
17
|
-
import * as facts from './facts'
|
|
18
|
-
import * as store from './store'
|
|
19
|
-
import { firstParagraph, type ParsedPage, type WikiLink } from './store'
|
|
20
|
-
import * as wikiIndex from './wikiIndex'
|
|
21
|
-
|
|
22
|
-
export interface WikiPageSummary {
|
|
23
|
-
id: string
|
|
24
|
-
title: string
|
|
25
|
-
type: string
|
|
26
|
-
path: string
|
|
27
|
-
namespace: string
|
|
28
|
-
updatedAt: string
|
|
29
|
-
sourceIds: string[]
|
|
30
|
-
links: WikiLink[]
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export interface WikiPage extends WikiPageSummary {
|
|
34
|
-
body: string
|
|
35
|
-
revisionId: string | null
|
|
36
|
-
file: string
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export function summary(page: ParsedPage): WikiPageSummary {
|
|
40
|
-
return {
|
|
41
|
-
id: page.frontmatter.id,
|
|
42
|
-
title: page.frontmatter.title,
|
|
43
|
-
type: page.frontmatter.type,
|
|
44
|
-
path: page.location.path,
|
|
45
|
-
namespace: page.location.namespace,
|
|
46
|
-
updatedAt: page.frontmatter.updatedAt,
|
|
47
|
-
sourceIds: page.frontmatter.sourceIds,
|
|
48
|
-
links: page.frontmatter.links
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
// ---- Index feed ----------------------------------------------------------------------
|
|
53
|
-
|
|
54
|
-
export type EntryKind = 'page' | 'memory' | 'block' | 'skill' | 'rule' | 'prompt'
|
|
55
|
-
|
|
56
|
-
/** One searchable entry as the index stores it. */
|
|
57
|
-
export interface IndexedEntry {
|
|
58
|
-
id: string
|
|
59
|
-
kind: EntryKind
|
|
60
|
-
title: string
|
|
61
|
-
entryType: string
|
|
62
|
-
path: string
|
|
63
|
-
namespace: string
|
|
64
|
-
file: string
|
|
65
|
-
updatedAt: string
|
|
66
|
-
body: string
|
|
67
|
-
sourceIds: string[]
|
|
68
|
-
/** `[to id, link type]`. */
|
|
69
|
-
links: [string, string][]
|
|
70
|
-
/** `[revision id, created at, actor]`. */
|
|
71
|
-
revisions: [string, string, string][]
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
export interface IndexedSource {
|
|
75
|
-
id: string
|
|
76
|
-
name: string
|
|
77
|
-
importedAt: string
|
|
78
|
-
version: number
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
function revisionsOf(wiki: string, id: string): [string, string, string][] {
|
|
82
|
-
try {
|
|
83
|
-
return store.readManifest(wiki, id).map((revision) => [revision.id, revision.createdAt, revision.actor])
|
|
84
|
-
} catch {
|
|
85
|
-
return []
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/** Registry kind (`skills`) → entry kind (`skill`). */
|
|
90
|
-
export function registryEntryKind(kind: string): 'skill' | 'rule' | 'prompt' | null {
|
|
91
|
-
return kind === 'skills' ? 'skill' : kind === 'rules' ? 'rule' : kind === 'prompts' ? 'prompt' : null
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
const linkPairs = (links: readonly WikiLink[]): [string, string][] => links.map((link) => [link.toId, link.type])
|
|
95
|
-
|
|
96
|
-
/** Everything the index holds: pages, facts, blocks, and registry skills/prompts/rules. */
|
|
97
|
-
export function scan(root: string): { entries: IndexedEntry[]; sources: IndexedSource[] } {
|
|
98
|
-
const wiki = store.wikiDir(root)
|
|
99
|
-
const entries: IndexedEntry[] = store.loadPages(wiki).map((page) => ({
|
|
100
|
-
id: page.frontmatter.id,
|
|
101
|
-
kind: 'page',
|
|
102
|
-
title: page.frontmatter.title,
|
|
103
|
-
entryType: page.frontmatter.type,
|
|
104
|
-
path: page.location.path,
|
|
105
|
-
namespace: page.location.namespace,
|
|
106
|
-
file: page.path,
|
|
107
|
-
updatedAt: page.frontmatter.updatedAt,
|
|
108
|
-
body: page.body,
|
|
109
|
-
sourceIds: page.frontmatter.sourceIds,
|
|
110
|
-
links: linkPairs(page.frontmatter.links),
|
|
111
|
-
revisions: revisionsOf(wiki, page.frontmatter.id)
|
|
112
|
-
}))
|
|
113
|
-
for (const memory of facts.loadMemories(wiki)) {
|
|
114
|
-
entries.push({
|
|
115
|
-
id: memory.front.id,
|
|
116
|
-
kind: 'memory',
|
|
117
|
-
title: memory.front.title,
|
|
118
|
-
entryType: memory.front.kind,
|
|
119
|
-
path: `/facts/${memory.front.id}`,
|
|
120
|
-
namespace: memory.front.namespace,
|
|
121
|
-
file: memory.path,
|
|
122
|
-
updatedAt: memory.front.updatedAt,
|
|
123
|
-
body: memory.body,
|
|
124
|
-
sourceIds: memory.front.sourceIds,
|
|
125
|
-
links: linkPairs(memory.front.links),
|
|
126
|
-
revisions: revisionsOf(wiki, memory.front.id)
|
|
127
|
-
})
|
|
128
|
-
}
|
|
129
|
-
for (const block of blocks.loadAll(wiki)) {
|
|
130
|
-
entries.push({
|
|
131
|
-
id: block.id,
|
|
132
|
-
kind: 'block',
|
|
133
|
-
title: block.front.label,
|
|
134
|
-
// The scope (`shared`, `agent`, `project`) decides who may see the block in a search.
|
|
135
|
-
entryType: block.scope,
|
|
136
|
-
path: `/${block.id}`,
|
|
137
|
-
namespace: block.scope === 'project' && block.owner ? `/projects/${block.owner}` : store.ROOT_NAMESPACE,
|
|
138
|
-
file: block.path,
|
|
139
|
-
updatedAt: block.front.updatedAt,
|
|
140
|
-
body: `${block.front.description}\n\n${block.body}`,
|
|
141
|
-
sourceIds: [],
|
|
142
|
-
links: [],
|
|
143
|
-
revisions: revisionsOf(wiki, block.id)
|
|
144
|
-
})
|
|
145
|
-
}
|
|
146
|
-
for (const item of registry.list(root)) {
|
|
147
|
-
const kind = registryEntryKind(item.kind)
|
|
148
|
-
if (!kind) continue
|
|
149
|
-
entries.push({
|
|
150
|
-
id: item.id,
|
|
151
|
-
kind,
|
|
152
|
-
title: item.name,
|
|
153
|
-
entryType: kind,
|
|
154
|
-
path: `/${item.id}`,
|
|
155
|
-
namespace: store.ROOT_NAMESPACE,
|
|
156
|
-
file: item.path,
|
|
157
|
-
updatedAt: '',
|
|
158
|
-
body: `${item.description}\n\n${registry.itemBody(item)}`,
|
|
159
|
-
sourceIds: [],
|
|
160
|
-
links: [],
|
|
161
|
-
revisions: revisionsOf(wiki, item.id)
|
|
162
|
-
})
|
|
163
|
-
}
|
|
164
|
-
const sources = store.listSources(wiki).map((source) => ({ id: source.id, name: source.name, importedAt: source.importedAt, version: source.version }))
|
|
165
|
-
return { entries, sources }
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
export function countPages(root: string): number {
|
|
169
|
-
return store.pageFiles(store.wikiDir(root)).length
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
// ---- Search ---------------------------------------------------------------------------
|
|
173
|
-
|
|
174
|
-
export interface WikiSearchHit {
|
|
175
|
-
id: string
|
|
176
|
-
kind: string
|
|
177
|
-
title: string
|
|
178
|
-
type: string
|
|
179
|
-
path: string
|
|
180
|
-
namespace: string
|
|
181
|
-
snippet: string
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
export interface SearchOptions {
|
|
185
|
-
kinds?: string[]
|
|
186
|
-
/** Only entries in this namespace (or below it). */
|
|
187
|
-
namespace?: string | null
|
|
188
|
-
/** Ranks this project's namespace higher; never filters. */
|
|
189
|
-
projectId?: string | null
|
|
190
|
-
limit?: number | null
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
export function inNamespace(namespace: string, filter: string | null | undefined): boolean {
|
|
194
|
-
const prefix = filter?.replace(/\/+$/, '')
|
|
195
|
-
return !prefix || namespace === prefix || namespace.startsWith(`${prefix}/`)
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
function snippet(body: string, needle: string): string {
|
|
199
|
-
const line = lines(body).map((entry) => entry.trim()).find((entry) => needle && entry.toLowerCase().includes(needle)) ?? firstParagraph(body)
|
|
200
|
-
return takeChars(line, 240)
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/** Ranked hits across the whole Wiki. Falls back to scanning files when the index is unavailable. */
|
|
204
|
-
export function searchEntries(root: string, query: string, options: SearchOptions = {}): WikiSearchHit[] {
|
|
205
|
-
const limit = Math.min(Math.max(options.limit ?? 20, 1), 100)
|
|
206
|
-
let boost: string | null = null
|
|
207
|
-
if (options.projectId) {
|
|
208
|
-
try {
|
|
209
|
-
boost = findProject(root, options.projectId).wikiNamespace
|
|
210
|
-
} catch {
|
|
211
|
-
boost = null
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
const { entries } = scan(root)
|
|
215
|
-
const kinds = options.kinds ?? []
|
|
216
|
-
let ranked: string[]
|
|
217
|
-
try {
|
|
218
|
-
ranked = wikiIndex.search(root, query, limit * 4, boost)
|
|
219
|
-
} catch (error) {
|
|
220
|
-
logLine(`[index] search fell back to files: ${error instanceof Error ? error.message : String(error)}`)
|
|
221
|
-
const needle = query.trim().toLowerCase()
|
|
222
|
-
if (!needle) return []
|
|
223
|
-
const rank = (entry: IndexedEntry): number => (entry.namespace === boost ? 0 : 2) + (entry.title.toLowerCase().includes(needle) ? 0 : 1)
|
|
224
|
-
ranked = entries
|
|
225
|
-
.filter((entry) => entry.title.toLowerCase().includes(needle) || entry.body.toLowerCase().includes(needle))
|
|
226
|
-
.map((entry, index) => ({ entry, index }))
|
|
227
|
-
.sort((left, right) => rank(left.entry) - rank(right.entry) || left.index - right.index)
|
|
228
|
-
.map(({ entry }) => entry.id)
|
|
229
|
-
}
|
|
230
|
-
// A volatile fact that has gone stale may no longer be true, so it ranks after everything else.
|
|
231
|
-
const stale = new Set(facts.loadMemories(store.wikiDir(root)).filter((fact) => facts.isStale(fact.front)).map((fact) => fact.front.id))
|
|
232
|
-
ranked = [...ranked.filter((id) => !stale.has(id)), ...ranked.filter((id) => stale.has(id))]
|
|
233
|
-
const needle = (query.split(/\s+/).find(Boolean) ?? '').toLowerCase()
|
|
234
|
-
const hits: WikiSearchHit[] = []
|
|
235
|
-
for (const id of ranked) {
|
|
236
|
-
if (hits.length >= limit) break
|
|
237
|
-
const entry = entries.find((candidate) => candidate.id === id)
|
|
238
|
-
if (!entry || (kinds.length && !kinds.includes(entry.kind)) || !inNamespace(entry.namespace, options.namespace)) continue
|
|
239
|
-
hits.push({ id: entry.id, kind: entry.kind, title: entry.title, type: entry.entryType, path: entry.path, namespace: entry.namespace, snippet: snippet(entry.body, needle) })
|
|
240
|
-
}
|
|
241
|
-
return hits
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// ---- Entries (pages, facts, blocks, registry items) ------------------------------------------
|
|
245
|
-
|
|
246
|
-
export type EntryFile =
|
|
247
|
-
| { kind: 'page'; page: ParsedPage }
|
|
248
|
-
| { kind: 'memory'; memory: facts.ParsedMemory }
|
|
249
|
-
| { kind: 'block'; block: blocks.Block }
|
|
250
|
-
| { kind: 'registry'; item: registry.RegistryItem }
|
|
251
|
-
|
|
252
|
-
/** Resolves a page id, fact id, registry id (`skills/<name>`), block id, or Wiki path (`/library/x`). */
|
|
253
|
-
export function resolveEntry(root: string, rawKey: string): EntryFile {
|
|
254
|
-
const wiki = store.wikiDir(root)
|
|
255
|
-
const key = rawKey.trim()
|
|
256
|
-
const registryKey = key.replace(/^\/+/, '')
|
|
257
|
-
if (registryKey.startsWith('blocks/')) return { kind: 'block', block: blocks.find(wiki, registryKey) }
|
|
258
|
-
const item = registry.list(root).find((entry) => entry.id === registryKey)
|
|
259
|
-
if (item) return { kind: 'registry', item }
|
|
260
|
-
const factId = key.startsWith('/facts/') ? key.slice('/facts/'.length) : key.startsWith('/memory/') ? key.slice('/memory/'.length) : null
|
|
261
|
-
if (factId !== null) return { kind: 'memory', memory: facts.findMemory(wiki, factId) }
|
|
262
|
-
const page = store.loadPages(wiki).find((entry) => entry.frontmatter.id === key || entry.location.path === key)
|
|
263
|
-
if (page) return { kind: 'page', page }
|
|
264
|
-
try {
|
|
265
|
-
return { kind: 'memory', memory: facts.findMemory(wiki, key) }
|
|
266
|
-
} catch {
|
|
267
|
-
throw notFound('Wiki entry')
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
export function entryId(entry: EntryFile): string {
|
|
272
|
-
return entry.kind === 'page' ? entry.page.frontmatter.id : entry.kind === 'memory' ? entry.memory.front.id : entry.kind === 'block' ? entry.block.id : entry.item.id
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
export function entryFile(entry: EntryFile): string {
|
|
276
|
-
return entry.kind === 'page' ? entry.page.path : entry.kind === 'memory' ? entry.memory.path : entry.kind === 'block' ? entry.block.path : entry.item.path
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
// ---- Reads for a person ------------------------------------------------------------------------
|
|
280
|
-
|
|
281
|
-
export function listPages(root: string, namespace?: string | null): WikiPageSummary[] {
|
|
282
|
-
return store.loadPages(store.wikiDir(root)).filter((page) => inNamespace(page.location.namespace, namespace)).map(summary)
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
export function loadPage(root: string, pageId: string): WikiPage {
|
|
286
|
-
const wiki = store.wikiDir(root)
|
|
287
|
-
const page = store.loadPages(wiki).find((entry) => entry.frontmatter.id === pageId || entry.location.path === pageId)
|
|
288
|
-
if (!page) throw notFound('Page')
|
|
289
|
-
return { ...summary(page), body: page.body, revisionId: store.currentRevision(wiki, page.frontmatter.id), file: page.path }
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
/** Revisions of an entry, newest first. */
|
|
293
|
-
export function listRevisions(root: string, id: string): store.WikiRevision[] {
|
|
294
|
-
return store.readManifest(store.wikiDir(root), id).reverse()
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
/** A revision's body without its frontmatter. */
|
|
298
|
-
export function loadRevision(root: string, id: string, revisionId: string): string {
|
|
299
|
-
const text = store.revisionText(store.wikiDir(root), id, revisionId)
|
|
300
|
-
return frontmatter.split(text)?.body ?? text
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
/**
|
|
304
|
-
* Restore never erases history: the current content is snapshotted, then the chosen revision
|
|
305
|
-
* becomes the content and is itself recorded as a new revision. Works for every kind of entry.
|
|
306
|
-
*/
|
|
307
|
-
export function restore(root: string, id: string, revisionId: string): void {
|
|
308
|
-
const wiki = store.wikiDir(root)
|
|
309
|
-
const entry = resolveEntry(root, id)
|
|
310
|
-
const resolved = entryId(entry)
|
|
311
|
-
const file = entryFile(entry)
|
|
312
|
-
const revisionText = store.revisionText(wiki, resolved, revisionId)
|
|
313
|
-
let nextText = revisionText
|
|
314
|
-
if (entry.kind === 'page') {
|
|
315
|
-
const restored = store.parsePage(revisionText, wiki, entry.page.path)
|
|
316
|
-
if (!restored) throw new HarnessError('invalid_data', 'The revision could not be read as a Wiki page.')
|
|
317
|
-
nextText = store.serializePage({ ...restored.frontmatter, id: entry.page.frontmatter.id, updatedAt: now() }, restored.body)
|
|
318
|
-
} else if (entry.kind === 'memory') {
|
|
319
|
-
if (!facts.parse(revisionText, file)) throw new HarnessError('invalid_data', 'The revision could not be read as a memory.')
|
|
320
|
-
} else if (entry.kind === 'block') {
|
|
321
|
-
const parsed = frontmatter.parse(revisionText)
|
|
322
|
-
if (!parsed || !blocks.parseFront(parsed.front)) throw new HarnessError('invalid_data', 'The revision could not be read as a memory block.')
|
|
323
|
-
}
|
|
324
|
-
let currentText = ''
|
|
325
|
-
try {
|
|
326
|
-
currentText = readFileSync(file, 'utf8')
|
|
327
|
-
} catch {
|
|
328
|
-
currentText = ''
|
|
329
|
-
}
|
|
330
|
-
store.snapshot(wiki, resolved, currentText, 'user', 'Before restore')
|
|
331
|
-
atomicWrite(file, nextText)
|
|
332
|
-
const revision = store.snapshot(wiki, resolved, nextText, 'user', `Restored revision ${revisionId.slice(0, 8)}`)
|
|
333
|
-
noteChange({ wiki, id: resolved, path: file, text: nextText, actor: 'user', reason: 'Restored an earlier revision' })
|
|
334
|
-
store.log(wiki, `- ${now()} · user · restore · \`${resolved}\` · revision \`${revision.id}\``)
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
/** Any entry's content in one shape. */
|
|
338
|
-
export interface WikiEntry {
|
|
339
|
-
id: string
|
|
340
|
-
kind: EntryKind
|
|
341
|
-
title: string
|
|
342
|
-
description: string
|
|
343
|
-
body: string
|
|
344
|
-
file: string
|
|
345
|
-
revisionId: string | null
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
export function loadEntry(root: string, id: string): WikiEntry {
|
|
349
|
-
const entry = resolveEntry(root, id)
|
|
350
|
-
const shared = { file: entryFile(entry), revisionId: store.currentRevision(store.wikiDir(root), entryId(entry)) }
|
|
351
|
-
if (entry.kind === 'page') return { id: entry.page.frontmatter.id, kind: 'page', title: entry.page.frontmatter.title, description: firstParagraph(entry.page.body), body: entry.page.body, ...shared }
|
|
352
|
-
if (entry.kind === 'memory') return { id: entry.memory.front.id, kind: 'memory', title: entry.memory.front.title, description: entry.memory.front.kind, body: entry.memory.body, ...shared }
|
|
353
|
-
if (entry.kind === 'block') return { id: entry.block.id, kind: 'block', title: entry.block.front.label, description: entry.block.front.description, body: entry.block.body, ...shared }
|
|
354
|
-
return { id: entry.item.id, kind: registryEntryKind(entry.item.kind) ?? 'skill', title: entry.item.name, description: entry.item.description, body: registry.itemBody(entry.item), ...shared }
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
export interface Backlink {
|
|
358
|
-
fromId: string
|
|
359
|
-
kind: string
|
|
360
|
-
title: string
|
|
361
|
-
type: string
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
/** Entries that link to `id`, from the index's link table (files when it is unavailable). */
|
|
365
|
-
export function backlinks(root: string, id: string): Backlink[] {
|
|
366
|
-
const { entries } = scan(root)
|
|
367
|
-
let pairs: [string, string][]
|
|
368
|
-
try {
|
|
369
|
-
pairs = wikiIndex.backlinks(root, id)
|
|
370
|
-
} catch {
|
|
371
|
-
pairs = entries.flatMap((entry) => entry.links.filter(([to]) => to === id).map(([, type]): [string, string] => [entry.id, type]))
|
|
372
|
-
}
|
|
373
|
-
return pairs.flatMap(([from, type]) => {
|
|
374
|
-
const entry = entries.find((candidate) => candidate.id === from)
|
|
375
|
-
return entry ? [{ fromId: from, kind: entry.kind, title: entry.title, type }] : []
|
|
376
|
-
})
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
export interface WikiNamespace {
|
|
380
|
-
namespace: string
|
|
381
|
-
label: string
|
|
382
|
-
projectId: string | null
|
|
383
|
-
pageCount: number
|
|
384
|
-
memoryCount: number
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
/** Project namespaces (registered projects first, then folders left by removed projects). */
|
|
388
|
-
export function namespaces(root: string): WikiNamespace[] {
|
|
389
|
-
const wiki = store.wikiDir(root)
|
|
390
|
-
const pages = store.loadPages(wiki)
|
|
391
|
-
const memories = facts.loadMemories(wiki)
|
|
392
|
-
const counts = (namespace: string) => ({
|
|
393
|
-
pageCount: pages.filter((page) => page.location.namespace === namespace).length,
|
|
394
|
-
memoryCount: memories.filter((memory) => memory.front.namespace === namespace).length
|
|
395
|
-
})
|
|
396
|
-
const seen = new Set<string>()
|
|
397
|
-
const result: WikiNamespace[] = []
|
|
398
|
-
for (const project of projectsOrNone(root)) {
|
|
399
|
-
seen.add(project.wikiNamespace)
|
|
400
|
-
result.push({ namespace: project.wikiNamespace, label: project.name, projectId: project.id, ...counts(project.wikiNamespace) })
|
|
401
|
-
}
|
|
402
|
-
let folders: string[] = []
|
|
403
|
-
try {
|
|
404
|
-
folders = readdirSync(join(wiki, 'projects')).filter((name) => store.isDir(join(wiki, 'projects', name)))
|
|
405
|
-
} catch {
|
|
406
|
-
folders = []
|
|
407
|
-
}
|
|
408
|
-
for (const namespace of folders.map((name) => `/projects/${name}`).sort(compareText)) {
|
|
409
|
-
if (!seen.has(namespace)) result.push({ namespace, label: namespace.slice('/projects/'.length), projectId: null, ...counts(namespace) })
|
|
410
|
-
}
|
|
411
|
-
return result
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
/** The newest 500 entries of `log.md`, newest first. */
|
|
415
|
-
export function recentLog(root: string): string[] {
|
|
416
|
-
let text = ''
|
|
417
|
-
try {
|
|
418
|
-
text = readFileSync(join(store.wikiDir(root), 'log.md'), 'utf8')
|
|
419
|
-
} catch {
|
|
420
|
-
text = ''
|
|
421
|
-
}
|
|
422
|
-
return lines(text).filter((line) => line.startsWith('- ')).map((line) => line.slice(2)).reverse().slice(0, 500)
|
|
423
|
-
}
|
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
2
|
-
import { tmpdir } from 'node:os'
|
|
3
|
-
import { join } from 'node:path'
|
|
4
|
-
import { DatabaseSync } from 'node:sqlite'
|
|
5
|
-
import { expect, test } from 'vitest'
|
|
6
|
-
|
|
7
|
-
import { writeFile } from '../../tests/helpers'
|
|
8
|
-
import { writeJson } from '../core/fs'
|
|
9
|
-
import { all, closeIndex, dbPath, migrate, withIndex } from '../core/index'
|
|
10
|
-
import { runRecordSchema, type RunRecord } from '../sessions/contracts'
|
|
11
|
-
import { recordRun, runsSince, syncRuns } from '../sessions/runIndex'
|
|
12
|
-
import { ensure, rebuild, search } from './wikiIndex'
|
|
13
|
-
|
|
14
|
-
const temp = () => mkdtempSync(join(tmpdir(), 'harness-index-'))
|
|
15
|
-
const page = (root: string, relative: string, text: string) => writeFile(join(root, 'wiki', relative), text)
|
|
16
|
-
const record = (fields: Partial<RunRecord> & { id: string }): RunRecord => runRecordSchema.parse({ projectId: null, mode: 'chat', harness: 'pi', sessionType: 'structured', sessionId: 'agent-1', nativeSessionId: null, workingDirectory: null, worktreeId: null, goal: null, startedAt: new Date().toISOString(), endedAt: null, status: 'completed', ...fields })
|
|
17
|
-
|
|
18
|
-
test('rebuild indexes the whole wiki and boosts the current project', () => {
|
|
19
|
-
const root = temp()
|
|
20
|
-
page(root, 'library/a.md', '---\nid: tauri\ntitle: Use Tauri for the desktop shell\ntype: decision\n---\n\nRust host and React renderer.\n')
|
|
21
|
-
page(root, 'projects/app/b.md', '---\nid: notes\ntitle: Meeting notes\ntype: summary\n---\n\nWe mentioned tauri once.\n')
|
|
22
|
-
page(root, 'facts/m.md', '---\nid: shell\ntitle: Shell choice\nkind: decision\n---\n\nTauri stays.\n')
|
|
23
|
-
expect(rebuild(root)).toBe(2)
|
|
24
|
-
const first = search(root, 'tauri', 10, null)
|
|
25
|
-
expect(first[0]).toBe('tauri')
|
|
26
|
-
expect(first).toHaveLength(3)
|
|
27
|
-
expect(search(root, 'rend', 10, null)).toEqual(['tauri'])
|
|
28
|
-
// The current project's namespace ranks higher but nothing else is hidden.
|
|
29
|
-
const boosted = search(root, 'tauri', 10, '/projects/app')
|
|
30
|
-
expect(boosted.indexOf('notes')).toBeLessThanOrEqual(first.indexOf('notes'))
|
|
31
|
-
expect(boosted).toHaveLength(3)
|
|
32
|
-
// Rebuilding from files after deleting the index gives the same results.
|
|
33
|
-
closeIndex(root)
|
|
34
|
-
rmSync(dbPath(root))
|
|
35
|
-
expect(ensure(root)).toBe(2)
|
|
36
|
-
expect(search(root, 'tauri', 10, null)).toEqual(first)
|
|
37
|
-
closeIndex(root)
|
|
38
|
-
})
|
|
39
|
-
|
|
40
|
-
test('runs are listed per account and agent and survive a rebuild', () => {
|
|
41
|
-
const root = temp()
|
|
42
|
-
const run = record({ id: 'run1', mode: 'code', harness: 'codex', sessionId: 's1', agentId: 'agent-7', accountId: 'acc1' })
|
|
43
|
-
writeJson(join(root, 'runs/run1/run.json'), run)
|
|
44
|
-
writeFileSync(join(root, 'runs/run1/events.jsonl'), '{"type":"usage.updated","payload":{"inputTokens":7,"outputTokens":3,"model":"gpt"}}\n')
|
|
45
|
-
recordRun(root, run)
|
|
46
|
-
expect(runsSince(root, { account: 'acc1' }, null)).toEqual(['run1'])
|
|
47
|
-
expect(runsSince(root, { agent: 'agent-7' }, null)).toEqual(['run1'])
|
|
48
|
-
rebuild(root)
|
|
49
|
-
expect(runsSince(root, { account: 'acc1' }, null)).toEqual(['run1'])
|
|
50
|
-
expect(runsSince(root, { agent: 'agent-7' }, null)).toEqual(['run1'])
|
|
51
|
-
expect(runsSince(root, { agent: 'agent-8' }, null)).toEqual([])
|
|
52
|
-
expect(withIndex(root, (database) => all<{ count: number }>(database, "SELECT COUNT(*) AS count FROM sqlite_master WHERE name = 'usage'")[0].count)).toBe(0)
|
|
53
|
-
closeIndex(root)
|
|
54
|
-
})
|
|
55
|
-
|
|
56
|
-
test('runs the index never saw are picked up from their files', () => {
|
|
57
|
-
const root = temp()
|
|
58
|
-
const run = (id: string, status: string) => record({ id, status, startedAt: `2026-10-01T10:00:0${id.length}.000Z`, accountId: 'acc1' })
|
|
59
|
-
// The index file exists with one run before the others are noticed, so `ensure` does not rebuild.
|
|
60
|
-
writeJson(join(root, 'runs/a/run.json'), run('a', 'running'))
|
|
61
|
-
recordRun(root, run('a', 'running'))
|
|
62
|
-
writeJson(join(root, 'runs/a/run.json'), run('a', 'completed'))
|
|
63
|
-
writeJson(join(root, 'runs/bb/run.json'), run('bb', 'completed'))
|
|
64
|
-
|
|
65
|
-
ensure(root)
|
|
66
|
-
expect(runsSince(root, { session: 'agent-1' }, null)).toEqual(['a', 'bb'])
|
|
67
|
-
expect(runsSince(root, { session: 'agent-1' }, '2026-10-01T10:00:02.000Z')).toEqual(['bb'])
|
|
68
|
-
// Chat runs from before the agent column count for the agent whose conversation they ran in.
|
|
69
|
-
expect(runsSince(root, { agent: 'agent-1' }, null)).toHaveLength(2)
|
|
70
|
-
expect(runsSince(root, { account: 'acc1' }, null)).toEqual(['a', 'bb'])
|
|
71
|
-
expect(runsSince(root, { account: 'other' }, null)).toEqual([])
|
|
72
|
-
expect(withIndex(root, (database) => all<{ status: string }>(database, "SELECT status FROM runs WHERE id = 'a'")[0].status)).toBe('completed')
|
|
73
|
-
// Nothing is rewritten once the tables match the files.
|
|
74
|
-
expect(syncRuns(root)).toBe(0)
|
|
75
|
-
closeIndex(root)
|
|
76
|
-
})
|
|
77
|
-
|
|
78
|
-
test('failed migration restores the backup', () => {
|
|
79
|
-
const path = join(temp(), 'index.sqlite')
|
|
80
|
-
const good = ["CREATE TABLE kept (value TEXT); INSERT INTO kept VALUES ('before');"]
|
|
81
|
-
migrate(path, good).close()
|
|
82
|
-
expect(() => migrate(path, [good[0], 'CREATE TABLE extra (value TEXT); THIS IS NOT SQL;'])).toThrow('Index migration 2 failed')
|
|
83
|
-
const database = new DatabaseSync(path)
|
|
84
|
-
expect((database.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get() as { value: string }).value).toBe('1')
|
|
85
|
-
expect((database.prepare('SELECT value FROM kept').get() as { value: string }).value).toBe('before')
|
|
86
|
-
expect(() => database.prepare('SELECT COUNT(*) FROM extra').get()).toThrow()
|
|
87
|
-
database.close()
|
|
88
|
-
})
|
package/src/memory/wikiIndex.ts
DELETED
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
// The Wiki's rows in the index: full-text entries, links, sources, and revisions, all derived
|
|
2
|
-
// from the canonical files.
|
|
3
|
-
import { existsSync } from 'node:fs'
|
|
4
|
-
import type { DatabaseSync } from 'node:sqlite'
|
|
5
|
-
|
|
6
|
-
import { logLine } from '../core/bus'
|
|
7
|
-
import { all, dbPath, deleteIndex, ftsQuery, run, transaction, withIndex } from '../core/index'
|
|
8
|
-
import { readAllRuns, syncRuns, upsertRun } from '../sessions/runIndex'
|
|
9
|
-
import { scan } from './wiki'
|
|
10
|
-
|
|
11
|
-
/** Replaces every Wiki row from canonical files. Returns the number of pages indexed. */
|
|
12
|
-
export function indexWiki(database: DatabaseSync, root: string): number {
|
|
13
|
-
const { entries, sources } = scan(root)
|
|
14
|
-
transaction(database, () => {
|
|
15
|
-
for (const table of ['entries', 'entries_fts', 'links', 'sources', 'revisions']) database.exec(`DELETE FROM ${table}`)
|
|
16
|
-
for (const entry of entries) {
|
|
17
|
-
run(
|
|
18
|
-
database,
|
|
19
|
-
'INSERT OR REPLACE INTO entries (id, kind, title, type, path, namespace, file, updated_at, source_ids) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)',
|
|
20
|
-
entry.id,
|
|
21
|
-
entry.kind,
|
|
22
|
-
entry.title,
|
|
23
|
-
entry.entryType,
|
|
24
|
-
entry.path,
|
|
25
|
-
entry.namespace,
|
|
26
|
-
entry.file,
|
|
27
|
-
entry.updatedAt,
|
|
28
|
-
JSON.stringify(entry.sourceIds)
|
|
29
|
-
)
|
|
30
|
-
const sourceNames = entry.sourceIds.flatMap((id) => sources.find((source) => source.id === id)?.name ?? [])
|
|
31
|
-
run(
|
|
32
|
-
database,
|
|
33
|
-
'INSERT INTO entries_fts (id, kind, namespace, title, body, links, sources) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)',
|
|
34
|
-
entry.id,
|
|
35
|
-
entry.kind,
|
|
36
|
-
entry.namespace,
|
|
37
|
-
entry.title,
|
|
38
|
-
entry.body,
|
|
39
|
-
entry.links.map(([to]) => to).join(' '),
|
|
40
|
-
`${entry.path.replaceAll('/', ' ')} ${entry.sourceIds.join(' ')} ${sourceNames.join(' ')}`
|
|
41
|
-
)
|
|
42
|
-
for (const [to, type] of entry.links) run(database, 'INSERT INTO links (from_id, to_id, type) VALUES (?1, ?2, ?3)', entry.id, to, type)
|
|
43
|
-
for (const [id, createdAt, actor] of entry.revisions) run(database, 'INSERT OR REPLACE INTO revisions (entry_id, id, created_at, actor) VALUES (?1, ?2, ?3, ?4)', entry.id, id, createdAt, actor)
|
|
44
|
-
}
|
|
45
|
-
for (const source of sources) run(database, 'INSERT OR REPLACE INTO sources (id, name, imported_at, version) VALUES (?1, ?2, ?3, ?4)', source.id, source.name, source.importedAt, source.version)
|
|
46
|
-
})
|
|
47
|
-
return entries.filter((entry) => entry.kind === 'page').length
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
export function reindexWiki(root: string): number {
|
|
51
|
-
return withIndex(root, (database) => indexWiki(database, root))
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/** `reindexWiki` for callers that must not fail because the index is unavailable. */
|
|
55
|
-
export function reindexQuietly(root: string): void {
|
|
56
|
-
try {
|
|
57
|
-
reindexWiki(root)
|
|
58
|
-
} catch (error) {
|
|
59
|
-
logLine(`[index] reindex failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/** Deletes the index and rebuilds it from canonical files. Returns pages indexed. */
|
|
64
|
-
export function rebuild(root: string): number {
|
|
65
|
-
deleteIndex(root)
|
|
66
|
-
return withIndex(root, (database) => {
|
|
67
|
-
const pages = indexWiki(database, root)
|
|
68
|
-
const records = readAllRuns(root)
|
|
69
|
-
transaction(database, () => records.forEach((record) => upsertRun(database, record)))
|
|
70
|
-
return pages
|
|
71
|
-
})
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* Builds the index when it is missing and otherwise refreshes the Wiki rows, which also
|
|
76
|
-
* catches edits made while nothing was running. `after` runs last, whatever happened before it.
|
|
77
|
-
*/
|
|
78
|
-
export function ensure(root: string, after?: () => void): number {
|
|
79
|
-
let pages: number
|
|
80
|
-
if (existsSync(dbPath(root))) {
|
|
81
|
-
pages = reindexWiki(root)
|
|
82
|
-
try {
|
|
83
|
-
syncRuns(root)
|
|
84
|
-
} catch (error) {
|
|
85
|
-
logLine(`[index] ${error instanceof Error ? error.message : String(error)}`)
|
|
86
|
-
}
|
|
87
|
-
} else {
|
|
88
|
-
pages = rebuild(root)
|
|
89
|
-
}
|
|
90
|
-
after?.()
|
|
91
|
-
return pages
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/**
|
|
95
|
-
* Ranked entry ids across the whole Wiki (best match first). Entries in `boostNamespace`
|
|
96
|
-
* rank higher; nothing is filtered out by namespace here.
|
|
97
|
-
*/
|
|
98
|
-
export function search(root: string, query: string, limit: number, boostNamespace: string | null): string[] {
|
|
99
|
-
const fts = ftsQuery(query)
|
|
100
|
-
if (!fts) return []
|
|
101
|
-
return withIndex(root, (database) =>
|
|
102
|
-
all<{ id: string }>(
|
|
103
|
-
database,
|
|
104
|
-
`SELECT id FROM entries_fts WHERE entries_fts MATCH ?1
|
|
105
|
-
ORDER BY bm25(entries_fts, 0, 0, 0, 10.0, 1.0, 2.0, 2.0) * (CASE WHEN ?2 IS NOT NULL AND namespace = ?2 THEN 1.5 ELSE 1.0 END)
|
|
106
|
-
LIMIT ?3`,
|
|
107
|
-
fts,
|
|
108
|
-
boostNamespace,
|
|
109
|
-
limit
|
|
110
|
-
).map((row) => row.id)
|
|
111
|
-
)
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
/** (from id, link type) for every entry linking to `entryId`. */
|
|
115
|
-
export function backlinks(root: string, entryId: string): [string, string][] {
|
|
116
|
-
return withIndex(root, (database) => all<{ from_id: string; type: string }>(database, 'SELECT from_id, type FROM links WHERE to_id = ?1 ORDER BY from_id', entryId).map((row) => [row.from_id, row.type]))
|
|
117
|
-
}
|