@iiiokojiadbi/shift 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/core/docs.ts ADDED
@@ -0,0 +1,396 @@
1
+ // Generated from core/src by scripts/sync-core.ts — edit the source, not this copy.
2
+ import { DIR, MOD } from './layout'
3
+ import { frontmatter, type Heading, headings, lineCount, type Pin, pinned } from './markdown'
4
+ import { basename, dirname, fromRoot, normalize } from './paths'
5
+ import type { Project } from './project'
6
+ import type { DocsSettings } from './settings'
7
+ import type { Store } from './store'
8
+
9
+ // The project's documents the mod keeps, the ones `[docs] dirs` names — folders, or a single document such as
10
+ // AGENTS.md at the root: a map of them in SHIFT.md, a toc inside each, checks after the agent writes a file, a
11
+ // document's sections with line numbers, and the documents that mention a file. With none named the mod touches no
12
+ // document. Every document is read by one reader: its description and its headings outside code fences.
13
+
14
+ export const TOC_START = '<!-- toc -->'
15
+ export const TOC_END = '<!-- /toc -->'
16
+
17
+ // `##` and `###` outside code fences, each with the `> ` line right under it as its note.
18
+ export function outline(text: string): Heading[] {
19
+ return headings(text).filter(heading => heading.level === 2 || heading.level === 3)
20
+ }
21
+
22
+ // The text with its toc block written from its headings: in place of the old block, else right under the `# ` title;
23
+ // null when there is neither.
24
+ export function withToc(text: string): string | null {
25
+ const rows = outline(text).map(
26
+ heading => `${' '.repeat(heading.level - 2)}- ${heading.title}${heading.note === '' ? '' : ` - ${heading.note}`}`,
27
+ )
28
+ const block = [TOC_START, ...rows, TOC_END].join('\n')
29
+ const start = text.indexOf(TOC_START)
30
+ const end = start === -1 ? -1 : text.indexOf(TOC_END, start)
31
+ if (end !== -1) return `${text.slice(0, start)}${block}${text.slice(end + TOC_END.length)}`
32
+ const title = headings(text).find(heading => heading.level === 1)
33
+ if (title === undefined) return null
34
+ const lines = text.split('\n')
35
+
36
+ return [...lines.slice(0, title.line), '', block, ...lines.slice(title.line)].join('\n')
37
+ }
38
+
39
+ export function isDoc(path: string, dirs: readonly string[]): boolean {
40
+ return path.endsWith('.md') && dirs.map(trimDir).some(entry => (isFile(entry) ? path === entry : path.startsWith(`${entry}/`)))
41
+ }
42
+
43
+ function description(text: string): string | null {
44
+ const found = frontmatter(text).get('description') ?? ''
45
+
46
+ return found === '' ? null : found
47
+ }
48
+
49
+ // What a document breaks of the settings: its description and its length.
50
+ export function docRemarks(text: string, docs: DocsSettings): string[] {
51
+ const remarks: string[] = []
52
+ const about = description(text)
53
+ if (about === null) remarks.push(NO_DESCRIPTION)
54
+ else if (docs.maxDescription !== null && about.length > docs.maxDescription) {
55
+ remarks.push(`a description of ${about.length} characters, over \`[docs] max_description = ${docs.maxDescription}\``)
56
+ }
57
+ const lines = lineCount(text)
58
+ if (docs.maxLines !== null && lines > docs.maxLines) {
59
+ remarks.push(`${lines} lines, over \`[docs] max_lines = ${docs.maxLines}\` — split it by topic`)
60
+ }
61
+ if (docs.toc && withToc(text) === null) remarks.push('no `# ` title line for the toc to go under')
62
+
63
+ return remarks
64
+ }
65
+
66
+ // Links to documents that are not there: a `<folder>/….md` path anywhere in the text, from the project's root, and in
67
+ // a Markdown file a `[text](file.md)` from the file's own folder. A path with `<` or `*` is a pattern, not a link.
68
+ export async function brokenLinks(store: Store, root: string, path: string, text: string, dirs: readonly string[]): Promise<string[]> {
69
+ const targets = new Map<string, string | null>()
70
+ for (const dir of dirs.map(trimDir).filter(entry => !isFile(entry))) {
71
+ const pattern = new RegExp(`(?<![\\w./-])${escapeRegExp(dir)}/[^\\s\`'"()\\[\\],:;»<>*]+?\\.md(?![\\w-])`, 'g')
72
+ for (const found of text.matchAll(pattern)) targets.set(found[0], found[0])
73
+ }
74
+ if (path.endsWith('.md')) {
75
+ for (const found of text.matchAll(/\]\(([^)#\s]+\.md)(?:#[^)]*)?\)/g)) {
76
+ const link = found[1] ?? ''
77
+ if (/^[a-z]+:/i.test(link) || link.includes('*') || targets.has(link)) continue
78
+ targets.set(link, normalize(`${dirname(path)}/${link}`))
79
+ }
80
+ }
81
+ const missing: string[] = []
82
+ for (const [link, target] of targets) {
83
+ if (target === null || !(await store.exists(`${root}/${target}`))) missing.push(link)
84
+ }
85
+
86
+ return missing
87
+ }
88
+
89
+ // After the agent wrote a file: a document gets its toc rewritten and the map follows it; a document's remarks and any
90
+ // file's broken links come back as lines of one note. None when all is well: a check that speaks on every edit is noise.
91
+ export async function docsAfterWrite(project: Project, file: string): Promise<string[]> {
92
+ const { store, root, settings } = project
93
+ const docs = settings.docs
94
+ if (!settings.enabled || docs.dirs.length === 0 || !file.startsWith(`${root}/`)) return []
95
+ const path = file.slice(root.length + 1)
96
+ if (path.startsWith(`${DIR}/`)) return []
97
+ let text = await store.read(file)
98
+ if (text === null) return []
99
+
100
+ const lines: string[] = []
101
+ const doc = isDoc(path, docs.dirs)
102
+ if (doc && docs.toc) {
103
+ const next = withToc(text)
104
+ if (next !== null && next !== text) {
105
+ await store.write(file, next)
106
+ text = next
107
+ lines.push(`The mod rewrote the toc of \`${path}\`: read the file before editing it again.`)
108
+ }
109
+ }
110
+ // Pinned blocks past the limit are told when the document written has some: the write may be what passed it.
111
+ const over = doc ? pinnedOver(await refreshDocs(project), docs) : null
112
+ if (over !== null && pinned(text).length > 0) lines.push(over)
113
+ const remarks = [
114
+ ...(doc ? docRemarks(text, docs) : []),
115
+ ...(await brokenLinks(store, root, path, text, docs.dirs)).map(link => `a link to \`${link}\` that is not there`),
116
+ ]
117
+ if (remarks.length > 0) lines.push(`\`${path}\`: ${remarks.join('; ')}.`)
118
+
119
+ return lines
120
+ }
121
+
122
+ // The documents with remarks and the root's Markdown files with a broken link, short enough to read after a shift:
123
+ // documents without a description as one line with a count, the other remarks for the first few documents and a count
124
+ // of the rest. Null when there are none.
125
+ export async function docsReport(project: Project): Promise<string | null> {
126
+ const { store, root, settings } = project
127
+ const docs = settings.docs
128
+ if (!settings.enabled || docs.dirs.length === 0) return null
129
+ const rootFiles = (await store.list(root, 'file')).filter(name => name.endsWith('.md'))
130
+ const undescribed: string[] = []
131
+ const lines: string[] = []
132
+ const kept: Doc[] = []
133
+ for (const path of [...(await docPaths(store, root, docs.dirs)), ...rootFiles]) {
134
+ const text = (await store.read(`${root}/${path}`)) ?? ''
135
+ if (isDoc(path, docs.dirs)) kept.push(readDoc(path, text))
136
+ const own = isDoc(path, docs.dirs) ? docRemarks(text, docs) : []
137
+ if (own.includes(NO_DESCRIPTION)) undescribed.push(path)
138
+ const remarks = [
139
+ ...own.filter(remark => remark !== NO_DESCRIPTION),
140
+ ...(await brokenLinks(store, root, path, text, docs.dirs)).map(link => `a link to \`${link}\` that is not there`),
141
+ ]
142
+ if (remarks.length > 0) lines.push(`- \`${path}\`: ${remarks.join('; ')}`)
143
+ }
144
+ const shown = lines.slice(0, REPORT_DOCS)
145
+ const pins = pinnedOver(kept, docs)
146
+ const report = [
147
+ ...(pins === null ? [] : [`- ${pins}`]),
148
+ ...(undescribed.length === 0
149
+ ? []
150
+ : [
151
+ `- ${undescribed.length} ${undescribed.length === 1 ? 'document' : 'documents'} without \`description:\`: ${undescribed
152
+ .slice(0, 3)
153
+ .map(path => `\`${path}\``)
154
+ .join(', ')}${undescribed.length > 3 ? ' and the rest the map marks' : ''}`,
155
+ ]),
156
+ ...shown,
157
+ ...(lines.length > shown.length
158
+ ? [`- and ${lines.length - shown.length} more documents with remarks: each comes after its next edit`]
159
+ : []),
160
+ ]
161
+
162
+ return report.length === 0 ? null : report.join('\n')
163
+ }
164
+
165
+ const NO_DESCRIPTION = 'no `description:` in its frontmatter'
166
+
167
+ // How many documents the report after a shift names one by one.
168
+ const REPORT_DOCS = 10
169
+
170
+ // A document's description and its `##` and `###` with line numbers, so the agent reads only the section it needs.
171
+ export async function docOutline({ store, root }: Project, file: string): Promise<string> {
172
+ const path = fromRoot(root, file)
173
+ const text = await store.read(`${root}/${path}`)
174
+ if (text === null) return `No file at ${path}.`
175
+ const rows = outline(text).map(
176
+ heading =>
177
+ `${' '.repeat(heading.level - 2)}${String(heading.line).padStart(5)} ${heading.title}${heading.note === '' ? '' : ` - ${heading.note}`}`,
178
+ )
179
+
180
+ return [`${path} — ${lineCount(text)} lines`, ` ${description(text) ?? 'no description'}`, ...rows].join('\n')
181
+ }
182
+
183
+ // The documents that name a file — its name without the folder, as a whole word — to check them against a change.
184
+ export async function docsMentioning({ store, root, settings }: Project, file: string): Promise<string> {
185
+ const dirs = settings.docs.dirs
186
+ if (dirs.length === 0) return 'The owner turned the docs off: `[docs] dirs = []`.'
187
+ const name = basename(file)
188
+ if (name === '') return 'Name a file.'
189
+ const pattern = new RegExp(`(?<![\\w-])${escapeRegExp(name)}(?![\\w-])`)
190
+ const found: string[] = []
191
+ for (const path of await docPaths(store, root, dirs)) {
192
+ if (path === fromRoot(root, file)) continue
193
+ if (pattern.test((await store.read(`${root}/${path}`)) ?? '')) found.push(path)
194
+ }
195
+ const where = dirs.map(dir => entryName(trimDir(dir))).join(', ')
196
+
197
+ return found.length === 0
198
+ ? `No document in ${where} mentions \`${name}\`.`
199
+ : [`\`${name}\` is mentioned in:`, ...found.map(path => `- ${path}`)].join('\n')
200
+ }
201
+
202
+ function escapeRegExp(text: string): string {
203
+ return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
204
+ }
205
+
206
+ // The map in SHIFT.md: per folder, each document's path, its `description:` and its `##` sections, so the agent knows
207
+ // what the docs hold without opening one. Built from the files every time, so it cannot go stale; the mod rewrites only
208
+ // the block between its markers.
209
+
210
+ export const MAP_START = `<!-- ${MOD}:docs — written by the mod from each document's description and ## headings; edit those, not this -->`
211
+ export const MAP_END = `<!-- /${MOD}:docs -->`
212
+
213
+ export type Doc = { path: string; description: string | null; sections: string[]; pins: Pin[] }
214
+
215
+ // Every document the settings name, from the project's root, sorted: each .md file under a folder — a missing folder
216
+ // has none, a hidden one is skipped — and a named document that is there.
217
+ export async function docPaths(store: Store, root: string, dirs: readonly string[]): Promise<string[]> {
218
+ const paths = new Set<string>()
219
+ const walk = async (dir: string): Promise<void> => {
220
+ for (const name of await store.list(`${root}/${dir}`, 'file')) if (name.endsWith('.md')) paths.add(`${dir}/${name}`)
221
+ for (const name of await store.list(`${root}/${dir}`, 'dir')) if (!name.startsWith('.')) await walk(`${dir}/${name}`)
222
+ }
223
+ for (const entry of dirs.map(trimDir)) {
224
+ if (!isFile(entry)) await walk(entry)
225
+ else if ((await store.read(`${root}/${entry}`)) !== null) paths.add(entry)
226
+ }
227
+
228
+ return [...paths].sort((a, b) => a.localeCompare(b))
229
+ }
230
+
231
+ export function trimDir(dir: string): string {
232
+ return dir.replace(/\/+$/, '')
233
+ }
234
+
235
+ // An entry of `[docs] dirs` that names one document rather than a folder.
236
+ function isFile(entry: string): boolean {
237
+ return entry.endsWith('.md')
238
+ }
239
+
240
+ // An entry as the mod names it to the agent: a folder with its slash, a document as it is.
241
+ function entryName(entry: string): string {
242
+ return isFile(entry) ? `\`${entry}\`` : `\`${entry}/\``
243
+ }
244
+
245
+ // The block itself; past `[docs] max_chars` first without the sections, then only folders and file names, so it never
246
+ // crowds the session and loses the descriptions last. A map cut down says so, and what this level dropped, so the
247
+ // agent knows what it does not see. A folder of the
248
+ // settings with nothing to list is named with why — no such folder, or no documents in it — so the agent knows the
249
+ // project has no docs there and does not go searching. `[docs] dirs = []` is the owner turning the map off — the
250
+ // project's docs may be shared with people and not the agent's to describe — and the block says just that.
251
+ export function renderMap(docs: readonly Doc[], settings: DocsSettings, missing: readonly string[]): string {
252
+ const { dirs, maxChars } = settings
253
+ if (dirs.length === 0) {
254
+ const off =
255
+ "off: the owner set `[docs] dirs = []` in the settings. No map of the documents; do not add descriptions, headings or anything else to the project's docs for it."
256
+
257
+ return [MAP_START, '', `## Documents`, '', off, '', MAP_END].join('\n')
258
+ }
259
+ const empty = dirs
260
+ .map(trimDir)
261
+ .filter(entry => !docs.some(doc => isDoc(doc.path, [entry])))
262
+ .map(entry => `- ${entryName(entry)} — ${isFile(entry) ? 'no such file' : missing.includes(entry) ? 'no such folder' : 'no documents'}`)
263
+ if (docs.length === 0) return [MAP_START, '', `## Documents`, '', 'none:', ...empty, '', MAP_END].join('\n')
264
+ const folders = new Map<string, Doc[]>()
265
+ for (const doc of docs) {
266
+ const folder = dirname(doc.path)
267
+ folders.set(folder, [...(folders.get(folder) ?? []), doc])
268
+ }
269
+ // A folder before the folders inside it.
270
+ const sorted = [...folders].sort(([a], [b]) => a.localeCompare(b))
271
+ const listed = (sections: boolean): string[] =>
272
+ sorted.map(([folder, inside]) =>
273
+ [
274
+ `### ${folder}/`,
275
+ '',
276
+ ...inside.flatMap(doc => [
277
+ `- \`${doc.path}\` — ${doc.description ?? 'no description: add `description:` to its frontmatter'}`,
278
+ ...(!sections || doc.sections.length === 0 ? [] : [` sections: ${doc.sections.join(' · ')}`]),
279
+ ...pinRows(doc.pins),
280
+ ]),
281
+ ].join('\n'),
282
+ )
283
+ const size = (body: string[]): number => body.join('\n\n').length
284
+ const full = listed(true)
285
+ const described = listed(false)
286
+ const over = `the full map is ${size(full)} characters, over \`[docs] max_chars = ${maxChars}\` in the settings`
287
+ const back =
288
+ 'Tell the owner: they may raise `[docs] max_chars`; shorter descriptions and fewer `##` sections also bring the map back whole.'
289
+ const [body, note] =
290
+ size(full) <= maxChars
291
+ ? [full, 'Open a document for numbers, commands and names: the map only points.']
292
+ : size(described) <= maxChars
293
+ ? [
294
+ described,
295
+ `Cut down: ${over}. Dropped at this level: the \`sections:\` line of every document — open a document for its \`##\` sections. Whole: every document with its description. ${back}`,
296
+ ]
297
+ : [
298
+ sorted.map(([folder, inside]) =>
299
+ [
300
+ `- \`${folder}/\` — ${inside.length} docs: ${inside.map(doc => basename(doc.path)).join(', ')}`,
301
+ ...inside.flatMap(doc => (doc.pins.length === 0 ? [] : [` \`${doc.path}\`:`, ...pinRows(doc.pins, ' ')])),
302
+ ].join('\n'),
303
+ ),
304
+ `Cut down: ${over}, and ${size(described)} without the sections. Dropped at this level: the description and the sections of every document — read a document's frontmatter \`description:\` for what it holds. Whole: every folder with its file names. ${back}`,
305
+ ]
306
+ const nothing = empty.length === 0 ? [] : ['', 'Nothing to list:', ...empty]
307
+ const pins = pinnedOver(docs, settings)
308
+
309
+ return [
310
+ MAP_START,
311
+ '',
312
+ `## Documents`,
313
+ '',
314
+ note,
315
+ ...(pins === null ? [] : ['', pins]),
316
+ '',
317
+ body.join('\n\n'),
318
+ ...nothing,
319
+ '',
320
+ MAP_END,
321
+ ].join('\n')
322
+ }
323
+
324
+ // SHIFT.md with its map block replaced, or added at the end; a new file starts with a heading and a line on what it is.
325
+ export function withMap(shift: string | null, map: string): string {
326
+ if (shift === null) {
327
+ return `# Shift\n\nHow this project keeps its knowledge. The rules are written on the owner's word; the map below is the mod's.\n\n${map}\n`
328
+ }
329
+ const start = shift.indexOf(MAP_START.slice(0, MAP_START.indexOf(' — ')))
330
+ const end = shift.indexOf(MAP_END)
331
+ if (start === -1 || end === -1) return `${shift.trimEnd()}\n\n${map}\n`
332
+
333
+ return `${shift.slice(0, start)}${map}${shift.slice(end + MAP_END.length)}`
334
+ }
335
+
336
+ // Every document read once: its toc rewritten when the settings ask for one, then the map in SHIFT.md written when it
337
+ // changed — nothing when there are no documents and no SHIFT.md yet. At the session's start, a shift and a write.
338
+ export async function refreshDocs(project: Project): Promise<Doc[]> {
339
+ const { store, root } = project
340
+ const settings = project.settings.docs
341
+ const docs: Doc[] = []
342
+ for (const path of await docPaths(store, root, settings.dirs)) {
343
+ let text = (await store.read(`${root}/${path}`)) ?? ''
344
+ const next = settings.toc ? withToc(text) : null
345
+ if (next !== null && next !== text) {
346
+ await store.write(`${root}/${path}`, next)
347
+ text = next
348
+ }
349
+ docs.push(readDoc(path, text))
350
+ }
351
+ const missing: string[] = []
352
+ for (const entry of settings.dirs.map(trimDir)) {
353
+ if (!(await store.exists(`${root}/${entry}`))) missing.push(entry)
354
+ }
355
+ const shift = `${root}/${project.settings.files.shift}`
356
+ const current = await store.read(shift)
357
+ if (docs.length === 0 && current === null) return docs
358
+ const map = withMap(current, renderMap(docs, settings, missing))
359
+ if (map !== current) await store.write(shift, map)
360
+
361
+ return docs
362
+ }
363
+
364
+ // A document's pinned blocks under its entry, each with the section it is in: the label on a line of its own, then
365
+ // every line of the block under it starting with " · ", a list item's own marker replaced.
366
+ function pinRows(pins: readonly Pin[], indent = ''): string[] {
367
+ return pins.flatMap(pin => [
368
+ `${indent} pinned${pin.section === null ? '' : ` (${pin.section})`}:`,
369
+ ...pin.text
370
+ .split('\n')
371
+ .map(line => line.trim().replace(/^[-*+] +/, ''))
372
+ .filter(line => line !== '')
373
+ .map(line => `${indent} · ${line}`),
374
+ ])
375
+ }
376
+
377
+ // The note when the pinned blocks together pass `[docs] max_pinned`; null within it. Never a refusal: they all show.
378
+ export function pinnedOver(docs: readonly Doc[], settings: DocsSettings): string | null {
379
+ const size = docs.reduce((sum, doc) => sum + doc.pins.reduce((each, pin) => each + pin.text.length, 0), 0)
380
+
381
+ return size <= settings.maxPinned
382
+ ? null
383
+ : `Pinned: ${size} characters, over \`[docs] max_pinned = ${settings.maxPinned}\` in the settings — every block still shows. Unpin what is not needed in every task, or tell the owner to raise the limit.`
384
+ }
385
+
386
+ // What the map shows of a document.
387
+ function readDoc(path: string, text: string): Doc {
388
+ return {
389
+ path,
390
+ description: description(text),
391
+ sections: outline(text)
392
+ .filter(h => h.level === 2)
393
+ .map(h => h.title),
394
+ pins: pinned(text),
395
+ }
396
+ }