mikser-io 9.41.0 → 9.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/api-reference.md +132 -0
- package/index.js +2 -0
- package/package.json +1 -1
- package/src/builtin-tools.js +39 -0
- package/src/search.js +284 -0
- package/src/utils.js +21 -3
- package/src/write.js +279 -0
package/docs/api-reference.md
CHANGED
|
@@ -483,6 +483,138 @@ Query types throughout: function, lodash match object, or `undefined` for all.
|
|
|
483
483
|
|
|
484
484
|
---
|
|
485
485
|
|
|
486
|
+
## Writing source files
|
|
487
|
+
|
|
488
|
+
`updateEntity` is a catalog operation. `writeEntitySource` writes the FILE, with
|
|
489
|
+
the checks that make a whole-file rewrite safe to perform without having watched
|
|
490
|
+
the file the whole time.
|
|
491
|
+
|
|
492
|
+
### `writeEntitySource(options)`
|
|
493
|
+
|
|
494
|
+
```js
|
|
495
|
+
import { writeEntitySource } from 'mikser-io'
|
|
496
|
+
|
|
497
|
+
const preview = await writeEntitySource({
|
|
498
|
+
id: '/documents/about.md',
|
|
499
|
+
content: next,
|
|
500
|
+
dryRun: true, // writes nothing; reports what it would re-render
|
|
501
|
+
})
|
|
502
|
+
|
|
503
|
+
const result = await writeEntitySource({
|
|
504
|
+
id: '/documents/about.md',
|
|
505
|
+
content: next,
|
|
506
|
+
ifChecksum: preview.currentChecksum,
|
|
507
|
+
})
|
|
508
|
+
```
|
|
509
|
+
|
|
510
|
+
| Option | Meaning |
|
|
511
|
+
| --- | --- |
|
|
512
|
+
| `id` | Catalog id of an existing entity. Alternative to the pair below. |
|
|
513
|
+
| `collection` + `relativePath` | Where to write. Required unless `id` is given. |
|
|
514
|
+
| `content` | The COMPLETE file. Anything omitted is deleted — there is no patch mode. |
|
|
515
|
+
| `ifChecksum` | Only write if the file's current DISK checksum equals this. |
|
|
516
|
+
| `dryRun` | Write nothing; return the blast radius and any advisory. |
|
|
517
|
+
| `awaitCycle` | Resolve once the cycle that picks the write up finishes, with its build report attached as `report`. |
|
|
518
|
+
|
|
519
|
+
**Never throws for an expected outcome.** A bad id, a path that escapes the
|
|
520
|
+
collection, a checksum that no longer matches — each returns
|
|
521
|
+
`{ ok: false, refused }` with the facts needed to retry, because those are
|
|
522
|
+
answers rather than faults. `refused` is one of `unresolvable-id`,
|
|
523
|
+
`collection-mismatch`, `incomplete-target`, `invalid-target`,
|
|
524
|
+
`checksum-mismatch`.
|
|
525
|
+
|
|
526
|
+
**Containment.** `relativePath` cannot leave the collection folder. This
|
|
527
|
+
matters because the path often comes from a request body or a CMS form, and
|
|
528
|
+
`path.join(folder, '../../x')` resolves outside the folder and writes there. It
|
|
529
|
+
is resolved and then contained rather than rejected on a literal `..`, so
|
|
530
|
+
`blog/../about.md` still works. The refusal happens before anything stats the
|
|
531
|
+
file — reporting a checksum for an out-of-tree path is a disclosure on its own.
|
|
532
|
+
|
|
533
|
+
**The precondition is not a lock.** A writer landing between the check and the
|
|
534
|
+
write still wins. It closes the window that matters in practice: read, think,
|
|
535
|
+
write back a whole file built from a copy that is now stale.
|
|
536
|
+
|
|
537
|
+
`ifChecksum` is compared against the DISK. `readEntity`'s `checksum` is the
|
|
538
|
+
catalog's, which lags between builds — pass `diskChecksum`, or the
|
|
539
|
+
`currentChecksum` a refusal hands back.
|
|
540
|
+
|
|
541
|
+
### Advisories
|
|
542
|
+
|
|
543
|
+
`contentAdvisories(entity, content)` names files a caller must not edit blind,
|
|
544
|
+
from `meta.specLocked` / `meta.generated` or from a header in the first 40
|
|
545
|
+
lines. Two kinds, kept apart because the instruction differs: `spec-locked`
|
|
546
|
+
means the bytes answer to a document outside the repo, `generated` means
|
|
547
|
+
editing the file is pointless because the next build overwrites it.
|
|
548
|
+
`advisoryWarning(advisories)` renders one line of prose for a response meant to
|
|
549
|
+
be read rather than parsed. Both are reported by `writeEntitySource` — on the
|
|
550
|
+
dry run and again on the way out, since a caller that never read the file is
|
|
551
|
+
exactly the one that needs telling.
|
|
552
|
+
|
|
553
|
+
`siblingDestinations(folder, relativePath)` reports files differing only by
|
|
554
|
+
extension, which may render to the same destination.
|
|
555
|
+
|
|
556
|
+
## Search
|
|
557
|
+
|
|
558
|
+
`queryEntities` sifts **meta**. `searchEntities` answers the other question —
|
|
559
|
+
*where does this text appear?* — across structured values, source files and
|
|
560
|
+
built output.
|
|
561
|
+
|
|
562
|
+
### `searchEntities(options)`
|
|
563
|
+
|
|
564
|
+
```js
|
|
565
|
+
import { searchEntities } from 'mikser-io'
|
|
566
|
+
|
|
567
|
+
const { hits, count, truncated } = await searchEntities({
|
|
568
|
+
query: 'NOVAPRESS',
|
|
569
|
+
in: ['meta', 'content'], // default
|
|
570
|
+
})
|
|
571
|
+
```
|
|
572
|
+
|
|
573
|
+
| Option | Meaning |
|
|
574
|
+
| --- | --- |
|
|
575
|
+
| `query` | Text to find. A plain substring unless `regex` is true. Required. |
|
|
576
|
+
| `in` | `'meta'`, `'content'`, `'output'`. Default `['meta', 'content']`. |
|
|
577
|
+
| `collection` | Restrict to one collection. |
|
|
578
|
+
| `filter` | Sift filter narrowing **what may be searched at all** — see below. |
|
|
579
|
+
| `regex` | Treat `query` as a JavaScript regular expression. |
|
|
580
|
+
| `ignoreCase` | Case-insensitive matching. Default false. |
|
|
581
|
+
| `limit` | Maximum hits (default 50). `truncated` says when it stopped early. |
|
|
582
|
+
|
|
583
|
+
The three scopes answer different questions and none implies another. `meta`
|
|
584
|
+
walks structured values as dotted paths and touches no files. `content` reads
|
|
585
|
+
the **source**. `output` walks the **built** folder and reports `occurrences`
|
|
586
|
+
per destination — the blast-radius question, and the one where a string can be
|
|
587
|
+
present because a layout writes it, with no source entity containing it
|
|
588
|
+
anywhere.
|
|
589
|
+
|
|
590
|
+
Whether a file is text is decided by reading its bytes, not by its extension,
|
|
591
|
+
so a `.njk` source or a `.webmanifest` output is searched without first being
|
|
592
|
+
added to a list of known formats.
|
|
593
|
+
|
|
594
|
+
`truncated` is load-bearing: it separates "these are the hits" from "these are
|
|
595
|
+
the first N", which a caller acting on a blast radius cannot afford to guess.
|
|
596
|
+
|
|
597
|
+
> **Unscoped by default.** A bare call reads every entity and every source
|
|
598
|
+
> file — drafts, unpublished documents, layouts, sidecar JavaScript. The `api`
|
|
599
|
+
> plugin narrows `list` through each endpoint's own sift scope; nothing
|
|
600
|
+
> narrows this. Anything that puts search behind a request **must** pass that
|
|
601
|
+
> same scope as `filter`, or a public endpoint listing only published
|
|
602
|
+
> documents will happily search the unpublished ones. There is deliberately no
|
|
603
|
+
> `search` operation in the `api` plugin and no route in this module.
|
|
604
|
+
|
|
605
|
+
### Primitives
|
|
606
|
+
|
|
607
|
+
Exported so a caller building a different question out of the same parts does
|
|
608
|
+
not reimplement them: `countMatches`, `snippetAround`, `lineOfFirstMatch`,
|
|
609
|
+
`flattenMeta`, `findOccurrences`, `walkFiles`.
|
|
610
|
+
|
|
611
|
+
`findOccurrences(text, needle)` returns every occurrence with `line`, `col`,
|
|
612
|
+
the line's `text`, and `leading` — whether the match begins its line. That one
|
|
613
|
+
flag separates the file **declaring** something from the files merely using
|
|
614
|
+
it, in any text format, with no per-language grammar involved. It returns the
|
|
615
|
+
line rather than a verdict about it, so where the heuristic is wrong the
|
|
616
|
+
evidence is in the result.
|
|
617
|
+
|
|
486
618
|
## Database
|
|
487
619
|
|
|
488
620
|
```js
|
package/index.js
CHANGED
|
@@ -11,6 +11,8 @@ export * from './src/lifecycle.js'
|
|
|
11
11
|
export * from './src/database/index.js'
|
|
12
12
|
export * from './src/journal.js'
|
|
13
13
|
export * from './src/catalog.js'
|
|
14
|
+
export * from './src/search.js'
|
|
15
|
+
export * from './src/write.js'
|
|
14
16
|
export * from './src/refs.js'
|
|
15
17
|
export * from './src/manifest.js'
|
|
16
18
|
export * from './src/provenance.js'
|
package/package.json
CHANGED
package/src/builtin-tools.js
CHANGED
|
@@ -96,6 +96,45 @@ export function registerBuiltinTools() {
|
|
|
96
96
|
},
|
|
97
97
|
)
|
|
98
98
|
|
|
99
|
+
registerTool(
|
|
100
|
+
'search',
|
|
101
|
+
{
|
|
102
|
+
description:
|
|
103
|
+
'Find a string across the catalog in ONE call — "where does this appear?". Searches entity meta '
|
|
104
|
+
+ 'values and, when asked, the source files themselves and the built output, returning '
|
|
105
|
+
+ '{ id, collection, path, field, snippet } per hit.\n\n'
|
|
106
|
+
+ 'This is how you locate content you can only describe by what it says: a menu label, a phone '
|
|
107
|
+
+ 'number, a sentence you were asked to change. Paging the catalog to find it means reading '
|
|
108
|
+
+ 'everything, most of which is fonts and image derivatives, and finding a SECOND copy of the same '
|
|
109
|
+
+ 'label somewhere else is then a matter of luck.\n\n'
|
|
110
|
+
+ 'in: ["meta"] searches structured values, no file I/O. in: ["content"] reads source files, '
|
|
111
|
+
+ 'binaries skipped. Default is both. in: ["output"] searches the BUILT files instead and reports '
|
|
112
|
+
+ '`occurrences` per destination — the blast-radius question, which is what you want before editing '
|
|
113
|
+
+ 'anything shared. The scopes answer different questions and none implies another: a string can be '
|
|
114
|
+
+ 'in the output because a layout writes it, with no source entity containing it anywhere.',
|
|
115
|
+
inputSchema: {
|
|
116
|
+
query: { type: 'string', required: true,
|
|
117
|
+
description: 'Text to find. A plain substring unless `regex` is true. Case-sensitive by default.' },
|
|
118
|
+
collection: { type: 'string',
|
|
119
|
+
description: 'Restrict to one collection (e.g. "documents"). Omit to search all.' },
|
|
120
|
+
in: { type: 'array',
|
|
121
|
+
description: 'Where to look: "meta", "content", "output". Default is meta + content.' },
|
|
122
|
+
regex: { type: 'boolean',
|
|
123
|
+
description: 'Treat `query` as a JavaScript regular expression rather than a literal substring.' },
|
|
124
|
+
ignoreCase: { type: 'boolean', description: 'Case-insensitive matching. Default false.' },
|
|
125
|
+
limit: { type: 'number', description: 'Maximum hits to return (default 50).' },
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
async (args) => {
|
|
129
|
+
try {
|
|
130
|
+
const { searchEntities } = await import('./search.js')
|
|
131
|
+
return ok(await searchEntities(args ?? {}))
|
|
132
|
+
} catch (err) {
|
|
133
|
+
return fail(err.message)
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
)
|
|
137
|
+
|
|
99
138
|
registerTool(
|
|
100
139
|
'sources',
|
|
101
140
|
{
|
package/src/search.js
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
// Find a string across the catalog and the built output.
|
|
2
|
+
//
|
|
3
|
+
// `queryEntities` sifts META. This answers the other question — "where does
|
|
4
|
+
// this text appear?" — over structured values, source files and built output,
|
|
5
|
+
// which is what locating a phone number in a document body or a class name in
|
|
6
|
+
// the shipped CSS requires.
|
|
7
|
+
//
|
|
8
|
+
// Three scopes, answering three different questions, none implying another:
|
|
9
|
+
//
|
|
10
|
+
// meta — structured values, walked as dotted paths. No file I/O.
|
|
11
|
+
// content — the SOURCE files. What an editor is going to change.
|
|
12
|
+
// output — the BUILT files. The blast radius: a string can be in the
|
|
13
|
+
// output because a layout writes it, with no source entity
|
|
14
|
+
// containing it anywhere.
|
|
15
|
+
//
|
|
16
|
+
// UNSCOPED BY DEFAULT, and that is a security property, not a detail. A bare
|
|
17
|
+
// call reads every entity and every source file — drafts, unpublished
|
|
18
|
+
// documents, layouts, sidecar JavaScript. The `api` plugin's public endpoints
|
|
19
|
+
// narrow `list` through the endpoint's own sift scope; nothing narrows this.
|
|
20
|
+
// So anything that puts search behind a request MUST pass that same scope as
|
|
21
|
+
// `filter`, or a public endpoint that lists only published documents will
|
|
22
|
+
// happily search the unpublished ones. There is deliberately no `search`
|
|
23
|
+
// operation in the api plugin, and no route here.
|
|
24
|
+
//
|
|
25
|
+
// The primitives are exported alongside `searchEntities` so a caller building
|
|
26
|
+
// a different question out of the same parts does not reimplement counting,
|
|
27
|
+
// snippets or line numbers to do it.
|
|
28
|
+
|
|
29
|
+
import path from 'node:path'
|
|
30
|
+
import { readdir, readFile } from 'node:fs/promises'
|
|
31
|
+
|
|
32
|
+
import runtime from './runtime.js'
|
|
33
|
+
import { findEntities } from './catalog.js'
|
|
34
|
+
import { readEntityContent, looksTextual } from './utils.js'
|
|
35
|
+
|
|
36
|
+
// How much of an output file to read before deciding it is text. Same rule
|
|
37
|
+
// the source read uses, so the two scopes cannot disagree about a file that
|
|
38
|
+
// appears in both.
|
|
39
|
+
const SNIFF_BYTES = 8 * 1024
|
|
40
|
+
|
|
41
|
+
// Every leaf value under meta as [dottedPath, value], arrays included.
|
|
42
|
+
// The same shape refs_inbound reports, so a hit here and a referrer there
|
|
43
|
+
// name the same field.
|
|
44
|
+
export function* flattenMeta(node, prefix = '') {
|
|
45
|
+
if (node === null || node === undefined) return
|
|
46
|
+
if (Array.isArray(node)) {
|
|
47
|
+
for (let i = 0; i < node.length; i++) yield* flattenMeta(node[i], `${prefix}[${i}]`)
|
|
48
|
+
return
|
|
49
|
+
}
|
|
50
|
+
if (typeof node === 'object') {
|
|
51
|
+
for (const [key, value] of Object.entries(node)) {
|
|
52
|
+
yield* flattenMeta(value, prefix ? `${prefix}.${key}` : key)
|
|
53
|
+
}
|
|
54
|
+
return
|
|
55
|
+
}
|
|
56
|
+
yield [prefix, node]
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// How many times the needle appears. A count, not a boolean, because "this
|
|
60
|
+
// class is on nine pages" and "this class is on nine pages and seven times on
|
|
61
|
+
// one of them" are different facts, and the second one names the component.
|
|
62
|
+
export function countMatches(text, query, regex, ignoreCase) {
|
|
63
|
+
if (regex) {
|
|
64
|
+
const re = new RegExp(query, ignoreCase ? 'gi' : 'g')
|
|
65
|
+
let n = 0
|
|
66
|
+
// Guard the zero-width case: /a*/g on a non-matching position returns
|
|
67
|
+
// an empty match forever and never advances.
|
|
68
|
+
for (let m = re.exec(text); m; m = re.exec(text)) {
|
|
69
|
+
n++
|
|
70
|
+
if (m.index === re.lastIndex) re.lastIndex++
|
|
71
|
+
}
|
|
72
|
+
return n
|
|
73
|
+
}
|
|
74
|
+
const haystack = ignoreCase ? text.toLowerCase() : text
|
|
75
|
+
const needle = ignoreCase ? query.toLowerCase() : query
|
|
76
|
+
if (!needle) return 0
|
|
77
|
+
let n = 0
|
|
78
|
+
for (let at = haystack.indexOf(needle); at !== -1; at = haystack.indexOf(needle, at + needle.length)) n++
|
|
79
|
+
return n
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Offset of the first match, or -1. The one place the regex/substring
|
|
83
|
+
// difference is resolved, so the line number and the snippet cannot end up
|
|
84
|
+
// pointing at different matches.
|
|
85
|
+
function firstMatchAt(text, query, regex, ignoreCase) {
|
|
86
|
+
if (regex) {
|
|
87
|
+
const m = new RegExp(query, ignoreCase ? 'i' : '').exec(text)
|
|
88
|
+
return m ? m.index : -1
|
|
89
|
+
}
|
|
90
|
+
const haystack = ignoreCase ? text.toLowerCase() : text
|
|
91
|
+
return haystack.indexOf(ignoreCase ? query.toLowerCase() : query)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// 1-based line of the first match, so a hit is somewhere to go rather than
|
|
95
|
+
// something to grep for again.
|
|
96
|
+
export function lineOfFirstMatch(text, query, regex, ignoreCase) {
|
|
97
|
+
const at = firstMatchAt(text, query, regex, ignoreCase)
|
|
98
|
+
if (at < 0) return null
|
|
99
|
+
let line = 1
|
|
100
|
+
for (let i = 0; i < at; i++) if (text.charCodeAt(i) === 10) line++
|
|
101
|
+
return line
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Enough text around the match to recognise it without returning the file.
|
|
105
|
+
export function snippetAround(text, query, regex, ignoreCase) {
|
|
106
|
+
const at = firstMatchAt(text, query, regex, ignoreCase)
|
|
107
|
+
if (at < 0) return text.slice(0, 120)
|
|
108
|
+
const start = Math.max(0, at - 60)
|
|
109
|
+
const end = Math.min(text.length, at + query.length + 60)
|
|
110
|
+
return (start > 0 ? '…' : '') + text.slice(start, end).replace(/\s+/g, ' ') + (end < text.length ? '…' : '')
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Every occurrence of a needle in a text, with the one signal that separates
|
|
114
|
+
// "this file DECLARES it" from "this file uses it" without knowing the
|
|
115
|
+
// language: a declaration begins its line in nearly every text format, while a
|
|
116
|
+
// use sits mid-line.
|
|
117
|
+
//
|
|
118
|
+
// The LINE is returned rather than a computed verdict about it. A caller can
|
|
119
|
+
// tell a declaration from something that merely starts with the same token by
|
|
120
|
+
// reading it, which needs no grammar here at all — and where the heuristic is
|
|
121
|
+
// wrong, the evidence for that is right there in the result.
|
|
122
|
+
export function findOccurrences(text, needle, { limit = 200 } = {}) {
|
|
123
|
+
const sites = []
|
|
124
|
+
if (!needle || typeof text !== 'string') return sites
|
|
125
|
+
let line = 1
|
|
126
|
+
let lineStart = 0
|
|
127
|
+
let scanned = 0
|
|
128
|
+
for (let at = text.indexOf(needle); at !== -1; at = text.indexOf(needle, at + needle.length)) {
|
|
129
|
+
while (scanned < at) {
|
|
130
|
+
if (text.charCodeAt(scanned) === 10) { line++; lineStart = scanned + 1 }
|
|
131
|
+
scanned++
|
|
132
|
+
}
|
|
133
|
+
const lineEnd = text.indexOf('\n', at)
|
|
134
|
+
sites.push({
|
|
135
|
+
line,
|
|
136
|
+
col: at - lineStart,
|
|
137
|
+
leading: text.slice(lineStart, at).trim() === '',
|
|
138
|
+
text: text.slice(lineStart, lineEnd === -1 ? text.length : lineEnd).trim().slice(0, 160),
|
|
139
|
+
})
|
|
140
|
+
if (sites.length >= limit) break
|
|
141
|
+
}
|
|
142
|
+
return sites
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Every file under a folder, depth-first. A generator so a search that hits
|
|
146
|
+
// its limit stops walking rather than materializing the tree.
|
|
147
|
+
export async function* walkFiles(folder) {
|
|
148
|
+
let entries
|
|
149
|
+
try {
|
|
150
|
+
entries = await readdir(folder, { withFileTypes: true })
|
|
151
|
+
} catch {
|
|
152
|
+
return
|
|
153
|
+
}
|
|
154
|
+
for (const entry of entries) {
|
|
155
|
+
const full = path.join(folder, entry.name)
|
|
156
|
+
if (entry.isDirectory()) yield* walkFiles(full)
|
|
157
|
+
else if (entry.isFile()) yield full
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Read a built file as text, or null when it is not text.
|
|
162
|
+
//
|
|
163
|
+
// Decided by the bytes, so a `.webmanifest`, a `.map` or whatever a renderer
|
|
164
|
+
// plugin emits is searchable without first being added to a list of known
|
|
165
|
+
// extensions — and a font still comes back null rather than as convincing
|
|
166
|
+
// garbage.
|
|
167
|
+
async function readOutputText(file) {
|
|
168
|
+
let buf
|
|
169
|
+
try {
|
|
170
|
+
buf = await readFile(file)
|
|
171
|
+
} catch {
|
|
172
|
+
return null
|
|
173
|
+
}
|
|
174
|
+
if (!looksTextual(buf.subarray(0, Math.min(buf.length, SNIFF_BYTES)))) return null
|
|
175
|
+
return buf.toString('utf8')
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Build the test used for meta values and for a whole-file contains check.
|
|
179
|
+
// Throws on an invalid regex so the caller can report it as bad input rather
|
|
180
|
+
// than as an empty result.
|
|
181
|
+
function buildMatcher(query, regex, ignoreCase) {
|
|
182
|
+
if (regex) {
|
|
183
|
+
const re = new RegExp(query, ignoreCase ? 'i' : '')
|
|
184
|
+
return { test: (value) => re.test(String(value)) }
|
|
185
|
+
}
|
|
186
|
+
const needle = ignoreCase ? query.toLowerCase() : query
|
|
187
|
+
return {
|
|
188
|
+
test: (value) => (ignoreCase ? String(value).toLowerCase() : String(value)).includes(needle),
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Find `query` across the catalog and, when asked, the built output.
|
|
193
|
+
//
|
|
194
|
+
// Returns { query, scopes, count, truncated, searched, hits }. Each hit
|
|
195
|
+
// carries where it was found and enough to go there:
|
|
196
|
+
//
|
|
197
|
+
// meta { id, collection, path, where, field, snippet }
|
|
198
|
+
// content { id, collection, path, where, field: null, occurrences, line, snippet }
|
|
199
|
+
// output { destination, where, occurrences, snippet }
|
|
200
|
+
//
|
|
201
|
+
// `truncated` says the limit stopped the walk, which is the difference
|
|
202
|
+
// between "these are the hits" and "these are the first N" — a distinction a
|
|
203
|
+
// caller acting on a blast radius cannot afford to guess at.
|
|
204
|
+
export async function searchEntities({
|
|
205
|
+
query,
|
|
206
|
+
collection,
|
|
207
|
+
filter,
|
|
208
|
+
in: where,
|
|
209
|
+
regex = false,
|
|
210
|
+
ignoreCase = false,
|
|
211
|
+
limit = 50,
|
|
212
|
+
} = {}) {
|
|
213
|
+
if (!query) throw new Error('searchEntities: query is required')
|
|
214
|
+
const scopes = where?.length ? where : ['meta', 'content']
|
|
215
|
+
const matcher = buildMatcher(query, regex, ignoreCase)
|
|
216
|
+
|
|
217
|
+
const hits = []
|
|
218
|
+
let truncated = false
|
|
219
|
+
|
|
220
|
+
// The output scope walks the deployed folder rather than the catalog, so
|
|
221
|
+
// it runs on its own. Counting is the point here rather than first-match:
|
|
222
|
+
// seven occurrences on one page and one on nine others is the shape of a
|
|
223
|
+
// shared component, and a list of nine equal-looking filenames hides it.
|
|
224
|
+
if (scopes.includes('output')) {
|
|
225
|
+
const outputFolder = runtime.options?.outputFolder
|
|
226
|
+
if (!outputFolder) throw new Error('No output folder configured, so there is no built output to search.')
|
|
227
|
+
for await (const file of walkFiles(outputFolder)) {
|
|
228
|
+
if (hits.length >= limit) { truncated = true; break }
|
|
229
|
+
const text = await readOutputText(file)
|
|
230
|
+
if (text === null) continue
|
|
231
|
+
const occurrences = countMatches(text, query, regex, ignoreCase)
|
|
232
|
+
if (!occurrences) continue
|
|
233
|
+
hits.push({
|
|
234
|
+
destination: '/' + path.relative(outputFolder, file).split(path.sep).join('/'),
|
|
235
|
+
where: 'output',
|
|
236
|
+
occurrences,
|
|
237
|
+
snippet: snippetAround(text, query, regex, ignoreCase),
|
|
238
|
+
})
|
|
239
|
+
}
|
|
240
|
+
hits.sort((a, b) => b.occurrences - a.occurrences || a.destination.localeCompare(b.destination))
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// `filter` is how a caller narrows what may be searched at all. It merges
|
|
244
|
+
// with `collection` and pushes into the catalog query, so a scoped caller
|
|
245
|
+
// never materializes a row it is not allowed to see rather than filtering
|
|
246
|
+
// one out after reading it off disk.
|
|
247
|
+
const searchesCatalog = scopes.some(scope => scope !== 'output')
|
|
248
|
+
const catalogQuery = { ...(filter ?? {}), ...(collection ? { collection } : {}) }
|
|
249
|
+
const entities = searchesCatalog
|
|
250
|
+
? await findEntities(Object.keys(catalogQuery).length ? catalogQuery : undefined)
|
|
251
|
+
: []
|
|
252
|
+
|
|
253
|
+
for (const entity of entities) {
|
|
254
|
+
if (hits.length >= limit) { truncated = true; break }
|
|
255
|
+
if (scopes.includes('meta') && entity.meta) {
|
|
256
|
+
for (const [field, value] of flattenMeta(entity.meta)) {
|
|
257
|
+
if (!matcher.test(value)) continue
|
|
258
|
+
hits.push({
|
|
259
|
+
id: entity.id, collection: entity.collection ?? null, path: entity.uri ?? null,
|
|
260
|
+
where: 'meta', field, snippet: snippetAround(String(value), query, regex, ignoreCase),
|
|
261
|
+
})
|
|
262
|
+
break
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
if (hits.length >= limit) { truncated = true; break }
|
|
266
|
+
if (scopes.includes('content')) {
|
|
267
|
+
// readEntityContent owns the text/binary decision and makes it by
|
|
268
|
+
// reading the bytes, so a .njk or .toml is searched like anything
|
|
269
|
+
// else. Deciding it a second time here on an extension would make
|
|
270
|
+
// a skipped format indistinguishable from a real absence.
|
|
271
|
+
const { content } = await readEntityContent(entity)
|
|
272
|
+
if (typeof content !== 'string' || !matcher.test(content)) continue
|
|
273
|
+
hits.push({
|
|
274
|
+
id: entity.id, collection: entity.collection ?? null, path: entity.uri ?? null,
|
|
275
|
+
where: 'content', field: null,
|
|
276
|
+
occurrences: countMatches(content, query, regex, ignoreCase),
|
|
277
|
+
line: lineOfFirstMatch(content, query, regex, ignoreCase),
|
|
278
|
+
snippet: snippetAround(content, query, regex, ignoreCase),
|
|
279
|
+
})
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return { query, scopes, count: hits.length, truncated, searched: entities.length, hits }
|
|
284
|
+
}
|
package/src/utils.js
CHANGED
|
@@ -1077,20 +1077,38 @@ export function useCollection(runtime, name) {
|
|
|
1077
1077
|
return folder
|
|
1078
1078
|
}
|
|
1079
1079
|
|
|
1080
|
+
// A path that cannot leave the collection folder.
|
|
1081
|
+
//
|
|
1082
|
+
// `path.join(folder, '../../x')` resolves outside the folder and writes
|
|
1083
|
+
// there, which turns a collection handle into an arbitrary-write
|
|
1084
|
+
// primitive the moment a relative path comes from a request body or a CMS
|
|
1085
|
+
// form. Resolved and then contained rather than rejected on a literal
|
|
1086
|
+
// `..`, so `a/../b.md` — which lands inside — still works.
|
|
1087
|
+
function resolveWithin(relativePath) {
|
|
1088
|
+
const folder = resolveFolder()
|
|
1089
|
+
const uri = path.resolve(folder, relativePath ?? '')
|
|
1090
|
+
const root = path.resolve(folder)
|
|
1091
|
+
if (uri !== root && !uri.startsWith(root + path.sep)) {
|
|
1092
|
+
throw new Error(
|
|
1093
|
+
`Path escapes the ${name} collection: ${JSON.stringify(relativePath)} resolves outside ${root}`)
|
|
1094
|
+
}
|
|
1095
|
+
return uri
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1080
1098
|
return {
|
|
1081
1099
|
name,
|
|
1082
1100
|
get folder() { return resolveFolder() },
|
|
1101
|
+
resolveWithin,
|
|
1083
1102
|
|
|
1084
1103
|
async write(relativePath, content = '') {
|
|
1085
|
-
const uri =
|
|
1104
|
+
const uri = resolveWithin(relativePath)
|
|
1086
1105
|
await mkdir(path.dirname(uri), { recursive: true })
|
|
1087
1106
|
await writeFile(uri, content, 'utf8')
|
|
1088
1107
|
return uri
|
|
1089
1108
|
},
|
|
1090
1109
|
|
|
1091
1110
|
async remove(relativePath) {
|
|
1092
|
-
|
|
1093
|
-
await unlink(uri)
|
|
1111
|
+
await unlink(resolveWithin(relativePath))
|
|
1094
1112
|
},
|
|
1095
1113
|
}
|
|
1096
1114
|
}
|
package/src/write.js
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// Write a source file back, with the checks that make a whole-file rewrite
|
|
2
|
+
// safe to perform without having watched the file the whole time.
|
|
3
|
+
//
|
|
4
|
+
// The only write mode is whole-file: there is no patch. That makes the write
|
|
5
|
+
// itself the easy part and everything around it the point —
|
|
6
|
+
//
|
|
7
|
+
// - a checksum precondition, so a rewrite built from a stale copy is
|
|
8
|
+
// refused instead of silently discarding whoever edited in between
|
|
9
|
+
// - a dry run reporting which destinations the write would re-render
|
|
10
|
+
// - advisories naming a file that is GENERATED or answers to an external
|
|
11
|
+
// spec, which a caller must not edit blind
|
|
12
|
+
// - siblings that could render to the same destination
|
|
13
|
+
// - containment, so a relative path from a form or a request body cannot
|
|
14
|
+
// write outside the collection folder
|
|
15
|
+
//
|
|
16
|
+
// An editing agent gets these through mikser-io-mcp. An application driving
|
|
17
|
+
// mikser as a CMS writes its own files and gets none of them, which is the
|
|
18
|
+
// gap this closes: the safety belongs to the write, not to one transport.
|
|
19
|
+
|
|
20
|
+
import path from 'node:path'
|
|
21
|
+
import { readdir } from 'node:fs/promises'
|
|
22
|
+
|
|
23
|
+
import runtime from './runtime.js'
|
|
24
|
+
import { readEntity, findEntities } from './catalog.js'
|
|
25
|
+
import { useCollection, checksum, readEntityContent } from './utils.js'
|
|
26
|
+
import { nextCycleId, whenCycleCompletes } from './report.js'
|
|
27
|
+
|
|
28
|
+
// How far into a file to look for a marker. A header nobody reads is not a
|
|
29
|
+
// header; one buried 200 lines down is not either.
|
|
30
|
+
const HEADER_SCAN_LINES = 40
|
|
31
|
+
|
|
32
|
+
// Files a caller must not edit blind, read from the bytes themselves.
|
|
33
|
+
//
|
|
34
|
+
// Two kinds, kept apart because the instruction differs. `spec-locked` means
|
|
35
|
+
// the bytes answer to a document outside the repo — change it and the site
|
|
36
|
+
// stops matching something a human signed off. `generated` means editing the
|
|
37
|
+
// file is pointless, because the next build overwrites it.
|
|
38
|
+
const HEADER_PATTERNS = [
|
|
39
|
+
{ kind: 'spec-locked', re: /^\W*spec source:\s*(.+?)\s*$/i },
|
|
40
|
+
{ kind: 'generated', re: /^\W*(?:generated by|do not edit)\b:?\s*(.*?)\s*$/i },
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
export function contentAdvisories(entity, content) {
|
|
44
|
+
const found = []
|
|
45
|
+
const push = (kind, detail, via, line) => {
|
|
46
|
+
if (found.some(a => a.kind === kind)) return
|
|
47
|
+
found.push({ kind, detail: detail || null, via, ...(line ? { line } : {}) })
|
|
48
|
+
}
|
|
49
|
+
// Explicit meta wins: someone wrote it down as data, on purpose.
|
|
50
|
+
if (entity?.meta?.specLocked) {
|
|
51
|
+
push('spec-locked', typeof entity.meta.specLocked === 'string' ? entity.meta.specLocked : null, 'meta.specLocked')
|
|
52
|
+
}
|
|
53
|
+
if (entity?.meta?.generated) {
|
|
54
|
+
push('generated', typeof entity.meta.generated === 'string' ? entity.meta.generated : null, 'meta.generated')
|
|
55
|
+
}
|
|
56
|
+
if (typeof content === 'string') {
|
|
57
|
+
const lines = content.split('\n', HEADER_SCAN_LINES)
|
|
58
|
+
for (let i = 0; i < lines.length; i++) {
|
|
59
|
+
for (const { kind, re } of HEADER_PATTERNS) {
|
|
60
|
+
const m = re.exec(lines[i])
|
|
61
|
+
if (m) push(kind, m[1], 'header', i + 1)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return found
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// One line of prose for a response that has to be read, not parsed.
|
|
69
|
+
export function advisoryWarning(advisories) {
|
|
70
|
+
if (!advisories?.length) return null
|
|
71
|
+
return advisories.map(a => a.kind === 'spec-locked'
|
|
72
|
+
? `SPEC-LOCKED: ${a.detail ?? 'this file answers to an external specification'}`
|
|
73
|
+
+ ' — changing it may break a signed-off design. Confirm against the spec before writing.'
|
|
74
|
+
: `GENERATED: ${a.detail ?? 'this file is produced by the build'}`
|
|
75
|
+
+ ' — edit its source instead; the next build overwrites this.').join(' ')
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Files beside this one that differ only by extension.
|
|
79
|
+
//
|
|
80
|
+
// An empty `index.md` sitting next to a real `index.yml` renders to the same
|
|
81
|
+
// destination; whichever renders last wins and the other output is discarded.
|
|
82
|
+
// The destination is not known until the cycle runs, but the COLLIDING SHAPE
|
|
83
|
+
// is visible at write time, and the write is the cheapest moment to say so.
|
|
84
|
+
export async function siblingDestinations(folder, relativePath) {
|
|
85
|
+
const dir = path.dirname(path.join(folder, relativePath))
|
|
86
|
+
const base = path.basename(relativePath, path.extname(relativePath))
|
|
87
|
+
try {
|
|
88
|
+
const entries = await readdir(dir, { withFileTypes: true })
|
|
89
|
+
return entries
|
|
90
|
+
.filter(e => e.isFile()
|
|
91
|
+
&& path.basename(e.name, path.extname(e.name)) === base
|
|
92
|
+
&& e.name !== path.basename(relativePath))
|
|
93
|
+
.map(e => ({
|
|
94
|
+
path: path.join(path.dirname(relativePath), e.name),
|
|
95
|
+
note: 'same name, different extension — may render to the same destination',
|
|
96
|
+
}))
|
|
97
|
+
} catch {
|
|
98
|
+
return []
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// The checksum of a file, or null when there is nothing there. Not an error
|
|
103
|
+
// path: "does not exist yet" is the normal case for a create.
|
|
104
|
+
async function fileChecksum(uri) {
|
|
105
|
+
try {
|
|
106
|
+
return await checksum(uri)
|
|
107
|
+
} catch {
|
|
108
|
+
return null
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// The catalog entity written from this file, when there is one.
|
|
113
|
+
async function findEntityAtUri(uri) {
|
|
114
|
+
if (!uri) return null
|
|
115
|
+
const matches = await findEntities({ uri })
|
|
116
|
+
return matches?.[0] ?? null
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Resolve a catalog id to the (collection, relativePath) pair a write needs.
|
|
120
|
+
//
|
|
121
|
+
// Taken from the entity rather than by splitting the id: the id prefix is
|
|
122
|
+
// `idPrefix ?? '/' + collection` and the extension may have been stripped, so
|
|
123
|
+
// splitting on the first segment is a guess that is usually right and
|
|
124
|
+
// silently wrong for any source configured either way.
|
|
125
|
+
export async function locateEntityFile(id) {
|
|
126
|
+
const entity = await readEntity({ id })
|
|
127
|
+
if (!entity) return { error: `No entity with id ${id}.` }
|
|
128
|
+
if (!entity.collection) {
|
|
129
|
+
return { error: `Entity ${id} has no collection, so its file location cannot be derived.` }
|
|
130
|
+
}
|
|
131
|
+
let folder
|
|
132
|
+
try {
|
|
133
|
+
folder = useCollection(runtime, entity.collection).folder
|
|
134
|
+
} catch (err) {
|
|
135
|
+
return { error: `Entity ${id} is in collection ${entity.collection}, which has no folder: ${err.message}` }
|
|
136
|
+
}
|
|
137
|
+
if (!entity.uri) {
|
|
138
|
+
return { error: `Entity ${id} has no uri — it is synthetic (emitted by a plugin, not read from a file) `
|
|
139
|
+
+ 'and has no file to rewrite.' }
|
|
140
|
+
}
|
|
141
|
+
const relativePath = path.relative(folder, entity.uri)
|
|
142
|
+
if (!relativePath || relativePath.startsWith('..') || path.isAbsolute(relativePath)) {
|
|
143
|
+
return { error: `Entity ${id} lives at ${entity.uri}, outside its collection folder ${folder}.` }
|
|
144
|
+
}
|
|
145
|
+
return { collection: entity.collection, relativePath }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Create or overwrite a source file inside a collection.
|
|
149
|
+
//
|
|
150
|
+
// Takes either `id` (an existing entity) or `collection` + `relativePath`.
|
|
151
|
+
// `content` is the COMPLETE file: anything omitted is deleted.
|
|
152
|
+
//
|
|
153
|
+
// Never throws for an expected outcome. A refusal — a bad id, a path that
|
|
154
|
+
// escapes the collection, a checksum that no longer matches — comes back as
|
|
155
|
+
// `{ ok: false, refused }` with the facts needed to retry, because those are
|
|
156
|
+
// answers rather than faults.
|
|
157
|
+
//
|
|
158
|
+
// dryRun write nothing; report what the write would touch
|
|
159
|
+
// ifChecksum only write if the file's current DISK checksum equals this
|
|
160
|
+
// awaitCycle resolve once the cycle that picks the write up has finished,
|
|
161
|
+
// with its build report attached
|
|
162
|
+
export async function writeEntitySource({
|
|
163
|
+
id,
|
|
164
|
+
collection,
|
|
165
|
+
relativePath,
|
|
166
|
+
content = '',
|
|
167
|
+
ifChecksum,
|
|
168
|
+
dryRun = false,
|
|
169
|
+
awaitCycle = false,
|
|
170
|
+
} = {}) {
|
|
171
|
+
if (id) {
|
|
172
|
+
const located = await locateEntityFile(id)
|
|
173
|
+
if (located.error) return { ok: false, refused: 'unresolvable-id', error: located.error }
|
|
174
|
+
// An explicit pair still wins if a caller passes both, but disagreeing
|
|
175
|
+
// with the id is a mistake worth refusing rather than silently
|
|
176
|
+
// resolving one way.
|
|
177
|
+
if (collection && collection !== located.collection) {
|
|
178
|
+
return {
|
|
179
|
+
ok: false,
|
|
180
|
+
refused: 'collection-mismatch',
|
|
181
|
+
error: `id ${id} is in collection ${located.collection}, not ${collection}. Pass one or the other.`,
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
collection ??= located.collection
|
|
185
|
+
relativePath ??= located.relativePath
|
|
186
|
+
}
|
|
187
|
+
if (!collection || !relativePath) {
|
|
188
|
+
return {
|
|
189
|
+
ok: false,
|
|
190
|
+
refused: 'incomplete-target',
|
|
191
|
+
error: 'Pass either `id` (for an existing entity) or both `collection` and `relativePath`.',
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
let handle
|
|
196
|
+
let uri
|
|
197
|
+
try {
|
|
198
|
+
handle = useCollection(runtime, collection)
|
|
199
|
+
// Containment before anything reads or writes. A relative path that
|
|
200
|
+
// escapes the collection must not even be STATTED — reporting a
|
|
201
|
+
// checksum for /etc/passwd is a disclosure on its own.
|
|
202
|
+
uri = handle.resolveWithin(relativePath)
|
|
203
|
+
} catch (err) {
|
|
204
|
+
return { ok: false, refused: 'invalid-target', collection, relativePath, error: err.message }
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Everything a caller should know BEFORE the bytes move. Computed for the
|
|
208
|
+
// dry run and the real write alike, so the preview and the thing it
|
|
209
|
+
// previews cannot disagree.
|
|
210
|
+
const existing = id ? await readEntity({ id }) : await findEntityAtUri(uri)
|
|
211
|
+
const onDisk = await readEntityContent({ uri }, { reload: true })
|
|
212
|
+
const advisories = contentAdvisories(existing, typeof onDisk.content === 'string' ? onDisk.content : null)
|
|
213
|
+
|
|
214
|
+
if (dryRun) {
|
|
215
|
+
const wouldAffect = existing?.id ? (runtime.manifest?.affectedBy?.(existing) ?? []) : []
|
|
216
|
+
const touched = new Set(wouldAffect.map(a => a.destination))
|
|
217
|
+
return {
|
|
218
|
+
ok: true, dryRun: true, collection, relativePath,
|
|
219
|
+
id: existing?.id ?? null,
|
|
220
|
+
exists: existing != null,
|
|
221
|
+
currentChecksum: await fileChecksum(uri),
|
|
222
|
+
advisories,
|
|
223
|
+
warning: advisoryWarning(advisories),
|
|
224
|
+
wouldAffect,
|
|
225
|
+
wouldAffectCount: wouldAffect.length,
|
|
226
|
+
siblingDestinations: await siblingDestinations(handle.folder, relativePath),
|
|
227
|
+
// Collisions ALREADY standing at the outputs this write would
|
|
228
|
+
// touch. A write cannot be blamed for them, but re-rendering into
|
|
229
|
+
// one is how the wrong half of a contested destination wins.
|
|
230
|
+
collisionsAtAffected: (runtime.manifest?.collisions?.() ?? [])
|
|
231
|
+
.filter(c => touched.has(c.destination)),
|
|
232
|
+
note: existing?.id
|
|
233
|
+
? 'Destinations are computed with the engine\'s own skip rule, so they match what a real cycle '
|
|
234
|
+
+ 'would do — EXCEPT for changes to the entity\'s own frontmatter, which is parsed during '
|
|
235
|
+
+ 'import and can move its destination.'
|
|
236
|
+
: 'This file is not in the catalog yet, so nothing depends on it and there is no blast radius '
|
|
237
|
+
+ 'to report.',
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Checked immediately before the write. Not a lock — a writer that lands
|
|
242
|
+
// between the check and the write still wins — but it closes the window
|
|
243
|
+
// that matters in practice: read, think, write back a whole file built
|
|
244
|
+
// from a copy that is now stale.
|
|
245
|
+
const before = await fileChecksum(uri)
|
|
246
|
+
if (ifChecksum !== undefined && ifChecksum !== before) {
|
|
247
|
+
return {
|
|
248
|
+
ok: false,
|
|
249
|
+
refused: 'checksum-mismatch',
|
|
250
|
+
collection, relativePath,
|
|
251
|
+
expectedChecksum: ifChecksum,
|
|
252
|
+
currentChecksum: before,
|
|
253
|
+
hint: before === null
|
|
254
|
+
? 'The file does not exist. Omit ifChecksum to create it.'
|
|
255
|
+
: 'The file on disk changed since you read it. Re-read it for the CONTENT, re-apply your change, '
|
|
256
|
+
+ 'and retry with `currentChecksum` from THIS response.',
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
const cycleId = nextCycleId()
|
|
261
|
+
await handle.write(relativePath, content)
|
|
262
|
+
|
|
263
|
+
const result = {
|
|
264
|
+
ok: true, collection, relativePath,
|
|
265
|
+
checksum: await fileChecksum(uri),
|
|
266
|
+
bytes: Buffer.byteLength(content),
|
|
267
|
+
cycleId,
|
|
268
|
+
siblingDestinations: await siblingDestinations(handle.folder, relativePath),
|
|
269
|
+
}
|
|
270
|
+
// Echoed on the way out, not only on read. A caller that never read the
|
|
271
|
+
// file — or read past the header — is exactly the one that needs telling,
|
|
272
|
+
// and telling it after the write still names what to check before deploy.
|
|
273
|
+
if (advisories.length) {
|
|
274
|
+
result.advisories = advisories
|
|
275
|
+
result.warning = advisoryWarning(advisories)
|
|
276
|
+
}
|
|
277
|
+
if (awaitCycle) result.report = await whenCycleCompletes(cycleId)
|
|
278
|
+
return result
|
|
279
|
+
}
|