explorbot 0.4.8 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/boat/api-tester/src/cli.ts +1 -1
- package/boat/api-tester/src/config.ts +34 -26
- package/dist/boat/api-tester/src/cli.js +1 -1
- package/dist/boat/api-tester/src/config.js +32 -26
- package/dist/package.json +1 -1
- package/dist/src/ai/fisherman/tools.js +10 -4
- package/dist/src/ai/pilot.js +11 -6
- package/dist/src/ai/tools.js +8 -1
- package/dist/src/commands/config-command.d.ts +2 -0
- package/dist/src/commands/config-command.js +8 -2
- package/dist/src/commands/init-command.js +6 -26
- package/dist/src/commands/sites-command.js +6 -1
- package/dist/src/config.d.ts +5 -4
- package/dist/src/config.js +25 -23
- package/dist/src/global-config.d.ts +6 -0
- package/dist/src/global-config.js +82 -7
- package/dist/src/utils/html.js +6 -0
- package/dist/src/utils/merge.d.ts +1 -0
- package/dist/src/utils/merge.js +11 -0
- package/docs/reference/commands.md +10 -2
- package/docs/reference/configuration.md +37 -4
- package/docs/superpowers/plans/2026-09-15-mdq-package.md +2029 -0
- package/docs/superpowers/specs/2026-09-14-mdq-package-design.md +397 -0
- package/package.json +1 -1
- package/src/ai/fisherman/tools.ts +10 -4
- package/src/ai/pilot.ts +11 -6
- package/src/ai/tools.ts +7 -1
- package/src/commands/config-command.ts +9 -2
- package/src/commands/init-command.ts +6 -27
- package/src/commands/sites-command.ts +6 -1
- package/src/config.ts +28 -25
- package/src/global-config.ts +81 -6
- package/src/utils/html.ts +6 -0
- package/src/utils/merge.ts +13 -0
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
# mdq — Markdown Query & Edit Package
|
|
2
|
+
|
|
3
|
+
Date: 2026-09-14
|
|
4
|
+
Status: Approved design, pending implementation
|
|
5
|
+
|
|
6
|
+
## Goal
|
|
7
|
+
|
|
8
|
+
Extract `src/utils/markdown-query.ts` into `src/utils/mdq/`, designed as a publishable
|
|
9
|
+
standalone package: query markdown *and* update it, with a jq-like CLI planned as a
|
|
10
|
+
second phase.
|
|
11
|
+
|
|
12
|
+
The name `mdq` is unclaimed on npm (verified 404). Publishing is deferred; this change
|
|
13
|
+
makes the package publish-ready but adds no `package.json` or build script.
|
|
14
|
+
|
|
15
|
+
## Constraints
|
|
16
|
+
|
|
17
|
+
- **Zero explorbot imports.** Two dependencies only: `marked` for markdown, `yaml` for
|
|
18
|
+
frontmatter. Both are already repo deps (`marked` ^16.2.0, `yaml` ^2.8.3).
|
|
19
|
+
- **Two files**, per the module split below.
|
|
20
|
+
- 54 in-repo call sites must keep working; a re-export shim carries them.
|
|
21
|
+
|
|
22
|
+
## Architecture
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
src/utils/mdq/
|
|
26
|
+
query.ts selector grammar - token index - MarkdownDoc - Selection (reads)
|
|
27
|
+
edit.ts pure edits over (source, ranges): splicing - whitespace - renderers
|
|
28
|
+
README.md public documentation
|
|
29
|
+
src/utils/markdown-query.ts re-export shim
|
|
30
|
+
tests/unit/mdq/*.test.ts
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
`edit.ts` exports pure functions taking source text plus ranges or tokens, and returning
|
|
34
|
+
new source text. It imports types from `query.ts` type-only and never references a class
|
|
35
|
+
value. `query.ts` owns both classes and delegates each write verb to exactly one `edit.ts`
|
|
36
|
+
call. This keeps the split acyclic by construction.
|
|
37
|
+
|
|
38
|
+
Two types:
|
|
39
|
+
|
|
40
|
+
- **`MarkdownDoc`** — a whole document. Returned by `mdq()` and by every write.
|
|
41
|
+
- **`Selection`** — a set of matched ranges. Returned by `query()` and the sugar methods.
|
|
42
|
+
|
|
43
|
+
One rule, stated in the README: **reads narrow, writes return the document.**
|
|
44
|
+
|
|
45
|
+
## API
|
|
46
|
+
|
|
47
|
+
### `mdq(source)`
|
|
48
|
+
|
|
49
|
+
`mdq(source: string | MarkdownDoc): MarkdownDoc`
|
|
50
|
+
|
|
51
|
+
Accepting a `MarkdownDoc` makes re-wrapping free.
|
|
52
|
+
|
|
53
|
+
### `MarkdownDoc`
|
|
54
|
+
|
|
55
|
+
| Method | Returns | Notes |
|
|
56
|
+
|---|---|---|
|
|
57
|
+
| `query(selector, matcher?)` | `Selection` | |
|
|
58
|
+
| `frontmatter()` | `Record<string, unknown>` | `{}` when absent |
|
|
59
|
+
| `setFrontmatter(key, value)` | `MarkdownDoc` | `null` value deletes the key |
|
|
60
|
+
| `append(md)` | `MarkdownDoc` | add a block at end of document |
|
|
61
|
+
| `prepend(md)` | `MarkdownDoc` | add a block at start of body, after frontmatter |
|
|
62
|
+
| `toString()` / `valueOf()` | `string` | full document, frontmatter included |
|
|
63
|
+
|
|
64
|
+
Plus the shared sugar layer.
|
|
65
|
+
|
|
66
|
+
`append`/`prepend` exist because "add a section to the end of the document" otherwise has
|
|
67
|
+
no clean path — only the `section().last().insertAfter(...)` workaround. There is a real
|
|
68
|
+
call site: `deep-analysis.ts:131` builds it by hand today as
|
|
69
|
+
`` `${cached.trimEnd()}\n\n# Extended Research\n\n...` ``.
|
|
70
|
+
|
|
71
|
+
### `Selection` — reads
|
|
72
|
+
|
|
73
|
+
Narrow or extract; never mutate.
|
|
74
|
+
|
|
75
|
+
| Method | Returns | Replaces |
|
|
76
|
+
|---|---|---|
|
|
77
|
+
| `query(selector, matcher?)` | `Selection` | sub-query, unchanged |
|
|
78
|
+
| `text()` | `string` | — (`get()` deprecated) |
|
|
79
|
+
| `count()` | `number` | |
|
|
80
|
+
| `exists()` | `boolean` | the `.count() > 0` idiom, 3 in-repo uses |
|
|
81
|
+
| `first()` / `last()` | `Selection` | |
|
|
82
|
+
| `at(n)` | `Selection` | new; sugar-path equivalent of DSL `[n]` |
|
|
83
|
+
| `slice(from?, to?)` | `Selection` | new; sugar-path equivalent of DSL `[a:b]` |
|
|
84
|
+
| `each()` | `Selection[]` | |
|
|
85
|
+
| `nodes()` | `NodeInfo[]` | `meta()` |
|
|
86
|
+
| `rows()` | `Record<string,string>[]` | `toJson()` — it only ever handled tables |
|
|
87
|
+
| `entries()` | `Record<string,string>` | `keyValue()` |
|
|
88
|
+
| `preceding()` / `following()` | `Selection` | `before()` / `after()` |
|
|
89
|
+
|
|
90
|
+
`before`/`after` are renamed specifically to free those names from colliding with
|
|
91
|
+
`insertBefore`/`insertAfter`.
|
|
92
|
+
|
|
93
|
+
### `Selection` — writes
|
|
94
|
+
|
|
95
|
+
Every write returns `MarkdownDoc`, so edits chain in one expression.
|
|
96
|
+
|
|
97
|
+
| Method | Signature | Notes |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| `replace(md)` | `(Markdown) => MarkdownDoc` | |
|
|
100
|
+
| `replaceEach(fn)` | `((Selection, number) => Markdown) => MarkdownDoc` | |
|
|
101
|
+
| `remove()` | `() => MarkdownDoc` | node **plus its adjacent `space` token** |
|
|
102
|
+
| `insertBefore(md)` | `(Markdown) => MarkdownDoc` | sibling |
|
|
103
|
+
| `insertAfter(md)` | `(Markdown) => MarkdownDoc` | sibling |
|
|
104
|
+
| `prepend(md)` | `(Markdown) => MarkdownDoc` | inside a section or list; `MdqOperationError` on a leaf node |
|
|
105
|
+
| `append(md)` | `(Markdown) => MarkdownDoc` | inside a section or list; `MdqOperationError` on a leaf node |
|
|
106
|
+
| `addRow(row)` | `(Record<string,string>) => MarkdownDoc` | table only; re-aligns columns |
|
|
107
|
+
| `addItem(text)` | `(string) => MarkdownDoc` | list only; matches marker + indent |
|
|
108
|
+
| `setEntry(key, value)` | `(string, string \| null) => MarkdownDoc` | `null` deletes |
|
|
109
|
+
|
|
110
|
+
Naming now pairs: `rows()`/`addRow()`, `entries()`/`setEntry()`, `nodes()`.
|
|
111
|
+
|
|
112
|
+
Every verb that *takes* markdown accepts `Markdown = string | MarkdownDoc`, mirroring
|
|
113
|
+
`mdq()` itself. A `replaceEach` callback may therefore return a `MarkdownDoc` built by a
|
|
114
|
+
nested edit, without a `.toString()` hop.
|
|
115
|
+
|
|
116
|
+
### Sugar layer
|
|
117
|
+
|
|
118
|
+
Eleven methods on both classes: `section` `heading` `paragraph` `table` `list` `item`
|
|
119
|
+
`code` `blockquote` `comment` `html` `hr`.
|
|
120
|
+
|
|
121
|
+
Each is `(matcher?, opts?) => Selection` and is *defined as* `query(sel, matcher)` —
|
|
122
|
+
documented as sugar, not a parallel implementation. Defined once on a shared base that
|
|
123
|
+
implements them in terms of an abstract `query()`, so the two classes do not duplicate it.
|
|
124
|
+
|
|
125
|
+
Depth is an option rather than 12 near-duplicate methods:
|
|
126
|
+
|
|
127
|
+
```js
|
|
128
|
+
mdq(src).section('API', { depth: 2 }) // DSL: query('section2("API")')
|
|
129
|
+
mdq(src).heading(/^f/i).at(0) // DSL: query('heading(/^f/i)[0]')
|
|
130
|
+
mdq(src).comment(/^test/) // DSL: query('comment(/^test/)')
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### Exported types
|
|
134
|
+
|
|
135
|
+
Declared at the end of their file, per repo convention.
|
|
136
|
+
|
|
137
|
+
```ts
|
|
138
|
+
type Markdown = string | MarkdownDoc;
|
|
139
|
+
type Matcher = string | RegExp | ((text: string) => boolean);
|
|
140
|
+
|
|
141
|
+
interface NodeInfo {
|
|
142
|
+
type: string; // 'heading' | 'paragraph' | 'table' | 'comment' | ...
|
|
143
|
+
depth: number | null; // heading level, else null
|
|
144
|
+
text: string; // unwrapped text; comment bodies without <!-- -->
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
interface SelectorOptions {
|
|
148
|
+
depth?: 1 | 2 | 3 | 4 | 5 | 6;
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
`Matcher` semantics:
|
|
153
|
+
|
|
154
|
+
- `string` — exact match (mirrors DSL `"x"`)
|
|
155
|
+
- `RegExp` — pattern, honoring its own flags
|
|
156
|
+
- function — predicate; needs no escaping at all
|
|
157
|
+
|
|
158
|
+
Note the consequence for `comment`: a `string` matcher is **exact**, and this repo's own
|
|
159
|
+
test-plan comments are multi-line (`<!-- test\n priority=critical\n-->`). So
|
|
160
|
+
`comment('test')` matches only a bare `<!-- test -->`; reaching the multi-line ones needs
|
|
161
|
+
`comment(/^test/)` or a predicate. Exactness is the consistent rule and is kept, but it is
|
|
162
|
+
the one place the sugar is likely to surprise.
|
|
163
|
+
|
|
164
|
+
This removes an existing wart. Today the repo hand-escapes to build selector strings:
|
|
165
|
+
|
|
166
|
+
```js
|
|
167
|
+
const escaped = section.name.replace(/"/g, '\\"'); // researcher/focus.ts:77
|
|
168
|
+
mdq(result.text).query(`section2(~"${escaped}")`);
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Deprecated aliases
|
|
172
|
+
|
|
173
|
+
`get` `toJson` `keyValue` `setKeyValue` `meta` `before` `after` are kept, marked
|
|
174
|
+
`@deprecated`, and omitted from the README so the published surface reads clean.
|
|
175
|
+
|
|
176
|
+
Aliases cover the read renames completely. They cannot shield the write return-type
|
|
177
|
+
change — see Migration.
|
|
178
|
+
|
|
179
|
+
## Selector grammar
|
|
180
|
+
|
|
181
|
+
Unchanged, plus one addition and three fixes.
|
|
182
|
+
|
|
183
|
+
### `comment` (new)
|
|
184
|
+
|
|
185
|
+
`html` tokens filtered to those that are comments. Not an alias for `html`: `<div>x</div>`
|
|
186
|
+
lexes as `html` too.
|
|
187
|
+
|
|
188
|
+
- `comment` matches on the **inner** body, trimmed. `html` matches on raw.
|
|
189
|
+
This is required for anchored patterns — `/^test/` against `<!-- test id=1 -->` only
|
|
190
|
+
works if the text is `test id=1`.
|
|
191
|
+
- Multi-line comments are a single token and keep their newlines in the matched text.
|
|
192
|
+
- **Inline comments are out of scope for 1.0.** `para with <!-- x --> comment` lexes the
|
|
193
|
+
comment inside the paragraph token; it is not reachable as a block. Documented, not faked.
|
|
194
|
+
|
|
195
|
+
`comment` and `html` together finish the `test-plan-markdown.ts` story: its hand-rolled
|
|
196
|
+
line parser (`src/utils/test-plan-markdown.ts:122+`) exists only because mdq could not
|
|
197
|
+
see `<!-- suite -->` and `<!-- test ... -->`.
|
|
198
|
+
|
|
199
|
+
### Fixes
|
|
200
|
+
|
|
201
|
+
1. **Regex flags are honored.** Today flags are parsed then discarded
|
|
202
|
+
(`markdown-query.ts:90`) and `'i'` is hardcoded (`markdown-query.ts:156`), so `/x/` is
|
|
203
|
+
case-insensitive while `"x"` and `~"x"` are case-sensitive. After the fix `/x/i` is
|
|
204
|
+
insensitive and `/x/` is not. One production call site relies on the old behavior:
|
|
205
|
+
`researcher.ts:316` `section2(/^summary/)` becomes `/^summary/i`. Tests already write
|
|
206
|
+
flags explicitly.
|
|
207
|
+
2. **Unknown selectors throw.** Today `query('secton("A")')` silently matches nothing
|
|
208
|
+
(`markdown-query.ts:103-106`, `:356`). Unacceptable for a CLI.
|
|
209
|
+
3. **Table text-match widens to headers plus cells.** Today `getTokenText` returns headers
|
|
210
|
+
only (`markdown-query.ts:181`), so `table(~"GET")` can never match a cell. No call site
|
|
211
|
+
uses table text-matching, so this is safe.
|
|
212
|
+
|
|
213
|
+
## Update semantics
|
|
214
|
+
|
|
215
|
+
`marked` separators are uneven, and every write rule follows from this:
|
|
216
|
+
|
|
217
|
+
| Token | `raw` |
|
|
218
|
+
|---|---|
|
|
219
|
+
| `heading` | `"# A\n\n"` — separators baked in |
|
|
220
|
+
| `paragraph` | `"para"` — no trailing newline |
|
|
221
|
+
| `space` | `"\n\n"` — a separate token |
|
|
222
|
+
|
|
223
|
+
The token index therefore records each node's range **and its adjacent `space` range**.
|
|
224
|
+
|
|
225
|
+
> **Invariant: mdq never leaves zero blank lines between blocks, and never more than one.**
|
|
226
|
+
|
|
227
|
+
- `remove()` takes the node plus its trailing `space` — or its leading `space` when it is
|
|
228
|
+
the last block. Without this, removing a paragraph leaves a four-newline crater. This is
|
|
229
|
+
the most likely bug in the feature and gets dedicated tests.
|
|
230
|
+
- `insertAfter` / `append` normalize inserted markdown to one trailing `\n` and splice at
|
|
231
|
+
the boundary, never inside a space token.
|
|
232
|
+
- `append` on a section inserts before the next same-or-shallower heading, reusing the
|
|
233
|
+
existing `computeSections` end boundary.
|
|
234
|
+
- `addRow` re-renders the whole table so column pipes stay aligned.
|
|
235
|
+
- `addItem` copies the list's existing marker (`-`, `*`, `1.`) and indent.
|
|
236
|
+
|
|
237
|
+
## Frontmatter
|
|
238
|
+
|
|
239
|
+
Every `knowledge/` and `experience/` file opens with `---\nurl: /login\n---`, which
|
|
240
|
+
`marked` lexes as a setext h2 titled `url: /login`.
|
|
241
|
+
|
|
242
|
+
mdq detects leading frontmatter, excludes it from the token index with offsets preserved
|
|
243
|
+
so edits splice correctly, and exposes it as data.
|
|
244
|
+
|
|
245
|
+
Reading and writing both go through `yaml`'s **Document API** (`YAML.parseDocument`), not
|
|
246
|
+
`parse`/`stringify`. That buys two things a hand-rolled parser cannot: correctness on
|
|
247
|
+
nested maps, lists and block scalars — the Jekyll/Astro/Obsidian files that justify the
|
|
248
|
+
feature — and **comment preservation through a write**, verified:
|
|
249
|
+
|
|
250
|
+
```yaml
|
|
251
|
+
# a leading comment <- survives setFrontmatter('wait', 2000)
|
|
252
|
+
url: /login
|
|
253
|
+
wait: 2000
|
|
254
|
+
tags:
|
|
255
|
+
- auth
|
|
256
|
+
- smoke
|
|
257
|
+
nested:
|
|
258
|
+
key: value # trailing note <- also survives
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
`gray-matter` is deliberately not used: `knowledge-tracker.ts` keeps it for its own
|
|
262
|
+
purposes, but a published package should not carry it to do what `yaml` already does.
|
|
263
|
+
|
|
264
|
+
```js
|
|
265
|
+
const doc = mdq(knowledgeFile);
|
|
266
|
+
doc.frontmatter(); // { url: '/login', wait: 1000, tags: ['auth'] }
|
|
267
|
+
doc.query('h2').count(); // 0 — the --- block is not a heading
|
|
268
|
+
doc.setFrontmatter('wait', 2000).toString();
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
## Errors
|
|
272
|
+
|
|
273
|
+
`MdqError` base, with:
|
|
274
|
+
|
|
275
|
+
- `MdqSelectorError` — malformed or unknown selector, carrying the offending index.
|
|
276
|
+
- `MdqOperationError` — a verb applied to the wrong node type, e.g. `addRow` on a paragraph.
|
|
277
|
+
|
|
278
|
+
An **empty selection is a safe no-op**: reads return `''` / `[]`, writes return the
|
|
279
|
+
document unchanged. This preserves the existing "returns source unchanged when no matches"
|
|
280
|
+
test.
|
|
281
|
+
|
|
282
|
+
## CLI (phase 2)
|
|
283
|
+
|
|
284
|
+
The selector is the program, the file or stdin is the input, markdown is the default output.
|
|
285
|
+
|
|
286
|
+
A leading `.` is accepted and ignored, so muscle memory from jq (`mdq '.h2'`) works. It is
|
|
287
|
+
sugar in the grammar, not a separate syntax — without it the new "unknown selectors throw"
|
|
288
|
+
rule would reject the most natural thing a jq user types first.
|
|
289
|
+
|
|
290
|
+
```bash
|
|
291
|
+
mdq 'h2' README.md # raw markdown of matches
|
|
292
|
+
cat plan.md | mdq 'section("API") table' -j # rows() as JSON
|
|
293
|
+
mdq 'comment(~"test")' plan.md --count
|
|
294
|
+
mdq 'section("FAQ")' doc.md --remove -i # edit in place
|
|
295
|
+
mdq 'table[0]' api.md --add-row '{"Method":"GET","Path":"/users"}' -i
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
Flags mirror library verbs exactly: `--remove` `--replace` `--insert-before`
|
|
299
|
+
`--insert-after` `--prepend` `--append` `--add-row` `--add-item` `--set k=v`, plus
|
|
300
|
+
`-i/--in-place`, `-j/--json`, `-c/--count`, `-t/--text`, `--frontmatter`.
|
|
301
|
+
|
|
302
|
+
Built with Commander, per repo convention. Exit codes compose like grep: **0** match,
|
|
303
|
+
**1** no match, **2** usage or selector error.
|
|
304
|
+
|
|
305
|
+
1.0 reads one file or stdin. Multi-file input is out of scope.
|
|
306
|
+
|
|
307
|
+
## Testing
|
|
308
|
+
|
|
309
|
+
Port the existing 801-line suite first — it is the regression net for all 54 call sites.
|
|
310
|
+
Then add coverage for what is new or newly specified:
|
|
311
|
+
|
|
312
|
+
- whitespace craters on `remove` (the invariant above)
|
|
313
|
+
- chained multi-edits through `MarkdownDoc`
|
|
314
|
+
- `addRow` column alignment; `addItem` marker and indent matching
|
|
315
|
+
- frontmatter round-trip, including a file whose body has its own `---`
|
|
316
|
+
- `comment` inner-text matching, multi-line comments, and `html` versus `comment`
|
|
317
|
+
- `MdqSelectorError` on unknown selectors and malformed input
|
|
318
|
+
- sugar equivalence: every sugar call equals its `query()` form
|
|
319
|
+
|
|
320
|
+
## Migration
|
|
321
|
+
|
|
322
|
+
Two steps. Only the second carries risk.
|
|
323
|
+
|
|
324
|
+
1. `src/utils/markdown-query.ts` becomes a re-export shim. **All 54 call sites keep
|
|
325
|
+
working untouched.**
|
|
326
|
+
2. A sweep updates imports, then fixes the call sites the return-type change breaks.
|
|
327
|
+
|
|
328
|
+
Deprecated read aliases mean **no read call site changes**. Writes are not shielded: a
|
|
329
|
+
verb that returned `string` now returns `MarkdownDoc`. That breaks four classes of site,
|
|
330
|
+
at least eleven in total.
|
|
331
|
+
|
|
332
|
+
**(a) Assignment into a `string`-typed target** — 7 sites, each needs `.toString()`:
|
|
333
|
+
|
|
334
|
+
| Site | Target |
|
|
335
|
+
|---|---|
|
|
336
|
+
| `experience-tracker.ts:265` | `content` (inferred `string`) |
|
|
337
|
+
| `experience-tracker.ts:289` | `combined` (inferred `string`) |
|
|
338
|
+
| `researcher/deep-analysis.ts:129` | `let updated: string` |
|
|
339
|
+
| `researcher/locators.ts:307` | `result.text` |
|
|
340
|
+
| `researcher/locators.ts:309` | `result.text` |
|
|
341
|
+
| `researcher/pagination.ts:61` | `result.text` |
|
|
342
|
+
| `researcher/research-result.ts:57` | `section.rawMarkdown` |
|
|
343
|
+
| `researcher/research-result.ts:58` | `this.text` |
|
|
344
|
+
|
|
345
|
+
**(b) A string method called on the result** — 2 sites:
|
|
346
|
+
|
|
347
|
+
- `planner.ts:304` — `.replace('').trim()`
|
|
348
|
+
- `planner.ts:322` — `const kept = ...replace('')`, then `kept.trimEnd()` on line 324
|
|
349
|
+
|
|
350
|
+
**(c) Returned from a `replaceEach` callback** — `deep-analysis.ts:542`. **Resolved by
|
|
351
|
+
design**, not by migration: callbacks accept `Markdown`, so returning a `MarkdownDoc` is
|
|
352
|
+
valid. No edit needed.
|
|
353
|
+
|
|
354
|
+
**(d) Compared against a string — the dangerous one.** `research-result.ts:56`:
|
|
355
|
+
|
|
356
|
+
```js
|
|
357
|
+
const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`);
|
|
358
|
+
if (updated === this.text) return; // MarkdownDoc === string is always false
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
This does not crash. The guard silently stops firing and the method starts doing work it
|
|
362
|
+
used to skip. `tsc` does flag it — comparing types with no overlap is an error — which is
|
|
363
|
+
precisely why the manual type-check below is not optional. Every `replace`/`setEntry`
|
|
364
|
+
result used in an equality or truthiness test must be audited, not just the ones that fail
|
|
365
|
+
to compile.
|
|
366
|
+
|
|
367
|
+
Sites that flow the result straight back into `mdq()` — `planner.ts:303`, `planner.ts:405`
|
|
368
|
+
— keep working unchanged, because `mdq()` accepts a `MarkdownDoc`.
|
|
369
|
+
|
|
370
|
+
`tsc` is the complete detector for these breaks: every one surfaces as a type error. The migration step is therefore *run `tsc` over the changed files and fix
|
|
371
|
+
what it reports*, with the table above as the expected result rather than the whole story.
|
|
372
|
+
|
|
373
|
+
Also in the sweep: `researcher.ts:316` gains its `i` flag (`section2(/^summary/i)`).
|
|
374
|
+
|
|
375
|
+
### Hazards
|
|
376
|
+
|
|
377
|
+
- **`.claude/worktrees/**` holds four stale copies** of `markdown-query.ts` and its
|
|
378
|
+
consumers. Every grep or sed sweep must exclude that path.
|
|
379
|
+
- **`tsc` runs with `--noCheck` in CI.** The nine breaks above are *exactly* the errors a
|
|
380
|
+
type-check would raise, and CI raises none of them — a fully green build proves nothing
|
|
381
|
+
here. Run `tsc` manually over the changed files before considering the sweep done.
|
|
382
|
+
|
|
383
|
+
## Style compliance
|
|
384
|
+
|
|
385
|
+
The current file violates several repo rules that the rewrite fixes: types belong at the
|
|
386
|
+
end of the file, ternaries are banned (`markdown-query.ts:30`, `:34`, `:129-130`, `:295`),
|
|
387
|
+
and the `switch` in `matchText` should be early returns.
|
|
388
|
+
|
|
389
|
+
## Deliberately out of scope
|
|
390
|
+
|
|
391
|
+
- Publishing: no `package.json`, no build script, no npm release in this change.
|
|
392
|
+
- Frontmatter formats other than YAML (TOML `+++`, JSON) — detected and skipped from the
|
|
393
|
+
token index, but not parsed.
|
|
394
|
+
- Row-level and item-level *selectors* (`addRow` has no `removeRow` partner). A future
|
|
395
|
+
`row(...)` selector is the right shape for that; guessing at it now is premature.
|
|
396
|
+
- Inline HTML comments.
|
|
397
|
+
- Multi-file CLI input.
|
package/package.json
CHANGED
|
@@ -221,13 +221,19 @@ export function createAskApiTool(fisherman: Fisherman | null, task: Test) {
|
|
|
221
221
|
return {
|
|
222
222
|
askApi: tool({
|
|
223
223
|
description: dedent`
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
224
|
+
Read the app's data over the API, changing nothing.
|
|
225
|
+
Answers questions about records: which ones are there, what they are called, whether a particular one exists.
|
|
226
|
+
|
|
227
|
+
Use it to:
|
|
228
|
+
- check whether suitable data already exists, before precondition() creates any
|
|
229
|
+
- get the exact name or id of a record a step must act on
|
|
230
|
+
- find out whether an action was stored, when the app reported success but the page does not show the result
|
|
231
|
+
- find out whether data exists at all, when a list or dropdown is empty
|
|
232
|
+
|
|
227
233
|
It never creates, edits or deletes anything — precondition() does that.
|
|
228
234
|
`,
|
|
229
235
|
inputSchema: z.object({
|
|
230
|
-
question: z.string().describe('What to find out about data
|
|
236
|
+
question: z.string().describe('What to find out about the data'),
|
|
231
237
|
}),
|
|
232
238
|
execute: async ({ question }) => {
|
|
233
239
|
tag('info').log(`Ask API: ${question}`);
|
package/src/ai/pilot.ts
CHANGED
|
@@ -1016,14 +1016,14 @@ export class Pilot implements Agent {
|
|
|
1016
1016
|
private formatSuccessfulAssertions(currentState: ActionResult, testerConversation: Conversation): string {
|
|
1017
1017
|
const lines: string[] = [];
|
|
1018
1018
|
for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
|
|
1019
|
-
if (passed) lines.push(`
|
|
1019
|
+
if (passed) lines.push(`verify: ${assertion}`);
|
|
1020
1020
|
}
|
|
1021
1021
|
|
|
1022
1022
|
for (const exec of testerConversation.getToolExecutions()) {
|
|
1023
1023
|
if (!EVIDENCE_TOOLS.includes(exec.toolName) || !exec.wasSuccessful) continue;
|
|
1024
1024
|
const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
|
|
1025
|
-
const
|
|
1026
|
-
lines.push(
|
|
1025
|
+
const analysis = exec.output?.analysis;
|
|
1026
|
+
lines.push(`${exec.toolName}: ${description}${analysis ? ` -> ${analysis}` : ''}`);
|
|
1027
1027
|
}
|
|
1028
1028
|
|
|
1029
1029
|
return [...new Set(lines)].join('\n');
|
|
@@ -1157,9 +1157,14 @@ export class Pilot implements Agent {
|
|
|
1157
1157
|
|
|
1158
1158
|
YOUR Pilot-only tools, both over the API:
|
|
1159
1159
|
|
|
1160
|
-
askApi(question) —
|
|
1161
|
-
|
|
1162
|
-
|
|
1160
|
+
askApi(question) — read the app's data over the API. It changes nothing. Use when:
|
|
1161
|
+
|
|
1162
|
+
- Before precondition() — check whether suitable data already exists.
|
|
1163
|
+
- A step needs the exact name or id of an existing record.
|
|
1164
|
+
- The app reported success but the page does not show the result — ask whether it was stored.
|
|
1165
|
+
- A list or dropdown is empty — ask whether the data exists at all.
|
|
1166
|
+
|
|
1167
|
+
The page is not the only witness. A record missing from the page may still exist.
|
|
1163
1168
|
|
|
1164
1169
|
precondition(description) — create FRESH disposable test data. Never request users. Use when:
|
|
1165
1170
|
|
package/src/ai/tools.ts
CHANGED
|
@@ -1368,16 +1368,22 @@ async function extractWebElements(error: Error | null | undefined): Promise<Matc
|
|
|
1368
1368
|
|
|
1369
1369
|
function formatElementList(matched: MatchedElement[] | null): string {
|
|
1370
1370
|
if (!matched) return 'Could not fetch element details. Repeat the action to get better info.';
|
|
1371
|
-
|
|
1371
|
+
const keys = matched.map((el) => `${el.text}::${el.visible}::${el.html}`);
|
|
1372
|
+
const list = matched
|
|
1372
1373
|
.map((el, i) => {
|
|
1373
1374
|
const lines = [`Element ${i + 1}:`, `Text: "${el.text}"`];
|
|
1374
1375
|
if (el.visible !== undefined) lines.push(`Visible: ${el.visible}`);
|
|
1375
1376
|
const wrapped = matched.map((_, j) => j).filter((j) => j !== i && matched[j].xpath.startsWith(`${el.xpath}/`));
|
|
1376
1377
|
if (wrapped.length) lines.push(`Wraps: element ${wrapped.map((j) => j + 1).join(', ')}`);
|
|
1378
|
+
const same = keys.map((_, j) => j).filter((j) => j !== i && keys[j] === keys[i]);
|
|
1379
|
+
if (same.length) lines.push(`Identical to element ${same.map((j) => j + 1).join(', ')}`);
|
|
1377
1380
|
lines.push(`XPath: ${el.xpath}`, `HTML: ${el.html}`);
|
|
1378
1381
|
return lines.join('\n');
|
|
1379
1382
|
})
|
|
1380
1383
|
.join('\n\n');
|
|
1384
|
+
|
|
1385
|
+
if (new Set(keys).size === matched.length) return list;
|
|
1386
|
+
return `${list}\n\nIdentical matches are not told apart by their number — picking one is a guess. Click the one you mean by appearance with visualClick().`;
|
|
1381
1387
|
}
|
|
1382
1388
|
|
|
1383
1389
|
export async function formatMatchedElements(error: Error | null | undefined): Promise<string | null> {
|
|
@@ -15,7 +15,7 @@ export class ConfigCommand extends BaseCommand {
|
|
|
15
15
|
|
|
16
16
|
async execute(): Promise<void> {
|
|
17
17
|
const parser = ConfigParser.getInstance();
|
|
18
|
-
tag('info').log(ConfigCommand.render(this.explorBot.getConfig(), { configPath: parser.getConfigPath(), root: parser.getProjectRoot() }));
|
|
18
|
+
tag('info').log(ConfigCommand.render(this.explorBot.getConfig(), { configPath: parser.getConfigPath(), siteConfigPath: parser.getSiteConfigPath(), root: parser.getProjectRoot() }));
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
static async summary(options: { config?: string; path?: string; url?: string; json?: boolean } = {}): Promise<string> {
|
|
@@ -28,13 +28,16 @@ export class ConfigCommand extends BaseCommand {
|
|
|
28
28
|
return load(site.url);
|
|
29
29
|
});
|
|
30
30
|
|
|
31
|
-
return ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json });
|
|
31
|
+
return ConfigCommand.render(config, { configPath: parser.getConfigPath(), siteConfigPath: parser.getSiteConfigPath(), root: parser.getProjectRoot(), json: options.json });
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
static data(config: SummarizedConfig, options: ConfigSummaryOptions = {}): ConfigData {
|
|
35
35
|
let configPath = '';
|
|
36
36
|
if (options.configPath && existsSync(options.configPath)) configPath = options.configPath;
|
|
37
37
|
|
|
38
|
+
let siteConfigPath = '';
|
|
39
|
+
if (options.siteConfigPath && existsSync(options.siteConfigPath)) siteConfigPath = options.siteConfigPath;
|
|
40
|
+
|
|
38
41
|
const dirs: Record<string, string> = {};
|
|
39
42
|
if (options.root) {
|
|
40
43
|
for (const [name, dir] of Object.entries({ output: 'output', ...config.dirs })) {
|
|
@@ -60,6 +63,7 @@ export class ConfigCommand extends BaseCommand {
|
|
|
60
63
|
|
|
61
64
|
return {
|
|
62
65
|
config: configPath,
|
|
66
|
+
siteConfig: siteConfigPath,
|
|
63
67
|
url: config.playwright?.url || config.web?.url || config.api?.baseEndpoint || '',
|
|
64
68
|
browser: config.playwright?.browser || '',
|
|
65
69
|
headless: !config.playwright?.show,
|
|
@@ -82,6 +86,7 @@ export class ConfigCommand extends BaseCommand {
|
|
|
82
86
|
const section = (title: string, entries: [string, string][]) => lines.push(...renderSection(title, entries));
|
|
83
87
|
|
|
84
88
|
const general: [string, string][] = [['config', data.config || 'EXPLORBOT_* environment variables']];
|
|
89
|
+
if (data.siteConfig) general.push(['site config', data.siteConfig]);
|
|
85
90
|
if (data.url) general.push(['url', data.url]);
|
|
86
91
|
if (data.browser) {
|
|
87
92
|
let window = 'visible';
|
|
@@ -118,12 +123,14 @@ export class ConfigCommand extends BaseCommand {
|
|
|
118
123
|
|
|
119
124
|
interface ConfigSummaryOptions {
|
|
120
125
|
configPath?: string | null;
|
|
126
|
+
siteConfigPath?: string | null;
|
|
121
127
|
root?: string;
|
|
122
128
|
json?: boolean;
|
|
123
129
|
}
|
|
124
130
|
|
|
125
131
|
export interface ConfigData {
|
|
126
132
|
config: string;
|
|
133
|
+
siteConfig: string;
|
|
127
134
|
url: string;
|
|
128
135
|
browser: string;
|
|
129
136
|
headless: boolean;
|
|
@@ -7,10 +7,7 @@ import { getCliName } from '../utils/cli-name.ts';
|
|
|
7
7
|
import { log, tag } from '../utils/logger.js';
|
|
8
8
|
import { relativeToCwd } from '../utils/next-steps.ts';
|
|
9
9
|
|
|
10
|
-
function defaultConfigTemplate(provider: string
|
|
11
|
-
let moduleExport = 'module.exports = config;';
|
|
12
|
-
if (esm) moduleExport = 'export default config;';
|
|
13
|
-
|
|
10
|
+
function defaultConfigTemplate(provider: string): string {
|
|
14
11
|
return `// 'provider/model-id' uses a bundled provider.
|
|
15
12
|
// It is also possible to import provider as a module from Vercel AI SDK.
|
|
16
13
|
// https://github.com/testomatio/explorbot/blob/main/docs/basics/providers.md
|
|
@@ -35,7 +32,7 @@ ${modelLines(provider)}
|
|
|
35
32
|
},
|
|
36
33
|
};
|
|
37
34
|
|
|
38
|
-
|
|
35
|
+
export default config;
|
|
39
36
|
`;
|
|
40
37
|
}
|
|
41
38
|
|
|
@@ -144,8 +141,7 @@ export function runInitCommand(options: InitCommandOptions): void {
|
|
|
144
141
|
process.exit(1);
|
|
145
142
|
}
|
|
146
143
|
|
|
147
|
-
|
|
148
|
-
writeFileSync(outPath, defaultConfigTemplate(provider, esm), 'utf8');
|
|
144
|
+
writeFileSync(outPath, defaultConfigTemplate(provider), 'utf8');
|
|
149
145
|
log(`Created config file: ${relativeToCwd(outPath)}`);
|
|
150
146
|
|
|
151
147
|
const envPath = resolve(process.cwd(), '.env');
|
|
@@ -271,6 +267,8 @@ function globalConfigTemplate(provider: string): string {
|
|
|
271
267
|
const { envKey } = PROVIDERS[provider];
|
|
272
268
|
|
|
273
269
|
return `// Global Explorbot configuration — used by every directory without its own explorbot.config.js.
|
|
270
|
+
// Settings shared by every site. Each site extends them in
|
|
271
|
+
// ~/.explorbot/sites/<host>/explorbot.config.js, written on its first run.
|
|
274
272
|
// Models are written as 'provider/model-id' so they resolve without a local node_modules.
|
|
275
273
|
// The key is read from ${envKey} in ~/.explorbot/.env
|
|
276
274
|
// Model ids are snapshotted from the recommendations of this Explorbot version.
|
|
@@ -288,29 +286,10 @@ ${modelLines(provider)}
|
|
|
288
286
|
},
|
|
289
287
|
};
|
|
290
288
|
|
|
291
|
-
|
|
289
|
+
export default config;
|
|
292
290
|
`;
|
|
293
291
|
}
|
|
294
292
|
|
|
295
|
-
function isModuleProject(configDir: string): boolean {
|
|
296
|
-
let currentDir = resolve(configDir);
|
|
297
|
-
|
|
298
|
-
while (true) {
|
|
299
|
-
const packagePath = join(currentDir, 'package.json');
|
|
300
|
-
if (existsSync(packagePath)) {
|
|
301
|
-
try {
|
|
302
|
-
return JSON.parse(readFileSync(packagePath, 'utf8')).type === 'module';
|
|
303
|
-
} catch {
|
|
304
|
-
return false;
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
const parentDir = dirname(currentDir);
|
|
309
|
-
if (parentDir === currentDir) return false;
|
|
310
|
-
currentDir = parentDir;
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
|
|
314
293
|
function writeEnvKey(key: string, value: string): void {
|
|
315
294
|
const envPath = globalEnvPath();
|
|
316
295
|
let content = '# AI provider API keys';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { listSites, sitesDir } from '../global-config.js';
|
|
1
|
+
import { findSiteConfig, listSites, sitesDir } from '../global-config.js';
|
|
2
2
|
import { getCliName } from '../utils/cli-name.js';
|
|
3
3
|
import { tag } from '../utils/logger.js';
|
|
4
4
|
import { BaseCommand } from './base-command.js';
|
|
@@ -20,6 +20,11 @@ export class SitesCommand extends BaseCommand {
|
|
|
20
20
|
tag('info').log(`Registered sites (${sites.length}):`);
|
|
21
21
|
for (const site of sites) {
|
|
22
22
|
tag('info').log(` ${site.folder.padEnd(width)} ${site.url} last run ${site.lastRunAt.slice(0, 16).replace('T', ' ')}`);
|
|
23
|
+
|
|
24
|
+
let config = 'inherits global config';
|
|
25
|
+
const sitePath = findSiteConfig(site.dir);
|
|
26
|
+
if (sitePath) config = sitePath;
|
|
27
|
+
tag('info').log(` ${' '.repeat(width)} ${config}`);
|
|
23
28
|
}
|
|
24
29
|
tag('info').log('');
|
|
25
30
|
tag('info').log(`Stored in ${sitesDir()}`);
|