@taurgis/bonsai 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -6
- package/agents/agents/salesforce-docs-researcher.agent.md +117 -0
- package/agents/hooks/README.md +62 -0
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +22 -0
- package/agents/hooks/claude-code/settings.json +16 -0
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +18 -0
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +22 -0
- package/agents/hooks/codex/hooks.json +28 -0
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +21 -0
- package/agents/hooks/cursor/hooks.json +12 -0
- package/agents/hooks/github-copilot/bonsai-web-fetch.json +23 -0
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +19 -0
- package/agents/instructions/salesforce-research.instructions.md +57 -0
- package/agents/instructions/web-research.instructions.md +2 -1
- package/bin/cli.mjs +13 -3
- package/dist/base-command.d.ts +20 -0
- package/dist/base-command.js +27 -1
- package/dist/commands/config/get.js +2 -2
- package/dist/commands/config/list.js +2 -2
- package/dist/commands/config/set.js +2 -2
- package/dist/commands/config/unset.js +2 -2
- package/dist/commands/fetch.d.ts +5 -0
- package/dist/commands/fetch.js +98 -27
- package/dist/commands/import.d.ts +1 -0
- package/dist/commands/import.js +51 -11
- package/dist/commands/list.js +18 -3
- package/dist/commands/prune.js +22 -6
- package/dist/commands/search.js +31 -30
- package/dist/commands/status.js +10 -9
- package/dist/hooks/command-not-found/suggest.d.ts +9 -0
- package/dist/hooks/command-not-found/suggest.js +93 -0
- package/dist/lib/config/index.d.ts +8 -3
- package/dist/lib/config/index.js +19 -4
- package/dist/lib/config/resolve.d.ts +21 -1
- package/dist/lib/config/resolve.js +42 -1
- package/dist/lib/config/schema.d.ts +3 -0
- package/dist/lib/config/schema.js +11 -1
- package/dist/lib/research/artifact.d.ts +13 -2
- package/dist/lib/research/artifact.js +27 -7
- package/dist/lib/research/browser.d.ts +12 -0
- package/dist/lib/research/browser.js +40 -3
- package/dist/lib/research/compress.d.ts +19 -0
- package/dist/lib/research/compress.js +31 -0
- package/dist/lib/research/docs/capabilities.d.ts +1 -1
- package/dist/lib/research/docs/code-blocks.js +24 -0
- package/dist/lib/research/docs/discovery-index.d.ts +14 -0
- package/dist/lib/research/docs/discovery-index.js +112 -0
- package/dist/lib/research/docs/machine-readable.d.ts +2 -2
- package/dist/lib/research/docs/machine-readable.js +3 -7
- package/dist/lib/research/docs/remote-search-runner.js +61 -1
- package/dist/lib/research/docs/section-artifacts.d.ts +3 -2
- package/dist/lib/research/docs/section-artifacts.js +5 -5
- package/dist/lib/research/docs/sections.d.ts +1 -0
- package/dist/lib/research/docs/sections.js +19 -1
- package/dist/lib/research/freshness.d.ts +9 -1
- package/dist/lib/research/freshness.js +24 -2
- package/dist/lib/research/keywords.d.ts +24 -0
- package/dist/lib/research/keywords.js +247 -0
- package/dist/lib/research/revalidate.d.ts +3 -1
- package/dist/lib/research/revalidate.js +8 -4
- package/dist/lib/research/search-index.d.ts +18 -0
- package/dist/lib/research/search-index.js +109 -0
- package/dist/lib/research/secret-scan.js +4 -2
- package/dist/lib/research/storage.d.ts +1 -0
- package/dist/lib/research/storage.js +1 -1
- package/dist/lib/research/summarize.d.ts +17 -0
- package/dist/lib/research/summarize.js +400 -0
- package/dist/lib/research/url.js +5 -2
- package/dist/lib/text.d.ts +33 -0
- package/dist/lib/text.js +63 -0
- package/oclif.manifest.json +8 -8
- package/package.json +17 -3
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
// Dependency-free extractive summarizer used as a fallback when structural compression
|
|
2
|
+
// (compressMarkdown) fails to meaningfully shrink prose-heavy pages. It selects the most important
|
|
3
|
+
// *original* sentences from prose paragraphs and drops the rest, while emitting headings, fenced
|
|
4
|
+
// code blocks, tables, and lists verbatim. Extractive only — it never rewrites text, so no LLM.
|
|
5
|
+
// ponytail: frequency scoring for small inputs, TextRank (token-overlap similarity + weighted
|
|
6
|
+
// PageRank) for mid-sized ones. Ceiling: English-only stopwords, regex sentence splitting, lexical
|
|
7
|
+
// (not semantic) similarity, O(n^2) TextRank capped by sentence count. Upgrade path: embedding-based
|
|
8
|
+
// similarity or an abstractive LLM pass behind the same summarizeMarkdown() API.
|
|
9
|
+
import { STOPWORDS, tokenize } from './keywords.js';
|
|
10
|
+
import { estimateTokens } from './token-estimate.js';
|
|
11
|
+
// Target output prose length as a fraction of input prose length, per level.
|
|
12
|
+
const TARGET_RATIO = {
|
|
13
|
+
conservative: 0.65,
|
|
14
|
+
balanced: 0.5,
|
|
15
|
+
aggressive: 0.35,
|
|
16
|
+
};
|
|
17
|
+
const DEFAULT_MIN_TOKENS = 200;
|
|
18
|
+
const DEFAULT_MIN_SENTENCES = 2;
|
|
19
|
+
// Below this, PageRank over a tiny graph is degenerate; above it the O(n^2) matrix gets expensive.
|
|
20
|
+
// Outside the window we fall back to linear frequency scoring.
|
|
21
|
+
const TEXTRANK_MIN_SENTENCES = 10;
|
|
22
|
+
const TEXTRANK_MAX_SENTENCES = 500;
|
|
23
|
+
// Abbreviations whose trailing period must not end a sentence. Lowercased, internal dots kept.
|
|
24
|
+
const ABBREVIATIONS = new Set([
|
|
25
|
+
'e.g',
|
|
26
|
+
'i.e',
|
|
27
|
+
'etc',
|
|
28
|
+
'vs',
|
|
29
|
+
'mr',
|
|
30
|
+
'mrs',
|
|
31
|
+
'dr',
|
|
32
|
+
'prof',
|
|
33
|
+
'fig',
|
|
34
|
+
'no',
|
|
35
|
+
'al',
|
|
36
|
+
'inc',
|
|
37
|
+
'ltd',
|
|
38
|
+
'co',
|
|
39
|
+
'vol',
|
|
40
|
+
'pp',
|
|
41
|
+
'eq',
|
|
42
|
+
'ref',
|
|
43
|
+
'cf',
|
|
44
|
+
'approx',
|
|
45
|
+
'est',
|
|
46
|
+
'dept',
|
|
47
|
+
'st',
|
|
48
|
+
'jr',
|
|
49
|
+
'sr',
|
|
50
|
+
]);
|
|
51
|
+
const isHeading = (line) => /^\s{0,3}#{1,6}\s/.test(line);
|
|
52
|
+
// Lines that must survive verbatim: structural markdown that a sentence splitter would mangle.
|
|
53
|
+
function isVerbatim(line) {
|
|
54
|
+
return (line.trim() === '' ||
|
|
55
|
+
isHeading(line) ||
|
|
56
|
+
/^\s*([-*+]|\d+\.)\s/.test(line) || // list item
|
|
57
|
+
/^\s*>/.test(line) || // blockquote
|
|
58
|
+
/^\s*([-*_])(\s*\1){2,}\s*$/.test(line) || // thematic break
|
|
59
|
+
/^\s*<!--/.test(line) || // single-line HTML comment (multi-line tracked in segment)
|
|
60
|
+
/^\s*<\/?[a-zA-Z]/.test(line) || // HTML block
|
|
61
|
+
line.includes('|') || // table row (contiguous runs preserved)
|
|
62
|
+
/^\s{2,}\S/.test(line) // indented: list continuation or indented code
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
// Single forward pass grouping contiguous same-kind lines. Fenced code and multi-line HTML comments
|
|
66
|
+
// are fully preserved; an unclosed fence/comment keeps everything to EOF verbatim (fail safe toward
|
|
67
|
+
// keeping content). Tracking comments as a block keeps their inner lines (e.g. Node's `<!-- YAML
|
|
68
|
+
// ... -->` changelog metadata) out of prose scoring and stops them being reflowed.
|
|
69
|
+
class BlockState {
|
|
70
|
+
fenceMarker = '';
|
|
71
|
+
inComment = false;
|
|
72
|
+
update(line) {
|
|
73
|
+
if (this.inComment) {
|
|
74
|
+
if (line.includes('-->'))
|
|
75
|
+
this.inComment = false;
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
const fenceMatch = /^\s*(```+|~~~+)/.exec(line);
|
|
79
|
+
if (this.fenceMarker) {
|
|
80
|
+
if (fenceMatch && line.trim().startsWith(this.fenceMarker))
|
|
81
|
+
this.fenceMarker = '';
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
if (fenceMatch) {
|
|
85
|
+
this.fenceMarker = fenceMatch[1];
|
|
86
|
+
return true;
|
|
87
|
+
}
|
|
88
|
+
if (line.trimStart().startsWith('<!--') && !line.includes('-->')) {
|
|
89
|
+
this.inComment = true;
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// Single forward pass grouping contiguous same-kind lines. Fenced code and multi-line HTML comments
|
|
96
|
+
// are fully preserved; an unclosed fence/comment keeps everything to EOF verbatim (fail safe toward
|
|
97
|
+
// keeping content). Tracking comments as a block keeps their inner lines (e.g. Node's `<!-- YAML
|
|
98
|
+
// ... -->` changelog metadata) out of prose scoring and stops them being reflowed.
|
|
99
|
+
function segment(markdown) {
|
|
100
|
+
const segments = [];
|
|
101
|
+
let current = null;
|
|
102
|
+
const state = new BlockState();
|
|
103
|
+
const push = (kind, line) => {
|
|
104
|
+
if (current && current.kind === kind)
|
|
105
|
+
current.lines.push(line);
|
|
106
|
+
else {
|
|
107
|
+
current = { kind, lines: [line] };
|
|
108
|
+
segments.push(current);
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
for (const line of markdown.split('\n')) {
|
|
112
|
+
const isSpecialBlock = state.update(line);
|
|
113
|
+
const kind = isSpecialBlock || isVerbatim(line) ? 'verbatim' : 'prose';
|
|
114
|
+
push(kind, line);
|
|
115
|
+
}
|
|
116
|
+
return segments;
|
|
117
|
+
}
|
|
118
|
+
// Replaces inline-code spans with NUL-delimited index placeholders so a '.' inside `code` cannot
|
|
119
|
+
// trigger a false sentence boundary; the returned restore() puts the originals back. Any NUL in the
|
|
120
|
+
// (untrusted) input is stripped first so the placeholder is collision-proof — otherwise a stray
|
|
121
|
+
// "NUL digit NUL" run already in the source would be mistaken for a placeholder and silently dropped.
|
|
122
|
+
function maskInlineCode(text) {
|
|
123
|
+
const spans = [];
|
|
124
|
+
const masked = text.replace(/\u0000/g, '').replace(/`[^`]*`/g, (m) => {
|
|
125
|
+
const token = `\u0000${spans.length}\u0000`;
|
|
126
|
+
spans.push(m);
|
|
127
|
+
return token;
|
|
128
|
+
});
|
|
129
|
+
const restore = (s) => s.replace(/\u0000(\d+)\u0000/g, (_, n) => spans[Number(n)] ?? '');
|
|
130
|
+
return { masked, restore };
|
|
131
|
+
}
|
|
132
|
+
function lastWhitespaceIndex(s, from) {
|
|
133
|
+
for (let k = from - 1; k >= 0; k--) {
|
|
134
|
+
if (/\s/.test(s[k]))
|
|
135
|
+
return k;
|
|
136
|
+
}
|
|
137
|
+
return -1;
|
|
138
|
+
}
|
|
139
|
+
// Splits a single prose paragraph into sentences, avoiding splits inside inline code, after known
|
|
140
|
+
// abbreviations or single-letter initials, and before digits. Exported for unit testing because
|
|
141
|
+
// sentence boundaries are not observable once kept sentences are rejoined.
|
|
142
|
+
function findBoundaryEnd(masked, startIdx) {
|
|
143
|
+
let end = startIdx;
|
|
144
|
+
while (end + 1 < masked.length && '.!?'.includes(masked[end + 1])) {
|
|
145
|
+
end++;
|
|
146
|
+
}
|
|
147
|
+
return end;
|
|
148
|
+
}
|
|
149
|
+
function hasSentenceStartLookahead(masked, fromIndex) {
|
|
150
|
+
return /^(\s+)["'(\[]?[A-Z0-9]/.test(masked.slice(fromIndex));
|
|
151
|
+
}
|
|
152
|
+
function isAbbreviationOrInitial(masked, periodIndex) {
|
|
153
|
+
const wordStart = lastWhitespaceIndex(masked, periodIndex) + 1;
|
|
154
|
+
const word = masked.slice(wordStart, periodIndex).toLowerCase().replace(/\.+$/, '');
|
|
155
|
+
return ABBREVIATIONS.has(word) || word.length === 1;
|
|
156
|
+
}
|
|
157
|
+
function shouldSkipBoundary(masked, i, end) {
|
|
158
|
+
if (!hasSentenceStartLookahead(masked, end + 1)) {
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
if (masked[i] === '.' && isAbbreviationOrInitial(masked, i)) {
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
// Splits a single prose paragraph into sentences, avoiding splits inside inline code, after known
|
|
167
|
+
// abbreviations or single-letter initials, and before digits. Exported for unit testing because
|
|
168
|
+
// sentence boundaries are not observable once kept sentences are rejoined.
|
|
169
|
+
export function splitSentences(paragraph) {
|
|
170
|
+
const { masked, restore } = maskInlineCode(paragraph);
|
|
171
|
+
const sentences = [];
|
|
172
|
+
let start = 0;
|
|
173
|
+
for (let i = 0; i < masked.length; i++) {
|
|
174
|
+
if (!'.!?'.includes(masked[i]))
|
|
175
|
+
continue;
|
|
176
|
+
const end = findBoundaryEnd(masked, i);
|
|
177
|
+
if (shouldSkipBoundary(masked, i, end)) {
|
|
178
|
+
i = end;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
const sentence = masked.slice(start, end + 1).trim();
|
|
182
|
+
if (sentence) {
|
|
183
|
+
sentences.push(restore(sentence));
|
|
184
|
+
}
|
|
185
|
+
start = end + 1;
|
|
186
|
+
i = end;
|
|
187
|
+
}
|
|
188
|
+
const tail = masked.slice(start).trim();
|
|
189
|
+
if (tail) {
|
|
190
|
+
sentences.push(restore(tail));
|
|
191
|
+
}
|
|
192
|
+
return sentences;
|
|
193
|
+
}
|
|
194
|
+
// Content-bearing tokens for scoring: code/URLs stripped, stopwords/short/numeric tokens dropped.
|
|
195
|
+
// Same filter as keywords.ts so scoring stays consistent with auto-tagging.
|
|
196
|
+
function contentTokens(text) {
|
|
197
|
+
const stripped = text.replace(/`[^`]*`/g, ' ').replace(/https?:\/\/\S+/g, ' ');
|
|
198
|
+
return tokenize(stripped).filter((t) => t.length >= 3 && !STOPWORDS.has(t) && !/^\d+$/.test(t));
|
|
199
|
+
}
|
|
200
|
+
function buildFrequencies(text, headingTerms) {
|
|
201
|
+
const counts = new Map();
|
|
202
|
+
for (const token of contentTokens(text)) {
|
|
203
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
204
|
+
}
|
|
205
|
+
// Terms echoed in headings are likely topical; weight them above body mentions.
|
|
206
|
+
for (const term of headingTerms) {
|
|
207
|
+
if (counts.has(term))
|
|
208
|
+
counts.set(term, counts.get(term) + 3);
|
|
209
|
+
}
|
|
210
|
+
return counts;
|
|
211
|
+
}
|
|
212
|
+
function frequencyScore(text, freq) {
|
|
213
|
+
const tokens = contentTokens(text);
|
|
214
|
+
if (tokens.length === 0)
|
|
215
|
+
return 0;
|
|
216
|
+
let sum = 0;
|
|
217
|
+
for (const t of tokens)
|
|
218
|
+
sum += freq.get(t) ?? 0;
|
|
219
|
+
return sum / tokens.length; // length-normalized → no long-sentence bias
|
|
220
|
+
}
|
|
221
|
+
function similarity(a, b) {
|
|
222
|
+
if (a.size === 0 || b.size === 0)
|
|
223
|
+
return 0;
|
|
224
|
+
let common = 0;
|
|
225
|
+
for (const t of a)
|
|
226
|
+
if (b.has(t))
|
|
227
|
+
common++;
|
|
228
|
+
if (common === 0)
|
|
229
|
+
return 0;
|
|
230
|
+
// Mihalcea TextRank normalization: overlap divided by the log sizes of both sentences.
|
|
231
|
+
return common / (Math.log(a.size + 1) + Math.log(b.size + 1));
|
|
232
|
+
}
|
|
233
|
+
// Weighted PageRank over the sentence-similarity graph. Fixed iterations + uniform init keep it
|
|
234
|
+
// deterministic. Returns one centrality score per sentence, index-aligned with `sets`.
|
|
235
|
+
function textRankScores(sets) {
|
|
236
|
+
const n = sets.length;
|
|
237
|
+
const weights = Array.from({ length: n }, () => new Array(n).fill(0));
|
|
238
|
+
const rowSums = new Array(n).fill(0);
|
|
239
|
+
for (let i = 0; i < n; i++) {
|
|
240
|
+
for (let j = i + 1; j < n; j++) {
|
|
241
|
+
const w = similarity(sets[i], sets[j]);
|
|
242
|
+
weights[i][j] = w;
|
|
243
|
+
weights[j][i] = w;
|
|
244
|
+
rowSums[i] += w;
|
|
245
|
+
rowSums[j] += w;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
const damping = 0.85;
|
|
249
|
+
let scores = new Array(n).fill(1 / n);
|
|
250
|
+
for (let iter = 0; iter < 30; iter++) {
|
|
251
|
+
const next = new Array(n).fill((1 - damping) / n);
|
|
252
|
+
for (let i = 0; i < n; i++) {
|
|
253
|
+
for (let j = 0; j < n; j++) {
|
|
254
|
+
const wji = weights[j][i];
|
|
255
|
+
const rsj = rowSums[j];
|
|
256
|
+
if (i === j || wji === 0 || rsj === 0)
|
|
257
|
+
continue;
|
|
258
|
+
next[i] += damping * (wji / rsj) * scores[j];
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
scores = next;
|
|
262
|
+
}
|
|
263
|
+
return scores;
|
|
264
|
+
}
|
|
265
|
+
// Lexical scoring ranks a "data corruption may occur" warning no higher than a bland description,
|
|
266
|
+
// so plain extraction can drop the sentences a reader can least afford to lose. Two tiers counter
|
|
267
|
+
// that, both kept tight enough not to fire on ordinary prose:
|
|
268
|
+
// - CRITICAL: catastrophic-if-lost cues (data loss, security, hard prohibitions, deprecation).
|
|
269
|
+
// These are force-kept in selection, never dropped — the same guarantee as a paragraph anchor.
|
|
270
|
+
// - CAUTION: broader API caveats (constraints, failure modes). These only get a score boost, so
|
|
271
|
+
// they are preferred when budget allows but can still yield on heavily anchor-saturated pages.
|
|
272
|
+
const CRITICAL = /\b(must not|mustn't|cannot|can't|do not|don't|should not|shouldn't|never|deprecat\w*|unsafe|security|secret|corrupt\w*|data loss|overwrit\w*|irreversible|not permitted|warning|caution|danger\w*)\b/i;
|
|
273
|
+
const CAUTION = /\b(must|required?|requires?|throws?|fails?|failure|invalid|important|only if|note that|otherwise)\b/i;
|
|
274
|
+
const SALIENCE_BOOST = 1.6;
|
|
275
|
+
// Raises the score of broad-caveat sentences so the budget cut preserves them preferentially.
|
|
276
|
+
// Multiplicative so it composes with either scorer's scale (frequency sum or PageRank centrality).
|
|
277
|
+
function applySalienceBoost(sentences) {
|
|
278
|
+
for (const s of sentences)
|
|
279
|
+
if (CAUTION.test(s.text))
|
|
280
|
+
s.score *= SALIENCE_BOOST;
|
|
281
|
+
}
|
|
282
|
+
function scoreSentences(sentences, headingTerms) {
|
|
283
|
+
const n = sentences.length;
|
|
284
|
+
if (n >= TEXTRANK_MIN_SENTENCES && n <= TEXTRANK_MAX_SENTENCES) {
|
|
285
|
+
const scores = textRankScores(sentences.map((s) => new Set(contentTokens(s.text))));
|
|
286
|
+
sentences.forEach((s, i) => (s.score = scores[i]));
|
|
287
|
+
}
|
|
288
|
+
else {
|
|
289
|
+
const freq = buildFrequencies(sentences.map((s) => s.text).join(' '), headingTerms);
|
|
290
|
+
for (const s of sentences)
|
|
291
|
+
s.score = frequencyScore(s.text, freq);
|
|
292
|
+
}
|
|
293
|
+
applySalienceBoost(sentences);
|
|
294
|
+
}
|
|
295
|
+
function collectHeadingTerms(segments) {
|
|
296
|
+
const terms = new Set();
|
|
297
|
+
for (const seg of segments) {
|
|
298
|
+
if (seg.kind !== 'verbatim')
|
|
299
|
+
continue;
|
|
300
|
+
for (const line of seg.lines) {
|
|
301
|
+
if (isHeading(line))
|
|
302
|
+
for (const t of tokenize(line))
|
|
303
|
+
terms.add(t);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return terms;
|
|
307
|
+
}
|
|
308
|
+
function collectSentences(segments) {
|
|
309
|
+
const sentences = [];
|
|
310
|
+
let globalIdx = 0;
|
|
311
|
+
segments.forEach((seg, segIdx) => {
|
|
312
|
+
if (seg.kind !== 'prose')
|
|
313
|
+
return;
|
|
314
|
+
const paragraph = seg.lines.join(' ').replace(/\s+/g, ' ').trim();
|
|
315
|
+
splitSentences(paragraph).forEach((text, idxInSeg) => {
|
|
316
|
+
sentences.push({
|
|
317
|
+
segIdx,
|
|
318
|
+
globalIdx: globalIdx++,
|
|
319
|
+
text,
|
|
320
|
+
chars: text.length,
|
|
321
|
+
isFirst: idxInSeg === 0,
|
|
322
|
+
score: 0,
|
|
323
|
+
});
|
|
324
|
+
});
|
|
325
|
+
});
|
|
326
|
+
return sentences;
|
|
327
|
+
}
|
|
328
|
+
// Picks which sentences to keep: every paragraph's first sentence (and every sentence of a
|
|
329
|
+
// short paragraph) is anchored; the remaining budget is filled by score, highest first.
|
|
330
|
+
function selectKept(sentences, level, minSentences) {
|
|
331
|
+
const perSeg = new Map();
|
|
332
|
+
for (const s of sentences)
|
|
333
|
+
perSeg.set(s.segIdx, (perSeg.get(s.segIdx) ?? 0) + 1);
|
|
334
|
+
const kept = new Set();
|
|
335
|
+
let keptChars = 0;
|
|
336
|
+
const keep = (s) => {
|
|
337
|
+
if (kept.has(s.globalIdx))
|
|
338
|
+
return;
|
|
339
|
+
kept.add(s.globalIdx);
|
|
340
|
+
keptChars += s.chars;
|
|
341
|
+
};
|
|
342
|
+
for (const s of sentences) {
|
|
343
|
+
// Anchor every paragraph's first sentence and every sentence of a short paragraph, and never
|
|
344
|
+
// drop a critical caveat regardless of level — these are kept before any budget is spent.
|
|
345
|
+
if (s.isFirst || perSeg.get(s.segIdx) < minSentences || CRITICAL.test(s.text))
|
|
346
|
+
keep(s);
|
|
347
|
+
}
|
|
348
|
+
const totalChars = sentences.reduce((sum, s) => sum + s.chars, 0);
|
|
349
|
+
const target = totalChars * TARGET_RATIO[level];
|
|
350
|
+
const candidates = sentences
|
|
351
|
+
.filter((s) => !kept.has(s.globalIdx))
|
|
352
|
+
.sort((a, b) => b.score - a.score || a.globalIdx - b.globalIdx);
|
|
353
|
+
for (const s of candidates) {
|
|
354
|
+
if (keptChars >= target)
|
|
355
|
+
break;
|
|
356
|
+
keep(s);
|
|
357
|
+
}
|
|
358
|
+
return kept;
|
|
359
|
+
}
|
|
360
|
+
function reassemble(segments, sentences, kept) {
|
|
361
|
+
const bySeg = new Map();
|
|
362
|
+
for (const s of sentences) {
|
|
363
|
+
if (!bySeg.has(s.segIdx))
|
|
364
|
+
bySeg.set(s.segIdx, []);
|
|
365
|
+
bySeg.get(s.segIdx).push(s);
|
|
366
|
+
}
|
|
367
|
+
const out = segments.map((seg, segIdx) => {
|
|
368
|
+
if (seg.kind === 'verbatim')
|
|
369
|
+
return seg.lines.join('\n');
|
|
370
|
+
return (bySeg.get(segIdx) ?? [])
|
|
371
|
+
.filter((s) => kept.has(s.globalIdx))
|
|
372
|
+
.map((s) => s.text)
|
|
373
|
+
.join(' ');
|
|
374
|
+
});
|
|
375
|
+
return out
|
|
376
|
+
.join('\n')
|
|
377
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
378
|
+
.trim();
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Extractively shortens the prose paragraphs of a Markdown document while preserving headings,
|
|
382
|
+
* fenced code blocks, tables, and lists verbatim. Returns the input unchanged when it is below
|
|
383
|
+
* `minTokens` or contains no prose to condense. Deterministic for identical input + options.
|
|
384
|
+
*/
|
|
385
|
+
export function summarizeMarkdown(markdown, options = {}) {
|
|
386
|
+
const level = options.level ?? 'conservative';
|
|
387
|
+
const minTokens = options.minTokens ?? DEFAULT_MIN_TOKENS;
|
|
388
|
+
const minSentences = options.minSentences ?? DEFAULT_MIN_SENTENCES;
|
|
389
|
+
if (!markdown || !markdown.trim())
|
|
390
|
+
return markdown;
|
|
391
|
+
if (estimateTokens(markdown) < minTokens)
|
|
392
|
+
return markdown;
|
|
393
|
+
const segments = segment(markdown);
|
|
394
|
+
const sentences = collectSentences(segments);
|
|
395
|
+
if (sentences.length === 0)
|
|
396
|
+
return markdown;
|
|
397
|
+
scoreSentences(sentences, collectHeadingTerms(segments));
|
|
398
|
+
const kept = selectKept(sentences, level, minSentences);
|
|
399
|
+
return reassemble(segments, sentences, kept);
|
|
400
|
+
}
|
package/dist/lib/research/url.js
CHANGED
|
@@ -92,8 +92,11 @@ export function normalizeUrl(input) {
|
|
|
92
92
|
try {
|
|
93
93
|
url = new URL(input);
|
|
94
94
|
}
|
|
95
|
-
catch
|
|
96
|
-
|
|
95
|
+
catch {
|
|
96
|
+
// new URL() only ever throws the constant "Invalid URL" (no structured cause worth keeping),
|
|
97
|
+
// and every caller prefixes its own "Invalid URL:" — forwarding the native text would stutter.
|
|
98
|
+
// Drop the binding and emit an actionable, self-contained reason instead.
|
|
99
|
+
throw new Error(`Could not parse "${input}" as a URL; provide a full http(s) URL like https://example.com/page.`);
|
|
97
100
|
}
|
|
98
101
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
99
102
|
throw new Error(`Unsupported protocol "${url.protocol}". Only http: and https: are supported.`);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pick the singular or plural noun for a count so human-readable output stays grammatical
|
|
3
|
+
* ("1 entry" / "2 entries"). English has irregular plurals, so the plural is passed explicitly
|
|
4
|
+
* rather than derived.
|
|
5
|
+
*/
|
|
6
|
+
export declare function pluralize(count: number, singular: string, plural: string): string;
|
|
7
|
+
/** How a command labels a result listing, so its heading and truncation notice stay consistent. */
|
|
8
|
+
export interface ResultListLabels {
|
|
9
|
+
/** Noun phrase after the count, e.g. "cached research"; the "entry/entries" plural is appended. */
|
|
10
|
+
noun: string;
|
|
11
|
+
/** Ordering word shown when truncated: "first" (recency order) or "top" (ranked order). */
|
|
12
|
+
order: 'first' | 'top';
|
|
13
|
+
/** The command's --limit ceiling, surfaced in the truncation hint. */
|
|
14
|
+
maxLimit: number;
|
|
15
|
+
}
|
|
16
|
+
/** Heading for a result listing; notes truncation inline when more matched than are shown. */
|
|
17
|
+
export declare function resultListHeading(total: number, shown: number, labels: ResultListLabels): string;
|
|
18
|
+
/**
|
|
19
|
+
* One-line truncation notice for stderr, or null when nothing was cut. Used under --json, where the
|
|
20
|
+
* stdout heading is suppressed and an agent would otherwise get no signal that results were capped.
|
|
21
|
+
*/
|
|
22
|
+
export declare function truncationNotice(total: number, shown: number, labels: ResultListLabels): string | null;
|
|
23
|
+
/**
|
|
24
|
+
* Levenshtein edit distance: the minimum number of single-character insertions, deletions, or
|
|
25
|
+
* substitutions to turn `s1` into `s2`. Shared by fuzzy search ranking and command-typo suggestion.
|
|
26
|
+
*/
|
|
27
|
+
export declare function levenshtein(s1: string, s2: string): number;
|
|
28
|
+
/**
|
|
29
|
+
* The candidate nearest to `input` by edit distance, or null when even the closest is farther than
|
|
30
|
+
* `maxDistance`. The threshold keeps suggestions to plausible typos — input that resembles nothing
|
|
31
|
+
* gets no correction rather than an absurd one.
|
|
32
|
+
*/
|
|
33
|
+
export declare function closestMatch(input: string, candidates: readonly string[], maxDistance: number): string | null;
|
package/dist/lib/text.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pick the singular or plural noun for a count so human-readable output stays grammatical
|
|
3
|
+
* ("1 entry" / "2 entries"). English has irregular plurals, so the plural is passed explicitly
|
|
4
|
+
* rather than derived.
|
|
5
|
+
*/
|
|
6
|
+
export function pluralize(count, singular, plural) {
|
|
7
|
+
return count === 1 ? singular : plural;
|
|
8
|
+
}
|
|
9
|
+
/** Heading for a result listing; notes truncation inline when more matched than are shown. */
|
|
10
|
+
export function resultListHeading(total, shown, labels) {
|
|
11
|
+
const noun = `${labels.noun} ${pluralize(total, 'entry', 'entries')}`;
|
|
12
|
+
return total > shown
|
|
13
|
+
? `Found ${total} ${noun} (showing ${labels.order} ${shown}; raise --limit to see more):`
|
|
14
|
+
: `Found ${total} ${noun}:`;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* One-line truncation notice for stderr, or null when nothing was cut. Used under --json, where the
|
|
18
|
+
* stdout heading is suppressed and an agent would otherwise get no signal that results were capped.
|
|
19
|
+
*/
|
|
20
|
+
export function truncationNotice(total, shown, labels) {
|
|
21
|
+
if (total <= shown)
|
|
22
|
+
return null;
|
|
23
|
+
return `${total} entries matched; returning the ${labels.order} ${shown}. Raise --limit (max ${labels.maxLimit}) to see more.`;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Levenshtein edit distance: the minimum number of single-character insertions, deletions, or
|
|
27
|
+
* substitutions to turn `s1` into `s2`. Shared by fuzzy search ranking and command-typo suggestion.
|
|
28
|
+
*/
|
|
29
|
+
export function levenshtein(s1, s2) {
|
|
30
|
+
if (s1.length < s2.length)
|
|
31
|
+
return levenshtein(s2, s1);
|
|
32
|
+
if (s2.length === 0)
|
|
33
|
+
return s1.length;
|
|
34
|
+
let prevRow = Array.from({ length: s2.length + 1 }, (_, i) => i);
|
|
35
|
+
for (let i = 0; i < s1.length; i++) {
|
|
36
|
+
const currRow = [i + 1];
|
|
37
|
+
for (let j = 0; j < s2.length; j++) {
|
|
38
|
+
const deletions = prevRow[j + 1] + 1;
|
|
39
|
+
const insertions = currRow[j] + 1;
|
|
40
|
+
const substitutions = prevRow[j] + (s1[i] === s2[j] ? 0 : 1);
|
|
41
|
+
currRow.push(Math.min(deletions, insertions, substitutions));
|
|
42
|
+
}
|
|
43
|
+
prevRow = currRow;
|
|
44
|
+
}
|
|
45
|
+
return prevRow[s2.length];
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The candidate nearest to `input` by edit distance, or null when even the closest is farther than
|
|
49
|
+
* `maxDistance`. The threshold keeps suggestions to plausible typos — input that resembles nothing
|
|
50
|
+
* gets no correction rather than an absurd one.
|
|
51
|
+
*/
|
|
52
|
+
export function closestMatch(input, candidates, maxDistance) {
|
|
53
|
+
let best = null;
|
|
54
|
+
let bestDistance = Infinity;
|
|
55
|
+
for (const candidate of candidates) {
|
|
56
|
+
const distance = levenshtein(input, candidate);
|
|
57
|
+
if (distance < bestDistance) {
|
|
58
|
+
bestDistance = distance;
|
|
59
|
+
best = candidate;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return bestDistance <= maxDistance ? best : null;
|
|
63
|
+
}
|
package/oclif.manifest.json
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"required": true
|
|
10
10
|
}
|
|
11
11
|
},
|
|
12
|
-
"description": "Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules
|
|
12
|
+
"description": "Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.\n\nUsually invoked via the shorthand `bonsai <url>` rather than `bonsai fetch <url>`.",
|
|
13
13
|
"examples": [
|
|
14
14
|
{
|
|
15
15
|
"description": "Research a URL with detailed output, tagged with topic and tags",
|
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
},
|
|
73
73
|
"ttl": {
|
|
74
74
|
"char": "l",
|
|
75
|
-
"description": "Predicted lifespan
|
|
75
|
+
"description": "Predicted lifespan: number + h/d/w/m/y (m = months), e.g. \"2h\", \"7d\", \"6m\".",
|
|
76
76
|
"name": "ttl",
|
|
77
77
|
"hasDynamicHelp": false,
|
|
78
78
|
"multiple": false,
|
|
@@ -525,7 +525,7 @@
|
|
|
525
525
|
"type": "option"
|
|
526
526
|
},
|
|
527
527
|
"remote": {
|
|
528
|
-
"description": "Discover uncached docs via a site’s public search (Algolia DocSearch, MkDocs/Sphinx/Just-the-Docs
|
|
528
|
+
"description": "Discover uncached docs via a site’s public search or index (Algolia DocSearch, MkDocs/Sphinx/Just-the-Docs, llms.txt, sitemap.xml). Pass a docs page URL; falls back to local cache search if no connector applies.",
|
|
529
529
|
"name": "remote",
|
|
530
530
|
"hasDynamicHelp": false,
|
|
531
531
|
"multiple": false,
|
|
@@ -610,7 +610,7 @@
|
|
|
610
610
|
"required": true
|
|
611
611
|
}
|
|
612
612
|
},
|
|
613
|
-
"description": "Print a config value. Without --global/--local, shows the merged effective value (flag > env > project > user > default).\n\nValid keys: storage.",
|
|
613
|
+
"description": "Print a config value. Without --global/--local, shows the merged effective value (flag > env > project > user > default).\n\nValid keys: storage, summary.",
|
|
614
614
|
"examples": [
|
|
615
615
|
{
|
|
616
616
|
"description": "Get the effective storage mode",
|
|
@@ -662,7 +662,7 @@
|
|
|
662
662
|
"config:list": {
|
|
663
663
|
"aliases": [],
|
|
664
664
|
"args": {},
|
|
665
|
-
"description": "Show every configuration key with its current value. Use --global/--local to show a single scope.\n\nValid keys: storage.",
|
|
665
|
+
"description": "Show every configuration key with its current value. Use --global/--local to show a single scope.\n\nValid keys: storage, summary.",
|
|
666
666
|
"examples": [
|
|
667
667
|
{
|
|
668
668
|
"description": "Show all effective config values",
|
|
@@ -725,7 +725,7 @@
|
|
|
725
725
|
"required": false
|
|
726
726
|
}
|
|
727
727
|
},
|
|
728
|
-
"description": "Persist a configuration value. Writes user-level config by default; pass --local to write the project-level config (.bonsai.json in cwd).\n\nValid keys: storage. The inline form `<key>=<value>` is also accepted.",
|
|
728
|
+
"description": "Persist a configuration value. Writes user-level config by default; pass --local to write the project-level config (.bonsai.json in cwd).\n\nValid keys: storage, summary. The inline form `<key>=<value>` is also accepted.",
|
|
729
729
|
"examples": [
|
|
730
730
|
{
|
|
731
731
|
"description": "Store research cache inside the current project",
|
|
@@ -789,7 +789,7 @@
|
|
|
789
789
|
"required": true
|
|
790
790
|
}
|
|
791
791
|
},
|
|
792
|
-
"description": "Delete a key from user-level config (default) or project-level config (--local), restoring the built-in default.\n\nValid keys: storage.",
|
|
792
|
+
"description": "Delete a key from user-level config (default) or project-level config (--local), restoring the built-in default.\n\nValid keys: storage, summary.",
|
|
793
793
|
"examples": [
|
|
794
794
|
{
|
|
795
795
|
"description": "Remove the user-level storage setting",
|
|
@@ -845,5 +845,5 @@
|
|
|
845
845
|
"stdoutIsPrimaryData": true
|
|
846
846
|
}
|
|
847
847
|
},
|
|
848
|
-
"version": "1.0.
|
|
848
|
+
"version": "1.0.5"
|
|
849
849
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@taurgis/bonsai",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.5",
|
|
4
4
|
"description": "Bonsai is a standalone local research cache CLI for AI agents",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "git+https://github.com/taurgis/bonsai.git"
|
|
8
|
+
},
|
|
9
|
+
"homepage": "https://bonsai.rhino-inquisitor.com/",
|
|
5
10
|
"type": "module",
|
|
6
11
|
"main": "./dist/index.js",
|
|
7
12
|
"bin": {
|
|
@@ -29,7 +34,10 @@
|
|
|
29
34
|
"nexus:sync": "npx forward-nexus update && npx forward-nexus sync",
|
|
30
35
|
"test": "vitest",
|
|
31
36
|
"test:coverage": "vitest run --coverage",
|
|
32
|
-
"type-check": "tsc --noEmit"
|
|
37
|
+
"type-check": "tsc --noEmit",
|
|
38
|
+
"docs:dev": "vitepress dev docs",
|
|
39
|
+
"docs:build": "vitepress build docs",
|
|
40
|
+
"docs:preview": "vitepress preview docs"
|
|
33
41
|
},
|
|
34
42
|
"lint-staged": {
|
|
35
43
|
"src/**/*.ts": "prettier --write",
|
|
@@ -55,7 +63,9 @@
|
|
|
55
63
|
"oclif": "^4.23.21",
|
|
56
64
|
"prettier": "^3.8.1",
|
|
57
65
|
"typescript": "^5.9.3",
|
|
58
|
-
"
|
|
66
|
+
"vitepress": "^1.6.4",
|
|
67
|
+
"vitest": "^4.0.17",
|
|
68
|
+
"vue": "^3.5.39"
|
|
59
69
|
},
|
|
60
70
|
"engines": {
|
|
61
71
|
"node": ">=22"
|
|
@@ -64,12 +74,16 @@
|
|
|
64
74
|
"oclif": {
|
|
65
75
|
"bin": "bonsai",
|
|
66
76
|
"dirname": "bonsai",
|
|
77
|
+
"description": "Bonsai is a standalone local research cache CLI for AI agents.\nResearch and cache a web page by passing its URL directly:\n\n $ bonsai https://nodejs.org/api/url.html\n\nThe URL form is shorthand for fetching and caching the page. The commands listed below search, inspect, list, import, and prune the local research cache.",
|
|
67
78
|
"commands": {
|
|
68
79
|
"strategy": "explicit",
|
|
69
80
|
"target": "./dist/commands.js",
|
|
70
81
|
"identifier": "commands"
|
|
71
82
|
},
|
|
72
83
|
"topicSeparator": " ",
|
|
84
|
+
"hooks": {
|
|
85
|
+
"command_not_found": "./dist/hooks/command-not-found/suggest.js"
|
|
86
|
+
},
|
|
73
87
|
"topics": {
|
|
74
88
|
"config": {
|
|
75
89
|
"description": "Manage research cache storage configuration (global vs project)"
|