codemini-cli 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,178 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { getProjectSkillsDir, getSkillsDir } from './paths.js';
4
+ import { createChatCompletion } from './provider/index.js';
5
+
6
+ const REFLECT_TIMEOUT_MS = 45000;
7
+
8
+ function slugifySkillName(value) {
9
+ const slug = String(value || '')
10
+ .trim()
11
+ .toLowerCase()
12
+ .replace(/[^a-z0-9\u4e00-\u9fa5]+/g, '-')
13
+ .replace(/^-+|-+$/g, '');
14
+ return slug || 'reflected-success-workflow';
15
+ }
16
+
17
+ function escapeFrontmatter(value) {
18
+ return String(value || '').replace(/\r?\n/g, ' ').replace(/"/g, '\\"').trim();
19
+ }
20
+
21
+ function hasFrontmatter(content) {
22
+ return /^---\r?\n[\s\S]*?\r?\n---\r?\n/.test(String(content || '').trimStart());
23
+ }
24
+
25
+ function renderSkillContent({ name, description, content }) {
26
+ const body = String(content || '').trim() || [
27
+ '## Workflow',
28
+ '',
29
+ '1. Recreate the successful chain from the recent task.',
30
+ '2. Preserve the key decision that made it work.',
31
+ '3. Verify with the narrowest relevant check.',
32
+ '',
33
+ '## Boundaries',
34
+ '',
35
+ 'Use this only when the current task matches the preserved workflow.'
36
+ ].join('\n');
37
+ if (hasFrontmatter(body)) return `${body.trim()}\n`;
38
+ return [
39
+ '---',
40
+ `name: ${name}`,
41
+ `description: ${escapeFrontmatter(description) || `Use when this reflected workflow applies.`}`,
42
+ '---',
43
+ '',
44
+ body
45
+ ].join('\n').trimEnd() + '\n';
46
+ }
47
+
48
+ export function normalizeReflectDraft(raw = {}) {
49
+ const name = slugifySkillName(raw.name || raw.skillName || raw.title);
50
+ const description = String(raw.description || raw.summary || `Use when the ${name} workflow applies.`).trim();
51
+ const confidence = Math.min(1, Math.max(0, Number(raw.confidence ?? 0.75)));
52
+ return {
53
+ id: Number(raw.id || 1),
54
+ name,
55
+ description,
56
+ confidence,
57
+ content: renderSkillContent({ name, description, content: raw.content || raw.markdown || raw.body })
58
+ };
59
+ }
60
+
61
+ export function buildReflectTargetPath({ scope = 'project', name, workspaceRoot = process.cwd() } = {}) {
62
+ const safeName = slugifySkillName(name);
63
+ const baseDir = String(scope || '').toLowerCase() === 'global'
64
+ ? getSkillsDir()
65
+ : getProjectSkillsDir(workspaceRoot);
66
+ return path.join(baseDir, safeName, 'SKILL.md');
67
+ }
68
+
69
+ export function parseReflectScope(args = []) {
70
+ let scope = 'project';
71
+ const requestParts = [];
72
+ for (let index = 0; index < args.length; index += 1) {
73
+ const arg = String(args[index] || '');
74
+ if (arg === '--scope') {
75
+ const next = String(args[index + 1] || '').toLowerCase();
76
+ if (next === 'global' || next === 'project') {
77
+ scope = next;
78
+ index += 1;
79
+ }
80
+ continue;
81
+ }
82
+ if (arg.startsWith('--scope=')) {
83
+ const value = arg.slice('--scope='.length).toLowerCase();
84
+ if (value === 'global' || value === 'project') scope = value;
85
+ continue;
86
+ }
87
+ requestParts.push(arg);
88
+ }
89
+ return { scope, request: requestParts.join(' ').trim() };
90
+ }
91
+
92
+ function parseModelDrafts(text) {
93
+ const raw = String(text || '').trim();
94
+ if (!raw) return [];
95
+ const unfenced = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/i, '').trim();
96
+ try {
97
+ const parsed = JSON.parse(unfenced);
98
+ if (Array.isArray(parsed?.candidates)) return parsed.candidates.map((item, index) => normalizeReflectDraft({ id: index + 1, ...item }));
99
+ if (Array.isArray(parsed)) return parsed.map((item, index) => normalizeReflectDraft({ id: index + 1, ...item }));
100
+ if (parsed && typeof parsed === 'object') return [normalizeReflectDraft(parsed)];
101
+ } catch {
102
+ // Fall back to wrapping plain markdown below.
103
+ }
104
+ return [normalizeReflectDraft({
105
+ name: 'reflected-success-workflow',
106
+ description: 'Use when the reflected successful workflow applies.',
107
+ content: raw
108
+ })];
109
+ }
110
+
111
+ function recentContext(session, limit = 10) {
112
+ const messages = Array.isArray(session?.messages) ? session.messages : [];
113
+ return messages
114
+ .slice(-limit)
115
+ .map((message) => `${message.role}: ${String(message.content || '').slice(0, 1200)}`)
116
+ .join('\n\n');
117
+ }
118
+
119
+ export async function buildReflectSkillDraft({
120
+ request = '',
121
+ scope = 'project',
122
+ session,
123
+ config = {},
124
+ model,
125
+ systemPrompt = '',
126
+ previousDraft = null,
127
+ feedback = ''
128
+ } = {}) {
129
+ const mode = String(request || '').trim() ? 'directed' : 'exploratory';
130
+ const prompt = [
131
+ 'Create a reusable Codex/CodeMini SKILL.md draft from a successful workflow.',
132
+ `Mode: ${mode}`,
133
+ `Target scope: ${scope}`,
134
+ request ? `User reflection request:\n${request}` : 'No explicit request was supplied. Be conservative and return no candidates if the recent context does not show a reusable success pattern.',
135
+ previousDraft ? `Existing draft to revise:\n${previousDraft.content || ''}` : '',
136
+ feedback ? `User edit feedback:\n${feedback}` : '',
137
+ 'Recent session context:',
138
+ recentContext(session),
139
+ 'Return valid JSON only, no markdown fences.',
140
+ 'Shape: {"candidates":[{"name":"kebab-case-name","description":"when to use this skill","confidence":0.0,"content":"full SKILL.md body or markdown body"}]}',
141
+ 'The content must include trigger conditions, workflow/toolchain, key decisions, pitfalls, verification, and boundaries.',
142
+ 'Do not write memory or inbox content. This is only a skill draft.'
143
+ ].filter(Boolean).join('\n\n');
144
+
145
+ const result = await createChatCompletion({
146
+ sdkProvider: config?.sdk?.provider,
147
+ baseUrl: config?.gateway?.base_url,
148
+ apiKey: config?.gateway?.api_key,
149
+ model: model || config?.model?.name,
150
+ messages: [
151
+ { role: 'system', content: systemPrompt || 'You draft concise, reusable coding workflow skills.' },
152
+ { role: 'user', content: prompt }
153
+ ],
154
+ temperature: 0,
155
+ timeoutMs: REFLECT_TIMEOUT_MS
156
+ });
157
+
158
+ return parseModelDrafts(result?.text || '');
159
+ }
160
+
161
+ export function attachReflectTargets({ candidates = [], scope = 'project', workspaceRoot = process.cwd() } = {}) {
162
+ return candidates.map((candidate, index) => {
163
+ const draft = normalizeReflectDraft({ id: index + 1, ...candidate });
164
+ return {
165
+ ...draft,
166
+ targetPath: buildReflectTargetPath({ scope, name: draft.name, workspaceRoot })
167
+ };
168
+ });
169
+ }
170
+
171
+ export async function writeReflectSkillDraft({ draft, scope = 'project', workspaceRoot = process.cwd() } = {}) {
172
+ const normalized = normalizeReflectDraft(draft);
173
+ const filePath = buildReflectTargetPath({ scope, name: normalized.name, workspaceRoot });
174
+ await fs.mkdir(path.dirname(filePath), { recursive: true });
175
+ await fs.writeFile(filePath, normalized.content, 'utf8');
176
+ return { filePath, draft: normalized };
177
+ }
178
+
@@ -0,0 +1,206 @@
1
+ import fs from 'node:fs/promises';
2
+ import os from 'node:os';
3
+ import path from 'node:path';
4
+ import { BoundedCache } from './bounded-cache.js';
5
+ import { trimInline } from './string-utils.js';
6
+
7
+ const TOOL_RESULT_DISK_THRESHOLD = 6000;
8
+ const PREVIEW_SIZE_BYTES = 2000;
9
+ const TOOL_RESULTS_SUBDIR = 'tool-results';
10
+
11
+ let currentResultDir = null;
12
+ let resultDirReady = false;
13
+
14
+ const storedResults = new BoundedCache({
15
+ maxSize: 64,
16
+ ttlMs: 30 * 60 * 1000,
17
+ onEvict(_key, value) {
18
+ if (value?.filePath) {
19
+ fs.unlink(value.filePath).catch(() => {});
20
+ }
21
+ }
22
+ });
23
+
24
+ const readCache = new BoundedCache({ maxSize: 128, ttlMs: 10 * 60 * 1000 });
25
+
26
+ function generatePreview(content) {
27
+ if (content.length <= PREVIEW_SIZE_BYTES) {
28
+ return { preview: content, hasMore: false };
29
+ }
30
+ const truncated = content.slice(0, PREVIEW_SIZE_BYTES);
31
+ const lastNewline = truncated.lastIndexOf('\n');
32
+ const cutPoint = lastNewline > PREVIEW_SIZE_BYTES * 0.5 ? lastNewline : PREVIEW_SIZE_BYTES;
33
+ return { preview: content.slice(0, cutPoint), hasMore: true };
34
+ }
35
+
36
+ function formatFileSize(chars) {
37
+ if (chars < 1024) return `${chars} B`;
38
+ return `${(chars / 1024).toFixed(1)} KB`;
39
+ }
40
+
41
+ export function setResultDir(dir) {
42
+ currentResultDir = dir ? path.join(dir, TOOL_RESULTS_SUBDIR) : null;
43
+ resultDirReady = false;
44
+ }
45
+
46
+ async function ensureResultDir() {
47
+ if (!currentResultDir) return false;
48
+ if (!resultDirReady) {
49
+ await fs.mkdir(currentResultDir, { recursive: true });
50
+ resultDirReady = true;
51
+ }
52
+ return true;
53
+ }
54
+
55
+ export async function storeResultIfNeeded(callId, formattedContent, rawResult) {
56
+ if (formattedContent.length <= TOOL_RESULT_DISK_THRESHOLD) {
57
+ return formattedContent;
58
+ }
59
+ try {
60
+ const ready = await ensureResultDir();
61
+ const dir = ready ? currentResultDir : path.join(os.tmpdir(), 'codemini-results');
62
+ if (!resultDirReady && dir === currentResultDir) {
63
+ await fs.mkdir(dir, { recursive: true });
64
+ } else if (!resultDirReady) {
65
+ await fs.mkdir(dir, { recursive: true });
66
+ }
67
+ const filePath = path.join(dir, `${callId}.txt`);
68
+ const payload = typeof rawResult === 'string' ? rawResult : JSON.stringify(rawResult, null, 2);
69
+ await fs.writeFile(filePath, payload, 'utf-8');
70
+ const summary = summarizeToolResult(rawResult);
71
+ const { preview, hasMore } = generatePreview(payload);
72
+ storedResults.set(callId, { filePath, summary });
73
+
74
+ return `<persisted-output>
75
+ Output too large (${formatFileSize(payload.length)}). Full output saved to: ${filePath}
76
+
77
+ Preview (first ${formatFileSize(PREVIEW_SIZE_BYTES)}):
78
+ ${preview}${hasMore ? '\n...' : ''}
79
+
80
+ Summary: ${summary}
81
+ </persisted-output>`;
82
+ } catch {
83
+ return formattedContent;
84
+ }
85
+ }
86
+
87
+ export function clearResultStore() {
88
+ const files = [];
89
+ for (const [, val] of storedResults.entries()) {
90
+ files.push(val.filePath);
91
+ }
92
+ storedResults.clear();
93
+ readCache.clear();
94
+ return Promise.allSettled(files.map((filePath) => fs.unlink(filePath).catch(() => {})));
95
+ }
96
+
97
+ export function checkReadDedup(filePath, startLine, endLine, mtimeMs) {
98
+ const key = `${filePath}:${startLine || 0}:${endLine || 0}:${mtimeMs}`;
99
+ if (readCache.has(key)) {
100
+ return true;
101
+ }
102
+ readCache.set(key, true);
103
+ return false;
104
+ }
105
+
106
+ export function summarizeToolResult(result) {
107
+ if (result === null || result === undefined) return 'no output';
108
+ if (typeof result === 'string') {
109
+ const oneLine = result.replace(/\s+/g, ' ').trim();
110
+ return oneLine.length > 90 ? `${oneLine.slice(0, 87)}...` : oneLine || 'empty string';
111
+ }
112
+ if (typeof result === 'object') {
113
+ const obj = result;
114
+ if (Array.isArray(obj)) return `array(${obj.length})`;
115
+ if ('deleted' in obj && 'path' in obj) {
116
+ const kind = trimInline(obj.type || 'item', 16);
117
+ const target = trimInline(obj.path || '', 96);
118
+ if (obj.deleted) return target ? `deleted ${kind} ${target}` : `deleted ${kind}`;
119
+ if (obj.cancelled) return target ? `cancelled delete ${target}` : 'cancelled delete';
120
+ }
121
+ if ('path' in obj && 'action' in obj) {
122
+ const p = String(obj.path || '');
123
+ const action = String(obj.action || 'write');
124
+ const line = Number(obj.changed_line || 1);
125
+ const suffix =
126
+ action === 'delete'
127
+ ? 'deleted'
128
+ : action === 'create'
129
+ ? 'created'
130
+ : action === 'patch'
131
+ ? 'patched'
132
+ : action === 'replace_block' || action === 'replace_text'
133
+ ? 'edited'
134
+ : action === 'append'
135
+ ? 'appended'
136
+ : 'updated';
137
+ return p ? `${suffix} ${p}${line > 0 ? ` @L${line}` : ''}` : suffix;
138
+ }
139
+ if ('path' in obj && 'phase' in obj) {
140
+ const phase = String(obj.phase || '');
141
+ const p = String(obj.path || '');
142
+ const total = Number(obj.total_lines);
143
+ const start =
144
+ Number(obj.suggested_start_line || obj.start_line) > 0
145
+ ? Number(obj.suggested_start_line || obj.start_line)
146
+ : 1;
147
+ const end =
148
+ Number(obj.suggested_end_line || obj.end_line) >= start
149
+ ? Number(obj.suggested_end_line || obj.end_line)
150
+ : start;
151
+ const rangeText = start > 0 && end >= start ? ` lines ${start}-${end}` : '';
152
+ const totalText = total > 0 ? ` of ${total}` : '';
153
+ const enclosingText = obj.enclosing_symbol ? ` in ${obj.enclosing_symbol}` : '';
154
+ const errorText = obj.error ? ` (${trimInline(obj.error, 64)})` : '';
155
+ const truncatedText = obj.truncated ? ' [truncated]' : '';
156
+ return phase === 'metadata'
157
+ ? `metadata for ${p}${rangeText}${totalText}${errorText}`
158
+ : `content from ${p}${rangeText}${totalText}${enclosingText}${truncatedText}`;
159
+ }
160
+ if ('stdout' in obj || 'stderr' in obj || 'code' in obj) {
161
+ const stdout = trimInline(obj.stdout || '', 96);
162
+ const stderr = trimInline(obj.stderr || '', 96);
163
+ const command = trimInline(obj.command || '', 72);
164
+ const lead = command ? `${command} -> ` : '';
165
+ if (stdout) return `${lead}exit ${obj.code ?? 0}\nstdout: ${stdout}`;
166
+ if (stderr) return `${lead}exit ${obj.code ?? 0}\nstderr: ${stderr}`;
167
+ return `${lead}exit ${obj.code ?? 0}`;
168
+ }
169
+ if ('task_id' in obj && 'startup_confirmed' in obj) {
170
+ const status = trimInline(obj.status || 'unknown', 32);
171
+ const taskId = trimInline(obj.task_id || '', 24);
172
+ const source = trimInline(obj.startup_source || '', 24);
173
+ const outputFile = trimInline(obj.output_file || '', 72);
174
+ const output = Array.isArray(obj.recent_output) ? trimInline(obj.recent_output.slice(-1)[0] || '', 96) : '';
175
+ return `${taskId || 'task'} ${status}${source ? ` (${source})` : ''}${outputFile ? ` -> ${outputFile}` : ''}${output ? `\n${output}` : ''}`;
176
+ }
177
+ if ('tasks' in obj && Array.isArray(obj.tasks)) {
178
+ const count = obj.tasks.length;
179
+ const first = obj.tasks[0];
180
+ const lead = first?.task_id ? `${trimInline(first.task_id, 24)} ${trimInline(first.status || 'unknown', 24)}` : '';
181
+ return `tasks(${count})${lead ? `\n${lead}` : ''}`;
182
+ }
183
+ if ('files' in obj && Array.isArray(obj.files)) {
184
+ return `patched ${obj.files.length} file(s)`;
185
+ }
186
+ if ('diff' in obj && 'new_hash' in obj && 'path' in obj) {
187
+ const p = String(obj.path || '');
188
+ return p ? `diff preview for ${p}` : 'diff preview';
189
+ }
190
+ if ('created' in obj && Array.isArray(obj.created)) {
191
+ return `created ${obj.created.length} task(s)`;
192
+ }
193
+ if ('tasks' in obj && Array.isArray(obj.tasks)) {
194
+ return `${obj.tasks.length} task(s)`;
195
+ }
196
+ if ('newTodos' in obj && Array.isArray(obj.newTodos)) {
197
+ return obj.newTodos.length > 0 ? `updated ${obj.newTodos.length} todo item(s)` : 'cleared todo list';
198
+ }
199
+ if ('newPlan' in obj) {
200
+ return obj.newPlan ? `updated plan state (${String(obj.newPlan.status || 'draft')})` : 'cleared plan state';
201
+ }
202
+ const keys = Object.keys(obj);
203
+ return keys.length > 0 ? `keys: ${keys.slice(0, 5).join(',')}` : 'object';
204
+ }
205
+ return String(result);
206
+ }
package/src/core/tools.js CHANGED
@@ -15,7 +15,7 @@ import {
15
15
  import { evaluateCommandPolicy } from './command-policy.js';
16
16
  import { findEnclosingSymbol, queryAst, readAstNode, resolveAstTarget } from './ast.js';
17
17
  import { initializeProjectIndex, queryProjectIndex, refreshIndexedFile } from './project-index.js';
18
- import { checkReadDedup } from './agent-loop.js';
18
+ import { checkReadDedup } from './tool-result-store.js';
19
19
  import { TOOL_SKIP_DIRS as SKIP_DIRS, TEXT_EXTENSIONS, CODE_WRITE_GUARD_EXTENSIONS, LANGUAGE_FILE_TYPES } from './constants.js';
20
20
  import { sha256Prefixed as sha256, sha256 as sha256Hash } from './crypto-utils.js';
21
21
  import { forgetMemory, listMemories, rememberMemory, searchMemories, captureToInbox } from './memory-store.js';
@@ -172,6 +172,55 @@ function collectPageLinks($, pageUrl, maxLinks = 20) {
172
172
  return links;
173
173
  }
174
174
 
175
+ function extractPageContent(cheerio, html, pageUrl, { maxLinks, status = null, contentType = '', fetchMode = 'static' } = {}) {
176
+ const $ = cheerio.load(html);
177
+ $('script, style, noscript').remove();
178
+ const bodyText = $('body').text() || $.root().text();
179
+ const text = String(bodyText || '').replace(/[^\S\n]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
180
+ const title = trimPreview($('title').first().text(), 240);
181
+ const description = extractHtmlMeta($, 'description') || extractHtmlMeta($, 'og:description');
182
+ const links = collectPageLinks($, pageUrl, maxLinks);
183
+
184
+ return {
185
+ final_url: pageUrl,
186
+ title,
187
+ description,
188
+ text,
189
+ links,
190
+ metadata: {
191
+ status,
192
+ fetched_at: new Date().toISOString(),
193
+ content_type: contentType,
194
+ fetch_mode: fetchMode,
195
+ lang: String($('html').attr('lang') || '').trim()
196
+ }
197
+ };
198
+ }
199
+
200
+ function shouldTryBrowserRender(html, text) {
201
+ if (String(text || '').trim().length >= 120) return false;
202
+ return /<script\b/i.test(html) ||
203
+ /id=["']__(?:next|nuxt)["']/i.test(html) ||
204
+ /data-reactroot|ng-version|window\.__/i.test(html);
205
+ }
206
+
207
+ function playwrightInstallHint() {
208
+ return 'For JavaScript-rendered pages, install Playwright for richer web_fetch results: npm install -g playwright && playwright install chromium';
209
+ }
210
+
211
+ async function loadOptionalPlaywright() {
212
+ try {
213
+ return await import('playwright');
214
+ } catch (error) {
215
+ const code = String(error?.code || '');
216
+ const message = String(error?.message || '');
217
+ if (code === 'ERR_MODULE_NOT_FOUND' || /Cannot find package 'playwright'|Cannot find module 'playwright'/i.test(message)) {
218
+ return null;
219
+ }
220
+ throw error;
221
+ }
222
+ }
223
+
175
224
  async function buildPlaywrightLaunchEnv() {
176
225
  const localLibDir = path.join(
177
226
  process.env.HOME || '',
@@ -204,44 +253,85 @@ async function webFetchPage(args = {}) {
204
253
  ? String(normalizedArgs.wait_until).trim()
205
254
  : 'domcontentloaded';
206
255
 
207
- const [{ chromium }, cheerio] = await Promise.all([import('playwright'), import('cheerio')]);
256
+ const cheerio = await import('cheerio');
257
+ let staticResult = null;
258
+ let staticHtml = '';
259
+ let staticError = null;
260
+ try {
261
+ const controller = new AbortController();
262
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
263
+ let response;
264
+ try {
265
+ response = await fetch(url, {
266
+ redirect: 'follow',
267
+ signal: controller.signal,
268
+ headers: {
269
+ 'user-agent': 'CodeMiniCLI/0.4 web_fetch'
270
+ }
271
+ });
272
+ } finally {
273
+ clearTimeout(timeout);
274
+ }
275
+ staticHtml = await response.text();
276
+ staticResult = {
277
+ url,
278
+ ...extractPageContent(cheerio, staticHtml, response.url || url, {
279
+ maxLinks,
280
+ status: response.status,
281
+ contentType: response.headers.get('content-type') || '',
282
+ fetchMode: 'static'
283
+ })
284
+ };
285
+ if (!shouldTryBrowserRender(staticHtml, staticResult.text)) {
286
+ return staticResult;
287
+ }
288
+ } catch (error) {
289
+ staticError = error;
290
+ }
208
291
 
209
- // Crawlee is intentionally disabled for now so single-page reads stay lightweight.
210
- // If we later need multi-URL crawl orchestration, retries, or request queues, we can re-enable it here.
292
+ const playwright = await loadOptionalPlaywright();
293
+ if (!playwright) {
294
+ if (staticResult) {
295
+ return {
296
+ ...staticResult,
297
+ warnings: [playwrightInstallHint()]
298
+ };
299
+ }
300
+ throw new Error(`web_fetch failed and browser rendering is unavailable. ${playwrightInstallHint()}. Static fetch error: ${staticError?.message || staticError}`);
301
+ }
211
302
 
212
- const browser = await chromium.launch({
213
- headless: true,
214
- env: await buildPlaywrightLaunchEnv()
215
- });
303
+ let browser;
216
304
  try {
305
+ browser = await playwright.chromium.launch({
306
+ headless: true,
307
+ env: await buildPlaywrightLaunchEnv()
308
+ });
217
309
  const page = await browser.newPage();
218
310
  const response = await page.goto(url, { waitUntil, timeout: timeoutMs });
219
311
  const finalUrl = page.url();
220
312
  const html = await page.content();
221
- const $ = cheerio.load(html);
222
- const bodyText = $('body').text() || $.root().text();
223
- const text = String(bodyText || '').replace(/[^\S\n]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
224
- const title = trimPreview($('title').first().text() || (await page.title()), 240);
225
- const description = extractHtmlMeta($, 'description') || extractHtmlMeta($, 'og:description');
226
- const links = collectPageLinks($, finalUrl, maxLinks);
227
-
228
- return {
313
+ const rendered = {
229
314
  url,
230
- final_url: finalUrl,
231
- title,
232
- description,
233
- text,
234
- links,
235
- metadata: {
315
+ ...extractPageContent(cheerio, html, finalUrl, {
316
+ maxLinks,
236
317
  status: response?.status?.() ?? null,
237
- fetched_at: new Date().toISOString(),
238
- content_type: response?.headers?.()['content-type'] || '',
239
- wait_until: waitUntil,
240
- lang: String($('html').attr('lang') || '').trim()
241
- }
318
+ contentType: response?.headers?.()['content-type'] || '',
319
+ fetchMode: 'browser'
320
+ })
242
321
  };
322
+ rendered.metadata.wait_until = waitUntil;
323
+ rendered.title = rendered.title || trimPreview(await page.title(), 240);
324
+ return rendered;
325
+ } catch (error) {
326
+ if (staticResult) {
327
+ return {
328
+ ...staticResult,
329
+ warnings: [`Browser rendering fallback failed: ${error?.message || error}`]
330
+ };
331
+ }
332
+ throw error;
243
333
  } finally {
244
- await browser.close();
334
+ if (browser) await browser.close();
245
335
  }
246
336
  }
247
337
 
@@ -2036,7 +2126,7 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
2036
2126
  function: {
2037
2127
  name: 'web_fetch',
2038
2128
  description:
2039
- 'Fetch and read a live web page. Uses Playwright to render the page, Cheerio to extract structured content, and Crawlee request handling to normalize the fetch flow. Use this for direct URL reads, not for keyword search.',
2129
+ 'Fetch and read a live web page. Uses a lightweight fetch + Cheerio reader by default, then falls back to optional Playwright browser rendering for JavaScript-heavy pages when Playwright is installed. Use this for direct URL reads, not for keyword search.',
2040
2130
  parameters: {
2041
2131
  type: 'object',
2042
2132
  properties: {
@@ -2140,7 +2230,7 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
2140
2230
  function: {
2141
2231
  name: 'dream_consolidate',
2142
2232
  description:
2143
- 'Run a dream loop consolidation pass over inbox entries. Reads recent inbox items, deduplicates, evaluates lifecycle progression (observed → candidate → operational/longterm), promotes stable patterns into persistent memory, archives expired items, and writes an audit report. Use during off-hours or explicit maintenance.',
2233
+ 'Run a dream loop pass over inbox entries and existing memory buckets. Reads recent inbox items, deduplicates, evaluates lifecycle progression (observed → candidate → operational/longterm), promotes stable patterns into persistent memory, then uses LLM maintenance to merge/summarize/clean stale user/global/project memories when their bucket changed since the last maintenance marker. Writes an audit report. Use during off-hours or explicit maintenance.',
2144
2234
  parameters: {
2145
2235
  type: 'object',
2146
2236
  properties: {
@@ -2742,6 +2832,12 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
2742
2832
  if (result.title) lines.push(`title: ${result.title}`);
2743
2833
  if (result.description) lines.push(`description: ${trimPreview(result.description, 200)}`);
2744
2834
  if (result.metadata?.status) lines.push(`status: ${result.metadata.status}`);
2835
+ if (result.metadata?.fetch_mode) lines.push(`mode: ${result.metadata.fetch_mode}`);
2836
+ if (Array.isArray(result.warnings)) {
2837
+ for (const warning of result.warnings.slice(0, 3)) {
2838
+ if (warning) lines.push(`warning: ${warning}`);
2839
+ }
2840
+ }
2745
2841
  if (Array.isArray(result.links) && result.links.length > 0) {
2746
2842
  lines.push(`links: ${result.links.slice(0, 5).map((item) => item.href).join(', ')}`);
2747
2843
  }