@jackwener/opencli 1.5.8 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +35 -1
- package/README.zh-CN.md +17 -1
- package/SKILL.md +31 -851
- package/autoresearch/baseline-browse.txt +1 -0
- package/autoresearch/baseline-skill.txt +1 -0
- package/autoresearch/browse-tasks.json +688 -0
- package/autoresearch/eval-browse.ts +185 -0
- package/autoresearch/eval-skill.ts +248 -0
- package/autoresearch/run-browse.sh +9 -0
- package/autoresearch/run-skill.sh +9 -0
- package/dist/browser/base-page.d.ts +48 -0
- package/dist/browser/base-page.js +160 -0
- package/dist/browser/cdp.js +4 -106
- package/dist/browser/daemon-client.d.ts +20 -7
- package/dist/browser/daemon-client.js +39 -39
- package/dist/browser/daemon-client.test.js +77 -0
- package/dist/browser/discover.d.ts +1 -4
- package/dist/browser/discover.js +9 -23
- package/dist/browser/errors.d.ts +4 -0
- package/dist/browser/errors.js +20 -0
- package/dist/browser/index.d.ts +1 -1
- package/dist/browser/index.js +1 -1
- package/dist/browser/page.d.ts +10 -35
- package/dist/browser/page.js +55 -187
- package/dist/browser/tabs.js +5 -5
- package/dist/browser.test.js +15 -15
- package/dist/cli-manifest.json +294 -22
- package/dist/cli.js +392 -0
- package/dist/clis/amazon/bestsellers.d.ts +21 -0
- package/dist/clis/amazon/bestsellers.js +130 -0
- package/dist/clis/amazon/bestsellers.test.js +20 -0
- package/dist/clis/amazon/discussion.d.ts +20 -0
- package/dist/clis/amazon/discussion.js +91 -0
- package/dist/clis/amazon/discussion.test.d.ts +1 -0
- package/dist/clis/amazon/discussion.test.js +36 -0
- package/dist/clis/amazon/offer.d.ts +23 -0
- package/dist/clis/amazon/offer.js +140 -0
- package/dist/clis/amazon/offer.test.d.ts +1 -0
- package/dist/clis/amazon/offer.test.js +29 -0
- package/dist/clis/amazon/product.d.ts +18 -0
- package/dist/clis/amazon/product.js +92 -0
- package/dist/clis/amazon/product.test.d.ts +1 -0
- package/dist/clis/amazon/product.test.js +24 -0
- package/dist/clis/amazon/search.d.ts +18 -0
- package/dist/clis/amazon/search.js +87 -0
- package/dist/clis/amazon/search.test.d.ts +1 -0
- package/dist/clis/amazon/search.test.js +22 -0
- package/dist/clis/amazon/shared.d.ts +64 -0
- package/dist/clis/amazon/shared.js +255 -0
- package/dist/clis/amazon/shared.test.d.ts +1 -0
- package/dist/clis/amazon/shared.test.js +33 -0
- package/dist/clis/gemini/ask.d.ts +1 -0
- package/dist/clis/gemini/ask.js +40 -0
- package/dist/clis/gemini/image.d.ts +1 -0
- package/dist/clis/gemini/image.js +105 -0
- package/dist/clis/gemini/new.d.ts +1 -0
- package/dist/clis/gemini/new.js +20 -0
- package/dist/clis/gemini/utils.d.ts +34 -0
- package/dist/clis/gemini/utils.js +463 -0
- package/dist/clis/gemini/utils.test.d.ts +1 -0
- package/dist/clis/gemini/utils.test.js +31 -0
- package/dist/clis/notebooklm/compat.test.d.ts +1 -1
- package/dist/clis/notebooklm/compat.test.js +3 -3
- package/dist/clis/notebooklm/current.js +2 -3
- package/dist/clis/notebooklm/get.js +2 -3
- package/dist/clis/notebooklm/history.js +2 -3
- package/dist/clis/notebooklm/note-list.js +2 -3
- package/dist/clis/notebooklm/notes-get.js +2 -3
- package/dist/clis/notebooklm/open.d.ts +1 -0
- package/dist/clis/notebooklm/open.js +41 -0
- package/dist/clis/notebooklm/open.test.d.ts +1 -0
- package/dist/clis/notebooklm/open.test.js +63 -0
- package/dist/clis/notebooklm/source-fulltext.js +2 -3
- package/dist/clis/notebooklm/source-get.js +2 -3
- package/dist/clis/notebooklm/source-guide.js +2 -3
- package/dist/clis/notebooklm/source-list.js +2 -3
- package/dist/clis/notebooklm/status.js +1 -2
- package/dist/clis/notebooklm/summary.js +2 -3
- package/dist/clis/notebooklm/utils.d.ts +2 -1
- package/dist/clis/notebooklm/utils.js +20 -21
- package/dist/clis/twitter/article.js +28 -1
- package/dist/clis/xiaohongshu/creator-note-detail.test.js +11 -11
- package/dist/clis/xiaohongshu/creator-notes-summary.test.js +6 -6
- package/dist/clis/xiaohongshu/creator-notes.test.js +22 -22
- package/dist/clis/xiaohongshu/note.js +11 -0
- package/dist/clis/xiaohongshu/note.test.js +49 -0
- package/dist/commanderAdapter.js +7 -4
- package/dist/commanderAdapter.test.js +76 -0
- package/dist/commands/daemon.js +8 -47
- package/dist/commands/daemon.test.js +45 -70
- package/dist/discovery.js +27 -0
- package/dist/doctor.d.ts +1 -2
- package/dist/doctor.js +7 -8
- package/dist/explore.js +1 -1
- package/dist/output.js +28 -0
- package/dist/output.test.js +15 -0
- package/dist/pipeline/executor.js +2 -7
- package/dist/pipeline/steps/browser.js +1 -1
- package/dist/pipeline/template.js +25 -3
- package/dist/record.d.ts +50 -0
- package/dist/record.js +298 -57
- package/dist/record.test.d.ts +1 -0
- package/dist/record.test.js +293 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +1 -0
- package/dist/registry.test.js +10 -0
- package/dist/runtime.js +3 -3
- package/dist/snapshotFormatter.d.ts +1 -1
- package/dist/snapshotFormatter.js +4 -4
- package/dist/snapshotFormatter.test.d.ts +1 -1
- package/dist/snapshotFormatter.test.js +2 -2
- package/dist/types.d.ts +11 -1
- package/dist/types.js +1 -1
- package/docs/.vitepress/config.mts +2 -0
- package/docs/adapters/browser/amazon.md +53 -0
- package/docs/adapters/browser/gemini.md +72 -0
- package/docs/adapters/browser/notebooklm.md +5 -5
- package/docs/adapters/index.md +3 -1
- package/docs/guide/getting-started.md +21 -0
- package/docs/superpowers/specs/2026-04-02-browse-skill-testing-design.md +144 -0
- package/docs/zh/guide/getting-started.md +21 -0
- package/extension/package-lock.json +2 -2
- package/extension/src/background.test.ts +7 -163
- package/extension/src/background.ts +58 -161
- package/extension/src/cdp.ts +77 -124
- package/extension/src/protocol.ts +5 -5
- package/package.json +1 -1
- package/skills/opencli-explorer/SKILL.md +853 -0
- package/skills/opencli-oneshot/SKILL.md +222 -0
- package/skills/opencli-operate/SKILL.md +213 -0
- package/skills/opencli-usage/SKILL.md +152 -0
- package/skills/opencli-usage/browser.md +429 -0
- package/skills/opencli-usage/desktop.md +118 -0
- package/skills/opencli-usage/plugins.md +82 -0
- package/skills/opencli-usage/public-api.md +149 -0
- package/src/browser/base-page.ts +197 -0
- package/src/browser/cdp.ts +7 -131
- package/src/browser/daemon-client.test.ts +103 -0
- package/src/browser/daemon-client.ts +55 -43
- package/src/browser/discover.ts +9 -21
- package/src/browser/errors.ts +22 -0
- package/src/browser/index.ts +1 -1
- package/src/browser/page.ts +57 -209
- package/src/browser/tabs.ts +5 -5
- package/src/browser.test.ts +15 -15
- package/src/cli.ts +392 -0
- package/src/clis/amazon/bestsellers.test.ts +22 -0
- package/src/clis/amazon/bestsellers.ts +180 -0
- package/src/clis/amazon/discussion.test.ts +38 -0
- package/src/clis/amazon/discussion.ts +131 -0
- package/src/clis/amazon/offer.test.ts +35 -0
- package/src/clis/amazon/offer.ts +185 -0
- package/src/clis/amazon/product.test.ts +26 -0
- package/src/clis/amazon/product.ts +131 -0
- package/src/clis/amazon/search.test.ts +24 -0
- package/src/clis/amazon/search.ts +128 -0
- package/src/clis/amazon/shared.test.ts +37 -0
- package/src/clis/amazon/shared.ts +316 -0
- package/src/clis/gemini/ask.ts +46 -0
- package/src/clis/gemini/image.ts +115 -0
- package/src/clis/gemini/new.ts +22 -0
- package/src/clis/gemini/utils.test.ts +36 -0
- package/src/clis/gemini/utils.ts +523 -0
- package/src/clis/notebooklm/compat.test.ts +3 -3
- package/src/clis/notebooklm/current.ts +2 -3
- package/src/clis/notebooklm/get.ts +1 -3
- package/src/clis/notebooklm/history.ts +1 -3
- package/src/clis/notebooklm/note-list.ts +1 -3
- package/src/clis/notebooklm/notes-get.ts +1 -3
- package/src/clis/notebooklm/open.test.ts +78 -0
- package/src/clis/notebooklm/open.ts +61 -0
- package/src/clis/notebooklm/source-fulltext.ts +1 -3
- package/src/clis/notebooklm/source-get.ts +1 -3
- package/src/clis/notebooklm/source-guide.ts +1 -3
- package/src/clis/notebooklm/source-list.ts +1 -3
- package/src/clis/notebooklm/status.ts +1 -2
- package/src/clis/notebooklm/summary.ts +1 -3
- package/src/clis/notebooklm/utils.ts +29 -20
- package/src/clis/twitter/article.ts +31 -1
- package/src/clis/xiaohongshu/creator-note-detail.test.ts +11 -11
- package/src/clis/xiaohongshu/creator-notes-summary.test.ts +6 -6
- package/src/clis/xiaohongshu/creator-notes.test.ts +22 -22
- package/src/clis/xiaohongshu/note.test.ts +51 -0
- package/src/clis/xiaohongshu/note.ts +18 -0
- package/src/commanderAdapter.test.ts +109 -0
- package/src/commanderAdapter.ts +8 -4
- package/src/commands/daemon.test.ts +50 -84
- package/src/commands/daemon.ts +8 -56
- package/src/discovery.ts +22 -0
- package/src/doctor.ts +8 -9
- package/src/explore.ts +1 -1
- package/src/output.test.ts +17 -0
- package/src/output.ts +27 -0
- package/src/pipeline/executor.ts +2 -7
- package/src/pipeline/steps/browser.ts +1 -1
- package/src/pipeline/template.ts +27 -4
- package/src/record.test.ts +362 -0
- package/src/record.ts +341 -62
- package/src/registry.test.ts +12 -0
- package/src/registry.ts +3 -0
- package/src/runtime.ts +3 -3
- package/src/snapshotFormatter.test.ts +2 -2
- package/src/snapshotFormatter.ts +4 -4
- package/src/types.ts +11 -1
- package/.agents/skills/cross-project-adapter-migration/SKILL.md +0 -249
- package/.agents/workflows/cross-project-adapter-migration.md +0 -54
- package/dist/clis/notebooklm/bind-current.js +0 -29
- package/dist/clis/notebooklm/bind-current.test.d.ts +0 -1
- package/dist/clis/notebooklm/bind-current.test.js +0 -35
- package/dist/clis/notebooklm/binding.test.js +0 -44
- package/extension/dist/background.js +0 -819
- package/src/clis/notebooklm/bind-current.test.ts +0 -43
- package/src/clis/notebooklm/bind-current.ts +0 -36
- package/src/clis/notebooklm/binding.test.ts +0 -53
- /package/dist/browser/{mcp.d.ts → bridge.d.ts} +0 -0
- /package/dist/browser/{mcp.js → bridge.js} +0 -0
- /package/dist/{clis/notebooklm/bind-current.d.ts → browser/daemon-client.test.d.ts} +0 -0
- /package/dist/clis/{notebooklm/binding.test.d.ts → amazon/bestsellers.test.d.ts} +0 -0
- /package/src/browser/{mcp.ts → bridge.ts} +0 -0
package/src/cli.ts
CHANGED
|
@@ -18,6 +18,13 @@ import { registerAllCommands } from './commanderAdapter.js';
|
|
|
18
18
|
import { EXIT_CODES, getErrorMessage } from './errors.js';
|
|
19
19
|
import { daemonStatus, daemonStop, daemonRestart } from './commands/daemon.js';
|
|
20
20
|
|
|
21
|
+
/** Create a browser page for operate commands. Uses 'operate' workspace for session persistence. */
|
|
22
|
+
async function getOperatePage(): Promise<import('./types.js').IPage> {
|
|
23
|
+
const { BrowserBridge } = await import('./browser/index.js');
|
|
24
|
+
const bridge = new BrowserBridge();
|
|
25
|
+
return bridge.connect({ timeout: 30, workspace: 'operate:default' });
|
|
26
|
+
}
|
|
27
|
+
|
|
21
28
|
export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
|
|
22
29
|
const program = new Command();
|
|
23
30
|
// enablePositionalOptions: prevents parent from consuming flags meant for subcommands;
|
|
@@ -229,6 +236,391 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
|
|
|
229
236
|
console.log(renderCascadeResult(result));
|
|
230
237
|
});
|
|
231
238
|
|
|
239
|
+
// ── Built-in: operate (browser control for Claude Code skill) ───────────────
|
|
240
|
+
//
|
|
241
|
+
// Make websites accessible for AI agents.
|
|
242
|
+
// All commands wrapped in operateAction() for consistent error handling.
|
|
243
|
+
|
|
244
|
+
const operate = program
|
|
245
|
+
.command('operate')
|
|
246
|
+
.description('Browser control — navigate, click, type, extract, wait (no LLM needed)');
|
|
247
|
+
|
|
248
|
+
/** Wrap operate actions with error handling and optional --json output */
|
|
249
|
+
function operateAction(fn: (page: Awaited<ReturnType<typeof getOperatePage>>, ...args: any[]) => Promise<unknown>) {
|
|
250
|
+
return async (...args: any[]) => {
|
|
251
|
+
try {
|
|
252
|
+
const page = await getOperatePage();
|
|
253
|
+
await fn(page, ...args);
|
|
254
|
+
} catch (err) {
|
|
255
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
256
|
+
if (msg.includes('Extension not connected') || msg.includes('Daemon')) {
|
|
257
|
+
console.error(`Browser not connected. Run 'opencli doctor' to diagnose.`);
|
|
258
|
+
} else if (msg.includes('attach failed') || msg.includes('chrome-extension://')) {
|
|
259
|
+
console.error(`Browser attach failed — another extension may be interfering. Try disabling 1Password.`);
|
|
260
|
+
} else {
|
|
261
|
+
console.error(`Error: ${msg}`);
|
|
262
|
+
}
|
|
263
|
+
process.exitCode = EXIT_CODES.GENERIC_ERROR;
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// ── Navigation ──
|
|
269
|
+
|
|
270
|
+
/** Network interceptor JS — injected on every open/navigate to capture fetch/XHR */
|
|
271
|
+
const NETWORK_INTERCEPTOR_JS = `(function(){if(window.__opencli_net)return;window.__opencli_net=[];var M=200,B=50000,F=window.fetch;window.fetch=async function(){var r=await F.apply(this,arguments);try{var ct=r.headers.get('content-type')||'';if(ct.includes('json')||ct.includes('text')){var c=r.clone(),t=await c.text();if(window.__opencli_net.length<M){var b=null;if(t.length<=B)try{b=JSON.parse(t)}catch(e){b=t}window.__opencli_net.push({url:r.url||(arguments[0]&&arguments[0].url)||String(arguments[0]),method:(arguments[1]&&arguments[1].method)||'GET',status:r.status,size:t.length,ct:ct,body:b})}}}catch(e){}return r};var X=XMLHttpRequest.prototype,O=X.open,S=X.send;X.open=function(m,u){this._om=m;this._ou=u;return O.apply(this,arguments)};X.send=function(){var x=this;x.addEventListener('load',function(){try{var ct=x.getResponseHeader('content-type')||'';if((ct.includes('json')||ct.includes('text'))&&window.__opencli_net.length<M){var t=x.responseText,b=null;if(t&&t.length<=B)try{b=JSON.parse(t)}catch(e){b=t}window.__opencli_net.push({url:x._ou,method:x._om||'GET',status:x.status,size:t?t.length:0,ct:ct,body:b})}}catch(e){}});return S.apply(this,arguments)}})()`;
|
|
272
|
+
|
|
273
|
+
operate.command('open').argument('<url>').description('Open URL in automation window')
|
|
274
|
+
.action(operateAction(async (page, url) => {
|
|
275
|
+
await page.goto(url);
|
|
276
|
+
await page.wait(2);
|
|
277
|
+
// Auto-inject network interceptor for API discovery
|
|
278
|
+
try { await page.evaluate(NETWORK_INTERCEPTOR_JS); } catch { /* non-fatal */ }
|
|
279
|
+
console.log(`Navigated to: ${await page.getCurrentUrl?.() ?? url}`);
|
|
280
|
+
}));
|
|
281
|
+
|
|
282
|
+
operate.command('back').description('Go back in browser history')
|
|
283
|
+
.action(operateAction(async (page) => {
|
|
284
|
+
await page.evaluate('history.back()');
|
|
285
|
+
await page.wait(2);
|
|
286
|
+
console.log('Navigated back');
|
|
287
|
+
}));
|
|
288
|
+
|
|
289
|
+
operate.command('scroll').argument('<direction>', 'up or down').option('--amount <pixels>', 'Pixels to scroll', '500')
|
|
290
|
+
.description('Scroll page')
|
|
291
|
+
.action(operateAction(async (page, direction, opts) => {
|
|
292
|
+
if (direction !== 'up' && direction !== 'down') {
|
|
293
|
+
console.error(`Invalid direction "${direction}". Use "up" or "down".`);
|
|
294
|
+
process.exitCode = EXIT_CODES.USAGE_ERROR;
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
await page.scroll(direction, parseInt(opts.amount, 10));
|
|
298
|
+
console.log(`Scrolled ${direction}`);
|
|
299
|
+
}));
|
|
300
|
+
|
|
301
|
+
// ── Inspect ──
|
|
302
|
+
|
|
303
|
+
operate.command('state').description('Page state: URL, title, interactive elements with [N] indices')
|
|
304
|
+
.action(operateAction(async (page) => {
|
|
305
|
+
const snapshot = await page.snapshot({ viewportExpand: 800 });
|
|
306
|
+
const url = await page.getCurrentUrl?.() ?? '';
|
|
307
|
+
console.log(`URL: ${url}\n`);
|
|
308
|
+
console.log(typeof snapshot === 'string' ? snapshot : JSON.stringify(snapshot, null, 2));
|
|
309
|
+
}));
|
|
310
|
+
|
|
311
|
+
operate.command('screenshot').argument('[path]', 'Save to file (base64 if omitted)')
|
|
312
|
+
.description('Take screenshot')
|
|
313
|
+
.action(operateAction(async (page, path) => {
|
|
314
|
+
if (path) {
|
|
315
|
+
await page.screenshot({ path });
|
|
316
|
+
console.log(`Screenshot saved to: ${path}`);
|
|
317
|
+
} else {
|
|
318
|
+
console.log(await page.screenshot({ format: 'png' }));
|
|
319
|
+
}
|
|
320
|
+
}));
|
|
321
|
+
|
|
322
|
+
// ── Get commands (structured data extraction) ──
|
|
323
|
+
|
|
324
|
+
const get = operate.command('get').description('Get page properties');
|
|
325
|
+
|
|
326
|
+
get.command('title').description('Page title')
|
|
327
|
+
.action(operateAction(async (page) => {
|
|
328
|
+
console.log(await page.evaluate('document.title'));
|
|
329
|
+
}));
|
|
330
|
+
|
|
331
|
+
get.command('url').description('Current page URL')
|
|
332
|
+
.action(operateAction(async (page) => {
|
|
333
|
+
console.log(await page.getCurrentUrl?.() ?? await page.evaluate('location.href'));
|
|
334
|
+
}));
|
|
335
|
+
|
|
336
|
+
get.command('text').argument('<index>', 'Element index').description('Element text content')
|
|
337
|
+
.action(operateAction(async (page, index) => {
|
|
338
|
+
const text = await page.evaluate(`document.querySelector('[data-opencli-ref="${index}"]')?.textContent?.trim()`);
|
|
339
|
+
console.log(text ?? '(empty)');
|
|
340
|
+
}));
|
|
341
|
+
|
|
342
|
+
get.command('value').argument('<index>', 'Element index').description('Input/textarea value')
|
|
343
|
+
.action(operateAction(async (page, index) => {
|
|
344
|
+
const val = await page.evaluate(`document.querySelector('[data-opencli-ref="${index}"]')?.value`);
|
|
345
|
+
console.log(val ?? '(empty)');
|
|
346
|
+
}));
|
|
347
|
+
|
|
348
|
+
get.command('html').option('--selector <css>', 'CSS selector scope').description('Page HTML (or scoped)')
|
|
349
|
+
.action(operateAction(async (page, opts) => {
|
|
350
|
+
const sel = opts.selector ? JSON.stringify(opts.selector) : 'null';
|
|
351
|
+
const html = await page.evaluate(`(${sel} ? document.querySelector(${sel})?.outerHTML : document.documentElement.outerHTML)?.slice(0, 50000)`);
|
|
352
|
+
console.log(html ?? '(empty)');
|
|
353
|
+
}));
|
|
354
|
+
|
|
355
|
+
get.command('attributes').argument('<index>', 'Element index').description('Element attributes')
|
|
356
|
+
.action(operateAction(async (page, index) => {
|
|
357
|
+
const attrs = await page.evaluate(`JSON.stringify(Object.fromEntries([...document.querySelector('[data-opencli-ref="${index}"]')?.attributes].map(a=>[a.name,a.value])))`);
|
|
358
|
+
console.log(attrs ?? '{}');
|
|
359
|
+
}));
|
|
360
|
+
|
|
361
|
+
// ── Interact ──
|
|
362
|
+
|
|
363
|
+
operate.command('click').argument('<index>', 'Element index from state').description('Click element by index')
|
|
364
|
+
.action(operateAction(async (page, index) => {
|
|
365
|
+
await page.click(index);
|
|
366
|
+
console.log(`Clicked element [${index}]`);
|
|
367
|
+
}));
|
|
368
|
+
|
|
369
|
+
operate.command('type').argument('<index>', 'Element index').argument('<text>', 'Text to type')
|
|
370
|
+
.description('Click element, then type text')
|
|
371
|
+
.action(operateAction(async (page, index, text) => {
|
|
372
|
+
await page.click(index);
|
|
373
|
+
await page.wait(0.3);
|
|
374
|
+
await page.typeText(index, text);
|
|
375
|
+
console.log(`Typed "${text}" into element [${index}]`);
|
|
376
|
+
}));
|
|
377
|
+
|
|
378
|
+
operate.command('select').argument('<index>', 'Element index of <select>').argument('<option>', 'Option text')
|
|
379
|
+
.description('Select dropdown option')
|
|
380
|
+
.action(operateAction(async (page, index, option) => {
|
|
381
|
+
const result = await page.evaluate(`
|
|
382
|
+
(function() {
|
|
383
|
+
var sel = document.querySelector('[data-opencli-ref="${index}"]');
|
|
384
|
+
if (!sel || sel.tagName !== 'SELECT') return { error: 'Not a <select>' };
|
|
385
|
+
var match = Array.from(sel.options).find(o => o.text.trim() === ${JSON.stringify(option)} || o.value === ${JSON.stringify(option)});
|
|
386
|
+
if (!match) return { error: 'Option not found', available: Array.from(sel.options).map(o => o.text.trim()) };
|
|
387
|
+
var setter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set;
|
|
388
|
+
if (setter) setter.call(sel, match.value); else sel.value = match.value;
|
|
389
|
+
sel.dispatchEvent(new Event('input', {bubbles:true}));
|
|
390
|
+
sel.dispatchEvent(new Event('change', {bubbles:true}));
|
|
391
|
+
return { selected: match.text };
|
|
392
|
+
})()
|
|
393
|
+
`) as { error?: string; selected?: string; available?: string[] } | null;
|
|
394
|
+
if (result?.error) {
|
|
395
|
+
console.error(`Error: ${result.error}${result.available ? ` — Available: ${result.available.join(', ')}` : ''}`);
|
|
396
|
+
process.exitCode = EXIT_CODES.GENERIC_ERROR;
|
|
397
|
+
} else {
|
|
398
|
+
console.log(`Selected "${result?.selected}" in element [${index}]`);
|
|
399
|
+
}
|
|
400
|
+
}));
|
|
401
|
+
|
|
402
|
+
operate.command('keys').argument('<key>', 'Key to press (Enter, Escape, Tab, Control+a)')
|
|
403
|
+
.description('Press keyboard key')
|
|
404
|
+
.action(operateAction(async (page, key) => {
|
|
405
|
+
await page.pressKey(key);
|
|
406
|
+
console.log(`Pressed: ${key}`);
|
|
407
|
+
}));
|
|
408
|
+
|
|
409
|
+
// ── Wait commands ──
|
|
410
|
+
|
|
411
|
+
operate.command('wait')
|
|
412
|
+
.argument('<type>', 'selector, text, or time')
|
|
413
|
+
.argument('[value]', 'CSS selector, text string, or seconds')
|
|
414
|
+
.option('--timeout <ms>', 'Timeout in milliseconds', '10000')
|
|
415
|
+
.description('Wait for selector, text, or time (e.g. wait selector ".loaded", wait text "Success", wait time 3)')
|
|
416
|
+
.action(operateAction(async (page, type, value, opts) => {
|
|
417
|
+
const timeout = parseInt(opts.timeout, 10);
|
|
418
|
+
if (type === 'time') {
|
|
419
|
+
const seconds = parseFloat(value ?? '2');
|
|
420
|
+
await page.wait(seconds);
|
|
421
|
+
console.log(`Waited ${seconds}s`);
|
|
422
|
+
} else if (type === 'selector') {
|
|
423
|
+
if (!value) { console.error('Missing CSS selector'); process.exitCode = EXIT_CODES.USAGE_ERROR; return; }
|
|
424
|
+
await page.wait({ selector: value, timeout: timeout / 1000 });
|
|
425
|
+
console.log(`Element "${value}" appeared`);
|
|
426
|
+
} else if (type === 'text') {
|
|
427
|
+
if (!value) { console.error('Missing text'); process.exitCode = EXIT_CODES.USAGE_ERROR; return; }
|
|
428
|
+
await page.wait({ text: value, timeout: timeout / 1000 });
|
|
429
|
+
console.log(`Text "${value}" appeared`);
|
|
430
|
+
} else {
|
|
431
|
+
console.error(`Unknown wait type "${type}". Use: selector, text, or time`);
|
|
432
|
+
process.exitCode = EXIT_CODES.USAGE_ERROR;
|
|
433
|
+
}
|
|
434
|
+
}));
|
|
435
|
+
|
|
436
|
+
// ── Extract ──
|
|
437
|
+
|
|
438
|
+
operate.command('eval').argument('<js>', 'JavaScript code').description('Execute JS in page context, return result')
|
|
439
|
+
.action(operateAction(async (page, js) => {
|
|
440
|
+
const result = await page.evaluate(js);
|
|
441
|
+
if (typeof result === 'string') console.log(result);
|
|
442
|
+
else console.log(JSON.stringify(result, null, 2));
|
|
443
|
+
}));
|
|
444
|
+
|
|
445
|
+
// ── Network (API discovery) ──
|
|
446
|
+
|
|
447
|
+
operate.command('network')
|
|
448
|
+
.option('--detail <index>', 'Show full response body of request at index')
|
|
449
|
+
.option('--all', 'Show all requests including static resources')
|
|
450
|
+
.description('Show captured network requests (auto-captured since last open)')
|
|
451
|
+
.action(operateAction(async (page, opts) => {
|
|
452
|
+
const requests = await page.evaluate(`(function(){
|
|
453
|
+
var reqs = window.__opencli_net || [];
|
|
454
|
+
return JSON.stringify(reqs);
|
|
455
|
+
})()`) as string;
|
|
456
|
+
|
|
457
|
+
let items: Array<{ url: string; method: string; status: number; size: number; ct: string; body: unknown }> = [];
|
|
458
|
+
try { items = JSON.parse(requests); } catch { console.log('No network data captured. Run "operate open <url>" first.'); return; }
|
|
459
|
+
|
|
460
|
+
if (items.length === 0) { console.log('No requests captured.'); return; }
|
|
461
|
+
|
|
462
|
+
// Filter out static resources unless --all
|
|
463
|
+
if (!opts.all) {
|
|
464
|
+
items = items.filter(r =>
|
|
465
|
+
(r.ct?.includes('json') || r.ct?.includes('xml') || r.ct?.includes('text/plain')) &&
|
|
466
|
+
!/\.(js|css|png|jpg|gif|svg|woff|ico|map)(\?|$)/i.test(r.url) &&
|
|
467
|
+
!/analytics|tracking|telemetry|beacon|pixel|gtag|fbevents/i.test(r.url)
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
if (opts.detail !== undefined) {
|
|
472
|
+
const idx = parseInt(opts.detail, 10);
|
|
473
|
+
const req = items[idx];
|
|
474
|
+
if (!req) { console.error(`Request #${idx} not found. ${items.length} requests available.`); process.exitCode = EXIT_CODES.USAGE_ERROR; return; }
|
|
475
|
+
console.log(`${req.method} ${req.url}`);
|
|
476
|
+
console.log(`Status: ${req.status} | Size: ${req.size} | Type: ${req.ct}`);
|
|
477
|
+
console.log('---');
|
|
478
|
+
console.log(typeof req.body === 'string' ? req.body : JSON.stringify(req.body, null, 2));
|
|
479
|
+
} else {
|
|
480
|
+
console.log(`Captured ${items.length} API requests:\n`);
|
|
481
|
+
items.forEach((r, i) => {
|
|
482
|
+
const bodyPreview = r.body ? (typeof r.body === 'string' ? r.body.slice(0, 60) : JSON.stringify(r.body).slice(0, 60)) : '';
|
|
483
|
+
console.log(` [${i}] ${r.method} ${r.status} ${r.url.slice(0, 80)}`);
|
|
484
|
+
if (bodyPreview) console.log(` ${bodyPreview}...`);
|
|
485
|
+
});
|
|
486
|
+
console.log(`\nUse --detail <index> to see full response body.`);
|
|
487
|
+
}
|
|
488
|
+
}));
|
|
489
|
+
|
|
490
|
+
// ── Init (adapter scaffolding) ──
|
|
491
|
+
|
|
492
|
+
operate.command('init')
|
|
493
|
+
.argument('<name>', 'Adapter name in site/command format (e.g. hn/top)')
|
|
494
|
+
.description('Generate adapter scaffold in ~/.opencli/clis/')
|
|
495
|
+
.action(async (name: string) => {
|
|
496
|
+
try {
|
|
497
|
+
const parts = name.split('/');
|
|
498
|
+
if (parts.length !== 2 || !parts[0] || !parts[1]) {
|
|
499
|
+
console.error('Name must be site/command format (e.g. hn/top)');
|
|
500
|
+
process.exitCode = EXIT_CODES.USAGE_ERROR;
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
const [site, command] = parts;
|
|
504
|
+
if (!/^[a-zA-Z0-9_-]+$/.test(site) || !/^[a-zA-Z0-9_-]+$/.test(command)) {
|
|
505
|
+
console.error('Name parts must be alphanumeric/dash/underscore only');
|
|
506
|
+
process.exitCode = EXIT_CODES.USAGE_ERROR;
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const os = await import('node:os');
|
|
511
|
+
const fs = await import('node:fs');
|
|
512
|
+
const path = await import('node:path');
|
|
513
|
+
const dir = path.join(os.homedir(), '.opencli', 'clis', site);
|
|
514
|
+
const filePath = path.join(dir, `${command}.ts`);
|
|
515
|
+
|
|
516
|
+
if (fs.existsSync(filePath)) {
|
|
517
|
+
console.log(`Adapter already exists: ${filePath}`);
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// Try to detect domain from last operate session
|
|
522
|
+
let domain = site;
|
|
523
|
+
try {
|
|
524
|
+
const page = await getOperatePage();
|
|
525
|
+
const url = await page.getCurrentUrl?.();
|
|
526
|
+
if (url) { try { domain = new URL(url).hostname; } catch {} }
|
|
527
|
+
} catch { /* no active session */ }
|
|
528
|
+
|
|
529
|
+
const template = `import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
530
|
+
|
|
531
|
+
cli({
|
|
532
|
+
site: '${site}',
|
|
533
|
+
name: '${command}',
|
|
534
|
+
description: '', // TODO: describe what this command does
|
|
535
|
+
domain: '${domain}',
|
|
536
|
+
strategy: Strategy.PUBLIC, // TODO: PUBLIC (no auth), COOKIE (needs login), UI (DOM interaction)
|
|
537
|
+
browser: false, // TODO: set true if needs browser
|
|
538
|
+
args: [
|
|
539
|
+
{ name: 'limit', type: 'int', default: 10, help: 'Number of items' },
|
|
540
|
+
],
|
|
541
|
+
columns: [], // TODO: field names for table output (e.g. ['title', 'score', 'url'])
|
|
542
|
+
func: async (page, kwargs) => {
|
|
543
|
+
// TODO: implement data fetching
|
|
544
|
+
// Prefer API calls (fetch) over browser automation
|
|
545
|
+
// page is available if browser: true
|
|
546
|
+
return [];
|
|
547
|
+
},
|
|
548
|
+
});
|
|
549
|
+
`;
|
|
550
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
551
|
+
fs.writeFileSync(filePath, template, 'utf-8');
|
|
552
|
+
console.log(`Created: ${filePath}`);
|
|
553
|
+
console.log(`Edit the file to implement your adapter, then run: opencli operate verify ${name}`);
|
|
554
|
+
} catch (err) {
|
|
555
|
+
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
556
|
+
process.exitCode = EXIT_CODES.GENERIC_ERROR;
|
|
557
|
+
}
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
// ── Verify (test adapter) ──
|
|
561
|
+
|
|
562
|
+
operate.command('verify')
|
|
563
|
+
.argument('<name>', 'Adapter name in site/command format (e.g. hn/top)')
|
|
564
|
+
.description('Execute an adapter and show results')
|
|
565
|
+
.action(async (name: string) => {
|
|
566
|
+
try {
|
|
567
|
+
const parts = name.split('/');
|
|
568
|
+
if (parts.length !== 2) { console.error('Name must be site/command format'); process.exitCode = EXIT_CODES.USAGE_ERROR; return; }
|
|
569
|
+
const [site, command] = parts;
|
|
570
|
+
if (!/^[a-zA-Z0-9_-]+$/.test(site) || !/^[a-zA-Z0-9_-]+$/.test(command)) {
|
|
571
|
+
console.error('Name parts must be alphanumeric/dash/underscore only');
|
|
572
|
+
process.exitCode = EXIT_CODES.USAGE_ERROR;
|
|
573
|
+
return;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
const { execSync } = await import('node:child_process');
|
|
577
|
+
const os = await import('node:os');
|
|
578
|
+
const path = await import('node:path');
|
|
579
|
+
const filePath = path.join(os.homedir(), '.opencli', 'clis', site, `${command}.ts`);
|
|
580
|
+
|
|
581
|
+
const fs = await import('node:fs');
|
|
582
|
+
if (!fs.existsSync(filePath)) {
|
|
583
|
+
console.error(`Adapter not found: ${filePath}`);
|
|
584
|
+
console.error(`Run "opencli operate init ${name}" to create it.`);
|
|
585
|
+
process.exitCode = EXIT_CODES.GENERIC_ERROR;
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
console.log(`🔍 Verifying ${name}...\n`);
|
|
590
|
+
console.log(` Loading: ${filePath}`);
|
|
591
|
+
|
|
592
|
+
try {
|
|
593
|
+
const output = execSync(`node dist/main.js ${site} ${command} --limit 3`, {
|
|
594
|
+
cwd: path.join(path.dirname(import.meta.url.replace('file://', '')), '..'),
|
|
595
|
+
timeout: 30000,
|
|
596
|
+
encoding: 'utf-8',
|
|
597
|
+
env: process.env,
|
|
598
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
599
|
+
});
|
|
600
|
+
console.log(` Executing: opencli ${site} ${command} --limit 3\n`);
|
|
601
|
+
console.log(output);
|
|
602
|
+
console.log(`\n ✓ Adapter works!`);
|
|
603
|
+
} catch (err: any) {
|
|
604
|
+
console.log(` Executing: opencli ${site} ${command} --limit 3\n`);
|
|
605
|
+
if (err.stdout) console.log(err.stdout);
|
|
606
|
+
if (err.stderr) console.error(err.stderr.slice(0, 500));
|
|
607
|
+
console.log(`\n ✗ Adapter failed. Fix the code and try again.`);
|
|
608
|
+
process.exitCode = EXIT_CODES.GENERIC_ERROR;
|
|
609
|
+
}
|
|
610
|
+
} catch (err) {
|
|
611
|
+
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
612
|
+
process.exitCode = EXIT_CODES.GENERIC_ERROR;
|
|
613
|
+
}
|
|
614
|
+
});
|
|
615
|
+
|
|
616
|
+
// ── Session ──
|
|
617
|
+
|
|
618
|
+
operate.command('close').description('Close the automation window')
|
|
619
|
+
.action(operateAction(async (page) => {
|
|
620
|
+
await page.closeWindow?.();
|
|
621
|
+
console.log('Automation window closed');
|
|
622
|
+
}));
|
|
623
|
+
|
|
232
624
|
// ── Built-in: doctor / completion ──────────────────────────────────────────
|
|
233
625
|
|
|
234
626
|
program
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { __test__ } from './bestsellers.js';
|
|
3
|
+
|
|
4
|
+
describe('amazon bestsellers normalization', () => {
|
|
5
|
+
it('normalizes bestseller cards and infers review counts from card text', () => {
|
|
6
|
+
const result = __test__.normalizeBestsellerCandidate({
|
|
7
|
+
asin: 'B0DR31GC3D',
|
|
8
|
+
title: '',
|
|
9
|
+
href: 'https://www.amazon.com/NUTIKAS-Shelves-Desktop-Orgnizer-Shlef/dp/B0DR31GC3D/ref=zg_bs',
|
|
10
|
+
price_text: '$25.92',
|
|
11
|
+
rating_text: '4.3 out of 5 stars',
|
|
12
|
+
review_count_text: '',
|
|
13
|
+
card_text: 'Desk Shelves Desktop Organizer Shlef\n4.3 out of 5 stars\n435\n$25.92',
|
|
14
|
+
}, 2, 'Amazon Best Sellers: Best Desktop & Off-Surface Shelves', 'https://www.amazon.com/example');
|
|
15
|
+
|
|
16
|
+
expect(result.rank).toBe(2);
|
|
17
|
+
expect(result.asin).toBe('B0DR31GC3D');
|
|
18
|
+
expect(result.title).toBe('Desk Shelves Desktop Organizer Shlef');
|
|
19
|
+
expect(result.review_count).toBe(435);
|
|
20
|
+
expect(result.list_title).toBe('Amazon Best Sellers: Best Desktop & Off-Surface Shelves');
|
|
21
|
+
});
|
|
22
|
+
});
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
2
|
+
import { cli, Strategy } from '../../registry.js';
|
|
3
|
+
import type { IPage } from '../../types.js';
|
|
4
|
+
import {
|
|
5
|
+
buildProvenance,
|
|
6
|
+
cleanText,
|
|
7
|
+
extractAsin,
|
|
8
|
+
extractReviewCountFromCardText,
|
|
9
|
+
firstMeaningfulLine,
|
|
10
|
+
normalizeProductUrl,
|
|
11
|
+
parsePriceText,
|
|
12
|
+
parseRatingValue,
|
|
13
|
+
parseReviewCount,
|
|
14
|
+
resolveBestsellersUrl,
|
|
15
|
+
uniqueNonEmpty,
|
|
16
|
+
assertUsableState,
|
|
17
|
+
gotoAndReadState,
|
|
18
|
+
} from './shared.js';
|
|
19
|
+
|
|
20
|
+
interface BestsellersPagePayload {
|
|
21
|
+
href?: string;
|
|
22
|
+
title?: string;
|
|
23
|
+
list_title?: string;
|
|
24
|
+
cards?: Array<{
|
|
25
|
+
rank_text?: string | null;
|
|
26
|
+
asin?: string | null;
|
|
27
|
+
title?: string | null;
|
|
28
|
+
href?: string | null;
|
|
29
|
+
price_text?: string | null;
|
|
30
|
+
rating_text?: string | null;
|
|
31
|
+
review_count_text?: string | null;
|
|
32
|
+
card_text?: string | null;
|
|
33
|
+
}>;
|
|
34
|
+
page_links?: string[];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function normalizeBestsellerCandidate(
|
|
38
|
+
candidate: NonNullable<BestsellersPagePayload['cards']>[number],
|
|
39
|
+
rank: number,
|
|
40
|
+
listTitle: string | null,
|
|
41
|
+
sourceUrl: string,
|
|
42
|
+
): Record<string, unknown> {
|
|
43
|
+
const productUrl = normalizeProductUrl(candidate.href);
|
|
44
|
+
const asin = extractAsin(candidate.asin ?? '') ?? extractAsin(productUrl ?? '') ?? null;
|
|
45
|
+
const title = cleanText(candidate.title) || firstMeaningfulLine(candidate.card_text);
|
|
46
|
+
const price = parsePriceText(cleanText(candidate.price_text) || candidate.card_text);
|
|
47
|
+
const ratingText = cleanText(candidate.rating_text) || null;
|
|
48
|
+
const reviewCountText = cleanText(candidate.review_count_text)
|
|
49
|
+
|| extractReviewCountFromCardText(candidate.card_text)
|
|
50
|
+
|| null;
|
|
51
|
+
const provenance = buildProvenance(sourceUrl);
|
|
52
|
+
|
|
53
|
+
return {
|
|
54
|
+
rank,
|
|
55
|
+
asin,
|
|
56
|
+
title: title || null,
|
|
57
|
+
product_url: productUrl,
|
|
58
|
+
list_title: listTitle,
|
|
59
|
+
...provenance,
|
|
60
|
+
price_text: price.price_text,
|
|
61
|
+
price_value: price.price_value,
|
|
62
|
+
currency: price.currency,
|
|
63
|
+
rating_text: ratingText,
|
|
64
|
+
rating_value: parseRatingValue(ratingText),
|
|
65
|
+
review_count_text: reviewCountText,
|
|
66
|
+
review_count: parseReviewCount(reviewCountText),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function readBestsellersPage(page: IPage, url: string): Promise<BestsellersPagePayload> {
|
|
71
|
+
const state = await gotoAndReadState(page, url, 2500, 'bestsellers');
|
|
72
|
+
assertUsableState(state, 'bestsellers');
|
|
73
|
+
|
|
74
|
+
return await page.evaluate(`
|
|
75
|
+
(() => ({
|
|
76
|
+
href: window.location.href,
|
|
77
|
+
title: document.title || '',
|
|
78
|
+
list_title:
|
|
79
|
+
document.querySelector('#zg_banner_text')?.textContent
|
|
80
|
+
|| document.querySelector('h1')?.textContent
|
|
81
|
+
|| '',
|
|
82
|
+
cards: Array.from(document.querySelectorAll('.p13n-sc-uncoverable-faceout'))
|
|
83
|
+
.map((card) => ({
|
|
84
|
+
rank_text:
|
|
85
|
+
card.querySelector('.zg-bdg-text')?.textContent
|
|
86
|
+
|| card.querySelector('[class*="rank"]')?.textContent
|
|
87
|
+
|| '',
|
|
88
|
+
asin: card.id || '',
|
|
89
|
+
title:
|
|
90
|
+
card.querySelector('[class*="line-clamp"]')?.textContent
|
|
91
|
+
|| card.querySelector('img')?.getAttribute('alt')
|
|
92
|
+
|| '',
|
|
93
|
+
href: card.querySelector('a[href*="/dp/"]')?.href || '',
|
|
94
|
+
price_text: card.querySelector('.a-price .a-offscreen')?.textContent || '',
|
|
95
|
+
rating_text: card.querySelector('[aria-label*="out of 5 stars"]')?.getAttribute('aria-label') || '',
|
|
96
|
+
review_count_text:
|
|
97
|
+
card.querySelector('a[href*="#customerReviews"]')?.textContent
|
|
98
|
+
|| card.querySelector('.a-size-small')?.textContent
|
|
99
|
+
|| '',
|
|
100
|
+
card_text: card.innerText || '',
|
|
101
|
+
})),
|
|
102
|
+
page_links: Array.from(document.querySelectorAll('li.a-normal a, li.a-selected a'))
|
|
103
|
+
.map((anchor) => anchor.href || '')
|
|
104
|
+
.filter((href) => /\\/zgbs\\//.test(href) && /(?:[?&]pg=|ref=zg_bs_pg_)/.test(href)),
|
|
105
|
+
}))()
|
|
106
|
+
`) as BestsellersPagePayload;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
cli({
|
|
110
|
+
site: 'amazon',
|
|
111
|
+
name: 'bestsellers',
|
|
112
|
+
description: 'Amazon Best Sellers pages for category candidate discovery',
|
|
113
|
+
domain: 'amazon.com',
|
|
114
|
+
strategy: Strategy.COOKIE,
|
|
115
|
+
navigateBefore: false,
|
|
116
|
+
args: [
|
|
117
|
+
{
|
|
118
|
+
name: 'input',
|
|
119
|
+
positional: true,
|
|
120
|
+
help: 'Best sellers URL or /zgbs path. Omit to use the root Best Sellers page.',
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
name: 'limit',
|
|
124
|
+
type: 'int',
|
|
125
|
+
default: 100,
|
|
126
|
+
help: 'Maximum number of ranked items to return (default 100)',
|
|
127
|
+
},
|
|
128
|
+
],
|
|
129
|
+
columns: ['rank', 'asin', 'title', 'price_text', 'rating_value', 'review_count'],
|
|
130
|
+
func: async (page, kwargs) => {
|
|
131
|
+
const limit = Math.max(1, Number(kwargs.limit) || 100);
|
|
132
|
+
const initialUrl = resolveBestsellersUrl(typeof kwargs.input === 'string' ? kwargs.input : undefined);
|
|
133
|
+
|
|
134
|
+
const queue = [initialUrl];
|
|
135
|
+
const visited = new Set<string>();
|
|
136
|
+
const seenAsins = new Set<string>();
|
|
137
|
+
const results: Record<string, unknown>[] = [];
|
|
138
|
+
let listTitle: string | null = null;
|
|
139
|
+
|
|
140
|
+
while (queue.length > 0 && results.length < limit) {
|
|
141
|
+
const nextUrl = queue.shift()!;
|
|
142
|
+
if (visited.has(nextUrl)) continue;
|
|
143
|
+
visited.add(nextUrl);
|
|
144
|
+
|
|
145
|
+
const payload = await readBestsellersPage(page, nextUrl);
|
|
146
|
+
const sourceUrl = cleanText(payload.href) || nextUrl;
|
|
147
|
+
listTitle = cleanText(payload.list_title) || cleanText(payload.title) || listTitle;
|
|
148
|
+
const cards = payload.cards ?? [];
|
|
149
|
+
|
|
150
|
+
for (const card of cards) {
|
|
151
|
+
const normalized = normalizeBestsellerCandidate(card, results.length + 1, listTitle, sourceUrl);
|
|
152
|
+
const asin = cleanText(String(normalized.asin ?? ''));
|
|
153
|
+
if (!asin || seenAsins.has(asin)) continue;
|
|
154
|
+
seenAsins.add(asin);
|
|
155
|
+
results.push(normalized);
|
|
156
|
+
if (results.length >= limit) break;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const pageLinks = uniqueNonEmpty(payload.page_links ?? []);
|
|
160
|
+
for (const href of pageLinks) {
|
|
161
|
+
if (!visited.has(href) && !queue.includes(href)) {
|
|
162
|
+
queue.push(href);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (results.length === 0) {
|
|
168
|
+
throw new CommandExecutionError(
|
|
169
|
+
'amazon bestsellers did not expose any ranked items',
|
|
170
|
+
'Open the same best sellers page in Chrome, verify it is a real Amazon ranking page, and retry.',
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return results.slice(0, limit);
|
|
175
|
+
},
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
export const __test__ = {
|
|
179
|
+
normalizeBestsellerCandidate,
|
|
180
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { __test__ } from './discussion.js';
|
|
3
|
+
|
|
4
|
+
describe('amazon discussion normalization', () => {
|
|
5
|
+
it('normalizes review summary and sample reviews', () => {
|
|
6
|
+
const result = __test__.normalizeDiscussionPayload({
|
|
7
|
+
href: 'https://www.amazon.com/product-reviews/B0FJS72893',
|
|
8
|
+
average_rating_text: '3.9 out of 5',
|
|
9
|
+
total_review_count_text: '27 global ratings',
|
|
10
|
+
qa_links: [],
|
|
11
|
+
review_samples: [
|
|
12
|
+
{
|
|
13
|
+
title: '5.0 out of 5 stars Great value and quality',
|
|
14
|
+
rating_text: '5.0 out of 5 stars',
|
|
15
|
+
author: 'GTreader2',
|
|
16
|
+
date_text: 'Reviewed in the United States on February 21, 2026',
|
|
17
|
+
body: 'Small but mighty.',
|
|
18
|
+
verified: true,
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
expect(result.asin).toBe('B0FJS72893');
|
|
24
|
+
expect(result.average_rating_value).toBe(3.9);
|
|
25
|
+
expect(result.total_review_count).toBe(27);
|
|
26
|
+
expect(result.review_samples).toEqual([
|
|
27
|
+
{
|
|
28
|
+
title: 'Great value and quality',
|
|
29
|
+
rating_text: '5.0 out of 5 stars',
|
|
30
|
+
rating_value: 5,
|
|
31
|
+
author: 'GTreader2',
|
|
32
|
+
date_text: 'Reviewed in the United States on February 21, 2026',
|
|
33
|
+
body: 'Small but mighty.',
|
|
34
|
+
verified_purchase: true,
|
|
35
|
+
},
|
|
36
|
+
]);
|
|
37
|
+
});
|
|
38
|
+
});
|