@jackwener/opencli 1.6.6 → 1.6.8

package/dist/src/diagnostic.test.js

@@ -0,0 +1,213 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { buildRepairContext, isDiagnosticEnabled, emitDiagnostic, truncate, redactUrl, redactText, resolveAdapterSourcePath, MAX_DIAGNOSTIC_BYTES, } from './diagnostic.js';
+import { SelectorError, CommandExecutionError } from './errors.js';
+function makeCmd(overrides = {}) {
+    return {
+        site: 'test-site',
+        name: 'test-cmd',
+        description: 'test',
+        args: [],
+        ...overrides,
+    };
+}
+describe('isDiagnosticEnabled', () => {
+    const origEnv = process.env.OPENCLI_DIAGNOSTIC;
+    afterEach(() => {
+        if (origEnv === undefined)
+            delete process.env.OPENCLI_DIAGNOSTIC;
+        else
+            process.env.OPENCLI_DIAGNOSTIC = origEnv;
+    });
+    it('returns false when env not set', () => {
+        delete process.env.OPENCLI_DIAGNOSTIC;
+        expect(isDiagnosticEnabled()).toBe(false);
+    });
+    it('returns true when env is "1"', () => {
+        process.env.OPENCLI_DIAGNOSTIC = '1';
+        expect(isDiagnosticEnabled()).toBe(true);
+    });
+    it('returns false for other values', () => {
+        process.env.OPENCLI_DIAGNOSTIC = 'true';
+        expect(isDiagnosticEnabled()).toBe(false);
+    });
+});
+describe('truncate', () => {
+    it('returns short strings unchanged', () => {
+        expect(truncate('hello', 100)).toBe('hello');
+    });
+    it('truncates long strings with marker', () => {
+        const long = 'a'.repeat(200);
+        const result = truncate(long, 50);
+        expect(result.length).toBeLessThan(200);
+        expect(result).toContain('...[truncated,');
+        expect(result).toContain('150 chars omitted]');
+    });
+});
+describe('redactUrl', () => {
+    it('redacts sensitive query parameters', () => {
+        expect(redactUrl('https://api.com/v1?token=abc123&q=test'))
+            .toBe('https://api.com/v1?token=[REDACTED]&q=test');
+    });
+    it('redacts multiple sensitive params', () => {
+        const url = 'https://api.com?api_key=xxx&secret=yyy&page=1';
+        const result = redactUrl(url);
+        expect(result).toContain('api_key=[REDACTED]');
+        expect(result).toContain('secret=[REDACTED]');
+        expect(result).toContain('page=1');
+    });
+    it('leaves clean URLs unchanged', () => {
+        expect(redactUrl('https://example.com/page?q=test')).toBe('https://example.com/page?q=test');
+    });
+});
+describe('redactText', () => {
+    it('redacts Bearer tokens', () => {
+        expect(redactText('Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.test'))
+            .toContain('Bearer [REDACTED]');
+    });
+    it('redacts JWT tokens', () => {
+        const jwt = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U';
+        expect(redactText(`token is ${jwt}`)).toContain('[REDACTED_JWT]');
+        expect(redactText(`token is ${jwt}`)).not.toContain('eyJhbGci');
+    });
+    it('redacts inline token=value patterns', () => {
+        expect(redactText('failed with token=abc123def456')).toContain('token=[REDACTED]');
+    });
+    it('redacts cookie values', () => {
+        const result = redactText('cookie: session=abc123; user=xyz789; path=/');
+        expect(result).toContain('[REDACTED]');
+        expect(result).not.toContain('session=abc123');
+    });
+    it('leaves normal text unchanged', () => {
+        expect(redactText('Error: element not found')).toBe('Error: element not found');
+    });
+});
+describe('resolveAdapterSourcePath', () => {
+    it('returns source when it is a real file path (not manifest:)', () => {
+        const cmd = makeCmd({ source: '/home/user/.opencli/clis/arxiv/search.yaml' });
+        expect(resolveAdapterSourcePath(cmd)).toBe('/home/user/.opencli/clis/arxiv/search.yaml');
+    });
+    it('skips manifest: pseudo-paths and falls back to _modulePath', () => {
+        const cmd = makeCmd({ source: 'manifest:arxiv/search', _modulePath: '/pkg/dist/clis/arxiv/search.js' });
+        // Should try to map dist→source, but since files don't exist on disk, returns _modulePath
+        const result = resolveAdapterSourcePath(cmd);
+        expect(result).toBeDefined();
+        expect(result).not.toContain('manifest:');
+    });
+    it('returns undefined when only manifest: pseudo-path and no _modulePath', () => {
+        const cmd = makeCmd({ source: 'manifest:test/cmd' });
+        expect(resolveAdapterSourcePath(cmd)).toBeUndefined();
+    });
+    it('prefers _modulePath mapped to .ts over dist .js', () => {
+        // This test verifies the mapping logic without requiring files on disk
+        const cmd = makeCmd({ _modulePath: '/project/dist/clis/site/cmd.js' });
+        const result = resolveAdapterSourcePath(cmd);
+        // Since neither .ts nor .js exists, returns _modulePath as best guess
+        expect(result).toBe('/project/dist/clis/site/cmd.js');
+    });
+});
+describe('buildRepairContext', () => {
+    it('captures CliError fields', () => {
+        const err = new SelectorError('.missing-element', 'Element removed');
+        const ctx = buildRepairContext(err, makeCmd());
+        expect(ctx.error.code).toBe('SELECTOR');
+        expect(ctx.error.message).toContain('.missing-element');
+        expect(ctx.error.hint).toBe('Element removed');
+        expect(ctx.error.stack).toBeDefined();
+        expect(ctx.adapter.site).toBe('test-site');
+        expect(ctx.adapter.command).toBe('test-site/test-cmd');
+        expect(ctx.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/);
+    });
+    it('handles non-CliError errors', () => {
+        const err = new TypeError('Cannot read property "x" of undefined');
+        const ctx = buildRepairContext(err, makeCmd());
+        expect(ctx.error.code).toBe('UNKNOWN');
+        expect(ctx.error.message).toContain('Cannot read property');
+        expect(ctx.error.hint).toBeUndefined();
+    });
+    it('includes page state when provided', () => {
+        const pageState = {
+            url: 'https://example.com/page',
+            snapshot: '<div>...</div>',
+            networkRequests: [{ url: '/api/data', status: 200 }],
+            consoleErrors: ['Uncaught TypeError'],
+        };
+        const ctx = buildRepairContext(new CommandExecutionError('boom'), makeCmd(), pageState);
+        expect(ctx.page).toEqual(pageState);
+    });
+    it('omits page when not provided', () => {
+        const ctx = buildRepairContext(new Error('boom'), makeCmd());
+        expect(ctx.page).toBeUndefined();
+    });
+    it('truncates long stack traces', () => {
+        const err = new Error('boom');
+        err.stack = 'x'.repeat(10_000);
+        const ctx = buildRepairContext(err, makeCmd());
+        expect(ctx.error.stack.length).toBeLessThan(10_000);
+        expect(ctx.error.stack).toContain('truncated');
+    });
+    it('redacts sensitive data in error message and stack', () => {
+        const err = new Error('Request failed with Bearer eyJhbGciOiJIUzI1NiJ9.test.sig');
+        const ctx = buildRepairContext(err, makeCmd());
+        expect(ctx.error.message).toContain('Bearer [REDACTED]');
+        expect(ctx.error.message).not.toContain('eyJhbGci');
+        // Stack also gets redacted
+        expect(ctx.error.stack).toContain('Bearer [REDACTED]');
+    });
+});
+describe('emitDiagnostic', () => {
+    it('writes delimited JSON to stderr', () => {
+        const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true);
+        const ctx = buildRepairContext(new CommandExecutionError('test error'), makeCmd());
+        emitDiagnostic(ctx);
+        const output = writeSpy.mock.calls.map(c => c[0]).join('');
+        expect(output).toContain('___OPENCLI_DIAGNOSTIC___');
+        expect(output).toContain('"code":"COMMAND_EXEC"');
+        expect(output).toContain('"message":"test error"');
+        // Verify JSON is parseable between markers
+        const match = output.match(/___OPENCLI_DIAGNOSTIC___\n(.*)\n___OPENCLI_DIAGNOSTIC___/);
+        expect(match).toBeTruthy();
+        const parsed = JSON.parse(match[1]);
+        expect(parsed.error.code).toBe('COMMAND_EXEC');
+        writeSpy.mockRestore();
+    });
+    it('drops page snapshot when over size budget', () => {
+        const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true);
+        const ctx = {
+            error: { code: 'COMMAND_EXEC', message: 'boom' },
+            adapter: { site: 'test', command: 'test/cmd' },
+            page: {
+                url: 'https://example.com',
+                snapshot: 'x'.repeat(MAX_DIAGNOSTIC_BYTES + 1000),
+                networkRequests: [],
+                consoleErrors: [],
+            },
+            timestamp: new Date().toISOString(),
+        };
+        emitDiagnostic(ctx);
+        const output = writeSpy.mock.calls.map(c => c[0]).join('');
+        const match = output.match(/___OPENCLI_DIAGNOSTIC___\n(.*)\n___OPENCLI_DIAGNOSTIC___/);
+        expect(match).toBeTruthy();
+        const parsed = JSON.parse(match[1]);
+        // Page snapshot should be replaced or page dropped entirely
+        expect(parsed.page?.snapshot !== ctx.page.snapshot || parsed.page === undefined).toBe(true);
+        expect(match[1].length).toBeLessThanOrEqual(MAX_DIAGNOSTIC_BYTES);
+        writeSpy.mockRestore();
+    });
+    it('redacts sensitive headers in network requests', () => {
+        const pageState = {
+            url: 'https://example.com',
+            snapshot: '<div/>',
+            networkRequests: [{
+                    url: 'https://api.com/data?token=secret123',
+                    headers: { authorization: 'Bearer xyz', 'content-type': 'application/json' },
+                    body: '{"data": "ok"}',
+                }],
+            consoleErrors: [],
+        };
+        // Build context manually to test redaction via collectPageState
+        // Since collectPageState is private, test the output of buildRepairContext
+        // with already-collected page state — redaction happens in collectPageState.
+        // For unit test, verify redactUrl directly (tested above) and trust integration.
+        expect(redactUrl('https://api.com/data?token=secret123')).toContain('[REDACTED]');
+    });
+});

package/dist/src/discovery.js

@@ -15,6 +15,7 @@ import yaml from 'js-yaml';
 import { Strategy, registerCommand } from './registry.js';
 import { getErrorMessage } from './errors.js';
 import { log } from './logger.js';
+import { findPackageRoot, getCliManifestPath, getFetchAdaptersScriptPath } from './package-paths.js';
 /** User runtime directory: ~/.opencli */
 export const USER_OPENCLI_DIR = path.join(os.homedir(), '.opencli');
 /** User CLIs directory: ~/.opencli/clis */
@@ -31,18 +32,7 @@ function parseStrategy(rawStrategy, fallback = Strategy.COOKIE) {
     return Strategy[key] ?? fallback;
 }
 import { isRecord } from './utils.js';
-/**
- * Find the package root (directory containing package.json).
- * Dev: import.meta.url is in src/ → one level up.
- * Prod: import.meta.url is in dist/src/ → two levels up.
- */
-function findPackageRoot() {
-    let dir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-    if (!fs.existsSync(path.join(dir, 'package.json'))) {
-        dir = path.resolve(dir, '..');
-    }
-    return dir;
-}
+const PACKAGE_ROOT = findPackageRoot(fileURLToPath(import.meta.url));
 /**
  * Ensure ~/.opencli/node_modules/@jackwener/opencli symlink exists so that
  * user CLIs in ~/.opencli/clis/ can `import { cli } from '@jackwener/opencli/registry'`.
@@ -65,7 +55,7 @@ export async function ensureUserCliCompatShims(baseDir = USER_OPENCLI_DIR) {
         await fs.promises.writeFile(pkgJsonPath, pkgJsonContent, 'utf-8');
     }
     // Create node_modules/@jackwener/opencli symlink pointing to the installed package root.
-    const opencliRoot = findPackageRoot();
+    const opencliRoot = PACKAGE_ROOT;
     const symlinkDir = path.join(baseDir, 'node_modules', '@jackwener');
     const symlinkPath = path.join(symlinkDir, 'opencli');
     try {
@@ -116,7 +106,7 @@ export async function ensureUserAdapters() {
     log.info('First run detected — copying adapters (one-time setup)...');
     try {
         const { execFileSync } = await import('node:child_process');
-        const scriptPath = path.join(findPackageRoot(), 'scripts', 'fetch-adapters.js');
+        const scriptPath = getFetchAdaptersScriptPath(PACKAGE_ROOT);
         execFileSync(process.execPath, [scriptPath], {
             stdio: 'inherit',
             env: { ...process.env, _OPENCLI_FIRST_RUN: '1' },
@@ -135,7 +125,7 @@ export async function ensureUserAdapters() {
 export async function discoverClis(...dirs) {
     // Fast path: try manifest first (production / post-build)
     for (const dir of dirs) {
-        const manifestPath = path.resolve(dir, '..', 'cli-manifest.json');
+        const manifestPath = getCliManifestPath(dir);
         try {
             await fs.promises.access(manifestPath);
             const loaded = await loadFromManifest(manifestPath, dir);
@@ -173,7 +163,7 @@ async function loadFromManifest(manifestPath, clisDir) {
                     columns: entry.columns,
                     pipeline: entry.pipeline,
                     timeoutSeconds: entry.timeout,
-                    source: `manifest:${entry.site}/${entry.name}`,
+                    source: entry.sourceFile ? path.resolve(clisDir, entry.sourceFile) : `manifest:${entry.site}/${entry.name}`,
                     deprecated: entry.deprecated,
                     replacedBy: entry.replacedBy,
                     navigateBefore: entry.navigateBefore,
@@ -196,7 +186,7 @@ async function loadFromManifest(manifestPath, clisDir) {
                     args: entry.args ?? [],
                     columns: entry.columns,
                     timeoutSeconds: entry.timeout,
-                    source: modulePath,
+                    source: entry.sourceFile ? path.resolve(clisDir, entry.sourceFile) : modulePath,
                     deprecated: entry.deprecated,
                     replacedBy: entry.replacedBy,
                     navigateBefore: entry.navigateBefore,

package/dist/src/download/progress.js

@@ -23,8 +23,13 @@ export function formatDuration(ms) {
     if (seconds < 60)
         return `${seconds}s`;
     const minutes = Math.floor(seconds / 60);
-    const remainingSeconds = seconds % 60;
-    return `${minutes}m ${remainingSeconds}s`;
+    if (minutes < 60) {
+        const remainingSeconds = seconds % 60;
+        return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`;
+    }
+    const hours = Math.floor(minutes / 60);
+    const remainingMinutes = minutes % 60;
+    return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;
 }
 /**
  * Create a simple progress bar for terminal display.

package/dist/src/execution.js

@@ -13,6 +13,7 @@ import { Strategy, getRegistry, fullName } from './registry.js';
 import { pathToFileURL } from 'node:url';
 import { executePipeline } from './pipeline/index.js';
 import { AdapterLoadError, ArgumentError, BrowserConnectError, CommandExecutionError, getErrorMessage } from './errors.js';
+import { isDiagnosticEnabled, collectDiagnostic, emitDiagnostic } from './diagnostic.js';
 import { shouldUseBrowserSession } from './capabilityRouting.js';
 import { getBrowserFactory, browserSession, runWithTimeout, DEFAULT_BROWSER_COMMAND_TIMEOUT } from './runtime.js';
 import { emitHook } from './hooks.js';
@@ -129,6 +130,7 @@ export async function executeCommand(cmd, rawKwargs, debug = false) {
     };
     await emitHook('onBeforeExecute', hookCtx);
     let result;
+    let diagnosticEmitted = false;
     try {
         if (shouldUseBrowserSession(cmd)) {
             const electron = isElectronApp(cmd.site);
@@ -176,10 +178,22 @@ export async function executeCommand(cmd, rawKwargs, debug = false) {
                             log.debug(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`);
                     }
                 }
-                return runWithTimeout(runCommand(cmd, page, kwargs, debug), {
-                    timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT,
-                    label: fullName(cmd),
-                });
+                try {
+                    return await runWithTimeout(runCommand(cmd, page, kwargs, debug), {
+                        timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT,
+                        label: fullName(cmd),
+                    });
+                }
+                catch (err) {
+                    // Collect diagnostic while page is still alive (before browserSession closes it).
+                    if (isDiagnosticEnabled()) {
+                        const internal = cmd;
+                        const ctx = await collectDiagnostic(err, internal, page);
+                        emitDiagnostic(ctx);
+                        diagnosticEmitted = true;
+                    }
+                    throw err;
+                }
             }, { workspace: `site:${cmd.site}`, cdpEndpoint });
         }
         else {
@@ -198,6 +212,13 @@ export async function executeCommand(cmd, rawKwargs, debug = false) {
         }
     }
     catch (err) {
+        // Emit diagnostic if not already emitted (browser session emits with page state;
+        // this fallback covers non-browser commands and pre-session failures like BrowserConnectError).
+        if (isDiagnosticEnabled() && !diagnosticEmitted) {
+            const internal = cmd;
+            const ctx = await collectDiagnostic(err, internal, null);
+            emitDiagnostic(ctx);
+        }
         hookCtx.error = err;
         hookCtx.finishedAt = Date.now();
         await emitHook('onAfterExecute', hookCtx);

package/dist/src/explore.d.ts

@@ -12,7 +12,6 @@ interface InferredCapability {
     name: string;
     description: string;
     strategy: string;
-    confidence: number;
     endpoint: string;
     itemPath: string | null;
     recommendedColumns: string[];
@@ -52,7 +51,6 @@ export interface ExploreEndpointArtifact {
     url: string;
     status: number | null;
     contentType: string;
-    score: number;
     queryParams: string[];
     itemPath: string | null;
     itemCount: number;

package/dist/src/explore.js

@@ -13,7 +13,7 @@ import { detectFramework } from './scripts/framework.js';
 import { discoverStores } from './scripts/store.js';
 import { interactFuzz } from './scripts/interact.js';
 import { log } from './logger.js';
-import { urlToPattern, findArrayPath, flattenFields, detectFieldRoles, inferCapabilityName, inferStrategy, detectAuthFromHeaders, classifyQueryParams, } from './analysis.js';
+import { urlToPattern, findArrayPath, flattenFields, detectFieldRoles, inferCapabilityName, inferStrategy, detectAuthFromHeaders, classifyQueryParams, isNoiseUrl, } from './analysis.js';
 // ── Site name detection ────────────────────────────────────────────────────
 const KNOWN_SITE_ALIASES = {
     'x.com': 'twitter', 'twitter.com': 'twitter',
@@ -66,13 +66,29 @@ function parseNetworkRequests(raw) {
         return entries;
     }
     if (Array.isArray(raw)) {
-        return raw.filter(e => e && typeof e === 'object').map(e => ({
-            method: (e.method ?? 'GET').toUpperCase(),
-            url: String(e.url ?? e.request?.url ?? e.requestUrl ?? ''),
-            status: e.status ?? e.statusCode ?? null,
-            contentType: e.contentType ?? e.response?.contentType ?? '',
-            responseBody: e.responseBody, requestHeaders: e.requestHeaders,
-        }));
+        return raw.filter(e => e && typeof e === 'object').map(e => {
+            // Handle both legacy shape (status/contentType/responseBody) and
+            // extension/CDP capture shape (responseStatus/responseContentType/responsePreview)
+            let body = e.responseBody;
+            if (body === undefined && e.responsePreview !== undefined) {
+                const preview = e.responsePreview;
+                if (typeof preview === 'string') {
+                    try {
+                        body = JSON.parse(preview);
+                    }
+                    catch {
+                        body = preview;
+                    }
+                }
+            }
+            return {
+                method: (e.method ?? 'GET').toUpperCase(),
+                url: String(e.url ?? e.request?.url ?? e.requestUrl ?? ''),
+                status: e.status ?? e.responseStatus ?? e.statusCode ?? null,
+                contentType: e.contentType ?? e.responseContentType ?? e.response?.contentType ?? '',
+                responseBody: body, requestHeaders: e.requestHeaders,
+            };
+        });
     }
     return [];
 }
@@ -91,29 +107,32 @@ function isBooleanRecord(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value)
         && Object.values(value).every(v => typeof v === 'boolean');
 }
-function scoreEndpoint(ep) {
-    let s = 0;
-    if (ep.contentType.includes('json'))
-        s += 10;
-    if (ep.responseAnalysis) {
-        s += 5;
-        s += Math.min(ep.responseAnalysis.itemCount, 10);
-        s += Object.keys(ep.responseAnalysis.detectedFields).length * 2;
-    }
+/**
+ * Deterministic sort key for endpoint ordering — transparent, observable signals only.
+ * Used by generate/synthesize to pick a stable default candidate.
+ * Not exposed externally; AI agents see the raw metadata and decide for themselves.
+ */
+function endpointSortKey(ep) {
+    let k = 0;
+    // Prefer endpoints with array data (list APIs are more useful for automation)
+    const items = ep.responseAnalysis?.itemCount ?? 0;
+    if (items > 0)
+        k += 100 + Math.min(items, 50);
+    // Prefer endpoints with detected semantic fields
+    k += Object.keys(ep.responseAnalysis?.detectedFields ?? {}).length * 10;
+    // Prefer API-style paths
     if (ep.pattern.includes('/api/') || ep.pattern.includes('/x/'))
-        s += 3;
-    if (ep.hasSearchParam)
-        s += 3;
-    if (ep.hasPaginationParam)
-        s += 2;
-    if (ep.hasLimitParam)
-        s += 2;
-    if (ep.status === 200)
-        s += 2;
-    // Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
-    if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json'))
-        s -= 3;
-    return s;
+        k += 5;
+    // Prefer endpoints with query params (more likely to be parameterized APIs)
+    if (ep.hasSearchParam || ep.hasPaginationParam || ep.hasLimitParam)
+        k += 5;
+    return k;
+}
+/** Check whether an endpoint carries useful structured data (any JSON response, not noise). */
+function isUsefulEndpoint(ep) {
+    if (isNoiseUrl(ep.url))
+        return false;
+    return ep.contentType.includes('json');
 }
 // ── Framework detection ────────────────────────────────────────────────────
 const FRAMEWORK_DETECT_JS = detectFramework.toString();
@@ -122,7 +141,7 @@ const STORE_DISCOVER_JS = discoverStores.toString();
 // ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
 const INTERACT_FUZZ_JS = interactFuzz.toString();
 // ── Analysis helpers (extracted from exploreUrl) ───────────────────────────
-/** Filter, deduplicate, and score network endpoints. */
+/** Filter and deduplicate network endpoints, keeping only useful structured-data APIs. */
 function analyzeEndpoints(networkEntries) {
     const seen = new Map();
     for (const entry of networkEntries) {
@@ -145,12 +164,13 @@ function analyzeEndpoints(networkEntries) {
             hasLimitParam: hasLimit || qp.some(p => LIMIT_PARAMS.has(p)),
             authIndicators: detectAuthFromHeaders(entry.requestHeaders),
             responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null,
-            score: 0,
         };
-        ep.score = scoreEndpoint(ep);
         seen.set(key, ep);
     }
-    const analyzed = [...seen.values()].filter(ep => ep.score >= 5).sort((a, b) => b.score - a.score);
+    // Filter to useful endpoints; deterministic ordering by observable metadata signals
+    const analyzed = [...seen.values()]
+        .filter(isUsefulEndpoint)
+        .sort((a, b) => endpointSortKey(b) - endpointSortKey(a));
     return { analyzed, totalCount: seen.size };
 }
 /** Infer CLI capabilities from analyzed endpoints. */
@@ -192,7 +212,7 @@ function inferCapabilitiesFromEndpoints(endpoints, stores, opts) {
         capabilities.push({
             name: capName, description: `${opts.site ?? detectSiteName(opts.url)} ${capName}`,
             strategy: storeHint ? 'store-action' : epStrategy,
-            confidence: Math.min(ep.score / 20, 1.0), endpoint: ep.pattern,
+            endpoint: ep.pattern,
             itemPath: ep.responseAnalysis?.itemPath ?? null,
             recommendedColumns: cols.length ? cols : ['title', 'url'],
             recommendedArgs: args,
@@ -216,7 +236,7 @@ async function writeExploreArtifacts(targetDir, result, analyzedEndpoints, store
         }, null, 2)),
         fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
             pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
-            contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
+            contentType: ep.contentType, queryParams: ep.queryParams,
             itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
             detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
         })), null, 2)),
@@ -237,6 +257,7 @@ export async function exploreUrl(url, opts) {
     return browserSession(opts.BrowserFactory, async (page) => {
         return runWithTimeout((async () => {
             // Step 1: Navigate
+            await page.startNetworkCapture?.();
             await page.goto(url);
             await page.wait(waitSeconds);
             // Step 2: Auto-scroll to trigger lazy loading intelligently
@@ -269,7 +290,9 @@ export async function exploreUrl(url, opts) {
             // Step 3: Read page metadata
             const metadata = await readPageMetadata(page);
             // Step 4: Capture network traffic
-            const rawNetwork = await page.networkRequests(false);
+            const rawNetwork = page.readNetworkCapture
+                ? await page.readNetworkCapture()
+                : await page.networkRequests(false);
             const networkEntries = parseNetworkRequests(rawNetwork);
             // Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe
             const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody);
@@ -348,7 +371,7 @@ export function renderExploreSummary(result) {
     ];
     for (const cap of (result.capabilities ?? []).slice(0, 5)) {
         const storeInfo = cap.storeHint ? ` → ${cap.storeHint.store}.${cap.storeHint.action}()` : '';
-        lines.push(`  • ${cap.name} (${cap.strategy}, ${(cap.confidence * 100).toFixed(0)}%)${storeInfo}`);
+        lines.push(`  • ${cap.name} (${cap.strategy})${storeInfo}`);
     }
     const fw = result.framework ?? {};
     const fwNames = Object.entries(fw).filter(([, v]) => v).map(([k]) => k);

package/dist/src/extension-manifest-regression.test.js

@@ -7,7 +7,6 @@ describe('extension manifest regression', () => {
         const raw = await fs.readFile(manifestPath, 'utf8');
         const manifest = JSON.parse(raw);
         expect(manifest.permissions).toContain('cookies');
-        expect(manifest.permissions).toContain('scripting');
         expect(manifest.host_permissions).toContain('<all_urls>');
     });
 });

package/dist/src/generate.d.ts

@@ -34,7 +34,7 @@ export interface GenerateCliResult {
     };
     synthesize: {
         candidate_count: number;
-        candidates: Array<Pick<SynthesizeCandidateSummary, 'name' | 'strategy' | 'confidence'>>;
+        candidates: Array<Pick<SynthesizeCandidateSummary, 'name' | 'strategy'>>;
     };
 }
 export declare function generateCliFromUrl(opts: GenerateCliOptions): Promise<GenerateCliResult>;

package/dist/src/generate.js

@@ -40,7 +40,7 @@ function selectCandidate(candidates, goal) {
     if (!candidates.length)
         return null;
     if (!goal)
-        return candidates[0]; // highest confidence first
+        return candidates[0];
     const normalized = normalizeGoal(goal);
     if (normalized) {
         const exact = candidates.find(c => c.name === normalized);
@@ -90,7 +90,6 @@ export async function generateCliFromUrl(opts) {
             candidates: (synthesizeResult.candidates ?? []).map((c) => ({
                 name: c.name,
                 strategy: c.strategy,
-                confidence: c.confidence,
             })),
         },
     };
@@ -111,7 +110,7 @@ export function renderGenerateSummary(r) {
         `  Candidates: ${r.synthesize?.candidate_count ?? 0}`,
     ];
     for (const c of r.synthesize?.candidates ?? []) {
-        lines.push(`    • ${c.name} (${c.strategy}, ${((c.confidence ?? 0) * 100).toFixed(0)}%)`);
+        lines.push(`    • ${c.name} (${c.strategy})`);
     }
     const fw = r.explore?.framework ?? {};
     const fwNames = Object.entries(fw).filter(([, v]) => v).map(([k]) => k);

package/dist/src/package-paths.d.ts

@@ -0,0 +1,8 @@
+export interface PackageJsonLike {
+    bin?: string | Record<string, string>;
+    main?: string;
+}
+export declare function findPackageRoot(startFile: string, fileExists?: (candidate: string) => boolean): string;
+export declare function getBuiltEntryCandidates(packageRoot: string, readFile?: (filePath: string) => string): string[];
+export declare function getCliManifestPath(clisDir: string): string;
+export declare function getFetchAdaptersScriptPath(packageRoot: string): string;

package/dist/src/package-paths.js

@@ -0,0 +1,41 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+export function findPackageRoot(startFile, fileExists = fs.existsSync) {
+    let dir = path.dirname(startFile);
+    while (true) {
+        if (fileExists(path.join(dir, 'package.json')))
+            return dir;
+        const parent = path.dirname(dir);
+        if (parent === dir) {
+            throw new Error(`Could not find package.json above ${startFile}`);
+        }
+        dir = parent;
+    }
+}
+export function getBuiltEntryCandidates(packageRoot, readFile = (filePath) => fs.readFileSync(filePath, 'utf-8')) {
+    const candidates = [];
+    try {
+        const pkg = JSON.parse(readFile(path.join(packageRoot, 'package.json')));
+        if (typeof pkg.bin === 'string') {
+            candidates.push(path.join(packageRoot, pkg.bin));
+        }
+        else if (pkg.bin && typeof pkg.bin === 'object' && typeof pkg.bin.opencli === 'string') {
+            candidates.push(path.join(packageRoot, pkg.bin.opencli));
+        }
+        if (typeof pkg.main === 'string') {
+            candidates.push(path.join(packageRoot, pkg.main));
+        }
+    }
+    catch {
+        // Fall through to compatibility candidates below.
+    }
+    // Compatibility fallback for partially-built trees or older layouts.
+    candidates.push(path.join(packageRoot, 'dist', 'src', 'main.js'), path.join(packageRoot, 'dist', 'main.js'));
+    return [...new Set(candidates)];
+}
+export function getCliManifestPath(clisDir) {
+    return path.resolve(clisDir, '..', 'cli-manifest.json');
+}
+export function getFetchAdaptersScriptPath(packageRoot) {
+    return path.join(packageRoot, 'scripts', 'fetch-adapters.js');
+}

package/dist/src/plugin-scaffold.js

@@ -68,9 +68,7 @@ pipeline:
   - fetch:
       url: "https://httpbin.org/get?greeting=hello"
       method: GET
-  - extract:
-      type: json
-      selector: "$.args"
+  - select: "args"
 `;
     writeFile(targetDir, 'hello.yaml', yamlContent);
     files.push('hello.yaml');