@jackwener/opencli 1.3.2 → 1.4.0

package/src/download/index.ts

@@ -11,12 +11,17 @@ import * as os from 'node:os';
 import { URL } from 'node:url';
 import type { ProgressBar } from './progress.js';
 import { isBinaryInstalled } from '../external.js';
+import type { BrowserCookie } from '../types.js';
+import { getErrorMessage } from '../errors.js';
+
+export type { BrowserCookie } from '../types.js';
 
 export interface DownloadOptions {
   cookies?: string;
   headers?: Record<string, string>;
   timeout?: number;
   onProgress?: (received: number, total: number) => void;
+  maxRedirects?: number;
 }
 
 export interface YtdlpOptions {
@@ -27,26 +32,11 @@ export interface YtdlpOptions {
   onProgress?: (percent: number) => void;
 }
 
-export interface BrowserCookie {
-  name: string;
-  value: string;
-  domain: string;
-  path?: string;
-  secure?: boolean;
-  httpOnly?: boolean;
-  expirationDate?: number;
-}
-
 /** Check if yt-dlp is available in PATH. */
 export function checkYtdlp(): boolean {
   return isBinaryInstalled('yt-dlp');
 }
 
-/** Check if ffmpeg is available in PATH. */
-export function checkFfmpeg(): boolean {
-  return isBinaryInstalled('ffmpeg');
-}
-
 /** Domains that host video content and can be downloaded via yt-dlp. */
 const VIDEO_PLATFORM_DOMAINS = [
   'youtube.com', 'youtu.be', 'bilibili.com', 'twitter.com',
@@ -92,15 +82,16 @@ export async function httpDownload(
   url: string,
   destPath: string,
   options: DownloadOptions = {},
+  redirectCount = 0,
 ): Promise<{ success: boolean; size: number; error?: string }> {
-  const { cookies, headers = {}, timeout = 30000, onProgress } = options;
+  const { cookies, headers = {}, timeout = 30000, onProgress, maxRedirects = 10 } = options;
 
   return new Promise((resolve) => {
     const parsedUrl = new URL(url);
     const protocol = parsedUrl.protocol === 'https:' ? https : http;
 
     const requestHeaders: Record<string, string> = {
-      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
       ...headers,
     };
 
@@ -120,7 +111,23 @@ export async function httpDownload(
       if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
         file.close();
         if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
-        httpDownload(resolveRedirectUrl(url, response.headers.location), destPath, options).then(resolve);
+        if (redirectCount >= maxRedirects) {
+          resolve({ success: false, size: 0, error: `Too many redirects (> ${maxRedirects})` });
+          return;
+        }
+        const redirectUrl = resolveRedirectUrl(url, response.headers.location);
+        const originalHost = new URL(url).hostname;
+        const redirectHost = new URL(redirectUrl).hostname;
+        // Do not forward cookies when a redirect crosses host boundaries.
+        const redirectOptions = originalHost === redirectHost
+          ? options
+          : { ...options, cookies: undefined, headers: stripCookieHeaders(options.headers) };
+        httpDownload(
+          redirectUrl,
+          destPath,
+          redirectOptions,
+          redirectCount + 1,
+        ).then(resolve);
         return;
       }
 
@@ -168,6 +175,13 @@ export function resolveRedirectUrl(currentUrl: string, location: string): string
   return new URL(location, currentUrl).toString();
 }
 
+function stripCookieHeaders(headers?: Record<string, string>): Record<string, string> | undefined {
+  if (!headers) return headers;
+  return Object.fromEntries(
+    Object.entries(headers).filter(([key]) => key.toLowerCase() !== 'cookie'),
+  );
+}
+
 /**
  * Export cookies to Netscape format for yt-dlp.
  */
@@ -188,7 +202,9 @@ export function exportCookiesToNetscape(
     const cookiePath = cookie.path || '/';
     const secure = cookie.secure ? 'TRUE' : 'FALSE';
     const expiry = Math.floor(Date.now() / 1000) + 86400 * 365; // 1 year from now
-    lines.push(`${domain}\t${includeSubdomains}\t${cookiePath}\t${secure}\t${expiry}\t${cookie.name}\t${cookie.value}`);
+    const safeName = cookie.name.replace(/[\t\n\r]/g, '');
+    const safeValue = cookie.value.replace(/[\t\n\r]/g, '');
+    lines.push(`${domain}\t${includeSubdomains}\t${cookiePath}\t${secure}\t${expiry}\t${safeName}\t${safeValue}`);
   }
 
   fs.mkdirSync(path.dirname(filePath), { recursive: true });
@@ -226,8 +242,13 @@ export async function ytdlpDownload(
       '--progress',
     ];
 
-    if (cookiesFile && fs.existsSync(cookiesFile)) {
-      args.push('--cookies', cookiesFile);
+    if (cookiesFile) {
+      if (fs.existsSync(cookiesFile)) {
+        args.push('--cookies', cookiesFile);
+      } else {
+        console.error(`[download] Cookies file not found: ${cookiesFile}, falling back to browser cookies`);
+        args.push('--cookies-from-browser', 'chrome');
+      }
     } else {
       // Try to use browser cookies
       args.push('--cookies-from-browser', 'chrome');
@@ -319,8 +340,8 @@ export async function saveDocument(
 
     fs.writeFileSync(destPath, output, 'utf-8');
     return { success: true, size: Buffer.byteLength(output, 'utf-8') };
-  } catch (err: any) {
-    return { success: false, size: 0, error: err.message };
+  } catch (err) {
+    return { success: false, size: 0, error: getErrorMessage(err) };
   }
 }
 

package/src/download/media-download.ts

@@ -9,6 +9,7 @@
 
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { getErrorMessage } from '../errors.js';
 import {
   httpDownload,
   ytdlpDownload,
@@ -153,15 +154,16 @@ export async function downloadMedia(
           status: result.success ? 'success' : 'failed',
           size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
         });
-      } catch (err: any) {
-        if (progressBar) progressBar.fail(err.message);
+      } catch (err) {
+        const msg = getErrorMessage(err);
+        if (progressBar) progressBar.fail(msg);
         tracker.onFileComplete(false);
 
         results.push({
           index: i + 1,
           type: media.type,
           status: 'failed',
-          size: err.message,
+          size: msg,
         });
       }
     }

package/src/engine.test.ts

@@ -2,17 +2,20 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
 import { discoverClis, discoverPlugins, PLUGINS_DIR } from './discovery.js';
 import { executeCommand } from './execution.js';
 import { getRegistry, cli, Strategy } from './registry.js';
+import { clearAllHooks, onAfterExecute } from './hooks.js';
 import * as fs from 'node:fs';
+import * as os from 'node:os';
 import * as path from 'node:path';
+import { pathToFileURL } from 'node:url';
 
 describe('discoverClis', () => {
   it('handles non-existent directories gracefully', async () => {
     // Should not throw for missing directories
-    await expect(discoverClis('/tmp/nonexistent-opencli-test-dir')).resolves.not.toThrow();
+    await expect(discoverClis(path.join(os.tmpdir(), 'nonexistent-opencli-test-dir'))).resolves.not.toThrow();
   });
 
   it('imports only CLI command modules during filesystem discovery', async () => {
-    const tempRoot = await fs.promises.mkdtemp(path.join('/tmp', 'opencli-discovery-'));
+    const tempRoot = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-discovery-'));
     const siteDir = path.join(tempRoot, 'temp-site');
     const helperPath = path.join(siteDir, 'helper.ts');
     const commandPath = path.join(siteDir, 'hello.ts');
@@ -24,7 +27,7 @@ globalThis.__opencli_helper_loaded__ = true;
 export const helper = true;
 `);
       await fs.promises.writeFile(commandPath, `
-import { cli, Strategy } from '${path.join(process.cwd(), 'src', 'registry.ts')}';
+import { cli, Strategy } from '${pathToFileURL(path.join(process.cwd(), 'src', 'registry.ts')).href}';
 cli({
   site: 'temp-site',
   name: 'hello',
@@ -45,6 +48,36 @@ cli({
       await fs.promises.rm(tempRoot, { recursive: true, force: true });
     }
   });
+
+  it('falls back to filesystem discovery when the manifest is invalid', async () => {
+    const tempBuildRoot = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-manifest-fallback-'));
+    const distDir = path.join(tempBuildRoot, 'dist');
+    const siteDir = path.join(distDir, 'fallback-site');
+    const commandPath = path.join(siteDir, 'hello.ts');
+    const manifestPath = path.join(tempBuildRoot, 'cli-manifest.json');
+
+    try {
+      await fs.promises.mkdir(siteDir, { recursive: true });
+      await fs.promises.writeFile(manifestPath, '{ invalid json');
+      await fs.promises.writeFile(commandPath, `
+import { cli, Strategy } from '${pathToFileURL(path.join(process.cwd(), 'src', 'registry.ts')).href}';
+cli({
+  site: 'fallback-site',
+  name: 'hello',
+  description: 'hello command',
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  func: async () => [{ ok: true }],
+});
+`);
+
+      await discoverClis(distDir);
+
+      expect(getRegistry().get('fallback-site/hello')).toBeDefined();
+    } finally {
+      await fs.promises.rm(tempBuildRoot, { recursive: true, force: true });
+    }
+  });
 });
 
 describe('discoverPlugins', () => {
@@ -88,6 +121,11 @@ columns: [message]
 });
 
 describe('executeCommand', () => {
+  beforeEach(() => {
+    clearAllHooks();
+    vi.unstubAllEnvs();
+  });
+
   it('accepts kebab-case option names after Commander camelCases them', async () => {
     const cmd = cli({
       site: 'test-engine',
@@ -165,4 +203,47 @@ describe('executeCommand', () => {
     await executeCommand(cmd, {}, true);
     expect(receivedDebug).toBe(true);
   });
+
+  it('fires onAfterExecute even when command execution throws', async () => {
+    const seen: Array<{ error?: unknown; finishedAt?: number }> = [];
+    onAfterExecute((ctx) => {
+      seen.push({ error: ctx.error, finishedAt: ctx.finishedAt });
+    });
+
+    const cmd = cli({
+      site: 'test-engine',
+      name: 'failing-test',
+      description: 'failing command',
+      browser: false,
+      strategy: Strategy.PUBLIC,
+      func: async () => {
+        throw new Error('boom');
+      },
+    });
+
+    await expect(executeCommand(cmd, {})).rejects.toThrow('boom');
+    expect(seen).toHaveLength(1);
+    expect(seen[0].error).toBeInstanceOf(Error);
+    expect((seen[0].error as Error).message).toBe('boom');
+    expect(typeof seen[0].finishedAt).toBe('number');
+  });
+
+  it('fails fast for chatwise commands when OPENCLI_CDP_ENDPOINT is missing', async () => {
+    const cmd = cli({
+      site: 'chatwise',
+      name: 'status',
+      description: 'chatwise status',
+      browser: true,
+      strategy: Strategy.PUBLIC,
+      requiredEnv: [
+        {
+          name: 'OPENCLI_CDP_ENDPOINT',
+          help: 'Set OPENCLI_CDP_ENDPOINT before running chatwise commands.',
+        },
+      ],
+      func: async () => [{ ok: true }],
+    });
+
+    await expect(executeCommand(cmd, {})).rejects.toThrow('requires environment variable OPENCLI_CDP_ENDPOINT');
+  });
 });

package/src/execution.ts

@@ -7,6 +7,7 @@
  * 3. Domain pre-navigation for cookie/header strategies
  * 4. Timeout enforcement
  * 5. Lazy-loading of TS modules from manifest
+ * 6. Lifecycle hooks (onBeforeExecute / onAfterExecute)
  */
 
 import { type CliCommand, type InternalCliCommand, type Arg, Strategy, getRegistry, fullName } from './registry.js';
@@ -16,22 +17,17 @@ import { executePipeline } from './pipeline/index.js';
 import { AdapterLoadError, ArgumentError, CommandExecutionError, getErrorMessage } from './errors.js';
 import { shouldUseBrowserSession } from './capabilityRouting.js';
 import { getBrowserFactory, browserSession, runWithTimeout, DEFAULT_BROWSER_COMMAND_TIMEOUT } from './runtime.js';
+import { emitHook, type HookContext } from './hooks.js';
 
-/** Set of TS module paths that have been loaded */
 const _loadedModules = new Set<string>();
 type CommandArgs = Record<string, unknown>;
 
-
-/**
- * Validates and coerces arguments based on the command's Arg definitions.
- */
 export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): CommandArgs {
   const result: CommandArgs = { ...kwargs };
 
   for (const argDef of cmdArgs) {
     const val = result[argDef.name];
-    
-    // 1. Check required
+
     if (argDef.required && (val === undefined || val === null || val === '')) {
       throw new ArgumentError(
         `Argument "${argDef.name}" is required.`,
@@ -40,7 +36,6 @@ export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): Comm
     }
 
     if (val !== undefined && val !== null) {
-      // 2. Type coercion
       if (argDef.type === 'int' || argDef.type === 'number') {
         const num = Number(val);
         if (Number.isNaN(num)) {
@@ -58,7 +53,6 @@ export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): Comm
         }
       }
 
-      // 3. Choices validation
       const coercedVal = result[argDef.name];
       if (argDef.choices && argDef.choices.length > 0) {
         if (!argDef.choices.map(String).includes(String(coercedVal))) {
@@ -72,16 +66,12 @@ export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): Comm
   return result;
 }
 
-/**
- * Run a command's func or pipeline against a page.
- */
 async function runCommand(
   cmd: CliCommand,
   page: IPage | null,
   kwargs: CommandArgs,
   debug: boolean,
 ): Promise<unknown> {
-  // Lazy-load TS module on first execution (manifest fast-path)
   const internal = cmd as InternalCliCommand;
   if (internal._lazy && internal._modulePath) {
     const modulePath = internal._modulePath;
@@ -96,13 +86,18 @@ async function runCommand(
         );
       }
     }
-    // After loading, the module's cli() call will have updated the registry.
+
     const updated = getRegistry().get(fullName(cmd));
-    if (updated?.func) return updated.func(page!, kwargs, debug);
+    if (updated?.func) {
+      if (!page && updated.browser !== false) {
+        throw new CommandExecutionError(`Command ${fullName(cmd)} requires a browser session but none was provided`);
+      }
+      return updated.func(page as IPage, kwargs, debug);
+    }
     if (updated?.pipeline) return executePipeline(page, updated.pipeline, { args: kwargs, debug });
   }
 
-  if (cmd.func) return cmd.func(page!, kwargs, debug);
+  if (cmd.func) return cmd.func(page as IPage, kwargs, debug);
   if (cmd.pipeline) return executePipeline(page, cmd.pipeline, { args: kwargs, debug });
   throw new CommandExecutionError(
     `Command ${fullName(cmd)} has no func or pipeline`,
@@ -110,30 +105,29 @@ async function runCommand(
   );
 }
 
-/**
- * Resolve the pre-navigation URL for a command, or null to skip.
- *
- * COOKIE/HEADER strategies need the browser on the target domain so
- * `fetch(url, { credentials: 'include' })` carries cookies.
- * Adapters that handle their own navigation set `navigateBefore: false`.
- */
 function resolvePreNav(cmd: CliCommand): string | null {
   if (cmd.navigateBefore === false) return null;
   if (typeof cmd.navigateBefore === 'string') return cmd.navigateBefore;
 
-  // Default: pre-navigate for COOKIE/HEADER strategies with a domain
   if ((cmd.strategy === Strategy.COOKIE || cmd.strategy === Strategy.HEADER) && cmd.domain) {
     return `https://${cmd.domain}`;
   }
   return null;
 }
 
-/**
- * Execute a CLI command. Automatically manages browser sessions when needed.
- *
- * This is the unified entry point — callers don't need to care about
- * whether the command requires a browser or not.
- */
+function ensureRequiredEnv(cmd: CliCommand): void {
+  const missing = (cmd.requiredEnv ?? []).find(({ name }) => {
+    const value = process.env[name];
+    return value === undefined || value === null || value === '';
+  });
+  if (!missing) return;
+
+  throw new CommandExecutionError(
+    `Command ${fullName(cmd)} requires environment variable ${missing.name}.`,
+    missing.help ?? `Set ${missing.name} before running ${fullName(cmd)}.`,
+  );
+}
+
 export async function executeCommand(
   cmd: CliCommand,
   rawKwargs: CommandArgs,
@@ -147,22 +141,44 @@ export async function executeCommand(
     throw new ArgumentError(getErrorMessage(err));
   }
 
-  if (shouldUseBrowserSession(cmd)) {
-    const BrowserFactory = getBrowserFactory();
-    return browserSession(BrowserFactory, async (page) => {
-      // Pre-navigate to target domain for cookie/header context if needed.
-      // Each adapter controls this via `navigateBefore` (see CliCommand docs).
-      const preNavUrl = resolvePreNav(cmd);
-      if (preNavUrl) {
-        try { await page.goto(preNavUrl); await page.wait(2); } catch {}
-      }
-      return runWithTimeout(runCommand(cmd, page, kwargs, debug), {
-        timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT,
-        label: fullName(cmd),
-      });
-    }, { workspace: `site:${cmd.site}` });
+  const hookCtx: HookContext = {
+    command: fullName(cmd),
+    args: kwargs,
+    startedAt: Date.now(),
+  };
+  await emitHook('onBeforeExecute', hookCtx);
+
+  let result: unknown;
+  try {
+    if (shouldUseBrowserSession(cmd)) {
+      ensureRequiredEnv(cmd);
+      const BrowserFactory = getBrowserFactory();
+      result = await browserSession(BrowserFactory, async (page) => {
+        const preNavUrl = resolvePreNav(cmd);
+        if (preNavUrl) {
+          try {
+            await page.goto(preNavUrl);
+            await page.wait(2);
+          } catch (err) {
+            if (debug) console.error(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`);
+          }
+        }
+        return runWithTimeout(runCommand(cmd, page, kwargs, debug), {
+          timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT,
+          label: fullName(cmd),
+        });
+      }, { workspace: `site:${cmd.site}` });
+    } else {
+      result = await runCommand(cmd, null, kwargs, debug);
+    }
+  } catch (err) {
+    hookCtx.error = err;
+    hookCtx.finishedAt = Date.now();
+    await emitHook('onAfterExecute', hookCtx);
+    throw err;
   }
 
-  // Non-browser commands run directly
-  return runCommand(cmd, null, kwargs, debug);
+  hookCtx.finishedAt = Date.now();
+  await emitHook('onAfterExecute', hookCtx, result);
+  return result;
 }

package/src/explore.ts

@@ -10,12 +10,22 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
 import type { IBrowserFactory } from './runtime.js';
-import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
+import { LIMIT_PARAMS } from './constants.js';
 import { detectFramework } from './scripts/framework.js';
 import { discoverStores } from './scripts/store.js';
 import { interactFuzz } from './scripts/interact.js';
 import type { IPage } from './types.js';
 import { log } from './logger.js';
+import {
+  urlToPattern,
+  findArrayPath,
+  flattenFields,
+  detectFieldRoles,
+  inferCapabilityName,
+  inferStrategy,
+  detectAuthFromHeaders,
+  classifyQueryParams,
+} from './analysis.js';
 
 // ── Site name detection ────────────────────────────────────────────────────
 
@@ -48,10 +58,6 @@ export function slugify(value: string): string {
   return value.trim().toLowerCase().replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-|-$/g, '') || 'site';
 }
 
-// ── Field & capability inference ───────────────────────────────────────────
-
-// (constants now imported from constants.ts)
-
 // ── Network analysis ───────────────────────────────────────────────────────
 
 interface NetworkEntry {
@@ -160,68 +166,16 @@ function parseNetworkRequests(raw: unknown): NetworkEntry[] {
   return [];
 }
 
-function urlToPattern(url: string): string {
-  try {
-    const p = new URL(url);
-    const pathNorm = p.pathname.replace(/\/\d+/g, '/{id}').replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}').replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}');
-    const params: string[] = [];
-    p.searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) params.push(k); });
-    return `${p.host}${pathNorm}${params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : ''}`;
-  } catch { return url; }
-}
-
-function detectAuthIndicators(headers?: Record<string, string>): string[] {
-  if (!headers) return [];
-  const indicators: string[] = [];
-  const keys = Object.keys(headers).map(k => k.toLowerCase());
-  if (keys.some(k => k === 'authorization')) indicators.push('bearer');
-  if (keys.some(k => k.startsWith('x-csrf') || k.startsWith('x-xsrf'))) indicators.push('csrf');
-  if (keys.some(k => k.startsWith('x-s') || k === 'x-t' || k === 'x-s-common')) indicators.push('signature');
-  return indicators;
-}
-
 function analyzeResponseBody(body: unknown): AnalyzedEndpoint['responseAnalysis'] {
   if (!body || typeof body !== 'object') return null;
-  const candidates: Array<{ path: string; items: unknown[] }> = [];
-
-  function findArrays(obj: unknown, path: string, depth: number) {
-    if (depth > 4) return;
-    if (Array.isArray(obj) && obj.length >= 2 && obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) {
-      candidates.push({ path, items: obj });
-    }
-    if (obj && typeof obj === 'object' && !Array.isArray(obj)) {
-      for (const [key, val] of Object.entries(obj)) findArrays(val, path ? `${path}.${key}` : key, depth + 1);
-    }
-  }
-  findArrays(body, '', 0);
-  if (!candidates.length) return null;
+  const result = findArrayPath(body);
+  if (!result) return null;
 
-  candidates.sort((a, b) => b.items.length - a.items.length);
-  const best = candidates[0];
-  const sample = best.items[0];
+  const sample = result.items[0];
   const sampleFields = sample && typeof sample === 'object' ? flattenFields(sample, '', 2) : [];
+  const detectedFields = detectFieldRoles(sampleFields);
 
-  const detectedFields: Record<string, string> = {};
-  for (const [role, aliases] of Object.entries(FIELD_ROLES)) {
-    for (const f of sampleFields) {
-      if (aliases.includes(f.split('.').pop()?.toLowerCase() ?? '')) { detectedFields[role] = f; break; }
-    }
-  }
-
-  return { itemPath: best.path || null, itemCount: best.items.length, detectedFields, sampleFields };
-}
-
-function flattenFields(obj: unknown, prefix: string, maxDepth: number): string[] {
-  if (maxDepth <= 0 || !obj || typeof obj !== 'object') return [];
-  const names: string[] = [];
-  const record = obj as Record<string, unknown>;
-  for (const key of Object.keys(record)) {
-    const full = prefix ? `${prefix}.${key}` : key;
-    names.push(full);
-    const val = record[key];
-    if (val && typeof val === 'object' && !Array.isArray(val)) names.push(...flattenFields(val, full, maxDepth - 1));
-  }
-  return names;
+  return { itemPath: result.path || null, itemCount: result.items.length, detectedFields, sampleFields };
 }
 
 function isBooleanRecord(value: unknown): value is Record<string, boolean> {
@@ -243,28 +197,6 @@ function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndp
   return s;
 }
 
-function inferCapabilityName(url: string, goal?: string): string {
-  if (goal) return goal;
-  const u = url.toLowerCase();
-  if (u.includes('hot') || u.includes('popular') || u.includes('ranking') || u.includes('trending')) return 'hot';
-  if (u.includes('search')) return 'search';
-  if (u.includes('feed') || u.includes('timeline') || u.includes('dynamic')) return 'feed';
-  if (u.includes('comment') || u.includes('reply')) return 'comments';
-  if (u.includes('history')) return 'history';
-  if (u.includes('profile') || u.includes('userinfo') || u.includes('/me')) return 'me';
-  if (u.includes('favorite') || u.includes('collect') || u.includes('bookmark')) return 'favorite';
-  try {
-    const segs = new URL(url).pathname.split('/').filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i));
-    if (segs.length) return segs[segs.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase();
-  } catch {}
-  return 'data';
-}
-
-function inferStrategy(authIndicators: string[]): string {
-  if (authIndicators.includes('signature')) return 'intercept';
-  if (authIndicators.includes('bearer') || authIndicators.includes('csrf')) return 'header';
-  return 'cookie';
-}
 
 // ── Framework detection ────────────────────────────────────────────────────
 
@@ -300,15 +232,14 @@ function analyzeEndpoints(networkEntries: NetworkEntry[]): { analyzed: AnalyzedE
     const key = `${entry.method}:${pattern}`;
     if (seen.has(key)) continue;
 
-    const qp: string[] = [];
-    try { new URL(entry.url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) qp.push(k); }); } catch {}
+    const { params: qp, hasSearch, hasPagination, hasLimit } = classifyQueryParams(entry.url);
 
     const ep: AnalyzedEndpoint = {
       pattern, method: entry.method, url: entry.url, status: entry.status, contentType: ct,
-      queryParams: qp, hasSearchParam: qp.some(p => SEARCH_PARAMS.has(p)),
-      hasPaginationParam: qp.some(p => PAGINATION_PARAMS.has(p)),
-      hasLimitParam: qp.some(p => LIMIT_PARAMS.has(p)),
-      authIndicators: detectAuthIndicators(entry.requestHeaders),
+      queryParams: qp, hasSearchParam: hasSearch,
+      hasPaginationParam: hasPagination,
+      hasLimitParam: hasLimit || qp.some(p => LIMIT_PARAMS.has(p)),
+      authIndicators: detectAuthFromHeaders(entry.requestHeaders),
       responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null,
       score: 0,
     };

package/src/external-clis.yaml

@@ -22,14 +22,6 @@
   install:
     default: "npm install -g @readwiseio/readwise-cli"
 
-- name: kubectl
-  binary: kubectl
-  description: "Kubernetes command-line tool"
-  homepage: "https://kubernetes.io/docs/reference/kubectl/"
-  tags: [kubernetes, k8s, devops]
-  install:
-    mac: "brew install kubectl"
-
 - name: docker
   binary: docker
   description: "Docker command-line interface"

package/src/external.test.ts

@@ -33,6 +33,15 @@ describe('parseCommand', () => {
       'Install command contains unsafe shell operators',
     );
   });
+
+  it('rejects command substitution and multiline input', () => {
+    expect(() => parseCommand('brew install $(whoami)')).toThrow(
+      'Install command contains unsafe shell operators',
+    );
+    expect(() => parseCommand('brew install gh\nrm -rf /')).toThrow(
+      'Install command contains unsafe shell operators',
+    );
+  });
 });
 
 describe('installExternalCli', () => {