omnikey-cli 1.0.24 → 1.0.26

package/backend-dist/web-search/llm-auth-check.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPageAuthenticated = isPageAuthenticated;
+const ai_client_1 = require("../ai-client");
+const ai_client_2 = require("../ai-client");
+const SYSTEM_PROMPT = 'You are an expert at detecting whether a web page is showing the real requested content. ' +
+    'Given a URL and the visible text content of a web page, answer "yes" if EITHER: ' +
+    '(1) The URL looks like a public resource that does not require authentication — such as documentation sites, ' +
+    'public wikis, news articles, open-source repos, package registries, developer references, or any URL whose ' +
+    'hostname/path strongly suggests publicly accessible content (e.g. docs.*, developer.*, wikipedia.org, github.com public repos, ' +
+    'stackoverflow.com, npmjs.com, medium.com, reddit.com, youtube.com, etc.). ' +
+    '(2) The page is showing the actual content that an authenticated user would see at that URL. ' +
+    'Answer "no" if the page is: a login/sign-in page, an access denied or unauthorized page, a redirect away from the requested URL, ' +
+    'a generic 404/not-found or error page that could be an auth redirect in disguise (e.g. shows a not-found message but ' +
+    'the URL was a valid authenticated route), or any page that does not correspond to the requested resource. ' +
+    'When in doubt about whether a URL is public, lean towards "yes". Reply with only one word: "yes" or "no".';
+const PUBLIC_URL_PATTERNS = [
+    /^https?:\/\/(www\.)?github\.com\/(?!.*\/settings|.*\/account)/,
+    /^https?:\/\/(www\.)?stackoverflow\.com/,
+    /^https?:\/\/(www\.)?wikipedia\.org/,
+    /^https?:\/\/docs\./,
+    /^https?:\/\/developer\./,
+    /^https?:\/\/(www\.)?npmjs\.com/,
+    /^https?:\/\/(www\.)?pypi\.org/,
+    /^https?:\/\/(www\.)?medium\.com/,
+    /^https?:\/\/(www\.)?reddit\.com/,
+    /^https?:\/\/(www\.)?youtube\.com/,
+    /^https?:\/\/(www\.)?news\.ycombinator\.com/,
+    // Package registries & language docs
+    /^https?:\/\/(www\.)?crates\.io/,
+    /^https?:\/\/(www\.)?rubygems\.org/,
+    /^https?:\/\/(www\.)?packagist\.org/,
+    /^https?:\/\/(www\.)?pkg\.go\.dev/,
+    /^https?:\/\/(www\.)?hex\.pm/,
+    /^https?:\/\/(www\.)?nuget\.org/,
+    /^https?:\/\/(www\.)?maven\.apache\.org/,
+    /^https?:\/\/central\.sonatype\.com/,
+    // Official language & runtime docs
+    /^https?:\/\/(www\.)?python\.org/,
+    /^https?:\/\/(www\.)?rust-lang\.org/,
+    /^https?:\/\/(www\.)?golang\.org/,
+    /^https?:\/\/(www\.)?go\.dev/,
+    /^https?:\/\/(www\.)?ruby-lang\.org/,
+    /^https?:\/\/(www\.)?php\.net/,
+    /^https?:\/\/(www\.)?kotlinlang\.org/,
+    /^https?:\/\/(www\.)?swift\.org/,
+    /^https?:\/\/learn\.microsoft\.com/,
+    /^https?:\/\/msdn\.microsoft\.com/,
+    /^https?:\/\/devblogs\.microsoft\.com/,
+    /^https?:\/\/(www\.)?w3\.org/,
+    /^https?:\/\/(www\.)?w3schools\.com/,
+    /^https?:\/\/(www\.)?mdn\./,
+    /^https?:\/\/developer\.mozilla\.org/,
+    // Source code & open-source platforms
+    /^https?:\/\/(www\.)?gitlab\.com\/(?!.*\/-\/settings)/,
+    /^https?:\/\/(www\.)?bitbucket\.org\/(?!.*\/admin)/,
+    /^https?:\/\/(www\.)?sourceforge\.net/,
+    /^https?:\/\/(www\.)?codepen\.io/,
+    /^https?:\/\/(www\.)?jsfiddle\.net/,
+    /^https?:\/\/(www\.)?codesandbox\.io/,
+    // Q&A, forums & community sites
+    /^https?:\/\/(www\.)?stackexchange\.com/,
+    /^https?:\/\/(www\.)?superuser\.com/,
+    /^https?:\/\/(www\.)?serverfault\.com/,
+    /^https?:\/\/(www\.)?askubuntu\.com/,
+    /^https?:\/\/(www\.)?quora\.com/,
+    /^https?:\/\/(www\.)?dev\.to/,
+    /^https?:\/\/(www\.)?hashnode\.com/,
+    /^https?:\/\/(www\.)?lobste\.rs/,
+    // News & tech media
+    /^https?:\/\/(www\.)?techcrunch\.com/,
+    /^https?:\/\/(www\.)?theverge\.com/,
+    /^https?:\/\/(www\.)?wired\.com/,
+    /^https?:\/\/(www\.)?arstechnica\.com/,
+    /^https?:\/\/(www\.)?thenextweb\.com/,
+    /^https?:\/\/(www\.)?infoq\.com/,
+    /^https?:\/\/(www\.)?smashingmagazine\.com/,
+    /^https?:\/\/(www\.)?css-tricks\.com/,
+    // Reference & encyclopedias
+    /^https?:\/\/[a-z-]+\.wikipedia\.org/,
+    /^https?:\/\/(www\.)?wikidata\.org/,
+    /^https?:\/\/(www\.)?wikimedia\.org/,
+    /^https?:\/\/(www\.)?archive\.org/,
+    // Cloud provider public docs
+    /^https?:\/\/cloud\.google\.com\/(?!.*\/console)/,
+    /^https?:\/\/aws\.amazon\.com\/(?!(.*\/console|.*\/signin))/,
+    /^https?:\/\/(www\.)?azure\.microsoft\.com/,
+    /^https?:\/\/registry\./,
+];
+const AUTH_PATH_PATTERN = /[/?#](login|log-in|signin|sign-in|signup|sign-up|register|auth|authenticate|oauth|sso|saml|forgot-password|reset-password|verify|two-factor|2fa|mfa)([/?#]|$)/i;
+function isPublicUrl(url) {
+    if (AUTH_PATH_PATTERN.test(url))
+        return false;
+    return PUBLIC_URL_PATTERNS.some((pattern) => pattern.test(url));
+}
+async function isPageAuthenticated(content, url, log, finalUrl) {
+    if (finalUrl) {
+        const normalize = (u) => u.replace(/#.*$/, '').replace(/\/$/, '');
+        if (normalize(finalUrl) !== normalize(url)) {
+            log.info('llm-auth-check: redirect detected, treating as not authenticated', {
+                requestUrl: url,
+                finalUrl,
+            });
+            return false;
+        }
+    }
+    if (isPublicUrl(url)) {
+        log.info('llm-auth-check: public URL, skipping auth check', { url });
+        return true;
+    }
+    const model = (0, ai_client_2.getDefaultModel)(ai_client_1.aiClient.getProvider(), 'fast');
+    const messages = [
+        { role: 'system', content: SYSTEM_PROMPT },
+        { role: 'user', content: `URL: ${url}\n\nPage content:\n${content}` },
+    ];
+    try {
+        const result = await ai_client_1.aiClient.complete(model, messages, { temperature: 0, maxTokens: 1 });
+        const answer = result.content.trim().toLowerCase();
+        log.info('llm-auth-check: LLM response', { url, answer });
+        return answer === 'yes';
+    }
+    catch (err) {
+        log.error('llm-auth-check: LLM call failed', { url, error: String(err) });
+        // If LLM call fails, default to not authorized
+        return false;
+    }
+}

package/backend-dist/{web-search-provider.js → web-search/web-search-provider.js}

@@ -7,7 +7,9 @@ exports.MAX_TOOL_CONTENT_CHARS = exports.MAX_WEB_FETCH_BYTES = exports.WEB_SEARC
 exports.executeWebSearch = executeWebSearch;
 exports.executeTool = executeTool;
 const axios_1 = __importDefault(require("axios"));
-const config_1 = require("./config");
+const config_1 = require("../config");
+const browser_playwright_1 = require("./browser-playwright");
+const llm_auth_check_1 = require("./llm-auth-check");
 exports.WEB_FETCH_TOOL = {
     name: 'web_fetch',
     description: "Fetch the text content of any publicly accessible URL. Use this to retrieve documentation, error references, API guides, release notes, or any web resource that would help answer the user's question.",
@@ -134,27 +136,107 @@ async function executeWebSearch(query, log) {
     log.info('web_search: using DuckDuckGo (free fallback)', { query });
     return formatSearchResults(await searchWithDuckDuckGo(query));
 }
+function stripHtml(raw) {
+    return raw
+        .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+const BASE_FETCH_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (compatible; OmniKeyAgent/1.0)',
+};
+// ── Step 1: plain HTTP fetch ──────────────────────────────────────────────────
+async function fetchPlainHttp(url, log) {
+    try {
+        const response = await axios_1.default.get(url, {
+            timeout: 15000,
+            responseType: 'text',
+            maxContentLength: exports.MAX_WEB_FETCH_BYTES,
+            headers: BASE_FETCH_HEADERS,
+        });
+        const finalUrl = response.request?.res?.responseUrl ?? url;
+        return { html: String(response.data), authBlocked: false, finalUrl };
+    }
+    catch (err) {
+        const status = axios_1.default.isAxiosError(err) ? err.response?.status : undefined;
+        log.warn('Initial fetch failed', {
+            url,
+            error: err instanceof Error ? err.message : String(err),
+            status,
+        });
+        // If a browser is running, any failure could be auth-related —
+        // sites use redirects, 302s, custom error pages, or soft-blocks
+        // rather than a clean 401/403, so checking status codes alone is
+        // unreliable. Fall through to the browser-session path instead.
+        if (isSelfHostedMacOS && (0, browser_playwright_1.isBrowserOpenWithUrl)(url, log)) {
+            return { html: null, authBlocked: true, finalUrl: url };
+        }
+        if (status === 401 || status === 403) {
+            return { html: null, authBlocked: true, finalUrl: url };
+        }
+        throw err;
+    }
+}
+// ── Step 2: LLM auth check on plain response ──────────────────────────────────
+async function checkPlainResponseAuth(plainText, url, log, finalUrl) {
+    const authenticated = await (0, llm_auth_check_1.isPageAuthenticated)(plainText.slice(0, 5000), url, log, finalUrl);
+    if (!authenticated) {
+        log.info('web_fetch: plain response failed auth check — trying active-tab strategy', { url });
+    }
+    return authenticated;
+}
+// ── Step 3: active-tab extraction (self-hosted macOS only) ───────────────────
+async function fetchFromActiveTab(url, log) {
+    log.info('web_fetch: falling back to active-tab extraction', { url });
+    return (0, browser_playwright_1.fetchWithPlaywright)(url, log);
+}
+const isSelfHostedMacOS = config_1.config.isSelfHosted && config_1.config.terminalPlatform === 'macos';
+async function executeWebFetch(url, log) {
+    log.info('Executing web_fetch tool', { url });
+    // ── Step 1: plain HTTP request ────────────────────────────────────────────
+    const { html, authBlocked, finalUrl } = await fetchPlainHttp(url, log);
+    const plainText = html ? stripHtml(html) : '';
+    if (!isSelfHostedMacOS) {
+        if (authBlocked) {
+            log.warn('Error: page requires authentication. Run OmniKey in self-hosted mode on macOS to enable browser-session access.');
+        }
+        return plainText.slice(0, exports.MAX_TOOL_CONTENT_CHARS) || 'No content retrieved';
+    }
+    // ── Step 2 (self-hosted macOS only): LLM auth check on plain response ─────
+    let looksUnauthenticated = false;
+    if (!authBlocked && plainText) {
+        log.info('web_fetch: performing LLM auth check on plain HTTP response', { url });
+        const authenticated = await checkPlainResponseAuth(plainText, url, log, finalUrl);
+        if (authenticated) {
+            return plainText.slice(0, exports.MAX_TOOL_CONTENT_CHARS) || 'No content retrieved';
+        }
+        looksUnauthenticated = true;
+    }
+    // ── Step 3 (self-hosted macOS only): active-tab extraction ───────────────
+    // Only attempted when there is evidence authentication is required.
+    const needsAuth = authBlocked || looksUnauthenticated;
+    if (needsAuth) {
+        log.info('web_fetch: evidence of authentication requirement, attempting active-tab extraction', { url });
+        const activeTabText = await fetchFromActiveTab(url, log);
+        if (activeTabText) {
+            return activeTabText.slice(0, exports.MAX_TOOL_CONTENT_CHARS);
+        }
+    }
+    // All strategies exhausted.
+    if (authBlocked) {
+        log.warn('Error: page requires authentication. Open the page in Chrome and ensure "Allow JavaScript from Apple Events" is enabled (View → Developer → Allow JavaScript from Apple Events).');
+    }
+    return plainText.slice(0, exports.MAX_TOOL_CONTENT_CHARS) || 'No content retrieved';
+}
 async function executeTool(name, args, log) {
     if (name === 'web_fetch') {
         const url = args.url;
         if (!url)
             return 'Error: url parameter is required';
         try {
-            log.info('Executing web_fetch tool', { url });
-            const response = await axios_1.default.get(url, {
-                timeout: 15000,
-                responseType: 'text',
-                maxContentLength: exports.MAX_WEB_FETCH_BYTES,
-                headers: { 'User-Agent': 'Mozilla/5.0 (compatible; OmniKeyAgent/1.0)' },
-            });
-            const text = String(response.data)
-                .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
-                .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
-                .replace(/<[^>]+>/g, ' ')
-                .replace(/\s+/g, ' ')
-                .trim()
-                .slice(0, exports.MAX_TOOL_CONTENT_CHARS);
-            return text || 'No content retrieved';
+            return await executeWebFetch(url, log);
         }
         catch (err) {
             log.warn('web_fetch tool failed', {

package/package.json

@@ -4,7 +4,7 @@
     "access": "public",
     "registry": "https://registry.npmjs.org/"
   },
-  "version": "1.0.24",
+  "version": "1.0.26",
   "description": "CLI for onboarding users to Omnikey AI and configuring OPENAI_API_KEY. Use Yarn for install/build.",
   "engines": {
     "node": ">=14.0.0",
@@ -44,7 +44,8 @@
     "sqlite3": "^5.1.6",
     "winston": "^3.19.0",
     "ws": "^8.18.0",
-    "zod": "^4.3.6"
+    "zod": "^4.3.6",
+    "playwright-core": "^1.50.0"
   },
   "devDependencies": {
     "@types/inquirer": "^9.0.9",