@nextpa/chat-core 0.1.0

package/dist/services/page-interaction/iframe-navigation-manager.js

@@ -0,0 +1,154 @@
+import { logger } from '../../utils/logger';
+import { NEXTPA_IFRAME_NAME } from './constants';
+const IFRAME_Z_INDEX = '2147483640';
+const WIDGET_Z_INDEX = '2147483641';
+const BLOCK_DETECTION_TIMEOUT_MS = 3000;
+export class IframeNavigationManager {
+    overlay = null;
+    iframe = null;
+    closeCallbacks = [];
+    widgetElevated = false;
+    previousOverflow = '';
+    navigateTo(url) {
+        const existingIframe = this.iframe
+            ?? document.querySelector(`iframe[name="${NEXTPA_IFRAME_NAME}"]`);
+        if (existingIframe) {
+            this.iframe = existingIframe;
+            this.overlay = existingIframe.parentElement;
+            return this.navigateExistingIframe(url);
+        }
+        return this.createIframeOverlay(url);
+    }
+    close() {
+        if (!this.overlay)
+            return;
+        this.cleanup();
+        for (const cb of this.closeCallbacks) {
+            try {
+                cb();
+            }
+            catch { /* swallow */ }
+        }
+    }
+    isActive() {
+        return this.iframe !== null && this.overlay !== null;
+    }
+    getTargetDocument() {
+        if (!this.iframe)
+            return document;
+        try {
+            return this.iframe.contentDocument ?? document;
+        }
+        catch {
+            return document;
+        }
+    }
+    getTargetWindow() {
+        if (!this.iframe)
+            return globalThis;
+        try {
+            const win = this.iframe.contentWindow;
+            if (win?.location.href) {
+                return win;
+            }
+        }
+        catch {
+            return globalThis;
+        }
+        return globalThis;
+    }
+    onClose(callback) {
+        this.closeCallbacks.push(callback);
+    }
+    createIframeOverlay(url) {
+        this.close();
+        this.elevateWidget();
+        this.hideParentScrollbar();
+        return new Promise((resolve) => {
+            const overlay = document.createElement('div');
+            overlay.dataset.nextpaIframeNav = 'true';
+            overlay.style.cssText = [
+                'position: fixed',
+                'inset: 0',
+                `z-index: ${IFRAME_Z_INDEX}`,
+                'background: #fff',
+            ].join('; ');
+            const iframe = document.createElement('iframe');
+            iframe.style.cssText = 'border: none; width: 100%; height: 100%';
+            iframe.name = NEXTPA_IFRAME_NAME;
+            iframe.setAttribute('sandbox', 'allow-same-origin allow-scripts allow-forms allow-popups allow-popups-to-escape-sandbox');
+            overlay.appendChild(iframe);
+            document.body.appendChild(overlay);
+            this.overlay = overlay;
+            this.iframe = iframe;
+            this.loadUrlInIframe(iframe, url, resolve);
+        });
+    }
+    navigateExistingIframe(url) {
+        return new Promise((resolve) => {
+            this.loadUrlInIframe(this.iframe, url, resolve);
+        });
+    }
+    loadUrlInIframe(iframe, url, resolve) {
+        let resolved = false;
+        const settle = (result) => {
+            if (resolved)
+                return;
+            resolved = true;
+            clearTimeout(timeoutId);
+            iframe.removeEventListener('load', onLoad);
+            iframe.removeEventListener('error', onError);
+            if (result.status === 'blocked') {
+                this.cleanup();
+            }
+            resolve(result);
+        };
+        const timeoutId = setTimeout(() => {
+            settle({ status: 'blocked', reason: 'Timeout: page did not load within detection window.' });
+        }, BLOCK_DETECTION_TIMEOUT_MS);
+        const onLoad = () => {
+            try {
+                const iframeDoc = iframe.contentDocument;
+                if (!iframeDoc) {
+                    throw new Error('contentDocument is null');
+                }
+                // Access title to trigger cross-origin exception if blocked
+                logger.debug('[IframeNav] Loaded in iframe:', url, 'title:', iframeDoc.title);
+                settle({ status: 'iframe-loaded' });
+            }
+            catch {
+                logger.debug('[IframeNav] Blocked by cross-origin policy:', url);
+                settle({ status: 'blocked', reason: 'Cross-origin restrictions prevent iframe access.' });
+            }
+        };
+        const onError = () => {
+            settle({ status: 'blocked', reason: 'Failed to load URL in iframe.' });
+        };
+        iframe.addEventListener('load', onLoad);
+        iframe.addEventListener('error', onError);
+        iframe.src = url;
+    }
+    elevateWidget() {
+        if (this.widgetElevated)
+            return;
+        const widget = document.querySelector('nextpa-widget');
+        if (!widget)
+            return;
+        widget.style.position = 'relative';
+        widget.style.zIndex = WIDGET_Z_INDEX;
+        this.widgetElevated = true;
+    }
+    hideParentScrollbar() {
+        this.previousOverflow = document.documentElement.style.overflow;
+        document.documentElement.style.overflow = 'hidden';
+    }
+    restoreParentScrollbar() {
+        document.documentElement.style.overflow = this.previousOverflow;
+    }
+    cleanup() {
+        this.restoreParentScrollbar();
+        this.overlay?.remove();
+        this.overlay = null;
+        this.iframe = null;
+    }
+}

package/dist/services/page-interaction/index.d.ts

@@ -0,0 +1,5 @@
+export { extractPageSnapshot, extractForms, extractButtons, extractLinks, extractTables, extractProductCards, extractInteractiveElements } from './dom-extractor';
+export { extractPageContent } from './content-extractor';
+export { fillField, fillFields, clickElement, scrollToElement, highlightElement, waitForElement } from './dom-executor';
+export { generateSelector } from './selector-generator';
+export type { PageSnapshot, FormElement, ButtonElement, LinkElement, TableElement, ProductCardElement, InteractiveElement, PageContent, PageInteractionRequest, PageInteractionResult, ActionResult } from './types';

package/dist/services/page-interaction/index.js

@@ -0,0 +1,4 @@
+export { extractPageSnapshot, extractForms, extractButtons, extractLinks, extractTables, extractProductCards, extractInteractiveElements } from './dom-extractor';
+export { extractPageContent } from './content-extractor';
+export { fillField, fillFields, clickElement, scrollToElement, highlightElement, waitForElement } from './dom-executor';
+export { generateSelector } from './selector-generator';

package/dist/services/page-interaction/page-interaction-handler.d.ts

@@ -0,0 +1,21 @@
+import type * as signalR from '@microsoft/signalr';
+import type { PageInteractionRequest, PageInteractionResult } from './types';
+export type DocumentProvider = () => Document;
+export type WindowProvider = () => Window;
+export type ExternalExecutor = (request: PageInteractionRequest) => Promise<PageInteractionResult | string>;
+export declare class PageInteractionHandler {
+    private readonly getConnection;
+    private requestTimestamps;
+    private getDocument;
+    private getWindow;
+    private externalExecutor;
+    constructor(getConnection: () => signalR.HubConnection | null);
+    setDocumentProvider(getDoc: DocumentProvider, getWin: WindowProvider): void;
+    setExternalExecutor(executor: ExternalExecutor | null): void;
+    handleRequest(requestJson: string): Promise<void>;
+    private executeAction;
+    private toResult;
+    private sendResponse;
+    private validateRequest;
+    private checkRateLimit;
+}

package/dist/services/page-interaction/page-interaction-handler.js

@@ -0,0 +1,129 @@
+import { logger } from '../../utils/logger';
+import { extractPageSnapshot } from './dom-extractor';
+import { extractPageContent } from './content-extractor';
+import { clickElement, fillField, fillFields, highlightElement, scrollToElement, waitForElement } from './dom-executor';
+const ALLOWED_ACTIONS = new Set(['read', 'read_content', 'fill', 'fill_multiple', 'click', 'scroll_to', 'highlight', 'wait_for']);
+const MAX_REQUESTS_PER_SECOND = 10;
+const DANGEROUS_SELECTOR_PATTERN = /javascript:|<script|on\w+\s*=/i;
+export class PageInteractionHandler {
+    getConnection;
+    requestTimestamps = [];
+    getDocument = () => document;
+    getWindow = () => globalThis;
+    externalExecutor = null;
+    constructor(getConnection) {
+        this.getConnection = getConnection;
+    }
+    setDocumentProvider(getDoc, getWin) {
+        this.getDocument = getDoc;
+        this.getWindow = getWin;
+    }
+    setExternalExecutor(executor) {
+        this.externalExecutor = executor;
+    }
+    async handleRequest(requestJson) {
+        let request;
+        try {
+            request = JSON.parse(requestJson);
+        }
+        catch {
+            logger.error('[PageInteraction] Failed to parse request JSON');
+            return;
+        }
+        if (!this.validateRequest(request))
+            return;
+        if (!this.checkRateLimit()) {
+            await this.sendResponse(request.requestId, {
+                success: false,
+                errorMessage: 'Rate limit exceeded. Too many requests.',
+            });
+            return;
+        }
+        const result = this.externalExecutor
+            ? await this.externalExecutor(request)
+            : await this.executeAction(request);
+        await this.sendResponse(request.requestId, result);
+    }
+    async executeAction(request) {
+        const doc = this.getDocument();
+        const win = this.getWindow();
+        switch (request.action) {
+            case 'read': {
+                const snapshot = extractPageSnapshot(doc, win);
+                return JSON.stringify(snapshot);
+            }
+            case 'read_content': {
+                const content = extractPageContent(doc, win);
+                return JSON.stringify(content);
+            }
+            case 'fill':
+                return this.toResult(fillField(request.selector, request.value, doc));
+            case 'fill_multiple':
+                return this.toResult(fillFields(request.fields, doc));
+            case 'click':
+                return this.toResult(clickElement(request.selector, doc));
+            case 'scroll_to':
+                return this.toResult(scrollToElement(request.selector, doc));
+            case 'highlight':
+                return this.toResult(highlightElement(request.selector, undefined, undefined, doc));
+            case 'wait_for':
+                return this.toResult(await waitForElement(request.selector, request.value ? Number(request.value) : undefined, doc));
+            default:
+                return { success: false, errorMessage: `Unknown action: ${request.action}` };
+        }
+    }
+    toResult(actionResult) {
+        return {
+            success: actionResult.success,
+            errorMessage: actionResult.errorMessage,
+        };
+    }
+    async sendResponse(requestId, result) {
+        const connection = this.getConnection();
+        if (!connection) {
+            logger.error('[PageInteraction] No SignalR connection available to send response');
+            return;
+        }
+        try {
+            const responseJson = typeof result === 'string' ? result : JSON.stringify(result);
+            await connection.invoke('SendPageInteractionResponse', requestId, responseJson);
+            logger.debug('[PageInteraction] Response sent for request:', requestId);
+        }
+        catch (error) {
+            logger.error('[PageInteraction] Failed to send response:', error);
+        }
+    }
+    validateRequest(request) {
+        if (!request.requestId || !request.action) {
+            logger.warn('[PageInteraction] Invalid request: missing requestId or action');
+            return false;
+        }
+        if (!ALLOWED_ACTIONS.has(request.action)) {
+            logger.warn('[PageInteraction] Unknown action:', request.action);
+            return false;
+        }
+        if (request.selector && DANGEROUS_SELECTOR_PATTERN.test(request.selector)) {
+            logger.warn('[PageInteraction] Dangerous selector rejected:', request.selector);
+            return false;
+        }
+        if (request.fields) {
+            for (const selector of Object.keys(request.fields)) {
+                if (DANGEROUS_SELECTOR_PATTERN.test(selector)) {
+                    logger.warn('[PageInteraction] Dangerous selector in fields rejected:', selector);
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    checkRateLimit() {
+        const now = Date.now();
+        this.requestTimestamps = this.requestTimestamps.filter(t => now - t < 1000);
+        if (this.requestTimestamps.length >= MAX_REQUESTS_PER_SECOND) {
+            logger.warn('[PageInteraction] Rate limit exceeded');
+            return false;
+        }
+        this.requestTimestamps.push(now);
+        return true;
+    }
+}

package/dist/services/page-interaction/selector-generator.d.ts

@@ -0,0 +1 @@
+export declare function generateSelector(element: Element, doc?: Document): string;

package/dist/services/page-interaction/selector-generator.js

@@ -0,0 +1,48 @@
+export function generateSelector(element, doc = document) {
+    const dataTestId = element.dataset?.testid;
+    if (dataTestId) {
+        const selector = `[data-testid="${CSS.escape(dataTestId)}"]`;
+        if (doc.querySelectorAll(selector).length === 1) {
+            return selector;
+        }
+    }
+    if (element.id) {
+        return `#${CSS.escape(element.id)}`;
+    }
+    const name = element.getAttribute('name');
+    if (name) {
+        const tag = element.tagName.toLowerCase();
+        const selector = `${tag}[name="${CSS.escape(name)}"]`;
+        if (doc.querySelectorAll(selector).length === 1) {
+            return selector;
+        }
+    }
+    return buildNthChildPath(element, doc);
+}
+function buildNthChildPath(element, doc) {
+    const parts = [];
+    let current = element;
+    while (current !== null && current !== doc.documentElement) {
+        if (current.id) {
+            parts.unshift(`#${CSS.escape(current.id)}`);
+            break;
+        }
+        const parentEl = current.parentElement;
+        if (!parentEl) {
+            parts.unshift(current.tagName.toLowerCase());
+            break;
+        }
+        const currentTag = current.tagName;
+        const siblings = Array.from(parentEl.children);
+        const sameTagSiblings = siblings.filter((s) => s.tagName === currentTag);
+        if (sameTagSiblings.length === 1) {
+            parts.unshift(current.tagName.toLowerCase());
+        }
+        else {
+            const index = sameTagSiblings.indexOf(current) + 1;
+            parts.unshift(`${current.tagName.toLowerCase()}:nth-of-type(${index})`);
+        }
+        current = parentEl;
+    }
+    return parts.join(' > ');
+}

package/dist/services/page-interaction/types.d.ts

@@ -0,0 +1,75 @@
+export interface PageSnapshot {
+    url: string;
+    title: string | null;
+    forms: FormElement[];
+    buttons: ButtonElement[];
+    links: LinkElement[];
+    tables: TableElement[];
+    productCards: ProductCardElement[];
+    otherInteractiveElements: InteractiveElement[];
+}
+export interface FormElement {
+    selector: string;
+    formSelector: string | null;
+    label: string | null;
+    context: string | null;
+    inputType: string;
+    currentValue: string | null;
+    isRequired: boolean;
+    isDisabled: boolean;
+    options: string[] | null;
+    placeholder: string | null;
+}
+export interface ButtonElement {
+    selector: string;
+    text: string | null;
+    isDisabled: boolean;
+    type: string | null;
+}
+export interface LinkElement {
+    selector: string;
+    text: string | null;
+    href: string | null;
+}
+export interface TableElement {
+    selector: string;
+    headers: string[];
+    rowCount: number;
+    sampleRows: string[][] | null;
+}
+export interface ProductCardElement {
+    selector: string;
+    title: string | null;
+    price: string | null;
+    imageUrl: string | null;
+    description: string | null;
+}
+export interface InteractiveElement {
+    selector: string;
+    tagName: string;
+    text: string | null;
+    role: string | null;
+    ariaLabel: string | null;
+}
+export interface PageContent {
+    url: string;
+    title: string | null;
+    content: string;
+    truncated: boolean;
+}
+export interface PageInteractionRequest {
+    requestId: string;
+    action: string;
+    selector?: string;
+    value?: string;
+    fields?: Record<string, string>;
+}
+export interface PageInteractionResult {
+    success: boolean;
+    errorMessage?: string;
+    data?: string;
+}
+export interface ActionResult {
+    success: boolean;
+    errorMessage?: string;
+}

package/dist/services/page-interaction/types.js

@@ -0,0 +1 @@
+export {};

package/dist/services/realtime/google/google-audio.d.ts

@@ -0,0 +1,44 @@
+export declare function resolveWorkletUrl(): string;
+export declare class GoogleAudioHandler {
+    private captureContext;
+    private mediaStream;
+    private workletNode;
+    private playbackContext;
+    private gainNode;
+    private aecStreamDestination;
+    private loopbackSource;
+    private loopbackSink;
+    private playbackElement;
+    private scheduledTime;
+    private audioQueue;
+    private isMuted;
+    private isPlaying;
+    private capturePaused;
+    private speechDetected;
+    private silenceTimerId;
+    private speechStartTimerId;
+    private consecutiveSpeechFrames;
+    onAudioData: ((base64Pcm: string) => void) | null;
+    onSpeechStarted: (() => void) | null;
+    onSpeechEnded: (() => void) | null;
+    onAudioLevel: ((level01: number) => void) | null;
+    get isPlaybackActive(): boolean;
+    startCapture(): Promise<void>;
+    playAudio(base64Pcm: string): void;
+    private setupPlaybackChain;
+    private connectGainOutput;
+    private tryRouteThroughAecLoopback;
+    private teardownAecLoopback;
+    private enqueueAudio;
+    private schedulePlayback;
+    resume(): void;
+    pauseCapture(): void;
+    resumeCapture(): void;
+    setMuted(muted: boolean): void;
+    stop(): void;
+    private detectSpeech;
+    private handleSpeechFrame;
+    private handleSilenceFrame;
+    restartCapture(): Promise<void>;
+    clearPlaybackQueue(): void;
+}