@myscheme/voice-navigation-sdk 0.1.0

package/dist/navigation-controller.js

@@ -0,0 +1,782 @@
+import { MicrophoneHandler } from "./microphone-handler.js";
+import { AzureSpeechService } from "./services/azure-speech.js";
+import { BedrockService } from "./services/bedrock.js";
+import { VectorSearchService, } from "./services/vector-search.js";
+import { performAgentAction, formatActionLabel, extractAgentAction, } from "./actions.js";
+import { createFloatingControl, setState, updateStatus, updateTranscript, updateActionIndicator, showError, emitEvent, } from "./ui.js";
+const AUTO_START_STORAGE_KEY = "__navigate_auto_start";
+const RESUME_STATE_STORAGE_KEY = "__navigate_resume_state";
+const flowLog = (...args) => {
+    if (typeof console !== "undefined" && typeof console.info === "function") {
+        console.info("[VoiceNavigation]", ...args);
+    }
+};
+const clipText = (value, limit = 120) => {
+    const trimmed = value.trim();
+    if (trimmed.length <= limit) {
+        return trimmed;
+    }
+    return `${trimmed.slice(0, limit)}...`;
+};
+const resolveOpenSearchConfig = (config) => {
+    const aliasConfig = config.openSearch;
+    const resolved = config.opensearch ?? aliasConfig;
+    if (!resolved || typeof resolved !== "object") {
+        return undefined;
+    }
+    return resolved;
+};
+export class VoiceNavigationController {
+    constructor(config) {
+        this.lastTranscript = "";
+        this.isProcessing = false;
+        this.resumeAfterNavigation = false;
+        this.lastStartReason = "user";
+        this.autoStartPendingUserInteraction = false;
+        this.removeAutoStartListeners = null;
+        this.vectorSearchService = null;
+        const openSearchConfig = resolveOpenSearchConfig(config);
+        this.config = {
+            ...config,
+            opensearch: openSearchConfig,
+        };
+        if (!this.config.azure?.subscriptionKey || !this.config.azure?.region) {
+            throw new Error("Azure Speech configuration is required");
+        }
+        if (!this.config.aws?.accessKeyId ||
+            !this.config.aws?.secretAccessKey ||
+            !this.config.aws?.modelId) {
+            throw new Error("AWS Bedrock configuration is required");
+        }
+        this.azureSpeechService = new AzureSpeechService({
+            subscriptionKey: this.config.azure.subscriptionKey,
+            region: this.config.azure.region,
+        });
+        this.bedrockService = new BedrockService({
+            region: this.config.aws.region || "ap-south-1",
+            accessKeyId: this.config.aws.accessKeyId,
+            secretAccessKey: this.config.aws.secretAccessKey,
+            modelId: this.config.aws.modelId,
+            embeddingModelId: this.config.aws.embeddingModelId === undefined
+                ? undefined
+                : this.config.aws.embeddingModelId,
+        });
+        this.microphoneHandler = new MicrophoneHandler({
+            azureSpeechService: this.azureSpeechService,
+            bedrockService: this.bedrockService,
+            language: this.config.language || "en-IN",
+        });
+        if (openSearchConfig) {
+            try {
+                this.vectorSearchService = new VectorSearchService(openSearchConfig);
+            }
+            catch (error) {
+                console.error("Failed to initialize OpenSearch client:", error);
+                this.vectorSearchService = null;
+            }
+        }
+        this.resumeAfterNavigation = this.consumeResumeState();
+        this.ui = createFloatingControl();
+        this.setupEventHandlers();
+        if (this.resumeAfterNavigation) {
+            void this.start({ reason: "auto" });
+        }
+        else if (this.shouldAutoStart()) {
+            void this.start({ reason: "auto" });
+        }
+    }
+    shouldAutoStart() {
+        if (this.config.autoStart === true) {
+            return true;
+        }
+        if (typeof localStorage === "undefined") {
+            return false;
+        }
+        try {
+            const stored = localStorage.getItem(AUTO_START_STORAGE_KEY);
+            return stored === "true";
+        }
+        catch {
+            return false;
+        }
+    }
+    setAutoStart(enabled) {
+        if (typeof localStorage === "undefined") {
+            return;
+        }
+        try {
+            localStorage.setItem(AUTO_START_STORAGE_KEY, String(enabled));
+        }
+        catch (error) {
+            console.error("Failed to save auto-start preference:", error);
+        }
+    }
+    prepareForNavigation(options = {}) {
+        this.persistResumeState(options.target);
+    }
+    setupEventHandlers() {
+        this.ui.button.addEventListener("click", () => {
+            this.toggleRecording();
+        });
+    }
+    async toggleRecording() {
+        const currentState = this.ui.root.dataset.state;
+        if (currentState === "listening" || currentState === "thinking") {
+            await this.stop();
+            return;
+        }
+        if (currentState === "loading") {
+            return;
+        }
+        await this.start({ reason: "user" });
+    }
+    async start(options = {}) {
+        const reason = options.reason ?? "user";
+        this.lastStartReason = reason;
+        if (this.ui.root.dataset.state === "listening") {
+            return;
+        }
+        if (reason === "auto") {
+            try {
+                const canAutoStart = await this.microphoneHandler.canStartAutomatically();
+                if (!canAutoStart) {
+                    this.handleAutoStartFallback("auto");
+                    return;
+                }
+            }
+            catch (error) {
+                this.handleAutoStartFallback("auto");
+                return;
+            }
+        }
+        if (reason === "user") {
+            this.clearAutoStartFallback();
+            try {
+                await this.microphoneHandler.unlockAudioContext(true);
+            }
+            catch (error) {
+                setState(this.ui, "idle");
+                updateStatus(this.ui, "Tap to enable voice navigation");
+                return;
+            }
+        }
+        this.isProcessing = false;
+        setState(this.ui, "loading");
+        emitEvent("state-change", { state: "loading" });
+        try {
+            const started = await this.microphoneHandler.startRecording({
+                onPartial: (text) => this.handlePartial(text),
+                onSegment: (text, info) => this.handleSegment(text, info),
+                onSilence: () => this.handleSilence(),
+                onError: (error) => this.handleError(error),
+            });
+            if (started) {
+                this.clearAutoStartFallback();
+                setState(this.ui, "listening");
+                emitEvent("state-change", { state: "listening" });
+            }
+            else {
+                setState(this.ui, "idle");
+                emitEvent("state-change", { state: "idle" });
+                this.handleAutoStartFallback(reason);
+            }
+        }
+        catch (error) {
+            console.error("Failed to start recording:", error);
+            showError(this.ui, error);
+            setState(this.ui, "error");
+            emitEvent("error", { error });
+        }
+    }
+    async stop() {
+        this.isProcessing = false;
+        setState(this.ui, "thinking");
+        emitEvent("state-change", { state: "thinking" });
+        try {
+            const finalText = await this.microphoneHandler.stopRecording();
+            if (finalText && finalText.trim()) {
+                await this.processTranscript(finalText);
+            }
+            this.microphoneHandler.resetTranscriptionBuffer();
+            setState(this.ui, "idle");
+            emitEvent("state-change", { state: "idle" });
+            updateTranscript(this.ui, "");
+        }
+        catch (error) {
+            console.error("Failed to stop recording:", error);
+            showError(this.ui, error);
+            setState(this.ui, "error");
+            emitEvent("error", { error });
+        }
+    }
+    handlePartial(text) {
+        if (this.isProcessing) {
+            return;
+        }
+        updateTranscript(this.ui, text);
+    }
+    handleSegment(_text, _info) {
+        if (this.isProcessing) {
+            return;
+        }
+        this.lastTranscript = this.microphoneHandler.getAggregatedText();
+        updateTranscript(this.ui, this.lastTranscript);
+    }
+    async handleSilence() {
+        await this.processPendingTranscript();
+    }
+    async processPendingTranscript() {
+        if (this.isProcessing) {
+            return;
+        }
+        const aggregatedText = this.microphoneHandler.getAggregatedText();
+        const transcript = aggregatedText.trim();
+        if (!transcript) {
+            return;
+        }
+        this.isProcessing = true;
+        this.lastTranscript = transcript;
+        this.microphoneHandler.clearSilenceTimer();
+        this.microphoneHandler.resetTranscriptionBuffer();
+        setState(this.ui, "thinking");
+        emitEvent("state-change", { state: "thinking" });
+        try {
+            await this.processTranscript(transcript);
+        }
+        finally {
+            const statusMessage = this.ui.status.textContent || "";
+            if (this.ui.root.dataset.state !== "idle") {
+                setState(this.ui, "listening");
+                emitEvent("state-change", { state: "listening" });
+                if (statusMessage) {
+                    updateStatus(this.ui, statusMessage);
+                }
+            }
+            updateTranscript(this.ui, "");
+            this.isProcessing = false;
+        }
+    }
+    handleError(error) {
+        console.error("Microphone error:", error);
+        if (this.lastStartReason === "auto" && this.isPermissionRelatedError(error)) {
+            setState(this.ui, "idle");
+            this.handleAutoStartFallback("auto");
+            updateStatus(this.ui, "Tap to start voice control");
+            return;
+        }
+        showError(this.ui, error);
+        setState(this.ui, "error");
+        emitEvent("error", { error });
+    }
+    async processTranscript(text) {
+        if (!text || !text.trim()) {
+            return;
+        }
+        try {
+            const transcriptSnippet = clipText(text);
+            flowLog("Transcript received", { transcript: transcriptSnippet });
+            updateStatus(this.ui, "Processing...");
+            flowLog("Sending transcript for action extraction", {
+                transcript: transcriptSnippet,
+            });
+            const result = await this.microphoneHandler.sendToBedrockAgent(text);
+            await this.handleAgentResult(text, result);
+        }
+        catch (error) {
+            console.error("Failed to process transcript:", error);
+            showError(this.ui, error);
+        }
+    }
+    async handleAgentResult(originalText, result) {
+        const transcriptSnippet = clipText(originalText);
+        const actionData = extractAgentAction(result);
+        if (!actionData) {
+            console.error("Could not extract action fromm result", originalText, result);
+            console.error("Extracted action data:", originalText, result);
+            updateActionIndicator(this.ui, "unknown", false);
+            updateStatus(this.ui, "Could not understand the request");
+            return;
+        }
+        const { action, query } = actionData;
+        flowLog("Action extraction completed", {
+            transcript: transcriptSnippet,
+            action,
+            query: query ?? null,
+        });
+        emitEvent("action-detected", { action, originalText });
+        if (action === "search_content") {
+            flowLog("search_content action detected", {
+                transcript: transcriptSnippet,
+                query: query ?? null,
+            });
+            const actionResult = await this.executeVectorSearchAction({
+                query,
+                originalText,
+            });
+            if (typeof this.config.actionHandlers?.[action] === "function") {
+                try {
+                    this.config.actionHandlers[action]({
+                        controller: this,
+                        transcript: originalText,
+                        actionResult,
+                        timestamp: Date.now(),
+                    });
+                }
+                catch (handlerError) {
+                    console.error("Custom action handler failed:", handlerError);
+                }
+            }
+            updateActionIndicator(this.ui, formatActionLabel(action), actionResult.performed, actionResult.info);
+            const statusMessage = typeof actionResult.info.message === "string"
+                ? actionResult.info.message
+                : actionResult.performed
+                    ? "Opening the best matching result."
+                    : "Could not open a relevant result.";
+            updateStatus(this.ui, statusMessage);
+            emitEvent("action-performed", {
+                action,
+                performed: actionResult.performed,
+                info: actionResult.info,
+            });
+            return;
+        }
+        const actionResult = performAgentAction(action, {
+            handler: this,
+            onStop: () => this.stop(),
+            transcript: originalText,
+        });
+        if (typeof this.config.actionHandlers?.[action] === "function") {
+            try {
+                this.config.actionHandlers[action]({
+                    controller: this,
+                    transcript: originalText,
+                    actionResult,
+                    timestamp: Date.now(),
+                });
+            }
+            catch (handlerError) {
+                console.error("Custom action handler failed:", handlerError);
+            }
+        }
+        updateActionIndicator(this.ui, formatActionLabel(action), actionResult.performed, actionResult.info);
+        if (action === "unknown" || !actionResult.performed) {
+            updateStatus(this.ui, "Say that again? I didn't catch the action.");
+        }
+        else {
+            updateStatus(this.ui, `Action completed: ${formatActionLabel(action).toLowerCase()}`);
+        }
+        emitEvent("action-performed", {
+            action,
+            performed: actionResult.performed,
+            info: actionResult.info,
+        });
+    }
+    setLanguage(language) {
+        this.microphoneHandler.setLanguage(language);
+    }
+    async executeVectorSearchAction(options) {
+        const vectorService = this.vectorSearchService;
+        const normalizedQuery = this.normalizeSearchQuery(options.query, options.originalText);
+        const transcriptSnippet = clipText(options.originalText);
+        flowLog("Vector search invoked", {
+            transcript: transcriptSnippet,
+            providedQuery: options.query ?? null,
+            normalizedQuery,
+        });
+        if (!vectorService) {
+            flowLog("Vector search unavailable", {
+                reason: "not_configured",
+            });
+            return {
+                performed: false,
+                info: {
+                    query: normalizedQuery,
+                    message: "Search is not configured.",
+                    reason: "vector_search_unavailable",
+                },
+            };
+        }
+        if (!normalizedQuery) {
+            flowLog("Vector search missing query", {
+                reason: "empty_query",
+            });
+            return {
+                performed: false,
+                info: {
+                    query: normalizedQuery,
+                    message: "Please mention what you would like to find.",
+                    reason: "missing_query",
+                },
+            };
+        }
+        updateStatus(this.ui, `Searching for “${normalizedQuery}”...`);
+        flowLog("Generating embeddings", { query: normalizedQuery });
+        let embedding = [];
+        try {
+            embedding = await this.bedrockService.embedText(normalizedQuery);
+            flowLog("Embeddings generated", {
+                query: normalizedQuery,
+                dimensions: embedding.length,
+            });
+        }
+        catch (error) {
+            console.error("Failed to generate embeddings:", error);
+            flowLog("Embedding generation failed", {
+                query: normalizedQuery,
+            });
+            showError(this.ui, new Error("Search is currently unavailable."));
+            return {
+                performed: false,
+                info: {
+                    query: normalizedQuery,
+                    message: "Search is currently unavailable.",
+                    reason: "embedding_failed",
+                },
+            };
+        }
+        if (!embedding.length) {
+            flowLog("Embedding result empty", { query: normalizedQuery });
+            return {
+                performed: false,
+                info: {
+                    query: normalizedQuery,
+                    message: "Search is currently unavailable.",
+                    reason: "empty_embedding",
+                },
+            };
+        }
+        let hits;
+        try {
+            flowLog("Calling vector search service", { query: normalizedQuery });
+            hits = await vectorService.search(embedding, normalizedQuery);
+            flowLog("Vector search completed", {
+                query: normalizedQuery,
+                hitCount: hits.length,
+                hits: hits,
+            });
+        }
+        catch (error) {
+            console.error("OpenSearch query failed:", error);
+            flowLog("Vector search call failed", { query: normalizedQuery });
+            showError(this.ui, new Error("Unable to reach the search service."));
+            return {
+                performed: false,
+                info: {
+                    query: normalizedQuery,
+                    message: "Unable to reach the search service.",
+                    reason: "opensearch_error",
+                },
+            };
+        }
+        const best = vectorService.findBestMatchWithUrl(hits);
+        if (!best) {
+            flowLog("Vector search returned no matching URL", {
+                query: normalizedQuery,
+            });
+            return {
+                performed: false,
+                info: {
+                    query: normalizedQuery,
+                    message: "I could not find a relevant result.",
+                    reason: "no_match",
+                },
+            };
+        }
+        const resolvedUrl = this.resolveUrl(best.url);
+        const sameDomain = this.isSameDomain(resolvedUrl);
+        const baseInfo = {
+            url: resolvedUrl,
+            query: normalizedQuery,
+            score: best.hit.score,
+            external: !sameDomain,
+        };
+        if (sameDomain) {
+            flowLog("Navigating to internal result", {
+                query: normalizedQuery,
+                url: resolvedUrl,
+                score: best.hit.score,
+            });
+            this.prepareForNavigation({ target: resolvedUrl });
+            baseInfo.message = "Opening the best matching result.";
+            window.location.href = resolvedUrl;
+            return {
+                performed: true,
+                info: baseInfo,
+            };
+        }
+        const hostname = this.safeHostname(resolvedUrl);
+        const prompt = hostname
+            ? `This will open an external page on ${hostname}. Continue?`
+            : "This will open an external page. Continue?";
+        flowLog("Prompting for external navigation", {
+            query: normalizedQuery,
+            url: resolvedUrl,
+            score: best.hit.score,
+        });
+        const confirmed = window.confirm(prompt);
+        if (!confirmed) {
+            flowLog("External navigation declined", {
+                query: normalizedQuery,
+                url: resolvedUrl,
+            });
+            baseInfo.reason = "user_declined";
+            baseInfo.message = "External navigation cancelled.";
+            return {
+                performed: false,
+                info: baseInfo,
+            };
+        }
+        this.prepareForNavigation({ target: resolvedUrl });
+        window.open(resolvedUrl, "_blank", "noopener");
+        this.pauseAfterExternalNavigation();
+        baseInfo.message = hostname
+            ? `Opened ${hostname} in a new tab.`
+            : "Opened external result in a new tab.";
+        flowLog("External navigation approved", {
+            query: normalizedQuery,
+            url: resolvedUrl,
+            score: best.hit.score,
+        });
+        return {
+            performed: true,
+            info: baseInfo,
+        };
+    }
+    normalizeSearchQuery(primary, fallback) {
+        const candidates = [primary, fallback];
+        for (const candidate of candidates) {
+            if (!candidate || typeof candidate !== "string") {
+                continue;
+            }
+            let working = candidate.trim();
+            if (!working) {
+                continue;
+            }
+            working = working.replace(/^["'](.+)["']$/u, "$1").trim();
+            const prefacePatterns = [
+                /^(please\s+)?(can|could|would)\s+you\s+/i,
+                /^i\s+(want|would\s+like)\s+to\s+/i,
+            ];
+            for (const pattern of prefacePatterns) {
+                working = working.replace(pattern, "").trim();
+            }
+            const searchPatterns = [
+                /^(search|find|look\s*up|show\s+me|tell\s+me\s+about)\s+(for\s+)?/i,
+            ];
+            for (const pattern of searchPatterns) {
+                const replaced = working.replace(pattern, "").trim();
+                if (replaced !== working) {
+                    working = replaced;
+                    break;
+                }
+            }
+            working = this.normalizeIndianSchemeName(working);
+            if (working) {
+                return working;
+            }
+        }
+        return "";
+    }
+    normalizeIndianSchemeName(query) {
+        let normalized = query.trim();
+        const replacements = [
+            [/\bpm\s+/gi, "Pradhan Mantri "],
+            [/\bpradhanmantri\b/gi, "Pradhan Mantri"],
+            [/\bpradhanmanthri\b/gi, "Pradhan Mantri"],
+            [/\bpradhaan\s+mantri\b/gi, "Pradhan Mantri"],
+            [/\baawas\b/gi, "Awas"],
+            [/\bavas\b/gi, "Awas"],
+            [/\baavas\b/gi, "Awas"],
+            [/\byojna\b/gi, "Yojana"],
+            [/\byojana\b/gi, "Yojana"],
+            [/\byojanaa\b/gi, "Yojana"],
+            [/\bpmay\b/gi, "Pradhan Mantri Awas Yojana"],
+            [/\bukiran\b/gi, "Ujjwala"],
+            [/\bjan\s*dhan\b/gi, "Jan Dhan"],
+            [/\bjandhan\b/gi, "Jan Dhan"],
+            [/\s+/g, " "],
+        ];
+        for (const [pattern, replacement] of replacements) {
+            normalized = normalized.replace(pattern, replacement);
+        }
+        return normalized.trim();
+    }
+    resolveUrl(url) {
+        try {
+            return new URL(url, window.location.href).toString();
+        }
+        catch (error) {
+            return url;
+        }
+    }
+    isSameDomain(url) {
+        try {
+            const current = new URL(window.location.href);
+            const target = new URL(url, current.href);
+            return current.host === target.host;
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    safeHostname(url) {
+        try {
+            const target = new URL(url, window.location.href);
+            return target.hostname || null;
+        }
+        catch {
+            return null;
+        }
+    }
+    pauseAfterExternalNavigation() {
+        try {
+            this.microphoneHandler.disposeRecognizer();
+        }
+        catch {
+        }
+        this.isProcessing = false;
+        setState(this.ui, "idle");
+        emitEvent("state-change", { state: "idle" });
+        this.microphoneHandler.resetTranscriptionBuffer();
+        updateTranscript(this.ui, "");
+    }
+    destroy() {
+        this.clearAutoStartFallback();
+        if (this.ui.root.parentElement) {
+            this.ui.root.parentElement.removeChild(this.ui.root);
+        }
+        const styles = document.getElementById("navigate-control-styles");
+        if (styles) {
+            styles.remove();
+        }
+    }
+    persistResumeState(target) {
+        if (typeof window === "undefined") {
+            return;
+        }
+        if (typeof sessionStorage === "undefined") {
+            return;
+        }
+        const shouldResume = this.isActiveState();
+        try {
+            if (!shouldResume) {
+                sessionStorage.removeItem(RESUME_STATE_STORAGE_KEY);
+                return;
+            }
+            const payload = {
+                shouldResume: true,
+                timestamp: Date.now(),
+                target: target ?? null,
+            };
+            sessionStorage.setItem(RESUME_STATE_STORAGE_KEY, JSON.stringify(payload));
+        }
+        catch (error) {
+        }
+    }
+    consumeResumeState() {
+        if (typeof window === "undefined") {
+            return false;
+        }
+        if (typeof sessionStorage === "undefined") {
+            return false;
+        }
+        try {
+            const raw = sessionStorage.getItem(RESUME_STATE_STORAGE_KEY);
+            if (!raw) {
+                return false;
+            }
+            sessionStorage.removeItem(RESUME_STATE_STORAGE_KEY);
+            const parsed = JSON.parse(raw);
+            if (!parsed?.shouldResume) {
+                return false;
+            }
+            const isRecent = typeof parsed.timestamp === "number"
+                ? Date.now() - parsed.timestamp < 60000
+                : true;
+            if (!isRecent) {
+                return false;
+            }
+            if (parsed.target) {
+                try {
+                    const targetUrl = new URL(parsed.target, window.location.href);
+                    if (targetUrl.origin !== window.location.origin) {
+                        return false;
+                    }
+                }
+                catch {
+                    return false;
+                }
+            }
+            return true;
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    isActiveState() {
+        if (!this.ui?.root?.dataset?.state) {
+            return false;
+        }
+        const state = this.ui.root.dataset.state;
+        return state === "listening" || state === "thinking" || state === "loading";
+    }
+    handleAutoStartFallback(reason) {
+        if (reason !== "auto") {
+            return;
+        }
+        if (typeof window === "undefined") {
+            return;
+        }
+        if (this.microphoneHandler.hasMicrophonePermission()) {
+            return;
+        }
+        if (this.autoStartPendingUserInteraction) {
+            return;
+        }
+        this.autoStartPendingUserInteraction = true;
+        setState(this.ui, "idle");
+        const resume = () => {
+            this.clearAutoStartFallback();
+            void this.microphoneHandler
+                .unlockAudioContext(true)
+                .finally(() => {
+                void this.start({ reason: "user" });
+            });
+        };
+        const events = [
+            "pointerdown",
+            "keydown",
+            "touchstart",
+            "click",
+        ];
+        events.forEach((event) => {
+            window.addEventListener(event, resume, { once: false, passive: true });
+        });
+        this.removeAutoStartListeners = () => {
+            events.forEach((event) => {
+                window.removeEventListener(event, resume);
+            });
+        };
+        updateStatus(this.ui, "Tap to enable voice navigation");
+    }
+    clearAutoStartFallback() {
+        if (this.removeAutoStartListeners) {
+            this.removeAutoStartListeners();
+            this.removeAutoStartListeners = null;
+        }
+        this.autoStartPendingUserInteraction = false;
+    }
+    isPermissionRelatedError(error) {
+        const message = (error?.message || "").toLowerCase();
+        if (message.includes("permission") || message.includes("denied")) {
+            return true;
+        }
+        if (message.includes("audiocontext") && message.includes("not allowed")) {
+            return true;
+        }
+        if (message.includes("must be resumed") && message.includes("user gesture")) {
+            return true;
+        }
+        return false;
+    }
+}