@myscheme/voice-navigation-sdk 0.1.6 → 0.1.8

package/dist/server/bedrock-handler.js

@@ -9,12 +9,7 @@ function getSignatureKey(key, dateStamp, region, service) {
     const kSigning = sign(kService, "aws4_request");
     return kSigning;
 }
-export async function bedrockHandler(req, res) {
-    if (req.method !== "POST") {
-        res.setHeader("Allow", ["POST"]);
-        res.status(405).json({ error: "Method not allowed" });
-        return;
-    }
+async function invokeBedrock(requestBody) {
     const region = process.env.AWS_REGION || "ap-south-1";
     const accessKey = process.env.AWS_ACCESS_KEY_ID;
     const secretKey = process.env.AWS_SECRET_ACCESS_KEY;
@@ -22,16 +17,21 @@ export async function bedrockHandler(req, res) {
     const modelId = process.env.AWS_MODEL_ID || "anthropic.claude-3-sonnet-20240229-v1:0";
     if (!accessKey || !secretKey) {
         console.error("[VoiceNavigation] Missing AWS credentials. Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in your .env file");
-        res.status(500).json({
-            error: "Missing AWS credentials",
-            hint: "Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables",
-        });
-        return;
+        return {
+            success: false,
+            statusCode: 500,
+            error: {
+                error: "Missing AWS credentials",
+                hint: "Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables",
+            },
+        };
     }
-    const requestBody = req.body?.body;
     if (!requestBody) {
-        res.status(400).json({ error: "Missing request body" });
-        return;
+        return {
+            success: false,
+            statusCode: 400,
+            error: { error: "Missing request body" },
+        };
     }
     const body = JSON.stringify(requestBody);
     const service = "bedrock";
@@ -95,20 +95,77 @@ export async function bedrockHandler(req, res) {
         if (!response.ok) {
             const errorText = await response.text();
             console.error("[VoiceNavigation] Bedrock API error:", response.status, errorText);
-            res.status(response.status).json({
-                error: "Bedrock API error",
-                details: errorText,
-            });
-            return;
+            return {
+                success: false,
+                statusCode: response.status,
+                error: {
+                    error: "Bedrock API error",
+                    details: errorText,
+                },
+            };
         }
         const json = await response.json();
-        res.status(200).json(json);
+        return {
+            success: true,
+            statusCode: 200,
+            data: json,
+        };
     }
     catch (error) {
         console.error("[VoiceNavigation] Bedrock action extraction error:", error);
-        res.status(500).json({
-            error: "Failed to extract action",
+        return {
+            success: false,
+            statusCode: 500,
+            error: {
+                error: "Failed to extract action",
+                message: error instanceof Error ? error.message : "Unknown error",
+            },
+        };
+    }
+}
+export async function bedrockHandler(req, res) {
+    if (req.method !== "POST") {
+        res.setHeader("Allow", ["POST"]);
+        res.status(405).json({ error: "Method not allowed" });
+        return;
+    }
+    const requestBody = req.body?.body;
+    const result = await invokeBedrock(requestBody);
+    if (result.success && result.data) {
+        res.status(result.statusCode).json(result.data);
+    }
+    else {
+        res.status(result.statusCode).json(result.error);
+    }
+}
+export async function BedrockPOST(request) {
+    try {
+        const body = await request.json();
+        const requestBody = body?.body;
+        const result = await invokeBedrock(requestBody);
+        const headers = {
+            "Content-Type": "application/json",
+        };
+        if (result.success && result.data) {
+            return new Response(JSON.stringify(result.data), {
+                status: result.statusCode,
+                headers,
+            });
+        }
+        else {
+            return new Response(JSON.stringify(result.error), {
+                status: result.statusCode,
+                headers,
+            });
+        }
+    }
+    catch (error) {
+        return new Response(JSON.stringify({
+            error: "Invalid request",
             message: error instanceof Error ? error.message : "Unknown error",
+        }), {
+            status: 400,
+            headers: { "Content-Type": "application/json" },
         });
     }
 }

package/dist/server/index.d.ts

@@ -2,6 +2,9 @@ export { azureSpeechTokenHandler } from "./azure-speech-handler.js";
 export { bedrockHandler } from "./bedrock-handler.js";
 export { bedrockEmbeddingHandler } from "./bedrock-embedding-handler.js";
 export { openSearchSearchHandler, openSearchSuggestHandler, } from "./opensearch-env-handler.js";
+export { POST } from "./azure-speech-handler.js";
+export { BedrockPOST } from "./bedrock-handler.js";
+export { BedrockEmbeddingPOST } from "./bedrock-embedding-handler.js";
 export { createOpenSearchProxyHandler, DEFAULT_OPENSEARCH_PROXY_PATH, } from "./opensearch-handler.js";
 export type { VectorSearchProxyRequest, VectorSearchProxyResponse, } from "./opensearch-handler.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/server/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,EACL,uBAAuB,EACvB,wBAAwB,GACzB,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EACL,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,wBAAwB,EACxB,yBAAyB,GAC1B,MAAM,yBAAyB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAkCA,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,EACL,uBAAuB,EACvB,wBAAwB,GACzB,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EAAE,IAAI,EAAE,MAAM,2BAA2B,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AAGtE,OAAO,EACL,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,wBAAwB,EACxB,yBAAyB,GAC1B,MAAM,yBAAyB,CAAC"}

package/dist/server/index.js

@@ -2,4 +2,7 @@ export { azureSpeechTokenHandler } from "./azure-speech-handler.js";
 export { bedrockHandler } from "./bedrock-handler.js";
 export { bedrockEmbeddingHandler } from "./bedrock-embedding-handler.js";
 export { openSearchSearchHandler, openSearchSuggestHandler, } from "./opensearch-env-handler.js";
+export { POST } from "./azure-speech-handler.js";
+export { BedrockPOST } from "./bedrock-handler.js";
+export { BedrockEmbeddingPOST } from "./bedrock-embedding-handler.js";
 export { createOpenSearchProxyHandler, DEFAULT_OPENSEARCH_PROXY_PATH, } from "./opensearch-handler.js";

package/dist/services/bedrock.d.ts

@@ -27,5 +27,8 @@ export declare class BedrockService {
     embedText(text: string): Promise<number[]>;
     private embedTextViaProxy;
     private embedTextDirect;
+    generateFeedbackMessage(prompt: string, language: string): Promise<string>;
+    private generateFeedbackViaProxy;
+    private generateFeedbackDirect;
 }
 //# sourceMappingURL=bedrock.d.ts.map

package/dist/services/bedrock.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../src/services/bedrock.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,mBAAmB,EAAmB,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,aAAa;IAE5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAmCD,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAqC;IACnD,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,QAAQ,CAAkB;gBAEtB,MAAM,EAAE,aAAa;IA6CjC,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAO7C,OAAO,CAAC,iBAAiB;IA+BnB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;YAWtD,qBAAqB;YA2ErB,mBAAmB;IAyEjC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,cAAc;IAO9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAWlC,iBAAiB;YAkFjB,eAAe;CAoF9B"}
+{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../src/services/bedrock.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,mBAAmB,EAAmB,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,aAAa;IAE5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAqDD,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAqC;IACnD,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,QAAQ,CAAkB;gBAEtB,MAAM,EAAE,aAAa;IA6CjC,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAO7C,OAAO,CAAC,iBAAiB;IA+BnB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;YAWtD,qBAAqB;YA2ErB,mBAAmB;IAyEjC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,cAAc;IAO9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAWlC,iBAAiB;YAkFjB,eAAe;IAwFvB,uBAAuB,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;YA0BJ,wBAAwB;YAwExB,sBAAsB;CA+DrC"}

package/dist/services/bedrock.js

@@ -23,11 +23,29 @@ EXAMPLES:
 - User says "scroll down" → {"action": "scroll_down"}
 - User says "find schemes" → {"action": "search_content", "query": "schemes"}
 - User says "type hello world" → {"action": "type_text", "text": "hello world"}
+- User says "tab" or "tab next" → {"action": "tab_next"}
+- User says "tab previous" or "tab back" → {"action": "tab_back"}
+- User says "focus next" → {"action": "focus_next"}
+- User says "focus previous" → {"action": "focus_previous"}
+- User says "go back" or "back" (browser history) → {"action": "go_back"}
+- User says "click" → {"action": "click"}
 
 MATCHING RULES FOR PAGE NAVIGATION:
 - Match user's words to page names and keywords
 - Choose the navigate_ action that best matches
-- If user says "go to X" or "open X" or "show me X", find the page with that name or keyword`;
+- If user says "go to X" or "open X" or "show me X", find the page with that name or keyword
+
+SPECIAL PRIORITY RULES FOR STATES AND MINISTRIES:
+- If user mentions a state/UT name (like "Assam", "Delhi", "Tamil Nadu", "Maharashtra"), ALWAYS use the state navigation action (navigate_state_*)
+- If user mentions a ministry name or ministry keywords (like "education", "health", "agriculture"), ALWAYS use the ministry navigation action (navigate_ministry_*)
+- User does NOT need to say "state of" or "ministry of" - the name alone is enough
+- Examples:
+  * "go to Assam" → {"action": "navigate_state_assam"}
+  * "open Delhi" → {"action": "navigate_state_delhi"}
+  * "show Kerala" → {"action": "navigate_state_kerala"}
+  * "go to education" → {"action": "navigate_ministry_education"}
+  * "open health ministry" → {"action": "navigate_ministry_health"}
+  * "ministry of agriculture" → {"action": "navigate_ministry_agriculture"}`;
 const DEFAULT_EMBEDDING_MODEL_ID = "cohere.embed-multilingual-v3";
 export class BedrockService {
     constructor(config) {
@@ -338,4 +356,120 @@ export class BedrockService {
             throw error;
         }
     }
+    async generateFeedbackMessage(prompt, language) {
+        console.log("[Bedrock] generateFeedbackMessage called for language:", language);
+        const systemPrompt = `You are a voice navigation assistant providing spoken feedback.
+Generate brief, natural, conversational feedback messages (maximum 10 words).
+Be specific and clear about what action was performed.
+Use the language: ${language}
+Respond with ONLY the feedback message text, no JSON, no explanations.`;
+        const userPrompt = prompt;
+        console.log("[Bedrock] Using proxy mode:", this.useProxy);
+        if (this.useProxy) {
+            return this.generateFeedbackViaProxy(systemPrompt, userPrompt);
+        }
+        else {
+            return this.generateFeedbackDirect(systemPrompt, userPrompt);
+        }
+    }
+    async generateFeedbackViaProxy(systemPrompt, userPrompt) {
+        if (!this.bedrockEndpoint) {
+            throw new Error("Bedrock endpoint not configured");
+        }
+        console.log("[Bedrock] Calling proxy endpoint:", this.bedrockEndpoint);
+        try {
+            const requestBody = {
+                anthropic_version: "bedrock-2023-05-31",
+                max_tokens: 50,
+                system: systemPrompt,
+                messages: [
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "text",
+                                text: userPrompt,
+                            },
+                        ],
+                    },
+                ],
+            };
+            const response = await fetch(this.bedrockEndpoint, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    body: requestBody,
+                }),
+            });
+            if (!response.ok) {
+                throw new Error(`Bedrock proxy error: ${response.status}`);
+            }
+            const result = (await response.json());
+            console.log("[Bedrock] Proxy response received:", result);
+            if (result.content &&
+                Array.isArray(result.content) &&
+                result.content[0]?.text) {
+                const message = result.content[0].text.trim();
+                console.log("[Bedrock] Extracted message from proxy:", message);
+                return message;
+            }
+            if (typeof result === "object" && result !== null && "text" in result) {
+                const message = String(result.text).trim();
+                console.log("[Bedrock] Extracted text from proxy:", message);
+                return message;
+            }
+            throw new Error("Unexpected feedback response format");
+        }
+        catch (error) {
+            console.error("[Bedrock] Proxy generation error:", error);
+            throw error;
+        }
+    }
+    async generateFeedbackDirect(systemPrompt, userPrompt) {
+        if (!this.client || !this.actionModelId) {
+            throw new Error("Bedrock client not configured");
+        }
+        console.log("[Bedrock] Using direct AWS SDK with model:", this.actionModelId);
+        const payload = {
+            anthropic_version: "bedrock-2023-05-31",
+            max_tokens: 50,
+            temperature: 0.7,
+            system: systemPrompt,
+            messages: [
+                {
+                    role: "user",
+                    content: userPrompt,
+                },
+            ],
+        };
+        const input = {
+            modelId: this.actionModelId,
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify(payload),
+        };
+        try {
+            const command = new InvokeModelCommand(input);
+            const response = await this.client.send(command);
+            if (!response.body) {
+                throw new Error("No response from Bedrock");
+            }
+            const decoded = JSON.parse(new TextDecoder().decode(response.body));
+            console.log("[Bedrock] Direct SDK response received");
+            if (decoded.content &&
+                Array.isArray(decoded.content) &&
+                decoded.content[0]?.text) {
+                const message = decoded.content[0].text.trim();
+                console.log("[Bedrock] Extracted message from direct:", message);
+                return message;
+            }
+            throw new Error("Unexpected feedback response format");
+        }
+        catch (error) {
+            console.error("[Bedrock] Direct generation error:", error);
+            throw error;
+        }
+    }
 }

package/dist/services/voice-feedback.d.ts

@@ -0,0 +1,46 @@
+import type { NavigationAction, ActionResult } from "../types.js";
+import type { AzureSpeechService } from "./azure-speech.js";
+import type { BedrockService } from "./bedrock.js";
+interface VoiceFeedbackConfig {
+    enabled?: boolean;
+    language?: string;
+    azureSpeechService: AzureSpeechService;
+    bedrockService: BedrockService;
+}
+interface FeedbackContext {
+    action: NavigationAction;
+    actionResult?: ActionResult;
+    transcript?: string;
+    elementInfo?: {
+        tag?: string;
+        label?: string;
+        text?: string;
+        type?: string;
+    };
+}
+export declare class VoiceFeedbackService {
+    private enabled;
+    private language;
+    private azureSpeechService;
+    private bedrockService;
+    private synthesizer;
+    private isSpeaking;
+    constructor(config: VoiceFeedbackConfig);
+    setEnabled(enabled: boolean): void;
+    isEnabled(): boolean;
+    setLanguage(language: string): void;
+    private initializeSynthesizer;
+    private getVoiceForLanguage;
+    private generateFeedbackMessage;
+    private buildPromptForAction;
+    private getFallbackMessage;
+    private speak;
+    private stopSpeaking;
+    provideFeedback(context: FeedbackContext): Promise<void>;
+    provideImmediateFeedback(message: string): Promise<void>;
+    announceFocusChange(element: HTMLElement, direction: "next" | "previous"): Promise<void>;
+    private extractElementInfo;
+    dispose(): void;
+}
+export {};
+//# sourceMappingURL=voice-feedback.d.ts.map

package/dist/services/voice-feedback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-feedback.d.ts","sourceRoot":"","sources":["../../src/services/voice-feedback.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,UAAU,mBAAmB;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,UAAU,eAAe;IACvB,MAAM,EAAE,gBAAgB,CAAC;IACzB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE;QACZ,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,UAAU,CAAkB;gBAExB,MAAM,EAAE,mBAAmB;IAiBvC,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAclC,SAAS,IAAI,OAAO;IAOpB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;YAQrB,qBAAqB;IAmCnC,OAAO,CAAC,mBAAmB;YAsBb,uBAAuB;IAsCrC,OAAO,CAAC,oBAAoB;IA4C5B,OAAO,CAAC,kBAAkB;YAgCZ,KAAK;IA0EnB,OAAO,CAAC,YAAY;IAkBd,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAoFxD,wBAAwB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBxD,mBAAmB,CACvB,OAAO,EAAE,WAAW,EACpB,SAAS,EAAE,MAAM,GAAG,UAAU,GAC7B,OAAO,CAAC,IAAI,CAAC;IAwBhB,OAAO,CAAC,kBAAkB;IAwF1B,OAAO,IAAI,IAAI;CAOhB"}