@nyris/nyris-webapp 0.3.90 → 0.3.92

package/src/hooks/useGroundedSearch.ts

@@ -0,0 +1,88 @@
+import { useCallback } from 'react';
+import {
+  groundedSearch,
+  groundedSearchFilter,
+  groundingSearchQuery,
+} from 'services/vizo';
+import useResultStore from 'stores/result/resultStore';
+import useRequestStore from 'stores/request/requestStore';
+import useUiStore from 'stores/ui/uiStore';
+import { RectCoords } from '@nyris/nyris-api';
+
+export const useGroundedSearch = () => {
+  const setGoogleGroundingResponse = useResultStore(
+    state => state.setGoogleGroundingResponse,
+  );
+  const setGroundingFilterResult = useResultStore(
+    state => state.setGroundingFilterResult,
+  );
+  const setGroundingError = useResultStore(state => state.setGroundingError);
+  const setIsGoogleGroundingLoading = useUiStore(
+    state => state.setIsGoogleGroundingLoading,
+  );
+
+  const groundedSearchCallback = useCallback(
+    (
+      requestImage: File | HTMLCanvasElement,
+      products: any[] = [],
+      region?: RectCoords,
+    ) => {
+      setIsGoogleGroundingLoading(true);
+      //   setGroundingError('Unable to search the Internet.');
+      setGroundingError(null);
+      groundedSearch(requestImage, region)
+        .then(groundingRes => {
+          const groundingSpecs = {
+            product_name: groundingRes.product_name,
+            brand: groundingRes.brand,
+            part_number: groundingRes.part_number,
+            key_specs: groundingRes.key_specs,
+          };
+
+          groundedSearchFilter({
+            products: products || [],
+            groundingSummary: groundingRes?.grounding_summary || '',
+            apiKey: window.settings.aiApiKey || '',
+          })
+            .then((filterRes: any) => {
+              setGroundingFilterResult(filterRes?.matches || []);
+
+              if (
+                filterRes?.matches?.length === 0 &&
+                window.settings.vizo?.fallbackToElasticSearch
+              ) {
+                groundingSearchQuery({
+                  groundingSummary: JSON.stringify(groundingRes),
+                  apiKey: window.settings.aiApiKey || '',
+                }).then((queryRes: any) => {
+                  useRequestStore.getState().setGroundingQuery(queryRes?.query);
+                });
+              }
+            })
+            .catch(error => {
+              console.log('Grounding Filter API error:', error);
+              setGroundingError('Unable to search the Internet.');
+            })
+            .finally(() => {
+              setIsGoogleGroundingLoading(false);
+            });
+
+          setGoogleGroundingResponse(groundingSpecs);
+        })
+        .catch(error => {
+          console.log('Grounding API error:', error);
+          setGroundingError('Unable to search the Internet.');
+          setIsGoogleGroundingLoading(false);
+        })
+        .finally(() => {});
+    },
+    [
+      setGoogleGroundingResponse,
+      setGroundingFilterResult,
+      setGroundingError,
+      setIsGoogleGroundingLoading,
+    ],
+  );
+
+  return { groundedSearch: groundedSearchCallback };
+};

package/src/hooks/useImageSearch.ts

@@ -2,7 +2,6 @@ import { useCallback } from 'react';
 import { RectCoords } from '@nyris/nyris-api';
 import { isEmpty, isUndefined } from 'lodash';
 
-import { decode } from 'tiff';
 import { createImage, find, findRegions } from 'services/visualSearch';
 
 import useResultStore from 'stores/result/resultStore';
@@ -10,25 +9,18 @@ import useRequestStore from 'stores/request/requestStore';
 
 import { AppSettings } from 'types';
 import { compressImage } from 'utils/compressImage';
+import { pdfToImage, tiffToJpg } from 'utils/imageConverters';
 import useUiStore from 'stores/ui/uiStore';
 import { useClearRefinements } from 'react-instantsearch';
 import { isHEIC } from 'utils/misc';
 
-import * as pdfjsLib from 'pdfjs-dist';
-import { GlobalWorkerOptions } from 'pdfjs-dist';
+import useResultEvaluator from './useResultEvaluator';
+import useConversationStore from 'stores/chat/conversationStore';
+import { useGroundedSearch } from './useGroundedSearch';
+import { generateSmartFilters } from '../services/vizo';
 
-// @ts-ignore
-import * as pdfjsWorker from 'pdfjs-dist/build/pdf.worker.min.mjs';
-import {
-  groundedSearch,
-  groundedSearchFilter,
-  groundingSearchQuery,
-} from 'services/vizo';
-
-GlobalWorkerOptions.workerSrc = new URL(
-  pdfjsWorker,
-  import.meta.url,
-).toString();
+const DEFAULT_SMART_FILTER_PROMPT =
+  'Extract at most 4 relevant product intent filters from the request image. Return strict JSON only.';
 
 export const useImageSearch = () => {
   const setRegions = useRequestStore(state => state.setRegions);
@@ -73,129 +65,18 @@ export const useImageSearch = () => {
   const setGoogleGroundingResponse = useResultStore(
     state => state.setGoogleGroundingResponse,
   );
-  const setIsGoogleGroundingLoading = useUiStore(
-    state => state.setIsGoogleGroundingLoading,
-  );
 
   const setGroundingFilterResult = useResultStore(
     state => state.setGroundingFilterResult,
   );
+  const setSmartFilters = useResultStore(state => state.setSmartFilters);
 
-  const { refine } = useClearRefinements();
-
-  const tiffToJpg = async (file: File): Promise<Blob> => {
-    return new Promise((resolve, reject) => {
-      const reader = new FileReader();
-
-      reader.onload = (event: ProgressEvent<FileReader>) => {
-        if (event.target?.result) {
-          try {
-            const tiffArray = new Uint8Array(
-              event.target.result as ArrayBuffer,
-            );
-            const tiffImages = decode(tiffArray);
-            const firstImage = tiffImages[0];
-            const { width, height, data } = firstImage;
-            const canvas = document.createElement('canvas');
-            const ctx = canvas.getContext('2d');
-            let firstImageData: any = data;
-
-            if (!ctx) {
-              reject(new Error('Failed to get canvas context.'));
-              return;
-            }
-            // Convert RGB to RGBA by adding an alpha channel
-            if (data.length === width * height * 3) {
-              const fixedData = new Uint8ClampedArray(width * height * 4);
-              for (let i = 0, j = 0; i < data.length; i += 3, j += 4) {
-                fixedData[j] = data[i];
-                fixedData[j + 1] = data[i + 1];
-                fixedData[j + 2] = data[i + 2];
-                fixedData[j + 3] = 255;
-              }
-              firstImageData = fixedData;
-            }
-
-            canvas.width = firstImage.width;
-            canvas.height = firstImage.height;
+  const setGroundingError = useResultStore(state => state.setGroundingError);
 
-            const imageData = new ImageData(
-              new Uint8ClampedArray(firstImageData),
-              width,
-              height,
-            );
-            ctx.putImageData(imageData, 0, 0);
-
-            canvas.toBlob(blob => {
-              if (blob) {
-                resolve(blob);
-              } else {
-                reject(new Error('Failed to convert TIFF to JPG.'));
-              }
-            }, 'image/jpeg');
-          } catch (error) {
-            console.log(error);
-            reject(new Error('Error decoding TIFF file.'));
-          }
-        } else {
-          reject(new Error('FileReader failed to load file.'));
-        }
-      };
-
-      reader.onerror = () => reject(new Error('Error reading TIFF file.'));
-      reader.readAsArrayBuffer(file);
-    });
-  };
-
-  const pdfToImage = async (file: File): Promise<Blob> => {
-    return new Promise((resolve, reject) => {
-      const reader = new FileReader();
-      reader.readAsArrayBuffer(file);
-      reader.onload = async () => {
-        if (!reader.result) {
-          reject(new Error('FileReader failed to load file.'));
-          return;
-        }
-
-        try {
-          const loadingTask = pdfjsLib.getDocument(
-            new Uint8Array(reader.result as ArrayBuffer),
-          );
-          const pdf = await loadingTask.promise;
-          const page = await pdf.getPage(1); // Get first page
-
-          const scale = 2; // Adjust for better resolution
-          const viewport = page.getViewport({ scale });
-
-          const canvas = document.createElement('canvas');
-          const context = canvas.getContext('2d');
-          if (!context) {
-            reject(new Error('Failed to get canvas context.'));
-            return;
-          }
-
-          canvas.width = viewport.width;
-          canvas.height = viewport.height;
-
-          await page.render({ canvasContext: context, viewport }).promise;
-
-          canvas.toBlob(blob => {
-            if (blob) {
-              resolve(blob);
-            } else {
-              reject(new Error('Failed to convert PDF to image.'));
-            }
-          }, 'image/png');
-        } catch (error) {
-          console.log({ error });
-
-          reject(new Error('Error processing PDF file.'));
-        }
-      };
-
-      reader.onerror = () => reject(new Error('Error reading PDF file.'));
-    });
-  };
+  const { refine } = useClearRefinements();
+  const { evaluateResults } = useResultEvaluator();
+  const { groundedSearch } = useGroundedSearch();
+  const startNewSession = useConversationStore(s => s.startNewSession);
 
   const singleImageSearch = useCallback(
     async ({
@@ -208,7 +89,6 @@ export const useImageSearch = () => {
       preFilterParams,
       clearPostFilter,
       text,
-      grounding,
     }: {
       image?: any;
       settings: AppSettings;
@@ -219,16 +99,33 @@ export const useImageSearch = () => {
       preFilterParams?: Record<string, boolean>;
       clearPostFilter?: boolean;
       text?: string;
-      grounding?: boolean;
     }) => {
+      const isSmartFiltersEnabled = !!settings.useSmartFilters;
       setIsFindApiLoading(true);
       // setAlgoliaProducts([]);
+      useRequestStore.getState().setGroundingQuery('');
+
+      useUiStore.getState().resetBanner();
+      startNewSession();
 
       if (newSearch) {
         setGroundingFilterResult([]);
         setGoogleGroundingResponse(null);
+        setGroundingError(null);
+        if (isSmartFiltersEnabled) {
+          setSmartFilters({
+            status: 'idle',
+            filters: [],
+            selectedFilterKey: null,
+            selectedFilterValue: null,
+            rerankedSkus: [],
+            rerankStatus: 'idle',
+            errorMessage: null,
+          });
+        }
         useRequestStore.getState().setGroundingQuery('');
         useResultStore.getState().setShowingGroundingFilterResult(false);
+        startNewSession();
       }
 
       let region: RectCoords | undefined = imageRegion;
@@ -297,6 +194,88 @@ export const useImageSearch = () => {
         },
       ];
 
+      let smartFilterPromise: Promise<void> | null = null;
+      if (isSmartFiltersEnabled && requestImage && newSearch) {
+        setSmartFilters({
+          status: 'loading',
+          filters: [],
+          selectedFilterKey: null,
+          selectedFilterValue: null,
+          rerankedSkus: [],
+          rerankStatus: 'idle',
+          errorMessage: null,
+        });
+
+        smartFilterPromise = generateSmartFilters({
+          image: requestImage,
+          prompt: DEFAULT_SMART_FILTER_PROMPT,
+        })
+          .then(response => {
+            const rawFilters = response?.filters;
+            if (!rawFilters || typeof rawFilters !== 'object') {
+              setSmartFilters({
+                status: 'error',
+                filters: [],
+                selectedFilterKey: null,
+                selectedFilterValue: null,
+                rerankedSkus: [],
+                rerankStatus: 'idle',
+                errorMessage: 'Could not generate smart filters.',
+              });
+              return;
+            }
+
+            const normalizedFilters = Object.entries(rawFilters)
+              .slice(0, 4)
+              .map(([key, value]) => {
+                const normalizedValue = Array.isArray(value)
+                  ? String(value[0] || '')
+                  : String(value || '');
+                return {
+                  key: String(key),
+                  label: String(key),
+                  value: normalizedValue,
+                  selected: false,
+                };
+              })
+              .filter(filter => filter.value.trim().length > 0);
+
+            const rerankedSkus =
+              response?.rerankedSkus ||
+              response?.filtered_items?.map(item => String(item.sku)) ||
+              [];
+            const rerankingApplied =
+              response?.rerankingApplied ??
+              (rerankedSkus.length > 0 ? true : false);
+
+            setSmartFilters({
+              status: 'success',
+              filters: normalizedFilters,
+              selectedFilterKey: null,
+              selectedFilterValue: null,
+              rerankedSkus,
+              rerankStatus: 'idle',
+              errorMessage: null,
+            });
+          })
+          .catch(error => {
+            const timeout = String(error?.message || '').includes(
+              'SMART_FILTER_TIMEOUT',
+            );
+            setSmartFilters({
+              status: 'error',
+              filters: [],
+              selectedFilterKey: null,
+              selectedFilterValue: null,
+              rerankedSkus: [],
+              rerankStatus: 'idle',
+              errorMessage: timeout
+                ? 'Smart filters are unavailable right now.'
+                : 'Could not generate smart filters.',
+            });
+          });
+      }
+
       try {
         res = await find({
           image: requestImage,
@@ -319,7 +298,7 @@ export const useImageSearch = () => {
                 : text
               : undefined,
         });
-
+        useResultStore.getState().resetGrounding();
         if (clearPostFilter) {
           refine();
           clearPostFilterSelections();
@@ -333,58 +312,23 @@ export const useImageSearch = () => {
         const specification = res?.image_analysis?.specification || {};
         const hasOcr = Object.keys(specification).length > 0;
 
-        if (hasOcr && newSearch && window.settings.vizo?.groundingEnabled) {
-          // call grounding API with ocr text
-          setIsGoogleGroundingLoading(true);
-
+        if (
+          hasOcr &&
+          window.settings.vizo?.groundingEnabled &&
+          res?.results.length > 0
+        ) {
           groundedSearch(
             requestImage || canvasImage,
-            window.settings.aiApiKey || '',
-          )
-            .then(groundingRes => {
-              const groundingSpecs = {
-                product_name: groundingRes.product_name,
-                brand: groundingRes.brand,
-                part_number: groundingRes.part_number,
-                key_specs: groundingRes.key_specs,
-              };
-
-              groundedSearchFilter({
-                products: res?.results || [],
-                groundingSummary: groundingRes?.grounding_summary || '',
-                apiKey: window.settings.aiApiKey || '',
-              })
-                .then((filterRes: any) => {
-                  setGroundingFilterResult(filterRes?.matches || []);
-
-                  if (
-                    filterRes?.matches?.length === 0 &&
-                    window.settings.vizo?.fallbackToElasticSearch
-                  ) {
-                    groundingSearchQuery({
-                      groundingSummary: JSON.stringify(groundingRes),
-                      apiKey: window.settings.aiApiKey || '',
-                    }).then((queryRes: any) => {
-                      useRequestStore
-                        .getState()
-                        .setGroundingQuery(queryRes?.query);
-                    });
-                  }
-                })
-                .catch(error => {
-                  console.log('Grounding Filter API error:', error);
-                });
-
-              setGoogleGroundingResponse(groundingSpecs);
-            })
-            .catch(error => {
-              console.log('Grounding API error:', error);
-            })
-            .finally(() => {
-              setIsGoogleGroundingLoading(false);
-            });
+            res?.results || [],
+            region,
+          );
+        }
+        if (window.settings.vizo.chat) {
+          evaluateResults({
+            activeProducts: res?.results || [],
+            imageAnalysis: res?.image_analysis,
+          });
         }
-
         const nonEmptyFilter: any[] = ['sku:DOES_NOT_EXIST<score=1> '];
         const filterSkus: any = res?.results
           ? res?.results
@@ -400,6 +344,8 @@ export const useImageSearch = () => {
         if (showFeedback) {
           setShowFeedback(true);
         }
+
+        // setGroundingError(null);
         // go back
         if (firstSearchResults.length === 0 || newSearch) {
           setFirstSearchResults(res?.results);
@@ -411,11 +357,17 @@ export const useImageSearch = () => {
         setIsFindApiLoading(false);
       }
 
+      if (smartFilterPromise) {
+        void smartFilterPromise;
+      }
+
       return res;
     },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
     [
       setIsFindApiLoading,
       preFilter,
+      startNewSession,
       setRequestImages,
       setSpecificationFilter,
       setDetectedRegions,
@@ -430,14 +382,14 @@ export const useImageSearch = () => {
       firstSearchResults.length,
       refine,
       clearPostFilterSelections,
-      setIsGoogleGroundingLoading,
-      setGoogleGroundingResponse,
-      setGroundingFilterResult,
+      groundedSearch,
+      evaluateResults,
       setShowFeedback,
       setFirstSearchResults,
       setFirstSearchImage,
       setFirstSearchPreFilter,
       setFirstRequestImageAnalysis,
+      setSmartFilters,
     ],
   );