@nyris/nyris-webapp 0.3.91 → 0.3.92

package/src/services/vizo.ts

@@ -1,4 +1,19 @@
+import { RectCoords } from '@nyris/nyris-api';
 import axios from 'axios';
+import { cropImage } from 'utils/cropImageToBase64';
+
+export interface ChatMultimodalInput {
+  metadata: string;
+  mimeType?: string;
+  image?: string;
+}
+
+interface ChatAnalysisParams {
+  promptKey: string;
+  multimodalInputs: ChatMultimodalInput[];
+  apiKey?: string;
+}
+
 // Utility: Convert HTMLCanvasElement to Blob
 export function canvasToBlob(
   canvas: HTMLCanvasElement,
@@ -20,6 +35,149 @@ export function canvasToBlob(
   });
 }
 
+export interface SmartFilterRerankItemMetadata {
+  title: string;
+  description: string;
+  language: string;
+  sku: string;
+  score: number | null;
+}
+
+export interface SmartFilterRerankItem {
+  sku: string;
+  metadata: SmartFilterRerankItemMetadata;
+  imageUrl: string;
+}
+
+export interface SmartFilterRerankPayload {
+  filters: Record<string, string>;
+  items: SmartFilterRerankItem[];
+}
+
+export interface SmartFilterRerankParams {
+  image: File | HTMLCanvasElement;
+  payload: SmartFilterRerankPayload;
+  timeoutMs?: number;
+  apiKey?: string;
+}
+
+export interface SmartFilterRerankResponse {
+  filtered_items: Array<{ sku: string }>;
+  // TODO: confirm final backend field name; some responses use `reranked`.
+  reranked?: boolean;
+  rerankingApplied?: boolean;
+  message?: string;
+}
+
+export interface SmartFilterGenerateParams {
+  image: File | HTMLCanvasElement;
+  prompt: string;
+  model?: string;
+  timeoutMs?: number;
+  apiKey?: string;
+}
+
+export interface SmartFilterGenerateResponse {
+  filters: Record<string, string | string[]>;
+  rerankingApplied?: boolean;
+  filtered_items?: Array<{ sku: string }>;
+  rerankedSkus?: string[];
+  message?: string;
+}
+
+export async function rerankSmartFilters({
+  image,
+  payload,
+  apiKey,
+}: SmartFilterRerankParams): Promise<SmartFilterRerankResponse> {
+  const settings = window.settings;
+  const isSmartFiltersEnabled = !!settings.useSmartFilters;
+  if (!isSmartFiltersEnabled) {
+    return {
+      filtered_items: [],
+      rerankingApplied: false,
+      message: 'SMART_FILTERS_DISABLED',
+    };
+  }
+
+  const formData = new FormData();
+  if (image instanceof HTMLCanvasElement) {
+    const imageBlob = await canvasToBlob(image);
+    formData.append('image', imageBlob, 'smart-filters-rerank.jpg');
+  } else {
+    formData.append('image', image);
+  }
+  formData.append('payload', JSON.stringify(payload));
+
+  try {
+    const response = await axios.post<SmartFilterRerankResponse>(
+      `${window.settings.baseUrl}/api/ai-analysis/smart-filters/rerank`,
+      formData,
+      {
+        headers: {
+          'x-api-key': apiKey || window.settings.aiApiKey || window.settings.apiKey,
+        },
+      },
+    );
+
+    return response.data;
+  } catch (error: any) {
+    if (error?.code === 'ECONNABORTED') {
+      throw new Error('SMART_FILTER_TIMEOUT');
+    }
+    if (error.response) {
+      throw new Error(`API error: ${error.response.statusText}`);
+    } else {
+      throw new Error(`API error: ${error.message}`);
+    }
+  }
+}
+
+export async function generateSmartFilters({
+  image,
+  prompt,
+}: SmartFilterGenerateParams): Promise<SmartFilterGenerateResponse> {
+  const settings = window.settings;
+  const isSmartFiltersEnabled = !!settings.useSmartFilters;
+  if (!isSmartFiltersEnabled) {
+    return {
+      filters: {},
+      rerankingApplied: false,
+      message: 'SMART_FILTERS_DISABLED',
+    };
+  }
+
+  const formData = new FormData();
+  if (image instanceof HTMLCanvasElement) {
+    const imageBlob = await canvasToBlob(image);
+    formData.append('image', imageBlob, 'smart-filters-generate.jpg');
+  } else {
+    formData.append('image', image);
+  }
+  formData.append('prompt', prompt);
+
+  try {
+    const response = await axios.post<SmartFilterGenerateResponse>(
+      `${window.settings.baseUrl}/api/ai-analysis/smart-filters/generate`,
+      formData,
+      {
+        headers: {
+          'x-api-key':  window.settings.aiApiKey || window.settings.apiKey,
+        },
+      },
+    );
+    return response.data;
+  } catch (error: any) {
+    if (error?.code === 'ECONNABORTED') {
+      throw new Error('SMART_FILTER_TIMEOUT');
+    }
+    if (error.response) {
+      throw new Error(`API error: ${error.response.statusText}`);
+    }
+    throw new Error(`API error: ${error.message}`);
+  }
+}
+
 function buildProductEntry(p: any, index: number): string {
   const title = p.Bezeichnung || p.title || p.name || '';
   const desc =
@@ -50,7 +208,7 @@ function buildProductEntry(p: any, index: number): string {
 
   const idStr = idParts.length > 0 ? ` | IDs: ${idParts.join(', ')}` : '';
 
-  return `[${index}] SKU: ${sku} | Brand: ${brand} | ${title} | ${desc}${idStr}`;
+  return `index: ${index} | SKU: ${sku} | Brand: ${brand} | ${title} | ${desc}${idStr}`;
 }
 
 /**
@@ -60,18 +218,18 @@ function buildProductEntry(p: any, index: number): string {
  */
 export async function groundedSearch(
   imageInput: File | HTMLCanvasElement,
-  apiKey: string,
+  region?: RectCoords,
 ): Promise<any> {
   const formData = new FormData();
   let imageBlobOrFile: File | Blob;
 
   if (imageInput instanceof HTMLCanvasElement) {
-    imageBlobOrFile = await canvasToBlob(imageInput);
+    const croppedCanvas = cropImage(imageInput, region);
+    imageBlobOrFile = await canvasToBlob(croppedCanvas);
     formData.append('image', imageBlobOrFile, 'canvas.jpg');
   } else {
     formData.append('image', imageInput);
   }
-
   try {
     const response = await axios.post(
       'https://api.nyris.io/api/ai-analysis/grounded-search',
@@ -159,3 +317,33 @@ export async function groundingSearchQuery({
     }
   }
 }
+
+export async function chatAnalysis({
+  promptKey,
+  multimodalInputs,
+  apiKey,
+}: ChatAnalysisParams): Promise<any> {
+  try {
+    const response = await axios.post(
+      'https://api.nyris.io/api/ai-analysis/chat',
+      {
+        promptKey,
+        multimodalInputs,
+      },
+      {
+        headers: {
+          'x-api-key': apiKey || window.settings.aiApiKey || '',
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+
+    return response.data;
+  } catch (error: any) {
+    if (error.response) {
+      throw new Error(`API error: ${error.response.statusText}`);
+    }
+
+    throw new Error(`API error: ${error.message}`);
+  }
+}

package/src/stores/chat/chatStore.ts

@@ -0,0 +1,150 @@
+import { create } from 'zustand';
+import { persist } from 'zustand/middleware';
+
+export interface ChatMessage {
+  id: string;
+  role: 'user' | 'assistant' | 'system';
+  content: string;
+  timestamp: number;
+  // Context that was included with this message
+  context?: {
+    productSkus?: string[];
+    imageDescription?: string;
+    filters?: string[];
+  };
+}
+
+export interface ProductContext {
+  sku: string;
+  title: string;
+  description?: string;
+  imageUrl?: string;
+  attributes?: Record<string, string>;
+}
+
+interface ChatState {
+  // Messages
+  messages: ChatMessage[];
+
+  // UI State
+  isOpen: boolean;
+  isLoading: boolean;
+  isStreaming: boolean;
+
+  // Error handling
+  error: string | null;
+
+  // Session
+  conversationId: string;
+}
+
+interface ChatActions {
+  // Message actions
+  addMessage: (message: Omit<ChatMessage, 'id' | 'timestamp'>) => void;
+  updateLastAssistantMessage: (content: string) => void;
+  clearMessages: () => void;
+
+  // UI actions
+  setIsOpen: (isOpen: boolean) => void;
+  toggleOpen: () => void;
+  setIsLoading: (isLoading: boolean) => void;
+  setIsStreaming: (isStreaming: boolean) => void;
+
+  // Error
+  setError: (error: string | null) => void;
+
+  // Session
+  newConversation: () => void;
+
+  // Reset
+  reset: () => void;
+}
+
+const generateId = () =>
+  `msg_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+const generateConversationId = () =>
+  `conv_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+
+const initialState: ChatState = {
+  messages: [],
+  isOpen: false,
+  isLoading: false,
+  isStreaming: false,
+  error: null,
+  conversationId: generateConversationId(),
+};
+
+const useChatStore = create<ChatState & ChatActions>()(
+  persist(
+    (set, _get) => ({
+      ...initialState,
+
+      addMessage: message =>
+        set(state => ({
+          messages: [
+            ...state.messages,
+            {
+              ...message,
+              id: generateId(),
+              timestamp: Date.now(),
+            },
+          ],
+          error: null,
+        })),
+
+      updateLastAssistantMessage: content =>
+        set(state => {
+          const messages = [...state.messages];
+          const lastIndex = messages.length - 1;
+
+          if (lastIndex >= 0 && messages[lastIndex].role === 'assistant') {
+            messages[lastIndex] = {
+              ...messages[lastIndex],
+              content,
+            };
+          }
+
+          return { messages };
+        }),
+
+      clearMessages: () =>
+        set({
+          messages: [],
+          error: null,
+        }),
+
+      setIsOpen: isOpen => set({ isOpen }),
+
+      toggleOpen: () => set(state => ({ isOpen: !state.isOpen })),
+
+      setIsLoading: isLoading => set({ isLoading }),
+
+      setIsStreaming: isStreaming => set({ isStreaming }),
+
+      setError: error => set({ error, isLoading: false, isStreaming: false }),
+
+      newConversation: () =>
+        set({
+          messages: [],
+          conversationId: generateConversationId(),
+          error: null,
+        }),
+
+      reset: () =>
+        set({
+          ...initialState,
+          conversationId: generateConversationId(),
+        }),
+    }),
+    {
+      name: 'chat-storage',
+      partialize: state => ({
+        // Only persist conversation history, not UI state
+        messages: state.messages.slice(-50), // Keep last 50 messages
+        conversationId: state.conversationId,
+      }),
+    },
+  ),
+);
+
+export default useChatStore;

package/src/stores/chat/conversationStore.ts

@@ -0,0 +1,300 @@
+import { create } from 'zustand';
+
+// Types for the conversation flow
+export type ConversationRoute = 'pending' | 'P3' | 'P5';
+export type ResultQuality = 'pending' | 'good' | 'bad';
+export type FailureMode =
+  | null
+  | 'wrong_category'
+  | 'not_in_catalog'
+  | 'ocr_mismatch'
+  | 'ambiguous_image'
+  | 'full_machine'
+  | 'multiple_objects';
+
+export type ActionType =
+  | 'filter_apply'
+  | 'filter_remove'
+  | 'filter_clear'
+  | 'text_search'
+  | 'catalog_search_image'
+  | 'catalog_search_text'
+  | 'none' // P3 actions
+  | 'ask'
+  | 'resurface'
+  | 'reset'
+  | 'exit'; // P5 actions
+
+export interface QuickReplyOption {
+  label: string;
+  action_type:
+    | 'filter'
+    | 'compare'
+    | 'identify'
+    | 'clarify'
+    | 'confirm'
+    | 'text_search_prompt'
+    | 'catalog_search_image'
+    | 'catalog_search_text'
+    | 'restore_previous'
+    | 'related_part'
+    | 'find_related_parts'
+    | 'restore_original';
+  value?: string;
+  key?: string;
+}
+
+export interface ChatMessage {
+  id: string;
+  role: 'user' | 'assistant' | 'system';
+  content: string;
+  timestamp: number;
+  imageUrl?: string;
+  action?: {
+    type: ActionType;
+    key?: string;
+    value?: string;
+    filtered_results?: number[];
+  };
+  options?: QuickReplyOption[];
+}
+
+export interface ActiveFilter {
+  key: string;
+  value: string;
+}
+
+export interface RelatedPartSuggestion {
+  label: string;
+  description: string;
+}
+
+interface ConversationState {
+  // Session state
+  sessionId: string;
+  isEvaluated: boolean;  // P2 has fired
+  // Routing
+  route: ConversationRoute;
+  quality: ResultQuality;
+  failureMode: FailureMode;
+
+  // P5 recovery
+  attemptCounter: number; // 1-3 for P5 recovery
+
+  // Conversation
+  messages: ChatMessage[];
+  activeFilters: ActiveFilter[];
+
+  // UI state
+  isOpen: boolean;
+  hasNotification: boolean; // Pulse badge on chat icon when new message arrives
+  isLoading: boolean;
+  error: string | null;
+
+  // Opening message from P2
+  openingMessage: string | null;
+  openingOptions: QuickReplyOption[];
+
+  // Description mode — set when user clicks "Describe what you're looking for"
+  awaitingDescription: boolean;
+
+  // Pending catalog search — stored for deferred API search after AI refinement
+  pendingCatalogImage: File | string | null;
+  pendingCatalogText: string | null;
+  catalogSearchReady: boolean;
+
+  // Pre-catalog results — saved before catalog search so user can go back
+  preCatalogResults: any[] | null;
+  relatedPartsUsed: boolean;
+  pendingRelatedPart: string | null;
+  relatedPartsTableSuggestions: RelatedPartSuggestion[];
+
+  requestImageKeyword?: string; // Added to track the keyword extracted from the image in P2
+}
+
+interface ConversationActions {
+  // Session management
+  startNewSession: () => void;
+
+  // P2 evaluation result
+  setEvaluationResult: (
+    quality: ResultQuality,
+    route: ConversationRoute,
+    openingMessage: string,
+    options: QuickReplyOption[],
+    requestImageKeyword?: string,
+  ) => void;
+
+  // Messages
+  addMessage: (message: Omit<ChatMessage, 'id' | 'timestamp'>) => void;
+  clearMessages: () => void;
+
+  // Filters (for P3)
+  addFilter: (key: string, value: string) => void;
+  removeFilter: (key: string) => void;
+  clearFilters: () => void;
+
+  // P5 recovery
+  incrementAttempt: () => void;
+
+  // UI state
+  setIsOpen: (isOpen: boolean) => void;
+  clearNotification: () => void;
+  setIsLoading: (isLoading: boolean) => void;
+  setError: (error: string | null) => void;
+  setAwaitingDescription: (awaiting: boolean) => void;
+  setPendingCatalogImage: (file: File | string | null) => void;
+  setPendingCatalogText: (text: string | null) => void;
+  setCatalogSearchReady: (ready: boolean) => void;
+  setPreCatalogResults: (results: any[] | null) => void;
+  setPendingRelatedPart: (part: string | null) => void;
+  setRelatedPartsTableSuggestions: (
+    suggestions: RelatedPartSuggestion[],
+  ) => void;
+
+  setRelatedPartsUsed: (used: boolean) => void;
+
+  // Reset
+  reset: () => void;
+}
+
+const generateId = () =>
+  `msg_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+const generateSessionId = () =>
+  `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+
+const initialState: ConversationState = {
+  sessionId: '',
+  isEvaluated: false,
+  route: 'pending',
+  quality: 'pending',
+  failureMode: null,
+  attemptCounter: 0,
+  messages: [],
+  activeFilters: [],
+  isOpen: false, // Start closed so left panel is open by default
+  hasNotification: false,
+  isLoading: false,
+  error: null,
+  openingMessage: null,
+  openingOptions: [],
+  awaitingDescription: false,
+  pendingCatalogImage: null,
+  pendingCatalogText: null,
+  catalogSearchReady: false,
+  preCatalogResults: null,
+  relatedPartsUsed: false,
+  pendingRelatedPart: null,
+  relatedPartsTableSuggestions: [],
+  requestImageKeyword: '',
+};
+
+const useConversationStore = create<ConversationState & ConversationActions>()(
+  (set, get) => ({
+    ...initialState,
+
+    startNewSession: () =>
+      set(state => ({
+        ...initialState,
+        isEvaluated: false,
+        sessionId: generateSessionId(),
+        isOpen: state.isOpen, // Preserve current open state across searches
+      })),
+
+    setEvaluationResult: (
+      quality,
+      route,
+      openingMessage,
+      options,
+      requestImageKeyword,
+    ) => {
+      const state = get();
+
+      // Only set if not already evaluated (P2 fires once)
+      if (state.isEvaluated) return;
+
+      set({
+        isEvaluated: true,
+        quality,
+        route,
+        openingMessage,
+        openingOptions: options,
+        attemptCounter: route === 'P5' ? 1 : 0,
+        isOpen: false, // Don't auto-open — show notification badge instead
+        hasNotification: true,
+        requestImageKeyword,
+      });
+
+      // Add opening message to chat
+      set(state => ({
+        messages: [
+          ...state.messages,
+          {
+            id: generateId(),
+            role: 'assistant' as const,
+            content: openingMessage,
+            timestamp: Date.now(),
+            options,
+          },
+        ],
+      }));
+    },
+
+    addMessage: message =>
+      set(state => ({
+        messages: [
+          ...state.messages,
+          {
+            ...message,
+            id: generateId(),
+            timestamp: Date.now(),
+          },
+        ],
+      })),
+
+    clearMessages: () => set({ messages: [] }),
+
+    addFilter: (key, value) =>
+      set(state => ({
+        activeFilters: [
+          ...state.activeFilters.filter(f => f.key !== key),
+          { key, value },
+        ],
+      })),
+
+    removeFilter: key =>
+      set(state => ({
+        activeFilters: state.activeFilters.filter(f => f.key !== key),
+      })),
+
+    clearFilters: () => set({ activeFilters: [] }),
+
+    incrementAttempt: () =>
+      set(state => ({
+        attemptCounter: Math.min(state.attemptCounter + 1, 3),
+      })),
+
+    setIsOpen: isOpen =>
+      set({ isOpen, ...(isOpen ? { hasNotification: false } : {}) }),
+    clearNotification: () => set({ hasNotification: false }),
+    setIsLoading: isLoading => set({ isLoading }),
+    setError: error => set({ error }),
+    setAwaitingDescription: awaiting => set({ awaitingDescription: awaiting }),
+    setPendingCatalogImage: file => set({ pendingCatalogImage: file }),
+    setPendingCatalogText: text => set({ pendingCatalogText: text }),
+    setCatalogSearchReady: ready => set({ catalogSearchReady: ready }),
+    setPreCatalogResults: results => set({ preCatalogResults: results }),
+    setRelatedPartsTableSuggestions: suggestions =>
+      set({ relatedPartsTableSuggestions: suggestions }),
+    setRelatedPartsUsed: used => set({ relatedPartsUsed: used }),
+    setPendingRelatedPart: part => set({ pendingRelatedPart: part }),
+
+    reset: () =>
+      set({
+        ...initialState,
+        sessionId: generateSessionId(),
+      }),
+  }),
+);
+
+export default useConversationStore;

package/src/stores/request/Misc/misc.slice.ts

@@ -4,8 +4,8 @@ import { initialState } from './misc.initialstate';
 
 const miscSlice: StateCreator<MiscState & MiscAction> = set => ({
   ...initialState,
-  setMetaFilter: filter => set(state => ({ metaFilter: filter })),
-  setGroundingQuery: query => set(state => ({ groundingQuery: query })),
+  setMetaFilter: filter => set(() => ({ metaFilter: filter })),
+  setGroundingQuery: query => set(() => ({ groundingQuery: query })),
 });
 
 export default miscSlice;