@modelnex/sdk 0.1.1

package/dist/index.d.mts

@@ -0,0 +1,562 @@
+import React$1 from 'react';
+import { z } from 'zod';
+import * as react_jsx_runtime from 'react/jsx-runtime';
+
+type TourStepType = 'narrate' | 'act' | 'ask_and_fill' | 'user_input' | 'ask_or_fill';
+interface TourStepElement {
+    /** data-testid value — primary anchor; survives DOM restructuring */
+    testId?: string;
+    /** DOM fingerprint captured at record time — first fallback */
+    fingerprint: string;
+    /** Visible label at record time — second fallback */
+    textContaining: string;
+}
+interface TourStep {
+    order: number;
+    type: TourStepType;
+    /** Route pattern for navigation (e.g. "/documents", "/documents/*") */
+    url?: string;
+    /** Target element for highlight / interaction */
+    element?: TourStepElement;
+    /** LLM-polished narration spoken via TTS to end user */
+    narration: string;
+    /** Subscriber's original spoken words — stored for re-editing */
+    rawNarration: string;
+    /** Question the AI speaks to the user (ask_and_fill / ask_or_fill) */
+    ask?: string;
+    /** Plain-English goal for act steps — AI plans DOM actions */
+    goal?: string;
+    /** Pacing for narrate steps */
+    waitFor?: 'voice_next' | 'auto_advance';
+}
+type TourTrigger = 'first_visit' | 'feature_unlock' | 'idle_on_empty_state' | 'return_visit' | 'new_feature' | 'manual';
+interface TourVoiceConfig {
+    language: string;
+    ttsVoice?: string;
+}
+interface Tour {
+    id: string;
+    name: string;
+    websiteId: string;
+    targetUserTypes: string[];
+    trigger: TourTrigger;
+    voice: TourVoiceConfig;
+    steps: TourStep[];
+    status: 'draft' | 'published';
+    createdAt: string;
+    updatedAt: string;
+}
+/** User profile passed to ModelNexProvider — used for tour targeting */
+interface UserProfile {
+    /** User type matched against tour targetUserTypes (e.g. "admin", "free") */
+    type: string;
+    /** Whether this is a new user — triggers first_visit tours */
+    isNewUser?: boolean;
+    /** User ID for per-user completion state */
+    userId?: string;
+}
+/** Playback state machine states */
+type TourPlaybackState = 'idle' | 'intro' | 'executing' | 'waiting_voice' | 'waiting_input' | 'complete';
+/** Internal recording step being built */
+interface RecordingStep {
+    order: number;
+    type: TourStepType;
+    url: string;
+    element?: TourStepElement & {
+        label?: string;
+    };
+    rawNarration: string;
+    narration: string;
+    ask?: string;
+    goal?: string;
+    waitFor?: 'voice_next' | 'auto_advance';
+}
+
+/**
+ * Automatic DOM element extraction with stable fingerprinting.
+ *
+ * Scans the live DOM for all interactive elements (buttons, links, inputs, forms, etc.)
+ * and assigns each a deterministic fingerprint based on semantic anchors — NOT fragile
+ * DOM paths. A MutationObserver re-scans on changes (debounced).
+ */
+interface ExtractedElement {
+    /** Stable fingerprint derived from semantic anchors */
+    fingerprint: string;
+    /** HTML tag name (lowercase) */
+    tagName: string;
+    /** Visible text content (truncated) */
+    text: string;
+    /** Element role (button, link, input, form, etc.) */
+    role: string;
+    /** Bounding box relative to viewport */
+    rect: {
+        top: number;
+        left: number;
+        width: number;
+        height: number;
+    };
+    /** Semantic attributes found on the element */
+    attributes: {
+        id?: string;
+        name?: string;
+        type?: string;
+        ariaLabel?: string;
+        dataTestId?: string;
+        href?: string;
+        placeholder?: string;
+    };
+    /** Nearest ancestor heading text (for context) */
+    nearestHeading: string | null;
+    /** Parent container context (e.g. "dialog:Confirm Delete", "menu", "popover") */
+    parentContainer: string | null;
+    /** Whether the element is currently disabled */
+    disabled: boolean;
+    /** Reference to the live DOM element (not serializable) */
+    element: HTMLElement;
+}
+declare function generateFingerprint(el: HTMLElement): string;
+declare function extractInteractiveElements(): ExtractedElement[];
+/**
+ * Hook that automatically extracts all interactive DOM elements and keeps
+ * the list up-to-date via MutationObserver (debounced at 500ms).
+ */
+declare function useAutoExtract(): ExtractedElement[];
+
+/**
+ * Tag store for human-in-the-loop tagging.
+ * Persists semantic descriptions for auto-extracted elements via localStorage.
+ */
+interface TagData {
+    /** The element fingerprint this tag is associated with (can be empty if selector is used) */
+    fingerprint: string;
+    /** Optional CSS selector for pattern-based matching across instances */
+    selector?: string;
+    /** Optional human-readable ID for the pattern (e.g. 'delete-btn') */
+    patternId?: string;
+    /** Human-provided semantic description */
+    description: string;
+    /** What happens when the user interacts with this element (e.g. "Opens a confirmation dialog") */
+    behavior?: string;
+    /** The page route where this element was discovered (e.g. "/documents") */
+    sourcePage?: string;
+    /** How this element becomes visible (e.g. "Inside dropdown after clicking '...' menu") */
+    displayContext?: string;
+    /** Optional category */
+    category?: 'button' | 'navigation' | 'form-field' | 'action' | 'display' | 'other';
+    /** Optional freeform metadata */
+    metadata?: Record<string, unknown>;
+    /** ISO timestamp of creation */
+    createdAt: string;
+    /** ISO timestamp of last update */
+    updatedAt: string;
+    /** Contextual signature (nearestHeading, parentRole) to help resolve if selector fails */
+    contextualSignature?: {
+        nearestHeading?: string;
+        parentRole?: string;
+    };
+    /** Whether the agent must ask for user confirmation before executing this action */
+    requiresConfirmation?: boolean;
+}
+interface TagStore {
+    tags: Map<string, TagData>;
+    getTag: (fingerprint: string) => TagData | undefined;
+    setTag: (fingerprint: string, description: string, category?: TagData['category'], metadata?: Record<string, unknown>, selector?: string, patternId?: string, behavior?: string, sourcePage?: string, displayContext?: string) => void;
+    deleteTag: (fingerprint: string) => void;
+    getAllTags: () => TagData[];
+    /** Export all tags as a JSON string (for debugging / backup) */
+    exportTags: () => string;
+    /** Import tags from a JSON string (merges with existing) */
+    importTags: (json: string) => void;
+}
+declare function useTagStore(options?: {
+    serverUrl?: string;
+    websiteId?: string;
+}): TagStore;
+
+/** LLM input for a single step */
+interface AgentTraceLlmInput {
+    systemPrompt?: string;
+    userMessage?: string;
+}
+/** Single step in agent execution trace */
+interface AgentTraceStep {
+    step: number;
+    /** Chain-of-thought reasoning from the agent */
+    reasoning?: string;
+    llmInput?: AgentTraceLlmInput;
+    llmOutput: string;
+    actions: Array<{
+        actionId: string;
+        params?: Record<string, unknown>;
+    }>;
+    results?: Array<{
+        actionId: string;
+        success?: boolean;
+        result?: unknown;
+        error?: string;
+    }>;
+}
+/** Debug payload from /agent/command */
+interface AgentDebug {
+    actions?: Array<{
+        actionId: string;
+        params?: Record<string, unknown>;
+    }>;
+    llmInput?: {
+        systemPrompt?: string;
+        systemPromptLength?: number;
+        userMessage?: string;
+    };
+    llmOutput?: string[];
+    traces?: AgentTraceStep[];
+}
+interface RunCommandResult {
+    success: boolean;
+    actionsExecuted?: number;
+    sessionEnded?: boolean;
+    /** User-facing summary of what the agent did (when session ends) */
+    summary?: string | null;
+    /** Suggested next steps for the user */
+    nextSteps?: string[] | null;
+    debug?: AgentDebug;
+    [key: string]: unknown;
+}
+/**
+ * Run a natural language command via the agent.
+ * Uses serverUrl from ModelNexProvider when available, else default.
+ * Pass an AbortSignal to cancel the request mid-flight.
+ *
+ * Before sending, pre-searches the full tag store for elements relevant to the
+ * command and attaches them as `relevantTaggedElements` so the server can inject
+ * them into the system prompt alongside the documentation context — giving the
+ * agent both doc knowledge and element discovery in a single context window.
+ */
+declare function useRunCommand(serverUrlOverride?: string): (command: string, signal?: AbortSignal) => Promise<RunCommandResult>;
+
+/** Chat message for persistent conversation history */
+interface ChatMessage {
+    role: 'user' | 'assistant';
+    content: string;
+    summary?: string;
+    nextSteps?: string[];
+    debug?: AgentDebug;
+}
+
+interface UseRegisterActionOptions<T extends z.ZodTypeAny = z.ZodTypeAny> {
+    id: string;
+    description: string;
+    schema: T;
+    execute: (params: z.infer<T>) => unknown | Promise<unknown>;
+}
+/**
+ * Register a custom action with the ModelNex agent.
+ * Use this to add app-specific actions (e.g. navigation, domain operations).
+ * The action is synced to the server with built-in actions and can be invoked by the agent.
+ *
+ * Must be used within ModelNexProvider.
+ * Memoize schema (define outside component) and execute (useCallback) to avoid unnecessary re-registration.
+ */
+declare function useRegisterAction<T extends z.ZodTypeAny = z.ZodTypeAny>(action: UseRegisterActionOptions<T>): void;
+
+interface UIStateProviderProps {
+    children: React$1.ReactNode;
+    /** Agent context id (default: "agent-ui-state") */
+    id?: string;
+    /** Agent context type (default: "ui") */
+    type?: string;
+    /** Initial state */
+    initialState?: Record<string, unknown>;
+}
+/**
+ * Provider that holds UI state and syncs it to the agent.
+ * Use within ModelNexProvider for agent visibility.
+ */
+declare function UIStateProvider({ children, id, type, initialState }: UIStateProviderProps): react_jsx_runtime.JSX.Element;
+/**
+ * Hook to read and update UI state that is synced to the agent.
+ */
+declare function useUIState<T extends Record<string, unknown> = Record<string, unknown>>(): [
+    T,
+    (update: Partial<T>) => void
+];
+
+/**
+ * Returns a ref to attach to an element. When the element enters/leaves the viewport,
+ * its id is added/removed from the visible set.
+ */
+declare function useViewportTrack(id: string): React.RefObject<HTMLDivElement | null>;
+/**
+ * Returns the list of ids currently visible in the viewport.
+ * Use with useViewportTrack - each tracked element adds its id when visible.
+ */
+declare function useVisibleIds(): string[];
+interface UseAgentViewportOptions {
+    /** Agent context id (default: "agent-viewport") */
+    id?: string;
+    /** Agent context type (default: "viewport") */
+    type?: string;
+}
+/**
+ * Registers visible ids with the agent context.
+ * Call this in a parent component; use useViewportTrack in child elements.
+ */
+declare function useAgentViewport(options?: UseAgentViewportOptions): {
+    visibleIds: string[];
+};
+
+interface ModelNexChatBubbleProps {
+    /** Placeholder for the command input */
+    placeholder?: string;
+    /** Default command when input is empty */
+    defaultCommand?: string;
+    /** Additional class names for the container */
+    className?: string;
+    /** Custom fetch - use your own inference/backend. Omit to use agent server from Provider. */
+    onCommand?: (command: string) => Promise<{
+        actionsExecuted?: number;
+        summary?: string;
+        nextSteps?: string[];
+        debug?: AgentDebug;
+        [key: string]: unknown;
+    }>;
+    /** Welcome message when chat is empty */
+    welcomeMessage?: string;
+    /** Display name of the app — used in tour intro narration */
+    appName?: string;
+}
+/**
+ * Chat interface for natural language commands.
+ * Shows conversation history; on exit, the agent summarizes what it did and suggests next steps.
+ * Use within ModelNexProvider. Omit to use your own UI with useRunCommand.
+ */
+declare function ModelNexChatBubble({ placeholder, defaultCommand, className, onCommand, welcomeMessage, appName, }: ModelNexChatBubbleProps): react_jsx_runtime.JSX.Element;
+
+/**
+ * Toggle and read the action-highlight overlay state.
+ * When enabled, all elements with `data-modelnex-action-id` are visually highlighted.
+ */
+declare function useActionHighlight(): {
+    enabled: boolean;
+    toggle: () => void;
+    set: (value: boolean) => void;
+};
+
+interface StudioOverlayProps {
+    elements: ExtractedElement[];
+    tagStore: TagStore;
+}
+declare function StudioOverlay({ elements, tagStore }: StudioOverlayProps): react_jsx_runtime.JSX.Element;
+
+interface SpeechRecognitionEvent {
+    results: SpeechRecognitionResultList;
+    resultIndex: number;
+}
+interface SpeechRecognitionResultList {
+    readonly length: number;
+    [index: number]: SpeechRecognitionResult;
+}
+interface SpeechRecognitionResult {
+    readonly length: number;
+    [index: number]: SpeechRecognitionAlternative;
+    readonly isFinal: boolean;
+}
+interface SpeechRecognitionAlternative {
+    readonly transcript: string;
+    readonly confidence: number;
+}
+interface SpeechRecognitionErrorEvent {
+    error: string;
+    message: string;
+}
+interface ISpeechRecognition {
+    continuous: boolean;
+    interimResults: boolean;
+    lang: string;
+    onresult: ((event: SpeechRecognitionEvent) => void) | null;
+    onerror: ((event: SpeechRecognitionErrorEvent) => void) | null;
+    onend: (() => void) | null;
+    start(): void;
+    stop(): void;
+}
+interface SpeechRecognitionConstructor {
+    new (): ISpeechRecognition;
+}
+declare global {
+    interface Window {
+        SpeechRecognition?: SpeechRecognitionConstructor;
+        webkitSpeechRecognition?: SpeechRecognitionConstructor;
+    }
+}
+interface VoiceHook {
+    /** Speak text via Deepgram TTS (proxied through server). Uses WebRTC loopback for AEC when available. */
+    speak: (text: string, voiceId?: string) => Promise<void>;
+    /** Stop any in-progress TTS playback immediately */
+    stopSpeaking: () => void;
+    /** Start STT listening — Deepgram STT with AEC when supported, else Web Speech API */
+    startListening: (onResult: (transcript: string) => void, onInterruption?: (transcript: string) => void, onError?: (err: string) => void, options?: {
+        continuous?: boolean;
+        lang?: string;
+    }) => void;
+    /** Stop STT listening */
+    stopListening: () => void;
+    isSpeaking: boolean;
+    isListening: boolean;
+    /** When muted, speak() is a no-op and no audio plays */
+    isMuted: boolean;
+    toggleMute: () => void;
+    /** Whether STT is supported (Deepgram or Web Speech API) */
+    sttSupported: boolean;
+}
+declare function useVoice(serverUrl: string): VoiceHook;
+
+interface TourPlaybackHook {
+    /** Whether a tour is currently running */
+    isActive: boolean;
+    /** Current step index (0-based) */
+    currentStepIndex: number;
+    /** Total steps in current tour */
+    totalSteps: number;
+    /** The active tour */
+    activeTour: Tour | null;
+    /** Playback state */
+    playbackState: TourPlaybackState;
+    /** Start a specific tour manually */
+    startTour: (tour: Tour) => void;
+    /** Advance to next step (voice "next" command) */
+    advanceStep: () => void;
+    /** Skip the entire tour */
+    skipTour: () => void;
+    /** Repeat current step narration */
+    repeatStep: () => void;
+    /** Handle a voice input during the tour */
+    handleVoiceInput: (transcript: string) => void;
+    /** Handle a text input during an ask step */
+    handleTextInput: (text: string) => void;
+}
+interface UseTourPlaybackOptions {
+    serverUrl: string;
+    /** Base URL for agent commands (e.g. /api/modelnex for same-origin). Default: serverUrl */
+    commandUrl?: string;
+    /** Same-origin path for tour API (e.g. /api/modelnex/api) — avoids CORS */
+    toursApiBase?: string;
+    /** Socket ID for agent/command (required for act/ask steps to execute) */
+    socketId?: string | null;
+    websiteId?: string;
+    userProfile?: UserProfile;
+    voice: VoiceHook;
+    appName?: string;
+    /** Pass the page's current state to the LLM agent */
+    extractedElements?: any[];
+    /** Called when a tour step changes — for updating progress panel */
+    onStepChange?: (stepIndex: number, total: number, tour: Tour) => void;
+    /** Called when tour completes or is skipped */
+    onTourEnd?: () => void;
+}
+declare function useTourPlayback({ serverUrl, commandUrl, toursApiBase, socketId, websiteId, userProfile, voice, appName, extractedElements, onStepChange, onTourEnd, }: UseTourPlaybackOptions): TourPlaybackHook;
+
+interface RecordedElement {
+    el: HTMLElement;
+    testId?: string;
+    fingerprint: string;
+    textContaining: string;
+    label: string;
+    rect: DOMRect;
+}
+type RecordingPhase = 'idle' | 'active' | 'selecting' | 'configuring' | 'narrating' | 'reviewing' | 'saved' | 'finishing';
+interface RecordingModeHook {
+    phase: RecordingPhase;
+    steps: RecordingStep[];
+    stepCount: number;
+    isRecording: boolean;
+    selectedElement: RecordedElement | null;
+    pendingNarration: string;
+    polishedNarration: string;
+    selectedStepType: TourStepType;
+    startRecording: () => void;
+    markStep: () => void;
+    selectElement: (recorded: RecordedElement) => void;
+    selectPageLevel: () => void;
+    cancelSelection: () => void;
+    setStepType: (type: TourStepType) => void;
+    startNarration: () => void;
+    submitTextNarration: (text: string) => Promise<void>;
+    approveNarration: () => void;
+    redoNarration: () => void;
+    editNarration: (text: string) => void;
+    continueRecording: () => void;
+    undoLastStep: () => void;
+    previewSteps: () => void;
+    stopRecording: (tourName: string, targetUserTypes: string[]) => Promise<string | null>;
+    /** Abandon recording — discard all steps and exit recording mode */
+    cancelRecording: () => void;
+    handleVoiceCommand: (transcript: string) => void;
+}
+interface UseRecordingModeOptions {
+    serverUrl: string;
+    websiteId?: string;
+    voice: VoiceHook;
+    onPreview?: (steps: RecordingStep[]) => void;
+}
+declare function useRecordingMode({ serverUrl, websiteId, voice, onPreview, }: UseRecordingModeOptions): RecordingModeHook;
+
+interface TourProgressPanelProps {
+    tour: Tour;
+    currentStepIndex: number;
+    onSkip: () => void;
+    /** When true, hide Skip button — user says "skip" instead (fully voice-driven) */
+    voiceOnly?: boolean;
+}
+/** Compact progress bar + Skip — users care about flow, not step details */
+declare function TourProgressPanel({ tour, currentStepIndex, onSkip, voiceOnly }: TourProgressPanelProps): react_jsx_runtime.JSX.Element;
+
+interface RecordingOverlayProps {
+    stepIndex: number;
+    stepCount: number;
+    phase: RecordingPhase;
+    pendingNarration: string;
+    polishedNarration: string;
+    isListening: boolean;
+    onMarkStep: () => void;
+    onElementSelected: (recorded: RecordedElement) => void;
+    onPageLevelStep: () => void;
+    onStepTypeConfirmed: (type: TourStepType) => void;
+    onStartNarration: () => void;
+    onSubmitTextNarration: (text: string) => void;
+    onNarrationApprove: () => void;
+    onNarrationRedo: () => void;
+    onNarrationEdit: (text: string) => void;
+    onAddNextStep: () => void;
+    onDoneRecording: () => void;
+    onCancel: () => void;
+    onCancelRecording: () => void;
+    onStopRecording: () => void;
+}
+declare function RecordingOverlay({ stepIndex, stepCount, phase, pendingNarration, polishedNarration, isListening, onMarkStep, onElementSelected, onPageLevelStep, onStepTypeConfirmed, onStartNarration, onSubmitTextNarration, onNarrationApprove, onNarrationRedo, onNarrationEdit, onAddNextStep, onDoneRecording, onCancel, onCancelRecording, onStopRecording, }: RecordingOverlayProps): react_jsx_runtime.JSX.Element;
+
+interface ModelNexProviderProps {
+    children: React$1.ReactNode;
+    serverUrl?: string;
+    /** Base URL for agent commands. Use same-origin path (e.g. /api/modelnex) to avoid CORS. Default: serverUrl */
+    commandUrl?: string;
+    /** Identifier for the integrated website, for multi-tenancy */
+    websiteId?: string;
+    /**
+     * End-user profile for Voice Tour targeting.
+     * type — matched against tour targetUserTypes (e.g. "admin", "free")
+     * isNewUser — triggers first_visit tours
+     * userId — used for per-user tour completion state
+     */
+    userProfile?: UserProfile;
+    /**
+     * Same-origin base for tour API (e.g. /api/modelnex/api).
+     * When set, ?modelnex_test_tour= fetches via this path to avoid CORS.
+     */
+    toursApiBase?: string;
+    /**
+     * Enable SDK dev tools unconditionally (tour recording, studio mode)
+     */
+    devMode?: boolean;
+}
+declare const ModelNexProvider: React$1.FC<ModelNexProviderProps>;
+
+export { type AgentDebug, type AgentTraceLlmInput, type AgentTraceStep, type ChatMessage, type ExtractedElement, ModelNexChatBubble, type ModelNexChatBubbleProps, ModelNexProvider, type ModelNexProviderProps, type RecordingModeHook, RecordingOverlay, type RunCommandResult, StudioOverlay, type TagData, type TagStore, type Tour, type TourPlaybackHook, type TourPlaybackState, TourProgressPanel, type TourStep, type TourStepType, type TourTrigger, UIStateProvider, type UseRegisterActionOptions, type UserProfile, type VoiceHook, extractInteractiveElements, generateFingerprint, useActionHighlight, useAgentViewport, useAutoExtract, useRecordingMode, useRegisterAction, useRunCommand, useTagStore, useTourPlayback, useUIState, useViewportTrack, useVisibleIds, useVoice };