donobu 5.20.0 → 5.21.0

package/dist/apis/FlowsAiQueriesApi.d.ts

@@ -0,0 +1,20 @@
+import type { Request, Response } from 'express';
+import type { DonobuFlowsManager } from '../managers/DonobuFlowsManager';
+/**
+ * API controller for retrieving AI query records within Donobu flows.
+ *
+ * AI queries represent the decision cycles where the AI analyzed the current
+ * page state (via clean and annotated screenshots) before deciding on the next
+ * action. Each record includes the screenshots the AI was shown, the
+ * interactable elements it could choose from, and any error that occurred
+ * during the query.
+ */
+export declare class FlowsAiQueriesApi {
+    private readonly donobuFlowsManager;
+    constructor(donobuFlowsManager: DonobuFlowsManager);
+    /**
+     * Retrieves all AI query records for a specific flow, ordered by query time.
+     */
+    getAiQueries(req: Request, res: Response): Promise<void>;
+}
+//# sourceMappingURL=FlowsAiQueriesApi.d.ts.map

package/dist/apis/FlowsAiQueriesApi.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FlowsAiQueriesApi = void 0;
+/**
+ * API controller for retrieving AI query records within Donobu flows.
+ *
+ * AI queries represent the decision cycles where the AI analyzed the current
+ * page state (via clean and annotated screenshots) before deciding on the next
+ * action. Each record includes the screenshots the AI was shown, the
+ * interactable elements it could choose from, and any error that occurred
+ * during the query.
+ */
+class FlowsAiQueriesApi {
+    constructor(donobuFlowsManager) {
+        this.donobuFlowsManager = donobuFlowsManager;
+    }
+    /**
+     * Retrieves all AI query records for a specific flow, ordered by query time.
+     */
+    async getAiQueries(req, res) {
+        const flowId = String(req.params.flowId);
+        const aiQueries = await this.donobuFlowsManager.getAiQueries(flowId);
+        res.json(aiQueries);
+    }
+}
+exports.FlowsAiQueriesApi = FlowsAiQueriesApi;
+//# sourceMappingURL=FlowsAiQueriesApi.js.map

package/dist/clients/OllamaGptClient.d.ts

@@ -1,4 +1,4 @@
-import type { ZodType } from 'zod/v4';
+import { type ZodType } from 'zod/v4';
 import type { OllamaConfig } from '../models/GptConfig';
 import type { AssistantMessage, GptMessage, ProposedToolCallsMessage, StructuredOutputMessage } from '../models/GptMessage';
 import type { ToolOption } from './GptClient';

package/dist/clients/OllamaGptClient.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OllamaGptClient = void 0;
+const v4_1 = require("zod/v4");
+const GptModelCapabilityException_1 = require("../exceptions/GptModelCapabilityException");
 const GptModelNotFoundException_1 = require("../exceptions/GptModelNotFoundException");
 const GptPlatformNotReachableException_1 = require("../exceptions/GptPlatformNotReachableException");
 const Logger_1 = require("../utils/Logger");
@@ -23,9 +25,15 @@ class OllamaGptClient extends GptClient_1.GptClient {
     }
     async ping(options) {
         const signal = options?.signal ?? AbortSignal.timeout(10_000);
+        const modelName = this.config.modelName;
         let resp;
         try {
-            resp = await fetch(`${this.apiUrl}/api/tags`, { signal });
+            resp = await fetch(`${this.apiUrl}/api/show`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ model: modelName }),
+                signal,
+            });
         }
         catch (error) {
             if (error instanceof TypeError) {
@@ -34,18 +42,19 @@ class OllamaGptClient extends GptClient_1.GptClient {
             }
             throw error;
         }
+        if (resp.status === 404) {
+            throw new GptModelNotFoundException_1.GptModelNotFoundException(this.config.type, modelName);
+        }
         if (!resp.ok) {
             throw new GptPlatformNotReachableException_1.GptPlatformNotReachableException(this.config.type);
         }
-        const data = (await resp.json());
-        const models = data.models ?? [];
-        const modelName = this.config.modelName;
-        const found = models.some((m) => {
-            const name = m.name ?? '';
-            return name === modelName || name === `${modelName}:latest`;
-        });
-        if (!found) {
-            throw new GptModelNotFoundException_1.GptModelNotFoundException(this.config.type, modelName);
+        const data = v4_1.z
+            .object({ capabilities: v4_1.z.array(v4_1.z.string()).default([]) })
+            .parse(await resp.json());
+        const requiredCapabilities = ['completion', 'tools', 'vision'];
+        const missingCapabilities = requiredCapabilities.filter((cap) => !data.capabilities.includes(cap));
+        if (missingCapabilities.length > 0) {
+            throw new GptModelCapabilityException_1.GptModelCapabilityException(this.config.type, modelName, missingCapabilities);
         }
     }
     async getMessage(messages, options) {

package/dist/esm/apis/FlowsAiQueriesApi.d.ts

@@ -0,0 +1,20 @@
+import type { Request, Response } from 'express';
+import type { DonobuFlowsManager } from '../managers/DonobuFlowsManager';
+/**
+ * API controller for retrieving AI query records within Donobu flows.
+ *
+ * AI queries represent the decision cycles where the AI analyzed the current
+ * page state (via clean and annotated screenshots) before deciding on the next
+ * action. Each record includes the screenshots the AI was shown, the
+ * interactable elements it could choose from, and any error that occurred
+ * during the query.
+ */
+export declare class FlowsAiQueriesApi {
+    private readonly donobuFlowsManager;
+    constructor(donobuFlowsManager: DonobuFlowsManager);
+    /**
+     * Retrieves all AI query records for a specific flow, ordered by query time.
+     */
+    getAiQueries(req: Request, res: Response): Promise<void>;
+}
+//# sourceMappingURL=FlowsAiQueriesApi.d.ts.map

package/dist/esm/apis/FlowsAiQueriesApi.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FlowsAiQueriesApi = void 0;
+/**
+ * API controller for retrieving AI query records within Donobu flows.
+ *
+ * AI queries represent the decision cycles where the AI analyzed the current
+ * page state (via clean and annotated screenshots) before deciding on the next
+ * action. Each record includes the screenshots the AI was shown, the
+ * interactable elements it could choose from, and any error that occurred
+ * during the query.
+ */
+class FlowsAiQueriesApi {
+    constructor(donobuFlowsManager) {
+        this.donobuFlowsManager = donobuFlowsManager;
+    }
+    /**
+     * Retrieves all AI query records for a specific flow, ordered by query time.
+     */
+    async getAiQueries(req, res) {
+        const flowId = String(req.params.flowId);
+        const aiQueries = await this.donobuFlowsManager.getAiQueries(flowId);
+        res.json(aiQueries);
+    }
+}
+exports.FlowsAiQueriesApi = FlowsAiQueriesApi;
+//# sourceMappingURL=FlowsAiQueriesApi.js.map

package/dist/esm/clients/OllamaGptClient.d.ts

@@ -1,4 +1,4 @@
-import type { ZodType } from 'zod/v4';
+import { type ZodType } from 'zod/v4';
 import type { OllamaConfig } from '../models/GptConfig';
 import type { AssistantMessage, GptMessage, ProposedToolCallsMessage, StructuredOutputMessage } from '../models/GptMessage';
 import type { ToolOption } from './GptClient';

package/dist/esm/clients/OllamaGptClient.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OllamaGptClient = void 0;
+const v4_1 = require("zod/v4");
+const GptModelCapabilityException_1 = require("../exceptions/GptModelCapabilityException");
 const GptModelNotFoundException_1 = require("../exceptions/GptModelNotFoundException");
 const GptPlatformNotReachableException_1 = require("../exceptions/GptPlatformNotReachableException");
 const Logger_1 = require("../utils/Logger");
@@ -23,9 +25,15 @@ class OllamaGptClient extends GptClient_1.GptClient {
     }
     async ping(options) {
         const signal = options?.signal ?? AbortSignal.timeout(10_000);
+        const modelName = this.config.modelName;
         let resp;
         try {
-            resp = await fetch(`${this.apiUrl}/api/tags`, { signal });
+            resp = await fetch(`${this.apiUrl}/api/show`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ model: modelName }),
+                signal,
+            });
         }
         catch (error) {
             if (error instanceof TypeError) {
@@ -34,18 +42,19 @@ class OllamaGptClient extends GptClient_1.GptClient {
             }
             throw error;
         }
+        if (resp.status === 404) {
+            throw new GptModelNotFoundException_1.GptModelNotFoundException(this.config.type, modelName);
+        }
         if (!resp.ok) {
             throw new GptPlatformNotReachableException_1.GptPlatformNotReachableException(this.config.type);
         }
-        const data = (await resp.json());
-        const models = data.models ?? [];
-        const modelName = this.config.modelName;
-        const found = models.some((m) => {
-            const name = m.name ?? '';
-            return name === modelName || name === `${modelName}:latest`;
-        });
-        if (!found) {
-            throw new GptModelNotFoundException_1.GptModelNotFoundException(this.config.type, modelName);
+        const data = v4_1.z
+            .object({ capabilities: v4_1.z.array(v4_1.z.string()).default([]) })
+            .parse(await resp.json());
+        const requiredCapabilities = ['completion', 'tools', 'vision'];
+        const missingCapabilities = requiredCapabilities.filter((cap) => !data.capabilities.includes(cap));
+        if (missingCapabilities.length > 0) {
+            throw new GptModelCapabilityException_1.GptModelCapabilityException(this.config.type, modelName, missingCapabilities);
         }
     }
     async getMessage(messages, options) {

package/dist/esm/exceptions/GptModelCapabilityException.d.ts

@@ -0,0 +1,12 @@
+import { DonobuException } from './DonobuException';
+/**
+ * Thrown when a model exists but lacks required capabilities
+ * (e.g. completion, tools, or vision).
+ */
+export declare class GptModelCapabilityException extends DonobuException {
+    readonly platform: string;
+    readonly gptModel: string;
+    readonly missingCapabilities: string[];
+    constructor(platform: string, gptModel: string, missingCapabilities: string[]);
+}
+//# sourceMappingURL=GptModelCapabilityException.d.ts.map

package/dist/esm/exceptions/GptModelCapabilityException.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GptModelCapabilityException = void 0;
+const DonobuException_1 = require("./DonobuException");
+/**
+ * Thrown when a model exists but lacks required capabilities
+ * (e.g. completion, tools, or vision).
+ */
+class GptModelCapabilityException extends DonobuException_1.DonobuException {
+    constructor(platform, gptModel, missingCapabilities) {
+        super(`The model '${gptModel}' on '${platform}' is missing required capabilities: ${missingCapabilities.join(', ')}. ` +
+            `Please choose a model that supports completion, tools, and vision.`);
+        this.platform = platform;
+        this.gptModel = gptModel;
+        this.missingCapabilities = missingCapabilities;
+    }
+}
+exports.GptModelCapabilityException = GptModelCapabilityException;
+//# sourceMappingURL=GptModelCapabilityException.js.map

package/dist/esm/managers/AdminApiController.js

@@ -9,6 +9,7 @@ const v4_1 = require("zod/v4");
 const AgentsApi_1 = require("../apis/AgentsApi");
 const AskAiApi_1 = require("../apis/AskAiApi");
 const EnvDataApi_1 = require("../apis/EnvDataApi");
+const FlowsAiQueriesApi_1 = require("../apis/FlowsAiQueriesApi");
 const FlowsApi_1 = require("../apis/FlowsApi");
 const FlowsFilesApi_1 = require("../apis/FlowsFilesApi");
 const FlowsToolCallsApi_1 = require("../apis/FlowsToolCallsApi");
@@ -237,6 +238,7 @@ class AdminApiController {
             flowsApi: new FlowsApi_1.FlowsApi(donobuStack.flowsManager),
             flowsFilesApi: new FlowsFilesApi_1.FlowsFilesApi(donobuStack.flowsPersistenceRegistry),
             flowsToolCallsApi: new FlowsToolCallsApi_1.FlowsToolCallsApi(donobuStack.flowsManager),
+            flowsAiQueriesApi: new FlowsAiQueriesApi_1.FlowsAiQueriesApi(donobuStack.flowsManager),
             pingApi: new PingApi_1.PingApi(),
             schemaApi: new SchemaApi_1.SchemaApi(),
             targetsApi: new TargetsApi_1.TargetsApi(donobuStack.targetRuntimePlugins),
@@ -308,6 +310,7 @@ class AdminApiController {
         app.get('/api/flows/:flowId/video', this.asyncHandler(apis.flowsFilesApi.getFlowVideo.bind(apis.flowsFilesApi)));
         app.get('/api/flows/:flowId/tool-calls', this.asyncHandler(apis.flowsToolCallsApi.getToolCalls.bind(apis.flowsToolCallsApi)));
         app.get('/api/flows/:flowId/tool-calls/:toolCallId', this.asyncHandler(apis.flowsToolCallsApi.getToolCall.bind(apis.flowsToolCallsApi)));
+        app.get('/api/flows/:flowId/ai-queries', this.asyncHandler(apis.flowsAiQueriesApi.getAiQueries.bind(apis.flowsAiQueriesApi)));
         app.get('/api/flows/:flowId/logs', this.asyncHandler(apis.flowsApi.getFlowLogs.bind(apis.flowsApi)));
     }
     /**

package/dist/esm/managers/DonobuFlow.d.ts

@@ -1,5 +1,6 @@
 import type { z } from 'zod/v4';
 import type { GptClient } from '../clients/GptClient';
+import type { AiQuery } from '../models/AiQuery';
 import type { ControlPanel } from '../models/ControlPanel';
 import type { FlowMetadata } from '../models/FlowMetadata';
 import type { GptMessage, StructuredOutputMessage, TextItem } from '../models/GptMessage';
@@ -40,6 +41,7 @@ export declare class DonobuFlow {
     private static readonly MAIN_MESSAGE_ELEMENT_LIST_MARKER;
     static readonly USER_INTERRUPT_MARKER = "[User interruption while flow was paused, this MUST be acknowledged]";
     inProgressToolCall: ToolCall | null;
+    readonly aiQueries: AiQuery[];
     constructor(flowsManager: DonobuFlowsManager, envData: Record<string, string>, persistence: FlowsPersistence, gptClient: GptClient | null, toolManager: ToolManager, interactionVisualizer: InteractionVisualizer, proposedToolCalls: ProposedToolCall[], invokedToolCalls: ToolCall[], gptMessages: GptMessage[], targetInspector: TargetInspector, metadata: FlowMetadata, controlPanel: ControlPanel);
     /**
      * Drives the entire Donobu flow state-machine until it reaches a

package/dist/esm/managers/DonobuFlow.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DonobuFlow = void 0;
 exports.extractFromPage = extractFromPage;
+const crypto_1 = require("crypto");
 const GptPlatformInternalErrorException_1 = require("../exceptions/GptPlatformInternalErrorException");
 const UserInterruptException_1 = require("../exceptions/UserInterruptException");
 const FlowMetadata_1 = require("../models/FlowMetadata");
@@ -104,6 +105,7 @@ class DonobuFlow {
         this.metadata = metadata;
         this.controlPanel = controlPanel;
         this.inProgressToolCall = null;
+        this.aiQueries = [];
     }
     /**
      * Drives the entire Donobu flow state-machine until it reaches a
@@ -909,18 +911,42 @@ Message: ${dialog.message()}`;
     }
     async queryGptForProposedToolCalls() {
         this.targetInspector.checkConnectedOrThrow();
+        // Initialise the AI query record immediately so the error handler always
+        // has a record to update — no conditional check needed.
+        let aiQuery = {
+            id: (0, crypto_1.randomUUID)(),
+            cleanScreenshotId: null,
+            annotatedScreenshotId: null,
+            interactableElements: null,
+            error: null,
+            startedAt: Date.now(),
+            completedAt: null,
+        };
+        this.aiQueries.push(aiQuery);
         try {
             // Discover and mark all interactable elements on the current screen/page.
             await this.targetInspector.attributeInteractableElements();
             // Capture clean and annotated screenshots. Each inspector implementation
             // handles the platform-specific details (DOM injection vs server-side compositing).
             const screenshotBytes = await this.targetInspector.takeCleanScreenshot();
-            await this.persistence.saveScreenShot(this.metadata.id, screenshotBytes);
+            const cleanScreenshotId = await this.persistence.saveScreenShot(this.metadata.id, screenshotBytes);
             await this.targetInspector.annotateInteractableElements();
             const annotatedScreenShotBytes = await this.targetInspector.takeAnnotatedScreenshot();
             await this.targetInspector.removeAnnotations();
-            await this.persistence.saveScreenShot(this.metadata.id, annotatedScreenShotBytes);
+            const annotatedScreenshotId = await this.persistence.saveScreenShot(this.metadata.id, annotatedScreenShotBytes);
             const interactableElements = await this.targetInspector.getAttributedInteractableElements();
+            // Fill in the remaining fields and persist so the frontend can display
+            // the record immediately.
+            aiQuery = {
+                ...aiQuery,
+                cleanScreenshotId,
+                annotatedScreenshotId,
+                interactableElements,
+            };
+            this.aiQueries[this.aiQueries.length - 1] = aiQuery;
+            await this.persistence
+                .setAiQuery(this.metadata.id, aiQuery)
+                .catch((err) => Logger_1.appLogger.error('Failed to persist AI query record', err));
             const mainMessage = DonobuFlow.createMainUserMessage(this.targetInspector, interactableElements);
             // Give the LLM both the pre and post annotated screenshots. It can
             // use the clean screenshot to decide what it wants to do, then map it to
@@ -945,9 +971,20 @@ Message: ${dialog.message()}`;
             }));
             Logger_1.appLogger.debug('LLM response:', JsonUtils_1.JsonUtils.objectToJson(proposedToolCallsMessage));
             MiscUtils_1.MiscUtils.updateTokenCounts(proposedToolCallsMessage, this.metadata);
+            aiQuery = { ...aiQuery, completedAt: Date.now() };
+            this.aiQueries[this.aiQueries.length - 1] = aiQuery;
+            await this.persistence
+                .setAiQuery(this.metadata.id, aiQuery)
+                .catch((err) => Logger_1.appLogger.error('Failed to persist AI query completion', err));
             return proposedToolCallsMessage;
         }
         catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            aiQuery = { ...aiQuery, error: errorMessage, completedAt: Date.now() };
+            this.aiQueries[this.aiQueries.length - 1] = aiQuery;
+            await this.persistence
+                .setAiQuery(this.metadata.id, aiQuery)
+                .catch((err) => Logger_1.appLogger.error('Failed to persist AI query error', err));
             if (this.targetInspector.isTargetClosedError(error)) {
                 this.targetInspector.checkConnectedOrThrow();
             }

package/dist/esm/managers/DonobuFlowsManager.d.ts

@@ -5,6 +5,7 @@ import type { GptClient } from '../clients/GptClient';
 import type { GptClientFactory } from '../clients/GptClientFactory';
 import type { GeneratedProject } from '../codegen/CodeGenerator';
 import type { env } from '../envVars';
+import type { AiQuery } from '../models/AiQuery';
 import type { BrowserStateReference } from '../models/BrowserStateFlowReference';
 import type { BrowserStorageState } from '../models/BrowserStorageState';
 import type { CodeGenerationOptions } from '../models/CodeGenerationOptions';
@@ -109,6 +110,8 @@ export declare class DonobuFlowsManager {
     getFlowByName(flowName: string): Promise<FlowMetadata>;
     /** Returns all the tool calls made by the given flow by ID. */
     getToolCalls(flowId: string): Promise<ToolCall[]>;
+    /** Returns all AI query records for the given flow by ID. */
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     /**
      * Attempts to delete a flow by ID. If the flow is active, then
      * `CannotDeleteRunningFlowException` is thrown. If the flow is not active,

package/dist/esm/managers/DonobuFlowsManager.js

@@ -374,6 +374,10 @@ class DonobuFlowsManager {
     async getToolCalls(flowId) {
         return this.flowCatalog.getToolCalls(flowId);
     }
+    /** Returns all AI query records for the given flow by ID. */
+    async getAiQueries(flowId) {
+        return this.flowCatalog.getAiQueries(flowId);
+    }
     /**
      * Attempts to delete a flow by ID. If the flow is active, then
      * `CannotDeleteRunningFlowException` is thrown. If the flow is not active,

package/dist/esm/managers/FlowCatalog.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../models/AiQuery';
 import type { BrowserStorageState } from '../models/BrowserStorageState';
 import type { DonobuDeploymentEnvironment } from '../models/DonobuDeploymentEnvironment';
 import type { FlowMetadata, FlowsQuery } from '../models/FlowMetadata';
@@ -21,6 +22,7 @@ export declare class FlowCatalog {
     getFlowById(flowId: string): Promise<FlowMetadata>;
     getFlowByName(flowName: string): Promise<FlowMetadata>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     deleteFlow(flowId: string): Promise<void>;
     getBrowserState(flowId: string): Promise<BrowserStorageState | null>;
     getFlows(query: FlowsQuery): Promise<PaginatedResult<FlowMetadata>>;

package/dist/esm/managers/FlowCatalog.js

@@ -71,6 +71,25 @@ class FlowCatalog {
         }
         throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
     }
+    async getAiQueries(flowId) {
+        if (this.isLocal()) {
+            const activeHandle = this.flowRuntime.get(flowId);
+            if (activeHandle) {
+                return [...activeHandle.donobuFlow.aiQueries].sort((a, b) => a.startedAt - b.startedAt);
+            }
+        }
+        for (const persistence of await this.flowsPersistenceRegistry.getAll()) {
+            try {
+                return await persistence.getAiQueries(flowId);
+            }
+            catch (error) {
+                if (!(error instanceof FlowNotFoundException_1.FlowNotFoundException)) {
+                    throw error;
+                }
+            }
+        }
+        throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
+    }
     async deleteFlow(flowId) {
         for (const persistence of await this.flowsPersistenceRegistry.getAll()) {
             await persistence.deleteFlow(flowId);

package/dist/esm/models/AiQuery.d.ts

@@ -0,0 +1,42 @@
+import type { InteractableElement } from './InteractableElement';
+/**
+ * Represents a single AI decision cycle within a flow. Created at the start of
+ * a query cycle with just `id` and `queriedAt`, then progressively filled in as
+ * screenshots are captured and elements are discovered. The `error` field is
+ * populated only if the query fails.
+ */
+export type AiQuery = {
+    /**
+     * Unique identifier for this AI query.
+     */
+    readonly id: string;
+    /**
+     * The ID of the clean (un-annotated) screenshot of the page at query time.
+     * Null until the screenshot is captured.
+     */
+    readonly cleanScreenshotId: string | null;
+    /**
+     * The ID of the annotated screenshot (with numbered element badges) sent to the AI.
+     * Null until the screenshot is captured.
+     */
+    readonly annotatedScreenshotId: string | null;
+    /**
+     * The interactable elements that were identified and sent to the AI.
+     * Null until element discovery completes.
+     */
+    readonly interactableElements: InteractableElement[] | null;
+    /**
+     * If the AI query failed, the error message. Null on success.
+     */
+    readonly error: string | null;
+    /**
+     * The Unix epoch millisecond timestamp of when the query was initiated.
+     */
+    readonly startedAt: number;
+    /**
+     * The Unix epoch millisecond timestamp of when the query completed (success
+     * or failure). Null while the query is still in progress.
+     */
+    readonly completedAt: number | null;
+};
+//# sourceMappingURL=AiQuery.d.ts.map

package/dist/esm/models/AiQuery.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=AiQuery.js.map

package/dist/esm/persistence/DonobuSqliteDb.js

@@ -395,6 +395,23 @@ CREATE INDEX IF NOT EXISTS idx_test_metadata_suite_id ON test_metadata(suite_id)
 
 ALTER TABLE flow_metadata ADD COLUMN test_id TEXT NULL REFERENCES test_metadata(id) ON DELETE CASCADE;
 CREATE INDEX IF NOT EXISTS idx_flow_metadata_test_id ON flow_metadata(test_id);
+`);
+        },
+    },
+    {
+        // Create the ai_queries table to store AI decision-cycle records (clean
+        // and annotated screenshots, interactable elements, optional error).
+        version: 11,
+        up: (db) => {
+            db.exec(`
+CREATE TABLE IF NOT EXISTS ai_queries (
+  id TEXT PRIMARY KEY,
+  flow_id TEXT NOT NULL,
+  started_at INTEGER NOT NULL,
+  ai_query TEXT NOT NULL,
+  FOREIGN KEY (flow_id) REFERENCES flow_metadata(id) ON DELETE CASCADE
+);
+CREATE INDEX IF NOT EXISTS idx_ai_queries_flow_id_started_at ON ai_queries(flow_id, started_at);
 `);
         },
     },

package/dist/esm/persistence/flows/FlowsPersistence.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../../models/AiQuery';
 import type { BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -50,6 +51,15 @@ export interface FlowsPersistence {
      * Delete a persisted tool call from a specific flow.
      */
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    /**
+     * Save an AI query record for a specific flow.
+     */
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    /**
+     * Load all AI query records for a specific flow, ordered by queriedAt.
+     * @throws FlowNotFoundException if the flow is not found
+     */
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     /**
      * Set video data for a specific flow.
      */

package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../../models/AiQuery';
 import { type BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -28,6 +29,8 @@ export declare class FlowsPersistenceDonobuApi implements FlowsPersistence {
     setToolCall(flowId: string, toolCall: ToolCall): Promise<void>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     saveScreenShot(flowId: string, bytes: Buffer): Promise<string>;
     getScreenShot(flowId: string, screenShotId: string): Promise<Buffer | null>;
     setVideo(_flowId: string, _bytes: Buffer): Promise<void>;

package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.js

@@ -134,6 +134,24 @@ class FlowsPersistenceDonobuApi {
             throw new Error(`Failed to delete tool call: ${response.status} ${response.statusText}`);
         }
     }
+    // -- AI Queries ----------------------------------------------------
+    async setAiQuery(flowId, aiQuery) {
+        const response = await this.jsonRequest(`/v1/flows/${encodeURIComponent(flowId)}/ai-queries/${encodeURIComponent(aiQuery.id)}`, 'PUT', { record: aiQuery });
+        if (!response.ok) {
+            throw new Error(`Failed to set AI query: ${response.status} ${response.statusText}`);
+        }
+    }
+    async getAiQueries(flowId) {
+        const response = await this.jsonRequest(`/v1/flows/${encodeURIComponent(flowId)}/ai-queries`, 'GET');
+        if (response.status === 404) {
+            throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
+        }
+        if (!response.ok) {
+            throw new Error(`Failed to get AI queries: ${response.status} ${response.statusText}`);
+        }
+        const body = (await response.json());
+        return body.items;
+    }
     // -- Screenshots ---------------------------------------------------
     async saveScreenShot(flowId, bytes) {
         const imageType = MiscUtils_1.MiscUtils.detectImageType(bytes);

package/dist/esm/persistence/flows/FlowsPersistenceSqlite.d.ts

@@ -1,4 +1,5 @@
 import type Database from 'better-sqlite3';
+import type { AiQuery } from '../../models/AiQuery';
 import { type BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -40,6 +41,8 @@ export declare class FlowsPersistenceSqlite implements FlowsPersistence {
     setToolCall(flowId: string, toolCall: ToolCall): Promise<void>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     setVideo(flowId: string, bytes: Buffer): Promise<void>;
     getVideoSegment(flowId: string, startOffset: number, length: number): Promise<VideoSegment | null>;
     getFlowFile(flowId: string, fileId: string): Promise<Buffer | null>;

package/dist/esm/persistence/flows/FlowsPersistenceSqlite.js

@@ -212,6 +212,16 @@ class FlowsPersistenceSqlite {
         const stmt = this.db.prepare('DELETE FROM tool_calls WHERE flow_id = ? AND id = ?');
         stmt.run(flowId, toolCallId);
     }
+    async setAiQuery(flowId, aiQuery) {
+        const stmt = this.db.prepare('INSERT OR REPLACE INTO ai_queries (id, flow_id, started_at, ai_query) VALUES (?, ?, ?, ?)');
+        stmt.run(aiQuery.id, flowId, aiQuery.startedAt, JSON.stringify(aiQuery));
+    }
+    async getAiQueries(flowId) {
+        await this.getFlowMetadataById(flowId);
+        const stmt = this.db.prepare('SELECT ai_query FROM ai_queries WHERE flow_id = ? ORDER BY started_at');
+        const rows = stmt.all(flowId);
+        return rows.map((row) => JSON.parse(row.ai_query));
+    }
     async setVideo(flowId, bytes) {
         // Ensure flow exists.
         await this.getFlowMetadataById(flowId);

package/dist/esm/persistence/flows/FlowsPersistenceVolatile.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../../models/AiQuery';
 import type { BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -14,7 +15,8 @@ export declare class FlowsPersistenceVolatile implements FlowsPersistence {
     private readonly videos;
     private readonly files;
     private readonly browserStates;
-    constructor(flows?: Map<string, FlowMetadata>, screenshots?: Map<string, Map<string, Buffer>>, toolCalls?: Map<string, ToolCall[]>, videos?: Map<string, Buffer>, files?: Map<string, Map<string, Buffer>>, browserStates?: Map<string, BrowserStorageState>);
+    private readonly aiQueries;
+    constructor(flows?: Map<string, FlowMetadata>, screenshots?: Map<string, Map<string, Buffer>>, toolCalls?: Map<string, ToolCall[]>, videos?: Map<string, Buffer>, files?: Map<string, Map<string, Buffer>>, browserStates?: Map<string, BrowserStorageState>, aiQueries?: Map<string, AiQuery[]>);
     setFlowMetadata(flowMetadata: FlowMetadata): Promise<void>;
     getFlowMetadataById(flowId: string): Promise<FlowMetadata>;
     getFlowMetadataByName(flowName: string): Promise<FlowMetadata>;
@@ -27,6 +29,8 @@ export declare class FlowsPersistenceVolatile implements FlowsPersistence {
     setToolCall(flowId: string, toolCall: ToolCall): Promise<void>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     setVideo(flowId: string, bytes: Buffer): Promise<void>;
     getVideoSegment(flowId: string, startOffset: number, length: number): Promise<VideoSegment | null>;
     getFlowFile(flowId: string, fileId: string): Promise<Buffer | null>;

package/dist/esm/persistence/flows/FlowsPersistenceVolatile.js

@@ -7,13 +7,14 @@ const MiscUtils_1 = require("../../utils/MiscUtils");
  * A volatile (in-memory) implementation of FlowsPersistence.
  */
 class FlowsPersistenceVolatile {
-    constructor(flows = new Map(), screenshots = new Map(), toolCalls = new Map(), videos = new Map(), files = new Map(), browserStates = new Map()) {
+    constructor(flows = new Map(), screenshots = new Map(), toolCalls = new Map(), videos = new Map(), files = new Map(), browserStates = new Map(), aiQueries = new Map()) {
         this.flows = flows;
         this.screenshots = screenshots;
         this.toolCalls = toolCalls;
         this.videos = videos;
         this.files = files;
         this.browserStates = browserStates;
+        this.aiQueries = aiQueries;
     }
     async setFlowMetadata(flowMetadata) {
         this.flows.set(flowMetadata.id, { ...flowMetadata });
@@ -126,6 +127,26 @@ class FlowsPersistenceVolatile {
             this.toolCalls.delete(flowId);
         }
     }
+    async setAiQuery(flowId, aiQuery) {
+        if (!this.aiQueries.has(flowId)) {
+            this.aiQueries.set(flowId, []);
+        }
+        const queries = this.aiQueries.get(flowId);
+        const existingIndex = queries.findIndex((q) => q.id === aiQuery.id);
+        if (existingIndex >= 0) {
+            queries[existingIndex] = { ...aiQuery };
+        }
+        else {
+            queries.push({ ...aiQuery });
+        }
+    }
+    async getAiQueries(flowId) {
+        if (!this.flows.has(flowId)) {
+            throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
+        }
+        const queries = this.aiQueries.get(flowId) || [];
+        return [...queries].sort((a, b) => a.startedAt - b.startedAt);
+    }
     async setVideo(flowId, bytes) {
         this.videos.set(flowId, Buffer.from(bytes));
     }
@@ -167,6 +188,7 @@ class FlowsPersistenceVolatile {
         this.flows.delete(flowId);
         this.screenshots.delete(flowId);
         this.toolCalls.delete(flowId);
+        this.aiQueries.delete(flowId);
         this.videos.delete(flowId);
         this.files.delete(flowId);
         this.browserStates.delete(flowId);

package/dist/exceptions/GptModelCapabilityException.d.ts

@@ -0,0 +1,12 @@
+import { DonobuException } from './DonobuException';
+/**
+ * Thrown when a model exists but lacks required capabilities
+ * (e.g. completion, tools, or vision).
+ */
+export declare class GptModelCapabilityException extends DonobuException {
+    readonly platform: string;
+    readonly gptModel: string;
+    readonly missingCapabilities: string[];
+    constructor(platform: string, gptModel: string, missingCapabilities: string[]);
+}
+//# sourceMappingURL=GptModelCapabilityException.d.ts.map

package/dist/exceptions/GptModelCapabilityException.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GptModelCapabilityException = void 0;
+const DonobuException_1 = require("./DonobuException");
+/**
+ * Thrown when a model exists but lacks required capabilities
+ * (e.g. completion, tools, or vision).
+ */
+class GptModelCapabilityException extends DonobuException_1.DonobuException {
+    constructor(platform, gptModel, missingCapabilities) {
+        super(`The model '${gptModel}' on '${platform}' is missing required capabilities: ${missingCapabilities.join(', ')}. ` +
+            `Please choose a model that supports completion, tools, and vision.`);
+        this.platform = platform;
+        this.gptModel = gptModel;
+        this.missingCapabilities = missingCapabilities;
+    }
+}
+exports.GptModelCapabilityException = GptModelCapabilityException;
+//# sourceMappingURL=GptModelCapabilityException.js.map

package/dist/managers/AdminApiController.js

@@ -9,6 +9,7 @@ const v4_1 = require("zod/v4");
 const AgentsApi_1 = require("../apis/AgentsApi");
 const AskAiApi_1 = require("../apis/AskAiApi");
 const EnvDataApi_1 = require("../apis/EnvDataApi");
+const FlowsAiQueriesApi_1 = require("../apis/FlowsAiQueriesApi");
 const FlowsApi_1 = require("../apis/FlowsApi");
 const FlowsFilesApi_1 = require("../apis/FlowsFilesApi");
 const FlowsToolCallsApi_1 = require("../apis/FlowsToolCallsApi");
@@ -237,6 +238,7 @@ class AdminApiController {
             flowsApi: new FlowsApi_1.FlowsApi(donobuStack.flowsManager),
             flowsFilesApi: new FlowsFilesApi_1.FlowsFilesApi(donobuStack.flowsPersistenceRegistry),
             flowsToolCallsApi: new FlowsToolCallsApi_1.FlowsToolCallsApi(donobuStack.flowsManager),
+            flowsAiQueriesApi: new FlowsAiQueriesApi_1.FlowsAiQueriesApi(donobuStack.flowsManager),
             pingApi: new PingApi_1.PingApi(),
             schemaApi: new SchemaApi_1.SchemaApi(),
             targetsApi: new TargetsApi_1.TargetsApi(donobuStack.targetRuntimePlugins),
@@ -308,6 +310,7 @@ class AdminApiController {
         app.get('/api/flows/:flowId/video', this.asyncHandler(apis.flowsFilesApi.getFlowVideo.bind(apis.flowsFilesApi)));
         app.get('/api/flows/:flowId/tool-calls', this.asyncHandler(apis.flowsToolCallsApi.getToolCalls.bind(apis.flowsToolCallsApi)));
         app.get('/api/flows/:flowId/tool-calls/:toolCallId', this.asyncHandler(apis.flowsToolCallsApi.getToolCall.bind(apis.flowsToolCallsApi)));
+        app.get('/api/flows/:flowId/ai-queries', this.asyncHandler(apis.flowsAiQueriesApi.getAiQueries.bind(apis.flowsAiQueriesApi)));
         app.get('/api/flows/:flowId/logs', this.asyncHandler(apis.flowsApi.getFlowLogs.bind(apis.flowsApi)));
     }
     /**

package/dist/managers/DonobuFlow.d.ts

@@ -1,5 +1,6 @@
 import type { z } from 'zod/v4';
 import type { GptClient } from '../clients/GptClient';
+import type { AiQuery } from '../models/AiQuery';
 import type { ControlPanel } from '../models/ControlPanel';
 import type { FlowMetadata } from '../models/FlowMetadata';
 import type { GptMessage, StructuredOutputMessage, TextItem } from '../models/GptMessage';
@@ -40,6 +41,7 @@ export declare class DonobuFlow {
     private static readonly MAIN_MESSAGE_ELEMENT_LIST_MARKER;
     static readonly USER_INTERRUPT_MARKER = "[User interruption while flow was paused, this MUST be acknowledged]";
     inProgressToolCall: ToolCall | null;
+    readonly aiQueries: AiQuery[];
     constructor(flowsManager: DonobuFlowsManager, envData: Record<string, string>, persistence: FlowsPersistence, gptClient: GptClient | null, toolManager: ToolManager, interactionVisualizer: InteractionVisualizer, proposedToolCalls: ProposedToolCall[], invokedToolCalls: ToolCall[], gptMessages: GptMessage[], targetInspector: TargetInspector, metadata: FlowMetadata, controlPanel: ControlPanel);
     /**
      * Drives the entire Donobu flow state-machine until it reaches a

package/dist/managers/DonobuFlow.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DonobuFlow = void 0;
 exports.extractFromPage = extractFromPage;
+const crypto_1 = require("crypto");
 const GptPlatformInternalErrorException_1 = require("../exceptions/GptPlatformInternalErrorException");
 const UserInterruptException_1 = require("../exceptions/UserInterruptException");
 const FlowMetadata_1 = require("../models/FlowMetadata");
@@ -104,6 +105,7 @@ class DonobuFlow {
         this.metadata = metadata;
         this.controlPanel = controlPanel;
         this.inProgressToolCall = null;
+        this.aiQueries = [];
     }
     /**
      * Drives the entire Donobu flow state-machine until it reaches a
@@ -909,18 +911,42 @@ Message: ${dialog.message()}`;
     }
     async queryGptForProposedToolCalls() {
         this.targetInspector.checkConnectedOrThrow();
+        // Initialise the AI query record immediately so the error handler always
+        // has a record to update — no conditional check needed.
+        let aiQuery = {
+            id: (0, crypto_1.randomUUID)(),
+            cleanScreenshotId: null,
+            annotatedScreenshotId: null,
+            interactableElements: null,
+            error: null,
+            startedAt: Date.now(),
+            completedAt: null,
+        };
+        this.aiQueries.push(aiQuery);
         try {
             // Discover and mark all interactable elements on the current screen/page.
             await this.targetInspector.attributeInteractableElements();
             // Capture clean and annotated screenshots. Each inspector implementation
             // handles the platform-specific details (DOM injection vs server-side compositing).
             const screenshotBytes = await this.targetInspector.takeCleanScreenshot();
-            await this.persistence.saveScreenShot(this.metadata.id, screenshotBytes);
+            const cleanScreenshotId = await this.persistence.saveScreenShot(this.metadata.id, screenshotBytes);
             await this.targetInspector.annotateInteractableElements();
             const annotatedScreenShotBytes = await this.targetInspector.takeAnnotatedScreenshot();
             await this.targetInspector.removeAnnotations();
-            await this.persistence.saveScreenShot(this.metadata.id, annotatedScreenShotBytes);
+            const annotatedScreenshotId = await this.persistence.saveScreenShot(this.metadata.id, annotatedScreenShotBytes);
             const interactableElements = await this.targetInspector.getAttributedInteractableElements();
+            // Fill in the remaining fields and persist so the frontend can display
+            // the record immediately.
+            aiQuery = {
+                ...aiQuery,
+                cleanScreenshotId,
+                annotatedScreenshotId,
+                interactableElements,
+            };
+            this.aiQueries[this.aiQueries.length - 1] = aiQuery;
+            await this.persistence
+                .setAiQuery(this.metadata.id, aiQuery)
+                .catch((err) => Logger_1.appLogger.error('Failed to persist AI query record', err));
             const mainMessage = DonobuFlow.createMainUserMessage(this.targetInspector, interactableElements);
             // Give the LLM both the pre and post annotated screenshots. It can
             // use the clean screenshot to decide what it wants to do, then map it to
@@ -945,9 +971,20 @@ Message: ${dialog.message()}`;
             }));
             Logger_1.appLogger.debug('LLM response:', JsonUtils_1.JsonUtils.objectToJson(proposedToolCallsMessage));
             MiscUtils_1.MiscUtils.updateTokenCounts(proposedToolCallsMessage, this.metadata);
+            aiQuery = { ...aiQuery, completedAt: Date.now() };
+            this.aiQueries[this.aiQueries.length - 1] = aiQuery;
+            await this.persistence
+                .setAiQuery(this.metadata.id, aiQuery)
+                .catch((err) => Logger_1.appLogger.error('Failed to persist AI query completion', err));
             return proposedToolCallsMessage;
         }
         catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            aiQuery = { ...aiQuery, error: errorMessage, completedAt: Date.now() };
+            this.aiQueries[this.aiQueries.length - 1] = aiQuery;
+            await this.persistence
+                .setAiQuery(this.metadata.id, aiQuery)
+                .catch((err) => Logger_1.appLogger.error('Failed to persist AI query error', err));
             if (this.targetInspector.isTargetClosedError(error)) {
                 this.targetInspector.checkConnectedOrThrow();
             }

package/dist/managers/DonobuFlowsManager.d.ts

@@ -5,6 +5,7 @@ import type { GptClient } from '../clients/GptClient';
 import type { GptClientFactory } from '../clients/GptClientFactory';
 import type { GeneratedProject } from '../codegen/CodeGenerator';
 import type { env } from '../envVars';
+import type { AiQuery } from '../models/AiQuery';
 import type { BrowserStateReference } from '../models/BrowserStateFlowReference';
 import type { BrowserStorageState } from '../models/BrowserStorageState';
 import type { CodeGenerationOptions } from '../models/CodeGenerationOptions';
@@ -109,6 +110,8 @@ export declare class DonobuFlowsManager {
     getFlowByName(flowName: string): Promise<FlowMetadata>;
     /** Returns all the tool calls made by the given flow by ID. */
     getToolCalls(flowId: string): Promise<ToolCall[]>;
+    /** Returns all AI query records for the given flow by ID. */
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     /**
      * Attempts to delete a flow by ID. If the flow is active, then
      * `CannotDeleteRunningFlowException` is thrown. If the flow is not active,

package/dist/managers/DonobuFlowsManager.js

@@ -374,6 +374,10 @@ class DonobuFlowsManager {
     async getToolCalls(flowId) {
         return this.flowCatalog.getToolCalls(flowId);
     }
+    /** Returns all AI query records for the given flow by ID. */
+    async getAiQueries(flowId) {
+        return this.flowCatalog.getAiQueries(flowId);
+    }
     /**
      * Attempts to delete a flow by ID. If the flow is active, then
      * `CannotDeleteRunningFlowException` is thrown. If the flow is not active,

package/dist/managers/FlowCatalog.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../models/AiQuery';
 import type { BrowserStorageState } from '../models/BrowserStorageState';
 import type { DonobuDeploymentEnvironment } from '../models/DonobuDeploymentEnvironment';
 import type { FlowMetadata, FlowsQuery } from '../models/FlowMetadata';
@@ -21,6 +22,7 @@ export declare class FlowCatalog {
     getFlowById(flowId: string): Promise<FlowMetadata>;
     getFlowByName(flowName: string): Promise<FlowMetadata>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     deleteFlow(flowId: string): Promise<void>;
     getBrowserState(flowId: string): Promise<BrowserStorageState | null>;
     getFlows(query: FlowsQuery): Promise<PaginatedResult<FlowMetadata>>;

package/dist/managers/FlowCatalog.js

@@ -71,6 +71,25 @@ class FlowCatalog {
         }
         throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
     }
+    async getAiQueries(flowId) {
+        if (this.isLocal()) {
+            const activeHandle = this.flowRuntime.get(flowId);
+            if (activeHandle) {
+                return [...activeHandle.donobuFlow.aiQueries].sort((a, b) => a.startedAt - b.startedAt);
+            }
+        }
+        for (const persistence of await this.flowsPersistenceRegistry.getAll()) {
+            try {
+                return await persistence.getAiQueries(flowId);
+            }
+            catch (error) {
+                if (!(error instanceof FlowNotFoundException_1.FlowNotFoundException)) {
+                    throw error;
+                }
+            }
+        }
+        throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
+    }
     async deleteFlow(flowId) {
         for (const persistence of await this.flowsPersistenceRegistry.getAll()) {
             await persistence.deleteFlow(flowId);

package/dist/models/AiQuery.d.ts

@@ -0,0 +1,42 @@
+import type { InteractableElement } from './InteractableElement';
+/**
+ * Represents a single AI decision cycle within a flow. Created at the start of
+ * a query cycle with just `id` and `queriedAt`, then progressively filled in as
+ * screenshots are captured and elements are discovered. The `error` field is
+ * populated only if the query fails.
+ */
+export type AiQuery = {
+    /**
+     * Unique identifier for this AI query.
+     */
+    readonly id: string;
+    /**
+     * The ID of the clean (un-annotated) screenshot of the page at query time.
+     * Null until the screenshot is captured.
+     */
+    readonly cleanScreenshotId: string | null;
+    /**
+     * The ID of the annotated screenshot (with numbered element badges) sent to the AI.
+     * Null until the screenshot is captured.
+     */
+    readonly annotatedScreenshotId: string | null;
+    /**
+     * The interactable elements that were identified and sent to the AI.
+     * Null until element discovery completes.
+     */
+    readonly interactableElements: InteractableElement[] | null;
+    /**
+     * If the AI query failed, the error message. Null on success.
+     */
+    readonly error: string | null;
+    /**
+     * The Unix epoch millisecond timestamp of when the query was initiated.
+     */
+    readonly startedAt: number;
+    /**
+     * The Unix epoch millisecond timestamp of when the query completed (success
+     * or failure). Null while the query is still in progress.
+     */
+    readonly completedAt: number | null;
+};
+//# sourceMappingURL=AiQuery.d.ts.map

package/dist/models/AiQuery.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=AiQuery.js.map

package/dist/persistence/DonobuSqliteDb.js

@@ -395,6 +395,23 @@ CREATE INDEX IF NOT EXISTS idx_test_metadata_suite_id ON test_metadata(suite_id)
 
 ALTER TABLE flow_metadata ADD COLUMN test_id TEXT NULL REFERENCES test_metadata(id) ON DELETE CASCADE;
 CREATE INDEX IF NOT EXISTS idx_flow_metadata_test_id ON flow_metadata(test_id);
+`);
+        },
+    },
+    {
+        // Create the ai_queries table to store AI decision-cycle records (clean
+        // and annotated screenshots, interactable elements, optional error).
+        version: 11,
+        up: (db) => {
+            db.exec(`
+CREATE TABLE IF NOT EXISTS ai_queries (
+  id TEXT PRIMARY KEY,
+  flow_id TEXT NOT NULL,
+  started_at INTEGER NOT NULL,
+  ai_query TEXT NOT NULL,
+  FOREIGN KEY (flow_id) REFERENCES flow_metadata(id) ON DELETE CASCADE
+);
+CREATE INDEX IF NOT EXISTS idx_ai_queries_flow_id_started_at ON ai_queries(flow_id, started_at);
 `);
         },
     },

package/dist/persistence/flows/FlowsPersistence.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../../models/AiQuery';
 import type { BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -50,6 +51,15 @@ export interface FlowsPersistence {
      * Delete a persisted tool call from a specific flow.
      */
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    /**
+     * Save an AI query record for a specific flow.
+     */
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    /**
+     * Load all AI query records for a specific flow, ordered by queriedAt.
+     * @throws FlowNotFoundException if the flow is not found
+     */
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     /**
      * Set video data for a specific flow.
      */

package/dist/persistence/flows/FlowsPersistenceDonobuApi.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../../models/AiQuery';
 import { type BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -28,6 +29,8 @@ export declare class FlowsPersistenceDonobuApi implements FlowsPersistence {
     setToolCall(flowId: string, toolCall: ToolCall): Promise<void>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     saveScreenShot(flowId: string, bytes: Buffer): Promise<string>;
     getScreenShot(flowId: string, screenShotId: string): Promise<Buffer | null>;
     setVideo(_flowId: string, _bytes: Buffer): Promise<void>;

package/dist/persistence/flows/FlowsPersistenceDonobuApi.js

@@ -134,6 +134,24 @@ class FlowsPersistenceDonobuApi {
             throw new Error(`Failed to delete tool call: ${response.status} ${response.statusText}`);
         }
     }
+    // -- AI Queries ----------------------------------------------------
+    async setAiQuery(flowId, aiQuery) {
+        const response = await this.jsonRequest(`/v1/flows/${encodeURIComponent(flowId)}/ai-queries/${encodeURIComponent(aiQuery.id)}`, 'PUT', { record: aiQuery });
+        if (!response.ok) {
+            throw new Error(`Failed to set AI query: ${response.status} ${response.statusText}`);
+        }
+    }
+    async getAiQueries(flowId) {
+        const response = await this.jsonRequest(`/v1/flows/${encodeURIComponent(flowId)}/ai-queries`, 'GET');
+        if (response.status === 404) {
+            throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
+        }
+        if (!response.ok) {
+            throw new Error(`Failed to get AI queries: ${response.status} ${response.statusText}`);
+        }
+        const body = (await response.json());
+        return body.items;
+    }
     // -- Screenshots ---------------------------------------------------
     async saveScreenShot(flowId, bytes) {
         const imageType = MiscUtils_1.MiscUtils.detectImageType(bytes);

package/dist/persistence/flows/FlowsPersistenceSqlite.d.ts

@@ -1,4 +1,5 @@
 import type Database from 'better-sqlite3';
+import type { AiQuery } from '../../models/AiQuery';
 import { type BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -40,6 +41,8 @@ export declare class FlowsPersistenceSqlite implements FlowsPersistence {
     setToolCall(flowId: string, toolCall: ToolCall): Promise<void>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     setVideo(flowId: string, bytes: Buffer): Promise<void>;
     getVideoSegment(flowId: string, startOffset: number, length: number): Promise<VideoSegment | null>;
     getFlowFile(flowId: string, fileId: string): Promise<Buffer | null>;

package/dist/persistence/flows/FlowsPersistenceSqlite.js

@@ -212,6 +212,16 @@ class FlowsPersistenceSqlite {
         const stmt = this.db.prepare('DELETE FROM tool_calls WHERE flow_id = ? AND id = ?');
         stmt.run(flowId, toolCallId);
     }
+    async setAiQuery(flowId, aiQuery) {
+        const stmt = this.db.prepare('INSERT OR REPLACE INTO ai_queries (id, flow_id, started_at, ai_query) VALUES (?, ?, ?, ?)');
+        stmt.run(aiQuery.id, flowId, aiQuery.startedAt, JSON.stringify(aiQuery));
+    }
+    async getAiQueries(flowId) {
+        await this.getFlowMetadataById(flowId);
+        const stmt = this.db.prepare('SELECT ai_query FROM ai_queries WHERE flow_id = ? ORDER BY started_at');
+        const rows = stmt.all(flowId);
+        return rows.map((row) => JSON.parse(row.ai_query));
+    }
     async setVideo(flowId, bytes) {
         // Ensure flow exists.
         await this.getFlowMetadataById(flowId);

package/dist/persistence/flows/FlowsPersistenceVolatile.d.ts

@@ -1,3 +1,4 @@
+import type { AiQuery } from '../../models/AiQuery';
 import type { BrowserStorageState } from '../../models/BrowserStorageState';
 import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
 import type { PaginatedResult } from '../../models/PaginatedResult';
@@ -14,7 +15,8 @@ export declare class FlowsPersistenceVolatile implements FlowsPersistence {
     private readonly videos;
     private readonly files;
     private readonly browserStates;
-    constructor(flows?: Map<string, FlowMetadata>, screenshots?: Map<string, Map<string, Buffer>>, toolCalls?: Map<string, ToolCall[]>, videos?: Map<string, Buffer>, files?: Map<string, Map<string, Buffer>>, browserStates?: Map<string, BrowserStorageState>);
+    private readonly aiQueries;
+    constructor(flows?: Map<string, FlowMetadata>, screenshots?: Map<string, Map<string, Buffer>>, toolCalls?: Map<string, ToolCall[]>, videos?: Map<string, Buffer>, files?: Map<string, Map<string, Buffer>>, browserStates?: Map<string, BrowserStorageState>, aiQueries?: Map<string, AiQuery[]>);
     setFlowMetadata(flowMetadata: FlowMetadata): Promise<void>;
     getFlowMetadataById(flowId: string): Promise<FlowMetadata>;
     getFlowMetadataByName(flowName: string): Promise<FlowMetadata>;
@@ -27,6 +29,8 @@ export declare class FlowsPersistenceVolatile implements FlowsPersistence {
     setToolCall(flowId: string, toolCall: ToolCall): Promise<void>;
     getToolCalls(flowId: string): Promise<ToolCall[]>;
     deleteToolCall(flowId: string, toolCallId: string): Promise<void>;
+    setAiQuery(flowId: string, aiQuery: AiQuery): Promise<void>;
+    getAiQueries(flowId: string): Promise<AiQuery[]>;
     setVideo(flowId: string, bytes: Buffer): Promise<void>;
     getVideoSegment(flowId: string, startOffset: number, length: number): Promise<VideoSegment | null>;
     getFlowFile(flowId: string, fileId: string): Promise<Buffer | null>;

package/dist/persistence/flows/FlowsPersistenceVolatile.js

@@ -7,13 +7,14 @@ const MiscUtils_1 = require("../../utils/MiscUtils");
  * A volatile (in-memory) implementation of FlowsPersistence.
  */
 class FlowsPersistenceVolatile {
-    constructor(flows = new Map(), screenshots = new Map(), toolCalls = new Map(), videos = new Map(), files = new Map(), browserStates = new Map()) {
+    constructor(flows = new Map(), screenshots = new Map(), toolCalls = new Map(), videos = new Map(), files = new Map(), browserStates = new Map(), aiQueries = new Map()) {
         this.flows = flows;
         this.screenshots = screenshots;
         this.toolCalls = toolCalls;
         this.videos = videos;
         this.files = files;
         this.browserStates = browserStates;
+        this.aiQueries = aiQueries;
     }
     async setFlowMetadata(flowMetadata) {
         this.flows.set(flowMetadata.id, { ...flowMetadata });
@@ -126,6 +127,26 @@ class FlowsPersistenceVolatile {
             this.toolCalls.delete(flowId);
         }
     }
+    async setAiQuery(flowId, aiQuery) {
+        if (!this.aiQueries.has(flowId)) {
+            this.aiQueries.set(flowId, []);
+        }
+        const queries = this.aiQueries.get(flowId);
+        const existingIndex = queries.findIndex((q) => q.id === aiQuery.id);
+        if (existingIndex >= 0) {
+            queries[existingIndex] = { ...aiQuery };
+        }
+        else {
+            queries.push({ ...aiQuery });
+        }
+    }
+    async getAiQueries(flowId) {
+        if (!this.flows.has(flowId)) {
+            throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
+        }
+        const queries = this.aiQueries.get(flowId) || [];
+        return [...queries].sort((a, b) => a.startedAt - b.startedAt);
+    }
     async setVideo(flowId, bytes) {
         this.videos.set(flowId, Buffer.from(bytes));
     }
@@ -167,6 +188,7 @@ class FlowsPersistenceVolatile {
         this.flows.delete(flowId);
         this.screenshots.delete(flowId);
         this.toolCalls.delete(flowId);
+        this.aiQueries.delete(flowId);
         this.videos.delete(flowId);
         this.files.delete(flowId);
         this.browserStates.delete(flowId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "donobu",
-  "version": "5.20.0",
+  "version": "5.21.0",
   "description": "Create browser automations with an LLM agent and replay them as Playwright scripts.",
   "main": "dist/main.js",
   "module": "dist/esm/main.js",