@iblai/iblai-js 1.26.3 → 1.26.4

package/dist/data-layer/playwright/evaluation-tab-helpers.d.ts

@@ -0,0 +1,396 @@
+import { Locator, Page } from '@playwright/test';
+/**
+ * Evals tab helpers — Playwright bindings for the `AgentEvaluationTab`
+ * component from `@iblai/web-containers` (edit-mentor modal > Evals tab),
+ * including the embedded tenant Benchmarks management dialog.
+ *
+ * All UI strings below mirror `AGENT_EVALUATION_TAB_LABELS` from
+ * `@iblai/web-containers/next` plus the `benchmarks*` i18n namespaces. If a
+ * consumer renames a label via the `labels` prop, override these constants
+ * per spec — don't edit this file.
+ *
+ * Every dialog spawned by this tab carries an explicit `aria-label` on its
+ * `DialogContent` (see `dialogName` entries). Helpers ALWAYS resolve the
+ * dialog once via `[role="dialog"][aria-label="…"]` — the attribute selector,
+ * not the role accessible name, because a visible `DialogTitle` makes Radix's
+ * `aria-labelledby` win the accessible-name computation — and then resolve
+ * every in-dialog control from that scoped locator. This matters here more
+ * than usual:
+ *   - the toolbar "New Evaluation" / "Manage benchmarks" buttons share their
+ *     text with the dialog titles they open;
+ *   - the LLM-judge dialog's title AND submit button both read "Evaluate";
+ *   - dialogs nest three deep (Manage benchmarks → benchmark items → add
+ *     Q&A), and the add-Q&A dialog has Manual / CSV Upload sub-tabs whose
+ *     footers each have their own Cancel button.
+ *
+ * The helpers assume the Edit Mentor dialog is already open. Call
+ * `switchToEvaluationTab(page)` first; from then on, every helper accepts
+ * either the `Page` or a scoping `Locator`.
+ */
+export declare const EVALS_LABELS: {
+    readonly tabName: "Evals";
+    readonly header: {
+        readonly title: "Evals";
+        readonly description: "Run this agent against a benchmark and score the responses.";
+    };
+    readonly benchmarkPicker: {
+        readonly placeholder: "Select a benchmark";
+        readonly searchPlaceholder: "Search benchmarks...";
+        readonly empty: "No benchmarks found";
+        /** `data-testid` on the combobox trigger button (its text is the selected benchmark, so it has no stable accessible name). */
+        readonly triggerTestId: "benchmark-combobox";
+        /** `data-testid` on the dropdown panel (rendered inline, not in a portal). */
+        readonly dropdownTestId: "benchmark-combobox-dropdown";
+    };
+    readonly actions: {
+        readonly newExperiment: "New Evaluation";
+        readonly manageBenchmarks: "Manage benchmarks";
+        readonly viewResults: "View results";
+        readonly evaluate: "New review";
+        readonly exportCsv: "Export CSV";
+        readonly checkStatus: "Check status";
+        readonly delete: "Delete";
+    };
+    readonly checkStatusToast: {
+        readonly pending: "Evaluation is still running";
+        readonly completed: "Evaluation completed";
+        readonly failed: "Evaluation failed";
+    };
+    readonly emptyBenchmarks: "No benchmarks yet. Use \"Manage benchmarks\" above to create one.";
+    readonly startDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "start-evaluation";
+        readonly experimentNameLabel: "Evaluation name";
+        readonly cancel: "Cancel";
+        readonly start: "Start Evaluation";
+    };
+    readonly manageBenchmarksDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "manage-benchmarks";
+        readonly searchAriaLabel: "Search benchmarks";
+        readonly newBenchmark: "New Benchmark";
+        readonly empty: "No benchmarks found";
+    };
+    readonly createBenchmarkDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "create-benchmark";
+        readonly nameLabel: "Name";
+        readonly descriptionLabel: "Description";
+        readonly cancel: "Cancel";
+        readonly create: "Create";
+    };
+    readonly benchmarkItemsDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "benchmark-items";
+        readonly addItems: "Add Q&A";
+        readonly deleteItemAriaLabel: "Delete Q&A";
+        readonly empty: "No Q&A in this benchmark yet";
+    };
+    readonly addItemsDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "add-benchmark-items";
+        readonly manualTab: "Manual";
+        readonly csvTab: "CSV Upload";
+        readonly questionLabel: "Question";
+        readonly answerLabel: "Answer";
+        readonly addRow: "Add row";
+        readonly removeRowAriaLabel: "Remove row";
+        readonly add: "Add";
+        readonly csvFileLabel: "CSV file";
+        readonly upload: "Upload";
+        readonly cancel: "Cancel";
+    };
+    readonly deleteItemDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "delete-benchmark-item";
+        readonly delete: "Delete";
+        readonly cancel: "Cancel";
+    };
+    readonly detailDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "evaluation-detail";
+        readonly refreshAriaLabel: "Refresh";
+        readonly newReview: "New review";
+        readonly waitingHint: "Waiting for the evaluation to produce traces. This page auto-refreshes.";
+        readonly noScoresHint: "No scores yet";
+        readonly scoreNameLabel: "Score name";
+        readonly scoreValueLabel: "Value";
+        readonly scoreCommentLabel: "Comment";
+        readonly addScore: "Add";
+        readonly removeScoreAriaLabel: "Remove score";
+        readonly reviewsHeading: "Reviews";
+        readonly reviewDetailsAriaLabel: "Toggle review details";
+        readonly reviewStatus: {
+            readonly pending: "Queued";
+            readonly running: "Running";
+            readonly completed: "Complete";
+            readonly failed: "Failed";
+        };
+    };
+    readonly judgeDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "llm-judge";
+        readonly criteriaLabel: "Criteria";
+        readonly scoreNameLabel: "Score name";
+        readonly maxConcurrencyLabel: "Max concurrency";
+        readonly selectorAriaLabel: "LLM Model Selector";
+        readonly cancel: "Cancel";
+        readonly submit: "Evaluate";
+    };
+    readonly llmPickerDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "evaluate-llm-picker";
+        readonly searchPlaceholder: "Search providers...";
+        /** Title of the follow-up model-selection modal (llm-tab's `LLMProviderModal`, no aria-label of its own). */
+        readonly modelDialogTitle: "LLM Selection";
+    };
+    readonly deleteDialog: {
+        /** `aria-label` on the dialog's `DialogContent` — scopes in-dialog locators. */
+        readonly dialogName: "delete-evaluation";
+        readonly cancel: "Cancel";
+        readonly confirm: "Delete";
+    };
+};
+/** Status badge text in the runs table (component upper-cases the raw status). */
+export type EvalRunStatus = 'COMPLETED' | 'FAILED' | 'PENDING' | 'IN_PROGRESS';
+/** Status pill states on a review row inside the detail dialog. */
+export type EvalReviewStatus = keyof typeof EVALS_LABELS.detailDialog.reviewStatus;
+/**
+ * Check whether the Evals tab is rendered in the Edit Mentor dialog.
+ * Returns false instead of throwing so callers can guard permission-gated
+ * tabs.
+ */
+export declare function isEvaluationTabVisible(page: Page): Promise<boolean>;
+/**
+ * Switch to the Evals tab. Assumes the Edit Mentor dialog is open. Waits
+ * for the toolbar "Manage benchmarks" button (unique to this pane) before
+ * returning, so callers can rely on the panel being interactive.
+ */
+export declare function switchToEvaluationTab(page: Page): Promise<void>;
+/** Locator for the benchmark combobox trigger. Its text is the selected benchmark name. */
+export declare function getBenchmarkCombobox(scope: Page | Locator): Locator;
+/** Locator for the combobox dropdown panel (rendered inline under the trigger, not in a portal). */
+export declare function getBenchmarkComboboxDropdown(scope: Page | Locator): Locator;
+/** Locator for the toolbar "New Evaluation" button (opens the start dialog). */
+export declare function getNewEvaluationButton(scope: Page | Locator): Locator;
+/** Locator for the toolbar "Manage benchmarks" button (opens the benchmarks dialog). */
+export declare function getManageBenchmarksButton(scope: Page | Locator): Locator;
+/** Assert the currently selected benchmark shown on the combobox trigger. */
+export declare function expectSelectedBenchmark(scope: Page | Locator, benchmarkName: string): Promise<void>;
+/**
+ * Open the benchmark combobox, optionally filter via its search input, and
+ * pick a benchmark by exact name. Waits for the dropdown to close and the
+ * trigger to reflect the selection.
+ */
+export declare function selectBenchmark(scope: Page | Locator, benchmarkName: string, opts?: {
+    search?: boolean;
+}): Promise<void>;
+/** Assert the "no benchmarks yet" empty notice is shown in the tab body. */
+export declare function expectNoBenchmarksNotice(scope: Page | Locator): Promise<void>;
+/**
+ * Locator for an evaluation run's table row, matched by an exact cell so a
+ * run whose name is a substring of another ("smoke" vs "smoke-2") can't
+ * cross-match.
+ */
+export declare function getRunRow(scope: Page | Locator, runName: string): Locator;
+export declare function expectRunInTable(scope: Page | Locator, runName: string): Promise<void>;
+export declare function expectRunNotInTable(scope: Page | Locator, runName: string): Promise<void>;
+/** Assert the status badge on a run's row (badge text is upper-cased by the UI). */
+export declare function expectRunStatus(scope: Page | Locator, runName: string, status: EvalRunStatus): Promise<void>;
+/** Assert the empty-state row shown when the selected benchmark has no runs for this agent. */
+export declare function expectRunsTableEmpty(scope: Page | Locator, benchmarkName: string): Promise<void>;
+/**
+ * Open the three-dots actions menu on a run's row. The trigger carries
+ * `aria-label="Actions for <runName>"`, so it can't collide with other rows.
+ * Returns the menu locator (Radix portals it outside the table — page scope).
+ */
+export declare function openRunActionsMenu(scope: Page | Locator, runName: string): Promise<Locator>;
+/**
+ * Row action: "View results" — opens the evaluation detail dialog.
+ * Only enabled once the run is COMPLETED. Returns the scoped detail dialog.
+ */
+export declare function openRunResults(scope: Page | Locator, runName: string): Promise<Locator>;
+/**
+ * Row action: "New review" — opens the LLM-judge dialog for the run.
+ * Only enabled once the run is COMPLETED. Returns the scoped judge dialog.
+ */
+export declare function openNewReviewForRun(scope: Page | Locator, runName: string): Promise<Locator>;
+/**
+ * Row action: "Check status" — triggers a fresh status fetch and waits for
+ * the outcome toast. Returns which toast appeared. The menu stays open by
+ * design (the item prevents default), so we close it with Escape.
+ */
+export declare function checkRunStatus(scope: Page | Locator, runName: string): Promise<'pending' | 'completed' | 'failed'>;
+/**
+ * Row action: "Export CSV" — waits for the browser download and returns its
+ * suggested filename (`<benchmark>_<run>_results.csv`).
+ */
+export declare function exportRunCsv(scope: Page | Locator, runName: string): Promise<string>;
+/** Locator for the "New Evaluation" dialog. Scoped by aria-label — its title text collides with the toolbar button that opens it. */
+export declare function getStartEvaluationDialog(scope: Page | Locator): Locator;
+/** Open the start dialog via the toolbar button and wait for it to be interactive. */
+export declare function openStartEvaluationDialog(scope: Page | Locator): Promise<Locator>;
+/**
+ * End-to-end: open the "New Evaluation" dialog, optionally name the run,
+ * submit, and wait for the dialog to close. When `name` is omitted the
+ * backend auto-generates one.
+ */
+export declare function startEvaluation(scope: Page | Locator, opts?: {
+    name?: string;
+}): Promise<void>;
+/** Cancel out of the start dialog and wait for it to close. */
+export declare function cancelStartEvaluation(scope: Page | Locator): Promise<void>;
+/** Locator for the "Delete Evaluation" confirm dialog. */
+export declare function getDeleteEvaluationDialog(scope: Page | Locator): Locator;
+/**
+ * Delete a run from its row actions menu and confirm in the follow-up
+ * dialog. The confirm button ("Delete") shares its name with the menu item
+ * that opened the dialog, so the confirm click MUST stay dialog-scoped.
+ */
+export declare function deleteEvaluation(scope: Page | Locator, runName: string): Promise<void>;
+/** Open the delete confirm from the row menu, then cancel out of it. */
+export declare function cancelDeleteEvaluation(scope: Page | Locator, runName: string): Promise<void>;
+/**
+ * Locator for the LLM-judge dialog. Scoped by aria-label — its title AND its
+ * submit button both read "Evaluate", and it can be stacked on top of the
+ * evaluation-detail dialog.
+ */
+export declare function getLlmJudgeDialog(scope: Page | Locator): Locator;
+/** Locator for the judge dialog's LLM provider-picker dialog (stacked above it). */
+export declare function getLlmPickerDialog(scope: Page | Locator): Locator;
+/**
+ * Inside the judge dialog: pick the judging LLM. Opens the provider picker,
+ * optionally filters, clicks the provider card, then picks the model in the
+ * follow-up "LLM Selection" modal (llm-tab's `LLMProviderModal`; matched by
+ * title text since it has no aria-label). Both pickers close on selection.
+ */
+export declare function selectJudgeLlm(scope: Page | Locator, providerName: string, modelName: string): Promise<void>;
+/**
+ * Fill and submit the LLM-judge dialog (assumes it is already open — see
+ * `openNewReviewForRun` / `openNewReviewFromDetail`). Waits for the dialog
+ * to close on success.
+ */
+export declare function submitLlmJudge(scope: Page | Locator, opts: {
+    criteria: string;
+    scoreName: string;
+    provider: string;
+    model: string;
+    maxConcurrency?: number;
+}): Promise<void>;
+/** Cancel out of the judge dialog and wait for it to close. */
+export declare function cancelLlmJudge(scope: Page | Locator): Promise<void>;
+/** Locator for the evaluation-detail dialog. Its title is the run name (dynamic), hence the aria-label hook. */
+export declare function getEvaluationDetailDialog(scope: Page | Locator): Locator;
+/** Click the detail dialog's refresh icon button (aria-label "Refresh"). */
+export declare function refreshEvaluationDetail(scope: Page | Locator): Promise<void>;
+/** Open the LLM-judge dialog from the detail dialog's "New review" CTA. Returns the judge dialog. */
+export declare function openNewReviewFromDetail(scope: Page | Locator): Promise<Locator>;
+/** Assert the detail dialog is still waiting for traces (auto-refreshing hint). */
+export declare function expectDetailWaitingForTraces(scope: Page | Locator): Promise<void>;
+/**
+ * Locator for a trace's expand/collapse row inside the detail dialog,
+ * matched by (part of) its question text.
+ */
+export declare function getTraceRow(scope: Page | Locator, questionText: string): Locator;
+/** Expand a trace by its question text so its outputs and scores are visible. */
+export declare function expandTrace(scope: Page | Locator, questionText: string): Promise<void>;
+/**
+ * Add a manual score to the currently expanded trace. `value` must be
+ * between 0 and 1 (the UI rejects anything else with an error toast).
+ */
+export declare function addManualScore(scope: Page | Locator, opts: {
+    name: string;
+    value: number;
+    comment?: string;
+}): Promise<void>;
+/** Assert a score chip with the given name is visible on the expanded trace. */
+export declare function expectTraceScore(scope: Page | Locator, scoreName: string): Promise<void>;
+/**
+ * Remove a score chip from the expanded trace. The chip's trash button
+ * carries `aria-label="Remove score"`; we scope through the chip matched by
+ * score name so multiple chips can't cross-match.
+ */
+export declare function removeTraceScore(scope: Page | Locator, scoreName: string): Promise<void>;
+/**
+ * Locator for a review row's expand toggle in the detail dialog's Reviews
+ * list. All toggles share the "Toggle review details" aria-label, so the
+ * row is disambiguated by the score name it displays.
+ */
+export declare function getReviewRow(scope: Page | Locator, scoreName: string): Locator;
+/** Assert a review with the given score name appears in the Reviews list. */
+export declare function expectReviewInList(scope: Page | Locator, scoreName: string): Promise<void>;
+/** Assert a review row's status pill (Queued / Running / Complete / Failed). */
+export declare function expectReviewStatus(scope: Page | Locator, scoreName: string, status: EvalReviewStatus): Promise<void>;
+/** Expand a review row to show its criteria and produced scores. */
+export declare function expandReview(scope: Page | Locator, scoreName: string): Promise<void>;
+/** Close the detail dialog via Escape and wait for it to be gone. */
+export declare function closeEvaluationDetailDialog(scope: Page | Locator): Promise<void>;
+/** Locator for the "Manage benchmarks" dialog. Scoped by aria-label — its title collides with the toolbar button that opens it. */
+export declare function getManageBenchmarksDialog(scope: Page | Locator): Locator;
+/** Open the Manage benchmarks dialog from the toolbar and wait for its table UI. Returns the scoped dialog. */
+export declare function openManageBenchmarksDialog(scope: Page | Locator): Promise<Locator>;
+/** Type into the benchmarks search input (aria-label "Search benchmarks") inside the manage dialog. */
+export declare function searchBenchmarks(scope: Page | Locator, query: string): Promise<void>;
+/** Assert a benchmark appears in the manage dialog's table. */
+export declare function expectBenchmarkListed(scope: Page | Locator, benchmarkName: string): Promise<void>;
+/** Locator for the "New Benchmark" dialog. Scoped by aria-label — its title collides with the button that opens it. */
+export declare function getCreateBenchmarkDialog(scope: Page | Locator): Locator;
+/**
+ * Create a benchmark from inside the manage dialog: open "New Benchmark",
+ * fill the form, submit, and wait for both the dialog to close and the new
+ * row to appear.
+ */
+export declare function createBenchmark(scope: Page | Locator, opts: {
+    name: string;
+    description?: string;
+}): Promise<void>;
+/** Locator for a benchmark's items dialog (Q&A list). Its title is the benchmark name (dynamic), hence the aria-label hook. */
+export declare function getBenchmarkItemsDialog(scope: Page | Locator): Locator;
+/**
+ * Open a benchmark's Q&A items dialog from its row's eye button
+ * (`aria-label="View items in <name>"`). Returns the scoped items dialog.
+ */
+export declare function openBenchmarkItems(scope: Page | Locator, benchmarkName: string): Promise<Locator>;
+/** Assert a Q&A item (matched by its question text) is listed in the items dialog. */
+export declare function expectQaItemListed(scope: Page | Locator, questionText: string): Promise<void>;
+/** Locator for the "Add Q&A" dialog (Manual / CSV Upload sub-tabs). */
+export declare function getAddItemsDialog(scope: Page | Locator): Locator;
+/** Open the "Add Q&A" dialog from the items dialog. Returns the scoped add dialog. */
+export declare function openAddItemsDialog(scope: Page | Locator): Promise<Locator>;
+/**
+ * Switch to a sub-tab INSIDE the Add Q&A dialog. The tab query must stay
+ * dialog-scoped: the edit-mentor modal behind it has its own `role="tab"`
+ * strip, and an unscoped `getByRole('tab')` can cross-match. Returns the
+ * active tabpanel so callers interact only with that sub-tab's controls
+ * (each panel has its own footer with its own Cancel/submit buttons).
+ */
+export declare function switchToAddItemsSubTab(scope: Page | Locator, subTab: 'manual' | 'csv'): Promise<Locator>;
+/**
+ * Add Q&A pairs via the Manual sub-tab. Fills one row per pair (clicking
+ * "Add row" as needed), submits, and waits for the dialog to close. Row
+ * inputs are matched by label WITHIN each row container so the Question /
+ * Answer labels of sibling rows can't collide.
+ */
+export declare function addQaPairsManually(scope: Page | Locator, pairs: Array<{
+    question: string;
+    answer?: string;
+}>): Promise<void>;
+/**
+ * Upload Q&A pairs via the CSV Upload sub-tab. `csvPath` is a path on the
+ * test machine; the file needs `input` (required) and `expected_output`
+ * (optional) columns. Waits for the dialog to close on success.
+ */
+export declare function uploadQaCsv(scope: Page | Locator, csvPath: string): Promise<void>;
+/** Locator for the Q&A delete-confirm dialog. */
+export declare function getDeleteQaItemDialog(scope: Page | Locator): Locator;
+/**
+ * Delete a Q&A item by its question text: click the row's trash button
+ * (`aria-label="Delete Q&A"`) and confirm. The confirm "Delete" button must
+ * stay scoped to the confirm dialog — the items dialog's rows keep their
+ * own delete buttons mounted behind it.
+ */
+export declare function deleteQaItem(scope: Page | Locator, questionText: string): Promise<void>;
+/** Close the benchmark items dialog via Escape and wait for it to be gone. */
+export declare function closeBenchmarkItemsDialog(scope: Page | Locator): Promise<void>;
+/** Close the Manage benchmarks dialog via Escape and wait for it to be gone. */
+export declare function closeManageBenchmarksDialog(scope: Page | Locator): Promise<void>;

package/dist/data-layer/playwright/index.d.ts

@@ -33,6 +33,7 @@ export { billingPlanSection, billingCreditsSection, billingAutoRechargeSection,
 export type { BillingAutoRechargeStatus } from './billing-tab-helpers';
 export { TASKS_LABELS, isTasksTabVisible, switchToTasksTab, getScheduleTaskButton, getSearchInput, getTaskRow, expectTotalTasks, expectCompletedTasks, expectFailedTasks, expectTasksEmpty, expectTaskInList, expectTaskNotInList, selectTaskInList, expectTaskStatus, searchTasks, openScheduleTaskDialog, scheduleTask, expectScheduleStartTimeInPastError, deleteTask, expectLogsForTask, expectNoLogsForSelectedTask, openFirstLogDetails, expectLogDetailsStatus, } from './tasks-tab-helpers';
 export type { TaskRepeat, TaskStatus } from './tasks-tab-helpers';
+export * from './evaluation-tab-helpers';
 export * from './lti-tab-helpers';
 export { createPlaywrightConfig, generateProjectConfig, generateBrowserSetupProjects, getBrowserKey, } from './playwright-config';
 export type { PlatformConfig, CreatePlaywrightConfigOptions } from './playwright-config';