@datalayer/core 1.1.60 → 1.2.10

package/lib/api/evals/client.js

@@ -0,0 +1,359 @@
+/*
+ * Copyright (c) 2023-2025 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/*
+ * Copyright (c) 2023-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * The evals API, one function per endpoint.
+ *
+ * Paths mirror `tests/test_evals_api_contract.py` in the AI Agents service
+ * and `evalsApiContract.spec.ts` in the landings UI. Downloads (export and
+ * report) are URLs rather than requests: the browser opens them and the
+ * service answers with an attachment.
+ *
+ * @module api/evals/client
+ */
+import { evalsRequest, evalsUrl, segment, } from './request.js';
+export const listEvalsets = (options, query = {}) => evalsRequest(options, '/evalsets', { query });
+export const getEvalset = (options, evalsetId) => evalsRequest(options, `/evalsets/${segment(evalsetId)}`);
+export const createEvalset = (options, body) => evalsRequest(options, '/evalsets', { method: 'POST', body });
+export const updateEvalset = (options, evalsetId, body) => evalsRequest(options, `/evalsets/${segment(evalsetId)}`, {
+    method: 'PATCH',
+    body,
+});
+export const setEvalsetPublic = (options, evalsetId, isPublic) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/public`, {
+    method: 'PATCH',
+    body: { is_public: isPublic },
+});
+/**
+ * Take a benchmark into the account the options name (B5-03): a private copy
+ * of its definition that says what it was taken from. A published benchmark
+ * is anybody's to take; an unpublished one, a Viewer's. `derivation` says why:
+ * `fork` to change it, `compare` to run it against one's own agent (B5-05).
+ */
+export const cloneEvalset = (options, evalsetId, body = {}) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/clone`, {
+    method: 'POST',
+    body,
+});
+/**
+ * Propose the next version of a benchmark's definition, from an investigation
+ * (B5-07): a task added or corrected, a corrected evaluator, a threshold
+ * moved. The version records the investigation that asked for it.
+ */
+export const reviseEvalset = (options, evalsetId, body) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/revisions`, {
+    method: 'POST',
+    body,
+});
+/**
+ * What publishing this benchmark would put in the library (B5-02): the
+ * contents of the package, the enumeration the publication review shows, what
+ * is left out and why, and everything that would refuse it — before anybody
+ * presses the button, because a publication cannot be taken back from whoever
+ * already read it.
+ */
+export const previewEvalsetPublication = (options, evalsetId, query = {}) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/publication/preview`, { query });
+/**
+ * Publish the package: the immutable snapshot of the definition, the data, the
+ * subjects, the environment, the results, the report, the evidence and the
+ * evaluators (section 14.3). Refused with `detail.problems` where section 14.5
+ * says it cannot be published.
+ */
+export const publishEvalsetPackage = (options, evalsetId, body = {}) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/publications`, {
+    method: 'POST',
+    body,
+});
+/** Every package published of this benchmark, withdrawn ones included. */
+export const listEvalsetPublications = (options, evalsetId, query = {}) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/publications`, { query });
+/**
+ * Take a package out of the library. What it holds stays as it was written:
+ * withdrawing a snapshot does not make it editable.
+ */
+export const withdrawEvalsetPublication = (options, evalsetId, publicationId) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/publications/${segment(publicationId)}`, { method: 'DELETE' });
+/** The package a published benchmark stands for, to anybody (B5-02). */
+export const getPublicEvalsetPublication = (options, evalsetId) => evalsRequest(options, `/public/evalsets/${segment(evalsetId)}/publication`);
+/**
+ * Launches of one benchmark side by side (B5-08): what each scored, what each
+ * subject scored in each, and which tasks went from passing to failing or
+ * back. Launch ids are compared in the order given, oldest first; the two
+ * newest launches where none is named.
+ */
+export const compareEvalsetLaunches = (options, evalsetId, query = {}) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/comparison`, { query });
+/**
+ * The subjects that ran a published benchmark, best first. Only runs a package
+ * published are counted (B5-02), and each row names the package its best run
+ * came from.
+ */
+export const getPublicEvalsetLeaderboard = (options, evalsetId) => evalsRequest(options, `/public/evalsets/${segment(evalsetId)}/leaderboard`);
+/**
+ * What an agent or a model has actually run (B5-12): the benchmarks that ran
+ * it, what its runs scored, what they cost, how long they took, and the
+ * launches still going. Empty where it has never been run.
+ *
+ * The ref keeps its slashes — an agentspec id has one — so each segment is
+ * encoded rather than the whole string.
+ */
+export const getSubjectUsage = (options, subjectRef) => evalsRequest(options, `/subjects/${subjectRef.split('/').map(encodeURIComponent).join('/')}/usage`);
+/**
+ * Whether anything will run the account's benchmarks (B5-12, section 15.2):
+ * the executor, whether a worker polls the benchmark queue, and the account's
+ * own queued and running launches.
+ */
+export const getBenchmarkCompute = (options) => evalsRequest(options, '/compute/operations');
+/**
+ * What benchmark work a sandbox is doing (B5-12, section 15.4).
+ *
+ * The ref is a pool slot's name (`benchmark-<run>-slot-<n>`) or the snapshot
+ * a sandbox was restored from. A sandbox of nobody's benchmark answers
+ * `found: false`, so a page asks without knowing in advance.
+ */
+export const getSandboxProvenance = (options, sandboxRef) => evalsRequest(options, `/sandboxes/${sandboxRef.split('/').map(encodeURIComponent).join('/')}/provenance`);
+/**
+ * Find things across the caller's benchmarks (B5-11, section 19): benchmarks,
+ * launches, tasks, reports and investigations, each row keeping its kind and
+ * what it belongs to.
+ *
+ * Nobody types a field name, so the service reads the words first: a status
+ * word becomes a status, `run 128` becomes a launch number, and what is left
+ * is the text. An empty query finds nothing rather than everything.
+ */
+export const searchEvals = (options, query = {}) => evalsRequest(options, '/search', { query });
+export const renameEvalset = (options, evalsetId, name) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/rename`, {
+    method: 'PATCH',
+    body: { name },
+});
+export const deleteEvalset = (options, evalsetId) => evalsRequest(options, `/evalsets/${segment(evalsetId)}`, {
+    method: 'DELETE',
+});
+// --- Cases ------------------------------------------------------------------
+export const listCases = (options, evalsetId) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/cases`);
+export const createCase = (options, evalsetId, body) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/cases`, {
+    method: 'POST',
+    body,
+});
+export const updateCase = (options, evalsetId, caseId, body) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/cases/${segment(caseId)}`, {
+    method: 'PATCH',
+    body,
+});
+export const deleteCase = (options, evalsetId, caseId) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/cases/${segment(caseId)}`, {
+    method: 'DELETE',
+});
+/** The URL the browser opens to download an evalset definition. */
+export const evalsetExportUrl = (options, evalsetId, format = 'json') => evalsUrl(options, `/evalsets/${segment(evalsetId)}/export`, { format });
+/** The URL the browser opens to download a generated report. */
+export const evalsetReportUrl = (options, evalsetId, format = 'markdown', runLimit) => evalsUrl(options, `/evalsets/${segment(evalsetId)}/report`, {
+    format,
+    run_limit: runLimit,
+});
+// --- Public reads -----------------------------------------------------------
+export const getPublicEvalset = (options, evalsetId) => evalsRequest(options, `/public/evalsets/${segment(evalsetId)}`);
+export const getPublicEvalsetDetails = (options, evalsetId, runsLimit) => evalsRequest(options, `/public/evalsets/${segment(evalsetId)}/details`, {
+    query: { runs_limit: runsLimit },
+});
+export const listPublicRuns = (options, evalsetId, experimentId, query = {}) => evalsRequest(options, `/public/evalsets/${segment(evalsetId)}/experiments/${segment(experimentId)}/runs`, { query });
+export const createExperiment = (options, body) => evalsRequest(options, '/experiments', {
+    method: 'POST',
+    body,
+});
+export const listExperiments = (options, query = {}) => evalsRequest(options, '/experiments', { query });
+export const getExperiment = (options, experimentId) => evalsRequest(options, `/experiments/${segment(experimentId)}`);
+export const updateExperiment = (options, experimentId, body) => evalsRequest(options, `/experiments/${segment(experimentId)}`, {
+    method: 'PATCH',
+    body,
+});
+export const deleteExperiment = (options, experimentId) => evalsRequest(options, `/experiments/${segment(experimentId)}`, {
+    method: 'DELETE',
+});
+// --- Runs -------------------------------------------------------------------
+export const createRun = (options, experimentId, body) => evalsRequest(options, `/experiments/${segment(experimentId)}/runs`, {
+    method: 'POST',
+    body,
+});
+export const listRuns = (options, experimentId, query = {}) => evalsRequest(options, `/experiments/${segment(experimentId)}/runs`, { query });
+export const getRun = (options, runId) => evalsRequest(options, `/runs/${segment(runId)}`);
+export const deleteRun = (options, runId) => evalsRequest(options, `/runs/${segment(runId)}`, {
+    method: 'DELETE',
+});
+export const compareRuns = (options, runIds) => evalsRequest(options, '/runs/compare', {
+    method: 'POST',
+    body: { run_ids: runIds },
+});
+export const createLiveEvent = (options, body) => evalsRequest(options, '/live/events', {
+    method: 'POST',
+    body,
+});
+export const listLiveTargets = (options, query = {}) => evalsRequest(options, '/live/targets', { query });
+export const listLiveEvents = (options, query) => evalsRequest(options, '/live/events', { query });
+/** The alerts the rolling windows raised: failure spikes and drifts (B2-13). */
+export const listLiveAlerts = (options, query = {}) => evalsRequest(options, '/live/alerts', { query });
+export const deleteLiveTarget = (options, targetId, targetType = 'agent') => evalsRequest(options, '/live/targets', {
+    method: 'DELETE',
+    query: { target_id: targetId, target_type: targetType },
+});
+export const listRunsAcross = (options, query = {}) => evalsRequest(options, '/runs', { query });
+export const cancelRun = (options, runId) => evalsRequest(options, `/runs/${segment(runId)}/cancel`, {
+    method: 'POST',
+});
+/** The task grid of a run, filtered and paged by the service. */
+export const listCaseResults = (options, runId, query = {}) => evalsRequest(options, `/runs/${segment(runId)}/cases`, { query });
+export const getCaseResult = (options, runId, caseId) => evalsRequest(options, `/runs/${segment(runId)}/cases/${segment(caseId)}`);
+/**
+ * The run as the network of agents that worked on it. `budget` is how many
+ * tasks' messages to answer: every task that did not pass, then passed ones
+ * while there is room.
+ */
+export const getRunNetwork = (options, runId, query = {}) => evalsRequest(options, `/runs/${segment(runId)}/network`, {
+    query,
+});
+/**
+ * One message of a run's network, in full. The id is the service's own and
+ * holds colons, which the route reads as a path: each segment is encoded, the
+ * separators are kept.
+ */
+export const getRunNetworkMessage = (options, runId, messageId) => evalsRequest(options, `/runs/${segment(runId)}/network/messages/${messageId.split('/').map(segment).join('/')}`);
+/** A launch's runs, each as one line of its own network. */
+export const getLaunchNetwork = (options, launchId) => evalsRequest(options, `/launches/${segment(launchId)}/network`);
+export const reviewCaseResult = (options, runId, caseId, body) => evalsRequest(options, `/runs/${segment(runId)}/cases/${segment(caseId)}/review`, {
+    method: 'PATCH',
+    body,
+});
+/**
+ * Give an anonymous trial's work to the person who just signed in (B2-14):
+ * the benchmark, the launch, its runs and everything under them, moved
+ * rather than copied so a link the visitor kept still opens.
+ *
+ * Takes two credentials: the person's, in the client's options, and the
+ * trial's own key, which travels in its header.
+ */
+export const claimTrial = (options, trialUid, trialToken) => evalsRequest(options, '/trials/claim', {
+    method: 'POST',
+    body: { trial_uid: trialUid },
+    headers: { 'X-Datalayer-Trial-Token': trialToken },
+});
+/** The subjects an experiment can have, and the models offered (B2-11). */
+export const listSubjects = (options) => evalsRequest(options, '/subjects');
+/**
+ * Count one of those.
+ *
+ * Nothing is stored, and the answer says only that it was counted. Never worth
+ * failing a page for: a measure that breaks what somebody was doing is worse
+ * than a measure nobody has.
+ */
+export const recordProductEvent = (options, event) => evalsRequest(options, '/measures', {
+    method: 'POST',
+    body: { event },
+});
+/**
+ * The plan of a launch before it is made (B2-07): estimated duration and
+ * cost, the compute, and the problems that would stop it. Same body as the
+ * launch; nothing is created.
+ */
+export const validateLaunch = (options, evalsetId, body) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/launches/validate`, {
+    method: 'POST',
+    body,
+});
+/** One submission of a benchmark across its experiments. */
+export const createLaunch = (options, evalsetId, body) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/launches`, {
+    method: 'POST',
+    body,
+});
+export const listLaunches = (options, query = {}) => evalsRequest(options, '/launches', { query });
+export const getLaunch = (options, launchId) => evalsRequest(options, `/launches/${segment(launchId)}`);
+export const cancelLaunch = (options, launchId) => evalsRequest(options, `/launches/${segment(launchId)}/cancel`, { method: 'POST' });
+export const archiveLaunch = (options, launchId) => evalsRequest(options, `/launches/${segment(launchId)}/archive`, { method: 'POST' });
+// --- Spec import and definition versions (B2-08, B2-09) --------------------
+/** An evalset from a spec file, the same body the CLI and the action send. */
+export const importEvalset = (options, body) => evalsRequest(options, '/evalsets/import', {
+    method: 'POST',
+    body,
+});
+export const listEvalsetVersions = (options, evalsetId, query = {}) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/versions`, { query });
+export const getEvalsetVersion = (options, evalsetId, version) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/versions/${segment(String(version))}`);
+// --- Reports of a launch and a run, and the report document (B3-03) --------
+/** The benchmark's whole report (every launch) as a serialized Lexical editor state. */
+export const getEvalsetReportDocument = (options, evalsetId, query = {}) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/report`, { query: { ...query, format: 'lexical' } });
+/** The launch's report as a serialized Lexical editor state. */
+export const getLaunchReportDocument = (options, launchId, query = {}) => evalsRequest(options, `/launches/${segment(launchId)}/report`, { query: { ...query, format: 'lexical' } });
+/** Where a launch's Markdown or CSV report is downloaded from. */
+export const launchReportUrl = (options, launchId, format) => evalsUrl(options, `/launches/${segment(launchId)}/report`, { format });
+/** One run's report as a serialized Lexical editor state, with its task documents. */
+export const getRunReportDocument = (options, runId) => evalsRequest(options, `/runs/${segment(runId)}/report`, { query: { format: 'lexical' } });
+export const runReportUrl = (options, runId, format) => evalsUrl(options, `/runs/${segment(runId)}/report`, { format });
+/** Write the launch's report into the account's benchmarks space. */
+export const writeLaunchReportDocument = (options, launchId, body = {}) => evalsRequest(options, `/launches/${segment(launchId)}/report-document`, { method: 'POST', body });
+// --- Investigations and the task sandbox (B3-02, B3-05) --------------------
+/** Open the investigation of one task, or get the one already open. */
+export const openTaskInvestigation = (options, runId, caseId) => evalsRequest(options, `/runs/${segment(runId)}/tasks/${segment(caseId)}/investigation`, { method: 'POST' });
+export const openLaunchInvestigation = (options, launchId) => evalsRequest(options, `/launches/${segment(launchId)}/investigation`, { method: 'POST' });
+export const listInvestigations = (options, query = {}) => evalsRequest(options, '/investigations', {
+    query,
+});
+export const getInvestigation = (options, investigationId) => evalsRequest(options, `/investigations/${segment(investigationId)}`);
+export const updateInvestigation = (options, investigationId, body) => evalsRequest(options, `/investigations/${segment(investigationId)}`, { method: 'PATCH', body });
+/**
+ * Publish an investigation to the library, or take it back: its owner's alone.
+ * The document and the notebooks it is written in go public and private with
+ * it.
+ */
+export const setInvestigationPublic = (options, investigationId, isPublic) => evalsRequest(options, `/investigations/${segment(investigationId)}/public`, { method: 'PATCH', body: { is_public: isPublic } });
+/**
+ * Keep the investigation's page as it is shown, under a name everybody on
+ * the investigation sees; a view of that name is replaced (B4-11).
+ */
+export const saveInvestigationView = (options, investigationId, body) => evalsRequest(options, `/investigations/${segment(investigationId)}/views`, { method: 'POST', body });
+/** Forget a saved view of the investigation's page. */
+export const forgetInvestigationView = (options, investigationId, name) => evalsRequest(options, `/investigations/${segment(investigationId)}/views`, { method: 'DELETE', query: { name } });
+/** Bring the task's sandbox back from its snapshot, bound to its investigation. */
+export const resumeTaskSandbox = (options, runId, caseId, body = {}) => evalsRequest(options, `/runs/${segment(runId)}/tasks/${segment(caseId)}/sandbox/resume`, { method: 'POST', body });
+// --- Sharing and permissions (B4-05) ----------------------------------------
+/** Who a benchmark, one of its runs or an investigation is shared with; its owner's to read. */
+export const getEvalsSharing = (options, record, uid) => evalsRequest(options, `/${record}/${segment(uid)}/sharing`);
+/** Replace the grants at the levels named; the others are kept. Its owner's to do. */
+export const updateEvalsSharing = (options, record, uid, access) => evalsRequest(options, `/${record}/${segment(uid)}/sharing`, { method: 'PUT', body: { access } });
+/** The caller's role on the record, and what it allows. */
+export const getEvalsPermissions = (options, record, uid) => evalsRequest(options, `/${record}/${segment(uid)}/permissions`);
+// --- Reports (B4-04) ----------------------------------------------------------
+/** A report over runs of a benchmark, written as a draft. */
+export const createReport = (options, body) => evalsRequest(options, '/reports', { method: 'POST', body });
+export const listReports = (options, query = {}) => evalsRequest(options, '/reports', { query });
+export const getReport = (options, reportId) => evalsRequest(options, `/reports/${segment(reportId)}`);
+/** Send a report for review, send it back, approve it or publish it. */
+export const moveReport = (options, reportId, body) => evalsRequest(options, `/reports/${segment(reportId)}/state`, {
+    method: 'POST',
+    body,
+});
+/** A new version over newer runs, or the same ones; the old report is superseded by it. */
+export const regenerateReport = (options, reportId, launchIds) => evalsRequest(options, `/reports/${segment(reportId)}/regenerate`, {
+    method: 'POST',
+    body: launchIds?.length ? { launch_ids: launchIds } : {},
+});
+// --- Review decisions (B4-03) -------------------------------------------------
+/** What a reviewer decided about part of a result, recorded on a report or in an investigation. */
+export const recordDecision = (options, subject, uid, body) => evalsRequest(options, `/${subject}/${segment(uid)}/decisions`, { method: 'POST', body });
+/** The decisions made on a report or in an investigation, in the order they were made. */
+export const listDecisions = (options, subject, uid) => evalsRequest(options, `/${subject}/${segment(uid)}/decisions`);
+/**
+ * What was decided about a benchmark, or about one of its launches (B6-04).
+ *
+ * `listDecisions` answers what was decided *in* one report or investigation;
+ * this answers what was decided about the benchmark, which is what something
+ * outside the product has to ask — CI knows the benchmark it ran and nothing
+ * else. Naming neither answers nothing rather than everything.
+ */
+export const listDecisionsAbout = (options, query = {}) => evalsRequest(options, '/decisions', { query });
+/** Where a report is downloaded as it was written, with its decisions (B4-08). */
+export const reportExportUrl = (options, reportId, format = 'markdown') => evalsUrl(options, `/reports/${segment(reportId)}/export`, { format });
+// --- Shared with the caller (B4-07) -------------------------------------------
+/** The reports and investigations other people shared with the caller. */
+export const listSharedWithMe = (options) => evalsRequest(options, '/shared');
+// --- CI reports imported as snapshots (B4-09) ---------------------------------
+/** Keep a report file CI produced on its benchmark, as its text. */
+export const importReport = (options, evalsetId, body) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/reports/import`, { method: 'POST', body });
+/** The reports imported on a benchmark, newest first, without their text. */
+export const listReportImports = (options, evalsetId) => evalsRequest(options, `/evalsets/${segment(evalsetId)}/reports/imports`);
+/** One imported report, with its text as it was. */
+export const getReportImport = (options, importId) => evalsRequest(options, `/report-imports/${segment(importId)}`);
+/** Continue investigation: a live report and an investigation over its runs. */
+export const continueReportImport = (options, importId) => evalsRequest(options, `/report-imports/${segment(importId)}/continue`, { method: 'POST' });

package/lib/api/evals/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * The evals API of the AI Agents service: evalsets (benchmarks), their
+ * cases, experiments, runs, comparisons and live targets.
+ *
+ * @module api/evals
+ */
+export * from './types';
+export * from './request';
+export * from './client';
+export * from './status';

package/lib/api/evals/index.js

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2023-2025 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/*
+ * Copyright (c) 2023-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * The evals API of the AI Agents service: evalsets (benchmarks), their
+ * cases, experiments, runs, comparisons and live targets.
+ *
+ * @module api/evals
+ */
+export * from './types.js';
+export * from './request.js';
+export * from './client.js';
+export * from './status.js';

package/lib/api/evals/request.d.ts

@@ -0,0 +1,43 @@
+/** What every evals call needs to know. */
+export interface EvalsClientOptions {
+    /** The AI Agents service origin, e.g. `https://prod1.datalayer.run`. */
+    baseUrl?: string;
+    /** The caller's IAM token; a public read may go without one. */
+    token?: string;
+    /** An organization or team uid to act for; the caller's own when absent. */
+    accountUid?: string;
+}
+export type EvalsQuery = Record<string, string | number | boolean | undefined | null>;
+export type EvalsMethod = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE';
+/** The AI Agents origin, with no trailing slash. */
+export declare const aiAgentsOrigin: (baseUrl?: string) => string;
+/**
+ * A full URL under `/api/ai-agents/v1`, with the account scope and any
+ * query merged in. Empty, null and undefined values are left out.
+ */
+export declare const aiAgentsUrl: (options: EvalsClientOptions, path: string, query?: EvalsQuery) => string;
+/** A URL under `/api/ai-agents/v1/evals`. */
+export declare const evalsUrl: (options: EvalsClientOptions, suffix: string, query?: EvalsQuery) => string;
+export interface AiAgentsRequestInit {
+    method?: EvalsMethod;
+    body?: unknown;
+    query?: EvalsQuery;
+    signal?: AbortSignal;
+    /**
+     * Headers beyond the caller's own credential. One call needs this: a
+     * trial's claim carries the trial's key beside the person's (B2-14).
+     */
+    headers?: Record<string, string>;
+}
+/**
+ * One request to the AI Agents service, `path` relative to `/api/ai-agents/v1`.
+ *
+ * The typed functions in `client.ts` are built on this; a view that needs a
+ * path the client does not name yet may call it directly, and the contract
+ * spec still sees the path.
+ */
+export declare const aiAgentsRequest: <T>(options: EvalsClientOptions, path: string, init?: AiAgentsRequestInit) => Promise<T>;
+/** One request under `/evals`. */
+export declare const evalsRequest: <T>(options: EvalsClientOptions, suffix: string, init?: AiAgentsRequestInit) => Promise<T>;
+/** A path segment, encoded once. */
+export declare const segment: (value: string) => string;

package/lib/api/evals/request.js

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2023-2025 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/*
+ * Copyright (c) 2023-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * How the evals API is reached.
+ *
+ * One place builds the AI Agents origin, the `/api/ai-agents/v1` prefix and
+ * the account scope. The landings UI used to assemble those in three views;
+ * a service origin that moves then broke three places, silently. Here it
+ * is one, and the contract tests on both sides read this module's paths.
+ *
+ * Account scope: the service answers for the caller unless `account_uid`
+ * names an organization or a team the caller belongs to, in which case the
+ * evalsets are that account's. The option rides on every request.
+ *
+ * @module api/evals/request
+ */
+import { requestDatalayerAPI } from '../DatalayerApi.js';
+import { API_BASE_PATHS, DEFAULT_SERVICE_URLS } from '../constants.js';
+/** The AI Agents origin, with no trailing slash. */
+export const aiAgentsOrigin = (baseUrl) => String(baseUrl || DEFAULT_SERVICE_URLS.AI_AGENTS)
+    .trim()
+    .replace(/\/+$/, '');
+/**
+ * A full URL under `/api/ai-agents/v1`, with the account scope and any
+ * query merged in. Empty, null and undefined values are left out.
+ */
+export const aiAgentsUrl = (options, path, query = {}) => {
+    const [pathname, inlineSearch = ''] = path.split('?');
+    const parameters = new URLSearchParams(inlineSearch);
+    for (const [key, value] of Object.entries(query)) {
+        if (value !== undefined && value !== null && value !== '') {
+            parameters.set(key, String(value));
+        }
+    }
+    if (options.accountUid && !parameters.has('account_uid')) {
+        parameters.set('account_uid', options.accountUid);
+    }
+    const search = parameters.toString();
+    return `${aiAgentsOrigin(options.baseUrl)}${API_BASE_PATHS.AI_AGENTS}${pathname}${search ? `?${search}` : ''}`;
+};
+/** A URL under `/api/ai-agents/v1/evals`. */
+export const evalsUrl = (options, suffix, query = {}) => aiAgentsUrl(options, `/evals${suffix}`, query);
+/**
+ * One request to the AI Agents service, `path` relative to `/api/ai-agents/v1`.
+ *
+ * The typed functions in `client.ts` are built on this; a view that needs a
+ * path the client does not name yet may call it directly, and the contract
+ * spec still sees the path.
+ */
+export const aiAgentsRequest = (options, path, init = {}) => requestDatalayerAPI({
+    url: aiAgentsUrl(options, path, init.query),
+    method: init.method ?? 'GET',
+    body: init.body,
+    token: options.token,
+    signal: init.signal,
+    ...(init.headers ? { headers: init.headers } : {}),
+});
+/** One request under `/evals`. */
+export const evalsRequest = (options, suffix, init = {}) => aiAgentsRequest(options, `/evals${suffix}`, init);
+/** A path segment, encoded once. */
+export const segment = (value) => encodeURIComponent(String(value ?? '').trim());

package/lib/api/evals/status.d.ts

@@ -0,0 +1,27 @@
+/**
+ * The one vocabulary for what a run and a case are doing.
+ *
+ * The service stores `status` as free text and its writers disagreed over
+ * time (`queued`, `init`, `pending`, `running`, `completed`, `failed`,
+ * `error`, `success`, `done`, …). This is the same mapping as
+ * `agent_runtimes.evals.status` in Python: every word a writer has used maps
+ * onto the product vocabulary of BENCHMARK.md section 11.4, and "finished" is
+ * answered once. A word nobody has used maps to itself, lower-cased, so it is
+ * visible rather than misread.
+ *
+ * @module api/evals/status
+ */
+export declare const RUN_STATUSES: readonly ["queued", "provisioning", "running", "scoring", "completed", "failed", "review", "blocked", "cancelled"];
+export type RunStatus = (typeof RUN_STATUSES)[number];
+export declare const CASE_STATUSES: readonly ["waiting", "provisioning", "running", "scoring", "passed", "failed", "review", "blocked", "cancelled"];
+export type CaseStatus = (typeof CASE_STATUSES)[number];
+export declare const TERMINAL_RUN_STATUSES: ReadonlySet<string>;
+export declare const TERMINAL_CASE_STATUSES: ReadonlySet<string>;
+/** The section 11.4 word for a run status, whatever a writer wrote. */
+export declare const normalizeRunStatus: (value: unknown) => RunStatus | string;
+/** The section 11.4 word for a case status. */
+export declare const normalizeCaseStatus: (value: unknown) => CaseStatus | string;
+/** Whether a run with this status is over. */
+export declare const isTerminalRunStatus: (value: unknown) => boolean;
+/** Whether a case with this status is over. */
+export declare const isTerminalCaseStatus: (value: unknown) => boolean;

package/lib/api/evals/status.js

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2023-2025 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/*
+ * Copyright (c) 2023-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * The one vocabulary for what a run and a case are doing.
+ *
+ * The service stores `status` as free text and its writers disagreed over
+ * time (`queued`, `init`, `pending`, `running`, `completed`, `failed`,
+ * `error`, `success`, `done`, …). This is the same mapping as
+ * `agent_runtimes.evals.status` in Python: every word a writer has used maps
+ * onto the product vocabulary of BENCHMARK.md section 11.4, and "finished" is
+ * answered once. A word nobody has used maps to itself, lower-cased, so it is
+ * visible rather than misread.
+ *
+ * @module api/evals/status
+ */
+export const RUN_STATUSES = [
+    'queued',
+    'provisioning',
+    'running',
+    'scoring',
+    'completed',
+    'failed',
+    'review',
+    'blocked',
+    'cancelled',
+];
+export const CASE_STATUSES = [
+    'waiting',
+    'provisioning',
+    'running',
+    'scoring',
+    'passed',
+    'failed',
+    'review',
+    'blocked',
+    'cancelled',
+];
+export const TERMINAL_RUN_STATUSES = new Set([
+    'completed',
+    'failed',
+    'cancelled',
+]);
+export const TERMINAL_CASE_STATUSES = new Set([
+    'passed',
+    'failed',
+    'cancelled',
+]);
+const LEGACY_RUN_STATUS = {
+    '': 'queued',
+    init: 'queued',
+    pending: 'queued',
+    queued: 'queued',
+    provisioning: 'provisioning',
+    starting: 'provisioning',
+    running: 'running',
+    in_progress: 'running',
+    scoring: 'scoring',
+    evaluating: 'scoring',
+    completed: 'completed',
+    complete: 'completed',
+    success: 'completed',
+    succeeded: 'completed',
+    passed: 'completed',
+    done: 'completed',
+    finished: 'completed',
+    failed: 'failed',
+    failure: 'failed',
+    error: 'failed',
+    errored: 'failed',
+    review: 'review',
+    needs_review: 'review',
+    blocked: 'blocked',
+    cancelled: 'cancelled',
+    canceled: 'cancelled',
+    aborted: 'cancelled',
+};
+const LEGACY_CASE_STATUS = {
+    '': 'waiting',
+    waiting: 'waiting',
+    queued: 'waiting',
+    pending: 'waiting',
+    provisioning: 'provisioning',
+    running: 'running',
+    scoring: 'scoring',
+    passed: 'passed',
+    pass: 'passed',
+    success: 'passed',
+    completed: 'passed',
+    failed: 'failed',
+    fail: 'failed',
+    error: 'failed',
+    review: 'review',
+    needs_review: 'review',
+    blocked: 'blocked',
+    cancelled: 'cancelled',
+    canceled: 'cancelled',
+    skipped: 'cancelled',
+};
+const word = (value) => String(value ?? '')
+    .trim()
+    .toLowerCase()
+    .replace(/[-\s]/g, '_');
+/** The section 11.4 word for a run status, whatever a writer wrote. */
+export const normalizeRunStatus = (value) => {
+    const key = word(value);
+    return LEGACY_RUN_STATUS[key] ?? key;
+};
+/** The section 11.4 word for a case status. */
+export const normalizeCaseStatus = (value) => {
+    const key = word(value);
+    return LEGACY_CASE_STATUS[key] ?? key;
+};
+/** Whether a run with this status is over. */
+export const isTerminalRunStatus = (value) => TERMINAL_RUN_STATUSES.has(normalizeRunStatus(value));
+/** Whether a case with this status is over. */
+export const isTerminalCaseStatus = (value) => TERMINAL_CASE_STATUSES.has(normalizeCaseStatus(value));