ashr-labs 0.1.0

package/dist/client.d.ts

@@ -0,0 +1,31 @@
+export declare class AshrLabsClient {
+    readonly apiKey: string;
+    readonly baseUrl: string;
+    readonly timeout: number;
+    private _tenantId;
+    private _userId;
+    private _session;
+    constructor(apiKey: string, baseUrl?: string, timeout?: number);
+    static fromEnv(timeout?: number): AshrLabsClient;
+    private _ensureSession;
+    private _resolveTenantId;
+    private _resolveUserId;
+    private _makeRequest;
+    private _raiseForStatus;
+    getDataset(datasetId: number, includeSignedUrls?: boolean, urlExpiresSeconds?: number): Promise<Record<string, unknown>>;
+    listDatasets(tenantId?: number | null, limit?: number, offset?: number, includeSignedUrls?: boolean, urlExpiresSeconds?: number): Promise<Record<string, unknown>>;
+    createRun(datasetId: number, result: Record<string, unknown>, tenantId?: number | null, runnerId?: number | null): Promise<Record<string, unknown>>;
+    deleteRun(runId: number): Promise<Record<string, unknown>>;
+    getRun(runId: number): Promise<Record<string, unknown>>;
+    listRuns(datasetId?: number | null, tenantId?: number | null, limit?: number, offset?: number): Promise<Record<string, unknown>>;
+    createRequest(requestName: string, request: Record<string, unknown>, requestInputSchema?: Record<string, unknown> | null, tenantId?: number | null, requestorId?: number | null): Promise<Record<string, unknown>>;
+    getRequest(requestId: number): Promise<Record<string, unknown>>;
+    listRequests(tenantId?: number | null, status?: string | null, limit?: number, offset?: number): Promise<Record<string, unknown>>;
+    listApiKeys(includeInactive?: boolean): Promise<Record<string, unknown>[]>;
+    revokeApiKey(apiKeyId: number): Promise<Record<string, unknown>>;
+    init(): Promise<Record<string, unknown>>;
+    healthCheck(): Promise<Record<string, unknown>>;
+    waitForRequest(requestId: number, timeout?: number, pollInterval?: number): Promise<Record<string, unknown>>;
+    generateDataset(requestName: string, config: Record<string, unknown>, requestInputSchema?: Record<string, unknown> | null, timeout?: number, pollInterval?: number): Promise<[number, Record<string, unknown>]>;
+    toString(): string;
+}

package/dist/client.js

@@ -0,0 +1,251 @@
+import { AshrLabsError, AuthenticationError, AuthorizationError, NotFoundError, ValidationError, RateLimitError, ServerError, } from "./exceptions.js";
+const DEFAULT_BASE_URL = "https://api.ashr.io/testing-platform-api";
+export class AshrLabsClient {
+    apiKey;
+    baseUrl;
+    timeout;
+    _tenantId = null;
+    _userId = null;
+    _session = null;
+    constructor(apiKey, baseUrl = DEFAULT_BASE_URL, timeout = 30) {
+        if (!apiKey || !apiKey.startsWith("tp_")) {
+            throw new Error("Invalid API key format. API keys must start with 'tp_'");
+        }
+        this.apiKey = apiKey;
+        this.baseUrl = baseUrl.replace(/\/+$/, "");
+        this.timeout = timeout;
+    }
+    static fromEnv(timeout = 30) {
+        const apiKey = process.env.ASHR_LABS_API_KEY;
+        if (!apiKey) {
+            throw new Error("ASHR_LABS_API_KEY environment variable is not set. " +
+                "Create an API key at https://app.ashr.io → API Keys.");
+        }
+        const baseUrl = process.env.ASHR_LABS_BASE_URL ?? DEFAULT_BASE_URL;
+        return new AshrLabsClient(apiKey, baseUrl, timeout);
+    }
+    async _ensureSession() {
+        if (this._session !== null)
+            return;
+        this._session = await this._makeRequest("init");
+        const tenant = this._session.tenant;
+        const user = this._session.user;
+        this._tenantId = tenant.id;
+        this._userId = user.id;
+    }
+    async _resolveTenantId(tenantId) {
+        if (tenantId != null)
+            return tenantId;
+        await this._ensureSession();
+        return this._tenantId;
+    }
+    async _resolveUserId(userId) {
+        if (userId != null)
+            return userId;
+        await this._ensureSession();
+        return this._userId;
+    }
+    async _makeRequest(func, params = {}) {
+        const payload = { function: func, ...params };
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), this.timeout * 1000);
+        let response;
+        try {
+            response = await fetch(this.baseUrl, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify(payload),
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            clearTimeout(timeoutId);
+            if (err instanceof DOMException && err.name === "AbortError") {
+                throw new AshrLabsError("Request timed out");
+            }
+            throw new AshrLabsError(`Network error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+        let responseData;
+        try {
+            responseData = (await response.json());
+        }
+        catch {
+            throw new AshrLabsError(`Invalid JSON response (status ${response.status})`);
+        }
+        if (!response.ok) {
+            const message = responseData.message ?? `HTTP ${response.status}`;
+            this._raiseForStatus(response.status, message, responseData);
+        }
+        if (responseData.status === "error") {
+            const message = responseData.message ?? "Unknown error";
+            this._raiseForStatus(400, message, responseData);
+        }
+        return responseData;
+    }
+    _raiseForStatus(statusCode, message, response) {
+        if (statusCode === 401)
+            throw new AuthenticationError(message, statusCode, response ?? null);
+        if (statusCode === 403)
+            throw new AuthorizationError(message, statusCode, response ?? null);
+        if (statusCode === 404)
+            throw new NotFoundError(message, statusCode, response ?? null);
+        if (statusCode === 422)
+            throw new ValidationError(message, statusCode, response ?? null);
+        if (statusCode === 429)
+            throw new RateLimitError(message, statusCode, response ?? null);
+        if (statusCode >= 500)
+            throw new ServerError(message, statusCode, response ?? null);
+        throw new AshrLabsError(message, statusCode, response ?? null);
+    }
+    // =========================================================================
+    // Dataset Operations
+    // =========================================================================
+    async getDataset(datasetId, includeSignedUrls = false, urlExpiresSeconds = 3600) {
+        const response = await this._makeRequest("get_dataset", {
+            dataset_id: datasetId,
+            include_signed_urls: includeSignedUrls,
+            url_expires_seconds: urlExpiresSeconds,
+        });
+        return response.dataset;
+    }
+    async listDatasets(tenantId, limit = 50, offset = 0, includeSignedUrls = false, urlExpiresSeconds = 3600) {
+        return this._makeRequest("list_datasets", {
+            tenant_id: await this._resolveTenantId(tenantId),
+            limit,
+            offset,
+            include_signed_urls: includeSignedUrls,
+            url_expires_seconds: urlExpiresSeconds,
+        });
+    }
+    // =========================================================================
+    // Run Operations
+    // =========================================================================
+    async createRun(datasetId, result, tenantId, runnerId) {
+        const params = {
+            tenant_id: await this._resolveTenantId(tenantId),
+            dataset_id: datasetId,
+            result,
+        };
+        if (runnerId != null)
+            params.runner_id = runnerId;
+        const response = await this._makeRequest("create_run", params);
+        return response.run;
+    }
+    async deleteRun(runId) {
+        return this._makeRequest("delete_run", { run_id: runId });
+    }
+    async getRun(runId) {
+        const response = await this._makeRequest("get_run", { run_id: runId });
+        return response.run;
+    }
+    async listRuns(datasetId, tenantId, limit = 50, offset = 0) {
+        const params = {
+            tenant_id: await this._resolveTenantId(tenantId),
+            limit,
+            offset,
+        };
+        if (datasetId != null)
+            params.dataset_id = datasetId;
+        return this._makeRequest("list_runs", params);
+    }
+    // =========================================================================
+    // Request Operations
+    // =========================================================================
+    async createRequest(requestName, request, requestInputSchema, tenantId, requestorId) {
+        if (requestInputSchema == null) {
+            requestInputSchema = { type: "object", properties: {} };
+            const agent = request.agent;
+            const tools = agent?.tools;
+            if (tools) {
+                requestInputSchema.tools = tools;
+            }
+        }
+        const params = {
+            tenant_id: await this._resolveTenantId(tenantId),
+            requestor_id: await this._resolveUserId(requestorId),
+            request_name: requestName,
+            request,
+            request_input_schema: requestInputSchema,
+        };
+        const response = await this._makeRequest("create_request", params);
+        return response.request;
+    }
+    async getRequest(requestId) {
+        const response = await this._makeRequest("get_request", {
+            request_id: requestId,
+        });
+        return response.request;
+    }
+    async listRequests(tenantId, status, limit = 50, offset = 0) {
+        const params = {
+            tenant_id: await this._resolveTenantId(tenantId),
+            limit,
+            offset,
+        };
+        if (status != null)
+            params.status = status;
+        return this._makeRequest("list_requests", params);
+    }
+    // =========================================================================
+    // API Key Operations
+    // =========================================================================
+    async listApiKeys(includeInactive = false) {
+        const response = await this._makeRequest("list_api_keys", {
+            include_inactive: includeInactive,
+        });
+        return response.api_keys;
+    }
+    async revokeApiKey(apiKeyId) {
+        return this._makeRequest("revoke_api_key", { api_key_id: apiKeyId });
+    }
+    // =========================================================================
+    // Session Operations
+    // =========================================================================
+    async init() {
+        return this._makeRequest("init");
+    }
+    async healthCheck() {
+        return this._makeRequest("keep_alive");
+    }
+    // =========================================================================
+    // Convenience / Workflow Methods
+    // =========================================================================
+    async waitForRequest(requestId, timeout = 600, pollInterval = 5) {
+        const start = Date.now();
+        while ((Date.now() - start) / 1000 < timeout) {
+            const req = await this.getRequest(requestId);
+            const reqStatus = req.request_status;
+            if (reqStatus === "completed")
+                return req;
+            if (reqStatus === "failed") {
+                const errorMsg = req.error ?? "Request failed";
+                throw new AshrLabsError(`Request ${requestId} failed: ${errorMsg}`);
+            }
+            await new Promise((r) => setTimeout(r, pollInterval * 1000));
+        }
+        throw new Error(`Request ${requestId} did not complete within ${timeout}s`);
+    }
+    async generateDataset(requestName, config, requestInputSchema, timeout = 600, pollInterval = 5) {
+        const req = await this.createRequest(requestName, config, requestInputSchema);
+        const requestId = req.id;
+        await this.waitForRequest(requestId, timeout, pollInterval);
+        const resp = await this.listDatasets(undefined, 1);
+        const datasets = resp.datasets;
+        if (!datasets || datasets.length === 0) {
+            throw new AshrLabsError("No datasets found after generation completed");
+        }
+        const datasetId = datasets[0].id;
+        const fullDs = await this.getDataset(datasetId, false);
+        const source = (fullDs.dataset_source ?? {});
+        return [datasetId, source];
+    }
+    toString() {
+        return `AshrLabsClient(baseUrl='${this.baseUrl}', apiKey='${this.apiKey.slice(0, 8)}...')`;
+    }
+}

package/dist/comparators.d.ts

@@ -0,0 +1,6 @@
+export declare function stripMarkdown(text: string): string;
+export declare function tokenize(text: string): string[];
+export declare function fuzzyStrMatch(a: string, b: string, threshold?: number): boolean;
+export declare function extractToolArgs(toolCall: Record<string, unknown>): Record<string, unknown>;
+export declare function compareToolArgs(expected: Record<string, unknown>, actual: Record<string, unknown>): [string, string | null];
+export declare function textSimilarity(textA: string, textB: string): number;

package/dist/comparators.js

@@ -0,0 +1,212 @@
+export function stripMarkdown(text) {
+    // Remove bold/italic markers
+    text = text.replace(/\*{1,3}([^*]+)\*{1,3}/g, "$1");
+    // Remove markdown headers
+    text = text.replace(/^#{1,6}\s+/gm, "");
+    // Remove bullet points
+    text = text.replace(/^[\s]*[-*+]\s+/gm, "");
+    // Remove markdown links [text](url) -> text
+    text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
+    // Collapse multiple whitespace
+    text = text.replace(/\s+/g, " ").trim();
+    return text;
+}
+export function tokenize(text) {
+    text = stripMarkdown(text);
+    text = text.toLowerCase();
+    // Strip punctuation
+    text = text.replace(/[^\w\s]/g, "");
+    const tokens = text.split(/\s+/);
+    return tokens.filter((t) => t.length > 0);
+}
+export function fuzzyStrMatch(a, b, threshold) {
+    const na = a
+        .toLowerCase()
+        .replace(/[^\w\s]/g, "")
+        .trim();
+    const nb = b
+        .toLowerCase()
+        .replace(/[^\w\s]/g, "")
+        .trim();
+    if (na === nb)
+        return true;
+    if (na.includes(nb) || nb.includes(na))
+        return true;
+    const wa = new Set(na.split(/\s+/).filter((w) => w.length > 0));
+    const wb = new Set(nb.split(/\s+/).filter((w) => w.length > 0));
+    if (wa.size === 0 || wb.size === 0)
+        return false;
+    let overlapCount = 0;
+    for (const w of wa) {
+        if (wb.has(w))
+            overlapCount++;
+    }
+    const overlap = overlapCount / Math.max(wa.size, wb.size);
+    if (threshold === undefined) {
+        const minLen = Math.min(wa.size, wb.size);
+        threshold = minLen <= 5 ? 0.35 : minLen <= 8 ? 0.4 : 0.55;
+    }
+    return overlap >= threshold;
+}
+export function extractToolArgs(toolCall) {
+    if (typeof toolCall.arguments === "object" &&
+        toolCall.arguments !== null &&
+        !Array.isArray(toolCall.arguments)) {
+        return toolCall.arguments;
+    }
+    const raw = toolCall.arguments_json ?? "{}";
+    if (typeof raw === "string") {
+        try {
+            return JSON.parse(raw);
+        }
+        catch {
+            return {};
+        }
+    }
+    return raw || {};
+}
+export function compareToolArgs(expected, actual) {
+    const expArgs = extractToolArgs(expected);
+    const actArgs = extractToolArgs(actual);
+    if (Object.keys(expArgs).length === 0 && Object.keys(actArgs).length === 0) {
+        return ["exact", null];
+    }
+    let allMatch = true;
+    let anyMatch = false;
+    for (const key of Object.keys(expArgs)) {
+        const expVal = expArgs[key];
+        const actVal = actArgs[key];
+        if (actVal === undefined || actVal === null) {
+            allMatch = false;
+            continue;
+        }
+        if (typeof expVal === "string" && typeof actVal === "string") {
+            if (fuzzyStrMatch(expVal, actVal)) {
+                anyMatch = true;
+            }
+            else {
+                allMatch = false;
+            }
+        }
+        else {
+            if (JSON.stringify(expVal) === JSON.stringify(actVal)) {
+                anyMatch = true;
+            }
+            else {
+                allMatch = false;
+            }
+        }
+    }
+    let status;
+    if (allMatch && Object.keys(expArgs).length > 0) {
+        status = "exact";
+    }
+    else if (anyMatch) {
+        status = "partial";
+    }
+    else if (Object.keys(expArgs).length > 0) {
+        status = "mismatch";
+    }
+    else {
+        status = "exact";
+    }
+    const notes = status !== "exact" ? argDiffNotes(expArgs, actArgs) : null;
+    return [status, notes];
+}
+export function textSimilarity(textA, textB) {
+    const tokensA = tokenize(textA);
+    const tokensB = tokenize(textB);
+    if (tokensA.length === 0 || tokensB.length === 0)
+        return 0.0;
+    const counterA = new Map();
+    for (const t of tokensA)
+        counterA.set(t, (counterA.get(t) ?? 0) + 1);
+    const counterB = new Map();
+    for (const t of tokensB)
+        counterB.set(t, (counterB.get(t) ?? 0) + 1);
+    const allWords = new Set([...counterA.keys(), ...counterB.keys()]);
+    let dot = 0;
+    for (const w of allWords) {
+        dot += (counterA.get(w) ?? 0) * (counterB.get(w) ?? 0);
+    }
+    let magA = 0;
+    for (const v of counterA.values())
+        magA += v * v;
+    magA = Math.sqrt(magA);
+    let magB = 0;
+    for (const v of counterB.values())
+        magB += v * v;
+    magB = Math.sqrt(magB);
+    let cosine = magA > 0 && magB > 0 ? dot / (magA * magB) : 0.0;
+    // Boost 1: Entity overlap
+    const entityPattern = /ORD-\d+|REF-\d+|\$[\d.]+|\d{4}-\d{2}-\d{2}|track\S+\.\w+\/\S+/gi;
+    const entitiesA = new Set((textA.toUpperCase().match(entityPattern) ?? []).map((e) => e.toUpperCase()));
+    const entitiesB = new Set((textB.toUpperCase().match(entityPattern) ?? []).map((e) => e.toUpperCase()));
+    if (entitiesA.size > 0 && entitiesB.size > 0) {
+        let entityOverlapCount = 0;
+        for (const e of entitiesA) {
+            if (entitiesB.has(e))
+                entityOverlapCount++;
+        }
+        const entityOverlap = entityOverlapCount / Math.max(entitiesA.size, entitiesB.size);
+        cosine = Math.min(1.0, cosine + entityOverlap * 0.2);
+    }
+    // Boost 2: Domain concept overlap
+    const concepts = [
+        new Set(["refund", "refunded", "credited"]),
+        new Set(["shipped", "shipping", "transit", "delivered", "delivery"]),
+        new Set(["tracking", "track"]),
+        new Set(["stock", "available", "availability", "inventory"]),
+        new Set(["processing", "processed"]),
+        new Set(["pickup", "store"]),
+        new Set(["manual", "review", "escalat"]),
+        new Set(["damaged", "defective", "cracked", "broken"]),
+    ];
+    const la = textA.toLowerCase();
+    const lb = textB.toLowerCase();
+    let conceptMatches = 0;
+    let conceptTotal = 0;
+    for (const conceptSet of concepts) {
+        let aHas = false;
+        let bHas = false;
+        for (const c of conceptSet) {
+            if (la.includes(c))
+                aHas = true;
+            if (lb.includes(c))
+                bHas = true;
+        }
+        if (aHas || bHas) {
+            conceptTotal++;
+            if (aHas && bHas)
+                conceptMatches++;
+        }
+    }
+    if (conceptTotal > 0) {
+        const conceptOverlap = conceptMatches / conceptTotal;
+        cosine = Math.min(1.0, cosine + conceptOverlap * 0.1);
+    }
+    return Math.round(cosine * 100) / 100;
+}
+function argDiffNotes(expArgs, actArgs) {
+    const diffs = [];
+    const allKeys = new Set([...Object.keys(expArgs), ...Object.keys(actArgs)]);
+    for (const key of allKeys) {
+        const ev = expArgs[key];
+        const av = actArgs[key];
+        if (ev === undefined) {
+            diffs.push(`extra arg '${key}'=${av}`);
+        }
+        else if (av === undefined) {
+            diffs.push(`missing arg '${key}'`);
+        }
+        else if (typeof ev === "string" && typeof av === "string") {
+            if (!fuzzyStrMatch(ev, av)) {
+                diffs.push(`'${key}': expected='${ev}' actual='${av}'`);
+            }
+        }
+        else if (JSON.stringify(ev) !== JSON.stringify(av)) {
+            diffs.push(`'${key}': expected=${ev} actual=${av}`);
+        }
+    }
+    return diffs.length > 0 ? diffs.join("; ") : null;
+}

package/dist/eval.d.ts

@@ -0,0 +1,42 @@
+import { RunBuilder } from "./run-builder.js";
+import type { AshrLabsClient } from "./client.js";
+export interface Agent {
+    respond(message: string): Record<string, unknown> | Promise<Record<string, unknown>>;
+    reset(): void | Promise<void>;
+}
+export type OnScenarioCallback = (scenarioId: string, scenario: Record<string, unknown>) => void;
+export type OnActionCallback = (actionIndex: number, action: Record<string, unknown>) => void;
+export type ToolComparator = (expected: Record<string, unknown>, actual: Record<string, unknown>) => [string, string | null];
+export type TextComparator = (a: string, b: string) => number;
+export declare class EvalRunner {
+    private _source;
+    private _toolComparator;
+    private _textComparator;
+    private _exactThreshold;
+    private _similarThreshold;
+    constructor(datasetSource: Record<string, unknown>, options?: {
+        toolComparator?: ToolComparator;
+        textComparator?: TextComparator;
+        similarityThresholds?: {
+            exact?: number;
+            similar?: number;
+        };
+    });
+    static fromDataset(client: AshrLabsClient, datasetId: number, options?: {
+        toolComparator?: ToolComparator;
+        textComparator?: TextComparator;
+        similarityThresholds?: {
+            exact?: number;
+            similar?: number;
+        };
+    }): Promise<EvalRunner>;
+    private _runScenario;
+    run(agent: Agent, options?: {
+        onScenario?: OnScenarioCallback;
+        onAction?: OnActionCallback;
+    }): Promise<RunBuilder>;
+    runAndDeploy(agent: Agent, client: AshrLabsClient, datasetId: number, options?: {
+        onScenario?: OnScenarioCallback;
+        onAction?: OnActionCallback;
+    }): Promise<Record<string, unknown>>;
+}

package/dist/eval.js

@@ -0,0 +1,119 @@
+import { compareToolArgs, extractToolArgs, textSimilarity, } from "./comparators.js";
+import { RunBuilder, TestBuilder } from "./run-builder.js";
+export class EvalRunner {
+    _source;
+    _toolComparator;
+    _textComparator;
+    _exactThreshold;
+    _similarThreshold;
+    constructor(datasetSource, options) {
+        this._source = datasetSource;
+        this._toolComparator = options?.toolComparator ?? compareToolArgs;
+        this._textComparator = options?.textComparator ?? textSimilarity;
+        const thresholds = options?.similarityThresholds ?? {};
+        this._exactThreshold = thresholds.exact ?? 0.7;
+        this._similarThreshold = thresholds.similar ?? 0.4;
+    }
+    static async fromDataset(client, datasetId, options) {
+        const ds = await client.getDataset(datasetId, false);
+        const source = (ds.dataset_source ?? {});
+        return new EvalRunner(source, options);
+    }
+    async _runScenario(agent, runId, scenario, onScenario, onAction) {
+        if (onScenario)
+            onScenario(runId, scenario);
+        await agent.reset();
+        const test = new TestBuilder(runId);
+        test.start();
+        let agentText = "";
+        let agentTools = [];
+        const actions = (scenario.actions ?? []);
+        for (let i = 0; i < actions.length; i++) {
+            const action = actions[i];
+            if (onAction)
+                onAction(i, action);
+            const actor = action.actor;
+            const content = (action.content ?? "");
+            if (actor === "user") {
+                test.addUserText(content, action.name ?? `user_action_${i}`, i);
+                try {
+                    const result = await agent.respond(content);
+                    agentText = (result.text ?? "");
+                    agentTools = [...(result.tool_calls ?? [])];
+                }
+                catch (e) {
+                    agentText = `[error: ${e}]`;
+                    agentTools = [];
+                }
+            }
+            else if (actor === "agent") {
+                const expected = (action.expected_response ?? {});
+                const expectedTools = (expected.tool_calls ?? []);
+                const expectedText = content;
+                for (const expTc of expectedTools) {
+                    const expName = (expTc.name ?? "");
+                    let matched = null;
+                    let matchedIdx = -1;
+                    for (let ti = 0; ti < agentTools.length; ti++) {
+                        if (agentTools[ti].name === expName) {
+                            matched = agentTools[ti];
+                            matchedIdx = ti;
+                            break;
+                        }
+                    }
+                    if (matched !== null) {
+                        const [status, notes] = this._toolComparator(expTc, matched);
+                        const actualArgs = extractToolArgs(matched);
+                        test.addToolCall(expTc, {
+                            name: (matched.name ?? ""),
+                            arguments_json: JSON.stringify(actualArgs),
+                        }, status, notes, i);
+                        agentTools.splice(matchedIdx, 1);
+                    }
+                    else {
+                        test.addToolCall(expTc, { name: "NOT_CALLED", arguments_json: "{}" }, "mismatch", `Agent never called ${expName}`, i);
+                    }
+                }
+                if (expectedText && agentText) {
+                    const similarity = this._textComparator(expectedText, agentText);
+                    let matchStatus;
+                    if (similarity > this._exactThreshold) {
+                        matchStatus = "exact";
+                    }
+                    else if (similarity > this._similarThreshold) {
+                        matchStatus = "similar";
+                    }
+                    else {
+                        matchStatus = "divergent";
+                    }
+                    test.addAgentResponse({ text: expectedText }, { text: agentText }, matchStatus, similarity, null, i);
+                }
+                agentText = "";
+            }
+        }
+        test.complete();
+        return test;
+    }
+    async run(agent, options) {
+        const runsData = (this._source.runs ?? {});
+        const run = new RunBuilder();
+        run.start();
+        const scenarios = [];
+        for (const [runId, scenario] of Object.entries(runsData)) {
+            const actions = scenario.actions;
+            if (actions && actions.length > 0) {
+                scenarios.push([runId, scenario]);
+            }
+        }
+        for (const [runId, scenario] of scenarios) {
+            const test = await this._runScenario(agent, runId, scenario, options?.onScenario, options?.onAction);
+            run._tests.push(test);
+        }
+        run.complete();
+        return run;
+    }
+    async runAndDeploy(agent, client, datasetId, options) {
+        const runResult = await this.run(agent, options);
+        return runResult.deploy(client, datasetId);
+    }
+}

package/dist/exceptions.d.ts

@@ -0,0 +1,24 @@
+export declare class AshrLabsError extends Error {
+    statusCode: number | null;
+    response: Record<string, unknown> | null;
+    constructor(message: string, statusCode?: number | null, response?: Record<string, unknown> | null);
+    toString(): string;
+}
+export declare class AuthenticationError extends AshrLabsError {
+    constructor(message: string, statusCode?: number | null, response?: Record<string, unknown> | null);
+}
+export declare class AuthorizationError extends AshrLabsError {
+    constructor(message: string, statusCode?: number | null, response?: Record<string, unknown> | null);
+}
+export declare class NotFoundError extends AshrLabsError {
+    constructor(message: string, statusCode?: number | null, response?: Record<string, unknown> | null);
+}
+export declare class ValidationError extends AshrLabsError {
+    constructor(message: string, statusCode?: number | null, response?: Record<string, unknown> | null);
+}
+export declare class RateLimitError extends AshrLabsError {
+    constructor(message: string, statusCode?: number | null, response?: Record<string, unknown> | null);
+}
+export declare class ServerError extends AshrLabsError {
+    constructor(message: string, statusCode?: number | null, response?: Record<string, unknown> | null);
+}

package/dist/exceptions.js

@@ -0,0 +1,52 @@
+export class AshrLabsError extends Error {
+    statusCode;
+    response;
+    constructor(message, statusCode = null, response = null) {
+        super(message);
+        this.name = "AshrLabsError";
+        this.statusCode = statusCode;
+        this.response = response;
+    }
+    toString() {
+        if (this.statusCode) {
+            return `[${this.statusCode}] ${this.message}`;
+        }
+        return this.message;
+    }
+}
+export class AuthenticationError extends AshrLabsError {
+    constructor(message, statusCode = null, response = null) {
+        super(message, statusCode, response);
+        this.name = "AuthenticationError";
+    }
+}
+export class AuthorizationError extends AshrLabsError {
+    constructor(message, statusCode = null, response = null) {
+        super(message, statusCode, response);
+        this.name = "AuthorizationError";
+    }
+}
+export class NotFoundError extends AshrLabsError {
+    constructor(message, statusCode = null, response = null) {
+        super(message, statusCode, response);
+        this.name = "NotFoundError";
+    }
+}
+export class ValidationError extends AshrLabsError {
+    constructor(message, statusCode = null, response = null) {
+        super(message, statusCode, response);
+        this.name = "ValidationError";
+    }
+}
+export class RateLimitError extends AshrLabsError {
+    constructor(message, statusCode = null, response = null) {
+        super(message, statusCode, response);
+        this.name = "RateLimitError";
+    }
+}
+export class ServerError extends AshrLabsError {
+    constructor(message, statusCode = null, response = null) {
+        super(message, statusCode, response);
+        this.name = "ServerError";
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { AshrLabsClient } from "./client.js";
+export { AshrLabsError, AuthenticationError, AuthorizationError, NotFoundError, ValidationError, RateLimitError, ServerError, } from "./exceptions.js";
+export type { User, Tenant, Session, Dataset, Run, Request, APIKey, ToolCall, ExpectedResponse, Action, Scenario, } from "./models.js";
+export { RunBuilder, TestBuilder } from "./run-builder.js";
+export { stripMarkdown, tokenize, fuzzyStrMatch, extractToolArgs, compareToolArgs, textSimilarity, } from "./comparators.js";
+export { EvalRunner } from "./eval.js";
+export type { Agent } from "./eval.js";

package/dist/index.js

@@ -0,0 +1,5 @@
+export { AshrLabsClient } from "./client.js";
+export { AshrLabsError, AuthenticationError, AuthorizationError, NotFoundError, ValidationError, RateLimitError, ServerError, } from "./exceptions.js";
+export { RunBuilder, TestBuilder } from "./run-builder.js";
+export { stripMarkdown, tokenize, fuzzyStrMatch, extractToolArgs, compareToolArgs, textSimilarity, } from "./comparators.js";
+export { EvalRunner } from "./eval.js";

package/dist/models.d.ts

@@ -0,0 +1,93 @@
+export interface User {
+    id?: number;
+    created_at?: string;
+    email?: string;
+    name?: string | null;
+    tenant?: number;
+    is_active?: boolean;
+}
+export interface Tenant {
+    id?: number;
+    created_at?: string;
+    tenant_name?: string;
+    is_active?: boolean;
+}
+export interface Session {
+    status: string;
+    user: User;
+    tenant: Tenant;
+}
+export interface APIKey {
+    id?: number;
+    key?: string;
+    key_prefix?: string;
+    name?: string;
+    scopes?: string[];
+    user_id?: number;
+    tenant_id?: number;
+    created_at?: string;
+    last_used_at?: string | null;
+    expires_at?: string | null;
+    is_active?: boolean;
+}
+export interface Dataset {
+    id?: number;
+    created_at?: string;
+    tenant?: number;
+    creator?: number;
+    name?: string;
+    description?: string | null;
+    dataset_source?: Record<string, unknown>;
+}
+export interface Run {
+    id?: number;
+    created_at?: string;
+    dataset?: number;
+    tenant?: number;
+    runner?: number;
+    result?: Record<string, unknown>;
+}
+export interface Request {
+    id?: number;
+    created_at?: string;
+    requestor_id?: number;
+    requestor_tenant?: number;
+    request_name?: string;
+    request_status?: string;
+    request_input_schema?: Record<string, unknown> | null;
+    request?: Record<string, unknown>;
+}
+export interface ListResponse {
+    status: string;
+    total: number;
+}
+export interface DatasetsListResponse extends ListResponse {
+    datasets: Dataset[];
+}
+export interface RunsListResponse extends ListResponse {
+    runs: Run[];
+}
+export interface RequestsListResponse extends ListResponse {
+    requests: Request[];
+}
+export interface APIKeysListResponse extends ListResponse {
+    api_keys: APIKey[];
+}
+export interface ToolCall {
+    name?: string;
+    arguments_json?: string;
+}
+export interface ExpectedResponse {
+    tool_calls?: ToolCall[];
+    text?: string;
+}
+export interface Action {
+    actor?: string;
+    content?: string;
+    name?: string;
+    expected_response?: ExpectedResponse;
+}
+export interface Scenario {
+    title?: string;
+    actions?: Action[];
+}

package/dist/models.js

@@ -0,0 +1 @@
+export {};

package/dist/run-builder.d.ts

@@ -0,0 +1,30 @@
+import type { AshrLabsClient } from "./client.js";
+export declare class TestBuilder {
+    private _testId;
+    private _status;
+    private _startedAt;
+    private _completedAt;
+    private _actionResults;
+    private _nextActionIndex;
+    constructor(testId: string);
+    start(): this;
+    addUserFile(filePath: string, description: string, actionIndex?: number): this;
+    addUserText(text: string, description: string, actionIndex?: number): this;
+    addToolCall(expected: Record<string, unknown>, actual: Record<string, unknown>, matchStatus: string, divergenceNotes?: string | null, actionIndex?: number): this;
+    addAgentResponse(expectedResponse: Record<string, unknown>, actualResponse: Record<string, unknown>, matchStatus: string, semanticSimilarity?: number | null, divergenceNotes?: string | null, actionIndex?: number): this;
+    complete(status?: string): this;
+    build(): Record<string, unknown>;
+    private _resolveIndex;
+}
+export declare class RunBuilder {
+    private _status;
+    private _startedAt;
+    private _completedAt;
+    /** @internal */
+    _tests: TestBuilder[];
+    start(): this;
+    addTest(testId: string): TestBuilder;
+    complete(status?: string): this;
+    build(): Record<string, unknown>;
+    deploy(client: AshrLabsClient, datasetId: number, tenantId?: number, runnerId?: number): Promise<Record<string, unknown>>;
+}

package/dist/run-builder.js

@@ -0,0 +1,221 @@
+function now() {
+    return new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+function normalizeToolCall(tc) {
+    const out = { ...tc };
+    if (!("tool_name" in out) && "name" in out) {
+        out.tool_name = out.name;
+    }
+    else if (!("name" in out) && "tool_name" in out) {
+        out.name = out.tool_name;
+    }
+    if (!("arguments" in out) && "arguments_json" in out) {
+        const val = out.arguments_json;
+        if (typeof val === "string") {
+            try {
+                out.arguments = JSON.parse(val);
+            }
+            catch {
+                out.arguments = {};
+            }
+        }
+        else {
+            out.arguments = val;
+        }
+    }
+    else if (!("arguments_json" in out) && "arguments" in out) {
+        const args = out.arguments;
+        out.arguments_json =
+            typeof args === "object" && args !== null
+                ? JSON.stringify(args)
+                : String(args);
+    }
+    return out;
+}
+function computeAggregate(tests) {
+    const total = tests.length;
+    let passed = 0;
+    let failed = 0;
+    let totalToolCallDivergence = 0;
+    let totalResponseDivergence = 0;
+    const similarityScores = [];
+    for (const test of tests) {
+        if (test.status === "completed")
+            passed++;
+        else if (test.status === "failed")
+            failed++;
+        const actionResults = (test.action_results ?? []);
+        for (const action of actionResults) {
+            if (action.action_type === "tool_call") {
+                const tcs = (action.tool_calls ?? []);
+                for (const tc of tcs) {
+                    if (tc.match_status && tc.match_status !== "exact") {
+                        totalToolCallDivergence++;
+                    }
+                }
+            }
+            else if (action.action_type === "text" && action.actor === "agent") {
+                if (action.match_status && action.match_status !== "exact") {
+                    totalResponseDivergence++;
+                }
+                if (typeof action.semantic_similarity === "number") {
+                    similarityScores.push(action.semantic_similarity);
+                }
+            }
+        }
+    }
+    const avgSimilarity = similarityScores.length > 0
+        ? Math.round((similarityScores.reduce((a, b) => a + b, 0) /
+            similarityScores.length) *
+            100) / 100
+        : null;
+    return {
+        total_tests: total,
+        tests_passed: passed,
+        tests_failed: failed,
+        average_similarity_score: avgSimilarity,
+        total_tool_call_divergence: totalToolCallDivergence,
+        total_response_divergence: totalResponseDivergence,
+    };
+}
+export class TestBuilder {
+    _testId;
+    _status = "pending";
+    _startedAt = null;
+    _completedAt = null;
+    _actionResults = [];
+    _nextActionIndex = 0;
+    constructor(testId) {
+        this._testId = testId;
+    }
+    start() {
+        this._status = "running";
+        this._startedAt = now();
+        return this;
+    }
+    addUserFile(filePath, description, actionIndex) {
+        const idx = this._resolveIndex(actionIndex);
+        this._actionResults.push({
+            actor: "user",
+            action_type: "file",
+            action_index: idx,
+            description,
+            file_path: filePath,
+            input_provided: true,
+        });
+        return this;
+    }
+    addUserText(text, description, actionIndex) {
+        const idx = this._resolveIndex(actionIndex);
+        this._actionResults.push({
+            actor: "user",
+            action_type: "text",
+            action_index: idx,
+            description,
+            text,
+            input_provided: true,
+        });
+        return this;
+    }
+    addToolCall(expected, actual, matchStatus, divergenceNotes, actionIndex) {
+        const idx = this._resolveIndex(actionIndex);
+        const tcEntry = {
+            expected: normalizeToolCall(expected),
+            actual: normalizeToolCall(actual),
+            match_status: matchStatus,
+            divergence_notes: divergenceNotes ?? null,
+        };
+        const existing = this._actionResults.find((ar) => ar.action_type === "tool_call" && ar.action_index === idx);
+        if (existing) {
+            existing.tool_calls.push(tcEntry);
+        }
+        else {
+            this._actionResults.push({
+                actor: "agent",
+                action_type: "tool_call",
+                action_index: idx,
+                tool_calls: [tcEntry],
+            });
+        }
+        return this;
+    }
+    addAgentResponse(expectedResponse, actualResponse, matchStatus, semanticSimilarity, divergenceNotes, actionIndex) {
+        const idx = this._resolveIndex(actionIndex);
+        const result = {
+            actor: "agent",
+            action_type: "text",
+            action_index: idx,
+            expected_response: expectedResponse,
+            actual_response: actualResponse,
+            match_status: matchStatus,
+            divergence_notes: divergenceNotes ?? null,
+        };
+        if (semanticSimilarity != null) {
+            result.semantic_similarity = semanticSimilarity;
+        }
+        this._actionResults.push(result);
+        return this;
+    }
+    complete(status = "completed") {
+        this._status = status;
+        this._completedAt = now();
+        return this;
+    }
+    build() {
+        const result = {
+            test_id: this._testId,
+            status: this._status,
+            action_results: [...this._actionResults],
+        };
+        if (this._startedAt)
+            result.started_at = this._startedAt;
+        if (this._completedAt)
+            result.completed_at = this._completedAt;
+        return result;
+    }
+    _resolveIndex(explicit) {
+        if (explicit !== undefined) {
+            this._nextActionIndex = explicit + 1;
+            return explicit;
+        }
+        const idx = this._nextActionIndex;
+        this._nextActionIndex++;
+        return idx;
+    }
+}
+export class RunBuilder {
+    _status = "pending";
+    _startedAt = null;
+    _completedAt = null;
+    /** @internal */
+    _tests = [];
+    start() {
+        this._status = "running";
+        this._startedAt = now();
+        return this;
+    }
+    addTest(testId) {
+        const test = new TestBuilder(testId);
+        this._tests.push(test);
+        return test;
+    }
+    complete(status = "completed") {
+        this._status = status;
+        this._completedAt = now();
+        return this;
+    }
+    build() {
+        const tests = this._tests.map((t) => t.build());
+        return {
+            tests,
+            status: this._status,
+            started_at: this._startedAt,
+            completed_at: this._completedAt,
+            aggregate_metrics: computeAggregate(tests),
+        };
+    }
+    deploy(client, datasetId, tenantId, runnerId) {
+        const result = this.build();
+        return client.createRun(datasetId, result, tenantId, runnerId);
+    }
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "ashr-labs",
+  "version": "0.1.0",
+  "description": "TypeScript SDK for the Ashr Labs API",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "keywords": [
+    "ashr",
+    "labs",
+    "api",
+    "sdk"
+  ],
+  "license": "MIT",
+  "devDependencies": {
+    "@types/node": "^25.3.5",
+    "typescript": "^5.4.0",
+    "vitest": "^3.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}