@xoxoai/checkmate 0.6.6 → 0.7.0

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 Dawid Dobrowolski SoftQA
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/README.md

@@ -1,294 +0,0 @@
-#### □ □ ■ □
-
-## _Checkmate - a harness for agentic acceptance_
-
-[![npm version](https://img.shields.io/npm/v/%40xoxoai%2Fcheckmate)](https://www.npmjs.com/package/@xoxoai/checkmate)
-
-<img src="docs/img/onboarding.gif" alt="onboarding" width="50%" centered/>
-
-#
-
-Think of it like QA contracts for software factories: policies define the bounds of execution, where scenarios describe the intent to verify through a domain driver. It aims to patch the gap between fully coded tests and freelance agent testing.
-
-## Install
-
-```bash
-npm install @xoxoai/checkmate
-```
-
-## Usage
-
-#### Scenario is a JSON document:
-
-```json
-{
-	"schemaVersion": 1,
-	"scenario": {
-		"id": "checkout spring promo",
-		"driver": {
-			"id": "web",
-			"target": {
-				"baseUrl": "https://staging.example.test"
-			}
-		},
-		"policy": "ci",
-		"steps": [
-			{
-				"id": "start checkout",
-				"action": "Add a product to the cart",
-				"expect": "The checkout page is displayed"
-			},
-			{
-				"id": "apply promo",
-				"action": "Apply promotion code SPRING25",
-				"expect": "The order total is reduced by 25%"
-			},
-			{
-				"id": "verify total",
-				"action": "Proceed to the payment step",
-				"expect": "The order total reflects the applied promotion"
-			}
-		]
-	}
-}
-```
-
-#### Run:
-
-```bash
-checkmate run scenario.json
-```
-
-Checkmate writes one JSON result to stdout. Outcomes distinguish application, model, infrastructure, and invalid-invocation failures.
-
-<details>
-<summary>Example Results</summary>
-
-A successful run of the scenario above:
-
-```json
-{
-	"kind": "run-result",
-	"schemaVersion": 1,
-	"runId": "0123456789abcdef",
-	"scenarioId": "checkout spring promo",
-	"status": "passed",
-	"category": "passed",
-	"reason": "scenario-complete",
-	"targetMutation": "possibly-mutated",
-	"startedAt": "2026-09-04T12:00:00.000Z",
-	"durationMs": 12600,
-	"driver": { "id": "web", "contractVersion": 1 },
-	"policy": {
-		"id": "ci",
-		"effectiveLimits": {
-			"scenarioTimeoutMs": 180000,
-			"stepTimeoutMs": 120000,
-			"turnsPerStep": 20,
-			"requestTimeoutMs": 60000,
-			"maxRetries": 3,
-			"loopMaxRepetitions": 5,
-			"cleanupTimeoutMs": 10000,
-			"budgetTokens": 200000
-		}
-	},
-	"usage": {
-		"promptTokens": 3600,
-		"cachedPromptTokens": 1200,
-		"completionTokens": 300,
-		"totalTokens": 3900,
-		"state": "complete"
-	},
-	"steps": [
-		{
-			"id": "start checkout",
-			"status": "passed",
-			"category": "app",
-			"reason": "met-expectation",
-			"actual": "The checkout page displays one product with a $40 order total.",
-			"turns": 3,
-			"durationMs": 4000,
-			"usage": { "promptTokens": 1200, "cachedPromptTokens": 400, "completionTokens": 100, "totalTokens": 1300 },
-			"toolCalls": [
-				{
-					"turn": 1,
-					"driverId": "web",
-					"name": "browser_click_or_hover",
-					"arguments": { "ref": "e12", "name": "Add to cart", "hover": false, "goal": "Add a product to the cart" },
-					"status": "ok"
-				},
-				{
-					"turn": 2,
-					"driverId": "web",
-					"name": "browser_click_or_hover",
-					"arguments": { "ref": "e18", "name": "Checkout", "hover": false, "goal": "Open the checkout page" },
-					"status": "ok"
-				}
-			]
-		},
-		{
-			"id": "apply promo",
-			"status": "passed",
-			"category": "app",
-			"reason": "met-expectation",
-			"actual": "SPRING25 is applied and the order total is reduced from $40 to $30.",
-			"turns": 3,
-			"durationMs": 4000,
-			"usage": { "promptTokens": 1200, "cachedPromptTokens": 400, "completionTokens": 100, "totalTokens": 1300 },
-			"toolCalls": [
-				{
-					"turn": 1,
-					"driverId": "web",
-					"name": "browser_type_or_select",
-					"arguments": {
-						"elements": [
-							{ "ref": "e24", "name": "Promotion code", "text": "SPRING25", "clear": true, "select": false }
-						],
-						"goal": "Enter the promotion code"
-					},
-					"status": "ok"
-				},
-				{
-					"turn": 2,
-					"driverId": "web",
-					"name": "browser_click_or_hover",
-					"arguments": { "ref": "e25", "name": "Apply", "hover": false, "goal": "Apply the promotion code" },
-					"status": "ok"
-				}
-			]
-		},
-		{
-			"id": "verify total",
-			"status": "passed",
-			"category": "app",
-			"reason": "met-expectation",
-			"actual": "The payment step displays the discounted order total of $30.",
-			"turns": 2,
-			"durationMs": 4000,
-			"usage": { "promptTokens": 1200, "cachedPromptTokens": 400, "completionTokens": 100, "totalTokens": 1300 },
-			"toolCalls": [
-				{
-					"turn": 1,
-					"driverId": "web",
-					"name": "browser_click_or_hover",
-					"arguments": {
-						"ref": "e30",
-						"name": "Continue to payment",
-						"hover": false,
-						"goal": "Verify the discounted total at the payment step"
-					},
-					"status": "ok"
-				}
-			]
-		}
-	],
-	"evidence": { "state": "complete", "references": [] },
-	"diagnostics": []
-}
-```
-
-</details>
-
-## Configuration
-
-Execution is configured through `checkmate.config.json`.
-
-Policies keep execution settings outside test intent. For example, a policy can define limits, evidence retention, and allowed driver tools:
-
-<details>
-<summary>Example Configuration</summary>
-
-```json
-{
-	"schemaVersion": 1,
-	"outputDirectory": ".checkmate/runs",
-	"defaultPolicy": "ci",
-	"secretBindings": {
-		"openai-api-key": {
-			"source": "environment",
-			"name": "CHECKMATE_OPENAI_API_KEY"
-		}
-	},
-	"policies": {
-		"ci": {
-			"modelEgress": {
-				"provider": {
-					"id": "openai",
-					"model": "gpt-5-mini",
-					"apiKeyBinding": "openai-api-key"
-				},
-				"textRedaction": "on",
-				"allowOpaque": false,
-				"maxStepBytes": 1048576,
-				"maxMessageBytes": 262144
-			},
-			"bounds": {
-				"scenarioTimeoutMs": 180000,
-				"stepTimeoutMs": 120000,
-				"turnsPerStep": 20,
-				"requestTimeoutMs": 60000,
-				"maxRetries": 3,
-				"loopMaxRepetitions": 5,
-				"cleanupTimeoutMs": 10000,
-				"budgetTokens": 200000
-			},
-			"evidence": {
-				"retention": "retain-on-failure",
-				"redaction": "on",
-				"allowOpaque": false
-			},
-			"drivers": {
-				"web": {
-					"settings": {
-						"headless": true,
-						"logLevel": "off",
-						"logsAsEvidence": false,
-						"snapshotFilter": false
-					},
-					"tools": { "allowed": ["*"] }
-				}
-			}
-		}
-	},
-	"drivers": {
-		"web": {
-			"package": "@xoxoai/checkmate/driver-web",
-			"secrets": {}
-		}
-	}
-}
-```
-
-</details>
-
-## Drivers
-
-#### Checkmate is designed around domain-specific drivers.
-
-A driver defines how the model interacts with a particular system while the harness keeps scenario execution, policies, results, and evidence consistent.
-
-The included `web` driver is a simple reference implementation. Other domains can be integrated through the same public driver contract.
-
-## Commands
-
-```text
-checkmate run <request.json|->
-checkmate validate <request.json>
-checkmate describe
-checkmate --help
-checkmate --version
-```
-
-`describe` returns the drivers, policies, and contract versions available in the current environment.
-
-## Docs
-
-[Configuration](docs/CONFIGURATION.md)  
-[Evidence](docs/EVIDENCE.md)  
-[Drivers](docs/DRIVERS.md)  
-[CLI](docs/CLI.md)  
-[Dev](docs/DEVELOPMENT.md)
-
-## License
-
-[MIT](LICENSE)

package/bin/checkmate.js

@@ -1,13 +0,0 @@
-#!/usr/bin/env node
-
-import { runCli } from '../dist/cli/main.js'
-
-runCli(process.argv.slice(2)).then(
-	(code) => {
-		process.exitCode = code
-	},
-	(error) => {
-		console.error(error instanceof Error ? error.message : String(error))
-		process.exitCode = 1
-	}
-)

package/dist/ai/client.d.ts

@@ -1,64 +0,0 @@
-import { ChatCompletion, ChatCompletionAssistantMessageParam, ChatCompletionMessageParam } from 'openai/resources/chat/completions';
-import { RuntimeConfig } from '../runtime/config.js';
-import type { ModelEgressPolicyV1 } from '../contracts/types.js';
-import type { RuntimeLogger } from '../logging/types.js';
-import { ToolRegistry } from '../tools/registry.js';
-import { Step } from '../runtime/types.js';
-export type AiClientDependencies = {
-    config: RuntimeConfig;
-    toolRegistry: ToolRegistry;
-    apiKey: string;
-    modelEgress?: ModelEgressPolicyV1;
-    exactSecrets?: Iterable<string>;
-    logger: RuntimeLogger;
-};
-export type AiSendOptions = {
-    step?: Step;
-    signal?: AbortSignal;
-};
-export type AiResponse = {
-    response: ChatCompletion;
-    assistantMessages: ChatCompletionAssistantMessageParam[];
-};
-export declare class AiClient {
-    private client;
-    private readonly config;
-    private readonly toolRegistry;
-    private readonly apiKey;
-    private readonly modelEgress;
-    private readonly exactSecrets;
-    private readonly runtimeLogger;
-    private readonly retryableStatus;
-    private sendsTemperature;
-    readonly temperature: number;
-    constructor({ config, toolRegistry, apiKey, modelEgress, exactSecrets, logger: runtimeLogger, }: AiClientDependencies);
-    send(messages: ChatCompletionMessageParam[], options?: AiSendOptions): Promise<AiResponse>;
-    countHistoryTokens(messages: ChatCompletionMessageParam[]): number;
-    private complete;
-    /**
-     * Whether the provider rejected the request because of the configured temperature.
-     *
-     * Some models — OpenAI's `gpt-5` family and its reasoning models among them — accept only
-     * their own default temperature and answer any other value with a 400. Detecting that from
-     * the response is what keeps the policy temperature working against any OpenAI-compatible
-     * endpoint; a hard-coded list of model families would go stale the week after it was written.
-     */
-    private rejectsTemperature;
-    private openai;
-    private retainAssistantMessages;
-    private sanitizeAssistantMessage;
-    private executeWithRetry;
-    private getStatus;
-    private isRetryable;
-    private getRetryAfterSeconds;
-    private calculateBackoff;
-    private sleep;
-    private enhanceError;
-    private isToolError;
-    private formatStepContext;
-    private formatRecentMessages;
-    private formatError;
-    private formatContentPreview;
-    private countContentChars;
-}
-//# sourceMappingURL=client.d.ts.map

package/dist/ai/client.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/ai/client.ts"],"names":[],"mappings":"AACA,OAAO,EACN,cAAc,EACd,mCAAmC,EAEnC,0BAA0B,EAC1B,MAAM,mCAAmC,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAChE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAA;AAE1C,MAAM,MAAM,oBAAoB,GAAG;IAClC,MAAM,EAAE,aAAa,CAAA;IACrB,YAAY,EAAE,YAAY,CAAA;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,mBAAmB,CAAA;IACjC,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;IAC/B,MAAM,EAAE,aAAa,CAAA;CACrB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,IAAI,CAAC,EAAE,IAAI,CAAA;IACX,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB,CAAA;AAED,MAAM,MAAM,UAAU,GAAG;IACxB,QAAQ,EAAE,cAAc,CAAA;IACxB,iBAAiB,EAAE,mCAAmC,EAAE,CAAA;CACxD,CAAA;AAED,qBAAa,QAAQ;IACpB,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAc;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAiC;IAC7D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAmB;IAChD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA2D;IAC3F,OAAO,CAAC,gBAAgB,CAAO;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;gBAEhB,EACX,MAAM,EACN,YAAY,EACZ,MAAM,EACN,WAAW,EACX,YAAiB,EACjB,MAAM,EAAE,aAAa,GACrB,EAAE,oBAAoB;IAUjB,IAAI,CAAC,QAAQ,EAAE,0BAA0B,EAAE,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,UAAU,CAAC;IAqBpG,kBAAkB,CAAC,QAAQ,EAAE,0BAA0B,EAAE,GAAG,MAAM;YAKpD,QAAQ;IAwBtB;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,MAAM;IAed,OAAO,CAAC,uBAAuB;IAM/B,OAAO,CAAC,wBAAwB;YAgClB,gBAAgB;IAmC9B,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,oBAAoB;IAqB5B,OAAO,CAAC,gBAAgB;IAKxB,OAAO,CAAC,KAAK;IAgBb,OAAO,CAAC,YAAY;IAkBpB,OAAO,CAAC,WAAW;IAqBnB,OAAO,CAAC,iBAAiB;IAIzB,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,WAAW;IA0CnB,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,iBAAiB;CA0BzB"}