@chatcops/cli 0.0.0-stage → 0.1.0

package/dist/test/types.d.ts

@@ -0,0 +1,116 @@
+import type { ToolResult } from '@chatcops/core';
+/** One assertion, as written in a fixture: `{ contains: "$19" }`. */
+export type Assertion = Record<string, unknown>;
+export interface PageContext {
+    url: string;
+    title: string;
+    description?: string;
+    contentSnippet?: string;
+}
+export interface Turn {
+    user: string;
+    expect?: Assertion[];
+}
+export interface TestCase {
+    name: string;
+    tags?: string[];
+    locale?: string;
+    pageContext?: PageContext;
+    userData?: Record<string, string>;
+    /** Server tools answered with a fixed result instead of running. */
+    mocks?: {
+        tools?: Record<string, ToolResult>;
+    };
+    turns: Turn[];
+    /** Run the test this many times. Default 1. */
+    repeat?: number;
+    /** Share of repeats that must pass. Default 1 (all of them). */
+    passRate?: number;
+}
+export interface Fixture {
+    version: 1;
+    name: string;
+    /** Test defaults; a test's own values win, and `mocks.tools` are merged. */
+    defaults?: Partial<Omit<TestCase, 'name' | 'turns'>>;
+    tests: TestCase[];
+}
+/** What a turn produced, as the assertions see it. */
+export interface TurnResult {
+    user: string;
+    /** The reply's text, streamed parts joined. */
+    text: string;
+    toolCalls: Array<{
+        name: string;
+        input: unknown;
+    }>;
+    sources: Array<{
+        url: string;
+        title?: string;
+    }>;
+    leads: Array<Record<string, unknown>>;
+    finishReason?: string;
+    usage?: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    /** From the request to the `finish` part. */
+    latencyMs: number;
+    /** An `error` part's code, when the reply failed. */
+    error?: string;
+    /** The bot's model id, when the provider says. */
+    model?: string;
+    /** What the turn cost in US dollars, from `prices` in the config. Unset when the model has no price. */
+    costUsd?: number;
+}
+export interface AssertionResult {
+    assertion: Assertion;
+    pass: boolean;
+    score?: number;
+    reason?: string;
+    /** What checking cost (a judge's call), in US dollars. */
+    costUsd?: number;
+}
+/** What an assertion can use besides the turn. */
+export interface AssertionContext {
+    /** The eval file, on disk. */
+    fixture: string;
+    /** The judge, when the run uses `judge` assertions. */
+    judge?: Judge;
+}
+/** Grades a reply against a rubric with a model. */
+export interface Judge {
+    /** `provider:model`, printed with every run that uses it. */
+    id: string;
+    model?: string;
+    grade(rubric: string, conversation: TurnResult[]): Promise<{
+        score: number;
+        reason: string;
+        costUsd?: number;
+    }>;
+}
+type CheckResult = AssertionResult | Omit<AssertionResult, 'assertion'>;
+/** Checks one turn. Registered by the assertion's key (`contains`). */
+export type AssertionCheck = (value: unknown, turn: TurnResult, conversation: TurnResult[], context: AssertionContext) => Promise<CheckResult> | CheckResult;
+export interface TestRunResult {
+    fixture: string;
+    test: string;
+    /** 1-based, when the test repeats. */
+    run: number;
+    pass: boolean;
+    turns: Array<{
+        turn: TurnResult;
+        results: AssertionResult[];
+    }>;
+    /** When the run couldn't finish: config or provider problems. */
+    error?: string;
+}
+export interface TestResult {
+    fixture: string;
+    test: string;
+    runs: TestRunResult[];
+    passRate: number;
+    pass: boolean;
+    durationMs?: number;
+}
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/test/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/test/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,qEAAqE;AACrE,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEhD,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,oEAAoE;IACpE,KAAK,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;KAAE,CAAC;IAC/C,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IACrD,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAED,sDAAsD;AACtD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACnD,OAAO,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wGAAwG;IACxG,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,oDAAoD;AACpD,MAAM,WAAW,KAAK;IACpB,6DAA6D;IAC7D,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjH;AAED,KAAK,WAAW,GAAG,eAAe,GAAG,IAAI,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;AAExE,uEAAuE;AACvE,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC;AAE7J,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC,CAAC;IAC/D,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,aAAa,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}

package/dist/test/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/test/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/test/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -1,6 +1,63 @@
 {
   "name": "@chatcops/cli",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.0",
+  "description": "The chatcops command: build a knowledge index from your site and search it",
+  "keywords": [
+    "chatcops",
+    "cli",
+    "rag",
+    "ingest"
+  ],
+  "homepage": "https://chat.codercops.com",
+  "bugs": "https://github.com/codercops/chatcops/issues",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "chatcops": "dist/bin.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "sideEffects": false,
+  "dependencies": {
+    "yaml": "^2.6.0",
+    "zod": "^3.24.0",
+    "@chatcops/ingest": "^0.1.0",
+    "@chatcops/server": "^0.7.0"
+  },
+  "peerDependencies": {
+    "@chatcops/core": "^0.6.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/codercops/chatcops.git",
+    "directory": "packages/cli"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=22.13"
+  },
+  "devDependencies": {
+    "@types/node": "^22",
+    "typescript": "^5.7.0",
+    "vitest": "^4.1.11",
+    "@chatcops/core": "^0.6.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc -p tsconfig.test.json --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clean": "rm -rf dist *.tsbuildinfo"
+  }
 }