@chatcops/cli 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 CoderCops
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,57 @@
-# Temporary Holding Version
+# @chatcops/cli
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+The `chatcops` command: build a knowledge index from your site, search it to check what your bot will find, and test your bot against conversations it has to get right.
+
+```bash
+npx chatcops ingest --source https://example.com/sitemap.xml
+npx chatcops search "how much is the Team plan?"
+```
+
+Both read `chatcops.config.ts` (or `.mts`, `.js`, `.mjs`) in the current folder:
+
+```ts
+import { openaiEmbedder } from '@chatcops/core';
+import { defineConfig } from '@chatcops/cli';
+import { postgresStorage, fromPostgresJs } from '@chatcops/store-postgres';
+import postgres from 'postgres';
+
+export default defineConfig({
+  embedder: openaiEmbedder({ apiKey: process.env.OPENAI_API_KEY! }),
+  storage: postgresStorage({ db: fromPostgresJs(postgres(process.env.DATABASE_URL!)), dimensions: 1024 }),
+  ingest: {
+    sources: [{ type: 'sitemap', url: 'https://example.com/sitemap.xml' }],
+    exclude: ['**/blog/**'],
+  },
+});
+```
+
+`chatcops ingest` options: `--source` (repeatable; a sitemap, a page URL, or a folder with `--base-url`), `--index`, `--dry-run`, `--max-pages`, `--concurrency`, `--prune`, `--force`, `--json`. `chatcops search` options: `--index`, `--top-k`, `--json`. Run `chatcops --help` for details.
+
+`chatcops test` runs eval files (`evals/**/*.chat.yaml` by default) against the bot in the config's `server` section:
+
+```yaml
+version: 1
+name: Pricing
+tests:
+  - name: quotes the Team price
+    turns:
+      - user: How much is the Team plan?
+        expect:
+          - contains: "$19"
+          - not-contains: "$29"
+```
+
+Options: `--filter` (a name, or `tag:<name>`), `--repeat`, `--concurrency`, `--verbose`, `--reporter pretty|json` (repeatable), `--json`. The pretty report groups tests by file and includes run duration, assertion counts, available bot-token usage and total elapsed time. By default, assertion details are shown for failed tests; `--verbose` shows them for every test. `--record` saves every API call the tests make to `__cassettes__/` next to the eval files, and `--replay` answers them from there. Replay is the default when `CI` is set, so CI needs no API keys. `judge` assertions have a model from another provider grade a reply against a rubric (`--judge-model` to choose it), and with `prices` in the config, `--max-cost` stops a run once it has spent that much. It exits 0 when all tests passed, 1 when some failed, 2 when it couldn't run. See [Testing your bot](https://chat.codercops.com/core/testing/).
+
+To inspect failure formatting without calling a live AI provider, build the CLI and run its deterministic fake-provider demo from the repository root:
+
+```bash
+pnpm --filter @chatcops/cli build
+node packages/cli/dist/bin.js test packages/cli/tests/fixtures/reporter-failure.chat.yaml --config packages/cli/tests/fixtures/reporter-failure.config.mjs --live --verbose
+```
+
+The demo intentionally fails an assertion and exits with code 1; that is expected.
+
+Exit codes for ingest and search: 0 when everything worked, 1 when it finished with failed pages or discovery errors, 2 when it couldn't run (bad options, config problems, a different embedder than the index was built with).
+
+TypeScript config files need Node 22.18 or later; on older Node, use `chatcops.config.mjs`.

package/dist/bin.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=bin.d.ts.map

package/dist/bin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":""}

package/dist/bin.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { main } from './main.js';
+process.exitCode = await main(process.argv.slice(2), { stdout: process.stdout, stderr: process.stderr, cwd: process.cwd() });
+//# sourceMappingURL=bin.js.map

package/dist/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC"}

package/dist/config.d.ts

@@ -0,0 +1,53 @@
+import type { AIProvider, ChatCopsStorage, Embedder, KeyValueStore, ProviderConfig, VectorStore } from '@chatcops/core';
+import type { IngestConfig } from '@chatcops/ingest';
+import type { ChatCopsServerConfig } from '@chatcops/server';
+import type { Prices } from './test/cost.js';
+/** `chatcops.config.ts`: what the commands index into and search. */
+export interface ChatCopsCliConfig {
+    /** The embedder the index is built and searched with. Needed by `ingest` and `search`. */
+    embedder?: Embedder;
+    /** Storage with `vectors` and `kv` (the manifest), such as `postgresStorage(...)`. */
+    storage?: ChatCopsStorage;
+    /** Or the vector store and manifest separately. */
+    store?: VectorStore;
+    manifest?: KeyValueStore;
+    /** What `chatcops ingest` reads, unless `--source` says otherwise. */
+    ingest?: Omit<IngestConfig, 'embedder' | 'store' | 'manifest' | 'onEvent' | 'dryRun' | 'force' | 'prune'>;
+    /**
+     * The bot `chatcops test` talks to: the same config you give the server
+     * (provider, system prompt, tools, knowledge). Each test gets its own
+     * in-memory storage, so `storage` here is ignored.
+     */
+    server?: ChatCopsServerConfig;
+    /**
+     * The model that grades `judge` assertions. Defaults to the cheap model of
+     * a provider other than the bot's, with its key from the environment.
+     * `--judge-model` overrides it.
+     */
+    judge?: ProviderConfig | AIProvider;
+    /** US dollars per million tokens, by model id, for costs and `--max-cost`. */
+    prices?: Prices;
+}
+/** Types a config file: `export default defineConfig({ ... })`. */
+export declare function defineConfig(config: ChatCopsCliConfig): ChatCopsCliConfig;
+export declare const CONFIG_FILES: string[];
+export declare class ConfigError extends Error {
+    constructor(message: string);
+}
+export interface LoadedConfig {
+    file: string;
+    config: ChatCopsCliConfig;
+    close(): Promise<void>;
+}
+export interface IndexConfig {
+    embedder: Embedder;
+    store: VectorStore;
+    manifest: KeyValueStore;
+}
+/** What `ingest` and `search` need from the config, or a message saying what's missing. */
+export declare function indexConfig({ file, config }: LoadedConfig): IndexConfig;
+/** What `test` needs from the config. */
+export declare function serverConfig({ file, config }: LoadedConfig): ChatCopsServerConfig;
+/** The config at `path`, or the first `chatcops.config.*` in `cwd`, checked. */
+export declare function loadConfig(cwd: string, path?: string): Promise<LoadedConfig>;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACxH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAE7C,qEAAqE;AACrE,MAAM,WAAW,iBAAiB;IAChC,0FAA0F;IAC1F,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,sFAAsF;IACtF,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,mDAAmD;IACnD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,sEAAsE;IACtE,MAAM,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC,CAAC;IAC1G;;;;OAIG;IACH,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B;;;;OAIG;IACH,KAAK,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC;IACpC,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,mEAAmE;AACnE,wBAAgB,YAAY,CAAC,MAAM,EAAE,iBAAiB,GAAG,iBAAiB,CAEzE;AAED,eAAO,MAAM,YAAY,UAA6F,CAAC;AAEvH,qBAAa,WAAY,SAAQ,KAAK;gBACxB,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,iBAAiB,CAAC;IAC1B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED,2FAA2F;AAC3F,wBAAgB,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,YAAY,GAAG,WAAW,CAOvE;AAED,yCAAyC;AACzC,wBAAgB,YAAY,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,YAAY,GAAG,oBAAoB,CAGjF;AAED,gFAAgF;AAChF,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAsBlF"}

package/dist/config.js

@@ -0,0 +1,55 @@
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+/** Types a config file: `export default defineConfig({ ... })`. */
+export function defineConfig(config) {
+    return config;
+}
+export const CONFIG_FILES = ['chatcops.config.ts', 'chatcops.config.mts', 'chatcops.config.js', 'chatcops.config.mjs'];
+export class ConfigError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ConfigError';
+    }
+}
+/** What `ingest` and `search` need from the config, or a message saying what's missing. */
+export function indexConfig({ file, config }) {
+    if (!config.embedder)
+        throw new ConfigError(`${file} needs an \`embedder\`, such as openaiEmbedder({ apiKey }).`);
+    const store = config.store ?? config.storage?.vectors;
+    const manifest = config.manifest ?? config.storage?.kv;
+    if (!store)
+        throw new ConfigError(`${file} needs a vector store: \`storage\` with vectors, or \`store\`.`);
+    if (!manifest)
+        throw new ConfigError(`${file} needs a manifest: \`storage\` with a key-value store, or \`manifest\`.`);
+    return { embedder: config.embedder, store, manifest };
+}
+/** What `test` needs from the config. */
+export function serverConfig({ file, config }) {
+    if (!config.server)
+        throw new ConfigError(`${file} needs a \`server\` config for chatcops test: the provider, system prompt and tools of your bot.`);
+    return config.server;
+}
+/** The config at `path`, or the first `chatcops.config.*` in `cwd`, checked. */
+export async function loadConfig(cwd, path) {
+    const file = path ? resolve(cwd, path) : CONFIG_FILES.map((name) => resolve(cwd, name)).find((candidate) => existsSync(candidate));
+    if (!file || !existsSync(file)) {
+        throw new ConfigError(path ? `No config file at ${file}.` : `No config file found. Create chatcops.config.ts (or .mjs) in ${cwd}, or pass --config.`);
+    }
+    let config;
+    try {
+        const module = (await import(pathToFileURL(file).href));
+        config = module.default;
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === 'ERR_UNKNOWN_FILE_EXTENSION' && /\.m?ts$/.test(file)) {
+            throw new ConfigError(`This Node.js can't load TypeScript config files. Use Node 22.18 or later, or rename ${file} to chatcops.config.mjs.`);
+        }
+        throw new ConfigError(`Couldn't load ${file}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (!config || typeof config !== 'object')
+        throw new ConfigError(`${file} has no default export. Export the config with \`export default defineConfig({ ... })\`.`);
+    return { file, config, close: async () => void (await config.storage?.close?.()) };
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAiCzC,mEAAmE;AACnE,MAAM,UAAU,YAAY,CAAC,MAAyB;IACpD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,oBAAoB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,qBAAqB,CAAC,CAAC;AAEvH,MAAM,OAAO,WAAY,SAAQ,KAAK;IACpC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAcD,2FAA2F;AAC3F,MAAM,UAAU,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAgB;IACxD,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,6DAA6D,CAAC,CAAC;IAClH,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC;IACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;IACvD,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,gEAAgE,CAAC,CAAC;IAC3G,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,yEAAyE,CAAC,CAAC;IACvH,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACxD,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,YAAY,CAAC,EAAE,IAAI,EAAE,MAAM,EAAgB;IACzD,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,kGAAkG,CAAC,CAAC;IACrJ,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,IAAa;IACzD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IACnI,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,WAAW,CACnB,IAAI,CAAC,CAAC,CAAC,qBAAqB,IAAI,GAAG,CAAC,CAAC,CAAC,gEAAgE,GAAG,qBAAqB,CAC/H,CAAC;IACJ,CAAC;IAED,IAAI,MAAqC,CAAC;IAC1C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAoC,CAAC;QAC3F,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,GAAI,KAA2B,CAAC,IAAI,CAAC;QAC/C,IAAI,IAAI,KAAK,4BAA4B,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,WAAW,CAAC,uFAAuF,IAAI,0BAA0B,CAAC,CAAC;QAC/I,CAAC;QACD,MAAM,IAAI,WAAW,CAAC,iBAAiB,IAAI,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,0FAA0F,CAAC,CAAC;IACpK,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC;AACrF,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export { defineConfig, loadConfig, ConfigError, CONFIG_FILES, type ChatCopsCliConfig, type LoadedConfig } from './config.js';
+export { runFixtures, type RunOptions } from './test/run.js';
+export { loadFixture, findFixtures } from './test/load.js';
+export { ASSERTIONS } from './test/assertions/index.js';
+export { REPORTERS, type Reporter, type ReporterFactory, type ReporterOutput } from './test/reporters/index.js';
+export type { Assertion, AssertionCheck, AssertionResult, Fixture, TestCase, TestResult, TestRunResult, Turn, TurnResult } from './test/types.js';
+export { main, toSource, EXIT, type Io } from './main.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,iBAAiB,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAC7H,OAAO,EAAE,WAAW,EAAE,KAAK,UAAU,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChH,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,eAAe,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClJ,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,WAAW,CAAC"}

package/dist/index.js

@@ -0,0 +1,7 @@
+export { defineConfig, loadConfig, ConfigError, CONFIG_FILES } from './config.js';
+export { runFixtures } from './test/run.js';
+export { loadFixture, findFixtures } from './test/load.js';
+export { ASSERTIONS } from './test/assertions/index.js';
+export { REPORTERS } from './test/reporters/index.js';
+export { main, toSource, EXIT } from './main.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAA6C,MAAM,aAAa,CAAC;AAC7H,OAAO,EAAE,WAAW,EAAmB,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,SAAS,EAA4D,MAAM,2BAA2B,CAAC;AAEhH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAW,MAAM,WAAW,CAAC"}

package/dist/main.d.ts

@@ -0,0 +1,53 @@
+import { type IngestSource } from '@chatcops/ingest';
+export interface Io {
+    stdout: {
+        write(text: string): unknown;
+    };
+    stderr: {
+        write(text: string): unknown;
+    };
+    cwd: string;
+    /** Defaults to `process.env`. `chatcops test` reads `CI`. */
+    env?: Record<string, string | undefined>;
+}
+/** Exit codes: 0 done, 1 finished with problems (failed pages, discovery errors), 2 couldn't run. */
+export declare const EXIT: {
+    readonly ok: 0;
+    readonly problems: 1;
+    readonly unusable: 2;
+};
+/** Runs the command line and returns the exit code. */
+export declare function main(argv: string[], io: Io): Promise<number>;
+declare function parse(argv: string[]): {
+    values: {
+        config?: string | undefined;
+        source?: string[] | undefined;
+        'base-url'?: string | undefined;
+        index?: string | undefined;
+        'dry-run'?: boolean | undefined;
+        'max-pages'?: string | undefined;
+        concurrency?: string | undefined;
+        prune?: boolean | undefined;
+        force?: boolean | undefined;
+        json?: boolean | undefined;
+        'top-k'?: string | undefined;
+        filter?: string | undefined;
+        repeat?: string | undefined;
+        verbose?: boolean | undefined;
+        reporter?: string[] | undefined;
+        record?: boolean | undefined;
+        live?: boolean | undefined;
+        'judge-model'?: string | undefined;
+        'max-cost'?: string | undefined;
+        replay?: boolean | undefined;
+        help?: boolean | undefined;
+        version?: boolean | undefined;
+    };
+    positionals: string[];
+};
+export type Values = ReturnType<typeof parse>['values'];
+/** A `--source` value: a sitemap URL, a page URL, or a folder (with `--base-url`). */
+export declare function toSource(value: string, baseUrl: string | undefined, cwd: string): IngestSource;
+export declare function wholeNumber(value: string | undefined, flag: string): number | undefined;
+export {};
+//# sourceMappingURL=main.d.ts.map

package/dist/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAIA,OAAO,EAA0C,KAAK,YAAY,EAAoB,MAAM,kBAAkB,CAAC;AAK/G,MAAM,WAAW,EAAE;IACjB,MAAM,EAAE;QAAE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IACzC,MAAM,EAAE;QAAE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,6DAA6D;IAC7D,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;CAC1C;AAED,qGAAqG;AACrG,eAAO,MAAM,IAAI;;;;CAA+C,CAAC;AAsDjE,uDAAuD;AACvD,wBAAsB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CA2ClE;AAED,iBAAS,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;EA6B5B;AAED,MAAM,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;AA6DxD,sFAAsF;AACtF,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAM9F;AA2BD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAIvF"}

package/dist/main.js

@@ -0,0 +1,247 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { parseArgs } from 'node:util';
+import { hybridRetriever, vectorRetriever } from '@chatcops/core';
+import { ingest, manifestKeys } from '@chatcops/ingest';
+import { ConfigError, indexConfig, loadConfig, serverConfig } from './config.js';
+import { Cassettes } from './test/cassette.js';
+import { cassetteMode, runTests } from './test/command.js';
+/** Exit codes: 0 done, 1 finished with problems (failed pages, discovery errors), 2 couldn't run. */
+export const EXIT = { ok: 0, problems: 1, unusable: 2 };
+const VERSION = (() => {
+    try {
+        return JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
+    }
+    catch {
+        return '0.0.0';
+    }
+})();
+const HELP = `chatcops ${VERSION}
+
+Usage:
+  chatcops ingest [options]          Build or update the knowledge index from your site
+  chatcops search "<question>" [options]   Show what retrieval finds for a question
+  chatcops test [files...] [options]        Run eval files against your bot
+
+Ingest options:
+  --source <url|path>   A sitemap (.xml, .xml.gz), a page URL, or a folder of .md/.mdx/.html
+                        files (with --base-url). Repeatable. Replaces the config's sources.
+  --base-url <url>      Where a folder source is published
+  --index <name>        Index name (default "default")
+  --dry-run             Discover, extract and chunk; write and embed nothing
+  --max-pages <n>       Index at most n pages
+  --concurrency <n>     Pages fetched at once (default 4)
+  --prune               Remove pages that are gone (never after a discovery error)
+  --force               Re-embed every page, even unchanged ones
+  --json                One JSON object per line, for CI
+
+Search options:
+  --index <name>        Index name (default "default")
+  --top-k <n>           Chunks to show (default 5)
+  --json                JSON output
+
+Test options:
+  [files...]            Globs of eval files (default "evals/**/*.chat.yaml")
+  --filter <text>       Only tests whose name contains text, or tag:<name>
+  --repeat <n>          Run each test n times
+  --concurrency <n>     Tests run at once (default 4)
+  --verbose             Show assertion details for every test
+  --reporter <name>     pretty (default) or json. Repeatable.
+  --json                Same as --reporter json
+  --record              Call the APIs and save what they answer to __cassettes__
+  --replay              Answer every API call from __cassettes__ (the default when CI is set)
+  --live                Call the APIs, even when CI is set
+  --judge-model <p:m>   The judge for judge assertions, as provider:model (gemini, openai, claude)
+  --max-cost <usd>      Stop starting turns once the run has spent this (needs prices in the config)
+
+All commands:
+  --config <file>       Config file (default: chatcops.config.ts, .mts, .js or .mjs here)
+  -h, --help            Show this help
+  -v, --version         Show the version
+`;
+/** Runs the command line and returns the exit code. */
+export async function main(argv, io) {
+    let parsed;
+    try {
+        parsed = parse(argv);
+    }
+    catch (error) {
+        io.stderr.write(`${message(error)}\n\nRun chatcops --help for usage.\n`);
+        return EXIT.unusable;
+    }
+    const { values, positionals } = parsed;
+    if (values.version) {
+        io.stdout.write(`${VERSION}\n`);
+        return EXIT.ok;
+    }
+    const [command, ...rest] = positionals;
+    if (values.help || !command) {
+        io.stdout.write(HELP);
+        return command || values.help ? EXIT.ok : EXIT.unusable;
+    }
+    try {
+        if (command === 'ingest')
+            return await runIngest(values, io);
+        if (command === 'search')
+            return await runSearch(rest.join(' '), values, io);
+        if (command === 'test') {
+            const cassettes = new Cassettes(cassetteMode(values, io.env ?? process.env));
+            // Before the config loads, so the providers and embedders it creates use it.
+            const uninstall = cassettes.mode === 'live' ? () => { } : cassettes.install();
+            try {
+                const loaded = await loadConfig(io.cwd, values.config);
+                try {
+                    return await runTests(serverConfig(loaded), rest, values, io, cassettes, loaded.config);
+                }
+                finally {
+                    await loaded.close();
+                }
+            }
+            finally {
+                uninstall();
+            }
+        }
+        io.stderr.write(`Unknown command "${command}". Run chatcops --help for usage.\n`);
+        return EXIT.unusable;
+    }
+    catch (error) {
+        io.stderr.write(`${error instanceof ConfigError ? '' : 'Error: '}${message(error)}\n`);
+        return EXIT.unusable;
+    }
+}
+function parse(argv) {
+    return parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            config: { type: 'string' },
+            source: { type: 'string', multiple: true },
+            'base-url': { type: 'string' },
+            index: { type: 'string' },
+            'dry-run': { type: 'boolean' },
+            'max-pages': { type: 'string' },
+            concurrency: { type: 'string' },
+            prune: { type: 'boolean' },
+            force: { type: 'boolean' },
+            json: { type: 'boolean' },
+            'top-k': { type: 'string' },
+            filter: { type: 'string' },
+            repeat: { type: 'string' },
+            verbose: { type: 'boolean' },
+            reporter: { type: 'string', multiple: true },
+            record: { type: 'boolean' },
+            live: { type: 'boolean' },
+            'judge-model': { type: 'string' },
+            'max-cost': { type: 'string' },
+            replay: { type: 'boolean' },
+            help: { type: 'boolean', short: 'h' },
+            version: { type: 'boolean', short: 'v' },
+        },
+    });
+}
+async function runIngest(values, io) {
+    const maxPages = wholeNumber(values['max-pages'], '--max-pages');
+    const concurrency = wholeNumber(values.concurrency, '--concurrency');
+    const loaded = await loadConfig(io.cwd, values.config);
+    try {
+        const target = indexConfig(loaded);
+        const sources = values.source?.length ? values.source.map((source) => toSource(source, values['base-url'], io.cwd)) : loaded.config.ingest?.sources;
+        if (!sources?.length)
+            throw new ConfigError('Nothing to ingest: pass --source, or set ingest.sources in the config.');
+        const json = !!values.json;
+        const stats = await ingest({
+            ...loaded.config.ingest,
+            sources,
+            ...(values.index ? { index: values.index } : {}),
+            ...(maxPages !== undefined ? { maxPages } : {}),
+            ...(concurrency !== undefined ? { concurrency } : {}),
+            embedder: target.embedder,
+            store: target.store,
+            manifest: target.manifest,
+            dryRun: !!values['dry-run'],
+            force: !!values.force,
+            prune: !!values.prune,
+            onEvent: (event) => (json ? io.stdout.write(`${JSON.stringify(event)}\n`) : io.stderr.write(describe(event))),
+        });
+        if (json)
+            io.stdout.write(`${JSON.stringify({ type: 'done', dryRun: !!values['dry-run'], stats })}\n`);
+        else
+            io.stdout.write(summary(stats, !!values['dry-run']));
+        return stats.failed || stats.errors.length ? EXIT.problems : EXIT.ok;
+    }
+    finally {
+        await loaded.close();
+    }
+}
+async function runSearch(question, values, io) {
+    if (!question.trim())
+        throw new ConfigError('What should I search for? chatcops search "how much is the Team plan?"');
+    const topK = wholeNumber(values['top-k'], '--top-k') ?? 5;
+    const loaded = await loadConfig(io.cwd, values.config);
+    try {
+        const index = values.index ?? 'default';
+        const { embedder, store, manifest } = indexConfig(loaded);
+        const meta = await manifest.get(manifestKeys(index).meta);
+        if (meta && meta.embedder !== embedder.id) {
+            throw new ConfigError(`Index "${index}" was built with ${meta.embedder}, but the config's embedder is ${embedder.id}. Search it with the embedder that built it.`);
+        }
+        if (!meta)
+            io.stderr.write(`Index "${index}" hasn't been built yet. Run chatcops ingest first.\n`);
+        const options = { store, embedder };
+        const retriever = store.keywordQuery ? hybridRetriever(options) : vectorRetriever(options);
+        const chunks = await retriever.retrieve({ text: question, topK });
+        if (values.json)
+            io.stdout.write(`${JSON.stringify({ question, index, chunks })}\n`);
+        else
+            io.stdout.write(chunks.length ? chunks.map(formatChunk).join('\n') : 'Nothing found.\n');
+        return EXIT.ok;
+    }
+    finally {
+        await loaded.close();
+    }
+}
+/** A `--source` value: a sitemap URL, a page URL, or a folder (with `--base-url`). */
+export function toSource(value, baseUrl, cwd) {
+    if (/^https?:\/\//i.test(value)) {
+        return /\.xml(\.gz)?(\?.*)?$/i.test(value) ? { type: 'sitemap', url: value } : { type: 'urls', urls: [value] };
+    }
+    if (!baseUrl)
+        throw new ConfigError(`--source ${value} is a folder, so it needs --base-url (where those pages are published).`);
+    return { type: 'directory', path: resolve(cwd, value), baseUrl };
+}
+function describe(event) {
+    if (event.type === 'discovered') {
+        return `Found ${event.pages} page${event.pages === 1 ? '' : 's'}.${event.errors.map((error) => `\n  ! ${error}`).join('')}\n`;
+    }
+    if (event.type === 'pruned')
+        return `  removed    ${event.url}\n`;
+    const detail = event.chunks !== undefined ? ` (${event.chunks} chunk${event.chunks === 1 ? '' : 's'})` : event.reason ? ` (${event.reason})` : '';
+    return `  ${event.status.padEnd(10)} ${event.url}${detail}\n`;
+}
+function summary(stats, dryRun) {
+    const lines = [
+        `${dryRun ? 'Would index' : 'Indexed'} ${stats.indexed} of ${stats.discovered} pages: ${stats.chunks} chunks, ${stats.embedded} embedded.`,
+        `${stats.unchanged} unchanged, ${stats.skipped} skipped, ${stats.failed} failed, ${stats.pruned} ${dryRun ? 'to remove' : 'removed'}.`,
+    ];
+    if (stats.pruneSkipped)
+        lines.push(`Didn't remove anything: ${stats.pruneSkipped}.`);
+    if (stats.errors.length)
+        lines.push(...stats.errors.map((error) => `! ${error}`));
+    return `${lines.join('\n')}\n`;
+}
+function formatChunk(chunk, i) {
+    const title = [chunk.source.title, chunk.source.section].filter(Boolean).join(' > ') || chunk.source.url;
+    const text = chunk.text.replace(/\s+/g, ' ').trim();
+    return `${i + 1}. ${title}  (${chunk.score.toFixed(4)})\n   ${chunk.source.url}\n   ${text.length > 240 ? `${text.slice(0, 239)}…` : text}\n`;
+}
+export function wholeNumber(value, flag) {
+    if (value === undefined)
+        return undefined;
+    if (!/^\d+$/.test(value))
+        throw new ConfigError(`${flag} takes a whole number, not "${value}".`);
+    return Number(value);
+}
+function message(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=main.js.map

package/dist/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,eAAe,EAAuB,MAAM,gBAAgB,CAAC;AACvF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAyD,MAAM,kBAAkB,CAAC;AAC/G,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAU3D,qGAAqG;AACrG,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAW,CAAC;AAEjE,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE;IACpB,IAAI,CAAC;QACH,OAAQ,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAyB,CAAC,OAAO,CAAC;IACxH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;AACH,CAAC,CAAC,EAAE,CAAC;AAEL,MAAM,IAAI,GAAG,YAAY,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0C/B,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAc,EAAE,EAAM;IAC/C,IAAI,MAAgC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IACvC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC,EAAE,CAAC;IACjB,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,WAAW,CAAC;IACvC,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC5B,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;IAC1D,CAAC;IAED,IAAI,CAAC;QACH,IAAI,OAAO,KAAK,QAAQ;YAAE,OAAO,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,KAAK,QAAQ;YAAE,OAAO,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7E,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7E,6EAA6E;YAC7E,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAC7E,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACvD,IAAI,CAAC;oBACH,OAAO,MAAM,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1F,CAAC;wBAAS,CAAC;oBACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,SAAS,EAAE,CAAC;YACd,CAAC;QACH,CAAC;QACD,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,OAAO,qCAAqC,CAAC,CAAC;QAClF,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,YAAY,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,IAAc;IAC3B,OAAO,SAAS,CAAC;QACf,IAAI,EAAE,IAAI;QACV,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;YAC1C,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC9B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC/B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC/B,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC1B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACzB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC5B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;YAC5C,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC3B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACzB,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC3B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;YACrC,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;SACzC;KACF,CAAC,CAAC;AACL,CAAC;AAID,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,EAAM;IAC7C,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,aAAa,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;QACpJ,IAAI,CAAC,OAAO,EAAE,MAAM;YAAE,MAAM,IAAI,WAAW,CAAC,wEAAwE,CAAC,CAAC;QAEtH,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC;YACzB,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM;YACvB,OAAO;YACP,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;YAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;YACrB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;YACrB,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;SAC9G,CAAC,CAAC;QAEH,IAAI,IAAI;YAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC;;YAClG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC1D,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;IACvE,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,MAAc,EAAE,EAAM;IAC/D,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,WAAW,CAAC,wEAAwE,CAAC,CAAC;IACtH,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;QACxC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAuB,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QAChF,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,IAAI,WAAW,CACnB,UAAU,KAAK,oBAAoB,IAAI,CAAC,QAAQ,kCAAkC,QAAQ,CAAC,EAAE,8CAA8C,CAC5I,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,IAAI;YAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,uDAAuD,CAAC,CAAC;QACnG,MAAM,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAElE,IAAI,MAAM,CAAC,IAAI;YAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;;YAChF,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;QAC9F,OAAO,IAAI,CAAC,EAAE,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,QAAQ,CAAC,KAAa,EAAE,OAA2B,EAAE,GAAW;IAC9E,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;IACjH,CAAC;IACD,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,WAAW,CAAC,YAAY,KAAK,yEAAyE,CAAC,CAAC;IAChI,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAkB;IAClC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QAChC,OAAO,SAAS,KAAK,CAAC,KAAK,QAAQ,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;IAChI,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,gBAAgB,KAAK,CAAC,GAAG,IAAI,CAAC;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,SAAS,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAClJ,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC;AAChE,CAAC;AAED,SAAS,OAAO,CAAC,KAAkB,EAAE,MAAe;IAClD,MAAM,KAAK,GAAG;QACZ,GAAG,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,OAAO,KAAK,CAAC,UAAU,WAAW,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,QAAQ,YAAY;QAC1I,GAAG,KAAK,CAAC,SAAS,eAAe,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG;KACvI,CAAC;IACF,IAAI,KAAK,CAAC,YAAY;QAAE,KAAK,CAAC,IAAI,CAAC,2BAA2B,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC;IACrF,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC;IAClF,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,KAAqB,EAAE,CAAS;IACnD,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC;IACzG,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACpD,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,MAAM,CAAC,GAAG,QAAQ,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AAChJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAyB,EAAE,IAAY;IACjE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,WAAW,CAAC,GAAG,IAAI,+BAA+B,KAAK,IAAI,CAAC,CAAC;IACjG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}

package/dist/test/assertions/index.d.ts

@@ -0,0 +1,18 @@
+import type { Assertion, AssertionCheck, AssertionContext, AssertionResult, TurnResult } from '../types.js';
+/**
+ * The checks an eval file can use, by key. Every key also works with a
+ * `not-` prefix, which passes when the check fails (`not-contains`).
+ *
+ * To add one, write it in its own file in this folder and add it here.
+ */
+export declare const ASSERTIONS: Map<string, AssertionCheck>;
+/** The base name of an assertion and whether it's negated: `not-contains` is `contains`, negated. */
+export declare function assertionName(assertion: Assertion): {
+    key: string;
+    name: string;
+    negated: boolean;
+};
+/** Assertion keys a fixture uses that no check exists for. */
+export declare function unknownAssertions(assertions: Assertion[]): string[];
+export declare function check(assertion: Assertion, turn: TurnResult, conversation: TurnResult[], context?: AssertionContext): Promise<AssertionResult>;
+//# sourceMappingURL=index.d.ts.map

package/dist/test/assertions/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,gBAAgB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQ5G;;;;;GAKG;AACH,eAAO,MAAM,UAAU,6BAWrB,CAAC;AAEH,qGAAqG;AACrG,wBAAgB,aAAa,CAAC,SAAS,EAAE,SAAS,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CAInG;AAED,8DAA8D;AAC9D,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAEnE;AAED,wBAAsB,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,EAAE,OAAO,GAAE,gBAAkC,GAAG,OAAO,CAAC,eAAe,CAAC,CAWrK"}