@plune-ai/cli 0.2.0

package/dist/index.d.cts

@@ -0,0 +1,215 @@
+interface Summary {
+    total: number;
+    passed: number;
+    failed: number;
+    errored: number;
+    cost_usd: number;
+    duration_ms: number;
+}
+interface AssertionResultRecord {
+    type: string;
+    passed: boolean;
+    score?: number;
+    reason?: string;
+}
+interface Usage {
+    input_tokens: number;
+    output_tokens: number;
+    cost_usd: number;
+}
+interface RunError {
+    code: string;
+    message: string;
+}
+interface RowResult {
+    vars: Record<string, unknown>;
+    output: string | null;
+    cached: boolean;
+    usage?: Usage;
+    latency_ms?: number;
+    error?: RunError;
+    assertions: AssertionResultRecord[];
+}
+interface EvalResult {
+    id: string;
+    tags: string[];
+    rows: RowResult[];
+    passed: boolean;
+}
+interface RunResult {
+    schema: 1;
+    plune_version: string;
+    started_at: string;
+    finished_at: string;
+    config_hash: string;
+    summary: Summary;
+    evals: EvalResult[];
+}
+
+interface ProviderConfig {
+    type: 'anthropic' | 'openai' | 'openrouter';
+    model: string;
+    temperature?: number;
+    max_tokens?: number;
+    concurrency?: number;
+    timeout?: number;
+    max_retries?: number;
+}
+interface ModelPrice {
+    input_per_1k_usd: number;
+    output_per_1k_usd: number;
+}
+type PricingMap = Record<string, ModelPrice>;
+interface DatasetRow {
+    vars: Record<string, string | number | boolean>;
+    expected?: string;
+}
+type DatasetRef = string | {
+    examples: DatasetRow[];
+};
+interface ExactMatchAssertion {
+    type: 'exact-match';
+    value: string;
+    trim?: boolean;
+    ignore_case?: boolean;
+}
+interface ContainsAssertion {
+    type: 'contains';
+    value: string;
+    ignore_case?: boolean;
+}
+interface ContainsAnyAssertion {
+    type: 'contains-any';
+    values: string[];
+    ignore_case?: boolean;
+}
+interface ContainsAllAssertion {
+    type: 'contains-all';
+    values: string[];
+    ignore_case?: boolean;
+}
+interface JsonSchemaAssertion {
+    type: 'json-schema';
+    schema: object;
+    extract?: 'auto' | 'strict';
+}
+interface LlmJudgeAssertion {
+    type: 'llm-judge';
+    criteria: string;
+    provider?: Partial<ProviderConfig>;
+    pass_threshold?: number;
+}
+interface SemanticSimilarityAssertion {
+    type: 'semantic-similarity';
+    reference: string;
+    threshold?: number;
+}
+interface FaithfulnessAssertion {
+    type: 'faithfulness';
+    context: string;
+    threshold?: number;
+}
+interface AnswerRelevanceAssertion {
+    type: 'answer-relevance';
+    question: string;
+    threshold?: number;
+}
+interface ContextPrecisionAssertion {
+    type: 'context-precision';
+    context: string;
+    question: string;
+    threshold?: number;
+}
+type AssertionConfig = ExactMatchAssertion | ContainsAssertion | ContainsAnyAssertion | ContainsAllAssertion | JsonSchemaAssertion | LlmJudgeAssertion | SemanticSimilarityAssertion | FaithfulnessAssertion | AnswerRelevanceAssertion | ContextPrecisionAssertion;
+interface EvalConfig {
+    id: string;
+    description?: string;
+    tags?: string[];
+    provider?: Partial<ProviderConfig>;
+    prompt?: string;
+    prompt_file?: string;
+    dataset: DatasetRef;
+    assertions: AssertionConfig[];
+}
+interface Config {
+    version: 1;
+    provider: ProviderConfig;
+    defaults?: {
+        assertions?: AssertionConfig[];
+    };
+    pricing?: PricingMap;
+    evals: EvalConfig[];
+}
+
+interface CompletionRequest {
+    provider: string;
+    model: string;
+    temperature: number;
+    max_tokens: number;
+    prompt_resolved: string;
+}
+interface CompletionResponse {
+    output: string;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+    /**
+     * Actual USD cost the provider reported for this call, if it reports one (e.g. OpenRouter via
+     * `usage.include`). Absent for providers that only return token counts — the cost is then
+     * estimated downstream (ADR-PRC02). Additive + optional: existing providers omit it.
+     */
+    cost_usd?: number;
+}
+interface CostEstimate {
+    cost_usd: number;
+}
+interface Provider<_TConfig extends ProviderConfig = ProviderConfig> {
+    complete(req: CompletionRequest): Promise<CompletionResponse>;
+    /**
+     * Resolve the USD cost for a call's usage. `reportedCostUsd` (optional) is the provider's actual
+     * reported cost from `complete()` — when passed it is preferred over the table estimate, unless a
+     * config `pricing` override exists (precedence in `resolveCost`, ADR-PRC01). Omitting it (dry-run,
+     * judge calls) yields a pure token-based estimate, as before — additive + backward-compatible.
+     */
+    estimateCost(usage: {
+        input_tokens: number;
+        output_tokens: number;
+    }, reportedCostUsd?: number): CostEstimate;
+}
+
+interface Embedder {
+    /** Embed a batch of texts into fixed-dimension vectors (one per input, in order). */
+    embed(texts: string[]): Promise<Float32Array[]>;
+}
+
+interface Cache {
+    /** Return the cached completion, or undefined on a miss. `maxAgeMs` (optional) expires old entries. */
+    get(key: string, opts?: {
+        maxAgeMs?: number;
+    }): CompletionResponse | undefined;
+    set(key: string, value: CompletionResponse): void;
+    clear(): void;
+    close(): void;
+}
+
+interface RunDeps {
+    resolveProvider(cfg: ProviderConfig): Provider;
+    embedder: Embedder;
+    cache: Cache;
+    now: () => number;
+    loadDataset(ref: DatasetRef, baseDir: string): DatasetRow[];
+    baseDir: string;
+}
+
+interface RunOptions {
+    dryRun: boolean;
+    configPath?: string;
+    only?: string[];
+    concurrency?: number;
+    noCache?: boolean;
+    bail?: boolean;
+}
+declare function handleRun(options: RunOptions, depsFactory?: (config: Config, baseDir: string) => RunDeps): Promise<RunResult>;
+
+export { type AssertionResultRecord, type Config, type EvalResult, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type Usage, handleRun as run };

package/dist/index.d.ts

@@ -0,0 +1,215 @@
+interface Summary {
+    total: number;
+    passed: number;
+    failed: number;
+    errored: number;
+    cost_usd: number;
+    duration_ms: number;
+}
+interface AssertionResultRecord {
+    type: string;
+    passed: boolean;
+    score?: number;
+    reason?: string;
+}
+interface Usage {
+    input_tokens: number;
+    output_tokens: number;
+    cost_usd: number;
+}
+interface RunError {
+    code: string;
+    message: string;
+}
+interface RowResult {
+    vars: Record<string, unknown>;
+    output: string | null;
+    cached: boolean;
+    usage?: Usage;
+    latency_ms?: number;
+    error?: RunError;
+    assertions: AssertionResultRecord[];
+}
+interface EvalResult {
+    id: string;
+    tags: string[];
+    rows: RowResult[];
+    passed: boolean;
+}
+interface RunResult {
+    schema: 1;
+    plune_version: string;
+    started_at: string;
+    finished_at: string;
+    config_hash: string;
+    summary: Summary;
+    evals: EvalResult[];
+}
+
+interface ProviderConfig {
+    type: 'anthropic' | 'openai' | 'openrouter';
+    model: string;
+    temperature?: number;
+    max_tokens?: number;
+    concurrency?: number;
+    timeout?: number;
+    max_retries?: number;
+}
+interface ModelPrice {
+    input_per_1k_usd: number;
+    output_per_1k_usd: number;
+}
+type PricingMap = Record<string, ModelPrice>;
+interface DatasetRow {
+    vars: Record<string, string | number | boolean>;
+    expected?: string;
+}
+type DatasetRef = string | {
+    examples: DatasetRow[];
+};
+interface ExactMatchAssertion {
+    type: 'exact-match';
+    value: string;
+    trim?: boolean;
+    ignore_case?: boolean;
+}
+interface ContainsAssertion {
+    type: 'contains';
+    value: string;
+    ignore_case?: boolean;
+}
+interface ContainsAnyAssertion {
+    type: 'contains-any';
+    values: string[];
+    ignore_case?: boolean;
+}
+interface ContainsAllAssertion {
+    type: 'contains-all';
+    values: string[];
+    ignore_case?: boolean;
+}
+interface JsonSchemaAssertion {
+    type: 'json-schema';
+    schema: object;
+    extract?: 'auto' | 'strict';
+}
+interface LlmJudgeAssertion {
+    type: 'llm-judge';
+    criteria: string;
+    provider?: Partial<ProviderConfig>;
+    pass_threshold?: number;
+}
+interface SemanticSimilarityAssertion {
+    type: 'semantic-similarity';
+    reference: string;
+    threshold?: number;
+}
+interface FaithfulnessAssertion {
+    type: 'faithfulness';
+    context: string;
+    threshold?: number;
+}
+interface AnswerRelevanceAssertion {
+    type: 'answer-relevance';
+    question: string;
+    threshold?: number;
+}
+interface ContextPrecisionAssertion {
+    type: 'context-precision';
+    context: string;
+    question: string;
+    threshold?: number;
+}
+type AssertionConfig = ExactMatchAssertion | ContainsAssertion | ContainsAnyAssertion | ContainsAllAssertion | JsonSchemaAssertion | LlmJudgeAssertion | SemanticSimilarityAssertion | FaithfulnessAssertion | AnswerRelevanceAssertion | ContextPrecisionAssertion;
+interface EvalConfig {
+    id: string;
+    description?: string;
+    tags?: string[];
+    provider?: Partial<ProviderConfig>;
+    prompt?: string;
+    prompt_file?: string;
+    dataset: DatasetRef;
+    assertions: AssertionConfig[];
+}
+interface Config {
+    version: 1;
+    provider: ProviderConfig;
+    defaults?: {
+        assertions?: AssertionConfig[];
+    };
+    pricing?: PricingMap;
+    evals: EvalConfig[];
+}
+
+interface CompletionRequest {
+    provider: string;
+    model: string;
+    temperature: number;
+    max_tokens: number;
+    prompt_resolved: string;
+}
+interface CompletionResponse {
+    output: string;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+    /**
+     * Actual USD cost the provider reported for this call, if it reports one (e.g. OpenRouter via
+     * `usage.include`). Absent for providers that only return token counts — the cost is then
+     * estimated downstream (ADR-PRC02). Additive + optional: existing providers omit it.
+     */
+    cost_usd?: number;
+}
+interface CostEstimate {
+    cost_usd: number;
+}
+interface Provider<_TConfig extends ProviderConfig = ProviderConfig> {
+    complete(req: CompletionRequest): Promise<CompletionResponse>;
+    /**
+     * Resolve the USD cost for a call's usage. `reportedCostUsd` (optional) is the provider's actual
+     * reported cost from `complete()` — when passed it is preferred over the table estimate, unless a
+     * config `pricing` override exists (precedence in `resolveCost`, ADR-PRC01). Omitting it (dry-run,
+     * judge calls) yields a pure token-based estimate, as before — additive + backward-compatible.
+     */
+    estimateCost(usage: {
+        input_tokens: number;
+        output_tokens: number;
+    }, reportedCostUsd?: number): CostEstimate;
+}
+
+interface Embedder {
+    /** Embed a batch of texts into fixed-dimension vectors (one per input, in order). */
+    embed(texts: string[]): Promise<Float32Array[]>;
+}
+
+interface Cache {
+    /** Return the cached completion, or undefined on a miss. `maxAgeMs` (optional) expires old entries. */
+    get(key: string, opts?: {
+        maxAgeMs?: number;
+    }): CompletionResponse | undefined;
+    set(key: string, value: CompletionResponse): void;
+    clear(): void;
+    close(): void;
+}
+
+interface RunDeps {
+    resolveProvider(cfg: ProviderConfig): Provider;
+    embedder: Embedder;
+    cache: Cache;
+    now: () => number;
+    loadDataset(ref: DatasetRef, baseDir: string): DatasetRow[];
+    baseDir: string;
+}
+
+interface RunOptions {
+    dryRun: boolean;
+    configPath?: string;
+    only?: string[];
+    concurrency?: number;
+    noCache?: boolean;
+    bail?: boolean;
+}
+declare function handleRun(options: RunOptions, depsFactory?: (config: Config, baseDir: string) => RunDeps): Promise<RunResult>;
+
+export { type AssertionResultRecord, type Config, type EvalResult, type RowResult, type RunError, type RunOptions, type RunResult, type Summary, type Usage, handleRun as run };