@unbrained/pm-cli 2026.9.25 → 2026.9.26

package/dist/core/search/embedding-batches.d.ts

@@ -1,38 +1,70 @@
 /**
  * @module core/search/embedding-batches
  *
- * Powers search, embeddings, and semantic retrieval behavior for Embedding Batches.
+ * Bounds embedding requests independently of per-input truncation and preserves
+ * input ordering through retries and adaptive splits.
  */
 import type { PmSettings } from "../../types/index.js";
 import type { EmbeddingProviderConfig } from "./providers.js";
-/** Documents the embedding batch execution result payload exchanged by command, SDK, and package integrations. */
+/** Completed vectors and execution evidence returned only after every input succeeds. */
 export interface EmbeddingBatchExecutionResult {
-    /** Value that configures or reports vectors for this contract. */
+    /** One vector for each input position, including repeated input text. */
     vectors: number[][];
-    /** Value that configures or reports warnings for this contract. */
+    /** Truncation, retry, and split diagnostics; never contains source input text. */
     warnings: string[];
+    /** Actual provider work, including retries and adaptive splits. */
+    receipt?: EmbeddingBatchReceipt;
 }
-/** Documents the embedding batch progress event payload exchanged by command, SDK, and package integrations. */
+/** Bounded summary of actual provider work; histogram keys are successful request sizes. */
+export interface EmbeddingBatchReceipt {
+    /** Provider and model used for this execution, without endpoint or credentials. */
+    provider: string;
+    /** Model requested from the provider. */
+    model: string;
+    /** Maximum encoded JSON request size, including model and escaping. */
+    max_request_bytes: number;
+    /** Whether the request ceiling was supplied by the caller or the runtime. */
+    limit_source: "caller" | "default";
+    /** Number of requests planned before adaptive splitting. */
+    planned_batches: number;
+    /** Total HTTP attempts, including unsuccessful requests. */
+    requests: number;
+    /** Requests that returned validated vectors. */
+    successful_batches: number;
+    /** Failed requests subdivided into smaller requests. */
+    split_batches: number;
+    /** Additional attempts of the same batch after a transient failure. */
+    retries: number;
+    /** Successful request count by number of input positions. */
+    batch_size_histogram: Record<string, number>;
+    /** Wall-clock duration including provider time and retry backoff. */
+    elapsed_ms: number;
+    /** Completed input positions divided by elapsed seconds. */
+    inputs_per_second: number;
+}
+/** Progress across planned batches; actual HTTP work is reported separately in the receipt. */
 export interface EmbeddingBatchProgressEvent {
-    /** Value that configures or reports batch index for this contract. */
+    /** One-based index of the current planned batch. */
     batch_index: number;
-    /** Value that configures or reports batch total for this contract. */
+    /** Number of planned batches before adaptive splitting. */
     batch_total: number;
-    /** Value that configures or reports batch size for this contract. */
+    /** Number of input positions in the current planned batch. */
     batch_size: number;
-    /** Value that configures or reports completed inputs for this contract. */
+    /** Input positions successfully embedded before or after this event. */
     completed_inputs: number;
-    /** Value that configures or reports total inputs for this contract. */
+    /** Total input positions in this execution. */
     total_inputs: number;
-    /** Value that configures or reports attempt for this contract. */
+    /** Optional attempt counter for callers extending progress reporting. */
     attempt?: number;
-    /** Value that configures or reports phase for this contract. */
+    /** Whether dispatch is starting or the complete planned batch has succeeded. */
     phase: "start" | "complete";
 }
-/** Documents the embedding batch execution options payload exchanged by command, SDK, and package integrations. */
+/** Host controls for progress reporting and transport payload bounds. */
 export interface EmbeddingBatchExecutionOptions {
-    /** Value that configures or reports on progress for this contract. */
+    /** Observe planned-batch boundaries without receiving input text. */
     onProgress?: (event: EmbeddingBatchProgressEvent) => void;
+    /** Positive JSON payload byte ceiling, independent of per-input corpus truncation; defaults to 256 KiB. */
+    maxRequestBytes?: number;
 }
-/** Implements execute embedding batches with retry for the public runtime surface of this module. */
+/** Execute count- and byte-bounded batches in order. Reject impossible inputs before dispatch, retry transient failures, and subdivide size or timeout failures without returning partial vectors. */
 export declare function executeEmbeddingBatchesWithRetry(provider: EmbeddingProviderConfig, settings: PmSettings, inputs: string[], options?: EmbeddingBatchExecutionOptions): Promise<EmbeddingBatchExecutionResult>;

package/dist/core/search/embedding-batches.js

@@ -1,2 +1,2 @@
-!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="f0edf7da-87cc-5e66-9f9f-c9a25e21808e")}catch(e2){}})();import{executeEmbeddingRequest}from"./providers.js";import{toErrorMessage}from"../shared/primitives.js";import{OLLAMA_SEMANTIC_CORPUS_INPUT_MAX_CHARACTERS,SEMANTIC_CORPUS_TRUNCATION_SUFFIX,resolveSemanticCorpusCharacterLimit}from"./corpus.js";function resolveBatchRuntime(settings,maxInputCharacters){const batchSizeCandidate=settings.search.embedding_batch_size;const timeoutMsCandidate=settings.search.embedding_timeout_ms;const maxRetriesCandidate=settings.search.scanner_max_batch_retries;const batchSize=Number.isFinite(batchSizeCandidate)&&batchSizeCandidate>0?Math.floor(batchSizeCandidate):1;const timeoutMs=Number.isFinite(timeoutMsCandidate)&&timeoutMsCandidate>0?Math.floor(timeoutMsCandidate):3e4;const maxRetries=Number.isFinite(maxRetriesCandidate)&&maxRetriesCandidate>=0?Math.floor(maxRetriesCandidate):0;return{batchSize,timeoutMs,maxRetries,maxBatchInputCharacters:Number.POSITIVE_INFINITY,maxInputCharacters}}function createBatches(inputs,batchSize,maxBatchInputCharacters){const batches=[];let currentBatch=[];let currentCharacters=0;for(const input of inputs){const wouldExceedCount=currentBatch.length>=batchSize;const wouldExceedCharacters=currentBatch.length>0&&Number.isFinite(maxBatchInputCharacters)&&currentCharacters+input.length>maxBatchInputCharacters;if(wouldExceedCount||wouldExceedCharacters){batches.push(currentBatch);currentBatch=[];currentCharacters=0}currentBatch.push(input);currentCharacters+=input.length}batches.push(currentBatch);return batches}function resolveProviderBatchRuntime(provider,runtime){if(provider.name!=="ollama"){return runtime}return{...runtime,maxBatchInputCharacters:OLLAMA_SEMANTIC_CORPUS_INPUT_MAX_CHARACTERS}}function truncateInputForRuntime(input,maxInputCharacters){if(!Number.isFinite(maxInputCharacters)||maxInputCharacters<=0||input.length<=maxInputCharacters){return input}const keepLength=Math.max(0,maxInputCharacters-SEMANTIC_CORPUS_TRUNCATION_SUFFIX.length);return`${input.slice(0,keepLength)}${SEMANTIC_CORPUS_TRUNCATION_SUFFIX}`.slice(0,maxInputCharacters)}function isEmbeddingTimeoutError(error){const message=toErrorMessage(error).toLowerCase();return message.includes("timed out")||message.includes("timeout")}function buildEmbeddingFailureMessage(provider,batchLabel,batchSize,timeoutMs,attempts,error){const base=`Embedding batch ${batchLabel} failed after ${attempts} attempt(s): ${toErrorMessage(error)}`;const details=`provider=${provider.name} model=${provider.model} batch_size=${batchSize} timeout_ms=${timeoutMs}`;if(!isEmbeddingTimeoutError(error)){return`${base} (${details})`}return`${base} (${details}; guidance=check provider availability or lower search.embedding_batch_size, raise search.embedding_timeout_ms, or run keyword search while semantic indexing catches up)`}async function executeBatchWithAdaptiveSplit(provider,batch,batchLabel,timeoutMs,maxRetries,warnings){let lastError=null;for(let attempt=0;attempt<=maxRetries;attempt+=1){try{const batchVectors=await executeEmbeddingRequest(provider,batch,{timeout_ms:timeoutMs});if(attempt>0){warnings.push(`search_embedding_batch_retry_succeeded:batch=${batchLabel}:attempt=${attempt+1}:size=${batch.length}`)}return batchVectors}catch(error){lastError=error;if(isEmbeddingTimeoutError(error)&&batch.length>1){const midpoint=Math.ceil(batch.length/2);const left=batch.slice(0,midpoint);const right=batch.slice(midpoint);warnings.push(`search_embedding_batch_split_after_timeout:batch=${batchLabel}:size=${batch.length}:parts=${left.length}|${right.length}`);return[...await executeBatchWithAdaptiveSplit(provider,left,`${batchLabel}.1`,timeoutMs,maxRetries,warnings),...await executeBatchWithAdaptiveSplit(provider,right,`${batchLabel}.2`,timeoutMs,maxRetries,warnings)]}if(attempt<maxRetries){const delayMs=Math.min(1e3*2**attempt,8e3);await new Promise(resolve=>setTimeout(resolve,delayMs))}}}throw new Error(buildEmbeddingFailureMessage(provider,batchLabel,batch.length,timeoutMs,maxRetries+1,lastError))}async function executeEmbeddingBatchesWithRetry(provider,settings,inputs,options={}){if(inputs.length===0){return{vectors:[],warnings:[]}}const warnings=[];const corpusLimitResolution=resolveSemanticCorpusCharacterLimit(provider.name,settings.search.embedding_corpus_max_characters);if(corpusLimitResolution.warning){warnings.push(corpusLimitResolution.warning)}const runtime=resolveProviderBatchRuntime(provider,resolveBatchRuntime(settings,corpusLimitResolution.maxCharacters));let truncatedInputCount=0;const normalizedInputs=inputs.map(input=>{const normalized=truncateInputForRuntime(input,runtime.maxInputCharacters);if(normalized.length<input.length){truncatedInputCount+=1}return normalized});if(truncatedInputCount>0){warnings.push(`search_embedding_input_truncated:count=${truncatedInputCount}:max_characters=${runtime.maxInputCharacters}`)}const batches=createBatches(normalizedInputs,runtime.batchSize,runtime.maxBatchInputCharacters);const vectors=[];let completedInputs=0;for(let batchIndex=0;batchIndex<batches.length;batchIndex+=1){const batch=batches[batchIndex];options.onProgress?.({batch_index:batchIndex+1,batch_total:batches.length,batch_size:batch.length,completed_inputs:completedInputs,total_inputs:normalizedInputs.length,phase:"start"});vectors.push(...await executeBatchWithAdaptiveSplit(provider,batch,String(batchIndex+1),runtime.timeoutMs,runtime.maxRetries,warnings));completedInputs+=batch.length;options.onProgress?.({batch_index:batchIndex+1,batch_total:batches.length,batch_size:batch.length,completed_inputs:completedInputs,total_inputs:normalizedInputs.length,phase:"complete"})}return{vectors,warnings}}export{executeEmbeddingBatchesWithRetry};
+!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="dff1d04a-bad4-51d7-ab46-41e49387a44d")}catch(e2){}})();import{executeEmbeddingRequest}from"./providers.js";import{SearchHttpError}from"./http-client.js";import{toErrorMessage}from"../shared/primitives.js";import{SEMANTIC_CORPUS_TRUNCATION_SUFFIX,resolveSemanticCorpusCharacterLimit}from"./corpus.js";function resolveBatchRuntime(settings,maxInputCharacters,maxRequestBytes){const batchSizeCandidate=settings.search.embedding_batch_size;const timeoutMsCandidate=settings.search.embedding_timeout_ms;const maxRetriesCandidate=settings.search.scanner_max_batch_retries;const batchSize=Number.isFinite(batchSizeCandidate)&&batchSizeCandidate>0?Math.max(1,Math.floor(batchSizeCandidate)):1;const timeoutMs=Number.isFinite(timeoutMsCandidate)&&timeoutMsCandidate>0?Math.max(1,Math.floor(timeoutMsCandidate)):3e4;const maxRetries=Number.isFinite(maxRetriesCandidate)&&maxRetriesCandidate>=0?Math.floor(maxRetriesCandidate):0;return{batchSize,timeoutMs,maxRetries,maxRequestBytes,maxInputCharacters}}function createBatches(provider,inputs,batchSize,maxRequestBytes){const batches=[];let currentBatch=[];const envelopeBytes=Buffer.byteLength(JSON.stringify({model:provider.model,input:[]}));let currentBytes=envelopeBytes;for(const input of inputs){const inputBytes=Buffer.byteLength(JSON.stringify(input.trim()));if(envelopeBytes+inputBytes>maxRequestBytes){throw new RangeError(`Embedding input exceeds request byte ceiling ${maxRequestBytes}; lower search.embedding_corpus_max_characters or raise maxRequestBytes`)}const separatorBytes=currentBatch.length>0?1:0;if(currentBatch.length>=batchSize||currentBytes+separatorBytes+inputBytes>maxRequestBytes){batches.push(currentBatch);currentBatch=[];currentBytes=envelopeBytes}currentBytes+=inputBytes+(currentBatch.length>0?1:0);currentBatch.push(input)}batches.push(currentBatch);return batches}function truncateInputForRuntime(input,maxInputCharacters){if(!Number.isFinite(maxInputCharacters)||maxInputCharacters<=0||input.length<=maxInputCharacters){return input}const keepLength=Math.max(0,maxInputCharacters-SEMANTIC_CORPUS_TRUNCATION_SUFFIX.length);return`${input.slice(0,keepLength)}${SEMANTIC_CORPUS_TRUNCATION_SUFFIX}`.slice(0,maxInputCharacters)}function isEmbeddingTimeoutError(error){const message=toErrorMessage(error).toLowerCase();return message.includes("timed out")||message.includes("timeout")}function buildEmbeddingFailureMessage(provider,batchLabel,batchSize,timeoutMs,attempts,error){const base=`Embedding batch ${batchLabel} failed after ${attempts} attempt(s): ${toErrorMessage(error)}`;const details=`provider=${provider.name} model=${provider.model} batch_size=${batchSize} timeout_ms=${timeoutMs}`;if(!isEmbeddingTimeoutError(error)){return`${base} (${details})`}return`${base} (${details}; guidance=check provider availability or lower search.embedding_batch_size, raise search.embedding_timeout_ms, or run keyword search while semantic indexing catches up)`}function resolveSplitReason(error){if(error instanceof SearchHttpError)return error.status===413?"payload_too_large":null;return isEmbeddingTimeoutError(error)?"timeout":null}async function executeBatchWithAdaptiveSplit(provider,batch,batchLabel,timeoutMs,maxRetries,warnings,receipt){let lastError=null;for(let attempt=0;attempt<=maxRetries;attempt+=1){try{receipt.requests+=1;const batchVectors=await executeEmbeddingRequest(provider,batch,{timeout_ms:timeoutMs});if(attempt>0){warnings.push(`search_embedding_batch_retry_succeeded:batch=${batchLabel}:attempt=${attempt+1}:size=${batch.length}`)}receipt.successful_batches+=1;receipt.batch_size_histogram[batch.length]=(receipt.batch_size_histogram[batch.length]??0)+1;return batchVectors}catch(error){lastError=error;const splitReason=resolveSplitReason(error);if(splitReason!==null&&batch.length>1){receipt.split_batches+=1;const midpoint=Math.ceil(batch.length/2);const left=batch.slice(0,midpoint);const right=batch.slice(midpoint);warnings.push(`search_embedding_batch_split_after_${splitReason}:batch=${batchLabel}:size=${batch.length}:parts=${left.length}|${right.length}`);return[...await executeBatchWithAdaptiveSplit(provider,left,`${batchLabel}.1`,timeoutMs,maxRetries,warnings,receipt),...await executeBatchWithAdaptiveSplit(provider,right,`${batchLabel}.2`,timeoutMs,maxRetries,warnings,receipt)]}if(attempt<maxRetries){receipt.retries+=1;const delayMs=Math.min(1e3*2**attempt,8e3);await new Promise(resolve=>setTimeout(resolve,delayMs))}}}throw new Error(buildEmbeddingFailureMessage(provider,batchLabel,batch.length,timeoutMs,maxRetries+1,lastError))}async function executeEmbeddingBatchesWithRetry(provider,settings,inputs,options={}){if(inputs.length===0){return{vectors:[],warnings:[]}}const startedAt=performance.now();const maxRequestBytes=options.maxRequestBytes??262144;if(!Number.isSafeInteger(maxRequestBytes)||maxRequestBytes<=0){throw new RangeError("maxRequestBytes must be a positive safe integer")}const receipt={provider:provider.name,model:provider.model,max_request_bytes:maxRequestBytes,limit_source:options.maxRequestBytes===void 0?"default":"caller",planned_batches:0,requests:0,successful_batches:0,split_batches:0,retries:0,batch_size_histogram:{},elapsed_ms:0,inputs_per_second:0};const warnings=[];const corpusLimitResolution=resolveSemanticCorpusCharacterLimit(provider.name,settings.search.embedding_corpus_max_characters);if(corpusLimitResolution.warning){warnings.push(corpusLimitResolution.warning)}const runtime=resolveBatchRuntime(settings,corpusLimitResolution.maxCharacters,maxRequestBytes);let truncatedInputCount=0;const normalizedInputs=inputs.map(input=>{if(input.trim().length===0)throw new TypeError("Embedding inputs must be non-empty strings");const normalized=truncateInputForRuntime(input,runtime.maxInputCharacters);if(normalized.length<input.length){truncatedInputCount+=1}return normalized});if(truncatedInputCount>0){warnings.push(`search_embedding_input_truncated:count=${truncatedInputCount}:max_characters=${runtime.maxInputCharacters}`)}const batches=createBatches(provider,normalizedInputs,runtime.batchSize,runtime.maxRequestBytes);receipt.planned_batches=batches.length;const vectors=[];let completedInputs=0;for(let batchIndex=0;batchIndex<batches.length;batchIndex+=1){const batch=batches[batchIndex];options.onProgress?.({batch_index:batchIndex+1,batch_total:batches.length,batch_size:batch.length,completed_inputs:completedInputs,total_inputs:normalizedInputs.length,phase:"start"});vectors.push(...await executeBatchWithAdaptiveSplit(provider,batch,String(batchIndex+1),runtime.timeoutMs,runtime.maxRetries,warnings,receipt));completedInputs+=batch.length;options.onProgress?.({batch_index:batchIndex+1,batch_total:batches.length,batch_size:batch.length,completed_inputs:completedInputs,total_inputs:normalizedInputs.length,phase:"complete"})}receipt.elapsed_ms=Math.max(1,performance.now()-startedAt);receipt.inputs_per_second=inputs.length*1e3/receipt.elapsed_ms;return{vectors,warnings,receipt}}export{executeEmbeddingBatchesWithRetry};
 //# sourceMappingURL=embedding-batches.js.map

package/dist/core/search/eval-thresholds.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @module core/search/eval-thresholds
+ *
+ * Validates and evaluates per-query retrieval floors independently of aggregate
+ * scores. Shared by the public evaluator and repository quality gates.
+ */
+/** Ranking metrics eligible for a declared quality floor. */
+export declare const EVAL_METRIC_NAMES: readonly ["ndcg", "mrr", "precision", "recall"];
+/** Optional normalized lower bounds; omitted metrics impose no constraint. */
+export type EvalMetricFloors = Partial<Record<typeof EVAL_METRIC_NAMES[number], number>>;
+/** Reject malformed or misspelled floors rather than silently weakening a quality contract. */
+export declare function parseEvalMetricFloors(value: unknown): EvalMetricFloors;
+/** Compare unrounded metrics to validated floors and return stable, actionable violations. */
+export declare function evaluateMetricFloors(metrics: Record<typeof EVAL_METRIC_NAMES[number], number>, floors: EvalMetricFloors): string[];

package/dist/core/search/eval-thresholds.js

@@ -0,0 +1,2 @@
+!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="b152a753-033c-593f-8b81-feb68b0a895f")}catch(e2){}})();const EVAL_METRIC_NAMES=["ndcg","mrr","precision","recall"];function parseEvalMetricFloors(value){if(typeof value!=="object"||value===null||Array.isArray(value)){throw new TypeError("Eval minimum must be an object of metric floors")}const result={};for(const[key,floor]of Object.entries(value)){if(!EVAL_METRIC_NAMES.some(metric=>metric===key)){throw new TypeError(`Unknown eval minimum metric: ${key}`)}if(typeof floor!=="number"||!Number.isFinite(floor)||floor<0||floor>1){throw new TypeError(`Eval minimum.${key} must be a finite number in [0, 1]`)}result[key]=floor}return result}function evaluateMetricFloors(metrics,floors){const validated=parseEvalMetricFloors(floors);const violations=[];for(const metric of EVAL_METRIC_NAMES){const minimum=validated[metric];if(minimum===void 0)continue;const actual=metrics[metric];if(!Number.isFinite(actual)||actual<0||actual>1){violations.push(`${metric}:invalid`)}else if(actual<minimum){violations.push(`${metric}:${actual}<${minimum}`)}}return violations}export{EVAL_METRIC_NAMES,evaluateMetricFloors,parseEvalMetricFloors};
+//# sourceMappingURL=eval-thresholds.js.map

package/dist/core/search/eval.d.ts

@@ -13,6 +13,7 @@
  * not. This keeps the golden set cheap to curate while still supporting the
  * graded-discount behavior of DCG (earlier hits are worth more).
  */
+import { type EvalMetricFloors } from "./eval-thresholds.js";
 /** Retrieval mode a golden query is evaluated under. Mirrors the `pm search` mode surface; omitting it on a query defers to the eval run's default mode. */
 export type EvalSearchMode = "keyword" | "semantic" | "hybrid";
 /** Default cutoff (`@k`) used for every metric when the caller does not override it. */
@@ -24,7 +25,7 @@ export declare const EVAL_QUERY_SET_CONTRACT: {
     readonly schema: "https://schema.unbrained.dev/pm/eval-query-set/v1";
     readonly accepted_top_level: readonly ["array", "object"];
     readonly required_query_fields: readonly ["query", "relevant_ids"];
-    readonly optional_fields: readonly ["mode", "description"];
+    readonly optional_fields: readonly ["mode", "description", "minimum"];
     readonly modes: readonly ["keyword", "semantic", "hybrid"];
 };
 /** Copy-ready minimal query set shared by help, diagnostics, and integrations. */
@@ -46,6 +47,8 @@ export interface EvalQuery {
     mode?: EvalSearchMode;
     /** Value that configures or reports description for this contract. */
     description?: string;
+    /** Per-query floors applied before rounding, independently of the aggregate gate. */
+    minimum?: EvalMetricFloors;
 }
 /** A parsed, validated golden-query set. */
 export interface EvalQuerySet {

package/dist/core/search/eval.js

@@ -1,2 +1,2 @@
-!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="51927137-53a7-5667-a341-c4e28a2c3964")}catch(e2){}})();const DEFAULT_EVAL_K=10;const EVAL_QUERY_SET_SCHEMA_ID="https://schema.unbrained.dev/pm/eval-query-set/v1";const EVAL_QUERY_SET_CONTRACT={schema:EVAL_QUERY_SET_SCHEMA_ID,accepted_top_level:["array","object"],required_query_fields:["query","relevant_ids"],optional_fields:["mode","description"],modes:["keyword","semantic","hybrid"]};const EVAL_QUERY_SET_EXAMPLE={schema:EVAL_QUERY_SET_SCHEMA_ID,queries:[{query:"offline search",relevant_ids:["pm-75k9"],mode:"keyword"}]};class EvalQuerySetError extends Error{constructor(message){super(message);this.name="EvalQuerySetError"}}const VALID_EVAL_MODES=new Set(["keyword","semantic","hybrid"]);function dcgAtK(relevanceGains,k){let dcg=0;const limit=Math.min(k,relevanceGains.length);for(let index=0;index<limit;index+=1){dcg+=relevanceGains[index]/Math.log2(index+2)}return dcg}function ndcgAtK(rankedIds,relevantIds,k){if(relevantIds.size===0){return 0}const gains=rankedIds.map(id=>relevantIds.has(id)?1:0);const dcg=dcgAtK(gains,k);const idealGains=Array.from({length:relevantIds.size},()=>1);const idealDcg=dcgAtK(idealGains,k);return idealDcg===0?0:dcg/idealDcg}function reciprocalRankAtK(rankedIds,relevantIds,k){const limit=Math.min(k,rankedIds.length);for(let index=0;index<limit;index+=1){if(relevantIds.has(rankedIds[index])){return 1/(index+1)}}return 0}function countRelevantInTopK(rankedIds,relevantIds,k){const limit=Math.min(k,rankedIds.length);let hits=0;for(let index=0;index<limit;index+=1){if(relevantIds.has(rankedIds[index])){hits+=1}}return hits}function precisionAtK(rankedIds,relevantIds,k){if(k<=0){return 0}return countRelevantInTopK(rankedIds,relevantIds,k)/k}function recallAtK(rankedIds,relevantIds,k){if(relevantIds.size===0){return 0}return countRelevantInTopK(rankedIds,relevantIds,k)/relevantIds.size}function evaluateRanking(rankedIds,relevantIds,k){return{ndcg:ndcgAtK(rankedIds,relevantIds,k),mrr:reciprocalRankAtK(rankedIds,relevantIds,k),precision:precisionAtK(rankedIds,relevantIds,k),recall:recallAtK(rankedIds,relevantIds,k),relevant_total:relevantIds.size,retrieved_relevant:countRelevantInTopK(rankedIds,relevantIds,k)}}function aggregateEvalMetrics(perQuery){const queryCount=perQuery.length;if(queryCount===0){return{ndcg:0,mrr:0,precision:0,recall:0,query_count:0}}const sum=perQuery.reduce((accumulator,metrics)=>({ndcg:accumulator.ndcg+metrics.ndcg,mrr:accumulator.mrr+metrics.mrr,precision:accumulator.precision+metrics.precision,recall:accumulator.recall+metrics.recall}),{ndcg:0,mrr:0,precision:0,recall:0});return{ndcg:sum.ndcg/queryCount,mrr:sum.mrr/queryCount,precision:sum.precision/queryCount,recall:sum.recall/queryCount,query_count:queryCount}}function parseEvalQueryEntry(raw,index){if(typeof raw!=="object"||raw===null){throw new EvalQuerySetError(`Eval query at index ${index} must be an object`)}const entry=raw;const query=typeof entry.query==="string"?entry.query.trim():"";if(query.length===0){throw new EvalQuerySetError(`Eval query at index ${index} must have a non-empty "query" string`)}if(!Array.isArray(entry.relevant_ids)){throw new EvalQuerySetError(`Eval query at index ${index} must have a "relevant_ids" array`)}if(entry.relevant_ids.some(value=>typeof value!=="string")){throw new EvalQuerySetError(`Eval query at index ${index} must have a "relevant_ids" array of strings`)}const relevantIds=[...new Set(entry.relevant_ids.map(value=>value.trim()).filter(value=>value.length>0))];if(relevantIds.length===0){throw new EvalQuerySetError(`Eval query at index ${index} must list at least one relevant id`)}if(entry.mode!==void 0&&(typeof entry.mode!=="string"||!VALID_EVAL_MODES.has(entry.mode))){throw new EvalQuerySetError(`Eval query at index ${index} has an invalid mode (expected keyword|semantic|hybrid)`)}return{query,relevant_ids:relevantIds,...entry.mode!==void 0?{mode:entry.mode}:{},...typeof entry.description==="string"&&entry.description.trim().length>0?{description:entry.description.trim()}:{}}}function parseEvalQuerySet(raw){const rawSchema=typeof raw==="object"&&raw!==null&&!Array.isArray(raw)?raw.schema:void 0;if(rawSchema!==void 0&&rawSchema!==EVAL_QUERY_SET_SCHEMA_ID){throw new EvalQuerySetError(`Unsupported eval query set schema; expected "${EVAL_QUERY_SET_SCHEMA_ID}"`)}const rawQueries=Array.isArray(raw)?raw:typeof raw==="object"&&raw!==null?raw.queries:void 0;if(!Array.isArray(rawQueries)){throw new EvalQuerySetError('Eval query set must be an array of queries or an object with a "queries" array')}if(rawQueries.length===0){throw new EvalQuerySetError("Eval query set must contain at least one query")}return{schema:EVAL_QUERY_SET_SCHEMA_ID,queries:rawQueries.map((entry,index)=>parseEvalQueryEntry(entry,index))}}export{DEFAULT_EVAL_K,EVAL_QUERY_SET_CONTRACT,EVAL_QUERY_SET_EXAMPLE,EVAL_QUERY_SET_SCHEMA_ID,EvalQuerySetError,aggregateEvalMetrics,dcgAtK,evaluateRanking,ndcgAtK,parseEvalQuerySet,precisionAtK,recallAtK,reciprocalRankAtK};
+!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="451970e8-8403-5527-b795-a873241c015e")}catch(e2){}})();import{parseEvalMetricFloors}from"./eval-thresholds.js";const DEFAULT_EVAL_K=10;const EVAL_QUERY_SET_SCHEMA_ID="https://schema.unbrained.dev/pm/eval-query-set/v1";const EVAL_QUERY_SET_CONTRACT={schema:EVAL_QUERY_SET_SCHEMA_ID,accepted_top_level:["array","object"],required_query_fields:["query","relevant_ids"],optional_fields:["mode","description","minimum"],modes:["keyword","semantic","hybrid"]};const EVAL_QUERY_SET_EXAMPLE={schema:EVAL_QUERY_SET_SCHEMA_ID,queries:[{query:"offline search",relevant_ids:["pm-75k9"],mode:"keyword"}]};class EvalQuerySetError extends Error{constructor(message){super(message);this.name="EvalQuerySetError"}}const VALID_EVAL_MODES=new Set(["keyword","semantic","hybrid"]);function dcgAtK(relevanceGains,k){let dcg=0;const limit=Math.min(k,relevanceGains.length);for(let index=0;index<limit;index+=1){dcg+=relevanceGains[index]/Math.log2(index+2)}return dcg}function ndcgAtK(rankedIds,relevantIds,k){if(relevantIds.size===0){return 0}const gains=rankedIds.map(id=>relevantIds.has(id)?1:0);const dcg=dcgAtK(gains,k);const idealGains=Array.from({length:relevantIds.size},()=>1);const idealDcg=dcgAtK(idealGains,k);return idealDcg===0?0:dcg/idealDcg}function reciprocalRankAtK(rankedIds,relevantIds,k){const limit=Math.min(k,rankedIds.length);for(let index=0;index<limit;index+=1){if(relevantIds.has(rankedIds[index])){return 1/(index+1)}}return 0}function countRelevantInTopK(rankedIds,relevantIds,k){const limit=Math.min(k,rankedIds.length);let hits=0;for(let index=0;index<limit;index+=1){if(relevantIds.has(rankedIds[index])){hits+=1}}return hits}function precisionAtK(rankedIds,relevantIds,k){if(k<=0){return 0}return countRelevantInTopK(rankedIds,relevantIds,k)/k}function recallAtK(rankedIds,relevantIds,k){if(relevantIds.size===0){return 0}return countRelevantInTopK(rankedIds,relevantIds,k)/relevantIds.size}function evaluateRanking(rankedIds,relevantIds,k){return{ndcg:ndcgAtK(rankedIds,relevantIds,k),mrr:reciprocalRankAtK(rankedIds,relevantIds,k),precision:precisionAtK(rankedIds,relevantIds,k),recall:recallAtK(rankedIds,relevantIds,k),relevant_total:relevantIds.size,retrieved_relevant:countRelevantInTopK(rankedIds,relevantIds,k)}}function aggregateEvalMetrics(perQuery){const queryCount=perQuery.length;if(queryCount===0){return{ndcg:0,mrr:0,precision:0,recall:0,query_count:0}}const sum=perQuery.reduce((accumulator,metrics)=>({ndcg:accumulator.ndcg+metrics.ndcg,mrr:accumulator.mrr+metrics.mrr,precision:accumulator.precision+metrics.precision,recall:accumulator.recall+metrics.recall}),{ndcg:0,mrr:0,precision:0,recall:0});return{ndcg:sum.ndcg/queryCount,mrr:sum.mrr/queryCount,precision:sum.precision/queryCount,recall:sum.recall/queryCount,query_count:queryCount}}function parseEvalQueryEntry(raw,index){if(typeof raw!=="object"||raw===null){throw new EvalQuerySetError(`Eval query at index ${index} must be an object`)}const entry=raw;const query=typeof entry.query==="string"?entry.query.trim():"";if(query.length===0){throw new EvalQuerySetError(`Eval query at index ${index} must have a non-empty "query" string`)}if(!Array.isArray(entry.relevant_ids)){throw new EvalQuerySetError(`Eval query at index ${index} must have a "relevant_ids" array`)}if(entry.relevant_ids.some(value=>typeof value!=="string")){throw new EvalQuerySetError(`Eval query at index ${index} must have a "relevant_ids" array of strings`)}const relevantIds=[...new Set(entry.relevant_ids.map(value=>value.trim()).filter(value=>value.length>0))];if(relevantIds.length===0){throw new EvalQuerySetError(`Eval query at index ${index} must list at least one relevant id`)}if(entry.mode!==void 0&&(typeof entry.mode!=="string"||!VALID_EVAL_MODES.has(entry.mode))){throw new EvalQuerySetError(`Eval query at index ${index} has an invalid mode (expected keyword|semantic|hybrid)`)}return{query,relevant_ids:relevantIds,...entry.minimum===void 0?{}:{minimum:parseEvalMetricFloors(entry.minimum)},...entry.mode!==void 0?{mode:entry.mode}:{},...typeof entry.description==="string"&&entry.description.trim().length>0?{description:entry.description.trim()}:{}}}function parseEvalQuerySet(raw){const rawSchema=typeof raw==="object"&&raw!==null&&!Array.isArray(raw)?raw.schema:void 0;if(rawSchema!==void 0&&rawSchema!==EVAL_QUERY_SET_SCHEMA_ID){throw new EvalQuerySetError(`Unsupported eval query set schema; expected "${EVAL_QUERY_SET_SCHEMA_ID}"`)}const rawQueries=Array.isArray(raw)?raw:typeof raw==="object"&&raw!==null?raw.queries:void 0;if(!Array.isArray(rawQueries)){throw new EvalQuerySetError('Eval query set must be an array of queries or an object with a "queries" array')}if(rawQueries.length===0){throw new EvalQuerySetError("Eval query set must contain at least one query")}return{schema:EVAL_QUERY_SET_SCHEMA_ID,queries:rawQueries.map((entry,index)=>parseEvalQueryEntry(entry,index))}}export{DEFAULT_EVAL_K,EVAL_QUERY_SET_CONTRACT,EVAL_QUERY_SET_EXAMPLE,EVAL_QUERY_SET_SCHEMA_ID,EvalQuerySetError,aggregateEvalMetrics,dcgAtK,evaluateRanking,ndcgAtK,parseEvalQuerySet,precisionAtK,recallAtK,reciprocalRankAtK};
 //# sourceMappingURL=eval.js.map

package/dist/core/search/http-client.d.ts

@@ -1,3 +1,10 @@
+/** HTTP failure with a status code that schedulers can classify without parsing provider text. */
+export declare class SearchHttpError extends Error {
+    /** HTTP status returned by the provider, retained independently of its response body. */
+    readonly status: number;
+    /** Preserve the existing user-facing message and the machine-readable HTTP status. */
+    constructor(message: string, status: number);
+}
 /** Documents the search http response payload exchanged by command, SDK, and package integrations. */
 export interface SearchHttpResponse {
     /** Whether the operation completed without a blocking failure. */

package/dist/core/search/http-client.js

@@ -1,2 +1,2 @@
-!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="74cee8e6-1785-50f4-94d8-fc0b29055dc2")}catch(e2){}})();import{toErrorMessage}from"../shared/primitives.js";const DEFAULT_SEARCH_HTTP_TIMEOUT_MS=3e4;function normalizeSearchHttpTimeoutMs(timeoutMs,requestLabel,defaultTimeoutMs=DEFAULT_SEARCH_HTTP_TIMEOUT_MS){const resolved=timeoutMs??defaultTimeoutMs;if(!Number.isFinite(resolved)||resolved<=0){throw new Error(`${requestLabel} timeout must be a positive finite number`)}return Math.floor(resolved)}function resolveSearchHttpFetcher(fetcher,requestLabel){if(fetcher){return fetcher}if(typeof globalThis.fetch==="function"){return globalThis.fetch.bind(globalThis)}throw new Error(`${requestLabel} execution requires a fetch implementation`)}async function readFailedSearchHttpResponseBody(response){try{return(await response.text()).replaceAll(/\s+/g," ").trim()}catch(error){return`(failed to read response body: ${toErrorMessage(error)})`}}async function executeSearchJsonRequest(options){const controller=new AbortController;const timeoutHandle=setTimeout(()=>{controller.abort()},options.timeoutMs);try{let response;try{response=await options.fetcher(options.endpoint,{method:options.method,headers:options.headers,body:JSON.stringify(options.body),signal:controller.signal})}catch(error){if(error instanceof Error&&error.name==="AbortError"){throw new Error(`${options.requestLabel} timed out after ${options.timeoutMs}ms`)}throw new Error(`${options.requestLabel} execution failed: ${toErrorMessage(error)}`)}if(!response.ok){const responseBody=await readFailedSearchHttpResponseBody(response);const detail=responseBody.length>0?`: ${responseBody}`:"";throw new Error(`${options.requestLabel} failed with status ${response.status} ${response.statusText}${detail}`)}try{return await response.json()}catch(error){throw new Error(`${options.responseLabel} JSON parse failed: ${toErrorMessage(error)}`)}}finally{clearTimeout(timeoutHandle)}}export{executeSearchJsonRequest,normalizeSearchHttpTimeoutMs,readFailedSearchHttpResponseBody,resolveSearchHttpFetcher};
+!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="01b280e5-31b5-532d-b569-67aab828358a")}catch(e2){}})();import{toErrorMessage}from"../shared/primitives.js";const DEFAULT_SEARCH_HTTP_TIMEOUT_MS=3e4;class SearchHttpError extends Error{status;constructor(message,status){super(message);this.name="SearchHttpError";this.status=status}}function normalizeSearchHttpTimeoutMs(timeoutMs,requestLabel,defaultTimeoutMs=DEFAULT_SEARCH_HTTP_TIMEOUT_MS){const resolved=timeoutMs??defaultTimeoutMs;if(!Number.isFinite(resolved)||resolved<=0){throw new Error(`${requestLabel} timeout must be a positive finite number`)}return Math.floor(resolved)}function resolveSearchHttpFetcher(fetcher,requestLabel){if(fetcher){return fetcher}if(typeof globalThis.fetch==="function"){return globalThis.fetch.bind(globalThis)}throw new Error(`${requestLabel} execution requires a fetch implementation`)}async function readFailedSearchHttpResponseBody(response){try{return(await response.text()).replaceAll(/\s+/g," ").trim()}catch(error){return`(failed to read response body: ${toErrorMessage(error)})`}}async function executeSearchJsonRequest(options){const controller=new AbortController;const timeoutHandle=setTimeout(()=>{controller.abort()},options.timeoutMs);try{let response;try{response=await options.fetcher(options.endpoint,{method:options.method,headers:options.headers,body:JSON.stringify(options.body),signal:controller.signal})}catch(error){if(error instanceof Error&&error.name==="AbortError"){throw new Error(`${options.requestLabel} timed out after ${options.timeoutMs}ms`)}throw new Error(`${options.requestLabel} execution failed: ${toErrorMessage(error)}`)}if(!response.ok){const responseBody=await readFailedSearchHttpResponseBody(response);const detail=responseBody.length>0?`: ${responseBody}`:"";throw new SearchHttpError(`${options.requestLabel} failed with status ${response.status} ${response.statusText}${detail}`,response.status)}try{return await response.json()}catch(error){throw new Error(`${options.responseLabel} JSON parse failed: ${toErrorMessage(error)}`)}}finally{clearTimeout(timeoutHandle)}}export{SearchHttpError,executeSearchJsonRequest,normalizeSearchHttpTimeoutMs,readFailedSearchHttpResponseBody,resolveSearchHttpFetcher};
 //# sourceMappingURL=http-client.js.map

package/dist/sdk/eval.d.ts

@@ -3,7 +3,7 @@ import type { GlobalOptions } from "../core/shared/command-types.js";
 export { EVAL_QUERY_SET_CONTRACT, EVAL_QUERY_SET_EXAMPLE, EVAL_QUERY_SET_SCHEMA_ID, parseEvalQuerySet, } from "../core/search/eval.js";
 /** Relative location (under the pm root) of the default golden-query set. A git-tracked, human-curated file so relevance ground truth lives alongside the tracker it evaluates. */
 export declare const DEFAULT_EVAL_QUERIES_RELATIVE_PATH: string;
-/** Documents the eval command options payload exchanged by command, SDK, and package integrations. */
+/** Evaluation inputs shared by SDK hosts and the CLI adapter. */
 export interface EvalOptions {
     /** Default retrieval mode for queries that do not set their own (keyword|semantic|hybrid). */
     mode?: string;
@@ -18,41 +18,43 @@ export interface EvalOptions {
 }
 /** Per-query relevance report row emitted by {@link runEval}. */
 export interface EvalQueryReport {
-    /** Value that configures or reports query for this contract. */
+    /** Original golden-query text used for retrieval. */
     query: string;
-    /** Value that configures or reports mode for this contract. */
+    /** Resolved retrieval mode after applying query and run defaults. */
     mode: EvalSearchMode;
-    /** Value that configures or reports relevant total for this contract. */
+    /** Number of unique items judged relevant to this query. */
     relevant_total: number;
-    /** Value that configures or reports retrieved relevant for this contract. */
+    /** Relevant items found within the ranking cutoff. */
     retrieved_relevant: number;
-    /** Value that configures or reports ndcg for this contract. */
+    /** Normalized discounted cumulative gain at the selected cutoff. */
     ndcg: number;
-    /** Value that configures or reports mrr for this contract. */
+    /** Reciprocal rank of the first relevant result, or zero when absent. */
     mrr: number;
-    /** Value that configures or reports precision for this contract. */
+    /** Fraction of positions at the selected cutoff containing relevant items. */
     precision: number;
-    /** Value that configures or reports recall for this contract. */
+    /** Fraction of judged relevant items retrieved at the selected cutoff. */
     recall: number;
+    /** Failed per-query floors, compared against unrounded scores. */
+    violations?: string[];
 }
-/** Documents the eval result payload exchanged by command, SDK, and package integrations. */
+/** Rounded ranking evidence and the verdict computed from unrounded metrics. */
 export interface EvalResult {
-    /** Value that configures or reports k for this contract. */
+    /** Maximum ranking positions scored for each query. */
     k: number;
     /** Number of query entries represented by this result. */
     query_count: number;
-    /** Value that configures or reports aggregate for this contract. */
+    /** Equal-weight macro averages across all evaluated queries. */
     aggregate: {
         ndcg: number;
         mrr: number;
         precision: number;
         recall: number;
     };
-    /** Value that configures or reports queries for this contract. */
+    /** Individual query scores and any failed declared floors. */
     queries: EvalQueryReport[];
     /** Present only when `--fail-under` was supplied. */
     fail_under?: number;
-    /** Whether the aggregate nDCG met the gate (always true when no gate is set). */
+    /** Whether every declared per-query floor and optional aggregate nDCG threshold passed. */
     passed: boolean;
 }
 /**
@@ -60,8 +62,8 @@ export interface EvalResult {
  * (default `<pmRoot>/search/eval-queries.json`, overridable via `--queries`),
  * runs each query through {@link runSearch} at the resolved mode, scores the
  * returned ranking with the nDCG/MRR/precision/recall metrics, and macro-averages
- * across queries. When `--fail-under` is supplied, `passed` reflects whether the
- * aggregate nDCG@k cleared the threshold; the CLI layer maps a failed gate to a
- * non-zero exit code.
+ * across queries. `passed` requires every declared per-query minimum and, when
+ * supplied, the aggregate `--fail-under` threshold. The CLI maps any failed
+ * threshold to a non-zero exit code.
  */
 export declare const runEval: (options: EvalOptions, global: GlobalOptions) => Promise<EvalResult>;

package/dist/sdk/eval.js

@@ -1,2 +1,2 @@
-!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="7f6cfdef-5456-5716-9067-8ccadf077f5f")}catch(e2){}})();import{assertInitializedTracker}from"./environment/tracker-preflight.js";import fs from"node:fs/promises";import path from"node:path";import{aggregateEvalMetrics,DEFAULT_EVAL_K,EVAL_QUERY_SET_EXAMPLE,EVAL_QUERY_SET_SCHEMA_ID,evaluateRanking,parseEvalQuerySet}from"../core/search/eval.js";import{EXIT_CODE}from"../core/shared/constants.js";import{PmCliError}from"../core/shared/errors.js";import{coercePositiveInteger}from"../core/shared/primitives.js";import{resolvePmRoot}from"../core/store/paths.js";import{runSearch}from"./query/search.js";import{EVAL_QUERY_SET_CONTRACT,EVAL_QUERY_SET_EXAMPLE as EVAL_QUERY_SET_EXAMPLE2,EVAL_QUERY_SET_SCHEMA_ID as EVAL_QUERY_SET_SCHEMA_ID2,parseEvalQuerySet as parseEvalQuerySet2}from"../core/search/eval.js";const DEFAULT_EVAL_QUERIES_RELATIVE_PATH=path.join("search","eval-queries.json");const EVAL_QUERY_SET_ERROR_CONTEXT={schema:EVAL_QUERY_SET_SCHEMA_ID,examples:[JSON.stringify(EVAL_QUERY_SET_EXAMPLE)],nextSteps:["Create a git-tracked golden-query JSON file that follows the schema, then re-run pm eval."]};const VALID_EVAL_MODES=new Set(["keyword","semantic","hybrid"]);const parseEvalMode=raw=>{if(raw===void 0){return"keyword"}const normalized=raw.trim().toLowerCase();if(!VALID_EVAL_MODES.has(normalized)){throw new PmCliError("Eval --mode must be one of keyword|semantic|hybrid",EXIT_CODE.USAGE)}return normalized};const parseEvalK=raw=>{if(raw===void 0||raw===""){return DEFAULT_EVAL_K}const parsed=coercePositiveInteger(raw);if(parsed===null){throw new PmCliError("Eval --k must be a positive integer",EXIT_CODE.USAGE)}return parsed};const parseFailUnder=raw=>{if(raw===void 0||raw===""){return void 0}const normalized=String(raw).trim();const parsed=Number(normalized);if([normalized.length===0,!Number.isFinite(parsed),parsed<0,parsed>1].includes(true)){throw new PmCliError("Eval --fail-under must be a number in the range [0, 1]",EXIT_CODE.USAGE)}return parsed};const roundMetric=value=>Math.round(value*1e4)/1e4;const readEvalQuerySetFile=async queriesPath=>{try{return await fs.readFile(queriesPath,"utf8")}catch{throw new PmCliError(`Eval query set not found at ${queriesPath}`,EXIT_CODE.NOT_FOUND,{...EVAL_QUERY_SET_ERROR_CONTEXT,examples:[`echo '${JSON.stringify(EVAL_QUERY_SET_EXAMPLE)}' > ${queriesPath}`,"pm eval --queries ./my-eval.json"]})}};const parseEvalQuerySetJson=(raw,queriesPath)=>{try{return JSON.parse(raw)}catch(error){throw new PmCliError(`Eval query set at ${queriesPath} is not valid JSON: ${error instanceof Error?error.message:String(error)}`,EXIT_CODE.USAGE,EVAL_QUERY_SET_ERROR_CONTEXT)}};const validateEvalQuerySet=parsed=>{try{return parseEvalQuerySet(parsed)}catch(error){throw new PmCliError(error instanceof Error?error.message:String(error),EXIT_CODE.USAGE,EVAL_QUERY_SET_ERROR_CONTEXT)}};const loadEvalQuerySet=async queriesPath=>validateEvalQuerySet(parseEvalQuerySetJson(await readEvalQuerySetFile(queriesPath),queriesPath));const resolveEvalQueriesPath=(pmRoot,queries)=>queries?path.resolve(process.cwd(),queries):path.join(pmRoot,DEFAULT_EVAL_QUERIES_RELATIVE_PATH);const evaluateEvalQuery=async(evalQuery,defaultMode,k,global2)=>{const mode=evalQuery.mode??defaultMode;const searchResult=await runSearch(evalQuery.query,{mode,limit:String(k),fields:"id"},global2);const rankedIds=searchResult.items.map(item=>item.id).filter(id=>typeof id==="string");const metrics=evaluateRanking(rankedIds,new Set(evalQuery.relevant_ids),k);return{metrics,report:{query:evalQuery.query,mode,relevant_total:metrics.relevant_total,retrieved_relevant:metrics.retrieved_relevant,ndcg:roundMetric(metrics.ndcg),mrr:roundMetric(metrics.mrr),precision:roundMetric(metrics.precision),recall:roundMetric(metrics.recall)}}};const buildEvalResult=(k,reports,rawMetrics,failUnder)=>{const aggregate=aggregateEvalMetrics(rawMetrics);return{k,query_count:reports.length,aggregate:{ndcg:roundMetric(aggregate.ndcg),mrr:roundMetric(aggregate.mrr),precision:roundMetric(aggregate.precision),recall:roundMetric(aggregate.recall)},queries:reports,...failUnder!==void 0?{fail_under:failUnder}:{},passed:failUnder===void 0||aggregate.ndcg>=failUnder}};const runEval=async(options,global2)=>{const pmRoot=resolvePmRoot(process.cwd(),global2.path);await assertInitializedTracker(pmRoot);const k=parseEvalK(options.k);const defaultMode=parseEvalMode(options.mode);const failUnder=parseFailUnder(options.failUnder);const queriesPath=resolveEvalQueriesPath(pmRoot,options.queries);const querySet=await loadEvalQuerySet(queriesPath);const reports=[];const rawMetrics=[];for(const evalQuery of querySet.queries){const{report,metrics}=await evaluateEvalQuery(evalQuery,defaultMode,k,global2);rawMetrics.push(metrics);reports.push(report)}return buildEvalResult(k,reports,rawMetrics,failUnder)};export{DEFAULT_EVAL_QUERIES_RELATIVE_PATH,EVAL_QUERY_SET_CONTRACT,EVAL_QUERY_SET_EXAMPLE2 as EVAL_QUERY_SET_EXAMPLE,EVAL_QUERY_SET_SCHEMA_ID2 as EVAL_QUERY_SET_SCHEMA_ID,parseEvalQuerySet2 as parseEvalQuerySet,runEval};
+!(function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=new e.Error().stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="d4020969-4ce5-5f87-ae41-f5a3dec0a197")}catch(e2){}})();import{evaluateMetricFloors}from"../core/search/eval-thresholds.js";import{assertInitializedTracker}from"./environment/tracker-preflight.js";import fs from"node:fs/promises";import path from"node:path";import{aggregateEvalMetrics,DEFAULT_EVAL_K,EVAL_QUERY_SET_EXAMPLE,EVAL_QUERY_SET_SCHEMA_ID,evaluateRanking,parseEvalQuerySet}from"../core/search/eval.js";import{EXIT_CODE}from"../core/shared/constants.js";import{PmCliError}from"../core/shared/errors.js";import{coercePositiveInteger}from"../core/shared/primitives.js";import{resolvePmRoot}from"../core/store/paths.js";import{runSearch}from"./query/search.js";import{EVAL_QUERY_SET_CONTRACT,EVAL_QUERY_SET_EXAMPLE as EVAL_QUERY_SET_EXAMPLE2,EVAL_QUERY_SET_SCHEMA_ID as EVAL_QUERY_SET_SCHEMA_ID2,parseEvalQuerySet as parseEvalQuerySet2}from"../core/search/eval.js";const DEFAULT_EVAL_QUERIES_RELATIVE_PATH=path.join("search","eval-queries.json");const EVAL_QUERY_SET_ERROR_CONTEXT={schema:EVAL_QUERY_SET_SCHEMA_ID,examples:[JSON.stringify(EVAL_QUERY_SET_EXAMPLE)],nextSteps:["Create a git-tracked golden-query JSON file that follows the schema, then re-run pm eval."]};const VALID_EVAL_MODES=new Set(["keyword","semantic","hybrid"]);const parseEvalMode=raw=>{if(raw===void 0){return"keyword"}const normalized=raw.trim().toLowerCase();if(!VALID_EVAL_MODES.has(normalized)){throw new PmCliError("Eval --mode must be one of keyword|semantic|hybrid",EXIT_CODE.USAGE)}return normalized};const parseEvalK=raw=>{if(raw===void 0||raw===""){return DEFAULT_EVAL_K}const parsed=coercePositiveInteger(raw);if(parsed===null){throw new PmCliError("Eval --k must be a positive integer",EXIT_CODE.USAGE)}return parsed};const parseFailUnder=raw=>{if(raw===void 0||raw===""){return void 0}const normalized=String(raw).trim();const parsed=Number(normalized);if([normalized.length===0,!Number.isFinite(parsed),parsed<0,parsed>1].includes(true)){throw new PmCliError("Eval --fail-under must be a number in the range [0, 1]",EXIT_CODE.USAGE)}return parsed};const roundMetric=value=>Math.round(value*1e4)/1e4;const readEvalQuerySetFile=async queriesPath=>{try{return await fs.readFile(queriesPath,"utf8")}catch{throw new PmCliError(`Eval query set not found at ${queriesPath}`,EXIT_CODE.NOT_FOUND,{...EVAL_QUERY_SET_ERROR_CONTEXT,examples:[`echo '${JSON.stringify(EVAL_QUERY_SET_EXAMPLE)}' > ${queriesPath}`,"pm eval --queries ./my-eval.json"]})}};const parseEvalQuerySetJson=(raw,queriesPath)=>{try{return JSON.parse(raw)}catch(error){throw new PmCliError(`Eval query set at ${queriesPath} is not valid JSON: ${error instanceof Error?error.message:String(error)}`,EXIT_CODE.USAGE,EVAL_QUERY_SET_ERROR_CONTEXT)}};const validateEvalQuerySet=parsed=>{try{return parseEvalQuerySet(parsed)}catch(error){throw new PmCliError(error instanceof Error?error.message:String(error),EXIT_CODE.USAGE,EVAL_QUERY_SET_ERROR_CONTEXT)}};const loadEvalQuerySet=async queriesPath=>validateEvalQuerySet(parseEvalQuerySetJson(await readEvalQuerySetFile(queriesPath),queriesPath));const evaluateEvalQuery=async(evalQuery,defaultMode,k,global2)=>{const mode=evalQuery.mode??defaultMode;const searchResult=await runSearch(evalQuery.query,{mode,limit:String(k),fields:"id"},global2);const rankedIds=searchResult.items.map(item=>item.id).filter(id=>typeof id==="string");const metrics=evaluateRanking(rankedIds,new Set(evalQuery.relevant_ids),k);return{metrics,report:{query:evalQuery.query,...evalQuery.minimum===void 0?{}:{violations:evaluateMetricFloors(metrics,evalQuery.minimum)},mode,relevant_total:metrics.relevant_total,retrieved_relevant:metrics.retrieved_relevant,ndcg:roundMetric(metrics.ndcg),mrr:roundMetric(metrics.mrr),precision:roundMetric(metrics.precision),recall:roundMetric(metrics.recall)}}};const buildEvalResult=(k,reports,rawMetrics,failUnder)=>{const aggregate=aggregateEvalMetrics(rawMetrics);return{k,query_count:reports.length,aggregate:{ndcg:roundMetric(aggregate.ndcg),mrr:roundMetric(aggregate.mrr),precision:roundMetric(aggregate.precision),recall:roundMetric(aggregate.recall)},queries:reports,...failUnder!==void 0?{fail_under:failUnder}:{},passed:(failUnder===void 0||aggregate.ndcg>=failUnder)&&reports.every(report=>(report.violations?.length??0)===0)}};const runEval=async(options,global2)=>{const pmRoot=resolvePmRoot(process.cwd(),global2.path);await assertInitializedTracker(pmRoot);const k=parseEvalK(options.k);const defaultMode=parseEvalMode(options.mode);const failUnder=parseFailUnder(options.failUnder);const queriesPath=options.queries?path.resolve(process.cwd(),options.queries):path.join(pmRoot,DEFAULT_EVAL_QUERIES_RELATIVE_PATH);const querySet=await loadEvalQuerySet(queriesPath);const reports=[];const rawMetrics=[];for(const evalQuery of querySet.queries){const{report,metrics}=await evaluateEvalQuery(evalQuery,defaultMode,k,global2);rawMetrics.push(metrics);reports.push(report)}return buildEvalResult(k,reports,rawMetrics,failUnder)};export{DEFAULT_EVAL_QUERIES_RELATIVE_PATH,EVAL_QUERY_SET_CONTRACT,EVAL_QUERY_SET_EXAMPLE2 as EVAL_QUERY_SET_EXAMPLE,EVAL_QUERY_SET_SCHEMA_ID2 as EVAL_QUERY_SET_SCHEMA_ID,parseEvalQuerySet2 as parseEvalQuerySet,runEval};
 //# sourceMappingURL=eval.js.map