evaliphy 1.0.1-beta.3 → 1.0.1-beta.5

package/dist/bin.mjs

@@ -3744,7 +3744,14 @@ var ConfigLoader = class ConfigLoader {
 		const configPath = this.findConfigFile(cwd);
 		if (!configPath) {
 			this.configLogger.debug("No config file found, using defaults if applicable");
-			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, "Provided config file is not correct or does not exist.");
+			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Could not load configuration. Please check the following:
+
+  - Does "evaliphy.config.ts" exist in your project root?
+  - Is the path correct if you specified a custom config location?
+  - Does the config file export a default defineConfig({}) call?
+
+  Expected location: ${process.cwd()}/evaliphy.config.ts
+  Visit https://evaliphy.com/docs/configuration for more details.`);
 		}
 		if (!this.cachedConfig) {
 			this.configLogger.debug({ configPath }, "Loading base configuration from file");
@@ -3757,7 +3764,10 @@ var ConfigLoader = class ConfigLoader {
 						error: parsed.error,
 						configPath
 					}, "Configuration validation failed");
-					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}: ${parsed.error.message}`);
+					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+            Docs: https://evaliphy.com/docs/configuration`);
 				}
 				this.cachedConfig = mergeConfigs(parsed.data, {}, this.cliOverrides);
 				this.cachedConfig.configFile = configPath;
@@ -3767,7 +3777,11 @@ var ConfigLoader = class ConfigLoader {
 					configPath
 				}, "Error loading configuration");
 				if (error instanceof EvaliphyError) throw error;
-				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}`);
+				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+              Docs: https://evaliphy.com/docs/configuration
+              `);
 			}
 		}
 		const finalConfig = mergeConfigs(this.cachedConfig, transientOverrides, {});
@@ -4006,50 +4020,99 @@ function handleFatalError(err) {
 //#endregion
 //#region packages/cli/src/initProject/createFolderStructure.ts
 const require$1 = createRequire(import.meta.url);
+const green = (s) => `\x1b[32m${s}\x1b[0m`;
+const bold = (s) => `\x1b[1m${s}\x1b[0m`;
+const dim = (s) => `\x1b[2m${s}\x1b[0m`;
+const red = (s) => `\x1b[31m${s}\x1b[0m`;
 function createProject(projectName) {
 	const __filename = fileURLToPath(import.meta.url);
 	const __dirname = path.dirname(__filename);
 	const { version } = require$1(path.join(__dirname, "../package.json"));
+	if (!/^[a-z0-9-_]+$/i.test(projectName)) {
+		console.error(red(`\n  ✗ Invalid project name "${projectName}".`));
+		console.error(dim("    Use only letters, numbers, hyphens and underscores.\n"));
+		process.exit(1);
+	}
 	const rootPath = path.join(process.cwd(), projectName);
 	if (fs.existsSync(rootPath)) {
-		console.error("Folder already exists.");
+		console.error(red(`\n  ✗ Directory "${projectName}" already exists.`));
+		console.error(dim(`    Choose a different name or remove the existing directory.\n`));
 		process.exit(1);
 	}
-	fs.mkdirSync(rootPath, { recursive: true });
-	createStructure(rootPath, {
-		evals: { "example.eval.ts": `import { evaluate } from 'evaliphy';
+	console.log(`\n  Creating project ${bold(projectName)}...\n`);
+	const structure = {
+		evals: { "example.eval.ts": `import { evaluate, expect } from 'evaliphy';
+
+/**
+ * Example evaluation — replace this with your own RAG endpoint and samples.
+ * Docs: https://evaliphy.com/docs/quick-start
+ */
+evaluate("context handling: multiple chunks", async ({ httpClient }) => {
+  const query   = "What are the support hours?";
+  const context = [
+    "Support is available 24/7 via email.",
+    "Live chat support is open from 9 AM to 5 PM EST.",
+    "Phone support is currently unavailable."
+  ];
+
+  const res  = await httpClient.post("/api/generate", { prompt: query });
+  const data = await res.json();
 
-evaluate('basic test', async ({ httpClient }) => {
-  // add your evaluate code
+  await expect({
+    query,
+    response: data.content,
+    context,
+  }).toBeFaithful({ threshold: 0.7 });
 });
 ` },
-		utils: {},
+		".env.example": `# Copy this file to .env and fill in your values
+OPENAI_API_KEY=your-api-key-here
+`,
+		".gitignore": `.env
+node_modules
+dist
+results
+`,
 		"evaliphy.config.ts": `import { defineConfig } from 'evaliphy';
 
 export default defineConfig({
-  baseUrl: 'http://localhost:8080',
-  testDir: './evals',
+  http: {
+    baseUrl: "http://localhost:8080",
+    timeout: 10_000,
+  },
+  evalDir: './evals',
   llmAsJudgeConfig: {
     model: 'gpt-4o-mini',
     provider: {
-      type: 'gateway',
-      url: 'https://openrouter.ai/api/v1',
-      apiKey: process.env.OPENROUTER_API_KEY,
+      type: 'openai',
+      apiKey: process.env.OPENAI_API_KEY,
     },
-    promptsDir: './prompts',
-    temperature: 0
+    temperature: 0,
   },
 });
 `,
-		"tsconfig.json": JSON.stringify({ compilerOptions: {
-			target: "ES2020",
-			module: "ESNext",
-			moduleResolution: "bundler",
-			strict: true,
-			esModuleInterop: true,
-			skipLibCheck: true
-		} }, null, 2)
-	});
+		"tsconfig.json": JSON.stringify({
+			compilerOptions: {
+				target: "ES2020",
+				module: "ESNext",
+				moduleResolution: "bundler",
+				strict: true,
+				esModuleInterop: true,
+				skipLibCheck: true,
+				outDir: "./dist"
+			},
+			include: ["**/*.ts"],
+			exclude: ["node_modules", "dist"]
+		}, null, 2)
+	};
+	try {
+		fs.mkdirSync(rootPath, { recursive: true });
+		createStructure(rootPath, structure);
+	} catch (err) {
+		console.error(red("\n  ✗ Failed to create project files."));
+		console.error(dim(`    ${err.message}\n`));
+		process.exit(1);
+	}
 	const pkg = {
 		name: projectName,
 		version: "1.0.0",
@@ -4064,19 +4127,42 @@ export default defineConfig({
 			typescript: "^5.0.0"
 		}
 	};
-	fs.writeFileSync(path.join(rootPath, "package.json"), JSON.stringify(pkg, null, 2));
-	console.log(`\nProject "${projectName}" is ready!`);
-	console.log(`\nNext steps:`);
-	console.log(`  cd ${projectName}`);
-	console.log(`  npm test`);
+	try {
+		fs.writeFileSync(path.join(rootPath, "package.json"), JSON.stringify(pkg, null, 2));
+	} catch (err) {
+		console.error(red("\n  ✗ Failed to write package.json."));
+		console.error(dim(`    ${err.message}\n`));
+		process.exit(1);
+	}
+	console.log(`  ${green("✓")} evaliphy.config.ts`);
+	console.log(`  ${green("✓")} evals/example.eval.ts`);
+	console.log(`  ${green("✓")} .env.example`);
+	console.log(`  ${green("✓")} .gitignore`);
+	console.log(`  ${green("✓")} tsconfig.json`);
+	console.log(`  ${green("✓")} package.json`);
+	console.log(`
+  ${green("✓")} ${bold(`Project "${projectName}" created successfully!`)}
+
+  ${bold("Next steps:")}
+    ${dim("$")} Ensuer you have Node JS either v24.0.0 or higher.
+    ${dim("$")} cd ${projectName}
+    ${dim("$")} cp .env.example .env     ${dim("← add your OPENAI_API_KEY")}
+    ${dim("$")} npm install
+    ${dim("$")} npm test
+
+  ${dim(`Docs     → https://evaliphy.com/docs`)}
+  ${dim(`Examples → https://evaliphy.com/docs/examples`)}
+`);
 }
 function createStructure(basePath, structure) {
-	for (const name in structure) {
+	for (const [name, content] of Object.entries(structure)) {
 		const fullPath = path.join(basePath, name);
-		if (typeof structure[name] === "string") fs.writeFileSync(fullPath, structure[name]);
-		else {
+		if (typeof content === "string") {
+			fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+			fs.writeFileSync(fullPath, content, "utf8");
+		} else {
 			fs.mkdirSync(fullPath, { recursive: true });
-			createStructure(fullPath, structure[name]);
+			createStructure(fullPath, content);
 		}
 	}
 }
@@ -31257,9 +31343,10 @@ var ConsoleReporter = class {
 		const duration = this.formatDuration(payload.duration);
 		console.log("\n" + this.getSeparator());
 		[
-			import_picocolors.default.bold("Tests:   ") + (payload.failed > 0 ? import_picocolors.default.red(`${payload.failed} failed`) + import_picocolors.default.dim(", ") : "") + import_picocolors.default.green(`${payload.passed} passed`) + import_picocolors.default.dim(`, ${total} total`),
-			import_picocolors.default.bold("Time:    ") + import_picocolors.default.white(duration),
-			import_picocolors.default.bold("Run ID:  ") + import_picocolors.default.dim(payload.runId)
+			import_picocolors.default.bold("Tests:       ") + (payload.failed > 0 ? import_picocolors.default.red(`${payload.failed} failed`) + import_picocolors.default.dim(", ") : "") + import_picocolors.default.green(`${payload.passed} passed`) + import_picocolors.default.dim(`, ${total} total`),
+			import_picocolors.default.bold("Time:        ") + import_picocolors.default.white(duration),
+			import_picocolors.default.bold("Run ID:      ") + import_picocolors.default.dim(payload.runId),
+			import_picocolors.default.bold("HTML Report: ") + import_picocolors.default.dim("report/report-" + payload.runId + ".html")
 		].forEach((line) => console.log(` ${line}`));
 		console.log(this.getSeparator() + "\n");
 		if (payload.failed > 0) console.log(`  ${import_picocolors.default.red(import_picocolors.default.bold("FAIL"))} ${import_picocolors.default.red("Run failed")}\n`);

package/dist/index.cjs

@@ -183,7 +183,14 @@ var ConfigLoader = class ConfigLoader {
 		const configPath = this.findConfigFile(cwd);
 		if (!configPath) {
 			this.configLogger.debug("No config file found, using defaults if applicable");
-			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, "Provided config file is not correct or does not exist.");
+			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Could not load configuration. Please check the following:
+
+  - Does "evaliphy.config.ts" exist in your project root?
+  - Is the path correct if you specified a custom config location?
+  - Does the config file export a default defineConfig({}) call?
+
+  Expected location: ${process.cwd()}/evaliphy.config.ts
+  Visit https://evaliphy.com/docs/configuration for more details.`);
 		}
 		if (!this.cachedConfig) {
 			this.configLogger.debug({ configPath }, "Loading base configuration from file");
@@ -196,7 +203,10 @@ var ConfigLoader = class ConfigLoader {
 						error: parsed.error,
 						configPath
 					}, "Configuration validation failed");
-					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}: ${parsed.error.message}`);
+					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+            Docs: https://evaliphy.com/docs/configuration`);
 				}
 				this.cachedConfig = mergeConfigs(parsed.data, {}, this.cliOverrides);
 				this.cachedConfig.configFile = configPath;
@@ -206,7 +216,11 @@ var ConfigLoader = class ConfigLoader {
 					configPath
 				}, "Error loading configuration");
 				if (error instanceof EvaliphyError) throw error;
-				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}`);
+				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+              Docs: https://evaliphy.com/docs/configuration
+              `);
 			}
 		}
 		const finalConfig = mergeConfigs(this.cachedConfig, transientOverrides, {});
@@ -672,12 +686,29 @@ var AssertionEngine = class {
 	static loadPrompt(matcherName, assertionDef, config) {
 		const configDir = config.configFile ? node_path.default.dirname(config.configFile) : process.cwd();
 		const customPromptsDir = config.llmAsJudgeConfig?.promptsDir ? node_path.default.resolve(configDir, config.llmAsJudgeConfig.promptsDir) : null;
-		const defaultPromptsDir = node_path.default.join(process.cwd(), "packages/prompts");
-		let promptPath = customPromptsDir ? node_path.default.join(customPromptsDir, `${matcherName}.md`) : null;
+		const localPromptsDir = node_path.default.join(process.cwd(), "prompts");
+		const __dirname = node_path.default.dirname((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
+		const sdkPromptsDirSource = node_path.default.resolve(__dirname, "../../prompts");
+		const sdkPromptsDirDist = node_path.default.resolve(__dirname, "./prompts");
+		[
+			customPromptsDir ? node_path.default.join(customPromptsDir, `${matcherName}.md`) : null,
+			node_path.default.join(localPromptsDir, `${matcherName}.md`),
+			node_path.default.join(sdkPromptsDirDist, `${matcherName}.md`),
+			node_path.default.join(sdkPromptsDirSource, `${matcherName}.md`)
+		].filter(Boolean);
 		try {
-			if (promptPath && PromptLoader.exists(promptPath)) return PromptLoader.load(promptPath, assertionDef);
-			promptPath = node_path.default.join(defaultPromptsDir, `${matcherName}.md`);
-			return PromptLoader.load(promptPath, assertionDef);
+			if (customPromptsDir) {
+				const customPath = node_path.default.join(customPromptsDir, `${matcherName}.md`);
+				if (PromptLoader.exists(customPath)) return PromptLoader.load(customPath, assertionDef);
+				else console.warn(`Custom prompt file not found at: ${customPath}. Falling back to defaults.`);
+			}
+			const fallbackPaths = [
+				node_path.default.join(localPromptsDir, `${matcherName}.md`),
+				node_path.default.join(sdkPromptsDirDist, `${matcherName}.md`),
+				node_path.default.join(sdkPromptsDirSource, `${matcherName}.md`)
+			];
+			for (const promptPath of fallbackPaths) if (PromptLoader.exists(promptPath)) return PromptLoader.load(promptPath, assertionDef);
+			return PromptLoader.load(node_path.default.join(sdkPromptsDirDist, `${matcherName}.md`), assertionDef);
 		} catch (error) {
 			throw new EvaliphyError(EvaliphyErrorCode.PROMPT_LOAD_ERROR, `Failed to load prompt for "${matcherName}": ${error.message}`, "Check your prompt file formatting and variables.", error);
 		}

package/dist/index.mjs

@@ -1,7 +1,7 @@
 import { i as __toESM, n as __commonJSMin, t as require_token_error } from "./token-error-TubvLIoa.mjs";
 import fs, { existsSync } from "node:fs";
 import path, { join } from "node:path";
-import { pathToFileURL } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
 import { pino } from "pino";
 import { z } from "zod";
 import { AsyncLocalStorage } from "node:async_hooks";
@@ -180,7 +180,14 @@ var ConfigLoader = class ConfigLoader {
 		const configPath = this.findConfigFile(cwd);
 		if (!configPath) {
 			this.configLogger.debug("No config file found, using defaults if applicable");
-			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, "Provided config file is not correct or does not exist.");
+			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Could not load configuration. Please check the following:
+
+  - Does "evaliphy.config.ts" exist in your project root?
+  - Is the path correct if you specified a custom config location?
+  - Does the config file export a default defineConfig({}) call?
+
+  Expected location: ${process.cwd()}/evaliphy.config.ts
+  Visit https://evaliphy.com/docs/configuration for more details.`);
 		}
 		if (!this.cachedConfig) {
 			this.configLogger.debug({ configPath }, "Loading base configuration from file");
@@ -193,7 +200,10 @@ var ConfigLoader = class ConfigLoader {
 						error: parsed.error,
 						configPath
 					}, "Configuration validation failed");
-					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}: ${parsed.error.message}`);
+					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+            Docs: https://evaliphy.com/docs/configuration`);
 				}
 				this.cachedConfig = mergeConfigs(parsed.data, {}, this.cliOverrides);
 				this.cachedConfig.configFile = configPath;
@@ -203,7 +213,11 @@ var ConfigLoader = class ConfigLoader {
 					configPath
 				}, "Error loading configuration");
 				if (error instanceof EvaliphyError) throw error;
-				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}`);
+				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+              Docs: https://evaliphy.com/docs/configuration
+              `);
 			}
 		}
 		const finalConfig = mergeConfigs(this.cachedConfig, transientOverrides, {});
@@ -669,12 +683,29 @@ var AssertionEngine = class {
 	static loadPrompt(matcherName, assertionDef, config) {
 		const configDir = config.configFile ? path.dirname(config.configFile) : process.cwd();
 		const customPromptsDir = config.llmAsJudgeConfig?.promptsDir ? path.resolve(configDir, config.llmAsJudgeConfig.promptsDir) : null;
-		const defaultPromptsDir = path.join(process.cwd(), "packages/prompts");
-		let promptPath = customPromptsDir ? path.join(customPromptsDir, `${matcherName}.md`) : null;
+		const localPromptsDir = path.join(process.cwd(), "prompts");
+		const __dirname = path.dirname(fileURLToPath(import.meta.url));
+		const sdkPromptsDirSource = path.resolve(__dirname, "../../prompts");
+		const sdkPromptsDirDist = path.resolve(__dirname, "./prompts");
+		[
+			customPromptsDir ? path.join(customPromptsDir, `${matcherName}.md`) : null,
+			path.join(localPromptsDir, `${matcherName}.md`),
+			path.join(sdkPromptsDirDist, `${matcherName}.md`),
+			path.join(sdkPromptsDirSource, `${matcherName}.md`)
+		].filter(Boolean);
 		try {
-			if (promptPath && PromptLoader.exists(promptPath)) return PromptLoader.load(promptPath, assertionDef);
-			promptPath = path.join(defaultPromptsDir, `${matcherName}.md`);
-			return PromptLoader.load(promptPath, assertionDef);
+			if (customPromptsDir) {
+				const customPath = path.join(customPromptsDir, `${matcherName}.md`);
+				if (PromptLoader.exists(customPath)) return PromptLoader.load(customPath, assertionDef);
+				else console.warn(`Custom prompt file not found at: ${customPath}. Falling back to defaults.`);
+			}
+			const fallbackPaths = [
+				path.join(localPromptsDir, `${matcherName}.md`),
+				path.join(sdkPromptsDirDist, `${matcherName}.md`),
+				path.join(sdkPromptsDirSource, `${matcherName}.md`)
+			];
+			for (const promptPath of fallbackPaths) if (PromptLoader.exists(promptPath)) return PromptLoader.load(promptPath, assertionDef);
+			return PromptLoader.load(path.join(sdkPromptsDirDist, `${matcherName}.md`), assertionDef);
 		} catch (error) {
 			throw new EvaliphyError(EvaliphyErrorCode.PROMPT_LOAD_ERROR, `Failed to load prompt for "${matcherName}": ${error.message}`, "Check your prompt file formatting and variables.", error);
 		}