evaliphy 1.0.1-beta.2 → 1.0.1-beta.4

package/dist/bin.mjs

@@ -1,15 +1,15 @@
 #!/usr/bin/env node
 import { createRequire } from "node:module";
-import fs, { existsSync, readFileSync } from "node:fs";
+import fs, { existsSync } from "node:fs";
 import path, { join } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { pino } from "pino";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { program } from "commander";
-import fs$1 from "fs";
 import path$1 from "path";
-import { execSync } from "child_process";
+import { fileURLToPath as fileURLToPath$1 } from "url";
 import fg from "fast-glob";
+import fs$1 from "fs";
 import fs$2 from "node:fs/promises";
 //#region \0rolldown/runtime.js
 var __create = Object.create;
@@ -3744,7 +3744,14 @@ var ConfigLoader = class ConfigLoader {
 		const configPath = this.findConfigFile(cwd);
 		if (!configPath) {
 			this.configLogger.debug("No config file found, using defaults if applicable");
-			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, "Provided config file is not correct or does not exist.");
+			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Could not load configuration. Please check the following:
+
+  - Does "evaliphy.config.ts" exist in your project root?
+  - Is the path correct if you specified a custom config location?
+  - Does the config file export a default defineConfig({}) call?
+
+  Expected location: ${process.cwd()}/evaliphy.config.ts
+  Visit https://evaliphy.com/docs/configuration for more details.`);
 		}
 		if (!this.cachedConfig) {
 			this.configLogger.debug({ configPath }, "Loading base configuration from file");
@@ -3757,7 +3764,10 @@ var ConfigLoader = class ConfigLoader {
 						error: parsed.error,
 						configPath
 					}, "Configuration validation failed");
-					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}: ${parsed.error.message}`);
+					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+            Docs: https://evaliphy.com/docs/configuration`);
 				}
 				this.cachedConfig = mergeConfigs(parsed.data, {}, this.cliOverrides);
 				this.cachedConfig.configFile = configPath;
@@ -3767,7 +3777,11 @@ var ConfigLoader = class ConfigLoader {
 					configPath
 				}, "Error loading configuration");
 				if (error instanceof EvaliphyError) throw error;
-				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}`);
+				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+              Docs: https://evaliphy.com/docs/configuration
+              `);
 			}
 		}
 		const finalConfig = mergeConfigs(this.cachedConfig, transientOverrides, {});
@@ -4005,48 +4019,100 @@ function handleFatalError(err) {
 }
 //#endregion
 //#region packages/cli/src/initProject/createFolderStructure.ts
+const require$1 = createRequire(import.meta.url);
+const green = (s) => `\x1b[32m${s}\x1b[0m`;
+const bold = (s) => `\x1b[1m${s}\x1b[0m`;
+const dim = (s) => `\x1b[2m${s}\x1b[0m`;
+const red = (s) => `\x1b[31m${s}\x1b[0m`;
 function createProject(projectName) {
-	const rootPath = path$1.join(process.cwd(), projectName);
-	if (fs$1.existsSync(rootPath)) {
-		console.error("Folder already exists.");
+	const __filename = fileURLToPath(import.meta.url);
+	const __dirname = path.dirname(__filename);
+	const { version } = require$1(path.join(__dirname, "../package.json"));
+	if (!/^[a-z0-9-_]+$/i.test(projectName)) {
+		console.error(red(`\n  ✗ Invalid project name "${projectName}".`));
+		console.error(dim("    Use only letters, numbers, hyphens and underscores.\n"));
+		process.exit(1);
+	}
+	const rootPath = path.join(process.cwd(), projectName);
+	if (fs.existsSync(rootPath)) {
+		console.error(red(`\n  ✗ Directory "${projectName}" already exists.`));
+		console.error(dim(`    Choose a different name or remove the existing directory.\n`));
 		process.exit(1);
 	}
-	fs$1.mkdirSync(rootPath, { recursive: true });
-	createStructure(rootPath, {
-		evals: { "example.eval.ts": `
-import { evaluate } from 'evaliphy';
+	console.log(`\n  Creating project ${bold(projectName)}...\n`);
+	const structure = {
+		evals: { "example.eval.ts": `import { evaluate, expect } from 'evaliphy';
+
+/**
+ * Example evaluation — replace this with your own RAG endpoint and samples.
+ * Docs: https://evaliphy.com/docs/quick-start
+ */
+evaluate("context handling: multiple chunks", async ({ httpClient }) => {
+  const query   = "What are the support hours?";
+  const context = [
+    "Support is available 24/7 via email.",
+    "Live chat support is open from 9 AM to 5 PM EST.",
+    "Phone support is currently unavailable."
+  ];
+
+  const res  = await httpClient.post("/api/generate", { prompt: query });
+  const data = await res.json();
 
-evaluate('basic test', async ({ httpClient }) => {
-  // add your evaluate code
+  await expect({
+    query,
+    response: data.content,
+    context,
+  }).toBeFaithful({ threshold: 0.7 });
 });
 ` },
-		utils: {},
-		"evaliphy.config.ts": `
-import { defineConfig } from 'evaliphy';
+		".env.example": `# Copy this file to .env and fill in your values
+OPENAI_API_KEY=your-api-key-here
+`,
+		".gitignore": `.env
+node_modules
+dist
+results
+`,
+		"evaliphy.config.ts": `import { defineConfig } from 'evaliphy';
 
 export default defineConfig({
-  baseUrl: 'http://localhost:8080',
-  testDir: './evals',
+  http: {
+    baseUrl: "http://localhost:8080",
+    timeout: 10_000,
+  },
+  evalDir: './evals',
   llmAsJudgeConfig: {
     model: 'gpt-4o-mini',
     provider: {
-      type: 'gateway',
-      url: 'https://openrouter.ai/api/v1',
-      apiKey: process.env.OPENROUTER_API_KEY,
+      type: 'openai',
+      apiKey: process.env.OPENAI_API_KEY,
     },
-    promptsDir: './prompts',
-    temperature: 0
+    temperature: 0,
   },
 });
 `,
-		"tsconfig.json": JSON.stringify({ compilerOptions: {
-			target: "ES2020",
-			module: "CommonJS",
-			strict: true,
-			esModuleInterop: true,
-			skipLibCheck: true
-		} }, null, 2)
-	});
+		"tsconfig.json": JSON.stringify({
+			compilerOptions: {
+				target: "ES2020",
+				module: "ESNext",
+				moduleResolution: "bundler",
+				strict: true,
+				esModuleInterop: true,
+				skipLibCheck: true,
+				outDir: "./dist"
+			},
+			include: ["**/*.ts"],
+			exclude: ["node_modules", "dist"]
+		}, null, 2)
+	};
+	try {
+		fs.mkdirSync(rootPath, { recursive: true });
+		createStructure(rootPath, structure);
+	} catch (err) {
+		console.error(red("\n  ✗ Failed to create project files."));
+		console.error(dim(`    ${err.message}\n`));
+		process.exit(1);
+	}
 	const pkg = {
 		name: projectName,
 		version: "1.0.0",
@@ -4057,29 +4123,46 @@ export default defineConfig({
 			build: "tsc"
 		},
 		devDependencies: {
-			"evaliphy": "^1.0.0",
-			"typescript": "^5.0.0"
+			"evaliphy": `^${version}`,
+			typescript: "^5.0.0"
 		}
 	};
-	fs$1.writeFileSync(path$1.join(rootPath, "package.json"), JSON.stringify(pkg, null, 2));
 	try {
-		console.log("📦 Installing dependencies...");
-		execSync("npm install", {
-			cwd: rootPath,
-			stdio: "inherit"
-		});
+		fs.writeFileSync(path.join(rootPath, "package.json"), JSON.stringify(pkg, null, 2));
 	} catch (err) {
-		console.error("Failed to install dependencies");
+		console.error(red("\n  ✗ Failed to write package.json."));
+		console.error(dim(`    ${err.message}\n`));
+		process.exit(1);
 	}
-	console.log(`\n✅ Project "${projectName}" is ready!`);
+	console.log(`  ${green("✓")} evaliphy.config.ts`);
+	console.log(`  ${green("✓")} evals/example.eval.ts`);
+	console.log(`  ${green("✓")} .env.example`);
+	console.log(`  ${green("✓")} .gitignore`);
+	console.log(`  ${green("✓")} tsconfig.json`);
+	console.log(`  ${green("✓")} package.json`);
+	console.log(`
+  ${green("✓")} ${bold(`Project "${projectName}" created successfully!`)}
+
+  ${bold("Next steps:")}
+    ${dim("$")} Ensuer you have Node JS either v24.0.0 or higher.
+    ${dim("$")} cd ${projectName}
+    ${dim("$")} cp .env.example .env     ${dim("← add your OPENAI_API_KEY")}
+    ${dim("$")} npm install
+    ${dim("$")} npm test
+
+  ${dim(`Docs     → https://evaliphy.com/docs`)}
+  ${dim(`Examples → https://evaliphy.com/docs/examples`)}
+`);
 }
 function createStructure(basePath, structure) {
-	for (const name in structure) {
-		const fullPath = path$1.join(basePath, name);
-		if (typeof structure[name] === "string") fs$1.writeFileSync(fullPath, structure[name]);
-		else {
-			fs$1.mkdirSync(fullPath, { recursive: true });
-			createStructure(fullPath, structure[name]);
+	for (const [name, content] of Object.entries(structure)) {
+		const fullPath = path.join(basePath, name);
+		if (typeof content === "string") {
+			fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+			fs.writeFileSync(fullPath, content, "utf8");
+		} else {
+			fs.mkdirSync(fullPath, { recursive: true });
+			createStructure(fullPath, content);
 		}
 	}
 }
@@ -31260,9 +31343,10 @@ var ConsoleReporter = class {
 		const duration = this.formatDuration(payload.duration);
 		console.log("\n" + this.getSeparator());
 		[
-			import_picocolors.default.bold("Tests:   ") + (payload.failed > 0 ? import_picocolors.default.red(`${payload.failed} failed`) + import_picocolors.default.dim(", ") : "") + import_picocolors.default.green(`${payload.passed} passed`) + import_picocolors.default.dim(`, ${total} total`),
-			import_picocolors.default.bold("Time:    ") + import_picocolors.default.white(duration),
-			import_picocolors.default.bold("Run ID:  ") + import_picocolors.default.dim(payload.runId)
+			import_picocolors.default.bold("Tests:       ") + (payload.failed > 0 ? import_picocolors.default.red(`${payload.failed} failed`) + import_picocolors.default.dim(", ") : "") + import_picocolors.default.green(`${payload.passed} passed`) + import_picocolors.default.dim(`, ${total} total`),
+			import_picocolors.default.bold("Time:        ") + import_picocolors.default.white(duration),
+			import_picocolors.default.bold("Run ID:      ") + import_picocolors.default.dim(payload.runId),
+			import_picocolors.default.bold("HTML Report: ") + import_picocolors.default.dim("report/report-" + payload.runId + ".html")
 		].forEach((line) => console.log(` ${line}`));
 		console.log(this.getSeparator() + "\n");
 		if (payload.failed > 0) console.log(`  ${import_picocolors.default.red(import_picocolors.default.bold("FAIL"))} ${import_picocolors.default.red("Run failed")}\n`);
@@ -32131,8 +32215,9 @@ async function runRegistry(explicitFile) {
 }
 //#endregion
 //#region packages/cli/src/bin.ts
-const packageJsonPath = join(fileURLToPath(new URL(".", import.meta.url)), "../package.json");
-const { version } = JSON.parse(readFileSync(packageJsonPath, "utf8"));
+const __filename = fileURLToPath$1(import.meta.url);
+const __dirname = path$1.dirname(__filename);
+const { version } = __require(path$1.join(__dirname, "../package.json"));
 program.name("evaliphy").version(version).description("Evaliphy — eval runner for LLM pipelines");
 program.command("init <project-name>").description("Create project structure as recommended by Evaliphy.").action((projectName) => {
 	createProject(projectName);

package/dist/index.cjs

@@ -183,7 +183,14 @@ var ConfigLoader = class ConfigLoader {
 		const configPath = this.findConfigFile(cwd);
 		if (!configPath) {
 			this.configLogger.debug("No config file found, using defaults if applicable");
-			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, "Provided config file is not correct or does not exist.");
+			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Could not load configuration. Please check the following:
+
+  - Does "evaliphy.config.ts" exist in your project root?
+  - Is the path correct if you specified a custom config location?
+  - Does the config file export a default defineConfig({}) call?
+
+  Expected location: ${process.cwd()}/evaliphy.config.ts
+  Visit https://evaliphy.com/docs/configuration for more details.`);
 		}
 		if (!this.cachedConfig) {
 			this.configLogger.debug({ configPath }, "Loading base configuration from file");
@@ -196,7 +203,10 @@ var ConfigLoader = class ConfigLoader {
 						error: parsed.error,
 						configPath
 					}, "Configuration validation failed");
-					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}: ${parsed.error.message}`);
+					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+            Docs: https://evaliphy.com/docs/configuration`);
 				}
 				this.cachedConfig = mergeConfigs(parsed.data, {}, this.cliOverrides);
 				this.cachedConfig.configFile = configPath;
@@ -206,7 +216,11 @@ var ConfigLoader = class ConfigLoader {
 					configPath
 				}, "Error loading configuration");
 				if (error instanceof EvaliphyError) throw error;
-				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}`);
+				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+              Docs: https://evaliphy.com/docs/configuration
+              `);
 			}
 		}
 		const finalConfig = mergeConfigs(this.cachedConfig, transientOverrides, {});

package/dist/index.mjs

@@ -180,7 +180,14 @@ var ConfigLoader = class ConfigLoader {
 		const configPath = this.findConfigFile(cwd);
 		if (!configPath) {
 			this.configLogger.debug("No config file found, using defaults if applicable");
-			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, "Provided config file is not correct or does not exist.");
+			throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Could not load configuration. Please check the following:
+
+  - Does "evaliphy.config.ts" exist in your project root?
+  - Is the path correct if you specified a custom config location?
+  - Does the config file export a default defineConfig({}) call?
+
+  Expected location: ${process.cwd()}/evaliphy.config.ts
+  Visit https://evaliphy.com/docs/configuration for more details.`);
 		}
 		if (!this.cachedConfig) {
 			this.configLogger.debug({ configPath }, "Loading base configuration from file");
@@ -193,7 +200,10 @@ var ConfigLoader = class ConfigLoader {
 						error: parsed.error,
 						configPath
 					}, "Configuration validation failed");
-					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}: ${parsed.error.message}`);
+					throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Invalid configuration in ${configPath}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+            Docs: https://evaliphy.com/docs/configuration`);
 				}
 				this.cachedConfig = mergeConfigs(parsed.data, {}, this.cliOverrides);
 				this.cachedConfig.configFile = configPath;
@@ -203,7 +213,11 @@ var ConfigLoader = class ConfigLoader {
 					configPath
 				}, "Error loading configuration");
 				if (error instanceof EvaliphyError) throw error;
-				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}`);
+				throw new EvaliphyError(EvaliphyErrorCode.INVALID_CONFIG, `Failed to load config from ${configPath}: ${error.message}
+
+            Please check your evaliphy.config.ts and fix the above fields.
+              Docs: https://evaliphy.com/docs/configuration
+              `);
 			}
 		}
 		const finalConfig = mergeConfigs(this.cachedConfig, transientOverrides, {});