@llmops/cli 1.0.0-beta.2 → 1.0.0-beta.23

package/dist/index.mjs

@@ -1,37 +1,29 @@
 #!/usr/bin/env node
-import { command, run, string } from "@drizzle-team/brocli";
+import { boolean, command, run, string } from "@drizzle-team/brocli";
 import { logger } from "@llmops/core";
-import { getMigrations } from "@llmops/core/db";
-import { existsSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs";
 import yoctoSpinner from "yocto-spinner";
-import chalk from "chalk";
 import prompts from "prompts";
-import { loadConfig } from "c12";
-import path from "node:path";
+import path, { basename, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { execSync } from "node:child_process";
+import chalk from "chalk";
 
 //#region src/lib/get-config.ts
 const getConfig = async ({ cwd, configPath }) => {
-	if (configPath) {
-		let resolvedPath = path.join(cwd, configPath);
-		if (existsSync(configPath)) resolvedPath = configPath;
-		const { config: loadedConfig } = await loadConfig({ configFile: resolvedPath });
-		return loadedConfig.config;
+	if (!configPath) return void 0;
+	const resolvedPath = existsSync(configPath) ? path.resolve(configPath) : path.resolve(cwd, configPath);
+	if (!existsSync(resolvedPath)) throw new Error(`Config file not found: ${resolvedPath}`);
+	const mod = await import(pathToFileURL(resolvedPath).href);
+	if (mod.config) return mod.config;
+	if (mod.default) {
+		if (typeof mod.default === "object" && mod.default !== null && "config" in mod.default && mod.default.config) return mod.default.config;
+		return mod.default;
 	}
 };
 
 //#endregion
 //#region src/commands/migrate.ts
-/**
-* @fileoverview This file defines the 'migrate' command for the CLI application.
-* Steps:
-* 1. Look for package.json and @llmops/sdk in the current directory.
-* 2. Check if the @llmops/sdk version works with the current CLI version.
-* 3. If compatible, check for the config file passed as an argument.
-* 4. If the config file exists, read and parse it.
-* 5. If not passed, look for default config file locations.
-* 6. Use zod to validate the existing configuration schema.
-* 7. If valid, get the telemetry store from the config.
-*/
 const migrateCommand = command({
 	name: "migrate",
 	desc: "Run database migrations based on LLMOps configuration",
@@ -61,43 +53,167 @@ const migrateCommand = command({
 		}
 		const telemetry = config.telemetry;
 		const store = Array.isArray(telemetry) ? telemetry[0] : telemetry;
-		if (!store || !store._db) {
+		if (!store || !store._pool) {
 			logger.error("No telemetry store with database found. Configure pgStore in your config.");
 			process.exit(1);
 		}
-		const db = store._db;
-		const spinner = yoctoSpinner({ text: "preparing migration..." }).start();
-		const { toBeAdded, toBeCreated, runMigrations } = await getMigrations(db, "postgres", { schema: "llmops" });
-		if (!toBeAdded.length && !toBeCreated.length) {
-			spinner.stop();
-			console.log("🚀 No migrations needed.");
-			process.exit(0);
-		}
-		spinner.stop();
-		console.log(`🔑 The migration will affect the following:`);
-		for (const table of [...toBeCreated, ...toBeAdded]) console.log("->", chalk.magenta(Object.keys(table.fields).join(", ")), chalk.white("fields on"), chalk.yellow(`${table.table}`), chalk.white("table."));
 		let migrate = opts.yes;
 		if (!opts.yes) migrate = (await prompts({
 			type: "confirm",
 			name: "migrate",
-			message: "Do you want to proceed with the migration?",
+			message: "Do you want to run pending migrations?",
 			initial: false
 		})).migrate;
 		if (!migrate) {
 			console.log("Migration cancelled.");
 			process.exit(0);
 		}
-		spinner.start("migrating...");
-		await runMigrations();
-		spinner.stop();
-		console.log("✅ Migration completed successfully.");
+		const spinner = yoctoSpinner({ text: "running migrations..." }).start();
+		try {
+			const { runMigrations } = await import("@llmops/sdk/store/pg");
+			const { applied } = await runMigrations(store._pool, store._schema ?? "llmops");
+			spinner.stop();
+			if (applied.length === 0) console.log("🚀 No pending migrations.");
+			else console.log(`✅ Applied ${applied.length} migration(s): ${applied.join(", ")}`);
+		} catch (error) {
+			spinner.stop();
+			logger.error(`Migration failed: ${error}`);
+			process.exit(1);
+		}
 		process.exit(0);
 	}
 });
 
+//#endregion
+//#region src/commands/eval.ts
+/**
+* Find eval files from a path (file or directory).
+* Default pattern: *.eval.ts, *.eval.js
+*/
+function findEvalFiles(target) {
+	const resolved = resolve(target);
+	if (!existsSync(resolved)) {
+		console.error(chalk.red(`Path not found: ${target}`));
+		process.exit(1);
+	}
+	const stat = statSync(resolved);
+	if (stat.isFile()) return [resolved];
+	if (stat.isDirectory()) return collectEvalFiles(resolved);
+	return [];
+}
+function collectEvalFiles(dir) {
+	const files = [];
+	const entries = readdirSync(dir, { withFileTypes: true });
+	for (const entry of entries) {
+		const fullPath = join(dir, entry.name);
+		if (entry.isDirectory() && !entry.name.startsWith(".") && entry.name !== "node_modules") files.push(...collectEvalFiles(fullPath));
+		else if (entry.isFile() && (entry.name.endsWith(".eval.ts") || entry.name.endsWith(".eval.js"))) files.push(fullPath);
+	}
+	return files.sort();
+}
+/**
+* Bundle and run an eval file using esbuild.
+* Bundles to a temp file, executes with node, then cleans up.
+*/
+async function bundleAndRun(file, env) {
+	const esbuild = await import("esbuild");
+	const tmpDir = join(process.cwd(), ".llmops-eval-tmp");
+	mkdirSync(tmpDir, { recursive: true });
+	const outFile = join(tmpDir, `${basename(file, ".ts").replace(".eval", "")}_eval.mjs`);
+	try {
+		await esbuild.build({
+			entryPoints: [file],
+			bundle: true,
+			platform: "node",
+			format: "esm",
+			outfile: outFile,
+			external: [
+				"@llmops/sdk",
+				"@llmops/sdk/*",
+				"@llmops/core",
+				"@llmops/core/*",
+				"openai",
+				"esbuild",
+				"fsevents",
+				"dotenv",
+				"dotenv/*"
+			],
+			treeShaking: true,
+			sourcemap: false,
+			banner: { js: "" }
+		});
+		execSync(`node ${outFile}`, {
+			stdio: "inherit",
+			env,
+			cwd: process.cwd()
+		});
+	} finally {
+		try {
+			rmSync(tmpDir, {
+				recursive: true,
+				force: true
+			});
+		} catch {}
+	}
+}
+const evalCommand = command({
+	name: "eval",
+	desc: "Run evaluation files",
+	options: {
+		target: string().default("./evals").desc("File or directory to run. Default: ./evals").alias("t"),
+		outputDir: string().default("./llmops-evals").desc("Output directory for results").alias("o"),
+		json: boolean().desc("Output results as JSON to stdout").alias("j")
+	},
+	handler: async (opts) => {
+		const target = opts.target;
+		const files = findEvalFiles(target);
+		if (files.length === 0) {
+			console.error(chalk.yellow(`No eval files found. Create files matching *.eval.ts or *.eval.js in ${target}`));
+			process.exit(1);
+		}
+		console.log(chalk.dim(`Found ${files.length} eval file${files.length > 1 ? "s" : ""}`));
+		const env = {
+			...process.env,
+			LLMOPS_EVAL_OUTPUT_DIR: resolve(opts.outputDir),
+			...opts.json ? { LLMOPS_EVAL_OUTPUT: "json" } : {}
+		};
+		let hasErrors = false;
+		for (const file of files) {
+			const name = basename(file);
+			console.log(chalk.dim(`\nRunning ${name}...`));
+			try {
+				await bundleAndRun(file, env);
+			} catch (err) {
+				hasErrors = true;
+				console.error(chalk.red(`\n✗ ${name} failed`));
+				if (err instanceof Error && !("status" in err)) console.error(err.message);
+			}
+		}
+		const outputDir = resolve(opts.outputDir);
+		if (existsSync(outputDir) && !opts.json) {
+			const evalDirs = readdirSync(outputDir, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name);
+			if (evalDirs.length > 0) {
+				console.log(chalk.dim("\n─────────────────────────────────"));
+				console.log(chalk.dim(`Results saved to ${opts.outputDir}/`));
+				for (const evalDir of evalDirs) {
+					const resultFiles = readdirSync(join(outputDir, evalDir)).filter((f) => f.endsWith(".json")).sort().reverse();
+					if (resultFiles.length > 0) try {
+						const scores = JSON.parse(readFileSync(join(outputDir, evalDir, resultFiles[0]), "utf-8")).scores;
+						const scoreStr = Object.entries(scores).map(([k, v]) => `${k}=${v.mean.toFixed(2)}`).join("  ");
+						console.log(`  ${chalk.white(evalDir)}  ${chalk.cyan(scoreStr)}`);
+					} catch {
+						console.log(`  ${chalk.white(evalDir)}  ${chalk.dim(resultFiles[0])}`);
+					}
+				}
+			}
+		}
+		if (hasErrors) process.exit(1);
+	}
+});
+
 //#endregion
 //#region src/index.ts
-run([migrateCommand], {
+run([migrateCommand, evalCommand], {
 	name: "llmops",
 	description: "LLMOps CLI - A pluggable LLMOps toolkit for TypeScript teams",
 	version: "0.0.1"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llmops/cli",
-  "version": "1.0.0-beta.2",
+  "version": "1.0.0-beta.23",
   "type": "module",
   "description": "LLMOps CLI - A pluggable LLMOps toolkit for TypeScript teams",
   "license": "Apache-2.0",
@@ -14,23 +14,13 @@
     "directory": "packages/cli"
   },
   "author": "chtushar",
+  "bin": {
+    "llmops": "./dist/index.mjs"
+  },
   "files": [
     "dist"
   ],
   "main": "./dist/index.mjs",
-  "module": "./dist/index.mjs",
-  "types": "./dist/index.d.mts",
-  "exports": "./dist/index.mjs",
-  "bin": {
-    "cli": "./dist/index.mjs",
-    "llmops": "./dist/index.mjs"
-  },
-  "publishConfig": {
-    "access": "public",
-    "executableFiles": [
-      "./dist/index.mjs"
-    ]
-  },
   "keywords": [
     "llmops",
     "cli",
@@ -38,15 +28,17 @@
     "llm",
     "typescript"
   ],
+  "publishConfig": {
+    "access": "public"
+  },
   "dependencies": {
     "@drizzle-team/brocli": "^0.11.0",
-    "c12": "^3.3.2",
     "chalk": "^5.6.2",
-    "kysely": "^0.28.8",
+    "esbuild": "^0.25.0",
     "prompts": "^2.4.2",
     "yocto-spinner": "^1.0.0",
-    "@llmops/core": "^1.0.0-beta.2",
-    "@llmops/sdk": "^1.0.0-beta.2"
+    "@llmops/core": "^1.0.0-beta.23",
+    "@llmops/sdk": "^1.0.0-beta.23"
   },
   "devDependencies": {
     "@types/pg": "^8.15.6",
@@ -55,9 +47,8 @@
   },
   "scripts": {
     "build": "tsdown",
-    "start": "node ./dist/index.mjs",
     "clean": "rm -rf dist",
-    "dev": "tsdown --watch",
+    "dev": "tsx src/index.ts",
     "test": "vitest",
     "typecheck": "tsc --noEmit"
   }

package/dist/index.d.mts

@@ -1 +0,0 @@
-export { };