evalsense 0.2.0 → 0.3.0

package/dist/cli.js

@@ -1,46 +1,49 @@
 #!/usr/bin/env node
-import { discoverFromPath, filterFiles, ExitCodes, ConsoleReporter, executeEvalFiles, JsonReporter, getExitCode } from './chunk-5P7LNNO6.js';
+import { discoverFromPath, filterFiles, ExitCodes, ConsoleReporter, executeEvalFiles, JsonReporter, getExitCode } from './chunk-JPVZL45G.js';
+import './chunk-DGUM43GV.js';
 import { Command } from 'commander';
 
 var program = new Command();
 program.name("evalsense").description("JS-native LLM evaluation framework with Jest-like API").version("0.1.0");
-program.command("run").description("Run evaluation tests").argument("[path]", "Path to eval file or directory", ".").option("-f, --filter <pattern>", "Filter tests by name pattern").option("-o, --output <file>", "Write JSON report to file").option("-r, --reporter <type>", "Reporter type: console, json, both", "console").option("-b, --bail", "Stop on first failure").option("-t, --timeout <ms>", "Test timeout in milliseconds", "30000").action(async (path, options) => {
-  try {
-    const files = await discoverFromPath(path);
-    const filtered = filterFiles(files, options.filter);
-    if (filtered.length === 0) {
-      console.error("No eval files found");
-      process.exit(ExitCodes.CONFIGURATION_ERROR);
-    }
-    const consoleReporter = new ConsoleReporter();
-    consoleReporter.printHeader(filtered.length);
-    const report = await executeEvalFiles(filtered, {
-      bail: options.bail,
-      timeout: parseInt(options.timeout, 10),
-      filter: options.filter
-    });
-    const reporterType = options.reporter.toLowerCase();
-    if (reporterType === "console" || reporterType === "both") {
-      consoleReporter.printReport(report);
-    }
-    if (reporterType === "json" || reporterType === "both" || options.output) {
-      const jsonReporter = new JsonReporter();
-      const json = jsonReporter.format(report);
-      if (options.output) {
-        await jsonReporter.writeToFile(report, options.output);
-        console.log(`
+program.command("run").description("Run evaluation tests").argument("[path]", "Path to eval file or directory", ".").option("-f, --filter <pattern>", "Filter tests by name pattern").option("-o, --output <file>", "Write JSON report to file").option("-r, --reporter <type>", "Reporter type: console, json, both", "console").option("-b, --bail", "Stop on first failure").option("-t, --timeout <ms>", "Test timeout in milliseconds", "30000").action(
+  async (path, options) => {
+    try {
+      const files = await discoverFromPath(path);
+      const filtered = filterFiles(files, options.filter);
+      if (filtered.length === 0) {
+        console.error("No eval files found");
+        process.exit(ExitCodes.CONFIGURATION_ERROR);
+      }
+      const consoleReporter = new ConsoleReporter();
+      consoleReporter.printHeader(filtered.length);
+      const report = await executeEvalFiles(filtered, {
+        bail: options.bail,
+        timeout: parseInt(options.timeout, 10),
+        filter: options.filter
+      });
+      const reporterType = options.reporter.toLowerCase();
+      if (reporterType === "console" || reporterType === "both") {
+        consoleReporter.printReport(report);
+      }
+      if (reporterType === "json" || reporterType === "both" || options.output) {
+        const jsonReporter = new JsonReporter();
+        const json = jsonReporter.format(report);
+        if (options.output) {
+          await jsonReporter.writeToFile(report, options.output);
+          console.log(`
 Report written to: ${options.output}`);
-      } else if (reporterType === "json") {
-        console.log(json);
+        } else if (reporterType === "json") {
+          console.log(json);
+        }
       }
+      const exitCode = getExitCode(report);
+      process.exit(exitCode);
+    } catch (error) {
+      console.error("Error:", error instanceof Error ? error.message : String(error));
+      process.exit(ExitCodes.EXECUTION_ERROR);
     }
-    const exitCode = getExitCode(report);
-    process.exit(exitCode);
-  } catch (error) {
-    console.error("Error:", error instanceof Error ? error.message : String(error));
-    process.exit(ExitCodes.EXECUTION_ERROR);
   }
-});
+);
 program.command("list").description("List discovered eval files").argument("[path]", "Path to search", ".").action(async (path) => {
   try {
     const files = await discoverFromPath(path);

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/runner/cli.ts"],"names":[],"mappings":";;;;AAaA,IAAM,OAAA,GAAU,IAAI,OAAA,EAAQ;AAE5B,OAAA,CACG,KAAK,WAAW,CAAA,CAChB,YAAY,uDAAuD,CAAA,CACnE,QAAQ,OAAO,CAAA;AAElB,OAAA,CACG,QAAQ,KAAK,CAAA,CACb,WAAA,CAAY,sBAAsB,EAClC,QAAA,CAAS,QAAA,EAAU,gCAAA,EAAkC,GAAG,EACxD,MAAA,CAAO,wBAAA,EAA0B,8BAA8B,CAAA,CAC/D,OAAO,qBAAA,EAAuB,2BAA2B,CAAA,CACzD,MAAA,CAAO,yBAAyB,oCAAA,EAAsC,SAAS,CAAA,CAC/E,MAAA,CAAO,cAAc,uBAAuB,CAAA,CAC5C,MAAA,CAAO,oBAAA,EAAsB,gCAAgC,OAAO,CAAA,CACpE,MAAA,CAAO,OAAO,MAAc,OAAA,KAMvB;AACJ,EAAA,IAAI;AAEF,IAAA,MAAM,KAAA,GAAQ,MAAM,gBAAA,CAAiB,IAAI,CAAA;AACzC,IAAA,MAAM,QAAA,GAAW,WAAA,CAAY,KAAA,EAAO,OAAA,CAAQ,MAAM,CAAA;AAElD,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAA,CAAQ,MAAM,qBAAqB,CAAA;AACnC,MAAA,OAAA,CAAQ,IAAA,CAAK,UAAU,mBAAmB,CAAA;AAAA,IAC5C;AAEA,IAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAG5C,IAAA,eAAA,CAAgB,WAAA,CAAY,SAAS,MAAM,CAAA;AAG3C,IAAA,MAAM,MAAA,GAAS,MAAM,gBAAA,CAAiB,QAAA,EAAU;AAAA,MAC9C,MAAM,OAAA,CAAQ,IAAA;AAAA,MACd,OAAA,EAAS,QAAA,CAAS,OAAA,CAAQ,OAAA,EAAS,EAAE,CAAA;AAAA,MACrC,QAAQ,OAAA,CAAQ;AAAA,KACjB,CAAA;AAGD,IAAA,MAAM,YAAA,GAAe,OAAA,CAAQ,QAAA,CAAS,WAAA,EAAY;AAElD,IAAA,IAAI,YAAA,KAAiB,SAAA,IAAa,YAAA,KAAiB,MAAA,EAAQ;AACzD,MAAA,eAAA,CAAgB,YAAY,MAAM,CAAA;AAAA,IACpC;AAEA,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAU,YAAA,KAAiB,MAAA,IAAU,QAAQ,MAAA,EAAQ;AACxE,MAAA,MAAM,YAAA,GAAe,IAAI,YAAA,EAAa;AACtC,MAAA,MAAM,IAAA,GAAO,YAAA,CAAa,MAAA,CAAO,MAAM,CAAA;AAEvC,MAAA,IAAI,QAAQ,MAAA,EAAQ;AAClB,QAAA,MAAM,YAAA,CAAa,WAAA,CAAY,MAAA,EAAQ,OAAA,CAAQ,MAAM,CAAA;AACrD,QAAA,OAAA,CAAQ,GAAA,CAAI;AAAA,mBAAA,EAAwB,OAAA,CAAQ,MAAM,CAAA,CAAE,CAAA;AAAA,MACtD,CAAA,MAAA,IAAW,iBAAiB,MAAA,EAAQ;AAClC,QAAA,OAAA,CAAQ,IAAI,IAAI,CAAA;AAAA,MAClB;AAAA,IACF;AAGA,IAAA,MAAM,QAAA,GAAW,YAAY,MAAM,CAAA;AACnC,IAAA,OAAA,CAAQ,KAAK,QAAQ,CAAA;AAAA,EACvB,SAAS,KAAA,EAAO;AACd,IAAA,OAAA,CAAQ,KAAA,CAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,MAAM,OAAA,GAAU,MAAA,CAAO,KAAK,CAAC,CAAA;AAC9E,IAAA,OAAA,CAAQ,IAAA,CAAK,UAAU,eAAe,CAAA;AAAA,EACxC;AACF,CAAC,CAAA;AAEH,OAAA,CACG,OAAA,CAAQ,MAAM,CAAA,CACd,WAAA,CAAY,4BAA4B,CAAA,CACxC,QAAA,CAAS,QAAA,EAAU,gBAAA,EAAkB,GAAG,CAAA,CACxC,MAAA,CAAO,OAAO,IAAA,KAAiB;AAC9B,EAAA,IAAI;AACF,IAAA,MAAM,KAAA,GAAQ,MAAM,gBAAA,CAAiB,IAAI,CAAA;AAEzC,IAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,IAAI,qBAAqB,CAAA;AACjC,MAAA;AAAA,IACF;AAEA,IAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,MAAA,EAAS,KAAA,CAAM,MAAM,CAAA;AAAA,CAAkB,CAAA;AACnD,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,EAAA,EAAK,IAAI,CAAA,CAAE,CAAA;AAAA,IACzB;AAAA,EACF,SAAS,KAAA,EAAO;AACd,IAAA,OAAA,CAAQ,KAAA,CAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,MAAM,OAAA,GAAU,MAAA,CAAO,KAAK,CAAC,CAAA;AAC9E,IAAA,OAAA,CAAQ,IAAA,CAAK,UAAU,mBAAmB,CAAA;AAAA,EAC5C;AACF,CAAC,CAAA;AAEH,OAAA,CAAQ,KAAA,EAAM","file":"cli.js","sourcesContent":["#!/usr/bin/env node\n\n/**\n * EvalSense CLI\n */\n\nimport { Command } from \"commander\";\nimport { discoverFromPath, filterFiles } from \"./discovery.js\";\nimport { executeEvalFiles, getExitCode } from \"./executor.js\";\nimport { ConsoleReporter } from \"../report/console-reporter.js\";\nimport { JsonReporter } from \"../report/json-reporter.js\";\nimport { ExitCodes } from \"../core/types.js\";\n\nconst program = new Command();\n\nprogram\n  .name(\"evalsense\")\n  .description(\"JS-native LLM evaluation framework with Jest-like API\")\n  .version(\"0.1.0\");\n\nprogram\n  .command(\"run\")\n  .description(\"Run evaluation tests\")\n  .argument(\"[path]\", \"Path to eval file or directory\", \".\")\n  .option(\"-f, --filter <pattern>\", \"Filter tests by name pattern\")\n  .option(\"-o, --output <file>\", \"Write JSON report to file\")\n  .option(\"-r, --reporter <type>\", \"Reporter type: console, json, both\", \"console\")\n  .option(\"-b, --bail\", \"Stop on first failure\")\n  .option(\"-t, --timeout <ms>\", \"Test timeout in milliseconds\", \"30000\")\n  .action(async (path: string, options: {\n    filter?: string;\n    output?: string;\n    reporter: string;\n    bail?: boolean;\n    timeout: string;\n  }) => {\n    try {\n      // Discover eval files\n      const files = await discoverFromPath(path);\n      const filtered = filterFiles(files, options.filter);\n\n      if (filtered.length === 0) {\n        console.error(\"No eval files found\");\n        process.exit(ExitCodes.CONFIGURATION_ERROR);\n      }\n\n      const consoleReporter = new ConsoleReporter();\n\n      // Print header\n      consoleReporter.printHeader(filtered.length);\n\n      // Execute tests\n      const report = await executeEvalFiles(filtered, {\n        bail: options.bail,\n        timeout: parseInt(options.timeout, 10),\n        filter: options.filter,\n      });\n\n      // Output results\n      const reporterType = options.reporter.toLowerCase();\n\n      if (reporterType === \"console\" || reporterType === \"both\") {\n        consoleReporter.printReport(report);\n      }\n\n      if (reporterType === \"json\" || reporterType === \"both\" || options.output) {\n        const jsonReporter = new JsonReporter();\n        const json = jsonReporter.format(report);\n\n        if (options.output) {\n          await jsonReporter.writeToFile(report, options.output);\n          console.log(`\\nReport written to: ${options.output}`);\n        } else if (reporterType === \"json\") {\n          console.log(json);\n        }\n      }\n\n      // Exit with appropriate code\n      const exitCode = getExitCode(report);\n      process.exit(exitCode);\n    } catch (error) {\n      console.error(\"Error:\", error instanceof Error ? error.message : String(error));\n      process.exit(ExitCodes.EXECUTION_ERROR);\n    }\n  });\n\nprogram\n  .command(\"list\")\n  .description(\"List discovered eval files\")\n  .argument(\"[path]\", \"Path to search\", \".\")\n  .action(async (path: string) => {\n    try {\n      const files = await discoverFromPath(path);\n\n      if (files.length === 0) {\n        console.log(\"No eval files found\");\n        return;\n      }\n\n      console.log(`Found ${files.length} eval file(s):\\n`);\n      for (const file of files) {\n        console.log(`  ${file}`);\n      }\n    } catch (error) {\n      console.error(\"Error:\", error instanceof Error ? error.message : String(error));\n      process.exit(ExitCodes.CONFIGURATION_ERROR);\n    }\n  });\n\nprogram.parse();\n"]}
+{"version":3,"sources":["../src/runner/cli.ts"],"names":[],"mappings":";;;;;AAaA,IAAM,OAAA,GAAU,IAAI,OAAA,EAAQ;AAE5B,OAAA,CACG,KAAK,WAAW,CAAA,CAChB,YAAY,uDAAuD,CAAA,CACnE,QAAQ,OAAO,CAAA;AAElB,OAAA,CACG,OAAA,CAAQ,KAAK,CAAA,CACb,WAAA,CAAY,sBAAsB,CAAA,CAClC,QAAA,CAAS,QAAA,EAAU,gCAAA,EAAkC,GAAG,CAAA,CACxD,MAAA,CAAO,wBAAA,EAA0B,8BAA8B,CAAA,CAC/D,MAAA,CAAO,qBAAA,EAAuB,2BAA2B,CAAA,CACzD,MAAA,CAAO,uBAAA,EAAyB,oCAAA,EAAsC,SAAS,CAAA,CAC/E,MAAA,CAAO,YAAA,EAAc,uBAAuB,CAAA,CAC5C,MAAA,CAAO,oBAAA,EAAsB,8BAAA,EAAgC,OAAO,CAAA,CACpE,MAAA;AAAA,EACC,OACE,MACA,OAAA,KAOG;AACH,IAAA,IAAI;AAEF,MAAA,MAAM,KAAA,GAAQ,MAAM,gBAAA,CAAiB,IAAI,CAAA;AACzC,MAAA,MAAM,QAAA,GAAW,WAAA,CAAY,KAAA,EAAO,OAAA,CAAQ,MAAM,CAAA;AAElD,MAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,QAAA,OAAA,CAAQ,MAAM,qBAAqB,CAAA;AACnC,QAAA,OAAA,CAAQ,IAAA,CAAK,UAAU,mBAAmB,CAAA;AAAA,MAC5C;AAEA,MAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAG5C,MAAA,eAAA,CAAgB,WAAA,CAAY,SAAS,MAAM,CAAA;AAG3C,MAAA,MAAM,MAAA,GAAS,MAAM,gBAAA,CAAiB,QAAA,EAAU;AAAA,QAC9C,MAAM,OAAA,CAAQ,IAAA;AAAA,QACd,OAAA,EAAS,QAAA,CAAS,OAAA,CAAQ,OAAA,EAAS,EAAE,CAAA;AAAA,QACrC,QAAQ,OAAA,CAAQ;AAAA,OACjB,CAAA;AAGD,MAAA,MAAM,YAAA,GAAe,OAAA,CAAQ,QAAA,CAAS,WAAA,EAAY;AAElD,MAAA,IAAI,YAAA,KAAiB,SAAA,IAAa,YAAA,KAAiB,MAAA,EAAQ;AACzD,QAAA,eAAA,CAAgB,YAAY,MAAM,CAAA;AAAA,MACpC;AAEA,MAAA,IAAI,YAAA,KAAiB,MAAA,IAAU,YAAA,KAAiB,MAAA,IAAU,QAAQ,MAAA,EAAQ;AACxE,QAAA,MAAM,YAAA,GAAe,IAAI,YAAA,EAAa;AACtC,QAAA,MAAM,IAAA,GAAO,YAAA,CAAa,MAAA,CAAO,MAAM,CAAA;AAEvC,QAAA,IAAI,QAAQ,MAAA,EAAQ;AAClB,UAAA,MAAM,YAAA,CAAa,WAAA,CAAY,MAAA,EAAQ,OAAA,CAAQ,MAAM,CAAA;AACrD,UAAA,OAAA,CAAQ,GAAA,CAAI;AAAA,mBAAA,EAAwB,OAAA,CAAQ,MAAM,CAAA,CAAE,CAAA;AAAA,QACtD,CAAA,MAAA,IAAW,iBAAiB,MAAA,EAAQ;AAClC,UAAA,OAAA,CAAQ,IAAI,IAAI,CAAA;AAAA,QAClB;AAAA,MACF;AAGA,MAAA,MAAM,QAAA,GAAW,YAAY,MAAM,CAAA;AACnC,MAAA,OAAA,CAAQ,KAAK,QAAQ,CAAA;AAAA,IACvB,SAAS,KAAA,EAAO;AACd,MAAA,OAAA,CAAQ,KAAA,CAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,MAAM,OAAA,GAAU,MAAA,CAAO,KAAK,CAAC,CAAA;AAC9E,MAAA,OAAA,CAAQ,IAAA,CAAK,UAAU,eAAe,CAAA;AAAA,IACxC;AAAA,EACF;AACF,CAAA;AAEF,OAAA,CACG,OAAA,CAAQ,MAAM,CAAA,CACd,WAAA,CAAY,4BAA4B,CAAA,CACxC,QAAA,CAAS,QAAA,EAAU,gBAAA,EAAkB,GAAG,CAAA,CACxC,MAAA,CAAO,OAAO,IAAA,KAAiB;AAC9B,EAAA,IAAI;AACF,IAAA,MAAM,KAAA,GAAQ,MAAM,gBAAA,CAAiB,IAAI,CAAA;AAEzC,IAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,IAAI,qBAAqB,CAAA;AACjC,MAAA;AAAA,IACF;AAEA,IAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,MAAA,EAAS,KAAA,CAAM,MAAM,CAAA;AAAA,CAAkB,CAAA;AACnD,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,EAAA,EAAK,IAAI,CAAA,CAAE,CAAA;AAAA,IACzB;AAAA,EACF,SAAS,KAAA,EAAO;AACd,IAAA,OAAA,CAAQ,KAAA,CAAM,UAAU,KAAA,YAAiB,KAAA,GAAQ,MAAM,OAAA,GAAU,MAAA,CAAO,KAAK,CAAC,CAAA;AAC9E,IAAA,OAAA,CAAQ,IAAA,CAAK,UAAU,mBAAmB,CAAA;AAAA,EAC5C;AACF,CAAC,CAAA;AAEH,OAAA,CAAQ,KAAA,EAAM","file":"cli.js","sourcesContent":["#!/usr/bin/env node\n\n/**\n * EvalSense CLI\n */\n\nimport { Command } from \"commander\";\nimport { discoverFromPath, filterFiles } from \"./discovery.js\";\nimport { executeEvalFiles, getExitCode } from \"./executor.js\";\nimport { ConsoleReporter } from \"../report/console-reporter.js\";\nimport { JsonReporter } from \"../report/json-reporter.js\";\nimport { ExitCodes } from \"../core/types.js\";\n\nconst program = new Command();\n\nprogram\n  .name(\"evalsense\")\n  .description(\"JS-native LLM evaluation framework with Jest-like API\")\n  .version(\"0.1.0\");\n\nprogram\n  .command(\"run\")\n  .description(\"Run evaluation tests\")\n  .argument(\"[path]\", \"Path to eval file or directory\", \".\")\n  .option(\"-f, --filter <pattern>\", \"Filter tests by name pattern\")\n  .option(\"-o, --output <file>\", \"Write JSON report to file\")\n  .option(\"-r, --reporter <type>\", \"Reporter type: console, json, both\", \"console\")\n  .option(\"-b, --bail\", \"Stop on first failure\")\n  .option(\"-t, --timeout <ms>\", \"Test timeout in milliseconds\", \"30000\")\n  .action(\n    async (\n      path: string,\n      options: {\n        filter?: string;\n        output?: string;\n        reporter: string;\n        bail?: boolean;\n        timeout: string;\n      }\n    ) => {\n      try {\n        // Discover eval files\n        const files = await discoverFromPath(path);\n        const filtered = filterFiles(files, options.filter);\n\n        if (filtered.length === 0) {\n          console.error(\"No eval files found\");\n          process.exit(ExitCodes.CONFIGURATION_ERROR);\n        }\n\n        const consoleReporter = new ConsoleReporter();\n\n        // Print header\n        consoleReporter.printHeader(filtered.length);\n\n        // Execute tests\n        const report = await executeEvalFiles(filtered, {\n          bail: options.bail,\n          timeout: parseInt(options.timeout, 10),\n          filter: options.filter,\n        });\n\n        // Output results\n        const reporterType = options.reporter.toLowerCase();\n\n        if (reporterType === \"console\" || reporterType === \"both\") {\n          consoleReporter.printReport(report);\n        }\n\n        if (reporterType === \"json\" || reporterType === \"both\" || options.output) {\n          const jsonReporter = new JsonReporter();\n          const json = jsonReporter.format(report);\n\n          if (options.output) {\n            await jsonReporter.writeToFile(report, options.output);\n            console.log(`\\nReport written to: ${options.output}`);\n          } else if (reporterType === \"json\") {\n            console.log(json);\n          }\n        }\n\n        // Exit with appropriate code\n        const exitCode = getExitCode(report);\n        process.exit(exitCode);\n      } catch (error) {\n        console.error(\"Error:\", error instanceof Error ? error.message : String(error));\n        process.exit(ExitCodes.EXECUTION_ERROR);\n      }\n    }\n  );\n\nprogram\n  .command(\"list\")\n  .description(\"List discovered eval files\")\n  .argument(\"[path]\", \"Path to search\", \".\")\n  .action(async (path: string) => {\n    try {\n      const files = await discoverFromPath(path);\n\n      if (files.length === 0) {\n        console.log(\"No eval files found\");\n        return;\n      }\n\n      console.log(`Found ${files.length} eval file(s):\\n`);\n      for (const file of files) {\n        console.log(`  ${file}`);\n      }\n    } catch (error) {\n      console.error(\"Error:\", error instanceof Error ? error.message : String(error));\n      process.exit(ExitCodes.CONFIGURATION_ERROR);\n    }\n  });\n\nprogram.parse();\n"]}