judgeval 0.1.33

package/dist/scorers/base-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-scorer.js","sourceRoot":"","sources":["../../src/scorers/base-scorer.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,4CAA4D;AAiB5D;;GAEG;AACH,MAAsB,iBAAiB;IAErC,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,6BAA6B;IAM3E,YAAY,IAAY,EAAE,SAAiB,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QAC9G,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,uCAAuC;QACvC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,6BAAiB,CAAC,CAAC,IAAI,CACpD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAC3D,CAAC;QAEF,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,6CAA6C,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,kCAAkC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAChG,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,MAAM,MAAM,GAAwB;YAClC,UAAU,EAAE,IAAI,CAAC,IAAI;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AA/DD,8CA+DC;AAED;;GAEG;AACH,MAAsB,cAAc;IAQlC,YAAY,IAAY,EAAE,SAAiB,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QAC9G,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,6BAA6B;QACpD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,uCAAuC;QACvC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,6BAAiB,CAAC,CAAC,IAAI,CACpD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAC3D,CAAC;QAEF,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,6CAA6C,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,kCAAkC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAChG,CAAC;QACH,CAAC;IACH,CAAC;IASD;;OAEG;IACH,MAAM;QACJ,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,IAAI;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;CACF;AAjED,wCAiEC;AAED;;GAEG;AACH,MAAa,aAAa;IASxB,YAAY,MAAW;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,6BAA6B;QACrF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAC;QACtD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,cAAuB,IAAI;QAC5C,+DAA+D;QAC/D,8CAA8C;QAC9C,IAAI,WAAW,EAAE,CAAC;YAChB,4BAA4B;YAC5B,OAAO,IAAI,CAAC,MAA2B,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,8BAA8B;YAC9B,OAAO,IAAI,CAAC,MAAwB,CAAC;QACvC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,uCAAuC;QACvC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,6BAAiB,CAAC,CAAC,IAAI,CACpD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAC3D,CAAC;QAEF,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,6CAA6C,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,kCAAkC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAChG,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,IAAI;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;CACF;AA5ED,sCA4EC"}

package/dist/scorers/exact-match-scorer.d.ts

@@ -0,0 +1,10 @@
+/**
+ * ExactMatchScorer - A custom scorer that checks if the actual output exactly matches the expected output
+ */
+import { Example } from '../data/example';
+import { JudgevalScorer } from './base-scorer';
+import { ScorerData } from '../data/result';
+export declare class ExactMatchScorer extends JudgevalScorer {
+    constructor(threshold?: number, additionalMetadata?: Record<string, any>, verbose?: boolean);
+    scoreExample(example: Example): Promise<ScorerData>;
+}

package/dist/scorers/exact-match-scorer.js

@@ -0,0 +1,84 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExactMatchScorer = void 0;
+const base_scorer_1 = require("./base-scorer");
+class ExactMatchScorer extends base_scorer_1.JudgevalScorer {
+    constructor(threshold = 1.0, additionalMetadata, verbose = false) {
+        super('exact_match', threshold, additionalMetadata, verbose);
+    }
+    scoreExample(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            try {
+                // Check if the example has expected output
+                if (!example.expectedOutput) {
+                    return {
+                        name: this.type,
+                        threshold: this.threshold,
+                        success: false,
+                        score: 0,
+                        reason: "Expected output is required for exact match scoring",
+                        strict_mode: null,
+                        evaluation_model: "exact-match",
+                        error: "Missing expected output",
+                        evaluation_cost: null,
+                        verbose_logs: null,
+                        additional_metadata: this.additional_metadata || {}
+                    };
+                }
+                // Compare the actual output with the expected output
+                const actualOutput = ((_a = example.actualOutput) === null || _a === void 0 ? void 0 : _a.trim()) || '';
+                const expectedOutput = example.expectedOutput.trim();
+                // Calculate the score (1 for exact match, 0 otherwise)
+                const isMatch = actualOutput === expectedOutput;
+                this.score = isMatch ? 1 : 0;
+                // Generate a reason for the score
+                const reason = isMatch
+                    ? "The actual output exactly matches the expected output."
+                    : `The actual output "${actualOutput}" does not match the expected output "${expectedOutput}".`;
+                // Return the scorer data
+                return {
+                    name: this.type,
+                    threshold: this.threshold,
+                    success: this.successCheck(),
+                    score: this.score,
+                    reason: reason,
+                    strict_mode: null,
+                    evaluation_model: "exact-match",
+                    error: null,
+                    evaluation_cost: null,
+                    verbose_logs: this.verbose ? `Comparing: "${actualOutput}" with "${expectedOutput}"` : null,
+                    additional_metadata: this.additional_metadata || {}
+                };
+            }
+            catch (error) {
+                // Handle any errors during scoring
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                return {
+                    name: this.type,
+                    threshold: this.threshold,
+                    success: false,
+                    score: 0,
+                    reason: `Error during scoring: ${errorMessage}`,
+                    strict_mode: null,
+                    evaluation_model: "exact-match",
+                    error: errorMessage,
+                    evaluation_cost: null,
+                    verbose_logs: null,
+                    additional_metadata: this.additional_metadata || {}
+                };
+            }
+        });
+    }
+}
+exports.ExactMatchScorer = ExactMatchScorer;
+//# sourceMappingURL=exact-match-scorer.js.map

package/dist/scorers/exact-match-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exact-match-scorer.js","sourceRoot":"","sources":["../../src/scorers/exact-match-scorer.ts"],"names":[],"mappings":";;;;;;;;;;;;AAIA,+CAA+C;AAG/C,MAAa,gBAAiB,SAAQ,4BAAc;IAClD,YAAY,YAAoB,GAAG,EAAE,kBAAwC,EAAE,UAAmB,KAAK;QACrG,KAAK,CAAC,aAAa,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;IAEK,YAAY,CAAC,OAAgB;;;YACjC,IAAI,CAAC;gBACH,2CAA2C;gBAC3C,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC5B,OAAO;wBACL,IAAI,EAAE,IAAI,CAAC,IAAI;wBACf,SAAS,EAAE,IAAI,CAAC,SAAS;wBACzB,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,qDAAqD;wBAC7D,WAAW,EAAE,IAAI;wBACjB,gBAAgB,EAAE,aAAa;wBAC/B,KAAK,EAAE,yBAAyB;wBAChC,eAAe,EAAE,IAAI;wBACrB,YAAY,EAAE,IAAI;wBAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,IAAI,EAAE;qBACpD,CAAC;gBACJ,CAAC;gBAED,qDAAqD;gBACrD,MAAM,YAAY,GAAG,CAAA,MAAA,OAAO,CAAC,YAAY,0CAAE,IAAI,EAAE,KAAI,EAAE,CAAC;gBACxD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;gBAErD,uDAAuD;gBACvD,MAAM,OAAO,GAAG,YAAY,KAAK,cAAc,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE7B,kCAAkC;gBAClC,MAAM,MAAM,GAAG,OAAO;oBACpB,CAAC,CAAC,wDAAwD;oBAC1D,CAAC,CAAC,sBAAsB,YAAY,yCAAyC,cAAc,IAAI,CAAC;gBAElG,yBAAyB;gBACzB,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE;oBAC5B,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,IAAI;oBACjB,gBAAgB,EAAE,aAAa;oBAC/B,KAAK,EAAE,IAAI;oBACX,eAAe,EAAE,IAAI;oBACrB,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,YAAY,WAAW,cAAc,GAAG,CAAC,CAAC,CAAC,IAAI;oBAC3F,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,IAAI,EAAE;iBACpD,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,mCAAmC;gBACnC,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAE5E,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,yBAAyB,YAAY,EAAE;oBAC/C,WAAW,EAAE,IAAI;oBACjB,gBAAgB,EAAE,aAAa;oBAC/B,KAAK,EAAE,YAAY;oBACnB,eAAe,EAAE,IAAI;oBACrB,YAAY,EAAE,IAAI;oBAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,IAAI,EAAE;iBACpD,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;CACF;AAtED,4CAsEC"}

package/package.json

@@ -0,0 +1,88 @@
+{
+  "name": "judgeval",
+  "version": "0.1.33",
+  "description": "Judgment SDK for TypeScript/JavaScript",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "directories": {
+    "doc": "docs",
+    "test": "tests"
+  },
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md"
+  ],
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.22.0",
+    "@langchain/core": "^0.3.44",
+    "@langchain/langgraph": "^0.2.63",
+    "@langchain/openai": "^0.5.5",
+    "@supabase/supabase-js": "^2.43.4",
+    "@types/node-fetch": "^2.6.12",
+    "@types/uuid": "^10.0.0",
+    "axios": "^1.7.2",
+    "chalk": "4.1.2",
+    "dotenv": "^16.4.7",
+    "eventsource-parser": "^1.1.2",
+    "json-schema-to-ts": "^3.1.0",
+    "json2csv": "^5.0.7",
+    "langchain": "^0.3.21",
+    "node-fetch": "^2.7.0",
+    "openai": "^4.47.3",
+    "papaparse": "^5.4.1",
+    "together-ai": "^0.5.1",
+    "undici-types": "^6.21.0",
+    "uuid": "^11.1.0",
+    "winston": "^3.17.0",
+    "ws": "^8.18.1"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.12.12",
+    "@typescript-eslint/eslint-plugin": "^7.10.0",
+    "@typescript-eslint/parser": "^7.10.0",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "jest": "^29.7.0",
+    "prettier": "^3.2.5",
+    "ts-jest": "^29.1.2",
+    "typedoc": "^0.25.13",
+    "typescript": "^5.4.5"
+  },
+  "scripts": {
+    "build": "tsc",
+    "build:examples": "tsc -p tsconfig.examples.json",
+    "test": "jest",
+    "lint": "eslint . --ext .ts",
+    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "prepublishOnly": "npm run build",
+    "docs": "typedoc --out docs src/index.ts",
+    "demo:basic": "npx ts-node src/demo/basic-bot.ts",
+    "demo:llm-wrap": "npx ts-node src/demo/llm-wrap-demo.ts",
+    "demo:openai-wrap": "npx ts-node src/demo/openai-wrap-demo.ts",
+    "demo:complex-async": "npx ts-node src/demo/demo_complex_async.ts",
+    "example:basic": "npm run build && node dist/examples/basic-usage.js",
+    "example:result-retrieval": "npm run build && node dist/examples/result-retrieval.js",
+    "example:custom-scorer": "npm run build && node dist/examples/custom-scorer.js",
+    "example:compare-python": "npm run build && node dist/examples/compare-with-python.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Judgment-AI/judgment-dev.git"
+  },
+  "keywords": [
+    "judgment",
+    "evaluation",
+    "llm",
+    "ai",
+    "sdk"
+  ],
+  "author": "Judgment AI",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://github.com/Judgment-AI/judgment-dev/issues"
+  },
+  "homepage": "https://github.com/Judgment-AI/judgment-dev#readme"
+}