@cef-ai/eval 0.0.0-stage → 0.2.0

package/dist/vault-store.d.ts

@@ -0,0 +1,46 @@
+/**
+ * `EvalStore` over a vault scope's object storage.
+ *
+ * Evaluations live in the vault the workflow runs in, in the workflow's scope:
+ * `/api/v1/vaults/<vault>/scopes/<scope>/objects/evals/workflows/<wf>/…`. So
+ * the vault decides who may read and write them, the same way it decides who
+ * may run the workflow: a read grant on the scope reads datasets and
+ * experiments, a write grant manages datasets and runs experiments.
+ *
+ * Takes the objects client STRUCTURALLY — vault-sdk's `vault.scope(s).objects`
+ * fits, and so does an in-memory fake — so this package keeps no dependency on
+ * the vault SDK and stays browser-safe.
+ */
+import type { EvalStore } from "./storage.js";
+/** What the store needs of a vault scope's objects client. */
+export interface VaultObjects {
+    upload(path: string, data: Uint8Array, opts: {
+        contentType?: string;
+        ifNoneMatch?: "*";
+    }): Promise<unknown>;
+    get(path: string): Promise<Uint8Array>;
+    list(opts: {
+        prefix?: string;
+    }): Promise<{
+        items: Array<{
+            path: string;
+        }>;
+    }>;
+}
+export interface VaultObjectStoreOptions {
+    /** Path prefix inside the scope's objects. Default `"evals/"`. */
+    root?: string;
+}
+/**
+ * The server or the client dropped `If-None-Match: *`: a create-only upload of
+ * a path that exists was accepted. Writing on would turn every write-once key
+ * (case revisions, dataset versions) into an overwrite that still answers
+ * "won", so the store refuses instead.
+ */
+export declare class ConditionalWriteUnsupportedError extends Error {
+    constructor();
+}
+/** The one key the write-once probe uses, so repeated sessions do not pile up keys nobody can delete. */
+export declare const CONDITIONAL_WRITE_PROBE_KEY = ".probe/conditional-write";
+export declare function vaultObjectStore(objects: VaultObjects, opts?: VaultObjectStoreOptions): EvalStore;
+//# sourceMappingURL=vault-store.d.ts.map

package/dist/vault-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vault-store.d.ts","sourceRoot":"","sources":["../src/vault-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9C,8DAA8D;AAC9D,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,UAAU,EAChB,IAAI,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,GAAG,CAAA;KAAE,GAChD,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;CAC9E;AAED,MAAM,WAAW,uBAAuB;IACtC,kEAAkE;IAClE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAwBD;;;;;GAKG;AACH,qBAAa,gCAAiC,SAAQ,KAAK;;CAS1D;AAED,yGAAyG;AACzG,eAAO,MAAM,2BAA2B,6BAA6B,CAAC;AAEtE,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,GAAE,uBAA4B,GAAG,SAAS,CA+FrG"}

package/dist/vault-store.js

@@ -0,0 +1,127 @@
+const errorField = (err, field) => err && typeof err === "object" ? err[field] : undefined;
+/**
+ * The object is not there: vault-api's `OBJECT_NOT_FOUND`, by its code. A bare
+ * 404 is not enough — a wrong vault-api URL, an unmounted route or a proxy
+ * answers one too, and reading that as "absent" would let `putCase`'s
+ * write-once check pass on a read that never happened.
+ */
+const isNotFound = (err) => errorField(err, "code") === "OBJECT_NOT_FOUND";
+/**
+ * A create-only upload lost: vault-api answers `If-None-Match: *` on a taken
+ * path with 409 `OBJECT_EXISTS`. Nothing else counts — any other refusal is a
+ * failure, not someone else having won.
+ */
+const isTaken = (err) => errorField(err, "status") === 409 && errorField(err, "code") === "OBJECT_EXISTS";
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+/**
+ * The server or the client dropped `If-None-Match: *`: a create-only upload of
+ * a path that exists was accepted. Writing on would turn every write-once key
+ * (case revisions, dataset versions) into an overwrite that still answers
+ * "won", so the store refuses instead.
+ */
+export class ConditionalWriteUnsupportedError extends Error {
+    constructor() {
+        super("This vault does not honour create-only uploads (If-None-Match: *): an upload of a path that already " +
+            "exists was accepted. Evaluations need vault-api with Cerebellum-Network/ddc-node#1375 and " +
+            "@cef-ai/vault-sdk >= 5.8.0; nothing was written.");
+        this.name = "ConditionalWriteUnsupportedError";
+    }
+}
+/** The one key the write-once probe uses, so repeated sessions do not pile up keys nobody can delete. */
+export const CONDITIONAL_WRITE_PROBE_KEY = ".probe/conditional-write";
+export function vaultObjectStore(objects, opts = {}) {
+    const root = (opts.root ?? "evals/").replace(/\/*$/, "/").replace(/^\/+/, "");
+    const full = (key) => `${root}${key}`;
+    const createOnly = async (path, body) => {
+        try {
+            await objects.upload(path, encoder.encode(body), { contentType: "application/json", ifNoneMatch: "*" });
+            return true;
+        }
+        catch (err) {
+            if (isTaken(err))
+                return false;
+            throw err;
+        }
+    };
+    /*
+     * ONCE PER STORE, BEFORE THE FIRST WRITE-ONCE WRITE: prove the create-only
+     * upload is real. Two create-only uploads of the same fixed key; the second
+     * must lose. An old vault-sdk (which drops the option) or a vault-api that
+     * ignores the header accepts it, and then `putIfAbsent` would overwrite
+     * silently while answering `true`. Two round trips per store; a reader never
+     * pays them. A probe that failed for any other reason (network, no grant) is
+     * not remembered, so the next write asks again.
+     */
+    let probe = null;
+    const proveCreateOnly = () => {
+        probe ??= (async () => {
+            const key = full(CONDITIONAL_WRITE_PROBE_KEY);
+            await createOnly(key, "{}");
+            if (await createOnly(key, "{}"))
+                throw new ConditionalWriteUnsupportedError();
+        })().catch((err) => {
+            if (!(err instanceof ConditionalWriteUnsupportedError))
+                probe = null;
+            throw err;
+        });
+        return probe;
+    };
+    const list = async (prefix) => {
+        // vault-api lists every page server-side and answers once, so there is
+        // no continuation to follow and nothing to be silently cut short. If its
+        // List handler ever starts paging (a cursor in the response), this must
+        // follow it in the same change, or listings truncate without a word.
+        const res = await objects.list({ prefix: full(prefix) });
+        return res.items
+            .map((i) => i.path)
+            .filter((p) => p.startsWith(root))
+            .map((p) => p.slice(root.length))
+            // The probe's key is the store's own, not eval data.
+            .filter((k) => k.startsWith(prefix) && k !== CONDITIONAL_WRITE_PROBE_KEY)
+            .sort();
+    };
+    return {
+        list,
+        /** Derived from the flat listing: the object API has no delimiter. */
+        async listPrefixes(prefix) {
+            const out = new Set();
+            for (const key of await list(prefix)) {
+                const slash = key.indexOf("/", prefix.length);
+                if (slash > prefix.length)
+                    out.add(key.slice(0, slash + 1));
+            }
+            return [...out].sort();
+        },
+        async get(key) {
+            try {
+                return decoder.decode(await objects.get(full(key)));
+            }
+            catch (err) {
+                // Only a missing object is `null`. Anything else (no grant, vault-api
+                // down) must throw, or `putCase`'s write-once check would pass on a
+                // failed read and overwrite a real case.
+                if (isNotFound(err))
+                    return null;
+                throw err;
+            }
+        },
+        async put(key, body) {
+            await objects.upload(full(key), encoder.encode(body), { contentType: "application/json" });
+        },
+        /**
+         * The server decides: `If-None-Match: *` on the upload, and a taken path
+         * answers `false`. No read first and no unconditional fallback: either
+         * could let two writers both believe they cut the same version. The first
+         * call proves the mechanism works (see `proveCreateOnly`) and refuses with
+         * `ConditionalWriteUnsupportedError` when it does not.
+         */
+        async putIfAbsent(key, body) {
+            await proveCreateOnly();
+            return createOnly(full(key), body);
+        },
+        /** Pinned by `vault-store.test.ts`, which reads `ifNoneMatch` off the upload this store sent. */
+        atomicity: "conditional-write",
+    };
+}
+//# sourceMappingURL=vault-store.js.map

package/dist/vault-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vault-store.js","sourceRoot":"","sources":["../src/vault-store.ts"],"names":[],"mappings":"AA+BA,MAAM,UAAU,GAAG,CAAC,GAAY,EAAE,KAAwB,EAAW,EAAE,CACrE,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,GAA+B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEvF;;;;;GAKG;AACH,MAAM,UAAU,GAAG,CAAC,GAAY,EAAW,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,kBAAkB,CAAC;AAE7F;;;;GAIG;AACH,MAAM,OAAO,GAAG,CAAC,GAAY,EAAW,EAAE,CACxC,UAAU,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,eAAe,CAAC;AAEnF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAElC;;;;;GAKG;AACH,MAAM,OAAO,gCAAiC,SAAQ,KAAK;IACzD;QACE,KAAK,CACH,sGAAsG;YACpG,4FAA4F;YAC5F,kDAAkD,CACrD,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kCAAkC,CAAC;IACjD,CAAC;CACF;AAED,yGAAyG;AACzG,MAAM,CAAC,MAAM,2BAA2B,GAAG,0BAA0B,CAAC;AAEtE,MAAM,UAAU,gBAAgB,CAAC,OAAqB,EAAE,OAAgC,EAAE;IACxF,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,IAAI,GAAG,GAAG,EAAE,CAAC;IAE9C,MAAM,UAAU,GAAG,KAAK,EAAE,IAAY,EAAE,IAAY,EAAoB,EAAE;QACxE,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YACxG,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/B,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC,CAAC;IAEF;;;;;;;;OAQG;IACH,IAAI,KAAK,GAAyB,IAAI,CAAC;IACvC,MAAM,eAAe,GAAG,GAAkB,EAAE;QAC1C,KAAK,KAAK,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,GAAG,GAAG,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC9C,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC5B,IAAI,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;gBAAE,MAAM,IAAI,gCAAgC,EAAE,CAAC;QAChF,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YAC1B,IAAI,CAAC,CAAC,GAAG,YAAY,gCAAgC,CAAC;gBAAE,KAAK,GAAG,IAAI,CAAC;YACrE,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,EAAE,MAAc,EAAqB,EAAE;QACvD,uEAAuE;QACvE,yEAAyE;QACzE,wEAAwE;QACxE,qEAAqE;QACrE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO,GAAG,CAAC,KAAK;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjC,qDAAqD;aACpD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,2BAA2B,CAAC;aACxE,IAAI,EAAE,CAAC;IACZ,CAAC,CAAC;IAEF,OAAO;QACL,IAAI;QAEJ,sEAAsE;QACtE,KAAK,CAAC,YAAY,CAAC,MAAM;YACvB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;YACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,GAAG;YACX,IAAI,CAAC;gBACH,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,sEAAsE;gBACtE,oEAAoE;gBACpE,yCAAyC;gBACzC,IAAI,UAAU,CAAC,GAAG,CAAC;oBAAE,OAAO,IAAI,CAAC;gBACjC,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI;YACjB,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED;;;;;;WAMG;QACH,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI;YACzB,MAAM,eAAe,EAAE,CAAC;YACxB,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,iGAAiG;QACjG,SAAS,EAAE,mBAAmB;KAC/B,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,30 @@
 {
   "name": "@cef-ai/eval",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.2.0",
+  "description": "Workflow evaluations for CEF: datasets, experiments, scoring and comparison over an injected store and workflow target.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.0",
+    "typescript": "^5.4.0",
+    "vitest": "^1.5.0"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "test": "tsc -p tsconfig.test.json && vitest run",
+    "lint": "eslint src"
+  }
 }