@workglow/eval 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -0
- package/dist/commands/dataset.d.ts +9 -0
- package/dist/commands/dataset.d.ts.map +1 -0
- package/dist/commands/report.d.ts +11 -0
- package/dist/commands/report.d.ts.map +1 -0
- package/dist/commands/run.d.ts +9 -0
- package/dist/commands/run.d.ts.map +1 -0
- package/dist/config.d.ts +29 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/evals/classify.d.ts +25 -0
- package/dist/evals/classify.d.ts.map +1 -0
- package/dist/evals/extract.d.ts +26 -0
- package/dist/evals/extract.d.ts.map +1 -0
- package/dist/evals/prompt.d.ts +12 -0
- package/dist/evals/prompt.d.ts.map +1 -0
- package/dist/evals/runner.d.ts +25 -0
- package/dist/evals/runner.d.ts.map +1 -0
- package/dist/evals/similarity.d.ts +15 -0
- package/dist/evals/similarity.d.ts.map +1 -0
- package/dist/evals/types.d.ts +39 -0
- package/dist/evals/types.d.ts.map +1 -0
- package/dist/hf/auth.d.ts +8 -0
- package/dist/hf/auth.d.ts.map +1 -0
- package/dist/hf/datasetsServer.d.ts +14 -0
- package/dist/hf/datasetsServer.d.ts.map +1 -0
- package/dist/hf/hubFiles.d.ts +38 -0
- package/dist/hf/hubFiles.d.ts.map +1 -0
- package/dist/hf/ids.d.ts +20 -0
- package/dist/hf/ids.d.ts.map +1 -0
- package/dist/hf/pullDataset.d.ts +26 -0
- package/dist/hf/pullDataset.d.ts.map +1 -0
- package/dist/hf/types.d.ts +33 -0
- package/dist/hf/types.d.ts.map +1 -0
- package/dist/models.d.ts +49 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/providers.d.ts +17 -0
- package/dist/providers.d.ts.map +1 -0
- package/dist/report/aggregate.d.ts +35 -0
- package/dist/report/aggregate.d.ts.map +1 -0
- package/dist/score/classification.d.ts +29 -0
- package/dist/score/classification.d.ts.map +1 -0
- package/dist/score/correlation.d.ts +12 -0
- package/dist/score/correlation.d.ts.map +1 -0
- package/dist/score/extraction.d.ts +47 -0
- package/dist/score/extraction.d.ts.map +1 -0
- package/dist/storage.d.ts +152 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/test/aggregate.test.d.ts +7 -0
- package/dist/test/aggregate.test.d.ts.map +1 -0
- package/dist/test/extractSmoke.e2e.test.d.ts +7 -0
- package/dist/test/extractSmoke.e2e.test.d.ts.map +1 -0
- package/dist/test/extraction.test.d.ts +7 -0
- package/dist/test/extraction.test.d.ts.map +1 -0
- package/dist/test/ggufSmoke.e2e.test.d.ts +7 -0
- package/dist/test/ggufSmoke.e2e.test.d.ts.map +1 -0
- package/dist/test/hubFiles.test.d.ts +7 -0
- package/dist/test/hubFiles.test.d.ts.map +1 -0
- package/dist/test/ids.test.d.ts +7 -0
- package/dist/test/ids.test.d.ts.map +1 -0
- package/dist/test/liveSimilarity.e2e.test.d.ts +7 -0
- package/dist/test/liveSimilarity.e2e.test.d.ts.map +1 -0
- package/dist/test/models.test.d.ts +7 -0
- package/dist/test/models.test.d.ts.map +1 -0
- package/dist/test/score.test.d.ts +7 -0
- package/dist/test/score.test.d.ts.map +1 -0
- package/dist/test/storage.test.d.ts +7 -0
- package/dist/test/storage.test.d.ts.map +1 -0
- package/dist/util.d.ts +13 -0
- package/dist/util.d.ts.map +1 -0
- package/dist/worker_hft.d.ts +7 -0
- package/dist/worker_hft.d.ts.map +1 -0
- package/dist/worker_hft.js +9 -0
- package/dist/worker_llamacpp.d.ts +7 -0
- package/dist/worker_llamacpp.d.ts.map +1 -0
- package/dist/worker_llamacpp.js +3 -0
- package/dist/workglow-eval.d.ts +8 -0
- package/dist/workglow-eval.d.ts.map +1 -0
- package/dist/workglow-eval.js +1266 -0
- package/package.json +55 -0
package/dist/models.d.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import type { ModelConfig } from "@workglow/ai";
|
|
7
|
+
export type EvalKind = "classify" | "similarity" | "extract";
|
|
8
|
+
/**
|
|
9
|
+
* Map a model id string to an inline {@link ModelConfig} by its shape, so any
|
|
10
|
+
* model can be named on the command line without a model repository:
|
|
11
|
+
*
|
|
12
|
+
* - `gguf:…` → local node-llama-cpp. The rest is a HuggingFace GGUF reference
|
|
13
|
+
* (`gguf:org/repo:Quant`, e.g. `gguf:prism-ml/Bonsai-27B-gguf:Q1_0`), an
|
|
14
|
+
* `hf:`/`https:` URL, or a path to a local `.gguf` file. The explicit prefix
|
|
15
|
+
* is needed because a GGUF repo id is indistinguishable from an ONNX one.
|
|
16
|
+
* - `org/name` (contains a slash) → local HuggingFace Transformers ONNX;
|
|
17
|
+
* the pipeline follows the eval kind (`text-generation` for classify,
|
|
18
|
+
* `feature-extraction` for similarity). Append `:dtype` to override the
|
|
19
|
+
* quantization (e.g. `onnx-community/Qwen3-0.6B-ONNX:q4`).
|
|
20
|
+
* - `claude-*` → Anthropic
|
|
21
|
+
* - `gpt-*`, `o<n>*`, `chatgpt-*`, `text-embedding-*` → OpenAI
|
|
22
|
+
* - `gemini-*` → Google Gemini
|
|
23
|
+
* - `grok-*` → xAI
|
|
24
|
+
*
|
|
25
|
+
* The slash check runs after the `gguf:` prefix but before the cloud
|
|
26
|
+
* prefixes: hub paths are unambiguous, and org names can legitimately start
|
|
27
|
+
* with a cloud prefix (e.g. `gpt-omni/mini-omni`).
|
|
28
|
+
*
|
|
29
|
+
* Inline configs skip the capability gate (treated as unverified-allow), so
|
|
30
|
+
* newly released model ids work without registering capability lists.
|
|
31
|
+
*/
|
|
32
|
+
export declare function resolveModelConfig(id: string, kind: EvalKind): ModelConfig;
|
|
33
|
+
/**
|
|
34
|
+
* Local ONNX embedding models must declare their vector width
|
|
35
|
+
* (`native_dimensions`) for the provider's dimension check. Look it up from
|
|
36
|
+
* the model's `config.json` on the hub (`hidden_size`, or `d_model` for
|
|
37
|
+
* encoder-decoder architectures) when the config doesn't carry one yet.
|
|
38
|
+
*/
|
|
39
|
+
export declare function ensureEmbeddingDimensions(config: ModelConfig): Promise<ModelConfig>;
|
|
40
|
+
/**
|
|
41
|
+
* GGUF weights are fetched by an explicit download step (the llama.cpp
|
|
42
|
+
* provider's generation run-fns load from disk only, unlike the ONNX provider
|
|
43
|
+
* which downloads inside its pipeline). Idempotent — an already-downloaded
|
|
44
|
+
* file is reused. No-op for every other provider.
|
|
45
|
+
*/
|
|
46
|
+
export declare function ensureModelDownloaded(config: ModelConfig): Promise<void>;
|
|
47
|
+
/** Parse the `--models "a,b,c"` flag into trimmed, de-duplicated ids. */
|
|
48
|
+
export declare function parseModelList(models: string): string[];
|
|
49
|
+
//# sourceMappingURL=models.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAMhD,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,YAAY,GAAG,SAAS,CAAC;AAI7D;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,GAAG,WAAW,CA2C1E;AAED;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAkBzF;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAI9E;AAQD,yEAAyE;AACzE,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CASvD"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import type { EvalConfig } from "./config";
|
|
7
|
+
/**
|
|
8
|
+
* Register task types and every AI provider the eval can route to. Cloud
|
|
9
|
+
* providers read their API keys from the environment (ANTHROPIC_API_KEY,
|
|
10
|
+
* OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY) at call time; the local
|
|
11
|
+
* HuggingFace Transformers (ONNX) and node-llama-cpp (GGUF) providers run in
|
|
12
|
+
* workers with their caches under the eval home. Each registration is
|
|
13
|
+
* defensive: a provider that fails to load only fails runs that target it,
|
|
14
|
+
* never the whole CLI.
|
|
15
|
+
*/
|
|
16
|
+
export declare function registerEvalProviders(config: EvalConfig): Promise<void>;
|
|
17
|
+
//# sourceMappingURL=providers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CA2C7E"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import type { EvalKind } from "../models";
|
|
7
|
+
import type { EvalResultRecord } from "../storage";
|
|
8
|
+
export interface ModelReport {
|
|
9
|
+
readonly model: string;
|
|
10
|
+
readonly rows: number;
|
|
11
|
+
readonly okRows: number;
|
|
12
|
+
/** classify: accuracy over ok rows (NaN when none). */
|
|
13
|
+
readonly accuracy: number;
|
|
14
|
+
/** similarity: correlations of predicted vs gold over ok rows (NaN when degenerate). */
|
|
15
|
+
readonly pearson: number;
|
|
16
|
+
readonly spearman: number;
|
|
17
|
+
/** extract: field agreement / entity recall / distinct-row precision (NaN when unscored). */
|
|
18
|
+
readonly score: number;
|
|
19
|
+
readonly found: number;
|
|
20
|
+
readonly prec: number;
|
|
21
|
+
readonly avgLatencyMs: number;
|
|
22
|
+
}
|
|
23
|
+
/** Per-kind scoring inputs beyond the stored results (extract only). */
|
|
24
|
+
export interface AggregateOptions {
|
|
25
|
+
readonly keyField?: string | undefined;
|
|
26
|
+
readonly fields?: readonly string[] | undefined;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Aggregate stored per-row results into one line per model, ranked best-first
|
|
30
|
+
* by the metric that matches the eval kind (accuracy for classify, Spearman
|
|
31
|
+
* for similarity, field agreement for extract), with average latency as the
|
|
32
|
+
* tie-breaker.
|
|
33
|
+
*/
|
|
34
|
+
export declare function aggregateResults(kind: EvalKind, results: readonly EvalResultRecord[], options?: AggregateOptions): ModelReport[];
|
|
35
|
+
//# sourceMappingURL=aggregate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aggregate.d.ts","sourceRoot":"","sources":["../../src/report/aggregate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAK1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,wFAAwF;IACxF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,6FAA6F;IAC7F,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,wEAAwE;AACxE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;CACjD;AAWD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,OAAO,GAAE,gBAAqB,GAC7B,WAAW,EAAE,CAmDf"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Case/whitespace/punctuation-insensitive label comparison key. Unicode-aware:
|
|
8
|
+
* letters and digits in any script survive (an ASCII-only class would collapse
|
|
9
|
+
* every accented or CJK label to the same empty key).
|
|
10
|
+
*/
|
|
11
|
+
export declare function normalizeLabel(label: string): string;
|
|
12
|
+
export interface ClassificationScore {
|
|
13
|
+
/** Rows where the model produced a prediction (no execution error). */
|
|
14
|
+
readonly scored: number;
|
|
15
|
+
readonly correct: number;
|
|
16
|
+
/** correct / scored, or NaN when nothing was scored. */
|
|
17
|
+
readonly accuracy: number;
|
|
18
|
+
/** Per-expected-label `correct/total` breakdown. */
|
|
19
|
+
readonly perLabel: ReadonlyMap<string, {
|
|
20
|
+
total: number;
|
|
21
|
+
correct: number;
|
|
22
|
+
}>;
|
|
23
|
+
}
|
|
24
|
+
/** Accuracy of predicted vs expected labels, normalized before comparison. */
|
|
25
|
+
export declare function scoreClassification(pairs: readonly {
|
|
26
|
+
expected: string;
|
|
27
|
+
predicted: string;
|
|
28
|
+
}[]): ClassificationScore;
|
|
29
|
+
//# sourceMappingURL=classification.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classification.d.ts","sourceRoot":"","sources":["../../src/score/classification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKpD;AAED,MAAM,WAAW,mBAAmB;IAClC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC5E;AAED,8EAA8E;AAC9E,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,SAAS;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,EAAE,GACxD,mBAAmB,CAmBrB"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
/** Pearson correlation coefficient of two equal-length series (NaN if degenerate). */
|
|
7
|
+
export declare function pearson(xs: readonly number[], ys: readonly number[]): number;
|
|
8
|
+
/** Fractional ranks (ties get the average rank), 1-based. */
|
|
9
|
+
export declare function ranks(values: readonly number[]): number[];
|
|
10
|
+
/** Spearman rank correlation: Pearson over fractional ranks. */
|
|
11
|
+
export declare function spearman(xs: readonly number[], ys: readonly number[]): number;
|
|
12
|
+
//# sourceMappingURL=correlation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"correlation.d.ts","sourceRoot":"","sources":["../../src/score/correlation.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,sFAAsF;AACtF,wBAAgB,OAAO,CAAC,EAAE,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAuB5E;AAED,6DAA6D;AAC7D,wBAAgB,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAazD;AAED,gEAAgE;AAChE,wBAAgB,QAAQ,CAAC,EAAE,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAG7E"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
/** One extracted entity/record: field name → value. */
|
|
7
|
+
export type ExtractedRow = Record<string, unknown>;
|
|
8
|
+
/**
|
|
9
|
+
* Counts for one (candidate, expected) comparison. Kept as raw counts so
|
|
10
|
+
* per-dataset-row results can be micro-averaged across a whole run.
|
|
11
|
+
*/
|
|
12
|
+
export interface ExtractionCounts {
|
|
13
|
+
/** Expected rows in total. */
|
|
14
|
+
readonly expected: number;
|
|
15
|
+
/** Expected rows a candidate row matched by key. */
|
|
16
|
+
readonly matched: number;
|
|
17
|
+
/** Distinct candidate rows (deduped on the key field). */
|
|
18
|
+
readonly candidateDistinct: number;
|
|
19
|
+
/** Distinct candidate rows that matched an expected row. */
|
|
20
|
+
readonly candidateMatched: number;
|
|
21
|
+
/** Scored fields across matched pairs. */
|
|
22
|
+
readonly fieldsTotal: number;
|
|
23
|
+
/** Scored fields whose normalized values agree. */
|
|
24
|
+
readonly fieldsAgreed: number;
|
|
25
|
+
}
|
|
26
|
+
export interface ExtractionScore extends ExtractionCounts {
|
|
27
|
+
/** Field-level agreement over matched rows (NaN when nothing matched). */
|
|
28
|
+
readonly score: number;
|
|
29
|
+
/** Entity recall: matched / expected (NaN when no expected rows). */
|
|
30
|
+
readonly found: number;
|
|
31
|
+
/** Precision: matching distinct candidate rows / distinct candidate rows. */
|
|
32
|
+
readonly prec: number;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Compare extracted rows against expected rows, aligning by `keyField`
|
|
36
|
+
* (case/punctuation-insensitive). `fields` limits which fields are scored for
|
|
37
|
+
* agreement; when omitted, every field present in the expected rows (other
|
|
38
|
+
* than empty values) is scored. The key field itself is never scored — a
|
|
39
|
+
* matched pair agrees on it by construction, so including it would inflate
|
|
40
|
+
* every score toward 1. A model that emits the same entity twice is
|
|
41
|
+
* over-producing rows, not inventing new ones, so precision is computed over
|
|
42
|
+
* distinct candidate rows.
|
|
43
|
+
*/
|
|
44
|
+
export declare function scoreExtraction(candidate: readonly ExtractedRow[], expected: readonly ExtractedRow[], keyField: string, fields?: readonly string[] | undefined): ExtractionScore;
|
|
45
|
+
/** Micro-average a set of per-row counts into one score. */
|
|
46
|
+
export declare function combineExtractionCounts(counts: readonly ExtractionCounts[]): ExtractionScore;
|
|
47
|
+
//# sourceMappingURL=extraction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/score/extraction.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,uDAAuD;AACvD,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,4DAA4D;IAC5D,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,0CAA0C;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,mDAAmD;IACnD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACvD,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAmBD;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,YAAY,EAAE,EAClC,QAAQ,EAAE,SAAS,YAAY,EAAE,EACjC,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GACrC,eAAe,CAoCjB;AAED,4DAA4D;AAC5D,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,SAAS,gBAAgB,EAAE,GAAG,eAAe,CAoB5F"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import type { ITabularStorage } from "@workglow/storage";
|
|
7
|
+
import type { FromSchema } from "@workglow/util/schema";
|
|
8
|
+
import type { EvalConfig } from "./config";
|
|
9
|
+
/**
|
|
10
|
+
* One pulled HuggingFace dataset split. `columns` and `label_names` are JSON
|
|
11
|
+
* strings so the record round-trips through any tabular backend unchanged.
|
|
12
|
+
*/
|
|
13
|
+
export declare const DatasetMetaSchema: {
|
|
14
|
+
readonly type: "object";
|
|
15
|
+
readonly properties: {
|
|
16
|
+
readonly dataset: {
|
|
17
|
+
readonly type: "string";
|
|
18
|
+
};
|
|
19
|
+
readonly split: {
|
|
20
|
+
readonly type: "string";
|
|
21
|
+
};
|
|
22
|
+
readonly config: {
|
|
23
|
+
readonly type: "string";
|
|
24
|
+
};
|
|
25
|
+
readonly num_rows: {
|
|
26
|
+
readonly type: "number";
|
|
27
|
+
};
|
|
28
|
+
readonly columns: {
|
|
29
|
+
readonly type: "string";
|
|
30
|
+
};
|
|
31
|
+
readonly label_names: {
|
|
32
|
+
readonly type: readonly ["string", "null"];
|
|
33
|
+
};
|
|
34
|
+
readonly source: {
|
|
35
|
+
readonly type: "string";
|
|
36
|
+
};
|
|
37
|
+
readonly fetched_at: {
|
|
38
|
+
readonly type: "string";
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
readonly required: readonly ["dataset", "split", "config", "num_rows", "columns", "source", "fetched_at"];
|
|
42
|
+
readonly additionalProperties: false;
|
|
43
|
+
};
|
|
44
|
+
export declare const DatasetMetaPrimaryKeyNames: readonly ["dataset", "split"];
|
|
45
|
+
/** One dataset row, stored as the raw JSON object the dataset provides. */
|
|
46
|
+
export declare const DatasetRowSchema: {
|
|
47
|
+
readonly type: "object";
|
|
48
|
+
readonly properties: {
|
|
49
|
+
readonly dataset: {
|
|
50
|
+
readonly type: "string";
|
|
51
|
+
};
|
|
52
|
+
readonly split: {
|
|
53
|
+
readonly type: "string";
|
|
54
|
+
};
|
|
55
|
+
readonly row_index: {
|
|
56
|
+
readonly type: "number";
|
|
57
|
+
};
|
|
58
|
+
readonly data: {
|
|
59
|
+
readonly type: "string";
|
|
60
|
+
};
|
|
61
|
+
};
|
|
62
|
+
readonly required: readonly ["dataset", "split", "row_index", "data"];
|
|
63
|
+
readonly additionalProperties: false;
|
|
64
|
+
};
|
|
65
|
+
export declare const DatasetRowPrimaryKeyNames: readonly ["dataset", "split", "row_index"];
|
|
66
|
+
export type DatasetRowRecord = FromSchema<typeof DatasetRowSchema>;
|
|
67
|
+
/** One eval sweep: a dataset split run through one workflow across N models. */
|
|
68
|
+
export declare const EvalRunSchema: {
|
|
69
|
+
readonly type: "object";
|
|
70
|
+
readonly properties: {
|
|
71
|
+
readonly run_id: {
|
|
72
|
+
readonly type: "string";
|
|
73
|
+
readonly "x-auto-generated": true;
|
|
74
|
+
};
|
|
75
|
+
readonly kind: {
|
|
76
|
+
readonly type: "string";
|
|
77
|
+
};
|
|
78
|
+
readonly dataset: {
|
|
79
|
+
readonly type: "string";
|
|
80
|
+
};
|
|
81
|
+
readonly split: {
|
|
82
|
+
readonly type: "string";
|
|
83
|
+
};
|
|
84
|
+
readonly models: {
|
|
85
|
+
readonly type: "string";
|
|
86
|
+
};
|
|
87
|
+
readonly options: {
|
|
88
|
+
readonly type: "string";
|
|
89
|
+
};
|
|
90
|
+
readonly created_at: {
|
|
91
|
+
readonly type: "string";
|
|
92
|
+
};
|
|
93
|
+
};
|
|
94
|
+
readonly required: readonly ["run_id", "kind", "dataset", "split", "models", "options", "created_at"];
|
|
95
|
+
readonly additionalProperties: false;
|
|
96
|
+
};
|
|
97
|
+
export declare const EvalRunPrimaryKeyNames: readonly ["run_id"];
|
|
98
|
+
/**
|
|
99
|
+
* One workflow execution result: (run, model, dataset row) → prediction.
|
|
100
|
+
* Classification-style evals fill `expected`/`predicted`; numeric evals fill
|
|
101
|
+
* `expected_value`/`predicted_value`. Failed executions set `ok: 0` and `error`.
|
|
102
|
+
*/
|
|
103
|
+
export declare const EvalResultSchema: {
|
|
104
|
+
readonly type: "object";
|
|
105
|
+
readonly properties: {
|
|
106
|
+
readonly run_id: {
|
|
107
|
+
readonly type: "string";
|
|
108
|
+
};
|
|
109
|
+
readonly model: {
|
|
110
|
+
readonly type: "string";
|
|
111
|
+
};
|
|
112
|
+
readonly row_index: {
|
|
113
|
+
readonly type: "number";
|
|
114
|
+
};
|
|
115
|
+
readonly ok: {
|
|
116
|
+
readonly type: "number";
|
|
117
|
+
};
|
|
118
|
+
readonly error: {
|
|
119
|
+
readonly type: readonly ["string", "null"];
|
|
120
|
+
};
|
|
121
|
+
readonly expected: {
|
|
122
|
+
readonly type: readonly ["string", "null"];
|
|
123
|
+
};
|
|
124
|
+
readonly predicted: {
|
|
125
|
+
readonly type: readonly ["string", "null"];
|
|
126
|
+
};
|
|
127
|
+
readonly expected_value: {
|
|
128
|
+
readonly type: readonly ["number", "null"];
|
|
129
|
+
};
|
|
130
|
+
readonly predicted_value: {
|
|
131
|
+
readonly type: readonly ["number", "null"];
|
|
132
|
+
};
|
|
133
|
+
readonly latency_ms: {
|
|
134
|
+
readonly type: "number";
|
|
135
|
+
};
|
|
136
|
+
};
|
|
137
|
+
readonly required: readonly ["run_id", "model", "row_index", "ok", "latency_ms"];
|
|
138
|
+
readonly additionalProperties: false;
|
|
139
|
+
};
|
|
140
|
+
export declare const EvalResultPrimaryKeyNames: readonly ["run_id", "model", "row_index"];
|
|
141
|
+
export type EvalResultRecord = FromSchema<typeof EvalResultSchema>;
|
|
142
|
+
export interface EvalStores {
|
|
143
|
+
readonly datasets: ITabularStorage<typeof DatasetMetaSchema, typeof DatasetMetaPrimaryKeyNames>;
|
|
144
|
+
readonly rows: ITabularStorage<typeof DatasetRowSchema, typeof DatasetRowPrimaryKeyNames>;
|
|
145
|
+
readonly runs: ITabularStorage<typeof EvalRunSchema, typeof EvalRunPrimaryKeyNames>;
|
|
146
|
+
readonly results: ITabularStorage<typeof EvalResultSchema, typeof EvalResultPrimaryKeyNames>;
|
|
147
|
+
}
|
|
148
|
+
/** Opens (creating on first use) the SQLite-backed eval stores. */
|
|
149
|
+
export declare function createSqliteStores(config: EvalConfig): Promise<EvalStores>;
|
|
150
|
+
/** In-memory stores with the same shape; used by the unit tests. */
|
|
151
|
+
export declare function createInMemoryStores(): Promise<EvalStores>;
|
|
152
|
+
//# sourceMappingURL=storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEzD,OAAO,KAAK,EAAwB,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC9E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C;;;GAGG;AACH,eAAO,MAAM,iBAAiB;mBACtB,QAAQ;;iBAEZ,OAAO;qBAAI,IAAI,EAAE,QAAQ;;iBACzB,KAAK;qBAAI,IAAI,EAAE,QAAQ;;iBACvB,MAAM;qBAAI,IAAI,EAAE,QAAQ;;iBACxB,QAAQ;qBAAI,IAAI,EAAE,QAAQ;;iBAC1B,OAAO;qBAAI,IAAI,EAAE,QAAQ;;iBACzB,WAAW;qBAAI,IAAI;;iBACnB,MAAM;qBAAI,IAAI,EAAE,QAAQ;;iBACxB,UAAU;qBAAI,IAAI,EAAE,QAAQ;;;;;CAIS,CAAC;AAE1C,eAAO,MAAM,0BAA0B,YAAI,SAAS,EAAE,OAAO,CAAU,CAAC;AAExE,2EAA2E;AAC3E,eAAO,MAAM,gBAAgB;mBACrB,QAAQ;;iBAEZ,OAAO;qBAAI,IAAI,EAAE,QAAQ;;iBACzB,KAAK;qBAAI,IAAI,EAAE,QAAQ;;iBACvB,SAAS;qBAAI,IAAI,EAAE,QAAQ;;iBAC3B,IAAI;qBAAI,IAAI,EAAE,QAAQ;;;;;CAIe,CAAC;AAE1C,eAAO,MAAM,yBAAyB,YAAI,SAAS,EAAE,OAAO,EAAE,WAAW,CAAU,CAAC;AACpF,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEnE,gFAAgF;AAChF,eAAO,MAAM,aAAa;mBAClB,QAAQ;;iBAEZ,MAAM;qBAAI,IAAI,EAAE,QAAQ;qBAAE,kBAAkB;;iBAC5C,IAAI;qBAAI,IAAI,EAAE,QAAQ;;iBACtB,OAAO;qBAAI,IAAI,EAAE,QAAQ;;iBACzB,KAAK;qBAAI,IAAI,EAAE,QAAQ;;iBACvB,MAAM;qBAAI,IAAI,EAAE,QAAQ;;iBACxB,OAAO;qBAAI,IAAI,EAAE,QAAQ;;iBACzB,UAAU;qBAAI,IAAI,EAAE,QAAQ;;;;;CAIS,CAAC;AAE1C,eAAO,MAAM,sBAAsB,YAAI,QAAQ,CAAU,CAAC;AAE1D;;;;GAIG;AACH,eAAO,MAAM,gBAAgB;mBACrB,QAAQ;;iBAEZ,MAAM;qBAAI,IAAI,EAAE,QAAQ;;iBACxB,KAAK;qBAAI,IAAI,EAAE,QAAQ;;iBACvB,SAAS;qBAAI,IAAI,EAAE,QAAQ;;iBAC3B,EAAE;qBAAI,IAAI,EAAE,QAAQ;;iBACpB,KAAK;qBAAI,IAAI;;iBACb,QAAQ;qBAAI,IAAI;;iBAChB,SAAS;qBAAI,IAAI;;iBACjB,cAAc;qBAAI,IAAI;;iBACtB,eAAe;qBAAI,IAAI;;iBACvB,UAAU;qBAAI,IAAI,EAAE,QAAQ;;;;;CAIS,CAAC;AAE1C,eAAO,MAAM,yBAAyB,YAAI,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAU,CAAC;AACnF,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEnE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC,OAAO,iBAAiB,EAAE,OAAO,0BAA0B,CAAC,CAAC;IAChG,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,gBAAgB,EAAE,OAAO,yBAAyB,CAAC,CAAC;IAC1F,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,aAAa,EAAE,OAAO,sBAAsB,CAAC,CAAC;IACpF,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC,OAAO,gBAAgB,EAAE,OAAO,yBAAyB,CAAC,CAAC;CAC9F;AAED,mEAAmE;AACnE,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CA0BhF;AAED,oEAAoE;AACpE,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,UAAU,CAAC,CAShE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aggregate.test.d.ts","sourceRoot":"","sources":["../../src/test/aggregate.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractSmoke.e2e.test.d.ts","sourceRoot":"","sources":["../../src/test/extractSmoke.e2e.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extraction.test.d.ts","sourceRoot":"","sources":["../../src/test/extraction.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ggufSmoke.e2e.test.d.ts","sourceRoot":"","sources":["../../src/test/ggufSmoke.e2e.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hubFiles.test.d.ts","sourceRoot":"","sources":["../../src/test/hubFiles.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ids.test.d.ts","sourceRoot":"","sources":["../../src/test/ids.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"liveSimilarity.e2e.test.d.ts","sourceRoot":"","sources":["../../src/test/liveSimilarity.e2e.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.test.d.ts","sourceRoot":"","sources":["../../src/test/models.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.test.d.ts","sourceRoot":"","sources":["../../src/test/score.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.test.d.ts","sourceRoot":"","sources":["../../src/test/storage.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
package/dist/util.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
/** Format an array of records as a simple aligned text table. */
|
|
7
|
+
export declare function formatTable(rows: Record<string, string>[], columns: string[]): string;
|
|
8
|
+
/** Fixed-precision number for table cells; NaN renders as "-". */
|
|
9
|
+
export declare function formatMetric(value: number, digits?: number): string;
|
|
10
|
+
export declare function formatError(err: unknown): string;
|
|
11
|
+
/** Parse an integer CLI flag, rejecting NaN and out-of-range values loudly. */
|
|
12
|
+
export declare function parseIntFlag(value: string, flag: string, minimum: number): number;
|
|
13
|
+
//# sourceMappingURL=util.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,iEAAiE;AACjE,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAcrF;AAED,kEAAkE;AAClE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,SAAI,GAAG,MAAM,CAE9D;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAEhD;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAMjF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker_hft.d.ts","sourceRoot":"","sources":["../src/worker_hft.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// src/worker_hft.ts
|
|
2
|
+
import {
|
|
3
|
+
registerHuggingFaceTransformersWorker,
|
|
4
|
+
setHftCacheDir
|
|
5
|
+
} from "@workglow/huggingface-transformers/ai-runtime";
|
|
6
|
+
if (process.env.WORKGLOW_MODEL_CACHE) {
|
|
7
|
+
setHftCacheDir(process.env.WORKGLOW_MODEL_CACHE);
|
|
8
|
+
}
|
|
9
|
+
await registerHuggingFaceTransformersWorker();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker_llamacpp.d.ts","sourceRoot":"","sources":["../src/worker_llamacpp.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workglow-eval.d.ts","sourceRoot":"","sources":["../src/workglow-eval.ts"],"names":[],"mappings":";AAEA;;;;GAIG"}
|