@tpsdev-ai/flair-bench 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +19 -0
- package/README.md +157 -0
- package/dist/cli-shim.cjs +49 -0
- package/dist/cli-shim.d.cts +21 -0
- package/dist/cli.d.ts +14 -0
- package/dist/cli.js +198 -0
- package/dist/corpus-v2.d.ts +102 -0
- package/dist/corpus-v2.js +1087 -0
- package/dist/cosine.d.ts +21 -0
- package/dist/cosine.js +42 -0
- package/dist/engine.d.ts +36 -0
- package/dist/engine.js +203 -0
- package/dist/format.d.ts +13 -0
- package/dist/format.js +69 -0
- package/dist/hostinfo.d.ts +14 -0
- package/dist/hostinfo.js +40 -0
- package/dist/index.d.ts +49 -0
- package/dist/index.js +82 -0
- package/dist/prefixes.d.ts +37 -0
- package/dist/prefixes.js +64 -0
- package/dist/recommend-heuristic.d.ts +22 -0
- package/dist/recommend-heuristic.js +48 -0
- package/dist/scorer.d.ts +32 -0
- package/dist/scorer.js +31 -0
- package/dist/share.d.ts +30 -0
- package/dist/share.js +75 -0
- package/dist/types.d.ts +156 -0
- package/dist/types.js +8 -0
- package/dist/version.d.ts +8 -0
- package/dist/version.js +8 -0
- package/package.json +68 -0
package/dist/prefixes.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prefixes.ts — the input-prefix convention table.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors resources/embeddings-provider.ts's Phase 2 behavior (flair#504):
|
|
5
|
+
* HFE 0.3.0's engine prepends a literal `"search_document: "` /
|
|
6
|
+
* `"search_query: "` string ahead of the text it hands to the model, for
|
|
7
|
+
* nomic-family models trained on that asymmetric-prefix convention. flair-
|
|
8
|
+
* bench talks to node-llama-cpp directly (no HFE in the loop — see the
|
|
9
|
+
* package README for why), so it has to replicate that string-prepend
|
|
10
|
+
* itself rather than pass an `inputType` option through to a wrapper that
|
|
11
|
+
* does it.
|
|
12
|
+
*
|
|
13
|
+
* KEYED ON MODEL FILENAME, for now — nomic is the only convention flair
|
|
14
|
+
* ships today, and there's exactly one production model family to match.
|
|
15
|
+
* This table is the seam for adding more: a future entry keys on another
|
|
16
|
+
* filename pattern (or, if a GGUF ever carries a documented convention-id
|
|
17
|
+
* in its own metadata, that becomes the preferred match — filename is a
|
|
18
|
+
* pragmatic fallback, not a design commitment). Nomic upstream has a request
|
|
19
|
+
* to carry this in the GGUF/HFE registration surface itself rather than
|
|
20
|
+
* every consumer re-deriving it from a filename convention — see
|
|
21
|
+
* harper-fabric-embeddings#4 (upstream proposal, not yet merged as of this
|
|
22
|
+
* writing).
|
|
23
|
+
*
|
|
24
|
+
* A model whose filename matches no entry gets `undefined` from
|
|
25
|
+
* `resolvePrefixConvention()` — texts are embedded unprefixed, and the
|
|
26
|
+
* result is labeled accordingly rather than silently guessing.
|
|
27
|
+
*/
|
|
28
|
+
const NOMIC_CONVENTION = {
|
|
29
|
+
id: "nomic-search-prefix",
|
|
30
|
+
documentPrefix: "search_document: ",
|
|
31
|
+
queryPrefix: "search_query: ",
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* The convention table. Order matters only in that the first match wins —
|
|
35
|
+
* kept a flat array (not a Map) since patterns can overlap in principle
|
|
36
|
+
* (e.g. a future family sharing a "nomic-embed" substring with a distinct
|
|
37
|
+
* suffix) and an explicit ordered scan makes that resolvable without a key
|
|
38
|
+
* collision.
|
|
39
|
+
*/
|
|
40
|
+
const CONVENTIONS = [
|
|
41
|
+
{
|
|
42
|
+
id: "nomic-embed-text-family",
|
|
43
|
+
// nomic-embed-text-v1.5.*, nomic-embed-text-v2-moe.* — both nomic
|
|
44
|
+
// families document the same search_document:/search_query: task-prefix
|
|
45
|
+
// convention on their model cards.
|
|
46
|
+
filenamePattern: /^nomic-embed-text/i,
|
|
47
|
+
convention: NOMIC_CONVENTION,
|
|
48
|
+
},
|
|
49
|
+
];
|
|
50
|
+
/** Resolve the prefix convention for a GGUF file, by basename. `undefined` = no known convention (embed unprefixed). */
|
|
51
|
+
export function resolvePrefixConvention(fileName) {
|
|
52
|
+
const base = fileName.replace(/^.*[/\\]/, "");
|
|
53
|
+
for (const entry of CONVENTIONS) {
|
|
54
|
+
if (entry.filenamePattern.test(base))
|
|
55
|
+
return entry.convention;
|
|
56
|
+
}
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
export function applyDocumentPrefix(text, convention) {
|
|
60
|
+
return convention ? `${convention.documentPrefix}${text}` : text;
|
|
61
|
+
}
|
|
62
|
+
export function applyQueryPrefix(text, convention) {
|
|
63
|
+
return convention ? `${convention.queryPrefix}${text}` : text;
|
|
64
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recommend-heuristic.ts — the pure "which model should I use" pick,
|
|
3
|
+
* factored out of index.ts's recommend() so it's unit-testable against
|
|
4
|
+
* fixture BatchResult objects (see test/recommend.test.ts) without ever
|
|
5
|
+
* touching node-llama-cpp or a real GGUF file.
|
|
6
|
+
*
|
|
7
|
+
* See recommend()'s own doc in index.ts for the honesty caveats this
|
|
8
|
+
* heuristic carries (single-request serial measurement, no concurrency
|
|
9
|
+
* model, fixed fraction/ceiling thresholds).
|
|
10
|
+
*/
|
|
11
|
+
import type { BatchResult, Recommendation } from "./types.js";
|
|
12
|
+
export declare const DEFAULT_RAM_HEADROOM_FRACTION = 0.5;
|
|
13
|
+
export declare const DEFAULT_LATENCY_THRESHOLD_MS = 500;
|
|
14
|
+
export interface RecommendHeuristicOptions {
|
|
15
|
+
ramHeadroomFraction?: number;
|
|
16
|
+
latencyThresholdMsPerEmbed?: number;
|
|
17
|
+
}
|
|
18
|
+
export interface RecommendHeuristicResult {
|
|
19
|
+
recommendation: Recommendation | null;
|
|
20
|
+
notes: string[];
|
|
21
|
+
}
|
|
22
|
+
export declare function pickRecommendation(batch: BatchResult, options?: RecommendHeuristicOptions): RecommendHeuristicResult;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recommend-heuristic.ts — the pure "which model should I use" pick,
|
|
3
|
+
* factored out of index.ts's recommend() so it's unit-testable against
|
|
4
|
+
* fixture BatchResult objects (see test/recommend.test.ts) without ever
|
|
5
|
+
* touching node-llama-cpp or a real GGUF file.
|
|
6
|
+
*
|
|
7
|
+
* See recommend()'s own doc in index.ts for the honesty caveats this
|
|
8
|
+
* heuristic carries (single-request serial measurement, no concurrency
|
|
9
|
+
* model, fixed fraction/ceiling thresholds).
|
|
10
|
+
*/
|
|
11
|
+
import { basename } from "node:path";
|
|
12
|
+
export const DEFAULT_RAM_HEADROOM_FRACTION = 0.5; // generous: a model may use up to half of currently-available RAM
|
|
13
|
+
export const DEFAULT_LATENCY_THRESHOLD_MS = 500; // generous: 500ms/embed serial is a very permissive ceiling
|
|
14
|
+
export function pickRecommendation(batch, options = {}) {
|
|
15
|
+
const ramHeadroomFraction = options.ramHeadroomFraction ?? DEFAULT_RAM_HEADROOM_FRACTION;
|
|
16
|
+
const latencyThresholdMsPerEmbed = options.latencyThresholdMsPerEmbed ?? DEFAULT_LATENCY_THRESHOLD_MS;
|
|
17
|
+
const notes = [
|
|
18
|
+
`Heuristic: best measured recall (MRR) among models whose peak RSS delta fits within ${Math.round(ramHeadroomFraction * 100)}% of available RAM (${(batch.host.availableRamGiB * ramHeadroomFraction).toFixed(1)} GiB budget on this host) and whose ms/embed is <= ${latencyThresholdMsPerEmbed}ms. Doesn't know your concurrency, batching, or serving framework — see README for the full limits list.`,
|
|
19
|
+
];
|
|
20
|
+
if (batch.models.length === 0) {
|
|
21
|
+
return { recommendation: null, notes: [...notes, "No models were benchmarked."] };
|
|
22
|
+
}
|
|
23
|
+
const budgetMiB = batch.host.availableRamGiB * ramHeadroomFraction * 1024;
|
|
24
|
+
const affordable = batch.models.filter((m) => m.peakRssDeltaMiB <= budgetMiB && m.msPerEmbedSerialWarm <= latencyThresholdMsPerEmbed);
|
|
25
|
+
let pool = affordable;
|
|
26
|
+
if (affordable.length === 0) {
|
|
27
|
+
pool = batch.models;
|
|
28
|
+
notes.push(`No model fit the RAM/latency budget above — falling back to ranking all ${batch.models.length} benchmarked model(s) by recall alone. Treat this pick as informational, not a "fits your host" claim.`);
|
|
29
|
+
}
|
|
30
|
+
const ranked = [...pool].sort((a, b) => b.aggregate.mrr - a.aggregate.mrr || a.msPerEmbedSerialWarm - b.msPerEmbedSerialWarm);
|
|
31
|
+
const best = ranked[0];
|
|
32
|
+
const runnerUp = ranked[1];
|
|
33
|
+
let reason;
|
|
34
|
+
if (runnerUp) {
|
|
35
|
+
reason =
|
|
36
|
+
`${basename(best.model.fileName)} because p@3 ${best.aggregate.p3.toFixed(3)} vs ${runnerUp.aggregate.p3.toFixed(3)} ` +
|
|
37
|
+
`(MRR ${best.aggregate.mrr.toFixed(3)} vs ${runnerUp.aggregate.mrr.toFixed(3)}) at ${best.msPerEmbedSerialWarm.toFixed(1)}ms/embed ` +
|
|
38
|
+
`and ${best.peakRssDeltaMiB.toFixed(0)} MiB peak RSS on your ${batch.host.availableRamGiB.toFixed(1)}GiB-available ${batch.host.backend} host.`;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
reason =
|
|
42
|
+
`${basename(best.model.fileName)} — the only model benchmarked (p@3 ${best.aggregate.p3.toFixed(3)}, MRR ${best.aggregate.mrr.toFixed(3)}) ` +
|
|
43
|
+
`at ${best.msPerEmbedSerialWarm.toFixed(1)}ms/embed and ${best.peakRssDeltaMiB.toFixed(0)} MiB peak RSS on your ${batch.host.availableRamGiB.toFixed(1)}GiB-available ${batch.host.backend} host. ` +
|
|
44
|
+
`Run with more --model-file flags to get a real comparison.`;
|
|
45
|
+
}
|
|
46
|
+
const recommendation = { fileName: best.model.fileName, reason };
|
|
47
|
+
return { recommendation, notes };
|
|
48
|
+
}
|
package/dist/scorer.d.ts
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scorer.ts — precision@3 / MRR scoring, faithfully replicated from
|
|
3
|
+
* test/bench/recall-harness/run.ts's `statsFor()` (that function is not
|
|
4
|
+
* exported, so it can't be imported directly — see test/scorer-sync.test.ts,
|
|
5
|
+
* which parses run.ts's source text and asserts it still contains this
|
|
6
|
+
* exact formula, so a change to the harness's scoring math trips a loud
|
|
7
|
+
* failure here instead of silently drifting).
|
|
8
|
+
*
|
|
9
|
+
* KEEP THIS FUNCTION BYTE-FOR-BYTE IDENTICAL (modulo variable/type names) to
|
|
10
|
+
* run.ts's `statsFor()`:
|
|
11
|
+
*
|
|
12
|
+
* function statsFor(rows: QueryRow[]): { p3: number; mrr: number; n: number } {
|
|
13
|
+
* if (!rows.length) return { p3: 0, mrr: 0, n: 0 };
|
|
14
|
+
* const hits3 = rows.filter(r => r.rank >= 0 && r.rank < 3).length;
|
|
15
|
+
* const rr = rows.reduce((s, r) => s + (r.rank >= 0 ? 1 / (r.rank + 1) : 0), 0);
|
|
16
|
+
* return { p3: hits3 / rows.length, mrr: rr / rows.length, n: rows.length };
|
|
17
|
+
* }
|
|
18
|
+
*/
|
|
19
|
+
import type { QueryKind } from "./corpus-v2.js";
|
|
20
|
+
import type { AggregateStat, PerKindStat } from "./types.js";
|
|
21
|
+
export interface ScoredRow {
|
|
22
|
+
/** 0-based rank of the expected record, or -1 if not found. */
|
|
23
|
+
rank: number;
|
|
24
|
+
kind: QueryKind;
|
|
25
|
+
}
|
|
26
|
+
export declare function statsFor(rows: readonly {
|
|
27
|
+
rank: number;
|
|
28
|
+
}[]): AggregateStat;
|
|
29
|
+
export declare function scoreRows(rows: readonly ScoredRow[]): {
|
|
30
|
+
aggregate: AggregateStat;
|
|
31
|
+
perKind: Record<QueryKind, PerKindStat>;
|
|
32
|
+
};
|
package/dist/scorer.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scorer.ts — precision@3 / MRR scoring, faithfully replicated from
|
|
3
|
+
* test/bench/recall-harness/run.ts's `statsFor()` (that function is not
|
|
4
|
+
* exported, so it can't be imported directly — see test/scorer-sync.test.ts,
|
|
5
|
+
* which parses run.ts's source text and asserts it still contains this
|
|
6
|
+
* exact formula, so a change to the harness's scoring math trips a loud
|
|
7
|
+
* failure here instead of silently drifting).
|
|
8
|
+
*
|
|
9
|
+
* KEEP THIS FUNCTION BYTE-FOR-BYTE IDENTICAL (modulo variable/type names) to
|
|
10
|
+
* run.ts's `statsFor()`:
|
|
11
|
+
*
|
|
12
|
+
* function statsFor(rows: QueryRow[]): { p3: number; mrr: number; n: number } {
|
|
13
|
+
* if (!rows.length) return { p3: 0, mrr: 0, n: 0 };
|
|
14
|
+
* const hits3 = rows.filter(r => r.rank >= 0 && r.rank < 3).length;
|
|
15
|
+
* const rr = rows.reduce((s, r) => s + (r.rank >= 0 ? 1 / (r.rank + 1) : 0), 0);
|
|
16
|
+
* return { p3: hits3 / rows.length, mrr: rr / rows.length, n: rows.length };
|
|
17
|
+
* }
|
|
18
|
+
*/
|
|
19
|
+
export function statsFor(rows) {
|
|
20
|
+
if (!rows.length)
|
|
21
|
+
return { p3: 0, mrr: 0, n: 0 };
|
|
22
|
+
const hits3 = rows.filter((r) => r.rank >= 0 && r.rank < 3).length;
|
|
23
|
+
const rr = rows.reduce((s, r) => s + (r.rank >= 0 ? 1 / (r.rank + 1) : 0), 0);
|
|
24
|
+
return { p3: hits3 / rows.length, mrr: rr / rows.length, n: rows.length };
|
|
25
|
+
}
|
|
26
|
+
const ALL_KINDS = ["stress", "trap", "hard", "clean"];
|
|
27
|
+
export function scoreRows(rows) {
|
|
28
|
+
const aggregate = statsFor(rows);
|
|
29
|
+
const perKind = Object.fromEntries(ALL_KINDS.map((k) => [k, statsFor(rows.filter((r) => r.kind === k))]));
|
|
30
|
+
return { aggregate, perKind };
|
|
31
|
+
}
|
package/dist/share.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* share.ts — canonical share-JSON schema + local write.
|
|
3
|
+
*
|
|
4
|
+
* PRIVACY CONTRACT: the document built here MUST NEVER contain a hostname,
|
|
5
|
+
* filesystem path, or username. `model.fileBasename` is a basename (see
|
|
6
|
+
* ModelIdentity.fileName's own doc — never the full path); `hardware.label`
|
|
7
|
+
* is a freeform string the CALLER chose (see types.ts's HostFingerprint —
|
|
8
|
+
* defaults to nothing, never auto-filled from os.hostname()). See
|
|
9
|
+
* test/share-schema.test.ts for the automated forbidden-field check this
|
|
10
|
+
* contract is gated on.
|
|
11
|
+
*
|
|
12
|
+
* The submission endpoint is a placeholder — see writeShareDocument's doc.
|
|
13
|
+
* No network call happens anywhere in this file.
|
|
14
|
+
*/
|
|
15
|
+
import type { ModelBenchResult, HostFingerprint, ShareDocument } from "./types.js";
|
|
16
|
+
/** Config placeholder for the future hosted submission endpoint (not implemented — see README "Stubbed"). */
|
|
17
|
+
export declare const SUBMISSION_ENDPOINT_PLACEHOLDER = "https://bench.tps.dev/api/submit";
|
|
18
|
+
export declare function buildShareDocument(result: ModelBenchResult, host: HostFingerprint): ShareDocument;
|
|
19
|
+
export interface WrittenShare {
|
|
20
|
+
filePath: string;
|
|
21
|
+
endpointNote: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Writes the share document locally and returns where. The hosted
|
|
25
|
+
* submission endpoint (SUBMISSION_ENDPOINT_PLACEHOLDER) is not wired up —
|
|
26
|
+
* this function never makes a network call; the CLI layer is what prints
|
|
27
|
+
* `endpointNote` to the user (core does no console output — see index.ts's
|
|
28
|
+
* header).
|
|
29
|
+
*/
|
|
30
|
+
export declare function writeShareDocument(doc: ShareDocument, outDir?: string): WrittenShare;
|
package/dist/share.js
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* share.ts — canonical share-JSON schema + local write.
|
|
3
|
+
*
|
|
4
|
+
* PRIVACY CONTRACT: the document built here MUST NEVER contain a hostname,
|
|
5
|
+
* filesystem path, or username. `model.fileBasename` is a basename (see
|
|
6
|
+
* ModelIdentity.fileName's own doc — never the full path); `hardware.label`
|
|
7
|
+
* is a freeform string the CALLER chose (see types.ts's HostFingerprint —
|
|
8
|
+
* defaults to nothing, never auto-filled from os.hostname()). See
|
|
9
|
+
* test/share-schema.test.ts for the automated forbidden-field check this
|
|
10
|
+
* contract is gated on.
|
|
11
|
+
*
|
|
12
|
+
* The submission endpoint is a placeholder — see writeShareDocument's doc.
|
|
13
|
+
* No network call happens anywhere in this file.
|
|
14
|
+
*/
|
|
15
|
+
import { writeFileSync, mkdirSync } from "node:fs";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { TOOL_VERSION } from "./version.js";
|
|
18
|
+
/** Config placeholder for the future hosted submission endpoint (not implemented — see README "Stubbed"). */
|
|
19
|
+
export const SUBMISSION_ENDPOINT_PLACEHOLDER = "https://bench.tps.dev/api/submit"; // not live; no network call is ever made against this
|
|
20
|
+
function deriveModelName(fileBasename) {
|
|
21
|
+
// Strip a trailing .gguf and a recognizable quant/variant suffix so the
|
|
22
|
+
// "name" reads as the model family, e.g. "nomic-embed-text-v1.5.Q4_K_M.gguf"
|
|
23
|
+
// -> "nomic-embed-text-v1.5". Best-effort — falls back to the basename
|
|
24
|
+
// minus extension if no quant suffix is recognized.
|
|
25
|
+
const withoutExt = fileBasename.replace(/\.gguf$/i, "");
|
|
26
|
+
const withoutQuant = withoutExt.replace(/\.((?:Q|IQ)[0-9][A-Z0-9_]*|F16|F32|BF16)$/i, "");
|
|
27
|
+
return withoutQuant;
|
|
28
|
+
}
|
|
29
|
+
export function buildShareDocument(result, host) {
|
|
30
|
+
return {
|
|
31
|
+
toolVersion: TOOL_VERSION,
|
|
32
|
+
timestamp: new Date().toISOString(),
|
|
33
|
+
model: {
|
|
34
|
+
name: deriveModelName(result.model.fileName),
|
|
35
|
+
fileBasename: result.model.fileName,
|
|
36
|
+
sha256: result.model.sha256,
|
|
37
|
+
quant: result.model.quant,
|
|
38
|
+
paramsApprox: result.model.paramsApprox,
|
|
39
|
+
dims: result.model.dims,
|
|
40
|
+
},
|
|
41
|
+
hardware: {
|
|
42
|
+
label: host.label,
|
|
43
|
+
platform: host.platform,
|
|
44
|
+
arch: host.arch,
|
|
45
|
+
cpuModel: host.cpuModel,
|
|
46
|
+
ramGiB: Math.round(host.totalRamGiB * 10) / 10,
|
|
47
|
+
gpu: host.gpuDeviceNames.length > 0 ? host.gpuDeviceNames.join(", ") : null,
|
|
48
|
+
backend: host.backend,
|
|
49
|
+
},
|
|
50
|
+
results: {
|
|
51
|
+
aggregate: result.aggregate,
|
|
52
|
+
perKind: result.perKind,
|
|
53
|
+
msPerEmbedSerialWarm: Math.round(result.msPerEmbedSerialWarm * 100) / 100,
|
|
54
|
+
peakRssMiB: Math.round(result.peakRssDeltaMiB * 10) / 10,
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Writes the share document locally and returns where. The hosted
|
|
60
|
+
* submission endpoint (SUBMISSION_ENDPOINT_PLACEHOLDER) is not wired up —
|
|
61
|
+
* this function never makes a network call; the CLI layer is what prints
|
|
62
|
+
* `endpointNote` to the user (core does no console output — see index.ts's
|
|
63
|
+
* header).
|
|
64
|
+
*/
|
|
65
|
+
export function writeShareDocument(doc, outDir = ".") {
|
|
66
|
+
mkdirSync(outDir, { recursive: true });
|
|
67
|
+
const safeModelName = doc.model.name.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
68
|
+
const fileName = `flair-bench-share-${safeModelName}-${doc.timestamp.replace(/[:.]/g, "-")}.json`;
|
|
69
|
+
const filePath = join(outDir, fileName);
|
|
70
|
+
writeFileSync(filePath, JSON.stringify(doc, null, 2) + "\n", "utf8");
|
|
71
|
+
return {
|
|
72
|
+
filePath,
|
|
73
|
+
endpointNote: `submission endpoint not yet configured — file saved at ${filePath}`,
|
|
74
|
+
};
|
|
75
|
+
}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* types.ts — shared types for the flair-bench public API.
|
|
3
|
+
*
|
|
4
|
+
* Kept dependency-free (no node-llama-cpp types leak here) so a future
|
|
5
|
+
* `flair bench` CLI subcommand (see README "Library use") can import just
|
|
6
|
+
* the shapes it needs without pulling in the embedding engine.
|
|
7
|
+
*/
|
|
8
|
+
import type { QueryKind } from "./corpus-v2.js";
|
|
9
|
+
export type { QueryKind };
|
|
10
|
+
/** "metal" | "cuda" | "vulkan" — the backend node-llama-cpp actually loaded, or "cpu" if none. */
|
|
11
|
+
export type ComputeBackend = "metal" | "cuda" | "vulkan" | "cpu";
|
|
12
|
+
/**
|
|
13
|
+
* Host fingerprint. `label` is a freeform, user-chosen grouping key (e.g.
|
|
14
|
+
* "fabric-free-gcp", "local-m4-mini") — NOT a hostname. Real hostnames never
|
|
15
|
+
* appear here; see share.ts for the privacy contract that leans on this.
|
|
16
|
+
*/
|
|
17
|
+
export interface HostFingerprint {
|
|
18
|
+
label?: string;
|
|
19
|
+
platform: string;
|
|
20
|
+
arch: string;
|
|
21
|
+
cpuModel: string;
|
|
22
|
+
totalRamGiB: number;
|
|
23
|
+
availableRamGiB: number;
|
|
24
|
+
backend: ComputeBackend;
|
|
25
|
+
/** GPU device name(s) node-llama-cpp reports, e.g. ["Apple M4 Pro"]. Empty when backend is "cpu". */
|
|
26
|
+
gpuDeviceNames: string[];
|
|
27
|
+
}
|
|
28
|
+
/** Everything about the GGUF file identifying the model under test. */
|
|
29
|
+
export interface ModelIdentity {
|
|
30
|
+
/** Basename only — never the full path (privacy: no local filesystem layout in shared results). */
|
|
31
|
+
fileName: string;
|
|
32
|
+
sha256: string;
|
|
33
|
+
sizeBytes: number;
|
|
34
|
+
/** e.g. "Q4_K_M" — read from GGUF metadata when present, else parsed from the filename. */
|
|
35
|
+
quant: string;
|
|
36
|
+
quantSource: "gguf-metadata" | "filename-fallback";
|
|
37
|
+
/** Embedding vector dimensionality. */
|
|
38
|
+
dims: number;
|
|
39
|
+
/**
|
|
40
|
+
* Approximate total parameter count, computed by summing every tensor's
|
|
41
|
+
* element count from the GGUF's own tensor index (the same method
|
|
42
|
+
* llama.cpp's own CLI reports "params" with) — not read from a
|
|
43
|
+
* general.parameter_count field, which embedding-architecture GGUFs don't
|
|
44
|
+
* reliably carry.
|
|
45
|
+
*/
|
|
46
|
+
paramsApprox: number;
|
|
47
|
+
/** Bits per weight: (file size in bits) / paramsApprox. */
|
|
48
|
+
bpw: number;
|
|
49
|
+
}
|
|
50
|
+
export interface PerKindStat {
|
|
51
|
+
n: number;
|
|
52
|
+
p3: number;
|
|
53
|
+
mrr: number;
|
|
54
|
+
}
|
|
55
|
+
export interface AggregateStat {
|
|
56
|
+
n: number;
|
|
57
|
+
p3: number;
|
|
58
|
+
mrr: number;
|
|
59
|
+
}
|
|
60
|
+
export interface ModelBenchResult {
|
|
61
|
+
model: ModelIdentity;
|
|
62
|
+
loadTimeMs: number;
|
|
63
|
+
/** Serial, warm (post-warmup), N>=64 embed calls; mean ms/embed. */
|
|
64
|
+
msPerEmbedSerialWarm: number;
|
|
65
|
+
/** process.memoryUsage().rss peak observed during this model's load+embed phase, minus the pre-load baseline. */
|
|
66
|
+
peakRssDeltaMiB: number;
|
|
67
|
+
aggregate: AggregateStat;
|
|
68
|
+
perKind: Record<QueryKind, PerKindStat>;
|
|
69
|
+
}
|
|
70
|
+
export interface CorpusInfo {
|
|
71
|
+
version: "v2";
|
|
72
|
+
records: number;
|
|
73
|
+
queries: number;
|
|
74
|
+
}
|
|
75
|
+
export interface BatchResult {
|
|
76
|
+
toolVersion: string;
|
|
77
|
+
timestamp: string;
|
|
78
|
+
corpus: CorpusInfo;
|
|
79
|
+
host: HostFingerprint;
|
|
80
|
+
models: ModelBenchResult[];
|
|
81
|
+
}
|
|
82
|
+
export interface BenchOptions {
|
|
83
|
+
/** One or more GGUF file paths. Batch = multiple entries. */
|
|
84
|
+
modelFiles: string[];
|
|
85
|
+
/** Freeform host-grouping label — see HostFingerprint.label. */
|
|
86
|
+
label?: string;
|
|
87
|
+
/** Number of corpus texts embedded during warmup before timing starts. Default 8. */
|
|
88
|
+
warmupN?: number;
|
|
89
|
+
/** Progress/diagnostic callback — core never calls console.*; a caller (e.g. the CLI) wires this to stderr. */
|
|
90
|
+
onProgress?: (event: BenchProgressEvent) => void;
|
|
91
|
+
}
|
|
92
|
+
export type BenchProgressEvent = {
|
|
93
|
+
type: "host-fingerprinted";
|
|
94
|
+
host: HostFingerprint;
|
|
95
|
+
} | {
|
|
96
|
+
type: "model-start";
|
|
97
|
+
fileName: string;
|
|
98
|
+
index: number;
|
|
99
|
+
total: number;
|
|
100
|
+
} | {
|
|
101
|
+
type: "model-loaded";
|
|
102
|
+
fileName: string;
|
|
103
|
+
loadTimeMs: number;
|
|
104
|
+
} | {
|
|
105
|
+
type: "model-embedding";
|
|
106
|
+
fileName: string;
|
|
107
|
+
done: number;
|
|
108
|
+
total: number;
|
|
109
|
+
} | {
|
|
110
|
+
type: "model-done";
|
|
111
|
+
fileName: string;
|
|
112
|
+
result: ModelBenchResult;
|
|
113
|
+
};
|
|
114
|
+
export interface RecommendOptions extends BenchOptions {
|
|
115
|
+
/** Fraction of *available* RAM a model's peak RSS delta may use. Default 0.5 (generous headroom). */
|
|
116
|
+
ramHeadroomFraction?: number;
|
|
117
|
+
/** ms/embed ceiling for a model to be considered "usable" on this host. Default 500 (generous). */
|
|
118
|
+
latencyThresholdMsPerEmbed?: number;
|
|
119
|
+
}
|
|
120
|
+
export interface Recommendation {
|
|
121
|
+
fileName: string;
|
|
122
|
+
reason: string;
|
|
123
|
+
}
|
|
124
|
+
export interface RecommendResult {
|
|
125
|
+
batch: BatchResult;
|
|
126
|
+
recommendation: Recommendation | null;
|
|
127
|
+
/** Human-readable lines explaining the heuristic's inputs/limits — always populated, even with no pick. */
|
|
128
|
+
notes: string[];
|
|
129
|
+
}
|
|
130
|
+
export interface ShareDocument {
|
|
131
|
+
toolVersion: string;
|
|
132
|
+
timestamp: string;
|
|
133
|
+
model: {
|
|
134
|
+
name: string;
|
|
135
|
+
fileBasename: string;
|
|
136
|
+
sha256: string;
|
|
137
|
+
quant: string;
|
|
138
|
+
paramsApprox: number;
|
|
139
|
+
dims: number;
|
|
140
|
+
};
|
|
141
|
+
hardware: {
|
|
142
|
+
label?: string;
|
|
143
|
+
platform: string;
|
|
144
|
+
arch: string;
|
|
145
|
+
cpuModel: string;
|
|
146
|
+
ramGiB: number;
|
|
147
|
+
gpu: string | null;
|
|
148
|
+
backend: ComputeBackend;
|
|
149
|
+
};
|
|
150
|
+
results: {
|
|
151
|
+
aggregate: AggregateStat;
|
|
152
|
+
perKind: Record<QueryKind, PerKindStat>;
|
|
153
|
+
msPerEmbedSerialWarm: number;
|
|
154
|
+
peakRssMiB: number;
|
|
155
|
+
};
|
|
156
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* types.ts — shared types for the flair-bench public API.
|
|
3
|
+
*
|
|
4
|
+
* Kept dependency-free (no node-llama-cpp types leak here) so a future
|
|
5
|
+
* `flair bench` CLI subcommand (see README "Library use") can import just
|
|
6
|
+
* the shapes it needs without pulling in the embedding engine.
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* version.ts — single source of truth for the tool version stamped into
|
|
3
|
+
* output/share documents. Kept as a plain constant (rather than a JSON
|
|
4
|
+
* import of package.json at runtime) to sidestep NodeNext JSON-import-
|
|
5
|
+
* attribute edge cases in a published dist/ build. test/version-sync.test.ts
|
|
6
|
+
* asserts this stays equal to package.json's "version" field.
|
|
7
|
+
*/
|
|
8
|
+
export declare const TOOL_VERSION = "0.22.0";
|
package/dist/version.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* version.ts — single source of truth for the tool version stamped into
|
|
3
|
+
* output/share documents. Kept as a plain constant (rather than a JSON
|
|
4
|
+
* import of package.json at runtime) to sidestep NodeNext JSON-import-
|
|
5
|
+
* attribute edge cases in a published dist/ build. test/version-sync.test.ts
|
|
6
|
+
* asserts this stays equal to package.json's "version" field.
|
|
7
|
+
*/
|
|
8
|
+
export const TOOL_VERSION = "0.22.0";
|
package/package.json
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@tpsdev-ai/flair-bench",
|
|
3
|
+
"version": "0.22.0",
|
|
4
|
+
"description": "Standalone embedding recall benchmark for flair — run real recall numbers (p@3/MRR) against any GGUF embedding model, batch-compare, get a host-aware recommendation, and share redacted results. No flair install required.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"import": "./dist/index.js",
|
|
11
|
+
"types": "./dist/index.d.ts"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"bin": {
|
|
15
|
+
"flair-bench": "dist/cli-shim.cjs"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"dist/",
|
|
19
|
+
"LICENSE",
|
|
20
|
+
"README.md"
|
|
21
|
+
],
|
|
22
|
+
"scripts": {
|
|
23
|
+
"build": "tsc --noCheck",
|
|
24
|
+
"test": "bun test",
|
|
25
|
+
"sync:corpus": "node scripts/sync-corpus.mjs",
|
|
26
|
+
"prepublishOnly": "npm run build",
|
|
27
|
+
"postinstall": "node -e \"try{const{chmodSync,statSync}=require('fs');for(const p of ['dist/cli-shim.cjs','dist/cli.js']){try{if(statSync(p).isFile()){chmodSync(p,0o755);console.error('@tpsdev-ai/flair-bench: chmod +x ' + p + ' OK')}}catch(e){if(e.code!=='ENOENT')console.error('postinstall warn:',e.message)}}}catch(e){console.error('postinstall warn:',e.message)}\""
|
|
28
|
+
},
|
|
29
|
+
"publishConfig": {
|
|
30
|
+
"access": "public"
|
|
31
|
+
},
|
|
32
|
+
"engines": {
|
|
33
|
+
"node": ">=22"
|
|
34
|
+
},
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"node-llama-cpp": "3.18.1"
|
|
37
|
+
},
|
|
38
|
+
"optionalDependencies": {
|
|
39
|
+
"@node-llama-cpp/linux-arm64": "3.18.1",
|
|
40
|
+
"@node-llama-cpp/linux-armv7l": "3.18.1",
|
|
41
|
+
"@node-llama-cpp/linux-x64": "3.18.1",
|
|
42
|
+
"@node-llama-cpp/mac-arm64-metal": "3.18.1",
|
|
43
|
+
"@node-llama-cpp/mac-x64": "3.18.1",
|
|
44
|
+
"@node-llama-cpp/win-arm64": "3.18.1",
|
|
45
|
+
"@node-llama-cpp/win-x64": "3.18.1"
|
|
46
|
+
},
|
|
47
|
+
"devDependencies": {
|
|
48
|
+
"@types/node": "24.11.0",
|
|
49
|
+
"typescript": "5.9.3"
|
|
50
|
+
},
|
|
51
|
+
"license": "Apache-2.0",
|
|
52
|
+
"repository": {
|
|
53
|
+
"type": "git",
|
|
54
|
+
"url": "git+https://github.com/tpsdev-ai/flair.git",
|
|
55
|
+
"directory": "packages/flair-bench"
|
|
56
|
+
},
|
|
57
|
+
"homepage": "https://tps.dev/#flair",
|
|
58
|
+
"keywords": [
|
|
59
|
+
"flair",
|
|
60
|
+
"embeddings",
|
|
61
|
+
"benchmark",
|
|
62
|
+
"recall",
|
|
63
|
+
"gguf",
|
|
64
|
+
"llama.cpp",
|
|
65
|
+
"ai",
|
|
66
|
+
"semantic-search"
|
|
67
|
+
]
|
|
68
|
+
}
|