@tpsdev-ai/flair-bench 0.22.0

package/dist/cosine.d.ts

@@ -0,0 +1,21 @@
+/**
+ * cosine.ts — exact cosine similarity + full-corpus ranking.
+ *
+ * Stands in for Harper's HNSW approximate-nearest-neighbor index (see
+ * README "Comparable to flair, with a caveat"): flair-bench has no HNSW
+ * graph, so instead of an approximate top-K it scores EVERY corpus vector
+ * against the query and sorts — the correct answer for a corpus this size
+ * (251 records), and the reason the tool can claim exact recall numbers
+ * rather than approximate ones.
+ */
+export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
+/**
+ * 0-based rank of `targetIndex` within `corpusVectors` when sorted by
+ * descending cosine similarity to `query`. Ties are broken by corpus
+ * insertion order (stable sort — Array.prototype.sort is guaranteed stable
+ * since ES2019/V8 since Node 11), the simplest documented tie-break
+ * available without reverse-engineering Harper's own HNSW/BM25-RRF
+ * tie-break internals. See README's "exact-vs-HNSW caveat" for what this
+ * does and doesn't guarantee to reproduce.
+ */
+export declare function rankOf(query: readonly number[], corpusVectors: readonly (readonly number[])[], targetIndex: number): number;

package/dist/cosine.js

@@ -0,0 +1,42 @@
+/**
+ * cosine.ts — exact cosine similarity + full-corpus ranking.
+ *
+ * Stands in for Harper's HNSW approximate-nearest-neighbor index (see
+ * README "Comparable to flair, with a caveat"): flair-bench has no HNSW
+ * graph, so instead of an approximate top-K it scores EVERY corpus vector
+ * against the query and sorts — the correct answer for a corpus this size
+ * (251 records), and the reason the tool can claim exact recall numbers
+ * rather than approximate ones.
+ */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        throw new Error(`cosineSimilarity: dimension mismatch (${a.length} vs ${b.length})`);
+    }
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        const x = a[i];
+        const y = b[i];
+        dot += x * y;
+        normA += x * x;
+        normB += y * y;
+    }
+    if (normA === 0 || normB === 0)
+        return 0;
+    return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+/**
+ * 0-based rank of `targetIndex` within `corpusVectors` when sorted by
+ * descending cosine similarity to `query`. Ties are broken by corpus
+ * insertion order (stable sort — Array.prototype.sort is guaranteed stable
+ * since ES2019/V8 since Node 11), the simplest documented tie-break
+ * available without reverse-engineering Harper's own HNSW/BM25-RRF
+ * tie-break internals. See README's "exact-vs-HNSW caveat" for what this
+ * does and doesn't guarantee to reproduce.
+ */
+export function rankOf(query, corpusVectors, targetIndex) {
+    const scored = corpusVectors.map((v, index) => ({ index, score: cosineSimilarity(query, v) }));
+    scored.sort((x, y) => y.score - x.score);
+    return scored.findIndex((s) => s.index === targetIndex);
+}

package/dist/engine.d.ts

@@ -0,0 +1,36 @@
+/**
+ * engine.ts — node-llama-cpp integration: model loading, identity, and
+ * corpus embedding.
+ *
+ * Uses node-llama-cpp DIRECTLY (the same engine dependency
+ * harper-fabric-embeddings/HFE wraps for Harper) rather than going through
+ * HFE — flair-bench has to run with no Flair/Harper installation at all, so
+ * it can't depend on the Harper-only wrapper package. See prefixes.ts for
+ * how the nomic search-prefix convention HFE applies is replicated here.
+ */
+import { type Llama, type LlamaModel } from "node-llama-cpp";
+import type { BenchProgressEvent, CorpusInfo, ModelBenchResult, ModelIdentity } from "./types.js";
+/** Shared Llama instance for the whole batch run — one backend/GPU detection, reused across every model load. */
+export declare function getSharedLlama(): Promise<Llama>;
+export declare function corpusInfo(): CorpusInfo;
+export declare function computeModelIdentity(model: LlamaModel, filePath: string): Promise<ModelIdentity>;
+/**
+ * Full single-model benchmark: load, embed the corpus, exact-cosine rank
+ * every query, score p@3/MRR overall and per-kind. Disposes the model
+ * before returning so a batch run's next model starts from a clean slate.
+ *
+ * Peak-RSS baseline is captured BEFORE `llama.loadModel()` runs — sampling
+ * only from inside the embedding phase (an earlier version of this
+ * function did that) would already have the model's own weights resident
+ * in the baseline, silently undercounting the number the spec actually
+ * wants ("how much memory does loading+using this model cost"). Verified
+ * against the Q4 baseline: an 80 MiB GGUF was reporting a ~24 MiB delta
+ * before this fix, because mmap'd model weight pages had already been
+ * counted into the baseline by the time sampling started.
+ */
+export declare function benchModel(llama: Llama, filePath: string, options?: {
+    warmupN?: number;
+    onProgress?: (e: BenchProgressEvent) => void;
+    index?: number;
+    total?: number;
+}): Promise<ModelBenchResult>;

package/dist/engine.js

@@ -0,0 +1,203 @@
+/**
+ * engine.ts — node-llama-cpp integration: model loading, identity, and
+ * corpus embedding.
+ *
+ * Uses node-llama-cpp DIRECTLY (the same engine dependency
+ * harper-fabric-embeddings/HFE wraps for Harper) rather than going through
+ * HFE — flair-bench has to run with no Flair/Harper installation at all, so
+ * it can't depend on the Harper-only wrapper package. See prefixes.ts for
+ * how the nomic search-prefix convention HFE applies is replicated here.
+ */
+import { createHash } from "node:crypto";
+import { createReadStream, statSync } from "node:fs";
+import { basename } from "node:path";
+import { getLlama, GgufFileType } from "node-llama-cpp";
+import { CORPUS, QUERIES } from "./corpus-v2.js";
+import { rankOf } from "./cosine.js";
+import { applyDocumentPrefix, applyQueryPrefix, resolvePrefixConvention } from "./prefixes.js";
+import { scoreRows } from "./scorer.js";
+const DEFAULT_WARMUP_N = 8;
+const MIN_TIMED_EMBEDS = 64;
+let _llama;
+/** Shared Llama instance for the whole batch run — one backend/GPU detection, reused across every model load. */
+export async function getSharedLlama() {
+    if (_llama)
+        return _llama;
+    _llama = await getLlama();
+    return _llama;
+}
+export function corpusInfo() {
+    return { version: "v2", records: CORPUS.length, queries: QUERIES.length };
+}
+async function sha256File(filePath) {
+    return new Promise((resolve, reject) => {
+        const hash = createHash("sha256");
+        const stream = createReadStream(filePath);
+        stream.on("data", (chunk) => hash.update(chunk));
+        stream.on("end", () => resolve(hash.digest("hex")));
+        stream.on("error", reject);
+    });
+}
+/**
+ * Total parameter count, approximated by summing every tensor's element
+ * count from the GGUF's own tensor index — the same method llama.cpp's CLI
+ * uses to report "params" in its load banner. Embedding-architecture GGUFs
+ * (BERT-family, unlike LLaMA-family) don't reliably carry a
+ * `general.parameter_count` metadata field, so this is computed rather than
+ * read.
+ */
+function approxParamCount(model) {
+    const tensors = model.fileInfo.tensorInfo ?? [];
+    let total = 0n;
+    for (const t of tensors) {
+        let elements = 1n;
+        for (const d of t.dimensions) {
+            elements *= typeof d === "bigint" ? d : BigInt(d);
+        }
+        total += elements;
+    }
+    return Number(total);
+}
+/** "MOSTLY_Q4_K_M" -> "Q4_K_M"; falls back to parsing the filename if GGUF metadata carries no file_type. */
+function resolveQuant(model, fileName) {
+    const fileType = model.fileInfo.metadata?.general?.file_type;
+    if (fileType !== undefined) {
+        const name = GgufFileType[fileType];
+        if (name)
+            return { quant: name.replace(/^MOSTLY_/, "").replace(/^ALL_/, ""), quantSource: "gguf-metadata" };
+    }
+    const match = fileName.match(/\.((?:Q|IQ)[0-9][A-Z0-9_]*|F16|F32|BF16)\.gguf$/i);
+    return { quant: match ? match[1].toUpperCase() : "unknown", quantSource: "filename-fallback" };
+}
+export async function computeModelIdentity(model, filePath) {
+    const fileName = basename(filePath);
+    const sizeBytes = statSync(filePath).size;
+    const sha256 = await sha256File(filePath);
+    const { quant, quantSource } = resolveQuant(model, fileName);
+    const dims = model.fileInsights.embeddingVectorSize ?? 0;
+    const paramsApprox = approxParamCount(model);
+    const bpw = paramsApprox > 0 ? (sizeBytes * 8) / paramsApprox : 0;
+    return { fileName, sha256, sizeBytes, quant, quantSource, dims, paramsApprox, bpw };
+}
+/**
+ * Embeds every CORPUS record (as "document") and every QUERIES entry (as
+ * "query") serially, applying the resolved prefix convention. The first
+ * `warmupN` embeds (drawn from CORPUS) run untimed before the stopwatch
+ * starts, so first-call allocation/JIT overhead doesn't skew the reported
+ * ms/embed — the timed set is CORPUS.length + QUERIES.length - warmupN,
+ * comfortably >= 64 for corpus-v2 (251 + 126 - 8 = 369).
+ */
+async function embedCorpus(model, fileName, warmupN, sample, onProgress) {
+    const convention = resolvePrefixConvention(fileName);
+    const embeddingContext = await model.createEmbeddingContext();
+    sample();
+    try {
+        const documentTexts = CORPUS.map((r) => applyDocumentPrefix(r.text, convention));
+        const queryTexts = QUERIES.map((q) => applyQueryPrefix(q.q, convention));
+        const warm = Math.min(Math.max(0, warmupN), documentTexts.length);
+        for (let i = 0; i < warm; i++) {
+            await embeddingContext.getEmbeddingFor(documentTexts[i]);
+            sample();
+        }
+        const documentVectors = new Array(documentTexts.length);
+        const queryVectors = new Array(queryTexts.length);
+        const totalToEmbed = documentTexts.length + queryTexts.length;
+        let done = 0;
+        let timedCount = 0;
+        let timedMs = 0;
+        for (let i = 0; i < documentTexts.length; i++) {
+            const timed = i >= warm;
+            const t0 = timed ? performance.now() : 0;
+            const emb = await embeddingContext.getEmbeddingFor(documentTexts[i]);
+            if (timed) {
+                timedMs += performance.now() - t0;
+                timedCount++;
+            }
+            documentVectors[i] = emb.vector;
+            sample();
+            done++;
+            onProgress?.({ type: "model-embedding", fileName, done, total: totalToEmbed });
+        }
+        for (let i = 0; i < queryTexts.length; i++) {
+            const t0 = performance.now();
+            const emb = await embeddingContext.getEmbeddingFor(queryTexts[i]);
+            timedMs += performance.now() - t0;
+            timedCount++;
+            queryVectors[i] = emb.vector;
+            sample();
+            done++;
+            onProgress?.({ type: "model-embedding", fileName, done, total: totalToEmbed });
+        }
+        const msPerEmbedSerialWarm = timedCount > 0 ? timedMs / timedCount : 0;
+        if (timedCount < MIN_TIMED_EMBEDS) {
+            // Should be unreachable with corpus-v2's fixed size, but fail loudly
+            // rather than silently report a latency figure the spec's own N>=64
+            // floor wasn't met by.
+            throw new Error(`embedCorpus: only ${timedCount} timed embeds (need >= ${MIN_TIMED_EMBEDS}) — corpus-v2 too small or warmupN too large`);
+        }
+        return { documentVectors, queryVectors, msPerEmbedSerialWarm };
+    }
+    finally {
+        await embeddingContext.dispose();
+    }
+}
+/**
+ * Full single-model benchmark: load, embed the corpus, exact-cosine rank
+ * every query, score p@3/MRR overall and per-kind. Disposes the model
+ * before returning so a batch run's next model starts from a clean slate.
+ *
+ * Peak-RSS baseline is captured BEFORE `llama.loadModel()` runs — sampling
+ * only from inside the embedding phase (an earlier version of this
+ * function did that) would already have the model's own weights resident
+ * in the baseline, silently undercounting the number the spec actually
+ * wants ("how much memory does loading+using this model cost"). Verified
+ * against the Q4 baseline: an 80 MiB GGUF was reporting a ~24 MiB delta
+ * before this fix, because mmap'd model weight pages had already been
+ * counted into the baseline by the time sampling started.
+ */
+export async function benchModel(llama, filePath, options = {}) {
+    const fileName = basename(filePath);
+    const { warmupN = DEFAULT_WARMUP_N, onProgress, index = 0, total = 1 } = options;
+    onProgress?.({ type: "model-start", fileName, index, total });
+    const baselineRss = process.memoryUsage().rss;
+    let peakRss = baselineRss;
+    const sample = () => {
+        const rss = process.memoryUsage().rss;
+        if (rss > peakRss)
+            peakRss = rss;
+    };
+    const loadStart = performance.now();
+    const model = await llama.loadModel({ modelPath: filePath });
+    const loadTimeMs = performance.now() - loadStart;
+    sample();
+    onProgress?.({ type: "model-loaded", fileName, loadTimeMs });
+    try {
+        const modelIdentity = await computeModelIdentity(model, filePath);
+        sample();
+        const { documentVectors, queryVectors, msPerEmbedSerialWarm } = await embedCorpus(model, fileName, warmupN, sample, onProgress);
+        const peakRssDeltaMiB = Math.max(0, (peakRss - baselineRss) / (1024 * 1024));
+        const markerToIndex = new Map(CORPUS.map((r, i) => [r.marker, i]));
+        const rows = QUERIES.map((q, qi) => {
+            const targetIndex = markerToIndex.get(q.expectMarker);
+            if (targetIndex === undefined) {
+                throw new Error(`benchModel: QUERIES[${qi}].expectMarker "${q.expectMarker}" has no matching CORPUS record`);
+            }
+            const rank = rankOf(queryVectors[qi], documentVectors, targetIndex);
+            return { rank, kind: q.kind };
+        });
+        const { aggregate, perKind } = scoreRows(rows);
+        const result = {
+            model: modelIdentity,
+            loadTimeMs,
+            msPerEmbedSerialWarm,
+            peakRssDeltaMiB,
+            aggregate,
+            perKind,
+        };
+        onProgress?.({ type: "model-done", fileName, result });
+        return result;
+    }
+    finally {
+        await model.dispose();
+    }
+}

package/dist/format.d.ts

@@ -0,0 +1,13 @@
+/**
+ * format.ts — rendering layer. Takes structured results (index.ts's return
+ * types) and produces strings; never calls process.exit or writes anywhere
+ * itself. cli.ts is the only place that actually console.logs these
+ * strings — keeping this module pure text-in/text-out means a library
+ * consumer (see index.ts's header) can call these directly or write its
+ * own renderer against the same BatchResult/RecommendResult shapes.
+ */
+import type { BatchResult, RecommendResult } from "./types.js";
+export declare function formatPretty(result: BatchResult): string;
+export declare function formatJson(result: BatchResult): string;
+export declare function formatRecommendPretty(result: RecommendResult): string;
+export declare function formatRecommendJson(result: RecommendResult): string;

package/dist/format.js

@@ -0,0 +1,69 @@
+/**
+ * format.ts — rendering layer. Takes structured results (index.ts's return
+ * types) and produces strings; never calls process.exit or writes anywhere
+ * itself. cli.ts is the only place that actually console.logs these
+ * strings — keeping this module pure text-in/text-out means a library
+ * consumer (see index.ts's header) can call these directly or write its
+ * own renderer against the same BatchResult/RecommendResult shapes.
+ */
+function fmt(n, d = 3) {
+    return n.toFixed(d);
+}
+function miB(n) {
+    return `${n.toFixed(1)} MiB`;
+}
+function modelLine(m) {
+    return (`${m.model.fileName}\n` +
+        `  quant=${m.model.quant} (${m.model.quantSource})  dims=${m.model.dims}  size=${(m.model.sizeBytes / (1024 * 1024)).toFixed(1)} MiB  ` +
+        `bpw=${fmt(m.model.bpw, 2)}  params~${(m.model.paramsApprox / 1e6).toFixed(1)}M\n` +
+        `  sha256=${m.model.sha256}\n` +
+        `  load=${fmt(m.loadTimeMs, 1)}ms  embed=${fmt(m.msPerEmbedSerialWarm, 2)}ms/ea (serial, warm)  peak-rss-delta=${miB(m.peakRssDeltaMiB)}\n` +
+        `  aggregate: p@3=${fmt(m.aggregate.p3)}  MRR=${fmt(m.aggregate.mrr)}  (n=${m.aggregate.n})\n` +
+        `  by kind:   ` +
+        ["stress", "trap", "hard", "clean"]
+            .map((k) => `${k}=${fmt(m.perKind[k].mrr)}(n=${m.perKind[k].n})`)
+            .join("  "));
+}
+export function formatPretty(result) {
+    const lines = [];
+    lines.push(`flair-bench ${result.toolVersion} — ${result.timestamp}`);
+    lines.push(`corpus: v2 (${result.corpus.records} records, ${result.corpus.queries} queries)`);
+    const label = result.host.label ? ` [${result.host.label}]` : "";
+    lines.push(`host${label}: ${result.host.platform}/${result.host.arch}  ${result.host.cpuModel}  ` +
+        `${result.host.totalRamGiB.toFixed(1)}GiB total / ${result.host.availableRamGiB.toFixed(1)}GiB available  ` +
+        `backend=${result.host.backend}${result.host.gpuDeviceNames.length ? ` gpu=${result.host.gpuDeviceNames.join(", ")}` : ""}`);
+    lines.push("");
+    for (const m of result.models) {
+        lines.push(modelLine(m));
+        lines.push("");
+    }
+    if (result.models.length > 1) {
+        lines.push("── ranked comparison (by MRR) ──");
+        const ranked = [...result.models].sort((a, b) => b.aggregate.mrr - a.aggregate.mrr);
+        for (const m of ranked) {
+            lines.push(`  ${m.model.fileName.padEnd(40)} p@3=${fmt(m.aggregate.p3)}  MRR=${fmt(m.aggregate.mrr)}  ` +
+                `${fmt(m.msPerEmbedSerialWarm, 1)}ms/embed  ${miB(m.peakRssDeltaMiB)}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+export function formatJson(result) {
+    return JSON.stringify(result, null, 2);
+}
+export function formatRecommendPretty(result) {
+    const lines = [formatPretty(result.batch), "── recommendation ──"];
+    if (result.recommendation) {
+        lines.push(result.recommendation.reason);
+    }
+    else {
+        lines.push("(no recommendation — see notes below)");
+    }
+    lines.push("");
+    for (const note of result.notes)
+        lines.push(`note: ${note}`);
+    return lines.join("\n");
+}
+export function formatRecommendJson(result) {
+    return JSON.stringify(result, null, 2);
+}

package/dist/hostinfo.d.ts

@@ -0,0 +1,14 @@
+/**
+ * hostinfo.ts — host fingerprinting.
+ *
+ * Compute backend + GPU device string come from node-llama-cpp's OWN report
+ * (`llama.gpu`, `llama.getGpuDeviceNames()`) — the engine that actually
+ * loaded the model, not inferred from `os.platform()`/`os.arch()`. Two
+ * hosts can share a platform/arch and still land on different backends
+ * (e.g. a Linux x64 box with no CUDA/Vulkan falls back to CPU) — this is
+ * what turns shared results into a real model × infra matrix rather than a
+ * model × OS-name one.
+ */
+import type { Llama } from "node-llama-cpp";
+import type { HostFingerprint } from "./types.js";
+export declare function fingerprintHost(llama: Llama, label?: string): Promise<HostFingerprint>;

package/dist/hostinfo.js

@@ -0,0 +1,40 @@
+/**
+ * hostinfo.ts — host fingerprinting.
+ *
+ * Compute backend + GPU device string come from node-llama-cpp's OWN report
+ * (`llama.gpu`, `llama.getGpuDeviceNames()`) — the engine that actually
+ * loaded the model, not inferred from `os.platform()`/`os.arch()`. Two
+ * hosts can share a platform/arch and still land on different backends
+ * (e.g. a Linux x64 box with no CUDA/Vulkan falls back to CPU) — this is
+ * what turns shared results into a real model × infra matrix rather than a
+ * model × OS-name one.
+ */
+import os from "node:os";
+const BYTES_PER_GIB = 1024 ** 3;
+export async function fingerprintHost(llama, label) {
+    const gpuType = llama.gpu; // "metal" | "cuda" | "vulkan" | false
+    const backend = gpuType === false ? "cpu" : gpuType;
+    let gpuDeviceNames = [];
+    if (backend !== "cpu") {
+        try {
+            gpuDeviceNames = await llama.getGpuDeviceNames();
+        }
+        catch {
+            // Some backends/drivers don't expose device enumeration — backend is
+            // still known and reported; device names degrade to empty, not a crash.
+            gpuDeviceNames = [];
+        }
+    }
+    const cpus = os.cpus();
+    const cpuModel = cpus[0]?.model?.trim() || "unknown";
+    return {
+        label,
+        platform: os.platform(),
+        arch: os.arch(),
+        cpuModel,
+        totalRamGiB: os.totalmem() / BYTES_PER_GIB,
+        availableRamGiB: os.freemem() / BYTES_PER_GIB,
+        backend,
+        gpuDeviceNames,
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,49 @@
+/**
+ * index.ts — flair-bench's public library API.
+ *
+ * CORE CONTRACT (so a future `flair bench` CLI subcommand — see README
+ * "Library use" — can consume this package directly): no `process.exit`,
+ * no `console.*` calls anywhere in this file or the modules it drives
+ * (engine.ts, hostinfo.ts, scorer.ts, cosine.ts, prefixes.ts, share.ts).
+ * Progress is reported via the optional `onProgress` callback
+ * (BenchOptions.onProgress); errors are thrown as real Error objects, never
+ * swallowed into an exit code. Rendering (pretty tables / JSON strings) and
+ * printing both live in cli.ts + format.ts — this file only computes and
+ * returns structured data.
+ */
+import type { BatchResult, BenchOptions, RecommendOptions, RecommendResult } from "./types.js";
+export * from "./types.js";
+export { buildShareDocument, writeShareDocument, SUBMISSION_ENDPOINT_PLACEHOLDER } from "./share.js";
+export type { WrittenShare } from "./share.js";
+export { TOOL_VERSION } from "./version.js";
+export { resolvePrefixConvention } from "./prefixes.js";
+export { pickRecommendation, DEFAULT_RAM_HEADROOM_FRACTION, DEFAULT_LATENCY_THRESHOLD_MS } from "./recommend-heuristic.js";
+export type { RecommendHeuristicOptions, RecommendHeuristicResult } from "./recommend-heuristic.js";
+/**
+ * Run the embedding benchmark for one or more GGUF files. Batch = more than
+ * one entry in `options.modelFiles`. Models are loaded and disposed one at
+ * a time (see engine.ts's benchModel) so peak-RSS measurement stays
+ * per-model rather than accumulating across the whole batch.
+ */
+export declare function runBenchmark(options: BenchOptions): Promise<BatchResult>;
+/**
+ * Runs the batch, then recommends the model with the best measured recall
+ * (MRR, tie-broken by faster ms/embed) among those that fit the host's RAM
+ * headroom and latency threshold. HONEST ABOUT WHAT IT DOESN'T KNOW (see
+ * README "Recommend heuristic, and its limits"):
+ *   - RSS is measured for THIS process's single-model-at-a-time load, not a
+ *     concurrent multi-request serving load — a real server's memory
+ *     footprint under concurrency will be higher.
+ *   - Latency is single-request serial ms/embed, not throughput under
+ *     concurrent load or with batching.
+ *   - The RAM/latency thresholds are simple fixed fractions/ceilings, not a
+ *     learned or host-class-specific model — see the two options below to
+ *     tune them.
+ * If nothing fits the budget, recommend() falls back to ranking the full
+ * set and says so explicitly in `notes` rather than silently returning null.
+ *
+ * The actual pick logic is pickRecommendation() (recommend-heuristic.ts) —
+ * factored out so it's unit-testable against fixture BatchResult objects
+ * without touching node-llama-cpp (see test/recommend.test.ts).
+ */
+export declare function recommend(options: RecommendOptions): Promise<RecommendResult>;

package/dist/index.js

@@ -0,0 +1,82 @@
+/**
+ * index.ts — flair-bench's public library API.
+ *
+ * CORE CONTRACT (so a future `flair bench` CLI subcommand — see README
+ * "Library use" — can consume this package directly): no `process.exit`,
+ * no `console.*` calls anywhere in this file or the modules it drives
+ * (engine.ts, hostinfo.ts, scorer.ts, cosine.ts, prefixes.ts, share.ts).
+ * Progress is reported via the optional `onProgress` callback
+ * (BenchOptions.onProgress); errors are thrown as real Error objects, never
+ * swallowed into an exit code. Rendering (pretty tables / JSON strings) and
+ * printing both live in cli.ts + format.ts — this file only computes and
+ * returns structured data.
+ */
+import { benchModel, corpusInfo, getSharedLlama } from "./engine.js";
+import { fingerprintHost } from "./hostinfo.js";
+import { TOOL_VERSION } from "./version.js";
+import { pickRecommendation } from "./recommend-heuristic.js";
+export * from "./types.js";
+export { buildShareDocument, writeShareDocument, SUBMISSION_ENDPOINT_PLACEHOLDER } from "./share.js";
+export { TOOL_VERSION } from "./version.js";
+export { resolvePrefixConvention } from "./prefixes.js";
+export { pickRecommendation, DEFAULT_RAM_HEADROOM_FRACTION, DEFAULT_LATENCY_THRESHOLD_MS } from "./recommend-heuristic.js";
+/**
+ * Run the embedding benchmark for one or more GGUF files. Batch = more than
+ * one entry in `options.modelFiles`. Models are loaded and disposed one at
+ * a time (see engine.ts's benchModel) so peak-RSS measurement stays
+ * per-model rather than accumulating across the whole batch.
+ */
+export async function runBenchmark(options) {
+    if (!options.modelFiles || options.modelFiles.length === 0) {
+        throw new Error("runBenchmark: options.modelFiles must contain at least one GGUF file path");
+    }
+    const llama = await getSharedLlama();
+    const host = await fingerprintHost(llama, options.label);
+    options.onProgress?.({ type: "host-fingerprinted", host });
+    const models = [];
+    for (let i = 0; i < options.modelFiles.length; i++) {
+        const filePath = options.modelFiles[i];
+        const result = await benchModel(llama, filePath, {
+            warmupN: options.warmupN,
+            onProgress: options.onProgress,
+            index: i,
+            total: options.modelFiles.length,
+        });
+        models.push(result);
+    }
+    return {
+        toolVersion: TOOL_VERSION,
+        timestamp: new Date().toISOString(),
+        corpus: corpusInfo(),
+        host,
+        models,
+    };
+}
+/**
+ * Runs the batch, then recommends the model with the best measured recall
+ * (MRR, tie-broken by faster ms/embed) among those that fit the host's RAM
+ * headroom and latency threshold. HONEST ABOUT WHAT IT DOESN'T KNOW (see
+ * README "Recommend heuristic, and its limits"):
+ *   - RSS is measured for THIS process's single-model-at-a-time load, not a
+ *     concurrent multi-request serving load — a real server's memory
+ *     footprint under concurrency will be higher.
+ *   - Latency is single-request serial ms/embed, not throughput under
+ *     concurrent load or with batching.
+ *   - The RAM/latency thresholds are simple fixed fractions/ceilings, not a
+ *     learned or host-class-specific model — see the two options below to
+ *     tune them.
+ * If nothing fits the budget, recommend() falls back to ranking the full
+ * set and says so explicitly in `notes` rather than silently returning null.
+ *
+ * The actual pick logic is pickRecommendation() (recommend-heuristic.ts) —
+ * factored out so it's unit-testable against fixture BatchResult objects
+ * without touching node-llama-cpp (see test/recommend.test.ts).
+ */
+export async function recommend(options) {
+    const batch = await runBenchmark(options);
+    const { recommendation, notes } = pickRecommendation(batch, {
+        ramHeadroomFraction: options.ramHeadroomFraction,
+        latencyThresholdMsPerEmbed: options.latencyThresholdMsPerEmbed,
+    });
+    return { batch, recommendation, notes };
+}

package/dist/prefixes.d.ts

@@ -0,0 +1,37 @@
+/**
+ * prefixes.ts — the input-prefix convention table.
+ *
+ * Mirrors resources/embeddings-provider.ts's Phase 2 behavior (flair#504):
+ * HFE 0.3.0's engine prepends a literal `"search_document: "` /
+ * `"search_query: "` string ahead of the text it hands to the model, for
+ * nomic-family models trained on that asymmetric-prefix convention. flair-
+ * bench talks to node-llama-cpp directly (no HFE in the loop — see the
+ * package README for why), so it has to replicate that string-prepend
+ * itself rather than pass an `inputType` option through to a wrapper that
+ * does it.
+ *
+ * KEYED ON MODEL FILENAME, for now — nomic is the only convention flair
+ * ships today, and there's exactly one production model family to match.
+ * This table is the seam for adding more: a future entry keys on another
+ * filename pattern (or, if a GGUF ever carries a documented convention-id
+ * in its own metadata, that becomes the preferred match — filename is a
+ * pragmatic fallback, not a design commitment). Nomic upstream has a request
+ * to carry this in the GGUF/HFE registration surface itself rather than
+ * every consumer re-deriving it from a filename convention — see
+ * harper-fabric-embeddings#4 (upstream proposal, not yet merged as of this
+ * writing).
+ *
+ * A model whose filename matches no entry gets `undefined` from
+ * `resolvePrefixConvention()` — texts are embedded unprefixed, and the
+ * result is labeled accordingly rather than silently guessing.
+ */
+export interface PrefixConvention {
+    /** Human-readable id, surfaced in output so a reader knows which convention (if any) applied. */
+    id: string;
+    documentPrefix: string;
+    queryPrefix: string;
+}
+/** Resolve the prefix convention for a GGUF file, by basename. `undefined` = no known convention (embed unprefixed). */
+export declare function resolvePrefixConvention(fileName: string): PrefixConvention | undefined;
+export declare function applyDocumentPrefix(text: string, convention: PrefixConvention | undefined): string;
+export declare function applyQueryPrefix(text: string, convention: PrefixConvention | undefined): string;