@elizaos/shared 2.0.0-beta.1 → 2.0.3-beta.3

package/local-inference-gpu/gpu-profile-schema.d.ts

@@ -0,0 +1,208 @@
+/**
+ * Zod schema for the per-GPU YAML profile files in
+ * `packages/shared/src/local-inference-gpu/profiles/*.yaml`.
+ *
+ * The YAML files are the source of truth for *per-bundle* deployment
+ * recommendations on a given card (n_gpu_layers, ctx_size, parallel,
+ * batch sizing, KV cache types, expected TPS). The card-level
+ * descriptive metadata (vram, bandwidth, FP8 support) is duplicated
+ * here so a single YAML round-trip captures everything an operator
+ * needs to deploy and verify.
+ *
+ * The runtime card-level descriptor in
+ * `packages/shared/src/local-inference/gpu-profiles.ts` is the *other*
+ * source of truth — it carries the same constants but is tree-shakable
+ * and import-safe in environments without YAML / file IO (mobile
+ * runtime, edge bundles). At load time we cross-validate the two so
+ * they cannot drift silently.
+ */
+import { z } from "zod";
+import { type Eliza1TierId } from "../local-inference/catalog.js";
+/**
+ * Card ids — must match `GpuProfileId` in
+ * `packages/shared/src/local-inference/gpu-profiles.ts`. Duplicated here
+ * to keep this schema module standalone (no runtime import cycle).
+ */
+export declare const GpuYamlId: z.ZodEnum<{
+    "rtx-3090": "rtx-3090";
+    "rtx-4090": "rtx-4090";
+    "rtx-5090": "rtx-5090";
+    h200: "h200";
+}>;
+export type GpuYamlId = z.infer<typeof GpuYamlId>;
+/** KV cache type strings accepted by `llama-server --cache-type-k/-v`. */
+export declare const KvCacheType: z.ZodEnum<{
+    q8_0: "q8_0";
+    f16: "f16";
+    q4_0: "q4_0";
+    qjl1_256: "qjl1_256";
+    q4_polar: "q4_polar";
+    turbo3_0: "turbo3_0";
+    turbo4_0: "turbo4_0";
+}>;
+export type KvCacheType = z.infer<typeof KvCacheType>;
+export declare const KernelName: z.ZodEnum<{
+    mtp: "mtp";
+    turbo3: "turbo3";
+    turbo4: "turbo4";
+    turbo3_tcq: "turbo3_tcq";
+    qjl_full: "qjl_full";
+    polarquant: "polarquant";
+}>;
+export type KernelName = z.infer<typeof KernelName>;
+/** Per-bundle llama-server flag bundle. */
+export declare const BundleRecommendation: z.ZodObject<{
+    n_gpu_layers: z.ZodNumber;
+    ctx_size: z.ZodNumber;
+    parallel: z.ZodNumber;
+    batch_size: z.ZodNumber;
+    ubatch_size: z.ZodNumber;
+    kv_cache_k: z.ZodEnum<{
+        q8_0: "q8_0";
+        f16: "f16";
+        q4_0: "q4_0";
+        qjl1_256: "qjl1_256";
+        q4_polar: "q4_polar";
+        turbo3_0: "turbo3_0";
+        turbo4_0: "turbo4_0";
+    }>;
+    kv_cache_v: z.ZodEnum<{
+        q8_0: "q8_0";
+        f16: "f16";
+        q4_0: "q4_0";
+        qjl1_256: "qjl1_256";
+        q4_polar: "q4_polar";
+        turbo3_0: "turbo3_0";
+        turbo4_0: "turbo4_0";
+    }>;
+    flash_attention: z.ZodBoolean;
+    mlock: z.ZodOptional<z.ZodBoolean>;
+    estimated_decode_tps: z.ZodNumber;
+    estimated_prefill_tps: z.ZodNumber;
+    notes: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type BundleRecommendation = z.infer<typeof BundleRecommendation>;
+export declare const MtpTuning: z.ZodObject<{
+    enabled: z.ZodBoolean;
+    draft_min: z.ZodNumber;
+    draft_max: z.ZodNumber;
+    draft_gpu_layers: z.ZodNumber;
+}, z.core.$strip>;
+export type MtpTuning = z.infer<typeof MtpTuning>;
+export declare const VerifyRecipe: z.ZodObject<{
+    build_target: z.ZodString;
+    cuda_arch: z.ZodNumber;
+    cmake_flags: z.ZodArray<z.ZodString>;
+    expected_kernels: z.ZodArray<z.ZodEnum<{
+        mtp: "mtp";
+        turbo3: "turbo3";
+        turbo4: "turbo4";
+        turbo3_tcq: "turbo3_tcq";
+        qjl_full: "qjl_full";
+        polarquant: "polarquant";
+    }>>;
+    unavailable_kernels: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+        mtp: "mtp";
+        turbo3: "turbo3";
+        turbo4: "turbo4";
+        turbo3_tcq: "turbo3_tcq";
+        qjl_full: "qjl_full";
+        polarquant: "polarquant";
+    }>>>;
+    warn_on_kernel_absent: z.ZodOptional<z.ZodBoolean>;
+    smoke_bundle: z.ZodString;
+    tolerance_pct: z.ZodNumber;
+}, z.core.$strip>;
+export type VerifyRecipe = z.infer<typeof VerifyRecipe>;
+/** Full per-GPU YAML profile. */
+export declare const GpuYamlProfile: z.ZodObject<{
+    gpu_id: z.ZodEnum<{
+        "rtx-3090": "rtx-3090";
+        "rtx-4090": "rtx-4090";
+        "rtx-5090": "rtx-5090";
+        h200: "h200";
+    }>;
+    gpu_arch: z.ZodString;
+    vram_gb: z.ZodNumber;
+    mem_bandwidth_gbps: z.ZodNumber;
+    fp8_supported: z.ZodBoolean;
+    fp4_supported: z.ZodBoolean;
+    nvlink: z.ZodBoolean;
+    bundle_recommendations: z.ZodRecord<z.ZodString, z.ZodObject<{
+        n_gpu_layers: z.ZodNumber;
+        ctx_size: z.ZodNumber;
+        parallel: z.ZodNumber;
+        batch_size: z.ZodNumber;
+        ubatch_size: z.ZodNumber;
+        kv_cache_k: z.ZodEnum<{
+            q8_0: "q8_0";
+            f16: "f16";
+            q4_0: "q4_0";
+            qjl1_256: "qjl1_256";
+            q4_polar: "q4_polar";
+            turbo3_0: "turbo3_0";
+            turbo4_0: "turbo4_0";
+        }>;
+        kv_cache_v: z.ZodEnum<{
+            q8_0: "q8_0";
+            f16: "f16";
+            q4_0: "q4_0";
+            qjl1_256: "qjl1_256";
+            q4_polar: "q4_polar";
+            turbo3_0: "turbo3_0";
+            turbo4_0: "turbo4_0";
+        }>;
+        flash_attention: z.ZodBoolean;
+        mlock: z.ZodOptional<z.ZodBoolean>;
+        estimated_decode_tps: z.ZodNumber;
+        estimated_prefill_tps: z.ZodNumber;
+        notes: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    mtp: z.ZodObject<{
+        enabled: z.ZodBoolean;
+        draft_min: z.ZodNumber;
+        draft_max: z.ZodNumber;
+        draft_gpu_layers: z.ZodNumber;
+    }, z.core.$strip>;
+    verify_recipe: z.ZodObject<{
+        build_target: z.ZodString;
+        cuda_arch: z.ZodNumber;
+        cmake_flags: z.ZodArray<z.ZodString>;
+        expected_kernels: z.ZodArray<z.ZodEnum<{
+            mtp: "mtp";
+            turbo3: "turbo3";
+            turbo4: "turbo4";
+            turbo3_tcq: "turbo3_tcq";
+            qjl_full: "qjl_full";
+            polarquant: "polarquant";
+        }>>;
+        unavailable_kernels: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+            mtp: "mtp";
+            turbo3: "turbo3";
+            turbo4: "turbo4";
+            turbo3_tcq: "turbo3_tcq";
+            qjl_full: "qjl_full";
+            polarquant: "polarquant";
+        }>>>;
+        warn_on_kernel_absent: z.ZodOptional<z.ZodBoolean>;
+        smoke_bundle: z.ZodString;
+        tolerance_pct: z.ZodNumber;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export type GpuYamlProfile = z.infer<typeof GpuYamlProfile>;
+/**
+ * Validate that every bundle id in `bundle_recommendations` is a real
+ * Eliza-1 tier id. Returns the list of offending keys (empty when OK).
+ *
+ * Kept out of the zod schema itself because `record(z.string(), …)` is
+ * the right shape for forward-compat — the YAML may reference a new
+ * tier id before the catalog enum is bumped, and we want a clear error
+ * rather than a cryptic union-mismatch.
+ */
+export declare function bundleIdsInProfileMatchCatalog(profile: GpuYamlProfile): {
+    ok: boolean;
+    unknown: string[];
+};
+/** Narrowed mapping of tier id -> recommendation, after catalog validation. */
+export declare function getRecommendationsByTier(profile: GpuYamlProfile): Partial<Record<Eliza1TierId, BundleRecommendation>>;
+//# sourceMappingURL=gpu-profile-schema.d.ts.map

package/local-inference-gpu/gpu-profile-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpu-profile-schema.d.ts","sourceRoot":"","sources":["../../src/local-inference-gpu/gpu-profile-schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,eAAO,MAAM,SAAS;;;;;EAAuD,CAAC;AAC9E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAC;AAElD,0EAA0E;AAC1E,eAAO,MAAM,WAAW;;;;;;;;EAQtB,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEtD,eAAO,MAAM,UAAU;;;;;;;EAOrB,CAAC;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC;AAEpD,2CAA2C;AAC3C,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAiB/B,CAAC;AACH,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAExE,eAAO,MAAM,SAAS;;;;;iBAKpB,CAAC;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAC;AAElD,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;iBAWvB,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAExD,iCAAiC;AACjC,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAWzB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE5D;;;;;;;;GAQG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,cAAc,GAAG;IACvE,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAOA;AAED,+EAA+E;AAC/E,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC,CAOrD"}

package/local-inference-gpu/gpu-profile-schema.js

@@ -0,0 +1,123 @@
+/**
+ * Zod schema for the per-GPU YAML profile files in
+ * `packages/shared/src/local-inference-gpu/profiles/*.yaml`.
+ *
+ * The YAML files are the source of truth for *per-bundle* deployment
+ * recommendations on a given card (n_gpu_layers, ctx_size, parallel,
+ * batch sizing, KV cache types, expected TPS). The card-level
+ * descriptive metadata (vram, bandwidth, FP8 support) is duplicated
+ * here so a single YAML round-trip captures everything an operator
+ * needs to deploy and verify.
+ *
+ * The runtime card-level descriptor in
+ * `packages/shared/src/local-inference/gpu-profiles.ts` is the *other*
+ * source of truth — it carries the same constants but is tree-shakable
+ * and import-safe in environments without YAML / file IO (mobile
+ * runtime, edge bundles). At load time we cross-validate the two so
+ * they cannot drift silently.
+ */
+import { z } from "zod";
+import { ELIZA_1_TIER_IDS, } from "../local-inference/catalog.js";
+/**
+ * Card ids — must match `GpuProfileId` in
+ * `packages/shared/src/local-inference/gpu-profiles.ts`. Duplicated here
+ * to keep this schema module standalone (no runtime import cycle).
+ */
+export const GpuYamlId = z.enum(["rtx-3090", "rtx-4090", "rtx-5090", "h200"]);
+/** KV cache type strings accepted by `llama-server --cache-type-k/-v`. */
+export const KvCacheType = z.enum([
+    "f16",
+    "q8_0",
+    "q4_0",
+    "qjl1_256",
+    "q4_polar",
+    "turbo3_0",
+    "turbo4_0",
+]);
+export const KernelName = z.enum([
+    "mtp",
+    "turbo3",
+    "turbo4",
+    "turbo3_tcq",
+    "qjl_full",
+    "polarquant",
+]);
+/** Per-bundle llama-server flag bundle. */
+export const BundleRecommendation = z.object({
+    n_gpu_layers: z.number().int(),
+    ctx_size: z.number().int().positive(),
+    parallel: z.number().int().positive(),
+    batch_size: z.number().int().positive(),
+    ubatch_size: z.number().int().positive(),
+    kv_cache_k: KvCacheType,
+    kv_cache_v: KvCacheType,
+    flash_attention: z.boolean(),
+    // Some profiles set `mlock`; others omit. Default = false at load time.
+    mlock: z.boolean().optional(),
+    // Throughput expectations — used by verify scripts' tolerance check.
+    // `estimated_decode_tps` is the steady-state generation rate; the
+    // prefill number is total tokens/sec across the batch on a long input.
+    estimated_decode_tps: z.number().positive(),
+    estimated_prefill_tps: z.number().positive(),
+    notes: z.string().optional(),
+});
+export const MtpTuning = z.object({
+    enabled: z.boolean(),
+    draft_min: z.number().int().positive(),
+    draft_max: z.number().int().positive(),
+    draft_gpu_layers: z.number().int(),
+});
+export const VerifyRecipe = z.object({
+    build_target: z.string(),
+    cuda_arch: z.number().int().positive(),
+    cmake_flags: z.array(z.string()).min(1),
+    expected_kernels: z.array(KernelName).min(1),
+    unavailable_kernels: z.array(KernelName).default([]),
+    // Optional flags some profiles set (e.g. 5090 warns instead of fails
+    // when a kernel is absent — Blackwell ports are still maturing).
+    warn_on_kernel_absent: z.boolean().optional(),
+    smoke_bundle: z.string(),
+    tolerance_pct: z.number().positive(),
+});
+/** Full per-GPU YAML profile. */
+export const GpuYamlProfile = z.object({
+    gpu_id: GpuYamlId,
+    gpu_arch: z.string().regex(/^sm_\d{2,3}$/, "expected sm_XX"),
+    vram_gb: z.number().positive(),
+    mem_bandwidth_gbps: z.number().positive(),
+    fp8_supported: z.boolean(),
+    fp4_supported: z.boolean(),
+    nvlink: z.boolean(),
+    bundle_recommendations: z.record(z.string(), BundleRecommendation),
+    mtp: MtpTuning,
+    verify_recipe: VerifyRecipe,
+});
+/**
+ * Validate that every bundle id in `bundle_recommendations` is a real
+ * Eliza-1 tier id. Returns the list of offending keys (empty when OK).
+ *
+ * Kept out of the zod schema itself because `record(z.string(), …)` is
+ * the right shape for forward-compat — the YAML may reference a new
+ * tier id before the catalog enum is bumped, and we want a clear error
+ * rather than a cryptic union-mismatch.
+ */
+export function bundleIdsInProfileMatchCatalog(profile) {
+    const known = new Set(ELIZA_1_TIER_IDS);
+    const unknown = [];
+    for (const id of Object.keys(profile.bundle_recommendations)) {
+        if (!known.has(id))
+            unknown.push(id);
+    }
+    return { ok: unknown.length === 0, unknown };
+}
+/** Narrowed mapping of tier id -> recommendation, after catalog validation. */
+export function getRecommendationsByTier(profile) {
+    const out = {};
+    const known = new Set(ELIZA_1_TIER_IDS);
+    for (const [k, v] of Object.entries(profile.bundle_recommendations)) {
+        if (known.has(k))
+            out[k] = v;
+    }
+    return out;
+}
+//# sourceMappingURL=gpu-profile-schema.js.map

package/local-inference-gpu/gpu-profile-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpu-profile-schema.js","sourceRoot":"","sources":["../../src/local-inference-gpu/gpu-profile-schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,gBAAgB,GAEjB,MAAM,+BAA+B,CAAC;AAEvC;;;;GAIG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;AAG9E,0EAA0E;AAC1E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC;IAChC,KAAK;IACL,MAAM;IACN,MAAM;IACN,UAAU;IACV,UAAU;IACV,UAAU;IACV,UAAU;CACX,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC;IAC/B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;CACb,CAAC,CAAC;AAGH,2CAA2C;AAC3C,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACrC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,WAAW;IACvB,UAAU,EAAE,WAAW;IACvB,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE;IAC5B,wEAAwE;IACxE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC7B,qEAAqE;IACrE,kEAAkE;IAClE,uEAAuE;IACvE,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3C,qBAAqB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;CACnC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACpD,qEAAqE;IACrE,iEAAiE;IACjE,qBAAqB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC7C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACrC,CAAC,CAAC;AAGH,iCAAiC;AACjC,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,MAAM,EAAE,SAAS;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAC5D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE;IAC1B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE;IAC1B,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,sBAAsB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC;IAClE,GAAG,EAAE,SAAS;IACd,aAAa,EAAE,YAAY;CAC5B,CAAC,CAAC;AAGH;;;;;;;;GAQG;AACH,MAAM,UAAU,8BAA8B,CAAC,OAAuB;IAIpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAC;IAChD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC7D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,wBAAwB,CACtC,OAAuB;IAEvB,MAAM,GAAG,GAAwD,EAAE,CAAC;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAC;IAChD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,EAAE,CAAC;QACpE,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,GAAG,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/local-inference-gpu/gpu-tier-detect.d.ts

@@ -0,0 +1,61 @@
+/**
+ * GPU detection utility for the simplified tier-profile system.
+ *
+ * Uses `nvidia-smi` to query the first GPU's name, total VRAM (MiB), and
+ * CUDA compute capability. On success the result feeds `selectBestProfile`
+ * to pick the best matching tier profile.
+ *
+ * **This module MUST NOT throw.** Any failure — missing binary, no GPU,
+ * parse error — returns `null`. Callers treat `null` as "run without GPU
+ * acceleration" and fall back to CPU/catalog defaults.
+ *
+ * **No model loading.** This file only spawns `nvidia-smi` for metadata
+ * queries; it never starts llama-server or loads any GGUF files.
+ *
+ * Override: set `ELIZA_GPU_PROFILE=<id>` (e.g. `ELIZA_GPU_PROFILE=rtx-4090`)
+ * to bypass detection entirely. `autoSelectProfile()` returns the named
+ * profile directly without running nvidia-smi.
+ */
+import type { GpuProfile } from "./gpu-tier-profiles.js";
+/** Raw data extracted from nvidia-smi for a single GPU. */
+export interface DetectedGpu {
+    /** Marketing name as reported by nvidia-smi, e.g. `"NVIDIA GeForce RTX 4090"`. */
+    name: string;
+    /** Total VRAM in MiB (as reported by `memory.total`). */
+    vram_mb: number;
+    /**
+     * CUDA compute capability in dotted form, e.g. `"8.9"`.
+     * `null` when nvidia-smi does not report `compute_cap` (older drivers).
+     */
+    cuda_compute: string | null;
+}
+/**
+ * Query the first detected NVIDIA GPU via `nvidia-smi`.
+ *
+ * Runs:
+ * ```
+ * nvidia-smi --query-gpu=name,memory.total,compute_cap --format=csv,noheader,nounits
+ * ```
+ *
+ * Parses the first output line. Returns `null` on any failure:
+ *   - nvidia-smi not found in PATH
+ *   - non-zero exit code (no NVIDIA driver / no GPU)
+ *   - unexpected output format
+ *   - any other exception
+ *
+ * The 2-second timeout prevents hangs in headless / CI environments.
+ */
+export declare function detectNvidiaGpu(): DetectedGpu | null;
+/**
+ * Auto-select the best GPU tier profile for the host.
+ *
+ * Resolution order:
+ *   1. If `ELIZA_GPU_PROFILE` is set, return that profile (or `null` if
+ *      the id is unrecognised).
+ *   2. Otherwise, run `detectNvidiaGpu()` and call `selectBestProfile`.
+ *   3. Return `null` when no GPU is detected or no profile fits.
+ *
+ * Never throws.
+ */
+export declare function autoSelectProfile(): GpuProfile | null;
+//# sourceMappingURL=gpu-tier-detect.d.ts.map

package/local-inference-gpu/gpu-tier-detect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpu-tier-detect.d.ts","sourceRoot":"","sources":["../../src/local-inference-gpu/gpu-tier-detect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAGzD,2DAA2D;AAC3D,MAAM,WAAW,WAAW;IAC1B,kFAAkF;IAClF,IAAI,EAAE,MAAM,CAAC;IACb,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,IAAI,WAAW,GAAG,IAAI,CAqCpD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,IAAI,UAAU,GAAG,IAAI,CAYrD"}

package/local-inference-gpu/gpu-tier-detect.js

@@ -0,0 +1,93 @@
+/**
+ * GPU detection utility for the simplified tier-profile system.
+ *
+ * Uses `nvidia-smi` to query the first GPU's name, total VRAM (MiB), and
+ * CUDA compute capability. On success the result feeds `selectBestProfile`
+ * to pick the best matching tier profile.
+ *
+ * **This module MUST NOT throw.** Any failure — missing binary, no GPU,
+ * parse error — returns `null`. Callers treat `null` as "run without GPU
+ * acceleration" and fall back to CPU/catalog defaults.
+ *
+ * **No model loading.** This file only spawns `nvidia-smi` for metadata
+ * queries; it never starts llama-server or loads any GGUF files.
+ *
+ * Override: set `ELIZA_GPU_PROFILE=<id>` (e.g. `ELIZA_GPU_PROFILE=rtx-4090`)
+ * to bypass detection entirely. `autoSelectProfile()` returns the named
+ * profile directly without running nvidia-smi.
+ */
+import { execSync } from "node:child_process";
+import { getGpuProfile, selectBestProfile } from "./gpu-tier-profiles.js";
+/**
+ * Query the first detected NVIDIA GPU via `nvidia-smi`.
+ *
+ * Runs:
+ * ```
+ * nvidia-smi --query-gpu=name,memory.total,compute_cap --format=csv,noheader,nounits
+ * ```
+ *
+ * Parses the first output line. Returns `null` on any failure:
+ *   - nvidia-smi not found in PATH
+ *   - non-zero exit code (no NVIDIA driver / no GPU)
+ *   - unexpected output format
+ *   - any other exception
+ *
+ * The 2-second timeout prevents hangs in headless / CI environments.
+ */
+export function detectNvidiaGpu() {
+    try {
+        const raw = execSync("nvidia-smi --query-gpu=name,memory.total,compute_cap --format=csv,noheader,nounits", { encoding: "utf8", timeout: 2_000, stdio: ["ignore", "pipe", "pipe"] });
+        const firstLine = raw.split(/\r?\n/).find((l) => l.trim().length > 0);
+        if (!firstLine)
+            return null;
+        // Format: "<name>, <memory_total_mib>, <compute_cap>"
+        // The name may contain commas (rare but possible), so split from the
+        // right to keep the name intact.
+        const parts = firstLine.split(",");
+        if (parts.length < 3)
+            return null;
+        // Last two parts are the numeric fields; everything before is the name.
+        const computeRaw = parts[parts.length - 1]?.trim() ?? "";
+        const memRaw = parts[parts.length - 2]?.trim() ?? "";
+        const name = parts
+            .slice(0, parts.length - 2)
+            .join(",")
+            .trim();
+        if (!name)
+            return null;
+        const vram_mb = parseInt(memRaw, 10);
+        if (Number.isNaN(vram_mb) || vram_mb <= 0)
+            return null;
+        // compute_cap may be "N/A" on very old drivers.
+        const cuda_compute = computeRaw.length > 0 && computeRaw !== "N/A" ? computeRaw : null;
+        return { name, vram_mb, cuda_compute };
+    }
+    catch {
+        // nvidia-smi not found, timed out, or any other error.
+        return null;
+    }
+}
+/**
+ * Auto-select the best GPU tier profile for the host.
+ *
+ * Resolution order:
+ *   1. If `ELIZA_GPU_PROFILE` is set, return that profile (or `null` if
+ *      the id is unrecognised).
+ *   2. Otherwise, run `detectNvidiaGpu()` and call `selectBestProfile`.
+ *   3. Return `null` when no GPU is detected or no profile fits.
+ *
+ * Never throws.
+ */
+export function autoSelectProfile() {
+    const envOverride = process.env.ELIZA_GPU_PROFILE;
+    if (envOverride) {
+        return getGpuProfile(envOverride);
+    }
+    const gpu = detectNvidiaGpu();
+    if (!gpu)
+        return null;
+    const vramGb = gpu.vram_mb / 1024;
+    const cudaCompute = gpu.cuda_compute ?? "0.0";
+    return selectBestProfile(vramGb, cudaCompute);
+}
+//# sourceMappingURL=gpu-tier-detect.js.map

package/local-inference-gpu/gpu-tier-detect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpu-tier-detect.js","sourceRoot":"","sources":["../../src/local-inference-gpu/gpu-tier-detect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAe1E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,eAAe;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,oFAAoF,EACpF,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACxE,CAAC;QACF,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtE,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,sDAAsD;QACtD,qEAAqE;QACrE,iCAAiC;QACjC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAElC,wEAAwE;QACxE,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrD,MAAM,IAAI,GAAG,KAAK;aACf,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;aAC1B,IAAI,CAAC,GAAG,CAAC;aACT,IAAI,EAAE,CAAC;QAEV,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAEvD,gDAAgD;QAChD,MAAM,YAAY,GAChB,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAEpE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,uDAAuD;QACvD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB;IAC/B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAClD,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,aAAa,CAAC,WAAW,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,GAAG,GAAG,eAAe,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;IAClC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,IAAI,KAAK,CAAC;IAC9C,OAAO,iBAAiB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;AAChD,CAAC"}

package/local-inference-gpu/gpu-tier-profiles.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Simplified per-GPU configuration profiles for the Eliza-1 model family.
+ *
+ * These profiles describe hardware capabilities and llama-server flag
+ * recommendations for four supported NVIDIA cards. They are intentionally
+ * separate from the richer YAML-backed `GpuYamlProfile` system in
+ * `gpu-profile-loader.ts` — the JSON-style data here is bundled inline
+ * (no fs reads, no YAML parser) so it is safe in every environment:
+ * mobile runtimes, edge bundles, and server contexts.
+ *
+ * Selection: use `selectBestProfile(vramGb, cudaCompute)` to pick the
+ * most capable profile the detected GPU qualifies for. The result feeds
+ * `buildLlamaCppArgs()` to produce a ready-to-use argv array for
+ * llama-server. Caller must fill in `--model <path>` separately — this
+ * module never references model files.
+ *
+ * Override: set `ELIZA_GPU_PROFILE=<id>` (e.g. `rtx-4090`) to bypass
+ * auto-detection and force a specific profile. `autoSelectProfile()` in
+ * `detect.ts` respects this variable.
+ *
+ * Supported ids: `rtx-3090`, `rtx-4090`, `rtx-5090`, `h200`.
+ */
+/** GPU feature flags that affect quantisation and kernel availability. */
+export type GpuFeature = "fp16" | "bf16" | "int8" | "int4" | "fp8" | "fp4";
+/** Simplified GPU profile — JSON-safe, bundle-friendly. */
+export interface GpuProfile {
+    /** Canonical profile id, e.g. `"rtx-4090"`. */
+    id: string;
+    /** Human-readable card name shown to the user. */
+    display_name: string;
+    /** Total VRAM in GiB. */
+    vram_gb: number;
+    /**
+     * CUDA compute capability as a dotted version string, e.g. `"8.9"`.
+     * Used for numeric comparison in `selectBestProfile`.
+     */
+    cuda_compute: string;
+    /** Quantisation / precision features the GPU supports. */
+    features: GpuFeature[];
+    /** Recommended Eliza-1 tier ids for common workloads. */
+    recommended_tiers: {
+        primary: string | null;
+        secondary: string | null;
+        heavy: string | null;
+    };
+    /** llama-server flags for this card. */
+    llama_cpp_flags: {
+        n_gpu_layers: number;
+        tensor_split: number[] | null;
+        flash_attn: boolean;
+        use_mmap: boolean;
+        numa: boolean;
+    };
+    /** MTP speculative-decoding settings. */
+    mtp: {
+        enabled: boolean;
+        drafter_tier: string;
+        speculative_window: number;
+    };
+    /** Maximum recommended context window in tokens. */
+    ctx_size_tokens: number;
+    /** Free-text notes for operators / documentation. */
+    notes: string;
+}
+/**
+ * All built-in profiles keyed by id.
+ *
+ * Data is inlined — no fs reads at runtime — so the map is available in
+ * every environment (mobile, edge, server) without extra IO.
+ */
+export declare const GPU_PROFILES: Record<string, GpuProfile>;
+/**
+ * Look up a profile by its canonical id. Returns `null` when the id is
+ * not in the built-in registry (including when the caller passes an
+ * unrecognised override id from `ELIZA_GPU_PROFILE`).
+ */
+export declare function getGpuProfile(id: string): GpuProfile | null;
+/**
+ * Return the best-fit profile for a detected GPU.
+ *
+ * "Best fit" = the profile with the highest `vram_gb` that still
+ * satisfies both of:
+ *   1. `profile.vram_gb <= vramGb`   (card has at least as much VRAM)
+ *   2. `profile.cuda_compute <= cudaCompute`  (card meets compute level)
+ *
+ * Returns `null` when no profile fits (e.g. only 8 GB VRAM — below all
+ * supported cards).
+ */
+export declare function selectBestProfile(vramGb: number, cudaCompute: string): GpuProfile | null;
+/**
+ * Produce a `llama-server` argv array from a profile.
+ *
+ * Flags produced (in order):
+ *   --n-gpu-layers <N>
+ *   --flash-attn             (when flash_attn is true)
+ *   --no-mmap                (when use_mmap is false)
+ *   --numa                   (when numa is true)
+ *   --ctx-size <N>
+ *
+ * Any value in `overrides` replaces the corresponding profile field
+ * before the flags are built — useful for per-call adjustments without
+ * modifying the shared profile object.
+ *
+ * Note: `--model <path>` is intentionally omitted. Callers must append
+ * it themselves so this function never needs to touch model files.
+ */
+export declare function buildLlamaCppArgs(profile: GpuProfile, overrides?: Partial<GpuProfile["llama_cpp_flags"]> & {
+    ctx_size_tokens?: number;
+}): string[];
+//# sourceMappingURL=gpu-tier-profiles.d.ts.map

package/local-inference-gpu/gpu-tier-profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpu-tier-profiles.d.ts","sourceRoot":"","sources":["../../src/local-inference-gpu/gpu-tier-profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,0EAA0E;AAC1E,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,CAAC;AAE3E,2DAA2D;AAC3D,MAAM,WAAW,UAAU;IACzB,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC;IACrB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,yDAAyD;IACzD,iBAAiB,EAAE;QACjB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;KACtB,CAAC;IACF,wCAAwC;IACxC,eAAe,EAAE;QACf,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAC9B,UAAU,EAAE,OAAO,CAAC;QACpB,QAAQ,EAAE,OAAO,CAAC;QAClB,IAAI,EAAE,OAAO,CAAC;KACf,CAAC;IACF,yCAAyC;IACzC,GAAG,EAAE;QACH,OAAO,EAAE,OAAO,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC;IACxB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;CACf;AA0HD;;;;;GAKG;AACH,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAKnD,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAE3D;AAiBD;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAClB,UAAU,GAAG,IAAI,CAmBnB;AAMD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,UAAU,EACnB,SAAS,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,GAAG;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GACA,MAAM,EAAE,CAyBV"}