@clutor/core 0.1.1 → 0.3.0

package/dist/index.d.cts

@@ -36,6 +36,8 @@ interface ArchitectureSpec {
     sharedExperts: number;
     expertFfn: number;
     maxContext: number;
+    /** The output projection is the embedding matrix itself (small Qwen2.5 models), so the vocabulary matrix is counted once. */
+    tiedEmbeddings?: boolean;
     /** The headline parameter count the vendor publishes, to check the computation against. */
     publishedParamsB: number;
     source: string;

package/dist/index.d.ts

@@ -36,6 +36,8 @@ interface ArchitectureSpec {
     sharedExperts: number;
     expertFfn: number;
     maxContext: number;
+    /** The output projection is the embedding matrix itself (small Qwen2.5 models), so the vocabulary matrix is counted once. */
+    tiedEmbeddings?: boolean;
     /** The headline parameter count the vendor publishes, to check the computation against. */
     publishedParamsB: number;
     source: string;

package/dist/index.js

@@ -238,6 +238,24 @@ var CATALOG = [
     publishedParamsB: 6.9,
     source: "OLMoE paper (Muennighoff et al., 2024)"
   },
+  {
+    id: "granite-3.0-3b-a800m",
+    name: "Granite 3.0 3B-A800M (IBM)",
+    family: "granite-moe",
+    layers: 32,
+    denseLayers: 0,
+    hidden: 1536,
+    vocab: 49155,
+    denseFfn: 0,
+    attention: { kind: "gqa", heads: 24, kvHeads: 8, headDim: 64 },
+    experts: 40,
+    expertsUsed: 8,
+    sharedExperts: 0,
+    expertFfn: 512,
+    maxContext: 4096,
+    publishedParamsB: 3.4,
+    source: "IBM Granite 3.0 model card; shapes read from the GGUF header of granite-3.0-3b-a800m-instruct"
+  },
   {
     id: "qwen2.5-coder-7b",
     name: "Qwen2.5-Coder-7B (dense)",
@@ -255,6 +273,25 @@ var CATALOG = [
     maxContext: 131072,
     publishedParamsB: 7.6,
     source: "Qwen2.5-Coder model card"
+  },
+  {
+    id: "qwen2.5-1.5b",
+    name: "Qwen2.5-1.5B-Instruct (dense)",
+    family: "qwen2",
+    layers: 28,
+    denseLayers: 28,
+    hidden: 1536,
+    vocab: 151936,
+    denseFfn: 8960,
+    attention: { kind: "gqa", heads: 12, kvHeads: 2, headDim: 128 },
+    experts: 0,
+    expertsUsed: 0,
+    sharedExperts: 0,
+    expertFfn: 0,
+    maxContext: 32768,
+    tiedEmbeddings: true,
+    publishedParamsB: 1.54,
+    source: "Qwen2.5 model card; shapes read from the GGUF header of qwen2.5-1.5b-instruct"
   }
 ];
 function layerParams(s, moe) {
@@ -271,7 +308,7 @@ function layerParams(s, moe) {
   };
 }
 function totalParams(s) {
-  let n = 2 * s.vocab * s.hidden;
+  let n = (s.tiedEmbeddings ? 1 : 2) * s.vocab * s.hidden;
   for (let l = 0; l < s.layers; l++) {
     const p = layerParams(s, s.experts > 0 && l >= s.denseLayers);
     n += p.attention + p.denseFfn + p.routed + p.shared + p.router;
@@ -284,7 +321,7 @@ function approximateLayout(s, bitsPerWeight2) {
   let offset = 0;
   const layers = [];
   let total = 0;
-  const resident = bytes(2 * s.vocab * s.hidden);
+  const resident = bytes((s.tiedEmbeddings ? 1 : 2) * s.vocab * s.hidden);
   for (let l = 0; l < s.layers; l++) {
     const moe = s.experts > 0 && l >= s.denseLayers;
     const p = layerParams(s, moe);