node-mlx 1.0.0

package/dist/cli.js

@@ -0,0 +1,639 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import * as readline from "readline";
+
+// src/index.ts
+import { platform, arch } from "os";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+import { existsSync, readFileSync } from "fs";
+import { createRequire } from "module";
+var __filename = fileURLToPath(import.meta.url);
+var __dirname = dirname(__filename);
+var require2 = createRequire(import.meta.url);
+var packageJsonPath = join(__dirname, "..", "package.json");
+var packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
+var VERSION = packageJson.version;
+var binding = null;
+var initialized = false;
+function loadNativeAddon() {
+  try {
+    const gypBuild = require2("node-gyp-build");
+    const nativeDir = join(__dirname, "..", "native");
+    if (existsSync(join(__dirname, "..", "prebuilds"))) {
+      return gypBuild(join(__dirname, ".."));
+    }
+    if (existsSync(join(nativeDir, "build"))) {
+      return gypBuild(nativeDir);
+    }
+  } catch {
+  }
+  const possibleAddonPaths = [
+    // From package dist/ (npm installed)
+    join(__dirname, "..", "prebuilds", "darwin-arm64", "node.napi.node"),
+    // From native/build (local development)
+    join(__dirname, "..", "native", "build", "Release", "node_mlx.node"),
+    // From project root (monorepo development)
+    join(process.cwd(), "packages", "node-mlx", "native", "build", "Release", "node_mlx.node")
+  ];
+  for (const p of possibleAddonPaths) {
+    if (existsSync(p)) {
+      return require2(p);
+    }
+  }
+  throw new Error(
+    `Native addon not found. Run 'pnpm build:native' first.
+Searched paths:
+${possibleAddonPaths.join("\n")}`
+  );
+}
+function findSwiftLibrary() {
+  const possibleDylibPaths = [
+    // From package swift/ (preferred - has metallib co-located)
+    join(__dirname, "..", "swift", "libNodeMLX.dylib"),
+    // From project root packages/node-mlx/swift/ (monorepo development)
+    join(process.cwd(), "packages", "node-mlx", "swift", "libNodeMLX.dylib"),
+    // Fallback to packages/swift/.build (monorepo dev)
+    join(__dirname, "..", "..", "swift", ".build", "release", "libNodeMLX.dylib"),
+    join(__dirname, "..", "..", "..", "swift", ".build", "release", "libNodeMLX.dylib"),
+    join(process.cwd(), "packages", "swift", ".build", "release", "libNodeMLX.dylib")
+  ];
+  for (const p of possibleDylibPaths) {
+    if (existsSync(p)) {
+      return p;
+    }
+  }
+  throw new Error(
+    `Swift library not found. Run 'pnpm build:swift' first.
+Searched paths:
+${possibleDylibPaths.join("\n")}`
+  );
+}
+function loadBinding() {
+  if (binding && initialized) {
+    return binding;
+  }
+  if (platform() !== "darwin" || arch() !== "arm64") {
+    throw new Error("node-mlx is only supported on macOS Apple Silicon (arm64)");
+  }
+  binding = loadNativeAddon();
+  const dylibPath = findSwiftLibrary();
+  const success = binding.initialize(dylibPath);
+  if (!success) {
+    throw new Error("Failed to initialize node-mlx native library");
+  }
+  initialized = true;
+  return binding;
+}
+var RECOMMENDED_MODELS = {
+  // Qwen 2.5 (Alibaba) - Working with proper RoPE support
+  // Using non-quantized models - quantized models have loading issues
+  qwen: "Qwen/Qwen2.5-1.5B-Instruct",
+  "qwen-2.5": "Qwen/Qwen2.5-1.5B-Instruct",
+  "qwen-2.5-0.5b": "Qwen/Qwen2.5-0.5B-Instruct",
+  "qwen-2.5-1.5b": "Qwen/Qwen2.5-1.5B-Instruct",
+  "qwen-2.5-3b": "Qwen/Qwen2.5-3B-Instruct",
+  // Phi (Microsoft) - Working with fused QKV and RoPE
+  phi: "microsoft/phi-4",
+  // Default to latest
+  phi4: "microsoft/phi-4",
+  "phi-4": "microsoft/phi-4",
+  phi3: "microsoft/Phi-3-mini-4k-instruct",
+  "phi-3": "microsoft/Phi-3-mini-4k-instruct",
+  "phi-3-mini": "microsoft/Phi-3-mini-4k-instruct",
+  // Llama 3.2 (Meta) - Requires HuggingFace authentication
+  // Note: meta-llama models require accepting license at huggingface.co
+  llama: "meta-llama/Llama-3.2-1B-Instruct",
+  "llama-3.2": "meta-llama/Llama-3.2-1B-Instruct",
+  "llama-3.2-1b": "meta-llama/Llama-3.2-1B-Instruct",
+  "llama-3.2-3b": "meta-llama/Llama-3.2-3B-Instruct",
+  // Gemma 3 (Google) - Standard transformer architecture with sliding window
+  gemma: "mlx-community/gemma-3-1b-it-4bit",
+  "gemma-3": "mlx-community/gemma-3-1b-it-4bit",
+  "gemma-3-1b": "mlx-community/gemma-3-1b-it-4bit",
+  "gemma-3-1b-bf16": "mlx-community/gemma-3-1b-it-bf16",
+  "gemma-3-4b": "mlx-community/gemma-3-4b-it-4bit",
+  "gemma-3-4b-bf16": "mlx-community/gemma-3-4b-it-bf16",
+  "gemma-3-12b": "mlx-community/gemma-3-12b-it-4bit",
+  "gemma-3-27b": "mlx-community/gemma-3-27b-it-4bit"
+};
+function isPlatformSupported() {
+  return platform() === "darwin" && arch() === "arm64";
+}
+function isSupported() {
+  if (!isPlatformSupported()) {
+    return false;
+  }
+  try {
+    const b = loadBinding();
+    return b.isAvailable();
+  } catch {
+    return false;
+  }
+}
+function loadModel(modelId) {
+  const b = loadBinding();
+  const handle = b.loadModel(modelId);
+  return {
+    handle,
+    generate(prompt, options) {
+      const jsonStr = b.generate(handle, prompt, {
+        maxTokens: options?.maxTokens ?? 256,
+        temperature: options?.temperature ?? 0.7,
+        topP: options?.topP ?? 0.9,
+        repetitionPenalty: options?.repetitionPenalty ?? 0,
+        repetitionContextSize: options?.repetitionContextSize ?? 20
+      });
+      const result = JSON.parse(jsonStr);
+      if (!result.success) {
+        throw new Error(result.error ?? "Generation failed");
+      }
+      return {
+        text: result.text ?? "",
+        tokenCount: result.tokenCount ?? 0,
+        tokensPerSecond: result.tokensPerSecond ?? 0
+      };
+    },
+    generateStreaming(prompt, options) {
+      const jsonStr = b.generateStreaming(handle, prompt, {
+        maxTokens: options?.maxTokens ?? 256,
+        temperature: options?.temperature ?? 0.7,
+        topP: options?.topP ?? 0.9,
+        repetitionPenalty: options?.repetitionPenalty ?? 0,
+        repetitionContextSize: options?.repetitionContextSize ?? 20
+      });
+      const result = JSON.parse(jsonStr);
+      if (!result.success) {
+        throw new Error(result.error ?? "Generation failed");
+      }
+      return {
+        tokenCount: result.tokenCount ?? 0,
+        tokensPerSecond: result.tokensPerSecond ?? 0
+      };
+    },
+    generateWithImage(prompt, imagePath, options) {
+      const jsonStr = b.generateWithImage(handle, prompt, imagePath, {
+        maxTokens: options?.maxTokens ?? 256,
+        temperature: options?.temperature ?? 0.7,
+        topP: options?.topP ?? 0.9,
+        repetitionPenalty: options?.repetitionPenalty ?? 0,
+        repetitionContextSize: options?.repetitionContextSize ?? 20
+      });
+      const result = JSON.parse(jsonStr);
+      if (!result.success) {
+        throw new Error(result.error ?? "Generation failed");
+      }
+      return {
+        tokenCount: result.tokenCount ?? 0,
+        tokensPerSecond: result.tokensPerSecond ?? 0
+      };
+    },
+    isVLM() {
+      return b.isVLM(handle);
+    },
+    unload() {
+      b.unloadModel(handle);
+    }
+  };
+}
+
+// src/cli.ts
+var colors = {
+  reset: "\x1B[0m",
+  bold: "\x1B[1m",
+  dim: "\x1B[2m",
+  cyan: "\x1B[36m",
+  green: "\x1B[32m",
+  yellow: "\x1B[33m",
+  magenta: "\x1B[35m",
+  red: "\x1B[31m"
+};
+function log(msg) {
+  console.log(msg);
+}
+function error(msg) {
+  console.error(`${colors.red}Error:${colors.reset} ${msg}`);
+}
+function printHeader() {
+  log("");
+  log(`${colors.bold}${colors.cyan}\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557${colors.reset}`);
+  log(
+    `${colors.bold}${colors.cyan}\u2551${colors.reset}  ${colors.bold}MLX CLI${colors.reset} - LLMs on Apple Silicon     ${colors.cyan}\u2551${colors.reset}`
+  );
+  log(`${colors.bold}${colors.cyan}\u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D${colors.reset}`);
+  log("");
+}
+function printHelp() {
+  log(`${colors.bold}Usage:${colors.reset}`);
+  log(`  mlx                              Interactive chat`);
+  log(`  mlx "prompt"                     One-shot generation`);
+  log(`  mlx --model <name>               Use specific model`);
+  log(`  mlx --image <path>               Include image (VLM only)`);
+  log(`  mlx --repetition-penalty <1-2>   Penalize repeated tokens (default: off)`);
+  log(`  mlx --list                       List available models`);
+  log(`  mlx --help                       Show this help`);
+  log("");
+  log(`${colors.bold}Vision models (VLM):${colors.reset}`);
+  log(`  mlx --model gemma-3-4b --image photo.jpg "What's in this image?"`);
+  log("");
+  log(`${colors.bold}Repetition penalty (for models that repeat):${colors.reset}`);
+  log(`  mlx --model gemma-3n --repetition-penalty 1.2 "Tell me about AI"`);
+  log("");
+  log(`${colors.bold}Interactive commands:${colors.reset}`);
+  log(`  /model <name>                    Switch model`);
+  log(`  /image <path>                    Set image for next prompt`);
+  log(`  /temp <0-2>                      Set temperature`);
+  log(`  /tokens <n>                      Set max tokens`);
+  log(`  /rep <1-2>                       Set repetition penalty`);
+  log(`  /clear                           Clear conversation`);
+  log(`  /help                            Show commands`);
+  log(`  /quit                            Exit`);
+  log("");
+}
+function printModels() {
+  log(`${colors.bold}Available models:${colors.reset}`);
+  log("");
+  const modelsByHfId = /* @__PURE__ */ new Map();
+  for (const [alias, hfId] of Object.entries(RECOMMENDED_MODELS)) {
+    if (!modelsByHfId.has(hfId)) {
+      modelsByHfId.set(hfId, []);
+    }
+    modelsByHfId.get(hfId)?.push(alias);
+  }
+  const families = [
+    {
+      name: "Phi (Microsoft)",
+      prefix: "Phi",
+      desc: "Reasoning & coding"
+    },
+    {
+      name: "Gemma (Google)",
+      prefix: "gemma",
+      desc: "Efficient on-device"
+    },
+    {
+      name: "Llama (Meta)",
+      prefix: "Llama",
+      desc: "General purpose"
+    },
+    {
+      name: "Qwen (Alibaba)",
+      prefix: "Qwen",
+      desc: "Multilingual"
+    },
+    {
+      name: "Mistral",
+      prefix: "Mistral",
+      desc: "Balanced performance"
+    },
+    {
+      name: "Ministral",
+      prefix: "Ministral",
+      desc: "Fast inference"
+    }
+  ];
+  for (const family of families) {
+    const familyModels = Array.from(modelsByHfId.entries()).filter(
+      ([hfId]) => hfId.toLowerCase().includes(family.prefix.toLowerCase())
+    );
+    if (familyModels.length === 0) continue;
+    log(`${colors.bold}${family.name}${colors.reset} ${colors.dim}\u2014 ${family.desc}${colors.reset}`);
+    for (const [hfId, aliases] of familyModels) {
+      const sortedAliases = aliases.sort((a, b) => a.length - b.length || a.localeCompare(b));
+      const primary = sortedAliases[0];
+      const others = sortedAliases.slice(1);
+      const aliasStr = others.length > 0 ? `${colors.green}${primary ?? ""}${colors.reset} ${colors.dim}(${others.join(", ")})${colors.reset}` : `${colors.green}${primary ?? ""}${colors.reset}`;
+      log(`  ${aliasStr.padEnd(45)} ${colors.dim}${hfId}${colors.reset}`);
+    }
+    log("");
+  }
+  log(`${colors.dim}Or use any mlx-community model:${colors.reset}`);
+  log(`  ${colors.cyan}node-mlx --model mlx-community/YourModel-4bit${colors.reset}`);
+  log("");
+}
+function resolveModel(name) {
+  if (name in RECOMMENDED_MODELS) {
+    return RECOMMENDED_MODELS[name];
+  }
+  if (name.includes("/")) {
+    return name;
+  }
+  return `mlx-community/${name}`;
+}
+function runInteractive(initialModel) {
+  const state = {
+    model: null,
+    modelName: initialModel,
+    options: {
+      maxTokens: 512,
+      temperature: 0.7,
+      topP: 0.9
+    },
+    history: [],
+    imagePath: null
+  };
+  log(`${colors.dim}Loading ${state.modelName}...${colors.reset}`);
+  const modelId = resolveModel(state.modelName);
+  try {
+    state.model = loadModel(modelId);
+    log(`${colors.green}\u2713${colors.reset} Model loaded`);
+  } catch (err) {
+    error(`Failed to load model: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+  }
+  log("");
+  log(`${colors.dim}Type your message or /help for commands${colors.reset}`);
+  log("");
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout
+  });
+  const promptUser = () => {
+    rl.question(`${colors.cyan}You:${colors.reset} `, (input) => {
+      void handleUserInput(input, state, rl, promptUser);
+    });
+  };
+  const handleUserInput = async (input, state2, rl2, next) => {
+    const trimmed = input.trim();
+    if (!trimmed) {
+      next();
+      return;
+    }
+    if (trimmed.startsWith("/")) {
+      await handleCommand(trimmed, state2, rl2);
+      next();
+      return;
+    }
+    if (!state2.model) {
+      error("No model loaded");
+      next();
+      return;
+    }
+    const fullPrompt = buildPrompt(state2.history, trimmed);
+    state2.history.push({ role: "user", content: trimmed });
+    process.stdout.write(`${colors.magenta}AI:${colors.reset} `);
+    try {
+      let result;
+      if (state2.imagePath && state2.model.isVLM()) {
+        result = state2.model.generateWithImage(fullPrompt, state2.imagePath, state2.options);
+        state2.imagePath = null;
+      } else {
+        result = state2.model.generateStreaming(fullPrompt, state2.options);
+      }
+      log("");
+      log(
+        `${colors.dim}(${String(result.tokenCount)} tokens, ${result.tokensPerSecond.toFixed(1)} tok/s)${colors.reset}`
+      );
+      log("");
+      state2.history.push({ role: "assistant", content: "[streamed response]" });
+    } catch (err) {
+      log("");
+      error(err instanceof Error ? err.message : String(err));
+    }
+    next();
+  };
+  rl.on("close", () => {
+    log("");
+    log(`${colors.dim}Goodbye!${colors.reset}`);
+    if (state.model) {
+      state.model.unload();
+    }
+    process.exit(0);
+  });
+  promptUser();
+}
+function buildPrompt(history, current) {
+  let prompt = "";
+  for (const msg of history.slice(-6)) {
+    if (msg.role === "user") {
+      prompt += `User: ${msg.content}
+`;
+    } else {
+      prompt += `Assistant: ${msg.content}
+`;
+    }
+  }
+  prompt += `User: ${current}
+Assistant:`;
+  return prompt;
+}
+async function handleCommand(input, state, rl) {
+  const [cmd, ...args] = input.slice(1).split(" ");
+  const arg = args.join(" ");
+  switch (cmd) {
+    case "help":
+    case "h":
+      printHelp();
+      break;
+    case "quit":
+    case "q":
+    case "exit":
+      rl.close();
+      break;
+    case "clear":
+    case "c":
+      state.history = [];
+      log(`${colors.dim}Conversation cleared${colors.reset}`);
+      break;
+    case "model":
+    case "m":
+      if (!arg) {
+        log(`${colors.dim}Current model: ${state.modelName}${colors.reset}`);
+        log(`${colors.dim}Use /model <name> to switch${colors.reset}`);
+      } else {
+        log(`${colors.dim}Loading ${arg}...${colors.reset}`);
+        if (state.model) {
+          state.model.unload();
+        }
+        try {
+          state.model = loadModel(resolveModel(arg));
+          state.modelName = arg;
+          state.history = [];
+          log(`${colors.green}\u2713${colors.reset} Switched to ${arg}`);
+        } catch (err) {
+          error(err instanceof Error ? err.message : String(err));
+        }
+      }
+      break;
+    case "temp":
+    case "t":
+      if (!arg) {
+        log(`${colors.dim}Temperature: ${String(state.options.temperature)}${colors.reset}`);
+      } else {
+        const temp = parseFloat(arg);
+        if (isNaN(temp) || temp < 0 || temp > 2) {
+          error("Temperature must be between 0 and 2");
+        } else {
+          state.options.temperature = temp;
+          log(`${colors.dim}Temperature set to ${String(temp)}${colors.reset}`);
+        }
+      }
+      break;
+    case "tokens":
+    case "n":
+      if (!arg) {
+        log(`${colors.dim}Max tokens: ${String(state.options.maxTokens)}${colors.reset}`);
+      } else {
+        const tokens = parseInt(arg, 10);
+        if (isNaN(tokens) || tokens < 1) {
+          error("Tokens must be a positive number");
+        } else {
+          state.options.maxTokens = tokens;
+          log(`${colors.dim}Max tokens set to ${String(tokens)}${colors.reset}`);
+        }
+      }
+      break;
+    case "rep":
+    case "r":
+      if (!arg) {
+        log(
+          `${colors.dim}Repetition penalty: ${state.options.repetitionPenalty != null ? String(state.options.repetitionPenalty) : "off"}${colors.reset}`
+        );
+      } else {
+        const penalty = parseFloat(arg);
+        if (isNaN(penalty) || penalty < 1 || penalty > 2) {
+          error("Repetition penalty must be between 1 and 2");
+        } else {
+          state.options.repetitionPenalty = penalty;
+          log(`${colors.dim}Repetition penalty set to ${String(penalty)}${colors.reset}`);
+        }
+      }
+      break;
+    case "list":
+    case "l":
+      printModels();
+      break;
+    case "image":
+    case "i":
+      if (!arg) {
+        if (state.imagePath) {
+          log(`${colors.dim}Current image: ${state.imagePath}${colors.reset}`);
+        } else {
+          log(`${colors.dim}No image set. Use /image <path> to set one.${colors.reset}`);
+        }
+      } else {
+        const fs = await import("fs");
+        if (!fs.existsSync(arg)) {
+          error(`Image not found: ${arg}`);
+        } else if (!state.model?.isVLM()) {
+          error(`Current model doesn't support images. Use a VLM like gemma-3-4b.`);
+        } else {
+          state.imagePath = arg;
+          log(`${colors.green}\u2713${colors.reset} Image set: ${arg}`);
+          log(`${colors.dim}The next message will include this image.${colors.reset}`);
+        }
+      }
+      break;
+    default:
+      error(`Unknown command: /${cmd ?? ""}. Type /help for commands.`);
+  }
+}
+function runOneShot(modelName, prompt, imagePath, options) {
+  log(`${colors.dim}Loading ${modelName}...${colors.reset}`);
+  const modelId = resolveModel(modelName);
+  try {
+    const model = loadModel(modelId);
+    let result;
+    if (imagePath) {
+      if (!model.isVLM()) {
+        error(`Model ${modelName} doesn't support images. Use a VLM like gemma-3-4b.`);
+        model.unload();
+        process.exit(1);
+      }
+      result = model.generateWithImage(prompt, imagePath, options);
+    } else {
+      result = model.generateStreaming(prompt, options);
+    }
+    log("");
+    log(
+      `${colors.dim}(${String(result.tokenCount)} tokens, ${result.tokensPerSecond.toFixed(1)} tok/s)${colors.reset}`
+    );
+    model.unload();
+  } catch (err) {
+    error(err instanceof Error ? err.message : String(err));
+    process.exit(1);
+  }
+}
+function parseArgs() {
+  const args = process.argv.slice(2);
+  let model = "qwen";
+  let prompt = null;
+  let imagePath = null;
+  const options = {
+    maxTokens: 512,
+    temperature: 0.7,
+    topP: 0.9
+  };
+  let command = "chat";
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "--help" || arg === "-h") {
+      command = "help";
+    } else if (arg === "--version" || arg === "-v") {
+      command = "version";
+    } else if (arg === "--list" || arg === "-l") {
+      command = "list";
+    } else if (arg === "--model" || arg === "-m") {
+      model = args[++i] || model;
+    } else if (arg === "--image" || arg === "-i") {
+      imagePath = args[++i] || null;
+    } else if (arg === "--temp" || arg === "-t") {
+      options.temperature = parseFloat(args[++i] || "0.7");
+    } else if (arg === "--tokens" || arg === "-n") {
+      options.maxTokens = parseInt(args[++i] || "512", 10);
+    } else if (arg === "--repetition-penalty" || arg === "-r") {
+      options.repetitionPenalty = parseFloat(args[++i] || "1.2");
+    } else if (arg && !arg.startsWith("-")) {
+      if (model === "qwen") {
+        model = arg;
+      } else if (prompt === null) {
+        prompt = arg;
+        command = "oneshot";
+      }
+    }
+  }
+  return { model, prompt, imagePath, options, command };
+}
+function main() {
+  const { model, prompt, imagePath, options, command } = parseArgs();
+  switch (command) {
+    case "help":
+      printHeader();
+      printHelp();
+      return;
+    case "version":
+      log(`node-mlx v${VERSION}`);
+      return;
+    case "list":
+      printHeader();
+      printModels();
+      return;
+  }
+  if (!isPlatformSupported()) {
+    error("node-mlx requires macOS on Apple Silicon (M1/M2/M3/M4)");
+    process.exit(1);
+  }
+  if (!isSupported()) {
+    error("Native libraries not found. Run 'pnpm build:swift && pnpm build:native' first.");
+    process.exit(1);
+  }
+  switch (command) {
+    case "oneshot":
+      if (prompt) {
+        runOneShot(model, prompt, imagePath, options);
+      }
+      break;
+    case "chat":
+      printHeader();
+      runInteractive(model);
+      break;
+  }
+}
+try {
+  main();
+} catch (err) {
+  error(err instanceof Error ? err.message : String(err));
+  process.exit(1);
+}
+//# sourceMappingURL=cli.js.map