@lloyal-labs/lloyal.node 1.0.5-alpha → 1.0.7

package/lib/index.js

@@ -1,6 +1,3 @@
-const path = require('path');
-const binary = require('node-gyp-build')(path.join(__dirname, '..'));
-
 /**
  * liblloyal-node - Thin N-API wrapper over liblloyal
  *
@@ -9,7 +6,7 @@ const binary = require('node-gyp-build')(path.join(__dirname, '..'));
  *
  * @example
  * ```js
- * const { createContext, withLogits } = require('lloyal.node');
+ * const { createContext, withLogits } = require('@lloyal-labs/lloyal.node');
  *
  * const ctx = await createContext({
  *   modelPath: './model.gguf',
@@ -24,20 +21,133 @@ const binary = require('node-gyp-build')(path.join(__dirname, '..'));
  * await ctx.decode(tokens, 0);
  *
  * // Safe logits access (Runtime Borrow Checker pattern)
- * const entropy = await withLogits(ctx, (logits) => {
+ * const entropy = withLogits(ctx, (logits) => {
  *   // logits is valid here - use synchronously only!
- *   return computeEntropy(logits);
+ *   return myComputeEntropy(logits);
  * });
  *
  * // Or with native reference implementations (for testing)
- * const nativeEntropy = ctx.computeEntropy();
+ * const entropy = ctx.modelEntropy();
  * const token = ctx.greedySample();
  *
  * // Cleanup
  * ctx.dispose();
  * ```
+ *
+ * @example GPU variant selection
+ * ```js
+ * // Option 1: Environment variable (affects all contexts)
+ * // Set LLOYAL_GPU=cuda before running
+ *
+ * // Option 2: Per-context selection (recommended)
+ * const ctx = await createContext(
+ *   { modelPath: './model.gguf', nCtx: 4096 },
+ *   { gpuVariant: 'cuda' }  // Falls back to CPU if CUDA unavailable
+ * );
+ * ```
  */
 
+/**
+ * Platform package naming: @lloyal-labs/lloyal.node-{platform}-{arch}[-{gpu}]
+ * @param {string} [variant] - GPU variant: 'cuda', 'vulkan', or undefined for CPU
+ * @returns {string} Platform package name
+ */
+const getPlatformPackageName = (variant) => {
+  const platform = process.platform;
+  const arch = process.arch;
+  // cpu/metal/default = no suffix, cuda/vulkan = suffix
+  const noSuffix = !variant || variant === 'default' || variant === 'cpu' || variant === 'metal';
+  const suffix = noSuffix ? '' : `-${variant}`;
+  return `@lloyal-labs/lloyal.node-${platform}-${arch}${suffix}`;
+};
+
+/**
+ * Try to load a platform package, return null on failure.
+ * Failures include: package not installed, missing GPU runtime libs (dlopen fails),
+ * or module doesn't export expected interface.
+ * @param {string} packageName - Package name to load
+ * @param {boolean} [verbose=false] - Log failure reasons
+ * @returns {object|null} The native binary module or null
+ */
+const tryLoadPackage = (packageName, verbose = false) => {
+  try {
+    const mod = require(packageName);
+    // Validate it's actually a native module with expected exports
+    if (mod && typeof mod.createContext === 'function') {
+      return mod;
+    }
+    if (verbose) {
+      console.warn(`[lloyal.node] ${packageName} loaded but missing createContext export`);
+    }
+    return null;
+  } catch (e) {
+    if (verbose) {
+      console.warn(`[lloyal.node] Failed to load ${packageName}: ${e.message}`);
+    }
+    return null;
+  }
+};
+
+/**
+ * Load the native binary with automatic fallback.
+ *
+ * Loading priority:
+ * 1. Requested GPU variant (if specified)
+ * 2. Default platform package (CPU)
+ * 3. Local build (development: build/Release/lloyal.node)
+ *
+ * @param {string} [variant] - GPU variant: 'cuda', 'vulkan', or undefined for CPU
+ * @returns {object} The native binary module
+ * @throws {Error} If no binary can be loaded
+ */
+const loadBinary = (variant) => {
+  // Use env var if no variant specified
+  variant = variant ?? process.env.LLOYAL_GPU;
+  // LLOYAL_NO_FALLBACK=1 disables fallback (for CI testing specific packages)
+  const noFallback = process.env.LLOYAL_NO_FALLBACK === '1';
+
+  // 1. Try requested variant (if specified)
+  if (variant && variant !== 'default') {
+    const pkgName = getPlatformPackageName(variant);
+    const binary = tryLoadPackage(pkgName, true); // verbose=true to see errors
+    if (binary) return binary;
+
+    if (noFallback) {
+      throw new Error(
+        `[lloyal.node] GPU variant "${variant}" failed to load. ` +
+        `Package: ${pkgName}. Check that runtime libraries are available.`
+      );
+    }
+    console.warn(`[lloyal.node] GPU variant "${variant}" unavailable, falling back to CPU`);
+  }
+
+  // 2. Try default platform package (CPU)
+  const defaultPkg = getPlatformPackageName();
+  const binary = tryLoadPackage(defaultPkg, true); // verbose=true
+  if (binary) return binary;
+
+  // 3. Try local build (development)
+  try {
+    return require('../build/Release/lloyal.node');
+  } catch (e) {
+    // ignore
+  }
+
+  throw new Error(
+    `No lloyal.node binary found for ${process.platform}-${process.arch}. ` +
+    `Tried: ${variant ? getPlatformPackageName(variant) + ', ' : ''}${defaultPkg}`
+  );
+};
+
+// Default binary (loaded lazily on first use)
+let _binary = null;
+const getBinary = () => {
+  if (!_binary) {
+    _binary = loadBinary(process.env.LLOYAL_GPU);
+  }
+  return _binary;
+};
+
 /**
  * Safe logits access with Runtime Borrow Checker pattern
  *
@@ -93,29 +203,65 @@ function withLogits(ctx, fn) {
   return result;
 }
 
+const { Branch } = require('./Branch');
+
 module.exports = {
+  /**
+   * Branch class for parallel generation
+   * @see Branch.create()
+   */
+  Branch,
   /**
    * Create a new inference context
    *
-   * @param {Object} options
-   * @param {string} options.modelPath - Path to .gguf model file
-   * @param {number} [options.nCtx=2048] - Context size
-   * @param {number} [options.nThreads=4] - Number of threads
-   * @returns {Promise<SessionContext>}
+   * @param {ContextOptions} options - Context configuration
+   * @param {LoadOptions} [loadOptions] - Binary loading options
+   * @returns {Promise<SessionContext>} The inference context
+   *
+   * @example
+   * ```js
+   * // Basic usage
+   * const ctx = await createContext({
+   *   modelPath: './model.gguf',
+   *   nCtx: 2048,
+   *   nThreads: 4
+   * });
+   *
+   * // With GPU variant
+   * const ctx = await createContext(
+   *   { modelPath: './model.gguf' },
+   *   { gpuVariant: 'cuda' }
+   * );
+   * ```
    */
-  createContext: async (options) => {
-    // For now, createContext is synchronous in C++
-    // Wrap in Promise for future async model loading
+  createContext: async (options, loadOptions) => {
+    const variant = loadOptions?.gpuVariant || process.env.LLOYAL_GPU;
+    const binary = variant ? loadBinary(variant) : getBinary();
     return binary.createContext(options);
   },
 
   /**
-   * Safe logits access with Runtime Borrow Checker pattern
+   * Load binary for a specific GPU variant.
+   * Useful for checking variant availability before creating context.
+   *
+   * @param {string} [variant] - 'cuda', 'vulkan', or undefined for CPU
+   * @returns {object} Native binary module
+   * @throws {Error} If no binary available for platform
    *
-   * Ensures logits are only accessed synchronously within the callback.
+   * @example
+   * ```js
+   * // Load default (CPU) binary
+   * const binary = loadBinary();
+   *
+   * // Load CUDA binary (falls back to CPU if unavailable)
+   * const binary = loadBinary('cuda');
+   * ```
+   */
+  loadBinary,
+
+  /**
+   * Safe logits access with Runtime Borrow Checker pattern.
    * See function JSDoc for full documentation.
    */
   withLogits,
-
-  SessionContext: binary.SessionContext
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lloyal-labs/lloyal.node",
-  "version": "1.0.5-alpha",
+  "version": "1.0.7",
   "description": "Node.js client for liblloyal+llama.cpp",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -10,7 +10,6 @@
   },
   "scripts": {
     "download-models": "bash scripts/download-test-models.sh",
-    "install": "node scripts/install.js",
     "build": "node scripts/build.js",
     "build:debug": "cmake-js compile --debug",
     "rebuild": "cmake-js rebuild",
@@ -20,6 +19,7 @@
     "test": "npm run test:api && npm run test:e2e",
     "test:api": "node test/api.js",
     "test:e2e": "node test/e2e.js",
+    "test:examples": "node test/examples.js",
     "example": "node examples/chat/chat.mjs"
   },
   "repository": {
@@ -43,28 +43,29 @@
   },
   "homepage": "https://github.com/lloyal-ai/lloyal.node#readme",
   "dependencies": {
-    "node-addon-api": "^8.5.0",
-    "node-gyp-build": "^4.8.4"
+    "@lloyal-labs/tsampler": "^0.2.0",
+    "node-addon-api": "^8.5.0"
   },
   "devDependencies": {
     "cmake-js": "^7.4.0",
     "glob": "^11.0.0",
-    "typedoc": "^0.27.5"
+    "typedoc": "^0.28.16",
+    "typedoc-rhineai-theme": "^1.2.0"
   },
   "optionalDependencies": {
-    "@lloyal-labs/lloyal.node-darwin-arm64": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-darwin-x64": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-linux-arm64": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-linux-arm64-cuda": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-linux-x64": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-linux-x64-cuda": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-linux-x64-vulkan": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-win32-arm64": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-win32-x64": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-win32-x64-cuda": "1.0.5-alpha",
-    "@lloyal-labs/lloyal.node-win32-x64-vulkan": "1.0.5-alpha"
+    "@lloyal-labs/lloyal.node-darwin-arm64": "1.0.7",
+    "@lloyal-labs/lloyal.node-darwin-x64": "1.0.7",
+    "@lloyal-labs/lloyal.node-linux-arm64": "1.0.7",
+    "@lloyal-labs/lloyal.node-linux-arm64-cuda": "1.0.7",
+    "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "1.0.7",
+    "@lloyal-labs/lloyal.node-linux-x64": "1.0.7",
+    "@lloyal-labs/lloyal.node-linux-x64-cuda": "1.0.7",
+    "@lloyal-labs/lloyal.node-linux-x64-vulkan": "1.0.7",
+    "@lloyal-labs/lloyal.node-win32-arm64": "1.0.7",
+    "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "1.0.7",
+    "@lloyal-labs/lloyal.node-win32-x64": "1.0.7",
+    "@lloyal-labs/lloyal.node-win32-x64-cuda": "1.0.7",
+    "@lloyal-labs/lloyal.node-win32-x64-vulkan": "1.0.7"
   },
   "engines": {
     "node": ">=22.0.0"

package/scripts/create-platform-package.js

@@ -108,52 +108,31 @@ if (osName === 'darwin') {
 // Create package.json from template
 console.log('\nGenerating package.json...');
 const mainPackageJson = require(path.join(ROOT, 'package.json'));
-const templatePath = path.join(ROOT, 'packages', 'template', 'package.json');
-
-let pkgJson;
-if (fs.existsSync(templatePath)) {
-  pkgJson = require(templatePath);
-} else {
-  // Fallback template if file doesn't exist yet
-  pkgJson = {
-    name: '@lloyal-labs/lloyal.node-PLATFORM',
-    version: '0.0.0',
-    description: 'Lloyal native binary for PLATFORM',
-    main: 'index.js',
-    files: ['bin/', 'index.js'],
-    repository: {
-      type: 'git',
-      url: 'git+https://github.com/lloyal-ai/lloyal.node.git'
-    },
-    license: 'Apache-2.0'
-  };
-}
 
-// Update with actual values
-pkgJson.name = `@lloyal-labs/lloyal.node-${packageName}`;
-pkgJson.version = mainPackageJson.version;
-pkgJson.description = `Lloyal native binary for ${packageName}`;
-pkgJson.os = [osName];
-pkgJson.cpu = [arch];
+// Platform package exports the binary directly (no index.js wrapper)
+// This enables runtime dynamic require with automatic fallback:
+//   require('@lloyal-labs/lloyal.node-linux-x64') → bin/lloyal.node
+const pkgJson = {
+  name: `@lloyal-labs/lloyal.node-${packageName}`,
+  version: mainPackageJson.version,
+  description: `Lloyal native binary for ${packageName}`,
+  main: 'bin/lloyal.node',
+  os: [osName],
+  cpu: [arch],
+  files: ['bin/'],
+  repository: {
+    type: 'git',
+    url: 'git+https://github.com/lloyal-ai/lloyal.node.git'
+  },
+  author: 'lloyal.ai',
+  license: 'Apache-2.0'
+};
 
 fs.writeFileSync(
   path.join(PKG_DIR, 'package.json'),
   JSON.stringify(pkgJson, null, 2) + '\n'
 );
-console.log(`  ✓ Created package.json`);
-
-// Create index.js
-console.log('\nGenerating index.js...');
-const indexJs = `// Platform-specific binary package for ${packageName}
-// This file resolves to the native binary in bin/
-
-const path = require('path');
-
-module.exports = path.join(__dirname, 'bin', 'lloyal.node');
-`;
-
-fs.writeFileSync(path.join(PKG_DIR, 'index.js'), indexJs);
-console.log(`  ✓ Created index.js`);
+console.log(`  ✓ Created package.json (main: bin/lloyal.node)`);
 
 // Summary
 console.log(`\n✅ Platform package created successfully!`);

package/scripts/download-test-models.sh

@@ -26,5 +26,15 @@ else
   echo "  ✓ nomic-embed-text already exists"
 fi
 
+# slim-summary-tool (1.7GB) - Summary sidecar for dynamic sinks
+if [ ! -f "slim-summarize.gguf" ]; then
+  echo "  → Downloading slim-summarize.gguf..."
+  curl -L -o "slim-summarize.gguf" \
+    "https://huggingface.co/llmware/slim-summary-tool/resolve/main/slim-summary-tool.gguf"
+  echo "  ✓ Downloaded slim-summarize"
+else
+  echo "  ✓ slim-summarize already exists"
+fi
+
 echo ""
 echo "✅ All test models ready"

package/scripts/install.js

@@ -1,138 +0,0 @@
-#!/usr/bin/env node
-/**
- * Smart installer for lloyal.node
- *
- * Strategy:
- * 1. Check if prebuilt binary exists for this platform
- * 2. If yes, copy to build/Release/ and exit
- * 3. If no, show helpful error with build-from-source instructions
- *
- * Respects LLOYAL_GPU environment variable for GPU variant selection
- */
-
-const fs = require('fs');
-const path = require('path');
-
-const PLATFORM = process.platform;
-const ARCH = process.arch;
-const ROOT = __dirname + '/..';
-const BUILD_DIR = path.join(ROOT, 'build', 'Release');
-
-// Logging helpers
-const log = (msg) => console.log(`[lloyal.node] ${msg}`);
-const error = (msg) => console.error(`[lloyal.node] ❌ ${msg}`);
-
-/**
- * Check if a platform package is installed and has binaries
- */
-function findPrebuilt(packageName) {
-  try {
-    const pkgPath = require.resolve(packageName);
-    const binPath = require(packageName); // index.js exports path to binary
-
-    if (fs.existsSync(binPath)) {
-      const binDir = path.dirname(binPath);
-      return binDir;
-    }
-  } catch (e) {
-    // Package not installed or doesn't export binary path
-  }
-  return null;
-}
-
-/**
- * Copy prebuilt binaries to build/Release/
- */
-function installPrebuilt(binDir, packageName) {
-  log(`Found prebuilt binaries in ${packageName}`);
-
-  try {
-    // Create build/Release directory
-    fs.mkdirSync(BUILD_DIR, { recursive: true });
-
-    // Copy all files from bin directory
-    const files = fs.readdirSync(binDir);
-    files.forEach(file => {
-      const src = path.join(binDir, file);
-      const dest = path.join(BUILD_DIR, file);
-
-      if (fs.statSync(src).isFile()) {
-        fs.copyFileSync(src, dest);
-        log(`  ✓ Copied ${file}`);
-      }
-    });
-
-    log(`✅ Installed prebuilt binaries successfully`);
-    process.exit(0);
-  } catch (e) {
-    error(`Failed to install prebuilt: ${e.message}`);
-    // Don't exit - fall through to source build
-  }
-}
-
-/**
- * Main installation logic
- */
-function main() {
-  log(`Platform: ${PLATFORM}-${ARCH}`);
-
-  // 1. Check for user-specified GPU variant via environment variable
-  if (process.env.LLOYAL_GPU) {
-    const gpu = process.env.LLOYAL_GPU.toLowerCase();
-    const packageName = `@lloyal-labs/lloyal.node-${PLATFORM}-${ARCH}-${gpu}`;
-
-    log(`LLOYAL_GPU=${gpu}, looking for ${packageName}...`);
-    const binDir = findPrebuilt(packageName);
-
-    if (binDir) {
-      installPrebuilt(binDir, packageName);
-      return; // exit(0) called in installPrebuilt
-    } else {
-      log(`  ⚠️  Package ${packageName} not found`);
-    }
-  }
-
-  // 2. Check for GPU variants in priority order
-  const gpuVariants = ['cuda', 'vulkan'];
-  for (const gpu of gpuVariants) {
-    const packageName = `@lloyal-labs/lloyal.node-${PLATFORM}-${ARCH}-${gpu}`;
-    const binDir = findPrebuilt(packageName);
-
-    if (binDir) {
-      log(`Auto-detected GPU variant: ${gpu}`);
-      installPrebuilt(binDir, packageName);
-      return; // exit(0) called in installPrebuilt
-    }
-  }
-
-  // 3. Check for default platform package (CPU or Metal on macOS)
-  const defaultPackage = `@lloyal-labs/lloyal.node-${PLATFORM}-${ARCH}`;
-  const binDir = findPrebuilt(defaultPackage);
-
-  if (binDir) {
-    installPrebuilt(binDir, defaultPackage);
-    return; // exit(0) called in installPrebuilt
-  }
-
-  // 4. No prebuilt found - error with helpful message
-  log('');
-  error('No prebuilt binary found for your platform');
-  log('');
-  log(`  Platform: ${PLATFORM}-${ARCH}`);
-  log('');
-  log('  Options:');
-  log('  1. Install a platform-specific package:');
-  log(`     npm install @lloyal-labs/lloyal.node-${PLATFORM}-${ARCH}`);
-  log('');
-  log('  2. Build from source (requires C++20, CMake 3.18+):');
-  log('     git clone --recursive https://github.com/lloyal-ai/lloyal.node.git');
-  log('     cd lloyal.node && npm run build');
-  log('');
-  log('  See: https://github.com/lloyal-ai/lloyal.node#building');
-  log('');
-  
-  process.exit(1);
-}
-
-// Run installer
-main();