@qvac/llm-llamacpp 0.1.0

package/addon.js

@@ -0,0 +1,95 @@
+const binding = require('./binding')
+
+/**
+ * An interface between Bare addon in C++ and JS runtime.
+ */
+class LlamaInterface {
+  /**
+   *
+   * @param {Object} configurationParams - all the required configuration for inference setup
+   * @param {Function} outputCb - to be called on any inference event ( started, new output, error, etc )
+   * @param {Function} transitionCb - to be called on addon state changes (LISTENING, IDLE, STOPPED, etc )
+   */
+  constructor (configurationParams, outputCb, transitionCb = null) {
+    this._handle = binding.createInstance(
+      this,
+      configurationParams,
+      outputCb,
+      transitionCb
+    )
+  }
+
+  /**
+   *
+   * @param {Object} weightsData
+   * @param {String} weightsData.filename
+   * @param {Uint8Array} weightsData.contents
+   * @param {Boolean} weightsData.completed
+   */
+  async loadWeights (weightsData) {
+    binding.loadWeights(this._handle, weightsData)
+  }
+
+  /**
+   * Moves addon to the LISTENING state after all the initialization is done
+   */
+  async activate () {
+    binding.activate(this._handle)
+  }
+
+  /**
+   * Pauses current inference process
+   */
+  async pause () {
+    binding.pause(this._handle)
+  }
+
+  /**
+   * Cancel a inference process by jobId, if no jobId is provided it cancel the whole queue
+   */
+  async cancel (jobId) {
+    binding.cancel(this._handle, jobId)
+  }
+
+  /**
+   * Adds new input to the processing queue
+   * @param {Object} data
+   * @param {String} data.type
+   * @param {String} data.input
+   * @returns {Number} - job ID
+   */
+  async append (data) {
+    return binding.append(this._handle, data)
+  }
+
+  /**
+   * Addon process status
+   * @returns {String}
+   */
+  async status () {
+    return binding.status(this._handle)
+  }
+
+  /**
+   * Stops addon process and clears resources (including memory).
+   */
+  async destroyInstance () {
+    if (!this._handle) return
+    binding.destroyInstance(this._handle)
+    this._handle = null
+  }
+
+  async unload () {
+    if (!this._handle) return
+    binding.destroyInstance(this._handle)
+    this._handle = null
+  }
+
+  async stop () {
+    binding.stop(this._handle)
+  }
+}
+
+module.exports = {
+  LlamaInterface
+}

package/binding.js

@@ -0,0 +1 @@
+module.exports = require.addon()

package/index.d.ts

@@ -0,0 +1,139 @@
+/// <reference types="node" />
+
+import BaseInference from '@tetherto/infer-base/WeightsProvider/BaseInference';
+import WeightsProvider from '@tetherto/infer-base/WeightsProvider/WeightsProvider';
+import type QvacResponse from '@tetherto/qvac-lib-response';
+import type Logger  from '@tetherto/qvac-lib-logging';
+import type Loader  from '@tetherto/qvac-lib-dl-base';
+import type { Readable } from 'stream';
+
+/**
+ * Loader interface provides the methods required by LlmLlamacpp to fetch and manage streams.
+ */
+export interface Loader {
+  /** Prepare the loader for operations */
+  ready(): Promise<void>;
+  /** Clean up or close any underlying resources */
+  close(): Promise<void>;
+  /** Obtain a readable stream for the specified path */
+  getStream(path: string): Promise<Readable>;
+  /** (Optional) Retrieve the size of a remote file in bytes */
+  getFileSize?(path: string): Promise<number>;
+}
+
+/**
+ * Arguments required to construct an instance of LlmLlamacpp
+ */
+export interface LlmLlamacppArgs {
+  /** External loader instance */
+  loader: Loader;
+  /** Optional structured logger */
+  logger?: Logger;
+  /** Optional inference options */
+  opts?: Record<string, any>;
+  /** Disk directory where model files are stored */
+  diskPath: string;
+  /** Name of the model directory or file */
+  modelName: string;
+  /** Name of the projection model directory or file */
+  projectionModel?: string;
+}
+
+/** Literal indicating end-of-input for the LLM job */
+export type EndOfInput = 'end of job';
+
+/** Input types accepted by the Llama addon */
+export type AppendInput =
+  | { type: 'text'; input: string }
+  | { type: 'media'; input: Uint8Array }
+  | { type: EndOfInput };
+
+/** Minimal interface for the native addon controlling the LLM */
+export interface Addon {
+  activate(): Promise<void>;
+  append(input: AppendInput): Promise<number>;
+  cancel(jobId: number): Promise<void>;
+}
+
+/** Callback invoked with the number of bytes processed during downloads */
+export type ProgressReportCallback = (bytes: number) => void;
+
+
+/** Structure of a message for chat-style prompts */
+export interface Message {
+  role: 'system' | 'user' | 'assistant' | 'tool' | string;
+  content: string;
+}
+
+/**
+ * GGML client implementation for the Llama LLM model.
+ */
+declare class LlmLlamacpp extends BaseInference {
+  protected readonly _config: Record<string, any>;
+  protected readonly _diskPath: string;
+  protected readonly _modelName: string;
+  protected addon!: Addon;
+  protected weightsProvider: WeightsProvider;
+
+  /**
+   * @param args - Setup parameters including loader, logger, disk path, and model name
+   * @param config - Model-specific configuration settings
+   */
+  constructor(args: LlmLlamacppArgs, config: Record<string, any>);
+
+  /**
+   * Implementation of load method, to load model weights, initialize the native addon, and activate the model.
+   * @param closeLoader - Whether to close the loader when complete (default true)
+   * @param onDownloadProgress - Optional byte-level progress callback
+   */
+  protected _load(
+    closeLoader?: boolean,
+    onDownloadProgress?: ProgressReportCallback
+  ): Promise<void>;
+
+  /**
+   * Load model weights, initialize the native addon, and activate the model.
+   * @param closeLoader - Whether to close the loader when complete (default true)
+   * @param onDownloadProgress - Optional byte-level progress callback
+   */
+  public load(
+    closeLoader?: boolean,
+    onDownloadProgress?: ProgressReportCallback
+  ): Promise<void>;
+
+  /**
+   * Download the model weight files and return the local path to the primary file.
+   * @param onDownloadProgress - Callback invoked with bytes downloaded
+   * @returns Local file path for the model weights
+   */
+  public downloadWeights(
+    onDownloadProgress?: ProgressReportCallback,
+    opts?: {
+      closeLoader?: boolean
+    }
+  ): Promise<string>;
+
+  /**
+   * Instantiate the native addon with the given parameters.
+   * @param params.path - Local file or directory path
+   * @param params.settings - LLM-specific settings
+   */
+  protected _createAddon(
+    params: { path: string; settings: Record<string, any> }
+  ): Addon;
+
+  /**
+   * Internal method to start inference with a text prompt.
+   * @param prompt - Input prompt string
+   * @returns A QvacResponse representing the inference job
+   */
+  protected _runInternal(prompt: Message[]): Promise<QvacResponse>;
+
+  /**
+   * Public API to run inference. Delegates to _runInternal.
+   * @param prompt - Input prompt string
+   */
+  run(prompt: Message[]): Promise<QvacResponse>;
+}
+
+export = LlmLlamacpp;

package/index.js

@@ -0,0 +1,160 @@
+'use strict'
+
+const path = require('bare-path')
+
+const BaseInference = require('@tetherto/infer-base/WeightsProvider/BaseInference')
+const WeightsProvider = require('@tetherto/infer-base/WeightsProvider/WeightsProvider')
+const { LlamaInterface } = require('./addon')
+
+const END_OF_INPUT = 'end of job'
+const noop = () => { }
+
+/**
+ * GGML client implementation for Llama LLM model
+ */
+class LlmLlamacpp extends BaseInference {
+  /**
+   * Creates an instance of LlmLlamacpp.
+   * @constructor
+   * @param {Object} args - Setup parameters including loader, logger, disk path, and model name
+   * @param {Loader} args.loader - External loader instance
+   * @param {Logger} [args.logger] - Optional structured logger
+   * @param {Object} [args.opts] - Optional inference options
+   * @param {string} args.diskPath - Disk directory where model files are stored
+   * @param {string} args.modelName - Name of the model directory or file
+   * @param {string} args.projectionModel - Name of the projection model directory or file
+   * @param {Object} config - Model-specific configuration settings
+   */
+  constructor ({ opts = {}, loader, logger = null, diskPath, modelName, projectionModel }, config) {
+    super({ logger, opts })
+    this._config = config
+    this._diskPath = diskPath
+    this._modelName = modelName
+    this._projectionModel = projectionModel
+    this.weightsProvider = new WeightsProvider(loader, this.logger)
+    this._runQueueWaiter = Promise.resolve()
+  }
+
+  /**
+   * Load model weights, initialize the native addon, and activate the model.
+   * @param {boolean} [closeLoader=true] - Whether to close the loader when complete
+   * @param {ProgressReportCallback} [onDownloadProgress] - Optional byte-level progress callback
+   * @returns {Promise<void>}
+   */
+  async _load (closeLoader = true, onDownloadProgress = noop) {
+    this.logger.info('Starting model load')
+
+    try {
+      await this.downloadWeights(onDownloadProgress, { closeLoader })
+
+      const configurationParams = {
+        path: path.join(this._diskPath, this._modelName),
+        projectionPath: this._projectionModel ? path.join(this._diskPath, this._projectionModel) : '',
+        settings: this._config
+      }
+
+      this.logger.info('Creating addon with configuration:', configurationParams)
+      this.addon = this._createAddon(configurationParams)
+
+      this.logger.info('Activating addon')
+      await this.addon.activate()
+
+      this.logger.info('Model load completed successfully')
+    } catch (error) {
+      this.logger.error('Error during model load:', error)
+      throw error
+    }
+  }
+
+  /**
+   * Download the model weight files and return the local path to the primary file.
+   * @param {ProgressReportCallback} [onDownloadProgress] - Callback invoked with bytes downloaded
+   * @returns {Promise<{filePath: string, completed: boolean, error: boolean}[]>} Local file path for the model weights
+   */
+  async _downloadWeights (onDownloadProgress, opts) {
+    return await this.weightsProvider.downloadFiles(
+      this._projectionModel ? [this._modelName, this._projectionModel] : [this._modelName],
+      this._diskPath,
+      {
+        closeLoader: opts.closeLoader,
+        onDownloadProgress
+      }
+    )
+  }
+
+  /**
+   * Instantiate the native addon with the given parameters.
+   * @param {Object} configurationParams - Configuration parameters for the addon
+   * @param {string} configurationParams.path - Local file or directory path
+   * @param {Object} configurationParams.settings - LLM-specific settings
+   * @returns {Addon} The instantiated addon interface
+   */
+  _createAddon (configurationParams) {
+    this.logger.info(
+      'Creating Llama interface with configuration:',
+      configurationParams
+    )
+    return new LlamaInterface(
+      configurationParams,
+      this._outputCallback.bind(this),
+      this.logger.info.bind(this.logger)
+    )
+  }
+
+  async _withExclusiveRun (fn) {
+    const prev = this._runQueueWaiter || Promise.resolve()
+    let release
+    this._runQueueWaiter = new Promise(resolve => { release = resolve })
+    await prev
+    try {
+      return await fn()
+    } finally {
+      release()
+    }
+  }
+
+  /**
+   * Internal method to start inference with a text prompt.
+   * @param {Message[]} prompt - Input prompt array of messages
+   * @returns {Promise<QvacResponse>} A QvacResponse representing the inference job
+   */
+  async _runInternal (prompt) {
+    this.logger.info('Starting inference with prompt:', prompt)
+    return this._withExclusiveRun(async () => {
+      // Process prompt to handle media content with user role
+      const processedPrompt = prompt.map(message => {
+        // Check if message has user role and media type with Uint8Array content
+        if (message.role === 'user' &&
+          message.type === 'media' &&
+          message.content instanceof Uint8Array) {
+          // Send media data as separate append call
+          this.addon.append({ type: 'media', input: message.content })
+            .catch(err => this.logger.error('Failed to send media data:', err))
+
+          // Return modified message with empty string for media content
+          return {
+            ...message,
+            content: ''
+          }
+        }
+
+        return message
+      })
+
+      const serializedPrompt = JSON.stringify(processedPrompt)
+
+      const jobId = await this.addon.append({ type: 'text', input: serializedPrompt })
+
+      this.logger.info('Created inference job with ID:', jobId)
+
+      const response = this._createResponse(jobId)
+      await this.addon.append({ type: END_OF_INPUT })
+
+      this.logger.info('Inference job started successfully')
+
+      return response
+    })
+  }
+}
+
+module.exports = LlmLlamacpp

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@qvac/llm-llamacpp",
+  "version": "0.1.0",
+  "description": "llama addon for qvac",
+  "addon": true,
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "scripts": {
+    "build": "bare-make generate && bare-make build && bare-make install",
+    "lint": "standard --ignore \"addon/**\"",
+    "lint:fix": "standard --ignore \"addon/**\" --fix",
+    "update:quickstart-section": "node ./scripts/quickstart-testing/quickstart_check.js && node ./scripts/quickstart-testing/update_readme.js",
+    "test": "npm run test:integration",
+    "test:integration": "npm run test:integration:generate && bare test/integration/all.js",
+    "model:init": "qvac-dev model init",
+    "model:new": "qvac-dev model new",
+    "model:generate": "qvac-dev model generate",
+    "test:dts": "tsc index.d.ts --noEmit",
+    "test:integration:generate": "brittle -r test/integration/all.js test/integration/*.test.js"
+  },
+  "standard": {
+    "ignore": [
+      "addon/lib/"
+    ]
+  },
+  "files": [
+    "binding.js",
+    "index.js",
+    "addon.js",
+    "prebuilds",
+    "index.d.ts"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/tetherto/qvac-lib-infer-llamacpp-llm.git"
+  },
+  "author": "Tether",
+  "license": "Apache-2.0",
+  "bugs": "https://github.com/tetherto/qvac-lib-infer-llamacpp-llm/issues",
+  "homepage": "https://github.com/tetherto/qvac-lib-infer-llamacpp-llm#readme",
+  "devDependencies": {
+    "@babel/parser": "^7.28.0",
+    "@babel/traverse": "^7.28.0",
+    "@tetherto/qvac-lib-dl-base": "1.0.2",
+    "@tetherto/qvac-lib-dl-filesystem": "1.0.2",
+    "@tetherto/qvac-lib-dl-hyperdrive": "^2.2.1",
+    "@tetherto/qvac-lib-logging": "1.0.1",
+    "hyperswarm": "^4.12.1",
+    "bare-buffer": "^3.1.0",
+    "bare-process": "^4.2.1",
+    "brittle": "^3.16.5",
+    "cmake-bare": "^1.6.5",
+    "cmake-vcpkg": "^1.1.0",
+    "corestore": "^7.4.5",
+    "standard": "^17.0.0"
+  },
+  "dependencies": {
+    "@tetherto/infer-base": "^2.4.0",
+    "@tetherto/qvac-lib-response": "1.0.2",
+    "bare-path": "^3.0.0",
+    "require-asset": "^1.1.0"
+  },
+  "peerDependencies": {
+    "@tetherto/qvac-lib-dl-hyperdrive": "^2.2.1"
+  },
+  "exports": {
+    "./package": "./package.json",
+    ".": "./index.js"
+  },
+  "types": "index.d.ts"
+}