@fugood/llama.node 0.0.1-alpha.1

package/CMakeLists.txt

@@ -0,0 +1,85 @@
+cmake_minimum_required(VERSION 3.15)
+cmake_policy(SET CMP0091 NEW)
+cmake_policy(SET CMP0042 NEW)
+
+project (llama-node)
+
+set(CMAKE_CXX_STANDARD 17)
+
+if(NOT DEFINED napi_build_version)
+  set(napi_build_version 6)
+endif()
+add_definitions(-DNAPI_VERSION=${napi_build_version})
+message(STATUS "NAPI_VERSION: ${napi_build_version}")
+
+set(CMAKE_SYSTEM_PROCESSOR ${NODE_ARCH})
+string(TOLOWER ${CMAKE_SYSTEM_NAME} PLATFORM)
+string(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} ARCH)
+
+# normalize platform to nodejs
+string(REPLACE "windows" "win32" PLATFORM ${PLATFORM})
+
+# normalize arch to nodejs: 'arm', 'arm64', 'ia32', 'loong64', 'mips', 'mipsel', 'ppc', 'ppc64', 'riscv64', 's390', 's390x', and 'x64'.
+string(REPLACE "amd64" "x64" ARCH ${ARCH})
+string(REPLACE "x86_64" "x64" ARCH ${ARCH})
+string(REPLACE "i686" "ia32" ARCH ${ARCH})
+string(REPLACE "i386" "ia32" ARCH ${ARCH})
+string(REPLACE "armv7l" "arm" ARCH ${ARCH})
+string(REPLACE "arm" "arm" ARCH ${ARCH})
+string(REPLACE "arm64ex" "arm64" ARCH ${ARCH})
+string(REPLACE "aarch64" "arm64" ARCH ${ARCH})
+
+if(DEFINED VARIANT)
+  set(VARIANT -${VARIANT})
+else()
+  set(VARIANT "")
+endif()
+
+set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/bin/${PLATFORM}${VARIANT}/${ARCH})
+
+message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
+message(STATUS "Platform: ${PLATFORM}")
+message(STATUS "Architecture: ${ARCH}")
+message(STATUS "PLATFORM_BINARY_DIR: ${PLATFORM_BINARY_DIR}")
+
+# set strip flags
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+  if(UNIX OR MINGW)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -s")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -s")
+  elseif(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /s")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /s")
+  endif()
+endif()
+
+include_directories(${CMAKE_JS_INC})
+
+# flags: -fPIC
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(LLAMA_STATIC ON CACHE BOOL "Build llama as static library")
+add_subdirectory("src/llama.cpp")
+
+file(GLOB SOURCE_FILES "src/addons.cpp")
+
+add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
+set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
+target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common)
+
+if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
+  # Generate node.lib
+  execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
+  # copy target to bin folder
+  get_filename_component(CMAKE_JS_NODELIB_TARGET_NAME ${CMAKE_JS_NODELIB_TARGET} NAME)
+  add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_JS_NODELIB_TARGET} ${PLATFORM_BINARY_DIR}/${CMAKE_JS_NODELIB_TARGET_NAME}
+    COMMENT "Copying to bin folder"
+  )
+endif()
+
+# copy target to bin folder
+add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${PROJECT_NAME}> ${PLATFORM_BINARY_DIR}/$<TARGET_FILE_NAME:${PROJECT_NAME}>
+  COMMENT "Copying to bin folder"
+)

package/README.md

@@ -0,0 +1,56 @@
+# llama.node
+
+Node binding of [llama.cpp](https://github.com/ggerganov/llama.cpp).
+
+[llama.cpp](https://github.com/ggerganov/llama.cpp): Inference of [LLaMA](https://arxiv.org/abs/2302.13971) model in pure C/C++
+
+## Installation
+
+```sh
+npm install llama.node
+```
+
+## Usage
+
+```js
+import { loadModel } from 'llama.node'
+
+// Initial a Llama context with the model (may take a while)
+const context = loadModel({
+  model: 'path/to/gguf/model',
+  use_mlock: true,
+  n_ctx: 2048,
+  n_gpu_layers: 1, // > 0: enable Metal on iOS
+  // embedding: true, // use embedding
+})
+
+// Do completion
+const { text, timings } = await context.completion(
+  {
+    prompt: 'This is a conversation between user and llama, a friendly chatbot. respond in simple markdown.\n\nUser: Hello!\nLlama:',
+    n_predict: 100,
+    stop: ['</s>', 'Llama:', 'User:'],
+    // n_threads: 4,
+  },
+  (data) => {
+    // This is a partial completion callback
+    const { token } = data
+  },
+)
+console.log('Result:', text)
+```
+
+## License
+
+MIT
+
+---
+
+<p align="center">
+  <a href="https://bricks.tools">
+    <img width="90px" src="https://avatars.githubusercontent.com/u/17320237?s=200&v=4">
+  </a>
+  <p align="center">
+    Built and maintained by <a href="https://bricks.tools">BRICKS</a>.
+  </p>
+</p>

package/lib/binding.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadModule = void 0;
+const loadModule = (variant) => {
+    try {
+        if (variant) {
+            return require(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`);
+        }
+    }
+    catch (_a) { } // ignore errors and try the common path
+    return require(`../bin/${process.platform}/${process.arch}/llama-node.node`);
+};
+exports.loadModule = loadModule;

package/lib/binding.ts

@@ -0,0 +1,57 @@
+export type LlamaModelOptions = {
+  model: string
+  embedding?: boolean
+  n_ctx?: number
+  n_batch?: number
+  n_threads?: number
+  n_gpu_layers?: number
+  use_mlock?: boolean
+  use_mmap?: boolean
+}
+
+export type LlamaCompletionOptions = {
+  prompt: string
+  n_samples?: number
+  temperature?: number
+  top_k?: number
+  top_p?: number
+  repetition_penalty?: number
+  n_predict?: number
+  max_length?: number
+  max_tokens?: number
+  seed?: number
+  stop?: string[]
+}
+
+export type LlamaCompletionResult = {
+  text: string
+  tokens_predicted: number
+  tokens_evaluated: number
+  truncated: boolean
+}
+
+export type LlamaCompletionToken = {
+  token: string
+}
+
+export interface LlamaContext {
+  new (options: LlamaModelOptions): LlamaContext
+  getSystemInfo(): string
+  completion(options: LlamaCompletionOptions, callback?: (token: LlamaCompletionToken) => void): Promise<LlamaCompletionResult>
+  stopCompletion(): void
+  saveSession(path: string): Promise<void>
+  loadSession(path: string): Promise<void>
+}
+
+export interface Module {
+  LlamaContext: LlamaContext
+}
+
+export const loadModule = (variant?: string): Module => {
+  try {
+    if (variant) {
+      return require(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
+    }
+  } catch {} // ignore errors and try the common path
+  return require(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
+}

package/lib/index.js

@@ -0,0 +1,24 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadModel = void 0;
+const binding_1 = require("./binding");
+__exportStar(require("./binding"), exports);
+const loadModel = (options) => {
+    const { LlamaContext } = (0, binding_1.loadModule)(options.lib_variant);
+    return new LlamaContext(options);
+};
+exports.loadModel = loadModel;

package/lib/index.ts

@@ -0,0 +1,13 @@
+import { loadModule, LlamaModelOptions } from './binding'
+import type { LlamaContext } from './binding'
+
+export * from './binding'
+
+export interface LlamaModelOptionsExtended extends LlamaModelOptions {
+  lib_variant?: string
+}
+
+export const loadModel = (options: LlamaModelOptionsExtended): LlamaContext => {
+    const { LlamaContext } = loadModule(options.lib_variant)
+    return new LlamaContext(options)
+}

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@fugood/llama.node",
+  "access": "public",
+  "version": "0.0.1-alpha.1",
+  "description": "Llama.cpp for Node.js",
+  "main": "lib/index.js",
+  "scripts": {
+    "test": "jest",
+    "build": "tsc",
+    "prepack": "yarn build",
+    "build-native": "cmake-js compile",
+    "clean": "rimraf build",
+    "prepare": "husky",
+    "commitlint": "commitlint --edit"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mybigday/llama.node.git"
+  },
+  "keywords": [
+    "llama",
+    "llm",
+    "ai",
+    "genai"
+  ],
+  "author": "Hans <hans.chen@bricks.tools>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/mybigday/llama.node/issues"
+  },
+  "homepage": "https://github.com/mybigday/llama.node#readme",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org"        
+  },
+  "binary": {
+    "napi_versions": [
+      5
+    ]
+  },
+  "files": [
+    "bin/**/*",
+    "scripts/*.js",
+    "scripts/*.ts",
+    "src/**/*.{c,cc,cpp,h,hh,hpp,txt,cmake}",
+    "lib/*.js",
+    "lib/*.ts",
+    "CMakeLists.txt"
+  ],
+  "devDependencies": {
+    "@babel/preset-env": "^7.24.4",
+    "@babel/preset-typescript": "^7.24.1",
+    "@commitlint/cli": "^19.3.0",
+    "@commitlint/config-conventional": "^19.2.2",
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.12.7",
+    "cmake-js": "^7.3.0",
+    "husky": "^9.0.11",
+    "jest": "^29.7.0",
+    "rimraf": "^5.0.5",
+    "typescript": "^5.4.5"
+  },
+  "dependencies": {
+    "node-addon-api": "^8.0.0"
+  }
+}