@openzim/libzim 3.5.0 → 4.0.0

package/.env

@@ -1 +1 @@
-LIBZIM_VERSION=9.3.0
+LIBZIM_VERSION=9.4.0

package/Changelog

@@ -1,3 +1,10 @@
+4.0.0
+* NEW: Use libzim 9.4.0
+* FIX: Segmentation Fault issues
+* FIX: Ensure addItem operation is thread-safe / protected by a mutex
+* NEW: Use new Illustration APIs
+* FIX: Add prebuilt libzim for Linux aarch64
+
 3.5.0
 * NEW: Build now supports ARM64 (`aarch64`) and ARM (32‑bit)
 * UPDATE: CI matrix extended with ARM64 Ubuntu runners

package/README.md

@@ -96,8 +96,44 @@ import { Archive, SuggestionSearcher, Searcher } from "@openzim/libzim";
     delete archive;
 })();
 
+## Local Development
+
+### Important Files
+`.env` - Set environment variables for local development. Only LIBZIM_VERSION for now
+`bindings.gyp` - Node-gyp build configuration file
+`src/` - Source code for the Node.js bindings
+`test/` - Test cases
+
+### Setup
+
+```bash
+git clone git@github.com:openzim/node-libzim.git
+cd node-libzim
+
+# Will install dependencies, download libzim binary and build the bindings
+npm run install
+
+# Required in order for local binding and tests to work.
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/download/lib/x86_64-linux-gnu
+```
+
+### Iterating during development
+
+Make your changes in `src/` and then run:
+```bash
+node-gyp rebuild --debug -v && npx jest ./test/zim.test.ts
 ```
 
+### Updating libzim version
+To update the libzim version used, change the `LIBZIM_VERSION` variable in
+`.env` file to the desired version and run:
+```bash
+npm run install
+```
+
+If you are upgrading libzim from a major version you will need to edit the `bundle-libzim.js` file
+and change the `libzim.so*` file names to match the new version.
+
 ## License
 
 [GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see

package/binding.gyp

@@ -34,8 +34,8 @@
                 ],
                 "libraries": [
                   "-Wl,-rpath,'$$ORIGIN'",
-                  "-L<(libzim_dir)/lib/aarch64-linux-gnu",
-                  "<(libzim_dir)/lib/aarch64-linux-gnu/libzim.so.9",
+                  "-L<(libzim_dir)/lib/aarch64-rpi3-linux-gnu",
+                  "<(libzim_dir)/lib/aarch64-rpi3-linux-gnu/libzim.so.9",
                 ],
               }],
               ["libzim_local!='true' and OS=='linux' and target_arch=='arm'", {

package/bundle-libzim.js

@@ -19,7 +19,7 @@ if (isLinux) {
   const rawArch = os.arch();
   let libDir;
   if (rawArch === "arm64") {
-    libDir = "aarch64-linux-gnu";
+    libDir = "aarch64-rpi3-linux-gnu";
   } else if (rawArch === "arm") {
     libDir = "arm-linux-gnueabihf";
   } else {

package/dist/index.d.ts

@@ -1,3 +1,7 @@
+export declare function getClusterCacheMaxSize(): number;
+export declare function getClusterCacheCurrentSize(): number;
+export declare function setClusterCacheMaxSize(nbClusters: number): void;
+
 export class IntegrityCheck {
   static CHECKSUM: symbol;
   static DIRENT_PTRS: symbol;
@@ -104,7 +108,10 @@ export class Creator {
     content: string | ContentProvider,
     mimetype?: string,
   ): void;
-  addIllustration(size: number, content: string | ContentProvider): void;
+  addIllustration(
+    sizeOrInfo: number | IIllustrationInfo,
+    content: string | ContentProvider,
+  ): void;
   addRedirection(
     path: string,
     title: string,
@@ -146,8 +153,35 @@ export interface EntryRange extends Iterable<Entry> {
   offset(start: number, maxResults: number): EntryRange;
 }
 
+export interface IIllustrationInfo {
+  width?: number;
+  height?: number;
+  scale?: number;
+  extraAttributes?: Record<string, string>;
+}
+
+export class IllustrationInfo implements IIllustrationInfo {
+  constructor(info?: IIllustrationInfo | IllustrationInfo);
+  get width(): number;
+  get height(): number;
+  get scale(): number;
+  get extraAttributes(): Record<string, string>;
+  asMetadataItemName(): string;
+  static fromMetadataItemName(name: string): IllustrationInfo;
+}
+
+export class OpenConfig {
+  constructor();
+
+  preloadXapianDb(preload: boolean): this;
+  preloadDirentRanges(nbRanges: number): this;
+
+  get m_preloadXapianDb(): boolean;
+  get m_preloadDirentRanges(): number;
+}
+
 export class Archive {
-  constructor(filepath: string);
+  constructor(filepath: string, config?: OpenConfig);
   get filename(): string;
   get filesize(): number | bigint;
   get allEntryCount(): number;
@@ -158,8 +192,14 @@ export class Archive {
   getMetadata(name: string): string;
   getMetadataItem(name: string): Item;
   get metadataKeys(): string[];
-  getIllustrationItem(size: number): Item;
+  getIllustrationItem(sizeOrInfo?: number | IIllustrationInfo): Item;
   get illustrationSizes(): Set<number>;
+  getIllustrationInfos(
+    width?: number,
+    height?: number,
+    minScale?: number,
+  ): IllustrationInfo[];
+  get illustrationInfos(): IllustrationInfo[];
   getEntryByPath(path_or_idx: string | number): Entry;
   getEntryByTitle(title_or_idx: string | number): Entry;
   getEntryByClusterOrder(idx: number): Entry;
@@ -182,14 +222,9 @@ export class Archive {
   checkIntegrity(checkType: symbol): boolean; // one of IntegrityCheck
   get isMultiPart(): boolean;
   get hasNewNamespaceScheme(): boolean;
-  getClusterCacheMaxSize(): number;
-  getClusterCacheCurrentSize(): number;
-  setClusterCacheMaxSize(nbClusters: number): void;
   getDirentCacheMaxSize(): number;
   getDirentCacheCurrentSize(): number;
   setDirentCacheMaxSize(nbDirents: number): void;
-  getDirentLookupCacheMaxSize(): number;
-  setDirentLookupCacheMaxSize(nbRanges: number): void;
 
   static validate(zimPath: string, checksToRun: symbol[]): boolean; // list of IntegrityCheck
 }

package/dist/index.js

@@ -2,9 +2,11 @@ import bindings from "bindings";
 
 export const {
   Archive,
+  OpenConfig,
   Entry,
   IntegrityCheck,
   Compression,
+  IllustrationInfo,
   Blob,
   Searcher,
   Query,
@@ -14,4 +16,7 @@ export const {
   FileProvider,
   StringItem,
   FileItem,
+  getClusterCacheMaxSize,
+  getClusterCacheCurrentSize,
+  setClusterCacheMaxSize,
 } = bindings("zim_binding");

package/download-libzim.js

@@ -5,7 +5,6 @@ import { mkdirp } from "mkdirp";
 import exec from "exec-then";
 import os from "os";
 import fs from "fs";
-import urlParser from "url";
 
 mkdirp.sync("./download");
 
@@ -46,7 +45,7 @@ const urls = [
 
 for (let url of urls) {
   console.info(`Downloading Libzim from: `, url);
-  const filename = urlParser.parse(url).pathname.split("/").slice(-1)[0];
+  const filename = new URL(url).pathname.split("/").slice(-1)[0];
   const dlFile = `./download/${filename}`;
 
   try {

package/package.json

@@ -2,7 +2,7 @@
   "name": "@openzim/libzim",
   "main": "dist/index.js",
   "types": "dist/index.d.js",
-  "version": "3.5.0",
+  "version": "4.0.0",
   "description": "Libzim bindings for NodeJS",
   "type": "module",
   "scripts": {
@@ -14,8 +14,8 @@
     "build": "node-gyp rebuild -v && npm run bundle",
     "download": "node ./download-libzim.js",
     "bundle": "node ./bundle-libzim.js",
-    "test": "jest --testPathIgnorePatterns=test/dist.test.ts",
-    "test-mem-leak": "node -r ts-node/register test/makeLargeZim.ts",
+    "test": "jest --testPathIgnorePatterns=test/dist.test.ts --testPathIgnorePatterns=dist/",
+    "test-mem-leak": "npx tsx test/makeLargeZim.ts",
     "test:dist": "npm run build && jest",
     "lint": "npx eslint .",
     "lint:fix": "npx eslint . --fix"
@@ -44,31 +44,31 @@
   "homepage": "https://github.com/openzim/node-libzim#readme",
   "gypfile": true,
   "dependencies": {
-    "@types/bindings": "^1.5.1",
-    "@types/jest": "^29.5.12",
-    "@types/node": "^18.0.6",
-    "axios": "^1.6.0",
+    "@types/bindings": "^1.5.5",
+    "@types/jest": "^30.0.0",
+    "@types/node": "^24.9.2",
+    "axios": "^1.13.1",
     "bindings": "^1.5.0",
-    "dotenv": "^16.0.1",
+    "dotenv": "^17.2.3",
     "exec-then": "^1.3.1",
     "mkdirp": "^3.0.1",
-    "node-addon-api": "^8.0.0",
-    "node-gyp": "^10.1.0",
+    "node-addon-api": "^8.5.0",
+    "node-gyp": "^11.5.0",
     "tqdm": "^2.0.3",
-    "ts-node": "^10.9.1",
-    "tsconfig-paths": "^4.0.0"
+    "ts-node": "^10.9.2",
+    "tsconfig-paths": "^4.2.0"
   },
   "devDependencies": {
-    "@faker-js/faker": "^8.4.1",
-    "@typescript-eslint/eslint-plugin": "^5.59.2",
-    "@typescript-eslint/parser": "^5.59.2",
+    "@faker-js/faker": "^10.1.0",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.57.0",
-    "eslint-config-prettier": "^8.10.0",
-    "eslint-plugin-prettier": "^5.1.3",
-    "nyc": "^17.0.0",
-    "prettier": "^3.3.2",
-    "ts-jest": "^29.1.5",
-    "typescript": "^5.1.6"
+    "eslint-config-prettier": "^8.10.2",
+    "eslint-plugin-prettier": "^5.5.4",
+    "nyc": "^17.1.0",
+    "prettier": "^3.6.2",
+    "ts-jest": "^29.4.5",
+    "typescript": "^5.9.3"
   },
   "jest": {
     "preset": "ts-jest/presets/js-with-ts"

package/src/archive.h

@@ -3,27 +3,52 @@
 #include <napi.h>
 #include <zim/archive.h>
 #include <exception>
+#include <iostream>
+#include <map>
 #include <memory>
-#include <sstream>
 #include <string>
 
 #include "entry.h"
+#include "illustration.h"
 #include "item.h"
+#include "openconfig.h"
 
 class Archive : public Napi::ObjectWrap<Archive> {
  public:
   explicit Archive(const Napi::CallbackInfo &info)
       : Napi::ObjectWrap<Archive>(info), archive_{nullptr} {
     Napi::Env env = info.Env();
-    Napi::HandleScope scope(env);
 
     if (info.Length() < 1) {
-      throw Napi::Error::New(info.Env(), "Archive requires arguments filepath");
+      throw Napi::Error::New(env, "Archive requires arguments filepath");
+    }
+
+    if (!info[0].IsString()) {
+      throw Napi::TypeError::New(env,
+                                 "First argument must be a string filepath.");
+    }
+
+    // Archive(filename: string)
+    // Archive(filepath: string, config: OpenConfig)
+    std::string filepath = info[0].As<Napi::String>();
+    zim::OpenConfig config{};
+    if (info[1].IsObject()) {
+      // @note: no bounds checking on info because it returns Undefined when out
+      // of bounds
+      auto obj = info[1].As<Napi::Object>();
+      // Check that the object is an instance of OpenConfig
+      // TODO(kelvinhammond): Update use of Unwrap everywhere to use
+      // InstanceOf and GetConstructor pattern
+      if (OpenConfig::InstanceOf(env, obj)) {
+        config = OpenConfig::Unwrap(obj)->getInternalConfig();
+      } else {
+        throw Napi::TypeError::New(
+            env, "Second argument must be an instance of OpenConfig.");
+      }
     }
 
     try {
-      std::string filepath = info[0].ToString();
-      archive_ = std::make_shared<zim::Archive>(filepath);
+      archive_ = std::make_shared<zim::Archive>(filepath, config);
     } catch (const std::exception &e) {
       throw Napi::Error::New(env, e.what());
     }
@@ -79,13 +104,8 @@ class Archive : public Napi::ObjectWrap<Archive> {
 
   Napi::Value getUuid(const Napi::CallbackInfo &info) {
     try {
-      // TODO(kelvinhammond): convert this to
-      // static_cast<std::string>(archive_->getUuid()) This didn't work when
-      // building because of the below error undefined symbol:
-      // _ZNK3zim4UuidcvNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEv
-      std::ostringstream out;
-      out << archive_->getUuid();
-      return Napi::Value::From(info.Env(), out.str());
+      return Napi::Value::From(info.Env(),
+                               static_cast<std::string>(archive_->getUuid()));
     } catch (const std::exception &err) {
       throw Napi::Error::New(info.Env(), err.what());
     }
@@ -112,7 +132,6 @@ class Archive : public Napi::ObjectWrap<Archive> {
   Napi::Value getMetadataKeys(const Napi::CallbackInfo &info) {
     try {
       auto env = info.Env();
-      Napi::HandleScope scope(env);
       auto res = Napi::Array::New(env);
       size_t idx = 0;
       for (const auto &key : archive_->getMetadataKeys()) {
@@ -125,22 +144,51 @@ class Archive : public Napi::ObjectWrap<Archive> {
   }
 
   Napi::Value getIllustrationItem(const Napi::CallbackInfo &info) {
+    auto env = info.Env();
     try {
-      if (info.Length() > 0) {
+      // getIllustrationItem()
+      if (info.Length() < 1) {
+        return Item::New(env, archive_->getIllustrationItem());
+      }
+
+      // getIllustrationItem(size: number)
+      if (info[0].IsNumber()) {
         auto size = static_cast<unsigned int>(info[0].ToNumber().Uint32Value());
-        return Item::New(info.Env(), archive_->getIllustrationItem(size));
+        return Item::New(env, archive_->getIllustrationItem(size));
       }
-      return Item::New(info.Env(), archive_->getIllustrationItem());
+
+      /// getIllustration(illusInfo: object)
+      if (info[0].IsObject()) {
+        auto obj = info[0].As<Napi::Object>();
+
+        // getIllustrationItem(illusInfo: IllustrationInfo)
+        if (IllustrationInfo::InstanceOf(env, obj)) {
+          auto illusInfo =
+              IllustrationInfo::Unwrap(obj)->getInternalIllustrationInfo();
+          return Item::New(env, archive_->getIllustrationItem(illusInfo));
+        }
+
+        // getIllustrationItem(illusInfo: object)
+        auto illusInfo = IllustrationInfo::infoFrom(obj);
+        return Item::New(env, archive_->getIllustrationItem(illusInfo));
+      }
+
+      throw Napi::TypeError::New(
+          env,
+          "getIllustrationItem expects no arguments, a number size, or an "
+          "IllustrationInfo object.");
     } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
+      throw Napi::Error::New(env, err.what());
     }
   }
 
   Napi::Value getIllustrationSizes(const Napi::CallbackInfo &info) {
+    // Warn: Deprecated, use illustrationSizes property instead.
+    std::cerr << "Warning: getIllustrationSizes() is deprecated, use "
+                 "illustrationSizes property instead."
+              << std::endl;
+    auto env = info.Env();
     try {
-      auto env = info.Env();
-      Napi::HandleScope scope(env);
-
       // returns a native Set object
       auto SetConstructor = env.Global().Get("Set").As<Napi::Function>();
       auto result = SetConstructor.New({});
@@ -150,24 +198,52 @@ class Archive : public Napi::ObjectWrap<Archive> {
       }
       return result;
     } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
+      throw Napi::Error::New(env, err.what());
     }
   }
 
+  Napi::Value getIllustrationInfos(const Napi::CallbackInfo &info) {
+    auto env = info.Env();
+
+    // getIllustrationInfos(w: number, h: number, minScale: number)
+    if (info.Length() >= 3) {
+      auto w = info[0].ToNumber().Uint32Value();
+      auto h = info[1].ToNumber().Uint32Value();
+      auto minScale = info[2].ToNumber().FloatValue();
+
+      auto infos = archive_->getIllustrationInfos(w, h, minScale);
+      auto array = Napi::Array::New(env, infos.size());
+      for (size_t i = 0; i < infos.size(); i++) {
+        array.Set(i, IllustrationInfo::New(env, infos[i]));
+      }
+      return array;
+    }
+
+    // getIllustrationInfos()
+    auto infos = archive_->getIllustrationInfos();
+    auto array = Napi::Array::New(env, infos.size());
+    for (size_t i = 0; i < infos.size(); i++) {
+      array.Set(i, IllustrationInfo::New(env, infos[i]));
+    }
+
+    return array;
+  }
+
   Napi::Value getEntryByPath(const Napi::CallbackInfo &info) {
+    auto env = info.Env();
     try {
       if (info[0].IsNumber()) {
         auto &&idx = info[0].ToNumber();
-        return Entry::New(info.Env(), archive_->getEntryByPath(idx));
+        return Entry::New(env, archive_->getEntryByPath(idx));
       } else if (info[0].IsString()) {
         auto &&path = info[0].ToString();
-        return Entry::New(info.Env(), archive_->getEntryByPath(path));
+        return Entry::New(env, archive_->getEntryByPath(path));
       }
 
       throw Napi::Error::New(
-          info.Env(), "Entry index must be a string (path) or number (index).");
+          env, "Entry index must be a string (path) or number (index).");
     } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
+      throw Napi::Error::New(env, err.what());
     }
   }
 
@@ -386,12 +462,12 @@ class Archive : public Napi::ObjectWrap<Archive> {
   }
 
   Napi::Value checkIntegrity(const Napi::CallbackInfo &info) {
+    auto env = info.Env();
     try {
-      auto env = info.Env();
       const auto &&checkType = IntegrityCheck::symbolToEnum(env, info[0]);
       return Napi::Value::From(env, archive_->checkIntegrity(checkType));
     } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
+      throw Napi::Error::New(env, err.what());
     }
   }
 
@@ -411,35 +487,6 @@ class Archive : public Napi::ObjectWrap<Archive> {
     }
   }
 
-  Napi::Value getClusterCacheMaxSize(const Napi::CallbackInfo &info) {
-    try {
-      return Napi::Value::From(info.Env(), archive_->getClusterCacheMaxSize());
-    } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
-    }
-  }
-
-  Napi::Value getClusterCacheCurrentSize(const Napi::CallbackInfo &info) {
-    try {
-      return Napi::Value::From(info.Env(),
-                               archive_->getClusterCacheCurrentSize());
-    } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
-    }
-  }
-
-  void setClusterCacheMaxSize(const Napi::CallbackInfo &info) {
-    try {
-      if (!info[0].IsNumber()) {
-        throw Napi::Error::New(info.Env(), "Expected a number");
-      }
-      auto nbClusters = info[0].As<Napi::Number>().Uint32Value();
-      archive_->setClusterCacheMaxSize(nbClusters);
-    } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
-    }
-  }
-
   Napi::Value getDirentCacheMaxSize(const Napi::CallbackInfo &info) {
     try {
       return Napi::Value::From(info.Env(), archive_->getDirentCacheMaxSize());
@@ -459,8 +506,9 @@ class Archive : public Napi::ObjectWrap<Archive> {
 
   void setDirentCacheMaxSize(const Napi::CallbackInfo &info) {
     try {
-      if (!info[0].IsNumber()) {
-        throw Napi::Error::New(info.Env(), "Expected a number");
+      if (info.Length() < 1 || !info[0].IsNumber()) {
+        throw Napi::TypeError::New(info.Env(),
+                                   "setDirentCacheMaxSize expects a number");
       }
       auto nbDirents = info[0].As<Napi::Number>().Uint32Value();
       archive_->setDirentCacheMaxSize(nbDirents);
@@ -469,30 +517,8 @@ class Archive : public Napi::ObjectWrap<Archive> {
     }
   }
 
-  Napi::Value getDirentLookupCacheMaxSize(const Napi::CallbackInfo &info) {
-    try {
-      return Napi::Value::From(info.Env(),
-                               archive_->getDirentLookupCacheMaxSize());
-    } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
-    }
-  }
-
-  void setDirentLookupCacheMaxSize(const Napi::CallbackInfo &info) {
-    try {
-      if (!info[0].IsNumber()) {
-        throw Napi::Error::New(info.Env(), "Expected a number");
-      }
-      auto nbRanges = info[0].As<Napi::Number>().Uint32Value();
-      archive_->setDirentLookupCacheMaxSize(nbRanges);
-    } catch (const std::exception &err) {
-      throw Napi::Error::New(info.Env(), err.what());
-    }
-  }
-
   static Napi::Value validate(const Napi::CallbackInfo &info) {
     Napi::Env env = info.Env();
-    Napi::HandleScope scope(env);
     try {
       if (info.Length() < 2) {
         throw Napi::Error::New(
@@ -524,7 +550,6 @@ class Archive : public Napi::ObjectWrap<Archive> {
 
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors &constructors) {
-    Napi::HandleScope scope(env);
     Napi::Function func = DefineClass(
         env, "Archive",
         {
@@ -542,6 +567,10 @@ class Archive : public Napi::ObjectWrap<Archive> {
                 "getIllustrationItem"),
             InstanceAccessor<&Archive::getIllustrationSizes>(
                 "illustrationSizes"),
+            InstanceMethod<&Archive::getIllustrationInfos>(
+                "getIllustrationInfos"),
+            InstanceAccessor<&Archive::getIllustrationInfos>(
+                "illustrationInfos"),
             InstanceMethod<&Archive::getEntryByPath>("getEntryByPath"),
             InstanceMethod<&Archive::getEntryByTitle>("getEntryByTitle"),
             InstanceMethod<&Archive::getEntryByClusterOrder>(
@@ -564,22 +593,12 @@ class Archive : public Napi::ObjectWrap<Archive> {
             InstanceAccessor<&Archive::getChecksum>("checksum"),
             InstanceMethod<&Archive::check>("check"),
             InstanceMethod<&Archive::checkIntegrity>("checkIntegrity"),
-            InstanceMethod<&Archive::getClusterCacheMaxSize>(
-                "getClusterCacheMaxSize"),
-            InstanceMethod<&Archive::getClusterCacheCurrentSize>(
-                "getClusterCacheCurrentSize"),
-            InstanceMethod<&Archive::setClusterCacheMaxSize>(
-                "setClusterCacheMaxSize"),
             InstanceMethod<&Archive::getDirentCacheMaxSize>(
                 "getDirentCacheMaxSize"),
             InstanceMethod<&Archive::getDirentCacheCurrentSize>(
                 "getDirentCacheCurrentSize"),
             InstanceMethod<&Archive::setDirentCacheMaxSize>(
                 "setDirentCacheMaxSize"),
-            InstanceMethod<&Archive::getDirentLookupCacheMaxSize>(
-                "getDirentLookupCacheMaxSize"),
-            InstanceMethod<&Archive::setDirentLookupCacheMaxSize>(
-                "setDirentLookupCacheMaxSize"),
             InstanceAccessor<&Archive::isMultiPart>("isMultiPart"),
             InstanceAccessor<&Archive::hasNewNamespaceScheme>(
                 "hasNewNamespaceScheme"),