@openzim/libzim 3.5.0 → 4.0.0

package/src/blob.h

@@ -12,13 +12,11 @@
 class Blob : public Napi::ObjectWrap<Blob> {
  public:
   explicit Blob(const Napi::CallbackInfo &info)
-      : Napi::ObjectWrap<Blob>(info), blob_{std::make_shared<zim::Blob>()} {
-    Napi::Env env = info.Env();
-    Napi::HandleScope scope(env);
-
-    if (info[0].IsExternal()) {  // handle internal zim::Blob
-      blob_ = std::make_shared<zim::Blob>(
-          *info[0].As<Napi::External<zim::Blob>>().Data());
+      : Napi::ObjectWrap<Blob>(info), blob_{} {
+    if (info[0].IsExternal()) {
+      // handle internal zim::Blob
+      // Copy blob (shared_ptr) from external
+      blob_ = zim::Blob(*info[0].As<Napi::External<zim::Blob>>().Data());
     } else if (info.Length() > 0) {  // use refcontent_ and copy content
       // TODO(kelvinhammond): avoid copying content somehow in certain scenarios
       // if possible Maybe use a reference object??? What is the lifecycle of
@@ -38,28 +36,31 @@ class Blob : public Napi::ObjectWrap<Blob> {
         data = std::shared_ptr<char>(new char[size],
                                      std::default_delete<char[]>());
         memcpy(data.get(), buf.Data(), size);
-      } else {                                      // all others toString()
-        auto str = info[0].ToString().Utf8Value();  // coerce to string
+      } else if (info[0].IsString()) {  // all others toString()
+        auto str = info[0].As<Napi::String>().Utf8Value();  // coerce to string
         size = str.size();
         data = std::shared_ptr<char>(new char[size],
                                      std::default_delete<char[]>());
         memcpy(data.get(), str.c_str(), size);
+      } else {
+        throw Napi::Error::New(
+            info.Env(),
+            "Blob constructor expects an ArrayBuffer, Buffer, or String");
       }
 
-      blob_ = std::make_shared<zim::Blob>(data, size);  // blob takes ownership
+      blob_ = zim::Blob(data, size);  // blob takes ownership
     }
   }
 
   static Napi::Object New(Napi::Env env, zim::Blob &blob) {
     auto external = Napi::External<zim::Blob>::New(env, &blob);
-    auto &constructor = env.GetInstanceData<ModuleConstructors>()->blob;
-    return constructor.New({external});
+    return GetConstructor(env).New({external});
   }
 
   Napi::Value getData(const Napi::CallbackInfo &info) {
     try {
       // TODO(kelvinhammond): find a way to have a readonly buffer in NodeJS
-      return Napi::Buffer<char>::Copy(info.Env(), blob_->data(), blob_->size());
+      return Napi::Buffer<char>::Copy(info.Env(), blob_.data(), blob_.size());
     } catch (const std::exception &err) {
       throw Napi::Error::New(info.Env(), err.what());
     }
@@ -67,7 +68,7 @@ class Blob : public Napi::ObjectWrap<Blob> {
 
   Napi::Value toString(const Napi::CallbackInfo &info) {
     try {
-      return Napi::Value::From(info.Env(), (std::string)*blob_);
+      return Napi::Value::From(info.Env(), (std::string)blob_);
     } catch (const std::exception &err) {
       throw Napi::Error::New(info.Env(), err.what());
     }
@@ -75,15 +76,27 @@ class Blob : public Napi::ObjectWrap<Blob> {
 
   Napi::Value getSize(const Napi::CallbackInfo &info) {
     try {
-      return Napi::Value::From(info.Env(), blob_->size());
+      return Napi::Value::From(info.Env(), blob_.size());
     } catch (const std::exception &err) {
       throw Napi::Error::New(info.Env(), err.what());
     }
   }
 
+  static bool InstanceOf(Napi::Env env, Napi::Value value) {
+    if (!value.IsObject()) {
+      return false;
+    }
+    Napi::Object obj = value.As<Napi::Object>();
+    Napi::FunctionReference &constructor = GetConstructor(env);
+    return obj.InstanceOf(constructor.Value());
+  }
+
+  static Napi::FunctionReference &GetConstructor(Napi::Env env) {
+    return env.GetInstanceData<ModuleConstructors>()->blob;
+  }
+
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors &constructors) {
-    Napi::HandleScope scope(env);
     Napi::Function func =
         DefineClass(env, "Blob",
                     {
@@ -97,8 +110,8 @@ class Blob : public Napi::ObjectWrap<Blob> {
   }
 
   // internal module methods
-  std::shared_ptr<zim::Blob> blob() const { return blob_; }
+  const zim::Blob &blob() const { return blob_; }
 
  private:
-  std::shared_ptr<zim::Blob> blob_;
+  zim::Blob blob_;
 };

package/src/common.h

@@ -17,6 +17,8 @@ using CompressionMap =
 
 struct ModuleConstructors {
   Napi::FunctionReference archive;
+  Napi::FunctionReference openConfig;
+  Napi::FunctionReference illustrationInfo;
   Napi::FunctionReference entry;
   Napi::FunctionReference item;
   Napi::FunctionReference blob;
@@ -60,7 +62,6 @@ class Compression : public Napi::ObjectWrap<Compression> {
 
     auto& compressionMap =
         env.GetInstanceData<ModuleConstructors>()->compressionMap;
-    Napi::HandleScope scope(env);
     for (const auto& [bit, symbolRef] : compressionMap) {
       if (!symbolRef.IsEmpty() && symbolRef.Value() == value) {
         return bit;
@@ -71,8 +72,6 @@ class Compression : public Napi::ObjectWrap<Compression> {
 
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors& constructors) {
-    Napi::HandleScope scope(env);
-
     constexpr auto attrs =
         static_cast<napi_property_attributes>(napi_default | napi_enumerable);
     std::vector<PropertyDescriptor> props;
@@ -107,7 +106,6 @@ class IntegrityCheck : public Napi::ObjectWrap<IntegrityCheck> {
     }
     auto& integrityCheckMap =
         env.GetInstanceData<ModuleConstructors>()->integrityCheckMap;
-    Napi::HandleScope scope(env);
     for (const auto& [bit, symbolRef] : integrityCheckMap) {
       if (!symbolRef.IsEmpty() && symbolRef.Value() == value) {
         return bit;
@@ -118,8 +116,6 @@ class IntegrityCheck : public Napi::ObjectWrap<IntegrityCheck> {
 
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors& constructors) {
-    Napi::HandleScope scope(env);
-
     constexpr auto attrs =
         static_cast<napi_property_attributes>(napi_default | napi_enumerable);
     std::vector<PropertyDescriptor> props;

package/src/contentProvider.h

@@ -6,7 +6,9 @@
 #include <exception>
 #include <functional>
 #include <future>
+#include <iostream>
 #include <memory>
+#include <string>
 #include <string_view>
 #include <thread>
 #include <utility>
@@ -14,31 +16,92 @@
 #include "blob.h"
 #include "common.h"
 
+/**
+ * Thread Safe Function wrapper for calling the ContentProvider.feed function
+ * asynchronously from libzim
+ */
+class FeedTSFN {
+ public:
+  using BlobPtr = zim::Blob;
+
+  FeedTSFN() = delete;
+
+  FeedTSFN(Napi::Env &env, Napi::Function &feedFunc) {
+    tsfn_ = TSFN::New(env,
+                      feedFunc,    // JavaScript function called asynchronously
+                      "FeedTSFN",  // name
+                      0,           // max queue size (0 = unlimited).
+                      1,           // initial thread count
+                      nullptr);    // context
+  }
+
+  ~FeedTSFN() { tsfn_.Release(); }
+
+  BlobPtr feed() {
+    try {
+      DataType promise;
+      auto future = promise.get_future();
+      tsfn_.NonBlockingCall(&promise);
+      return future.get();
+    } catch (const std::exception &e) {
+      std::cerr << "FeedTSFN feed() exception: " << e.what() << std::endl;
+      throw std::runtime_error(std::string("Error in FeedTSFN feed(): ") +
+                               e.what());
+    }
+  }
+
+ private:
+  using DataType = std::promise<BlobPtr>;
+  using Context = void;
+
+  static void CallJs(Napi::Env env, Napi::Function callback, Context *context,
+                     DataType *data) {
+    // Is the JavaScript environment still available to call into, eg. the TSFN
+    // is not aborted
+    if (env != nullptr) {
+      try {
+        // call feed(): object
+        auto result = callback.Call({});
+        if (Blob::InstanceOf(env, result)) {
+          auto blob = Blob::Unwrap(result.As<Napi::Object>())->blob();
+          // Note: Cannot move, blob could be used in nodejs world still
+          data->set_value(blob);
+        } else {
+          data->set_exception(std::make_exception_ptr(std::runtime_error(
+              "Expected an object of type Blob from feed()")));
+        }
+      } catch (const std::exception &e) {
+        data->set_exception(std::make_exception_ptr(e));
+      }
+    } else {
+      data->set_exception(std::make_exception_ptr(
+          std::runtime_error("Environment is shut down")));
+    }
+  }
+
+  using TSFN = Napi::TypedThreadSafeFunction<Context, DataType, CallJs>;
+
+ private:
+  TSFN tsfn_;
+};
+
 /**
  * Wraps the js world ObjectWrap and Objects to a proper content provider for
  * use with libzim
  */
 class ContentProviderWrapper : public zim::writer::ContentProvider {
  public:
-  explicit ContentProviderWrapper(Napi::Env env, const Napi::Object &provider)
-      : MAIN_THREAD_ID{} {
-    MAIN_THREAD_ID = std::this_thread::get_id();
+  explicit ContentProviderWrapper(Napi::Env env, const Napi::Object &provider) {
+    size_ = parseSize(provider.Get("size"));
 
     if (!provider.Get("feed").IsFunction()) {
       throw std::runtime_error("ContentProvider.feed must be a function.");
     }
 
     auto feedFunc = provider.Get("feed").As<Napi::Function>();
-    feed_ = Napi::Persistent(feedFunc);
-    size_ = parseSize(provider.Get("size"));
-    provider_ = Napi::Persistent(provider);
-
-    tsfn_ = Napi::ThreadSafeFunction::New(env, feedFunc,
-                                          "getContentProvider.feed", 0, 1);
+    feedTSFN_ = std::make_unique<FeedTSFN>(env, feedFunc);
   }
 
-  ~ContentProviderWrapper() { tsfn_.Release(); }
-
   zim::size_type getSize() const override { return size_; }
 
   /** Parse the size, supports BigInt */
@@ -60,46 +123,13 @@ class ContentProviderWrapper : public zim::writer::ContentProvider {
     return static_cast<uint64_t>(val);
   }
 
-  zim::Blob feed() override {
-    if (MAIN_THREAD_ID == std::this_thread::get_id()) {
-      // on main thread for some reason, do it here
-      auto blobObj = feed_.Call(provider_.Value(), {});
-      if (!blobObj.IsObject()) {
-        throw std::runtime_error("ContentProvider.feed must return a blob");
-      }
-      auto blob = Napi::ObjectWrap<Blob>::Unwrap(blobObj.ToObject());
-      return *(blob->blob());
-    }
-
-    // called from a thread
-    std::promise<zim::Blob> promise;
-    auto future = promise.get_future();
-
-    auto callback = [&promise, this](Napi::Env env, Napi::Function feedFunc) {
-      auto blobObj = feedFunc.Call(provider_.Value(), {});
-      if (!blobObj.IsObject()) {
-        throw std::runtime_error("ContentProvider.feed must return a blob");
-      }
-      auto blob = Napi::ObjectWrap<Blob>::Unwrap(blobObj.ToObject());
-      promise.set_value(*(blob->blob()));
-    };
-
-    auto status = tsfn_.BlockingCall(callback);
-    if (status != napi_ok) {
-      throw std::runtime_error("Error calling ThreadSafeFunction");
-    }
-
-    return future.get();
-  }
+  zim::Blob feed() override { return feedTSFN_->feed(); }
 
  private:
-  // track the main thread
-  std::thread::id MAIN_THREAD_ID;
-  // js world reference, could be an ObjectWrap provider or custom js object
-  Napi::ObjectReference provider_;
-  Napi::FunctionReference feed_;
-  Napi::ThreadSafeFunction tsfn_;
   zim::size_type size_;
+  // Unique pointer so that it isn't copied and the destructor will only be
+  // called once
+  std::unique_ptr<FeedTSFN> feedTSFN_;
 };
 
 class StringProvider : public Napi::ObjectWrap<StringProvider> {
@@ -144,6 +174,11 @@ class StringProvider : public Napi::ObjectWrap<StringProvider> {
   }
 
   Napi::Value getSize(const Napi::CallbackInfo &info) {
+    if (!provider_) {
+      throw Napi::Error::New(
+          info.Env(), "StringProvider has been moved and is no longer valid.");
+    }
+
     try {
       return Napi::Value::From(info.Env(), provider_->getSize());
     } catch (const std::exception &err) {
@@ -152,8 +187,12 @@ class StringProvider : public Napi::ObjectWrap<StringProvider> {
   }
 
   Napi::Value feed(const Napi::CallbackInfo &info) {
+    if (!provider_) {
+      throw Napi::Error::New(
+          info.Env(), "StringProvider has been moved and is no longer valid.");
+    }
+
     try {
-      // TODO(kelvinhammond): need a way to move this to avoid copying
       auto blob = provider_->feed();
       return Blob::New(info.Env(), blob);
     } catch (const std::exception &err) {
@@ -161,9 +200,21 @@ class StringProvider : public Napi::ObjectWrap<StringProvider> {
     }
   }
 
+  static bool InstanceOf(Napi::Env env, Napi::Value value) {
+    if (!value.IsObject()) {
+      return false;
+    }
+    Napi::Object obj = value.As<Napi::Object>();
+    Napi::FunctionReference &constructor = GetConstructor(env);
+    return obj.InstanceOf(constructor.Value());
+  }
+
+  static Napi::FunctionReference &GetConstructor(Napi::Env env) {
+    return env.GetInstanceData<ModuleConstructors>()->stringProvider;
+  }
+
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors &constructors) {
-    Napi::HandleScope scope(env);
     Napi::Function func =
         DefineClass(env, "StringProvider",
                     {
@@ -176,6 +227,11 @@ class StringProvider : public Napi::ObjectWrap<StringProvider> {
     constructors.stringProvider = Napi::Persistent(func);
   }
 
+  // Internal use only
+  std::unique_ptr<zim::writer::StringProvider> &&unwrapProvider() {
+    return std::move(provider_);
+  }
+
  private:
   std::unique_ptr<zim::writer::StringProvider> provider_;
 };
@@ -220,6 +276,11 @@ class FileProvider : public Napi::ObjectWrap<FileProvider> {
   }
 
   Napi::Value getSize(const Napi::CallbackInfo &info) {
+    if (!provider_) {
+      throw Napi::Error::New(
+          info.Env(), "FileProvider has been moved and is no longer valid.");
+    }
+
     try {
       return Napi::Value::From(info.Env(), provider_->getSize());
     } catch (const std::exception &err) {
@@ -228,8 +289,12 @@ class FileProvider : public Napi::ObjectWrap<FileProvider> {
   }
 
   Napi::Value feed(const Napi::CallbackInfo &info) {
+    if (!provider_) {
+      throw Napi::Error::New(
+          info.Env(), "FileProvider has been moved and is no longer valid.");
+    }
+
     try {
-      // TODO(kelvinhammond): need a way to move this to avoid copying
       auto blob = provider_->feed();
       return Blob::New(info.Env(), blob);
     } catch (const std::exception &err) {
@@ -237,9 +302,21 @@ class FileProvider : public Napi::ObjectWrap<FileProvider> {
     }
   }
 
+  static bool InstanceOf(Napi::Env env, Napi::Value value) {
+    if (!value.IsObject()) {
+      return false;
+    }
+    Napi::Object obj = value.As<Napi::Object>();
+    Napi::FunctionReference &constructor = GetConstructor(env);
+    return obj.InstanceOf(constructor.Value());
+  }
+
+  static Napi::FunctionReference &GetConstructor(Napi::Env env) {
+    return env.GetInstanceData<ModuleConstructors>()->fileProvider;
+  }
+
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors &constructors) {
-    Napi::HandleScope scope(env);
     Napi::Function func =
         DefineClass(env, "FileProvider",
                     {
@@ -252,6 +329,11 @@ class FileProvider : public Napi::ObjectWrap<FileProvider> {
     constructors.fileProvider = Napi::Persistent(func);
   }
 
+  // Internal use only
+  std::unique_ptr<zim::writer::FileProvider> &&unwrapProvider() {
+    return std::move(provider_);
+  }
+
  private:
   std::unique_ptr<zim::writer::FileProvider> provider_;
 };

package/src/creator.h

@@ -1,15 +1,19 @@
 #pragma once
 
 #include <napi.h>
+#include <zim/illustration.h>
 #include <zim/writer/creator.h>
 
 #include <exception>
 #include <functional>
+#include <iostream>
+#include <map>
 #include <memory>
 #include <string>
 #include <utility>
 
 #include "common.h"
+#include "illustration.h"
 #include "writerItem.h"
 
 // Handles creator_->finishZimCreation() operations in the background off the
@@ -24,7 +28,14 @@ class CreatorAsyncWorker : public Napi::AsyncWorker {
 
   ~CreatorAsyncWorker() {}
 
-  void Execute() override { creator_->finishZimCreation(); }
+  void Execute() override {
+    try {
+      creator_->finishZimCreation();
+    } catch (const std::exception &e) {
+      std::cerr << "Error: finishZimCreation failed: " << e.what() << std::endl;
+      SetError(e.what());
+    }
+  }
 
   void OnOK() override {
     auto env = Env();
@@ -54,7 +65,15 @@ class AddItemAsyncWorker : public Napi::AsyncWorker {
 
   Napi::Promise Promise() const { return promise_.Promise(); };
 
-  void Execute() override { creator_->addItem(item_); }
+  void Execute() override {
+    try {
+      creator_->addItem(item_);
+    } catch (const std::exception &e) {
+      std::cerr << "Error: AddItemAsyncWorker failed: " << e.what()
+                << std::endl;
+      SetError(e.what());
+    }
+  }
 
   void OnOK() override {
     auto env = Env();
@@ -163,16 +182,11 @@ class Creator : public Napi::ObjectWrap<Creator> {
         throw Napi::Error::New(env, "addItem requires an item object");
       }
 
-      const auto &stringItem =
-          env.GetInstanceData<ModuleConstructors>()->stringItem.Value();
-      const auto &fileItem =
-          env.GetInstanceData<ModuleConstructors>()->fileItem.Value();
-
       std::shared_ptr<zim::writer::Item> item{};
-      auto obj = info[0].ToObject();
-      if (obj.InstanceOf(stringItem)) {
+      auto obj = info[0].As<Napi::Object>();
+      if (StringItem::InstanceOf(env, obj)) {
         item = Napi::ObjectWrap<StringItem>::Unwrap(obj)->getItem();
-      } else if (obj.InstanceOf(fileItem)) {
+      } else if (FileItem::InstanceOf(env, obj)) {
         item = Napi::ObjectWrap<FileItem>::Unwrap(obj)->getItem();
       } else {
         item = std::make_shared<ItemWrapper>(env, info[0].ToObject());
@@ -218,9 +232,28 @@ class Creator : public Napi::ObjectWrap<Creator> {
 
       auto name = info[0].ToString().Utf8Value();
       auto content = info[1];
-      if (content.IsObject()) {  // content provider
-        std::unique_ptr<zim::writer::ContentProvider> provider =
-            std::make_unique<ContentProviderWrapper>(env, content.ToObject());
+      if (content.IsObject()) {
+        // addMetadata(name: string, content: ContentProvider, [mimetype])
+        auto obj = content.As<Napi::Object>();
+        std::unique_ptr<zim::writer::ContentProvider> provider{nullptr};
+
+        // Determine the type of ContentProvider
+        if (StringProvider::InstanceOf(env, content)) {
+          auto wrapped = Napi::ObjectWrap<StringProvider>::Unwrap(obj);
+          provider = wrapped->unwrapProvider();
+        } else if (FileProvider::InstanceOf(env, content)) {
+          auto wrapped = Napi::ObjectWrap<FileProvider>::Unwrap(obj);
+          provider = wrapped->unwrapProvider();
+        } else {
+          // Fallback to generic ContentProviderWrapper
+          provider = std::make_unique<ContentProviderWrapper>(env, obj);
+        }
+
+        if (provider == nullptr) {
+          throw Napi::Error::New(
+              env, "addMetadata failed to create ContentProvider from object");
+        }
+
         if (info.Length() > 2) {  // preserves default argument
           auto mimetype = info[2].ToString().Utf8Value();
           creator_->addMetadata(name, std::move(provider), mimetype);
@@ -228,7 +261,7 @@ class Creator : public Napi::ObjectWrap<Creator> {
           const std::string mimetype = "text/plain;charset=utf-8";
           creator_->addMetadata(name, std::move(provider), mimetype);
         }
-      } else {  // string version
+      } else {  // addMetadata(name: string, content: string, [mimetype])
         auto str = content.ToString().Utf8Value();
         if (info.Length() > 2) {
           auto mimetype = info[2].ToString().Utf8Value();
@@ -238,6 +271,7 @@ class Creator : public Napi::ObjectWrap<Creator> {
         }
       }
     } catch (const std::exception &err) {
+      std::cerr << "Error: addMetadata failed: " << err.what() << std::endl;
       throw Napi::Error::New(info.Env(), err.what());
     }
   }
@@ -245,15 +279,40 @@ class Creator : public Napi::ObjectWrap<Creator> {
   void addIllustration(const Napi::CallbackInfo &info) {
     try {
       auto env = info.Env();
-      auto size = info[0].ToNumber().Uint32Value();
-      auto content = info[1];
-      if (content.IsObject()) {
-        std::unique_ptr<zim::writer::ContentProvider> provider =
-            std::make_unique<ContentProviderWrapper>(env, content.ToObject());
-        creator_->addIllustration(size, std::move(provider));
+
+      // Inline template function to handle both size and IllustrationInfo
+      const auto addIllusWithContent = [&](auto &opt1) {
+        auto content = info[1];
+        if (content.IsObject()) {
+          std::unique_ptr<zim::writer::ContentProvider> provider =
+              std::make_unique<ContentProviderWrapper>(env, content.ToObject());
+          creator_->addIllustration(opt1, std::move(provider));
+        } else {
+          auto str = content.ToString().Utf8Value();
+          creator_->addIllustration(opt1, str);
+        }
+      };
+
+      auto arg0 = info[0];
+      if (arg0.IsNumber()) {
+        auto size = arg0.ToNumber().Uint32Value();
+        addIllusWithContent(size);
+      } else if (arg0.IsObject()) {
+        // Parse as IllustrationInfo
+        auto obj = arg0.ToObject();
+
+        // getIllustrationItem(illusInfo: IllustrationInfo)
+        // getIllustrationItem(illusInfo: object)
+        auto illusInfo =
+            IllustrationInfo::InstanceOf(env, obj)
+                ? IllustrationInfo::Unwrap(obj)->getInternalIllustrationInfo()
+                : IllustrationInfo::infoFrom(obj);
+        addIllusWithContent(illusInfo);
       } else {
-        auto str = content.ToString().Utf8Value();
-        creator_->addIllustration(size, str);
+        throw Napi::Error::New(
+            env,
+            "addIllustration first argument must be size[number] or "
+            "IllustrationInfo[object]");
       }
     } catch (const std::exception &err) {
       throw Napi::Error::New(info.Env(), err.what());
@@ -315,7 +374,6 @@ class Creator : public Napi::ObjectWrap<Creator> {
 
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors &constructors) {
-    Napi::HandleScope scope(env);
     Napi::Function func = DefineClass(
         env, "Creator",
         {

package/src/entry.h

@@ -93,7 +93,6 @@ class Entry : public Napi::ObjectWrap<Entry> {
 
   static void Init(Napi::Env env, Napi::Object exports,
                    ModuleConstructors &constructors) {
-    Napi::HandleScope scope(env);
     Napi::Function func = DefineClass(
         env, "Entry",
         {