@openzim/libzim 2.4.4 → 3.1.0

package/.env

@@ -1 +1 @@
-LIBZIM_VERSION=6.3.2
+LIBZIM_VERSION=8.2.1

package/.eslintignore

@@ -0,0 +1,3 @@
+.eslintrc.js
+node_modules
+dist/*

package/.eslintrc.js

@@ -0,0 +1,39 @@
+module.exports = {
+  env: {
+    es6: true,
+    node: true,
+  },
+  rules: {
+    "prettier/prettier": "error",
+    eqeqeq: "error",
+  },
+  extends: [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier",
+  ],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    project: "tsconfig.json",
+    sourceType: "module",
+  },
+  plugins: ["prettier"],
+  root: true,
+  overrides: [
+    {
+      files: ["*.ts", "*.js"],
+      rules: {
+        "@typescript-eslint/no-var-requires": "off",
+        "no-import-assign": "off",
+        "no-useless-escape": "off",
+      },
+      parserOptions: {
+        sourceType: "module",
+      },
+      extends: ["plugin:@typescript-eslint/recommended"],
+      parser: "@typescript-eslint/parser",
+      plugins: ["@typescript-eslint"],
+    },
+  ],
+};

package/Changelog

@@ -1,3 +1,16 @@
+3.1.0:
+* NEW: Use libzim v8.2.1
+* FIX: Handling of binary data in the creator
+* FIX: Use ESLint instead of TSLint
+* REMOVE: Node.js 16 from the CI
+
+3.0.0:
+* NEW: Use libzim v8.2.0
+* NEW: Calling API (to follow libzim changes)
+* NEW: Add support of ARM(64) for both Linux and macOS
+* UPDATE: Most of the dependencies
+* UPDATE: Many improvements around the CI
+
 2.4.4:
 * NEW: Use libzim v6.3.2
 

package/README.md

@@ -6,8 +6,8 @@ This is the Node.js binding to the
 [ZIM](https://openzim.org) files easily in Javascript.
 
 [![npm](https://img.shields.io/npm/v/@openzim/libzim.svg)](https://www.npmjs.com/package/@openzim/libzim)
-[![Build Status](https://github.com/openzim/node-libzim/workflows/CI/badge.svg?branch=master)](https://github.com/openzim/node-libzim/actions?query=branch%3Amaster)
-[![codecov](https://codecov.io/gh/openzim/node-libzim/branch/master/graph/badge.svg)](https://codecov.io/gh/openzim/node-libzim)
+[![Build Status](https://github.com/openzim/node-libzim/workflows/CI/badge.svg?branch=main)](https://github.com/openzim/node-libzim/actions?query=branch%3Amain)
+[![codecov](https://codecov.io/gh/openzim/node-libzim/branch/main/graph/badge.svg)](https://codecov.io/gh/openzim/node-libzim)
 [![CodeFactor](https://www.codefactor.io/repository/github/openzim/node-libzim/badge)](https://www.codefactor.io/repository/github/openzim/node-libzim)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
@@ -22,55 +22,77 @@ here](https://github.com/openzim/libzim/)).
 ## Usage
 
 ```bash
-npm i openzim/libzim
+npm i @openzim/libzim
 ```
 
-### Writing a Zim file
+### Writing a ZIM file
 ```javascript
 // write.js
-const { ZimArticle, ZimCreator } = require("@openzim/libzim");
+import { Creator, StringItem } from "@openzim/libzim";
 
 (async () => {
-
     console.info('Starting');
-    const creator = new ZimCreator({ fileName: 'test.zim' }, { welcome: 'index.html' });
-
-    for (let i = 100; i > 0; i--) {
-        const a = new ZimArticle({ url: `file${i}`, data: `Content ${i}` });
-        await creator.addArticle(a);
+    const outFile = "./test.zim";
+    const creator = new Creator()
+        .configNbWorkers(1)
+        .configIndexing(true, "en")
+        .configClusterSize(2048)
+        .startZimCreation(outFile);
+
+    for (let i = 0; i < 100; i++) {
+        const item = new StringItem(
+            `file${i}`,                       // path url
+            "text/plain",                     // content-type
+            `Title ${i}`,                     // title
+            {FRONT_ARTICLE: 1, COMPRESS: 1},  // hint option flags
+            `<h1>Content / Data ${i}</h1>`    // content
+        );
+        await creator.addItem(item);
     }
 
-    const welcome = new ZimArticle({ url: `index.html`, data: `<h1>Welcome!</h1>` });
-    await creator.addArticle(welcome);
-
-    await creator.finalise();
+    creator.setMainPath("file0");
+    await creator.finishZimCreation();
 
     console.log('Done Writing');
-
 })();
 ```
 
-### Reading a Zim file
+### Reading a ZIM file
 ```javascript
 // read.js
-
-const { ZimArticle, ZimReader } = require("@openzim/libzim");
+import { Archive, SuggestionSearcher, Searcher } from "@openzim/libzim";
 
 (async () => {
+    const outFile = "./test.zim";
+    const archive = new Archive(outFile);
+    console.log(`Archive opened: main entry path - ${archive.mainEntry.path}`);
 
-    const zimFile = new ZimReader(path.join(__dirname, '../test.zim'));
-
-    const suggestResults = await zimFile.suggest('laborum');
-    console.info(`Suggest Results:`, suggestResults);
+    for (const entry of archive.iterByPath()) {
+        console.log(`entry: ${entry.path} - ${entry.title}`);
+    }
 
-    const searchResults = await zimFile.search('rem');
-    console.info(`Search Results:`, searchResults);
+    const suggestionSearcher = new SuggestionSearcher(archive);
+    const suggestion = suggestionSearcher.suggest('laborum');
+    let results = suggestion.getResults(0, 10);
+    console.log("Suggestion results:");
+    for(const entry of results) {
+        console.log(`\t- ${entry.path} - ${entry.title}`);
+    }
 
-    const readArticleContent = await zimFile.getArticleByUrl('A/laborum');
-    console.info(`Article by url (laborum):`, readArticleContent);
+    const searcher = new Searcher(archive);
+    const search = searcher.search(new Query('rem'));
+    results = search.getResults(0, 10);
+    console.log("Search results:");
+    for(const entry of results) {
+        console.log(`\t- ${entry.path} - ${entry.title}`);
+    }
 
-    await zimFile.destroy();
 
+    const entry = await archive.getEntryByPath("A/laborum");
+    const item = entry.item;
+    const blob = item.data;
+    console.info(`Entry by url (laborum):`, blob.data);
+    delete archive;
 })();
 
 ```

package/binding.gyp

@@ -24,7 +24,7 @@
                 "libraries": [
                   "-Wl,-rpath,'$$ORIGIN'",
                   "-L<(libzim_dir)/lib/x86_64-linux-gnu",
-                  "<(libzim_dir)/lib/x86_64-linux-gnu/libzim.so.6",
+                  "<(libzim_dir)/lib/x86_64-linux-gnu/libzim.so.8",
                 ],
               }],
               ["libzim_local!='true' and OS=='mac'", {
@@ -33,7 +33,8 @@
                   "xcode_settings": {
                       "GCC_SYMBOLS_PRIVATE_EXTERN": "YES",  # -fvisibility=hidden
                       "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
-                      "LD_RUNPATH_SEARCH_PATHS": "@loader_path/"
+                      "LD_RUNPATH_SEARCH_PATHS": "@loader_path/",
+                      "OTHER_CFLAGS": [ "-std=c++17", "-fexceptions" ],
                   },
                   "libraries": ["-Wl,-rpath,@loader_path/", "-L<(libzim_dir)/lib", "-lzim"],
               }],
@@ -45,9 +46,6 @@
             "cflags_cc": [ "-std=c++17", "-fexceptions" ],
             "sources": [
                 "src/module.cc",
-                "src/article.cc",
-                "src/reader.cc",
-                "src/writer.cc",
             ],
             "include_dirs": [
                 "<!@(node -p \"require('node-addon-api').include\")",

package/bundle-libzim.js

@@ -1,26 +1,32 @@
-require('dotenv').config();
-const mkdirp = require('mkdirp');
-const exec = require('exec-then');
-const os = require('os');
+require("dotenv").config();
+const mkdirp = require("mkdirp");
+const exec = require("exec-then");
+const os = require("os");
 
-mkdirp.sync('./build/Release');
+mkdirp.sync("./build/Release");
 
-const isMacOS = os.type() === 'Darwin'
-const isLinux = os.type() === 'Linux'
+const isMacOS = os.type() === "Darwin";
+const isLinux = os.type() === "Linux";
 
 if (!isMacOS && !isLinux) {
-    console.warn(`\x1b[41m\n================================ README \n\nlibzim bundle with prebuilt binaries only available for macOS and Linux:\n\n\thttps://github.com/openzim/libzim/\n\n================================\x1b[0m\n`);
+  console.warn(
+    "\x1b[41m\n================================ README \n\nlibzim bundle with prebuilt binaries only available for macOS and Linux:\n\n\thttps://github.com/openzim/libzim/\n\n================================\x1b[0m\n"
+  );
 }
 
 if (isLinux) {
-    console.info(`Copying libzim.so.6 to build folder`)
-    exec(`cp download/lib/x86_64-linux-gnu/libzim.so.6 build/Release/libzim.so.6`)
-    exec(`ln -sf build/Release/libzim.so.6 build/Release/libzim.so`)  // convienience only, not required
+  console.info("Copying libzim.so.8 to build folder");
+  exec(
+    "cp download/lib/x86_64-linux-gnu/libzim.so.8 build/Release/libzim.so.8"
+  );
+  exec("ln -sf build/Release/libzim.so.8 build/Release/libzim.so"); // convienience only, not required
 }
 if (isMacOS) {
-    console.info(`Copying libzim.6.dylib to build folder`);
-    exec(`cp download/lib/libzim.6.dylib build/Release/libzim.6.dylib`)
-    exec(`ln -sf build/Release/libzim.6.dylib build/Release/libzim.dylib`)   // convienience only, not required
-    console.info(`Fixing rpath`)
-    exec(`install_name_tool -change libzim.6.dylib @loader_path/libzim.6.dylib build/Release/zim_binding.node`)
+  console.info("Copying libzim.8.dylib to build folder");
+  exec("cp download/lib/libzim.8.dylib build/Release/libzim.8.dylib");
+  exec("ln -sf build/Release/libzim.8.dylib build/Release/libzim.dylib"); // convienience only, not required
+  console.info("Fixing rpath");
+  exec(
+    "install_name_tool -change libzim.8.dylib @loader_path/libzim.8.dylib build/Release/zim_binding.node"
+  );
 }

package/dist/index.d.ts

@@ -1,4 +1,254 @@
-import { ZimArticle } from './zim';
-import { ZimReader } from './ZimReader';
-import { ZimCreator } from './ZimCreator';
-export { ZimArticle, ZimCreator, ZimReader, };
+export class IntegrityCheck {
+  static CHECKSUM: symbol;
+  static DIRENT_PTRS: symbol;
+  static DIRENT_ORDER: symbol;
+  static TITLE_INDEX: symbol;
+  static CLUSTER_PTRS: symbol;
+  static DIRENT_MIMETYPES: symbol;
+  static COUNT: symbol; // DO NOT USE THIS. See libzim docs.
+}
+
+export class Compression {
+  static None: symbol;
+  static Zstd: symbol;
+}
+
+export class Blob {
+  constructor(buf?: ArrayBuffer | Buffer | string);
+  get data(): Buffer;
+  get size(): number | bigint;
+  toString(): string;
+}
+
+export type ContentProvider = {
+  size: number | bigint;
+  feed(): Blob;
+};
+
+export class StringProvider {
+  constructor(content: string);
+  get size(): number;
+  feed(): Blob;
+}
+
+export class FileProvider {
+  constructor(filepath: string);
+  get size(): number | bigint;
+  feed(): Blob;
+}
+
+export type Hint = {
+  COMPRESS?: number;
+  FRONT_ARTICLE?: number;
+};
+
+export interface IndexData {
+  hasIndexData?: boolean;
+  title?: string;
+  content?: string;
+  keywords?: string;
+  wordcount?: number;
+  position?: [boolean, number, number];
+}
+
+export interface WriterItem {
+  path: string;
+  title: string;
+  mimeType: string;
+  getContentProvider(): ContentProvider;
+  hints: Hint;
+  getIndexData?: () => IndexData;
+}
+
+export class StringItem {
+  constructor(
+    path: string,
+    mimeType: string,
+    title: string,
+    hint: Hint,
+    content: ArrayBuffer | Buffer | string
+  );
+  readonly path: string;
+  readonly title: string;
+  readonly mimeType: string;
+  getContentProvider(): StringProvider;
+  readonly hints: Hint;
+}
+
+export class FileItem {
+  constructor(
+    path: string,
+    mimeType: string,
+    title: string,
+    hints: Hint,
+    filePath: string
+  );
+  readonly path: string;
+  readonly title: string;
+  readonly mimeType: string;
+  getContentProvider(): StringProvider;
+  readonly hints: Hint;
+}
+
+export class Creator {
+  constructor();
+  configVerbose(verbose: boolean): this;
+  configCompression(value: Compression): this;
+  configClusterSize(size: number): this;
+  configIndexing(indexing: boolean, language: string): this;
+  configNbWorkers(num: number): this;
+  startZimCreation(filepath: string): this;
+  addItem(item: WriterItem): Promise<void>;
+  addMetadata(
+    name: string,
+    content: string | ContentProvider,
+    mimetype?: string
+  ): void;
+  addIllustration(size: number, content: string | ContentProvider): void;
+  addRedirection(
+    path: string,
+    title: string,
+    targetPath: string,
+    hints?: Hint
+  ): void;
+  setMainPath(mainPath: string): void;
+  setUuid(uuid: string): void;
+  finishZimCreation(): Promise<void>;
+}
+
+export class Item {
+  get title(): string;
+  get path(): string;
+  get mimetype(): string;
+  get data(): Blob;
+  getData(offset?: number | bigint, limit?: number | bigint): Blob;
+  get size(): number | bigint;
+  get directAccessInformation(): {
+    filename: string;
+    offset: number;
+  };
+  get index(): number | bigint;
+}
+
+export class Entry {
+  get isRedirect(): boolean;
+  get title(): string;
+  get path(): string;
+  get item(): Item;
+  getItem(followRedirect?: boolean): Item;
+  get redirect(): Item;
+  get redirectEntry(): Entry;
+  get index(): number;
+}
+
+export interface EntryRange extends Iterable<Entry> {
+  size: number;
+  offset(start: number, maxResults: number): EntryRange;
+}
+
+export class Archive {
+  constructor(filepath: string);
+  get filename(): string;
+  get filesize(): number | bigint;
+  get allEntryCount(): number;
+  get entryCount(): number;
+  get articleCount(): number;
+  get uuid(): string;
+  getMetadata(name: string): string;
+  getMetadataItem(name: string): Item;
+  get metadataKeys(): string[];
+  getIllustrationItem(size: number): Item;
+  get illustrationSizes(): Set<number>;
+  getEntryByPath(path_or_idx: string | number): Entry;
+  getEntryByTitle(title_or_idx: string | number): Entry;
+  getEntryByClusterOrder(idx: number): Entry;
+  get mainEntry(): Entry;
+  get randomEntry(): Entry;
+  hasEntryByPath(path: string): boolean;
+  hasEntryByTitle(title: string): boolean;
+  hasMainEntry(): boolean;
+  hasIllustration(size: number): boolean;
+  hasFulltextIndex(): boolean;
+  hasTitleIndex(): boolean;
+  iterByPath(): EntryRange;
+  iterByTitle(): EntryRange;
+  iterEfficient(): EntryRange;
+  findByPath(path: string): EntryRange;
+  findByTitle(title: string): EntryRange;
+  get hasChecksum(): boolean;
+  get checksum(): string;
+  check(): boolean;
+  checkIntegrity(checkType: symbol): boolean; // one of IntegrityCheck
+  get isMultiPart(): boolean;
+  get hasNewNamespaceScheme(): boolean;
+
+  static validate(zimPath: string, checksToRun: symbol[]): boolean; // list of IntegrityCheck
+}
+
+interface Georange {
+  latitude: number;
+  longitude: number;
+  distance: number;
+}
+
+export class Query {
+  constructor(query: string);
+  setQuery(query: string): this;
+  setGeorange(latitude: number, longitude: number, distance: number): this;
+  get query(): string;
+  set query(query: string);
+  toString(): string;
+  get georange(): Georange;
+  set georange(range: Georange);
+}
+
+export class SearchIterator {
+  get path(): string;
+  get title(): string;
+  get score(): number;
+  get snippet(): string;
+  get wordCount(): number;
+  get size(): number;
+  get fileIndex(): number;
+  get zimId(): string;
+  get entry(): Entry;
+}
+
+export interface SearchResultSet extends Iterable<SearchIterator> {
+  readonly size: number;
+}
+
+export class Search {
+  getResults(start: number, maxResults: number): SearchResultSet;
+  get estimatedMatches(): number;
+}
+
+export class Searcher {
+  constructor(archives: Archive | Archive[]);
+  addArchive(archive: Archive): this;
+  search(query: string | Query): Search;
+  setVerbose(verbose: boolean): this;
+}
+
+export class SuggestionIterator {
+  get entry(): Entry;
+  get title(): string;
+  get path(): string;
+  get snippet(): string;
+  get hasSnippet(): boolean;
+}
+
+export interface SuggestionResultSet extends Iterable<SuggestionIterator> {
+  readonly size: number;
+}
+
+export class SuggestionSearch {
+  getResults(start: number, maxResults: number): SuggestionResultSet;
+  get estimatedMatches(): number;
+}
+
+export class SuggestionSearcher {
+  constructor(archives: Archive);
+  suggest(query: string): SuggestionSearch;
+  setVerbose(verbose: boolean): this;
+}

package/dist/index.js

@@ -1,9 +1,33 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ZimReader = exports.ZimCreator = exports.ZimArticle = void 0;
-var zim_1 = require("./zim");
-Object.defineProperty(exports, "ZimArticle", { enumerable: true, get: function () { return zim_1.ZimArticle; } });
-var ZimReader_1 = require("./ZimReader");
-Object.defineProperty(exports, "ZimReader", { enumerable: true, get: function () { return ZimReader_1.ZimReader; } });
-var ZimCreator_1 = require("./ZimCreator");
-Object.defineProperty(exports, "ZimCreator", { enumerable: true, get: function () { return ZimCreator_1.ZimCreator; } });
+import bindings from "bindings";
+
+const {
+  Archive,
+  Entry,
+  IntegrityCheck,
+  Compression,
+  Blob,
+  Searcher,
+  Query,
+  SuggestionSearcher,
+  Creator,
+  StringProvider,
+  FileProvider,
+  StringItem,
+  FileItem,
+} = bindings("zim_binding");
+
+module.exports = {
+  Archive,
+  Entry,
+  IntegrityCheck,
+  Compression,
+  Blob,
+  Searcher,
+  Query,
+  SuggestionSearcher,
+  Creator,
+  StringProvider,
+  FileProvider,
+  StringItem,
+  FileItem,
+};