@openzim/libzim 2.4.4 → 3.0.0

package/.env

@@ -1 +1 @@
-LIBZIM_VERSION=6.3.2
+LIBZIM_VERSION=8.2.0

package/Changelog

@@ -1,3 +1,10 @@
+3.0.0:
+* NEW: Use libzim v8.2.0
+* NEW: Calling API (to follow libzim changes)
+* NEW: Add support of ARM(64) for both Linux and macOS
+* UPDATE: Most of the dependencies
+* UPDATE: Many improvements around the CI
+
 2.4.4:
 * NEW: Use libzim v6.3.2
 

package/README.md

@@ -6,8 +6,8 @@ This is the Node.js binding to the
 [ZIM](https://openzim.org) files easily in Javascript.
 
 [![npm](https://img.shields.io/npm/v/@openzim/libzim.svg)](https://www.npmjs.com/package/@openzim/libzim)
-[![Build Status](https://github.com/openzim/node-libzim/workflows/CI/badge.svg?branch=master)](https://github.com/openzim/node-libzim/actions?query=branch%3Amaster)
-[![codecov](https://codecov.io/gh/openzim/node-libzim/branch/master/graph/badge.svg)](https://codecov.io/gh/openzim/node-libzim)
+[![Build Status](https://github.com/openzim/node-libzim/workflows/CI/badge.svg?branch=main)](https://github.com/openzim/node-libzim/actions?query=branch%3Amain)
+[![codecov](https://codecov.io/gh/openzim/node-libzim/branch/main/graph/badge.svg)](https://codecov.io/gh/openzim/node-libzim)
 [![CodeFactor](https://www.codefactor.io/repository/github/openzim/node-libzim/badge)](https://www.codefactor.io/repository/github/openzim/node-libzim)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
@@ -25,52 +25,74 @@ here](https://github.com/openzim/libzim/)).
 npm i openzim/libzim
 ```
 
-### Writing a Zim file
+### Writing a ZIM file
 ```javascript
 // write.js
-const { ZimArticle, ZimCreator } = require("@openzim/libzim");
+import { Creator, StringItem } from "@openzim/libzim";
 
 (async () => {
-
     console.info('Starting');
-    const creator = new ZimCreator({ fileName: 'test.zim' }, { welcome: 'index.html' });
-
-    for (let i = 100; i > 0; i--) {
-        const a = new ZimArticle({ url: `file${i}`, data: `Content ${i}` });
-        await creator.addArticle(a);
+    const outFile = "./test.zim";
+    const creator = new Creator()
+        .configNbWorkers(1)
+        .configIndexing(true, "en")
+        .configClusterSize(2048)
+        .startZimCreation(outFile);
+
+    for (let i = 0; i < 100; i++) {
+        const item = new StringItem(
+            `file${i}`,                       // path url
+            "text/plain",                     // content-type
+            `Title ${i}`,                     // title
+            {FRONT_ARTICLE: 1, COMPRESS: 1},  // hint option flags
+            `<h1>Content / Data ${i}</h1>`    // content
+        );
+        await creator.addItem(item);
     }
 
-    const welcome = new ZimArticle({ url: `index.html`, data: `<h1>Welcome!</h1>` });
-    await creator.addArticle(welcome);
-
-    await creator.finalise();
+    creator.setMainPath("file0");
+    await creator.finishZimCreation();
 
     console.log('Done Writing');
-
 })();
 ```
 
-### Reading a Zim file
+### Reading a ZIM file
 ```javascript
 // read.js
-
-const { ZimArticle, ZimReader } = require("@openzim/libzim");
+import { Archive, SuggestionSearcher, Searcher } from "@openzim/libzim";
 
 (async () => {
+    const outFile = "./test.zim";
+    const archive = new Archive(outFile);
+    console.log(`Archive opened: main entry path - ${archive.mainEntry.path}`);
 
-    const zimFile = new ZimReader(path.join(__dirname, '../test.zim'));
-
-    const suggestResults = await zimFile.suggest('laborum');
-    console.info(`Suggest Results:`, suggestResults);
+    for (const entry of archive.iterByPath()) {
+        console.log(`entry: ${entry.path} - ${entry.title}`);
+    }
 
-    const searchResults = await zimFile.search('rem');
-    console.info(`Search Results:`, searchResults);
+    const suggestionSearcher = new SuggestionSearcher(archive);
+    const suggestion = suggestionSearcher.suggest('laborum');
+    let results = suggestion.getResults(0, 10);
+    console.log("Suggestion results:");
+    for(const entry of results) {
+        console.log(`\t- ${entry.path} - ${entry.title}`);
+    }
 
-    const readArticleContent = await zimFile.getArticleByUrl('A/laborum');
-    console.info(`Article by url (laborum):`, readArticleContent);
+    const searcher = new Searcher(archive);
+    const search = searcher.search(new Query('rem'));
+    results = search.getResults(0, 10);
+    console.log("Search results:");
+    for(const entry of results) {
+        console.log(`\t- ${entry.path} - ${entry.title}`);
+    }
 
-    await zimFile.destroy();
 
+    const entry = await archive.getEntryByPath("A/laborum");
+    const item = entry.item;
+    const blob = item.data;
+    console.info(`Entry by url (laborum):`, blob.data);
+    delete archive;
 })();
 
 ```

package/binding.gyp

@@ -24,7 +24,7 @@
                 "libraries": [
                   "-Wl,-rpath,'$$ORIGIN'",
                   "-L<(libzim_dir)/lib/x86_64-linux-gnu",
-                  "<(libzim_dir)/lib/x86_64-linux-gnu/libzim.so.6",
+                  "<(libzim_dir)/lib/x86_64-linux-gnu/libzim.so.8",
                 ],
               }],
               ["libzim_local!='true' and OS=='mac'", {
@@ -33,7 +33,8 @@
                   "xcode_settings": {
                       "GCC_SYMBOLS_PRIVATE_EXTERN": "YES",  # -fvisibility=hidden
                       "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
-                      "LD_RUNPATH_SEARCH_PATHS": "@loader_path/"
+                      "LD_RUNPATH_SEARCH_PATHS": "@loader_path/",
+                      "OTHER_CFLAGS": [ "-std=c++17", "-fexceptions" ],
                   },
                   "libraries": ["-Wl,-rpath,@loader_path/", "-L<(libzim_dir)/lib", "-lzim"],
               }],
@@ -45,9 +46,6 @@
             "cflags_cc": [ "-std=c++17", "-fexceptions" ],
             "sources": [
                 "src/module.cc",
-                "src/article.cc",
-                "src/reader.cc",
-                "src/writer.cc",
             ],
             "include_dirs": [
                 "<!@(node -p \"require('node-addon-api').include\")",

package/bundle-libzim.js

@@ -13,14 +13,14 @@ if (!isMacOS && !isLinux) {
 }
 
 if (isLinux) {
-    console.info(`Copying libzim.so.6 to build folder`)
-    exec(`cp download/lib/x86_64-linux-gnu/libzim.so.6 build/Release/libzim.so.6`)
-    exec(`ln -sf build/Release/libzim.so.6 build/Release/libzim.so`)  // convienience only, not required
+    console.info(`Copying libzim.so.8 to build folder`)
+    exec(`cp download/lib/x86_64-linux-gnu/libzim.so.8 build/Release/libzim.so.8`)
+    exec(`ln -sf build/Release/libzim.so.8 build/Release/libzim.so`)  // convienience only, not required
 }
 if (isMacOS) {
-    console.info(`Copying libzim.6.dylib to build folder`);
-    exec(`cp download/lib/libzim.6.dylib build/Release/libzim.6.dylib`)
-    exec(`ln -sf build/Release/libzim.6.dylib build/Release/libzim.dylib`)   // convienience only, not required
+    console.info(`Copying libzim.8.dylib to build folder`);
+    exec(`cp download/lib/libzim.8.dylib build/Release/libzim.8.dylib`)
+    exec(`ln -sf build/Release/libzim.8.dylib build/Release/libzim.dylib`)   // convienience only, not required
     console.info(`Fixing rpath`)
-    exec(`install_name_tool -change libzim.6.dylib @loader_path/libzim.6.dylib build/Release/zim_binding.node`)
+    exec(`install_name_tool -change libzim.8.dylib @loader_path/libzim.8.dylib build/Release/zim_binding.node`)
 }

package/dist/index.d.ts

@@ -1,4 +1,234 @@
-import { ZimArticle } from './zim';
-import { ZimReader } from './ZimReader';
-import { ZimCreator } from './ZimCreator';
-export { ZimArticle, ZimCreator, ZimReader, };
+
+export class IntegrityCheck {
+  static CHECKSUM: symbol;
+  static DIRENT_PTRS: symbol;
+  static DIRENT_ORDER: symbol;
+  static TITLE_INDEX: symbol;
+  static CLUSTER_PTRS: symbol;
+  static DIRENT_MIMETYPES: symbol;
+  static COUNT: symbol;  // DO NOT USE THIS. See libzim docs.
+}
+
+export class Compression {
+  static None: symbol;
+  static Zstd: symbol;
+}
+
+export class Blob {
+  constructor(buf?: ArrayBuffer | Buffer | string);
+  get data(): Buffer;
+  get size(): number | bigint;
+  toString(): string;
+}
+
+export type ContentProvider = {
+  size: number | bigint;
+  feed() : Blob;
+}
+
+export class StringProvider {
+  constructor(content: string);
+  get size(): number;
+  feed() : Blob;
+}
+
+export class FileProvider {
+  constructor(filepath: string);
+  get size(): number | bigint;
+  feed() : Blob;
+}
+
+export type Hint = {
+  COMPRESS?: number;
+  FRONT_ARTICLE?: number;
+}
+
+export interface IndexData {
+  hasIndexData?: boolean;
+  title?: string;
+  content?: string;
+  keywords?: string;
+  wordcount?: number;
+  position?: [boolean, number, number];
+}
+
+export interface WriterItem {
+  path: string;
+  title: string;
+  mimeType: string;
+  getContentProvider() : ContentProvider;
+  hints: Hint;
+  getIndexData?: () => IndexData;
+}
+
+export class StringItem {
+  constructor(path: string, mimeType: string, title: string, hint: Hint, content: string);
+  readonly path: string;
+  readonly title: string;
+  readonly mimeType: string;
+  getContentProvider() : StringProvider;
+  readonly hints: Hint;
+}
+
+export class FileItem {
+  constructor(path: string, mimeType: string, title: string, hints: Hint, filePath: string);
+  readonly path: string;
+  readonly title: string;
+  readonly mimeType: string;
+  getContentProvider() : StringProvider;
+  readonly hints: Hint;
+}
+
+export class Creator {
+  constructor();
+  configVerbose(verbose: boolean) : this;
+  configCompression(value: Compression) : this;
+  configClusterSize(size: number) : this;
+  configIndexing(indexing: boolean, language: string) : this;
+  configNbWorkers(num: number) : this;
+  startZimCreation(filepath: string): this;
+  addItem(item: WriterItem): Promise<void>;
+  addMetadata(name: string, content: string | ContentProvider, mimetype?: string): void;
+  addIllustration(size: number, content: string | ContentProvider): void;
+  addRedirection(path: string, title: string, targetPath: string, hints?: Hint): void;
+  setMainPath(mainPath: string): void;
+  setUuid(uuid: string): void;
+  finishZimCreation() : Promise<void>;
+}
+
+export class Item {
+  get title(): string;
+  get path(): string;
+  get mimetype(): string;
+  get data(): Blob;
+  getData(offset?: number | bigint, limit?: number | bigint) : Blob;
+  get size(): number | bigint;
+  get directAccessInformation(): {
+    filename: string,
+      offset: number,
+  };
+  get index(): number | bigint;
+}
+
+export class Entry {
+  get isRedirect(): boolean;
+  get title(): string;
+  get path(): string;
+  get item(): Item;
+  getItem(followRedirect?: boolean) : Item;
+  get redirect(): Item;
+  get redirectEntry(): Entry;
+  get index(): number;
+}
+
+export interface EntryRange extends Iterable<Entry> {
+  size: number;
+  offset(start: number, maxResults: number) : EntryRange;
+}
+
+export class Archive {
+  constructor(filepath: string);
+  get filename(): string;
+  get filesize(): number | bigint;
+  get allEntryCount(): number;
+  get entryCount(): number;
+  get articleCount(): number;
+  get uuid(): string;
+  getMetadata(name: string) : string;
+  getMetadataItem(name: string) : Item;
+  get metadataKeys(): string[];
+  getIllustrationItem(size: number) : Item;
+  get illustrationSizes() : Set<number>;
+  getEntryByPath(path_or_idx: string | number) : Entry;
+  getEntryByTitle(title_or_idx: string | number) : Entry;
+  getEntryByClusterOrder(idx: number) : Entry;
+  get mainEntry(): Entry;
+  get randomEntry(): Entry;
+  hasEntryByPath(path: string) : boolean;
+  hasEntryByTitle(title: string) : boolean;
+  hasMainEntry(): boolean;
+  hasIllustration(size: number) : boolean;
+  hasFulltextIndex() : boolean;
+  hasTitleIndex(): boolean;
+  iterByPath() : EntryRange;
+  iterByTitle() : EntryRange;
+  iterEfficient() : EntryRange;
+  findByPath(path: string) : EntryRange;
+  findByTitle(title: string) : EntryRange;
+  get hasChecksum(): boolean;
+  get checksum(): string;
+  check() : boolean;
+  checkIntegrity(checkType: symbol) : boolean;  // one of IntegrityCheck
+  get isMultiPart(): boolean;
+  get hasNewNamespaceScheme(): boolean;
+
+  static validate(zimPath: string, checksToRun: symbol[]) : boolean; // list of IntegrityCheck
+}
+
+interface Georange {
+  latitude: number;
+  longitude: number;
+  distance: number;
+}
+
+export class Query {
+  constructor(query: string);
+  setQuery(query: string): this;
+  setGeorange(latitude: number, longitude: number, distance: number): this;
+  get query() : string;
+  set query(query: string);
+  toString(): string;
+  get georange() : Georange;
+  set georange(range: Georange);
+}
+
+export class SearchIterator {
+  get path(): string;
+  get title(): string;
+  get score(): number;
+  get snippet(): string;
+  get wordCount(): number;
+  get size() : number;
+  get fileIndex(): number;
+  get zimId(): string;
+  get entry(): Entry;
+}
+
+export interface SearchResultSet extends Iterable<SearchIterator> {
+  readonly size: number;
+}
+
+export class Search {
+  getResults(start: number, maxResults: number) : SearchResultSet;
+  get estimatedMatches(): number;
+}
+
+export class Searcher {
+  constructor(archives: Archive | Archive[]);
+  addArchive(archive: Archive): this;
+  search(query: string | Query) : Search;
+  setVerbose(verbose: boolean) : this;
+}
+
+export class SuggestionIterator {
+  get entry(): Entry;
+  get title(): string;
+  get path(): string;
+  get snippet(): string;
+  get hasSnippet(): boolean;
+}
+
+export interface SuggestionResultSet extends Iterable<SuggestionIterator> {
+  readonly size: number;
+}
+
+export class SuggestionSearch {
+  getResults(start: number, maxResults: number) : SuggestionResultSet;
+  get estimatedMatches(): number;
+}
+
+export class SuggestionSearcher {
+  constructor(archives: Archive);
+  suggest(query: string): SuggestionSearch;
+  setVerbose(verbose: boolean) : this;
+}

package/dist/index.js

@@ -1,9 +1,34 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ZimReader = exports.ZimCreator = exports.ZimArticle = void 0;
-var zim_1 = require("./zim");
-Object.defineProperty(exports, "ZimArticle", { enumerable: true, get: function () { return zim_1.ZimArticle; } });
-var ZimReader_1 = require("./ZimReader");
-Object.defineProperty(exports, "ZimReader", { enumerable: true, get: function () { return ZimReader_1.ZimReader; } });
-var ZimCreator_1 = require("./ZimCreator");
-Object.defineProperty(exports, "ZimCreator", { enumerable: true, get: function () { return ZimCreator_1.ZimCreator; } });
+
+const bindings = require('bindings');
+
+const {
+  Archive,
+  Entry,
+  IntegrityCheck,
+  Compression,
+  Blob,
+  Searcher,
+  Query,
+  SuggestionSearcher,
+  Creator,
+  StringProvider,
+  FileProvider,
+  StringItem,
+  FileItem,
+} = bindings('zim_binding');
+
+module.exports = {
+  Archive,
+  Entry,
+  IntegrityCheck,
+  Compression,
+  Blob,
+  Searcher,
+  Query,
+  SuggestionSearcher,
+  Creator,
+  StringProvider,
+  FileProvider,
+  StringItem,
+  FileItem,
+}

package/download-libzim.js

@@ -10,14 +10,29 @@ mkdirp.sync('./download');
 
 const isMacOS = os.type() === 'Darwin'
 const isLinux = os.type() === 'Linux'
+const rawArch = os.arch()
+const isAvailableArch = rawArch === 'x64' || rawArch == 'arm' || rawArch == 'arm64'
 
 if (!isMacOS && !isLinux) {
     console.warn(`\x1b[41m\n================================ README \n\nPre-built binaries only available on Linux and MacOS for now...\nPlease ensure you have libzim installed globally on this machine:\n\n\thttps://github.com/openzim/libzim/\n\n================================\x1b[0m\n`);
 }
+if (!isAvailableArch) {
+    console.warn(`\x1b[41m\n================================ README \n\nPre-built binaries only available on x86_64, arm and arm64 for now...\nPlease ensure you have libzim installed globally on this machine:\n\n\thttps://github.com/openzim/libzim/\n\n================================\x1b[0m\n`);
+}
 
 let osPrefix = (isMacOS) ? 'macos' : 'linux';
+let osArch = (isLinux) ? 'x86_64-bionic' : 'x86_64';
+
+if (rawArch != 'x64'){
+    if (isLinux) {
+        osArch = rawArch == 'arm64' ? 'aarch64-bionic' : 'armhf'
+    } else {
+        osArch = rawArch
+    }
+}
+
 const urls = [
-    `http://download.openzim.org/release/libzim/libzim_${osPrefix}-x86_64-${process.env.LIBZIM_VERSION}.tar.gz`,
+    `https://download.openzim.org/release/libzim/libzim_${osPrefix}-${osArch}-${process.env.LIBZIM_VERSION}.tar.gz`,
 ].filter(a => a);
 
 for (let url of urls) {

package/package.json

@@ -1,13 +1,16 @@
 {
   "name": "@openzim/libzim",
-  "version": "2.4.4",
+  "main": "dist/index.js",
+  "types": "dist/index.d.js",
+  "version": "3.0.0",
   "description": "Libzim bindings for NodeJS",
   "scripts": {
     "clean": "rm -rf dist build/native/build",
     "tsc": "tsc",
-    "prepare": "npm run tsc",
+    "prepare": "mkdir -p dist/ && cp -v src/index.js src/index.d.ts dist/",
     "codecov": "nyc --reporter=lcov npm t",
     "install": "npm run download && node-gyp rebuild -v && npm run bundle",
+    "build": "node-gyp rebuild -v && npm run bundle",
     "download": "node ./download-libzim.js",
     "bundle": "node ./bundle-libzim.js",
     "test": "jest",
@@ -22,7 +25,6 @@
       "node_modules"
     ]
   },
-  "main": "dist/index.js",
   "author": "Joseph Reeve",
   "license": "GPL-3.0",
   "repository": {
@@ -35,32 +37,26 @@
   "homepage": "https://github.com/openzim/node-libzim#readme",
   "gypfile": true,
   "dependencies": {
-    "@types/bindings": "^1.5.0",
-    "@types/faker": "^4.1.12",
-    "@types/jest": "^25.2.3",
-    "@types/mime": "^2.0.3",
-    "@types/node": "^13.13.52",
-    "@types/rimraf": "^3.0.0",
-    "axios": "^0.21.1",
+    "@types/bindings": "^1.5.1",
+    "@types/jest": "^28.1.6",
+    "@types/node": "^18.0.6",
+    "axios": "^0.27.2",
     "bindings": "^1.5.0",
-    "dotenv": "^8.6.0",
+    "dotenv": "^16.0.1",
     "exec-then": "^1.3.1",
-    "faker": "^4.1.0",
-    "jest": "^25.5.4",
-    "mime": "^2.5.2",
-    "minimist": "^1.2.5",
     "mkdirp": "^1.0.4",
-    "node-addon-api": "^2.0.2",
-    "node-gyp": "^6.1.0",
-    "rimraf": "^3.0.2",
-    "tar": "^6.1.0",
-    "ts-jest": "^25.4.0",
-    "ts-node": "^8.10.2",
-    "tsconfig-paths": "^3.9.0"
+    "node-addon-api": "^5.0.0",
+    "node-gyp": "^9.3.1",
+    "tqdm": "^2.0.3",
+    "ts-node": "^10.9.1",
+    "tsconfig-paths": "^4.0.0"
   },
   "devDependencies": {
+    "@faker-js/faker": "^7.6.0",
+    "jest": "^28.1.3",
     "nyc": "^15.1.0",
-    "typescript": "^3.9.9"
+    "ts-jest": "^28.0.7",
+    "typescript": "^4.7.4"
   },
   "jest": {
     "preset": "ts-jest/presets/js-with-ts"