@openzim/libzim 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env +1 -1
- package/.eslintignore +3 -0
- package/.eslintrc.js +39 -0
- package/Changelog +6 -0
- package/README.md +1 -1
- package/bundle-libzim.js +22 -16
- package/dist/index.d.ts +72 -52
- package/dist/index.js +3 -4
- package/download-libzim.js +60 -56
- package/package.json +11 -3
- package/src/index.d.ts +72 -52
- package/src/index.js +3 -4
- package/src/writerItem.h +16 -1
package/.env
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
LIBZIM_VERSION=8.2.
|
|
1
|
+
LIBZIM_VERSION=8.2.1
|
package/.eslintignore
ADDED
package/.eslintrc.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
module.exports = {
|
|
2
|
+
env: {
|
|
3
|
+
es6: true,
|
|
4
|
+
node: true,
|
|
5
|
+
},
|
|
6
|
+
rules: {
|
|
7
|
+
"prettier/prettier": "error",
|
|
8
|
+
eqeqeq: "error",
|
|
9
|
+
},
|
|
10
|
+
extends: [
|
|
11
|
+
"eslint:recommended",
|
|
12
|
+
"plugin:@typescript-eslint/eslint-recommended",
|
|
13
|
+
"plugin:@typescript-eslint/recommended",
|
|
14
|
+
"prettier",
|
|
15
|
+
],
|
|
16
|
+
parser: "@typescript-eslint/parser",
|
|
17
|
+
parserOptions: {
|
|
18
|
+
project: "tsconfig.json",
|
|
19
|
+
sourceType: "module",
|
|
20
|
+
},
|
|
21
|
+
plugins: ["prettier"],
|
|
22
|
+
root: true,
|
|
23
|
+
overrides: [
|
|
24
|
+
{
|
|
25
|
+
files: ["*.ts", "*.js"],
|
|
26
|
+
rules: {
|
|
27
|
+
"@typescript-eslint/no-var-requires": "off",
|
|
28
|
+
"no-import-assign": "off",
|
|
29
|
+
"no-useless-escape": "off",
|
|
30
|
+
},
|
|
31
|
+
parserOptions: {
|
|
32
|
+
sourceType: "module",
|
|
33
|
+
},
|
|
34
|
+
extends: ["plugin:@typescript-eslint/recommended"],
|
|
35
|
+
parser: "@typescript-eslint/parser",
|
|
36
|
+
plugins: ["@typescript-eslint"],
|
|
37
|
+
},
|
|
38
|
+
],
|
|
39
|
+
};
|
package/Changelog
CHANGED
package/README.md
CHANGED
package/bundle-libzim.js
CHANGED
|
@@ -1,26 +1,32 @@
|
|
|
1
|
-
require(
|
|
2
|
-
const mkdirp = require(
|
|
3
|
-
const exec = require(
|
|
4
|
-
const os = require(
|
|
1
|
+
require("dotenv").config();
|
|
2
|
+
const mkdirp = require("mkdirp");
|
|
3
|
+
const exec = require("exec-then");
|
|
4
|
+
const os = require("os");
|
|
5
5
|
|
|
6
|
-
mkdirp.sync(
|
|
6
|
+
mkdirp.sync("./build/Release");
|
|
7
7
|
|
|
8
|
-
const isMacOS = os.type() ===
|
|
9
|
-
const isLinux = os.type() ===
|
|
8
|
+
const isMacOS = os.type() === "Darwin";
|
|
9
|
+
const isLinux = os.type() === "Linux";
|
|
10
10
|
|
|
11
11
|
if (!isMacOS && !isLinux) {
|
|
12
|
-
|
|
12
|
+
console.warn(
|
|
13
|
+
"\x1b[41m\n================================ README \n\nlibzim bundle with prebuilt binaries only available for macOS and Linux:\n\n\thttps://github.com/openzim/libzim/\n\n================================\x1b[0m\n"
|
|
14
|
+
);
|
|
13
15
|
}
|
|
14
16
|
|
|
15
17
|
if (isLinux) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
console.info("Copying libzim.so.8 to build folder");
|
|
19
|
+
exec(
|
|
20
|
+
"cp download/lib/x86_64-linux-gnu/libzim.so.8 build/Release/libzim.so.8"
|
|
21
|
+
);
|
|
22
|
+
exec("ln -sf build/Release/libzim.so.8 build/Release/libzim.so"); // convienience only, not required
|
|
19
23
|
}
|
|
20
24
|
if (isMacOS) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
console.info("Copying libzim.8.dylib to build folder");
|
|
26
|
+
exec("cp download/lib/libzim.8.dylib build/Release/libzim.8.dylib");
|
|
27
|
+
exec("ln -sf build/Release/libzim.8.dylib build/Release/libzim.dylib"); // convienience only, not required
|
|
28
|
+
console.info("Fixing rpath");
|
|
29
|
+
exec(
|
|
30
|
+
"install_name_tool -change libzim.8.dylib @loader_path/libzim.8.dylib build/Release/zim_binding.node"
|
|
31
|
+
);
|
|
26
32
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
|
|
2
1
|
export class IntegrityCheck {
|
|
3
2
|
static CHECKSUM: symbol;
|
|
4
3
|
static DIRENT_PTRS: symbol;
|
|
@@ -6,7 +5,7 @@ export class IntegrityCheck {
|
|
|
6
5
|
static TITLE_INDEX: symbol;
|
|
7
6
|
static CLUSTER_PTRS: symbol;
|
|
8
7
|
static DIRENT_MIMETYPES: symbol;
|
|
9
|
-
static COUNT: symbol;
|
|
8
|
+
static COUNT: symbol; // DO NOT USE THIS. See libzim docs.
|
|
10
9
|
}
|
|
11
10
|
|
|
12
11
|
export class Compression {
|
|
@@ -23,25 +22,25 @@ export class Blob {
|
|
|
23
22
|
|
|
24
23
|
export type ContentProvider = {
|
|
25
24
|
size: number | bigint;
|
|
26
|
-
feed()
|
|
27
|
-
}
|
|
25
|
+
feed(): Blob;
|
|
26
|
+
};
|
|
28
27
|
|
|
29
28
|
export class StringProvider {
|
|
30
29
|
constructor(content: string);
|
|
31
30
|
get size(): number;
|
|
32
|
-
feed()
|
|
31
|
+
feed(): Blob;
|
|
33
32
|
}
|
|
34
33
|
|
|
35
34
|
export class FileProvider {
|
|
36
35
|
constructor(filepath: string);
|
|
37
36
|
get size(): number | bigint;
|
|
38
|
-
feed()
|
|
37
|
+
feed(): Blob;
|
|
39
38
|
}
|
|
40
39
|
|
|
41
40
|
export type Hint = {
|
|
42
41
|
COMPRESS?: number;
|
|
43
42
|
FRONT_ARTICLE?: number;
|
|
44
|
-
}
|
|
43
|
+
};
|
|
45
44
|
|
|
46
45
|
export interface IndexData {
|
|
47
46
|
hasIndexData?: boolean;
|
|
@@ -56,44 +55,65 @@ export interface WriterItem {
|
|
|
56
55
|
path: string;
|
|
57
56
|
title: string;
|
|
58
57
|
mimeType: string;
|
|
59
|
-
getContentProvider()
|
|
58
|
+
getContentProvider(): ContentProvider;
|
|
60
59
|
hints: Hint;
|
|
61
60
|
getIndexData?: () => IndexData;
|
|
62
61
|
}
|
|
63
62
|
|
|
64
63
|
export class StringItem {
|
|
65
|
-
constructor(
|
|
64
|
+
constructor(
|
|
65
|
+
path: string,
|
|
66
|
+
mimeType: string,
|
|
67
|
+
title: string,
|
|
68
|
+
hint: Hint,
|
|
69
|
+
content: ArrayBuffer | Buffer | string
|
|
70
|
+
);
|
|
66
71
|
readonly path: string;
|
|
67
72
|
readonly title: string;
|
|
68
73
|
readonly mimeType: string;
|
|
69
|
-
getContentProvider()
|
|
74
|
+
getContentProvider(): StringProvider;
|
|
70
75
|
readonly hints: Hint;
|
|
71
76
|
}
|
|
72
77
|
|
|
73
78
|
export class FileItem {
|
|
74
|
-
constructor(
|
|
79
|
+
constructor(
|
|
80
|
+
path: string,
|
|
81
|
+
mimeType: string,
|
|
82
|
+
title: string,
|
|
83
|
+
hints: Hint,
|
|
84
|
+
filePath: string
|
|
85
|
+
);
|
|
75
86
|
readonly path: string;
|
|
76
87
|
readonly title: string;
|
|
77
88
|
readonly mimeType: string;
|
|
78
|
-
getContentProvider()
|
|
89
|
+
getContentProvider(): StringProvider;
|
|
79
90
|
readonly hints: Hint;
|
|
80
91
|
}
|
|
81
92
|
|
|
82
93
|
export class Creator {
|
|
83
94
|
constructor();
|
|
84
|
-
configVerbose(verbose: boolean)
|
|
85
|
-
configCompression(value: Compression)
|
|
86
|
-
configClusterSize(size: number)
|
|
87
|
-
configIndexing(indexing: boolean, language: string)
|
|
88
|
-
configNbWorkers(num: number)
|
|
95
|
+
configVerbose(verbose: boolean): this;
|
|
96
|
+
configCompression(value: Compression): this;
|
|
97
|
+
configClusterSize(size: number): this;
|
|
98
|
+
configIndexing(indexing: boolean, language: string): this;
|
|
99
|
+
configNbWorkers(num: number): this;
|
|
89
100
|
startZimCreation(filepath: string): this;
|
|
90
101
|
addItem(item: WriterItem): Promise<void>;
|
|
91
|
-
addMetadata(
|
|
102
|
+
addMetadata(
|
|
103
|
+
name: string,
|
|
104
|
+
content: string | ContentProvider,
|
|
105
|
+
mimetype?: string
|
|
106
|
+
): void;
|
|
92
107
|
addIllustration(size: number, content: string | ContentProvider): void;
|
|
93
|
-
addRedirection(
|
|
108
|
+
addRedirection(
|
|
109
|
+
path: string,
|
|
110
|
+
title: string,
|
|
111
|
+
targetPath: string,
|
|
112
|
+
hints?: Hint
|
|
113
|
+
): void;
|
|
94
114
|
setMainPath(mainPath: string): void;
|
|
95
115
|
setUuid(uuid: string): void;
|
|
96
|
-
finishZimCreation()
|
|
116
|
+
finishZimCreation(): Promise<void>;
|
|
97
117
|
}
|
|
98
118
|
|
|
99
119
|
export class Item {
|
|
@@ -101,11 +121,11 @@ export class Item {
|
|
|
101
121
|
get path(): string;
|
|
102
122
|
get mimetype(): string;
|
|
103
123
|
get data(): Blob;
|
|
104
|
-
getData(offset?: number | bigint, limit?: number | bigint)
|
|
124
|
+
getData(offset?: number | bigint, limit?: number | bigint): Blob;
|
|
105
125
|
get size(): number | bigint;
|
|
106
126
|
get directAccessInformation(): {
|
|
107
|
-
filename: string
|
|
108
|
-
|
|
127
|
+
filename: string;
|
|
128
|
+
offset: number;
|
|
109
129
|
};
|
|
110
130
|
get index(): number | bigint;
|
|
111
131
|
}
|
|
@@ -115,7 +135,7 @@ export class Entry {
|
|
|
115
135
|
get title(): string;
|
|
116
136
|
get path(): string;
|
|
117
137
|
get item(): Item;
|
|
118
|
-
getItem(followRedirect?: boolean)
|
|
138
|
+
getItem(followRedirect?: boolean): Item;
|
|
119
139
|
get redirect(): Item;
|
|
120
140
|
get redirectEntry(): Entry;
|
|
121
141
|
get index(): number;
|
|
@@ -123,7 +143,7 @@ export class Entry {
|
|
|
123
143
|
|
|
124
144
|
export interface EntryRange extends Iterable<Entry> {
|
|
125
145
|
size: number;
|
|
126
|
-
offset(start: number, maxResults: number)
|
|
146
|
+
offset(start: number, maxResults: number): EntryRange;
|
|
127
147
|
}
|
|
128
148
|
|
|
129
149
|
export class Archive {
|
|
@@ -134,35 +154,35 @@ export class Archive {
|
|
|
134
154
|
get entryCount(): number;
|
|
135
155
|
get articleCount(): number;
|
|
136
156
|
get uuid(): string;
|
|
137
|
-
getMetadata(name: string)
|
|
138
|
-
getMetadataItem(name: string)
|
|
157
|
+
getMetadata(name: string): string;
|
|
158
|
+
getMetadataItem(name: string): Item;
|
|
139
159
|
get metadataKeys(): string[];
|
|
140
|
-
getIllustrationItem(size: number)
|
|
141
|
-
get illustrationSizes()
|
|
142
|
-
getEntryByPath(path_or_idx: string | number)
|
|
143
|
-
getEntryByTitle(title_or_idx: string | number)
|
|
144
|
-
getEntryByClusterOrder(idx: number)
|
|
160
|
+
getIllustrationItem(size: number): Item;
|
|
161
|
+
get illustrationSizes(): Set<number>;
|
|
162
|
+
getEntryByPath(path_or_idx: string | number): Entry;
|
|
163
|
+
getEntryByTitle(title_or_idx: string | number): Entry;
|
|
164
|
+
getEntryByClusterOrder(idx: number): Entry;
|
|
145
165
|
get mainEntry(): Entry;
|
|
146
166
|
get randomEntry(): Entry;
|
|
147
|
-
hasEntryByPath(path: string)
|
|
148
|
-
hasEntryByTitle(title: string)
|
|
167
|
+
hasEntryByPath(path: string): boolean;
|
|
168
|
+
hasEntryByTitle(title: string): boolean;
|
|
149
169
|
hasMainEntry(): boolean;
|
|
150
|
-
hasIllustration(size: number)
|
|
151
|
-
hasFulltextIndex()
|
|
170
|
+
hasIllustration(size: number): boolean;
|
|
171
|
+
hasFulltextIndex(): boolean;
|
|
152
172
|
hasTitleIndex(): boolean;
|
|
153
|
-
iterByPath()
|
|
154
|
-
iterByTitle()
|
|
155
|
-
iterEfficient()
|
|
156
|
-
findByPath(path: string)
|
|
157
|
-
findByTitle(title: string)
|
|
173
|
+
iterByPath(): EntryRange;
|
|
174
|
+
iterByTitle(): EntryRange;
|
|
175
|
+
iterEfficient(): EntryRange;
|
|
176
|
+
findByPath(path: string): EntryRange;
|
|
177
|
+
findByTitle(title: string): EntryRange;
|
|
158
178
|
get hasChecksum(): boolean;
|
|
159
179
|
get checksum(): string;
|
|
160
|
-
check()
|
|
161
|
-
checkIntegrity(checkType: symbol)
|
|
180
|
+
check(): boolean;
|
|
181
|
+
checkIntegrity(checkType: symbol): boolean; // one of IntegrityCheck
|
|
162
182
|
get isMultiPart(): boolean;
|
|
163
183
|
get hasNewNamespaceScheme(): boolean;
|
|
164
184
|
|
|
165
|
-
static validate(zimPath: string, checksToRun: symbol[])
|
|
185
|
+
static validate(zimPath: string, checksToRun: symbol[]): boolean; // list of IntegrityCheck
|
|
166
186
|
}
|
|
167
187
|
|
|
168
188
|
interface Georange {
|
|
@@ -175,10 +195,10 @@ export class Query {
|
|
|
175
195
|
constructor(query: string);
|
|
176
196
|
setQuery(query: string): this;
|
|
177
197
|
setGeorange(latitude: number, longitude: number, distance: number): this;
|
|
178
|
-
get query()
|
|
198
|
+
get query(): string;
|
|
179
199
|
set query(query: string);
|
|
180
200
|
toString(): string;
|
|
181
|
-
get georange()
|
|
201
|
+
get georange(): Georange;
|
|
182
202
|
set georange(range: Georange);
|
|
183
203
|
}
|
|
184
204
|
|
|
@@ -188,7 +208,7 @@ export class SearchIterator {
|
|
|
188
208
|
get score(): number;
|
|
189
209
|
get snippet(): string;
|
|
190
210
|
get wordCount(): number;
|
|
191
|
-
get size()
|
|
211
|
+
get size(): number;
|
|
192
212
|
get fileIndex(): number;
|
|
193
213
|
get zimId(): string;
|
|
194
214
|
get entry(): Entry;
|
|
@@ -199,15 +219,15 @@ export interface SearchResultSet extends Iterable<SearchIterator> {
|
|
|
199
219
|
}
|
|
200
220
|
|
|
201
221
|
export class Search {
|
|
202
|
-
getResults(start: number, maxResults: number)
|
|
222
|
+
getResults(start: number, maxResults: number): SearchResultSet;
|
|
203
223
|
get estimatedMatches(): number;
|
|
204
224
|
}
|
|
205
225
|
|
|
206
226
|
export class Searcher {
|
|
207
227
|
constructor(archives: Archive | Archive[]);
|
|
208
228
|
addArchive(archive: Archive): this;
|
|
209
|
-
search(query: string | Query)
|
|
210
|
-
setVerbose(verbose: boolean)
|
|
229
|
+
search(query: string | Query): Search;
|
|
230
|
+
setVerbose(verbose: boolean): this;
|
|
211
231
|
}
|
|
212
232
|
|
|
213
233
|
export class SuggestionIterator {
|
|
@@ -223,12 +243,12 @@ export interface SuggestionResultSet extends Iterable<SuggestionIterator> {
|
|
|
223
243
|
}
|
|
224
244
|
|
|
225
245
|
export class SuggestionSearch {
|
|
226
|
-
getResults(start: number, maxResults: number)
|
|
246
|
+
getResults(start: number, maxResults: number): SuggestionResultSet;
|
|
227
247
|
get estimatedMatches(): number;
|
|
228
248
|
}
|
|
229
249
|
|
|
230
250
|
export class SuggestionSearcher {
|
|
231
251
|
constructor(archives: Archive);
|
|
232
252
|
suggest(query: string): SuggestionSearch;
|
|
233
|
-
setVerbose(verbose: boolean)
|
|
253
|
+
setVerbose(verbose: boolean): this;
|
|
234
254
|
}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
const bindings = require('bindings');
|
|
1
|
+
import bindings from "bindings";
|
|
3
2
|
|
|
4
3
|
const {
|
|
5
4
|
Archive,
|
|
@@ -15,7 +14,7 @@ const {
|
|
|
15
14
|
FileProvider,
|
|
16
15
|
StringItem,
|
|
17
16
|
FileItem,
|
|
18
|
-
} = bindings(
|
|
17
|
+
} = bindings("zim_binding");
|
|
19
18
|
|
|
20
19
|
module.exports = {
|
|
21
20
|
Archive,
|
|
@@ -31,4 +30,4 @@ module.exports = {
|
|
|
31
30
|
FileProvider,
|
|
32
31
|
StringItem,
|
|
33
32
|
FileItem,
|
|
34
|
-
}
|
|
33
|
+
};
|
package/download-libzim.js
CHANGED
|
@@ -1,74 +1,78 @@
|
|
|
1
|
-
require(
|
|
2
|
-
const axios = require(
|
|
3
|
-
const mkdirp = require(
|
|
4
|
-
const exec = require(
|
|
5
|
-
const os = require(
|
|
6
|
-
const fs = require(
|
|
7
|
-
const urlParser = require(
|
|
1
|
+
require("dotenv").config();
|
|
2
|
+
const axios = require("axios");
|
|
3
|
+
const mkdirp = require("mkdirp");
|
|
4
|
+
const exec = require("exec-then");
|
|
5
|
+
const os = require("os");
|
|
6
|
+
const fs = require("fs");
|
|
7
|
+
const urlParser = require("url");
|
|
8
8
|
|
|
9
|
-
mkdirp.sync(
|
|
9
|
+
mkdirp.sync("./download");
|
|
10
10
|
|
|
11
|
-
const isMacOS = os.type() ===
|
|
12
|
-
const isLinux = os.type() ===
|
|
13
|
-
const rawArch = os.arch()
|
|
14
|
-
const isAvailableArch =
|
|
11
|
+
const isMacOS = os.type() === "Darwin";
|
|
12
|
+
const isLinux = os.type() === "Linux";
|
|
13
|
+
const rawArch = os.arch();
|
|
14
|
+
const isAvailableArch =
|
|
15
|
+
rawArch === "x64" || rawArch === "arm" || rawArch === "arm64";
|
|
15
16
|
|
|
16
17
|
if (!isMacOS && !isLinux) {
|
|
17
|
-
|
|
18
|
+
console.warn(
|
|
19
|
+
`\x1b[41m\n================================ README \n\nPre-built binaries only available on Linux and MacOS for now...\nPlease ensure you have libzim installed globally on this machine:\n\n\thttps://github.com/openzim/libzim/\n\n================================\x1b[0m\n`
|
|
20
|
+
);
|
|
18
21
|
}
|
|
19
22
|
if (!isAvailableArch) {
|
|
20
|
-
|
|
23
|
+
console.warn(
|
|
24
|
+
`\x1b[41m\n================================ README \n\nPre-built binaries only available on x86_64, arm and arm64 for now...\nPlease ensure you have libzim installed globally on this machine:\n\n\thttps://github.com/openzim/libzim/\n\n================================\x1b[0m\n`
|
|
25
|
+
);
|
|
21
26
|
}
|
|
22
27
|
|
|
23
|
-
let osPrefix =
|
|
24
|
-
let osArch =
|
|
28
|
+
let osPrefix = isMacOS ? "macos" : "linux";
|
|
29
|
+
let osArch = isLinux ? "x86_64-bionic" : "x86_64";
|
|
25
30
|
|
|
26
|
-
if (rawArch
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
if (rawArch !== "x64") {
|
|
32
|
+
if (isLinux) {
|
|
33
|
+
osArch = rawArch === "arm64" ? "aarch64-bionic" : "armhf";
|
|
34
|
+
} else {
|
|
35
|
+
osArch = rawArch;
|
|
36
|
+
}
|
|
32
37
|
}
|
|
33
38
|
|
|
34
39
|
const urls = [
|
|
35
|
-
|
|
36
|
-
].filter(a => a);
|
|
40
|
+
`https://download.openzim.org/release/libzim/libzim_${osPrefix}-${osArch}-${process.env.LIBZIM_VERSION}.tar.gz`,
|
|
41
|
+
].filter((a) => a);
|
|
37
42
|
|
|
38
43
|
for (let url of urls) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
44
|
+
console.info(`Downloading Libzim from: `, url);
|
|
45
|
+
const filename = urlParser.parse(url).pathname.split("/").slice(-1)[0];
|
|
46
|
+
const dlFile = `./download/${filename}`;
|
|
42
47
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
+
try {
|
|
49
|
+
fs.statSync(dlFile);
|
|
50
|
+
console.warn(`File [${dlFile}] already exists, not downloading`);
|
|
51
|
+
return;
|
|
52
|
+
} catch (err) {
|
|
53
|
+
//
|
|
54
|
+
}
|
|
48
55
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
56
|
+
axios({
|
|
57
|
+
url,
|
|
58
|
+
method: "get",
|
|
59
|
+
responseType: "stream",
|
|
60
|
+
})
|
|
61
|
+
.then(function (response) {
|
|
62
|
+
const ws = fs.createWriteStream(dlFile);
|
|
63
|
+
return new Promise((resolve, reject) => {
|
|
64
|
+
response.data.pipe(ws).on("error", reject).on("close", resolve);
|
|
65
|
+
});
|
|
53
66
|
})
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
console.log(`Running Extract:`, `[${cmd}]`);
|
|
66
|
-
return exec(cmd);
|
|
67
|
-
})
|
|
68
|
-
.then(() => {
|
|
69
|
-
console.info(`Successfully downloaded and extracted file`);
|
|
70
|
-
})
|
|
71
|
-
.catch(err => {
|
|
72
|
-
console.error(`Failed to download and extract file:`, err);
|
|
73
|
-
});
|
|
67
|
+
.then(() => {
|
|
68
|
+
const cmd = `tar --strip-components 1 -xf ${dlFile} -C ./download`;
|
|
69
|
+
console.log(`Running Extract:`, `[${cmd}]`);
|
|
70
|
+
return exec(cmd);
|
|
71
|
+
})
|
|
72
|
+
.then(() => {
|
|
73
|
+
console.info(`Successfully downloaded and extracted file`);
|
|
74
|
+
})
|
|
75
|
+
.catch((err) => {
|
|
76
|
+
console.error(`Failed to download and extract file:`, err);
|
|
77
|
+
});
|
|
74
78
|
}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@openzim/libzim",
|
|
3
3
|
"main": "dist/index.js",
|
|
4
4
|
"types": "dist/index.d.js",
|
|
5
|
-
"version": "3.
|
|
5
|
+
"version": "3.1.0",
|
|
6
6
|
"description": "Libzim bindings for NodeJS",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"clean": "rm -rf dist build/native/build",
|
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
"download": "node ./download-libzim.js",
|
|
15
15
|
"bundle": "node ./bundle-libzim.js",
|
|
16
16
|
"test": "jest",
|
|
17
|
-
"test-mem-leak": "node -r ts-node/register test/makeLargeZim.ts"
|
|
17
|
+
"test-mem-leak": "node -r ts-node/register test/makeLargeZim.ts",
|
|
18
|
+
"lint": "npx eslint .",
|
|
19
|
+
"lint:fix": "npx eslint . --fix"
|
|
18
20
|
},
|
|
19
21
|
"nyc": {
|
|
20
22
|
"extension": [
|
|
@@ -40,7 +42,7 @@
|
|
|
40
42
|
"@types/bindings": "^1.5.1",
|
|
41
43
|
"@types/jest": "^28.1.6",
|
|
42
44
|
"@types/node": "^18.0.6",
|
|
43
|
-
"axios": "^
|
|
45
|
+
"axios": "^1.6.0",
|
|
44
46
|
"bindings": "^1.5.0",
|
|
45
47
|
"dotenv": "^16.0.1",
|
|
46
48
|
"exec-then": "^1.3.1",
|
|
@@ -53,8 +55,14 @@
|
|
|
53
55
|
},
|
|
54
56
|
"devDependencies": {
|
|
55
57
|
"@faker-js/faker": "^7.6.0",
|
|
58
|
+
"@typescript-eslint/eslint-plugin": "^5.59.2",
|
|
59
|
+
"@typescript-eslint/parser": "^5.59.2",
|
|
60
|
+
"eslint": "^8.39.0",
|
|
61
|
+
"eslint-config-prettier": "^8.8.0",
|
|
62
|
+
"eslint-plugin-prettier": "^4.2.1",
|
|
56
63
|
"jest": "^28.1.3",
|
|
57
64
|
"nyc": "^15.1.0",
|
|
65
|
+
"prettier": "^2.8.8",
|
|
58
66
|
"ts-jest": "^28.0.7",
|
|
59
67
|
"typescript": "^4.7.4"
|
|
60
68
|
},
|
package/src/index.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
|
|
2
1
|
export class IntegrityCheck {
|
|
3
2
|
static CHECKSUM: symbol;
|
|
4
3
|
static DIRENT_PTRS: symbol;
|
|
@@ -6,7 +5,7 @@ export class IntegrityCheck {
|
|
|
6
5
|
static TITLE_INDEX: symbol;
|
|
7
6
|
static CLUSTER_PTRS: symbol;
|
|
8
7
|
static DIRENT_MIMETYPES: symbol;
|
|
9
|
-
static COUNT: symbol;
|
|
8
|
+
static COUNT: symbol; // DO NOT USE THIS. See libzim docs.
|
|
10
9
|
}
|
|
11
10
|
|
|
12
11
|
export class Compression {
|
|
@@ -23,25 +22,25 @@ export class Blob {
|
|
|
23
22
|
|
|
24
23
|
export type ContentProvider = {
|
|
25
24
|
size: number | bigint;
|
|
26
|
-
feed()
|
|
27
|
-
}
|
|
25
|
+
feed(): Blob;
|
|
26
|
+
};
|
|
28
27
|
|
|
29
28
|
export class StringProvider {
|
|
30
29
|
constructor(content: string);
|
|
31
30
|
get size(): number;
|
|
32
|
-
feed()
|
|
31
|
+
feed(): Blob;
|
|
33
32
|
}
|
|
34
33
|
|
|
35
34
|
export class FileProvider {
|
|
36
35
|
constructor(filepath: string);
|
|
37
36
|
get size(): number | bigint;
|
|
38
|
-
feed()
|
|
37
|
+
feed(): Blob;
|
|
39
38
|
}
|
|
40
39
|
|
|
41
40
|
export type Hint = {
|
|
42
41
|
COMPRESS?: number;
|
|
43
42
|
FRONT_ARTICLE?: number;
|
|
44
|
-
}
|
|
43
|
+
};
|
|
45
44
|
|
|
46
45
|
export interface IndexData {
|
|
47
46
|
hasIndexData?: boolean;
|
|
@@ -56,44 +55,65 @@ export interface WriterItem {
|
|
|
56
55
|
path: string;
|
|
57
56
|
title: string;
|
|
58
57
|
mimeType: string;
|
|
59
|
-
getContentProvider()
|
|
58
|
+
getContentProvider(): ContentProvider;
|
|
60
59
|
hints: Hint;
|
|
61
60
|
getIndexData?: () => IndexData;
|
|
62
61
|
}
|
|
63
62
|
|
|
64
63
|
export class StringItem {
|
|
65
|
-
constructor(
|
|
64
|
+
constructor(
|
|
65
|
+
path: string,
|
|
66
|
+
mimeType: string,
|
|
67
|
+
title: string,
|
|
68
|
+
hint: Hint,
|
|
69
|
+
content: ArrayBuffer | Buffer | string
|
|
70
|
+
);
|
|
66
71
|
readonly path: string;
|
|
67
72
|
readonly title: string;
|
|
68
73
|
readonly mimeType: string;
|
|
69
|
-
getContentProvider()
|
|
74
|
+
getContentProvider(): StringProvider;
|
|
70
75
|
readonly hints: Hint;
|
|
71
76
|
}
|
|
72
77
|
|
|
73
78
|
export class FileItem {
|
|
74
|
-
constructor(
|
|
79
|
+
constructor(
|
|
80
|
+
path: string,
|
|
81
|
+
mimeType: string,
|
|
82
|
+
title: string,
|
|
83
|
+
hints: Hint,
|
|
84
|
+
filePath: string
|
|
85
|
+
);
|
|
75
86
|
readonly path: string;
|
|
76
87
|
readonly title: string;
|
|
77
88
|
readonly mimeType: string;
|
|
78
|
-
getContentProvider()
|
|
89
|
+
getContentProvider(): StringProvider;
|
|
79
90
|
readonly hints: Hint;
|
|
80
91
|
}
|
|
81
92
|
|
|
82
93
|
export class Creator {
|
|
83
94
|
constructor();
|
|
84
|
-
configVerbose(verbose: boolean)
|
|
85
|
-
configCompression(value: Compression)
|
|
86
|
-
configClusterSize(size: number)
|
|
87
|
-
configIndexing(indexing: boolean, language: string)
|
|
88
|
-
configNbWorkers(num: number)
|
|
95
|
+
configVerbose(verbose: boolean): this;
|
|
96
|
+
configCompression(value: Compression): this;
|
|
97
|
+
configClusterSize(size: number): this;
|
|
98
|
+
configIndexing(indexing: boolean, language: string): this;
|
|
99
|
+
configNbWorkers(num: number): this;
|
|
89
100
|
startZimCreation(filepath: string): this;
|
|
90
101
|
addItem(item: WriterItem): Promise<void>;
|
|
91
|
-
addMetadata(
|
|
102
|
+
addMetadata(
|
|
103
|
+
name: string,
|
|
104
|
+
content: string | ContentProvider,
|
|
105
|
+
mimetype?: string
|
|
106
|
+
): void;
|
|
92
107
|
addIllustration(size: number, content: string | ContentProvider): void;
|
|
93
|
-
addRedirection(
|
|
108
|
+
addRedirection(
|
|
109
|
+
path: string,
|
|
110
|
+
title: string,
|
|
111
|
+
targetPath: string,
|
|
112
|
+
hints?: Hint
|
|
113
|
+
): void;
|
|
94
114
|
setMainPath(mainPath: string): void;
|
|
95
115
|
setUuid(uuid: string): void;
|
|
96
|
-
finishZimCreation()
|
|
116
|
+
finishZimCreation(): Promise<void>;
|
|
97
117
|
}
|
|
98
118
|
|
|
99
119
|
export class Item {
|
|
@@ -101,11 +121,11 @@ export class Item {
|
|
|
101
121
|
get path(): string;
|
|
102
122
|
get mimetype(): string;
|
|
103
123
|
get data(): Blob;
|
|
104
|
-
getData(offset?: number | bigint, limit?: number | bigint)
|
|
124
|
+
getData(offset?: number | bigint, limit?: number | bigint): Blob;
|
|
105
125
|
get size(): number | bigint;
|
|
106
126
|
get directAccessInformation(): {
|
|
107
|
-
filename: string
|
|
108
|
-
|
|
127
|
+
filename: string;
|
|
128
|
+
offset: number;
|
|
109
129
|
};
|
|
110
130
|
get index(): number | bigint;
|
|
111
131
|
}
|
|
@@ -115,7 +135,7 @@ export class Entry {
|
|
|
115
135
|
get title(): string;
|
|
116
136
|
get path(): string;
|
|
117
137
|
get item(): Item;
|
|
118
|
-
getItem(followRedirect?: boolean)
|
|
138
|
+
getItem(followRedirect?: boolean): Item;
|
|
119
139
|
get redirect(): Item;
|
|
120
140
|
get redirectEntry(): Entry;
|
|
121
141
|
get index(): number;
|
|
@@ -123,7 +143,7 @@ export class Entry {
|
|
|
123
143
|
|
|
124
144
|
export interface EntryRange extends Iterable<Entry> {
|
|
125
145
|
size: number;
|
|
126
|
-
offset(start: number, maxResults: number)
|
|
146
|
+
offset(start: number, maxResults: number): EntryRange;
|
|
127
147
|
}
|
|
128
148
|
|
|
129
149
|
export class Archive {
|
|
@@ -134,35 +154,35 @@ export class Archive {
|
|
|
134
154
|
get entryCount(): number;
|
|
135
155
|
get articleCount(): number;
|
|
136
156
|
get uuid(): string;
|
|
137
|
-
getMetadata(name: string)
|
|
138
|
-
getMetadataItem(name: string)
|
|
157
|
+
getMetadata(name: string): string;
|
|
158
|
+
getMetadataItem(name: string): Item;
|
|
139
159
|
get metadataKeys(): string[];
|
|
140
|
-
getIllustrationItem(size: number)
|
|
141
|
-
get illustrationSizes()
|
|
142
|
-
getEntryByPath(path_or_idx: string | number)
|
|
143
|
-
getEntryByTitle(title_or_idx: string | number)
|
|
144
|
-
getEntryByClusterOrder(idx: number)
|
|
160
|
+
getIllustrationItem(size: number): Item;
|
|
161
|
+
get illustrationSizes(): Set<number>;
|
|
162
|
+
getEntryByPath(path_or_idx: string | number): Entry;
|
|
163
|
+
getEntryByTitle(title_or_idx: string | number): Entry;
|
|
164
|
+
getEntryByClusterOrder(idx: number): Entry;
|
|
145
165
|
get mainEntry(): Entry;
|
|
146
166
|
get randomEntry(): Entry;
|
|
147
|
-
hasEntryByPath(path: string)
|
|
148
|
-
hasEntryByTitle(title: string)
|
|
167
|
+
hasEntryByPath(path: string): boolean;
|
|
168
|
+
hasEntryByTitle(title: string): boolean;
|
|
149
169
|
hasMainEntry(): boolean;
|
|
150
|
-
hasIllustration(size: number)
|
|
151
|
-
hasFulltextIndex()
|
|
170
|
+
hasIllustration(size: number): boolean;
|
|
171
|
+
hasFulltextIndex(): boolean;
|
|
152
172
|
hasTitleIndex(): boolean;
|
|
153
|
-
iterByPath()
|
|
154
|
-
iterByTitle()
|
|
155
|
-
iterEfficient()
|
|
156
|
-
findByPath(path: string)
|
|
157
|
-
findByTitle(title: string)
|
|
173
|
+
iterByPath(): EntryRange;
|
|
174
|
+
iterByTitle(): EntryRange;
|
|
175
|
+
iterEfficient(): EntryRange;
|
|
176
|
+
findByPath(path: string): EntryRange;
|
|
177
|
+
findByTitle(title: string): EntryRange;
|
|
158
178
|
get hasChecksum(): boolean;
|
|
159
179
|
get checksum(): string;
|
|
160
|
-
check()
|
|
161
|
-
checkIntegrity(checkType: symbol)
|
|
180
|
+
check(): boolean;
|
|
181
|
+
checkIntegrity(checkType: symbol): boolean; // one of IntegrityCheck
|
|
162
182
|
get isMultiPart(): boolean;
|
|
163
183
|
get hasNewNamespaceScheme(): boolean;
|
|
164
184
|
|
|
165
|
-
static validate(zimPath: string, checksToRun: symbol[])
|
|
185
|
+
static validate(zimPath: string, checksToRun: symbol[]): boolean; // list of IntegrityCheck
|
|
166
186
|
}
|
|
167
187
|
|
|
168
188
|
interface Georange {
|
|
@@ -175,10 +195,10 @@ export class Query {
|
|
|
175
195
|
constructor(query: string);
|
|
176
196
|
setQuery(query: string): this;
|
|
177
197
|
setGeorange(latitude: number, longitude: number, distance: number): this;
|
|
178
|
-
get query()
|
|
198
|
+
get query(): string;
|
|
179
199
|
set query(query: string);
|
|
180
200
|
toString(): string;
|
|
181
|
-
get georange()
|
|
201
|
+
get georange(): Georange;
|
|
182
202
|
set georange(range: Georange);
|
|
183
203
|
}
|
|
184
204
|
|
|
@@ -188,7 +208,7 @@ export class SearchIterator {
|
|
|
188
208
|
get score(): number;
|
|
189
209
|
get snippet(): string;
|
|
190
210
|
get wordCount(): number;
|
|
191
|
-
get size()
|
|
211
|
+
get size(): number;
|
|
192
212
|
get fileIndex(): number;
|
|
193
213
|
get zimId(): string;
|
|
194
214
|
get entry(): Entry;
|
|
@@ -199,15 +219,15 @@ export interface SearchResultSet extends Iterable<SearchIterator> {
|
|
|
199
219
|
}
|
|
200
220
|
|
|
201
221
|
export class Search {
|
|
202
|
-
getResults(start: number, maxResults: number)
|
|
222
|
+
getResults(start: number, maxResults: number): SearchResultSet;
|
|
203
223
|
get estimatedMatches(): number;
|
|
204
224
|
}
|
|
205
225
|
|
|
206
226
|
export class Searcher {
|
|
207
227
|
constructor(archives: Archive | Archive[]);
|
|
208
228
|
addArchive(archive: Archive): this;
|
|
209
|
-
search(query: string | Query)
|
|
210
|
-
setVerbose(verbose: boolean)
|
|
229
|
+
search(query: string | Query): Search;
|
|
230
|
+
setVerbose(verbose: boolean): this;
|
|
211
231
|
}
|
|
212
232
|
|
|
213
233
|
export class SuggestionIterator {
|
|
@@ -223,12 +243,12 @@ export interface SuggestionResultSet extends Iterable<SuggestionIterator> {
|
|
|
223
243
|
}
|
|
224
244
|
|
|
225
245
|
export class SuggestionSearch {
|
|
226
|
-
getResults(start: number, maxResults: number)
|
|
246
|
+
getResults(start: number, maxResults: number): SuggestionResultSet;
|
|
227
247
|
get estimatedMatches(): number;
|
|
228
248
|
}
|
|
229
249
|
|
|
230
250
|
export class SuggestionSearcher {
|
|
231
251
|
constructor(archives: Archive);
|
|
232
252
|
suggest(query: string): SuggestionSearch;
|
|
233
|
-
setVerbose(verbose: boolean)
|
|
253
|
+
setVerbose(verbose: boolean): this;
|
|
234
254
|
}
|
package/src/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
const bindings = require('bindings');
|
|
1
|
+
import bindings from "bindings";
|
|
3
2
|
|
|
4
3
|
const {
|
|
5
4
|
Archive,
|
|
@@ -15,7 +14,7 @@ const {
|
|
|
15
14
|
FileProvider,
|
|
16
15
|
StringItem,
|
|
17
16
|
FileItem,
|
|
18
|
-
} = bindings(
|
|
17
|
+
} = bindings("zim_binding");
|
|
19
18
|
|
|
20
19
|
module.exports = {
|
|
21
20
|
Archive,
|
|
@@ -31,4 +30,4 @@ module.exports = {
|
|
|
31
30
|
FileProvider,
|
|
32
31
|
StringItem,
|
|
33
32
|
FileItem,
|
|
34
|
-
}
|
|
33
|
+
};
|
package/src/writerItem.h
CHANGED
|
@@ -262,7 +262,22 @@ class StringItem : public Napi::ObjectWrap<StringItem> {
|
|
|
262
262
|
auto mimetype = info[1].ToString().Utf8Value();
|
|
263
263
|
auto title = info[2].ToString().Utf8Value();
|
|
264
264
|
auto hints = Object2Hints(info[3].ToObject());
|
|
265
|
-
|
|
265
|
+
|
|
266
|
+
const auto &&cval = info[4];
|
|
267
|
+
std::string content;
|
|
268
|
+
|
|
269
|
+
if (cval.IsArrayBuffer()) { // handle ArrayBuffer
|
|
270
|
+
auto buf = cval.As<Napi::ArrayBuffer>();
|
|
271
|
+
auto size = buf.ByteLength();
|
|
272
|
+
content = std::string(reinterpret_cast<const char *>(buf.Data()), size);
|
|
273
|
+
} else if (cval.IsBuffer()) { // handle Buffer
|
|
274
|
+
auto buf = cval.As<Napi::Buffer<char>>();
|
|
275
|
+
auto size = buf.Length();
|
|
276
|
+
content = std::string(reinterpret_cast<const char *>(buf.Data()), size);
|
|
277
|
+
} else {
|
|
278
|
+
content = cval.ToString().Utf8Value();
|
|
279
|
+
}
|
|
280
|
+
|
|
266
281
|
item_ = zim::writer::StringItem::create(path, mimetype, title, hints,
|
|
267
282
|
content);
|
|
268
283
|
} catch (const std::exception &e) {
|