@lgrammel/kiwix-tool 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -13
- package/dist/index.d.ts +49 -50
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +111 -73
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -1,32 +1,105 @@
|
|
|
1
1
|
# @lgrammel/kiwix-tool
|
|
2
2
|
|
|
3
|
-
AI SDK 7
|
|
3
|
+
AI SDK 7 tool package for searching and reading local [Kiwix](https://kiwix.org/) `.zim` archives from Node.js.
|
|
4
|
+
|
|
5
|
+
Use it when a local agent should answer from an offline knowledge base such as Wikipedia, Stack Exchange, project docs, or another ZIM archive.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
bun add @lgrammel/kiwix-tool
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
This package uses `@openzim/libzim` under the hood and must run in a Node.js-compatible environment.
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
4
16
|
|
|
5
17
|
```ts
|
|
6
|
-
import {
|
|
18
|
+
import { kiwixReadPage, kiwixSearch } from "@lgrammel/kiwix-tool";
|
|
19
|
+
import { openai } from "@ai-sdk/openai";
|
|
7
20
|
import { ToolLoopAgent } from "ai";
|
|
8
21
|
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
22
|
+
const prompt =
|
|
23
|
+
process.argv.slice(2).join(" ") || "Explain what Kiwix is in three concise bullet points.";
|
|
24
|
+
|
|
25
|
+
const kiwixArchiveContext = {
|
|
26
|
+
zimPath: process.env.WIKIPEDIA_ZIM_PATH ?? "~/opt/zim/wikipedia.zim"
|
|
27
|
+
};
|
|
12
28
|
|
|
13
29
|
const agent = new ToolLoopAgent({
|
|
14
|
-
model,
|
|
15
|
-
instructions:
|
|
30
|
+
model: openai("gpt-5.5"),
|
|
31
|
+
instructions:
|
|
32
|
+
"Answer using the local Wikipedia archive. Search before reading pages, and cite the page titles you used.",
|
|
16
33
|
tools: {
|
|
17
|
-
wikipediaSearch:
|
|
18
|
-
wikipediaRead:
|
|
34
|
+
wikipediaSearch: kiwixSearch,
|
|
35
|
+
wikipediaRead: kiwixReadPage
|
|
36
|
+
},
|
|
37
|
+
toolsContext: {
|
|
38
|
+
wikipediaSearch: {
|
|
39
|
+
...kiwixArchiveContext,
|
|
40
|
+
searchResultLimit: 5
|
|
41
|
+
},
|
|
42
|
+
wikipediaRead: {
|
|
43
|
+
...kiwixArchiveContext,
|
|
44
|
+
readMaxBytes: 80 * 1024
|
|
45
|
+
}
|
|
19
46
|
}
|
|
20
47
|
});
|
|
21
48
|
|
|
22
49
|
const result = await agent.generate({
|
|
23
|
-
prompt
|
|
50
|
+
prompt
|
|
24
51
|
});
|
|
52
|
+
|
|
53
|
+
console.log(result.text);
|
|
25
54
|
```
|
|
26
55
|
|
|
27
56
|
## Tools
|
|
28
57
|
|
|
29
|
-
- `
|
|
30
|
-
- `
|
|
58
|
+
- `kiwixSearch`: full-text search over the archive. Input is `{ query }`. Output is `results` with `title`, `path`, and optional `snippet`.
|
|
59
|
+
- `kiwixReadPage`: read a page by exact path returned from search. Input is `{ path }`. Output is `title`, `path`, `content`, and `truncated`.
|
|
60
|
+
|
|
61
|
+
The model cannot choose result or read limits. Search result count and page byte limits are configured only on the tool that uses them through each tool's `toolsContext` entry, validated by that tool's `contextSchema`, so tool outputs stay predictable for agents. HTML pages are converted to UTF-8 text before returning them to the model.
|
|
62
|
+
|
|
63
|
+
## API
|
|
64
|
+
|
|
65
|
+
Use the exported tools directly and pass tool-specific configuration through `toolsContext`:
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
import { kiwixReadPage, kiwixSearch } from "@lgrammel/kiwix-tool";
|
|
69
|
+
|
|
70
|
+
const kiwixArchiveContext = {
|
|
71
|
+
zimPath: "~/opt/zim/wikipedia.zim"
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
const tools = {
|
|
75
|
+
wikipediaSearch: kiwixSearch,
|
|
76
|
+
wikipediaRead: kiwixReadPage
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const toolsContext = {
|
|
80
|
+
wikipediaSearch: {
|
|
81
|
+
...kiwixArchiveContext,
|
|
82
|
+
searchResultLimit: 5
|
|
83
|
+
},
|
|
84
|
+
wikipediaRead: {
|
|
85
|
+
...kiwixArchiveContext,
|
|
86
|
+
readMaxBytes: 80 * 1024
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Each tool keeps its archive connection private and refreshes it if the archive context changes.
|
|
92
|
+
|
|
93
|
+
## Shared Archive Context
|
|
94
|
+
|
|
95
|
+
- `zimPath`: path to the `.zim` file. `~/` is expanded to the current user's home directory.
|
|
96
|
+
- `preloadXapianDb`: preload the full-text index when opening the archive. Defaults to `true`.
|
|
97
|
+
- `preloadDirentRanges`: number of directory entry ranges to preload when opening the archive.
|
|
98
|
+
|
|
99
|
+
## Search Context
|
|
100
|
+
|
|
101
|
+
- `searchResultLimit`: fixed number of search results returned to the agent. Defaults to `5` and is capped at `10`.
|
|
102
|
+
|
|
103
|
+
## Read Context
|
|
31
104
|
|
|
32
|
-
|
|
105
|
+
- `readMaxBytes`: maximum page bytes read before conversion to text. Defaults to `81920` and is capped at `524288`.
|
package/dist/index.d.ts
CHANGED
|
@@ -1,71 +1,70 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { ToolExecutionOptions } from "ai";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
-
|
|
3
|
+
declare const kiwixSearchInputSchema: z.ZodObject<{
|
|
4
4
|
query: z.ZodString;
|
|
5
5
|
}, z.core.$strip>;
|
|
6
|
-
|
|
6
|
+
declare const kiwixReadInputSchema: z.ZodObject<{
|
|
7
7
|
path: z.ZodString;
|
|
8
8
|
}, z.core.$strip>;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Preload the full-text index when opening the archive.
|
|
28
|
-
*/
|
|
29
|
-
preloadXapianDb?: boolean;
|
|
30
|
-
/**
|
|
31
|
-
* Number of directory entry ranges to preload when opening the archive.
|
|
32
|
-
*/
|
|
33
|
-
preloadDirentRanges?: number;
|
|
34
|
-
}
|
|
35
|
-
export interface KiwixSearchResult {
|
|
9
|
+
declare const kiwixSearchContextSchema: z.ZodObject<{
|
|
10
|
+
zimPath: z.ZodString;
|
|
11
|
+
preloadXapianDb: z.ZodOptional<z.ZodBoolean>;
|
|
12
|
+
preloadDirentRanges: z.ZodOptional<z.ZodNumber>;
|
|
13
|
+
searchResultLimit: z.ZodOptional<z.ZodNumber>;
|
|
14
|
+
}, z.core.$strip>;
|
|
15
|
+
declare const kiwixReadPageContextSchema: z.ZodObject<{
|
|
16
|
+
zimPath: z.ZodString;
|
|
17
|
+
preloadXapianDb: z.ZodOptional<z.ZodBoolean>;
|
|
18
|
+
preloadDirentRanges: z.ZodOptional<z.ZodNumber>;
|
|
19
|
+
readMaxBytes: z.ZodOptional<z.ZodNumber>;
|
|
20
|
+
}, z.core.$strip>;
|
|
21
|
+
type ParsedKiwixSearchInput = z.output<typeof kiwixSearchInputSchema>;
|
|
22
|
+
type ParsedKiwixReadInput = z.output<typeof kiwixReadInputSchema>;
|
|
23
|
+
type KiwixSearchContext = z.output<typeof kiwixSearchContextSchema>;
|
|
24
|
+
type KiwixReadPageContext = z.output<typeof kiwixReadPageContextSchema>;
|
|
25
|
+
interface KiwixSearchResult {
|
|
36
26
|
title: string;
|
|
37
27
|
path: string;
|
|
38
28
|
snippet?: string;
|
|
39
29
|
}
|
|
40
|
-
|
|
30
|
+
interface KiwixSearchOutput {
|
|
41
31
|
results: KiwixSearchResult[];
|
|
42
32
|
}
|
|
43
|
-
|
|
33
|
+
interface KiwixReadOutput {
|
|
44
34
|
title: string;
|
|
45
35
|
path: string;
|
|
46
36
|
content: string;
|
|
47
37
|
truncated: boolean;
|
|
48
38
|
}
|
|
49
|
-
|
|
50
|
-
export type KiwixReadTool = Tool<ParsedKiwixReadInput, KiwixReadOutput>;
|
|
51
|
-
export declare class KiwixReader {
|
|
39
|
+
declare class KiwixSearchTool {
|
|
52
40
|
#private;
|
|
53
|
-
readonly
|
|
54
|
-
readonly
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
41
|
+
readonly description = "Search the local Kiwix archive. Returns page paths, titles, and short snippets. Use the read tool with a returned path to read a page.";
|
|
42
|
+
readonly inputSchema: z.ZodObject<{
|
|
43
|
+
query: z.ZodString;
|
|
44
|
+
}, z.core.$strip>;
|
|
45
|
+
readonly contextSchema: z.ZodObject<{
|
|
46
|
+
zimPath: z.ZodString;
|
|
47
|
+
preloadXapianDb: z.ZodOptional<z.ZodBoolean>;
|
|
48
|
+
preloadDirentRanges: z.ZodOptional<z.ZodNumber>;
|
|
49
|
+
searchResultLimit: z.ZodOptional<z.ZodNumber>;
|
|
50
|
+
}, z.core.$strip>;
|
|
51
|
+
execute: ({ query }: ParsedKiwixSearchInput, { context }: ToolExecutionOptions<KiwixSearchContext>) => Promise<KiwixSearchOutput>;
|
|
59
52
|
}
|
|
60
|
-
|
|
53
|
+
declare class KiwixReadPageTool {
|
|
61
54
|
#private;
|
|
62
|
-
readonly
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
55
|
+
readonly description = "Read one page from the local Kiwix archive by exact path from the search tool.";
|
|
56
|
+
readonly inputSchema: z.ZodObject<{
|
|
57
|
+
path: z.ZodString;
|
|
58
|
+
}, z.core.$strip>;
|
|
59
|
+
readonly contextSchema: z.ZodObject<{
|
|
60
|
+
zimPath: z.ZodString;
|
|
61
|
+
preloadXapianDb: z.ZodOptional<z.ZodBoolean>;
|
|
62
|
+
preloadDirentRanges: z.ZodOptional<z.ZodNumber>;
|
|
63
|
+
readMaxBytes: z.ZodOptional<z.ZodNumber>;
|
|
64
|
+
}, z.core.$strip>;
|
|
65
|
+
execute: ({ path }: ParsedKiwixReadInput, { context }: ToolExecutionOptions<KiwixReadPageContext>) => Promise<KiwixReadOutput>;
|
|
66
66
|
}
|
|
67
|
-
export declare
|
|
68
|
-
export declare
|
|
69
|
-
export
|
|
70
|
-
export declare function kiwixReadTool(reader: KiwixReader): KiwixReadTool;
|
|
67
|
+
export declare const kiwixSearch: KiwixSearchTool;
|
|
68
|
+
export declare const kiwixReadPage: KiwixReadPageTool;
|
|
69
|
+
export {};
|
|
71
70
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAQ,oBAAoB,EAAE,MAAM,IAAI,CAAC;AAKrD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,QAAA,MAAM,sBAAsB;;iBAE1B,CAAC;AAEH,QAAA,MAAM,oBAAoB;;iBAExB,CAAC;AAqBH,QAAA,MAAM,wBAAwB;;;;;iBAU5B,CAAC;AAEH,QAAA,MAAM,0BAA0B;;;;;iBAQ9B,CAAC;AAEH,KAAK,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,KAAK,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,KAAK,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,wBAAwB,CAAC,CAAC;AACpE,KAAK,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAExE,UAAU,iBAAiB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,iBAAiB;IACzB,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;CACpB;AAsCD,cAAM,eAAe;;IACnB,QAAQ,CAAC,WAAW,4IACuH;IAC3I,QAAQ,CAAC,WAAW;;sBAA0B;IAC9C,QAAQ,CAAC,aAAa;;;;;sBAA4B;IAMlD,OAAO,GACL,WAAW,sBAAsB,EACjC,aAAa,oBAAoB,CAAC,kBAAkB,CAAC,KACpD,OAAO,CAAC,iBAAiB,CAAC,CAiB3B;CAiBH;AAED,cAAM,iBAAiB;;IACrB,QAAQ,CAAC,WAAW,oFAC+D;IACnF,QAAQ,CAAC,WAAW;;sBAAwB;IAC5C,QAAQ,CAAC,aAAa;;;;;sBAA8B;IAKpD,OAAO,GACL,UAAU,oBAAoB,EAC9B,aAAa,oBAAoB,CAAC,oBAAoB,CAAC,KACtD,OAAO,CAAC,eAAe,CAAC,CAazB;CAgBH;AAED,eAAO,MAAM,WAAW,iBAIvB,CAAC;AAEF,eAAO,MAAM,aAAa,mBAIzB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { tool } from "ai";
|
|
2
1
|
import { convert as htmlToText } from "html-to-text";
|
|
3
2
|
import { homedir } from "node:os";
|
|
4
3
|
import { isAbsolute, join } from "node:path";
|
|
@@ -8,105 +7,144 @@ const MAX_SEARCH_RESULT_LIMIT = 10;
|
|
|
8
7
|
const DEFAULT_READ_MAX_BYTES = 80 * 1024;
|
|
9
8
|
const HARD_READ_MAX_BYTES = 512 * 1024;
|
|
10
9
|
const MAX_SNIPPET_LENGTH = 600;
|
|
11
|
-
|
|
10
|
+
const kiwixSearchInputSchema = z.object({
|
|
12
11
|
query: z.string().min(1).describe("Full-text search query for the Kiwix archive."),
|
|
13
12
|
});
|
|
14
|
-
|
|
13
|
+
const kiwixReadInputSchema = z.object({
|
|
15
14
|
path: z.string().min(1).describe("Exact ZIM page path returned by the search tool."),
|
|
16
15
|
});
|
|
17
|
-
|
|
16
|
+
const kiwixArchiveContextSchema = z.object({
|
|
17
|
+
zimPath: z
|
|
18
|
+
.string()
|
|
19
|
+
.min(1)
|
|
20
|
+
.describe("Path to the `.zim` file. A leading `~/` is expanded to the current user's home directory."),
|
|
21
|
+
preloadXapianDb: z
|
|
22
|
+
.boolean()
|
|
23
|
+
.optional()
|
|
24
|
+
.describe("Preload the full-text index when opening the archive. Defaults to true."),
|
|
25
|
+
preloadDirentRanges: z
|
|
26
|
+
.number()
|
|
27
|
+
.int()
|
|
28
|
+
.nonnegative()
|
|
29
|
+
.optional()
|
|
30
|
+
.describe("Number of directory entry ranges to preload when opening the archive."),
|
|
31
|
+
});
|
|
32
|
+
const kiwixSearchContextSchema = kiwixArchiveContextSchema.extend({
|
|
33
|
+
searchResultLimit: z
|
|
34
|
+
.number()
|
|
35
|
+
.int()
|
|
36
|
+
.positive()
|
|
37
|
+
.max(MAX_SEARCH_RESULT_LIMIT)
|
|
38
|
+
.optional()
|
|
39
|
+
.describe("Fixed number of search results returned to the agent. Defaults to 5 and is capped at 10."),
|
|
40
|
+
});
|
|
41
|
+
const kiwixReadPageContextSchema = kiwixArchiveContextSchema.extend({
|
|
42
|
+
readMaxBytes: z
|
|
43
|
+
.number()
|
|
44
|
+
.int()
|
|
45
|
+
.positive()
|
|
46
|
+
.max(HARD_READ_MAX_BYTES)
|
|
47
|
+
.optional()
|
|
48
|
+
.describe("Fixed maximum page bytes read before HTML-to-text conversion. Defaults to 81920."),
|
|
49
|
+
});
|
|
50
|
+
class KiwixArchive {
|
|
18
51
|
zimPath;
|
|
19
|
-
searchResultLimit;
|
|
20
|
-
readMaxBytes;
|
|
21
52
|
#archive;
|
|
22
53
|
#libzim;
|
|
23
|
-
#
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
this
|
|
27
|
-
this.searchResultLimit = clampSearchResultLimit(options.searchResultLimit ?? DEFAULT_SEARCH_RESULT_LIMIT);
|
|
28
|
-
this.readMaxBytes = clampReadMaxBytes(options.readMaxBytes ?? DEFAULT_READ_MAX_BYTES);
|
|
29
|
-
this.#options = options;
|
|
54
|
+
#context;
|
|
55
|
+
constructor(context) {
|
|
56
|
+
this.zimPath = resolveHomePath(context.zimPath);
|
|
57
|
+
this.#context = context;
|
|
30
58
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
59
|
+
get() {
|
|
60
|
+
this.#archive ??= this.#openArchive();
|
|
61
|
+
return this.#archive;
|
|
62
|
+
}
|
|
63
|
+
loadLibzim() {
|
|
64
|
+
this.#libzim ??= import("@openzim/libzim");
|
|
65
|
+
return this.#libzim;
|
|
66
|
+
}
|
|
67
|
+
async #openArchive() {
|
|
68
|
+
const { Archive, OpenConfig } = await this.loadLibzim();
|
|
69
|
+
const config = new OpenConfig();
|
|
70
|
+
config.preloadXapianDb(this.#context.preloadXapianDb ?? true);
|
|
71
|
+
if (this.#context.preloadDirentRanges !== undefined) {
|
|
72
|
+
config.preloadDirentRanges(this.#context.preloadDirentRanges);
|
|
73
|
+
}
|
|
74
|
+
return new Archive(this.zimPath, config);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
class KiwixSearchTool {
|
|
78
|
+
description = "Search the local Kiwix archive. Returns page paths, titles, and short snippets. Use the read tool with a returned path to read a page.";
|
|
79
|
+
inputSchema = kiwixSearchInputSchema;
|
|
80
|
+
contextSchema = kiwixSearchContextSchema;
|
|
81
|
+
#searcher;
|
|
82
|
+
#archive;
|
|
83
|
+
#archiveCacheKey;
|
|
84
|
+
execute = async ({ query }, { context }) => {
|
|
85
|
+
const archive = this.#getArchive(context);
|
|
86
|
+
const { Searcher } = await archive.loadLibzim();
|
|
87
|
+
this.#searcher ??= new Searcher(await archive.get());
|
|
34
88
|
const search = this.#searcher.search(query);
|
|
35
|
-
const
|
|
89
|
+
const searchResultLimit = clampSearchResultLimit(context.searchResultLimit ?? DEFAULT_SEARCH_RESULT_LIMIT);
|
|
90
|
+
const results = Array.from(search.getResults(0, searchResultLimit)).map((result) => ({
|
|
36
91
|
title: result.title,
|
|
37
92
|
path: result.path,
|
|
38
93
|
snippet: cleanSnippet(result.snippet),
|
|
39
94
|
}));
|
|
40
95
|
return { results };
|
|
96
|
+
};
|
|
97
|
+
#getArchive(context) {
|
|
98
|
+
const archiveCacheKey = getArchiveCacheKey(context);
|
|
99
|
+
if (archiveCacheKey !== this.#archiveCacheKey) {
|
|
100
|
+
this.#archive = new KiwixArchive(context);
|
|
101
|
+
this.#archiveCacheKey = archiveCacheKey;
|
|
102
|
+
this.#searcher = undefined;
|
|
103
|
+
}
|
|
104
|
+
if (!this.#archive) {
|
|
105
|
+
throw new Error("Kiwix archive was not initialized.");
|
|
106
|
+
}
|
|
107
|
+
return this.#archive;
|
|
41
108
|
}
|
|
42
|
-
|
|
43
|
-
|
|
109
|
+
}
|
|
110
|
+
class KiwixReadPageTool {
|
|
111
|
+
description = "Read one page from the local Kiwix archive by exact path from the search tool.";
|
|
112
|
+
inputSchema = kiwixReadInputSchema;
|
|
113
|
+
contextSchema = kiwixReadPageContextSchema;
|
|
114
|
+
#archive;
|
|
115
|
+
#archiveCacheKey;
|
|
116
|
+
execute = async ({ path }, { context }) => {
|
|
117
|
+
const entry = (await this.#getArchive(context).get()).getEntryByPath(path);
|
|
44
118
|
const item = entry.getItem(true);
|
|
45
119
|
const size = toSafeNumber(item.size);
|
|
46
|
-
const
|
|
120
|
+
const readMaxBytes = clampReadMaxBytes(context.readMaxBytes ?? DEFAULT_READ_MAX_BYTES);
|
|
121
|
+
const data = item.getData(0, Math.min(readMaxBytes, size)).data;
|
|
47
122
|
return {
|
|
48
123
|
title: entry.title,
|
|
49
124
|
path: entry.path,
|
|
50
125
|
content: formatPageContent(item, data),
|
|
51
126
|
truncated: data.byteLength < size,
|
|
52
127
|
};
|
|
53
|
-
}
|
|
54
|
-
#getArchive() {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
const { Archive, OpenConfig } = await this.#loadLibzim();
|
|
60
|
-
const config = new OpenConfig();
|
|
61
|
-
if (this.#options.preloadXapianDb !== undefined) {
|
|
62
|
-
config.preloadXapianDb(this.#options.preloadXapianDb);
|
|
128
|
+
};
|
|
129
|
+
#getArchive(context) {
|
|
130
|
+
const archiveCacheKey = getArchiveCacheKey(context);
|
|
131
|
+
if (archiveCacheKey !== this.#archiveCacheKey) {
|
|
132
|
+
this.#archive = new KiwixArchive(context);
|
|
133
|
+
this.#archiveCacheKey = archiveCacheKey;
|
|
63
134
|
}
|
|
64
|
-
if (this.#
|
|
65
|
-
|
|
135
|
+
if (!this.#archive) {
|
|
136
|
+
throw new Error("Kiwix archive was not initialized.");
|
|
66
137
|
}
|
|
67
|
-
return
|
|
68
|
-
}
|
|
69
|
-
#loadLibzim() {
|
|
70
|
-
this.#libzim ??= import("@openzim/libzim");
|
|
71
|
-
return this.#libzim;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
export class KiwixTools {
|
|
75
|
-
reader;
|
|
76
|
-
#searchTool;
|
|
77
|
-
#readTool;
|
|
78
|
-
constructor(options) {
|
|
79
|
-
this.reader = new KiwixReader(options);
|
|
80
|
-
}
|
|
81
|
-
get searchTool() {
|
|
82
|
-
this.#searchTool ??= kiwixSearchTool(this.reader);
|
|
83
|
-
return this.#searchTool;
|
|
84
|
-
}
|
|
85
|
-
get readTool() {
|
|
86
|
-
this.#readTool ??= kiwixReadTool(this.reader);
|
|
87
|
-
return this.#readTool;
|
|
138
|
+
return this.#archive;
|
|
88
139
|
}
|
|
89
140
|
}
|
|
90
|
-
export
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
}
|
|
98
|
-
export function kiwixSearchTool(reader) {
|
|
99
|
-
return tool({
|
|
100
|
-
description: "Search the local Kiwix archive. Returns page paths, titles, and short snippets. Use the read tool with a returned path to read a page.",
|
|
101
|
-
inputSchema: kiwixSearchInputSchema,
|
|
102
|
-
execute: async ({ query }) => reader.search(query),
|
|
103
|
-
});
|
|
104
|
-
}
|
|
105
|
-
export function kiwixReadTool(reader) {
|
|
106
|
-
return tool({
|
|
107
|
-
description: "Read one page from the local Kiwix archive by exact path from the search tool.",
|
|
108
|
-
inputSchema: kiwixReadInputSchema,
|
|
109
|
-
execute: async ({ path }) => reader.readPath(path),
|
|
141
|
+
export const kiwixSearch = new KiwixSearchTool();
|
|
142
|
+
export const kiwixReadPage = new KiwixReadPageTool();
|
|
143
|
+
function getArchiveCacheKey(context) {
|
|
144
|
+
return JSON.stringify({
|
|
145
|
+
zimPath: resolveHomePath(context.zimPath),
|
|
146
|
+
preloadXapianDb: context.preloadXapianDb,
|
|
147
|
+
preloadDirentRanges: context.preloadDirentRanges,
|
|
110
148
|
});
|
|
111
149
|
}
|
|
112
150
|
function formatPageContent(item, data) {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,2BAA2B,GAAG,CAAC,CAAC;AACtC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAC;AACzC,MAAM,mBAAmB,GAAG,GAAG,GAAG,IAAI,CAAC;AACvC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAI/B,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,+CAA+C,CAAC;CACnF,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,kDAAkD,CAAC;CACrF,CAAC,CAAC;AAEH,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,2FAA2F,CAC5F;IACH,eAAe,EAAE,CAAC;SACf,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,yEAAyE,CAAC;IACtF,mBAAmB,EAAE,CAAC;SACnB,MAAM,EAAE;SACR,GAAG,EAAE;SACL,WAAW,EAAE;SACb,QAAQ,EAAE;SACV,QAAQ,CAAC,uEAAuE,CAAC;CACrF,CAAC,CAAC;AAEH,MAAM,wBAAwB,GAAG,yBAAyB,CAAC,MAAM,CAAC;IAChE,iBAAiB,EAAE,CAAC;SACjB,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,GAAG,CAAC,uBAAuB,CAAC;SAC5B,QAAQ,EAAE;SACV,QAAQ,CACP,0FAA0F,CAC3F;CACJ,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAAG,yBAAyB,CAAC,MAAM,CAAC;IAClE,YAAY,EAAE,CAAC;SACZ,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,GAAG,CAAC,mBAAmB,CAAC;SACxB,QAAQ,EAAE;SACV,QAAQ,CAAC,kFAAkF,CAAC;CAChG,CAAC,CAAC;AAyBH,MAAM,YAAY;IACP,OAAO,CAAS;IAEzB,QAAQ,CAAoB;IAC5B,OAAO,CAAyB;IACvB,QAAQ,CAAsB;IAEvC,YAAY,OAA4B;QACtC,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC1B,CAAC;IAED,GAAG;QACD,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,UAAU;QACR,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAEhC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC;QAE9D,IAAI,IAAI,CAAC,QAAQ,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;YACpD,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QAChE,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,eAAe;IACV,WAAW,GAClB,wIAAwI,CAAC;IAClI,WAAW,GAAG,sBAAsB,CAAC;IACrC,aAAa,GAAG,wBAAwB,CAAC;IAElD,SAAS,CAAY;IACrB,QAAQ,CAAgB;IACxB,gBAAgB,CAAU;IAE1B,OAAO,GAAG,KAAK,EACb,EAAE,KAAK,EAA0B,EACjC,EAAE,OAAO,EAA4C,EACzB,EAAE;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAChD,IAAI,CAAC,SAAS,KAAK,IAAI,QAAQ,CAAC,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,iBAAiB,GAAG,sBAAsB,CAC9C,OAAO,CAAC,iBAAiB,IAAI,2BAA2B,CACzD,CAAC;QACF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,iBAAiB,CAA6B,CACpE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACjB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC;SACtC,CAAC,CAAC,CAAC;QAEJ,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC,CAAC;IAEF,WAAW,CAAC,OAA2B;QACrC,MAAM,eAAe,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEpD,IAAI,eAAe,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;YACxC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;CACF;AAED,MAAM,iBAAiB;IACZ,WAAW,GAClB,gFAAgF,CAAC;IAC1E,WAAW,GAAG,oBAAoB,CAAC;IACnC,aAAa,GAAG,0BAA0B,CAAC;IAEpD,QAAQ,CAAgB;IACxB,gBAAgB,CAAU;IAE1B,OAAO,GAAG,KAAK,EACb,EAAE,IAAI,EAAwB,EAC9B,EAAE,OAAO,EAA8C,EAC7B,EAAE;QAC5B,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,iBAAiB,CAAC,OAAO,CAAC,YAAY,IAAI,sBAAsB,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAEhE,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC;YACtC,SAAS,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI;SAClC,CAAC;IACJ,CAAC,CAAC;IAEF,WAAW,CAAC,OAA6B;QACvC,MAAM,eAAe,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEpD,IAAI,eAAe,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;CACF;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,eAAe,EAI7C,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,iBAAiB,EAIjD,CAAC;AAEF,SAAS,kBAAkB,CAAC,OAA4B;IACtD,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;KACjD,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAU,EAAE,IAAY;IACjD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,OAAO,IAAI,IAAI,CAAC,QAAQ,yDAAyD,CAAC;IACpF,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEnC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,EAAE;QACtB,SAAS,EAAE;YACT,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;YAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;YACnC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;YACtC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;SACtC;QACD,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAElF,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AACpG,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACtC,OAAO,CACL,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC;QAC5B,cAAc,CAAC,QAAQ,CAAC;QACxB,QAAQ,KAAK,kBAAkB;QAC/B,QAAQ,KAAK,iBAAiB;QAC9B,QAAQ,KAAK,uBAAuB;QACpC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1B,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC1B,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACtC,OAAO,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,uBAAuB,CAAC;AAC1E,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;QACjB,OAAO,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,YAAY,CAAC,KAAsB;IAC1C,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC3D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lgrammel/kiwix-tool",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "AI SDK 7 tool for reading Kiwix/ZIM archives with Node.js libzim bindings.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"files": [
|
|
@@ -31,9 +31,9 @@
|
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@types/html-to-text": "^9.0.4",
|
|
34
|
-
"ai": "7.0.0-canary.
|
|
34
|
+
"ai": "^7.0.0-canary.133"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
|
-
"ai": "7.0.0-canary.
|
|
37
|
+
"ai": "^7.0.0-canary.133"
|
|
38
38
|
}
|
|
39
39
|
}
|