@delali/narsil 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/dist/adapters/filesystem.d.ts +8 -0
- package/dist/adapters/filesystem.mjs +107 -0
- package/dist/adapters/indexeddb.d.ts +9 -0
- package/dist/adapters/indexeddb.mjs +153 -0
- package/dist/adapters/memory.d.ts +5 -0
- package/dist/adapters/memory.mjs +30 -0
- package/dist/adapters-XODT2F3h.d.ts +30 -0
- package/dist/chunk-5HXMVMSM.mjs +3147 -0
- package/dist/chunk-JKIBUPAY.mjs +42 -0
- package/dist/chunk-N37VMC62.mjs +18 -0
- package/dist/chunk-WYC635GI.mjs +335 -0
- package/dist/direct-executor-MQD7JRD4.mjs +3 -0
- package/dist/index.d.ts +386 -0
- package/dist/index.mjs +1099 -0
- package/dist/invalidation/broadcast-channel.d.ts +8 -0
- package/dist/invalidation/broadcast-channel.mjs +46 -0
- package/dist/invalidation/filesystem.d.ts +10 -0
- package/dist/invalidation/filesystem.mjs +159 -0
- package/dist/invalidation/noop.d.ts +5 -0
- package/dist/invalidation/noop.mjs +13 -0
- package/dist/language-ClUGLuoC.d.ts +13 -0
- package/dist/languages/arabic.d.ts +5 -0
- package/dist/languages/arabic.mjs +631 -0
- package/dist/languages/armenian.d.ts +5 -0
- package/dist/languages/armenian.mjs +306 -0
- package/dist/languages/bulgarian.d.ts +5 -0
- package/dist/languages/bulgarian.mjs +330 -0
- package/dist/languages/chinese.d.ts +5 -0
- package/dist/languages/chinese.mjs +250 -0
- package/dist/languages/dagbani.d.ts +5 -0
- package/dist/languages/dagbani.mjs +85 -0
- package/dist/languages/danish.d.ts +5 -0
- package/dist/languages/danish.mjs +224 -0
- package/dist/languages/dutch.d.ts +5 -0
- package/dist/languages/dutch.mjs +296 -0
- package/dist/languages/english.d.ts +5 -0
- package/dist/languages/english.mjs +1 -0
- package/dist/languages/ewe.d.ts +5 -0
- package/dist/languages/ewe.mjs +85 -0
- package/dist/languages/finnish.d.ts +5 -0
- package/dist/languages/finnish.mjs +536 -0
- package/dist/languages/french.d.ts +5 -0
- package/dist/languages/french.mjs +635 -0
- package/dist/languages/ga.d.ts +5 -0
- package/dist/languages/ga.mjs +69 -0
- package/dist/languages/german.d.ts +5 -0
- package/dist/languages/german.mjs +414 -0
- package/dist/languages/greek.d.ts +5 -0
- package/dist/languages/greek.mjs +888 -0
- package/dist/languages/hausa.d.ts +5 -0
- package/dist/languages/hausa.mjs +243 -0
- package/dist/languages/hindi.d.ts +5 -0
- package/dist/languages/hindi.mjs +333 -0
- package/dist/languages/hungarian.d.ts +5 -0
- package/dist/languages/hungarian.mjs +600 -0
- package/dist/languages/igbo.d.ts +5 -0
- package/dist/languages/igbo.mjs +96 -0
- package/dist/languages/indonesian.d.ts +5 -0
- package/dist/languages/indonesian.mjs +896 -0
- package/dist/languages/irish.d.ts +5 -0
- package/dist/languages/irish.mjs +268 -0
- package/dist/languages/italian.d.ts +5 -0
- package/dist/languages/italian.mjs +713 -0
- package/dist/languages/japanese.d.ts +5 -0
- package/dist/languages/japanese.mjs +210 -0
- package/dist/languages/nepali.d.ts +5 -0
- package/dist/languages/nepali.mjs +262 -0
- package/dist/languages/norwegian.d.ts +5 -0
- package/dist/languages/norwegian.mjs +274 -0
- package/dist/languages/portuguese.d.ts +5 -0
- package/dist/languages/portuguese.mjs +606 -0
- package/dist/languages/romanian.d.ts +5 -0
- package/dist/languages/romanian.mjs +634 -0
- package/dist/languages/russian.d.ts +5 -0
- package/dist/languages/russian.mjs +424 -0
- package/dist/languages/sanskrit.d.ts +5 -0
- package/dist/languages/sanskrit.mjs +155 -0
- package/dist/languages/serbian.d.ts +5 -0
- package/dist/languages/serbian.mjs +277 -0
- package/dist/languages/slovenian.d.ts +5 -0
- package/dist/languages/slovenian.mjs +522 -0
- package/dist/languages/spanish.d.ts +5 -0
- package/dist/languages/spanish.mjs +606 -0
- package/dist/languages/swahili.d.ts +5 -0
- package/dist/languages/swahili.mjs +212 -0
- package/dist/languages/swedish.d.ts +5 -0
- package/dist/languages/swedish.mjs +224 -0
- package/dist/languages/tamil.d.ts +5 -0
- package/dist/languages/tamil.mjs +186 -0
- package/dist/languages/turkish.d.ts +5 -0
- package/dist/languages/turkish.mjs +675 -0
- package/dist/languages/twi.d.ts +5 -0
- package/dist/languages/twi.mjs +84 -0
- package/dist/languages/ukrainian.d.ts +5 -0
- package/dist/languages/ukrainian.mjs +444 -0
- package/dist/languages/yoruba.d.ts +5 -0
- package/dist/languages/yoruba.mjs +103 -0
- package/dist/languages/zulu.d.ts +5 -0
- package/dist/languages/zulu.mjs +113 -0
- package/dist/workers/entry.d.ts +3 -0
- package/dist/workers/entry.mjs +106 -0
- package/package.json +87 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// src/languages/zulu.ts
|
|
2
|
+
var stopWords = /* @__PURE__ */ new Set([
|
|
3
|
+
"ukuthi",
|
|
4
|
+
"kodwa",
|
|
5
|
+
"futhi",
|
|
6
|
+
"kakhulu",
|
|
7
|
+
"wakhe",
|
|
8
|
+
"kusho",
|
|
9
|
+
"uma",
|
|
10
|
+
"wathi",
|
|
11
|
+
"kanye",
|
|
12
|
+
"phansi",
|
|
13
|
+
"ngesikhathi",
|
|
14
|
+
"lapho",
|
|
15
|
+
"u",
|
|
16
|
+
"zakhe",
|
|
17
|
+
"khona",
|
|
18
|
+
"ukuba",
|
|
19
|
+
"nje",
|
|
20
|
+
"phezulu",
|
|
21
|
+
"yakhe",
|
|
22
|
+
"kungani",
|
|
23
|
+
"wase",
|
|
24
|
+
"zonke",
|
|
25
|
+
"la",
|
|
26
|
+
"mina",
|
|
27
|
+
"wami",
|
|
28
|
+
"ukuze",
|
|
29
|
+
"wahamba",
|
|
30
|
+
"kahle",
|
|
31
|
+
"lakhe",
|
|
32
|
+
"i",
|
|
33
|
+
"ku",
|
|
34
|
+
"e",
|
|
35
|
+
"ba",
|
|
36
|
+
"si",
|
|
37
|
+
"ni",
|
|
38
|
+
"ngi",
|
|
39
|
+
"li",
|
|
40
|
+
"lu",
|
|
41
|
+
"bu",
|
|
42
|
+
"a",
|
|
43
|
+
"ka",
|
|
44
|
+
"wa",
|
|
45
|
+
"ya",
|
|
46
|
+
"za",
|
|
47
|
+
"na",
|
|
48
|
+
"nga",
|
|
49
|
+
"nge",
|
|
50
|
+
"no",
|
|
51
|
+
"ne",
|
|
52
|
+
"lo",
|
|
53
|
+
"le",
|
|
54
|
+
"lesi",
|
|
55
|
+
"leli",
|
|
56
|
+
"lolu",
|
|
57
|
+
"lobu",
|
|
58
|
+
"lawa",
|
|
59
|
+
"layo",
|
|
60
|
+
"leyo",
|
|
61
|
+
"lelo",
|
|
62
|
+
"noma",
|
|
63
|
+
"ngoba",
|
|
64
|
+
"kanti",
|
|
65
|
+
"nokho",
|
|
66
|
+
"ngakho",
|
|
67
|
+
"njalo",
|
|
68
|
+
"wena",
|
|
69
|
+
"yena",
|
|
70
|
+
"thina",
|
|
71
|
+
"nina",
|
|
72
|
+
"bona",
|
|
73
|
+
"ubani",
|
|
74
|
+
"ini",
|
|
75
|
+
"phi",
|
|
76
|
+
"nini",
|
|
77
|
+
"kanjani",
|
|
78
|
+
"konke",
|
|
79
|
+
"bonke",
|
|
80
|
+
"wonke",
|
|
81
|
+
"sonke",
|
|
82
|
+
"kuphela",
|
|
83
|
+
"se",
|
|
84
|
+
"sa",
|
|
85
|
+
"kwa",
|
|
86
|
+
"ngeke",
|
|
87
|
+
"hayi",
|
|
88
|
+
"yebo",
|
|
89
|
+
"kancane",
|
|
90
|
+
"yi",
|
|
91
|
+
"ngu",
|
|
92
|
+
"wu",
|
|
93
|
+
"oku",
|
|
94
|
+
"loku",
|
|
95
|
+
"loko",
|
|
96
|
+
"lapha",
|
|
97
|
+
"phakathi",
|
|
98
|
+
"phandle",
|
|
99
|
+
"emva",
|
|
100
|
+
"phambi"
|
|
101
|
+
]);
|
|
102
|
+
var zulu = {
|
|
103
|
+
name: "zulu",
|
|
104
|
+
stemmer: null,
|
|
105
|
+
stopWords,
|
|
106
|
+
tokenizer: {
|
|
107
|
+
splitPattern: /[^a-zA-Z0-9]+/gi,
|
|
108
|
+
normalizeDiacritics: false,
|
|
109
|
+
minTokenLength: 1
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
export { zulu };
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import '../chunk-N37VMC62.mjs';
|
|
2
|
+
|
|
3
|
+
// src/workers/protocol.ts
|
|
4
|
+
var KNOWN_ACTION_TYPES = [
|
|
5
|
+
"insert",
|
|
6
|
+
"remove",
|
|
7
|
+
"update",
|
|
8
|
+
"query",
|
|
9
|
+
"preflight",
|
|
10
|
+
"get",
|
|
11
|
+
"has",
|
|
12
|
+
"count",
|
|
13
|
+
"createIndex",
|
|
14
|
+
"dropIndex",
|
|
15
|
+
"getStats",
|
|
16
|
+
"clear",
|
|
17
|
+
"serialize",
|
|
18
|
+
"deserialize",
|
|
19
|
+
"memoryReport",
|
|
20
|
+
"shutdown"
|
|
21
|
+
];
|
|
22
|
+
function isValidWorkerAction(msg) {
|
|
23
|
+
if (msg === null || msg === void 0 || typeof msg !== "object") {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
const candidate = msg;
|
|
27
|
+
if (typeof candidate.type !== "string" || typeof candidate.requestId !== "string") {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
return KNOWN_ACTION_TYPES.includes(candidate.type);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// src/workers/worker-entry.ts
|
|
34
|
+
function startWorker() {
|
|
35
|
+
setup().catch((err) => {
|
|
36
|
+
console.error("Narsil worker setup failed:", err);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
async function setup() {
|
|
40
|
+
let parentPort = null;
|
|
41
|
+
try {
|
|
42
|
+
const workerThreads = await import('worker_threads');
|
|
43
|
+
parentPort = workerThreads.parentPort ?? null;
|
|
44
|
+
} catch {
|
|
45
|
+
parentPort = null;
|
|
46
|
+
}
|
|
47
|
+
const { createDirectExecutor } = await import('../direct-executor-MQD7JRD4.mjs');
|
|
48
|
+
const executor = createDirectExecutor();
|
|
49
|
+
function buildErrorResponse(requestId, code, message) {
|
|
50
|
+
return { type: "error", requestId, code, message };
|
|
51
|
+
}
|
|
52
|
+
function buildSuccessResponse(requestId, data) {
|
|
53
|
+
return { type: "success", requestId, data };
|
|
54
|
+
}
|
|
55
|
+
async function handleAction(action, postFn) {
|
|
56
|
+
if (action.type === "shutdown") {
|
|
57
|
+
await executor.shutdown();
|
|
58
|
+
postFn(buildSuccessResponse(action.requestId, void 0));
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
try {
|
|
62
|
+
const result = await executor.execute(action);
|
|
63
|
+
postFn(buildSuccessResponse(action.requestId, result));
|
|
64
|
+
} catch (err) {
|
|
65
|
+
const code = err?.code ?? "UNKNOWN_ERROR";
|
|
66
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
67
|
+
postFn(buildErrorResponse(action.requestId, code, message));
|
|
68
|
+
}
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
if (parentPort) {
|
|
72
|
+
const port = parentPort;
|
|
73
|
+
port.on("message", (raw) => {
|
|
74
|
+
if (!isValidWorkerAction(raw)) {
|
|
75
|
+
const requestId = raw?.requestId ?? "unknown";
|
|
76
|
+
port.postMessage(buildErrorResponse(requestId, "INVALID_ACTION", "Received an invalid worker action"));
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
handleAction(raw, (msg) => port.postMessage(msg)).then((shouldClose) => {
|
|
80
|
+
if (shouldClose) {
|
|
81
|
+
port.close();
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
const globalSelf = typeof self !== "undefined" ? self : void 0;
|
|
88
|
+
if (globalSelf && typeof globalSelf.postMessage === "function") {
|
|
89
|
+
const webSelf = globalSelf;
|
|
90
|
+
webSelf.onmessage = (event) => {
|
|
91
|
+
const raw = event.data;
|
|
92
|
+
if (!isValidWorkerAction(raw)) {
|
|
93
|
+
const requestId = raw?.requestId ?? "unknown";
|
|
94
|
+
webSelf.postMessage(buildErrorResponse(requestId, "INVALID_ACTION", "Received an invalid worker action"));
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
handleAction(raw, (msg) => webSelf.postMessage(msg)).then((shouldClose) => {
|
|
98
|
+
if (shouldClose) {
|
|
99
|
+
webSelf.close();
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export { startWorker };
|
package/package.json
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@delali/narsil",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "A distributed full-text search engine with cross-language portable binary format.",
|
|
6
|
+
"author": "Delali (https://sondelali.com)",
|
|
7
|
+
"license": "Apache-2.0",
|
|
8
|
+
"homepage": "https://github.com/assetcorp/narsil",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "https://github.com/assetcorp/narsil.git",
|
|
12
|
+
"directory": "packages/ts"
|
|
13
|
+
},
|
|
14
|
+
"bugs": {
|
|
15
|
+
"url": "https://github.com/assetcorp/narsil/issues"
|
|
16
|
+
},
|
|
17
|
+
"sideEffects": false,
|
|
18
|
+
"keywords": [
|
|
19
|
+
"search",
|
|
20
|
+
"full-text-search",
|
|
21
|
+
"search-engine",
|
|
22
|
+
"indexing",
|
|
23
|
+
"text-search",
|
|
24
|
+
"distributed",
|
|
25
|
+
"typescript"
|
|
26
|
+
],
|
|
27
|
+
"engines": {
|
|
28
|
+
"node": ">=22"
|
|
29
|
+
},
|
|
30
|
+
"scripts": {
|
|
31
|
+
"build": "tsup",
|
|
32
|
+
"test": "vitest run",
|
|
33
|
+
"test:watch": "vitest",
|
|
34
|
+
"test:coverage": "vitest run --coverage",
|
|
35
|
+
"lint": "biome check src/",
|
|
36
|
+
"typecheck": "tsc --noEmit"
|
|
37
|
+
},
|
|
38
|
+
"exports": {
|
|
39
|
+
".": {
|
|
40
|
+
"import": "./dist/index.mjs",
|
|
41
|
+
"types": "./dist/index.d.ts"
|
|
42
|
+
},
|
|
43
|
+
"./adapters/memory": {
|
|
44
|
+
"import": "./dist/adapters/memory.mjs",
|
|
45
|
+
"types": "./dist/adapters/memory.d.ts"
|
|
46
|
+
},
|
|
47
|
+
"./adapters/filesystem": {
|
|
48
|
+
"import": "./dist/adapters/filesystem.mjs",
|
|
49
|
+
"types": "./dist/adapters/filesystem.d.ts"
|
|
50
|
+
},
|
|
51
|
+
"./adapters/indexeddb": {
|
|
52
|
+
"import": "./dist/adapters/indexeddb.mjs",
|
|
53
|
+
"types": "./dist/adapters/indexeddb.d.ts"
|
|
54
|
+
},
|
|
55
|
+
"./invalidation/noop": {
|
|
56
|
+
"import": "./dist/invalidation/noop.mjs",
|
|
57
|
+
"types": "./dist/invalidation/noop.d.ts"
|
|
58
|
+
},
|
|
59
|
+
"./invalidation/filesystem": {
|
|
60
|
+
"import": "./dist/invalidation/filesystem.mjs",
|
|
61
|
+
"types": "./dist/invalidation/filesystem.d.ts"
|
|
62
|
+
},
|
|
63
|
+
"./invalidation/broadcast-channel": {
|
|
64
|
+
"import": "./dist/invalidation/broadcast-channel.mjs",
|
|
65
|
+
"types": "./dist/invalidation/broadcast-channel.d.ts"
|
|
66
|
+
},
|
|
67
|
+
"./languages/*": {
|
|
68
|
+
"import": "./dist/languages/*.mjs",
|
|
69
|
+
"types": "./dist/languages/*.d.ts"
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
"files": [
|
|
73
|
+
"dist/",
|
|
74
|
+
"LICENSE"
|
|
75
|
+
],
|
|
76
|
+
"dependencies": {
|
|
77
|
+
"@msgpack/msgpack": "3.1.1"
|
|
78
|
+
},
|
|
79
|
+
"devDependencies": {
|
|
80
|
+
"@biomejs/biome": "2.4.4",
|
|
81
|
+
"@types/node": "22.15.32",
|
|
82
|
+
"@vitest/coverage-v8": "4.0.18",
|
|
83
|
+
"tsup": "8.5.1",
|
|
84
|
+
"typescript": "5.9.3",
|
|
85
|
+
"vitest": "4.0.18"
|
|
86
|
+
}
|
|
87
|
+
}
|