knolo-core 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builder.js +8 -10
- package/dist/index.d.ts +3 -3
- package/dist/index.js +1 -1
- package/dist/pack.js +8 -8
- package/package.json +9 -2
package/dist/builder.js
CHANGED
|
@@ -46,22 +46,20 @@ export async function buildPack(docs) {
|
|
|
46
46
|
offset += 4;
|
|
47
47
|
out.set(lexBytes, offset);
|
|
48
48
|
offset += lexBytes.length;
|
|
49
|
-
// postings
|
|
49
|
+
// postings (alignment-safe via DataView)
|
|
50
50
|
dv.setUint32(offset, postings.length, true);
|
|
51
51
|
offset += 4;
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
for (let i = 0; i < postings.length; i++) {
|
|
53
|
+
dv.setUint32(offset, postings[i], true);
|
|
54
|
+
offset += 4;
|
|
55
|
+
}
|
|
54
56
|
// blocks
|
|
55
57
|
dv.setUint32(offset, blocksBytes.length, true);
|
|
56
58
|
offset += 4;
|
|
57
59
|
out.set(blocksBytes, offset);
|
|
58
60
|
return out;
|
|
59
61
|
}
|
|
60
|
-
/** Strip Markdown syntax
|
|
61
|
-
* `marked` library is used for parsing and rendering. A very naive HTML tag
|
|
62
|
-
* stripper removes tags by dropping anything between `<` and `>`. This is
|
|
63
|
-
* simplistic but adequate for plain text extraction.
|
|
64
|
-
*/
|
|
62
|
+
/** Strip Markdown syntax with lightweight regexes (no deps). */
|
|
65
63
|
function stripMd(md) {
|
|
66
64
|
// Remove code fences
|
|
67
65
|
let text = md.replace(/```[^```]*```/g, ' ');
|
|
@@ -71,8 +69,8 @@ function stripMd(md) {
|
|
|
71
69
|
text = text.replace(/[\*_~]+/g, ' ');
|
|
72
70
|
// Remove headings (#)
|
|
73
71
|
text = text.replace(/^#+\s*/gm, '');
|
|
74
|
-
//
|
|
75
|
-
text = text.replace(/\[([^\]]+)\]\([
|
|
72
|
+
// Links [text](url) -> text
|
|
73
|
+
text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1');
|
|
76
74
|
// Remove any remaining brackets
|
|
77
75
|
text = text.replace(/[\[\]()]/g, ' ');
|
|
78
76
|
// Collapse whitespace
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export type { MountOptions, PackMeta, Pack } from './pack';
|
|
2
|
-
export type { QueryOptions, Hit } from './query';
|
|
3
|
-
export type { ContextPatch } from './patch';
|
|
4
1
|
export { mountPack } from './pack.js';
|
|
5
2
|
export { query } from './query.js';
|
|
6
3
|
export { makeContextPatch } from './patch.js';
|
|
7
4
|
export { buildPack } from './builder.js';
|
|
5
|
+
export type { MountOptions, PackMeta, Pack } from './pack.js';
|
|
6
|
+
export type { QueryOptions, Hit } from './query.js';
|
|
7
|
+
export type { ContextPatch } from './patch.js';
|
package/dist/index.js
CHANGED
package/dist/pack.js
CHANGED
|
@@ -39,11 +39,14 @@ export async function mountPack(opts) {
|
|
|
39
39
|
offset += lexLen;
|
|
40
40
|
const lexEntries = JSON.parse(lexJson);
|
|
41
41
|
const lexicon = new Map(lexEntries);
|
|
42
|
-
// Read postings
|
|
42
|
+
// Read postings (alignment-safe via DataView)
|
|
43
43
|
const postCount = dv.getUint32(offset, true);
|
|
44
44
|
offset += 4;
|
|
45
|
-
const postings = new Uint32Array(
|
|
46
|
-
|
|
45
|
+
const postings = new Uint32Array(postCount);
|
|
46
|
+
for (let i = 0; i < postCount; i++) {
|
|
47
|
+
postings[i] = dv.getUint32(offset, true);
|
|
48
|
+
offset += 4;
|
|
49
|
+
}
|
|
47
50
|
// Read blocks
|
|
48
51
|
const blocksLen = dv.getUint32(offset, true);
|
|
49
52
|
offset += 4;
|
|
@@ -57,9 +60,6 @@ export async function mountPack(opts) {
|
|
|
57
60
|
*/
|
|
58
61
|
async function resolveToBuffer(src) {
|
|
59
62
|
if (typeof src === 'string') {
|
|
60
|
-
// Use fetch for browser and Node environments. For Node this requires the
|
|
61
|
-
// global fetch API (available since Node 18). Error handling is delegated
|
|
62
|
-
// to the caller.
|
|
63
63
|
const res = await fetch(src);
|
|
64
64
|
const ab = await res.arrayBuffer();
|
|
65
65
|
return ab;
|
|
@@ -70,8 +70,8 @@ async function resolveToBuffer(src) {
|
|
|
70
70
|
return src.buffer;
|
|
71
71
|
}
|
|
72
72
|
// Otherwise, copy to a new buffer so we return exactly the bytes for this view.
|
|
73
|
-
const copy = src.slice();
|
|
74
|
-
return copy.buffer;
|
|
73
|
+
const copy = src.slice();
|
|
74
|
+
return copy.buffer;
|
|
75
75
|
}
|
|
76
76
|
return src;
|
|
77
77
|
}
|
package/package.json
CHANGED
|
@@ -1,14 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "knolo-core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
|
+
"type": "module",
|
|
4
5
|
"description": "Local-first knowledge packs for small LLMs.",
|
|
5
6
|
"keywords": ["llm", "knowledge-base", "rag", "local", "expo"],
|
|
6
|
-
"author": "
|
|
7
|
+
"author": "Your Name",
|
|
7
8
|
"license": "MIT",
|
|
8
9
|
"main": "./dist/index.js",
|
|
9
10
|
"types": "./dist/index.d.ts",
|
|
10
11
|
"files": ["dist", "bin", "README.md", "LICENSE"],
|
|
11
12
|
"bin": { "knolo": "./bin/knolo.mjs" },
|
|
13
|
+
"exports": {
|
|
14
|
+
".": {
|
|
15
|
+
"import": "./dist/index.js",
|
|
16
|
+
"types": "./dist/index.d.ts"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
12
19
|
"scripts": {
|
|
13
20
|
"build": "tsc -p tsconfig.json",
|
|
14
21
|
"prepublishOnly": "npm run build"
|