@aidy-dev/knowledge 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/index.js +14 -4
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -63,8 +63,11 @@ const knowledge = createKnowledge({
|
|
|
63
63
|
|
|
64
64
|
- **Chunks.** Markdown is split at h1–h3 headings; each section is a chunk, and a section over
|
|
65
65
|
~800 tokens (one per CJK character, one per four other characters) is cut at paragraphs, then
|
|
66
|
-
sentences. A chunk's URL is the document URL plus the section's
|
|
67
|
-
|
|
66
|
+
sentences. A chunk's URL is the document URL plus the section's anchor: the heading's
|
|
67
|
+
explicit id (`## Rotate keys {#rotate}`, stripped from the title) or else its GitHub-style
|
|
68
|
+
slug; its title is `document title › heading path`. Text is stored as plain text.
|
|
69
|
+
- **Hits.** Each hit carries an `excerpt` (≤ 300 characters around the matches, for people)
|
|
70
|
+
and `content`, the chunk's text up to 4000 characters, which only the model reads.
|
|
68
71
|
- **Lexical.** SQLite (and D1) uses FTS5 with the `unicode61` tokenizer and ranks by `bm25()`.
|
|
69
72
|
Postgres uses a `tsvector` with the `simple` configuration and ranks by `ts_rank` (Postgres
|
|
70
73
|
has no BM25). CJK text has no spaces, so the overlapping bigrams of CJK runs are indexed in a
|
package/dist/index.js
CHANGED
|
@@ -108,7 +108,8 @@ function findMatches(text, query) {
|
|
|
108
108
|
}));
|
|
109
109
|
}
|
|
110
110
|
/**
|
|
111
|
-
* Splits Markdown at h1–h3 headings, so every chunk links to its own section
|
|
111
|
+
* Splits Markdown at h1–h3 headings, so every chunk links to its own section: a heading's
|
|
112
|
+
* explicit `{#id}` is its anchor, otherwise its GitHub slug. A section over
|
|
112
113
|
* 800 tokens is cut into pieces of up to 800 at paragraph, then sentence boundaries. A heading
|
|
113
114
|
* with nothing under it but deeper headings makes no chunk of its own; it stays in their paths.
|
|
114
115
|
*/
|
|
@@ -151,12 +152,18 @@ function sections(markdown) {
|
|
|
151
152
|
if (heading) {
|
|
152
153
|
endBlock();
|
|
153
154
|
const level = heading[1].length;
|
|
154
|
-
const
|
|
155
|
+
const explicit = /^(.*?)[ \t]*\{#([\p{L}\p{N}_][\p{L}\p{N}_:.-]*)\}$/u.exec(heading[2]);
|
|
156
|
+
const title = plainText(explicit ? explicit[1] : heading[2]);
|
|
155
157
|
stack.length = level - 1;
|
|
156
158
|
stack[level - 1] = title;
|
|
159
|
+
let anchor;
|
|
160
|
+
if (explicit) {
|
|
161
|
+
anchor = explicit[2];
|
|
162
|
+
slugs.set(anchor, (slugs.get(anchor) ?? 0) + 1);
|
|
163
|
+
} else anchor = uniqueSlug(slugs, title);
|
|
157
164
|
current = {
|
|
158
165
|
path: stack.filter(Boolean),
|
|
159
|
-
anchor
|
|
166
|
+
anchor,
|
|
160
167
|
blocks: []
|
|
161
168
|
};
|
|
162
169
|
all.push(current);
|
|
@@ -278,6 +285,8 @@ const migrations = [{
|
|
|
278
285
|
END;`
|
|
279
286
|
}];
|
|
280
287
|
// src/index.ts
|
|
288
|
+
/** `SearchHit.content` is at most 4000 characters (spec/search.md). */
|
|
289
|
+
const MAX_CONTENT_CHARS = 4e3;
|
|
281
290
|
/** BM25 candidates, and vector candidates, ranked before fusion. */
|
|
282
291
|
const CANDIDATES = 200;
|
|
283
292
|
/** Chunks scanned for vector similarity, newest first. */
|
|
@@ -407,7 +416,8 @@ function createKnowledge(options) {
|
|
|
407
416
|
title: row.title,
|
|
408
417
|
url: row.url,
|
|
409
418
|
excerpt: excerpt(row.body, query),
|
|
410
|
-
...row.updated_at ? { updatedAt: row.updated_at } : {}
|
|
419
|
+
...row.updated_at ? { updatedAt: row.updated_at } : {},
|
|
420
|
+
content: row.body.slice(0, MAX_CONTENT_CHARS)
|
|
411
421
|
}];
|
|
412
422
|
});
|
|
413
423
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aidy-dev/knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "aidy's knowledge index: Markdown ingestion, chunking and hybrid BM25 + vector search over any SqlDriver.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://aidy.lpythu.workers.dev",
|
|
@@ -25,8 +25,8 @@
|
|
|
25
25
|
"access": "public"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@aidy-dev/sql": "0.
|
|
29
|
-
"@aidy-dev/spec": "0.
|
|
28
|
+
"@aidy-dev/sql": "0.2.0",
|
|
29
|
+
"@aidy-dev/spec": "0.2.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"postgres": "^4.1.1"
|