@aidy-dev/knowledge 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/README.md +90 -2
- package/dist/index.d.ts +53 -0
- package/dist/index.js +460 -0
- package/package.json +36 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
Copyright 2026 Aidy Contributors
|
|
6
|
+
|
|
7
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
you may not use this file except in compliance with the License.
|
|
9
|
+
You may obtain a copy of the License at
|
|
10
|
+
|
|
11
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
|
|
13
|
+
Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
See the License for the specific language governing permissions and
|
|
17
|
+
limitations under the License.
|
|
18
|
+
|
package/README.md
CHANGED
|
@@ -1,3 +1,91 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @aidy-dev/knowledge
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
aidy's knowledge index: Markdown documents in collections, chunked by heading, searched with
|
|
4
|
+
BM25 (and vectors when you give an embedder), on any `SqlDriver` from `@aidy-dev/sql`
|
|
5
|
+
(Postgres, SQLite, Cloudflare D1). It is also a search source for aidy, named `knowledge`.
|
|
6
|
+
|
|
7
|
+
## Quickstart
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
import { createKnowledge } from '@aidy-dev/knowledge';
|
|
11
|
+
import { sqliteDriver } from '@aidy-dev/sql/sqlite';
|
|
12
|
+
|
|
13
|
+
const knowledge = createKnowledge({ driver: sqliteDriver('knowledge.sqlite') });
|
|
14
|
+
await knowledge.migrate();
|
|
15
|
+
|
|
16
|
+
await knowledge.ingest('docs', [
|
|
17
|
+
{
|
|
18
|
+
id: 'deploy',
|
|
19
|
+
title: 'Deploying TurboMesh',
|
|
20
|
+
url: 'https://docs.turbomesh.dev/deploy',
|
|
21
|
+
markdown: '# Deploying TurboMesh\n\n## Rotate certificates\n\nRun `tm certs rotate`.',
|
|
22
|
+
updatedAt: '2026-11-01T00:00:00Z',
|
|
23
|
+
},
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
const hits = await knowledge.search('rotate certificates', { collections: ['docs'], limit: 5 });
|
|
27
|
+
// [{ type: 'doc', id: 'docs/deploy/0',
|
|
28
|
+
// title: 'Deploying TurboMesh › Rotate certificates',
|
|
29
|
+
// url: 'https://docs.turbomesh.dev/deploy#rotate-certificates',
|
|
30
|
+
// excerpt: '«Rotate» «certificates» Run tm certs «rotate».', ... }]
|
|
31
|
+
|
|
32
|
+
// A source for createAidy({ sources: [...] }): each user searches the collections you pick.
|
|
33
|
+
const source = knowledge.source({
|
|
34
|
+
collections: (identity) => ['docs', `manuals/${identity.tenant}`],
|
|
35
|
+
});
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
With embeddings:
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
const knowledge = createKnowledge({
|
|
42
|
+
driver,
|
|
43
|
+
embed: async (texts) => (await openai.embeddings.create({ model, input: texts })).data.map((d) => d.embedding),
|
|
44
|
+
dimensions: 1536,
|
|
45
|
+
});
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## API
|
|
49
|
+
|
|
50
|
+
- `createKnowledge({ driver, embed?, dimensions? })`. `dimensions` is checked against every
|
|
51
|
+
vector `embed` returns.
|
|
52
|
+
- `migrate()` creates the tables (`aidy_knowledge_*`, namespace `knowledge`).
|
|
53
|
+
- `ingest(collection, documents)` adds or replaces documents `{ id, title, url, markdown,
|
|
54
|
+
updatedAt? }` and returns `{ updated, unchanged }`. A document whose content hash is unchanged
|
|
55
|
+
is skipped; a changed one has its chunks replaced, and chunks whose text is unchanged keep
|
|
56
|
+
their embeddings. `url` is a product path or an absolute http(s) URL.
|
|
57
|
+
- `remove(collection, ids)` deletes documents and their chunks.
|
|
58
|
+
- `search(query, { collections, limit })` returns `SearchHit`s (`@aidy-dev/spec/search`), best
|
|
59
|
+
first, at most 50.
|
|
60
|
+
- `source({ name = 'knowledge', collections: (identity) => string[] })` is a `SearchSource`.
|
|
61
|
+
|
|
62
|
+
## How it works
|
|
63
|
+
|
|
64
|
+
- **Chunks.** Markdown is split at h1–h3 headings; each section is a chunk, and a section over
|
|
65
|
+
~800 tokens (one per CJK character, one per four other characters) is cut at paragraphs, then
|
|
66
|
+
sentences. A chunk's URL is the document URL plus the section's GitHub-style anchor; its
|
|
67
|
+
title is `document title › heading path`. Text is stored as plain text.
|
|
68
|
+
- **Lexical.** SQLite (and D1) uses FTS5 with the `unicode61` tokenizer and ranks by `bm25()`.
|
|
69
|
+
Postgres uses a `tsvector` with the `simple` configuration and ranks by `ts_rank` (Postgres
|
|
70
|
+
has no BM25). CJK text has no spaces, so the overlapping bigrams of CJK runs are indexed in a
|
|
71
|
+
second column and queries are turned into the same bigrams. Query terms are OR-ed.
|
|
72
|
+
- **Vectors.** When `embed` is set, embeddings are stored as JSON text (SQLite) or `real[]`
|
|
73
|
+
(Postgres) and scored by cosine similarity in JavaScript, over the top 200 lexical hits plus
|
|
74
|
+
the newest 20,000 chunks of the searched collections. The 200 best are fused with the lexical
|
|
75
|
+
ranking by reciprocal rank fusion (k = 60).
|
|
76
|
+
- **Excerpts** are at most 300 characters around the densest cluster of matches, matches
|
|
77
|
+
wrapped in `«…»`.
|
|
78
|
+
|
|
79
|
+
## Limits
|
|
80
|
+
|
|
81
|
+
- Vector search reads every embedding of the searched collections up to 20,000 chunks per
|
|
82
|
+
query; beyond that, older chunks are only found lexically. With 1,536 dimensions that is about
|
|
83
|
+
a few hundred MB read per query at the cap, so keep embedded collections well under it or
|
|
84
|
+
search fewer collections at once. pgvector is not used.
|
|
85
|
+
- On D1, statements are atomic one by one (see `@aidy-dev/sql/d1`): a document whose ingest
|
|
86
|
+
fails halfway may be left without chunks; ingesting it again repairs it.
|
|
87
|
+
- Postgres ranking is `ts_rank`, close to but not BM25.
|
|
88
|
+
|
|
89
|
+
## Tests
|
|
90
|
+
|
|
91
|
+
Postgres tests run when `AIDY_TEST_POSTGRES_URL` is set (see `@aidy-dev/sql`).
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { SearchHit } from "@aidy-dev/spec/search";
|
|
2
|
+
import { SqlDriver } from "@aidy-dev/sql";
|
|
3
|
+
import { Identity } from "@aidy-dev/spec/catalog";
|
|
4
|
+
export interface KnowledgeDocument {
|
|
5
|
+
/** Unique within its collection. */
|
|
6
|
+
readonly id: string;
|
|
7
|
+
readonly title: string;
|
|
8
|
+
/** A path in the product or an absolute http(s) URL; chunk anchors are appended to it. */
|
|
9
|
+
readonly url: string;
|
|
10
|
+
readonly markdown: string;
|
|
11
|
+
/** RFC 3339 time, passed on to hits. */
|
|
12
|
+
readonly updatedAt?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface KnowledgeOptions {
|
|
15
|
+
readonly driver: SqlDriver;
|
|
16
|
+
/** Embeds texts, one vector per text. Without it, search is BM25 alone. */
|
|
17
|
+
readonly embed?: (texts: string[]) => Promise<number[][]>;
|
|
18
|
+
/** The length of `embed`'s vectors; every vector is checked against it. */
|
|
19
|
+
readonly dimensions?: number;
|
|
20
|
+
}
|
|
21
|
+
/** A search source as aidy's core consumes it (structurally the same as core's type). */
|
|
22
|
+
export interface SearchSource {
|
|
23
|
+
readonly name: string;
|
|
24
|
+
search(query: string, options: {
|
|
25
|
+
limit: number;
|
|
26
|
+
identity: Identity;
|
|
27
|
+
}): Promise<SearchHit[]>;
|
|
28
|
+
}
|
|
29
|
+
export interface Knowledge {
|
|
30
|
+
/** Creates or updates the index's tables (namespace `knowledge` in `aidy_migrations`). */
|
|
31
|
+
migrate(): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Adds or replaces documents of a collection. A document whose content is unchanged is
|
|
34
|
+
* skipped; a changed one has its chunks replaced, reusing embeddings of unchanged chunks.
|
|
35
|
+
*/
|
|
36
|
+
ingest(collection: string, documents: readonly KnowledgeDocument[]): Promise<{
|
|
37
|
+
updated: number;
|
|
38
|
+
unchanged: number;
|
|
39
|
+
}>;
|
|
40
|
+
/** Removes documents, with their chunks, from a collection. */
|
|
41
|
+
remove(collection: string, ids: readonly string[]): Promise<void>;
|
|
42
|
+
/** The best chunks of the given collections for `query`, best first. */
|
|
43
|
+
search(query: string, options: {
|
|
44
|
+
collections: readonly string[];
|
|
45
|
+
limit: number;
|
|
46
|
+
}): Promise<SearchHit[]>;
|
|
47
|
+
/** A search source over the collections `collections` picks for the caller. */
|
|
48
|
+
source(options: {
|
|
49
|
+
name?: string;
|
|
50
|
+
collections: (identity: Identity) => readonly string[];
|
|
51
|
+
}): SearchSource;
|
|
52
|
+
}
|
|
53
|
+
export declare function createKnowledge(options: KnowledgeOptions): Knowledge;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
import { KNOWLEDGE_SOURCE, MAX_SEARCH_LIMIT } from "@aidy-dev/spec/search";
|
|
2
|
+
import { migrate } from "@aidy-dev/sql";
|
|
3
|
+
// src/text.ts
|
|
4
|
+
/** Han, Hiragana, Katakana and Hangul: scripts written without spaces between words. */
|
|
5
|
+
const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
|
|
6
|
+
const CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
|
7
|
+
const WORD = /[\p{L}\p{N}]+/gu;
|
|
8
|
+
/**
|
|
9
|
+
* Overlapping bigrams of every CJK run, space separated, so a word tokenizer indexes them as
|
|
10
|
+
* terms: "部署集群" → "部署 署集 集群". A run of one character is kept as it is.
|
|
11
|
+
*/
|
|
12
|
+
function cjkBigrams(text) {
|
|
13
|
+
const grams = [];
|
|
14
|
+
for (const [run] of text.matchAll(CJK)) {
|
|
15
|
+
const chars = [...run];
|
|
16
|
+
if (chars.length === 1) grams.push(run);
|
|
17
|
+
for (let i = 0; i + 1 < chars.length; i++) grams.push(`${chars[i]}${chars[i + 1]}`);
|
|
18
|
+
}
|
|
19
|
+
return grams.join(" ");
|
|
20
|
+
}
|
|
21
|
+
/** The terms of a query: lowercased words outside CJK runs, and the bigrams of CJK runs. */
|
|
22
|
+
function queryTerms(query) {
|
|
23
|
+
const latin = query.replace(CJK, " ").toLowerCase().match(WORD) ?? [];
|
|
24
|
+
const cjk = cjkBigrams(query).split(" ").filter(Boolean);
|
|
25
|
+
return [.../* @__PURE__ */ new Set([...latin, ...cjk])];
|
|
26
|
+
}
|
|
27
|
+
/** A rough token count: one per CJK character, one per four other characters. */
|
|
28
|
+
function estimateTokens(text) {
|
|
29
|
+
return splitTokens(text, Number.POSITIVE_INFINITY)[0]?.tokens ?? 0;
|
|
30
|
+
}
|
|
31
|
+
/** Cuts `text` into consecutive parts of at most `max` estimated tokens each. */
|
|
32
|
+
function splitTokens(text, max) {
|
|
33
|
+
const parts = [];
|
|
34
|
+
let start = 0;
|
|
35
|
+
let cjk = 0;
|
|
36
|
+
let other = 0;
|
|
37
|
+
let i = 0;
|
|
38
|
+
for (const char of text) {
|
|
39
|
+
const isCjk = CJK_CHAR.test(char);
|
|
40
|
+
const nextOther = other + (!isCjk && char.trim() ? 1 : 0);
|
|
41
|
+
if (cjk + (isCjk ? 1 : 0) + Math.ceil(nextOther / 4) > max && i > start) {
|
|
42
|
+
parts.push({
|
|
43
|
+
text: text.slice(start, i),
|
|
44
|
+
tokens: cjk + Math.ceil(other / 4)
|
|
45
|
+
});
|
|
46
|
+
start = i;
|
|
47
|
+
cjk = 0;
|
|
48
|
+
other = 0;
|
|
49
|
+
}
|
|
50
|
+
if (isCjk) cjk++;
|
|
51
|
+
else if (char.trim()) other++;
|
|
52
|
+
i += char.length;
|
|
53
|
+
}
|
|
54
|
+
parts.push({
|
|
55
|
+
text: text.slice(start),
|
|
56
|
+
tokens: cjk + Math.ceil(other / 4)
|
|
57
|
+
});
|
|
58
|
+
return parts;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* At most 300 characters of `text` around the densest cluster of query matches, each match
|
|
62
|
+
* wrapped in `«…»`, with `…` where the text is cut.
|
|
63
|
+
*/
|
|
64
|
+
function excerpt(text, query) {
|
|
65
|
+
const matches = findMatches(text, query);
|
|
66
|
+
let best = 0;
|
|
67
|
+
let bestCount = 0;
|
|
68
|
+
matches.forEach((match, i) => {
|
|
69
|
+
const count = matches.filter((m) => m.start >= match.start && m.end <= match.start + 200).length;
|
|
70
|
+
if (count > bestCount) [best, bestCount] = [i, count];
|
|
71
|
+
});
|
|
72
|
+
let start = matches.length > 0 ? Math.max(0, (matches[best]?.start ?? 0) - 40) : 0;
|
|
73
|
+
if (start > 0) {
|
|
74
|
+
const space = text.lastIndexOf(" ", start);
|
|
75
|
+
start = space > start - 20 ? space + 1 : start;
|
|
76
|
+
}
|
|
77
|
+
let out = start > 0 ? "…" : "";
|
|
78
|
+
let i = start;
|
|
79
|
+
const budget = 299;
|
|
80
|
+
for (const match of matches.filter((m) => m.start >= start)) {
|
|
81
|
+
const plain = text.slice(i, match.start);
|
|
82
|
+
if (out.length + plain.length >= budget) break;
|
|
83
|
+
const marked = `«${text.slice(match.start, match.end)}»`;
|
|
84
|
+
if (out.length + plain.length + marked.length > budget) break;
|
|
85
|
+
out += plain + marked;
|
|
86
|
+
i = match.end;
|
|
87
|
+
}
|
|
88
|
+
const rest = text.slice(i, i + budget - out.length);
|
|
89
|
+
out += rest;
|
|
90
|
+
if (i + rest.length < text.length) out += "…";
|
|
91
|
+
return out;
|
|
92
|
+
}
|
|
93
|
+
/** Non-overlapping matches of the query's words (whole words), CJK runs and their bigrams. */
|
|
94
|
+
function findMatches(text, query) {
|
|
95
|
+
const latin = query.replace(CJK, " ").match(WORD) ?? [];
|
|
96
|
+
const cjk = [...query.matchAll(CJK)].map(([run]) => run);
|
|
97
|
+
const alternatives = [
|
|
98
|
+
...latin.map((word) => `(?<![\\p{L}\\p{N}])${word}(?![\\p{L}\\p{N}])`),
|
|
99
|
+
...cjk,
|
|
100
|
+
...cjkBigrams(query).split(" ").filter(Boolean)
|
|
101
|
+
];
|
|
102
|
+
if (alternatives.length === 0) return [];
|
|
103
|
+
alternatives.sort((a, b) => b.length - a.length);
|
|
104
|
+
const pattern = new RegExp(alternatives.join("|"), "giu");
|
|
105
|
+
return [...text.matchAll(pattern)].map((m) => ({
|
|
106
|
+
start: m.index,
|
|
107
|
+
end: m.index + m[0].length
|
|
108
|
+
}));
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Splits Markdown at h1–h3 headings, so every chunk links to its own section. A section over
|
|
112
|
+
* 800 tokens is cut into pieces of up to 800 at paragraph, then sentence boundaries. A heading
|
|
113
|
+
* with nothing under it but deeper headings makes no chunk of its own; it stays in their paths.
|
|
114
|
+
*/
|
|
115
|
+
function chunkMarkdown(markdown) {
|
|
116
|
+
return sections(markdown).flatMap((section) => pack(section.blocks.flatMap(splitBlock)).map((text) => ({
|
|
117
|
+
path: section.path,
|
|
118
|
+
anchor: section.anchor,
|
|
119
|
+
text
|
|
120
|
+
})));
|
|
121
|
+
}
|
|
122
|
+
function sections(markdown) {
|
|
123
|
+
const slugs = /* @__PURE__ */ new Map();
|
|
124
|
+
const stack = [];
|
|
125
|
+
let current = {
|
|
126
|
+
path: [],
|
|
127
|
+
anchor: void 0,
|
|
128
|
+
blocks: []
|
|
129
|
+
};
|
|
130
|
+
const all = [current];
|
|
131
|
+
let block = [];
|
|
132
|
+
let fence;
|
|
133
|
+
const endBlock = () => {
|
|
134
|
+
const text = plainText(block.join("\n"));
|
|
135
|
+
if (text) current.blocks.push(text);
|
|
136
|
+
block = [];
|
|
137
|
+
};
|
|
138
|
+
for (const line of markdown.split(/\r?\n/)) {
|
|
139
|
+
const fenceMark = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
|
|
140
|
+
if (fence) {
|
|
141
|
+
if (fenceMark?.startsWith(fence)) fence = void 0;
|
|
142
|
+
block.push(line);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (fenceMark) {
|
|
146
|
+
fence = fenceMark;
|
|
147
|
+
block.push(line);
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const heading = /^ {0,3}(#{1,3})[ \t]+(.*?)(?:[ \t]+#+)?[ \t]*$/.exec(line);
|
|
151
|
+
if (heading) {
|
|
152
|
+
endBlock();
|
|
153
|
+
const level = heading[1].length;
|
|
154
|
+
const title = plainText(heading[2]);
|
|
155
|
+
stack.length = level - 1;
|
|
156
|
+
stack[level - 1] = title;
|
|
157
|
+
current = {
|
|
158
|
+
path: stack.filter(Boolean),
|
|
159
|
+
anchor: uniqueSlug(slugs, title),
|
|
160
|
+
blocks: []
|
|
161
|
+
};
|
|
162
|
+
all.push(current);
|
|
163
|
+
} else if (!line.trim()) endBlock();
|
|
164
|
+
else block.push(line);
|
|
165
|
+
}
|
|
166
|
+
endBlock();
|
|
167
|
+
return all.filter((section) => section.blocks.length > 0).map((section) => {
|
|
168
|
+
const heading = section.path.at(-1);
|
|
169
|
+
return heading ? {
|
|
170
|
+
...section,
|
|
171
|
+
blocks: [heading, ...section.blocks]
|
|
172
|
+
} : section;
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
/** GitHub's heading anchor: lowercased, punctuation dropped, spaces to hyphens, deduplicated. */
|
|
176
|
+
function slug(heading) {
|
|
177
|
+
return heading.toLowerCase().replace(/[^\p{L}\p{M}\p{N}\p{Pc}\- ]/gu, "").replace(/ /g, "-");
|
|
178
|
+
}
|
|
179
|
+
function uniqueSlug(seen, heading) {
|
|
180
|
+
const base = slug(heading);
|
|
181
|
+
const count = seen.get(base) ?? 0;
|
|
182
|
+
seen.set(base, count + 1);
|
|
183
|
+
return count === 0 ? base : `${base}-${count}`;
|
|
184
|
+
}
|
|
185
|
+
/** Markdown to plain text: code fences, emphasis, links, images, HTML and markers removed. */
|
|
186
|
+
function plainText(markdown) {
|
|
187
|
+
return markdown.replace(/^\s*(`{3,}|~{3,}).*$/gm, "").replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/<(https?:\/\/[^>]+)>/g, "$1").replace(/<\/?[A-Za-z][^>]*>/g, "").replace(/^\s{0,3}(#{1,6}\s+|>\s?|[-*+]\s+|\d+[.)]\s+)/gm, "").replace(/\*\*|__|\*|`|~~/g, "").replace(/\s+/g, " ").trim();
|
|
188
|
+
}
|
|
189
|
+
/** A block over the limit, cut at sentence ends, and a sentence over it by characters. */
|
|
190
|
+
function splitBlock(block) {
|
|
191
|
+
if (estimateTokens(block) <= 800) return [block];
|
|
192
|
+
return pack(block.split(/(?<=[.!?。!?])\s*/u)).flatMap((piece) => {
|
|
193
|
+
if (estimateTokens(piece) <= 800) return [piece];
|
|
194
|
+
return splitTokens(piece, 800).map((part) => part.text);
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
/** Joins consecutive parts while the result stays within the limit. */
|
|
198
|
+
function pack(parts) {
|
|
199
|
+
const packed = [];
|
|
200
|
+
let tokens = 0;
|
|
201
|
+
for (const part of parts) {
|
|
202
|
+
const size = estimateTokens(part);
|
|
203
|
+
if (packed.length > 0 && tokens + size <= 800) {
|
|
204
|
+
packed[packed.length - 1] += ` ${part}`;
|
|
205
|
+
tokens += size;
|
|
206
|
+
} else {
|
|
207
|
+
packed.push(part);
|
|
208
|
+
tokens = size;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return packed;
|
|
212
|
+
}
|
|
213
|
+
// src/schema.ts
|
|
214
|
+
const NAMESPACE = "knowledge";
|
|
215
|
+
/**
|
|
216
|
+
* Documents keep a content hash; chunks keep the hit's fields, the plain text, the CJK bigrams
|
|
217
|
+
* of title and text, and an optional embedding. Title and text are indexed together. SQLite indexes text in an FTS5 table that triggers keep in step with the
|
|
218
|
+
* chunks (by `seq`);
|
|
219
|
+
* Postgres in a generated `tsvector` column.
|
|
220
|
+
*/
|
|
221
|
+
const migrations = [{
|
|
222
|
+
id: "001_knowledge",
|
|
223
|
+
postgres: `
|
|
224
|
+
CREATE TABLE aidy_knowledge_documents (
|
|
225
|
+
collection TEXT NOT NULL,
|
|
226
|
+
id TEXT NOT NULL,
|
|
227
|
+
hash TEXT NOT NULL,
|
|
228
|
+
PRIMARY KEY (collection, id)
|
|
229
|
+
);
|
|
230
|
+
CREATE TABLE aidy_knowledge_chunks (
|
|
231
|
+
seq BIGSERIAL PRIMARY KEY,
|
|
232
|
+
id TEXT NOT NULL UNIQUE,
|
|
233
|
+
collection TEXT NOT NULL,
|
|
234
|
+
document_id TEXT NOT NULL,
|
|
235
|
+
title TEXT NOT NULL,
|
|
236
|
+
url TEXT NOT NULL,
|
|
237
|
+
updated_at TEXT,
|
|
238
|
+
body TEXT NOT NULL,
|
|
239
|
+
cjk TEXT NOT NULL,
|
|
240
|
+
hash TEXT NOT NULL,
|
|
241
|
+
embedding REAL[],
|
|
242
|
+
tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('simple', title || ' ' || body || ' ' || cjk)) STORED,
|
|
243
|
+
FOREIGN KEY (collection, document_id)
|
|
244
|
+
REFERENCES aidy_knowledge_documents (collection, id) ON DELETE CASCADE
|
|
245
|
+
);
|
|
246
|
+
CREATE INDEX aidy_knowledge_chunks_document ON aidy_knowledge_chunks (collection, document_id);
|
|
247
|
+
CREATE INDEX aidy_knowledge_chunks_tsv ON aidy_knowledge_chunks USING gin (tsv);`,
|
|
248
|
+
sqlite: `
|
|
249
|
+
CREATE TABLE aidy_knowledge_documents (
|
|
250
|
+
collection TEXT NOT NULL,
|
|
251
|
+
id TEXT NOT NULL,
|
|
252
|
+
hash TEXT NOT NULL,
|
|
253
|
+
PRIMARY KEY (collection, id)
|
|
254
|
+
);
|
|
255
|
+
CREATE TABLE aidy_knowledge_chunks (
|
|
256
|
+
seq INTEGER PRIMARY KEY,
|
|
257
|
+
id TEXT NOT NULL UNIQUE,
|
|
258
|
+
collection TEXT NOT NULL,
|
|
259
|
+
document_id TEXT NOT NULL,
|
|
260
|
+
title TEXT NOT NULL,
|
|
261
|
+
url TEXT NOT NULL,
|
|
262
|
+
updated_at TEXT,
|
|
263
|
+
body TEXT NOT NULL,
|
|
264
|
+
cjk TEXT NOT NULL,
|
|
265
|
+
hash TEXT NOT NULL,
|
|
266
|
+
embedding TEXT,
|
|
267
|
+
FOREIGN KEY (collection, document_id)
|
|
268
|
+
REFERENCES aidy_knowledge_documents (collection, id) ON DELETE CASCADE
|
|
269
|
+
);
|
|
270
|
+
CREATE INDEX aidy_knowledge_chunks_document ON aidy_knowledge_chunks (collection, document_id);
|
|
271
|
+
CREATE VIRTUAL TABLE aidy_knowledge_fts USING fts5(text, cjk, tokenize = 'unicode61');
|
|
272
|
+
CREATE TRIGGER aidy_knowledge_chunks_insert AFTER INSERT ON aidy_knowledge_chunks BEGIN
|
|
273
|
+
INSERT INTO aidy_knowledge_fts (rowid, text, cjk)
|
|
274
|
+
VALUES (new.seq, new.title || ' ' || new.body, new.cjk);
|
|
275
|
+
END;
|
|
276
|
+
CREATE TRIGGER aidy_knowledge_chunks_delete AFTER DELETE ON aidy_knowledge_chunks BEGIN
|
|
277
|
+
DELETE FROM aidy_knowledge_fts WHERE rowid = old.seq;
|
|
278
|
+
END;`
|
|
279
|
+
}];
|
|
280
|
+
// src/index.ts
|
|
281
|
+
/** BM25 candidates, and vector candidates, ranked before fusion. */
|
|
282
|
+
const CANDIDATES = 200;
|
|
283
|
+
/** Chunks scanned for vector similarity, newest first. */
|
|
284
|
+
const VECTOR_SCAN = 2e4;
|
|
285
|
+
const RRF_K = 60;
|
|
286
|
+
const URL_PATTERN = /^(\/(?!\/)|https?:\/\/)/;
|
|
287
|
+
function createKnowledge(options) {
|
|
288
|
+
const { driver, embed, dimensions } = options;
|
|
289
|
+
async function vectors(texts) {
|
|
290
|
+
if (!embed || texts.length === 0) return [];
|
|
291
|
+
const result = await embed(texts);
|
|
292
|
+
if (result.length !== texts.length || dimensions !== void 0 && result.some((vector) => vector.length !== dimensions)) throw new Error(`embed returned ${result.length} vectors for ${texts.length} texts` + (dimensions === void 0 ? "" : `, each must have ${dimensions} dimensions`));
|
|
293
|
+
return result;
|
|
294
|
+
}
|
|
295
|
+
const encode = (vector) => driver.dialect === "postgres" ? vector : JSON.stringify(vector);
|
|
296
|
+
const decode = (value) => typeof value === "string" ? JSON.parse(value) : value;
|
|
297
|
+
async function ingestOne(collection, doc) {
|
|
298
|
+
if (!doc.title.trim()) throw new TypeError(`document ${doc.id} has no title`);
|
|
299
|
+
if (!URL_PATTERN.test(doc.url)) throw new TypeError(`document ${doc.id}: url must be a product path or an http(s) URL`);
|
|
300
|
+
const hash = await sha256(JSON.stringify([
|
|
301
|
+
doc.title,
|
|
302
|
+
doc.url,
|
|
303
|
+
doc.updatedAt ?? null,
|
|
304
|
+
doc.markdown,
|
|
305
|
+
embed !== void 0
|
|
306
|
+
]));
|
|
307
|
+
const [existing] = await driver.query("SELECT hash FROM aidy_knowledge_documents WHERE collection = $1 AND id = $2", [collection, doc.id]);
|
|
308
|
+
if (existing?.hash === hash) return false;
|
|
309
|
+
const chunks = await Promise.all(chunkMarkdown(doc.markdown).map(async (chunk, position) => {
|
|
310
|
+
const title = [doc.title, ...chunk.path.filter((h, i) => i > 0 || h !== doc.title)].join(" › ");
|
|
311
|
+
return {
|
|
312
|
+
id: `${collection}/${doc.id}/${position}`,
|
|
313
|
+
title,
|
|
314
|
+
url: chunk.anchor ? `${doc.url}#${chunk.anchor}` : doc.url,
|
|
315
|
+
body: chunk.text,
|
|
316
|
+
hash: await sha256(`${title}\n${chunk.text}`)
|
|
317
|
+
};
|
|
318
|
+
}));
|
|
319
|
+
const known = /* @__PURE__ */ new Map();
|
|
320
|
+
if (embed) {
|
|
321
|
+
const rows = await driver.query(`SELECT hash, embedding FROM aidy_knowledge_chunks
|
|
322
|
+
WHERE collection = $1 AND document_id = $2 AND embedding IS NOT NULL`, [collection, doc.id]);
|
|
323
|
+
for (const row of rows) known.set(row.hash, decode(row.embedding));
|
|
324
|
+
const missing = chunks.filter((chunk) => !known.has(chunk.hash));
|
|
325
|
+
const fresh = await vectors(missing.map((chunk) => `${chunk.title}\n${chunk.body}`));
|
|
326
|
+
missing.forEach((chunk, i) => {
|
|
327
|
+
known.set(chunk.hash, fresh[i]);
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
await driver.transaction(async (tx) => {
|
|
331
|
+
await tx.query("DELETE FROM aidy_knowledge_documents WHERE collection = $1 AND id = $2", [collection, doc.id]);
|
|
332
|
+
await tx.query("INSERT INTO aidy_knowledge_documents (collection, id, hash) VALUES ($1, $2, $3)", [
|
|
333
|
+
collection,
|
|
334
|
+
doc.id,
|
|
335
|
+
hash
|
|
336
|
+
]);
|
|
337
|
+
for (const chunk of chunks) {
|
|
338
|
+
const vector = known.get(chunk.hash);
|
|
339
|
+
await tx.query(`INSERT INTO aidy_knowledge_chunks
|
|
340
|
+
(id, collection, document_id, title, url, updated_at, body, cjk, hash, embedding)
|
|
341
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, [
|
|
342
|
+
chunk.id,
|
|
343
|
+
collection,
|
|
344
|
+
doc.id,
|
|
345
|
+
chunk.title,
|
|
346
|
+
chunk.url,
|
|
347
|
+
doc.updatedAt ?? null,
|
|
348
|
+
chunk.body,
|
|
349
|
+
cjkBigrams(`${chunk.title} ${chunk.body}`),
|
|
350
|
+
chunk.hash,
|
|
351
|
+
vector ? encode(vector) : null
|
|
352
|
+
]);
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
return true;
|
|
356
|
+
}
|
|
357
|
+
async function lexical(terms, collections) {
|
|
358
|
+
if (terms.length === 0) return [];
|
|
359
|
+
const within = collections.map((_, i) => `$${i + 2}`).join(", ");
|
|
360
|
+
const columns = "c.id, c.title, c.url, c.updated_at, c.body, c.embedding";
|
|
361
|
+
if (driver.dialect === "postgres") return driver.query(`SELECT ${columns} FROM aidy_knowledge_chunks c, to_tsquery('simple', $1) q
|
|
362
|
+
WHERE c.tsv @@ q AND c.collection IN (${within})
|
|
363
|
+
ORDER BY ts_rank(c.tsv, q) DESC, c.seq LIMIT ${CANDIDATES}`, [terms.map((term) => `'${term}'`).join(" | "), ...collections]);
|
|
364
|
+
return driver.query(`SELECT ${columns} FROM aidy_knowledge_fts f
|
|
365
|
+
JOIN aidy_knowledge_chunks c ON c.seq = f.rowid
|
|
366
|
+
WHERE aidy_knowledge_fts MATCH $1 AND c.collection IN (${within})
|
|
367
|
+
ORDER BY bm25(aidy_knowledge_fts), c.seq LIMIT ${CANDIDATES}`, [terms.map((term) => `"${term}"`).join(" OR "), ...collections]);
|
|
368
|
+
}
|
|
369
|
+
async function semantic(query, collections, lexicalRows) {
|
|
370
|
+
const [target] = await vectors([query]);
|
|
371
|
+
if (!target) return [];
|
|
372
|
+
const within = collections.map((_, i) => `$${i + 1}`).join(", ");
|
|
373
|
+
const scanned = await driver.query(`SELECT id, embedding FROM aidy_knowledge_chunks
|
|
374
|
+
WHERE collection IN (${within}) AND embedding IS NOT NULL
|
|
375
|
+
ORDER BY seq DESC LIMIT ${VECTOR_SCAN}`, [...collections]);
|
|
376
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
377
|
+
for (const row of lexicalRows) if (row.embedding) candidates.set(row.id, row.embedding);
|
|
378
|
+
for (const row of scanned) candidates.set(row.id, row.embedding);
|
|
379
|
+
return [...candidates].map(([id, embedding]) => ({
|
|
380
|
+
id,
|
|
381
|
+
score: cosine(target, decode(embedding))
|
|
382
|
+
})).sort((a, b) => b.score - a.score).slice(0, CANDIDATES).map(({ id }) => id);
|
|
383
|
+
}
|
|
384
|
+
async function search(query, { collections, limit }) {
|
|
385
|
+
if (collections.length === 0 || !query.trim()) return [];
|
|
386
|
+
const lexicalRows = await lexical(queryTerms(query), collections);
|
|
387
|
+
const rankings = [lexicalRows.map((row) => row.id)];
|
|
388
|
+
if (embed) rankings.push(await semantic(query, collections, lexicalRows));
|
|
389
|
+
const scores = /* @__PURE__ */ new Map();
|
|
390
|
+
for (const ranking of rankings) ranking.forEach((id, rank) => {
|
|
391
|
+
scores.set(id, (scores.get(id) ?? 0) + 1 / (RRF_K + rank + 1));
|
|
392
|
+
});
|
|
393
|
+
const ids = [...scores].sort((a, b) => b[1] - a[1]).slice(0, Math.min(limit, MAX_SEARCH_LIMIT)).map(([id]) => id);
|
|
394
|
+
const rows = new Map(lexicalRows.map((row) => [row.id, row]));
|
|
395
|
+
const missing = ids.filter((id) => !rows.has(id));
|
|
396
|
+
if (missing.length > 0) {
|
|
397
|
+
const fetched = await driver.query(`SELECT id, title, url, updated_at, body FROM aidy_knowledge_chunks
|
|
398
|
+
WHERE id IN (${missing.map((_, i) => `$${i + 1}`).join(", ")})`, missing);
|
|
399
|
+
for (const row of fetched) rows.set(row.id, row);
|
|
400
|
+
}
|
|
401
|
+
return ids.flatMap((id) => {
|
|
402
|
+
const row = rows.get(id);
|
|
403
|
+
if (!row) return [];
|
|
404
|
+
return [{
|
|
405
|
+
type: "doc",
|
|
406
|
+
id: row.id,
|
|
407
|
+
title: row.title,
|
|
408
|
+
url: row.url,
|
|
409
|
+
excerpt: excerpt(row.body, query),
|
|
410
|
+
...row.updated_at ? { updatedAt: row.updated_at } : {}
|
|
411
|
+
}];
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
return {
|
|
415
|
+
async migrate() {
|
|
416
|
+
await migrate(driver, NAMESPACE, migrations);
|
|
417
|
+
},
|
|
418
|
+
async ingest(collection, documents) {
|
|
419
|
+
let updated = 0;
|
|
420
|
+
for (const doc of documents) if (await ingestOne(collection, doc)) updated++;
|
|
421
|
+
return {
|
|
422
|
+
updated,
|
|
423
|
+
unchanged: documents.length - updated
|
|
424
|
+
};
|
|
425
|
+
},
|
|
426
|
+
async remove(collection, ids) {
|
|
427
|
+
if (ids.length === 0) return;
|
|
428
|
+
await driver.query(`DELETE FROM aidy_knowledge_documents
|
|
429
|
+
WHERE collection = $1 AND id IN (${ids.map((_, i) => `$${i + 2}`).join(", ")})`, [collection, ...ids]);
|
|
430
|
+
},
|
|
431
|
+
search,
|
|
432
|
+
source({ name = KNOWLEDGE_SOURCE, collections }) {
|
|
433
|
+
return {
|
|
434
|
+
name,
|
|
435
|
+
search: (query, { limit, identity }) => search(query, {
|
|
436
|
+
collections: collections(identity),
|
|
437
|
+
limit
|
|
438
|
+
})
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
function cosine(a, b) {
|
|
444
|
+
let dot = 0;
|
|
445
|
+
let na = 0;
|
|
446
|
+
let nb = 0;
|
|
447
|
+
for (let i = 0; i < a.length; i++) {
|
|
448
|
+
const x = a[i];
|
|
449
|
+
const y = b[i] ?? 0;
|
|
450
|
+
dot += x * y;
|
|
451
|
+
na += x * x;
|
|
452
|
+
nb += y * y;
|
|
453
|
+
}
|
|
454
|
+
return na === 0 || nb === 0 ? 0 : dot / Math.sqrt(na * nb);
|
|
455
|
+
}
|
|
456
|
+
async function sha256(text) {
|
|
457
|
+
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
|
|
458
|
+
return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
459
|
+
}
|
|
460
|
+
export { createKnowledge };
|
package/package.json
CHANGED
|
@@ -1,6 +1,39 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aidy-dev/knowledge",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "aidy's knowledge index: Markdown ingestion, chunking and hybrid BM25 + vector search over any SqlDriver.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"homepage": "https://aidy.lpythu.workers.dev",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/aidy-dev/aidy.git",
|
|
10
|
+
"directory": "packages/knowledge"
|
|
11
|
+
},
|
|
12
|
+
"type": "module",
|
|
13
|
+
"sideEffects": false,
|
|
14
|
+
"files": [
|
|
15
|
+
"dist"
|
|
16
|
+
],
|
|
17
|
+
"exports": {
|
|
18
|
+
".": {
|
|
19
|
+
"@aidy-dev/source": "./src/index.ts",
|
|
20
|
+
"default": "./dist/index.js"
|
|
21
|
+
},
|
|
22
|
+
"./package.json": "./package.json"
|
|
23
|
+
},
|
|
24
|
+
"publishConfig": {
|
|
25
|
+
"access": "public"
|
|
26
|
+
},
|
|
27
|
+
"dependencies": {
|
|
28
|
+
"@aidy-dev/sql": "0.1.0",
|
|
29
|
+
"@aidy-dev/spec": "0.1.0"
|
|
30
|
+
},
|
|
31
|
+
"devDependencies": {
|
|
32
|
+
"postgres": "^4.1.1"
|
|
33
|
+
},
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "tsdown",
|
|
36
|
+
"typecheck": "tsc -p .",
|
|
37
|
+
"test": "vitest run"
|
|
38
|
+
}
|
|
6
39
|
}
|