@aidy-dev/knowledge 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,18 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ Copyright 2026 Aidy Contributors
6
+
7
+ Licensed under the Apache License, Version 2.0 (the "License");
8
+ you may not use this file except in compliance with the License.
9
+ You may obtain a copy of the License at
10
+
11
+ http://www.apache.org/licenses/LICENSE-2.0
12
+
13
+ Unless required by applicable law or agreed to in writing, software
14
+ distributed under the License is distributed on an "AS IS" BASIS,
15
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ See the License for the specific language governing permissions and
17
+ limitations under the License.
18
+
package/README.md CHANGED
@@ -1,3 +1,94 @@
1
- # Temporary Holding Version
1
+ # @aidy-dev/knowledge
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ aidy's knowledge index: Markdown documents in collections, chunked by heading, searched with
4
+ BM25 (and vectors when you give an embedder), on any `SqlDriver` from `@aidy-dev/sql`
5
+ (Postgres, SQLite, Cloudflare D1). It is also a search source for aidy, named `knowledge`.
6
+
7
+ ## Quickstart
8
+
9
+ ```ts
10
+ import { createKnowledge } from '@aidy-dev/knowledge';
11
+ import { sqliteDriver } from '@aidy-dev/sql/sqlite';
12
+
13
+ const knowledge = createKnowledge({ driver: sqliteDriver('knowledge.sqlite') });
14
+ await knowledge.migrate();
15
+
16
+ await knowledge.ingest('docs', [
17
+ {
18
+ id: 'deploy',
19
+ title: 'Deploying TurboMesh',
20
+ url: 'https://docs.turbomesh.dev/deploy',
21
+ markdown: '# Deploying TurboMesh\n\n## Rotate certificates\n\nRun `tm certs rotate`.',
22
+ updatedAt: '2026-11-01T00:00:00Z',
23
+ },
24
+ ]);
25
+
26
+ const hits = await knowledge.search('rotate certificates', { collections: ['docs'], limit: 5 });
27
+ // [{ type: 'doc', id: 'docs/deploy/0',
28
+ // title: 'Deploying TurboMesh › Rotate certificates',
29
+ // url: 'https://docs.turbomesh.dev/deploy#rotate-certificates',
30
+ // excerpt: '«Rotate» «certificates» Run tm certs «rotate».', ... }]
31
+
32
+ // A source for createAidy({ sources: [...] }): each user searches the collections you pick.
33
+ const source = knowledge.source({
34
+ collections: (identity) => ['docs', `manuals/${identity.tenant}`],
35
+ });
36
+ ```
37
+
38
+ With embeddings:
39
+
40
+ ```ts
41
+ const knowledge = createKnowledge({
42
+ driver,
43
+ embed: async (texts) => (await openai.embeddings.create({ model, input: texts })).data.map((d) => d.embedding),
44
+ dimensions: 1536,
45
+ });
46
+ ```
47
+
48
+ ## API
49
+
50
+ - `createKnowledge({ driver, embed?, dimensions? })`. `dimensions` is checked against every
51
+ vector `embed` returns.
52
+ - `migrate()` creates the tables (`aidy_knowledge_*`, namespace `knowledge`).
53
+ - `ingest(collection, documents)` adds or replaces documents `{ id, title, url, markdown,
54
+ updatedAt? }` and returns `{ updated, unchanged }`. A document whose content hash is unchanged
55
+ is skipped; a changed one has its chunks replaced, and chunks whose text is unchanged keep
56
+ their embeddings. `url` is a product path or an absolute http(s) URL.
57
+ - `remove(collection, ids)` deletes documents and their chunks.
58
+ - `search(query, { collections, limit })` returns `SearchHit`s (`@aidy-dev/spec/search`), best
59
+ first, at most 50.
60
+ - `source({ name = 'knowledge', collections: (identity) => string[] })` is a `SearchSource`.
61
+
62
+ ## How it works
63
+
64
+ - **Chunks.** Markdown is split at h1–h3 headings; each section is a chunk, and a section over
65
+ ~800 tokens (one per CJK character, one per four other characters) is cut at paragraphs, then
66
+ sentences. A chunk's URL is the document URL plus the section's anchor: the heading's
67
+ explicit id (`## Rotate keys {#rotate}`, stripped from the title) or else its GitHub-style
68
+ slug; its title is `document title › heading path`. Text is stored as plain text.
69
+ - **Hits.** Each hit carries an `excerpt` (≤ 300 characters around the matches, for people)
70
+ and `content`, the chunk's text up to 4000 characters, which only the model reads.
71
+ - **Lexical.** SQLite (and D1) uses FTS5 with the `unicode61` tokenizer and ranks by `bm25()`.
72
+ Postgres uses a `tsvector` with the `simple` configuration and ranks by `ts_rank` (Postgres
73
+ has no BM25). CJK text has no spaces, so the overlapping bigrams of CJK runs are indexed in a
74
+ second column and queries are turned into the same bigrams. Query terms are OR-ed.
75
+ - **Vectors.** When `embed` is set, embeddings are stored as JSON text (SQLite) or `real[]`
76
+ (Postgres) and scored by cosine similarity in JavaScript, over the top 200 lexical hits plus
77
+ the newest 20,000 chunks of the searched collections. The 200 best are fused with the lexical
78
+ ranking by reciprocal rank fusion (k = 60).
79
+ - **Excerpts** are at most 300 characters around the densest cluster of matches, matches
80
+ wrapped in `«…»`.
81
+
82
+ ## Limits
83
+
84
+ - Vector search reads every embedding of the searched collections up to 20,000 chunks per
85
+ query; beyond that, older chunks are only found lexically. With 1,536 dimensions that is about
86
+ a few hundred MB read per query at the cap, so keep embedded collections well under it or
87
+ search fewer collections at once. pgvector is not used.
88
+ - On D1, statements are atomic one by one (see `@aidy-dev/sql/d1`): a document whose ingest
89
+ fails halfway may be left without chunks; ingesting it again repairs it.
90
+ - Postgres ranking is `ts_rank`, close to but not BM25.
91
+
92
+ ## Tests
93
+
94
+ Postgres tests run when `AIDY_TEST_POSTGRES_URL` is set (see `@aidy-dev/sql`).
@@ -0,0 +1,53 @@
1
+ import { SearchHit } from "@aidy-dev/spec/search";
2
+ import { SqlDriver } from "@aidy-dev/sql";
3
+ import { Identity } from "@aidy-dev/spec/catalog";
4
+ export interface KnowledgeDocument {
5
+ /** Unique within its collection. */
6
+ readonly id: string;
7
+ readonly title: string;
8
+ /** A path in the product or an absolute http(s) URL; chunk anchors are appended to it. */
9
+ readonly url: string;
10
+ readonly markdown: string;
11
+ /** RFC 3339 time, passed on to hits. */
12
+ readonly updatedAt?: string;
13
+ }
14
+ export interface KnowledgeOptions {
15
+ readonly driver: SqlDriver;
16
+ /** Embeds texts, one vector per text. Without it, search is BM25 alone. */
17
+ readonly embed?: (texts: string[]) => Promise<number[][]>;
18
+ /** The length of `embed`'s vectors; every vector is checked against it. */
19
+ readonly dimensions?: number;
20
+ }
21
+ /** A search source as aidy's core consumes it (structurally the same as core's type). */
22
+ export interface SearchSource {
23
+ readonly name: string;
24
+ search(query: string, options: {
25
+ limit: number;
26
+ identity: Identity;
27
+ }): Promise<SearchHit[]>;
28
+ }
29
+ export interface Knowledge {
30
+ /** Creates or updates the index's tables (namespace `knowledge` in `aidy_migrations`). */
31
+ migrate(): Promise<void>;
32
+ /**
33
+ * Adds or replaces documents of a collection. A document whose content is unchanged is
34
+ * skipped; a changed one has its chunks replaced, reusing embeddings of unchanged chunks.
35
+ */
36
+ ingest(collection: string, documents: readonly KnowledgeDocument[]): Promise<{
37
+ updated: number;
38
+ unchanged: number;
39
+ }>;
40
+ /** Removes documents, with their chunks, from a collection. */
41
+ remove(collection: string, ids: readonly string[]): Promise<void>;
42
+ /** The best chunks of the given collections for `query`, best first. */
43
+ search(query: string, options: {
44
+ collections: readonly string[];
45
+ limit: number;
46
+ }): Promise<SearchHit[]>;
47
+ /** A search source over the collections `collections` picks for the caller. */
48
+ source(options: {
49
+ name?: string;
50
+ collections: (identity: Identity) => readonly string[];
51
+ }): SearchSource;
52
+ }
53
+ export declare function createKnowledge(options: KnowledgeOptions): Knowledge;
package/dist/index.js ADDED
@@ -0,0 +1,470 @@
1
+ import { KNOWLEDGE_SOURCE, MAX_SEARCH_LIMIT } from "@aidy-dev/spec/search";
2
+ import { migrate } from "@aidy-dev/sql";
3
+ // src/text.ts
4
+ /** Han, Hiragana, Katakana and Hangul: scripts written without spaces between words. */
5
+ const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
6
+ const CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
7
+ const WORD = /[\p{L}\p{N}]+/gu;
8
+ /**
9
+ * Overlapping bigrams of every CJK run, space separated, so a word tokenizer indexes them as
10
+ * terms: "部署集群" → "部署 署集 集群". A run of one character is kept as it is.
11
+ */
12
+ function cjkBigrams(text) {
13
+ const grams = [];
14
+ for (const [run] of text.matchAll(CJK)) {
15
+ const chars = [...run];
16
+ if (chars.length === 1) grams.push(run);
17
+ for (let i = 0; i + 1 < chars.length; i++) grams.push(`${chars[i]}${chars[i + 1]}`);
18
+ }
19
+ return grams.join(" ");
20
+ }
21
+ /** The terms of a query: lowercased words outside CJK runs, and the bigrams of CJK runs. */
22
+ function queryTerms(query) {
23
+ const latin = query.replace(CJK, " ").toLowerCase().match(WORD) ?? [];
24
+ const cjk = cjkBigrams(query).split(" ").filter(Boolean);
25
+ return [.../* @__PURE__ */ new Set([...latin, ...cjk])];
26
+ }
27
+ /** A rough token count: one per CJK character, one per four other characters. */
28
+ function estimateTokens(text) {
29
+ return splitTokens(text, Number.POSITIVE_INFINITY)[0]?.tokens ?? 0;
30
+ }
31
+ /** Cuts `text` into consecutive parts of at most `max` estimated tokens each. */
32
+ function splitTokens(text, max) {
33
+ const parts = [];
34
+ let start = 0;
35
+ let cjk = 0;
36
+ let other = 0;
37
+ let i = 0;
38
+ for (const char of text) {
39
+ const isCjk = CJK_CHAR.test(char);
40
+ const nextOther = other + (!isCjk && char.trim() ? 1 : 0);
41
+ if (cjk + (isCjk ? 1 : 0) + Math.ceil(nextOther / 4) > max && i > start) {
42
+ parts.push({
43
+ text: text.slice(start, i),
44
+ tokens: cjk + Math.ceil(other / 4)
45
+ });
46
+ start = i;
47
+ cjk = 0;
48
+ other = 0;
49
+ }
50
+ if (isCjk) cjk++;
51
+ else if (char.trim()) other++;
52
+ i += char.length;
53
+ }
54
+ parts.push({
55
+ text: text.slice(start),
56
+ tokens: cjk + Math.ceil(other / 4)
57
+ });
58
+ return parts;
59
+ }
60
+ /**
61
+ * At most 300 characters of `text` around the densest cluster of query matches, each match
62
+ * wrapped in `«…»`, with `…` where the text is cut.
63
+ */
64
+ function excerpt(text, query) {
65
+ const matches = findMatches(text, query);
66
+ let best = 0;
67
+ let bestCount = 0;
68
+ matches.forEach((match, i) => {
69
+ const count = matches.filter((m) => m.start >= match.start && m.end <= match.start + 200).length;
70
+ if (count > bestCount) [best, bestCount] = [i, count];
71
+ });
72
+ let start = matches.length > 0 ? Math.max(0, (matches[best]?.start ?? 0) - 40) : 0;
73
+ if (start > 0) {
74
+ const space = text.lastIndexOf(" ", start);
75
+ start = space > start - 20 ? space + 1 : start;
76
+ }
77
+ let out = start > 0 ? "…" : "";
78
+ let i = start;
79
+ const budget = 299;
80
+ for (const match of matches.filter((m) => m.start >= start)) {
81
+ const plain = text.slice(i, match.start);
82
+ if (out.length + plain.length >= budget) break;
83
+ const marked = `«${text.slice(match.start, match.end)}»`;
84
+ if (out.length + plain.length + marked.length > budget) break;
85
+ out += plain + marked;
86
+ i = match.end;
87
+ }
88
+ const rest = text.slice(i, i + budget - out.length);
89
+ out += rest;
90
+ if (i + rest.length < text.length) out += "…";
91
+ return out;
92
+ }
93
+ /** Non-overlapping matches of the query's words (whole words), CJK runs and their bigrams. */
94
+ function findMatches(text, query) {
95
+ const latin = query.replace(CJK, " ").match(WORD) ?? [];
96
+ const cjk = [...query.matchAll(CJK)].map(([run]) => run);
97
+ const alternatives = [
98
+ ...latin.map((word) => `(?<![\\p{L}\\p{N}])${word}(?![\\p{L}\\p{N}])`),
99
+ ...cjk,
100
+ ...cjkBigrams(query).split(" ").filter(Boolean)
101
+ ];
102
+ if (alternatives.length === 0) return [];
103
+ alternatives.sort((a, b) => b.length - a.length);
104
+ const pattern = new RegExp(alternatives.join("|"), "giu");
105
+ return [...text.matchAll(pattern)].map((m) => ({
106
+ start: m.index,
107
+ end: m.index + m[0].length
108
+ }));
109
+ }
110
+ /**
111
+ * Splits Markdown at h1–h3 headings, so every chunk links to its own section: a heading's
112
+ * explicit `{#id}` is its anchor, otherwise its GitHub slug. A section over
113
+ * 800 tokens is cut into pieces of up to 800 at paragraph, then sentence boundaries. A heading
114
+ * with nothing under it but deeper headings makes no chunk of its own; it stays in their paths.
115
+ */
116
+ function chunkMarkdown(markdown) {
117
+ return sections(markdown).flatMap((section) => pack(section.blocks.flatMap(splitBlock)).map((text) => ({
118
+ path: section.path,
119
+ anchor: section.anchor,
120
+ text
121
+ })));
122
+ }
123
+ function sections(markdown) {
124
+ const slugs = /* @__PURE__ */ new Map();
125
+ const stack = [];
126
+ let current = {
127
+ path: [],
128
+ anchor: void 0,
129
+ blocks: []
130
+ };
131
+ const all = [current];
132
+ let block = [];
133
+ let fence;
134
+ const endBlock = () => {
135
+ const text = plainText(block.join("\n"));
136
+ if (text) current.blocks.push(text);
137
+ block = [];
138
+ };
139
+ for (const line of markdown.split(/\r?\n/)) {
140
+ const fenceMark = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
141
+ if (fence) {
142
+ if (fenceMark?.startsWith(fence)) fence = void 0;
143
+ block.push(line);
144
+ continue;
145
+ }
146
+ if (fenceMark) {
147
+ fence = fenceMark;
148
+ block.push(line);
149
+ continue;
150
+ }
151
+ const heading = /^ {0,3}(#{1,3})[ \t]+(.*?)(?:[ \t]+#+)?[ \t]*$/.exec(line);
152
+ if (heading) {
153
+ endBlock();
154
+ const level = heading[1].length;
155
+ const explicit = /^(.*?)[ \t]*\{#([\p{L}\p{N}_][\p{L}\p{N}_:.-]*)\}$/u.exec(heading[2]);
156
+ const title = plainText(explicit ? explicit[1] : heading[2]);
157
+ stack.length = level - 1;
158
+ stack[level - 1] = title;
159
+ let anchor;
160
+ if (explicit) {
161
+ anchor = explicit[2];
162
+ slugs.set(anchor, (slugs.get(anchor) ?? 0) + 1);
163
+ } else anchor = uniqueSlug(slugs, title);
164
+ current = {
165
+ path: stack.filter(Boolean),
166
+ anchor,
167
+ blocks: []
168
+ };
169
+ all.push(current);
170
+ } else if (!line.trim()) endBlock();
171
+ else block.push(line);
172
+ }
173
+ endBlock();
174
+ return all.filter((section) => section.blocks.length > 0).map((section) => {
175
+ const heading = section.path.at(-1);
176
+ return heading ? {
177
+ ...section,
178
+ blocks: [heading, ...section.blocks]
179
+ } : section;
180
+ });
181
+ }
182
+ /** GitHub's heading anchor: lowercased, punctuation dropped, spaces to hyphens, deduplicated. */
183
+ function slug(heading) {
184
+ return heading.toLowerCase().replace(/[^\p{L}\p{M}\p{N}\p{Pc}\- ]/gu, "").replace(/ /g, "-");
185
+ }
186
+ function uniqueSlug(seen, heading) {
187
+ const base = slug(heading);
188
+ const count = seen.get(base) ?? 0;
189
+ seen.set(base, count + 1);
190
+ return count === 0 ? base : `${base}-${count}`;
191
+ }
192
+ /** Markdown to plain text: code fences, emphasis, links, images, HTML and markers removed. */
193
+ function plainText(markdown) {
194
+ return markdown.replace(/^\s*(`{3,}|~{3,}).*$/gm, "").replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/<(https?:\/\/[^>]+)>/g, "$1").replace(/<\/?[A-Za-z][^>]*>/g, "").replace(/^\s{0,3}(#{1,6}\s+|>\s?|[-*+]\s+|\d+[.)]\s+)/gm, "").replace(/\*\*|__|\*|`|~~/g, "").replace(/\s+/g, " ").trim();
195
+ }
196
+ /** A block over the limit, cut at sentence ends, and a sentence over it by characters. */
197
+ function splitBlock(block) {
198
+ if (estimateTokens(block) <= 800) return [block];
199
+ return pack(block.split(/(?<=[.!?。!?])\s*/u)).flatMap((piece) => {
200
+ if (estimateTokens(piece) <= 800) return [piece];
201
+ return splitTokens(piece, 800).map((part) => part.text);
202
+ });
203
+ }
204
+ /** Joins consecutive parts while the result stays within the limit. */
205
+ function pack(parts) {
206
+ const packed = [];
207
+ let tokens = 0;
208
+ for (const part of parts) {
209
+ const size = estimateTokens(part);
210
+ if (packed.length > 0 && tokens + size <= 800) {
211
+ packed[packed.length - 1] += ` ${part}`;
212
+ tokens += size;
213
+ } else {
214
+ packed.push(part);
215
+ tokens = size;
216
+ }
217
+ }
218
+ return packed;
219
+ }
220
+ // src/schema.ts
221
+ const NAMESPACE = "knowledge";
222
+ /**
223
+ * Documents keep a content hash; chunks keep the hit's fields, the plain text, the CJK bigrams
224
+ * of title and text, and an optional embedding. Title and text are indexed together. SQLite indexes text in an FTS5 table that triggers keep in step with the
225
+ * chunks (by `seq`);
226
+ * Postgres in a generated `tsvector` column.
227
+ */
228
+ const migrations = [{
229
+ id: "001_knowledge",
230
+ postgres: `
231
+ CREATE TABLE aidy_knowledge_documents (
232
+ collection TEXT NOT NULL,
233
+ id TEXT NOT NULL,
234
+ hash TEXT NOT NULL,
235
+ PRIMARY KEY (collection, id)
236
+ );
237
+ CREATE TABLE aidy_knowledge_chunks (
238
+ seq BIGSERIAL PRIMARY KEY,
239
+ id TEXT NOT NULL UNIQUE,
240
+ collection TEXT NOT NULL,
241
+ document_id TEXT NOT NULL,
242
+ title TEXT NOT NULL,
243
+ url TEXT NOT NULL,
244
+ updated_at TEXT,
245
+ body TEXT NOT NULL,
246
+ cjk TEXT NOT NULL,
247
+ hash TEXT NOT NULL,
248
+ embedding REAL[],
249
+ tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('simple', title || ' ' || body || ' ' || cjk)) STORED,
250
+ FOREIGN KEY (collection, document_id)
251
+ REFERENCES aidy_knowledge_documents (collection, id) ON DELETE CASCADE
252
+ );
253
+ CREATE INDEX aidy_knowledge_chunks_document ON aidy_knowledge_chunks (collection, document_id);
254
+ CREATE INDEX aidy_knowledge_chunks_tsv ON aidy_knowledge_chunks USING gin (tsv);`,
255
+ sqlite: `
256
+ CREATE TABLE aidy_knowledge_documents (
257
+ collection TEXT NOT NULL,
258
+ id TEXT NOT NULL,
259
+ hash TEXT NOT NULL,
260
+ PRIMARY KEY (collection, id)
261
+ );
262
+ CREATE TABLE aidy_knowledge_chunks (
263
+ seq INTEGER PRIMARY KEY,
264
+ id TEXT NOT NULL UNIQUE,
265
+ collection TEXT NOT NULL,
266
+ document_id TEXT NOT NULL,
267
+ title TEXT NOT NULL,
268
+ url TEXT NOT NULL,
269
+ updated_at TEXT,
270
+ body TEXT NOT NULL,
271
+ cjk TEXT NOT NULL,
272
+ hash TEXT NOT NULL,
273
+ embedding TEXT,
274
+ FOREIGN KEY (collection, document_id)
275
+ REFERENCES aidy_knowledge_documents (collection, id) ON DELETE CASCADE
276
+ );
277
+ CREATE INDEX aidy_knowledge_chunks_document ON aidy_knowledge_chunks (collection, document_id);
278
+ CREATE VIRTUAL TABLE aidy_knowledge_fts USING fts5(text, cjk, tokenize = 'unicode61');
279
+ CREATE TRIGGER aidy_knowledge_chunks_insert AFTER INSERT ON aidy_knowledge_chunks BEGIN
280
+ INSERT INTO aidy_knowledge_fts (rowid, text, cjk)
281
+ VALUES (new.seq, new.title || ' ' || new.body, new.cjk);
282
+ END;
283
+ CREATE TRIGGER aidy_knowledge_chunks_delete AFTER DELETE ON aidy_knowledge_chunks BEGIN
284
+ DELETE FROM aidy_knowledge_fts WHERE rowid = old.seq;
285
+ END;`
286
+ }];
287
+ // src/index.ts
288
+ /** `SearchHit.content` is at most 4000 characters (spec/search.md). */
289
+ const MAX_CONTENT_CHARS = 4e3;
290
+ /** BM25 candidates, and vector candidates, ranked before fusion. */
291
+ const CANDIDATES = 200;
292
+ /** Chunks scanned for vector similarity, newest first. */
293
+ const VECTOR_SCAN = 2e4;
294
+ const RRF_K = 60;
295
+ const URL_PATTERN = /^(\/(?!\/)|https?:\/\/)/;
296
+ function createKnowledge(options) {
297
+ const { driver, embed, dimensions } = options;
298
+ async function vectors(texts) {
299
+ if (!embed || texts.length === 0) return [];
300
+ const result = await embed(texts);
301
+ if (result.length !== texts.length || dimensions !== void 0 && result.some((vector) => vector.length !== dimensions)) throw new Error(`embed returned ${result.length} vectors for ${texts.length} texts` + (dimensions === void 0 ? "" : `, each must have ${dimensions} dimensions`));
302
+ return result;
303
+ }
304
+ const encode = (vector) => driver.dialect === "postgres" ? vector : JSON.stringify(vector);
305
+ const decode = (value) => typeof value === "string" ? JSON.parse(value) : value;
306
+ async function ingestOne(collection, doc) {
307
+ if (!doc.title.trim()) throw new TypeError(`document ${doc.id} has no title`);
308
+ if (!URL_PATTERN.test(doc.url)) throw new TypeError(`document ${doc.id}: url must be a product path or an http(s) URL`);
309
+ const hash = await sha256(JSON.stringify([
310
+ doc.title,
311
+ doc.url,
312
+ doc.updatedAt ?? null,
313
+ doc.markdown,
314
+ embed !== void 0
315
+ ]));
316
+ const [existing] = await driver.query("SELECT hash FROM aidy_knowledge_documents WHERE collection = $1 AND id = $2", [collection, doc.id]);
317
+ if (existing?.hash === hash) return false;
318
+ const chunks = await Promise.all(chunkMarkdown(doc.markdown).map(async (chunk, position) => {
319
+ const title = [doc.title, ...chunk.path.filter((h, i) => i > 0 || h !== doc.title)].join(" › ");
320
+ return {
321
+ id: `${collection}/${doc.id}/${position}`,
322
+ title,
323
+ url: chunk.anchor ? `${doc.url}#${chunk.anchor}` : doc.url,
324
+ body: chunk.text,
325
+ hash: await sha256(`${title}\n${chunk.text}`)
326
+ };
327
+ }));
328
+ const known = /* @__PURE__ */ new Map();
329
+ if (embed) {
330
+ const rows = await driver.query(`SELECT hash, embedding FROM aidy_knowledge_chunks
331
+ WHERE collection = $1 AND document_id = $2 AND embedding IS NOT NULL`, [collection, doc.id]);
332
+ for (const row of rows) known.set(row.hash, decode(row.embedding));
333
+ const missing = chunks.filter((chunk) => !known.has(chunk.hash));
334
+ const fresh = await vectors(missing.map((chunk) => `${chunk.title}\n${chunk.body}`));
335
+ missing.forEach((chunk, i) => {
336
+ known.set(chunk.hash, fresh[i]);
337
+ });
338
+ }
339
+ await driver.transaction(async (tx) => {
340
+ await tx.query("DELETE FROM aidy_knowledge_documents WHERE collection = $1 AND id = $2", [collection, doc.id]);
341
+ await tx.query("INSERT INTO aidy_knowledge_documents (collection, id, hash) VALUES ($1, $2, $3)", [
342
+ collection,
343
+ doc.id,
344
+ hash
345
+ ]);
346
+ for (const chunk of chunks) {
347
+ const vector = known.get(chunk.hash);
348
+ await tx.query(`INSERT INTO aidy_knowledge_chunks
349
+ (id, collection, document_id, title, url, updated_at, body, cjk, hash, embedding)
350
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, [
351
+ chunk.id,
352
+ collection,
353
+ doc.id,
354
+ chunk.title,
355
+ chunk.url,
356
+ doc.updatedAt ?? null,
357
+ chunk.body,
358
+ cjkBigrams(`${chunk.title} ${chunk.body}`),
359
+ chunk.hash,
360
+ vector ? encode(vector) : null
361
+ ]);
362
+ }
363
+ });
364
+ return true;
365
+ }
366
+ async function lexical(terms, collections) {
367
+ if (terms.length === 0) return [];
368
+ const within = collections.map((_, i) => `$${i + 2}`).join(", ");
369
+ const columns = "c.id, c.title, c.url, c.updated_at, c.body, c.embedding";
370
+ if (driver.dialect === "postgres") return driver.query(`SELECT ${columns} FROM aidy_knowledge_chunks c, to_tsquery('simple', $1) q
371
+ WHERE c.tsv @@ q AND c.collection IN (${within})
372
+ ORDER BY ts_rank(c.tsv, q) DESC, c.seq LIMIT ${CANDIDATES}`, [terms.map((term) => `'${term}'`).join(" | "), ...collections]);
373
+ return driver.query(`SELECT ${columns} FROM aidy_knowledge_fts f
374
+ JOIN aidy_knowledge_chunks c ON c.seq = f.rowid
375
+ WHERE aidy_knowledge_fts MATCH $1 AND c.collection IN (${within})
376
+ ORDER BY bm25(aidy_knowledge_fts), c.seq LIMIT ${CANDIDATES}`, [terms.map((term) => `"${term}"`).join(" OR "), ...collections]);
377
+ }
378
+ async function semantic(query, collections, lexicalRows) {
379
+ const [target] = await vectors([query]);
380
+ if (!target) return [];
381
+ const within = collections.map((_, i) => `$${i + 1}`).join(", ");
382
+ const scanned = await driver.query(`SELECT id, embedding FROM aidy_knowledge_chunks
383
+ WHERE collection IN (${within}) AND embedding IS NOT NULL
384
+ ORDER BY seq DESC LIMIT ${VECTOR_SCAN}`, [...collections]);
385
+ const candidates = /* @__PURE__ */ new Map();
386
+ for (const row of lexicalRows) if (row.embedding) candidates.set(row.id, row.embedding);
387
+ for (const row of scanned) candidates.set(row.id, row.embedding);
388
+ return [...candidates].map(([id, embedding]) => ({
389
+ id,
390
+ score: cosine(target, decode(embedding))
391
+ })).sort((a, b) => b.score - a.score).slice(0, CANDIDATES).map(({ id }) => id);
392
+ }
393
+ async function search(query, { collections, limit }) {
394
+ if (collections.length === 0 || !query.trim()) return [];
395
+ const lexicalRows = await lexical(queryTerms(query), collections);
396
+ const rankings = [lexicalRows.map((row) => row.id)];
397
+ if (embed) rankings.push(await semantic(query, collections, lexicalRows));
398
+ const scores = /* @__PURE__ */ new Map();
399
+ for (const ranking of rankings) ranking.forEach((id, rank) => {
400
+ scores.set(id, (scores.get(id) ?? 0) + 1 / (RRF_K + rank + 1));
401
+ });
402
+ const ids = [...scores].sort((a, b) => b[1] - a[1]).slice(0, Math.min(limit, MAX_SEARCH_LIMIT)).map(([id]) => id);
403
+ const rows = new Map(lexicalRows.map((row) => [row.id, row]));
404
+ const missing = ids.filter((id) => !rows.has(id));
405
+ if (missing.length > 0) {
406
+ const fetched = await driver.query(`SELECT id, title, url, updated_at, body FROM aidy_knowledge_chunks
407
+ WHERE id IN (${missing.map((_, i) => `$${i + 1}`).join(", ")})`, missing);
408
+ for (const row of fetched) rows.set(row.id, row);
409
+ }
410
+ return ids.flatMap((id) => {
411
+ const row = rows.get(id);
412
+ if (!row) return [];
413
+ return [{
414
+ type: "doc",
415
+ id: row.id,
416
+ title: row.title,
417
+ url: row.url,
418
+ excerpt: excerpt(row.body, query),
419
+ ...row.updated_at ? { updatedAt: row.updated_at } : {},
420
+ content: row.body.slice(0, MAX_CONTENT_CHARS)
421
+ }];
422
+ });
423
+ }
424
+ return {
425
+ async migrate() {
426
+ await migrate(driver, NAMESPACE, migrations);
427
+ },
428
+ async ingest(collection, documents) {
429
+ let updated = 0;
430
+ for (const doc of documents) if (await ingestOne(collection, doc)) updated++;
431
+ return {
432
+ updated,
433
+ unchanged: documents.length - updated
434
+ };
435
+ },
436
+ async remove(collection, ids) {
437
+ if (ids.length === 0) return;
438
+ await driver.query(`DELETE FROM aidy_knowledge_documents
439
+ WHERE collection = $1 AND id IN (${ids.map((_, i) => `$${i + 2}`).join(", ")})`, [collection, ...ids]);
440
+ },
441
+ search,
442
+ source({ name = KNOWLEDGE_SOURCE, collections }) {
443
+ return {
444
+ name,
445
+ search: (query, { limit, identity }) => search(query, {
446
+ collections: collections(identity),
447
+ limit
448
+ })
449
+ };
450
+ }
451
+ };
452
+ }
453
+ function cosine(a, b) {
454
+ let dot = 0;
455
+ let na = 0;
456
+ let nb = 0;
457
+ for (let i = 0; i < a.length; i++) {
458
+ const x = a[i];
459
+ const y = b[i] ?? 0;
460
+ dot += x * y;
461
+ na += x * x;
462
+ nb += y * y;
463
+ }
464
+ return na === 0 || nb === 0 ? 0 : dot / Math.sqrt(na * nb);
465
+ }
466
+ async function sha256(text) {
467
+ const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
468
+ return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
469
+ }
470
+ export { createKnowledge };
package/package.json CHANGED
@@ -1,6 +1,39 @@
1
1
  {
2
2
  "name": "@aidy-dev/knowledge",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
3
+ "version": "0.2.0",
4
+ "description": "aidy's knowledge index: Markdown ingestion, chunking and hybrid BM25 + vector search over any SqlDriver.",
5
+ "license": "Apache-2.0",
6
+ "homepage": "https://aidy.lpythu.workers.dev",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/aidy-dev/aidy.git",
10
+ "directory": "packages/knowledge"
11
+ },
12
+ "type": "module",
13
+ "sideEffects": false,
14
+ "files": [
15
+ "dist"
16
+ ],
17
+ "exports": {
18
+ ".": {
19
+ "@aidy-dev/source": "./src/index.ts",
20
+ "default": "./dist/index.js"
21
+ },
22
+ "./package.json": "./package.json"
23
+ },
24
+ "publishConfig": {
25
+ "access": "public"
26
+ },
27
+ "dependencies": {
28
+ "@aidy-dev/sql": "0.2.0",
29
+ "@aidy-dev/spec": "0.2.0"
30
+ },
31
+ "devDependencies": {
32
+ "postgres": "^4.1.1"
33
+ },
34
+ "scripts": {
35
+ "build": "tsdown",
36
+ "typecheck": "tsc -p .",
37
+ "test": "vitest run"
38
+ }
6
39
  }