@aidy-dev/knowledge 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -63,8 +63,11 @@ const knowledge = createKnowledge({
63
63
 
64
64
  - **Chunks.** Markdown is split at h1–h3 headings; each section is a chunk, and a section over
65
65
  ~800 tokens (one per CJK character, one per four other characters) is cut at paragraphs, then
66
- sentences. A chunk's URL is the document URL plus the section's GitHub-style anchor; its
67
- title is `document title › heading path`. Text is stored as plain text.
66
+ sentences. A chunk's URL is the document URL plus the section's anchor: the heading's
67
+ explicit id (`## Rotate keys {#rotate}`, stripped from the title) or else its GitHub-style
68
+ slug; its title is `document title › heading path`. Text is stored as plain text.
69
+ - **Hits.** Each hit carries an `excerpt` (≤ 300 characters around the matches, for people)
70
+ and `content`, the chunk's text up to 4000 characters, which only the model reads.
68
71
  - **Lexical.** SQLite (and D1) uses FTS5 with the `unicode61` tokenizer and ranks by `bm25()`.
69
72
  Postgres uses a `tsvector` with the `simple` configuration and ranks by `ts_rank` (Postgres
70
73
  has no BM25). CJK text has no spaces, so the overlapping bigrams of CJK runs are indexed in a
package/dist/index.js CHANGED
@@ -108,7 +108,8 @@ function findMatches(text, query) {
108
108
  }));
109
109
  }
110
110
  /**
111
- * Splits Markdown at h1–h3 headings, so every chunk links to its own section. A section over
111
+ * Splits Markdown at h1–h3 headings, so every chunk links to its own section: a heading's
112
+ * explicit `{#id}` is its anchor, otherwise its GitHub slug. A section over
112
113
  * 800 tokens is cut into pieces of up to 800 at paragraph, then sentence boundaries. A heading
113
114
  * with nothing under it but deeper headings makes no chunk of its own; it stays in their paths.
114
115
  */
@@ -151,12 +152,18 @@ function sections(markdown) {
151
152
  if (heading) {
152
153
  endBlock();
153
154
  const level = heading[1].length;
154
- const title = plainText(heading[2]);
155
+ const explicit = /^(.*?)[ \t]*\{#([\p{L}\p{N}_][\p{L}\p{N}_:.-]*)\}$/u.exec(heading[2]);
156
+ const title = plainText(explicit ? explicit[1] : heading[2]);
155
157
  stack.length = level - 1;
156
158
  stack[level - 1] = title;
159
+ let anchor;
160
+ if (explicit) {
161
+ anchor = explicit[2];
162
+ slugs.set(anchor, (slugs.get(anchor) ?? 0) + 1);
163
+ } else anchor = uniqueSlug(slugs, title);
157
164
  current = {
158
165
  path: stack.filter(Boolean),
159
- anchor: uniqueSlug(slugs, title),
166
+ anchor,
160
167
  blocks: []
161
168
  };
162
169
  all.push(current);
@@ -278,6 +285,8 @@ const migrations = [{
278
285
  END;`
279
286
  }];
280
287
  // src/index.ts
288
+ /** `SearchHit.content` is at most 4000 characters (spec/search.md). */
289
+ const MAX_CONTENT_CHARS = 4e3;
281
290
  /** BM25 candidates, and vector candidates, ranked before fusion. */
282
291
  const CANDIDATES = 200;
283
292
  /** Chunks scanned for vector similarity, newest first. */
@@ -407,7 +416,8 @@ function createKnowledge(options) {
407
416
  title: row.title,
408
417
  url: row.url,
409
418
  excerpt: excerpt(row.body, query),
410
- ...row.updated_at ? { updatedAt: row.updated_at } : {}
419
+ ...row.updated_at ? { updatedAt: row.updated_at } : {},
420
+ content: row.body.slice(0, MAX_CONTENT_CHARS)
411
421
  }];
412
422
  });
413
423
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aidy-dev/knowledge",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "description": "aidy's knowledge index: Markdown ingestion, chunking and hybrid BM25 + vector search over any SqlDriver.",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://aidy.lpythu.workers.dev",
@@ -25,8 +25,8 @@
25
25
  "access": "public"
26
26
  },
27
27
  "dependencies": {
28
- "@aidy-dev/sql": "0.1.0",
29
- "@aidy-dev/spec": "0.1.0"
28
+ "@aidy-dev/sql": "0.2.0",
29
+ "@aidy-dev/spec": "0.2.0"
30
30
  },
31
31
  "devDependencies": {
32
32
  "postgres": "^4.1.1"