@dikolab/kbdb 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -15
- package/dist/cli.cjs +461 -72
- package/dist/cli.cjs.map +4 -4
- package/dist/cli.mjs +453 -64
- package/dist/cli.mjs.map +4 -4
- package/dist/kbdb-worker.cjs +395 -60
- package/dist/kbdb-worker.cjs.map +4 -4
- package/dist/mod.cjs +17 -0
- package/dist/mod.cjs.map +2 -2
- package/dist/mod.mjs +17 -0
- package/dist/mod.mjs.map +2 -2
- package/dist/src/shared/cli/functions/dispatch-command.function.d.ts +3 -1
- package/dist/src/shared/cli/functions/install-default-skills.function.d.ts +12 -0
- package/dist/src/shared/cli/functions/run-check.function.d.ts +47 -5
- package/dist/src/shared/cli/functions/run-export.function.d.ts +34 -0
- package/dist/src/shared/cli/functions/run-gc.function.d.ts +6 -0
- package/dist/src/shared/cli/functions/run-learn.function.d.ts +35 -6
- package/dist/src/shared/cli/functions/run-recall.function.d.ts +45 -0
- package/dist/src/shared/cli/functions/run-search.function.d.ts +52 -15
- package/dist/src/shared/cli/index.d.ts +4 -0
- package/dist/src/shared/cli/typings/cli-options.interface.d.ts +13 -0
- package/dist/src/shared/cli/typings/learn-result.model.d.ts +19 -0
- package/dist/src/shared/cli/typings/search-display.model.d.ts +18 -0
- package/dist/src/shared/mcp/classes/mcp-server.class.d.ts +8 -0
- package/dist/src/shared/mcp/typings/mcp-client.interface.d.ts +43 -0
- package/dist/src/shared/wasm-codec/functions/decode-add-section-result.function.d.ts +48 -0
- package/dist/src/shared/wasm-codec/functions/decode-scored-results.function.d.ts +58 -0
- package/dist/src/shared/wasm-codec/functions/decode-section-records.function.d.ts +33 -0
- package/dist/src/shared/wasm-codec/functions/encode-add-section-params.function.d.ts +52 -0
- package/dist/src/shared/wasm-codec/index.d.ts +2 -0
- package/dist/src/shared/worker-client/classes/worker-client.class.d.ts +16 -0
- package/dist/src/shared/worker-client/index.d.ts +1 -1
- package/dist/src/shared/worker-client/typings/add-section-result.model.d.ts +29 -15
- package/dist/src/shared/worker-client/typings/export-result.model.d.ts +14 -0
- package/dist/src/shared/worker-client/typings/gc-result.model.d.ts +6 -0
- package/dist/src/shared/worker-client/typings/search-result.model.d.ts +12 -0
- package/dist/src/shared/worker-client/typings/section.model.d.ts +16 -0
- package/dist/src/shared/worker-daemon/classes/worker-daemon.class.d.ts +35 -0
- package/dist/wasm/fs-database/kbdb_fs_database.d.ts +22 -1
- package/dist/wasm/fs-database/kbdb_fs_database.js +28 -1
- package/dist/wasm/fs-database/kbdb_fs_database_bg.wasm +0 -0
- package/dist/wasm/fs-database/kbdb_fs_database_bg.wasm.d.ts +1 -0
- package/dist/wasm/kb-worker/kbdb_worker.d.ts +48 -1
- package/dist/wasm/kb-worker/kbdb_worker.js +60 -1
- package/dist/wasm/kb-worker/kbdb_worker_bg.wasm +0 -0
- package/dist/wasm/kb-worker/kbdb_worker_bg.wasm.d.ts +2 -0
- package/dist/wasm/query-parser/kbdb_query_parser_bg.wasm +0 -0
- package/dist/worker.mjs +396 -60
- package/dist/worker.mjs.map +4 -4
- package/docs/details/cli.md +102 -15
- package/docs/details/library-api.md +55 -6
- package/docs/details/mcp-server.md +8 -4
- package/docs/details/search-and-ranking.md +63 -5
- package/docs/details/storage.md +2 -0
- package/docs/goals/cli.md +255 -14
- package/docs/goals/database.md +244 -17
- package/docs/goals/document.md +1 -1
- package/docs/goals/mcp.md +83 -14
- package/docs/goals/query-parser.md +98 -4
- package/docs/goals/query-result.md +177 -10
- package/docs/goals/skills.md +72 -0
- package/docs/goals/worker-daemon.md +54 -7
- package/docs/modules/rust/fs-database/functions/integrity.md +11 -0
- package/docs/modules/rust/fs-database/functions/section.md +23 -6
- package/docs/modules/rust/fs-database/module.md +1 -0
- package/docs/modules/rust/fs-database/types/section.md +27 -1
- package/docs/modules/rust/overview.md +2 -1
- package/docs/modules/rust/query-parser/functions/module.md +2 -2
- package/docs/modules/rust/query-parser/functions/pipeline.md +44 -6
- package/docs/modules/rust/query-parser/module.md +5 -4
- package/docs/modules/rust/query-parser/types/keyphrase.md +11 -0
- package/docs/modules/rust/query-parser/types/module.md +3 -3
- package/docs/modules/typescript/shared/cli/functions.md +68 -3
- package/package.json +1 -1
- package/src/shared/cli/functions/dispatch-command.function.ts +53 -1
- package/src/shared/cli/functions/format-output.function.ts +36 -12
- package/src/shared/cli/functions/generate-help.function.ts +56 -1
- package/src/shared/cli/functions/install-default-skills.function.ts +296 -0
- package/src/shared/cli/functions/parse-args.function.ts +13 -0
- package/src/shared/cli/functions/run-check.function.ts +87 -6
- package/src/shared/cli/functions/run-export.function.ts +40 -0
- package/src/shared/cli/functions/run-gc.function.ts +6 -0
- package/src/shared/cli/functions/run-init.function.ts +2 -0
- package/src/shared/cli/functions/run-learn.function.ts +68 -16
- package/src/shared/cli/functions/run-recall.function.ts +66 -0
- package/src/shared/cli/functions/run-search.function.ts +58 -15
- package/src/shared/cli/index.ts +4 -0
- package/src/shared/cli/typings/cli-options.interface.ts +13 -0
- package/src/shared/cli/typings/learn-result.model.ts +16 -0
- package/src/shared/cli/typings/search-display.model.ts +18 -0
- package/src/shared/mcp/classes/mcp-server.class.ts +30 -1
- package/src/shared/mcp/functions/create-tool-definitions.function.ts +46 -5
- package/src/shared/mcp/functions/handle-tool-call.function.ts +70 -2
- package/src/shared/mcp/typings/mcp-client.interface.ts +45 -0
- package/src/shared/wasm-codec/functions/decode-add-section-result.function.ts +112 -0
- package/src/shared/wasm-codec/functions/decode-scored-results.function.ts +86 -0
- package/src/shared/wasm-codec/functions/decode-section-records.function.ts +87 -0
- package/src/shared/wasm-codec/functions/encode-add-section-params.function.ts +68 -0
- package/src/shared/wasm-codec/index.ts +5 -0
- package/src/shared/worker-client/classes/worker-client.class.ts +19 -0
- package/src/shared/worker-client/index.ts +4 -1
- package/src/shared/worker-client/typings/add-section-result.model.ts +30 -15
- package/src/shared/worker-client/typings/export-result.model.ts +14 -0
- package/src/shared/worker-client/typings/gc-result.model.ts +6 -0
- package/src/shared/worker-client/typings/search-result.model.ts +12 -0
- package/src/shared/worker-client/typings/section.model.ts +16 -0
- package/src/shared/worker-daemon/classes/worker-daemon.class.ts +315 -46
- package/src/shared/worker-daemon/functions/create-host-imports.function.ts +19 -4
- package/src/shared/worker-daemon/functions/create-ipc-server.function.ts +5 -1
- package/src/wasm/fs-database/kbdb_fs_database.d.ts +22 -1
- package/src/wasm/fs-database/kbdb_fs_database.js +28 -1
- package/src/wasm/fs-database/kbdb_fs_database_bg.wasm +0 -0
- package/src/wasm/fs-database/kbdb_fs_database_bg.wasm.d.ts +1 -0
- package/src/wasm/kb-worker/kbdb_worker.d.ts +48 -1
- package/src/wasm/kb-worker/kbdb_worker.js +60 -1
- package/src/wasm/kb-worker/kbdb_worker_bg.wasm +0 -0
- package/src/wasm/kb-worker/kbdb_worker_bg.wasm.d.ts +2 -0
- package/src/wasm/query-parser/kbdb_query_parser_bg.wasm +0 -0
|
@@ -434,6 +434,34 @@ order** (highest relevance first). Ties are broken
|
|
|
434
434
|
by kbid lexicographic order for deterministic
|
|
435
435
|
ordering.
|
|
436
436
|
|
|
437
|
+
## Score Normalization
|
|
438
|
+
|
|
439
|
+
Raw BM25F scores vary in magnitude across queries and
|
|
440
|
+
corpora, making them difficult to interpret in
|
|
441
|
+
isolation. The ranking pipeline produces a
|
|
442
|
+
**normalized confidence** score alongside the raw
|
|
443
|
+
score:
|
|
444
|
+
|
|
445
|
+
```
|
|
446
|
+
confidence = final_score / max_score_in_result_set
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
- The top-ranked result always has `confidence = 1.0`.
|
|
450
|
+
- A result with half the raw score of the top result
|
|
451
|
+
has `confidence = 0.5`.
|
|
452
|
+
- When the result set is empty, no normalization is
|
|
453
|
+
needed.
|
|
454
|
+
|
|
455
|
+
`confidence` is a relative measure within a single
|
|
456
|
+
query's result set -- it does not compare across
|
|
457
|
+
different queries. Its purpose is to let the caller
|
|
458
|
+
distinguish strong matches from weak tail results
|
|
459
|
+
without needing to understand BM25F score magnitudes.
|
|
460
|
+
|
|
461
|
+
The CLI exposes `confidence` in `SearchDisplay` for
|
|
462
|
+
human inspection. The MCP projection omits it --
|
|
463
|
+
result ordering is the ranking signal for AI agents.
|
|
464
|
+
|
|
437
465
|
## Result Truncation and Pagination
|
|
438
466
|
|
|
439
467
|
After sorting, results are windowed by `offset` and
|
|
@@ -452,6 +480,7 @@ interface PagedSearchResult {
|
|
|
452
480
|
offset: number;
|
|
453
481
|
limit: number;
|
|
454
482
|
has_more: boolean;
|
|
483
|
+
relaxed: boolean;
|
|
455
484
|
}
|
|
456
485
|
```
|
|
457
486
|
|
|
@@ -462,6 +491,7 @@ interface PagedSearchResult {
|
|
|
462
491
|
| `offset` | Echo of requested offset (or 0) |
|
|
463
492
|
| `limit` | Echo of effective limit (or 20) |
|
|
464
493
|
| `has_more` | `true` when `offset + items.length < total` |
|
|
494
|
+
| `relaxed` | `true` when results came from relaxed-match fallback (see below) |
|
|
465
495
|
|
|
466
496
|
`has_more` is a convenience boolean so the caller
|
|
467
497
|
does not need to compute whether more pages exist.
|
|
@@ -489,14 +519,26 @@ interface SectionMatch {
|
|
|
489
519
|
type: string;
|
|
490
520
|
heading: string;
|
|
491
521
|
score: number;
|
|
522
|
+
confidence: number;
|
|
492
523
|
snippet: string;
|
|
493
524
|
matched_terms: string[];
|
|
525
|
+
matched_fields: string[];
|
|
526
|
+
created_at: string;
|
|
494
527
|
}
|
|
495
528
|
```
|
|
496
529
|
|
|
497
530
|
`heading` is `string | null` -- `null` when the
|
|
498
|
-
section has no title. `
|
|
499
|
-
|
|
531
|
+
section has no title. `confidence` is the normalized
|
|
532
|
+
score (0.0--1.0) relative to the top result (see
|
|
533
|
+
Score Normalization above). `matched_fields` is a
|
|
534
|
+
subset of `["heading", "body", "code"]` indicating
|
|
535
|
+
which fields contained term matches -- lets the
|
|
536
|
+
caller see whether the match was in a title vs body
|
|
537
|
+
text. `created_at` is the section's ISO 8601
|
|
538
|
+
creation timestamp.
|
|
539
|
+
|
|
540
|
+
`snippet` is always populated for text and code
|
|
541
|
+
sections using KWIC
|
|
500
542
|
(keyword-in-context): ~150 characters centered on
|
|
501
543
|
the highest-scoring matched term position, with
|
|
502
544
|
`...` ellipsis at boundaries. If the best match is
|
|
@@ -576,17 +618,25 @@ interface McpSearchResult {
|
|
|
576
618
|
docids: string[];
|
|
577
619
|
snippet: string;
|
|
578
620
|
matched_terms: string[];
|
|
621
|
+
matched_fields: string[];
|
|
622
|
+
created_at: string;
|
|
579
623
|
content?: string;
|
|
580
624
|
}
|
|
581
625
|
```
|
|
582
626
|
|
|
583
|
-
The `score`
|
|
584
|
-
intentionally omitted. Results
|
|
585
|
-
relevance descending; the position
|
|
586
|
-
is the ranking signal. Exposing
|
|
587
|
-
scores provides no actionable
|
|
588
|
-
agent and risks confusing score
|
|
589
|
-
different queries.
|
|
627
|
+
The `score` and `confidence` fields from
|
|
628
|
+
`SectionMatch` are intentionally omitted. Results
|
|
629
|
+
are ordered by relevance descending; the position
|
|
630
|
+
in the array is the ranking signal. Exposing
|
|
631
|
+
uncalibrated BM25F scores provides no actionable
|
|
632
|
+
information to the agent and risks confusing score
|
|
633
|
+
magnitudes across different queries.
|
|
634
|
+
|
|
635
|
+
`matched_fields` is included because it tells the
|
|
636
|
+
agent whether the match was in a heading (strong
|
|
637
|
+
topical signal) vs body text (weaker). `created_at`
|
|
638
|
+
is included to let agents prefer recent results
|
|
639
|
+
when multiple sections cover the same topic.
|
|
590
640
|
|
|
591
641
|
`matched_terms` is a flat `string[]` of the query
|
|
592
642
|
terms that matched -- no per-term frequency counts.
|
|
@@ -609,7 +659,40 @@ inspection and debugging.
|
|
|
609
659
|
|
|
610
660
|
The MCP result uses the same `PagedSearchResult`
|
|
611
661
|
envelope with `items`, `total`, `offset`, `limit`,
|
|
612
|
-
and `
|
|
662
|
+
`has_more`, and `relaxed`.
|
|
663
|
+
|
|
664
|
+
## Relaxed-Match Fallback
|
|
665
|
+
|
|
666
|
+
When the primary query (implicit AND -- all terms
|
|
667
|
+
required) returns **zero results**, the ranking
|
|
668
|
+
engine automatically retries with relaxed matching:
|
|
669
|
+
|
|
670
|
+
1. **OR mode** -- any query term may match, not all
|
|
671
|
+
required. The query plan is re-executed with OR
|
|
672
|
+
semantics using the same tokenized terms.
|
|
673
|
+
|
|
674
|
+
2. Results from the relaxed retry are flagged:
|
|
675
|
+
`relaxed: true` in the `PagedSearchResult`
|
|
676
|
+
envelope. Normal (AND) results have
|
|
677
|
+
`relaxed: false`.
|
|
678
|
+
|
|
679
|
+
Relaxed mode is transparent to the caller -- no
|
|
680
|
+
special syntax or parameters are needed. The engine
|
|
681
|
+
tries AND first and falls back to OR only when AND
|
|
682
|
+
produces zero matches. This converts a silent miss
|
|
683
|
+
("nothing found") into a soft hit ("weaker matches
|
|
684
|
+
found") that the caller can reason about.
|
|
685
|
+
|
|
686
|
+
The AI agent (or human user) can check the `relaxed`
|
|
687
|
+
flag to know that the results are partial-term
|
|
688
|
+
matches rather than full-term matches. Relaxed
|
|
689
|
+
results typically have lower relevance -- the
|
|
690
|
+
ordering and confidence scores still reflect relative
|
|
691
|
+
quality within the relaxed set.
|
|
692
|
+
|
|
693
|
+
When the primary AND query returns at least one
|
|
694
|
+
result, no fallback is attempted. The `relaxed` flag
|
|
695
|
+
is always `false` for non-empty AND results.
|
|
613
696
|
|
|
614
697
|
## References Resolution
|
|
615
698
|
|
|
@@ -691,3 +774,87 @@ of the ranking coin:
|
|
|
691
774
|
|
|
692
775
|
See [Query Parser](query-parser.md) for the full
|
|
693
776
|
tokenization pipeline and extraction algorithms.
|
|
777
|
+
|
|
778
|
+
## Content Authoring Best Practices
|
|
779
|
+
|
|
780
|
+
### How Field Weights Affect Searchability
|
|
781
|
+
|
|
782
|
+
kbdb uses three weighted fields for ranking (see
|
|
783
|
+
the [Field Weight Mapping](#field-weight-mapping)
|
|
784
|
+
table above). Content authors can influence how
|
|
785
|
+
well their content is found by understanding these
|
|
786
|
+
weights:
|
|
787
|
+
|
|
788
|
+
- **Title / heading text** carries **2.0x** the
|
|
789
|
+
ranking weight of body text. A match in a heading
|
|
790
|
+
has twice the influence on ranking as the same
|
|
791
|
+
match in the body.
|
|
792
|
+
- **Code blocks** carry **1.5x** weight. Code
|
|
793
|
+
identifiers (function names, variable names, class
|
|
794
|
+
names) are split on camelCase/snake_case boundaries
|
|
795
|
+
and indexed as separate terms.
|
|
796
|
+
- **Body text and descriptions** are the baseline at
|
|
797
|
+
**1.0x**.
|
|
798
|
+
- **Binary content** (images, attachments) is not
|
|
799
|
+
indexed at all -- searchability depends entirely on
|
|
800
|
+
the `title` (required) and `description` (optional)
|
|
801
|
+
attributes.
|
|
802
|
+
|
|
803
|
+
### Writing Index-Friendly Headings
|
|
804
|
+
|
|
805
|
+
The heading is the single highest-impact field for
|
|
806
|
+
searchability. Follow these guidelines:
|
|
807
|
+
|
|
808
|
+
- Use **descriptive, keyword-rich headings**. Prefer
|
|
809
|
+
"Authentication and Login Flow" over just "Auth
|
|
810
|
+
Flow" -- the expanded heading gives the search
|
|
811
|
+
engine more term surface.
|
|
812
|
+
- Include **synonyms and alternative phrasing** in
|
|
813
|
+
headings when natural. If a section covers logging
|
|
814
|
+
in, mention both "login" and "sign-in" if
|
|
815
|
+
applicable.
|
|
816
|
+
- Avoid generic headings like "Overview",
|
|
817
|
+
"Introduction", or "Notes" that carry no
|
|
818
|
+
discriminative value for search.
|
|
819
|
+
- Stemming is automatic ("connections" matches
|
|
820
|
+
"connect"), but different root words are NOT
|
|
821
|
+
equivalent without synonym expansion --
|
|
822
|
+
"login" does not match "sign-in" by default.
|
|
823
|
+
|
|
824
|
+
### Structuring Content for Recall
|
|
825
|
+
|
|
826
|
+
Content structure affects the progressive context
|
|
827
|
+
expansion provided by the `recall` command:
|
|
828
|
+
|
|
829
|
+
- Place the most important terms in the **first 20%**
|
|
830
|
+
of a section. The zone scoring algorithm gives
|
|
831
|
+
extra weight to terms in this heading zone.
|
|
832
|
+
- Use explicit cross-references (`$(link:kbid)`
|
|
833
|
+
markers) between related sections. The reference
|
|
834
|
+
graph enables progressive context expansion via
|
|
835
|
+
`recall` at depth 1+.
|
|
836
|
+
- Group related sections into documents. The `recall`
|
|
837
|
+
command at depth 2+ returns sibling sections,
|
|
838
|
+
giving AI agents full surrounding context.
|
|
839
|
+
- For code sections: use meaningful identifiers.
|
|
840
|
+
camelCase and snake_case names are automatically
|
|
841
|
+
split and indexed. `getUserById` produces four
|
|
842
|
+
searchable terms: "get", "user", "by", "id".
|
|
843
|
+
|
|
844
|
+
### Leveraging the Query Syntax
|
|
845
|
+
|
|
846
|
+
Guidance for AI agents and human users on writing
|
|
847
|
+
effective queries:
|
|
848
|
+
|
|
849
|
+
- Use **multiple short keyword queries** rather than
|
|
850
|
+
one long natural-language question. Lexical search
|
|
851
|
+
matches on token overlap, not semantic meaning.
|
|
852
|
+
- Use **exact phrases** (`"error handling"`) when term
|
|
853
|
+
order matters.
|
|
854
|
+
- Use **boolean operators** (`auth AND token`,
|
|
855
|
+
`cache OR redis`) to broaden or narrow results.
|
|
856
|
+
- Use **term exclusion** (`auth -oauth`) to filter out
|
|
857
|
+
irrelevant matches.
|
|
858
|
+
- After finding results with `search`, use `recall`
|
|
859
|
+
with increasing depth to progressively expand
|
|
860
|
+
context.
|
package/docs/goals/skills.md
CHANGED
|
@@ -719,12 +719,84 @@ dangling kbid. The agent is not deleted. See
|
|
|
719
719
|
[Agents](agents.md) for how dangling references are
|
|
720
720
|
handled at retrieval time.
|
|
721
721
|
|
|
722
|
+
## Default Skills
|
|
723
|
+
|
|
724
|
+
### Overview
|
|
725
|
+
|
|
726
|
+
When a new database is created via `kbdb db init`, a
|
|
727
|
+
set of default skills are automatically installed.
|
|
728
|
+
These skills provide baseline capabilities that
|
|
729
|
+
improve AI agent effectiveness. Default skills are
|
|
730
|
+
stored as regular skill sections (no special treatment
|
|
731
|
+
in the engine) and can be deleted by users if not
|
|
732
|
+
needed.
|
|
733
|
+
|
|
734
|
+
### `query-guidance` Default Skill
|
|
735
|
+
|
|
736
|
+
A built-in skill that teaches AI agents how to
|
|
737
|
+
effectively search the knowledge base. This addresses
|
|
738
|
+
the limitation that lexical search requires
|
|
739
|
+
disciplined query generation for good recall.
|
|
740
|
+
|
|
741
|
+
**Skill definition:**
|
|
742
|
+
|
|
743
|
+
```json
|
|
744
|
+
{
|
|
745
|
+
"name": "query-guidance",
|
|
746
|
+
"description": "How to effectively search this knowledge base. Read this before issuing search queries.",
|
|
747
|
+
"arguments": [],
|
|
748
|
+
"body": "This knowledge base uses lexical search (stemmed BM25F), not semantic/vector search. Follow these guidelines for effective retrieval:\n\n1. Issue MULTIPLE short keyword queries rather than one long natural-language question. Each query should target 2-3 specific terms.\n\n2. EXPAND SYNONYMS yourself. \"login\" will not match \"sign-in\" unless both terms appear in the content. Try alternative phrasings: search for \"login\", then \"sign-in\", then \"authentication\".\n\n3. USE BOOLEAN AND PHRASE SYNTAX:\n - Exact phrases: \"error handling\"\n - Boolean AND: auth AND token\n - Boolean OR: cache OR redis\n - Exclusion: auth -oauth\n\n4. LEAN ON HEADINGS. Headings carry 2x weight in ranking. Search for terms that are likely in section titles.\n\n5. STEMMING IS AUTOMATIC. \"connections\" matches \"connect\", \"running\" matches \"run\". You do not need to try inflected forms.\n\n6. USE RECALL AFTER SEARCH. After finding relevant sections with search, use recall with increasing depth (0-3) to progressively expand context: depth 0 = full content, depth 1 = parent documents + back-references, depth 2 = siblings + forward references, depth 3 = referenced sections.\n\n7. CODE IDENTIFIERS ARE SPLIT. camelCase and snake_case names are decomposed: getUserById becomes searchable as \"get\", \"user\", \"by\", \"id\".\n\n8. PAGINATION. Default limit is 20 results. Use offset for additional pages if total > limit."
|
|
749
|
+
}
|
|
750
|
+
```
|
|
751
|
+
|
|
752
|
+
### Installation Behaviour
|
|
753
|
+
|
|
754
|
+
Default skills are installed during `kbdb db init`.
|
|
755
|
+
The `runDbInit` function installs each default skill
|
|
756
|
+
via `client.addSection({ sectionType: 'skill', ... })`
|
|
757
|
+
after creating the database scaffold.
|
|
758
|
+
|
|
759
|
+
Default skills receive kbids via the same
|
|
760
|
+
content-addressing scheme as user-created skills.
|
|
761
|
+
Re-running `db init` on a fresh database produces
|
|
762
|
+
identical kbids for the same default skill content.
|
|
763
|
+
|
|
764
|
+
### `memory-protocol` Default Skill
|
|
765
|
+
|
|
766
|
+
A built-in skill that teaches AI agents *when* and
|
|
767
|
+
*what* to store in the knowledge base. This addresses
|
|
768
|
+
the risk of the brain filling with transient junk or
|
|
769
|
+
failing to capture durable facts.
|
|
770
|
+
|
|
771
|
+
**Skill definition:**
|
|
772
|
+
|
|
773
|
+
```json
|
|
774
|
+
{
|
|
775
|
+
"name": "memory-protocol",
|
|
776
|
+
"description": "Guidelines for what to store in this knowledge base and when. Read this before using the learn tool.",
|
|
777
|
+
"arguments": [],
|
|
778
|
+
"body": "Follow these guidelines to keep the knowledge base accurate and useful:\n\n1. STORE DURABLE FACTS: decisions, architecture choices, configuration values, corrections to prior knowledge, API contracts, and resolved incidents. These are the facts that matter weeks later.\n\n2. DO NOT STORE: transient conversation context, debugging output, ephemeral task state, raw log dumps, or information that changes every session. The brain should contain knowledge, not activity logs.\n\n3. SEARCH BEFORE STORING. Always search the knowledge base before adding new content. If a section already covers the topic, use learn with --replace to update it rather than creating a duplicate.\n\n4. USE --replace FOR CORRECTIONS. When a fact changes (e.g., rate limit updated from 100 to 200 req/min), re-learn the corrected content with --replace so the old version is superseded. Without --replace, both the old and new versions coexist with equal ranking.\n\n5. ATTACH TAGS for scoping. Use tags like project name, domain, or topic (e.g., --tags auth,api,v2) so queries can be filtered by scope later.\n\n6. PREFER RECENT RESULTS. When search returns multiple sections covering the same topic, prefer the one with the most recent created_at timestamp -- it is more likely to reflect the current state.\n\n7. USE RECALL AFTER SEARCH. After finding relevant sections, use recall with increasing depth (0-3) to expand context progressively before making decisions.\n\n8. INCLUDE HEADINGS. Always provide a descriptive title when storing content. Headings carry 2x ranking weight -- omitting them significantly reduces searchability."
|
|
779
|
+
}
|
|
780
|
+
```
|
|
781
|
+
|
|
782
|
+
### Future Default Skills
|
|
783
|
+
|
|
784
|
+
Additional default skills may be added in future
|
|
785
|
+
versions (e.g. `content-authoring-tips`,
|
|
786
|
+
`search-syntax-reference`). Each follows the same
|
|
787
|
+
installation pattern and is a regular skill section.
|
|
788
|
+
|
|
722
789
|
## Design Constraints
|
|
723
790
|
|
|
724
791
|
- **Skills are plain sections.** No separate entity
|
|
725
792
|
type, storage schema, or index is introduced. Skills
|
|
726
793
|
use `sectionType = 'skill'` within the existing
|
|
727
794
|
section infrastructure.
|
|
795
|
+
- **Default skills are regular sections.** They use
|
|
796
|
+
the same storage, indexing, and retrieval
|
|
797
|
+
infrastructure. There is no distinction between a
|
|
798
|
+
default skill and a user-created skill beyond the
|
|
799
|
+
installation source.
|
|
728
800
|
- **No skill versioning.** Updating a skill means
|
|
729
801
|
creating a new skill section (new kbid) and
|
|
730
802
|
optionally deleting the old one. The kbid changes
|
|
@@ -107,6 +107,34 @@ Each database directory is an independent context with
|
|
|
107
107
|
its own daemon, its own cache, and its own WASM module
|
|
108
108
|
instance. There is no shared state across contexts.
|
|
109
109
|
|
|
110
|
+
## Write Serialization
|
|
111
|
+
|
|
112
|
+
The daemon serializes write operations using an
|
|
113
|
+
exclusive file lock on `kbdb-{ctx}.lock` in the OS
|
|
114
|
+
temporary directory. The lock file uses the same
|
|
115
|
+
context ID as the PID file and IPC socket.
|
|
116
|
+
|
|
117
|
+
- **Read operations** proceed without acquiring the
|
|
118
|
+
lock. Concurrent reads are fully parallel.
|
|
119
|
+
- **Write operations** acquire the exclusive lock
|
|
120
|
+
before calling the WASM write export. If another
|
|
121
|
+
write is already in progress, the new write queues
|
|
122
|
+
behind the lock and waits.
|
|
123
|
+
- **Lock duration** -- the lock is held only for the
|
|
124
|
+
duration of the single write operation and released
|
|
125
|
+
immediately after it completes. It is not held
|
|
126
|
+
across multiple operations or across client
|
|
127
|
+
connections.
|
|
128
|
+
|
|
129
|
+
This serialization model is equivalent to SQLite's
|
|
130
|
+
default locking: readers never block, and writers
|
|
131
|
+
take turns one at a time.
|
|
132
|
+
|
|
133
|
+
Write methods subject to this lock are:
|
|
134
|
+
`addSection`, `updateSection`, `removeSection`,
|
|
135
|
+
`groupSections`, `removeGrouping`, `initDirectory`,
|
|
136
|
+
`gc`, and `rebuildIndexes`.
|
|
137
|
+
|
|
110
138
|
## Daemon Startup
|
|
111
139
|
|
|
112
140
|
When the worker client spawns a new daemon (see
|
|
@@ -127,15 +155,23 @@ daemon process performs the following startup sequence:
|
|
|
127
155
|
module (see WASM Module below).
|
|
128
156
|
5. Calls `worker_init` on the WASM module to open the
|
|
129
157
|
database context.
|
|
130
|
-
6.
|
|
158
|
+
6. Calls `db_check_integrity` on the WASM module to
|
|
159
|
+
run a startup health check. If `ok` is false, the
|
|
160
|
+
daemon logs a structured warning to stderr listing
|
|
161
|
+
the error count and error types. The daemon
|
|
162
|
+
continues operating -- health issues are not fatal
|
|
163
|
+
to startup. The health status (`integrity_ok`,
|
|
164
|
+
`integrity_errors`) is included in subsequent
|
|
165
|
+
`status` responses.
|
|
166
|
+
7. Writes its PID to `kbdb-{ctx}.pid` in the OS
|
|
131
167
|
temporary directory.
|
|
132
|
-
|
|
168
|
+
8. Creates the IPC socket (`kbdb-{ctx}.sock` on Unix,
|
|
133
169
|
named pipe on Windows).
|
|
134
|
-
|
|
170
|
+
9. Begins listening for connections.
|
|
135
171
|
|
|
136
172
|
The daemon only writes its PID file and creates the
|
|
137
173
|
IPC socket after all preconditions pass and the WASM
|
|
138
|
-
module is loaded. If the daemon exits before step
|
|
174
|
+
module is loaded. If the daemon exits before step 7,
|
|
139
175
|
the worker client's PID-file poll times out, and the
|
|
140
176
|
worker client reads the crash log (or the daemon's
|
|
141
177
|
stderr) to report the failure reason.
|
|
@@ -494,6 +530,7 @@ for each WASM operation:
|
|
|
494
530
|
| `listByType`, `list_by_type` | `worker_list_by_type` |
|
|
495
531
|
| `listDocumentsByType` | `worker_list_docs_by_type` |
|
|
496
532
|
| `recall` | `worker_recall` |
|
|
533
|
+
| `export` | `db_export` |
|
|
497
534
|
|
|
498
535
|
`migrateDatabase` is listed in the map but throws
|
|
499
536
|
`IpcError(-32601, 'migrateDatabase is not yet
|
|
@@ -520,12 +557,22 @@ Returns the daemon's current state.
|
|
|
520
557
|
// Response
|
|
521
558
|
{
|
|
522
559
|
"jsonrpc": "2.0", "id": 1,
|
|
523
|
-
"result":
|
|
560
|
+
"result": {
|
|
561
|
+
"state": "busy",
|
|
562
|
+
"integrity_ok": true,
|
|
563
|
+
"integrity_errors": []
|
|
564
|
+
}
|
|
524
565
|
}
|
|
525
566
|
```
|
|
526
567
|
|
|
527
|
-
Result:
|
|
528
|
-
|
|
568
|
+
Result fields:
|
|
569
|
+
|
|
570
|
+
| Field | Type | Description |
|
|
571
|
+
|--------------------|----------|-------------------------------------------|
|
|
572
|
+
| `state` | string | `"busy"` if connections > 0, else `"idle"` |
|
|
573
|
+
| `integrity_ok` | boolean | `true` if startup health check passed |
|
|
574
|
+
| `integrity_errors` | string[] | Error types from `db_check_integrity`; |
|
|
575
|
+
| | | empty when `integrity_ok` is `true` |
|
|
529
576
|
|
|
530
577
|
### `idle_at`
|
|
531
578
|
|
|
@@ -44,12 +44,23 @@ pub fn rebuild_indexes()
|
|
|
44
44
|
manifest must exist.
|
|
45
45
|
- Detect circular references in `$(text:)` and
|
|
46
46
|
`$(document:)` marker chains.
|
|
47
|
+
- Detect **divergent content**: sections sharing the
|
|
48
|
+
same `source_key` that have different content.
|
|
49
|
+
Indicates contradictory versions that should be
|
|
50
|
+
resolved via supersession. Reported as
|
|
51
|
+
`divergent_content` entries in `IntegrityReport`.
|
|
47
52
|
|
|
48
53
|
### `garbage_collect`
|
|
49
54
|
|
|
50
55
|
Remove section files not referenced by any document
|
|
51
56
|
manifest. Returns the number of removed files.
|
|
52
57
|
|
|
58
|
+
Also reports `stale_sources` -- an advisory count of
|
|
59
|
+
sections whose `source_path` points to a file that no
|
|
60
|
+
longer contains their heading text. Stale sections
|
|
61
|
+
are not automatically removed (they may still be
|
|
62
|
+
referenced by documents); the count is informational.
|
|
63
|
+
|
|
53
64
|
### `rebuild_indexes`
|
|
54
65
|
|
|
55
66
|
Reconstruct all index files (`terms.idx`,
|
|
@@ -30,17 +30,34 @@ pub fn remove_section(kbid: &KbId)
|
|
|
30
30
|
### `add_section`
|
|
31
31
|
|
|
32
32
|
1. Compute kb-id from content via `compute_kbid`.
|
|
33
|
-
2.
|
|
34
|
-
|
|
33
|
+
2. **Supersession check**: if `source_key` is
|
|
34
|
+
provided and a section with the same `source_key`
|
|
35
|
+
already exists, set `supersedes` to the old
|
|
36
|
+
section's kbid, set `updated_at` to the current
|
|
37
|
+
timestamp, remove the old kbid from document
|
|
38
|
+
manifests, and mark the old section for GC. The
|
|
39
|
+
new section replaces the old one as the current
|
|
40
|
+
version.
|
|
41
|
+
3. Create shard directory if needed (`host_mkdir`).
|
|
42
|
+
4. Serialize TOML header + `---` separator + body.
|
|
35
43
|
TOML header includes: kbid, docids, section_type,
|
|
36
44
|
title, description, size, created_at, checksum,
|
|
37
45
|
token_count, heading_token_count,
|
|
38
|
-
body_token_count, code_token_count
|
|
39
|
-
|
|
40
|
-
|
|
46
|
+
body_token_count, code_token_count, source_key,
|
|
47
|
+
source_path, source_mtime, updated_at, tags,
|
|
48
|
+
supersedes.
|
|
49
|
+
5. Write atomically via `atomic_write`.
|
|
50
|
+
6. **Near-duplicate detection**: compute Jaccard
|
|
51
|
+
coefficient on stemmed term sets against existing
|
|
52
|
+
sections. Sections with similarity >= 0.85 are
|
|
53
|
+
reported as `near_duplicates` in the result
|
|
54
|
+
(advisory, not a block).
|
|
55
|
+
7. Update catalog via `increment_sections(1)`.
|
|
41
56
|
|
|
42
57
|
Validation: image sections (`image/*` type) require
|
|
43
|
-
a `title` in `AddSectionParams`.
|
|
58
|
+
a `title` in `AddSectionParams`. Text sections
|
|
59
|
+
without a `title` emit a `heading_missing` warning
|
|
60
|
+
in the result.
|
|
44
61
|
|
|
45
62
|
### `update_section`
|
|
46
63
|
|
|
@@ -120,6 +120,7 @@ engine/crates/fs-database/
|
|
|
120
120
|
| `db_status` | Return database metadata |
|
|
121
121
|
| `db_check_integrity` | Run integrity checks |
|
|
122
122
|
| `db_garbage_collect` | Remove orphaned sections |
|
|
123
|
+
| `db_export` | Export database snapshot |
|
|
123
124
|
| `alloc` / `free` | Memory management (from `kbdb-shared`) |
|
|
124
125
|
|
|
125
126
|
## Dependencies
|
|
@@ -26,6 +26,12 @@ pub struct SectionHeader {
|
|
|
26
26
|
pub heading_token_count: u32,
|
|
27
27
|
pub body_token_count: u32,
|
|
28
28
|
pub code_token_count: u32,
|
|
29
|
+
pub source_key: Option<String>,
|
|
30
|
+
pub source_path: Option<String>,
|
|
31
|
+
pub source_mtime: Option<String>,
|
|
32
|
+
pub updated_at: Option<String>,
|
|
33
|
+
pub tags: Vec<String>,
|
|
34
|
+
pub supersedes: Option<KbId>,
|
|
29
35
|
}
|
|
30
36
|
```
|
|
31
37
|
|
|
@@ -43,6 +49,21 @@ and stored in the header for query-time retrieval.
|
|
|
43
49
|
See [FieldTokenCounts](../../shared/query/types.md)
|
|
44
50
|
for the shared struct.
|
|
45
51
|
|
|
52
|
+
`source_key` is a stable logical identity derived
|
|
53
|
+
from `sha256("${source_path}\0${heading}")`,
|
|
54
|
+
truncated to 16 hex chars. Separate from `kbid`
|
|
55
|
+
(content hash) -- identifies *what the section is
|
|
56
|
+
about* rather than *what it currently says*. `null`
|
|
57
|
+
for stdin input. `source_path` is the original file
|
|
58
|
+
path that was learned. `source_mtime` is the ISO 8601
|
|
59
|
+
modified time of the source file at learn time.
|
|
60
|
+
`updated_at` is the ISO 8601 timestamp of the last
|
|
61
|
+
supersession (`null` on first create). `tags` are
|
|
62
|
+
user-supplied strings for scoping and filtering.
|
|
63
|
+
`supersedes` is the kbid of the section this one
|
|
64
|
+
replaced (`null` on first create). See
|
|
65
|
+
[Database -- Logical Identity](../../../goals/database.md).
|
|
66
|
+
|
|
46
67
|
### `IntegrityReport`
|
|
47
68
|
|
|
48
69
|
```rust
|
|
@@ -53,6 +74,7 @@ pub struct IntegrityReport {
|
|
|
53
74
|
pub broken_refs: Vec<(KbId, String, KbId)>,
|
|
54
75
|
pub type_mismatches: Vec<(KbId, String, KbId)>,
|
|
55
76
|
pub circular_refs: Vec<(KbId, Vec<KbId>)>,
|
|
77
|
+
pub divergent_content: Vec<(String, Vec<KbId>)>,
|
|
56
78
|
}
|
|
57
79
|
```
|
|
58
80
|
|
|
@@ -64,7 +86,11 @@ covering all six marker types (`text`, `code`,
|
|
|
64
86
|
target_kbid)` where the target section type does
|
|
65
87
|
not match the marker type. `circular_refs` are
|
|
66
88
|
`(start_kbid, cycle_path)` for reference chains
|
|
67
|
-
that form loops.
|
|
89
|
+
that form loops. `divergent_content` entries are
|
|
90
|
+
`(source_key, conflicting_kbids)` -- sections
|
|
91
|
+
sharing the same `source_key` that have different
|
|
92
|
+
content, indicating contradictory versions that
|
|
93
|
+
should be resolved via supersession.
|
|
68
94
|
|
|
69
95
|
## Functions
|
|
70
96
|
|
|
@@ -266,7 +266,7 @@ engine/crates/query-parser/
|
|
|
266
266
|
├── lib.rs ← WASM exports
|
|
267
267
|
├── mem.rs ← shared-memory helpers
|
|
268
268
|
│
|
|
269
|
-
├── pipeline/ ←
|
|
269
|
+
├── pipeline/ ← 9-stage tokenizer
|
|
270
270
|
│ ├── mod.rs
|
|
271
271
|
│ ├── markers.rs
|
|
272
272
|
│ ├── unicode.rs
|
|
@@ -274,6 +274,7 @@ engine/crates/query-parser/
|
|
|
274
274
|
│ ├── identifiers.rs
|
|
275
275
|
│ ├── case.rs
|
|
276
276
|
│ ├── stopwords.rs
|
|
277
|
+
│ ├── synonyms.rs
|
|
277
278
|
│ ├── stem.rs
|
|
278
279
|
│ └── ngram.rs
|
|
279
280
|
│
|
|
@@ -5,9 +5,9 @@ extraction, and ranking.*
|
|
|
5
5
|
|
|
6
6
|
## Modules
|
|
7
7
|
|
|
8
|
-
- **[pipeline](pipeline.md)** --
|
|
8
|
+
- **[pipeline](pipeline.md)** -- 9-stage tokenization
|
|
9
9
|
pipeline. Converts raw text into normalized, stemmed
|
|
10
|
-
tokens with n-grams.
|
|
10
|
+
tokens with synonym expansion and n-grams.
|
|
11
11
|
- **[parse](parse.md)** -- Query string parsing into
|
|
12
12
|
structured `QueryPlan`.
|
|
13
13
|
- **[extract](extract.md)** -- RAKE keyphrase
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# functions::pipeline -- Tokenization Pipeline
|
|
2
2
|
|
|
3
|
-
*
|
|
4
|
-
normalized, stemmed tokens
|
|
5
|
-
and query processing.*
|
|
3
|
+
*9-stage pipeline that converts raw text into
|
|
4
|
+
normalized, stemmed tokens with synonym expansion.
|
|
5
|
+
Used for both indexing and query processing.*
|
|
6
6
|
|
|
7
7
|
## Source
|
|
8
8
|
|
|
@@ -14,7 +14,7 @@ and query processing.*
|
|
|
14
14
|
pub fn tokenize(input: &str) -> TokenStream;
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
-
Runs all
|
|
17
|
+
Runs all nine stages in sequence. Each stage is a
|
|
18
18
|
pure function.
|
|
19
19
|
|
|
20
20
|
## Stages
|
|
@@ -83,7 +83,32 @@ pub fn remove_stop_words(
|
|
|
83
83
|
Filters common function words. Single-character
|
|
84
84
|
tokens also removed.
|
|
85
85
|
|
|
86
|
-
### 7.
|
|
86
|
+
### 7. Synonym Expansion (`synonyms.rs`)
|
|
87
|
+
|
|
88
|
+
```rust
|
|
89
|
+
pub fn expand_synonyms(
|
|
90
|
+
word: &str,
|
|
91
|
+
map: &SynonymMap,
|
|
92
|
+
) -> Vec<String>;
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Looks up the lowercased word in the built-in synonym
|
|
96
|
+
map. Returns a vector containing the original word
|
|
97
|
+
plus any synonyms. When no synonyms exist, returns a
|
|
98
|
+
single-element vector containing only the original.
|
|
99
|
+
|
|
100
|
+
The synonym map is a bidirectional
|
|
101
|
+
`HashMap<String, Vec<String>>` built at compile time.
|
|
102
|
+
Expansion is applied after stop-word removal and
|
|
103
|
+
before stemming, so both original and synonym tokens
|
|
104
|
+
are stemmed consistently.
|
|
105
|
+
|
|
106
|
+
The map is small (~30-50 entries) and focused on
|
|
107
|
+
common IT/development vocabulary mismatches:
|
|
108
|
+
login/sign-in, auth/authentication, remove/delete,
|
|
109
|
+
config/configuration, credential/password, etc.
|
|
110
|
+
|
|
111
|
+
### 8. Stemming (`stem.rs`)
|
|
87
112
|
|
|
88
113
|
```rust
|
|
89
114
|
pub fn stem_word(word: &str) -> StemResult;
|
|
@@ -92,7 +117,7 @@ pub fn stem_word(word: &str) -> StemResult;
|
|
|
92
117
|
Snowball / Porter2. Preserves both original and
|
|
93
118
|
stemmed forms.
|
|
94
119
|
|
|
95
|
-
###
|
|
120
|
+
### 9. N-gram Extraction (`ngram.rs`)
|
|
96
121
|
|
|
97
122
|
```rust
|
|
98
123
|
pub fn extract_ngrams(
|
|
@@ -101,3 +126,16 @@ pub fn extract_ngrams(
|
|
|
101
126
|
```
|
|
102
127
|
|
|
103
128
|
Bigrams and trigrams for phrase indexing.
|
|
129
|
+
|
|
130
|
+
## Relaxed Query Mode
|
|
131
|
+
|
|
132
|
+
The tokenization pipeline is used for both the
|
|
133
|
+
primary AND-mode query plan and the relaxed OR-mode
|
|
134
|
+
fallback plan. `qp_parse_query` produces a
|
|
135
|
+
`QueryPlan` with a `match_mode` field (`All` | `Any`).
|
|
136
|
+
When the AND plan returns zero matches,
|
|
137
|
+
`kb-worker.wasm` retries with the OR plan using the
|
|
138
|
+
same tokenized terms. No additional pipeline stage or
|
|
139
|
+
WASM export is needed. See
|
|
140
|
+
[Query Parser -- Relaxed Query
|
|
141
|
+
Mode](../../../goals/query-parser.md).
|