@dikolab/kbdb 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +59 -24
  2. package/dist/cli.cjs +462 -73
  3. package/dist/cli.cjs.map +4 -4
  4. package/dist/cli.mjs +454 -65
  5. package/dist/cli.mjs.map +4 -4
  6. package/dist/kbdb-worker.cjs +395 -60
  7. package/dist/kbdb-worker.cjs.map +4 -4
  8. package/dist/mod.cjs +18 -1
  9. package/dist/mod.cjs.map +2 -2
  10. package/dist/mod.mjs +18 -1
  11. package/dist/mod.mjs.map +2 -2
  12. package/dist/src/shared/cli/functions/dispatch-command.function.d.ts +3 -1
  13. package/dist/src/shared/cli/functions/install-default-skills.function.d.ts +12 -0
  14. package/dist/src/shared/cli/functions/run-check.function.d.ts +47 -5
  15. package/dist/src/shared/cli/functions/run-export.function.d.ts +34 -0
  16. package/dist/src/shared/cli/functions/run-gc.function.d.ts +6 -0
  17. package/dist/src/shared/cli/functions/run-learn.function.d.ts +35 -6
  18. package/dist/src/shared/cli/functions/run-recall.function.d.ts +45 -0
  19. package/dist/src/shared/cli/functions/run-search.function.d.ts +52 -15
  20. package/dist/src/shared/cli/index.d.ts +4 -0
  21. package/dist/src/shared/cli/typings/cli-options.interface.d.ts +13 -0
  22. package/dist/src/shared/cli/typings/learn-result.model.d.ts +19 -0
  23. package/dist/src/shared/cli/typings/search-display.model.d.ts +18 -0
  24. package/dist/src/shared/mcp/classes/mcp-server.class.d.ts +8 -0
  25. package/dist/src/shared/mcp/typings/mcp-client.interface.d.ts +43 -0
  26. package/dist/src/shared/version/constants/version.constant.d.ts +1 -1
  27. package/dist/src/shared/wasm-codec/functions/decode-add-section-result.function.d.ts +48 -0
  28. package/dist/src/shared/wasm-codec/functions/decode-scored-results.function.d.ts +58 -0
  29. package/dist/src/shared/wasm-codec/functions/decode-section-records.function.d.ts +33 -0
  30. package/dist/src/shared/wasm-codec/functions/encode-add-section-params.function.d.ts +52 -0
  31. package/dist/src/shared/wasm-codec/index.d.ts +2 -0
  32. package/dist/src/shared/worker-client/classes/worker-client.class.d.ts +16 -0
  33. package/dist/src/shared/worker-client/index.d.ts +1 -1
  34. package/dist/src/shared/worker-client/typings/add-section-result.model.d.ts +29 -15
  35. package/dist/src/shared/worker-client/typings/export-result.model.d.ts +14 -0
  36. package/dist/src/shared/worker-client/typings/gc-result.model.d.ts +6 -0
  37. package/dist/src/shared/worker-client/typings/search-result.model.d.ts +12 -0
  38. package/dist/src/shared/worker-client/typings/section.model.d.ts +16 -0
  39. package/dist/src/shared/worker-daemon/classes/worker-daemon.class.d.ts +35 -0
  40. package/dist/wasm/fs-database/kbdb_fs_database.d.ts +22 -1
  41. package/dist/wasm/fs-database/kbdb_fs_database.js +28 -1
  42. package/dist/wasm/fs-database/kbdb_fs_database_bg.wasm +0 -0
  43. package/dist/wasm/fs-database/kbdb_fs_database_bg.wasm.d.ts +1 -0
  44. package/dist/wasm/kb-worker/kbdb_worker.d.ts +48 -1
  45. package/dist/wasm/kb-worker/kbdb_worker.js +60 -1
  46. package/dist/wasm/kb-worker/kbdb_worker_bg.wasm +0 -0
  47. package/dist/wasm/kb-worker/kbdb_worker_bg.wasm.d.ts +2 -0
  48. package/dist/wasm/query-parser/kbdb_query_parser_bg.wasm +0 -0
  49. package/dist/worker.mjs +396 -60
  50. package/dist/worker.mjs.map +4 -4
  51. package/docs/details/cli.md +102 -15
  52. package/docs/details/library-api.md +55 -6
  53. package/docs/details/mcp-server.md +8 -4
  54. package/docs/details/search-and-ranking.md +63 -5
  55. package/docs/details/storage.md +2 -0
  56. package/docs/goals/cli.md +255 -14
  57. package/docs/goals/database.md +244 -17
  58. package/docs/goals/document.md +1 -1
  59. package/docs/goals/mcp.md +83 -14
  60. package/docs/goals/query-parser.md +98 -4
  61. package/docs/goals/query-result.md +177 -10
  62. package/docs/goals/skills.md +72 -0
  63. package/docs/goals/worker-daemon.md +54 -7
  64. package/docs/modules/rust/fs-database/functions/integrity.md +11 -0
  65. package/docs/modules/rust/fs-database/functions/section.md +23 -6
  66. package/docs/modules/rust/fs-database/module.md +1 -0
  67. package/docs/modules/rust/fs-database/types/section.md +27 -1
  68. package/docs/modules/rust/overview.md +2 -1
  69. package/docs/modules/rust/query-parser/functions/module.md +2 -2
  70. package/docs/modules/rust/query-parser/functions/pipeline.md +44 -6
  71. package/docs/modules/rust/query-parser/module.md +5 -4
  72. package/docs/modules/rust/query-parser/types/keyphrase.md +11 -0
  73. package/docs/modules/rust/query-parser/types/module.md +3 -3
  74. package/docs/modules/typescript/shared/cli/functions.md +68 -3
  75. package/package.json +1 -1
  76. package/src/shared/cli/functions/dispatch-command.function.ts +53 -1
  77. package/src/shared/cli/functions/format-output.function.ts +36 -12
  78. package/src/shared/cli/functions/generate-help.function.ts +56 -1
  79. package/src/shared/cli/functions/install-default-skills.function.ts +296 -0
  80. package/src/shared/cli/functions/parse-args.function.ts +13 -0
  81. package/src/shared/cli/functions/run-check.function.ts +87 -6
  82. package/src/shared/cli/functions/run-export.function.ts +40 -0
  83. package/src/shared/cli/functions/run-gc.function.ts +6 -0
  84. package/src/shared/cli/functions/run-init.function.ts +2 -0
  85. package/src/shared/cli/functions/run-learn.function.ts +68 -16
  86. package/src/shared/cli/functions/run-recall.function.ts +66 -0
  87. package/src/shared/cli/functions/run-search.function.ts +58 -15
  88. package/src/shared/cli/index.ts +4 -0
  89. package/src/shared/cli/typings/cli-options.interface.ts +13 -0
  90. package/src/shared/cli/typings/learn-result.model.ts +16 -0
  91. package/src/shared/cli/typings/search-display.model.ts +18 -0
  92. package/src/shared/mcp/classes/mcp-server.class.ts +30 -1
  93. package/src/shared/mcp/functions/create-tool-definitions.function.ts +46 -5
  94. package/src/shared/mcp/functions/handle-tool-call.function.ts +70 -2
  95. package/src/shared/mcp/typings/mcp-client.interface.ts +45 -0
  96. package/src/shared/version/constants/version.constant.ts +1 -1
  97. package/src/shared/wasm-codec/functions/decode-add-section-result.function.ts +112 -0
  98. package/src/shared/wasm-codec/functions/decode-scored-results.function.ts +86 -0
  99. package/src/shared/wasm-codec/functions/decode-section-records.function.ts +87 -0
  100. package/src/shared/wasm-codec/functions/encode-add-section-params.function.ts +68 -0
  101. package/src/shared/wasm-codec/index.ts +5 -0
  102. package/src/shared/worker-client/classes/worker-client.class.ts +19 -0
  103. package/src/shared/worker-client/index.ts +4 -1
  104. package/src/shared/worker-client/typings/add-section-result.model.ts +30 -15
  105. package/src/shared/worker-client/typings/export-result.model.ts +14 -0
  106. package/src/shared/worker-client/typings/gc-result.model.ts +6 -0
  107. package/src/shared/worker-client/typings/search-result.model.ts +12 -0
  108. package/src/shared/worker-client/typings/section.model.ts +16 -0
  109. package/src/shared/worker-daemon/classes/worker-daemon.class.ts +315 -46
  110. package/src/shared/worker-daemon/functions/create-host-imports.function.ts +19 -4
  111. package/src/shared/worker-daemon/functions/create-ipc-server.function.ts +5 -1
  112. package/src/wasm/fs-database/kbdb_fs_database.d.ts +22 -1
  113. package/src/wasm/fs-database/kbdb_fs_database.js +28 -1
  114. package/src/wasm/fs-database/kbdb_fs_database_bg.wasm +0 -0
  115. package/src/wasm/fs-database/kbdb_fs_database_bg.wasm.d.ts +1 -0
  116. package/src/wasm/kb-worker/kbdb_worker.d.ts +48 -1
  117. package/src/wasm/kb-worker/kbdb_worker.js +60 -1
  118. package/src/wasm/kb-worker/kbdb_worker_bg.wasm +0 -0
  119. package/src/wasm/kb-worker/kbdb_worker_bg.wasm.d.ts +2 -0
  120. package/src/wasm/query-parser/kbdb_query_parser_bg.wasm +0 -0
@@ -434,6 +434,34 @@ order** (highest relevance first). Ties are broken
434
434
  by kbid lexicographic order for deterministic
435
435
  ordering.
436
436
 
437
+ ## Score Normalization
438
+
439
+ Raw BM25F scores vary in magnitude across queries and
440
+ corpora, making them difficult to interpret in
441
+ isolation. The ranking pipeline produces a
442
+ **normalized confidence** score alongside the raw
443
+ score:
444
+
445
+ ```
446
+ confidence = final_score / max_score_in_result_set
447
+ ```
448
+
449
+ - The top-ranked result always has `confidence = 1.0`.
450
+ - A result with half the raw score of the top result
451
+ has `confidence = 0.5`.
452
+ - When the result set is empty, no normalization is
453
+ needed.
454
+
455
+ `confidence` is a relative measure within a single
456
+ query's result set -- it does not compare across
457
+ different queries. Its purpose is to let the caller
458
+ distinguish strong matches from weak tail results
459
+ without needing to understand BM25F score magnitudes.
460
+
461
+ The CLI exposes `confidence` in `SearchDisplay` for
462
+ human inspection. The MCP projection omits it --
463
+ result ordering is the ranking signal for AI agents.
464
+
437
465
  ## Result Truncation and Pagination
438
466
 
439
467
  After sorting, results are windowed by `offset` and
@@ -452,6 +480,7 @@ interface PagedSearchResult {
452
480
  offset: number;
453
481
  limit: number;
454
482
  has_more: boolean;
483
+ relaxed: boolean;
455
484
  }
456
485
  ```
457
486
 
@@ -462,6 +491,7 @@ interface PagedSearchResult {
462
491
  | `offset` | Echo of requested offset (or 0) |
463
492
  | `limit` | Echo of effective limit (or 20) |
464
493
  | `has_more` | `true` when `offset + items.length < total` |
494
+ | `relaxed` | `true` when results came from relaxed-match fallback (see below) |
465
495
 
466
496
  `has_more` is a convenience boolean so the caller
467
497
  does not need to compute whether more pages exist.
@@ -489,14 +519,26 @@ interface SectionMatch {
489
519
  type: string;
490
520
  heading: string;
491
521
  score: number;
522
+ confidence: number;
492
523
  snippet: string;
493
524
  matched_terms: string[];
525
+ matched_fields: string[];
526
+ created_at: string;
494
527
  }
495
528
  ```
496
529
 
497
530
  `heading` is `string | null` -- `null` when the
498
- section has no title. `snippet` is always populated
499
- for text and code sections using KWIC
531
+ section has no title. `confidence` is the normalized
532
+ score (0.0--1.0) relative to the top result (see
533
+ Score Normalization above). `matched_fields` is a
534
+ subset of `["heading", "body", "code"]` indicating
535
+ which fields contained term matches -- lets the
536
+ caller see whether the match was in a title vs body
537
+ text. `created_at` is the section's ISO 8601
538
+ creation timestamp.
539
+
540
+ `snippet` is always populated for text and code
541
+ sections using KWIC
500
542
  (keyword-in-context): ~150 characters centered on
501
543
  the highest-scoring matched term position, with
502
544
  `...` ellipsis at boundaries. If the best match is
@@ -576,17 +618,25 @@ interface McpSearchResult {
576
618
  docids: string[];
577
619
  snippet: string;
578
620
  matched_terms: string[];
621
+ matched_fields: string[];
622
+ created_at: string;
579
623
  content?: string;
580
624
  }
581
625
  ```
582
626
 
583
- The `score` field from `SectionMatch` is
584
- intentionally omitted. Results are ordered by
585
- relevance descending; the position in the array
586
- is the ranking signal. Exposing uncalibrated BM25F
587
- scores provides no actionable information to the
588
- agent and risks confusing score magnitudes across
589
- different queries.
627
+ The `score` and `confidence` fields from
628
+ `SectionMatch` are intentionally omitted. Results
629
+ are ordered by relevance descending; the position
630
+ in the array is the ranking signal. Exposing
631
+ uncalibrated BM25F scores provides no actionable
632
+ information to the agent and risks confusing score
633
+ magnitudes across different queries.
634
+
635
+ `matched_fields` is included because it tells the
636
+ agent whether the match was in a heading (strong
637
+ topical signal) vs body text (weaker). `created_at`
638
+ is included to let agents prefer recent results
639
+ when multiple sections cover the same topic.
590
640
 
591
641
  `matched_terms` is a flat `string[]` of the query
592
642
  terms that matched -- no per-term frequency counts.
@@ -609,7 +659,40 @@ inspection and debugging.
609
659
 
610
660
  The MCP result uses the same `PagedSearchResult`
611
661
  envelope with `items`, `total`, `offset`, `limit`,
612
- and `has_more`.
662
+ `has_more`, and `relaxed`.
663
+
664
+ ## Relaxed-Match Fallback
665
+
666
+ When the primary query (implicit AND -- all terms
667
+ required) returns **zero results**, the ranking
668
+ engine automatically retries with relaxed matching:
669
+
670
+ 1. **OR mode** -- any query term may match, not all
671
+ required. The query plan is re-executed with OR
672
+ semantics using the same tokenized terms.
673
+
674
+ 2. Results from the relaxed retry are flagged:
675
+ `relaxed: true` in the `PagedSearchResult`
676
+ envelope. Normal (AND) results have
677
+ `relaxed: false`.
678
+
679
+ Relaxed mode is transparent to the caller -- no
680
+ special syntax or parameters are needed. The engine
681
+ tries AND first and falls back to OR only when AND
682
+ produces zero matches. This converts a silent miss
683
+ ("nothing found") into a soft hit ("weaker matches
684
+ found") that the caller can reason about.
685
+
686
+ The AI agent (or human user) can check the `relaxed`
687
+ flag to know that the results are partial-term
688
+ matches rather than full-term matches. Relaxed
689
+ results typically have lower relevance -- the
690
+ ordering and confidence scores still reflect relative
691
+ quality within the relaxed set.
692
+
693
+ When the primary AND query returns at least one
694
+ result, no fallback is attempted. The `relaxed` flag
695
+ is always `false` for non-empty AND results.
613
696
 
614
697
  ## References Resolution
615
698
 
@@ -691,3 +774,87 @@ of the ranking coin:
691
774
 
692
775
  See [Query Parser](query-parser.md) for the full
693
776
  tokenization pipeline and extraction algorithms.
777
+
778
+ ## Content Authoring Best Practices
779
+
780
+ ### How Field Weights Affect Searchability
781
+
782
+ kbdb uses three weighted fields for ranking (see
783
+ the [Field Weight Mapping](#field-weight-mapping)
784
+ table above). Content authors can influence how
785
+ well their content is found by understanding these
786
+ weights:
787
+
788
+ - **Title / heading text** carries **2.0x** the
789
+ ranking weight of body text. A match in a heading
790
+ has twice the influence on ranking as the same
791
+ match in the body.
792
+ - **Code blocks** carry **1.5x** weight. Code
793
+ identifiers (function names, variable names, class
794
+ names) are split on camelCase/snake_case boundaries
795
+ and indexed as separate terms.
796
+ - **Body text and descriptions** are the baseline at
797
+ **1.0x**.
798
+ - **Binary content** (images, attachments) is not
799
+ indexed at all -- searchability depends entirely on
800
+ the `title` (required) and `description` (optional)
801
+ attributes.
802
+
803
+ ### Writing Index-Friendly Headings
804
+
805
+ The heading is the single highest-impact field for
806
+ searchability. Follow these guidelines:
807
+
808
+ - Use **descriptive, keyword-rich headings**. Prefer
809
+ "Authentication and Login Flow" over just "Auth
810
+ Flow" -- the expanded heading gives the search
811
+ engine more term surface.
812
+ - Include **synonyms and alternative phrasing** in
813
+ headings when natural. If a section covers logging
814
+ in, mention both "login" and "sign-in" if
815
+ applicable.
816
+ - Avoid generic headings like "Overview",
817
+ "Introduction", or "Notes" that carry no
818
+ discriminative value for search.
819
+ - Stemming is automatic ("connections" matches
820
+ "connect"), but different root words are NOT
821
+ equivalent without synonym expansion --
822
+ "login" does not match "sign-in" by default.
823
+
824
+ ### Structuring Content for Recall
825
+
826
+ Content structure affects the progressive context
827
+ expansion provided by the `recall` command:
828
+
829
+ - Place the most important terms in the **first 20%**
830
+ of a section. The zone scoring algorithm gives
831
+ extra weight to terms in this heading zone.
832
+ - Use explicit cross-references (`$(link:kbid)`
833
+ markers) between related sections. The reference
834
+ graph enables progressive context expansion via
835
+ `recall` at depth 1+.
836
+ - Group related sections into documents. The `recall`
837
+ command at depth 2+ returns sibling sections,
838
+ giving AI agents full surrounding context.
839
+ - For code sections: use meaningful identifiers.
840
+ camelCase and snake_case names are automatically
841
+ split and indexed. `getUserById` produces four
842
+ searchable terms: "get", "user", "by", "id".
843
+
844
+ ### Leveraging the Query Syntax
845
+
846
+ Guidance for AI agents and human users on writing
847
+ effective queries:
848
+
849
+ - Use **multiple short keyword queries** rather than
850
+ one long natural-language question. Lexical search
851
+ matches on token overlap, not semantic meaning.
852
+ - Use **exact phrases** (`"error handling"`) when term
853
+ order matters.
854
+ - Use **boolean operators** (`auth AND token`,
855
+ `cache OR redis`) to broaden or narrow results.
856
+ - Use **term exclusion** (`auth -oauth`) to filter out
857
+ irrelevant matches.
858
+ - After finding results with `search`, use `recall`
859
+ with increasing depth to progressively expand
860
+ context.
@@ -719,12 +719,84 @@ dangling kbid. The agent is not deleted. See
719
719
  [Agents](agents.md) for how dangling references are
720
720
  handled at retrieval time.
721
721
 
722
+ ## Default Skills
723
+
724
+ ### Overview
725
+
726
+ When a new database is created via `kbdb db init`, a
727
+ set of default skills are automatically installed.
728
+ These skills provide baseline capabilities that
729
+ improve AI agent effectiveness. Default skills are
730
+ stored as regular skill sections (no special treatment
731
+ in the engine) and can be deleted by users if not
732
+ needed.
733
+
734
+ ### `query-guidance` Default Skill
735
+
736
+ A built-in skill that teaches AI agents how to
737
+ effectively search the knowledge base. This addresses
738
+ the limitation that lexical search requires
739
+ disciplined query generation for good recall.
740
+
741
+ **Skill definition:**
742
+
743
+ ```json
744
+ {
745
+ "name": "query-guidance",
746
+ "description": "How to effectively search this knowledge base. Read this before issuing search queries.",
747
+ "arguments": [],
748
+ "body": "This knowledge base uses lexical search (stemmed BM25F), not semantic/vector search. Follow these guidelines for effective retrieval:\n\n1. Issue MULTIPLE short keyword queries rather than one long natural-language question. Each query should target 2-3 specific terms.\n\n2. EXPAND SYNONYMS yourself. \"login\" will not match \"sign-in\" unless both terms appear in the content. Try alternative phrasings: search for \"login\", then \"sign-in\", then \"authentication\".\n\n3. USE BOOLEAN AND PHRASE SYNTAX:\n - Exact phrases: \"error handling\"\n - Boolean AND: auth AND token\n - Boolean OR: cache OR redis\n - Exclusion: auth -oauth\n\n4. LEAN ON HEADINGS. Headings carry 2x weight in ranking. Search for terms that are likely in section titles.\n\n5. STEMMING IS AUTOMATIC. \"connections\" matches \"connect\", \"running\" matches \"run\". You do not need to try inflected forms.\n\n6. USE RECALL AFTER SEARCH. After finding relevant sections with search, use recall with increasing depth (0-3) to progressively expand context: depth 0 = full content, depth 1 = parent documents + back-references, depth 2 = siblings + forward references, depth 3 = referenced sections.\n\n7. CODE IDENTIFIERS ARE SPLIT. camelCase and snake_case names are decomposed: getUserById becomes searchable as \"get\", \"user\", \"by\", \"id\".\n\n8. PAGINATION. Default limit is 20 results. Use offset for additional pages if total > limit."
749
+ }
750
+ ```
751
+
752
+ ### Installation Behaviour
753
+
754
+ Default skills are installed during `kbdb db init`.
755
+ The `runDbInit` function installs each default skill
756
+ via `client.addSection({ sectionType: 'skill', ... })`
757
+ after creating the database scaffold.
758
+
759
+ Default skills receive kbids via the same
760
+ content-addressing scheme as user-created skills.
761
+ Re-running `db init` on a fresh database produces
762
+ identical kbids for the same default skill content.
763
+
764
+ ### `memory-protocol` Default Skill
765
+
766
+ A built-in skill that teaches AI agents *when* and
767
+ *what* to store in the knowledge base. This addresses
768
+ the risk of the brain filling with transient junk or
769
+ failing to capture durable facts.
770
+
771
+ **Skill definition:**
772
+
773
+ ```json
774
+ {
775
+ "name": "memory-protocol",
776
+ "description": "Guidelines for what to store in this knowledge base and when. Read this before using the learn tool.",
777
+ "arguments": [],
778
+ "body": "Follow these guidelines to keep the knowledge base accurate and useful:\n\n1. STORE DURABLE FACTS: decisions, architecture choices, configuration values, corrections to prior knowledge, API contracts, and resolved incidents. These are the facts that matter weeks later.\n\n2. DO NOT STORE: transient conversation context, debugging output, ephemeral task state, raw log dumps, or information that changes every session. The brain should contain knowledge, not activity logs.\n\n3. SEARCH BEFORE STORING. Always search the knowledge base before adding new content. If a section already covers the topic, use learn with --replace to update it rather than creating a duplicate.\n\n4. USE --replace FOR CORRECTIONS. When a fact changes (e.g., rate limit updated from 100 to 200 req/min), re-learn the corrected content with --replace so the old version is superseded. Without --replace, both the old and new versions coexist with equal ranking.\n\n5. ATTACH TAGS for scoping. Use tags like project name, domain, or topic (e.g., --tags auth,api,v2) so queries can be filtered by scope later.\n\n6. PREFER RECENT RESULTS. When search returns multiple sections covering the same topic, prefer the one with the most recent created_at timestamp -- it is more likely to reflect the current state.\n\n7. USE RECALL AFTER SEARCH. After finding relevant sections, use recall with increasing depth (0-3) to expand context progressively before making decisions.\n\n8. INCLUDE HEADINGS. Always provide a descriptive title when storing content. Headings carry 2x ranking weight -- omitting them significantly reduces searchability."
779
+ }
780
+ ```
781
+
782
+ ### Future Default Skills
783
+
784
+ Additional default skills may be added in future
785
+ versions (e.g. `content-authoring-tips`,
786
+ `search-syntax-reference`). Each follows the same
787
+ installation pattern and is a regular skill section.
788
+
722
789
  ## Design Constraints
723
790
 
724
791
  - **Skills are plain sections.** No separate entity
725
792
  type, storage schema, or index is introduced. Skills
726
793
  use `sectionType = 'skill'` within the existing
727
794
  section infrastructure.
795
+ - **Default skills are regular sections.** They use
796
+ the same storage, indexing, and retrieval
797
+ infrastructure. There is no distinction between a
798
+ default skill and a user-created skill beyond the
799
+ installation source.
728
800
  - **No skill versioning.** Updating a skill means
729
801
  creating a new skill section (new kbid) and
730
802
  optionally deleting the old one. The kbid changes
@@ -107,6 +107,34 @@ Each database directory is an independent context with
107
107
  its own daemon, its own cache, and its own WASM module
108
108
  instance. There is no shared state across contexts.
109
109
 
110
+ ## Write Serialization
111
+
112
+ The daemon serializes write operations using an
113
+ exclusive file lock on `kbdb-{ctx}.lock` in the OS
114
+ temporary directory. The lock file uses the same
115
+ context ID as the PID file and IPC socket.
116
+
117
+ - **Read operations** proceed without acquiring the
118
+ lock. Concurrent reads are fully parallel.
119
+ - **Write operations** acquire the exclusive lock
120
+ before calling the WASM write export. If another
121
+ write is already in progress, the new write queues
122
+ behind the lock and waits.
123
+ - **Lock duration** -- the lock is held only for the
124
+ duration of the single write operation and released
125
+ immediately after it completes. It is not held
126
+ across multiple operations or across client
127
+ connections.
128
+
129
+ This serialization model is equivalent to SQLite's
130
+ default locking: readers never block, and writers
131
+ take turns one at a time.
132
+
133
+ Write methods subject to this lock are:
134
+ `addSection`, `updateSection`, `removeSection`,
135
+ `groupSections`, `removeGrouping`, `initDirectory`,
136
+ `gc`, and `rebuildIndexes`.
137
+
110
138
  ## Daemon Startup
111
139
 
112
140
  When the worker client spawns a new daemon (see
@@ -127,15 +155,23 @@ daemon process performs the following startup sequence:
127
155
  module (see WASM Module below).
128
156
  5. Calls `worker_init` on the WASM module to open the
129
157
  database context.
130
- 6. Writes its PID to `kbdb-{ctx}.pid` in the OS
158
+ 6. Calls `db_check_integrity` on the WASM module to
159
+ run a startup health check. If `ok` is false, the
160
+ daemon logs a structured warning to stderr listing
161
+ the error count and error types. The daemon
162
+ continues operating -- health issues are not fatal
163
+ to startup. The health status (`integrity_ok`,
164
+ `integrity_errors`) is included in subsequent
165
+ `status` responses.
166
+ 7. Writes its PID to `kbdb-{ctx}.pid` in the OS
131
167
  temporary directory.
132
- 7. Creates the IPC socket (`kbdb-{ctx}.sock` on Unix,
168
+ 8. Creates the IPC socket (`kbdb-{ctx}.sock` on Unix,
133
169
  named pipe on Windows).
134
- 8. Begins listening for connections.
170
+ 9. Begins listening for connections.
135
171
 
136
172
  The daemon only writes its PID file and creates the
137
173
  IPC socket after all preconditions pass and the WASM
138
- module is loaded. If the daemon exits before step 6,
174
+ module is loaded. If the daemon exits before step 7,
139
175
  the worker client's PID-file poll times out, and the
140
176
  worker client reads the crash log (or the daemon's
141
177
  stderr) to report the failure reason.
@@ -494,6 +530,7 @@ for each WASM operation:
494
530
  | `listByType`, `list_by_type` | `worker_list_by_type` |
495
531
  | `listDocumentsByType` | `worker_list_docs_by_type` |
496
532
  | `recall` | `worker_recall` |
533
+ | `export` | `db_export` |
497
534
 
498
535
  `migrateDatabase` is listed in the map but throws
499
536
  `IpcError(-32601, 'migrateDatabase is not yet
@@ -520,12 +557,22 @@ Returns the daemon's current state.
520
557
  // Response
521
558
  {
522
559
  "jsonrpc": "2.0", "id": 1,
523
- "result": "busy"
560
+ "result": {
561
+ "state": "busy",
562
+ "integrity_ok": true,
563
+ "integrity_errors": []
564
+ }
524
565
  }
525
566
  ```
526
567
 
527
- Result: `"busy"` if the connection count is greater
528
- than zero, `"idle"` otherwise.
568
+ Result fields:
569
+
570
+ | Field | Type | Description |
571
+ |--------------------|----------|-------------------------------------------|
572
+ | `state` | string | `"busy"` if connections > 0, else `"idle"` |
573
+ | `integrity_ok` | boolean | `true` if startup health check passed |
574
+ | `integrity_errors` | string[] | Error types from `db_check_integrity`; |
575
+ | | | empty when `integrity_ok` is `true` |
529
576
 
530
577
  ### `idle_at`
531
578
 
@@ -44,12 +44,23 @@ pub fn rebuild_indexes()
44
44
  manifest must exist.
45
45
  - Detect circular references in `$(text:)` and
46
46
  `$(document:)` marker chains.
47
+ - Detect **divergent content**: sections sharing the
48
+ same `source_key` that have different content.
49
+ Indicates contradictory versions that should be
50
+ resolved via supersession. Reported as
51
+ `divergent_content` entries in `IntegrityReport`.
47
52
 
48
53
  ### `garbage_collect`
49
54
 
50
55
  Remove section files not referenced by any document
51
56
  manifest. Returns the number of removed files.
52
57
 
58
+ Also reports `stale_sources` -- an advisory count of
59
+ sections whose `source_path` points to a file that no
60
+ longer contains their heading text. Stale sections
61
+ are not automatically removed (they may still be
62
+ referenced by documents); the count is informational.
63
+
53
64
  ### `rebuild_indexes`
54
65
 
55
66
  Reconstruct all index files (`terms.idx`,
@@ -30,17 +30,34 @@ pub fn remove_section(kbid: &KbId)
30
30
  ### `add_section`
31
31
 
32
32
  1. Compute kb-id from content via `compute_kbid`.
33
- 2. Create shard directory if needed (`host_mkdir`).
34
- 3. Serialize TOML header + `---` separator + body.
33
+ 2. **Supersession check**: if `source_key` is
34
+ provided and a section with the same `source_key`
35
+ already exists, set `supersedes` to the old
36
+ section's kbid, set `updated_at` to the current
37
+ timestamp, remove the old kbid from document
38
+ manifests, and mark the old section for GC. The
39
+ new section replaces the old one as the current
40
+ version.
41
+ 3. Create shard directory if needed (`host_mkdir`).
42
+ 4. Serialize TOML header + `---` separator + body.
35
43
  TOML header includes: kbid, docids, section_type,
36
44
  title, description, size, created_at, checksum,
37
45
  token_count, heading_token_count,
38
- body_token_count, code_token_count.
39
- 4. Write atomically via `atomic_write`.
40
- 5. Update catalog via `increment_sections(1)`.
46
+ body_token_count, code_token_count, source_key,
47
+ source_path, source_mtime, updated_at, tags,
48
+ supersedes.
49
+ 5. Write atomically via `atomic_write`.
50
+ 6. **Near-duplicate detection**: compute Jaccard
51
+ coefficient on stemmed term sets against existing
52
+ sections. Sections with similarity >= 0.85 are
53
+ reported as `near_duplicates` in the result
54
+ (advisory, not a block).
55
+ 7. Update catalog via `increment_sections(1)`.
41
56
 
42
57
  Validation: image sections (`image/*` type) require
43
- a `title` in `AddSectionParams`.
58
+ a `title` in `AddSectionParams`. Text sections
59
+ without a `title` emit a `heading_missing` warning
60
+ in the result.
44
61
 
45
62
  ### `update_section`
46
63
 
@@ -120,6 +120,7 @@ engine/crates/fs-database/
120
120
  | `db_status` | Return database metadata |
121
121
  | `db_check_integrity` | Run integrity checks |
122
122
  | `db_garbage_collect` | Remove orphaned sections |
123
+ | `db_export` | Export database snapshot |
123
124
  | `alloc` / `free` | Memory management (from `kbdb-shared`) |
124
125
 
125
126
  ## Dependencies
@@ -26,6 +26,12 @@ pub struct SectionHeader {
26
26
  pub heading_token_count: u32,
27
27
  pub body_token_count: u32,
28
28
  pub code_token_count: u32,
29
+ pub source_key: Option<String>,
30
+ pub source_path: Option<String>,
31
+ pub source_mtime: Option<String>,
32
+ pub updated_at: Option<String>,
33
+ pub tags: Vec<String>,
34
+ pub supersedes: Option<KbId>,
29
35
  }
30
36
  ```
31
37
 
@@ -43,6 +49,21 @@ and stored in the header for query-time retrieval.
43
49
  See [FieldTokenCounts](../../shared/query/types.md)
44
50
  for the shared struct.
45
51
 
52
+ `source_key` is a stable logical identity derived
53
+ from `sha256("${source_path}\0${heading}")`,
54
+ truncated to 16 hex chars. Separate from `kbid`
55
+ (content hash) -- identifies *what the section is
56
+ about* rather than *what it currently says*. `null`
57
+ for stdin input. `source_path` is the original file
58
+ path that was learned. `source_mtime` is the ISO 8601
59
+ modified time of the source file at learn time.
60
+ `updated_at` is the ISO 8601 timestamp of the last
61
+ supersession (`null` on first create). `tags` are
62
+ user-supplied strings for scoping and filtering.
63
+ `supersedes` is the kbid of the section this one
64
+ replaced (`null` on first create). See
65
+ [Database -- Logical Identity](../../../goals/database.md).
66
+
46
67
  ### `IntegrityReport`
47
68
 
48
69
  ```rust
@@ -53,6 +74,7 @@ pub struct IntegrityReport {
53
74
  pub broken_refs: Vec<(KbId, String, KbId)>,
54
75
  pub type_mismatches: Vec<(KbId, String, KbId)>,
55
76
  pub circular_refs: Vec<(KbId, Vec<KbId>)>,
77
+ pub divergent_content: Vec<(String, Vec<KbId>)>,
56
78
  }
57
79
  ```
58
80
 
@@ -64,7 +86,11 @@ covering all six marker types (`text`, `code`,
64
86
  target_kbid)` where the target section type does
65
87
  not match the marker type. `circular_refs` are
66
88
  `(start_kbid, cycle_path)` for reference chains
67
- that form loops.
89
+ that form loops. `divergent_content` entries are
90
+ `(source_key, conflicting_kbids)` -- sections
91
+ sharing the same `source_key` that have different
92
+ content, indicating contradictory versions that
93
+ should be resolved via supersession.
68
94
 
69
95
  ## Functions
70
96
 
@@ -266,7 +266,7 @@ engine/crates/query-parser/
266
266
  ├── lib.rs ← WASM exports
267
267
  ├── mem.rs ← shared-memory helpers
268
268
 
269
- ├── pipeline/ ← 8-stage tokenizer
269
+ ├── pipeline/ ← 9-stage tokenizer
270
270
  │ ├── mod.rs
271
271
  │ ├── markers.rs
272
272
  │ ├── unicode.rs
@@ -274,6 +274,7 @@ engine/crates/query-parser/
274
274
  │ ├── identifiers.rs
275
275
  │ ├── case.rs
276
276
  │ ├── stopwords.rs
277
+ │ ├── synonyms.rs
277
278
  │ ├── stem.rs
278
279
  │ └── ngram.rs
279
280
 
@@ -5,9 +5,9 @@ extraction, and ranking.*
5
5
 
6
6
  ## Modules
7
7
 
8
- - **[pipeline](pipeline.md)** -- 8-stage tokenization
8
+ - **[pipeline](pipeline.md)** -- 9-stage tokenization
9
9
  pipeline. Converts raw text into normalized, stemmed
10
- tokens with n-grams.
10
+ tokens with synonym expansion and n-grams.
11
11
  - **[parse](parse.md)** -- Query string parsing into
12
12
  structured `QueryPlan`.
13
13
  - **[extract](extract.md)** -- RAKE keyphrase
@@ -1,8 +1,8 @@
1
1
  # functions::pipeline -- Tokenization Pipeline
2
2
 
3
- *8-stage pipeline that converts raw text into
4
- normalized, stemmed tokens. Used for both indexing
5
- and query processing.*
3
+ *9-stage pipeline that converts raw text into
4
+ normalized, stemmed tokens with synonym expansion.
5
+ Used for both indexing and query processing.*
6
6
 
7
7
  ## Source
8
8
 
@@ -14,7 +14,7 @@ and query processing.*
14
14
  pub fn tokenize(input: &str) -> TokenStream;
15
15
  ```
16
16
 
17
- Runs all eight stages in sequence. Each stage is a
17
+ Runs all nine stages in sequence. Each stage is a
18
18
  pure function.
19
19
 
20
20
  ## Stages
@@ -83,7 +83,32 @@ pub fn remove_stop_words(
83
83
  Filters common function words. Single-character
84
84
  tokens also removed.
85
85
 
86
- ### 7. Stemming (`stem.rs`)
86
+ ### 7. Synonym Expansion (`synonyms.rs`)
87
+
88
+ ```rust
89
+ pub fn expand_synonyms(
90
+ word: &str,
91
+ map: &SynonymMap,
92
+ ) -> Vec<String>;
93
+ ```
94
+
95
+ Looks up the lowercased word in the built-in synonym
96
+ map. Returns a vector containing the original word
97
+ plus any synonyms. When no synonyms exist, returns a
98
+ single-element vector containing only the original.
99
+
100
+ The synonym map is a bidirectional
101
+ `HashMap<String, Vec<String>>` built at compile time.
102
+ Expansion is applied after stop-word removal and
103
+ before stemming, so both original and synonym tokens
104
+ are stemmed consistently.
105
+
106
+ The map is small (~30-50 entries) and focused on
107
+ common IT/development vocabulary mismatches:
108
+ login/sign-in, auth/authentication, remove/delete,
109
+ config/configuration, credential/password, etc.
110
+
111
+ ### 8. Stemming (`stem.rs`)
87
112
 
88
113
  ```rust
89
114
  pub fn stem_word(word: &str) -> StemResult;
@@ -92,7 +117,7 @@ pub fn stem_word(word: &str) -> StemResult;
92
117
  Snowball / Porter2. Preserves both original and
93
118
  stemmed forms.
94
119
 
95
- ### 8. N-gram Extraction (`ngram.rs`)
120
+ ### 9. N-gram Extraction (`ngram.rs`)
96
121
 
97
122
  ```rust
98
123
  pub fn extract_ngrams(
@@ -101,3 +126,16 @@ pub fn extract_ngrams(
101
126
  ```
102
127
 
103
128
  Bigrams and trigrams for phrase indexing.
129
+
130
+ ## Relaxed Query Mode
131
+
132
+ The tokenization pipeline is used for both the
133
+ primary AND-mode query plan and the relaxed OR-mode
134
+ fallback plan. `qp_parse_query` produces a
135
+ `QueryPlan` with a `match_mode` field (`All` | `Any`).
136
+ When the AND plan returns zero matches,
137
+ `kb-worker.wasm` retries with the OR plan using the
138
+ same tokenized terms. No additional pipeline stage or
139
+ WASM export is needed. See
140
+ [Query Parser -- Relaxed Query
141
+ Mode](../../../goals/query-parser.md).