@dikolab/kbdb 0.3.2 → 0.4.1

package/docs/modules/typescript/shared/cli/functions.md

@@ -13,9 +13,12 @@ utility helpers, and one function per CLI sub-command.*
 
 **File:** `run-cli.function.ts`
 
-Full CLI orchestrator. Parses arguments, routes to the
-appropriate sub-flow, and returns a `CommandResult`
-without touching process globals.
+Full CLI orchestrator. Parses arguments, configures the
+log level from `-v`/`-vv`/`-vvv` or `KBDB_LOG_LEVEL`,
+resolves non-interactive mode from `--non-interactive`
+or `KBDB_NON_INTERACTIVE` before dispatching commands,
+routes to the appropriate sub-flow, and returns a
+`CommandResult` without touching process globals.
 
 ```ts
 async function runCli(
@@ -41,14 +44,16 @@ interface CliDeps {
 
 Routes:
 
-| Command     | Handler                          |
-|-------------|----------------------------------|
-| `help`      | returns `HELP_TEXT`              |
-| `version`   | returns `VERSION`                |
-| `init`      | calls `deps.runInit`             |
-| `mcp`       | calls `deps.startMcpServer`      |
-| `migrate`   | version check + migrate daemon   |
-| *(other)*   | `dispatchCommand(client, opts)`  |
+| Command          | Handler                          |
+|------------------|----------------------------------|
+| `help`           | returns `HELP_TEXT`              |
+| `version`        | returns `VERSION`                |
+| `init`           | calls `deps.runInit`             |
+| `mcp`            | calls `deps.startMcpServer`      |
+| `migrate`        | version check + migrate daemon   |
+| `worker-stop`    | `runWorkerStop(dbDir)`           |
+| `worker-restart` | `runWorkerRestart(dbDir)`        |
+| *(other)*        | `dispatchCommand(client, opts)`  |
 
 ### `dispatchCommand`
 
@@ -69,8 +74,8 @@ async function dispatchCommand(
 `DispatchClient` is the intersection of all per-command
 client interfaces (`SearchClient & LearnClient &
 UnlearnClient & RecallClient & ContentClient &
-CheckClient & GcClient & RebuildClient & ExportClient &
-SkillClient & AgentClient`).
+CheckClient & GcClient & RebuildClient & StatusClient &
+ExportClient & SkillClient & AgentClient`).
 
 | Command        | Handler                          |
 |----------------|----------------------------------|
@@ -82,6 +87,7 @@ SkillClient & AgentClient`).
 | `content`      | `runContent(client, options)`    |
 | `check`        | `runCheck(client, options)`      |
 | `gc`           | `runGc(client, options)`         |
+| `status`       | `runStatus(client, options)`     |
 | `rebuild`      | `runRebuild(client, options)`    |
 | `skill-learn`  | `runSkillLearn(client, options)` |
 | `skill-list`   | `runSkillList(client, options)`  |
@@ -107,12 +113,19 @@ function parseArgs(
 ```
 
 Throws on invalid or missing required arguments.
-Prints help text and exits for `--help` / `--version`.
+Prints help text and exits for `--help`. Prints the
+version string and exits for `--version`.
+
+`-v` maps to `--verbose` (log level WARNING), not to
+`--version`. Use `--version` (long form only) to print
+the version. `-vv` sets log level INFO; `-vvv` sets
+DEBUG. The resolved level is stored in
+`CliOptions.logLevel`.
 
 **Compound command handling**: when the first positional
-argument is `db`, `skill`, or `agent`, the parser
-consumes the next positional as the subcommand and sets
-`command` to the hyphenated form:
+argument is `db`, `skill`, `agent`, or `worker`, the
+parser consumes the next positional as the subcommand
+and sets `command` to the hyphenated form:
 
 | First positional | Second positional | `command` value  |
 |------------------|-------------------|------------------|
@@ -126,6 +139,8 @@ consumes the next positional as the subcommand and sets
 | `agent`          | `list`            | `agent-list`     |
 | `agent`          | `get`             | `agent-get`      |
 | `agent`          | `delete`          | `agent-delete`   |
+| `worker`         | `stop`            | `worker-stop`    |
+| `worker`         | `restart`         | `worker-restart` |
 
 **New options parsed:**
 
@@ -135,6 +150,7 @@ consumes the next positional as the subcommand and sets
 | `--persona <p>` | `persona`          | Agent persona file    |
 | `--body <b>`    | `body`             | Skill body file       |
 | `--skills <s>`  | `skills`           | Comma-separated kbids |
+| `--non-interactive` | `nonInteractive` | Suppress prompts  |
 
 ### `findDb`
 
@@ -384,6 +400,70 @@ Output: JSON `{ "path": "...",
 "documents_exported": number,
 "elapsed_ms": number }`.
 
+### `runStatus`
+
+**File:** `run-status.function.ts`
+
+Retrieves the database status summary report from the
+worker daemon.
+
+```ts
+async function runStatus(
+   client: StatusClient,
+   options: CliOptions,
+): Promise<unknown>;
+```
+
+`StatusClient` interface:
+
+```ts
+interface StatusClient {
+   dbStatus(): Promise<unknown>;
+}
+```
+
+Delegates to `client.dbStatus()`. The caller is
+responsible for serialising the result and setting
+the process exit code.
+
+### `runWorkerStop`
+
+**File:** `run-worker-stop.function.ts`
+
+Stops the background worker daemon for the current
+database. Discovers the daemon via its PID file and
+sends `SIGTERM`. Returns success when the daemon was
+stopped or was not running.
+
+```ts
+async function runWorkerStop(
+   dbDir: string,
+): Promise<CommandResult>;
+```
+
+Routed directly by `runCli` (not via
+`dispatchCommand`) because the worker-stop command
+does not require a live worker client connection.
+
+### `runWorkerRestart`
+
+**File:** `run-worker-restart.function.ts`
+
+Stops and restarts the background worker daemon.
+Calls `runWorkerStop` internally; the daemon
+auto-spawns on the next CLI command, so restart only
+needs to stop the current instance.
+
+```ts
+async function runWorkerRestart(
+   dbDir: string,
+): Promise<CommandResult>;
+```
+
+Routed directly by `runCli` (not via
+`dispatchCommand`) for the same reason as
+`runWorkerStop`.
+
 ### `ensureDb`
 
 **File:** `ensure-db.function.ts`
@@ -470,6 +550,35 @@ interface PromptIO {
 
 Defaults to readline on `process.stdin`/`process.stdout`.
 
+### `resolveKbidPrefix`
+
+**File:** `resolve-kbid-prefix.function.ts`
+
+Resolves a short kbid prefix to a full 26-character
+kbid. Scans all section types (`text`, `code`, `image`)
+for kbids matching the prefix and returns the full
+identifier when exactly one match exists. Full-length
+identifiers (>= 26 chars) are returned as-is. Throws
+when the prefix is ambiguous (multiple matches) or
+unrecognized (no matches).
+
+```ts
+async function resolveKbidPrefix(
+   prefix: string,
+   client: PrefixResolver,
+): Promise<string>;
+```
+
+`PrefixResolver` interface:
+
+```ts
+interface PrefixResolver {
+   listByType(
+      sectionType: string,
+   ): Promise<readonly { kbid: string }[]>;
+}
+```
+
 ### `runDbInit`
 
 **File:** `run-db-init.function.ts`
@@ -675,10 +784,15 @@ src/shared/cli/
 ├── run-export.spec.ts
 ├── run-unlearn.spec.ts
 ├── run-search.spec.ts
+├── run-recall.spec.ts
 ├── run-content.spec.ts
+├── run-status.spec.ts
 ├── run-check.spec.ts
 ├── run-gc.spec.ts
 ├── run-rebuild.spec.ts
+├── run-worker-stop.spec.ts
+├── run-worker-restart.spec.ts
+├── resolve-kbid-prefix.spec.ts
 ├── run-skill-learn.spec.ts
 ├── run-skill-list.spec.ts
 ├── run-skill-get.spec.ts

package/docs/modules/typescript/shared/cli/module.md

@@ -40,6 +40,7 @@ src/shared/cli/
 │   ├── is-stdin-tty.function.ts
 │   ├── parse-args.function.ts
 │   ├── prompt-user.function.ts
+│   ├── resolve-kbid-prefix.function.ts
 │   ├── run-agent-create.function.ts
 │   ├── run-agent-delete.function.ts
 │   ├── run-agent-get.function.ts
@@ -49,17 +50,22 @@ src/shared/cli/
 │   ├── run-content.function.ts
 │   ├── run-db-init.function.ts
 │   ├── run-db-migrate.function.ts
+│   ├── run-export.function.ts
 │   ├── run-gc.function.ts
 │   ├── run-init.function.ts
 │   ├── run-learn.function.ts
 │   ├── run-migrate.function.ts
+│   ├── run-recall.function.ts
 │   ├── run-rebuild.function.ts
 │   ├── run-search.function.ts
 │   ├── run-skill-delete.function.ts
 │   ├── run-skill-get.function.ts
 │   ├── run-skill-learn.function.ts
 │   ├── run-skill-list.function.ts
-│   └── run-unlearn.function.ts
+│   ├── run-status.function.ts
+│   ├── run-unlearn.function.ts
+│   ├── run-worker-restart.function.ts
+│   └── run-worker-stop.function.ts
 ├── typings/
 │   ├── cli-options.interface.ts
 │   ├── command-result.model.ts
@@ -72,14 +78,16 @@ src/shared/cli/
 
 - **[functions.md](functions.md)** -- `runCli`,
   `dispatchCommand`, `parseArgs`, `findDb`,
-  `runInit`, `runMigrate`, `runLearn`, `runUnlearn`,
-  `runSearch`, `runContent`, `runCheck`, `runGc`,
-  `runRebuild`, `ensureDb`, `formatOutput`,
+  `resolveKbidPrefix`, `runInit`, `runMigrate`,
+  `runLearn`, `runUnlearn`, `runSearch`, `runRecall`,
+  `runContent`, `runExport`, `runStatus`, `runCheck`,
+  `runGc`, `runRebuild`, `ensureDb`, `formatOutput`,
   `errorToMessage`, `isStdinTty`, `promptUser`,
   `runDbInit`, `runDbMigrate`, `runSkillLearn`,
   `runSkillList`, `runSkillGet`, `runSkillDelete`,
   `runAgentCreate`, `runAgentList`, `runAgentGet`,
-  `runAgentDelete`.
+  `runAgentDelete`, `runWorkerStop`,
+  `runWorkerRestart`.
 - **[typings.md](typings.md)** -- `CliOptions`,
   `CommandResult`, `LearnResult`, `SearchDisplay`,
   `SearchMode`, `OutputFormat`.
@@ -100,26 +108,31 @@ cli.ts (entry point)
   │    ├─ command === 'migrate'    → runMigrate(client, options)
   │    ├─ command === 'db-init'    → runDbInit(options)
   │    ├─ command === 'db-migrate' → runDbMigrate(client, options)
+  │    ├─ command === 'worker-stop'    → runWorkerStop(dbDir)
+  │    ├─ command === 'worker-restart' → runWorkerRestart(dbDir)
   │    └─ data command:
   │         ├─ findDb(options.dbPath) → dbDir
   │         ├─ if not found: isStdinTty() / promptUser()
   │         ├─ createWorkerClient({ contextPath: dbDir })
   │         ├─ version gate (checkVersion)
   │         └─ dispatchCommand(client, options)
-  │              ├─ learn         → runLearn(client, options)
-  │              ├─ unlearn       → runUnlearn(client, options)
-  │              ├─ search        → runSearch(client, options)
-  │              ├─ content       → runContent(client, options)
-  │              ├─ check         → runCheck(client, options)
-  │              ├─ gc            → runGc(client, options)
-  │              ├─ rebuild       → runRebuild(client, options)
-  │              ├─ skill-learn   → runSkillLearn(client, options)
-  │              ├─ skill-list    → runSkillList(client, options)
-  │              ├─ skill-get     → runSkillGet(client, options)
-  │              ├─ skill-delete  → runSkillDelete(client, options)
-  │              ├─ agent-create  → runAgentCreate(client, options)
-  │              ├─ agent-list    → runAgentList(client, options)
-  │              ├─ agent-get     → runAgentGet(client, options)
-  │              └─ agent-delete  → runAgentDelete(client, options)
+  │              ├─ learn           → runLearn(client, options)
+  │              ├─ unlearn         → runUnlearn(client, options)
+  │              ├─ search          → runSearch(client, options)
+  │              ├─ recall          → runRecall(client, options)
+  │              ├─ content         → runContent(client, options)
+  │              ├─ export          → runExport(client, options)
+  │              ├─ status          → runStatus(client, options)
+  │              ├─ check           → runCheck(client, options)
+  │              ├─ gc              → runGc(client, options)
+  │              ├─ rebuild         → runRebuild(client, options)
+  │              ├─ skill-learn     → runSkillLearn(client, options)
+  │              ├─ skill-list      → runSkillList(client, options)
+  │              ├─ skill-get       → runSkillGet(client, options)
+  │              ├─ skill-delete    → runSkillDelete(client, options)
+  │              ├─ agent-create    → runAgentCreate(client, options)
+  │              ├─ agent-list      → runAgentList(client, options)
+  │              ├─ agent-get       → runAgentGet(client, options)
+  │              └─ agent-delete    → runAgentDelete(client, options)
   └─ returns CommandResult { output?, error?, exitCode }
 ```

package/docs/modules/typescript/shared/cli/typings.md

@@ -32,6 +32,9 @@ interface CliOptions {
    persona?: string;
    body?: string;
    skills?: string[];
+   timeoutMs?: number;
+   retry?: number;
+   logLevel?: LogLevel;
 }
 ```
 
@@ -61,6 +64,10 @@ interface CliOptions {
 | `persona`     | Agent persona file path (`--persona`) |
 | `body`        | Skill body file path (`--body`)       |
 | `skills`      | Ordered skill kbid list (`--skills`)  |
+| `timeoutMs`   | Request timeout in ms                 |
+| `retry`       | Max retry count on timeout            |
+| `logLevel`    | Log verbosity level from `-v`/`-vv`/`-vvv` |
+| `nonInteractive` | Suppress prompts (`--non-interactive`) |
 
 ### `OutputFormat`
 
@@ -89,6 +96,24 @@ query.
 type SearchMode = 'sections' | 'documents' | 'stats';
 ```
 
+### `LogLevel`
+
+**File:** `cli-options.interface.ts`
+
+Log verbosity level. Set via `-v`/`-vv`/`-vvv` flags
+or the `KBDB_LOG_LEVEL` environment variable.
+
+```ts
+type LogLevel = 'debug' | 'info' | 'warning' | 'error';
+```
+
+| Value     | Description                          |
+|-----------|--------------------------------------|
+| `error`   | Errors only (default)                |
+| `warning` | Errors and warnings (`-v`)           |
+| `info`    | Errors, warnings, and info (`-vv`)   |
+| `debug`   | All messages including debug (`-vvv`)|
+
 ### `CommandResult`
 
 **File:** `command-result.model.ts`

package/docs/modules/typescript/shared/embedding/functions.md

@@ -2,35 +2,46 @@
 
 **Source:** `src/shared/embedding/functions/`
 
-## embed
+## createEmbeddingProvider
 
 ```ts
-embed(
-   provider: EmbeddingProvider,
-   text: string,
-): Promise<Float32Array>
+createEmbeddingProvider(
+   config: EmbeddingConfig,
+): EmbeddingProvider
 ```
 
-Computes a single embedding vector for the given text.
-Delegates to the provider's `embed()` method. Returns a
-`Float32Array` of dimensions matching `provider.dimensions()`.
+Factory that returns the appropriate `EmbeddingProvider`
+implementation based on the given config. Returns an
+`OnnxEmbeddingProvider` for `provider: 'onnx'`, an
+`ApiEmbeddingProvider` for `provider: 'api'`, or a
+`TfidfEmbeddingProvider` when config is undefined, or a
+`NullEmbeddingProvider` when `provider: 'none'`. The
+TF-IDF provider is the default when no embedding config
+is present, enabling hybrid search without external
+configuration.
+
+## hashToken
 
-Used during `learn` to embed section content and during
-`search` to embed the query string.
+```ts
+hashToken(token: string): number
+```
 
-## batchEmbed
+Computes an FNV-1a hash of a string token, returning
+a non-negative 32-bit unsigned integer. Used by
+`TfidfEmbeddingProvider` to map tokens to fixed-size
+vector buckets (feature hashing).
+
+## tokenizeText
 
 ```ts
-batchEmbed(
-   provider: EmbeddingProvider,
-   texts: string[],
-): Promise<Float32Array[]>
+tokenizeText(text: string): string[]
 ```
 
-Computes embeddings for multiple texts in a single batch.
-More efficient than calling `embed()` in a loop -- providers
-can batch API calls or parallelize ONNX inference. Used
-during `rebuild` to re-embed all sections.
+Splits text into lowercase tokens by non-word
+characters, filtering out tokens shorter than 3
+characters. A simple JS tokenizer used by
+`TfidfEmbeddingProvider` for both document and query
+embedding.
 
 ## cosineSimilarity
 
@@ -92,3 +103,98 @@ Note: This is a brute-force linear scan. For corpus sizes
 under 100k sections, this is fast enough (< 50ms). A future
 version may add approximate nearest neighbor (ANN) indexing
 if scale demands it.
+
+## appendVectorEntry
+
+```ts
+appendVectorEntry(
+   provider: EmbeddingProvider,
+   kbid: string,
+   content: string,
+   indexPath: string,
+): Promise<void>
+```
+
+Appends a single vector entry to an existing vector index.
+Used during incremental `learn` to add new sections without
+a full rebuild.
+
+## readVectorIndexMeta
+
+```ts
+readVectorIndexMeta(
+   indexPath: string,
+): Promise<VectorIndexMeta | null>
+```
+
+Reads the `vectors-meta.toml` file for an existing vector
+index. Returns `null` if the index does not exist.
+
+## rrfFuse
+
+```ts
+rrfFuse(
+   lexical: { kbid: string; rank: number }[],
+   vector: { kbid: string; score: number }[],
+   k?: number,
+): { kbid: string; score: number }[]
+```
+
+Merges lexical and vector result lists using Reciprocal
+Rank Fusion. The `k` parameter controls rank smoothing
+(default: 60).
+
+## jaccardCoefficient
+
+```ts
+jaccardCoefficient(
+   a: Set<string>,
+   b: Set<string>,
+): number
+```
+
+Computes the Jaccard similarity coefficient between two
+term sets. Returns a value in [0, 1]. Used as the first
+pass in near-duplicate detection.
+
+## detectNearDuplicates
+
+```ts
+detectNearDuplicates(
+   sections: SectionRecord[],
+   provider?: EmbeddingProvider,
+): Promise<NearDuplicateResult[]>
+```
+
+Two-pass near-duplicate detection: Jaccard coefficient on
+term sets (always), then cosine similarity on embeddings
+(when a provider is available). Returns pairs of sections
+that exceed the similarity thresholds.
+
+## detectStaleVectors
+
+```ts
+detectStaleVectors(
+   indexPath: string,
+   currentKbids: Set<string>,
+): Promise<StaleVectorsReport>
+```
+
+Compares the vector index against the current section set
+and reports entries whose kbid no longer exists. Used by
+`check` to flag stale vector data.
+
+## rebuildVectors
+
+```ts
+rebuildVectors(
+   provider: EmbeddingProvider,
+   sections: SectionRecord[],
+   outPath: string,
+): Promise<VectorIndexMeta>
+```
+
+Rebuilds the entire vector index from scratch. Embeds all
+sections and writes a fresh `vectors.idx` and
+`vectors-meta.toml`. Called by `rebuild` when an embedding
+provider is configured.

package/docs/modules/typescript/shared/embedding/module.md

@@ -40,11 +40,17 @@ src/shared/embedding/
 
 ## Providers
 
-Three implementations of the `EmbeddingProvider` interface:
+Four implementations of the `EmbeddingProvider` interface:
 
-- **NullEmbeddingProvider** -- No-op provider used when no embedding
-  backend is configured. Returns empty vectors. Allows all
-  lexical-only paths to work unchanged.
+- **TfidfEmbeddingProvider** -- Built-in default provider backed by
+  the `kbdb-default-embedding` WASM crate. Delegates `embed()`
+  calls to WASM exports that compute feature-hashed sublinear TF
+  vectors (256 dimensions, L2-normalized) using kbdb-shared's
+  tokenization pipeline. Zero external dependencies, zero
+  configuration. Enables hybrid search out of the box.
+- **NullEmbeddingProvider** -- No-op provider (dimensions = 0).
+  Available for explicit use when embeddings should be fully
+  disabled.
 - **OnnxEmbeddingProvider** -- Local inference using ONNX Runtime.
   Loads a model from disk (e.g., `all-MiniLM-L6-v2`). Zero network
   calls, fully offline.

package/docs/modules/typescript/shared/embedding/typings.md

@@ -16,7 +16,8 @@ results.*
 
 Core abstraction for text embedding. Accepts a batch
 of texts and returns a vector for each. Implemented
-by `NullEmbeddingProvider`, `OnnxEmbeddingProvider`,
+by `TfidfEmbeddingProvider` (built-in default),
+`NullEmbeddingProvider`, `OnnxEmbeddingProvider`,
 and `ApiEmbeddingProvider`.
 
 ```ts
@@ -58,10 +59,11 @@ interface EmbeddingConfig {
 
 | Property      | Description                              |
 |---------------|------------------------------------------|
-| `provider`    | Selects the backend: `'none'` →          |
-|               | `NullEmbeddingProvider`; `'onnx'` →      |
-|               | `OnnxEmbeddingProvider`; `'api'` →       |
-|               | `ApiEmbeddingProvider`                   |
+| `provider`    | Selects the backend: missing →            |
+|               | `TfidfEmbeddingProvider`; `'none'` →     |
+|               | `NullEmbeddingProvider`;                 |
+|               | `'onnx'` → `OnnxEmbeddingProvider`;      |
+|               | `'api'` → `ApiEmbeddingProvider`         |
 | `model`       | Path to the local ONNX model file;       |
 |               | required when `provider` is `'onnx'`     |
 | `apiEndpoint` | Remote HTTP API URL; required when       |

package/docs/modules/typescript/shared/log/constants.md

@@ -0,0 +1,82 @@
+# log/constants -- Log Constants
+
+*Default log level, per-level Unicode icons, and per-level
+ANSI color codes.*
+
+## Source
+
+`src/shared/log/constants/`
+
+## Constants
+
+### `DEFAULT_LOG_LEVEL`
+
+**File:** `log-defaults.constant.ts`
+
+Default log level used when no CLI flag or `KBDB_LOG_LEVEL`
+environment variable is set.
+
+```ts
+const DEFAULT_LOG_LEVEL = LogLevel.ERROR;
+```
+
+Evaluated by `resolveLogLevel` as the lowest-priority
+fallback.
+
+### `LOG_ICONS`
+
+**File:** `log-icons.constant.ts`
+
+Single-character Unicode icon used as the visual prefix
+for each log level's output line.
+
+```ts
+const LOG_ICONS: Record<LogLevel, string> = {
+   [LogLevel.DEBUG]:   '▸',
+   [LogLevel.INFO]:    'ℹ',
+   [LogLevel.WARNING]: '⚠',
+   [LogLevel.ERROR]:   '✖',
+};
+```
+
+| Level     | Icon | Meaning           |
+|-----------|------|-------------------|
+| `DEBUG`   | `▸`  | Right arrow       |
+| `INFO`    | `ℹ`  | Information sign  |
+| `WARNING` | `⚠`  | Warning sign      |
+| `ERROR`   | `✖`  | Multiplication X  |
+
+### `LOG_COLORS`
+
+**File:** `log-colors.constant.ts`
+
+ANSI foreground color escape sequence applied before
+each log line's content.
+
+```ts
+const LOG_COLORS: Record<LogLevel, string> = {
+   [LogLevel.DEBUG]:   '\x1b[37m',
+   [LogLevel.INFO]:    '\x1b[36m',
+   [LogLevel.WARNING]: '\x1b[33m',
+   [LogLevel.ERROR]:   '\x1b[91m',
+};
+```
+
+| Level     | Code       | Color       |
+|-----------|------------|-------------|
+| `DEBUG`   | `\x1b[37m` | White       |
+| `INFO`    | `\x1b[36m` | Cyan        |
+| `WARNING` | `\x1b[33m` | Yellow      |
+| `ERROR`   | `\x1b[91m` | Bright red  |
+
+### `ANSI_RESET`
+
+**File:** `log-colors.constant.ts`
+
+ANSI escape sequence that resets all text formatting.
+Appended at the end of every log line so subsequent
+terminal output is unaffected.
+
+```ts
+const ANSI_RESET = '\x1b[0m';
+```