opencode-rag-plugin 2.10.1 → 2.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/ReadMe.md +40 -28
  2. package/dist/api.js +1 -0
  3. package/dist/chunker/grammar.d.ts +4 -1
  4. package/dist/chunker/grammar.js +9 -3
  5. package/dist/chunker/skeleton.js +61 -4
  6. package/dist/cli/commands/clickhouse.d.ts +61 -0
  7. package/dist/cli/commands/clickhouse.js +335 -0
  8. package/dist/cli/commands/eval-tuner.js +1 -1
  9. package/dist/cli/commands/eval.js +1 -1
  10. package/dist/cli/commands/index-command.js +2 -1
  11. package/dist/cli/commands/index.d.ts +1 -0
  12. package/dist/cli/commands/index.js +1 -0
  13. package/dist/cli/commands/init-helpers.js +2 -2
  14. package/dist/cli/commands/init.js +49 -11
  15. package/dist/cli/commands/quirk.js +6 -6
  16. package/dist/cli/commands/status.js +70 -6
  17. package/dist/cli/commands/ui.js +2 -2
  18. package/dist/cli/index.js +2 -1
  19. package/dist/cli/types.d.ts +14 -0
  20. package/dist/core/bootstrap.d.ts +7 -0
  21. package/dist/core/bootstrap.js +40 -9
  22. package/dist/core/config.d.ts +128 -5
  23. package/dist/core/config.js +205 -8
  24. package/dist/core/interfaces.d.ts +54 -0
  25. package/dist/core/runtime-overrides.d.ts +9 -0
  26. package/dist/core/runtime-overrides.js +44 -0
  27. package/dist/core/state-sync.d.ts +91 -0
  28. package/dist/core/state-sync.js +273 -0
  29. package/dist/eval/fast-index.js +2 -2
  30. package/dist/eval/rerank-eval.js +2 -2
  31. package/dist/eval/run-branch-compare.js +2 -2
  32. package/dist/eval/run-token-test.js +2 -2
  33. package/dist/index.d.ts +5 -2
  34. package/dist/index.js +2 -1
  35. package/dist/indexer/pipeline.d.ts +6 -0
  36. package/dist/indexer/pipeline.js +173 -56
  37. package/dist/indexer/progress.d.ts +75 -0
  38. package/dist/indexer/progress.js +114 -0
  39. package/dist/indexer/stats.d.ts +14 -0
  40. package/dist/plugin.d.ts +6 -3
  41. package/dist/plugin.js +39 -20
  42. package/dist/quirks/quirk-store.d.ts +10 -0
  43. package/dist/quirks/quirk-store.js +110 -15
  44. package/dist/quirks/types.d.ts +7 -0
  45. package/dist/tui.js +128 -111
  46. package/dist/vectorstore/clickhouse.d.ts +150 -0
  47. package/dist/vectorstore/clickhouse.js +931 -0
  48. package/dist/vectorstore/dimension.d.ts +19 -0
  49. package/dist/vectorstore/dimension.js +36 -0
  50. package/dist/vectorstore/factory.d.ts +9 -3
  51. package/dist/vectorstore/factory.js +11 -2
  52. package/dist/vectorstore/lancedb.d.ts +2 -2
  53. package/dist/watcher.d.ts +29 -0
  54. package/dist/watcher.js +76 -2
  55. package/dist/web/api.d.ts +3 -4
  56. package/dist/web/api.js +18 -3
  57. package/dist/web/server.js +28 -2
  58. package/dist/web/ui/assets/{ScatterPlot3D-CKwyh1Ec.js → ScatterPlot3D-XjXIwavM.js} +5 -5
  59. package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
  60. package/dist/web/ui/assets/index-FFPRbgLx.js +4 -0
  61. package/dist/web/ui/index.html +2 -2
  62. package/package.json +7 -2
  63. package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
  64. package/dist/web/ui/assets/index-DbXFrn53.js +0 -4
@@ -0,0 +1,19 @@
1
+ /**
2
+ * @fileoverview Backend-neutral dimension probe.
3
+ *
4
+ * The pipeline needs the dimension of an existing store before it can decide
5
+ * whether to rebuild. LanceDB keeps it in the table schema; ClickHouse keeps
6
+ * it in the project's `_meta` state row. Both probes are read-only.
7
+ */
8
+ import type { RagConfig } from "../core/config.js";
9
+ /**
10
+ * Read the embedding dimension of the configured store.
11
+ *
12
+ * @param config - Resolved RAG configuration.
13
+ * @param storePath - Local store directory (LanceDB only).
14
+ * @param workspaceDir - Workspace root (used to derive the ClickHouse namespace).
15
+ * @returns The stored dimension, or undefined when the store does not exist yet.
16
+ */
17
+ export declare function readConfiguredStoreDimension(config: RagConfig, storePath: string, workspaceDir: string): Promise<number | undefined>;
18
+ /** Read the dimension from a ClickHouse store without creating any schema. */
19
+ export declare function readClickHouseDimension(config: RagConfig, workspaceDir: string): Promise<number | undefined>;
@@ -0,0 +1,36 @@
1
+ import { readStoreDimension } from "./lancedb.js";
2
+ import { ClickHouseStore, resolveClickHouseOptions } from "./clickhouse.js";
3
+ /**
4
+ * Read the embedding dimension of the configured store.
5
+ *
6
+ * @param config - Resolved RAG configuration.
7
+ * @param storePath - Local store directory (LanceDB only).
8
+ * @param workspaceDir - Workspace root (used to derive the ClickHouse namespace).
9
+ * @returns The stored dimension, or undefined when the store does not exist yet.
10
+ */
11
+ export async function readConfiguredStoreDimension(config, storePath, workspaceDir) {
12
+ const provider = config.vectorStore.provider ?? "lancedb";
13
+ if (provider === "lancedb") {
14
+ return readStoreDimension(storePath);
15
+ }
16
+ if (provider === "clickhouse") {
17
+ return readClickHouseDimension(config, workspaceDir);
18
+ }
19
+ return undefined;
20
+ }
21
+ /** Read the dimension from a ClickHouse store without creating any schema. */
22
+ export async function readClickHouseDimension(config, workspaceDir) {
23
+ const options = resolveClickHouseOptions(config, workspaceDir);
24
+ const store = new ClickHouseStore(options, config.embedding.vectorDimension ?? 384);
25
+ try {
26
+ return await store.peekDimension();
27
+ }
28
+ catch (err) {
29
+ console.warn(`[bootstrap] Could not read ClickHouse store dimension: ${err.message}`);
30
+ return undefined;
31
+ }
32
+ finally {
33
+ await store.close().catch(() => { });
34
+ }
35
+ }
36
+ //# sourceMappingURL=dimension.js.map
@@ -1,5 +1,6 @@
1
1
  /**
2
- * @fileoverview Factory function to create the configured VectorStore instance (LanceDB or in-memory).
2
+ * @fileoverview Factory function to create the configured VectorStore instance
3
+ * (LanceDB, in-memory, or ClickHouse).
3
4
  */
4
5
  import type { VectorStore } from "../core/interfaces.js";
5
6
  import type { RagConfig } from "../core/config.js";
@@ -9,11 +10,16 @@ import type { RagConfig } from "../core/config.js";
9
10
  * Supported providers:
10
11
  * - `"lancedb"`: Persistent LanceDB-backed store (default).
11
12
  * - `"memory"`: Ephemeral in-memory store.
13
+ * - `"clickhouse"`: Central ClickHouse-backed store; `storePath` is only a
14
+ * local cache directory for workspace state.
12
15
  *
13
16
  * @param config - The RAG configuration containing the vector store provider setting.
14
- * @param storePath - The filesystem path for persistent stores (LanceDB).
17
+ * @param storePath - The filesystem path for persistent local stores (LanceDB)
18
+ * and for the disposable state cache when ClickHouse is active.
15
19
  * @param dimension - The embedding vector dimension.
20
+ * @param workspaceDir - Workspace root, used to derive the default ClickHouse
21
+ * namespace. Defaults to `process.cwd()`.
16
22
  * @returns A VectorStore instance matching the configured provider.
17
23
  * @throws If the provider name is unknown.
18
24
  */
19
- export declare function createVectorStore(config: RagConfig, storePath: string, dimension: number): VectorStore;
25
+ export declare function createVectorStore(config: RagConfig, storePath: string, dimension: number, workspaceDir?: string): VectorStore;
@@ -1,19 +1,25 @@
1
1
  import { LanceDbStore } from "./lancedb.js";
2
2
  import { InMemoryVectorStore } from "./memory.js";
3
+ import { ClickHouseStore, resolveClickHouseOptions } from "./clickhouse.js";
3
4
  /**
4
5
  * Create a VectorStore instance based on the configured provider.
5
6
  *
6
7
  * Supported providers:
7
8
  * - `"lancedb"`: Persistent LanceDB-backed store (default).
8
9
  * - `"memory"`: Ephemeral in-memory store.
10
+ * - `"clickhouse"`: Central ClickHouse-backed store; `storePath` is only a
11
+ * local cache directory for workspace state.
9
12
  *
10
13
  * @param config - The RAG configuration containing the vector store provider setting.
11
- * @param storePath - The filesystem path for persistent stores (LanceDB).
14
+ * @param storePath - The filesystem path for persistent local stores (LanceDB)
15
+ * and for the disposable state cache when ClickHouse is active.
12
16
  * @param dimension - The embedding vector dimension.
17
+ * @param workspaceDir - Workspace root, used to derive the default ClickHouse
18
+ * namespace. Defaults to `process.cwd()`.
13
19
  * @returns A VectorStore instance matching the configured provider.
14
20
  * @throws If the provider name is unknown.
15
21
  */
16
- export function createVectorStore(config, storePath, dimension) {
22
+ export function createVectorStore(config, storePath, dimension, workspaceDir = process.cwd()) {
17
23
  const provider = config.vectorStore.provider ?? "lancedb";
18
24
  if (provider === "lancedb") {
19
25
  return new LanceDbStore(storePath, dimension);
@@ -21,6 +27,9 @@ export function createVectorStore(config, storePath, dimension) {
21
27
  if (provider === "memory") {
22
28
  return new InMemoryVectorStore();
23
29
  }
30
+ if (provider === "clickhouse") {
31
+ return new ClickHouseStore(resolveClickHouseOptions(config, workspaceDir), dimension);
32
+ }
24
33
  throw new Error(`Unknown vector store provider: ${provider}`);
25
34
  }
26
35
  //# sourceMappingURL=factory.js.map
@@ -1,4 +1,4 @@
1
- import type { VectorStore, Chunk, ChunkSummary, SearchResult, MetadataFilter } from "../core/interfaces.js";
1
+ import type { Chunk, ChunkSummary, SearchResult, MetadataFilter, ChunkBrowseStore } from "../core/interfaces.js";
2
2
  /** Minimal warning sink for store diagnostics (defaults to console.warn). */
3
3
  export type StoreWarn = (message: string) => void;
4
4
  /**
@@ -112,7 +112,7 @@ export interface BulkChunkWrite {
112
112
  * and chunk metadata queries. Supports automatic corruption recovery by falling
113
113
  * back to prior table versions or dropping and rebuilding the table.
114
114
  */
115
- export declare class LanceDbStore implements VectorStore {
115
+ export declare class LanceDbStore implements ChunkBrowseStore {
116
116
  private dbPath;
117
117
  private readonly vectorDimension;
118
118
  private db;
package/dist/watcher.d.ts CHANGED
@@ -5,6 +5,7 @@
5
5
  */
6
6
  import type { RagConfig } from "./core/config.js";
7
7
  import type { DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "./core/interfaces.js";
8
+ import type { StateSync } from "./core/state-sync.js";
8
9
  /** A background indexer that can be shut down gracefully. */
9
10
  export interface BackgroundIndexer {
10
11
  /** Stop the watcher, cancel pending index passes, and clean up status files. */
@@ -32,6 +33,8 @@ export interface CreateBackgroundIndexerOptions {
32
33
  descriptionProvider?: DescriptionProvider;
33
34
  /** Embedding vector dimension — enables atomic rebuilds instead of destructive clear. */
34
35
  dimension?: number;
36
+ /** Durable-state synchronizer (ClickHouse); flushed at the end of each pass. */
37
+ stateSync?: StateSync;
35
38
  }
36
39
  /** The current operational status of the background indexer watcher. */
37
40
  export type WatcherStatus = {
@@ -73,6 +76,32 @@ export declare function deriveWatcherHealth(status: WatcherStatus): {
73
76
  export declare function tryAcquireWatcherLock(storePath: string): boolean;
74
77
  /** Release the watcher lock — only if this process owns it. */
75
78
  export declare function releaseWatcherLock(storePath: string): void;
79
+ /** A watcher observed on disk: its lock, its persisted status, and liveness. */
80
+ export interface ObservedWatcher {
81
+ /** Store directory holding `watcher.lock` / `watcher-status.json`. */
82
+ storePath: string;
83
+ /** Contents of `watcher.lock`, when present and parseable. */
84
+ lock?: {
85
+ pid: number;
86
+ startedAt: number;
87
+ };
88
+ /** PID in the lock is alive — the watcher process is currently running. */
89
+ alive: boolean;
90
+ /** Parsed `watcher-status.json`, when present and parseable. */
91
+ status?: WatcherStatus;
92
+ }
93
+ /** Read one store directory's watcher lock + status. Returns undefined when the dir holds neither. */
94
+ export declare function readWatcherState(storePath: string): ObservedWatcher | undefined;
95
+ /**
96
+ * Find every watcher observable for a workspace — including one running in a
97
+ * DIFFERENT store dir than the configured `vectorStore.path` (the classic
98
+ * symptom of a config change the already-running plugin never picked up).
99
+ *
100
+ * Scans the configured store dir, its sibling directories, and every child of
101
+ * `<workspaceRoot>/.opencode/` for `watcher.lock` / `watcher-status.json`.
102
+ * Results are sorted live-first, then by most recent activity.
103
+ */
104
+ export declare function discoverWatcherStates(configuredStorePath: string, workspaceRoot: string): ObservedWatcher[];
76
105
  /**
77
106
  * Create a background file watcher that automatically re-indexes the
78
107
  * workspace when files change. Uses chokidar for file system events and
package/dist/watcher.js CHANGED
@@ -5,7 +5,7 @@
5
5
  */
6
6
  import chokidar from "chokidar";
7
7
  import path from "node:path";
8
- import { writeFileSync, unlinkSync, existsSync, readFileSync, openSync, closeSync } from "node:fs";
8
+ import { writeFileSync, unlinkSync, existsSync, readFileSync, openSync, closeSync, readdirSync } from "node:fs";
9
9
  import { appendDebugLog } from "./core/fileLogger.js";
10
10
  import { isCorruptionError } from "./vectorstore/lancedb.js";
11
11
  import { createWatchPassScheduler, createWatchIgnore, runIndexPass, } from "./indexer.js";
@@ -113,6 +113,79 @@ export function releaseWatcherLock(storePath) {
113
113
  catch { /* ignore */ }
114
114
  }
115
115
  }
116
+ /** Read one store directory's watcher lock + status. Returns undefined when the dir holds neither. */
117
+ export function readWatcherState(storePath) {
118
+ let lock;
119
+ try {
120
+ lock = readWatcherLock(storePath);
121
+ }
122
+ catch {
123
+ /* unreadable dir */
124
+ }
125
+ let status;
126
+ try {
127
+ const statusPath = path.join(storePath, "watcher-status.json");
128
+ if (existsSync(statusPath)) {
129
+ status = JSON.parse(readFileSync(statusPath, "utf-8"));
130
+ }
131
+ }
132
+ catch {
133
+ /* corrupt status file — ignore */
134
+ }
135
+ if (!lock && !status)
136
+ return undefined;
137
+ return {
138
+ storePath,
139
+ lock,
140
+ alive: lock ? isPidAlive(lock.pid) : false,
141
+ status,
142
+ };
143
+ }
144
+ /**
145
+ * Find every watcher observable for a workspace — including one running in a
146
+ * DIFFERENT store dir than the configured `vectorStore.path` (the classic
147
+ * symptom of a config change the already-running plugin never picked up).
148
+ *
149
+ * Scans the configured store dir, its sibling directories, and every child of
150
+ * `<workspaceRoot>/.opencode/` for `watcher.lock` / `watcher-status.json`.
151
+ * Results are sorted live-first, then by most recent activity.
152
+ */
153
+ export function discoverWatcherStates(configuredStorePath, workspaceRoot) {
154
+ const candidates = new Set();
155
+ const add = (dir) => {
156
+ try {
157
+ if (existsSync(dir))
158
+ candidates.add(path.resolve(dir));
159
+ }
160
+ catch {
161
+ /* skip unreadable candidate */
162
+ }
163
+ };
164
+ add(configuredStorePath);
165
+ add(path.dirname(configuredStorePath));
166
+ const dotDir = path.join(workspaceRoot, ".opencode");
167
+ add(dotDir);
168
+ try {
169
+ if (existsSync(dotDir)) {
170
+ for (const entry of readdirSync(dotDir, { withFileTypes: true })) {
171
+ if (entry.isDirectory())
172
+ add(path.join(dotDir, entry.name));
173
+ }
174
+ }
175
+ }
176
+ catch {
177
+ /* unreadable .opencode dir */
178
+ }
179
+ const found = [];
180
+ for (const dir of candidates) {
181
+ const state = readWatcherState(dir);
182
+ if (state)
183
+ found.push(state);
184
+ }
185
+ const lastActivity = (w) => Math.max(w.status?.lastSuccessAt ?? 0, w.status?.lastRunAt ?? 0, w.status?.lastChangeAt ?? 0, w.lock?.startedAt ?? 0);
186
+ found.sort((a, b) => (b.alive ? 1 : 0) - (a.alive ? 1 : 0) || lastActivity(b) - lastActivity(a));
187
+ return found;
188
+ }
116
189
  /**
117
190
  * Create a background file watcher that automatically re-indexes the
118
191
  * workspace when files change. Uses chokidar for file system events and
@@ -123,7 +196,7 @@ export function releaseWatcherLock(storePath) {
123
196
  * @returns A BackgroundIndexer handle with a close() method for shutdown.
124
197
  */
125
198
  export function createBackgroundIndexer(options) {
126
- const { cwd, storePath, config, store, embedder, logFilePath, logLevel, keywordIndex, descriptionProvider, dimension } = options;
199
+ const { cwd, storePath, config, store, embedder, logFilePath, logLevel, keywordIndex, descriptionProvider, dimension, stateSync } = options;
127
200
  const autoIndexCfg = config.openCode.autoIndex ?? { enabled: false, debounceMs: 5000, intervalMs: 300000 };
128
201
  let active = false;
129
202
  let recheckTimer;
@@ -159,6 +232,7 @@ export function createBackgroundIndexer(options) {
159
232
  embedder,
160
233
  keywordIndex,
161
234
  descriptionProvider,
235
+ stateSync,
162
236
  dimension,
163
237
  abortSignal: ac.signal,
164
238
  logger: {
package/dist/web/api.d.ts CHANGED
@@ -2,10 +2,9 @@
2
2
  * @fileoverview REST API handler for the OpenCodeRAG Web UI with search, file, chunk, eval, and token analysis endpoints.
3
3
  */
4
4
  import type { IncomingMessage, ServerResponse } from "node:http";
5
- import { LanceDbStore } from "../vectorstore/lancedb.js";
6
5
  import { KeywordIndex } from "../retriever/keyword-index.js";
7
6
  import { type RagConfig } from "../core/config.js";
8
- import { type EmbeddingProvider } from "../core/interfaces.js";
7
+ import { type ChunkBrowseStore, type EmbeddingProvider } from "../core/interfaces.js";
9
8
  /** Internal shape for a JSON API response: an HTTP status code and a serialisable body. */
10
9
  interface ApiResponse {
11
10
  status: number;
@@ -21,7 +20,7 @@ interface ApiResponse {
21
20
  * - `/api/eval/sessions`, `/api/eval/sessions/:id`, `/api/eval/compare`
22
21
  * - `/api/eval/sessions/:id/analysis`, `/api/eval/token-compare`, `/api/eval/project-savings`
23
22
  *
24
- * @param store - The LanceDB vector store instance.
23
+ * @param store - The vector store instance (LanceDB or ClickHouse).
25
24
  * @param keywordIndex - The keyword-index instance for text search.
26
25
  * @param storePath - Filesystem path to the store directory (used by eval endpoints).
27
26
  * @param cwd - Optional workspace root for resolving file paths.
@@ -31,7 +30,7 @@ interface ApiResponse {
31
30
  * @param configPath - Path to the opencode-rag.json config file, used by PUT /api/config.
32
31
  * @returns An async handler that returns `true` when a route matched or `false` otherwise.
33
32
  */
34
- export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>, token?: string, configPath?: string): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
33
+ export declare function createApiHandler(store: ChunkBrowseStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>, token?: string, configPath?: string): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
35
34
  /**
36
35
  * Apply a validated config patch (`{ indexing: { includeDirs: [...] } }`)
37
36
  * to the on-disk config file and refresh the in-memory config used by
package/dist/web/api.js CHANGED
@@ -6,7 +6,7 @@ import { analyzeTokenUsage, compareTokenAnalyses, projectTokenSavings } from "..
6
6
  import { listQuirks, lintQuirks, removeQuirk } from "../quirks/quirk-store.js";
7
7
  import { retrieve } from "../retriever/retriever.js";
8
8
  import { getRerankerFor } from "../reranker/factory.js";
9
- import { updateConfigValue } from "../core/config.js";
9
+ import { updateConfigValue, ensureConfigFile } from "../core/config.js";
10
10
  import { createExcludeMatcher } from "../core/exclude.js";
11
11
  import { CODE_SEARCH_FILTER } from "../core/interfaces.js";
12
12
  const FILE_MIME_TYPES = {
@@ -92,7 +92,7 @@ function sendJson(res, response, origin) {
92
92
  * - `/api/eval/sessions`, `/api/eval/sessions/:id`, `/api/eval/compare`
93
93
  * - `/api/eval/sessions/:id/analysis`, `/api/eval/token-compare`, `/api/eval/project-savings`
94
94
  *
95
- * @param store - The LanceDB vector store instance.
95
+ * @param store - The vector store instance (LanceDB or ClickHouse).
96
96
  * @param keywordIndex - The keyword-index instance for text search.
97
97
  * @param storePath - Filesystem path to the store directory (used by eval endpoints).
98
98
  * @param cwd - Optional workspace root for resolving file paths.
@@ -629,10 +629,17 @@ async function handleReindex(cwd, cfg, storePath, store, getEmbedder) {
629
629
  }
630
630
  reindexInFlight = true;
631
631
  runIndexPass({ cwd, storePath, config: cfg, store, embedder })
632
- .then(() => {
632
+ .then(async () => {
633
633
  reindexInFlight = false;
634
634
  statusCache = null; // invalidate the status cache after a pass
635
635
  projectionCache = null;
636
+ try {
637
+ const { StateSync } = await import("../core/state-sync.js");
638
+ await new StateSync({ store, storePath, config: cfg }).flush(["manifest", "keyword-index", "quirks", "runtime-overrides", "doc-progress", ".desc-cache"]);
639
+ }
640
+ catch {
641
+ // durable-state upload is best effort; the pass itself succeeded
642
+ }
636
643
  })
637
644
  .catch((err) => {
638
645
  reindexInFlight = false;
@@ -745,6 +752,14 @@ export function handleConfigUpdate(configPath, configRef, body) {
745
752
  }
746
753
  }
747
754
  }
755
+ // Writers target the canonical `.opencode/opencode-rag.json`; a workspace
756
+ // without any config file yet gets one created on the first write.
757
+ try {
758
+ ensureConfigFile(configPath);
759
+ }
760
+ catch {
761
+ // fall through — updateConfigValue below reports the failure as 500
762
+ }
748
763
  if (indexingPatch) {
749
764
  for (const [key, value] of Object.entries(indexingPatch)) {
750
765
  const ok = updateConfigValue(configPath, ["indexing", key], value);
@@ -6,7 +6,9 @@ import { existsSync, readFileSync } from "node:fs";
6
6
  import { dirname, extname, join, sep } from "node:path";
7
7
  import { randomBytes } from "node:crypto";
8
8
  import { fileURLToPath } from "node:url";
9
+ import { createVectorStore } from "../vectorstore/factory.js";
9
10
  import { LanceDbStore } from "../vectorstore/lancedb.js";
11
+ import { StateSync } from "../core/state-sync.js";
10
12
  import { KeywordIndex } from "../retriever/keyword-index.js";
11
13
  import { createApiHandler } from "./api.js";
12
14
  import { getStaticHtml, resolveDistAsset } from "./static.js";
@@ -25,6 +27,17 @@ const MIME_TYPES = {
25
27
  ".webp": "image/webp",
26
28
  ".bmp": "image/bmp",
27
29
  };
30
+ /**
31
+ * Narrow a store to the browse-capable interface used by the Web UI.
32
+ * The in-memory store does not implement pagination/count queries.
33
+ */
34
+ function asBrowseStore(store) {
35
+ const candidate = store;
36
+ if (typeof candidate.getChunksFiltered !== "function") {
37
+ throw new Error("The Web UI requires a vector store with browse support (lancedb or clickhouse).");
38
+ }
39
+ return store;
40
+ }
28
41
  /** Serve an HTML string as the HTTP response with UTF-8 content type. */
29
42
  function serveStatic(res, html) {
30
43
  res.writeHead(200, {
@@ -70,8 +83,21 @@ function serveUiAsset(res, filePath) {
70
83
  * @returns A {@link WebUiServer} handle for the running server.
71
84
  */
72
85
  export async function startWebUi(storePath, port, cwd, vectorDimension = 384, cfg, configPath) {
73
- const store = new LanceDbStore(storePath, vectorDimension);
86
+ const store = cfg
87
+ ? createVectorStore(cfg, storePath, vectorDimension, cwd ?? process.cwd())
88
+ : new LanceDbStore(storePath, vectorDimension);
89
+ if (cfg) {
90
+ // ClickHouse: pull the durable manifest/keyword-index/quirks into the
91
+ // local cache before loading the keyword index below.
92
+ try {
93
+ await new StateSync({ store, storePath, config: cfg }).hydrate();
94
+ }
95
+ catch (err) {
96
+ console.warn(`[web] State hydration failed: ${err.message}`);
97
+ }
98
+ }
74
99
  const keywordIndex = await KeywordIndex.load(storePath);
100
+ const browseStore = asBrowseStore(store);
75
101
  // Lazy embedder for /api/retrieve — initialized on first use
76
102
  let embedderPromise = null;
77
103
  async function getEmbedder() {
@@ -83,7 +109,7 @@ export async function startWebUi(storePath, port, cwd, vectorDimension = 384, cf
83
109
  }
84
110
  const html = getStaticHtml();
85
111
  const token = randomBytes(24).toString("hex");
86
- const apiHandler = createApiHandler(store, keywordIndex, storePath, cwd, cfg, getEmbedder, token, configPath);
112
+ const apiHandler = createApiHandler(browseStore, keywordIndex, storePath, cwd, cfg, getEmbedder, token, configPath);
87
113
  const server = createServer(async (req, res) => {
88
114
  try {
89
115
  // Strip the query string before routing — the auth token arrives as