@vellumai/assistant 0.11.1-dev.202608031736.2d0bed8 → 0.11.1-dev.202608032034.aebd2da
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/eslint.config.mjs +28 -8
- package/package.json +1 -1
- package/src/__tests__/cli-logger-boundary-guard.test.ts +87 -0
- package/src/config/__tests__/deployment-context-defaults.test.ts +27 -5
- package/src/config/loader.ts +18 -11
- package/src/persistence/bookmark-crud.ts +18 -9
- package/src/persistence/db-init.ts +14 -1
- package/src/persistence/db-maintenance.ts +5 -4
- package/src/persistence/embeddings/__tests__/plugin-index-qdrant-init.test.ts +219 -0
- package/src/persistence/embeddings/__tests__/plugin-index.test.ts +22 -0
- package/src/persistence/embeddings/plugin-index.ts +84 -12
- package/src/persistence/planner-statistics.ts +14 -0
- package/src/plugins/defaults/memory/__tests__/bookmark-crud.test.ts +34 -0
- package/src/tools/browser/pinned-tabs.ts +3 -1
package/eslint.config.mjs
CHANGED
|
@@ -49,14 +49,29 @@ const eslintConfig = defineConfig([
|
|
|
49
49
|
"@typescript-eslint/no-explicit-any": "off",
|
|
50
50
|
},
|
|
51
51
|
},
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
52
|
+
// Two unrelated import bans share one `no-restricted-imports` entry on
|
|
53
|
+
// purpose: flat config replaces rule options rather than merging them, so a
|
|
54
|
+
// second block setting the same rule over overlapping files would silently
|
|
55
|
+
// drop the first one's patterns.
|
|
56
|
+
//
|
|
57
|
+
// 1. Managed-column profile resolution (`getEffectiveProfile(s)`) resolves
|
|
58
|
+
// default profiles against the vellum column only, ignoring
|
|
59
|
+
// `llm.defaultProvider`; on a BYO install that is not the body that
|
|
60
|
+
// dispatches. Runtime code must use `resolveDefaultProfileForProvider` /
|
|
61
|
+
// `getEffectiveProfilesForProvider` instead. The allowlist below is the
|
|
62
|
+
// full set of intentional managed-column consumers: the catalog itself,
|
|
63
|
+
// hatch-time seeding (writes managed stubs by design), and write-path
|
|
64
|
+
// validation.
|
|
65
|
+
//
|
|
66
|
+
// 2. `cli/logger` is the plain-stdout writer for short-lived `assistant …`
|
|
67
|
+
// invocations. It has no level filtering and no log-file routing, so a
|
|
68
|
+
// daemon-side module reaching for it writes debug output straight to a
|
|
69
|
+
// stdout the daemon may not own. Daemon code uses `getLogger()` from
|
|
70
|
+
// `util/logger.js`. The pattern matches on the import specifier, so
|
|
71
|
+
// CLI-internal files (which reach their logger relatively, as
|
|
72
|
+
// `../logger.js`) are unaffected. The companion guard test
|
|
73
|
+
// `src/__tests__/cli-logger-boundary-guard.test.ts` enforces the same
|
|
74
|
+
// boundary by resolved path, independent of how a specifier is spelled.
|
|
60
75
|
{
|
|
61
76
|
files: ["src/**/*.ts"],
|
|
62
77
|
ignores: [
|
|
@@ -77,6 +92,11 @@ const eslintConfig = defineConfig([
|
|
|
77
92
|
message:
|
|
78
93
|
"Managed-column resolution ignores llm.defaultProvider. Use resolveDefaultProfileForProvider / getEffectiveProfilesForProvider with the parsed config's llm.defaultProvider ?? null.",
|
|
79
94
|
},
|
|
95
|
+
{
|
|
96
|
+
group: ["**/cli/logger.js"],
|
|
97
|
+
message:
|
|
98
|
+
"cli/logger writes unfiltered plain text to stdout and is for src/cli/ only. Daemon and shared modules must use getLogger() from util/logger.js so output is level-filtered and routed to the log file. Within src/cli/, import the logger relatively (../logger.js).",
|
|
99
|
+
},
|
|
80
100
|
],
|
|
81
101
|
},
|
|
82
102
|
],
|
package/package.json
CHANGED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join, resolve } from "node:path";
|
|
3
|
+
import { describe, expect, test } from "bun:test";
|
|
4
|
+
|
|
5
|
+
import { Glob } from "bun";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Guard test for the CLI-logger boundary.
|
|
9
|
+
*
|
|
10
|
+
* `src/cli/logger.ts` is the plain-stdout writer for short-lived `assistant …`
|
|
11
|
+
* invocations. It has no level filtering and no log-file routing, so a
|
|
12
|
+
* daemon-side module that reaches for it writes debug output straight to a
|
|
13
|
+
* stdout the daemon may not own. When the daemon runs detached and its
|
|
14
|
+
* spawning parent has closed the read end of the stdout pipe, those writes
|
|
15
|
+
* fail with `EPIPE` and take the process down through the fatal-error handler.
|
|
16
|
+
*
|
|
17
|
+
* Daemon and shared modules use `getLogger()` from `util/logger.js`, which
|
|
18
|
+
* routes to the rotating log file and skips stdout entirely when stdout is not
|
|
19
|
+
* a TTY.
|
|
20
|
+
*
|
|
21
|
+
* `no-restricted-imports` in `eslint.config.mjs` bans this too, but matches on
|
|
22
|
+
* the import specifier. This guard resolves each relative import to a path, so
|
|
23
|
+
* it holds regardless of how the specifier is spelled.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** Tests run from `assistant/`. */
|
|
27
|
+
const ASSISTANT_ROOT = process.cwd();
|
|
28
|
+
const CLI_LOGGER_PATH = resolve(ASSISTANT_ROOT, "src/cli/logger.ts");
|
|
29
|
+
|
|
30
|
+
/** Modules allowed to import the CLI logger: the CLI itself, and tests. */
|
|
31
|
+
function isExempt(relPath: string): boolean {
|
|
32
|
+
return (
|
|
33
|
+
relPath.startsWith("src/cli/") ||
|
|
34
|
+
relPath.includes("/__tests__/") ||
|
|
35
|
+
relPath.endsWith(".test.ts")
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function findCliLoggerImporters(): string[] {
|
|
40
|
+
const pattern = /\b(?:from|import)\s*\(?\s*["'](\.[^"']*logger\.js)["']/g;
|
|
41
|
+
const violations = new Set<string>();
|
|
42
|
+
|
|
43
|
+
for (const relPath of new Glob("src/**/*.ts").scanSync({
|
|
44
|
+
cwd: ASSISTANT_ROOT,
|
|
45
|
+
})) {
|
|
46
|
+
if (isExempt(relPath)) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const filePath = join(ASSISTANT_ROOT, relPath);
|
|
50
|
+
for (const match of readFileSync(filePath, "utf-8").matchAll(pattern)) {
|
|
51
|
+
// Imports carry the compiled `.js` extension; sources are `.ts`.
|
|
52
|
+
const resolved = resolve(
|
|
53
|
+
dirname(filePath),
|
|
54
|
+
match[1]!.replace(/\.js$/, ".ts"),
|
|
55
|
+
);
|
|
56
|
+
if (resolved === CLI_LOGGER_PATH) {
|
|
57
|
+
violations.add(relPath);
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return Array.from(violations).sort();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
describe("CLI logger boundary", () => {
|
|
66
|
+
test("no production module outside src/cli/ imports cli/logger", () => {
|
|
67
|
+
expect(findCliLoggerImporters()).toEqual([]);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("the guard resolves the CLI logger it is guarding", () => {
|
|
71
|
+
// A typo in CLI_LOGGER_PATH would make the guard above vacuously pass.
|
|
72
|
+
expect(readFileSync(CLI_LOGGER_PATH, "utf-8")).toContain("getCliLogger");
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("pinned-tabs logs through the daemon logger", () => {
|
|
76
|
+
// The incident site: pinned-tab mutations run inside daemon host-browser
|
|
77
|
+
// request handling, on every tab pin, select, close, and invalidation.
|
|
78
|
+
const source = readFileSync(
|
|
79
|
+
join(ASSISTANT_ROOT, "src/tools/browser/pinned-tabs.ts"),
|
|
80
|
+
"utf-8",
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
expect(source).toContain('from "../../util/logger.js"');
|
|
84
|
+
expect(source).toContain('getLogger("pinned-tabs")');
|
|
85
|
+
expect(source).not.toContain("cli/logger");
|
|
86
|
+
});
|
|
87
|
+
});
|
|
@@ -52,6 +52,7 @@ afterAll(() => {
|
|
|
52
52
|
});
|
|
53
53
|
|
|
54
54
|
import { setStorePathForTesting } from "../../__tests__/encrypted-store-test-helpers.js";
|
|
55
|
+
import { enableMemoryV3LiveForNewWorkspacesMigration } from "../../workspace/migrations/105-enable-memory-v3-live-for-new-workspaces.js";
|
|
55
56
|
import { invalidateConfigCache, loadConfig } from "../loader.js";
|
|
56
57
|
|
|
57
58
|
// ---------------------------------------------------------------------------
|
|
@@ -221,7 +222,7 @@ describe("deployment-context embedding-provider default (via loadConfig)", () =>
|
|
|
221
222
|
expect(config.memory.qdrant.vectorSize).toBe(384);
|
|
222
223
|
});
|
|
223
224
|
|
|
224
|
-
test("first launch
|
|
225
|
+
test("first launch persists nothing under memory.v3, not even `live`", () => {
|
|
225
226
|
if (existsSync(CONFIG_PATH)) {
|
|
226
227
|
rmSync(CONFIG_PATH, { force: true });
|
|
227
228
|
}
|
|
@@ -233,16 +234,37 @@ describe("deployment-context embedding-provider default (via loadConfig)", () =>
|
|
|
233
234
|
expect(config.memory.v3.gate.denseThreshold).toBe(0.66);
|
|
234
235
|
expect(config.memory.v3.needleK).toBe(100);
|
|
235
236
|
|
|
236
|
-
//
|
|
237
|
-
//
|
|
238
|
-
//
|
|
237
|
+
// Nothing under memory.v3 is frozen to disk. Tuning knobs stay absent so a
|
|
238
|
+
// shipped schema-default change reaches this assistant on its next load
|
|
239
|
+
// (mirrors the embedding-provider strip above), and `live` stays absent so
|
|
240
|
+
// migration 105 can record the initial choice: 105 bails on any value
|
|
241
|
+
// already present, and this seed runs before workspace migrations.
|
|
239
242
|
const raw = readConfig();
|
|
240
243
|
const v3Raw = ((raw.memory as Record<string, unknown>).v3 ?? {}) as Record<
|
|
241
244
|
string,
|
|
242
245
|
unknown
|
|
243
246
|
>;
|
|
244
|
-
expect(Object.keys(v3Raw)).toEqual([
|
|
247
|
+
expect(Object.keys(v3Raw)).toEqual([]);
|
|
245
248
|
expect(v3Raw.gate).toBeUndefined();
|
|
246
249
|
expect(v3Raw.needleK).toBeUndefined();
|
|
247
250
|
});
|
|
251
|
+
|
|
252
|
+
test("migration 105 turns v3 on for a workspace this seed just created", () => {
|
|
253
|
+
if (existsSync(CONFIG_PATH)) {
|
|
254
|
+
rmSync(CONFIG_PATH, { force: true });
|
|
255
|
+
}
|
|
256
|
+
delete process.env.IS_PLATFORM;
|
|
257
|
+
|
|
258
|
+
loadConfig();
|
|
259
|
+
enableMemoryV3LiveForNewWorkspacesMigration.run(WORKSPACE_DIR, {
|
|
260
|
+
isNewWorkspace: true,
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
const raw = readConfig();
|
|
264
|
+
const v3Raw = ((raw.memory as Record<string, unknown>).v3 ?? {}) as Record<
|
|
265
|
+
string,
|
|
266
|
+
unknown
|
|
267
|
+
>;
|
|
268
|
+
expect(v3Raw.live).toBe(true);
|
|
269
|
+
});
|
|
248
270
|
});
|
package/src/config/loader.ts
CHANGED
|
@@ -1119,17 +1119,24 @@ export function loadConfig(): AssistantConfig {
|
|
|
1119
1119
|
// Drop the deployment-context embedding provider so it is never
|
|
1120
1120
|
// persisted (see above); the schema default re-applies in memory.
|
|
1121
1121
|
delete (seed.memory.embeddings as { provider?: unknown }).provider;
|
|
1122
|
-
//
|
|
1123
|
-
//
|
|
1124
|
-
//
|
|
1125
|
-
//
|
|
1126
|
-
//
|
|
1127
|
-
//
|
|
1128
|
-
//
|
|
1129
|
-
//
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1122
|
+
// Nothing under memory.v3 is persisted at seed time.
|
|
1123
|
+
//
|
|
1124
|
+
// The tuning knobs are globally-shipped defaults: freezing them to
|
|
1125
|
+
// disk would stop a shipped default change from reaching this
|
|
1126
|
+
// assistant (mirrors the embedding-provider strip above; migration 119
|
|
1127
|
+
// strips them from configs that already carry them).
|
|
1128
|
+
//
|
|
1129
|
+
// `live` stays absent because an absent leaf is the only way to
|
|
1130
|
+
// express "no decision recorded". This seed runs before workspace
|
|
1131
|
+
// migrations, and migration 105 defers to any value already present
|
|
1132
|
+
// (`if ("live" in v3Config) return`), unable to tell a deliberate
|
|
1133
|
+
// choice from a schema default. Writing the leaf here would pin a
|
|
1134
|
+
// brand-new workspace to the schema default; leaving it absent lets
|
|
1135
|
+
// 105 decide regardless of boot ordering. The schema default still
|
|
1136
|
+
// applies in memory on every load, and a hatch-time
|
|
1137
|
+
// `memory.v3.live=false` override still wins by merging after
|
|
1138
|
+
// migrations.
|
|
1139
|
+
seed.memory.v3 = {} as (typeof seed.memory)["v3"];
|
|
1133
1140
|
// Strip dataDir (runtime-derived) from the persisted config
|
|
1134
1141
|
const { dataDir: _, ...persistable } = seed;
|
|
1135
1142
|
writeFileSync(configPath, JSON.stringify(persistable, null, 2) + "\n");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { desc, eq } from "drizzle-orm";
|
|
1
|
+
import { and, desc, eq, type SQL } from "drizzle-orm";
|
|
2
2
|
import { v4 as uuid } from "uuid";
|
|
3
3
|
|
|
4
4
|
import type { DrizzleDb } from "./db-connection.js";
|
|
@@ -79,21 +79,30 @@ function rowToSummary(row: BookmarkJoinRow): BookmarkSummary {
|
|
|
79
79
|
};
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
/**
|
|
83
|
+
* CROSS JOIN constrains SQLite's join order to start from message_bookmarks,
|
|
84
|
+
* which the planner would otherwise skip in favor of a full messages scan
|
|
85
|
+
* because an empty table never has statistics. The join predicates live in
|
|
86
|
+
* WHERE with identical semantics; callers pass extra conditions through
|
|
87
|
+
* `filter` because chaining `.where()` would replace the predicates.
|
|
88
|
+
*/
|
|
89
|
+
function selectBookmarkJoin(db: DrizzleDb, filter?: SQL) {
|
|
90
|
+
const joinMatch = and(
|
|
91
|
+
eq(messages.id, messageBookmarks.messageId),
|
|
92
|
+
eq(conversations.id, messageBookmarks.conversationId),
|
|
93
|
+
);
|
|
83
94
|
return db
|
|
84
95
|
.select(BOOKMARK_JOIN_COLUMNS)
|
|
85
96
|
.from(messageBookmarks)
|
|
86
|
-
.
|
|
87
|
-
.
|
|
88
|
-
|
|
89
|
-
eq(conversations.id, messageBookmarks.conversationId),
|
|
90
|
-
);
|
|
97
|
+
.crossJoin(messages)
|
|
98
|
+
.crossJoin(conversations)
|
|
99
|
+
.where(filter ? and(joinMatch, filter) : joinMatch);
|
|
91
100
|
}
|
|
92
101
|
|
|
93
102
|
/**
|
|
94
103
|
* List all bookmarks newest-first, joined against `messages` and
|
|
95
104
|
* `conversations`. Bookmarks whose parent message or conversation has
|
|
96
|
-
* been deleted are naturally excluded by the
|
|
105
|
+
* been deleted are naturally excluded by the join predicates; the
|
|
97
106
|
* `ON DELETE CASCADE` on the FKs means rows should never end up in this
|
|
98
107
|
* orphan state, but the join provides a defense-in-depth guarantee.
|
|
99
108
|
*/
|
|
@@ -179,7 +188,7 @@ function readBookmarkSummaryOrThrow(
|
|
|
179
188
|
db: DrizzleDb,
|
|
180
189
|
id: string,
|
|
181
190
|
): BookmarkSummary {
|
|
182
|
-
const row = selectBookmarkJoin(db
|
|
191
|
+
const row = selectBookmarkJoin(db, eq(messageBookmarks.id, id)).get();
|
|
183
192
|
if (!row) {
|
|
184
193
|
// Unreachable: caller just observed (or inserted) this id.
|
|
185
194
|
throw new Error(`Bookmark ${id} disappeared between insert and read`);
|
|
@@ -3,9 +3,10 @@ import { existsSync } from "node:fs";
|
|
|
3
3
|
import { getLogger } from "../util/logger.js";
|
|
4
4
|
import { getDbPath } from "../util/platform.js";
|
|
5
5
|
import { runAsyncSqlite } from "./db-async-query.js";
|
|
6
|
-
import { getDb } from "./db-connection.js";
|
|
6
|
+
import { getDb, getSqliteFrom } from "./db-connection.js";
|
|
7
7
|
import { runMigrationSteps } from "./migrations/run-migrations.js";
|
|
8
8
|
import { validateMigrationState } from "./migrations/validate-migration-state.js";
|
|
9
|
+
import { PLANNER_OPTIMIZE_PRAGMA } from "./planner-statistics.js";
|
|
9
10
|
import { migrationSteps } from "./steps.js";
|
|
10
11
|
|
|
11
12
|
/**
|
|
@@ -101,6 +102,18 @@ export async function initializeDb(): Promise<{ migrationsOk: boolean }> {
|
|
|
101
102
|
"DB migration steps complete",
|
|
102
103
|
);
|
|
103
104
|
|
|
105
|
+
// An index a migration creates has no sqlite_stat1 entry until something
|
|
106
|
+
// analyzes it, which SQLite calls out as a case to handle after a schema
|
|
107
|
+
// change. Only boots that applied a step pay for it, and optimize analyzes
|
|
108
|
+
// just the tables that need it.
|
|
109
|
+
if (applied.length > 0) {
|
|
110
|
+
try {
|
|
111
|
+
getSqliteFrom(database).exec(PLANNER_OPTIMIZE_PRAGMA);
|
|
112
|
+
} catch (err) {
|
|
113
|
+
log.warn({ err }, "Post-migration PRAGMA optimize failed (non-fatal)");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
104
117
|
if (failed.length > 0) {
|
|
105
118
|
log.error(
|
|
106
119
|
{ failedMigrations: failed, count: failed.length },
|
|
@@ -8,6 +8,7 @@ import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
|
|
|
8
8
|
import { getLastInteractiveUserMessageTimestamp } from "./conversation-crud.js";
|
|
9
9
|
import { runAsyncSqlite } from "./db-async-query.js";
|
|
10
10
|
import { getSqlite } from "./db-connection.js";
|
|
11
|
+
import { PLANNER_OPTIMIZE_PRAGMA } from "./planner-statistics.js";
|
|
11
12
|
|
|
12
13
|
const log = getLogger("db-maintenance");
|
|
13
14
|
|
|
@@ -80,11 +81,11 @@ async function runDbMaintenance(): Promise<void> {
|
|
|
80
81
|
log.warn({ err }, "Workflow run pruning failed (non-fatal)");
|
|
81
82
|
}
|
|
82
83
|
|
|
83
|
-
// Refresh the query planner's statistics
|
|
84
|
-
//
|
|
85
|
-
// thread when the sqlite3 CLI backend is available.
|
|
84
|
+
// Refresh the query planner's statistics (see PLANNER_OPTIMIZE_PRAGMA for
|
|
85
|
+
// what the mask buys here). Routed through the async path to keep it off the
|
|
86
|
+
// main thread when the sqlite3 CLI backend is available.
|
|
86
87
|
const optimizeResult = await runAsyncSqlite(
|
|
87
|
-
|
|
88
|
+
PLANNER_OPTIMIZE_PRAGMA,
|
|
88
89
|
"db-maintenance:optimize",
|
|
89
90
|
);
|
|
90
91
|
if (!optimizeResult.ok) {
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type { SparseEmbedding } from "../embedding-types.js";
|
|
4
|
+
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// Regression: the plugin index must not depend on the memory tier — or on the
|
|
7
|
+
// daemon — having initialized the shared Qdrant client.
|
|
8
|
+
//
|
|
9
|
+
// `runMemoryStartup` calls `initQdrantClient` only in the daemon process, and
|
|
10
|
+
// only while memory v1 is the live tier; on a default workspace
|
|
11
|
+
// (`memory.v2.enabled` defaults true, or `memory.v3.live`) it is skipped, so
|
|
12
|
+
// `getQdrantClient()` throws and every plugin Index op used to fail with
|
|
13
|
+
// `BackendUnavailableError: Qdrant client not initialized`. The ops must
|
|
14
|
+
// initialize the client themselves from the live workspace config, and rebuild
|
|
15
|
+
// it when that config changes rather than serving a stale one.
|
|
16
|
+
//
|
|
17
|
+
// The fake singleton below behaves like the real one — absent until
|
|
18
|
+
// `initQdrantClient` sets it — where `plugin-index.test.ts` mocks a client that
|
|
19
|
+
// is already up. Each test file runs in its own Bun process (see
|
|
20
|
+
// scripts/test.ts), so the two mocks cannot collide.
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
interface InitCall {
|
|
24
|
+
url: string;
|
|
25
|
+
collection: string;
|
|
26
|
+
vectorSize: number;
|
|
27
|
+
onDisk: boolean;
|
|
28
|
+
quantization: string;
|
|
29
|
+
embeddingModel?: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const calls = {
|
|
33
|
+
init: [] as InitCall[],
|
|
34
|
+
upserts: [] as Array<{ targetType: string; targetId: string }>,
|
|
35
|
+
get: [] as Array<{ targetId: string }>,
|
|
36
|
+
deleteScoped: [] as Array<{ targetId: string }>,
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const fakeQdrant = {
|
|
40
|
+
async upsert(targetType: string, targetId: string) {
|
|
41
|
+
calls.upserts.push({ targetType, targetId });
|
|
42
|
+
},
|
|
43
|
+
async getByTarget(_targetType: string, targetId: string) {
|
|
44
|
+
calls.get.push({ targetId });
|
|
45
|
+
return null;
|
|
46
|
+
},
|
|
47
|
+
async deleteByTargetAndPlugin(_targetType: string, targetId: string) {
|
|
48
|
+
calls.deleteScoped.push({ targetId });
|
|
49
|
+
},
|
|
50
|
+
async hybridSearch() {
|
|
51
|
+
return [];
|
|
52
|
+
},
|
|
53
|
+
async search() {
|
|
54
|
+
return [];
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/** Stands in for the module singleton in `qdrant-client.ts`: null until set. */
|
|
59
|
+
let instance: typeof fakeQdrant | null = null;
|
|
60
|
+
|
|
61
|
+
mock.module("../qdrant-client.js", () => ({
|
|
62
|
+
getQdrantClient: () => {
|
|
63
|
+
if (!instance) {
|
|
64
|
+
throw new Error("Qdrant client not initialized. Call initQdrantClient()");
|
|
65
|
+
}
|
|
66
|
+
return instance;
|
|
67
|
+
},
|
|
68
|
+
initQdrantClient: (cfg: InitCall) => {
|
|
69
|
+
calls.init.push(cfg);
|
|
70
|
+
instance = fakeQdrant;
|
|
71
|
+
return instance;
|
|
72
|
+
},
|
|
73
|
+
resolveQdrantUrl: () => "http://127.0.0.1:7777",
|
|
74
|
+
}));
|
|
75
|
+
mock.module("../qdrant-circuit-breaker.js", () => ({
|
|
76
|
+
withQdrantBreaker: <T>(fn: () => Promise<T>) => fn(),
|
|
77
|
+
}));
|
|
78
|
+
mock.module("../embed.js", () => ({
|
|
79
|
+
embedWithRetry: async () => ({
|
|
80
|
+
provider: "gemini",
|
|
81
|
+
model: "test-embed-model",
|
|
82
|
+
vectors: [[0.1, 0.2, 0.3]],
|
|
83
|
+
}),
|
|
84
|
+
}));
|
|
85
|
+
mock.module("../embedding-backend.js", () => ({
|
|
86
|
+
selectEmbeddingBackend: async () => ({
|
|
87
|
+
backend: { provider: "gemini", model: liveConfig.memory.embeddings.model },
|
|
88
|
+
reason: null,
|
|
89
|
+
}),
|
|
90
|
+
getMemoryBackendStatus: async () => ({
|
|
91
|
+
enabled: true,
|
|
92
|
+
degraded: false,
|
|
93
|
+
provider: "gemini",
|
|
94
|
+
model: liveConfig.memory.embeddings.model,
|
|
95
|
+
reason: null,
|
|
96
|
+
}),
|
|
97
|
+
generateSparseEmbedding: (): SparseEmbedding => ({
|
|
98
|
+
indices: [1, 2],
|
|
99
|
+
values: [0.5, 0.5],
|
|
100
|
+
}),
|
|
101
|
+
}));
|
|
102
|
+
|
|
103
|
+
/** The workspace config `getConfig()` returns; mutated to simulate a reload. */
|
|
104
|
+
let liveConfig = {
|
|
105
|
+
memory: {
|
|
106
|
+
qdrant: {
|
|
107
|
+
collection: "vellum_memory",
|
|
108
|
+
vectorSize: 768,
|
|
109
|
+
onDisk: true,
|
|
110
|
+
quantization: "scalar",
|
|
111
|
+
},
|
|
112
|
+
embeddings: { provider: "gemini", model: "test-embed-model" },
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
mock.module("../../../config/loader.js", () => ({
|
|
117
|
+
getConfig: () => liveConfig,
|
|
118
|
+
}));
|
|
119
|
+
|
|
120
|
+
const { indexDocument, queryIndex, getDocument, removeDocument } =
|
|
121
|
+
await import("../plugin-index.js");
|
|
122
|
+
|
|
123
|
+
beforeEach(() => {
|
|
124
|
+
calls.init.length = 0;
|
|
125
|
+
calls.upserts.length = 0;
|
|
126
|
+
calls.get.length = 0;
|
|
127
|
+
calls.deleteScoped.length = 0;
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
afterAll(() => {
|
|
131
|
+
mock.restore();
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
describe("plugin index without a pre-initialized Qdrant client", () => {
|
|
135
|
+
test("indexDocument initializes the client from config instead of throwing", async () => {
|
|
136
|
+
const res = await indexDocument(
|
|
137
|
+
liveConfig as never,
|
|
138
|
+
"ledger",
|
|
139
|
+
"Morning coffee",
|
|
140
|
+
);
|
|
141
|
+
|
|
142
|
+
expect(res.documentId).toBeTruthy();
|
|
143
|
+
expect(calls.upserts).toEqual([
|
|
144
|
+
{ targetType: "plugin_index", targetId: `ledger:${res.documentId}` },
|
|
145
|
+
]);
|
|
146
|
+
// Same shared collection, same dimension, and the dense model identity so
|
|
147
|
+
// the collection keeps its create/migrate semantics.
|
|
148
|
+
expect(calls.init).toEqual([
|
|
149
|
+
{
|
|
150
|
+
url: "http://127.0.0.1:7777",
|
|
151
|
+
collection: "vellum_memory",
|
|
152
|
+
vectorSize: 768,
|
|
153
|
+
onDisk: true,
|
|
154
|
+
quantization: "scalar",
|
|
155
|
+
embeddingModel: "gemini:test-embed-model",
|
|
156
|
+
},
|
|
157
|
+
]);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("query/get/remove also work without a pre-initialized client", async () => {
|
|
161
|
+
await expect(
|
|
162
|
+
queryIndex(liveConfig as never, "ledger", "coffee"),
|
|
163
|
+
).resolves.toEqual([]);
|
|
164
|
+
await expect(getDocument("ledger", "doc-1")).resolves.toBeNull();
|
|
165
|
+
await expect(removeDocument("ledger", "doc-1")).resolves.toBeUndefined();
|
|
166
|
+
|
|
167
|
+
expect(calls.get).toEqual([{ targetId: "ledger:doc-1" }]);
|
|
168
|
+
expect(calls.deleteScoped).toEqual([{ targetId: "ledger:doc-1" }]);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("an unchanged config reuses the client instead of rebuilding it", async () => {
|
|
172
|
+
await indexDocument(liveConfig as never, "ledger", "one");
|
|
173
|
+
await indexDocument(liveConfig as never, "ledger", "two");
|
|
174
|
+
await getDocument("ledger", "doc-1");
|
|
175
|
+
|
|
176
|
+
expect(calls.upserts).toHaveLength(2);
|
|
177
|
+
expect(calls.init).toHaveLength(0); // already built by an earlier test
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("a live embedding-config change rebuilds the client", async () => {
|
|
181
|
+
await indexDocument(liveConfig as never, "ledger", "before");
|
|
182
|
+
expect(calls.init).toHaveLength(0);
|
|
183
|
+
|
|
184
|
+
// A model swap at the same dimension: the client's captured model identity
|
|
185
|
+
// is now wrong, so reusing it would mix two models' vectors in one
|
|
186
|
+
// collection without tripping the sentinel migration.
|
|
187
|
+
liveConfig = {
|
|
188
|
+
memory: {
|
|
189
|
+
...liveConfig.memory,
|
|
190
|
+
embeddings: { provider: "gemini", model: "swapped-embed-model" },
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
await indexDocument(liveConfig as never, "ledger", "after");
|
|
194
|
+
|
|
195
|
+
expect(calls.init).toEqual([
|
|
196
|
+
{
|
|
197
|
+
url: "http://127.0.0.1:7777",
|
|
198
|
+
collection: "vellum_memory",
|
|
199
|
+
vectorSize: 768,
|
|
200
|
+
onDisk: true,
|
|
201
|
+
quantization: "scalar",
|
|
202
|
+
embeddingModel: "gemini:swapped-embed-model",
|
|
203
|
+
},
|
|
204
|
+
]);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
test("a dimension change rebuilds the client", async () => {
|
|
208
|
+
liveConfig = {
|
|
209
|
+
memory: {
|
|
210
|
+
...liveConfig.memory,
|
|
211
|
+
qdrant: { ...liveConfig.memory.qdrant, vectorSize: 1536 },
|
|
212
|
+
},
|
|
213
|
+
};
|
|
214
|
+
await indexDocument(liveConfig as never, "ledger", "resized");
|
|
215
|
+
|
|
216
|
+
expect(calls.init).toHaveLength(1);
|
|
217
|
+
expect(calls.init[0].vectorSize).toBe(1536);
|
|
218
|
+
});
|
|
219
|
+
});
|
|
@@ -123,8 +123,26 @@ const fakeQdrant = {
|
|
|
123
123
|
},
|
|
124
124
|
};
|
|
125
125
|
|
|
126
|
+
// The client is already initialized in this process, so the ops resolve it
|
|
127
|
+
// straight from the singleton. `plugin-index-qdrant-init.test.ts` covers the
|
|
128
|
+
// other side: a process where it is not.
|
|
126
129
|
mock.module("../qdrant-client.js", () => ({
|
|
127
130
|
getQdrantClient: () => fakeQdrant,
|
|
131
|
+
initQdrantClient: () => fakeQdrant,
|
|
132
|
+
resolveQdrantUrl: () => "http://127.0.0.1:6333",
|
|
133
|
+
}));
|
|
134
|
+
mock.module("../../../config/loader.js", () => ({
|
|
135
|
+
getConfig: () => ({
|
|
136
|
+
memory: {
|
|
137
|
+
qdrant: {
|
|
138
|
+
collection: "vellum_memory",
|
|
139
|
+
vectorSize: 3,
|
|
140
|
+
onDisk: true,
|
|
141
|
+
quantization: "none",
|
|
142
|
+
},
|
|
143
|
+
embeddings: { provider: "gemini" },
|
|
144
|
+
},
|
|
145
|
+
}),
|
|
128
146
|
}));
|
|
129
147
|
mock.module("../qdrant-circuit-breaker.js", () => ({
|
|
130
148
|
withQdrantBreaker: <T>(fn: () => Promise<T>) => fn(),
|
|
@@ -137,6 +155,10 @@ mock.module("../embed.js", () => ({
|
|
|
137
155
|
}),
|
|
138
156
|
}));
|
|
139
157
|
mock.module("../embedding-backend.js", () => ({
|
|
158
|
+
selectEmbeddingBackend: async () => ({
|
|
159
|
+
backend: { provider: "gemini", model: "test-embed-model" },
|
|
160
|
+
reason: null,
|
|
161
|
+
}),
|
|
140
162
|
getMemoryBackendStatus: async () => ({
|
|
141
163
|
enabled: true,
|
|
142
164
|
degraded: false,
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
|
|
30
30
|
import { randomUUID } from "node:crypto";
|
|
31
31
|
|
|
32
|
+
import { getConfig } from "../../config/loader.js";
|
|
32
33
|
import type { AssistantConfig } from "../../config/types.js";
|
|
33
34
|
import { BackendUnavailableError } from "../../util/errors.js";
|
|
34
35
|
import { getLogger } from "../../util/logger.js";
|
|
@@ -36,6 +37,7 @@ import { embedWithRetry } from "./embed.js";
|
|
|
36
37
|
import {
|
|
37
38
|
generateSparseEmbedding,
|
|
38
39
|
getMemoryBackendStatus,
|
|
40
|
+
selectEmbeddingBackend,
|
|
39
41
|
} from "./embedding-backend.js";
|
|
40
42
|
import {
|
|
41
43
|
type EmbeddingInput,
|
|
@@ -43,7 +45,12 @@ import {
|
|
|
43
45
|
type SparseEmbedding,
|
|
44
46
|
} from "./embedding-types.js";
|
|
45
47
|
import { withQdrantBreaker } from "./qdrant-circuit-breaker.js";
|
|
46
|
-
import {
|
|
48
|
+
import {
|
|
49
|
+
getQdrantClient,
|
|
50
|
+
initQdrantClient,
|
|
51
|
+
resolveQdrantUrl,
|
|
52
|
+
type VellumQdrantClient,
|
|
53
|
+
} from "./qdrant-client.js";
|
|
47
54
|
|
|
48
55
|
const log = getLogger("plugin-index");
|
|
49
56
|
|
|
@@ -124,13 +131,78 @@ export interface IndexedDocument {
|
|
|
124
131
|
|
|
125
132
|
// ── Internal helpers ───────────────────────────────────────────────────────
|
|
126
133
|
|
|
127
|
-
/**
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
134
|
+
/**
|
|
135
|
+
* Config fingerprint of the client {@link resolveQdrant} last built, so a live
|
|
136
|
+
* config change is not served by a stale client. `VellumQdrantClient` captures
|
|
137
|
+
* its collection, dimension, and model identity at construction, so a client
|
|
138
|
+
* built before the change would keep writing at the old dimension (rejected by
|
|
139
|
+
* a resized collection) or quietly mix vectors from two models into one
|
|
140
|
+
* collection without triggering the sentinel migration.
|
|
141
|
+
*/
|
|
142
|
+
let clientConfigIdentity: string | null = null;
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* The Qdrant client for the shared collection, initializing it from the live
|
|
146
|
+
* workspace config when this process has not.
|
|
147
|
+
*
|
|
148
|
+
* The eager `initQdrantClient` in `runMemoryStartup`
|
|
149
|
+
* (`plugins/defaults/memory/startup.ts`) cannot be relied on here: it runs in
|
|
150
|
+
* the daemon process only, and only while memory v1 is the live tier — on a
|
|
151
|
+
* default workspace (`memory.v2.enabled`, or `memory.v3.live`) it is skipped
|
|
152
|
+
* entirely, because no v1 lane reads or writes the collection in that state.
|
|
153
|
+
* The plugin index is not a memory tier: it is plugin-owned search that must
|
|
154
|
+
* work on every tier and in any process that runs plugin code, so it resolves
|
|
155
|
+
* the client itself rather than depending on a memory-tier decision. Mirrors
|
|
156
|
+
* `resolveLexicalIndex` in `persistence/job-handlers/message-lexical.ts`.
|
|
157
|
+
*
|
|
158
|
+
* Cheap: `initQdrantClient` only constructs a client — the collection is
|
|
159
|
+
* created lazily inside each operation — and the steady-state path is a
|
|
160
|
+
* fingerprint comparison, with the embedding-backend lookup reached only when
|
|
161
|
+
* the config it derives from has actually changed.
|
|
162
|
+
*
|
|
163
|
+
* The dense embedding identity is passed through so a collection created or
|
|
164
|
+
* reused from here keeps the same model-sentinel semantics as the v1 path: a
|
|
165
|
+
* model or dimension change recreates the collection, which is the durability
|
|
166
|
+
* contract documented at the top of this file.
|
|
167
|
+
*/
|
|
168
|
+
async function resolveQdrant(): Promise<VellumQdrantClient> {
|
|
169
|
+
const config = getConfig();
|
|
170
|
+
const url = resolveQdrantUrl(config);
|
|
171
|
+
const { collection, vectorSize, onDisk, quantization } = config.memory.qdrant;
|
|
172
|
+
// `memory.embeddings` rather than the resolved backend: it is what the
|
|
173
|
+
// resolution below reads, and comparing it keeps that lookup off the hot
|
|
174
|
+
// path. Over-sensitive by a field or two (a `required` flip re-inits), which
|
|
175
|
+
// costs one redundant construction and never a wrong client.
|
|
176
|
+
const identity = JSON.stringify([
|
|
177
|
+
url,
|
|
178
|
+
collection,
|
|
179
|
+
vectorSize,
|
|
180
|
+
onDisk,
|
|
181
|
+
quantization,
|
|
182
|
+
config.memory.embeddings,
|
|
183
|
+
]);
|
|
184
|
+
|
|
185
|
+
if (identity === clientConfigIdentity) {
|
|
186
|
+
try {
|
|
187
|
+
return getQdrantClient();
|
|
188
|
+
} catch {
|
|
189
|
+
// Fingerprint outlived the singleton (another module reset it) — rebuild.
|
|
190
|
+
}
|
|
133
191
|
}
|
|
192
|
+
|
|
193
|
+
const selection = await selectEmbeddingBackend(config);
|
|
194
|
+
const client = initQdrantClient({
|
|
195
|
+
url,
|
|
196
|
+
collection,
|
|
197
|
+
vectorSize,
|
|
198
|
+
onDisk,
|
|
199
|
+
quantization,
|
|
200
|
+
embeddingModel: selection.backend
|
|
201
|
+
? `${selection.backend.provider}:${selection.backend.model}`
|
|
202
|
+
: undefined,
|
|
203
|
+
});
|
|
204
|
+
clientConfigIdentity = identity;
|
|
205
|
+
return client;
|
|
134
206
|
}
|
|
135
207
|
|
|
136
208
|
/** Embed a single input to a dense vector, failing loudly if no backend is up. */
|
|
@@ -208,7 +280,7 @@ export async function indexDocument(
|
|
|
208
280
|
const normalized = normalizeEmbeddingInput(input);
|
|
209
281
|
const documentId = opts?.documentId ?? randomUUID();
|
|
210
282
|
const now = opts?.createdAt ?? Date.now();
|
|
211
|
-
const qdrant =
|
|
283
|
+
const qdrant = await resolveQdrant();
|
|
212
284
|
|
|
213
285
|
await withQdrantBreaker(() =>
|
|
214
286
|
qdrant.upsert(
|
|
@@ -249,7 +321,7 @@ export async function queryIndex(
|
|
|
249
321
|
const embedding = await embedDense(config, query);
|
|
250
322
|
const sparseVector = sparseFor(query);
|
|
251
323
|
const limit = opts?.limit ?? 10;
|
|
252
|
-
const qdrant =
|
|
324
|
+
const qdrant = await resolveQdrant();
|
|
253
325
|
|
|
254
326
|
const filter = {
|
|
255
327
|
must: [
|
|
@@ -290,7 +362,7 @@ export async function getDocument(
|
|
|
290
362
|
plugin: string,
|
|
291
363
|
documentId: string,
|
|
292
364
|
): Promise<IndexedDocument | null> {
|
|
293
|
-
const qdrant =
|
|
365
|
+
const qdrant = await resolveQdrant();
|
|
294
366
|
const found = await withQdrantBreaker(() =>
|
|
295
367
|
qdrant.getByTarget(
|
|
296
368
|
PLUGIN_INDEX_TARGET_TYPE,
|
|
@@ -315,7 +387,7 @@ export async function removeDocument(
|
|
|
315
387
|
plugin: string,
|
|
316
388
|
documentId: string,
|
|
317
389
|
): Promise<void> {
|
|
318
|
-
const qdrant =
|
|
390
|
+
const qdrant = await resolveQdrant();
|
|
319
391
|
await withQdrantBreaker(() =>
|
|
320
392
|
qdrant.deleteByTargetAndPlugin(
|
|
321
393
|
PLUGIN_INDEX_TARGET_TYPE,
|
|
@@ -333,7 +405,7 @@ export async function removeDocument(
|
|
|
333
405
|
*/
|
|
334
406
|
export async function purgeEmbeddingsForPlugin(plugin: string): Promise<void> {
|
|
335
407
|
try {
|
|
336
|
-
const qdrant =
|
|
408
|
+
const qdrant = await resolveQdrant();
|
|
337
409
|
await withQdrantBreaker(() => qdrant.deleteByPlugin(plugin));
|
|
338
410
|
} catch (err) {
|
|
339
411
|
log.warn({ err, plugin }, "Failed to purge plugin embeddings");
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `PRAGMA optimize` mask run by the daily maintenance sweep and by
|
|
3
|
+
* `db-init` after a boot that applied migrations.
|
|
4
|
+
*
|
|
5
|
+
* Both run on connections with no useful query history, so `0x10000` is what
|
|
6
|
+
* lets optimize look past tables missing from `sqlite_stat1` and reconsider
|
|
7
|
+
* ones whose entries have gone stale. `0x2` keeps the default "analyze where
|
|
8
|
+
* it helps" behavior and `0x10` bounds each ANALYZE with a temporary
|
|
9
|
+
* `analysis_limit`. Releases before 3.46 ignore bits they do not recognize
|
|
10
|
+
* rather than failing, so the mask is safe to send to any version.
|
|
11
|
+
*
|
|
12
|
+
* https://sqlite.org/lang_analyze.html#recommended_usage_patterns
|
|
13
|
+
*/
|
|
14
|
+
export const PLANNER_OPTIMIZE_PRAGMA = "PRAGMA optimize=0x10012";
|
|
@@ -261,4 +261,38 @@ describe("bookmark-crud", () => {
|
|
|
261
261
|
/Message ghost not found/,
|
|
262
262
|
);
|
|
263
263
|
});
|
|
264
|
+
|
|
265
|
+
test("listBookmarks drives the join from message_bookmarks even with planner stats that predate the table", () => {
|
|
266
|
+
const sqlite = new Database(":memory:");
|
|
267
|
+
sqlite.exec("PRAGMA foreign_keys = ON");
|
|
268
|
+
const captured: string[] = [];
|
|
269
|
+
const db = drizzle(sqlite, {
|
|
270
|
+
schema,
|
|
271
|
+
logger: { logQuery: (query: string) => captured.push(query) },
|
|
272
|
+
});
|
|
273
|
+
const raw = getSqliteFrom(db);
|
|
274
|
+
bootstrapMessageTables(raw);
|
|
275
|
+
migrateMessageBookmarks(db);
|
|
276
|
+
for (let i = 0; i < 30; i++) {
|
|
277
|
+
seedConversationAndMessage(raw, {
|
|
278
|
+
conversationId: `conv-${i}`,
|
|
279
|
+
messageId: `msg-${i}`,
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
// ANALYZE writes no sqlite_stat1 rows for an empty table, leaving
|
|
283
|
+
// message_bookmarks without statistics while messages gets them. This is
|
|
284
|
+
// the production state that made the planner scan messages.
|
|
285
|
+
raw.exec("ANALYZE");
|
|
286
|
+
|
|
287
|
+
listBookmarks(db);
|
|
288
|
+
const sql = captured.at(-1);
|
|
289
|
+
expect(sql).toBeDefined();
|
|
290
|
+
const plan = raw.query(`EXPLAIN QUERY PLAN ${sql}`).all() as Array<{
|
|
291
|
+
detail: string;
|
|
292
|
+
}>;
|
|
293
|
+
expect(plan[0]?.detail ?? "").toMatch(/^SCAN message_bookmarks/);
|
|
294
|
+
expect(plan.some((row) => row.detail.includes("SCAN messages"))).toBe(
|
|
295
|
+
false,
|
|
296
|
+
);
|
|
297
|
+
});
|
|
264
298
|
});
|
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
* `host_browser_session_invalidated`.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import {
|
|
17
|
+
import { getLogger } from "../../util/logger.js";
|
|
18
|
+
|
|
19
|
+
const log = getLogger("pinned-tabs");
|
|
18
20
|
|
|
19
21
|
const pinnedTabs = new Map<string, Map<string, string>>();
|
|
20
22
|
|