@vellumai/assistant 0.10.4-dev.202607010541.092b857 → 0.10.4-dev.202607010743.70cd9f0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +1 -0
- package/src/plugins/defaults/memory/embeddings.ts +77 -0
- package/src/plugins/defaults/memory/graph/bootstrap.ts +3 -5
- package/src/plugins/defaults/memory/graph/extraction.ts +1 -1
- package/src/plugins/defaults/memory/graph/graph-search.ts +12 -23
- package/src/plugins/defaults/memory/graph/retriever.ts +3 -5
- package/src/plugins/defaults/memory/indexer.ts +2 -3
- package/src/plugins/defaults/memory/job-handlers/embedding.test.ts +6 -19
- package/src/plugins/defaults/memory/job-handlers/embedding.ts +9 -22
- package/src/plugins/defaults/memory/job-handlers/index-maintenance.ts +2 -3
- package/src/plugins/defaults/memory/job-handlers/index-message-lexical.ts +2 -2
- package/src/plugins/defaults/memory/job-handlers.ts +5 -5
- package/src/plugins/defaults/memory/jobs/embed-concept-page.ts +1 -1
- package/src/plugins/defaults/memory/pkb/pkb-index.ts +1 -5
- package/src/plugins/defaults/memory/routes/memory-item-routes.ts +1 -1
- package/src/plugins/defaults/memory/startup.ts +3 -5
- package/src/plugins/defaults/memory/v2/activation.ts +2 -4
- package/src/plugins/defaults/memory/v2/cli-command-store.ts +2 -4
- package/src/plugins/defaults/memory/v2/qdrant.ts +2 -3
- package/src/plugins/defaults/memory/v2/sim.ts +2 -4
- package/src/plugins/defaults/memory/v2/skill-store.ts +2 -4
- package/src/plugins/defaults/memory/v3/dense.ts +7 -12
- package/src/plugins/defaults/memory/v3/maintain-job.ts +2 -4
- package/src/plugins/defaults/memory/v3/section-dense-store.ts +8 -11
package/package.json
CHANGED
|
@@ -187,6 +187,7 @@ const BASELINE: Record<string, readonly string[]> = {
|
|
|
187
187
|
"../../../persistence/embeddings/messages-lexical-index.js",
|
|
188
188
|
"../../../persistence/embeddings/qdrant-client.js",
|
|
189
189
|
"../../../persistence/embeddings/qdrant-manager.js",
|
|
190
|
+
"../../../persistence/job-utils.js",
|
|
190
191
|
"../../../persistence/jobs-store.js",
|
|
191
192
|
"../../../persistence/jobs-worker.js",
|
|
192
193
|
"../../../persistence/memory-lifecycle-hooks.js",
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { getConfig } from "../../../config/loader.js";
|
|
2
|
+
import type { AssistantConfig } from "../../../config/types.js";
|
|
3
|
+
import type {
|
|
4
|
+
EmbeddingInput,
|
|
5
|
+
EmbeddingRequestOptions,
|
|
6
|
+
} from "../../../persistence/embeddings/embedding-types.js";
|
|
7
|
+
import { resolveQdrantUrl as resolveQdrantUrlWithConfig } from "../../../persistence/embeddings/qdrant-client.js";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Memory-internal accessor for the shared embeddings / vector subsystem.
|
|
11
|
+
*
|
|
12
|
+
* The embeddings operations live in `persistence/embeddings` (and
|
|
13
|
+
* `persistence/job-utils`) and take the full `AssistantConfig` as their first
|
|
14
|
+
* argument — they read both the `memory` slice (Qdrant collection and vector
|
|
15
|
+
* settings) and the `llm` slice (embedding-backend selection). Memory code
|
|
16
|
+
* reaches those operations through this accessor: the self-contained ones
|
|
17
|
+
* (`embedAndUpsert`, `selectedBackendSupportsMultimodal`, `resolveQdrantUrl`)
|
|
18
|
+
* resolve the live config internally, so their call sites need not hold it.
|
|
19
|
+
* `embedWithBackend` keeps its `config` parameter — it is a primitive whose
|
|
20
|
+
* vectors the caller stores alongside its own config-derived metadata (cache
|
|
21
|
+
* keys, vector-size checks, Qdrant collection), so it must embed with the
|
|
22
|
+
* caller's exact config snapshot rather than a re-read that could diverge from
|
|
23
|
+
* that metadata mid-operation.
|
|
24
|
+
*
|
|
25
|
+
* The embed operations are loaded via dynamic `import()` inside each wrapper so
|
|
26
|
+
* that importing this module for a single operation does not eagerly pull the
|
|
27
|
+
* whole embed/vector import graph (`job-utils`, `embedding-backend`) into the
|
|
28
|
+
* consumer. An eager pull would force every one of those modules' named exports
|
|
29
|
+
* to resolve at instantiation, which breaks the intentional partial module
|
|
30
|
+
* mocks in memory tests (a store that mocks only the exports it uses). Only the
|
|
31
|
+
* synchronous `resolveQdrantUrl` is imported statically.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
type EmbeddingTargetType = Parameters<
|
|
35
|
+
typeof import("../../../persistence/job-utils.js").embedAndUpsert
|
|
36
|
+
>[1];
|
|
37
|
+
|
|
38
|
+
/** Embed a target and upsert its vector to Qdrant. */
|
|
39
|
+
export async function embedAndUpsert(
|
|
40
|
+
targetType: EmbeddingTargetType,
|
|
41
|
+
targetId: string,
|
|
42
|
+
input: EmbeddingInput,
|
|
43
|
+
extraPayload?: Record<string, unknown>,
|
|
44
|
+
): Promise<void> {
|
|
45
|
+
const { embedAndUpsert: withConfig } =
|
|
46
|
+
await import("../../../persistence/job-utils.js");
|
|
47
|
+
return withConfig(getConfig(), targetType, targetId, input, extraPayload);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Embed one or more inputs via the selected embedding backend. */
|
|
51
|
+
export async function embedWithBackend(
|
|
52
|
+
config: AssistantConfig,
|
|
53
|
+
inputs: EmbeddingInput[],
|
|
54
|
+
options?: EmbeddingRequestOptions,
|
|
55
|
+
): Promise<
|
|
56
|
+
Awaited<
|
|
57
|
+
ReturnType<
|
|
58
|
+
typeof import("../../../persistence/embeddings/embedding-backend.js").embedWithBackend
|
|
59
|
+
>
|
|
60
|
+
>
|
|
61
|
+
> {
|
|
62
|
+
const { embedWithBackend: withConfig } =
|
|
63
|
+
await import("../../../persistence/embeddings/embedding-backend.js");
|
|
64
|
+
return withConfig(config, inputs, options);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Whether the active embedding backend handles multimodal inputs. */
|
|
68
|
+
export async function selectedBackendSupportsMultimodal(): Promise<boolean> {
|
|
69
|
+
const { selectedBackendSupportsMultimodal: withConfig } =
|
|
70
|
+
await import("../../../persistence/embeddings/embedding-backend.js");
|
|
71
|
+
return withConfig(getConfig());
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Resolve the Qdrant base URL for this process. */
|
|
75
|
+
export function resolveQdrantUrl(): string {
|
|
76
|
+
return resolveQdrantUrlWithConfig(getConfig());
|
|
77
|
+
}
|
|
@@ -21,10 +21,7 @@ import {
|
|
|
21
21
|
setMemoryCheckpoint,
|
|
22
22
|
} from "../../../../persistence/checkpoints.js";
|
|
23
23
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
24
|
-
import {
|
|
25
|
-
initQdrantClient,
|
|
26
|
-
resolveQdrantUrl,
|
|
27
|
-
} from "../../../../persistence/embeddings/qdrant-client.js";
|
|
24
|
+
import { initQdrantClient } from "../../../../persistence/embeddings/qdrant-client.js";
|
|
28
25
|
import {
|
|
29
26
|
enqueueMemoryJob,
|
|
30
27
|
hasActiveJobOfType,
|
|
@@ -38,6 +35,7 @@ import {
|
|
|
38
35
|
} from "../../../../persistence/schema/index.js";
|
|
39
36
|
import { getLogger } from "../../../../util/logger.js";
|
|
40
37
|
import { getWorkspaceDir } from "../../../../util/platform.js";
|
|
38
|
+
import { resolveQdrantUrl } from "../embeddings.js";
|
|
41
39
|
import { runGraphExtraction } from "./extraction.js";
|
|
42
40
|
import { countNodes } from "./store.js";
|
|
43
41
|
|
|
@@ -86,7 +84,7 @@ export async function bootstrapFromHistory(
|
|
|
86
84
|
// Initialize Qdrant client for inline embedding
|
|
87
85
|
try {
|
|
88
86
|
initQdrantClient({
|
|
89
|
-
url: resolveQdrantUrl(
|
|
87
|
+
url: resolveQdrantUrl(),
|
|
90
88
|
collection: config.memory.qdrant.collection,
|
|
91
89
|
vectorSize: config.memory.qdrant.vectorSize,
|
|
92
90
|
onDisk: config.memory.qdrant.onDisk ?? true,
|
|
@@ -1157,7 +1157,7 @@ export async function runGraphExtraction(
|
|
|
1157
1157
|
const { embedGraphNodeDirect } = await import("./graph-search.js");
|
|
1158
1158
|
for (const node of createdNodes) {
|
|
1159
1159
|
try {
|
|
1160
|
-
await embedGraphNodeDirect(node
|
|
1160
|
+
await embedGraphNodeDirect(node);
|
|
1161
1161
|
} catch (err) {
|
|
1162
1162
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1163
1163
|
log.warn(
|
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
|
|
5
5
|
import { getConfig } from "../../../../config/loader.js";
|
|
6
6
|
import type { AssistantConfig } from "../../../../config/types.js";
|
|
7
|
-
import { selectedBackendSupportsMultimodal } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
8
7
|
import type { EmbeddingInput } from "../../../../persistence/embeddings/embedding-types.js";
|
|
9
8
|
import { isQdrantBreakerOpen } from "../../../../persistence/embeddings/qdrant-circuit-breaker.js";
|
|
10
9
|
import { withQdrantBreaker } from "../../../../persistence/embeddings/qdrant-circuit-breaker.js";
|
|
@@ -13,7 +12,6 @@ import {
|
|
|
13
12
|
type QdrantSearchResult,
|
|
14
13
|
type QdrantSparseVector,
|
|
15
14
|
} from "../../../../persistence/embeddings/qdrant-client.js";
|
|
16
|
-
import { embedAndUpsert } from "../../../../persistence/job-utils.js";
|
|
17
15
|
import { asString } from "../../../../persistence/job-utils.js";
|
|
18
16
|
import {
|
|
19
17
|
enqueueMemoryJob,
|
|
@@ -21,6 +19,10 @@ import {
|
|
|
21
19
|
type MemoryJob,
|
|
22
20
|
} from "../../../../persistence/jobs-store.js";
|
|
23
21
|
import { getLogger } from "../../../../util/logger.js";
|
|
22
|
+
import {
|
|
23
|
+
embedAndUpsert,
|
|
24
|
+
selectedBackendSupportsMultimodal,
|
|
25
|
+
} from "../embeddings.js";
|
|
24
26
|
import { loadImageRefData } from "./image-ref-utils.js";
|
|
25
27
|
import { getNode } from "./store.js";
|
|
26
28
|
import type { MemoryNode } from "./types.js";
|
|
@@ -165,10 +167,7 @@ function formatNodeForEmbedding(node: MemoryNode): string {
|
|
|
165
167
|
* (text queries match image memories in the same vector space). Falls back
|
|
166
168
|
* to text embedding with image description suffixes otherwise.
|
|
167
169
|
*/
|
|
168
|
-
export async function embedGraphNodeDirect(
|
|
169
|
-
node: MemoryNode,
|
|
170
|
-
config: AssistantConfig,
|
|
171
|
-
): Promise<void> {
|
|
170
|
+
export async function embedGraphNodeDirect(node: MemoryNode): Promise<void> {
|
|
172
171
|
if (node.fidelity === "gone") return;
|
|
173
172
|
|
|
174
173
|
const text = formatNodeForEmbedding(node);
|
|
@@ -181,7 +180,7 @@ export async function embedGraphNodeDirect(
|
|
|
181
180
|
};
|
|
182
181
|
|
|
183
182
|
if (node.imageRefs && node.imageRefs.length > 0) {
|
|
184
|
-
const multimodalAvailable = await selectedBackendSupportsMultimodal(
|
|
183
|
+
const multimodalAvailable = await selectedBackendSupportsMultimodal();
|
|
185
184
|
if (multimodalAvailable) {
|
|
186
185
|
const imageData = await loadImageRefData(node.imageRefs[0]);
|
|
187
186
|
if (imageData) {
|
|
@@ -191,7 +190,7 @@ export async function embedGraphNodeDirect(
|
|
|
191
190
|
data: imageData.data,
|
|
192
191
|
mimeType: imageData.mimeType,
|
|
193
192
|
};
|
|
194
|
-
await embedAndUpsert(
|
|
193
|
+
await embedAndUpsert("graph_node", node.id, input, {
|
|
195
194
|
...extraPayload,
|
|
196
195
|
has_image: true,
|
|
197
196
|
});
|
|
@@ -209,33 +208,24 @@ export async function embedGraphNodeDirect(
|
|
|
209
208
|
// Fallback: text embedding with image description suffix
|
|
210
209
|
const descSuffix = node.imageRefs.map((r) => r.description).join("; ");
|
|
211
210
|
const textWithImages = `${text}\n[images: ${descSuffix}]`;
|
|
212
|
-
await embedAndUpsert(
|
|
213
|
-
config,
|
|
214
|
-
"graph_node",
|
|
215
|
-
node.id,
|
|
216
|
-
textWithImages,
|
|
217
|
-
extraPayload,
|
|
218
|
-
);
|
|
211
|
+
await embedAndUpsert("graph_node", node.id, textWithImages, extraPayload);
|
|
219
212
|
return;
|
|
220
213
|
}
|
|
221
214
|
|
|
222
|
-
await embedAndUpsert(
|
|
215
|
+
await embedAndUpsert("graph_node", node.id, text, extraPayload);
|
|
223
216
|
}
|
|
224
217
|
|
|
225
218
|
/**
|
|
226
219
|
* Job handler: embed a graph node and upsert to Qdrant.
|
|
227
220
|
*/
|
|
228
|
-
export async function embedGraphNodeJob(
|
|
229
|
-
job: MemoryJob,
|
|
230
|
-
config: AssistantConfig,
|
|
231
|
-
): Promise<void> {
|
|
221
|
+
export async function embedGraphNodeJob(job: MemoryJob): Promise<void> {
|
|
232
222
|
const nodeId = asString(job.payload.nodeId);
|
|
233
223
|
if (!nodeId) return;
|
|
234
224
|
|
|
235
225
|
const node = getNode(nodeId);
|
|
236
226
|
if (!node) return;
|
|
237
227
|
|
|
238
|
-
await embedGraphNodeDirect(node
|
|
228
|
+
await embedGraphNodeDirect(node);
|
|
239
229
|
}
|
|
240
230
|
|
|
241
231
|
/**
|
|
@@ -262,8 +252,7 @@ export async function embedGraphTriggerJob(
|
|
|
262
252
|
const { eq } = await import("drizzle-orm");
|
|
263
253
|
const { memoryGraphTriggers } =
|
|
264
254
|
await import("../../../../persistence/schema/index.js");
|
|
265
|
-
const { embedWithBackend } =
|
|
266
|
-
await import("../../../../persistence/embeddings/embedding-backend.js");
|
|
255
|
+
const { embedWithBackend } = await import("../embeddings.js");
|
|
267
256
|
|
|
268
257
|
const db = getDb();
|
|
269
258
|
const row = db
|
|
@@ -11,12 +11,10 @@ import { getConfiguredProvider } from "@vellumai/plugin-api";
|
|
|
11
11
|
|
|
12
12
|
import type { AssistantConfig } from "../../../../config/types.js";
|
|
13
13
|
import { embedWithRetry } from "../../../../persistence/embeddings/embed.js";
|
|
14
|
-
import {
|
|
15
|
-
generateSparseEmbedding,
|
|
16
|
-
selectedBackendSupportsMultimodal,
|
|
17
|
-
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
14
|
+
import { generateSparseEmbedding } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
18
15
|
import type { QdrantSparseVector } from "../../../../persistence/embeddings/qdrant-client.js";
|
|
19
16
|
import { getLogger } from "../../../../util/logger.js";
|
|
17
|
+
import { selectedBackendSupportsMultimodal } from "../embeddings.js";
|
|
20
18
|
import { extractToolUse, userMessage } from "../llm-helpers.js";
|
|
21
19
|
import { searchGraphNodes } from "./graph-search.js";
|
|
22
20
|
import type { InContextTracker } from "./injection.js";
|
|
@@ -973,7 +971,7 @@ export async function retrieveForTurn(
|
|
|
973
971
|
|
|
974
972
|
if (imageBlocks.length > 0) {
|
|
975
973
|
try {
|
|
976
|
-
const isMultimodal = await selectedBackendSupportsMultimodal(
|
|
974
|
+
const isMultimodal = await selectedBackendSupportsMultimodal();
|
|
977
975
|
if (isMultimodal) {
|
|
978
976
|
const maxImageQueries = 2;
|
|
979
977
|
for (
|
|
@@ -9,7 +9,6 @@ import {
|
|
|
9
9
|
setMemoryCheckpoint,
|
|
10
10
|
} from "../../../persistence/checkpoints.js";
|
|
11
11
|
import { getDb } from "../../../persistence/db-connection.js";
|
|
12
|
-
import { selectedBackendSupportsMultimodal } from "../../../persistence/embeddings/embedding-backend.js";
|
|
13
12
|
import {
|
|
14
13
|
enqueueMemoryJob,
|
|
15
14
|
isMemoryEnabled,
|
|
@@ -24,6 +23,7 @@ import type { TrustClass } from "../../../runtime/actor-trust-resolver.js";
|
|
|
24
23
|
import { enqueueAutoAnalysisIfEnabled } from "../../../runtime/services/auto-analysis-enqueue.js";
|
|
25
24
|
import { isAutoAnalysisConversation } from "../../../runtime/services/auto-analysis-guard.js";
|
|
26
25
|
import { getLogger } from "../../../util/logger.js";
|
|
26
|
+
import { selectedBackendSupportsMultimodal } from "./embeddings.js";
|
|
27
27
|
import { isMemoryRetrospectiveConversation } from "./memory-retrospective-enqueue.js";
|
|
28
28
|
import { maybeEnqueueRetrospective } from "./memory-retrospective-trigger-check.js";
|
|
29
29
|
import { segmentText } from "./segmenter.js";
|
|
@@ -93,8 +93,7 @@ export async function indexMessageNow(
|
|
|
93
93
|
(b) => b.type === "image",
|
|
94
94
|
);
|
|
95
95
|
const mediaBlocks =
|
|
96
|
-
candidateMediaMeta.length > 0 &&
|
|
97
|
-
(await selectedBackendSupportsMultimodal(getConfig()))
|
|
96
|
+
candidateMediaMeta.length > 0 && (await selectedBackendSupportsMultimodal())
|
|
98
97
|
? candidateMediaMeta
|
|
99
98
|
: [];
|
|
100
99
|
|
|
@@ -41,25 +41,12 @@ mock.module("../../../../persistence/job-utils.js", () => ({
|
|
|
41
41
|
}));
|
|
42
42
|
|
|
43
43
|
import { resetDbForTesting } from "../../../../__tests__/db-test-helpers.js";
|
|
44
|
-
import { DEFAULT_CONFIG } from "../../../../config/defaults.js";
|
|
45
|
-
import type { AssistantConfig } from "../../../../config/types.js";
|
|
46
44
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
47
45
|
import { initializeDb } from "../../../../persistence/db-init.js";
|
|
48
46
|
import type { MemoryJob } from "../../../../persistence/jobs-store.js";
|
|
49
47
|
import { mediaAssets } from "../../../../persistence/schema/index.js";
|
|
50
48
|
import { embedMediaJob } from "./embedding.js";
|
|
51
49
|
|
|
52
|
-
const TEST_CONFIG: AssistantConfig = {
|
|
53
|
-
...DEFAULT_CONFIG,
|
|
54
|
-
memory: {
|
|
55
|
-
...DEFAULT_CONFIG.memory,
|
|
56
|
-
extraction: {
|
|
57
|
-
...DEFAULT_CONFIG.memory.extraction,
|
|
58
|
-
useLLM: false,
|
|
59
|
-
},
|
|
60
|
-
},
|
|
61
|
-
};
|
|
62
|
-
|
|
63
50
|
describe("embedMediaJob", () => {
|
|
64
51
|
// initializeDb runs the full migration chain (hundreds of steps); under
|
|
65
52
|
// parallel CI load it can exceed bun's default 5s hook timeout, so allow more.
|
|
@@ -90,12 +77,12 @@ describe("embedMediaJob", () => {
|
|
|
90
77
|
}
|
|
91
78
|
|
|
92
79
|
test("skips when assetId is missing", async () => {
|
|
93
|
-
await embedMediaJob(makeJob({})
|
|
80
|
+
await embedMediaJob(makeJob({}));
|
|
94
81
|
expect(embedAndUpsertCalls).toHaveLength(0);
|
|
95
82
|
});
|
|
96
83
|
|
|
97
84
|
test("skips when asset is not found", async () => {
|
|
98
|
-
await embedMediaJob(makeJob({ assetId: "nonexistent" })
|
|
85
|
+
await embedMediaJob(makeJob({ assetId: "nonexistent" }));
|
|
99
86
|
expect(embedAndUpsertCalls).toHaveLength(0);
|
|
100
87
|
});
|
|
101
88
|
|
|
@@ -122,7 +109,7 @@ describe("embedMediaJob", () => {
|
|
|
122
109
|
})
|
|
123
110
|
.run();
|
|
124
111
|
|
|
125
|
-
await embedMediaJob(makeJob({ assetId: "asset-registered" })
|
|
112
|
+
await embedMediaJob(makeJob({ assetId: "asset-registered" }));
|
|
126
113
|
expect(embedAndUpsertCalls).toHaveLength(0);
|
|
127
114
|
});
|
|
128
115
|
|
|
@@ -149,7 +136,7 @@ describe("embedMediaJob", () => {
|
|
|
149
136
|
})
|
|
150
137
|
.run();
|
|
151
138
|
|
|
152
|
-
await embedMediaJob(makeJob({ assetId: "asset-image" })
|
|
139
|
+
await embedMediaJob(makeJob({ assetId: "asset-image" }));
|
|
153
140
|
|
|
154
141
|
expect(embedAndUpsertCalls).toHaveLength(1);
|
|
155
142
|
const call = embedAndUpsertCalls[0];
|
|
@@ -191,7 +178,7 @@ describe("embedMediaJob", () => {
|
|
|
191
178
|
})
|
|
192
179
|
.run();
|
|
193
180
|
|
|
194
|
-
await embedMediaJob(makeJob({ assetId: "asset-audio" })
|
|
181
|
+
await embedMediaJob(makeJob({ assetId: "asset-audio" }));
|
|
195
182
|
|
|
196
183
|
expect(embedAndUpsertCalls).toHaveLength(1);
|
|
197
184
|
const call = embedAndUpsertCalls[0];
|
|
@@ -224,7 +211,7 @@ describe("embedMediaJob", () => {
|
|
|
224
211
|
})
|
|
225
212
|
.run();
|
|
226
213
|
|
|
227
|
-
await embedMediaJob(makeJob({ assetId: "asset-video" })
|
|
214
|
+
await embedMediaJob(makeJob({ assetId: "asset-video" }));
|
|
228
215
|
|
|
229
216
|
expect(embedAndUpsertCalls).toHaveLength(1);
|
|
230
217
|
const call = embedAndUpsertCalls[0];
|
|
@@ -2,10 +2,9 @@ import { readFile } from "node:fs/promises";
|
|
|
2
2
|
|
|
3
3
|
import { eq } from "drizzle-orm";
|
|
4
4
|
|
|
5
|
-
import type { AssistantConfig } from "../../../../config/types.js";
|
|
6
5
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
7
6
|
import type { EmbeddingInput } from "../../../../persistence/embeddings/embedding-types.js";
|
|
8
|
-
import { asString
|
|
7
|
+
import { asString } from "../../../../persistence/job-utils.js";
|
|
9
8
|
import type { MemoryJob } from "../../../../persistence/jobs-store.js";
|
|
10
9
|
import { extractMediaBlocks } from "../../../../persistence/message-content.js";
|
|
11
10
|
import {
|
|
@@ -14,11 +13,9 @@ import {
|
|
|
14
13
|
memorySummaries,
|
|
15
14
|
messages,
|
|
16
15
|
} from "../../../../persistence/schema/index.js";
|
|
16
|
+
import { embedAndUpsert } from "../embeddings.js";
|
|
17
17
|
|
|
18
|
-
export async function embedSegmentJob(
|
|
19
|
-
job: MemoryJob,
|
|
20
|
-
config: AssistantConfig,
|
|
21
|
-
): Promise<void> {
|
|
18
|
+
export async function embedSegmentJob(job: MemoryJob): Promise<void> {
|
|
22
19
|
const segmentId = asString(job.payload.segmentId);
|
|
23
20
|
if (!segmentId) return;
|
|
24
21
|
const db = getDb();
|
|
@@ -28,7 +25,7 @@ export async function embedSegmentJob(
|
|
|
28
25
|
.where(eq(memorySegments.id, segmentId))
|
|
29
26
|
.get();
|
|
30
27
|
if (!segment) return;
|
|
31
|
-
await embedAndUpsert(
|
|
28
|
+
await embedAndUpsert("segment", segment.id, segment.text, {
|
|
32
29
|
conversation_id: segment.conversationId,
|
|
33
30
|
message_id: segment.messageId,
|
|
34
31
|
created_at: segment.createdAt,
|
|
@@ -36,10 +33,7 @@ export async function embedSegmentJob(
|
|
|
36
33
|
});
|
|
37
34
|
}
|
|
38
35
|
|
|
39
|
-
export async function embedSummaryJob(
|
|
40
|
-
job: MemoryJob,
|
|
41
|
-
config: AssistantConfig,
|
|
42
|
-
): Promise<void> {
|
|
36
|
+
export async function embedSummaryJob(job: MemoryJob): Promise<void> {
|
|
43
37
|
const summaryId = asString(job.payload.summaryId);
|
|
44
38
|
if (!summaryId) return;
|
|
45
39
|
const db = getDb();
|
|
@@ -50,7 +44,6 @@ export async function embedSummaryJob(
|
|
|
50
44
|
.get();
|
|
51
45
|
if (!summary) return;
|
|
52
46
|
await embedAndUpsert(
|
|
53
|
-
config,
|
|
54
47
|
"summary",
|
|
55
48
|
summary.id,
|
|
56
49
|
`[${summary.scope}] ${summary.summary}`,
|
|
@@ -63,10 +56,7 @@ export async function embedSummaryJob(
|
|
|
63
56
|
);
|
|
64
57
|
}
|
|
65
58
|
|
|
66
|
-
export async function embedMediaJob(
|
|
67
|
-
job: MemoryJob,
|
|
68
|
-
config: AssistantConfig,
|
|
69
|
-
): Promise<void> {
|
|
59
|
+
export async function embedMediaJob(job: MemoryJob): Promise<void> {
|
|
70
60
|
const assetId = asString(job.payload.assetId);
|
|
71
61
|
if (!assetId) return;
|
|
72
62
|
|
|
@@ -88,7 +78,7 @@ export async function embedMediaJob(
|
|
|
88
78
|
mimeType: asset.mimeType,
|
|
89
79
|
};
|
|
90
80
|
|
|
91
|
-
await embedAndUpsert(
|
|
81
|
+
await embedAndUpsert("media", asset.id, input, {
|
|
92
82
|
created_at: asset.createdAt,
|
|
93
83
|
kind: asset.mediaType,
|
|
94
84
|
subject: asset.title,
|
|
@@ -96,10 +86,7 @@ export async function embedMediaJob(
|
|
|
96
86
|
});
|
|
97
87
|
}
|
|
98
88
|
|
|
99
|
-
export async function embedAttachmentJob(
|
|
100
|
-
job: MemoryJob,
|
|
101
|
-
config: AssistantConfig,
|
|
102
|
-
): Promise<void> {
|
|
89
|
+
export async function embedAttachmentJob(job: MemoryJob): Promise<void> {
|
|
103
90
|
const messageId = asString(job.payload.messageId);
|
|
104
91
|
const blockIndex = job.payload.blockIndex as number;
|
|
105
92
|
if (!messageId || typeof blockIndex !== "number") return;
|
|
@@ -124,7 +111,7 @@ export async function embedAttachmentJob(
|
|
|
124
111
|
|
|
125
112
|
// Use messageId + blockIndex as targetId for uniqueness
|
|
126
113
|
const targetId = `${messageId}:${blockIndex}`;
|
|
127
|
-
await embedAndUpsert(
|
|
114
|
+
await embedAndUpsert("media", targetId, input, {
|
|
128
115
|
created_at: message.createdAt,
|
|
129
116
|
message_id: messageId,
|
|
130
117
|
conversation_id: message.conversationId,
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { eq, isNotNull, like, ne } from "drizzle-orm";
|
|
2
2
|
|
|
3
|
-
import { getConfig } from "../../../../config/loader.js";
|
|
4
3
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
5
|
-
import { selectedBackendSupportsMultimodal } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
6
4
|
import { withQdrantBreaker } from "../../../../persistence/embeddings/qdrant-circuit-breaker.js";
|
|
7
5
|
import { getQdrantClient } from "../../../../persistence/embeddings/qdrant-client.js";
|
|
8
6
|
import {
|
|
@@ -24,6 +22,7 @@ import {
|
|
|
24
22
|
messages,
|
|
25
23
|
} from "../../../../persistence/schema/index.js";
|
|
26
24
|
import { getLogger } from "../../../../util/logger.js";
|
|
25
|
+
import { selectedBackendSupportsMultimodal } from "../embeddings.js";
|
|
27
26
|
|
|
28
27
|
const log = getLogger("memory-jobs-worker");
|
|
29
28
|
|
|
@@ -50,7 +49,7 @@ export async function rebuildIndexJob(): Promise<void> {
|
|
|
50
49
|
// Re-enqueue multimodal embedding jobs only when the resolved embedding
|
|
51
50
|
// backend supports multimodal inputs. Without this gate, embed_media and
|
|
52
51
|
// embed_attachment jobs would all fail for text-only backends.
|
|
53
|
-
if (await selectedBackendSupportsMultimodal(
|
|
52
|
+
if (await selectedBackendSupportsMultimodal()) {
|
|
54
53
|
const assets = db
|
|
55
54
|
.select({ id: mediaAssets.id })
|
|
56
55
|
.from(mediaAssets)
|
|
@@ -11,7 +11,6 @@ import {
|
|
|
11
11
|
type MessagesLexicalIndex,
|
|
12
12
|
} from "../../../../persistence/embeddings/messages-lexical-index.js";
|
|
13
13
|
import { withQdrantBreaker } from "../../../../persistence/embeddings/qdrant-circuit-breaker.js";
|
|
14
|
-
import { resolveQdrantUrl } from "../../../../persistence/embeddings/qdrant-client.js";
|
|
15
14
|
import { asString } from "../../../../persistence/job-utils.js";
|
|
16
15
|
import {
|
|
17
16
|
enqueueMemoryJob,
|
|
@@ -21,6 +20,7 @@ import {
|
|
|
21
20
|
import { messages } from "../../../../persistence/schema/index.js";
|
|
22
21
|
import { getLogger } from "../../../../util/logger.js";
|
|
23
22
|
import { isPluginDisabled } from "../../../disabled-state.js";
|
|
23
|
+
import { resolveQdrantUrl } from "../embeddings.js";
|
|
24
24
|
import memoryPkg from "../package.json" with { type: "json" };
|
|
25
25
|
|
|
26
26
|
const log = getLogger("messages-lexical-enqueue");
|
|
@@ -62,7 +62,7 @@ export function resolveLexicalIndex(
|
|
|
62
62
|
return getMessagesLexicalIndex();
|
|
63
63
|
} catch {
|
|
64
64
|
return initMessagesLexicalIndex({
|
|
65
|
-
url: resolveQdrantUrl(
|
|
65
|
+
url: resolveQdrantUrl(),
|
|
66
66
|
collection: MESSAGES_LEXICAL_COLLECTION,
|
|
67
67
|
onDisk: config.memory.qdrant.onDisk,
|
|
68
68
|
});
|
|
@@ -122,11 +122,11 @@ async function graphNarrativeRefineJob(
|
|
|
122
122
|
export const memoryJobHandlers: readonly JobHandlerEntry[] = [
|
|
123
123
|
{
|
|
124
124
|
type: "embed_segment",
|
|
125
|
-
handler: (job
|
|
125
|
+
handler: (job) => embedSegmentJob(job),
|
|
126
126
|
},
|
|
127
127
|
{
|
|
128
128
|
type: "embed_summary",
|
|
129
|
-
handler: (job
|
|
129
|
+
handler: (job) => embedSummaryJob(job),
|
|
130
130
|
},
|
|
131
131
|
{ type: "backfill", handler: (job, config) => backfillJob(job, config) },
|
|
132
132
|
{ type: "rebuild_index", handler: () => rebuildIndexJob() },
|
|
@@ -134,14 +134,14 @@ export const memoryJobHandlers: readonly JobHandlerEntry[] = [
|
|
|
134
134
|
type: "delete_qdrant_vectors",
|
|
135
135
|
handler: (job) => deleteQdrantVectorsJob(job),
|
|
136
136
|
},
|
|
137
|
-
{ type: "embed_media", handler: (job
|
|
137
|
+
{ type: "embed_media", handler: (job) => embedMediaJob(job) },
|
|
138
138
|
{
|
|
139
139
|
type: "embed_attachment",
|
|
140
|
-
handler: (job
|
|
140
|
+
handler: (job) => embedAttachmentJob(job),
|
|
141
141
|
},
|
|
142
142
|
{
|
|
143
143
|
type: "embed_graph_node",
|
|
144
|
-
handler: (job
|
|
144
|
+
handler: (job) => embedGraphNodeJob(job),
|
|
145
145
|
},
|
|
146
146
|
{
|
|
147
147
|
type: "embed_pkb_file",
|
|
@@ -23,7 +23,6 @@ import { and, eq } from "drizzle-orm";
|
|
|
23
23
|
import type { AssistantConfig } from "../../../../config/types.js";
|
|
24
24
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
25
25
|
import {
|
|
26
|
-
embedWithBackend,
|
|
27
26
|
generateSparseEmbedding,
|
|
28
27
|
getMemoryBackendStatus,
|
|
29
28
|
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
@@ -43,6 +42,7 @@ import { BackendUnavailableError } from "../../../../util/errors.js";
|
|
|
43
42
|
import { getLogger } from "../../../../util/logger.js";
|
|
44
43
|
import { getWorkspaceDir } from "../../../../util/platform.js";
|
|
45
44
|
import { applyCorrectionIfCalibrated } from "../anisotropy.js";
|
|
45
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
46
46
|
import { readPage } from "../v2/page-store.js";
|
|
47
47
|
import {
|
|
48
48
|
deleteConceptPageEmbedding,
|
|
@@ -15,10 +15,9 @@ import { createHash } from "node:crypto";
|
|
|
15
15
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
16
16
|
import { join, relative } from "node:path";
|
|
17
17
|
|
|
18
|
-
import { getConfig } from "../../../../config/loader.js";
|
|
19
18
|
import { withQdrantBreaker } from "../../../../persistence/embeddings/qdrant-circuit-breaker.js";
|
|
20
19
|
import { getQdrantClient } from "../../../../persistence/embeddings/qdrant-client.js";
|
|
21
|
-
import { embedAndUpsert } from "
|
|
20
|
+
import { embedAndUpsert } from "../embeddings.js";
|
|
22
21
|
import type { PkbIndexEntry } from "./types.js";
|
|
23
22
|
import { PKB_TARGET_TYPE } from "./types.js";
|
|
24
23
|
|
|
@@ -189,8 +188,6 @@ export async function indexPkbFile(
|
|
|
189
188
|
const relPath = relative(pkbRoot, absPath);
|
|
190
189
|
const chunks = chunkPkbFile(content);
|
|
191
190
|
|
|
192
|
-
const config = getConfig();
|
|
193
|
-
|
|
194
191
|
const qdrant = getQdrantClient();
|
|
195
192
|
const existing = await withQdrantBreaker(() =>
|
|
196
193
|
qdrant.scrollByTargetType(PKB_TARGET_TYPE, {
|
|
@@ -209,7 +206,6 @@ export async function indexPkbFile(
|
|
|
209
206
|
const targetId = `${memoryScopeId}:${relPath}#${chunkIndex}`;
|
|
210
207
|
newTargetIds.add(targetId);
|
|
211
208
|
await embedAndUpsert(
|
|
212
|
-
config,
|
|
213
209
|
PKB_TARGET_TYPE,
|
|
214
210
|
targetId,
|
|
215
211
|
{ type: "text", text: chunk },
|
|
@@ -27,7 +27,6 @@ import { z } from "zod";
|
|
|
27
27
|
import { getConfig } from "../../../../config/loader.js";
|
|
28
28
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
29
29
|
import {
|
|
30
|
-
embedWithBackend,
|
|
31
30
|
generateSparseEmbedding,
|
|
32
31
|
getMemoryBackendStatus,
|
|
33
32
|
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
@@ -46,6 +45,7 @@ import {
|
|
|
46
45
|
} from "../../../../runtime/routes/errors.js";
|
|
47
46
|
import type { RouteDefinition } from "../../../../runtime/routes/types.js";
|
|
48
47
|
import { getLogger } from "../../../../util/logger.js";
|
|
48
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
49
49
|
import { createNode, deleteNode, getNode, updateNode } from "../graph/store.js";
|
|
50
50
|
import type {
|
|
51
51
|
Fidelity,
|
|
@@ -20,10 +20,7 @@ import {
|
|
|
20
20
|
initMessagesLexicalIndex,
|
|
21
21
|
MESSAGES_LEXICAL_COLLECTION,
|
|
22
22
|
} from "../../../persistence/embeddings/messages-lexical-index.js";
|
|
23
|
-
import {
|
|
24
|
-
initQdrantClient,
|
|
25
|
-
resolveQdrantUrl,
|
|
26
|
-
} from "../../../persistence/embeddings/qdrant-client.js";
|
|
23
|
+
import { initQdrantClient } from "../../../persistence/embeddings/qdrant-client.js";
|
|
27
24
|
import { createQdrantManager } from "../../../persistence/embeddings/qdrant-manager.js";
|
|
28
25
|
import {
|
|
29
26
|
enqueueMemoryJob,
|
|
@@ -32,6 +29,7 @@ import {
|
|
|
32
29
|
import { startMemoryJobsWorker } from "../../../persistence/jobs-worker.js";
|
|
33
30
|
import { getLogger } from "../../../util/logger.js";
|
|
34
31
|
import { getWorkspaceDir } from "../../../util/platform.js";
|
|
32
|
+
import { resolveQdrantUrl } from "./embeddings.js";
|
|
35
33
|
import { sweepConceptPageFrontmatter } from "./v2/frontmatter-sweep.js";
|
|
36
34
|
import {
|
|
37
35
|
maybeRebuildMemoryV2Concepts,
|
|
@@ -41,7 +39,7 @@ import {
|
|
|
41
39
|
const log = getLogger("memory-startup");
|
|
42
40
|
|
|
43
41
|
export async function runMemoryStartup(config: AssistantConfig): Promise<void> {
|
|
44
|
-
const qdrantUrl = resolveQdrantUrl(
|
|
42
|
+
const qdrantUrl = resolveQdrantUrl();
|
|
45
43
|
log.info({ qdrantUrl }, "Daemon startup: initializing Qdrant");
|
|
46
44
|
const manager = createQdrantManager({ url: qdrantUrl });
|
|
47
45
|
const QDRANT_START_MAX_ATTEMPTS = 3;
|
|
@@ -35,11 +35,9 @@
|
|
|
35
35
|
// for the next turn and drop below `epsilon` if no longer relevant.
|
|
36
36
|
|
|
37
37
|
import type { AssistantConfig } from "../../../../config/types.js";
|
|
38
|
-
import {
|
|
39
|
-
embedWithBackend,
|
|
40
|
-
isEmbeddingDimensionAvailable,
|
|
41
|
-
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
38
|
+
import { isEmbeddingDimensionAvailable } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
42
39
|
import { applyCorrectionIfCalibrated } from "../anisotropy.js";
|
|
40
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
43
41
|
import { clampUnitInterval } from "../validation.js";
|
|
44
42
|
import type { EdgeIndex } from "./edge-index.js";
|
|
45
43
|
import { hybridQueryConceptPages } from "./qdrant.js";
|
|
@@ -21,12 +21,10 @@
|
|
|
21
21
|
// summary.
|
|
22
22
|
|
|
23
23
|
import { getConfig } from "../../../../config/loader.js";
|
|
24
|
-
import {
|
|
25
|
-
embedWithBackend,
|
|
26
|
-
generateSparseEmbedding,
|
|
27
|
-
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
24
|
+
import { generateSparseEmbedding } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
28
25
|
import { getLogger } from "../../../../util/logger.js";
|
|
29
26
|
import { applyCorrectionIfCalibrated } from "../anisotropy.js";
|
|
27
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
30
28
|
import { buildCliCommandContent } from "./cli-command-content.js";
|
|
31
29
|
import { invalidatePageIndex } from "./page-index.js";
|
|
32
30
|
import {
|
|
@@ -27,9 +27,9 @@ import { v5 as uuidv5 } from "uuid";
|
|
|
27
27
|
|
|
28
28
|
import { getConfig } from "../../../../config/loader.js";
|
|
29
29
|
import type { SparseEmbedding } from "../../../../persistence/embeddings/embedding-types.js";
|
|
30
|
-
import { resolveQdrantUrl } from "../../../../persistence/embeddings/qdrant-client.js";
|
|
31
30
|
import { getLogger } from "../../../../util/logger.js";
|
|
32
31
|
import { getDataDir } from "../../../../util/platform.js";
|
|
32
|
+
import { resolveQdrantUrl } from "../embeddings.js";
|
|
33
33
|
|
|
34
34
|
const log = getLogger("memory-v2-qdrant");
|
|
35
35
|
|
|
@@ -137,9 +137,8 @@ export async function clearReembedSentinel(): Promise<void> {
|
|
|
137
137
|
/** Lazily create a Qdrant REST client bound to the resolved URL. */
|
|
138
138
|
function getClient(): QdrantRestClient {
|
|
139
139
|
if (_client) return _client;
|
|
140
|
-
const config = getConfig();
|
|
141
140
|
_client = new QdrantRestClient({
|
|
142
|
-
url: resolveQdrantUrl(
|
|
141
|
+
url: resolveQdrantUrl(),
|
|
143
142
|
checkCompatibility: false,
|
|
144
143
|
});
|
|
145
144
|
return _client;
|
|
@@ -33,11 +33,9 @@
|
|
|
33
33
|
// across turns.
|
|
34
34
|
|
|
35
35
|
import type { AssistantConfig } from "../../../../config/types.js";
|
|
36
|
-
import {
|
|
37
|
-
embedWithBackend,
|
|
38
|
-
isEmbeddingDimensionAvailable,
|
|
39
|
-
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
36
|
+
import { isEmbeddingDimensionAvailable } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
40
37
|
import { applyCorrectionIfCalibrated } from "../anisotropy.js";
|
|
38
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
41
39
|
import { clampUnitInterval } from "../validation.js";
|
|
42
40
|
import { hybridQueryConceptPages } from "./qdrant.js";
|
|
43
41
|
import { generateBm25QueryEmbedding } from "./sparse-bm25.js";
|
|
@@ -26,10 +26,7 @@ import { isAssistantFeatureFlagEnabled } from "../../../../config/assistant-feat
|
|
|
26
26
|
import { getConfig } from "../../../../config/loader.js";
|
|
27
27
|
import { resolveSkillStates } from "../../../../config/skill-state.js";
|
|
28
28
|
import { loadSkillCatalog } from "../../../../config/skills.js";
|
|
29
|
-
import {
|
|
30
|
-
embedWithBackend,
|
|
31
|
-
generateSparseEmbedding,
|
|
32
|
-
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
29
|
+
import { generateSparseEmbedding } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
33
30
|
import { getCatalog } from "../../../../skills/catalog-cache.js";
|
|
34
31
|
import {
|
|
35
32
|
fromCatalogSkill,
|
|
@@ -37,6 +34,7 @@ import {
|
|
|
37
34
|
} from "../../../../skills/skill-memory.js";
|
|
38
35
|
import { getLogger } from "../../../../util/logger.js";
|
|
39
36
|
import { applyCorrectionIfCalibrated } from "../anisotropy.js";
|
|
37
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
40
38
|
import { invalidatePageIndex } from "./page-index.js";
|
|
41
39
|
import {
|
|
42
40
|
backfillKindOnPointsWithPrefix,
|
|
@@ -16,11 +16,9 @@
|
|
|
16
16
|
// forward regardless, so a dense outage narrows recall but never breaks a turn.
|
|
17
17
|
|
|
18
18
|
import type { AssistantConfig } from "../../../../config/types.js";
|
|
19
|
-
import {
|
|
20
|
-
embedWithBackend,
|
|
21
|
-
isEmbeddingDimensionAvailable,
|
|
22
|
-
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
19
|
+
import { isEmbeddingDimensionAvailable } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
23
20
|
import { getLogger } from "../../../../util/logger.js";
|
|
21
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
24
22
|
import {
|
|
25
23
|
getSectionDenseClient,
|
|
26
24
|
SECTION_COLLECTION,
|
|
@@ -72,14 +70,11 @@ export async function denseLane(
|
|
|
72
70
|
const vector = vectors[0];
|
|
73
71
|
if (!vector || vector.length === 0) return [];
|
|
74
72
|
|
|
75
|
-
const result = await getSectionDenseClient(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
with_payload: true,
|
|
81
|
-
},
|
|
82
|
-
);
|
|
73
|
+
const result = await getSectionDenseClient().query(SECTION_COLLECTION, {
|
|
74
|
+
query: vector,
|
|
75
|
+
limit: k * OVERSAMPLE,
|
|
76
|
+
with_payload: true,
|
|
77
|
+
});
|
|
83
78
|
points = result.points;
|
|
84
79
|
} catch (err) {
|
|
85
80
|
log.warn({ err }, "memory v3 dense lane failed; degrading to no hits");
|
|
@@ -68,10 +68,7 @@ import {
|
|
|
68
68
|
getMemoryCheckpoint,
|
|
69
69
|
setMemoryCheckpoint,
|
|
70
70
|
} from "../../../../persistence/checkpoints.js";
|
|
71
|
-
import {
|
|
72
|
-
EmbeddingBackendUnavailableError,
|
|
73
|
-
embedWithBackend,
|
|
74
|
-
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
71
|
+
import { EmbeddingBackendUnavailableError } from "../../../../persistence/embeddings/embedding-backend.js";
|
|
75
72
|
import { EmbeddingBillingBlockError } from "../../../../persistence/embeddings/embedding-billing-breaker.js";
|
|
76
73
|
import type { MemoryJob } from "../../../../persistence/jobs-store.js";
|
|
77
74
|
import {
|
|
@@ -81,6 +78,7 @@ import {
|
|
|
81
78
|
import { executeDeleteManagedSkill } from "../../../../tools/skills/delete-managed.js";
|
|
82
79
|
import { getLogger } from "../../../../util/logger.js";
|
|
83
80
|
import { getWorkspaceDir } from "../../../../util/platform.js";
|
|
81
|
+
import { embedWithBackend } from "../embeddings.js";
|
|
84
82
|
import { getPageIndex } from "../v2/page-index.js";
|
|
85
83
|
import { readPage } from "../v2/page-store.js";
|
|
86
84
|
import { skillSlugFor } from "../v2/skill-store.js";
|
|
@@ -28,7 +28,6 @@ import type { AssistantConfig } from "../../../../config/types.js";
|
|
|
28
28
|
import { deleteMemoryCheckpoint } from "../../../../persistence/checkpoints.js";
|
|
29
29
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
30
30
|
import {
|
|
31
|
-
embedWithBackend,
|
|
32
31
|
geminiCacheExtras,
|
|
33
32
|
getMemoryBackendStatus,
|
|
34
33
|
} from "../../../../persistence/embeddings/embedding-backend.js";
|
|
@@ -37,8 +36,8 @@ import {
|
|
|
37
36
|
writeEmbeddingCache,
|
|
38
37
|
} from "../../../../persistence/embeddings/embedding-cache.js";
|
|
39
38
|
import { embeddingInputContentHash } from "../../../../persistence/embeddings/embedding-types.js";
|
|
40
|
-
import { resolveQdrantUrl } from "../../../../persistence/embeddings/qdrant-client.js";
|
|
41
39
|
import { getLogger } from "../../../../util/logger.js";
|
|
40
|
+
import { embedWithBackend, resolveQdrantUrl } from "../embeddings.js";
|
|
42
41
|
import type { Section } from "./types.js";
|
|
43
42
|
|
|
44
43
|
const log = getLogger("memory-v3-section-dense-store");
|
|
@@ -77,12 +76,10 @@ let _collectionReady = false;
|
|
|
77
76
|
* Shared by both section-dense lanes (this store and the dense read lane in
|
|
78
77
|
* `dense.ts`) so they reuse one client instance and one reset hook.
|
|
79
78
|
*/
|
|
80
|
-
export function getSectionDenseClient(
|
|
81
|
-
config: AssistantConfig,
|
|
82
|
-
): QdrantRestClient {
|
|
79
|
+
export function getSectionDenseClient(): QdrantRestClient {
|
|
83
80
|
if (_client) return _client;
|
|
84
81
|
_client = new QdrantRestClient({
|
|
85
|
-
url: resolveQdrantUrl(
|
|
82
|
+
url: resolveQdrantUrl(),
|
|
86
83
|
checkCompatibility: false,
|
|
87
84
|
});
|
|
88
85
|
return _client;
|
|
@@ -108,7 +105,7 @@ export async function ensureSectionCollection(
|
|
|
108
105
|
): Promise<void> {
|
|
109
106
|
if (_collectionReady) return;
|
|
110
107
|
|
|
111
|
-
const client = getSectionDenseClient(
|
|
108
|
+
const client = getSectionDenseClient();
|
|
112
109
|
const vectorSize = config.memory.qdrant.vectorSize;
|
|
113
110
|
const onDisk = config.memory.qdrant.onDisk;
|
|
114
111
|
|
|
@@ -218,7 +215,7 @@ export async function ensureSectionCollection(
|
|
|
218
215
|
export async function recreateSectionCollection(
|
|
219
216
|
config: AssistantConfig,
|
|
220
217
|
): Promise<void> {
|
|
221
|
-
const client = getSectionDenseClient(
|
|
218
|
+
const client = getSectionDenseClient();
|
|
222
219
|
const exists = await client.collectionExists(SECTION_COLLECTION);
|
|
223
220
|
if (exists.exists) {
|
|
224
221
|
await client.deleteCollection(SECTION_COLLECTION);
|
|
@@ -296,7 +293,7 @@ export async function upsertSections(
|
|
|
296
293
|
|
|
297
294
|
if (points.length === 0) return;
|
|
298
295
|
|
|
299
|
-
await getSectionDenseClient(
|
|
296
|
+
await getSectionDenseClient().upsert(SECTION_COLLECTION, {
|
|
300
297
|
wait: true,
|
|
301
298
|
points,
|
|
302
299
|
});
|
|
@@ -418,7 +415,7 @@ export async function deleteSectionsForArticle(
|
|
|
418
415
|
): Promise<void> {
|
|
419
416
|
await ensureSectionCollection(config);
|
|
420
417
|
|
|
421
|
-
await getSectionDenseClient(
|
|
418
|
+
await getSectionDenseClient().delete(SECTION_COLLECTION, {
|
|
422
419
|
wait: true,
|
|
423
420
|
filter: { must: [{ key: "article", match: { value: article } }] },
|
|
424
421
|
});
|
|
@@ -440,7 +437,7 @@ export async function listSectionArticles(
|
|
|
440
437
|
): Promise<string[]> {
|
|
441
438
|
await ensureSectionCollection(config);
|
|
442
439
|
|
|
443
|
-
const client = getSectionDenseClient(
|
|
440
|
+
const client = getSectionDenseClient();
|
|
444
441
|
const articles = new Set<string>();
|
|
445
442
|
let offset: string | number | undefined = undefined;
|
|
446
443
|
const maxIterations = 10_000;
|