@orquentra/codebuddy 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -1090
- package/dist/chunk-3XYQGFPB.js +91 -0
- package/dist/chunk-3XYQGFPB.js.map +1 -0
- package/dist/chunk-4BYGTEJT.js +103 -0
- package/dist/chunk-4BYGTEJT.js.map +1 -0
- package/dist/{chunk-ITGRANYW.js → chunk-6MKEIDX5.js} +2 -2
- package/dist/{chunk-K63EOA4B.js → chunk-776EQXDV.js} +25 -6
- package/dist/chunk-776EQXDV.js.map +1 -0
- package/dist/{chunk-5OWZVZSO.js → chunk-CQPCHMBS.js} +8 -165
- package/dist/chunk-CQPCHMBS.js.map +1 -0
- package/dist/chunk-JDZ3VK26.js +4006 -0
- package/dist/chunk-JDZ3VK26.js.map +1 -0
- package/dist/chunk-JMWJ7V6Y.js +99 -0
- package/dist/chunk-JMWJ7V6Y.js.map +1 -0
- package/dist/chunk-KYK6RPAJ.js +99 -0
- package/dist/chunk-KYK6RPAJ.js.map +1 -0
- package/dist/{chunk-IBMJYIEP.js → chunk-L5AYZC3F.js} +7 -7
- package/dist/{chunk-3ZELDCR3.js → chunk-PCE4STAY.js} +7 -7
- package/dist/{chunk-TI5TQD6D.js → chunk-QDBDYCM2.js} +2 -2
- package/dist/{chunk-DSN5CEHO.js → chunk-ROQTT5ZO.js} +3 -3
- package/dist/{chunk-IE2OUO54.js → chunk-UFZD3JAA.js} +3 -3
- package/dist/chunk-Z5QB367W.js +5028 -0
- package/dist/chunk-Z5QB367W.js.map +1 -0
- package/dist/cli/index.js +2 -2
- package/dist/cloud-GP6G3EAK.js +17 -0
- package/dist/{codebuddy-D2xJ6hFk.d.ts → codebuddy-DM-i0Fu5.d.ts} +49 -18
- package/dist/{config-S2TULOMK.js → config-KFEHLV6W.js} +3 -3
- package/dist/index.d.ts +23 -3
- package/dist/index.js +34 -25
- package/dist/langgraph/index.d.ts +1 -1
- package/dist/lean-command-DA4ZNKG3.js +15 -0
- package/dist/local-3NGNESNB.js +13 -0
- package/dist/{main-ZZ3EQLYA.js → main-BDVQAGAS.js} +1261 -242
- package/dist/main-BDVQAGAS.js.map +1 -0
- package/dist/{memory-NIBFAMY4.js → memory-4RGVLPKG.js} +6 -6
- package/dist/operations-EQGOFCBD.js +24 -0
- package/dist/operations-EQGOFCBD.js.map +1 -0
- package/dist/{rerank-Q7MIFNKA.js → rerank-3P3C2UHN.js} +4 -4
- package/dist/{summaries-TSE44KUQ.js → summaries-ZBV5IN7Y.js} +6 -6
- package/dist/summaries-ZBV5IN7Y.js.map +1 -0
- package/dist/{zvec-F27AR357.js → zvec-MAPSJQSR.js} +3 -3
- package/dist/zvec-MAPSJQSR.js.map +1 -0
- package/package.json +7 -3
- package/dist/chunk-5OWZVZSO.js.map +0 -1
- package/dist/chunk-K63EOA4B.js.map +0 -1
- package/dist/chunk-SYHB3AMM.js +0 -8580
- package/dist/chunk-SYHB3AMM.js.map +0 -1
- package/dist/lean-command-D6TZJEFX.js +0 -13
- package/dist/main-ZZ3EQLYA.js.map +0 -1
- /package/dist/{chunk-ITGRANYW.js.map → chunk-6MKEIDX5.js.map} +0 -0
- /package/dist/{chunk-IBMJYIEP.js.map → chunk-L5AYZC3F.js.map} +0 -0
- /package/dist/{chunk-3ZELDCR3.js.map → chunk-PCE4STAY.js.map} +0 -0
- /package/dist/{chunk-TI5TQD6D.js.map → chunk-QDBDYCM2.js.map} +0 -0
- /package/dist/{chunk-DSN5CEHO.js.map → chunk-ROQTT5ZO.js.map} +0 -0
- /package/dist/{chunk-IE2OUO54.js.map → chunk-UFZD3JAA.js.map} +0 -0
- /package/dist/{config-S2TULOMK.js.map → cloud-GP6G3EAK.js.map} +0 -0
- /package/dist/{lean-command-D6TZJEFX.js.map → config-KFEHLV6W.js.map} +0 -0
- /package/dist/{summaries-TSE44KUQ.js.map → lean-command-DA4ZNKG3.js.map} +0 -0
- /package/dist/{zvec-F27AR357.js.map → local-3NGNESNB.js.map} +0 -0
- /package/dist/{memory-NIBFAMY4.js.map → memory-4RGVLPKG.js.map} +0 -0
- /package/dist/{rerank-Q7MIFNKA.js.map → rerank-3P3C2UHN.js.map} +0 -0
|
@@ -0,0 +1,4006 @@
|
|
|
1
|
+
import {
|
|
2
|
+
REDACTED,
|
|
3
|
+
redactPersisted
|
|
4
|
+
} from "./chunk-GDJH7XJC.js";
|
|
5
|
+
import {
|
|
6
|
+
clampBudgetForCallerModel,
|
|
7
|
+
computeContentHash,
|
|
8
|
+
computeFactHash,
|
|
9
|
+
countTokens,
|
|
10
|
+
generateEntityId,
|
|
11
|
+
generateSessionId
|
|
12
|
+
} from "./chunk-4BYGTEJT.js";
|
|
13
|
+
import {
|
|
14
|
+
OpenRouterEmbeddingProvider
|
|
15
|
+
} from "./chunk-KYK6RPAJ.js";
|
|
16
|
+
import {
|
|
17
|
+
LocalModelProvider
|
|
18
|
+
} from "./chunk-3XYQGFPB.js";
|
|
19
|
+
import {
|
|
20
|
+
DEFAULT_CLOUD_EMBEDDING_MODELS,
|
|
21
|
+
DEFAULT_EMBEDDING_MODELS,
|
|
22
|
+
DEFAULT_LLM_MODELS,
|
|
23
|
+
EMBEDDING_DIMENSIONS,
|
|
24
|
+
assertCompatibleEmbeddingModels,
|
|
25
|
+
getModelDefinition
|
|
26
|
+
} from "./chunk-JMWJ7V6Y.js";
|
|
27
|
+
import {
|
|
28
|
+
MISSING_DATABASE_URL,
|
|
29
|
+
assertNoSymlinkBetween,
|
|
30
|
+
checkConfigPermissions,
|
|
31
|
+
checkProjectScaffold,
|
|
32
|
+
composeMarkdown,
|
|
33
|
+
defaultNamespaceForCwd,
|
|
34
|
+
loadRuntimeConfig,
|
|
35
|
+
parseFrontMatter,
|
|
36
|
+
readConfigFile,
|
|
37
|
+
redactSecrets,
|
|
38
|
+
writeAtomic
|
|
39
|
+
} from "./chunk-776EQXDV.js";
|
|
40
|
+
import {
|
|
41
|
+
__export
|
|
42
|
+
} from "./chunk-PZ5AY32C.js";
|
|
43
|
+
|
|
44
|
+
// src/db/schema.ts
|
|
45
|
+
var schema_exports = {};
|
|
46
|
+
__export(schema_exports, {
|
|
47
|
+
HNSW_EF_CONSTRUCTION: () => HNSW_EF_CONSTRUCTION,
|
|
48
|
+
HNSW_M: () => HNSW_M,
|
|
49
|
+
VECTOR_DIMENSIONS: () => VECTOR_DIMENSIONS,
|
|
50
|
+
auditActionEnum: () => auditActionEnum,
|
|
51
|
+
auditLog: () => auditLog,
|
|
52
|
+
checkpoints: () => checkpoints,
|
|
53
|
+
embeddingOwnerTypeEnum: () => embeddingOwnerTypeEnum,
|
|
54
|
+
embeddingStatusEnum: () => embeddingStatusEnum,
|
|
55
|
+
embeddings: () => embeddings,
|
|
56
|
+
facts: () => facts,
|
|
57
|
+
interactionRoleEnum: () => interactionRoleEnum,
|
|
58
|
+
interactions: () => interactions,
|
|
59
|
+
memoryTypeEnum: () => memoryTypeEnum,
|
|
60
|
+
modelCallStatusEnum: () => modelCallStatusEnum,
|
|
61
|
+
modelCallTypeEnum: () => modelCallTypeEnum,
|
|
62
|
+
modelCalls: () => modelCalls,
|
|
63
|
+
namespaces: () => namespaces,
|
|
64
|
+
sessionSummaries: () => sessionSummaries,
|
|
65
|
+
shareModeEnum: () => shareModeEnum,
|
|
66
|
+
shares: () => shares
|
|
67
|
+
});
|
|
68
|
+
import { sql } from "drizzle-orm";
|
|
69
|
+
import {
|
|
70
|
+
boolean,
|
|
71
|
+
customType,
|
|
72
|
+
index,
|
|
73
|
+
integer,
|
|
74
|
+
jsonb,
|
|
75
|
+
numeric,
|
|
76
|
+
pgEnum,
|
|
77
|
+
pgTable,
|
|
78
|
+
text,
|
|
79
|
+
timestamp,
|
|
80
|
+
uniqueIndex
|
|
81
|
+
} from "drizzle-orm/pg-core";
|
|
82
|
+
var VECTOR_DIMENSIONS = 384;
|
|
83
|
+
var HNSW_M = 16;
|
|
84
|
+
var HNSW_EF_CONSTRUCTION = 64;
|
|
85
|
+
var vector = customType({
|
|
86
|
+
dataType() {
|
|
87
|
+
return `vector(${VECTOR_DIMENSIONS})`;
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
var memoryTypeEnum = pgEnum("memory_type", ["interaction", "fact", "summary"]);
|
|
91
|
+
var interactionRoleEnum = pgEnum("interaction_role", ["user", "assistant", "system"]);
|
|
92
|
+
var embeddingOwnerTypeEnum = pgEnum("embedding_owner_type", [
|
|
93
|
+
"interaction",
|
|
94
|
+
"fact",
|
|
95
|
+
"summary"
|
|
96
|
+
]);
|
|
97
|
+
var embeddingStatusEnum = pgEnum("embedding_status", [
|
|
98
|
+
"pending",
|
|
99
|
+
"processing",
|
|
100
|
+
"ready",
|
|
101
|
+
"failed"
|
|
102
|
+
]);
|
|
103
|
+
var shareModeEnum = pgEnum("share_mode", ["reference", "snapshot"]);
|
|
104
|
+
var modelCallTypeEnum = pgEnum("model_call_type", ["embedding", "llm"]);
|
|
105
|
+
var modelCallStatusEnum = pgEnum("model_call_status", [
|
|
106
|
+
"success",
|
|
107
|
+
"failed",
|
|
108
|
+
"timeout",
|
|
109
|
+
"rate_limited",
|
|
110
|
+
"gated",
|
|
111
|
+
"fallback"
|
|
112
|
+
]);
|
|
113
|
+
var auditActionEnum = pgEnum("audit_action", [
|
|
114
|
+
"remember",
|
|
115
|
+
"remember_batch",
|
|
116
|
+
"share",
|
|
117
|
+
"forget",
|
|
118
|
+
"config"
|
|
119
|
+
]);
|
|
120
|
+
var createdAt = timestamp("created_at", { withTimezone: true }).notNull().defaultNow();
|
|
121
|
+
var updatedAt = timestamp("updated_at", { withTimezone: true }).notNull().defaultNow();
|
|
122
|
+
var namespaces = pgTable(
|
|
123
|
+
"namespaces",
|
|
124
|
+
{
|
|
125
|
+
id: text("id").primaryKey(),
|
|
126
|
+
name: text("name").notNull(),
|
|
127
|
+
createdAt,
|
|
128
|
+
updatedAt
|
|
129
|
+
},
|
|
130
|
+
(table) => ({
|
|
131
|
+
nameUnique: uniqueIndex("namespaces_name_unique").on(table.name)
|
|
132
|
+
})
|
|
133
|
+
);
|
|
134
|
+
var interactions = pgTable(
|
|
135
|
+
"interactions",
|
|
136
|
+
{
|
|
137
|
+
id: text("id").primaryKey(),
|
|
138
|
+
namespaceId: text("namespace_id").notNull().references(() => namespaces.id, { onDelete: "cascade" }),
|
|
139
|
+
sessionId: text("session_id").notNull(),
|
|
140
|
+
role: interactionRoleEnum("role").notNull().default("user"),
|
|
141
|
+
content: text("content").notNull(),
|
|
142
|
+
contentHash: text("content_hash").notNull(),
|
|
143
|
+
tokenCount: integer("token_count").notNull().default(0),
|
|
144
|
+
createdByAgent: text("created_by_agent"),
|
|
145
|
+
createdAt
|
|
146
|
+
},
|
|
147
|
+
(table) => ({
|
|
148
|
+
namespaceContentHashUnique: uniqueIndex("interactions_namespace_content_hash_unique").on(
|
|
149
|
+
table.namespaceId,
|
|
150
|
+
table.contentHash
|
|
151
|
+
),
|
|
152
|
+
namespaceSessionCreatedIdx: index("interactions_namespace_session_created_idx").on(
|
|
153
|
+
table.namespaceId,
|
|
154
|
+
table.sessionId,
|
|
155
|
+
table.createdAt
|
|
156
|
+
)
|
|
157
|
+
})
|
|
158
|
+
);
|
|
159
|
+
var sessionSummaries = pgTable(
|
|
160
|
+
"session_summaries",
|
|
161
|
+
{
|
|
162
|
+
id: text("id").primaryKey(),
|
|
163
|
+
namespaceId: text("namespace_id").notNull().references(() => namespaces.id, { onDelete: "cascade" }),
|
|
164
|
+
sessionId: text("session_id").notNull(),
|
|
165
|
+
content: text("content").notNull(),
|
|
166
|
+
version: integer("version").notNull().default(1),
|
|
167
|
+
tokenCount: integer("token_count").notNull().default(0),
|
|
168
|
+
createdByAgent: text("created_by_agent"),
|
|
169
|
+
/** See facts.fileBacked. */
|
|
170
|
+
fileBacked: boolean("file_backed").notNull().default(false),
|
|
171
|
+
createdAt
|
|
172
|
+
},
|
|
173
|
+
(table) => ({
|
|
174
|
+
namespaceSessionVersionIdx: index("session_summaries_namespace_session_version_idx").on(
|
|
175
|
+
table.namespaceId,
|
|
176
|
+
table.sessionId,
|
|
177
|
+
table.version
|
|
178
|
+
)
|
|
179
|
+
})
|
|
180
|
+
);
|
|
181
|
+
var facts = pgTable(
|
|
182
|
+
"facts",
|
|
183
|
+
{
|
|
184
|
+
id: text("id").primaryKey(),
|
|
185
|
+
namespaceId: text("namespace_id").notNull().references(() => namespaces.id, { onDelete: "cascade" }),
|
|
186
|
+
subject: text("subject").notNull(),
|
|
187
|
+
predicate: text("predicate").notNull(),
|
|
188
|
+
object: text("object").notNull(),
|
|
189
|
+
propositionKey: text("proposition_key"),
|
|
190
|
+
content: text("content").notNull(),
|
|
191
|
+
contentHash: text("content_hash").notNull(),
|
|
192
|
+
confidence: numeric("confidence", { precision: 5, scale: 4 }).notNull().default("1"),
|
|
193
|
+
/**
|
|
194
|
+
* Historical provenance only — deliberately not a foreign key. Interactions
|
|
195
|
+
* are ephemeral DB rows; Markdown facts must re-import after a rebuild that
|
|
196
|
+
* lost them, and the ID must survive so `sourceDeleted` can be tracked.
|
|
197
|
+
*/
|
|
198
|
+
sourceInteractionId: text("source_interaction_id"),
|
|
199
|
+
sourceDeleted: boolean("source_deleted").notNull().default(false),
|
|
200
|
+
supersededBy: text("superseded_by"),
|
|
201
|
+
/**
|
|
202
|
+
* True once a reindex imported this row from its Markdown file. Only such
|
|
203
|
+
* rows are deleted when the file disappears; DB-only legacy rows are kept.
|
|
204
|
+
*/
|
|
205
|
+
fileBacked: boolean("file_backed").notNull().default(false),
|
|
206
|
+
embeddingModel: text("embedding_model"),
|
|
207
|
+
createdByAgent: text("created_by_agent"),
|
|
208
|
+
createdAt,
|
|
209
|
+
updatedAt
|
|
210
|
+
},
|
|
211
|
+
(table) => ({
|
|
212
|
+
namespaceContentHashUnique: uniqueIndex("facts_namespace_content_hash_unique").on(
|
|
213
|
+
table.namespaceId,
|
|
214
|
+
table.contentHash
|
|
215
|
+
),
|
|
216
|
+
namespaceSubjectIdx: index("facts_namespace_subject_idx").on(table.namespaceId, table.subject),
|
|
217
|
+
sourceInteractionIdx: index("facts_source_interaction_idx").on(table.sourceInteractionId)
|
|
218
|
+
})
|
|
219
|
+
);
|
|
220
|
+
var embeddings = pgTable(
|
|
221
|
+
"embeddings",
|
|
222
|
+
{
|
|
223
|
+
id: text("id").primaryKey(),
|
|
224
|
+
namespaceId: text("namespace_id").notNull().references(() => namespaces.id, { onDelete: "cascade" }),
|
|
225
|
+
ownerType: embeddingOwnerTypeEnum("owner_type").notNull(),
|
|
226
|
+
ownerId: text("owner_id").notNull(),
|
|
227
|
+
vector: vector("vector").notNull(),
|
|
228
|
+
embeddingModel: text("embedding_model").notNull(),
|
|
229
|
+
embeddingSpace: text("embedding_space"),
|
|
230
|
+
status: embeddingStatusEnum("status").notNull().default("pending"),
|
|
231
|
+
attempts: integer("attempts").notNull().default(0),
|
|
232
|
+
lastError: text("last_error"),
|
|
233
|
+
/** Lease deadline while status='processing'; null otherwise. */
|
|
234
|
+
leaseUntil: timestamp("lease_until", { withTimezone: true }),
|
|
235
|
+
leaseToken: text("lease_token"),
|
|
236
|
+
createdAt,
|
|
237
|
+
updatedAt
|
|
238
|
+
},
|
|
239
|
+
(table) => ({
|
|
240
|
+
ownerUnique: uniqueIndex("embeddings_owner_unique").on(table.ownerType, table.ownerId),
|
|
241
|
+
namespaceModelStatusIdx: index("embeddings_namespace_model_status_idx").on(
|
|
242
|
+
table.namespaceId,
|
|
243
|
+
table.embeddingModel,
|
|
244
|
+
table.status
|
|
245
|
+
),
|
|
246
|
+
namespaceSpaceStatusIdx: index("embeddings_namespace_space_status_idx").on(
|
|
247
|
+
table.namespaceId,
|
|
248
|
+
table.embeddingSpace,
|
|
249
|
+
table.status
|
|
250
|
+
),
|
|
251
|
+
vectorHnswIdx: index("embeddings_vector_hnsw_idx").using(
|
|
252
|
+
"hnsw",
|
|
253
|
+
sql`${table.vector} vector_cosine_ops`
|
|
254
|
+
)
|
|
255
|
+
})
|
|
256
|
+
);
|
|
257
|
+
var shares = pgTable(
|
|
258
|
+
"shares",
|
|
259
|
+
{
|
|
260
|
+
id: text("id").primaryKey(),
|
|
261
|
+
fromNamespaceId: text("from_namespace_id").notNull().references(() => namespaces.id, { onDelete: "cascade" }),
|
|
262
|
+
toNamespaceId: text("to_namespace_id").notNull().references(() => namespaces.id, { onDelete: "cascade" }),
|
|
263
|
+
sourceFactId: text("source_fact_id").notNull().references(() => facts.id, { onDelete: "cascade" }),
|
|
264
|
+
snapshotFactId: text("snapshot_fact_id").references(() => facts.id, { onDelete: "set null" }),
|
|
265
|
+
mode: shareModeEnum("mode").notNull().default("reference"),
|
|
266
|
+
tombstoned: boolean("tombstoned").notNull().default(false),
|
|
267
|
+
createdByAgent: text("created_by_agent"),
|
|
268
|
+
createdAt
|
|
269
|
+
},
|
|
270
|
+
(table) => ({
|
|
271
|
+
shareUnique: uniqueIndex("shares_target_source_unique").on(
|
|
272
|
+
table.toNamespaceId,
|
|
273
|
+
table.sourceFactId,
|
|
274
|
+
table.mode
|
|
275
|
+
),
|
|
276
|
+
fromNamespaceIdx: index("shares_from_namespace_idx").on(table.fromNamespaceId)
|
|
277
|
+
})
|
|
278
|
+
);
|
|
279
|
+
var checkpoints = pgTable(
|
|
280
|
+
"checkpoints",
|
|
281
|
+
{
|
|
282
|
+
id: text("id").primaryKey(),
|
|
283
|
+
namespaceId: text("namespace_id").notNull().references(() => namespaces.id, { onDelete: "cascade" }),
|
|
284
|
+
threadId: text("thread_id").notNull(),
|
|
285
|
+
checkpointId: text("checkpoint_id").notNull(),
|
|
286
|
+
parentId: text("parent_id"),
|
|
287
|
+
channelValues: jsonb("channel_values").notNull().default(sql`'{}'::jsonb`),
|
|
288
|
+
metadata: jsonb("metadata").notNull().default(sql`'{}'::jsonb`),
|
|
289
|
+
createdAt
|
|
290
|
+
},
|
|
291
|
+
(table) => ({
|
|
292
|
+
checkpointUnique: uniqueIndex("checkpoints_thread_checkpoint_unique").on(
|
|
293
|
+
table.namespaceId,
|
|
294
|
+
table.threadId,
|
|
295
|
+
table.checkpointId
|
|
296
|
+
)
|
|
297
|
+
})
|
|
298
|
+
);
|
|
299
|
+
var modelCalls = pgTable(
|
|
300
|
+
"model_calls",
|
|
301
|
+
{
|
|
302
|
+
id: text("id").primaryKey(),
|
|
303
|
+
namespaceId: text("namespace_id").references(() => namespaces.id, { onDelete: "set null" }),
|
|
304
|
+
model: text("model").notNull(),
|
|
305
|
+
type: modelCallTypeEnum("type").notNull(),
|
|
306
|
+
latencyMs: integer("latency_ms"),
|
|
307
|
+
tokens: integer("tokens"),
|
|
308
|
+
status: modelCallStatusEnum("status").notNull(),
|
|
309
|
+
error: text("error"),
|
|
310
|
+
metadata: jsonb("metadata").notNull().default(sql`'{}'::jsonb`),
|
|
311
|
+
createdAt
|
|
312
|
+
},
|
|
313
|
+
(table) => ({
|
|
314
|
+
modelCreatedIdx: index("model_calls_model_created_idx").on(table.model, table.createdAt),
|
|
315
|
+
namespaceCreatedIdx: index("model_calls_namespace_created_idx").on(
|
|
316
|
+
table.namespaceId,
|
|
317
|
+
table.createdAt
|
|
318
|
+
)
|
|
319
|
+
})
|
|
320
|
+
);
|
|
321
|
+
var auditLog = pgTable(
|
|
322
|
+
"audit_log",
|
|
323
|
+
{
|
|
324
|
+
id: text("id").primaryKey(),
|
|
325
|
+
namespaceId: text("namespace_id").references(() => namespaces.id, { onDelete: "set null" }),
|
|
326
|
+
action: auditActionEnum("action").notNull(),
|
|
327
|
+
entityType: text("entity_type").notNull(),
|
|
328
|
+
entityId: text("entity_id"),
|
|
329
|
+
createdByAgent: text("created_by_agent"),
|
|
330
|
+
metadata: jsonb("metadata").notNull().default(sql`'{}'::jsonb`),
|
|
331
|
+
createdAt
|
|
332
|
+
},
|
|
333
|
+
(table) => ({
|
|
334
|
+
namespaceActionCreatedIdx: index("audit_log_namespace_action_created_idx").on(
|
|
335
|
+
table.namespaceId,
|
|
336
|
+
table.action,
|
|
337
|
+
table.createdAt
|
|
338
|
+
)
|
|
339
|
+
})
|
|
340
|
+
);
|
|
341
|
+
|
|
342
|
+
// src/db/bootstrap.ts
|
|
343
|
+
async function enablePgvector(sql3) {
|
|
344
|
+
await sql3`create extension if not exists vector`;
|
|
345
|
+
}
|
|
346
|
+
async function checkPgvector(sql3) {
|
|
347
|
+
const result = await sql3`
|
|
348
|
+
select extversion from pg_extension where extname = 'vector'
|
|
349
|
+
`;
|
|
350
|
+
return {
|
|
351
|
+
available: result.length > 0,
|
|
352
|
+
installedVersion: result[0]?.extversion ?? null
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
async function bootstrapDatabase(sql3) {
|
|
356
|
+
await enablePgvector(sql3);
|
|
357
|
+
return checkPgvector(sql3);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// src/db/client.ts
|
|
361
|
+
import { drizzle } from "drizzle-orm/postgres-js";
|
|
362
|
+
import postgres from "postgres";
|
|
363
|
+
function createDatabaseClient(config) {
|
|
364
|
+
const sql3 = postgres(config.postgresUrl, {
|
|
365
|
+
max: config.maxConnections ?? 10,
|
|
366
|
+
...config.connectTimeoutSeconds !== void 0 ? { connect_timeout: config.connectTimeoutSeconds } : {},
|
|
367
|
+
// postgres.js logs NOTICE messages (e.g. "extension already exists,
|
|
368
|
+
// skipping") to stdout by default. Over the MCP stdio transport stdout
|
|
369
|
+
// carries JSON-RPC, so any notice corrupts the stream and the client
|
|
370
|
+
// reports "Unexpected token … is not valid JSON". Route notices to
|
|
371
|
+
// stderr instead, where logs belong.
|
|
372
|
+
onnotice: (notice) => {
|
|
373
|
+
process.stderr.write(`${JSON.stringify(notice)}
|
|
374
|
+
`);
|
|
375
|
+
}
|
|
376
|
+
});
|
|
377
|
+
return {
|
|
378
|
+
sql: sql3,
|
|
379
|
+
db: drizzle(sql3, { schema: schema_exports }),
|
|
380
|
+
async close() {
|
|
381
|
+
await sql3.end();
|
|
382
|
+
}
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
async function pingDatabase(client) {
|
|
386
|
+
const result = await client.sql`select 1 as ok`;
|
|
387
|
+
return result[0]?.ok === 1;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// src/db/migrator.ts
|
|
391
|
+
import { existsSync } from "fs";
|
|
392
|
+
import { dirname, resolve } from "path";
|
|
393
|
+
import { fileURLToPath } from "url";
|
|
394
|
+
import { migrate } from "drizzle-orm/postgres-js/migrator";
|
|
395
|
+
function resolveMigrationsFolder() {
|
|
396
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
397
|
+
const candidates = [
|
|
398
|
+
// Co-located (src/db/ -> src/db/migrations)
|
|
399
|
+
resolve(here, "migrations"),
|
|
400
|
+
// Built dist (dist/<chunk>.js -> ../src/db/migrations relative paths)
|
|
401
|
+
resolve(here, "..", "src", "db", "migrations"),
|
|
402
|
+
resolve(here, "..", "..", "src", "db", "migrations"),
|
|
403
|
+
resolve(here, "..", "..", "..", "src", "db", "migrations"),
|
|
404
|
+
resolve(here, "..", "..", "..", "..", "src", "db", "migrations"),
|
|
405
|
+
// CWD fallback (developer running from repo root)
|
|
406
|
+
resolve(process.cwd(), "src", "db", "migrations")
|
|
407
|
+
];
|
|
408
|
+
for (const candidate of candidates) {
|
|
409
|
+
if (existsSync(candidate)) return candidate;
|
|
410
|
+
}
|
|
411
|
+
throw new Error(
|
|
412
|
+
`Could not locate migrations folder. Tried:
|
|
413
|
+
${candidates.join("\n ")}
|
|
414
|
+
If this is an npm-installed package, make sure src/db/migrations was included in the tarball.`
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
async function runMigrations(client) {
|
|
418
|
+
const migrationsFolder = resolveMigrationsFolder();
|
|
419
|
+
await migrate(client.db, { migrationsFolder });
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// src/db/repository.ts
|
|
423
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
424
|
+
import { createHash } from "crypto";
|
|
425
|
+
import { and, desc, eq, inArray, isNull, lt, ne, or, sql as sql2 } from "drizzle-orm";
|
|
426
|
+
|
|
427
|
+
// src/core/memory-search.ts
|
|
428
|
+
function candidateLimit(limit) {
|
|
429
|
+
if (!Number.isFinite(limit)) return 256;
|
|
430
|
+
return Math.max(1, Math.min(1024, Math.trunc(limit)));
|
|
431
|
+
}
|
|
432
|
+
function validSearchVector(vector2) {
|
|
433
|
+
return vector2.length === VECTOR_DIMENSIONS && vector2.every((value) => Number.isFinite(Math.fround(value))) && vector2.some((value) => value !== 0);
|
|
434
|
+
}
|
|
435
|
+
var QUERY_STOPWORDS = /* @__PURE__ */ new Set([
|
|
436
|
+
"a",
|
|
437
|
+
"about",
|
|
438
|
+
"after",
|
|
439
|
+
"again",
|
|
440
|
+
"all",
|
|
441
|
+
"also",
|
|
442
|
+
"always",
|
|
443
|
+
"an",
|
|
444
|
+
"and",
|
|
445
|
+
"any",
|
|
446
|
+
"are",
|
|
447
|
+
"as",
|
|
448
|
+
"at",
|
|
449
|
+
"back",
|
|
450
|
+
"be",
|
|
451
|
+
"been",
|
|
452
|
+
"before",
|
|
453
|
+
"being",
|
|
454
|
+
"but",
|
|
455
|
+
"by",
|
|
456
|
+
"can",
|
|
457
|
+
"could",
|
|
458
|
+
"did",
|
|
459
|
+
"do",
|
|
460
|
+
"does",
|
|
461
|
+
"done",
|
|
462
|
+
"each",
|
|
463
|
+
"even",
|
|
464
|
+
"ever",
|
|
465
|
+
"for",
|
|
466
|
+
"from",
|
|
467
|
+
"get",
|
|
468
|
+
"gets",
|
|
469
|
+
"got",
|
|
470
|
+
"had",
|
|
471
|
+
"has",
|
|
472
|
+
"have",
|
|
473
|
+
"here",
|
|
474
|
+
"how",
|
|
475
|
+
"i",
|
|
476
|
+
"if",
|
|
477
|
+
"in",
|
|
478
|
+
"into",
|
|
479
|
+
"is",
|
|
480
|
+
"it",
|
|
481
|
+
"its",
|
|
482
|
+
"just",
|
|
483
|
+
"like",
|
|
484
|
+
"make",
|
|
485
|
+
"me",
|
|
486
|
+
"more",
|
|
487
|
+
"most",
|
|
488
|
+
"my",
|
|
489
|
+
"never",
|
|
490
|
+
"no",
|
|
491
|
+
"not",
|
|
492
|
+
"now",
|
|
493
|
+
"of",
|
|
494
|
+
"on",
|
|
495
|
+
"only",
|
|
496
|
+
"or",
|
|
497
|
+
"other",
|
|
498
|
+
"our",
|
|
499
|
+
"out",
|
|
500
|
+
"over",
|
|
501
|
+
"please",
|
|
502
|
+
"really",
|
|
503
|
+
"same",
|
|
504
|
+
"should",
|
|
505
|
+
"so",
|
|
506
|
+
"some",
|
|
507
|
+
"sometimes",
|
|
508
|
+
"still",
|
|
509
|
+
"such",
|
|
510
|
+
"than",
|
|
511
|
+
"that",
|
|
512
|
+
"the",
|
|
513
|
+
"their",
|
|
514
|
+
"them",
|
|
515
|
+
"then",
|
|
516
|
+
"there",
|
|
517
|
+
"these",
|
|
518
|
+
"they",
|
|
519
|
+
"this",
|
|
520
|
+
"those",
|
|
521
|
+
"through",
|
|
522
|
+
"to",
|
|
523
|
+
"too",
|
|
524
|
+
"under",
|
|
525
|
+
"up",
|
|
526
|
+
"us",
|
|
527
|
+
"very",
|
|
528
|
+
"was",
|
|
529
|
+
"we",
|
|
530
|
+
"were",
|
|
531
|
+
"what",
|
|
532
|
+
"when",
|
|
533
|
+
"where",
|
|
534
|
+
"which",
|
|
535
|
+
"while",
|
|
536
|
+
"who",
|
|
537
|
+
"why",
|
|
538
|
+
"will",
|
|
539
|
+
"with",
|
|
540
|
+
"would",
|
|
541
|
+
"yet",
|
|
542
|
+
"you",
|
|
543
|
+
"your"
|
|
544
|
+
]);
|
|
545
|
+
function lexicalTerms(query) {
|
|
546
|
+
return [
|
|
547
|
+
...new Set(
|
|
548
|
+
query.slice(0, 4096).toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? []
|
|
549
|
+
)
|
|
550
|
+
].filter((term) => !QUERY_STOPWORDS.has(term)).slice(0, 32).map((term) => term.slice(0, 64));
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// src/providers/embedding-space.ts
|
|
554
|
+
function embeddingSpace(response) {
|
|
555
|
+
const space = {
|
|
556
|
+
model: response.model,
|
|
557
|
+
revision: response.revision ?? "unversioned",
|
|
558
|
+
schema: response.schema ?? "provider-v1",
|
|
559
|
+
dimensions: response.dimensions
|
|
560
|
+
};
|
|
561
|
+
if (!space.model || !space.revision || !space.schema || !Number.isSafeInteger(space.dimensions) || space.dimensions <= 0) {
|
|
562
|
+
throw new Error("Invalid embedding space identity.");
|
|
563
|
+
}
|
|
564
|
+
return space;
|
|
565
|
+
}
|
|
566
|
+
function embeddingSpaceKey(space) {
|
|
567
|
+
return JSON.stringify([space.model, space.revision, space.schema, space.dimensions]);
|
|
568
|
+
}
|
|
569
|
+
function validEmbeddingVector(vector2, dimensions) {
|
|
570
|
+
return dimensions > 0 && vector2.length === dimensions && vector2.every(Number.isFinite);
|
|
571
|
+
}
|
|
572
|
+
function validateEmbeddingResponse(response, count) {
|
|
573
|
+
embeddingSpace(response);
|
|
574
|
+
if (response.vectors.length !== count || response.vectors.some((vector2) => !validEmbeddingVector(vector2, response.dimensions))) {
|
|
575
|
+
throw new Error("Provider returned invalid embedding dimensions, values, or batch size.");
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
// src/db/repository.ts
|
|
580
|
+
var PostgresMemoryRepository = class {
|
|
581
|
+
client;
|
|
582
|
+
sessionTransaction = new AsyncLocalStorage();
|
|
583
|
+
constructor(client) {
|
|
584
|
+
this.client = client;
|
|
585
|
+
}
|
|
586
|
+
get db() {
|
|
587
|
+
return this.sessionTransaction.getStore() ?? this.client.db;
|
|
588
|
+
}
|
|
589
|
+
async ensureNamespace(name) {
|
|
590
|
+
const existing = await this.getNamespaceByName(name);
|
|
591
|
+
if (existing) return existing;
|
|
592
|
+
const id = generateEntityId("ns");
|
|
593
|
+
await this.db.insert(namespaces).values({ id, name }).onConflictDoNothing({ target: namespaces.name });
|
|
594
|
+
const row = await this.getNamespaceByName(name);
|
|
595
|
+
if (!row) throw new Error(`Failed to ensure namespace ${name}.`);
|
|
596
|
+
return row;
|
|
597
|
+
}
|
|
598
|
+
async getNamespaceByName(name) {
|
|
599
|
+
const rows = await this.db.select({ id: namespaces.id, name: namespaces.name }).from(namespaces).where(eq(namespaces.name, name)).limit(1);
|
|
600
|
+
return rows[0] ?? null;
|
|
601
|
+
}
|
|
602
|
+
async withSessionLock(namespaceId, sessionId, fn) {
|
|
603
|
+
const [k1, k2] = advisoryLockKeys(namespaceId, sessionId);
|
|
604
|
+
return this.client.db.transaction(async (tx) => {
|
|
605
|
+
await tx.execute(sql2`select pg_advisory_xact_lock(${k1}, ${k2})`);
|
|
606
|
+
return this.sessionTransaction.run(tx, fn);
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
async writeInteraction(input) {
|
|
610
|
+
return this.db.transaction(async (tx) => {
|
|
611
|
+
const existing = await tx.select({ id: interactions.id, sessionId: interactions.sessionId }).from(interactions).where(
|
|
612
|
+
and(
|
|
613
|
+
eq(interactions.namespaceId, input.interaction.namespaceId),
|
|
614
|
+
eq(interactions.contentHash, input.interaction.contentHash)
|
|
615
|
+
)
|
|
616
|
+
).limit(1);
|
|
617
|
+
const existingRow = existing[0];
|
|
618
|
+
if (existingRow) {
|
|
619
|
+
return {
|
|
620
|
+
id: existingRow.id,
|
|
621
|
+
embeddingId: "",
|
|
622
|
+
sessionId: existingRow.sessionId,
|
|
623
|
+
deduplicated: true
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
await tx.insert(interactions).values({
|
|
627
|
+
id: input.interaction.id,
|
|
628
|
+
namespaceId: input.interaction.namespaceId,
|
|
629
|
+
sessionId: input.interaction.sessionId,
|
|
630
|
+
content: input.interaction.content,
|
|
631
|
+
contentHash: input.interaction.contentHash,
|
|
632
|
+
tokenCount: input.interaction.tokenCount,
|
|
633
|
+
createdByAgent: input.interaction.createdByAgent ?? null
|
|
634
|
+
});
|
|
635
|
+
await tx.insert(embeddings).values({
|
|
636
|
+
id: input.embedding.id,
|
|
637
|
+
namespaceId: input.embedding.namespaceId,
|
|
638
|
+
ownerType: input.embedding.ownerType,
|
|
639
|
+
ownerId: input.embedding.ownerId,
|
|
640
|
+
vector: zeroVectorLiteral(),
|
|
641
|
+
embeddingModel: input.embedding.embeddingModel,
|
|
642
|
+
status: "pending"
|
|
643
|
+
});
|
|
644
|
+
await tx.insert(auditLog).values(toAuditRow(input.audit));
|
|
645
|
+
return {
|
|
646
|
+
id: input.interaction.id,
|
|
647
|
+
embeddingId: input.embedding.id,
|
|
648
|
+
sessionId: input.interaction.sessionId,
|
|
649
|
+
deduplicated: false
|
|
650
|
+
};
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
async writeFact(input) {
|
|
654
|
+
return this.db.transaction(async (tx) => {
|
|
655
|
+
const existing = await tx.select({ id: facts.id }).from(facts).where(
|
|
656
|
+
and(
|
|
657
|
+
eq(facts.namespaceId, input.fact.namespaceId),
|
|
658
|
+
eq(facts.contentHash, input.fact.contentHash)
|
|
659
|
+
)
|
|
660
|
+
).limit(1);
|
|
661
|
+
const existingRow = existing[0];
|
|
662
|
+
if (existingRow) {
|
|
663
|
+
return {
|
|
664
|
+
id: existingRow.id,
|
|
665
|
+
embeddingId: "",
|
|
666
|
+
sessionId: "",
|
|
667
|
+
deduplicated: true
|
|
668
|
+
};
|
|
669
|
+
}
|
|
670
|
+
const sameId = await tx.select({ id: facts.id }).from(facts).where(eq(facts.id, input.fact.id)).limit(1);
|
|
671
|
+
if (sameId[0]) {
|
|
672
|
+
return { id: sameId[0].id, embeddingId: "", sessionId: "", deduplicated: true };
|
|
673
|
+
}
|
|
674
|
+
await tx.insert(facts).values({
|
|
675
|
+
id: input.fact.id,
|
|
676
|
+
namespaceId: input.fact.namespaceId,
|
|
677
|
+
subject: input.fact.subject,
|
|
678
|
+
predicate: input.fact.predicate,
|
|
679
|
+
object: input.fact.object,
|
|
680
|
+
content: input.fact.content,
|
|
681
|
+
contentHash: input.fact.contentHash,
|
|
682
|
+
propositionKey: input.fact.propositionKey ?? null,
|
|
683
|
+
confidence: String(input.fact.confidence ?? 1),
|
|
684
|
+
sourceDeleted: input.fact.sourceDeleted ?? false,
|
|
685
|
+
supersededBy: input.fact.supersededBy ?? null,
|
|
686
|
+
...input.fact.createdAt ? { createdAt: input.fact.createdAt } : {},
|
|
687
|
+
sourceInteractionId: input.fact.sourceInteractionId ?? null,
|
|
688
|
+
createdByAgent: input.fact.createdByAgent ?? null
|
|
689
|
+
});
|
|
690
|
+
await tx.insert(embeddings).values({
|
|
691
|
+
id: input.embedding.id,
|
|
692
|
+
namespaceId: input.embedding.namespaceId,
|
|
693
|
+
ownerType: input.embedding.ownerType,
|
|
694
|
+
ownerId: input.embedding.ownerId,
|
|
695
|
+
vector: zeroVectorLiteral(),
|
|
696
|
+
embeddingModel: input.embedding.embeddingModel,
|
|
697
|
+
status: "pending"
|
|
698
|
+
});
|
|
699
|
+
await tx.insert(auditLog).values(toAuditRow(input.audit));
|
|
700
|
+
return {
|
|
701
|
+
id: input.fact.id,
|
|
702
|
+
embeddingId: input.embedding.id,
|
|
703
|
+
sessionId: "",
|
|
704
|
+
deduplicated: false
|
|
705
|
+
};
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
async writeSummary(input) {
|
|
709
|
+
return this.db.transaction(async (tx) => {
|
|
710
|
+
const existing = await tx.select({ id: sessionSummaries.id, sessionId: sessionSummaries.sessionId }).from(sessionSummaries).where(eq(sessionSummaries.id, input.summary.id)).limit(1);
|
|
711
|
+
const existingRow = existing[0];
|
|
712
|
+
if (existingRow) {
|
|
713
|
+
return {
|
|
714
|
+
id: existingRow.id,
|
|
715
|
+
embeddingId: "",
|
|
716
|
+
sessionId: existingRow.sessionId,
|
|
717
|
+
deduplicated: true
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
await tx.insert(sessionSummaries).values({
|
|
721
|
+
id: input.summary.id,
|
|
722
|
+
namespaceId: input.summary.namespaceId,
|
|
723
|
+
sessionId: input.summary.sessionId,
|
|
724
|
+
content: input.summary.content,
|
|
725
|
+
version: input.summary.version ?? 1,
|
|
726
|
+
tokenCount: input.summary.tokenCount,
|
|
727
|
+
createdByAgent: input.summary.createdByAgent ?? null
|
|
728
|
+
});
|
|
729
|
+
await tx.insert(embeddings).values({
|
|
730
|
+
id: input.embedding.id,
|
|
731
|
+
namespaceId: input.embedding.namespaceId,
|
|
732
|
+
ownerType: input.embedding.ownerType,
|
|
733
|
+
ownerId: input.embedding.ownerId,
|
|
734
|
+
vector: zeroVectorLiteral(),
|
|
735
|
+
embeddingModel: input.embedding.embeddingModel,
|
|
736
|
+
status: "pending"
|
|
737
|
+
});
|
|
738
|
+
await tx.insert(auditLog).values(toAuditRow(input.audit));
|
|
739
|
+
return {
|
|
740
|
+
id: input.summary.id,
|
|
741
|
+
embeddingId: input.embedding.id,
|
|
742
|
+
sessionId: input.summary.sessionId,
|
|
743
|
+
deduplicated: false
|
|
744
|
+
};
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
async upsertFactFromFile(input) {
|
|
748
|
+
const fact = input.fact;
|
|
749
|
+
return this.db.transaction(async (tx) => {
|
|
750
|
+
const [existing] = await tx.select().from(facts).where(eq(facts.id, fact.id)).for("update").limit(1);
|
|
751
|
+
if (existing && existing.namespaceId !== fact.namespaceId) {
|
|
752
|
+
throw new Error(`fact ${fact.id} already belongs to another namespace`);
|
|
753
|
+
}
|
|
754
|
+
const [duplicate] = await tx.select({ id: facts.id }).from(facts).where(
|
|
755
|
+
and(
|
|
756
|
+
eq(facts.namespaceId, fact.namespaceId),
|
|
757
|
+
eq(facts.contentHash, fact.contentHash),
|
|
758
|
+
ne(facts.id, fact.id)
|
|
759
|
+
)
|
|
760
|
+
).limit(1);
|
|
761
|
+
if (duplicate) throw new Error(`content duplicates fact ${duplicate.id}`);
|
|
762
|
+
const values = {
|
|
763
|
+
subject: fact.subject,
|
|
764
|
+
predicate: fact.predicate,
|
|
765
|
+
object: fact.object,
|
|
766
|
+
content: fact.content,
|
|
767
|
+
contentHash: fact.contentHash,
|
|
768
|
+
confidence: String(fact.confidence),
|
|
769
|
+
sourceInteractionId: fact.sourceInteractionId ?? null,
|
|
770
|
+
sourceDeleted: fact.sourceDeleted,
|
|
771
|
+
supersededBy: fact.supersededBy,
|
|
772
|
+
propositionKey: fact.propositionKey ?? null,
|
|
773
|
+
createdByAgent: fact.createdByAgent ?? null,
|
|
774
|
+
createdAt: fact.createdAt,
|
|
775
|
+
fileBacked: true
|
|
776
|
+
};
|
|
777
|
+
if (!existing) {
|
|
778
|
+
await tx.insert(facts).values({ id: fact.id, namespaceId: fact.namespaceId, ...values });
|
|
779
|
+
await upsertPendingEmbedding(tx, input.embedding, true);
|
|
780
|
+
await tx.insert(auditLog).values(toAuditRow(input.audit));
|
|
781
|
+
return { status: "inserted" };
|
|
782
|
+
}
|
|
783
|
+
const contentChanged = existing.content !== fact.content;
|
|
784
|
+
const metadataChanged = existing.subject !== values.subject || existing.predicate !== values.predicate || existing.object !== values.object || Number(existing.confidence) !== Number(fact.confidence.toFixed(4)) || existing.sourceInteractionId !== values.sourceInteractionId || existing.sourceDeleted !== values.sourceDeleted || existing.supersededBy !== values.supersededBy || existing.propositionKey !== values.propositionKey || existing.createdByAgent !== values.createdByAgent || existing.createdAt.getTime() !== values.createdAt.getTime();
|
|
785
|
+
if (contentChanged || metadataChanged || !existing.fileBacked) {
|
|
786
|
+
await tx.update(facts).set({ ...values, updatedAt: sql2`now()` }).where(and(eq(facts.namespaceId, fact.namespaceId), eq(facts.id, fact.id)));
|
|
787
|
+
}
|
|
788
|
+
await upsertPendingEmbedding(tx, input.embedding, contentChanged);
|
|
789
|
+
if (!contentChanged && !metadataChanged) return { status: "unchanged" };
|
|
790
|
+
await tx.insert(auditLog).values(toAuditRow(input.audit));
|
|
791
|
+
return { status: "updated" };
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
async upsertSummaryFromFile(input) {
|
|
795
|
+
const summary = input.summary;
|
|
796
|
+
return this.db.transaction(async (tx) => {
|
|
797
|
+
const [existing] = await tx.select().from(sessionSummaries).where(eq(sessionSummaries.id, summary.id)).for("update").limit(1);
|
|
798
|
+
if (existing && existing.namespaceId !== summary.namespaceId) {
|
|
799
|
+
throw new Error(`summary ${summary.id} already belongs to another namespace`);
|
|
800
|
+
}
|
|
801
|
+
const values = {
|
|
802
|
+
sessionId: summary.sessionId,
|
|
803
|
+
content: summary.content,
|
|
804
|
+
version: summary.version ?? 1,
|
|
805
|
+
tokenCount: summary.tokenCount,
|
|
806
|
+
createdByAgent: summary.createdByAgent ?? null,
|
|
807
|
+
createdAt: summary.createdAt,
|
|
808
|
+
fileBacked: true
|
|
809
|
+
};
|
|
810
|
+
if (!existing) {
|
|
811
|
+
await tx.insert(sessionSummaries).values({
|
|
812
|
+
id: summary.id,
|
|
813
|
+
namespaceId: summary.namespaceId,
|
|
814
|
+
...values
|
|
815
|
+
});
|
|
816
|
+
await upsertPendingEmbedding(tx, input.embedding, true);
|
|
817
|
+
await tx.insert(auditLog).values(toAuditRow(input.audit));
|
|
818
|
+
return { status: "inserted" };
|
|
819
|
+
}
|
|
820
|
+
const contentChanged = existing.content !== summary.content;
|
|
821
|
+
const metadataChanged = existing.sessionId !== values.sessionId || existing.version !== values.version || existing.tokenCount !== values.tokenCount || existing.createdByAgent !== values.createdByAgent || existing.createdAt.getTime() !== values.createdAt.getTime();
|
|
822
|
+
if (contentChanged || metadataChanged || !existing.fileBacked) {
|
|
823
|
+
await tx.update(sessionSummaries).set(values).where(
|
|
824
|
+
and(
|
|
825
|
+
eq(sessionSummaries.namespaceId, summary.namespaceId),
|
|
826
|
+
eq(sessionSummaries.id, summary.id)
|
|
827
|
+
)
|
|
828
|
+
);
|
|
829
|
+
}
|
|
830
|
+
await upsertPendingEmbedding(tx, input.embedding, contentChanged);
|
|
831
|
+
if (!contentChanged && !metadataChanged) return { status: "unchanged" };
|
|
832
|
+
await tx.insert(auditLog).values(toAuditRow(input.audit));
|
|
833
|
+
return { status: "updated" };
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
async listFileBackedIds(namespaceId) {
|
|
837
|
+
const [factRows, summaryRows] = await Promise.all([
|
|
838
|
+
this.db.select({ id: facts.id }).from(facts).where(and(eq(facts.namespaceId, namespaceId), eq(facts.fileBacked, true))),
|
|
839
|
+
this.db.select({ id: sessionSummaries.id }).from(sessionSummaries).where(
|
|
840
|
+
and(eq(sessionSummaries.namespaceId, namespaceId), eq(sessionSummaries.fileBacked, true))
|
|
841
|
+
)
|
|
842
|
+
]);
|
|
843
|
+
return { facts: factRows.map((row) => row.id), summaries: summaryRows.map((row) => row.id) };
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* Claim up to `limit` embeddable jobs.
|
|
847
|
+
*
|
|
848
|
+
* A claim is a LEASE, not just a row lock: claimed rows move to
|
|
849
|
+
* status='processing' with a `lease_until` deadline, so protection survives
|
|
850
|
+
* the committing transaction. Concurrent workers (e.g. Claude Desktop and
|
|
851
|
+
* Codex both running `codebuddy serve`) therefore never process the same
|
|
852
|
+
* item twice. Expired leases are re-claimable, so a crashed worker's work is
|
|
853
|
+
* picked up again instead of being stuck forever.
|
|
854
|
+
*/
|
|
855
|
+
async claimPendingEmbeddings(limit) {
|
|
856
|
+
const rows = await this.client.sql`
|
|
857
|
+
with claimed as (
|
|
858
|
+
select id from embeddings
|
|
859
|
+
where status = 'pending'
|
|
860
|
+
or (status = 'processing' and (lease_until is null or lease_until < now()))
|
|
861
|
+
order by created_at asc
|
|
862
|
+
for update skip locked
|
|
863
|
+
limit ${limit}
|
|
864
|
+
)
|
|
865
|
+
update embeddings e
|
|
866
|
+
set status = 'processing',
|
|
867
|
+
lease_until = now() + interval '120 seconds',
|
|
868
|
+
lease_token = ${generateEntityId("lease")},
|
|
869
|
+
updated_at = now()
|
|
870
|
+
from claimed
|
|
871
|
+
where e.id = claimed.id
|
|
872
|
+
returning
|
|
873
|
+
e.id,
|
|
874
|
+
e.lease_token,
|
|
875
|
+
e.namespace_id,
|
|
876
|
+
e.owner_type,
|
|
877
|
+
e.owner_id,
|
|
878
|
+
e.embedding_model,
|
|
879
|
+
e.attempts,
|
|
880
|
+
coalesce(
|
|
881
|
+
(select content from interactions where id = e.owner_id),
|
|
882
|
+
(select content from facts where id = e.owner_id),
|
|
883
|
+
(select content from session_summaries where id = e.owner_id)
|
|
884
|
+
) as content
|
|
885
|
+
`;
|
|
886
|
+
return rows.map((row) => ({
|
|
887
|
+
id: row.id,
|
|
888
|
+
leaseToken: row.lease_token,
|
|
889
|
+
leaseMs: 12e4,
|
|
890
|
+
namespaceId: row.namespace_id,
|
|
891
|
+
ownerType: row.owner_type,
|
|
892
|
+
ownerId: row.owner_id,
|
|
893
|
+
embeddingModel: row.embedding_model,
|
|
894
|
+
attempts: row.attempts,
|
|
895
|
+
content: row.content ?? ""
|
|
896
|
+
}));
|
|
897
|
+
}
|
|
898
|
+
async renewEmbeddingLease(id, leaseToken) {
|
|
899
|
+
const rows = await this.db.update(embeddings).set({ leaseUntil: sql2`now() + interval '120 seconds'` }).where(
|
|
900
|
+
and(
|
|
901
|
+
eq(embeddings.id, id),
|
|
902
|
+
eq(embeddings.leaseToken, leaseToken),
|
|
903
|
+
eq(embeddings.status, "processing"),
|
|
904
|
+
sql2`${embeddings.leaseUntil} > now()`
|
|
905
|
+
)
|
|
906
|
+
).returning({ id: embeddings.id });
|
|
907
|
+
return rows.length > 0;
|
|
908
|
+
}
|
|
909
|
+
async markEmbeddingReady(id, leaseToken, vector2, model, space) {
|
|
910
|
+
const literal = `[${vector2.join(",")}]`;
|
|
911
|
+
const rows = await this.db.update(embeddings).set({
|
|
912
|
+
status: "ready",
|
|
913
|
+
vector: literal,
|
|
914
|
+
embeddingModel: model,
|
|
915
|
+
embeddingSpace: space ?? embeddingSpaceKey(embeddingSpace({ model, dimensions: vector2.length })),
|
|
916
|
+
lastError: null,
|
|
917
|
+
leaseUntil: null,
|
|
918
|
+
leaseToken: null,
|
|
919
|
+
updatedAt: sql2`now()`
|
|
920
|
+
}).where(
|
|
921
|
+
and(
|
|
922
|
+
eq(embeddings.id, id),
|
|
923
|
+
eq(embeddings.leaseToken, leaseToken),
|
|
924
|
+
eq(embeddings.status, "processing"),
|
|
925
|
+
sql2`${embeddings.leaseUntil} > now()`
|
|
926
|
+
)
|
|
927
|
+
).returning({ id: embeddings.id });
|
|
928
|
+
return rows.length > 0;
|
|
929
|
+
}
|
|
930
|
+
/**
|
|
931
|
+
* Record a failure. Below `maxAttempts`, the row goes back to
|
|
932
|
+
* 'processing' with an exponentially growing lease (2s · 2^(attempts-1),
|
|
933
|
+
* capped at 60s) that acts as a retry backoff timer — a rate-limited
|
|
934
|
+
* endpoint is no longer hammered every poll tick. At `maxAttempts` the job
|
|
935
|
+
* is parked as 'failed'.
|
|
936
|
+
*/
|
|
937
|
+
async markEmbeddingFailed(id, leaseToken, error, attempts, options) {
|
|
938
|
+
const maxAttempts = options?.maxAttempts ?? 5;
|
|
939
|
+
const backoffSeconds = Math.min(60, 2 ** Math.max(0, attempts - 1));
|
|
940
|
+
const rows = await this.db.update(embeddings).set({
|
|
941
|
+
status: sql2`case when ${attempts} >= ${maxAttempts} then 'failed'::embedding_status else 'processing'::embedding_status end`,
|
|
942
|
+
leaseUntil: sql2`case when ${attempts} >= ${maxAttempts} then null else now() + (${backoffSeconds} || ' seconds')::interval end`,
|
|
943
|
+
leaseToken: null,
|
|
944
|
+
attempts,
|
|
945
|
+
lastError: error,
|
|
946
|
+
updatedAt: sql2`now()`
|
|
947
|
+
}).where(
|
|
948
|
+
and(
|
|
949
|
+
eq(embeddings.id, id),
|
|
950
|
+
eq(embeddings.leaseToken, leaseToken),
|
|
951
|
+
eq(embeddings.status, "processing"),
|
|
952
|
+
sql2`${embeddings.leaseUntil} > now()`
|
|
953
|
+
)
|
|
954
|
+
).returning({ id: embeddings.id });
|
|
955
|
+
return rows.length > 0;
|
|
956
|
+
}
|
|
957
|
+
async getEmbeddingStatus(id) {
|
|
958
|
+
const rows = await this.db.select({
|
|
959
|
+
id: embeddings.id,
|
|
960
|
+
status: embeddings.status,
|
|
961
|
+
attempts: embeddings.attempts,
|
|
962
|
+
lastError: embeddings.lastError
|
|
963
|
+
}).from(embeddings).where(eq(embeddings.id, id)).limit(1);
|
|
964
|
+
const row = rows[0];
|
|
965
|
+
if (!row) return null;
|
|
966
|
+
return {
|
|
967
|
+
id: row.id,
|
|
968
|
+
status: row.status,
|
|
969
|
+
attempts: row.attempts,
|
|
970
|
+
lastError: row.lastError
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
async shareReference(input) {
|
|
974
|
+
await this.db.insert(shares).values({
|
|
975
|
+
id: input.id,
|
|
976
|
+
fromNamespaceId: input.fromNamespaceId,
|
|
977
|
+
toNamespaceId: input.toNamespaceId,
|
|
978
|
+
sourceFactId: input.sourceFactId,
|
|
979
|
+
mode: "reference",
|
|
980
|
+
createdByAgent: input.createdByAgent ?? null
|
|
981
|
+
}).onConflictDoUpdate({
|
|
982
|
+
target: [shares.toNamespaceId, shares.sourceFactId, shares.mode],
|
|
983
|
+
set: { tombstoned: false }
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
async revokeShares(fromNamespaceId, toNamespaceId, factIds) {
|
|
987
|
+
if (factIds.length === 0) return 0;
|
|
988
|
+
const revoked = await this.db.update(shares).set({ tombstoned: true }).where(
|
|
989
|
+
and(
|
|
990
|
+
eq(shares.fromNamespaceId, fromNamespaceId),
|
|
991
|
+
eq(shares.toNamespaceId, toNamespaceId),
|
|
992
|
+
eq(shares.mode, "reference"),
|
|
993
|
+
eq(shares.tombstoned, false),
|
|
994
|
+
inArray(shares.sourceFactId, factIds)
|
|
995
|
+
)
|
|
996
|
+
).returning({ id: shares.id });
|
|
997
|
+
return revoked.length;
|
|
998
|
+
}
|
|
999
|
+
async shareSnapshot(input) {
|
|
1000
|
+
return this.db.transaction(async (tx) => {
|
|
1001
|
+
const prior = await tx.select({ snapshotFactId: shares.snapshotFactId }).from(shares).where(
|
|
1002
|
+
and(
|
|
1003
|
+
eq(shares.toNamespaceId, input.share.toNamespaceId),
|
|
1004
|
+
eq(shares.sourceFactId, input.share.sourceFactId),
|
|
1005
|
+
eq(shares.mode, "snapshot")
|
|
1006
|
+
)
|
|
1007
|
+
).limit(1);
|
|
1008
|
+
const priorId = prior[0]?.snapshotFactId;
|
|
1009
|
+
if (priorId) return { snapshotFactId: priorId };
|
|
1010
|
+
const copy = await tx.select({ id: facts.id }).from(facts).where(
|
|
1011
|
+
and(
|
|
1012
|
+
eq(facts.namespaceId, input.snapshotFact.namespaceId),
|
|
1013
|
+
eq(facts.contentHash, input.snapshotFact.contentHash)
|
|
1014
|
+
)
|
|
1015
|
+
).limit(1);
|
|
1016
|
+
const snapshotFactId = copy[0]?.id ?? input.snapshotFact.id;
|
|
1017
|
+
if (!copy[0]) {
|
|
1018
|
+
await tx.insert(facts).values({
|
|
1019
|
+
id: input.snapshotFact.id,
|
|
1020
|
+
namespaceId: input.snapshotFact.namespaceId,
|
|
1021
|
+
subject: input.snapshotFact.subject,
|
|
1022
|
+
predicate: input.snapshotFact.predicate,
|
|
1023
|
+
object: input.snapshotFact.object,
|
|
1024
|
+
content: input.snapshotFact.content,
|
|
1025
|
+
contentHash: input.snapshotFact.contentHash,
|
|
1026
|
+
sourceInteractionId: input.snapshotFact.sourceInteractionId ?? null,
|
|
1027
|
+
createdByAgent: input.snapshotFact.createdByAgent ?? null
|
|
1028
|
+
});
|
|
1029
|
+
await tx.insert(embeddings).values({
|
|
1030
|
+
id: input.embedding.id,
|
|
1031
|
+
namespaceId: input.embedding.namespaceId,
|
|
1032
|
+
ownerType: input.embedding.ownerType,
|
|
1033
|
+
ownerId: input.embedding.ownerId,
|
|
1034
|
+
vector: zeroVectorLiteral(),
|
|
1035
|
+
embeddingModel: input.embedding.embeddingModel,
|
|
1036
|
+
status: "pending"
|
|
1037
|
+
});
|
|
1038
|
+
}
|
|
1039
|
+
await tx.insert(shares).values({
|
|
1040
|
+
id: input.share.id,
|
|
1041
|
+
fromNamespaceId: input.share.fromNamespaceId,
|
|
1042
|
+
toNamespaceId: input.share.toNamespaceId,
|
|
1043
|
+
sourceFactId: input.share.sourceFactId,
|
|
1044
|
+
snapshotFactId,
|
|
1045
|
+
mode: "snapshot",
|
|
1046
|
+
createdByAgent: input.share.createdByAgent ?? null
|
|
1047
|
+
}).onConflictDoUpdate({
|
|
1048
|
+
target: [shares.toNamespaceId, shares.sourceFactId, shares.mode],
|
|
1049
|
+
set: { snapshotFactId }
|
|
1050
|
+
});
|
|
1051
|
+
return { snapshotFactId };
|
|
1052
|
+
});
|
|
1053
|
+
}
|
|
1054
|
+
async forgetInteraction(namespaceId, id, audit) {
|
|
1055
|
+
return this.db.transaction(async (tx) => {
|
|
1056
|
+
const existing = await tx.select({ id: interactions.id }).from(interactions).where(and(eq(interactions.namespaceId, namespaceId), eq(interactions.id, id))).for("update");
|
|
1057
|
+
if (existing.length === 0) return { found: false, sourceDeletedFactIds: [] };
|
|
1058
|
+
const marked = await tx.update(facts).set({ sourceDeleted: true }).where(and(eq(facts.namespaceId, namespaceId), eq(facts.sourceInteractionId, id))).returning({ id: facts.id });
|
|
1059
|
+
await tx.delete(interactions).where(and(eq(interactions.namespaceId, namespaceId), eq(interactions.id, id)));
|
|
1060
|
+
await tx.delete(embeddings).where(
|
|
1061
|
+
and(
|
|
1062
|
+
eq(embeddings.namespaceId, namespaceId),
|
|
1063
|
+
eq(embeddings.ownerType, "interaction"),
|
|
1064
|
+
eq(embeddings.ownerId, id)
|
|
1065
|
+
)
|
|
1066
|
+
);
|
|
1067
|
+
await tx.insert(auditLog).values(toAuditRow(audit));
|
|
1068
|
+
return { found: true, sourceDeletedFactIds: marked.map((row) => row.id) };
|
|
1069
|
+
});
|
|
1070
|
+
}
|
|
1071
|
+
async forgetFact(namespaceId, id, audit) {
|
|
1072
|
+
return this.db.transaction(async (tx) => {
|
|
1073
|
+
const deleted = await tx.delete(facts).where(and(eq(facts.namespaceId, namespaceId), eq(facts.id, id))).returning({ id: facts.id });
|
|
1074
|
+
if (deleted.length === 0) return { found: false };
|
|
1075
|
+
await tx.delete(embeddings).where(
|
|
1076
|
+
and(
|
|
1077
|
+
eq(embeddings.namespaceId, namespaceId),
|
|
1078
|
+
eq(embeddings.ownerType, "fact"),
|
|
1079
|
+
eq(embeddings.ownerId, id)
|
|
1080
|
+
)
|
|
1081
|
+
);
|
|
1082
|
+
await tx.update(shares).set({ tombstoned: true }).where(
|
|
1083
|
+
and(
|
|
1084
|
+
eq(shares.fromNamespaceId, namespaceId),
|
|
1085
|
+
eq(shares.sourceFactId, id),
|
|
1086
|
+
eq(shares.mode, "reference")
|
|
1087
|
+
)
|
|
1088
|
+
);
|
|
1089
|
+
await tx.insert(auditLog).values(toAuditRow(audit));
|
|
1090
|
+
return { found: true };
|
|
1091
|
+
});
|
|
1092
|
+
}
|
|
1093
|
+
async forgetSummary(namespaceId, id, audit) {
|
|
1094
|
+
return this.db.transaction(async (tx) => {
|
|
1095
|
+
const deleted = await tx.delete(sessionSummaries).where(and(eq(sessionSummaries.namespaceId, namespaceId), eq(sessionSummaries.id, id))).returning({ id: sessionSummaries.id });
|
|
1096
|
+
if (deleted.length === 0) return { found: false };
|
|
1097
|
+
await tx.delete(embeddings).where(
|
|
1098
|
+
and(
|
|
1099
|
+
eq(embeddings.namespaceId, namespaceId),
|
|
1100
|
+
eq(embeddings.ownerType, "summary"),
|
|
1101
|
+
eq(embeddings.ownerId, id)
|
|
1102
|
+
)
|
|
1103
|
+
);
|
|
1104
|
+
await tx.insert(auditLog).values(toAuditRow(audit));
|
|
1105
|
+
return { found: true };
|
|
1106
|
+
});
|
|
1107
|
+
}
|
|
1108
|
+
async recordModelCall(entry) {
|
|
1109
|
+
await this.db.insert(modelCalls).values({
|
|
1110
|
+
id: entry.id,
|
|
1111
|
+
namespaceId: entry.namespaceId ?? null,
|
|
1112
|
+
model: entry.metadata.model,
|
|
1113
|
+
type: entry.metadata.type,
|
|
1114
|
+
latencyMs: entry.metadata.latencyMs,
|
|
1115
|
+
tokens: entry.metadata.tokens ?? null,
|
|
1116
|
+
status: entry.metadata.status,
|
|
1117
|
+
error: entry.metadata.error ?? null,
|
|
1118
|
+
metadata: {
|
|
1119
|
+
retries: entry.metadata.retries,
|
|
1120
|
+
fallbackTriggered: entry.metadata.fallbackTriggered,
|
|
1121
|
+
coldStartWaitMs: entry.metadata.coldStartWaitMs
|
|
1122
|
+
}
|
|
1123
|
+
});
|
|
1124
|
+
}
|
|
1125
|
+
async recordAudit(entry) {
|
|
1126
|
+
await this.db.insert(auditLog).values(toAuditRow(entry));
|
|
1127
|
+
}
|
|
1128
|
+
async getReadyEmbeddings(namespaceId, limit, space) {
|
|
1129
|
+
const rows = await this.client.sql`
|
|
1130
|
+
select e.id, e.owner_type, e.owner_id, e.embedding_model, e.embedding_space,
|
|
1131
|
+
e.vector::text as vector
|
|
1132
|
+
from embeddings e
|
|
1133
|
+
where e.status = 'ready'
|
|
1134
|
+
and (${space ?? null}::text is null or e.embedding_space = ${space ?? null})
|
|
1135
|
+
and (
|
|
1136
|
+
e.namespace_id = ${namespaceId}
|
|
1137
|
+
or (
|
|
1138
|
+
e.owner_type = 'fact'
|
|
1139
|
+
and exists (
|
|
1140
|
+
select 1 from shares s
|
|
1141
|
+
where s.to_namespace_id = ${namespaceId}
|
|
1142
|
+
and s.from_namespace_id = e.namespace_id
|
|
1143
|
+
and s.source_fact_id = e.owner_id
|
|
1144
|
+
and s.mode = 'reference'
|
|
1145
|
+
and not s.tombstoned
|
|
1146
|
+
)
|
|
1147
|
+
)
|
|
1148
|
+
)
|
|
1149
|
+
order by e.updated_at desc
|
|
1150
|
+
limit ${limit}
|
|
1151
|
+
`;
|
|
1152
|
+
return rows.map((row) => ({
|
|
1153
|
+
embeddingId: row.id,
|
|
1154
|
+
ownerType: row.owner_type,
|
|
1155
|
+
ownerId: row.owner_id,
|
|
1156
|
+
embeddingModel: row.embedding_model,
|
|
1157
|
+
embeddingSpace: row.embedding_space,
|
|
1158
|
+
vector: parseVectorLiteral(row.vector)
|
|
1159
|
+
}));
|
|
1160
|
+
}
|
|
1161
|
+
async requeueEmbeddingsOutsideSpace(namespaceId, space) {
|
|
1162
|
+
const rows = await this.db.update(embeddings).set({
|
|
1163
|
+
status: "pending",
|
|
1164
|
+
vector: zeroVectorLiteral(),
|
|
1165
|
+
attempts: 0,
|
|
1166
|
+
lastError: null,
|
|
1167
|
+
leaseUntil: null,
|
|
1168
|
+
leaseToken: null,
|
|
1169
|
+
updatedAt: sql2`now()`
|
|
1170
|
+
}).where(
|
|
1171
|
+
and(
|
|
1172
|
+
eq(embeddings.namespaceId, namespaceId),
|
|
1173
|
+
inArray(embeddings.status, ["ready", "failed"]),
|
|
1174
|
+
sql2`${embeddings.embeddingSpace} is distinct from ${space}`
|
|
1175
|
+
)
|
|
1176
|
+
).returning({ id: embeddings.id });
|
|
1177
|
+
return rows.length;
|
|
1178
|
+
}
|
|
1179
|
+
async searchSimilarEmbeddings(input) {
|
|
1180
|
+
if (!validSearchVector(input.vector))
|
|
1181
|
+
return { method: "exact_pgvector", candidates: [], compatibleReady: 0 };
|
|
1182
|
+
const rows = await this.db.execute(sql2`
|
|
1183
|
+
with compatible as materialized (
|
|
1184
|
+
select id, owner_type, owner_id, embedding_model, embedding_space, vector
|
|
1185
|
+
from embeddings
|
|
1186
|
+
where status = 'ready'
|
|
1187
|
+
and (
|
|
1188
|
+
namespace_id = ${input.namespaceId}
|
|
1189
|
+
or (
|
|
1190
|
+
owner_type = 'fact'
|
|
1191
|
+
and ${liveReference(input.namespaceId, sql2`embeddings.namespace_id`, sql2`embeddings.owner_id`)}
|
|
1192
|
+
)
|
|
1193
|
+
)
|
|
1194
|
+
and embedding_space = ${input.embeddingSpace}
|
|
1195
|
+
and embedding_model = ${input.embeddingModel}
|
|
1196
|
+
and vector_norm(vector) > 0
|
|
1197
|
+
and not exists (
|
|
1198
|
+
select 1 from facts f
|
|
1199
|
+
where owner_type = 'fact' and f.namespace_id = embeddings.namespace_id
|
|
1200
|
+
and f.id = embeddings.owner_id
|
|
1201
|
+
and (f.source_deleted or f.superseded_by is not null)
|
|
1202
|
+
)
|
|
1203
|
+
)
|
|
1204
|
+
select id, owner_type, owner_id, embedding_model, embedding_space,
|
|
1205
|
+
1 - (vector <=> ${JSON.stringify(input.vector)}::vector) as score,
|
|
1206
|
+
count(*) over () as total
|
|
1207
|
+
from compatible
|
|
1208
|
+
order by vector <=> ${JSON.stringify(input.vector)}::vector, id collate "C"
|
|
1209
|
+
limit ${candidateLimit(input.limit)}
|
|
1210
|
+
`);
|
|
1211
|
+
return {
|
|
1212
|
+
method: "exact_pgvector",
|
|
1213
|
+
compatibleReady: Number(rows[0]?.total ?? 0),
|
|
1214
|
+
candidates: rows.map((row) => ({
|
|
1215
|
+
embeddingId: row.id,
|
|
1216
|
+
ownerType: row.owner_type,
|
|
1217
|
+
ownerId: row.owner_id,
|
|
1218
|
+
embeddingModel: row.embedding_model,
|
|
1219
|
+
embeddingSpace: row.embedding_space,
|
|
1220
|
+
score: Number(row.score)
|
|
1221
|
+
}))
|
|
1222
|
+
};
|
|
1223
|
+
}
|
|
1224
|
+
async searchFactsLexically(namespaceId, query, limit) {
|
|
1225
|
+
const terms = lexicalTerms(query);
|
|
1226
|
+
if (!terms.length) return { candidates: [], matchedFacts: 0 };
|
|
1227
|
+
const rows = await this.db.execute(sql2`
|
|
1228
|
+
with scored as materialized (
|
|
1229
|
+
select id, content, subject, predicate, object, proposition_key, confidence,
|
|
1230
|
+
created_by_agent, created_at,
|
|
1231
|
+
(select coalesce(sum(
|
|
1232
|
+
(case when strpos(' ' || regexp_replace(lower(subject || ' ' || object), '[^[:alnum:]_]+', ' ', 'g') || ' ', ' ' || term || ' ') > 0 then 2 else 0 end) +
|
|
1233
|
+
(case when strpos(' ' || regexp_replace(lower(content), '[^[:alnum:]_]+', ' ', 'g') || ' ', ' ' || term || ' ') > 0 then 1 else 0 end)
|
|
1234
|
+
), 0)::float / ${3 * terms.length} from
|
|
1235
|
+
jsonb_array_elements_text(${JSON.stringify(terms)}::jsonb) term
|
|
1236
|
+
) as score
|
|
1237
|
+
from facts
|
|
1238
|
+
where not source_deleted and superseded_by is null
|
|
1239
|
+
and (
|
|
1240
|
+
namespace_id = ${namespaceId}
|
|
1241
|
+
or ${liveReference(namespaceId, sql2`facts.namespace_id`, sql2`facts.id`)}
|
|
1242
|
+
)
|
|
1243
|
+
)
|
|
1244
|
+
select *, count(*) over () as total from scored where score > 0
|
|
1245
|
+
order by score desc, created_at desc, id collate "C"
|
|
1246
|
+
limit ${candidateLimit(limit)}
|
|
1247
|
+
`);
|
|
1248
|
+
return {
|
|
1249
|
+
matchedFacts: Number(rows[0]?.total ?? 0),
|
|
1250
|
+
candidates: rows.map((row) => ({
|
|
1251
|
+
id: row.id,
|
|
1252
|
+
content: row.content,
|
|
1253
|
+
subject: row.subject,
|
|
1254
|
+
predicate: row.predicate,
|
|
1255
|
+
object: row.object,
|
|
1256
|
+
propositionKey: row.proposition_key,
|
|
1257
|
+
confidence: Number(row.confidence),
|
|
1258
|
+
sourceDeleted: false,
|
|
1259
|
+
supersededBy: null,
|
|
1260
|
+
createdByAgent: row.created_by_agent,
|
|
1261
|
+
createdAt: new Date(row.created_at),
|
|
1262
|
+
score: Number(row.score)
|
|
1263
|
+
}))
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
async getRecentInteractions(namespaceId, sessionId, limit) {
|
|
1267
|
+
const rows = await this.db.select({
|
|
1268
|
+
id: interactions.id,
|
|
1269
|
+
sessionId: interactions.sessionId,
|
|
1270
|
+
content: interactions.content,
|
|
1271
|
+
tokenCount: interactions.tokenCount,
|
|
1272
|
+
createdAt: interactions.createdAt
|
|
1273
|
+
}).from(interactions).where(and(eq(interactions.namespaceId, namespaceId), eq(interactions.sessionId, sessionId))).orderBy(desc(interactions.createdAt)).limit(limit);
|
|
1274
|
+
return rows.map((row) => ({ ...row, createdAt: row.createdAt }));
|
|
1275
|
+
}
|
|
1276
|
+
async getRecentFacts(namespaceId, limit) {
|
|
1277
|
+
const rows = await this.db.select(recentFactColumns).from(facts).where(
|
|
1278
|
+
and(
|
|
1279
|
+
eq(facts.namespaceId, namespaceId),
|
|
1280
|
+
eq(facts.sourceDeleted, false),
|
|
1281
|
+
isNull(facts.supersededBy)
|
|
1282
|
+
)
|
|
1283
|
+
).orderBy(desc(facts.createdAt)).limit(limit);
|
|
1284
|
+
return rows.map((row) => ({ ...row, confidence: Number(row.confidence ?? 1) }));
|
|
1285
|
+
}
|
|
1286
|
+
async getLatestSummary(namespaceId, sessionId) {
|
|
1287
|
+
const rows = await this.db.select({
|
|
1288
|
+
id: sessionSummaries.id,
|
|
1289
|
+
sessionId: sessionSummaries.sessionId,
|
|
1290
|
+
content: sessionSummaries.content,
|
|
1291
|
+
tokenCount: sessionSummaries.tokenCount,
|
|
1292
|
+
createdAt: sessionSummaries.createdAt
|
|
1293
|
+
}).from(sessionSummaries).where(
|
|
1294
|
+
and(
|
|
1295
|
+
eq(sessionSummaries.namespaceId, namespaceId),
|
|
1296
|
+
eq(sessionSummaries.sessionId, sessionId)
|
|
1297
|
+
)
|
|
1298
|
+
).orderBy(desc(sessionSummaries.version)).limit(1);
|
|
1299
|
+
return rows[0] ?? null;
|
|
1300
|
+
}
|
|
1301
|
+
async getInteractionsByIds(namespaceId, ids) {
|
|
1302
|
+
if (ids.length === 0) return [];
|
|
1303
|
+
const rows = await this.db.select({
|
|
1304
|
+
id: interactions.id,
|
|
1305
|
+
sessionId: interactions.sessionId,
|
|
1306
|
+
content: interactions.content,
|
|
1307
|
+
tokenCount: interactions.tokenCount,
|
|
1308
|
+
createdAt: interactions.createdAt
|
|
1309
|
+
}).from(interactions).where(and(eq(interactions.namespaceId, namespaceId), inArray(interactions.id, ids)));
|
|
1310
|
+
return rows;
|
|
1311
|
+
}
|
|
1312
|
+
async getFactsByIds(namespaceId, ids) {
|
|
1313
|
+
if (ids.length === 0) return [];
|
|
1314
|
+
const rows = await this.db.select(recentFactColumns).from(facts).where(
|
|
1315
|
+
and(
|
|
1316
|
+
inArray(facts.id, ids),
|
|
1317
|
+
isNull(facts.supersededBy),
|
|
1318
|
+
or(
|
|
1319
|
+
eq(facts.namespaceId, namespaceId),
|
|
1320
|
+
liveReference(namespaceId, sql2`${facts.namespaceId}`, sql2`${facts.id}`)
|
|
1321
|
+
)
|
|
1322
|
+
)
|
|
1323
|
+
);
|
|
1324
|
+
return rows.map((row) => ({ ...row, confidence: Number(row.confidence ?? 1) }));
|
|
1325
|
+
}
|
|
1326
|
+
async getSummariesByIds(namespaceId, ids) {
|
|
1327
|
+
if (ids.length === 0) return [];
|
|
1328
|
+
const rows = await this.db.select({
|
|
1329
|
+
id: sessionSummaries.id,
|
|
1330
|
+
sessionId: sessionSummaries.sessionId,
|
|
1331
|
+
content: sessionSummaries.content,
|
|
1332
|
+
tokenCount: sessionSummaries.tokenCount,
|
|
1333
|
+
createdAt: sessionSummaries.createdAt
|
|
1334
|
+
}).from(sessionSummaries).where(and(eq(sessionSummaries.namespaceId, namespaceId), inArray(sessionSummaries.id, ids)));
|
|
1335
|
+
return rows;
|
|
1336
|
+
}
|
|
1337
|
+
async getFactById(namespaceId, id) {
|
|
1338
|
+
const rows = await this.db.select({
|
|
1339
|
+
id: facts.id,
|
|
1340
|
+
subject: facts.subject,
|
|
1341
|
+
predicate: facts.predicate,
|
|
1342
|
+
object: facts.object,
|
|
1343
|
+
content: facts.content,
|
|
1344
|
+
confidence: facts.confidence,
|
|
1345
|
+
sourceInteractionId: facts.sourceInteractionId,
|
|
1346
|
+
sourceDeleted: facts.sourceDeleted,
|
|
1347
|
+
supersededBy: facts.supersededBy,
|
|
1348
|
+
createdByAgent: facts.createdByAgent,
|
|
1349
|
+
createdAt: facts.createdAt
|
|
1350
|
+
}).from(facts).where(and(eq(facts.namespaceId, namespaceId), eq(facts.id, id))).limit(1);
|
|
1351
|
+
const row = rows[0];
|
|
1352
|
+
if (!row) return null;
|
|
1353
|
+
return { ...row, confidence: Number(row.confidence ?? 1) };
|
|
1354
|
+
}
|
|
1355
|
+
async countEmbeddingStatuses(namespaceId) {
|
|
1356
|
+
const rows = await this.client.sql`
|
|
1357
|
+
select status::text as status, count(*)::text as count
|
|
1358
|
+
from embeddings
|
|
1359
|
+
where namespace_id = ${namespaceId}
|
|
1360
|
+
group by status
|
|
1361
|
+
`;
|
|
1362
|
+
let pending = 0;
|
|
1363
|
+
let processing = 0;
|
|
1364
|
+
let failed = 0;
|
|
1365
|
+
let ready = 0;
|
|
1366
|
+
for (const row of rows) {
|
|
1367
|
+
const value = Number(row.count);
|
|
1368
|
+
if (row.status === "pending") pending = value;
|
|
1369
|
+
else if (row.status === "processing") processing = value;
|
|
1370
|
+
else if (row.status === "failed") failed = value;
|
|
1371
|
+
else if (row.status === "ready") ready = value;
|
|
1372
|
+
}
|
|
1373
|
+
return { pending, processing, failed, ready };
|
|
1374
|
+
}
|
|
1375
|
+
async listFacts(input) {
|
|
1376
|
+
const where = [
|
|
1377
|
+
eq(facts.namespaceId, input.namespaceId),
|
|
1378
|
+
eq(facts.sourceDeleted, false),
|
|
1379
|
+
isNull(facts.supersededBy),
|
|
1380
|
+
input.subject ? eq(facts.subject, input.subject) : void 0,
|
|
1381
|
+
input.cursor ? or(
|
|
1382
|
+
lt(facts.createdAt, input.cursor.createdAt),
|
|
1383
|
+
and(eq(facts.createdAt, input.cursor.createdAt), lt(facts.id, input.cursor.id))
|
|
1384
|
+
) : void 0
|
|
1385
|
+
].filter((clause) => Boolean(clause));
|
|
1386
|
+
const rows = await this.db.select({
|
|
1387
|
+
id: facts.id,
|
|
1388
|
+
subject: facts.subject,
|
|
1389
|
+
predicate: facts.predicate,
|
|
1390
|
+
object: facts.object,
|
|
1391
|
+
content: facts.content,
|
|
1392
|
+
confidence: facts.confidence,
|
|
1393
|
+
sourceInteractionId: facts.sourceInteractionId,
|
|
1394
|
+
sourceDeleted: facts.sourceDeleted,
|
|
1395
|
+
supersededBy: facts.supersededBy,
|
|
1396
|
+
createdByAgent: facts.createdByAgent,
|
|
1397
|
+
createdAt: facts.createdAt
|
|
1398
|
+
}).from(facts).where(and(...where)).orderBy(desc(facts.createdAt), desc(facts.id)).limit(input.limit);
|
|
1399
|
+
return rows.map((row) => ({ ...row, confidence: Number(row.confidence ?? 1) }));
|
|
1400
|
+
}
|
|
1401
|
+
async listSummaries(input) {
|
|
1402
|
+
return this.db.select({
|
|
1403
|
+
id: sessionSummaries.id,
|
|
1404
|
+
sessionId: sessionSummaries.sessionId,
|
|
1405
|
+
content: sessionSummaries.content,
|
|
1406
|
+
version: sessionSummaries.version,
|
|
1407
|
+
tokenCount: sessionSummaries.tokenCount,
|
|
1408
|
+
createdByAgent: sessionSummaries.createdByAgent,
|
|
1409
|
+
createdAt: sessionSummaries.createdAt
|
|
1410
|
+
}).from(sessionSummaries).where(eq(sessionSummaries.namespaceId, input.namespaceId)).orderBy(desc(sessionSummaries.createdAt)).limit(input.limit);
|
|
1411
|
+
}
|
|
1412
|
+
async listNamespaces() {
|
|
1413
|
+
const rows = await this.client.sql`
|
|
1414
|
+
select
|
|
1415
|
+
n.name,
|
|
1416
|
+
count(f.id)::text as fact_count,
|
|
1417
|
+
greatest(
|
|
1418
|
+
max(f.created_at),
|
|
1419
|
+
max(i.created_at),
|
|
1420
|
+
max(s.created_at),
|
|
1421
|
+
n.updated_at
|
|
1422
|
+
) as last_activity
|
|
1423
|
+
from namespaces n
|
|
1424
|
+
left join facts f on f.namespace_id = n.id and f.source_deleted = false
|
|
1425
|
+
left join interactions i on i.namespace_id = n.id
|
|
1426
|
+
left join session_summaries s on s.namespace_id = n.id
|
|
1427
|
+
group by n.id, n.name, n.updated_at
|
|
1428
|
+
order by last_activity desc nulls last, n.name asc
|
|
1429
|
+
`;
|
|
1430
|
+
return rows.map((row) => ({
|
|
1431
|
+
name: row.name,
|
|
1432
|
+
factCount: Number(row.fact_count),
|
|
1433
|
+
lastActivity: row.last_activity
|
|
1434
|
+
}));
|
|
1435
|
+
}
|
|
1436
|
+
async getAgentWriteBreakdown(namespaceId) {
|
|
1437
|
+
const rows = namespaceId ? await this.client.sql`
|
|
1438
|
+
select agent_id, sum(writes)::text as writes
|
|
1439
|
+
from (
|
|
1440
|
+
select created_by_agent as agent_id, count(*) as writes from interactions where namespace_id = ${namespaceId} group by created_by_agent
|
|
1441
|
+
union all
|
|
1442
|
+
select created_by_agent as agent_id, count(*) as writes from facts where namespace_id = ${namespaceId} group by created_by_agent
|
|
1443
|
+
union all
|
|
1444
|
+
select created_by_agent as agent_id, count(*) as writes from session_summaries where namespace_id = ${namespaceId} group by created_by_agent
|
|
1445
|
+
) t
|
|
1446
|
+
group by agent_id
|
|
1447
|
+
order by writes desc
|
|
1448
|
+
` : await this.client.sql`
|
|
1449
|
+
select agent_id, sum(writes)::text as writes
|
|
1450
|
+
from (
|
|
1451
|
+
select created_by_agent as agent_id, count(*) as writes from interactions group by created_by_agent
|
|
1452
|
+
union all
|
|
1453
|
+
select created_by_agent as agent_id, count(*) as writes from facts group by created_by_agent
|
|
1454
|
+
union all
|
|
1455
|
+
select created_by_agent as agent_id, count(*) as writes from session_summaries group by created_by_agent
|
|
1456
|
+
) t
|
|
1457
|
+
group by agent_id
|
|
1458
|
+
order by writes desc
|
|
1459
|
+
`;
|
|
1460
|
+
return rows.map((row) => ({ agentId: row.agent_id, writes: Number(row.writes) }));
|
|
1461
|
+
}
|
|
1462
|
+
async getStats(namespaceId) {
|
|
1463
|
+
const namespaceRows = await this.client.sql`select count(*)::text as count from namespaces`;
|
|
1464
|
+
const interactionRows = namespaceId ? await this.client.sql`select count(*)::text as count from interactions where namespace_id = ${namespaceId}` : await this.client.sql`select count(*)::text as count from interactions`;
|
|
1465
|
+
const factRows = namespaceId ? await this.client.sql`select count(*)::text as count from facts where namespace_id = ${namespaceId}` : await this.client.sql`select count(*)::text as count from facts`;
|
|
1466
|
+
const summaryRows = namespaceId ? await this.client.sql`select count(*)::text as count from session_summaries where namespace_id = ${namespaceId}` : await this.client.sql`select count(*)::text as count from session_summaries`;
|
|
1467
|
+
const embeddingRows = namespaceId ? await this.client.sql`
|
|
1468
|
+
select status::text as status, count(*)::text as count from embeddings where namespace_id = ${namespaceId} group by status
|
|
1469
|
+
` : await this.client.sql`
|
|
1470
|
+
select status::text as status, count(*)::text as count from embeddings group by status
|
|
1471
|
+
`;
|
|
1472
|
+
const callRows = namespaceId ? await this.client.sql`
|
|
1473
|
+
select
|
|
1474
|
+
count(*) filter (where created_at >= now() - interval '60 seconds')::text as last_60s,
|
|
1475
|
+
count(*) filter (where created_at >= now() - interval '1 hour')::text as last_hour,
|
|
1476
|
+
count(*) filter (where created_at >= now() - interval '24 hours')::text as last_24h,
|
|
1477
|
+
count(*) filter (where status = 'gated' and created_at >= now() - interval '24 hours')::text as gated_failures
|
|
1478
|
+
from model_calls
|
|
1479
|
+
where namespace_id = ${namespaceId}
|
|
1480
|
+
` : await this.client.sql`
|
|
1481
|
+
select
|
|
1482
|
+
count(*) filter (where created_at >= now() - interval '60 seconds')::text as last_60s,
|
|
1483
|
+
count(*) filter (where created_at >= now() - interval '1 hour')::text as last_hour,
|
|
1484
|
+
count(*) filter (where created_at >= now() - interval '24 hours')::text as last_24h,
|
|
1485
|
+
count(*) filter (where status = 'gated' and created_at >= now() - interval '24 hours')::text as gated_failures
|
|
1486
|
+
from model_calls
|
|
1487
|
+
`;
|
|
1488
|
+
const embeddingsCount = { pending: 0, processing: 0, failed: 0, ready: 0 };
|
|
1489
|
+
for (const row of embeddingRows) {
|
|
1490
|
+
if (row.status === "pending") embeddingsCount.pending = Number(row.count);
|
|
1491
|
+
if (row.status === "processing") embeddingsCount.processing = Number(row.count);
|
|
1492
|
+
if (row.status === "failed") embeddingsCount.failed = Number(row.count);
|
|
1493
|
+
if (row.status === "ready") embeddingsCount.ready = Number(row.count);
|
|
1494
|
+
}
|
|
1495
|
+
const calls = callRows[0] ?? {
|
|
1496
|
+
last_60s: "0",
|
|
1497
|
+
last_hour: "0",
|
|
1498
|
+
last_24h: "0",
|
|
1499
|
+
gated_failures: "0"
|
|
1500
|
+
};
|
|
1501
|
+
return {
|
|
1502
|
+
namespaces: Number(namespaceRows[0]?.count ?? 0),
|
|
1503
|
+
interactions: Number(interactionRows[0]?.count ?? 0),
|
|
1504
|
+
facts: Number(factRows[0]?.count ?? 0),
|
|
1505
|
+
summaries: Number(summaryRows[0]?.count ?? 0),
|
|
1506
|
+
embeddings: embeddingsCount,
|
|
1507
|
+
modelCalls: {
|
|
1508
|
+
last60s: Number(calls.last_60s),
|
|
1509
|
+
lastHour: Number(calls.last_hour),
|
|
1510
|
+
last24h: Number(calls.last_24h),
|
|
1511
|
+
gatedFailures: Number(calls.gated_failures)
|
|
1512
|
+
}
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
async pruneBefore(cutoff) {
|
|
1516
|
+
return this.db.transaction(async (tx) => {
|
|
1517
|
+
const deletedInteractions = await tx.delete(interactions).where(lt(interactions.createdAt, cutoff)).returning({ id: interactions.id });
|
|
1518
|
+
if (deletedInteractions.length > 0) {
|
|
1519
|
+
await tx.delete(embeddings).where(
|
|
1520
|
+
and(
|
|
1521
|
+
eq(embeddings.ownerType, "interaction"),
|
|
1522
|
+
inArray(
|
|
1523
|
+
embeddings.ownerId,
|
|
1524
|
+
deletedInteractions.map((row) => row.id)
|
|
1525
|
+
)
|
|
1526
|
+
)
|
|
1527
|
+
);
|
|
1528
|
+
}
|
|
1529
|
+
const deletedFacts = await tx.delete(facts).where(lt(facts.createdAt, cutoff)).returning({ id: facts.id });
|
|
1530
|
+
if (deletedFacts.length > 0) {
|
|
1531
|
+
await tx.delete(embeddings).where(
|
|
1532
|
+
and(
|
|
1533
|
+
eq(embeddings.ownerType, "fact"),
|
|
1534
|
+
inArray(
|
|
1535
|
+
embeddings.ownerId,
|
|
1536
|
+
deletedFacts.map((row) => row.id)
|
|
1537
|
+
)
|
|
1538
|
+
)
|
|
1539
|
+
);
|
|
1540
|
+
await tx.update(shares).set({ tombstoned: true }).where(
|
|
1541
|
+
and(
|
|
1542
|
+
eq(shares.mode, "reference"),
|
|
1543
|
+
inArray(
|
|
1544
|
+
shares.sourceFactId,
|
|
1545
|
+
deletedFacts.map((row) => row.id)
|
|
1546
|
+
)
|
|
1547
|
+
)
|
|
1548
|
+
);
|
|
1549
|
+
}
|
|
1550
|
+
const deletedSummaries = await tx.delete(sessionSummaries).where(lt(sessionSummaries.createdAt, cutoff)).returning({ id: sessionSummaries.id });
|
|
1551
|
+
if (deletedSummaries.length > 0) {
|
|
1552
|
+
await tx.delete(embeddings).where(
|
|
1553
|
+
and(
|
|
1554
|
+
eq(embeddings.ownerType, "summary"),
|
|
1555
|
+
inArray(
|
|
1556
|
+
embeddings.ownerId,
|
|
1557
|
+
deletedSummaries.map((row) => row.id)
|
|
1558
|
+
)
|
|
1559
|
+
)
|
|
1560
|
+
);
|
|
1561
|
+
}
|
|
1562
|
+
return {
|
|
1563
|
+
interactions: deletedInteractions.length,
|
|
1564
|
+
facts: deletedFacts.length,
|
|
1565
|
+
summaries: deletedSummaries.length
|
|
1566
|
+
};
|
|
1567
|
+
});
|
|
1568
|
+
}
|
|
1569
|
+
async getVectorIndexHealth(namespaceId) {
|
|
1570
|
+
const countRows = namespaceId ? await this.client.sql`select count(*)::text as count from embeddings where namespace_id = ${namespaceId}` : await this.client.sql`select count(*)::text as count from embeddings`;
|
|
1571
|
+
const indexRows = await this.client.sql`
|
|
1572
|
+
select indexname, indexdef from pg_indexes
|
|
1573
|
+
where schemaname = current_schema()
|
|
1574
|
+
and tablename = 'embeddings'
|
|
1575
|
+
and indexname = 'embeddings_vector_hnsw_idx'
|
|
1576
|
+
limit 1
|
|
1577
|
+
`;
|
|
1578
|
+
const vectorCount = Number(countRows[0]?.count ?? 0);
|
|
1579
|
+
const index2 = indexRows[0];
|
|
1580
|
+
return {
|
|
1581
|
+
vectorCount,
|
|
1582
|
+
indexName: index2?.indexname ?? null,
|
|
1583
|
+
indexDefinition: index2?.indexdef ?? null,
|
|
1584
|
+
needsTuning: vectorCount >= 1e5
|
|
1585
|
+
};
|
|
1586
|
+
}
|
|
1587
|
+
};
|
|
1588
|
+
function createPostgresRepository(client) {
|
|
1589
|
+
return new PostgresMemoryRepository(client);
|
|
1590
|
+
}
|
|
1591
|
+
function toAuditRow(entry) {
|
|
1592
|
+
return {
|
|
1593
|
+
id: entry.id,
|
|
1594
|
+
namespaceId: entry.namespaceId,
|
|
1595
|
+
action: entry.action,
|
|
1596
|
+
entityType: entry.entityType,
|
|
1597
|
+
entityId: entry.entityId ?? null,
|
|
1598
|
+
createdByAgent: entry.createdByAgent ?? null,
|
|
1599
|
+
metadata: entry.metadata ?? {}
|
|
1600
|
+
};
|
|
1601
|
+
}
|
|
1602
|
+
async function upsertPendingEmbedding(tx, embedding, reset) {
|
|
1603
|
+
const insert = tx.insert(embeddings).values({
|
|
1604
|
+
id: embedding.id,
|
|
1605
|
+
namespaceId: embedding.namespaceId,
|
|
1606
|
+
ownerType: embedding.ownerType,
|
|
1607
|
+
ownerId: embedding.ownerId,
|
|
1608
|
+
vector: zeroVectorLiteral(),
|
|
1609
|
+
embeddingModel: embedding.embeddingModel,
|
|
1610
|
+
status: "pending"
|
|
1611
|
+
});
|
|
1612
|
+
const target = [embeddings.ownerType, embeddings.ownerId];
|
|
1613
|
+
if (!reset) {
|
|
1614
|
+
await insert.onConflictDoNothing({ target });
|
|
1615
|
+
return;
|
|
1616
|
+
}
|
|
1617
|
+
await insert.onConflictDoUpdate({
|
|
1618
|
+
target,
|
|
1619
|
+
set: {
|
|
1620
|
+
vector: zeroVectorLiteral(),
|
|
1621
|
+
embeddingModel: embedding.embeddingModel,
|
|
1622
|
+
status: "pending",
|
|
1623
|
+
attempts: 0,
|
|
1624
|
+
lastError: null,
|
|
1625
|
+
leaseUntil: null,
|
|
1626
|
+
leaseToken: null,
|
|
1627
|
+
updatedAt: sql2`now()`
|
|
1628
|
+
}
|
|
1629
|
+
});
|
|
1630
|
+
}
|
|
1631
|
+
function zeroVectorLiteral() {
|
|
1632
|
+
return `[${Array.from({ length: 384 }, () => 0).join(",")}]`;
|
|
1633
|
+
}
|
|
1634
|
+
function parseVectorLiteral(literal) {
|
|
1635
|
+
const trimmed = literal.startsWith("[") ? literal.slice(1, -1) : literal;
|
|
1636
|
+
if (!trimmed) return [];
|
|
1637
|
+
return trimmed.split(",").map((value) => Number(value));
|
|
1638
|
+
}
|
|
1639
|
+
function advisoryLockKeys(namespaceId, sessionId) {
|
|
1640
|
+
const digest = createHash("sha256").update(`${namespaceId}${sessionId}`).digest();
|
|
1641
|
+
const k1 = digest.readInt32BE(0);
|
|
1642
|
+
const k2 = digest.readInt32BE(4);
|
|
1643
|
+
return [k1, k2];
|
|
1644
|
+
}
|
|
1645
|
+
var recentFactColumns = {
|
|
1646
|
+
id: facts.id,
|
|
1647
|
+
content: facts.content,
|
|
1648
|
+
subject: facts.subject,
|
|
1649
|
+
predicate: facts.predicate,
|
|
1650
|
+
object: facts.object,
|
|
1651
|
+
propositionKey: facts.propositionKey,
|
|
1652
|
+
confidence: facts.confidence,
|
|
1653
|
+
sourceDeleted: facts.sourceDeleted,
|
|
1654
|
+
supersededBy: facts.supersededBy,
|
|
1655
|
+
createdByAgent: facts.createdByAgent,
|
|
1656
|
+
createdAt: facts.createdAt
|
|
1657
|
+
};
|
|
1658
|
+
function liveReference(namespaceId, ownerNamespace, factId) {
|
|
1659
|
+
return sql2`exists (
|
|
1660
|
+
select 1 from shares s
|
|
1661
|
+
where s.to_namespace_id = ${namespaceId}
|
|
1662
|
+
and s.from_namespace_id = ${ownerNamespace}
|
|
1663
|
+
and s.source_fact_id = ${factId}
|
|
1664
|
+
and s.mode = 'reference'
|
|
1665
|
+
and not s.tombstoned
|
|
1666
|
+
)`;
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
// src/core/codebuddy.ts
|
|
1670
|
+
import pTimeout2 from "p-timeout";
|
|
1671
|
+
|
|
1672
|
+
// src/core/memory-lifecycle.ts
|
|
1673
|
+
function isEligibleMemory(fact, options = {}) {
|
|
1674
|
+
if (options.namespace && fact.namespace !== options.namespace) return false;
|
|
1675
|
+
if (fact.sourceDeleted || fact.supersededBy) return false;
|
|
1676
|
+
if (fact.category === "incident" && fact.resolvedBy && !options.includeResolved) return false;
|
|
1677
|
+
return true;
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
// src/planner/default-policy.ts
|
|
1681
|
+
var DefaultPolicy = class {
|
|
1682
|
+
recencyFloorSize;
|
|
1683
|
+
embeddingCandidateLimit;
|
|
1684
|
+
minVectorScore;
|
|
1685
|
+
constructor(options = {}) {
|
|
1686
|
+
this.recencyFloorSize = options.recencyFloorSize ?? 3;
|
|
1687
|
+
this.embeddingCandidateLimit = candidateLimit(options.embeddingCandidateLimit ?? 256);
|
|
1688
|
+
this.minVectorScore = options.minVectorScore ?? 0.1;
|
|
1689
|
+
}
|
|
1690
|
+
async plan(input) {
|
|
1691
|
+
const repo = input.repository;
|
|
1692
|
+
const counts = await repo.countEmbeddingStatuses(input.namespaceId);
|
|
1693
|
+
const query = input.queryEmbedding;
|
|
1694
|
+
const space = query ? embeddingSpaceKey(query.space) : null;
|
|
1695
|
+
let fallbackReason;
|
|
1696
|
+
if (!query) fallbackReason = "query_embedding_unavailable";
|
|
1697
|
+
else if (!validSearchVector(query.vector) || !validEmbeddingVector(query.vector, query.space.dimensions))
|
|
1698
|
+
fallbackReason = "invalid_query_vector";
|
|
1699
|
+
const recencyInteractions = await repo.getRecentInteractions(
|
|
1700
|
+
input.namespaceId,
|
|
1701
|
+
input.sessionId,
|
|
1702
|
+
this.recencyFloorSize
|
|
1703
|
+
);
|
|
1704
|
+
const coverage = {
|
|
1705
|
+
scope: "full_namespace",
|
|
1706
|
+
method: "lexical",
|
|
1707
|
+
compatibleReady: 0,
|
|
1708
|
+
candidateLimit: this.embeddingCandidateLimit,
|
|
1709
|
+
vectorCandidates: 0,
|
|
1710
|
+
lexicalCandidates: 0,
|
|
1711
|
+
lexicalMatchedFacts: 0,
|
|
1712
|
+
hydratedCandidates: 0,
|
|
1713
|
+
spaceFilter: space
|
|
1714
|
+
};
|
|
1715
|
+
let candidates = [];
|
|
1716
|
+
let fallback = "none";
|
|
1717
|
+
if (!fallbackReason && query && space) {
|
|
1718
|
+
const search = await repo.searchSimilarEmbeddings({
|
|
1719
|
+
namespaceId: input.namespaceId,
|
|
1720
|
+
vector: query.vector,
|
|
1721
|
+
embeddingSpace: space,
|
|
1722
|
+
embeddingModel: query.space.model,
|
|
1723
|
+
limit: this.embeddingCandidateLimit
|
|
1724
|
+
});
|
|
1725
|
+
coverage.compatibleReady = search.compatibleReady;
|
|
1726
|
+
coverage.vectorCandidates = search.candidates.length;
|
|
1727
|
+
coverage.method = search.method;
|
|
1728
|
+
const scored = search.candidates.map((row) => ({ row, score: row.score }));
|
|
1729
|
+
const ownerIdsByType = groupOwnerIdsByType(scored);
|
|
1730
|
+
const [factRows, interactionRows, summaryRows] = await Promise.all([
|
|
1731
|
+
repo.getFactsByIds(input.namespaceId, ownerIdsByType.fact),
|
|
1732
|
+
repo.getInteractionsByIds(input.namespaceId, ownerIdsByType.interaction),
|
|
1733
|
+
repo.getSummariesByIds(input.namespaceId, ownerIdsByType.summary)
|
|
1734
|
+
]);
|
|
1735
|
+
candidates = this.hydrateScored(scored, factRows, interactionRows, summaryRows);
|
|
1736
|
+
coverage.hydratedCandidates = candidates.length;
|
|
1737
|
+
if (search.compatibleReady === 0)
|
|
1738
|
+
fallbackReason = counts.ready === 0 ? "no_ready_embeddings" : "no_compatible_embeddings";
|
|
1739
|
+
else if (candidates.length === 0) fallbackReason = "no_hydratable_candidates";
|
|
1740
|
+
}
|
|
1741
|
+
const useFallback = fallbackReason !== void 0;
|
|
1742
|
+
const lexical = await repo.searchFactsLexically(
|
|
1743
|
+
input.namespaceId,
|
|
1744
|
+
input.query,
|
|
1745
|
+
useFallback ? this.embeddingCandidateLimit : Math.min(this.embeddingCandidateLimit, 32)
|
|
1746
|
+
);
|
|
1747
|
+
coverage.lexicalCandidates = lexical.candidates.length;
|
|
1748
|
+
coverage.lexicalMatchedFacts = lexical.matchedFacts;
|
|
1749
|
+
if (useFallback) {
|
|
1750
|
+
const [summary, fallbackInteractions] = await Promise.all([
|
|
1751
|
+
repo.getLatestSummary(input.namespaceId, input.sessionId),
|
|
1752
|
+
repo.getRecentInteractions(
|
|
1753
|
+
input.namespaceId,
|
|
1754
|
+
input.sessionId,
|
|
1755
|
+
Math.max(this.recencyFloorSize, 10)
|
|
1756
|
+
)
|
|
1757
|
+
]);
|
|
1758
|
+
candidates = this.buildFallbackCandidates(fallbackInteractions, summary);
|
|
1759
|
+
if (summary && fallbackInteractions.length > 0) fallback = "recency_summary";
|
|
1760
|
+
else if (summary) fallback = "summary";
|
|
1761
|
+
else fallback = "recency";
|
|
1762
|
+
}
|
|
1763
|
+
const byKey = new Map(candidates.map((candidate) => [candidate.key, candidate]));
|
|
1764
|
+
for (const fact of lexical.candidates) {
|
|
1765
|
+
if (!isEligibleMemory(fact)) continue;
|
|
1766
|
+
const candidate = factCandidate(fact, useFallback ? fact.score : 0.35 * fact.score);
|
|
1767
|
+
const existing = byKey.get(candidate.key);
|
|
1768
|
+
if (existing) existing.score = Math.max(existing.score, candidate.score);
|
|
1769
|
+
else byKey.set(candidate.key, candidate);
|
|
1770
|
+
}
|
|
1771
|
+
candidates = [...byKey.values()];
|
|
1772
|
+
candidates = this.applyRecencyFloor(candidates, recencyInteractions);
|
|
1773
|
+
candidates = this.applyConflictMode(candidates, input.conflictMode);
|
|
1774
|
+
candidates.sort((a, b) => {
|
|
1775
|
+
if (a.isRecencyFloor !== b.isRecencyFloor) return a.isRecencyFloor ? -1 : 1;
|
|
1776
|
+
return b.score - a.score || b.createdAt - a.createdAt || a.key.localeCompare(b.key);
|
|
1777
|
+
});
|
|
1778
|
+
const skipReasons = {};
|
|
1779
|
+
const included = [];
|
|
1780
|
+
let system = buildSystemPrompt(input.namespace, fallback, useFallback);
|
|
1781
|
+
let tokensUsed = countTokens(system);
|
|
1782
|
+
let itemsSkipped = 0;
|
|
1783
|
+
for (const candidate of candidates) {
|
|
1784
|
+
if (tokensUsed + candidate.tokens > input.budget) {
|
|
1785
|
+
bump(skipReasons, "budget_exhausted");
|
|
1786
|
+
itemsSkipped += 1;
|
|
1787
|
+
continue;
|
|
1788
|
+
}
|
|
1789
|
+
if (!candidate.isRecencyFloor && candidate.score < this.minVectorScore && !useFallback) {
|
|
1790
|
+
bump(skipReasons, "low_score");
|
|
1791
|
+
itemsSkipped += 1;
|
|
1792
|
+
continue;
|
|
1793
|
+
}
|
|
1794
|
+
included.push(candidate);
|
|
1795
|
+
tokensUsed += candidate.tokens;
|
|
1796
|
+
}
|
|
1797
|
+
const unresolvedKeys = [
|
|
1798
|
+
...new Set(
|
|
1799
|
+
included.flatMap(
|
|
1800
|
+
(c) => c.fact?.conflict?.unresolved ? [c.fact.propositionKey ?? ""] : []
|
|
1801
|
+
)
|
|
1802
|
+
)
|
|
1803
|
+
].sort();
|
|
1804
|
+
if (unresolvedKeys.length > 0) {
|
|
1805
|
+
const noted = `${system} Unresolved fact conflicts (compare fact provenance): ${unresolvedKeys.join(", ")}.`;
|
|
1806
|
+
const withNote = tokensUsed - countTokens(system) + countTokens(noted);
|
|
1807
|
+
if (withNote <= input.budget) {
|
|
1808
|
+
system = noted;
|
|
1809
|
+
tokensUsed = withNote;
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
const messages = included.map((candidate) => ({
|
|
1813
|
+
role: candidate.role,
|
|
1814
|
+
content: candidate.content,
|
|
1815
|
+
...candidate.fact ? { fact: candidate.fact } : {}
|
|
1816
|
+
}));
|
|
1817
|
+
const stats = {
|
|
1818
|
+
tokensUsed,
|
|
1819
|
+
itemsIncluded: included.length,
|
|
1820
|
+
itemsSkipped,
|
|
1821
|
+
budgetHeadroom: Math.max(0, input.budget - tokensUsed),
|
|
1822
|
+
fallback,
|
|
1823
|
+
skipReasons,
|
|
1824
|
+
embeddingCoverage: counts,
|
|
1825
|
+
candidateCoverage: coverage,
|
|
1826
|
+
...fallbackReason ? { fallbackReason } : {},
|
|
1827
|
+
explain: [
|
|
1828
|
+
`Candidate search covers the full namespace; limit ${this.embeddingCandidateLimit} applies after ranking.`,
|
|
1829
|
+
`Embedding space filter: ${coverage.spaceFilter ?? "unavailable"}; ${coverage.compatibleReady} compatible ready vectors; ${coverage.vectorCandidates} vector candidates; ${coverage.hydratedCandidates} hydrated.`,
|
|
1830
|
+
`Durable lexical facts: ${coverage.lexicalCandidates} candidates of ${coverage.lexicalMatchedFacts} matches.`,
|
|
1831
|
+
...useFallback ? [
|
|
1832
|
+
`Fallback: ${fallbackReason}; ${coverage.lexicalCandidates} durable lexical candidates of ${coverage.lexicalMatchedFacts} matches.`
|
|
1833
|
+
] : []
|
|
1834
|
+
],
|
|
1835
|
+
budgetClamped: false,
|
|
1836
|
+
budgetStatus: tokensUsed > input.budget ? "overflow" : "within_budget",
|
|
1837
|
+
...tokensUsed > input.budget ? { budgetWarning: "Required memory system prompt exceeds budget." } : {}
|
|
1838
|
+
};
|
|
1839
|
+
return { system, messages, stats };
|
|
1840
|
+
}
|
|
1841
|
+
hydrateScored(scored, factRows, interactionRows, summaryRows) {
|
|
1842
|
+
const factById = new Map(factRows.map((row) => [row.id, row]));
|
|
1843
|
+
const interactionById = new Map(interactionRows.map((row) => [row.id, row]));
|
|
1844
|
+
const summaryById = new Map(summaryRows.map((row) => [row.id, row]));
|
|
1845
|
+
const out = [];
|
|
1846
|
+
for (const { row, score } of scored) {
|
|
1847
|
+
if (row.ownerType === "fact") {
|
|
1848
|
+
const fact = factById.get(row.ownerId);
|
|
1849
|
+
if (!fact || !isEligibleMemory(fact)) continue;
|
|
1850
|
+
out.push(factCandidate(fact, score));
|
|
1851
|
+
} else if (row.ownerType === "interaction") {
|
|
1852
|
+
const interaction = interactionById.get(row.ownerId);
|
|
1853
|
+
if (!interaction) continue;
|
|
1854
|
+
out.push({
|
|
1855
|
+
key: `interaction:${interaction.id}`,
|
|
1856
|
+
role: "user",
|
|
1857
|
+
content: interaction.content,
|
|
1858
|
+
tokens: countTokens(interaction.content),
|
|
1859
|
+
score,
|
|
1860
|
+
source: "interaction",
|
|
1861
|
+
ownerId: interaction.id,
|
|
1862
|
+
createdAt: interaction.createdAt.getTime(),
|
|
1863
|
+
confidence: 1,
|
|
1864
|
+
isRecencyFloor: false
|
|
1865
|
+
});
|
|
1866
|
+
} else {
|
|
1867
|
+
const summary = summaryById.get(row.ownerId);
|
|
1868
|
+
if (!summary) continue;
|
|
1869
|
+
out.push({
|
|
1870
|
+
key: `summary:${summary.id}`,
|
|
1871
|
+
role: "system",
|
|
1872
|
+
content: summary.content,
|
|
1873
|
+
tokens: countTokens(summary.content),
|
|
1874
|
+
score,
|
|
1875
|
+
source: "summary",
|
|
1876
|
+
ownerId: summary.id,
|
|
1877
|
+
createdAt: summary.createdAt.getTime(),
|
|
1878
|
+
confidence: 1,
|
|
1879
|
+
isRecencyFloor: false
|
|
1880
|
+
});
|
|
1881
|
+
}
|
|
1882
|
+
}
|
|
1883
|
+
return out;
|
|
1884
|
+
}
|
|
1885
|
+
applyRecencyFloor(candidates, floor) {
|
|
1886
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1887
|
+
for (const candidate of candidates) byKey.set(candidate.key, candidate);
|
|
1888
|
+
for (const row of floor) {
|
|
1889
|
+
const key = `interaction:${row.id}`;
|
|
1890
|
+
const existing = byKey.get(key);
|
|
1891
|
+
if (existing) {
|
|
1892
|
+
existing.isRecencyFloor = true;
|
|
1893
|
+
continue;
|
|
1894
|
+
}
|
|
1895
|
+
byKey.set(key, {
|
|
1896
|
+
key,
|
|
1897
|
+
role: "user",
|
|
1898
|
+
content: row.content,
|
|
1899
|
+
tokens: countTokens(row.content),
|
|
1900
|
+
score: 1,
|
|
1901
|
+
source: "interaction",
|
|
1902
|
+
ownerId: row.id,
|
|
1903
|
+
createdAt: row.createdAt.getTime(),
|
|
1904
|
+
confidence: 1,
|
|
1905
|
+
isRecencyFloor: true
|
|
1906
|
+
});
|
|
1907
|
+
}
|
|
1908
|
+
return Array.from(byKey.values());
|
|
1909
|
+
}
|
|
1910
|
+
buildFallbackCandidates(interactions2, summary) {
|
|
1911
|
+
const out = [];
|
|
1912
|
+
if (summary) {
|
|
1913
|
+
out.push({
|
|
1914
|
+
key: `summary:${summary.id}`,
|
|
1915
|
+
role: "system",
|
|
1916
|
+
content: summary.content,
|
|
1917
|
+
tokens: countTokens(summary.content),
|
|
1918
|
+
score: 1,
|
|
1919
|
+
source: "summary",
|
|
1920
|
+
ownerId: summary.id,
|
|
1921
|
+
createdAt: summary.createdAt.getTime(),
|
|
1922
|
+
confidence: 1,
|
|
1923
|
+
isRecencyFloor: true
|
|
1924
|
+
});
|
|
1925
|
+
}
|
|
1926
|
+
for (const row of interactions2) {
|
|
1927
|
+
out.push({
|
|
1928
|
+
key: `interaction:${row.id}`,
|
|
1929
|
+
role: "user",
|
|
1930
|
+
content: row.content,
|
|
1931
|
+
tokens: countTokens(row.content),
|
|
1932
|
+
score: 1,
|
|
1933
|
+
source: "interaction",
|
|
1934
|
+
ownerId: row.id,
|
|
1935
|
+
createdAt: row.createdAt.getTime(),
|
|
1936
|
+
confidence: 1,
|
|
1937
|
+
isRecencyFloor: true
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
return out;
|
|
1941
|
+
}
|
|
1942
|
+
/**
|
|
1943
|
+
* Resolve rival assertions of the same explicit proposition key. Facts without
|
|
1944
|
+
* a key are treated as multi-valued and never grouped, so identical or
|
|
1945
|
+
* different surface text alone never merges or splits claims.
|
|
1946
|
+
*/
|
|
1947
|
+
applyConflictMode(candidates, mode) {
|
|
1948
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1949
|
+
for (const candidate of candidates) {
|
|
1950
|
+
const key = candidate.fact?.propositionKey;
|
|
1951
|
+
if (!key) continue;
|
|
1952
|
+
groups.set(key, [...groups.get(key) ?? [], candidate]);
|
|
1953
|
+
}
|
|
1954
|
+
const dropped = /* @__PURE__ */ new Set();
|
|
1955
|
+
for (const bucket of groups.values()) {
|
|
1956
|
+
if (bucket.length < 2) continue;
|
|
1957
|
+
bucket.sort((a, b) => compareRivals(a, b, mode));
|
|
1958
|
+
const winner = bucket[0];
|
|
1959
|
+
const contenders = bucket.filter(
|
|
1960
|
+
(c) => mode === "all" || c.createdAt === winner.createdAt && c.confidence === winner.confidence
|
|
1961
|
+
);
|
|
1962
|
+
const objects = new Set(contenders.map((c) => c.fact?.object.trim().toLowerCase()));
|
|
1963
|
+
const conflict = {
|
|
1964
|
+
candidateIds: bucket.map((c) => c.ownerId).sort(),
|
|
1965
|
+
unresolved: objects.size > 1
|
|
1966
|
+
};
|
|
1967
|
+
for (const candidate of bucket) {
|
|
1968
|
+
if (mode !== "all" && candidate !== winner) dropped.add(candidate.key);
|
|
1969
|
+
else if (candidate.fact) candidate.fact = { ...candidate.fact, conflict };
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
return candidates.filter((candidate) => !dropped.has(candidate.key));
|
|
1973
|
+
}
|
|
1974
|
+
};
|
|
1975
|
+
function compareRivals(a, b, mode) {
|
|
1976
|
+
const byTime = b.createdAt - a.createdAt;
|
|
1977
|
+
const byConfidence = b.confidence - a.confidence;
|
|
1978
|
+
const ordered = mode === "highest_confidence" ? byConfidence || byTime : byTime || byConfidence;
|
|
1979
|
+
return ordered || a.ownerId.localeCompare(b.ownerId);
|
|
1980
|
+
}
|
|
1981
|
+
function groupOwnerIdsByType(scored) {
|
|
1982
|
+
const out = { fact: [], interaction: [], summary: [] };
|
|
1983
|
+
for (const { row } of scored) {
|
|
1984
|
+
out[row.ownerType].push(row.ownerId);
|
|
1985
|
+
}
|
|
1986
|
+
return out;
|
|
1987
|
+
}
|
|
1988
|
+
function bump(map, key) {
|
|
1989
|
+
map[key] = (map[key] ?? 0) + 1;
|
|
1990
|
+
}
|
|
1991
|
+
function buildSystemPrompt(namespace, fallback, isFallback) {
|
|
1992
|
+
const base = `Memory context for namespace "${namespace}".`;
|
|
1993
|
+
if (isFallback) {
|
|
1994
|
+
return `${base} Fallback active (${fallback}); vector search unavailable or empty.`;
|
|
1995
|
+
}
|
|
1996
|
+
return base;
|
|
1997
|
+
}
|
|
1998
|
+
function factCandidate(fact, score) {
|
|
1999
|
+
return {
|
|
2000
|
+
key: `fact:${fact.id}`,
|
|
2001
|
+
role: "system",
|
|
2002
|
+
content: fact.content,
|
|
2003
|
+
tokens: countTokens(fact.content),
|
|
2004
|
+
score,
|
|
2005
|
+
source: "fact",
|
|
2006
|
+
ownerId: fact.id,
|
|
2007
|
+
createdAt: fact.createdAt.getTime(),
|
|
2008
|
+
confidence: fact.confidence,
|
|
2009
|
+
isRecencyFloor: false,
|
|
2010
|
+
fact: {
|
|
2011
|
+
id: fact.id,
|
|
2012
|
+
subject: fact.subject,
|
|
2013
|
+
predicate: fact.predicate,
|
|
2014
|
+
object: fact.object,
|
|
2015
|
+
propositionKey: fact.propositionKey,
|
|
2016
|
+
confidence: fact.confidence,
|
|
2017
|
+
createdAt: fact.createdAt.toISOString(),
|
|
2018
|
+
createdByAgent: fact.createdByAgent
|
|
2019
|
+
}
|
|
2020
|
+
};
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
// src/providers/huggingface.ts
|
|
2024
|
+
import { HfInference, setLogger } from "@huggingface/inference";
|
|
2025
|
+
|
|
2026
|
+
// src/providers/resilience.ts
|
|
2027
|
+
import PQueue from "p-queue";
|
|
2028
|
+
import pRetry, { AbortError } from "p-retry";
|
|
2029
|
+
import pTimeout from "p-timeout";
|
|
2030
|
+
var DEFAULT_RESILIENCE_OPTIONS = {
|
|
2031
|
+
retries: 5,
|
|
2032
|
+
concurrency: 2,
|
|
2033
|
+
rateLimitPerMinute: 10,
|
|
2034
|
+
timeoutMs: 9e4,
|
|
2035
|
+
warmCacheTtlMs: 5 * 6e4,
|
|
2036
|
+
coldStartBufferMs: 2e3,
|
|
2037
|
+
jitterMs: 250,
|
|
2038
|
+
minRetryTimeoutMs: 1e3,
|
|
2039
|
+
maxRetryTimeoutMs: 3e4
|
|
2040
|
+
};
|
|
2041
|
+
var GatedModelError = class extends Error {
|
|
2042
|
+
model;
|
|
2043
|
+
constructor(model) {
|
|
2044
|
+
super(
|
|
2045
|
+
`Model ${model} is gated. Accept the license at https://huggingface.co/${model} and retry.`
|
|
2046
|
+
);
|
|
2047
|
+
this.name = "GatedModelError";
|
|
2048
|
+
this.model = model;
|
|
2049
|
+
}
|
|
2050
|
+
};
|
|
2051
|
+
var UnauthorizedError = class extends Error {
|
|
2052
|
+
constructor() {
|
|
2053
|
+
super("Hugging Face rejected the API token (401). Set a valid HF token and retry.");
|
|
2054
|
+
this.name = "UnauthorizedError";
|
|
2055
|
+
}
|
|
2056
|
+
};
|
|
2057
|
+
var ModelUnavailableError = class extends Error {
|
|
2058
|
+
model;
|
|
2059
|
+
constructor(model, cause) {
|
|
2060
|
+
super(`Model ${model} is unavailable.`);
|
|
2061
|
+
this.name = "ModelUnavailableError";
|
|
2062
|
+
this.model = model;
|
|
2063
|
+
this.cause = cause;
|
|
2064
|
+
}
|
|
2065
|
+
};
|
|
2066
|
+
var HuggingFaceResilience = class {
|
|
2067
|
+
options;
|
|
2068
|
+
queues = /* @__PURE__ */ new Map();
|
|
2069
|
+
warmUntil = /* @__PURE__ */ new Map();
|
|
2070
|
+
constructor(options = {}) {
|
|
2071
|
+
this.options = {
|
|
2072
|
+
...DEFAULT_RESILIENCE_OPTIONS,
|
|
2073
|
+
...options
|
|
2074
|
+
};
|
|
2075
|
+
}
|
|
2076
|
+
isWarm(model, now = Date.now()) {
|
|
2077
|
+
return (this.warmUntil.get(model) ?? 0) > now;
|
|
2078
|
+
}
|
|
2079
|
+
markWarm(model, now = Date.now()) {
|
|
2080
|
+
this.warmUntil.set(model, now + this.options.warmCacheTtlMs);
|
|
2081
|
+
}
|
|
2082
|
+
async execute(input) {
|
|
2083
|
+
const models = [input.model, ...input.fallbackModels ?? []];
|
|
2084
|
+
let lastError;
|
|
2085
|
+
for (const [index2, model] of models.entries()) {
|
|
2086
|
+
const fallbackTriggered = index2 > 0;
|
|
2087
|
+
try {
|
|
2088
|
+
return await this.executeForModel(model, input, fallbackTriggered);
|
|
2089
|
+
} catch (error) {
|
|
2090
|
+
lastError = error;
|
|
2091
|
+
if (error instanceof GatedModelError || error instanceof UnauthorizedError) {
|
|
2092
|
+
throw error;
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
throw lastError instanceof Error ? lastError : new Error("All provider models failed.");
|
|
2097
|
+
}
|
|
2098
|
+
async executeForModel(model, input, fallbackTriggered) {
|
|
2099
|
+
const queue = this.getQueue(model);
|
|
2100
|
+
const startedAt = Date.now();
|
|
2101
|
+
let retries = 0;
|
|
2102
|
+
let coldStartWaitMs = 0;
|
|
2103
|
+
let status = fallbackTriggered ? "fallback" : "success";
|
|
2104
|
+
try {
|
|
2105
|
+
const result = await queue.add(
|
|
2106
|
+
async () => pRetry(
|
|
2107
|
+
async () => {
|
|
2108
|
+
try {
|
|
2109
|
+
return await pTimeout(input.call(model), {
|
|
2110
|
+
milliseconds: this.options.timeoutMs
|
|
2111
|
+
});
|
|
2112
|
+
} catch (error) {
|
|
2113
|
+
const classified = classifyHuggingFaceError(error, model);
|
|
2114
|
+
if (classified.kind === "gated") {
|
|
2115
|
+
status = "gated";
|
|
2116
|
+
throw new AbortError(new GatedModelError(model));
|
|
2117
|
+
}
|
|
2118
|
+
if (classified.kind === "unauthorized") {
|
|
2119
|
+
throw new AbortError(new UnauthorizedError());
|
|
2120
|
+
}
|
|
2121
|
+
if (classified.kind === "cold_start") {
|
|
2122
|
+
const waitMs = this.calculateColdStartWaitMs(classified.estimatedTimeSeconds);
|
|
2123
|
+
coldStartWaitMs += waitMs;
|
|
2124
|
+
await (input.sleep ?? sleep)(waitMs);
|
|
2125
|
+
throw error;
|
|
2126
|
+
}
|
|
2127
|
+
if (classified.kind === "rate_limited") {
|
|
2128
|
+
status = "rate_limited";
|
|
2129
|
+
if (classified.retryAfterMs) {
|
|
2130
|
+
await (input.sleep ?? sleep)(classified.retryAfterMs);
|
|
2131
|
+
}
|
|
2132
|
+
throw error;
|
|
2133
|
+
}
|
|
2134
|
+
if (classified.kind === "unavailable") {
|
|
2135
|
+
throw new ModelUnavailableError(model, error);
|
|
2136
|
+
}
|
|
2137
|
+
if (classified.kind === "timeout") {
|
|
2138
|
+
status = "timeout";
|
|
2139
|
+
}
|
|
2140
|
+
throw error;
|
|
2141
|
+
}
|
|
2142
|
+
},
|
|
2143
|
+
{
|
|
2144
|
+
retries: this.options.retries,
|
|
2145
|
+
minTimeout: this.options.minRetryTimeoutMs,
|
|
2146
|
+
maxTimeout: this.options.maxRetryTimeoutMs,
|
|
2147
|
+
factor: 2,
|
|
2148
|
+
randomize: true,
|
|
2149
|
+
onFailedAttempt: (error) => {
|
|
2150
|
+
retries = error.attemptNumber;
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
),
|
|
2154
|
+
{ throwOnTimeout: true }
|
|
2155
|
+
);
|
|
2156
|
+
this.markWarm(model);
|
|
2157
|
+
status = fallbackTriggered ? "fallback" : "success";
|
|
2158
|
+
await input.onDiagnostic?.({
|
|
2159
|
+
model,
|
|
2160
|
+
type: input.type,
|
|
2161
|
+
status,
|
|
2162
|
+
latencyMs: Date.now() - startedAt,
|
|
2163
|
+
retries,
|
|
2164
|
+
fallbackTriggered,
|
|
2165
|
+
coldStartWaitMs
|
|
2166
|
+
});
|
|
2167
|
+
return result;
|
|
2168
|
+
} catch (error) {
|
|
2169
|
+
const finalError = error instanceof AbortError && error.originalError ? error.originalError : error;
|
|
2170
|
+
const classified = classifyHuggingFaceError(finalError, model);
|
|
2171
|
+
const finalStatus = statusForClassifiedError(classified.kind, status);
|
|
2172
|
+
await input.onDiagnostic?.({
|
|
2173
|
+
model,
|
|
2174
|
+
type: input.type,
|
|
2175
|
+
status: finalStatus,
|
|
2176
|
+
latencyMs: Date.now() - startedAt,
|
|
2177
|
+
error: finalError instanceof Error ? finalError.message : String(finalError),
|
|
2178
|
+
retries,
|
|
2179
|
+
fallbackTriggered,
|
|
2180
|
+
coldStartWaitMs
|
|
2181
|
+
});
|
|
2182
|
+
throw finalError;
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
getQueue(model) {
|
|
2186
|
+
const existing = this.queues.get(model);
|
|
2187
|
+
if (existing) {
|
|
2188
|
+
return existing;
|
|
2189
|
+
}
|
|
2190
|
+
const queue = new PQueue({
|
|
2191
|
+
concurrency: this.options.concurrency,
|
|
2192
|
+
intervalCap: this.options.rateLimitPerMinute,
|
|
2193
|
+
interval: 6e4
|
|
2194
|
+
});
|
|
2195
|
+
this.queues.set(model, queue);
|
|
2196
|
+
return queue;
|
|
2197
|
+
}
|
|
2198
|
+
calculateColdStartWaitMs(estimatedTimeSeconds) {
|
|
2199
|
+
return Math.ceil(
|
|
2200
|
+
estimatedTimeSeconds * 1e3 + this.options.coldStartBufferMs + this.randomJitter()
|
|
2201
|
+
);
|
|
2202
|
+
}
|
|
2203
|
+
randomJitter() {
|
|
2204
|
+
return Math.floor(Math.random() * this.options.jitterMs);
|
|
2205
|
+
}
|
|
2206
|
+
};
|
|
2207
|
+
function classifyHuggingFaceError(error, model) {
|
|
2208
|
+
if (error instanceof GatedModelError) {
|
|
2209
|
+
return { kind: "gated" };
|
|
2210
|
+
}
|
|
2211
|
+
if (error instanceof ModelUnavailableError) {
|
|
2212
|
+
return { kind: "unavailable" };
|
|
2213
|
+
}
|
|
2214
|
+
if (error instanceof UnauthorizedError) {
|
|
2215
|
+
return { kind: "unauthorized" };
|
|
2216
|
+
}
|
|
2217
|
+
if (error instanceof Error && error.name === "TimeoutError") {
|
|
2218
|
+
return { kind: "timeout" };
|
|
2219
|
+
}
|
|
2220
|
+
const hfError = error;
|
|
2221
|
+
const status = hfError.status ?? hfError.statusCode ?? hfError.response?.status ?? // @huggingface/inference 4.x errors carry the code here.
|
|
2222
|
+
hfError.httpResponse?.status;
|
|
2223
|
+
if (status === 401) {
|
|
2224
|
+
return { kind: "unauthorized" };
|
|
2225
|
+
}
|
|
2226
|
+
if (status === 403) {
|
|
2227
|
+
return { kind: "gated" };
|
|
2228
|
+
}
|
|
2229
|
+
if (status === 404) {
|
|
2230
|
+
return { kind: "unavailable" };
|
|
2231
|
+
}
|
|
2232
|
+
if (status === 429) {
|
|
2233
|
+
const retryAfterMs = readRetryAfterMs(hfError);
|
|
2234
|
+
return retryAfterMs === void 0 ? { kind: "rate_limited" } : { kind: "rate_limited", retryAfterMs };
|
|
2235
|
+
}
|
|
2236
|
+
if (status === 503) {
|
|
2237
|
+
const estimatedTimeSeconds = readEstimatedTimeSeconds(hfError);
|
|
2238
|
+
if (estimatedTimeSeconds !== void 0) {
|
|
2239
|
+
return { kind: "cold_start", estimatedTimeSeconds };
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
if (error instanceof Error && error.message.includes("timed out")) {
|
|
2243
|
+
return { kind: "timeout" };
|
|
2244
|
+
}
|
|
2245
|
+
if (error instanceof Error && error.message.includes(model) && error.message.includes("gated")) {
|
|
2246
|
+
return { kind: "gated" };
|
|
2247
|
+
}
|
|
2248
|
+
return { kind: "unknown" };
|
|
2249
|
+
}
|
|
2250
|
+
function statusForClassifiedError(kind, currentStatus) {
|
|
2251
|
+
if (kind === "gated") {
|
|
2252
|
+
return "gated";
|
|
2253
|
+
}
|
|
2254
|
+
if (kind === "rate_limited") {
|
|
2255
|
+
return "rate_limited";
|
|
2256
|
+
}
|
|
2257
|
+
if (kind === "timeout") {
|
|
2258
|
+
return "timeout";
|
|
2259
|
+
}
|
|
2260
|
+
if (currentStatus === "fallback") {
|
|
2261
|
+
return "fallback";
|
|
2262
|
+
}
|
|
2263
|
+
return "failed";
|
|
2264
|
+
}
|
|
2265
|
+
function readEstimatedTimeSeconds(error) {
|
|
2266
|
+
if (typeof error.estimated_time === "number") {
|
|
2267
|
+
return error.estimated_time;
|
|
2268
|
+
}
|
|
2269
|
+
if (typeof error.estimatedTime === "number") {
|
|
2270
|
+
return error.estimatedTime;
|
|
2271
|
+
}
|
|
2272
|
+
if (typeof error.cause === "object" && error.cause !== null && "estimated_time" in error.cause) {
|
|
2273
|
+
const estimatedTime = error.cause.estimated_time;
|
|
2274
|
+
return typeof estimatedTime === "number" ? estimatedTime : void 0;
|
|
2275
|
+
}
|
|
2276
|
+
return void 0;
|
|
2277
|
+
}
|
|
2278
|
+
function readRetryAfterMs(error) {
|
|
2279
|
+
const retryAfter = error.response?.headers?.get?.("retry-after");
|
|
2280
|
+
if (!retryAfter) {
|
|
2281
|
+
return void 0;
|
|
2282
|
+
}
|
|
2283
|
+
const retryAfterSeconds = Number(retryAfter);
|
|
2284
|
+
return Number.isFinite(retryAfterSeconds) ? retryAfterSeconds * 1e3 : void 0;
|
|
2285
|
+
}
|
|
2286
|
+
function sleep(ms) {
|
|
2287
|
+
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
2288
|
+
}
|
|
2289
|
+
|
|
2290
|
+
// src/providers/huggingface.ts
|
|
2291
|
+
var toStderr = (...args) => console.error(...args);
|
|
2292
|
+
setLogger({ debug: toStderr, info: toStderr, log: toStderr, warn: toStderr, error: toStderr });
|
|
2293
|
+
var HuggingFaceProvider = class {
|
|
2294
|
+
client;
|
|
2295
|
+
embeddingModels;
|
|
2296
|
+
llmModels;
|
|
2297
|
+
resilience;
|
|
2298
|
+
onDiagnostic;
|
|
2299
|
+
/** The primary model's space: `reindex` re-embeds rows from any other (incl. fallback) space. */
|
|
2300
|
+
space;
|
|
2301
|
+
constructor(options = {}) {
|
|
2302
|
+
this.client = options.client ?? new HfInference(options.apiKey);
|
|
2303
|
+
this.embeddingModels = options.embeddingModels ?? DEFAULT_EMBEDDING_MODELS;
|
|
2304
|
+
this.llmModels = options.llmModels ?? DEFAULT_LLM_MODELS;
|
|
2305
|
+
this.resilience = new HuggingFaceResilience({
|
|
2306
|
+
...DEFAULT_RESILIENCE_OPTIONS,
|
|
2307
|
+
...options.resilience
|
|
2308
|
+
});
|
|
2309
|
+
this.onDiagnostic = options.onDiagnostic;
|
|
2310
|
+
assertCompatibleEmbeddingModels(this.embeddingModels);
|
|
2311
|
+
const primary = this.embeddingModels[0];
|
|
2312
|
+
if (primary) {
|
|
2313
|
+
this.space = {
|
|
2314
|
+
model: primary.id,
|
|
2315
|
+
revision: primary.revision ?? "unversioned",
|
|
2316
|
+
schema: primary.schema ?? "hf-feature-extraction-v1",
|
|
2317
|
+
dimensions: primary.dimensions ?? EMBEDDING_DIMENSIONS
|
|
2318
|
+
};
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
async embed(request) {
|
|
2322
|
+
const primaryModel = request.model ?? this.embeddingModels[0]?.id;
|
|
2323
|
+
if (!primaryModel) {
|
|
2324
|
+
throw new Error("No embedding model configured.");
|
|
2325
|
+
}
|
|
2326
|
+
const fallbackModels = this.embeddingModels.map((model) => model.id).filter((model) => model !== primaryModel);
|
|
2327
|
+
return this.resilience.execute({
|
|
2328
|
+
model: primaryModel,
|
|
2329
|
+
fallbackModels,
|
|
2330
|
+
type: "embedding",
|
|
2331
|
+
...this.onDiagnostic ? { onDiagnostic: this.onDiagnostic } : {},
|
|
2332
|
+
call: async (model) => {
|
|
2333
|
+
const response = await this.client.featureExtraction({
|
|
2334
|
+
model,
|
|
2335
|
+
inputs: request.input
|
|
2336
|
+
});
|
|
2337
|
+
const definition = this.embeddingModels.find((entry) => entry.id === model);
|
|
2338
|
+
const vectors = normalizeEmbeddingResponse(response);
|
|
2339
|
+
const dimensions = vectors[0]?.length ?? 0;
|
|
2340
|
+
const result = {
|
|
2341
|
+
model,
|
|
2342
|
+
revision: definition?.revision ?? "unversioned",
|
|
2343
|
+
schema: definition?.schema ?? "hf-feature-extraction-v1",
|
|
2344
|
+
dimensions,
|
|
2345
|
+
vectors
|
|
2346
|
+
};
|
|
2347
|
+
validateEmbeddingResponse(
|
|
2348
|
+
result,
|
|
2349
|
+
typeof request.input === "string" ? 1 : request.input.length
|
|
2350
|
+
);
|
|
2351
|
+
if (dimensions !== (definition?.dimensions ?? EMBEDDING_DIMENSIONS)) {
|
|
2352
|
+
throw new Error(
|
|
2353
|
+
`Embedding model ${model} returned unexpected dimensions: ${dimensions}.`
|
|
2354
|
+
);
|
|
2355
|
+
}
|
|
2356
|
+
return result;
|
|
2357
|
+
}
|
|
2358
|
+
});
|
|
2359
|
+
}
|
|
2360
|
+
async generateText(request) {
|
|
2361
|
+
const primaryModel = request.model ?? this.llmModels[0]?.id;
|
|
2362
|
+
if (!primaryModel) {
|
|
2363
|
+
throw new Error("No LLM model configured.");
|
|
2364
|
+
}
|
|
2365
|
+
const fallbackModels = this.llmModels.map((model) => model.id).filter((model) => model !== primaryModel);
|
|
2366
|
+
return this.resilience.execute({
|
|
2367
|
+
model: primaryModel,
|
|
2368
|
+
fallbackModels,
|
|
2369
|
+
type: "llm",
|
|
2370
|
+
...this.onDiagnostic ? { onDiagnostic: this.onDiagnostic } : {},
|
|
2371
|
+
call: async (model) => {
|
|
2372
|
+
const response = await this.client.textGeneration({
|
|
2373
|
+
model,
|
|
2374
|
+
inputs: request.prompt
|
|
2375
|
+
});
|
|
2376
|
+
return {
|
|
2377
|
+
model,
|
|
2378
|
+
text: response.generated_text
|
|
2379
|
+
};
|
|
2380
|
+
}
|
|
2381
|
+
});
|
|
2382
|
+
}
|
|
2383
|
+
};
|
|
2384
|
+
function normalizeEmbeddingResponse(response) {
|
|
2385
|
+
if (!Array.isArray(response)) {
|
|
2386
|
+
throw new Error("Unexpected Hugging Face embedding response.");
|
|
2387
|
+
}
|
|
2388
|
+
if (response.every((value) => typeof value === "number")) {
|
|
2389
|
+
return [response];
|
|
2390
|
+
}
|
|
2391
|
+
if (response.every(
|
|
2392
|
+
(vector2) => Array.isArray(vector2) && vector2.every((value) => typeof value === "number")
|
|
2393
|
+
)) {
|
|
2394
|
+
return response;
|
|
2395
|
+
}
|
|
2396
|
+
throw new Error("Unexpected Hugging Face embedding vector shape.");
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
// src/tracing/langsmith.ts
|
|
2400
|
+
function isLangSmithTracingEnabled(env = process.env) {
|
|
2401
|
+
return env.LANGCHAIN_TRACING_V2 === "true" || env.LANGSMITH_TRACING === "true";
|
|
2402
|
+
}
|
|
2403
|
+
function createTracer(env = process.env) {
|
|
2404
|
+
const enabled = isLangSmithTracingEnabled(env);
|
|
2405
|
+
if (!enabled) {
|
|
2406
|
+
return {
|
|
2407
|
+
enabled: false,
|
|
2408
|
+
startSpan: () => noopSpan,
|
|
2409
|
+
withSpan: async (_name, _input, fn) => fn()
|
|
2410
|
+
};
|
|
2411
|
+
}
|
|
2412
|
+
return {
|
|
2413
|
+
enabled: true,
|
|
2414
|
+
startSpan(name, input) {
|
|
2415
|
+
const started = Date.now();
|
|
2416
|
+
log("span:start", { name, input });
|
|
2417
|
+
return {
|
|
2418
|
+
end(metadata) {
|
|
2419
|
+
log("span:end", { name, durationMs: Date.now() - started, metadata });
|
|
2420
|
+
},
|
|
2421
|
+
fail(error, metadata) {
|
|
2422
|
+
log("span:fail", {
|
|
2423
|
+
name,
|
|
2424
|
+
durationMs: Date.now() - started,
|
|
2425
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2426
|
+
metadata
|
|
2427
|
+
});
|
|
2428
|
+
}
|
|
2429
|
+
};
|
|
2430
|
+
},
|
|
2431
|
+
async withSpan(name, input, fn) {
|
|
2432
|
+
const span = this.startSpan(name, input);
|
|
2433
|
+
try {
|
|
2434
|
+
const result = await fn();
|
|
2435
|
+
span.end();
|
|
2436
|
+
return result;
|
|
2437
|
+
} catch (error) {
|
|
2438
|
+
span.fail(error);
|
|
2439
|
+
throw error;
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
};
|
|
2443
|
+
}
|
|
2444
|
+
var noopSpan = {
|
|
2445
|
+
end: () => void 0,
|
|
2446
|
+
fail: () => void 0
|
|
2447
|
+
};
|
|
2448
|
+
function log(event, payload) {
|
|
2449
|
+
if (process.env.NODE_ENV === "test") return;
|
|
2450
|
+
process.stderr.write(`${JSON.stringify({ at: "tracer", event, ...payload })}
|
|
2451
|
+
`);
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
// src/core/config.ts
|
|
2455
|
+
import { z } from "zod";
|
|
2456
|
+
var providerSchema = z.object({
|
|
2457
|
+
type: z.enum(["local", "huggingface", "openrouter"]),
|
|
2458
|
+
apiKey: z.string().min(1).optional(),
|
|
2459
|
+
model: z.string().min(1).optional(),
|
|
2460
|
+
relevanceFloor: z.number().min(-1).max(1).optional()
|
|
2461
|
+
});
|
|
2462
|
+
var codeBuddyConfigSchema = z.object({
|
|
2463
|
+
postgresUrl: z.string().min(1, "postgresUrl is required"),
|
|
2464
|
+
provider: providerSchema,
|
|
2465
|
+
namespace: z.string().min(1, "namespace is required").max(128, "namespace must be 128 chars or fewer").regex(/^[a-zA-Z0-9_.:-]+$/, "namespace must match [a-zA-Z0-9_.:-]+"),
|
|
2466
|
+
projectRoot: z.string().min(1).optional(),
|
|
2467
|
+
tokenBudget: z.number().int().positive().max(1e6).optional(),
|
|
2468
|
+
graphify: z.object({
|
|
2469
|
+
enabled: z.boolean(),
|
|
2470
|
+
graphPath: z.string().min(1).optional()
|
|
2471
|
+
}).optional(),
|
|
2472
|
+
worker: z.object({
|
|
2473
|
+
pollIntervalMs: z.number().int().min(50).max(6e4).optional(),
|
|
2474
|
+
batchSize: z.number().int().min(1).max(64).optional(),
|
|
2475
|
+
maxAttempts: z.number().int().min(1).max(20).optional()
|
|
2476
|
+
}).optional()
|
|
2477
|
+
});
|
|
2478
|
+
function validateConfig(config) {
|
|
2479
|
+
return codeBuddyConfigSchema.parse(config);
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2482
|
+
// src/core/memory-file-store.ts
|
|
2483
|
+
import { mkdir, readdir, readFile, rm } from "fs/promises";
|
|
2484
|
+
import { dirname as dirname2, join, relative, resolve as resolve2, sep } from "path";
|
|
2485
|
+
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
|
|
2486
|
+
import { z as z2 } from "zod";
|
|
2487
|
+
var FACT_CATEGORIES = ["general", "incident"];
|
|
2488
|
+
var INCIDENT_SEVERITIES = ["low", "medium", "high", "critical"];
|
|
2489
|
+
var factFrontMatterSchema = z2.object({
|
|
2490
|
+
schemaVersion: z2.literal(1),
|
|
2491
|
+
id: z2.string().regex(/^fact_[a-z0-9]+$/),
|
|
2492
|
+
namespace: z2.string().min(1),
|
|
2493
|
+
type: z2.literal("fact"),
|
|
2494
|
+
subject: z2.string(),
|
|
2495
|
+
predicate: z2.string(),
|
|
2496
|
+
object: z2.string(),
|
|
2497
|
+
/** Explicit single-valued claim identity; facts sharing it are rival assertions. */
|
|
2498
|
+
propositionKey: z2.string().min(1).nullable().default(null),
|
|
2499
|
+
confidence: z2.number().min(0).max(1),
|
|
2500
|
+
createdAt: z2.preprocess(
|
|
2501
|
+
(value) => value instanceof Date ? value.toISOString() : value,
|
|
2502
|
+
z2.string().datetime()
|
|
2503
|
+
),
|
|
2504
|
+
createdByAgent: z2.string().nullable(),
|
|
2505
|
+
sourceInteractionId: z2.string().nullable(),
|
|
2506
|
+
sourceDeleted: z2.boolean(),
|
|
2507
|
+
category: z2.enum(FACT_CATEGORIES).default("general"),
|
|
2508
|
+
paths: z2.array(z2.string().min(1)).default([]),
|
|
2509
|
+
severity: z2.enum(INCIDENT_SEVERITIES).nullable().default(null),
|
|
2510
|
+
introducedBy: z2.string().nullable().default(null),
|
|
2511
|
+
resolvedBy: z2.string().nullable().default(null),
|
|
2512
|
+
fingerprint: z2.string().regex(/^[a-f0-9]{64}$/).nullable().default(null),
|
|
2513
|
+
sourcePlanId: z2.string().nullable().default(null),
|
|
2514
|
+
sourceSessionId: z2.string().nullable().default(null),
|
|
2515
|
+
verification: z2.array(z2.string()).default([]),
|
|
2516
|
+
supersededBy: z2.string().nullable().default(null),
|
|
2517
|
+
/** Namespaces granted a live reference to this fact (durable form of reference shares). */
|
|
2518
|
+
sharedWith: z2.array(z2.string().min(1)).default([])
|
|
2519
|
+
});
|
|
2520
|
+
var summaryFrontMatterSchema = z2.object({
|
|
2521
|
+
schemaVersion: z2.literal(1),
|
|
2522
|
+
id: z2.string().regex(/^sum_[a-z0-9]+$/),
|
|
2523
|
+
namespace: z2.string().min(1),
|
|
2524
|
+
type: z2.literal("summary"),
|
|
2525
|
+
sessionId: z2.string().min(1),
|
|
2526
|
+
version: z2.number().int().positive(),
|
|
2527
|
+
tokenCount: z2.number().int().nonnegative(),
|
|
2528
|
+
createdAt: z2.preprocess(
|
|
2529
|
+
(value) => value instanceof Date ? value.toISOString() : value,
|
|
2530
|
+
z2.string().datetime()
|
|
2531
|
+
),
|
|
2532
|
+
createdByAgent: z2.string().nullable()
|
|
2533
|
+
});
|
|
2534
|
+
var MemoryFileStore = class {
|
|
2535
|
+
repositoryRoot;
|
|
2536
|
+
factsDirectory;
|
|
2537
|
+
summariesDirectory;
|
|
2538
|
+
constructor(repositoryRoot = process.cwd()) {
|
|
2539
|
+
this.repositoryRoot = resolve2(repositoryRoot);
|
|
2540
|
+
this.factsDirectory = join(this.repositoryRoot, ".codebuddy", "memory", "facts");
|
|
2541
|
+
this.summariesDirectory = join(this.repositoryRoot, ".codebuddy", "memory", "summaries");
|
|
2542
|
+
}
|
|
2543
|
+
async writeFact(raw) {
|
|
2544
|
+
const input = redactPersisted(raw);
|
|
2545
|
+
const path = this.factPath(input.id);
|
|
2546
|
+
await this.prepareDirectory(this.factsDirectory);
|
|
2547
|
+
await assertNoSymlinkBetween(this.repositoryRoot, dirname2(path));
|
|
2548
|
+
const metadata = {
|
|
2549
|
+
schemaVersion: 1,
|
|
2550
|
+
id: input.id,
|
|
2551
|
+
namespace: input.namespace,
|
|
2552
|
+
type: "fact",
|
|
2553
|
+
subject: input.subject,
|
|
2554
|
+
predicate: input.predicate,
|
|
2555
|
+
object: input.object,
|
|
2556
|
+
propositionKey: input.propositionKey,
|
|
2557
|
+
confidence: input.confidence,
|
|
2558
|
+
createdAt: input.createdAt,
|
|
2559
|
+
createdByAgent: input.createdByAgent,
|
|
2560
|
+
sourceInteractionId: input.sourceInteractionId,
|
|
2561
|
+
sourceDeleted: input.sourceDeleted,
|
|
2562
|
+
category: input.category,
|
|
2563
|
+
paths: input.paths,
|
|
2564
|
+
severity: input.severity,
|
|
2565
|
+
introducedBy: input.introducedBy,
|
|
2566
|
+
resolvedBy: input.resolvedBy,
|
|
2567
|
+
fingerprint: input.fingerprint,
|
|
2568
|
+
sourcePlanId: input.sourcePlanId,
|
|
2569
|
+
sourceSessionId: input.sourceSessionId,
|
|
2570
|
+
verification: input.verification,
|
|
2571
|
+
supersededBy: input.supersededBy,
|
|
2572
|
+
sharedWith: input.sharedWith
|
|
2573
|
+
};
|
|
2574
|
+
const frontMatter = stringifyYaml(stripUndefinedAndDefaultIncidentFields(metadata));
|
|
2575
|
+
await writeAtomic(path, composeMarkdown(frontMatter, input.content));
|
|
2576
|
+
return path;
|
|
2577
|
+
}
|
|
2578
|
+
async writeSummary(input) {
|
|
2579
|
+
const path = this.summaryPath(input.id);
|
|
2580
|
+
await this.prepareDirectory(this.summariesDirectory);
|
|
2581
|
+
await assertNoSymlinkBetween(this.repositoryRoot, dirname2(path));
|
|
2582
|
+
const metadata = stringifyYaml({
|
|
2583
|
+
schemaVersion: 1,
|
|
2584
|
+
id: input.id,
|
|
2585
|
+
namespace: input.namespace,
|
|
2586
|
+
type: "summary",
|
|
2587
|
+
sessionId: input.sessionId,
|
|
2588
|
+
version: input.version,
|
|
2589
|
+
tokenCount: input.tokenCount,
|
|
2590
|
+
createdAt: input.createdAt,
|
|
2591
|
+
createdByAgent: input.createdByAgent
|
|
2592
|
+
});
|
|
2593
|
+
await writeAtomic(path, composeMarkdown(metadata, input.content));
|
|
2594
|
+
return path;
|
|
2595
|
+
}
|
|
2596
|
+
async readFact(pathOrId) {
|
|
2597
|
+
const path = pathOrId.endsWith(".md") ? this.safePath(pathOrId) : this.factPath(pathOrId);
|
|
2598
|
+
await assertNoSymlinkBetween(this.repositoryRoot, path);
|
|
2599
|
+
const parsed = parseFrontMatter(await readFile(path, "utf8"));
|
|
2600
|
+
const metadata = factFrontMatterSchema.parse(parseYaml(parsed.frontMatter));
|
|
2601
|
+
return {
|
|
2602
|
+
...metadata,
|
|
2603
|
+
content: parsed.content.trim(),
|
|
2604
|
+
path
|
|
2605
|
+
};
|
|
2606
|
+
}
|
|
2607
|
+
async listFacts() {
|
|
2608
|
+
return (await this.scanFacts()).facts;
|
|
2609
|
+
}
|
|
2610
|
+
/**
|
|
2611
|
+
* Parse every managed fact file. Hand-written markdown is ignored, and a
|
|
2612
|
+
* schema-invalid/corrupt fact is reported in `invalid` (not thrown) so one
|
|
2613
|
+
* bad file can't break recall, capture or reindexing for the whole project.
|
|
2614
|
+
* Invalid files stay on disk for manual repair.
|
|
2615
|
+
*/
|
|
2616
|
+
async scanFacts() {
|
|
2617
|
+
const scan = await this.scan(
|
|
2618
|
+
this.factsDirectory,
|
|
2619
|
+
/^fact_[a-z0-9]+\.md$/,
|
|
2620
|
+
(path) => this.readFact(path)
|
|
2621
|
+
);
|
|
2622
|
+
return { facts: scan.items, invalid: scan.invalid };
|
|
2623
|
+
}
|
|
2624
|
+
/**
|
|
2625
|
+
* List incident facts for the namespace, ranked by severity then recency.
|
|
2626
|
+
* Unlike {@link findIncidentFactsForPaths} this does not filter by path, so
|
|
2627
|
+
* callers (e.g. the context engine's project bootstrap) can surface the
|
|
2628
|
+
* repository's incident hotspots before any specific edit target is known.
|
|
2629
|
+
*/
|
|
2630
|
+
async listIncidentFacts(input = {}) {
|
|
2631
|
+
const incidents = (await this.listFacts()).filter(
|
|
2632
|
+
(fact) => fact.category === "incident" && Boolean(fact.severity) && isEligibleMemory(fact, input)
|
|
2633
|
+
);
|
|
2634
|
+
return incidents.sort((left, right) => {
|
|
2635
|
+
const severity = severityRank(right.severity) - severityRank(left.severity);
|
|
2636
|
+
if (severity !== 0) return severity;
|
|
2637
|
+
return right.createdAt.localeCompare(left.createdAt);
|
|
2638
|
+
});
|
|
2639
|
+
}
|
|
2640
|
+
async findIncidentFactsForPaths(input) {
|
|
2641
|
+
const wanted = new Set(input.paths.map(normalizeMemoryPath));
|
|
2642
|
+
if (wanted.size === 0) return [];
|
|
2643
|
+
return (await this.listIncidentFacts({
|
|
2644
|
+
...input.namespace !== void 0 ? { namespace: input.namespace } : {},
|
|
2645
|
+
...input.includeResolved !== void 0 ? { includeResolved: input.includeResolved } : {}
|
|
2646
|
+
})).filter((fact) => fact.paths.map(normalizeMemoryPath).some((path) => wanted.has(path)));
|
|
2647
|
+
}
|
|
2648
|
+
async readSummary(pathOrId) {
|
|
2649
|
+
const path = pathOrId.endsWith(".md") ? this.safePath(pathOrId) : this.summaryPath(pathOrId);
|
|
2650
|
+
await assertNoSymlinkBetween(this.repositoryRoot, path);
|
|
2651
|
+
const parsed = parseFrontMatter(await readFile(path, "utf8"));
|
|
2652
|
+
return {
|
|
2653
|
+
...summaryFrontMatterSchema.parse(parseYaml(parsed.frontMatter)),
|
|
2654
|
+
content: parsed.content.trim(),
|
|
2655
|
+
path
|
|
2656
|
+
};
|
|
2657
|
+
}
|
|
2658
|
+
async listSummaries() {
|
|
2659
|
+
return (await this.scanSummaries()).summaries;
|
|
2660
|
+
}
|
|
2661
|
+
async scanSummaries() {
|
|
2662
|
+
const scan = await this.scan(
|
|
2663
|
+
this.summariesDirectory,
|
|
2664
|
+
/^sum_[a-z0-9]+\.md$/,
|
|
2665
|
+
(path) => this.readSummary(path)
|
|
2666
|
+
);
|
|
2667
|
+
return { summaries: scan.items, invalid: scan.invalid };
|
|
2668
|
+
}
|
|
2669
|
+
async scan(directory, managedName, read) {
|
|
2670
|
+
try {
|
|
2671
|
+
await this.prepareDirectory(directory);
|
|
2672
|
+
const entries = await readdir(directory, { withFileTypes: true });
|
|
2673
|
+
const result = { items: [], invalid: [] };
|
|
2674
|
+
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
2675
|
+
if (!entry.isFile() || !managedName.test(entry.name)) continue;
|
|
2676
|
+
const path = join(directory, entry.name);
|
|
2677
|
+
try {
|
|
2678
|
+
const item = await read(path);
|
|
2679
|
+
const id = entry.name.slice(0, -".md".length);
|
|
2680
|
+
if (item.id !== id)
|
|
2681
|
+
throw new Error(`file ID ${id} differs from front matter ID ${item.id}`);
|
|
2682
|
+
result.items.push(item);
|
|
2683
|
+
} catch (error) {
|
|
2684
|
+
result.invalid.push({
|
|
2685
|
+
id: entry.name.slice(0, -".md".length),
|
|
2686
|
+
path,
|
|
2687
|
+
error: error.message.split("\n")[0]?.slice(0, 300) ?? "unreadable"
|
|
2688
|
+
});
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2691
|
+
return result;
|
|
2692
|
+
} catch (error) {
|
|
2693
|
+
if (error.code === "ENOENT") return { items: [], invalid: [] };
|
|
2694
|
+
throw error;
|
|
2695
|
+
}
|
|
2696
|
+
}
|
|
2697
|
+
async deleteFact(id) {
|
|
2698
|
+
await rm(this.factPath(id), { force: true });
|
|
2699
|
+
}
|
|
2700
|
+
async markIncidentResolved(id, resolvedBy) {
|
|
2701
|
+
const fact = await this.readFact(id);
|
|
2702
|
+
await this.writeFact(copyFact(fact, { resolvedBy }));
|
|
2703
|
+
}
|
|
2704
|
+
async markFactSuperseded(id, supersededBy) {
|
|
2705
|
+
const fact = await this.readFact(id);
|
|
2706
|
+
await this.writeFact(copyFact(fact, { supersededBy }));
|
|
2707
|
+
}
|
|
2708
|
+
async setFactSharedWith(id, sharedWith) {
|
|
2709
|
+
const fact = await this.readFact(id);
|
|
2710
|
+
await this.writeFact(copyFact(fact, { sharedWith: [...new Set(sharedWith)].sort() }));
|
|
2711
|
+
}
|
|
2712
|
+
/** Persist interaction deletion in Markdown before the derived DB is updated. */
|
|
2713
|
+
async markSourceInteractionDeleted(namespace, sourceInteractionId) {
|
|
2714
|
+
const { facts: facts2 } = await this.scanFacts();
|
|
2715
|
+
for (const fact of facts2) {
|
|
2716
|
+
if (fact.namespace !== namespace || fact.sourceInteractionId !== sourceInteractionId)
|
|
2717
|
+
continue;
|
|
2718
|
+
if (fact.sourceDeleted) continue;
|
|
2719
|
+
await this.writeFact(copyFact(fact, { sourceDeleted: true }));
|
|
2720
|
+
}
|
|
2721
|
+
}
|
|
2722
|
+
/** Persist that a fact's source interaction was forgotten, keeping its provenance id. */
|
|
2723
|
+
async markFactSourceDeleted(id) {
|
|
2724
|
+
const fact = await this.readFact(id);
|
|
2725
|
+
await this.writeFact(copyFact(fact, { sourceDeleted: true }));
|
|
2726
|
+
}
|
|
2727
|
+
factPath(id) {
|
|
2728
|
+
if (!/^fact_[a-z0-9]+$/.test(id)) {
|
|
2729
|
+
throw new Error(`Invalid fact id: ${id}`);
|
|
2730
|
+
}
|
|
2731
|
+
return this.safePath(join(this.factsDirectory, `${id}.md`));
|
|
2732
|
+
}
|
|
2733
|
+
summaryPath(id) {
|
|
2734
|
+
if (!/^sum_[a-z0-9]+$/.test(id)) throw new Error(`Invalid summary id: ${id}`);
|
|
2735
|
+
return this.safePath(join(this.summariesDirectory, `${id}.md`));
|
|
2736
|
+
}
|
|
2737
|
+
safePath(path) {
|
|
2738
|
+
const absolute = resolve2(path);
|
|
2739
|
+
const relativePath = relative(this.repositoryRoot, absolute);
|
|
2740
|
+
if (relativePath === ".." || relativePath.startsWith(`..${sep}`) || relativePath === "") {
|
|
2741
|
+
throw new Error(`Memory path escapes repository root: ${path}`);
|
|
2742
|
+
}
|
|
2743
|
+
return absolute;
|
|
2744
|
+
}
|
|
2745
|
+
async prepareDirectory(directory) {
|
|
2746
|
+
await assertNoSymlinkBetween(this.repositoryRoot, dirname2(directory));
|
|
2747
|
+
await mkdir(directory, { recursive: true, mode: 448 });
|
|
2748
|
+
}
|
|
2749
|
+
};
|
|
2750
|
+
function copyFact(fact, patch) {
|
|
2751
|
+
const { path: _path, schemaVersion: _schemaVersion, type: _type, ...write } = fact;
|
|
2752
|
+
return { ...write, ...patch };
|
|
2753
|
+
}
|
|
2754
|
+
function stripUndefinedAndDefaultIncidentFields(metadata) {
|
|
2755
|
+
return Object.fromEntries(
|
|
2756
|
+
Object.entries(metadata).filter(([key, value]) => {
|
|
2757
|
+
if (value === void 0) return false;
|
|
2758
|
+
if (key === "category" && value === "general") return false;
|
|
2759
|
+
if ((key === "paths" || key === "sharedWith") && Array.isArray(value) && value.length === 0)
|
|
2760
|
+
return false;
|
|
2761
|
+
if (["severity", "introducedBy", "resolvedBy", "propositionKey"].includes(key) && (value === null || value === void 0)) {
|
|
2762
|
+
return false;
|
|
2763
|
+
}
|
|
2764
|
+
return true;
|
|
2765
|
+
})
|
|
2766
|
+
);
|
|
2767
|
+
}
|
|
2768
|
+
function normalizeMemoryPath(path) {
|
|
2769
|
+
return path.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
2770
|
+
}
|
|
2771
|
+
function severityRank(severity) {
|
|
2772
|
+
switch (severity) {
|
|
2773
|
+
case "critical":
|
|
2774
|
+
return 4;
|
|
2775
|
+
case "high":
|
|
2776
|
+
return 3;
|
|
2777
|
+
case "medium":
|
|
2778
|
+
return 2;
|
|
2779
|
+
case "low":
|
|
2780
|
+
return 1;
|
|
2781
|
+
}
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
// src/core/worker.ts
|
|
2785
|
+
var DEFAULT_POLL_MS = 250;
|
|
2786
|
+
var DEFAULT_BATCH_SIZE = 8;
|
|
2787
|
+
var DEFAULT_MAX_ATTEMPTS = 5;
|
|
2788
|
+
var ERROR_BACKOFF_MS = 2e3;
|
|
2789
|
+
var EmbeddingWorker = class {
|
|
2790
|
+
repo;
|
|
2791
|
+
provider;
|
|
2792
|
+
pollIntervalMs;
|
|
2793
|
+
batchSize;
|
|
2794
|
+
maxAttempts;
|
|
2795
|
+
onDiagnostic;
|
|
2796
|
+
state = "idle";
|
|
2797
|
+
loopPromise = null;
|
|
2798
|
+
wake = null;
|
|
2799
|
+
idleResolvers = [];
|
|
2800
|
+
inFlight = 0;
|
|
2801
|
+
constructor(options) {
|
|
2802
|
+
this.repo = options.repository;
|
|
2803
|
+
this.provider = options.provider;
|
|
2804
|
+
this.pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_MS;
|
|
2805
|
+
this.batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
2806
|
+
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
2807
|
+
this.onDiagnostic = options.onDiagnostic;
|
|
2808
|
+
}
|
|
2809
|
+
getState() {
|
|
2810
|
+
return this.state;
|
|
2811
|
+
}
|
|
2812
|
+
start() {
|
|
2813
|
+
if (this.loopPromise !== null) return;
|
|
2814
|
+
this.state = "running";
|
|
2815
|
+
this.loopPromise = this.loop().finally(() => {
|
|
2816
|
+
this.loopPromise = null;
|
|
2817
|
+
if (this.state === "running") this.state = "idle";
|
|
2818
|
+
});
|
|
2819
|
+
this.loopPromise.catch(() => {
|
|
2820
|
+
});
|
|
2821
|
+
}
|
|
2822
|
+
notify() {
|
|
2823
|
+
if (this.state === "idle" && this.loopPromise === null) this.start();
|
|
2824
|
+
if (this.wake) {
|
|
2825
|
+
const fn = this.wake;
|
|
2826
|
+
this.wake = null;
|
|
2827
|
+
fn();
|
|
2828
|
+
}
|
|
2829
|
+
}
|
|
2830
|
+
/**
|
|
2831
|
+
* Resolve once the worker reaches an empty poll (no in-flight jobs and
|
|
2832
|
+
* nothing left to claim). Retained for API compatibility: with
|
|
2833
|
+
* repository-side leases and backoff there is no unbounded claim window to
|
|
2834
|
+
* wait out anymore.
|
|
2835
|
+
*/
|
|
2836
|
+
async drain() {
|
|
2837
|
+
await new Promise((resolve3) => {
|
|
2838
|
+
this.idleResolvers.push(resolve3);
|
|
2839
|
+
this.notify();
|
|
2840
|
+
});
|
|
2841
|
+
}
|
|
2842
|
+
async stop() {
|
|
2843
|
+
if (this.state === "stopped" || this.state === "idle") {
|
|
2844
|
+
this.state = "stopped";
|
|
2845
|
+
return;
|
|
2846
|
+
}
|
|
2847
|
+
this.state = "draining";
|
|
2848
|
+
this.notify();
|
|
2849
|
+
if (this.loopPromise) {
|
|
2850
|
+
await this.loopPromise;
|
|
2851
|
+
}
|
|
2852
|
+
this.state = "stopped";
|
|
2853
|
+
}
|
|
2854
|
+
async shutdown() {
|
|
2855
|
+
await this.stop();
|
|
2856
|
+
}
|
|
2857
|
+
getInFlight() {
|
|
2858
|
+
return this.inFlight;
|
|
2859
|
+
}
|
|
2860
|
+
async loop() {
|
|
2861
|
+
while (this.state === "running") {
|
|
2862
|
+
let jobs;
|
|
2863
|
+
try {
|
|
2864
|
+
jobs = await this.repo.claimPendingEmbeddings(this.batchSize);
|
|
2865
|
+
} catch (error) {
|
|
2866
|
+
console.error(
|
|
2867
|
+
`[worker] claimPendingEmbeddings failed: ${error instanceof Error ? error.message : String(error)}; retrying`
|
|
2868
|
+
);
|
|
2869
|
+
if (this.state !== "running") break;
|
|
2870
|
+
await new Promise((resolve3) => setTimeout(resolve3, ERROR_BACKOFF_MS));
|
|
2871
|
+
continue;
|
|
2872
|
+
}
|
|
2873
|
+
if (jobs.length === 0) {
|
|
2874
|
+
this.flushIdleResolvers();
|
|
2875
|
+
if (this.state !== "running") break;
|
|
2876
|
+
await this.sleep();
|
|
2877
|
+
continue;
|
|
2878
|
+
}
|
|
2879
|
+
this.inFlight += jobs.length;
|
|
2880
|
+
try {
|
|
2881
|
+
const outcomes = await Promise.allSettled(jobs.map((job) => this.processJob(job)));
|
|
2882
|
+
for (const outcome of outcomes) {
|
|
2883
|
+
if (outcome.status === "rejected") {
|
|
2884
|
+
console.error(`[worker] embedding job failed unexpectedly: ${String(outcome.reason)}`);
|
|
2885
|
+
}
|
|
2886
|
+
}
|
|
2887
|
+
} finally {
|
|
2888
|
+
this.inFlight -= jobs.length;
|
|
2889
|
+
}
|
|
2890
|
+
}
|
|
2891
|
+
this.flushIdleResolvers();
|
|
2892
|
+
}
|
|
2893
|
+
flushIdleResolvers() {
|
|
2894
|
+
if (this.inFlight > 0) return;
|
|
2895
|
+
const resolvers = this.idleResolvers;
|
|
2896
|
+
this.idleResolvers = [];
|
|
2897
|
+
for (const resolve3 of resolvers) resolve3();
|
|
2898
|
+
}
|
|
2899
|
+
async processJob(job) {
|
|
2900
|
+
let leaseLost = false;
|
|
2901
|
+
const renewal = setInterval(
|
|
2902
|
+
() => {
|
|
2903
|
+
void this.repo.renewEmbeddingLease(job.id, job.leaseToken).then((renewed) => {
|
|
2904
|
+
if (!renewed) leaseLost = true;
|
|
2905
|
+
}).catch(() => {
|
|
2906
|
+
leaseLost = true;
|
|
2907
|
+
});
|
|
2908
|
+
},
|
|
2909
|
+
Math.max(250, Math.floor(job.leaseMs / 3))
|
|
2910
|
+
);
|
|
2911
|
+
try {
|
|
2912
|
+
const response = await this.provider.embed({
|
|
2913
|
+
input: job.content,
|
|
2914
|
+
model: job.embeddingModel
|
|
2915
|
+
});
|
|
2916
|
+
validateEmbeddingResponse(response, 1);
|
|
2917
|
+
const vector2 = response.vectors[0];
|
|
2918
|
+
if (!vector2 || vector2.length === 0) {
|
|
2919
|
+
throw new Error("Provider returned no embedding vector.");
|
|
2920
|
+
}
|
|
2921
|
+
if (leaseLost || !await this.repo.markEmbeddingReady(
|
|
2922
|
+
job.id,
|
|
2923
|
+
job.leaseToken,
|
|
2924
|
+
vector2,
|
|
2925
|
+
response.model,
|
|
2926
|
+
embeddingSpaceKey(embeddingSpace(response))
|
|
2927
|
+
))
|
|
2928
|
+
return;
|
|
2929
|
+
await this.emitDiagnostic({
|
|
2930
|
+
id: generateEntityId("mc"),
|
|
2931
|
+
namespaceId: job.namespaceId,
|
|
2932
|
+
metadata: {
|
|
2933
|
+
model: response.model,
|
|
2934
|
+
type: "embedding",
|
|
2935
|
+
status: "success",
|
|
2936
|
+
latencyMs: 0,
|
|
2937
|
+
retries: 0,
|
|
2938
|
+
fallbackTriggered: false,
|
|
2939
|
+
coldStartWaitMs: 0
|
|
2940
|
+
}
|
|
2941
|
+
});
|
|
2942
|
+
} catch (error) {
|
|
2943
|
+
const attempts = job.attempts + 1;
|
|
2944
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2945
|
+
if (leaseLost) return;
|
|
2946
|
+
try {
|
|
2947
|
+
if (!await this.repo.markEmbeddingFailed(job.id, job.leaseToken, message, attempts, {
|
|
2948
|
+
maxAttempts: this.maxAttempts
|
|
2949
|
+
}))
|
|
2950
|
+
return;
|
|
2951
|
+
} catch (persistenceError) {
|
|
2952
|
+
console.error(
|
|
2953
|
+
`[worker] could not persist embedding failure: ${persistenceError instanceof Error ? persistenceError.message : String(persistenceError)}`
|
|
2954
|
+
);
|
|
2955
|
+
return;
|
|
2956
|
+
}
|
|
2957
|
+
await this.emitDiagnostic({
|
|
2958
|
+
id: generateEntityId("mc"),
|
|
2959
|
+
namespaceId: job.namespaceId,
|
|
2960
|
+
metadata: {
|
|
2961
|
+
model: job.embeddingModel,
|
|
2962
|
+
type: "embedding",
|
|
2963
|
+
status: attempts >= this.maxAttempts ? "failed" : "fallback",
|
|
2964
|
+
latencyMs: 0,
|
|
2965
|
+
error: message,
|
|
2966
|
+
retries: attempts,
|
|
2967
|
+
fallbackTriggered: false,
|
|
2968
|
+
coldStartWaitMs: 0
|
|
2969
|
+
}
|
|
2970
|
+
});
|
|
2971
|
+
} finally {
|
|
2972
|
+
clearInterval(renewal);
|
|
2973
|
+
}
|
|
2974
|
+
}
|
|
2975
|
+
emitDiagnostic(entry) {
|
|
2976
|
+
const sink = this.onDiagnostic;
|
|
2977
|
+
if (!sink) return Promise.resolve();
|
|
2978
|
+
return Promise.resolve().then(() => sink(entry)).catch(() => {
|
|
2979
|
+
});
|
|
2980
|
+
}
|
|
2981
|
+
sleep(ms) {
|
|
2982
|
+
return new Promise((resolve3) => {
|
|
2983
|
+
const timer = setTimeout(() => {
|
|
2984
|
+
this.wake = null;
|
|
2985
|
+
resolve3();
|
|
2986
|
+
}, ms ?? this.pollIntervalMs);
|
|
2987
|
+
this.wake = () => {
|
|
2988
|
+
clearTimeout(timer);
|
|
2989
|
+
resolve3();
|
|
2990
|
+
};
|
|
2991
|
+
});
|
|
2992
|
+
}
|
|
2993
|
+
};
|
|
2994
|
+
|
|
2995
|
+
// src/core/codebuddy.ts
|
|
2996
|
+
var RECALL_EMBED_TIMEOUT_MS = 1e4;
|
|
2997
|
+
var CodeBuddy = class {
|
|
2998
|
+
config;
|
|
2999
|
+
repo;
|
|
3000
|
+
provider;
|
|
3001
|
+
worker;
|
|
3002
|
+
policy;
|
|
3003
|
+
tracer;
|
|
3004
|
+
memoryFileStore;
|
|
3005
|
+
namespaceId = null;
|
|
3006
|
+
initialized = false;
|
|
3007
|
+
constructor(config, deps) {
|
|
3008
|
+
this.config = validateConfig(config);
|
|
3009
|
+
if (!deps?.repository) {
|
|
3010
|
+
throw new Error(
|
|
3011
|
+
"CodeBuddy requires a MemoryRepository. Pass deps.repository (use createPostgresRepository for production)."
|
|
3012
|
+
);
|
|
3013
|
+
}
|
|
3014
|
+
this.repo = deps.repository;
|
|
3015
|
+
this.memoryFileStore = deps.memoryFileStore;
|
|
3016
|
+
this.provider = deps.provider ?? buildProviderFromConfig(this.config.provider);
|
|
3017
|
+
this.worker = deps.worker ?? new EmbeddingWorker({
|
|
3018
|
+
repository: this.repo,
|
|
3019
|
+
provider: this.provider,
|
|
3020
|
+
...this.config.worker?.pollIntervalMs !== void 0 ? { pollIntervalMs: this.config.worker.pollIntervalMs } : {},
|
|
3021
|
+
...this.config.worker?.batchSize !== void 0 ? { batchSize: this.config.worker.batchSize } : {},
|
|
3022
|
+
...this.config.worker?.maxAttempts !== void 0 ? { maxAttempts: this.config.worker.maxAttempts } : {}
|
|
3023
|
+
});
|
|
3024
|
+
this.policy = deps.policy ?? new DefaultPolicy();
|
|
3025
|
+
this.tracer = deps.tracer ?? createTracer();
|
|
3026
|
+
}
|
|
3027
|
+
async init(options = {}) {
|
|
3028
|
+
const namespace = await this.repo.ensureNamespace(this.config.namespace);
|
|
3029
|
+
this.namespaceId = namespace.id;
|
|
3030
|
+
this.initialized = true;
|
|
3031
|
+
if (options.startWorker !== false) this.worker.start();
|
|
3032
|
+
return {
|
|
3033
|
+
ready: true,
|
|
3034
|
+
phase: "runtime",
|
|
3035
|
+
message: `CodeBuddy ready in namespace ${namespace.name}.`,
|
|
3036
|
+
namespaceId: namespace.id
|
|
3037
|
+
};
|
|
3038
|
+
}
|
|
3039
|
+
async shutdown() {
|
|
3040
|
+
await this.worker.shutdown();
|
|
3041
|
+
}
|
|
3042
|
+
getWorker() {
|
|
3043
|
+
return this.worker;
|
|
3044
|
+
}
|
|
3045
|
+
async remember(input) {
|
|
3046
|
+
const namespaceId = this.requireNamespaceId();
|
|
3047
|
+
const type = input.type ?? "interaction";
|
|
3048
|
+
const sessionId = input.sessionId ?? generateSessionId();
|
|
3049
|
+
const content = input.content;
|
|
3050
|
+
if (!content || content.length === 0) {
|
|
3051
|
+
throw new Error("remember: content must not be empty.");
|
|
3052
|
+
}
|
|
3053
|
+
const result = await this.repo.withSessionLock(
|
|
3054
|
+
namespaceId,
|
|
3055
|
+
sessionId,
|
|
3056
|
+
() => this.writeOne({
|
|
3057
|
+
namespaceId,
|
|
3058
|
+
sessionId,
|
|
3059
|
+
type,
|
|
3060
|
+
content,
|
|
3061
|
+
idempotencyKey: input.idempotencyKey,
|
|
3062
|
+
...input.agentId !== void 0 ? { agentId: input.agentId } : {}
|
|
3063
|
+
})
|
|
3064
|
+
);
|
|
3065
|
+
this.worker.notify();
|
|
3066
|
+
return {
|
|
3067
|
+
id: result.id,
|
|
3068
|
+
sessionId: result.sessionId,
|
|
3069
|
+
deduplicated: result.deduplicated
|
|
3070
|
+
};
|
|
3071
|
+
}
|
|
3072
|
+
async rememberBatch(items) {
|
|
3073
|
+
if (!Array.isArray(items) || items.length === 0) return [];
|
|
3074
|
+
const results = [];
|
|
3075
|
+
for (const item of items) {
|
|
3076
|
+
results.push(await this.remember(item));
|
|
3077
|
+
}
|
|
3078
|
+
return results;
|
|
3079
|
+
}
|
|
3080
|
+
async recall(input) {
|
|
3081
|
+
const namespaceId = this.requireNamespaceId();
|
|
3082
|
+
if (!input.query || input.query.length === 0) {
|
|
3083
|
+
throw new Error("recall: query must not be empty.");
|
|
3084
|
+
}
|
|
3085
|
+
const requested = input.budget ?? this.config.tokenBudget ?? 4e3;
|
|
3086
|
+
const clamp = clampBudgetForCallerModel(requested, input.callerModel);
|
|
3087
|
+
const queryEmbedding = await this.embedQueryWithFallback(input.query);
|
|
3088
|
+
const planInput = {
|
|
3089
|
+
namespace: this.config.namespace,
|
|
3090
|
+
namespaceId,
|
|
3091
|
+
sessionId: input.sessionId,
|
|
3092
|
+
query: input.query,
|
|
3093
|
+
budget: clamp.budget,
|
|
3094
|
+
conflictMode: input.conflictMode ?? "all",
|
|
3095
|
+
queryEmbedding,
|
|
3096
|
+
repository: this.repo,
|
|
3097
|
+
...input.callerModel !== void 0 ? { callerModel: input.callerModel } : {}
|
|
3098
|
+
};
|
|
3099
|
+
const planned = await this.tracer.withSpan(
|
|
3100
|
+
"codebuddy.recall",
|
|
3101
|
+
{
|
|
3102
|
+
namespace: this.config.namespace,
|
|
3103
|
+
sessionId: input.sessionId,
|
|
3104
|
+
budget: clamp.budget,
|
|
3105
|
+
conflictMode: planInput.conflictMode,
|
|
3106
|
+
callerModel: input.callerModel ?? null
|
|
3107
|
+
},
|
|
3108
|
+
() => this.policy.plan(planInput)
|
|
3109
|
+
);
|
|
3110
|
+
planned.stats.budgetClamped = clamp.clamped;
|
|
3111
|
+
if (clamp.clamped && clamp.reason) {
|
|
3112
|
+
planned.stats.budgetClampReason = clamp.reason;
|
|
3113
|
+
}
|
|
3114
|
+
return planned;
|
|
3115
|
+
}
|
|
3116
|
+
/**
|
|
3117
|
+
* Dense evidence for file facts supplied by the context engine: the cosine of
|
|
3118
|
+
* each fact whose stored vector clears the embedding model's calibrated
|
|
3119
|
+
* `relevanceFloor` against the task (exact pgvector search over the
|
|
3120
|
+
* namespace). Facts below the floor, of unknown models, or on any failure or
|
|
3121
|
+
* timeout are simply absent, so callers fall back to lexical evidence.
|
|
3122
|
+
*/
|
|
3123
|
+
async rankContextFacts(task, factIds, options = {}) {
|
|
3124
|
+
if (!task.trim() || factIds.length === 0) return {};
|
|
3125
|
+
const allowed = new Set(factIds.slice(0, 256));
|
|
3126
|
+
try {
|
|
3127
|
+
return await pTimeout2(
|
|
3128
|
+
(async () => {
|
|
3129
|
+
const query = await this.embedQueryWithFallback(task.slice(0, 4096));
|
|
3130
|
+
const floor = query ? this.config.provider.relevanceFloor ?? getModelDefinition(query.space.model)?.relevanceFloor : void 0;
|
|
3131
|
+
if (!query || floor === void 0) return {};
|
|
3132
|
+
const result = await this.repo.searchSimilarEmbeddings({
|
|
3133
|
+
namespaceId: this.requireNamespaceId(),
|
|
3134
|
+
vector: query.vector,
|
|
3135
|
+
embeddingSpace: embeddingSpaceKey(query.space),
|
|
3136
|
+
embeddingModel: query.space.model,
|
|
3137
|
+
limit: 256
|
|
3138
|
+
});
|
|
3139
|
+
const scores = {};
|
|
3140
|
+
for (const row of result.candidates) {
|
|
3141
|
+
if (row.ownerType === "fact" && allowed.has(row.ownerId) && Number.isFinite(row.score) && row.score >= floor) {
|
|
3142
|
+
scores[row.ownerId] = Math.max(scores[row.ownerId] ?? 0, Math.min(1, row.score));
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
return scores;
|
|
3146
|
+
})(),
|
|
3147
|
+
{ milliseconds: options.timeoutMs ?? 1500 }
|
|
3148
|
+
);
|
|
3149
|
+
} catch {
|
|
3150
|
+
return {};
|
|
3151
|
+
}
|
|
3152
|
+
}
|
|
3153
|
+
async embedQueryWithFallback(query) {
|
|
3154
|
+
try {
|
|
3155
|
+
const response = await this.tracer.withSpan(
|
|
3156
|
+
"codebuddy.recall.embed_query",
|
|
3157
|
+
{ length: query.length },
|
|
3158
|
+
() => pTimeout2(this.provider.embed({ input: query }), {
|
|
3159
|
+
milliseconds: RECALL_EMBED_TIMEOUT_MS
|
|
3160
|
+
})
|
|
3161
|
+
);
|
|
3162
|
+
validateEmbeddingResponse(response, 1);
|
|
3163
|
+
const vector2 = response.vectors[0];
|
|
3164
|
+
return vector2 ? { vector: vector2, space: embeddingSpace(response) } : null;
|
|
3165
|
+
} catch {
|
|
3166
|
+
return null;
|
|
3167
|
+
}
|
|
3168
|
+
}
|
|
3169
|
+
/**
|
|
3170
|
+
* Share facts owned by the configured namespace. A reference is a live grant
|
|
3171
|
+
* recorded durably in the source fact's Markdown (`sharedWith`); a snapshot is
|
|
3172
|
+
* a copy owned by the target and written to the target project's own store.
|
|
3173
|
+
*/
|
|
3174
|
+
async share(input) {
|
|
3175
|
+
if (!input.from || !input.to) {
|
|
3176
|
+
throw new Error("share: both 'from' and 'to' namespaces are required.");
|
|
3177
|
+
}
|
|
3178
|
+
if (input.from === input.to) {
|
|
3179
|
+
throw new Error("share: 'from' and 'to' must differ.");
|
|
3180
|
+
}
|
|
3181
|
+
const fromNamespaceId = this.requireNamespaceId();
|
|
3182
|
+
if (input.from !== this.config.namespace) {
|
|
3183
|
+
throw new Error(
|
|
3184
|
+
`share: only facts of the configured namespace "${this.config.namespace}" can be shared.`
|
|
3185
|
+
);
|
|
3186
|
+
}
|
|
3187
|
+
if (!Array.isArray(input.factIds) || input.factIds.length === 0) {
|
|
3188
|
+
return { shared: 0 };
|
|
3189
|
+
}
|
|
3190
|
+
const mode = input.mode ?? "reference";
|
|
3191
|
+
let targetStore;
|
|
3192
|
+
if (mode === "snapshot") {
|
|
3193
|
+
if (input.targetProjectRoot) {
|
|
3194
|
+
targetStore = await resolveTargetStore(input.targetProjectRoot, input.to);
|
|
3195
|
+
} else if (this.memoryFileStore) {
|
|
3196
|
+
throw new Error(
|
|
3197
|
+
"share: snapshot mode requires targetProjectRoot so the copy is durable in the target project's Markdown store."
|
|
3198
|
+
);
|
|
3199
|
+
}
|
|
3200
|
+
}
|
|
3201
|
+
const toNs = await this.repo.ensureNamespace(input.to);
|
|
3202
|
+
let shared = 0;
|
|
3203
|
+
const embeddingModel = DEFAULT_EMBEDDING_MODELS[0]?.id ?? "unknown";
|
|
3204
|
+
for (const factId of input.factIds) {
|
|
3205
|
+
const auditId = generateEntityId("aud");
|
|
3206
|
+
const shareId = generateEntityId("shr");
|
|
3207
|
+
const source = await this.repo.getFactById(fromNamespaceId, factId);
|
|
3208
|
+
if (!source) {
|
|
3209
|
+
throw new Error(`share: fact ${factId} not found in namespace "${input.from}".`);
|
|
3210
|
+
}
|
|
3211
|
+
if (mode === "reference") {
|
|
3212
|
+
const write = {
|
|
3213
|
+
id: shareId,
|
|
3214
|
+
fromNamespaceId,
|
|
3215
|
+
toNamespaceId: toNs.id,
|
|
3216
|
+
sourceFactId: factId,
|
|
3217
|
+
mode: "reference",
|
|
3218
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {}
|
|
3219
|
+
};
|
|
3220
|
+
await this.repo.shareReference(write);
|
|
3221
|
+
await this.updateSharedWith(factId, (current) => [...current, toNs.name]);
|
|
3222
|
+
} else {
|
|
3223
|
+
const snapshotContent = [
|
|
3224
|
+
`[shared from ${input.from}] ${source.subject} ${source.predicate} ${source.object}`,
|
|
3225
|
+
source.content
|
|
3226
|
+
].join("\n");
|
|
3227
|
+
const snapshotFactId = generateEntityId("fact");
|
|
3228
|
+
const snapshotEmbeddingId = generateEntityId("emb");
|
|
3229
|
+
const snapshotFact = {
|
|
3230
|
+
id: snapshotFactId,
|
|
3231
|
+
namespaceId: toNs.id,
|
|
3232
|
+
content: snapshotContent,
|
|
3233
|
+
contentHash: computeFactHash(toNs.name, snapshotContent),
|
|
3234
|
+
subject: source.subject,
|
|
3235
|
+
predicate: "snapshot_of",
|
|
3236
|
+
object: `${input.from}:${factId}`,
|
|
3237
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {}
|
|
3238
|
+
};
|
|
3239
|
+
const embedding = {
|
|
3240
|
+
id: snapshotEmbeddingId,
|
|
3241
|
+
namespaceId: toNs.id,
|
|
3242
|
+
ownerType: "fact",
|
|
3243
|
+
ownerId: snapshotFactId,
|
|
3244
|
+
embeddingModel
|
|
3245
|
+
};
|
|
3246
|
+
const write = {
|
|
3247
|
+
id: shareId,
|
|
3248
|
+
fromNamespaceId,
|
|
3249
|
+
toNamespaceId: toNs.id,
|
|
3250
|
+
sourceFactId: factId,
|
|
3251
|
+
mode: "snapshot",
|
|
3252
|
+
snapshotFactId,
|
|
3253
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {}
|
|
3254
|
+
};
|
|
3255
|
+
const result = await this.repo.shareSnapshot({ share: write, snapshotFact, embedding });
|
|
3256
|
+
if (targetStore)
|
|
3257
|
+
await writeSnapshotFile(this.repo, targetStore, toNs, result.snapshotFactId);
|
|
3258
|
+
}
|
|
3259
|
+
const audit = {
|
|
3260
|
+
id: auditId,
|
|
3261
|
+
namespaceId: fromNamespaceId,
|
|
3262
|
+
action: "share",
|
|
3263
|
+
entityType: "fact",
|
|
3264
|
+
entityId: factId,
|
|
3265
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {},
|
|
3266
|
+
metadata: { mode, toNamespace: toNs.name, shareId }
|
|
3267
|
+
};
|
|
3268
|
+
await this.repo.recordAudit(audit);
|
|
3269
|
+
shared += 1;
|
|
3270
|
+
}
|
|
3271
|
+
this.worker.notify();
|
|
3272
|
+
return { shared };
|
|
3273
|
+
}
|
|
3274
|
+
/** Revoke live reference shares of configured-namespace facts to `to`. */
|
|
3275
|
+
async unshare(input) {
|
|
3276
|
+
const fromNamespaceId = this.requireNamespaceId();
|
|
3277
|
+
const toNs = await this.repo.getNamespaceByName(input.to);
|
|
3278
|
+
if (!toNs || input.factIds.length === 0) return { revoked: 0 };
|
|
3279
|
+
const revoked = await this.repo.revokeShares(fromNamespaceId, toNs.id, input.factIds);
|
|
3280
|
+
for (const factId of input.factIds) {
|
|
3281
|
+
await this.updateSharedWith(factId, (current) => current.filter((ns) => ns !== toNs.name));
|
|
3282
|
+
}
|
|
3283
|
+
if (revoked > 0) {
|
|
3284
|
+
await this.repo.recordAudit({
|
|
3285
|
+
id: generateEntityId("aud"),
|
|
3286
|
+
namespaceId: fromNamespaceId,
|
|
3287
|
+
action: "share",
|
|
3288
|
+
entityType: "fact",
|
|
3289
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {},
|
|
3290
|
+
metadata: { mode: "reference", toNamespace: toNs.name, revoked: input.factIds }
|
|
3291
|
+
});
|
|
3292
|
+
}
|
|
3293
|
+
return { revoked };
|
|
3294
|
+
}
|
|
3295
|
+
/** Keep the source fact's durable `sharedWith` list in step with the shares table. */
|
|
3296
|
+
async updateSharedWith(factId, next) {
|
|
3297
|
+
if (!this.memoryFileStore) return;
|
|
3298
|
+
let file;
|
|
3299
|
+
try {
|
|
3300
|
+
file = await this.memoryFileStore.readFact(factId);
|
|
3301
|
+
} catch (error) {
|
|
3302
|
+
if (error.code === "ENOENT") return;
|
|
3303
|
+
throw error;
|
|
3304
|
+
}
|
|
3305
|
+
if (file.namespace !== this.config.namespace) return;
|
|
3306
|
+
await this.memoryFileStore.setFactSharedWith(factId, next(file.sharedWith));
|
|
3307
|
+
}
|
|
3308
|
+
async forget(id) {
|
|
3309
|
+
if (!id) throw new Error("forget: id is required.");
|
|
3310
|
+
const namespaceId = this.requireNamespaceId();
|
|
3311
|
+
const interactionAudit = {
|
|
3312
|
+
id: generateEntityId("aud"),
|
|
3313
|
+
namespaceId,
|
|
3314
|
+
action: "forget",
|
|
3315
|
+
entityType: "interaction",
|
|
3316
|
+
entityId: id
|
|
3317
|
+
};
|
|
3318
|
+
const interactionResult = await this.repo.withSessionLock(
|
|
3319
|
+
namespaceId,
|
|
3320
|
+
REINDEX_LOCK_KEY,
|
|
3321
|
+
async () => {
|
|
3322
|
+
const existing = await this.repo.getInteractionsByIds(namespaceId, [id]);
|
|
3323
|
+
if (existing.length === 0) return { found: false, sourceDeletedFactIds: [] };
|
|
3324
|
+
await this.memoryFileStore?.markSourceInteractionDeleted(this.config.namespace, id);
|
|
3325
|
+
return this.repo.forgetInteraction(namespaceId, id, interactionAudit);
|
|
3326
|
+
}
|
|
3327
|
+
);
|
|
3328
|
+
if (interactionResult.found) {
|
|
3329
|
+
for (const factId of interactionResult.sourceDeletedFactIds) {
|
|
3330
|
+
const file = await this.memoryFileStore?.readFact(factId).catch(() => null);
|
|
3331
|
+
if (file?.namespace === this.config.namespace) {
|
|
3332
|
+
await this.memoryFileStore?.markFactSourceDeleted(factId);
|
|
3333
|
+
}
|
|
3334
|
+
}
|
|
3335
|
+
return { ok: true, entityType: "interaction" };
|
|
3336
|
+
}
|
|
3337
|
+
const factAudit = {
|
|
3338
|
+
id: generateEntityId("aud"),
|
|
3339
|
+
namespaceId,
|
|
3340
|
+
action: "forget",
|
|
3341
|
+
entityType: "fact",
|
|
3342
|
+
entityId: id
|
|
3343
|
+
};
|
|
3344
|
+
const factResult = await this.repo.withSessionLock(namespaceId, REINDEX_LOCK_KEY, async () => {
|
|
3345
|
+
const result = await this.repo.forgetFact(namespaceId, id, factAudit);
|
|
3346
|
+
if (result.found) {
|
|
3347
|
+
const file = await this.memoryFileStore?.readFact(id).catch(() => null);
|
|
3348
|
+
if (file?.namespace === this.config.namespace) await this.memoryFileStore?.deleteFact(id);
|
|
3349
|
+
}
|
|
3350
|
+
return result;
|
|
3351
|
+
});
|
|
3352
|
+
if (factResult.found) return { ok: true, entityType: "fact" };
|
|
3353
|
+
return { ok: false, entityType: "unknown" };
|
|
3354
|
+
}
|
|
3355
|
+
/**
|
|
3356
|
+
* Reconcile the derived database with the Markdown memory files (the source
|
|
3357
|
+
* of truth) for this namespace: upsert by stable ID, preserving the file's
|
|
3358
|
+
* lifecycle metadata; re-embed only changed content; delete rows whose file
|
|
3359
|
+
* is gone. Only rows a reindex previously imported from Markdown
|
|
3360
|
+
* (`file_backed`) are ever deleted, so DB-only legacy memory survives. Each
|
|
3361
|
+
* file is applied independently: malformed or conflicting files are reported
|
|
3362
|
+
* in `failures` and never abort healthy imports (nor count as deletions).
|
|
3363
|
+
*/
|
|
3364
|
+
async reindexMemory() {
|
|
3365
|
+
const namespaceId = this.requireNamespaceId();
|
|
3366
|
+
const store = this.memoryFileStore;
|
|
3367
|
+
if (!store) {
|
|
3368
|
+
throw new Error("Fact reindexing requires a repository-rooted MemoryFileStore.");
|
|
3369
|
+
}
|
|
3370
|
+
const namespace = this.config.namespace;
|
|
3371
|
+
const result = await this.repo.withSessionLock(namespaceId, REINDEX_LOCK_KEY, async () => {
|
|
3372
|
+
const [factScan, summaryScan] = await Promise.all([store.scanFacts(), store.scanSummaries()]);
|
|
3373
|
+
const facts2 = factScan.facts.filter((file) => isEligibleMemory(file, { namespace }));
|
|
3374
|
+
const summaries = summaryScan.summaries.filter((file) => file.namespace === namespace);
|
|
3375
|
+
const failures = [
|
|
3376
|
+
...factScan.invalid.map((file) => ({ kind: "fact", ...file })),
|
|
3377
|
+
...summaryScan.invalid.map((file) => ({ kind: "summary", ...file }))
|
|
3378
|
+
];
|
|
3379
|
+
const factCounts = emptyCounts(
|
|
3380
|
+
facts2.length + factScan.invalid.length,
|
|
3381
|
+
factScan.invalid.length
|
|
3382
|
+
);
|
|
3383
|
+
const summaryCounts = emptyCounts(
|
|
3384
|
+
summaries.length + summaryScan.invalid.length,
|
|
3385
|
+
summaryScan.invalid.length
|
|
3386
|
+
);
|
|
3387
|
+
const apply = async (kind, file, counts, write) => {
|
|
3388
|
+
try {
|
|
3389
|
+
const { status } = await write();
|
|
3390
|
+
counts[status === "inserted" ? "imported" : status] += 1;
|
|
3391
|
+
return true;
|
|
3392
|
+
} catch (error) {
|
|
3393
|
+
counts.failed += 1;
|
|
3394
|
+
failures.push({ kind, id: file.id, path: file.path, error: describeError(error) });
|
|
3395
|
+
return false;
|
|
3396
|
+
}
|
|
3397
|
+
};
|
|
3398
|
+
const present = /* @__PURE__ */ new Set([
|
|
3399
|
+
...facts2.map((file) => file.id),
|
|
3400
|
+
...summaries.map((file) => file.id),
|
|
3401
|
+
...factScan.invalid.map((file) => file.id),
|
|
3402
|
+
...summaryScan.invalid.map((file) => file.id)
|
|
3403
|
+
]);
|
|
3404
|
+
const owned = await this.repo.listFileBackedIds(namespaceId);
|
|
3405
|
+
for (const id of owned.facts.filter((id2) => !present.has(id2))) {
|
|
3406
|
+
const removed = await this.repo.forgetFact(
|
|
3407
|
+
namespaceId,
|
|
3408
|
+
id,
|
|
3409
|
+
reindexAudit(namespaceId, "fact", id)
|
|
3410
|
+
);
|
|
3411
|
+
if (removed.found) factCounts.deleted += 1;
|
|
3412
|
+
}
|
|
3413
|
+
for (const id of owned.summaries.filter((id2) => !present.has(id2))) {
|
|
3414
|
+
const removed = await this.repo.forgetSummary(
|
|
3415
|
+
namespaceId,
|
|
3416
|
+
id,
|
|
3417
|
+
reindexAudit(namespaceId, "summary", id)
|
|
3418
|
+
);
|
|
3419
|
+
if (removed.found) summaryCounts.deleted += 1;
|
|
3420
|
+
}
|
|
3421
|
+
for (const file of facts2) {
|
|
3422
|
+
const imported = await apply(
|
|
3423
|
+
"fact",
|
|
3424
|
+
file,
|
|
3425
|
+
factCounts,
|
|
3426
|
+
() => this.importFactFile(namespaceId, file)
|
|
3427
|
+
);
|
|
3428
|
+
if (imported) await this.restoreShares(namespaceId, file);
|
|
3429
|
+
}
|
|
3430
|
+
for (const file of summaries) {
|
|
3431
|
+
await apply(
|
|
3432
|
+
"summary",
|
|
3433
|
+
file,
|
|
3434
|
+
summaryCounts,
|
|
3435
|
+
() => this.importSummaryFile(namespaceId, file)
|
|
3436
|
+
);
|
|
3437
|
+
}
|
|
3438
|
+
return {
|
|
3439
|
+
facts: factCounts,
|
|
3440
|
+
summaries: summaryCounts,
|
|
3441
|
+
failures: failures.slice(0, MAX_REPORTED_FAILURES)
|
|
3442
|
+
};
|
|
3443
|
+
});
|
|
3444
|
+
const space = this.provider.space;
|
|
3445
|
+
const canEmbed = space !== void 0 && await this.provider.embed({ input: "codebuddy" }).then(
|
|
3446
|
+
() => true,
|
|
3447
|
+
() => false
|
|
3448
|
+
);
|
|
3449
|
+
const reembedding = space && canEmbed ? await this.repo.requeueEmbeddingsOutsideSpace(namespaceId, embeddingSpaceKey(space)) : 0;
|
|
3450
|
+
this.worker.notify();
|
|
3451
|
+
return { ...result, reembedding };
|
|
3452
|
+
}
|
|
3453
|
+
async reindexFacts() {
|
|
3454
|
+
return (await this.reindexMemory()).facts;
|
|
3455
|
+
}
|
|
3456
|
+
requireNamespaceId() {
|
|
3457
|
+
if (!this.initialized || !this.namespaceId) {
|
|
3458
|
+
throw new Error("CodeBuddy.init() must complete before this method is called.");
|
|
3459
|
+
}
|
|
3460
|
+
return this.namespaceId;
|
|
3461
|
+
}
|
|
3462
|
+
async writeOne(input) {
|
|
3463
|
+
const embeddingModel = DEFAULT_EMBEDDING_MODELS[0]?.id ?? "unknown";
|
|
3464
|
+
const ownerId = generateEntityId(prefixForType(input.type));
|
|
3465
|
+
const claim = redactPersisted({
|
|
3466
|
+
subject: "__inline__",
|
|
3467
|
+
predicate: "asserts",
|
|
3468
|
+
object: input.content.slice(0, 200),
|
|
3469
|
+
content: input.content
|
|
3470
|
+
});
|
|
3471
|
+
const content = input.type === "fact" ? claim.content : input.content;
|
|
3472
|
+
const contentHash = input.idempotencyKey ?? (input.type === "fact" ? factContentHash(this.config.namespace, ownerId, content) : computeContentHash(this.config.namespace, input.sessionId, input.content));
|
|
3473
|
+
const tokenCount = estimateTokenCount(content);
|
|
3474
|
+
const embeddingId = generateEntityId("emb");
|
|
3475
|
+
const embedding = {
|
|
3476
|
+
id: embeddingId,
|
|
3477
|
+
namespaceId: input.namespaceId,
|
|
3478
|
+
ownerType: input.type,
|
|
3479
|
+
ownerId,
|
|
3480
|
+
embeddingModel
|
|
3481
|
+
};
|
|
3482
|
+
const audit = {
|
|
3483
|
+
id: generateEntityId("aud"),
|
|
3484
|
+
namespaceId: input.namespaceId,
|
|
3485
|
+
action: "remember",
|
|
3486
|
+
entityType: input.type,
|
|
3487
|
+
entityId: ownerId,
|
|
3488
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {},
|
|
3489
|
+
metadata: { sessionId: input.sessionId, type: input.type }
|
|
3490
|
+
};
|
|
3491
|
+
if (input.type === "interaction") {
|
|
3492
|
+
const interaction = {
|
|
3493
|
+
id: ownerId,
|
|
3494
|
+
namespaceId: input.namespaceId,
|
|
3495
|
+
sessionId: input.sessionId,
|
|
3496
|
+
content: input.content,
|
|
3497
|
+
contentHash,
|
|
3498
|
+
tokenCount,
|
|
3499
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {}
|
|
3500
|
+
};
|
|
3501
|
+
return this.repo.writeInteraction({ interaction, embedding, audit });
|
|
3502
|
+
}
|
|
3503
|
+
if (input.type === "fact") {
|
|
3504
|
+
const fact = {
|
|
3505
|
+
id: ownerId,
|
|
3506
|
+
namespaceId: input.namespaceId,
|
|
3507
|
+
content,
|
|
3508
|
+
contentHash,
|
|
3509
|
+
subject: claim.subject,
|
|
3510
|
+
predicate: claim.predicate,
|
|
3511
|
+
object: claim.object,
|
|
3512
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {}
|
|
3513
|
+
};
|
|
3514
|
+
const result2 = await this.repo.writeFact({ fact, embedding, audit });
|
|
3515
|
+
if (this.memoryFileStore) {
|
|
3516
|
+
if (result2.deduplicated) {
|
|
3517
|
+
try {
|
|
3518
|
+
await this.memoryFileStore.readFact(result2.id);
|
|
3519
|
+
return result2;
|
|
3520
|
+
} catch (error) {
|
|
3521
|
+
if (error.code !== "ENOENT") throw error;
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3524
|
+
await this.memoryFileStore.writeFact({
|
|
3525
|
+
id: result2.id,
|
|
3526
|
+
namespace: this.config.namespace,
|
|
3527
|
+
subject: fact.subject,
|
|
3528
|
+
predicate: fact.predicate,
|
|
3529
|
+
object: fact.object,
|
|
3530
|
+
confidence: 1,
|
|
3531
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3532
|
+
createdByAgent: input.agentId ?? null,
|
|
3533
|
+
sourceInteractionId: fact.sourceInteractionId ?? null,
|
|
3534
|
+
sourceDeleted: false,
|
|
3535
|
+
content: fact.content
|
|
3536
|
+
});
|
|
3537
|
+
}
|
|
3538
|
+
return result2;
|
|
3539
|
+
}
|
|
3540
|
+
const summary = {
|
|
3541
|
+
id: ownerId,
|
|
3542
|
+
namespaceId: input.namespaceId,
|
|
3543
|
+
sessionId: input.sessionId,
|
|
3544
|
+
content: input.content,
|
|
3545
|
+
tokenCount,
|
|
3546
|
+
...input.agentId !== void 0 ? { createdByAgent: input.agentId } : {}
|
|
3547
|
+
};
|
|
3548
|
+
const result = await this.repo.writeSummary({ summary, embedding, audit });
|
|
3549
|
+
if (this.memoryFileStore) {
|
|
3550
|
+
if (result.deduplicated) {
|
|
3551
|
+
try {
|
|
3552
|
+
await this.memoryFileStore.readSummary(result.id);
|
|
3553
|
+
return result;
|
|
3554
|
+
} catch (error) {
|
|
3555
|
+
if (error.code !== "ENOENT") throw error;
|
|
3556
|
+
}
|
|
3557
|
+
}
|
|
3558
|
+
await this.memoryFileStore.writeSummary({
|
|
3559
|
+
id: result.id,
|
|
3560
|
+
namespace: this.config.namespace,
|
|
3561
|
+
sessionId: result.sessionId,
|
|
3562
|
+
version: summary.version ?? 1,
|
|
3563
|
+
tokenCount: summary.tokenCount,
|
|
3564
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3565
|
+
createdByAgent: input.agentId ?? null,
|
|
3566
|
+
content: summary.content
|
|
3567
|
+
});
|
|
3568
|
+
}
|
|
3569
|
+
return result;
|
|
3570
|
+
}
|
|
3571
|
+
async restoreShares(namespaceId, file) {
|
|
3572
|
+
for (const to of file.sharedWith) {
|
|
3573
|
+
if (to === this.config.namespace) continue;
|
|
3574
|
+
const toNs = await this.repo.ensureNamespace(to);
|
|
3575
|
+
await this.repo.shareReference({
|
|
3576
|
+
id: generateEntityId("shr"),
|
|
3577
|
+
fromNamespaceId: namespaceId,
|
|
3578
|
+
toNamespaceId: toNs.id,
|
|
3579
|
+
sourceFactId: file.id,
|
|
3580
|
+
mode: "reference"
|
|
3581
|
+
});
|
|
3582
|
+
}
|
|
3583
|
+
}
|
|
3584
|
+
importFactFile(namespaceId, file) {
|
|
3585
|
+
return this.repo.upsertFactFromFile({
|
|
3586
|
+
fact: {
|
|
3587
|
+
id: file.id,
|
|
3588
|
+
namespaceId,
|
|
3589
|
+
content: file.content,
|
|
3590
|
+
contentHash: factContentHash(this.config.namespace, file.id, file.content),
|
|
3591
|
+
subject: file.subject,
|
|
3592
|
+
predicate: file.predicate,
|
|
3593
|
+
object: file.object,
|
|
3594
|
+
confidence: file.confidence,
|
|
3595
|
+
sourceInteractionId: file.sourceInteractionId ?? void 0,
|
|
3596
|
+
sourceDeleted: file.sourceDeleted,
|
|
3597
|
+
supersededBy: file.supersededBy,
|
|
3598
|
+
propositionKey: file.propositionKey ?? null,
|
|
3599
|
+
createdByAgent: file.createdByAgent ?? void 0,
|
|
3600
|
+
createdAt: new Date(file.createdAt)
|
|
3601
|
+
},
|
|
3602
|
+
embedding: pendingEmbedding(namespaceId, "fact", file.id),
|
|
3603
|
+
audit: {
|
|
3604
|
+
id: generateEntityId("aud"),
|
|
3605
|
+
namespaceId,
|
|
3606
|
+
action: "remember",
|
|
3607
|
+
entityType: "fact",
|
|
3608
|
+
entityId: file.id,
|
|
3609
|
+
metadata: { source: "reindex", path: file.path }
|
|
3610
|
+
}
|
|
3611
|
+
});
|
|
3612
|
+
}
|
|
3613
|
+
importSummaryFile(namespaceId, file) {
|
|
3614
|
+
return this.repo.upsertSummaryFromFile({
|
|
3615
|
+
summary: {
|
|
3616
|
+
id: file.id,
|
|
3617
|
+
namespaceId,
|
|
3618
|
+
sessionId: file.sessionId,
|
|
3619
|
+
content: file.content,
|
|
3620
|
+
version: file.version,
|
|
3621
|
+
tokenCount: file.tokenCount,
|
|
3622
|
+
createdByAgent: file.createdByAgent ?? void 0,
|
|
3623
|
+
createdAt: new Date(file.createdAt)
|
|
3624
|
+
},
|
|
3625
|
+
embedding: pendingEmbedding(namespaceId, "summary", file.id),
|
|
3626
|
+
audit: {
|
|
3627
|
+
id: generateEntityId("aud"),
|
|
3628
|
+
namespaceId,
|
|
3629
|
+
action: "remember",
|
|
3630
|
+
entityType: "summary",
|
|
3631
|
+
entityId: file.id,
|
|
3632
|
+
metadata: { source: "reindex", path: file.path }
|
|
3633
|
+
}
|
|
3634
|
+
});
|
|
3635
|
+
}
|
|
3636
|
+
};
|
|
3637
|
+
async function resolveTargetStore(root, namespace) {
|
|
3638
|
+
const resolved = (await readConfigFile(root)).namespace ?? defaultNamespaceForCwd(root);
|
|
3639
|
+
if (resolved !== namespace) {
|
|
3640
|
+
throw new Error(
|
|
3641
|
+
`share: targetProjectRoot ${root} resolves to namespace "${resolved}", not "${namespace}".`
|
|
3642
|
+
);
|
|
3643
|
+
}
|
|
3644
|
+
return new MemoryFileStore(root);
|
|
3645
|
+
}
|
|
3646
|
+
async function writeSnapshotFile(repo, store, namespace, factId) {
|
|
3647
|
+
try {
|
|
3648
|
+
await store.readFact(factId);
|
|
3649
|
+
return;
|
|
3650
|
+
} catch (error) {
|
|
3651
|
+
if (error.code !== "ENOENT") throw error;
|
|
3652
|
+
}
|
|
3653
|
+
const fact = await repo.getFactById(namespace.id, factId);
|
|
3654
|
+
if (!fact)
|
|
3655
|
+
throw new Error(`share: snapshot ${factId} missing from namespace "${namespace.name}".`);
|
|
3656
|
+
await store.writeFact({
|
|
3657
|
+
id: fact.id,
|
|
3658
|
+
namespace: namespace.name,
|
|
3659
|
+
subject: fact.subject,
|
|
3660
|
+
predicate: fact.predicate,
|
|
3661
|
+
object: fact.object,
|
|
3662
|
+
confidence: fact.confidence,
|
|
3663
|
+
createdAt: fact.createdAt.toISOString(),
|
|
3664
|
+
createdByAgent: fact.createdByAgent,
|
|
3665
|
+
sourceInteractionId: null,
|
|
3666
|
+
sourceDeleted: false,
|
|
3667
|
+
content: fact.content
|
|
3668
|
+
});
|
|
3669
|
+
}
|
|
3670
|
+
var REINDEX_LOCK_KEY = "__codebuddy_markdown_reindex__";
|
|
3671
|
+
var MAX_REPORTED_FAILURES = 50;
|
|
3672
|
+
function emptyCounts(scanned, failed) {
|
|
3673
|
+
return { scanned, imported: 0, updated: 0, unchanged: 0, deleted: 0, failed };
|
|
3674
|
+
}
|
|
3675
|
+
function pendingEmbedding(namespaceId, ownerType, ownerId) {
|
|
3676
|
+
return {
|
|
3677
|
+
id: generateEntityId("emb"),
|
|
3678
|
+
namespaceId,
|
|
3679
|
+
ownerType,
|
|
3680
|
+
ownerId,
|
|
3681
|
+
embeddingModel: DEFAULT_EMBEDDING_MODELS[0]?.id ?? "unknown"
|
|
3682
|
+
};
|
|
3683
|
+
}
|
|
3684
|
+
function reindexAudit(namespaceId, entityType, entityId) {
|
|
3685
|
+
return {
|
|
3686
|
+
id: generateEntityId("aud"),
|
|
3687
|
+
namespaceId,
|
|
3688
|
+
action: "forget",
|
|
3689
|
+
entityType,
|
|
3690
|
+
entityId,
|
|
3691
|
+
metadata: { source: "reindex", reason: "no eligible markdown file" }
|
|
3692
|
+
};
|
|
3693
|
+
}
|
|
3694
|
+
function describeError(error) {
|
|
3695
|
+
const cause = error.cause;
|
|
3696
|
+
const message = cause instanceof Error ? cause.message : error.message;
|
|
3697
|
+
return String(message).split("\n")[0]?.slice(0, 300) ?? "unknown error";
|
|
3698
|
+
}
|
|
3699
|
+
function factContentHash(namespace, id, content) {
|
|
3700
|
+
return computeFactHash(namespace, content === REDACTED ? `${REDACTED}${id}` : content);
|
|
3701
|
+
}
|
|
3702
|
+
function prefixForType(type) {
|
|
3703
|
+
if (type === "interaction") return "int";
|
|
3704
|
+
if (type === "fact") return "fact";
|
|
3705
|
+
return "sum";
|
|
3706
|
+
}
|
|
3707
|
+
function estimateTokenCount(content) {
|
|
3708
|
+
return countTokens(content);
|
|
3709
|
+
}
|
|
3710
|
+
function buildProviderFromConfig(provider) {
|
|
3711
|
+
if (provider.type === "local") return new LocalModelProvider();
|
|
3712
|
+
if (provider.type === "openrouter") {
|
|
3713
|
+
const model = provider.model ?? DEFAULT_CLOUD_EMBEDDING_MODELS.openrouter;
|
|
3714
|
+
const apiKey = process.env.OPENROUTER_API_KEY;
|
|
3715
|
+
if (!apiKey) return missingKeyProvider("OPENROUTER_API_KEY", model);
|
|
3716
|
+
return new OpenRouterEmbeddingProvider({ apiKey, model });
|
|
3717
|
+
}
|
|
3718
|
+
return new HuggingFaceProvider({
|
|
3719
|
+
...provider.apiKey ? { apiKey: provider.apiKey } : {},
|
|
3720
|
+
...provider.model ? {
|
|
3721
|
+
embeddingModels: [
|
|
3722
|
+
{
|
|
3723
|
+
id: provider.model,
|
|
3724
|
+
role: "primary",
|
|
3725
|
+
kind: "embedding",
|
|
3726
|
+
dimensions: EMBEDDING_DIMENSIONS
|
|
3727
|
+
}
|
|
3728
|
+
]
|
|
3729
|
+
} : {}
|
|
3730
|
+
});
|
|
3731
|
+
}
|
|
3732
|
+
function missingKeyProvider(variable, model) {
|
|
3733
|
+
const fail = async () => {
|
|
3734
|
+
throw new Error(
|
|
3735
|
+
`${variable} is not set; ${model} cannot embed. Export it or run \`codebuddy models use local\`.`
|
|
3736
|
+
);
|
|
3737
|
+
};
|
|
3738
|
+
return { embed: fail, generateText: fail };
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
// src/core/operations.ts
|
|
3742
|
+
async function createRuntime(config, options = {}) {
|
|
3743
|
+
const runtimeConfig = config ?? await loadRuntimeConfig();
|
|
3744
|
+
if (!runtimeConfig.postgresUrl) throw new Error(MISSING_DATABASE_URL);
|
|
3745
|
+
const client = createDatabaseClient({
|
|
3746
|
+
postgresUrl: runtimeConfig.postgresUrl,
|
|
3747
|
+
...options.connectTimeoutSeconds !== void 0 ? { connectTimeoutSeconds: options.connectTimeoutSeconds } : {}
|
|
3748
|
+
});
|
|
3749
|
+
try {
|
|
3750
|
+
if (options.autoBootstrap) {
|
|
3751
|
+
await bootstrapDatabase(client.sql);
|
|
3752
|
+
await runMigrations(client);
|
|
3753
|
+
}
|
|
3754
|
+
const repository = createPostgresRepository(client);
|
|
3755
|
+
const memory = new CodeBuddy(runtimeConfig, {
|
|
3756
|
+
repository,
|
|
3757
|
+
memoryFileStore: new MemoryFileStore(
|
|
3758
|
+
options.repositoryRoot ?? runtimeConfig.projectRoot ?? process.cwd()
|
|
3759
|
+
)
|
|
3760
|
+
});
|
|
3761
|
+
await memory.init(options.startWorker === false ? { startWorker: false } : {});
|
|
3762
|
+
return {
|
|
3763
|
+
memory,
|
|
3764
|
+
repository,
|
|
3765
|
+
async close() {
|
|
3766
|
+
await memory.shutdown();
|
|
3767
|
+
await client.close();
|
|
3768
|
+
}
|
|
3769
|
+
};
|
|
3770
|
+
} catch (error) {
|
|
3771
|
+
await client.close().catch(() => void 0);
|
|
3772
|
+
throw error;
|
|
3773
|
+
}
|
|
3774
|
+
}
|
|
3775
|
+
function encodeFactCursor(fact) {
|
|
3776
|
+
return Buffer.from(JSON.stringify([fact.createdAt.toISOString(), fact.id]), "utf8").toString(
|
|
3777
|
+
"base64url"
|
|
3778
|
+
);
|
|
3779
|
+
}
|
|
3780
|
+
function decodeFactCursor(cursor) {
|
|
3781
|
+
let parsed;
|
|
3782
|
+
try {
|
|
3783
|
+
parsed = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
|
3784
|
+
} catch {
|
|
3785
|
+
throw new Error("Invalid cursor: expected a base64url-encoded [createdAt, id] pair.");
|
|
3786
|
+
}
|
|
3787
|
+
if (!Array.isArray(parsed) || parsed.length !== 2 || typeof parsed[0] !== "string" || typeof parsed[1] !== "string" || Number.isNaN(Date.parse(parsed[0]))) {
|
|
3788
|
+
throw new Error("Invalid cursor: expected a base64url-encoded [createdAt, id] pair.");
|
|
3789
|
+
}
|
|
3790
|
+
const [createdAt2, id] = parsed;
|
|
3791
|
+
return { createdAt: new Date(createdAt2), id };
|
|
3792
|
+
}
|
|
3793
|
+
async function listFactsPage(memory, repository, input) {
|
|
3794
|
+
const namespace = await repository.ensureNamespace(memory.config.namespace);
|
|
3795
|
+
const requestedLimit = input.limit ?? 50;
|
|
3796
|
+
const limit = Math.max(1, Math.min(100, requestedLimit));
|
|
3797
|
+
const facts2 = await repository.listFacts({
|
|
3798
|
+
namespaceId: namespace.id,
|
|
3799
|
+
...input.subject ? { subject: input.subject } : {},
|
|
3800
|
+
limit: limit + 1,
|
|
3801
|
+
...input.cursor ? { cursor: decodeFactCursor(input.cursor) } : {}
|
|
3802
|
+
});
|
|
3803
|
+
const page = facts2.slice(0, limit);
|
|
3804
|
+
const next = facts2.length > limit ? page.at(-1) : void 0;
|
|
3805
|
+
return {
|
|
3806
|
+
facts: page,
|
|
3807
|
+
...next ? { nextCursor: encodeFactCursor(next) } : {}
|
|
3808
|
+
};
|
|
3809
|
+
}
|
|
3810
|
+
async function inspectNamespace(repository, namespace) {
|
|
3811
|
+
if (!namespace) {
|
|
3812
|
+
return {
|
|
3813
|
+
namespaces: await repository.listNamespaces(),
|
|
3814
|
+
agents: await repository.getAgentWriteBreakdown()
|
|
3815
|
+
};
|
|
3816
|
+
}
|
|
3817
|
+
const row = await repository.getNamespaceByName(namespace);
|
|
3818
|
+
if (!row) return { namespace, exists: false, facts: [], agents: [] };
|
|
3819
|
+
return {
|
|
3820
|
+
namespace,
|
|
3821
|
+
exists: true,
|
|
3822
|
+
facts: await repository.listFacts({ namespaceId: row.id, limit: 50 }),
|
|
3823
|
+
agents: await repository.getAgentWriteBreakdown(row.id)
|
|
3824
|
+
};
|
|
3825
|
+
}
|
|
3826
|
+
async function runDoctor(options = {}) {
|
|
3827
|
+
const fileConfig = await readConfigFile();
|
|
3828
|
+
const scaffold = await checkProjectScaffold();
|
|
3829
|
+
const configPermissions = await checkConfigPermissions();
|
|
3830
|
+
const expectedDefaultNamespace = defaultNamespaceForCwd();
|
|
3831
|
+
const runtimeNamespace = process.env.CODEBUDDY_NAMESPACE ?? fileConfig.namespace ?? options.config?.namespace ?? null;
|
|
3832
|
+
const namespaceReport = {
|
|
3833
|
+
expectedDefault: expectedDefaultNamespace,
|
|
3834
|
+
configured: fileConfig.namespace ?? null,
|
|
3835
|
+
runtime: runtimeNamespace,
|
|
3836
|
+
ok: !fileConfig.namespace || fileConfig.namespace === expectedDefaultNamespace || process.env.CODEBUDDY_NAMESPACE !== void 0,
|
|
3837
|
+
...!fileConfig.namespace || fileConfig.namespace === expectedDefaultNamespace || process.env.CODEBUDDY_NAMESPACE !== void 0 ? {} : {
|
|
3838
|
+
warning: `Configured namespace "${fileConfig.namespace}" differs from this folder's default "${expectedDefaultNamespace}". Confirm your MCP entries point at the intended namespace.`
|
|
3839
|
+
}
|
|
3840
|
+
};
|
|
3841
|
+
let config = options.config;
|
|
3842
|
+
if (!config) {
|
|
3843
|
+
try {
|
|
3844
|
+
config = await loadRuntimeConfig();
|
|
3845
|
+
} catch (error) {
|
|
3846
|
+
const message = String(redactSecrets(error.message));
|
|
3847
|
+
return {
|
|
3848
|
+
db: {
|
|
3849
|
+
ok: false,
|
|
3850
|
+
error: message,
|
|
3851
|
+
canConnect: false,
|
|
3852
|
+
postgresUrlSource: "missing"
|
|
3853
|
+
},
|
|
3854
|
+
pgvector: { ok: false, error: "Postgres is not configured yet." },
|
|
3855
|
+
vectors: { count: 0, index: null, needsTuning: false },
|
|
3856
|
+
models: await checkModels(options.provider, true),
|
|
3857
|
+
usage: {
|
|
3858
|
+
last60s: 0,
|
|
3859
|
+
lastHour: 0,
|
|
3860
|
+
last24h: 0,
|
|
3861
|
+
estimatedDailyCapRemaining: 1e4,
|
|
3862
|
+
warning80Percent: false
|
|
3863
|
+
},
|
|
3864
|
+
scaffold,
|
|
3865
|
+
config: configPermissions,
|
|
3866
|
+
namespace: namespaceReport
|
|
3867
|
+
};
|
|
3868
|
+
}
|
|
3869
|
+
}
|
|
3870
|
+
let repository = options.repository;
|
|
3871
|
+
let close;
|
|
3872
|
+
let db = { ok: true };
|
|
3873
|
+
let pgvector = { ok: true };
|
|
3874
|
+
if (!repository) {
|
|
3875
|
+
const postgresUrlSource = options.config?.postgresUrl !== void 0 ? "explicit" : process.env.DATABASE_URL ? "environment" : fileConfig.postgresUrl ? "config" : "missing";
|
|
3876
|
+
const client = createDatabaseClient({ postgresUrl: config.postgresUrl });
|
|
3877
|
+
repository = createPostgresRepository(client);
|
|
3878
|
+
close = () => client.close();
|
|
3879
|
+
try {
|
|
3880
|
+
const canConnect = await pingDatabase(client);
|
|
3881
|
+
db = { ok: canConnect, canConnect, postgresUrlSource };
|
|
3882
|
+
if (!canConnect) db.error = "Postgres did not respond to a simple ping.";
|
|
3883
|
+
} catch (error) {
|
|
3884
|
+
db = {
|
|
3885
|
+
ok: false,
|
|
3886
|
+
canConnect: false,
|
|
3887
|
+
postgresUrlSource,
|
|
3888
|
+
error: String(redactSecrets(error.message))
|
|
3889
|
+
};
|
|
3890
|
+
}
|
|
3891
|
+
try {
|
|
3892
|
+
const health = await checkPgvector(client.sql);
|
|
3893
|
+
pgvector = {
|
|
3894
|
+
ok: health.available,
|
|
3895
|
+
...health.available ? {} : { error: "pgvector extension is not installed." }
|
|
3896
|
+
};
|
|
3897
|
+
} catch (error) {
|
|
3898
|
+
pgvector = { ok: false, error: String(redactSecrets(error.message)) };
|
|
3899
|
+
}
|
|
3900
|
+
}
|
|
3901
|
+
try {
|
|
3902
|
+
if (!db.ok) {
|
|
3903
|
+
return {
|
|
3904
|
+
db,
|
|
3905
|
+
pgvector,
|
|
3906
|
+
vectors: { count: 0, index: null, needsTuning: false },
|
|
3907
|
+
models: await checkModels(options.provider, true),
|
|
3908
|
+
usage: {
|
|
3909
|
+
last60s: 0,
|
|
3910
|
+
lastHour: 0,
|
|
3911
|
+
last24h: 0,
|
|
3912
|
+
estimatedDailyCapRemaining: 1e4,
|
|
3913
|
+
warning80Percent: false
|
|
3914
|
+
},
|
|
3915
|
+
scaffold,
|
|
3916
|
+
config: configPermissions,
|
|
3917
|
+
namespace: namespaceReport
|
|
3918
|
+
};
|
|
3919
|
+
}
|
|
3920
|
+
const namespace = await repository.ensureNamespace(config.namespace);
|
|
3921
|
+
const [stats, vectorHealth] = await Promise.all([
|
|
3922
|
+
repository.getStats(namespace.id),
|
|
3923
|
+
repository.getVectorIndexHealth(namespace.id)
|
|
3924
|
+
]);
|
|
3925
|
+
const models = await checkModels(options.provider, options.skipModelCheck);
|
|
3926
|
+
const dailyCap = 1e4;
|
|
3927
|
+
const used = stats.modelCalls.last24h;
|
|
3928
|
+
return {
|
|
3929
|
+
db,
|
|
3930
|
+
pgvector,
|
|
3931
|
+
vectors: {
|
|
3932
|
+
count: vectorHealth.vectorCount,
|
|
3933
|
+
index: vectorHealth.indexName,
|
|
3934
|
+
needsTuning: vectorHealth.needsTuning
|
|
3935
|
+
},
|
|
3936
|
+
models,
|
|
3937
|
+
usage: {
|
|
3938
|
+
last60s: stats.modelCalls.last60s,
|
|
3939
|
+
lastHour: stats.modelCalls.lastHour,
|
|
3940
|
+
last24h: used,
|
|
3941
|
+
estimatedDailyCapRemaining: Math.max(0, dailyCap - used),
|
|
3942
|
+
warning80Percent: used >= dailyCap * 0.8
|
|
3943
|
+
},
|
|
3944
|
+
scaffold,
|
|
3945
|
+
config: configPermissions,
|
|
3946
|
+
namespace: namespaceReport
|
|
3947
|
+
};
|
|
3948
|
+
} finally {
|
|
3949
|
+
await close?.();
|
|
3950
|
+
}
|
|
3951
|
+
}
|
|
3952
|
+
async function checkModels(provider, skip = false) {
|
|
3953
|
+
const ids = [...DEFAULT_EMBEDDING_MODELS, ...DEFAULT_LLM_MODELS].map((model) => model.id);
|
|
3954
|
+
if (!provider || skip) {
|
|
3955
|
+
return ids.map((id) => ({ id, reachable: false, status: "unknown" }));
|
|
3956
|
+
}
|
|
3957
|
+
const results = [];
|
|
3958
|
+
for (const id of ids) {
|
|
3959
|
+
try {
|
|
3960
|
+
if (DEFAULT_EMBEDDING_MODELS.some((model) => model.id === id)) {
|
|
3961
|
+
await provider.embed({ model: id, input: "doctor" });
|
|
3962
|
+
} else {
|
|
3963
|
+
await provider.generateText({ model: id, prompt: "doctor" });
|
|
3964
|
+
}
|
|
3965
|
+
results.push({ id, reachable: true, status: "warm" });
|
|
3966
|
+
} catch (error) {
|
|
3967
|
+
const message = String(redactSecrets(error.message));
|
|
3968
|
+
results.push({
|
|
3969
|
+
id,
|
|
3970
|
+
reachable: false,
|
|
3971
|
+
status: /gated|403/i.test(message) ? "gated" : /cold|503/i.test(message) ? "cold" : "failed",
|
|
3972
|
+
error: message
|
|
3973
|
+
});
|
|
3974
|
+
}
|
|
3975
|
+
}
|
|
3976
|
+
return results;
|
|
3977
|
+
}
|
|
3978
|
+
|
|
3979
|
+
export {
|
|
3980
|
+
isEligibleMemory,
|
|
3981
|
+
lexicalTerms,
|
|
3982
|
+
embeddingSpaceKey,
|
|
3983
|
+
DefaultPolicy,
|
|
3984
|
+
HuggingFaceProvider,
|
|
3985
|
+
isLangSmithTracingEnabled,
|
|
3986
|
+
createTracer,
|
|
3987
|
+
codeBuddyConfigSchema,
|
|
3988
|
+
FACT_CATEGORIES,
|
|
3989
|
+
INCIDENT_SEVERITIES,
|
|
3990
|
+
MemoryFileStore,
|
|
3991
|
+
EmbeddingWorker,
|
|
3992
|
+
CodeBuddy,
|
|
3993
|
+
buildProviderFromConfig,
|
|
3994
|
+
bootstrapDatabase,
|
|
3995
|
+
createDatabaseClient,
|
|
3996
|
+
pingDatabase,
|
|
3997
|
+
runMigrations,
|
|
3998
|
+
createPostgresRepository,
|
|
3999
|
+
createRuntime,
|
|
4000
|
+
encodeFactCursor,
|
|
4001
|
+
decodeFactCursor,
|
|
4002
|
+
listFactsPage,
|
|
4003
|
+
inspectNamespace,
|
|
4004
|
+
runDoctor
|
|
4005
|
+
};
|
|
4006
|
+
//# sourceMappingURL=chunk-JDZ3VK26.js.map
|