bikky 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/config.d.ts +11 -1
- package/dist/config.js +88 -20
- package/dist/daemon/capture-policy.d.ts +0 -1
- package/dist/daemon/capture-policy.js +0 -1
- package/dist/daemon/consolidation.d.ts +2 -1
- package/dist/daemon/consolidation.js +28 -11
- package/dist/daemon/entity-typing.js +10 -0
- package/dist/daemon/episode-summary.d.ts +4 -0
- package/dist/daemon/episode-summary.js +39 -8
- package/dist/daemon/extraction.d.ts +1 -1
- package/dist/daemon/extraction.js +52 -17
- package/dist/daemon/qdrant.d.ts +32 -10
- package/dist/daemon/qdrant.js +177 -60
- package/dist/daemon/relations.d.ts +3 -3
- package/dist/daemon/relations.js +27 -15
- package/dist/daemon/session-index.d.ts +5 -0
- package/dist/daemon/session-index.js +36 -9
- package/dist/daemon/session-summary.d.ts +3 -0
- package/dist/daemon/session-summary.js +48 -15
- package/dist/daemon/staleness.js +2 -2
- package/dist/daemon/transcript-sources.js +3 -2
- package/dist/daemon/watcher.js +2 -0
- package/dist/daemon/workstream-summary.d.ts +4 -0
- package/dist/daemon/workstream-summary.js +58 -16
- package/dist/install.d.ts +11 -0
- package/dist/install.js +38 -0
- package/dist/llm/embedding/index.js +2 -1
- package/dist/llm/embedding/providers/openai.js +8 -2
- package/dist/llm/embedding/providers/portkey.js +9 -2
- package/dist/llm/inference/index.js +2 -1
- package/dist/llm/util.d.ts +12 -0
- package/dist/llm/util.js +18 -0
- package/dist/mcp/helpers.d.ts +5 -0
- package/dist/mcp/helpers.js +27 -3
- package/dist/mcp/taxonomy.js +12 -1
- package/dist/mcp/tools.js +161 -57
- package/dist/mcp/types.d.ts +12 -0
- package/dist/package-verifier.d.ts +19 -0
- package/dist/package-verifier.js +83 -0
- package/dist/provenance/origin.d.ts +57 -0
- package/dist/provenance/origin.js +254 -0
- package/docs/config/fully-hosted.md +33 -13
- package/docs/config/hosted-models.md +33 -13
- package/docs/configuration.md +18 -0
- package/package.json +2 -2
package/dist/daemon/qdrant.js
CHANGED
|
@@ -8,35 +8,84 @@
|
|
|
8
8
|
* self-hosted instances.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash, randomUUID } from "node:crypto";
|
|
11
|
-
import { loadConfig } from "../config.js";
|
|
11
|
+
import { getEffectiveDestinations, loadConfig } from "../config.js";
|
|
12
12
|
import { embed, initEmbedding, getEmbeddingConfig } from "../llm/index.js";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
13
|
+
import { QdrantPool } from "../lib/qdrant-pool.js";
|
|
14
|
+
import { buildResolver } from "../routing.js";
|
|
15
|
+
import { DEFAULT_DOMAIN, QDRANT_INDEXES, categoryForMemorySubtype, layerForMemorySubtype, normalizeCategory, normalizeDomain, normalizeKind, validateMemorySubtype, } from "../mcp/taxonomy.js";
|
|
15
16
|
import { combineRedactions, redactStorageText, } from "../privacy/redaction.js";
|
|
17
|
+
import { buildOperationOrigin } from "../provenance/origin.js";
|
|
16
18
|
// ---------------------------------------------------------------------------
|
|
17
19
|
// State
|
|
18
20
|
// ---------------------------------------------------------------------------
|
|
19
|
-
let qdrantUrl = null;
|
|
20
|
-
let qdrantApiKey = null;
|
|
21
21
|
let collection = "bikky";
|
|
22
22
|
let logFn = () => { };
|
|
23
|
-
let
|
|
23
|
+
let destinations = [];
|
|
24
|
+
let pool = null;
|
|
25
|
+
let resolver = null;
|
|
24
26
|
const setLogger = (fn) => { logFn = fn; };
|
|
25
27
|
const setEmbeddingConfig = (overrides) => {
|
|
26
28
|
if (overrides && overrides.provider)
|
|
27
29
|
initEmbedding(overrides);
|
|
28
30
|
};
|
|
29
31
|
const clientLogAdapter = (level, msg) => logFn(level, msg);
|
|
32
|
+
const fallbackDestination = () => {
|
|
33
|
+
if (destinations.length === 0) {
|
|
34
|
+
throw new Error("Qdrant client not initialized — call init() first");
|
|
35
|
+
}
|
|
36
|
+
return destinations.find((destination) => destination.default === true) ?? destinations[0];
|
|
37
|
+
};
|
|
38
|
+
const resolveDestination = (input = {}) => {
|
|
39
|
+
if (!resolver)
|
|
40
|
+
return fallbackDestination();
|
|
41
|
+
return resolver(input);
|
|
42
|
+
};
|
|
43
|
+
const destinationFromRef = (ref) => {
|
|
44
|
+
if (!ref)
|
|
45
|
+
return fallbackDestination();
|
|
46
|
+
if (typeof ref !== "string")
|
|
47
|
+
return ref;
|
|
48
|
+
const found = destinations.find((destination) => destination.name === ref);
|
|
49
|
+
if (!found) {
|
|
50
|
+
throw new Error(`Unknown Qdrant destination '${ref}'. Configured destinations: ${destinations.map((d) => d.name).join(", ") || "(none)"}`);
|
|
51
|
+
}
|
|
52
|
+
return found;
|
|
53
|
+
};
|
|
54
|
+
const pathForDestination = (urlPath, destination) => {
|
|
55
|
+
if (!urlPath.startsWith("/collections/"))
|
|
56
|
+
return urlPath;
|
|
57
|
+
return urlPath.replace(/^\/collections\/[^/]+/, `/collections/${destination.collection}`);
|
|
58
|
+
};
|
|
59
|
+
const routingInputForFact = (fact, normalizedContent, normalizedEntities, extraMetadata = {}) => ({
|
|
60
|
+
content: normalizedContent,
|
|
61
|
+
entities: normalizedEntities,
|
|
62
|
+
metadata: {
|
|
63
|
+
...(fact.metadata ?? {}),
|
|
64
|
+
...extraMetadata,
|
|
65
|
+
category: fact.category,
|
|
66
|
+
domain: fact.domain ?? DEFAULT_DOMAIN,
|
|
67
|
+
kind: fact.kind ?? "fact",
|
|
68
|
+
...(fact.memory_subtype ? { memory_subtype: fact.memory_subtype } : {}),
|
|
69
|
+
...(fact.source ? { source: fact.source } : {}),
|
|
70
|
+
...(fact.actor_id ? { actor_id: fact.actor_id } : {}),
|
|
71
|
+
...(fact.session_id ? { session_id: fact.session_id } : {}),
|
|
72
|
+
...(fact.episode_id ? { episode_id: fact.episode_id } : {}),
|
|
73
|
+
...(fact.workstream_key ? { workstream_key: fact.workstream_key } : {}),
|
|
74
|
+
...(fact.task_key ? { task_key: fact.task_key } : {}),
|
|
75
|
+
...(fact.repo ? { repo: fact.repo } : {}),
|
|
76
|
+
...(fact.branch ? { branch: fact.branch } : {}),
|
|
77
|
+
...(fact.surface ? { surface: fact.surface } : {}),
|
|
78
|
+
},
|
|
79
|
+
});
|
|
30
80
|
// ---------------------------------------------------------------------------
|
|
31
81
|
// Init — reads credentials from loadConfig()
|
|
32
82
|
// ---------------------------------------------------------------------------
|
|
33
83
|
const init = () => {
|
|
34
84
|
const cfg = loadConfig();
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
qdrantUrl = qdrantUrl.replace(/\/+$/, "");
|
|
85
|
+
destinations = getEffectiveDestinations(cfg);
|
|
86
|
+
collection = (destinations.find((destination) => destination.default === true) ?? destinations[0])?.collection
|
|
87
|
+
?? cfg.collection
|
|
88
|
+
?? "bikky";
|
|
40
89
|
// Initialize embedding provider from config
|
|
41
90
|
const embCfg = initEmbedding({
|
|
42
91
|
provider: cfg.embedding.provider,
|
|
@@ -50,48 +99,72 @@ const init = () => {
|
|
|
50
99
|
retryBaseDelayMs: cfg.embedding.retry_base_delay_ms,
|
|
51
100
|
});
|
|
52
101
|
logFn("INFO", `Embedding provider: ${embCfg.provider}/${embCfg.model} (${embCfg.dimensions}d) @ ${embCfg.baseUrl}`);
|
|
53
|
-
const ready =
|
|
102
|
+
const ready = destinations.length > 0;
|
|
54
103
|
if (ready) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
apiKey: qdrantApiKey,
|
|
58
|
-
collection,
|
|
59
|
-
timeoutMs: cfg.qdrant_client.timeout_ms,
|
|
60
|
-
retries: cfg.qdrant_client.retries,
|
|
61
|
-
retryBaseDelayMs: cfg.qdrant_client.retry_base_delay_ms,
|
|
104
|
+
pool = new QdrantPool(destinations, {
|
|
105
|
+
client: cfg.qdrant_client,
|
|
62
106
|
log: clientLogAdapter,
|
|
63
107
|
});
|
|
108
|
+
resolver = buildResolver(destinations);
|
|
109
|
+
logFn("INFO", `Qdrant destinations: ${destinations.map((destination) => `${destination.name}/${destination.collection}`).join(", ")}`);
|
|
64
110
|
}
|
|
65
111
|
else {
|
|
66
|
-
|
|
112
|
+
pool = null;
|
|
113
|
+
resolver = null;
|
|
67
114
|
logFn("WARN", "Qdrant client: missing URL (some memory features disabled)");
|
|
68
115
|
}
|
|
69
116
|
return ready;
|
|
70
117
|
};
|
|
71
|
-
const isReady = () => !!(
|
|
118
|
+
const isReady = () => !!(pool && destinations.length > 0);
|
|
72
119
|
const ensureCollection = async () => {
|
|
73
|
-
if (!
|
|
120
|
+
if (!pool) {
|
|
74
121
|
throw new Error("Qdrant client not initialized — call init() first");
|
|
75
122
|
}
|
|
76
123
|
const embCfg = getEmbeddingConfig();
|
|
77
|
-
await
|
|
78
|
-
|
|
124
|
+
const results = await Promise.all(destinations.map(async (destination) => {
|
|
125
|
+
try {
|
|
126
|
+
await pool.ensureCollection(destination.name, embCfg.dimensions, QDRANT_INDEXES);
|
|
127
|
+
logFn("INFO", `Qdrant destination '${destination.name}' collection '${destination.collection}' ready (${QDRANT_INDEXES.length} indexes)`);
|
|
128
|
+
return { destination, ok: true, error: null };
|
|
129
|
+
}
|
|
130
|
+
catch (e) {
|
|
131
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
132
|
+
logFn("WARN", `Qdrant destination '${destination.name}' readiness check failed: ${message}`);
|
|
133
|
+
return { destination, ok: false, error: message };
|
|
134
|
+
}
|
|
135
|
+
}));
|
|
136
|
+
if (!results.some((result) => result.ok)) {
|
|
137
|
+
throw new Error(`No Qdrant destinations ready: ${results.map((result) => `${result.destination.name}: ${result.error}`).join("; ")}`);
|
|
138
|
+
}
|
|
79
139
|
};
|
|
80
140
|
// ---------------------------------------------------------------------------
|
|
81
141
|
// HTTP requests
|
|
82
142
|
// ---------------------------------------------------------------------------
|
|
83
|
-
const qdrantRequest = async (method, urlPath, body) => {
|
|
84
|
-
if (!
|
|
143
|
+
const qdrantRequest = async (method, urlPath, body, destinationRef) => {
|
|
144
|
+
if (!pool) {
|
|
85
145
|
throw new Error(`Qdrant client not initialized — call init() first (${method} ${urlPath})`);
|
|
86
146
|
}
|
|
87
|
-
const
|
|
147
|
+
const destination = destinationFromRef(destinationRef);
|
|
148
|
+
const result = await pool.client(destination.name).request(method, pathForDestination(urlPath, destination), body);
|
|
88
149
|
// Some Qdrant endpoints return empty bodies on success — preserve old return shape.
|
|
89
150
|
return result ?? {};
|
|
90
151
|
};
|
|
152
|
+
const collectionForDestination = (destinationRef) => destinationFromRef(destinationRef).collection;
|
|
153
|
+
const destinationNames = () => destinations.map((destination) => destination.name);
|
|
154
|
+
const readyDestinations = () => {
|
|
155
|
+
if (!pool)
|
|
156
|
+
return [];
|
|
157
|
+
return destinations.filter((destination) => pool.isCollectionReady(destination.name));
|
|
158
|
+
};
|
|
159
|
+
const activeDestinations = () => {
|
|
160
|
+
const ready = readyDestinations();
|
|
161
|
+
return ready.length > 0 ? ready : destinations;
|
|
162
|
+
};
|
|
91
163
|
// ---------------------------------------------------------------------------
|
|
92
164
|
// Read methods
|
|
93
165
|
// ---------------------------------------------------------------------------
|
|
94
|
-
const searchFacts = async (query, filters = {}, limit = 10) => {
|
|
166
|
+
const searchFacts = async (query, filters = {}, limit = 10, destinationRef) => {
|
|
167
|
+
const destination = destinationFromRef(destinationRef);
|
|
95
168
|
const vector = await embed(query);
|
|
96
169
|
const must = [
|
|
97
170
|
{ is_null: { key: "superseded_by" } },
|
|
@@ -114,14 +187,15 @@ const searchFacts = async (query, filters = {}, limit = 10) => {
|
|
|
114
187
|
if (filters.workspaceId) {
|
|
115
188
|
must.push({ key: "workspace_id", match: { value: filters.workspaceId } });
|
|
116
189
|
}
|
|
117
|
-
const result = await qdrantRequest("POST", `/collections/${collection}/points/search`, {
|
|
190
|
+
const result = await qdrantRequest("POST", `/collections/${destination.collection}/points/search`, {
|
|
118
191
|
vector,
|
|
119
192
|
filter: { must },
|
|
120
193
|
limit,
|
|
121
194
|
with_payload: true,
|
|
122
|
-
});
|
|
195
|
+
}, destination);
|
|
123
196
|
return (result.result || []).map((hit) => ({
|
|
124
197
|
id: hit.id,
|
|
198
|
+
destination: destination.name,
|
|
125
199
|
score: hit.score,
|
|
126
200
|
content: hit.payload?.content ?? "",
|
|
127
201
|
category: hit.payload?.category ?? "",
|
|
@@ -131,7 +205,8 @@ const searchFacts = async (query, filters = {}, limit = 10) => {
|
|
|
131
205
|
created_at: hit.payload?.created_at ?? "",
|
|
132
206
|
}));
|
|
133
207
|
};
|
|
134
|
-
const scrollFacts = async (filters = {}, limit = 10) => {
|
|
208
|
+
const scrollFacts = async (filters = {}, limit = 10, destinationRef) => {
|
|
209
|
+
const destination = destinationFromRef(destinationRef);
|
|
135
210
|
const must = [
|
|
136
211
|
{ is_null: { key: "superseded_by" } },
|
|
137
212
|
];
|
|
@@ -179,14 +254,15 @@ const scrollFacts = async (filters = {}, limit = 10) => {
|
|
|
179
254
|
if (filters.workspaceId) {
|
|
180
255
|
must.push({ key: "workspace_id", match: { value: filters.workspaceId } });
|
|
181
256
|
}
|
|
182
|
-
const result = await qdrantRequest("POST", `/collections/${collection}/points/scroll`, {
|
|
257
|
+
const result = await qdrantRequest("POST", `/collections/${destination.collection}/points/scroll`, {
|
|
183
258
|
filter: { must, must_not },
|
|
184
259
|
limit,
|
|
185
260
|
...(filters.orderBy ? { order_by: { key: filters.orderBy.key, direction: filters.orderBy.direction } } : {}),
|
|
186
261
|
with_payload: true,
|
|
187
|
-
});
|
|
262
|
+
}, destination);
|
|
188
263
|
return (result.result?.points || []).map((pt) => ({
|
|
189
264
|
id: pt.id,
|
|
265
|
+
destination: destination.name,
|
|
190
266
|
content: pt.payload?.content ?? "",
|
|
191
267
|
category: pt.payload?.category ?? "",
|
|
192
268
|
entities: pt.payload?.entities || [],
|
|
@@ -203,10 +279,17 @@ const scrollFacts = async (filters = {}, limit = 10) => {
|
|
|
203
279
|
source_fact_ids: pt.payload?.source_fact_ids ?? [],
|
|
204
280
|
}));
|
|
205
281
|
};
|
|
282
|
+
const scrollFactsAcrossDestinations = async (filters = {}, limit = 10) => {
|
|
283
|
+
const results = await Promise.all(activeDestinations().map((destination) => scrollFacts(filters, limit, destination).catch((e) => {
|
|
284
|
+
logFn("WARN", `Qdrant scroll failed for destination '${destination.name}': ${e.message}`);
|
|
285
|
+
return [];
|
|
286
|
+
})));
|
|
287
|
+
return results.flat();
|
|
288
|
+
};
|
|
206
289
|
// ---------------------------------------------------------------------------
|
|
207
290
|
// Write methods
|
|
208
291
|
// ---------------------------------------------------------------------------
|
|
209
|
-
const storeFact = async (fact) => {
|
|
292
|
+
const storeFact = async (fact, routeInput) => {
|
|
210
293
|
const normalizedKind = normalizeKind(fact.kind);
|
|
211
294
|
const normalizedSubtype = validateMemorySubtype(normalizedKind, fact.memory_subtype);
|
|
212
295
|
const normalizedCategory = normalizedSubtype
|
|
@@ -226,9 +309,18 @@ const storeFact = async (fact) => {
|
|
|
226
309
|
...redactedEntities,
|
|
227
310
|
...(redactedRelation ? [redactedRelation.from, redactedRelation.type, redactedRelation.to] : []),
|
|
228
311
|
]);
|
|
229
|
-
const vector = await embed(redactedContent.text);
|
|
230
312
|
const now = new Date().toISOString();
|
|
231
313
|
const id = randomUUID();
|
|
314
|
+
const origin = fact.origin ?? buildOperationOrigin({
|
|
315
|
+
interface: "daemon",
|
|
316
|
+
action: "create",
|
|
317
|
+
subsystem: "qdrant.store_fact",
|
|
318
|
+
metadata: {
|
|
319
|
+
category: normalizedCategory,
|
|
320
|
+
kind: normalizedKind,
|
|
321
|
+
...(normalizedSubtype ? { memory_subtype: normalizedSubtype } : {}),
|
|
322
|
+
},
|
|
323
|
+
});
|
|
232
324
|
const payload = {
|
|
233
325
|
content: redactedContent.text,
|
|
234
326
|
category: normalizedCategory,
|
|
@@ -236,9 +328,9 @@ const storeFact = async (fact) => {
|
|
|
236
328
|
kind: normalizedKind,
|
|
237
329
|
...(normalizedLayer ? { layer: normalizedLayer } : {}),
|
|
238
330
|
...(normalizedSubtype ? { memory_subtype: normalizedSubtype } : {}),
|
|
239
|
-
|
|
331
|
+
origin,
|
|
332
|
+
...(fact.last_operation_origin ? { last_operation_origin: fact.last_operation_origin } : {}),
|
|
240
333
|
entities: redactedEntities.map((entity) => entity.text.toLowerCase()),
|
|
241
|
-
source: normalizeSource(fact.source ?? "system"),
|
|
242
334
|
confidence: fact.confidence ?? 0.7,
|
|
243
335
|
importance: fact.importance ?? 0.5,
|
|
244
336
|
content_hash: redactedContent.redacted
|
|
@@ -281,37 +373,58 @@ const storeFact = async (fact) => {
|
|
|
281
373
|
if (redaction.redacted) {
|
|
282
374
|
payload.redaction = redaction;
|
|
283
375
|
}
|
|
284
|
-
|
|
376
|
+
const destination = resolveDestination(routeInput ?? routingInputForFact(fact, redactedContent.text, payload.entities, {
|
|
377
|
+
category: normalizedCategory,
|
|
378
|
+
domain: normalizedDomain,
|
|
379
|
+
kind: normalizedKind,
|
|
380
|
+
...(normalizedSubtype ? { memory_subtype: normalizedSubtype } : {}),
|
|
381
|
+
}));
|
|
382
|
+
const vector = await embed(redactedContent.text);
|
|
383
|
+
await qdrantRequest("PUT", `/collections/${destination.collection}/points`, {
|
|
285
384
|
points: [{ id, vector, payload }],
|
|
286
|
-
});
|
|
287
|
-
logFn("DEBUG", `Qdrant: stored fact ${id} [${normalizedCategory}] ${redactedContent.text.slice(0, 60)}`);
|
|
385
|
+
}, destination);
|
|
386
|
+
logFn("DEBUG", `Qdrant: stored fact ${id} in '${destination.name}' [${normalizedCategory}] ${redactedContent.text.slice(0, 60)}`);
|
|
288
387
|
return id;
|
|
289
388
|
};
|
|
290
|
-
const supersedeFact = async (oldFactId, newFactId) => {
|
|
389
|
+
const supersedeFact = async (oldFactId, newFactId, destinationRef, origin) => {
|
|
390
|
+
const destination = destinationFromRef(destinationRef);
|
|
291
391
|
const now = new Date().toISOString();
|
|
292
|
-
await qdrantRequest("POST", `/collections/${collection}/points/payload`, {
|
|
392
|
+
await qdrantRequest("POST", `/collections/${destination.collection}/points/payload`, {
|
|
293
393
|
payload: {
|
|
294
394
|
superseded_by: newFactId,
|
|
295
395
|
superseded_at: now,
|
|
296
396
|
updated_at: now,
|
|
397
|
+
last_operation_origin: origin ?? buildOperationOrigin({
|
|
398
|
+
interface: "daemon",
|
|
399
|
+
action: "supersede",
|
|
400
|
+
subsystem: "qdrant.supersede_fact",
|
|
401
|
+
metadata: { new_fact_id: newFactId },
|
|
402
|
+
}),
|
|
297
403
|
},
|
|
298
404
|
points: [oldFactId],
|
|
299
|
-
});
|
|
300
|
-
logFn("DEBUG", `Qdrant: superseded fact ${oldFactId} → ${newFactId}`);
|
|
405
|
+
}, destination);
|
|
406
|
+
logFn("DEBUG", `Qdrant: superseded fact ${oldFactId} → ${newFactId} in '${destination.name}'`);
|
|
301
407
|
};
|
|
302
|
-
const reinforceFact = async (factId, currentCount) => {
|
|
408
|
+
const reinforceFact = async (factId, currentCount, destinationRef, origin) => {
|
|
409
|
+
const destination = destinationFromRef(destinationRef);
|
|
303
410
|
const now = new Date().toISOString();
|
|
304
|
-
await qdrantRequest("POST", `/collections/${collection}/points/payload`, {
|
|
411
|
+
await qdrantRequest("POST", `/collections/${destination.collection}/points/payload`, {
|
|
305
412
|
payload: {
|
|
306
413
|
reinforcement_count: (currentCount || 1) + 1,
|
|
307
414
|
last_reinforced_at: now,
|
|
308
415
|
updated_at: now,
|
|
416
|
+
last_operation_origin: origin ?? buildOperationOrigin({
|
|
417
|
+
interface: "daemon",
|
|
418
|
+
action: "reinforce",
|
|
419
|
+
subsystem: "qdrant.reinforce_fact",
|
|
420
|
+
}),
|
|
309
421
|
},
|
|
310
422
|
points: [factId],
|
|
311
|
-
});
|
|
312
|
-
logFn("DEBUG", `Qdrant: reinforced fact ${factId}`);
|
|
423
|
+
}, destination);
|
|
424
|
+
logFn("DEBUG", `Qdrant: reinforced fact ${factId} in '${destination.name}'`);
|
|
313
425
|
};
|
|
314
|
-
const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supersedeThreshold = 0.80 } = {}, workspaceId) => {
|
|
426
|
+
const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supersedeThreshold = 0.80 } = {}, workspaceId, routeInput) => {
|
|
427
|
+
const destination = resolveDestination(routeInput ?? { content });
|
|
315
428
|
// First: hash-based exact check (fast, no embedding)
|
|
316
429
|
try {
|
|
317
430
|
const must = [
|
|
@@ -320,15 +433,16 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
|
|
|
320
433
|
];
|
|
321
434
|
if (workspaceId)
|
|
322
435
|
must.push({ key: "workspace_id", match: { value: workspaceId } });
|
|
323
|
-
const hashResult = await qdrantRequest("POST", `/collections/${collection}/points/scroll`, {
|
|
436
|
+
const hashResult = await qdrantRequest("POST", `/collections/${destination.collection}/points/scroll`, {
|
|
324
437
|
filter: { must },
|
|
325
438
|
limit: 1,
|
|
326
439
|
with_payload: true,
|
|
327
|
-
});
|
|
440
|
+
}, destination);
|
|
328
441
|
const existing = hashResult.result?.points?.[0];
|
|
329
442
|
if (existing) {
|
|
330
443
|
return {
|
|
331
444
|
action: "skip",
|
|
445
|
+
destination: destination.name,
|
|
332
446
|
existingId: existing.id,
|
|
333
447
|
existingCount: existing.payload?.reinforcement_count || 1,
|
|
334
448
|
score: 1.0,
|
|
@@ -345,18 +459,19 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
|
|
|
345
459
|
const must = [{ is_null: { key: "superseded_by" } }];
|
|
346
460
|
if (workspaceId)
|
|
347
461
|
must.push({ key: "workspace_id", match: { value: workspaceId } });
|
|
348
|
-
const searchResult = await qdrantRequest("POST", `/collections/${collection}/points/search`, {
|
|
462
|
+
const searchResult = await qdrantRequest("POST", `/collections/${destination.collection}/points/search`, {
|
|
349
463
|
vector,
|
|
350
464
|
filter: { must },
|
|
351
465
|
limit: 1,
|
|
352
466
|
with_payload: true,
|
|
353
|
-
});
|
|
467
|
+
}, destination);
|
|
354
468
|
const top = searchResult.result?.[0];
|
|
355
469
|
if (!top)
|
|
356
|
-
return { action: "insert" };
|
|
470
|
+
return { action: "insert", destination: destination.name };
|
|
357
471
|
if (top.score >= exactThreshold) {
|
|
358
472
|
return {
|
|
359
473
|
action: "skip",
|
|
474
|
+
destination: destination.name,
|
|
360
475
|
existingId: top.id,
|
|
361
476
|
existingCount: top.payload?.reinforcement_count || 1,
|
|
362
477
|
score: top.score,
|
|
@@ -365,17 +480,18 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
|
|
|
365
480
|
if (top.score >= supersedeThreshold) {
|
|
366
481
|
return {
|
|
367
482
|
action: "supersede",
|
|
483
|
+
destination: destination.name,
|
|
368
484
|
existingId: top.id,
|
|
369
485
|
existingCount: top.payload?.reinforcement_count || 1,
|
|
370
486
|
score: top.score,
|
|
371
487
|
};
|
|
372
488
|
}
|
|
373
|
-
return { action: "insert" };
|
|
489
|
+
return { action: "insert", destination: destination.name };
|
|
374
490
|
}
|
|
375
491
|
catch (e) {
|
|
376
492
|
const msg = e instanceof Error ? e.message : String(e);
|
|
377
493
|
logFn("WARN", `Qdrant dedup vector check failed: ${msg}`);
|
|
378
|
-
return { action: "insert" }; // fail open — better to duplicate than lose
|
|
494
|
+
return { action: "insert", destination: destination.name }; // fail open — better to duplicate than lose
|
|
379
495
|
}
|
|
380
496
|
};
|
|
381
497
|
/**
|
|
@@ -387,7 +503,8 @@ const dedupCheck = async (content, contentHashVal, { exactThreshold = 0.92, supe
|
|
|
387
503
|
* No hardcoded vocabulary — purely embedding-similarity based, and the
|
|
388
504
|
* exemplar set grows organically every time a user calls memory_forget.
|
|
389
505
|
*/
|
|
390
|
-
const badExemplarCheck = async (content, workspaceId) => {
|
|
506
|
+
const badExemplarCheck = async (content, workspaceId, routeInput) => {
|
|
507
|
+
const destination = resolveDestination(routeInput ?? { content });
|
|
391
508
|
try {
|
|
392
509
|
const vector = await embed(content);
|
|
393
510
|
const must = [
|
|
@@ -395,12 +512,12 @@ const badExemplarCheck = async (content, workspaceId) => {
|
|
|
395
512
|
];
|
|
396
513
|
if (workspaceId)
|
|
397
514
|
must.push({ key: "workspace_id", match: { value: workspaceId } });
|
|
398
|
-
const result = await qdrantRequest("POST", `/collections/${collection}/points/search`, {
|
|
515
|
+
const result = await qdrantRequest("POST", `/collections/${destination.collection}/points/search`, {
|
|
399
516
|
vector,
|
|
400
517
|
filter: { must },
|
|
401
518
|
limit: 1,
|
|
402
519
|
with_payload: true,
|
|
403
|
-
});
|
|
520
|
+
}, destination);
|
|
404
521
|
const top = result.result?.[0];
|
|
405
522
|
if (!top)
|
|
406
523
|
return null;
|
|
@@ -419,5 +536,5 @@ const badExemplarCheck = async (content, workspaceId) => {
|
|
|
419
536
|
// ---------------------------------------------------------------------------
|
|
420
537
|
// Exports
|
|
421
538
|
// ---------------------------------------------------------------------------
|
|
422
|
-
export { init, isReady, ensureCollection, setLogger, setEmbeddingConfig, qdrantRequest, embed, searchFacts, scrollFacts, storeFact, supersedeFact, reinforceFact, dedupCheck, badExemplarCheck, collection, };
|
|
539
|
+
export { init, isReady, ensureCollection, setLogger, setEmbeddingConfig, qdrantRequest, resolveDestination, collectionForDestination, destinationNames, readyDestinations, activeDestinations, embed, searchFacts, scrollFacts, scrollFactsAcrossDestinations, storeFact, supersedeFact, reinforceFact, dedupCheck, badExemplarCheck, collection, };
|
|
423
540
|
//# sourceMappingURL=qdrant.js.map
|
|
@@ -44,8 +44,8 @@ declare const buildChangedCoOccurrenceCandidates: (facts: QdrantScrollResult[])
|
|
|
44
44
|
* Get the set of entity pairs that already have a system-inferred relation.
|
|
45
45
|
* Returns a Set of pairKeys.
|
|
46
46
|
*/
|
|
47
|
-
declare const getExistingRelations: () => Promise<Set<string>>;
|
|
48
|
-
declare const fetchSupportingFacts: (entityA: string, entityB: string) => Promise<RelationFact[]>;
|
|
47
|
+
declare const getExistingRelations: (destination?: string) => Promise<Set<string>>;
|
|
48
|
+
declare const fetchSupportingFacts: (entityA: string, entityB: string, destination?: string) => Promise<RelationFact[]>;
|
|
49
49
|
declare const inferRelation: (candidate: RelationCandidate) => Promise<{
|
|
50
50
|
from: string;
|
|
51
51
|
type: string;
|
|
@@ -60,7 +60,7 @@ declare const inferRelation: (candidate: RelationCandidate) => Promise<{
|
|
|
60
60
|
directionality_clarity?: string;
|
|
61
61
|
};
|
|
62
62
|
} | null>;
|
|
63
|
-
declare const storeRelation: (fromEntity: string, toEntity: string, relationType: string, content: string, candidate: RelationCandidate, extras?: {
|
|
63
|
+
declare const storeRelation: (fromEntity: string, toEntity: string, relationType: string, content: string, candidate: RelationCandidate, destination?: string, extras?: {
|
|
64
64
|
evidence?: string;
|
|
65
65
|
confidence?: number;
|
|
66
66
|
inVocabulary?: boolean;
|
package/dist/daemon/relations.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { createHash } from "node:crypto";
|
|
9
9
|
import * as qdrant from "./qdrant.js";
|
|
10
|
+
import { buildOperationOrigin } from "../provenance/origin.js";
|
|
10
11
|
import { chatCompletion } from "../llm/index.js";
|
|
11
12
|
import { relationsPrompt, RELATIONS_PROMPT_DESCRIPTOR, safeParseJson, } from "../prompts/index.js";
|
|
12
13
|
import { DEFAULT_CAPTURE_CONTEXT } from "./capture-policy.js";
|
|
@@ -72,8 +73,9 @@ const buildChangedCoOccurrenceCandidates = (facts) => {
|
|
|
72
73
|
* Get the set of entity pairs that already have a system-inferred relation.
|
|
73
74
|
* Returns a Set of pairKeys.
|
|
74
75
|
*/
|
|
75
|
-
const getExistingRelations = async () => {
|
|
76
|
+
const getExistingRelations = async (destination) => {
|
|
76
77
|
const existing = new Set();
|
|
78
|
+
const collection = qdrant.collectionForDestination(destination);
|
|
77
79
|
let offset = null;
|
|
78
80
|
for (;;) {
|
|
79
81
|
const body = {
|
|
@@ -89,7 +91,7 @@ const getExistingRelations = async () => {
|
|
|
89
91
|
};
|
|
90
92
|
if (offset)
|
|
91
93
|
body.offset = offset;
|
|
92
|
-
const result = await qdrant.qdrantRequest("POST", `/collections/${
|
|
94
|
+
const result = await qdrant.qdrantRequest("POST", `/collections/${collection}/points/scroll`, body, destination);
|
|
93
95
|
const points = result.result?.points || [];
|
|
94
96
|
if (points.length === 0)
|
|
95
97
|
break;
|
|
@@ -106,8 +108,9 @@ const getExistingRelations = async () => {
|
|
|
106
108
|
logFn("DEBUG", `Relations: ${existing.size} existing daemon-inferred relations`);
|
|
107
109
|
return existing;
|
|
108
110
|
};
|
|
109
|
-
const fetchSupportingFacts = async (entityA, entityB) => {
|
|
110
|
-
const
|
|
111
|
+
const fetchSupportingFacts = async (entityA, entityB, destination) => {
|
|
112
|
+
const collection = qdrant.collectionForDestination(destination);
|
|
113
|
+
const result = await qdrant.qdrantRequest("POST", `/collections/${collection}/points/scroll`, {
|
|
111
114
|
filter: {
|
|
112
115
|
must: [
|
|
113
116
|
{ is_null: { key: "superseded_by" } },
|
|
@@ -122,7 +125,7 @@ const fetchSupportingFacts = async (entityA, entityB) => {
|
|
|
122
125
|
order_by: { key: "updated_at", direction: "desc" },
|
|
123
126
|
limit: SUPPORTING_FACTS_LIMIT,
|
|
124
127
|
with_payload: true,
|
|
125
|
-
});
|
|
128
|
+
}, destination);
|
|
126
129
|
return (result.result?.points ?? []).map((point) => ({
|
|
127
130
|
id: point.id,
|
|
128
131
|
content: point.payload?.content ?? "",
|
|
@@ -133,8 +136,8 @@ const fetchSupportingFacts = async (entityA, entityB) => {
|
|
|
133
136
|
metadata: point.payload?.metadata ?? {},
|
|
134
137
|
}));
|
|
135
138
|
};
|
|
136
|
-
const buildRelationCandidate = async (changed) => {
|
|
137
|
-
const supportingFacts = await fetchSupportingFacts(changed.entityA, changed.entityB);
|
|
139
|
+
const buildRelationCandidate = async (changed, destination) => {
|
|
140
|
+
const supportingFacts = await fetchSupportingFacts(changed.entityA, changed.entityB, destination);
|
|
138
141
|
if (supportingFacts.length < MIN_SHARED_FACTS)
|
|
139
142
|
return null;
|
|
140
143
|
return {
|
|
@@ -224,7 +227,7 @@ const inferRelation = async (candidate) => {
|
|
|
224
227
|
judgment: parsed.judgment,
|
|
225
228
|
};
|
|
226
229
|
};
|
|
227
|
-
const storeRelation = async (fromEntity, toEntity, relationType, content, candidate, extras = {}) => {
|
|
230
|
+
const storeRelation = async (fromEntity, toEntity, relationType, content, candidate, destination, extras = {}) => {
|
|
228
231
|
const hash = createHash("sha256")
|
|
229
232
|
.update(`daemon-relation:${pairKey(fromEntity, toEntity)}:${relationType}`)
|
|
230
233
|
.digest("hex");
|
|
@@ -256,11 +259,19 @@ const storeRelation = async (fromEntity, toEntity, relationType, content, candid
|
|
|
256
259
|
domain: DEFAULT_CAPTURE_CONTEXT.domain,
|
|
257
260
|
kind: "relation",
|
|
258
261
|
entities: [fromEntity, toEntity],
|
|
259
|
-
source: "system",
|
|
260
262
|
confidence: extras.confidence ?? 0.7,
|
|
261
263
|
importance: 0.6,
|
|
262
264
|
content_hash: hash,
|
|
263
265
|
metadata,
|
|
266
|
+
origin: buildOperationOrigin({
|
|
267
|
+
interface: "daemon",
|
|
268
|
+
action: "create",
|
|
269
|
+
subsystem: "relations",
|
|
270
|
+
metadata: {
|
|
271
|
+
relation_type: relationType,
|
|
272
|
+
supporting_fact_count: candidate.supportingFactIds.length,
|
|
273
|
+
},
|
|
274
|
+
}),
|
|
264
275
|
source_fact_ids: candidate.supportingFactIds,
|
|
265
276
|
...(candidate.workstreamKeys.length > 0 ? { workstream_key: candidate.workstreamKeys[0] } : {}),
|
|
266
277
|
relation: {
|
|
@@ -268,7 +279,7 @@ const storeRelation = async (fromEntity, toEntity, relationType, content, candid
|
|
|
268
279
|
type: relationType,
|
|
269
280
|
to: toEntity,
|
|
270
281
|
},
|
|
271
|
-
});
|
|
282
|
+
}, destination ? { destination } : undefined);
|
|
272
283
|
logFn("INFO", `Relations: inferred ${fromEntity} —[${relationType}]→ ${toEntity} (id: ${id})`);
|
|
273
284
|
return id;
|
|
274
285
|
};
|
|
@@ -287,12 +298,13 @@ const tick = async (config) => {
|
|
|
287
298
|
const attempts = pruneRecentAttempts(job.recent_attempts, now, RELATION_ATTEMPT_BACKOFF_MS);
|
|
288
299
|
const maxPairs = config.daemon.relation_inference_max_pairs_per_run ?? 3;
|
|
289
300
|
const since = job.cursor_updated_at ?? new Date(now.getTime() - DEFAULT_LOOKBACK_MS).toISOString();
|
|
301
|
+
const destination = qdrant.resolveDestination({}).name;
|
|
290
302
|
try {
|
|
291
303
|
const changedFacts = await qdrant.scrollFacts({
|
|
292
304
|
sinceUpdated: since,
|
|
293
305
|
excludeKinds: ["relation"],
|
|
294
306
|
orderBy: { key: "updated_at", direction: "asc" },
|
|
295
|
-
}, CHANGED_FACTS_LIMIT);
|
|
307
|
+
}, CHANGED_FACTS_LIMIT, destination);
|
|
296
308
|
if (changedFacts.length === 0) {
|
|
297
309
|
recordMaintenanceRun("relation_inference", {
|
|
298
310
|
job: "relation_inference",
|
|
@@ -318,14 +330,14 @@ const tick = async (config) => {
|
|
|
318
330
|
}, { cursorUpdatedAt: changedFacts.map((fact) => fact.updated_at || fact.created_at).filter(Boolean).sort().at(-1) ?? nowIso, recentAttempts: attempts }, logFn);
|
|
319
331
|
return;
|
|
320
332
|
}
|
|
321
|
-
const existing = await getExistingRelations();
|
|
333
|
+
const existing = await getExistingRelations(destination);
|
|
322
334
|
const touchedPairs = changedPairs
|
|
323
335
|
.filter((pair) => !existing.has(pairKey(pair.entityA, pair.entityB)))
|
|
324
336
|
.filter((pair) => !isAttemptBackedOff(attempts, pairKey(pair.entityA, pair.entityB), now, RELATION_ATTEMPT_BACKOFF_MS));
|
|
325
337
|
const supportLookupLimit = Math.max(maxPairs * 5, maxPairs);
|
|
326
338
|
const relationCandidates = [];
|
|
327
339
|
for (const changed of touchedPairs.slice(0, supportLookupLimit)) {
|
|
328
|
-
const candidate = await buildRelationCandidate(changed);
|
|
340
|
+
const candidate = await buildRelationCandidate(changed, destination);
|
|
329
341
|
if (candidate)
|
|
330
342
|
relationCandidates.push(candidate);
|
|
331
343
|
if (relationCandidates.length >= maxPairs)
|
|
@@ -345,12 +357,12 @@ const tick = async (config) => {
|
|
|
345
357
|
const hash = createHash("sha256")
|
|
346
358
|
.update(`daemon-relation:${pairKey(result.from, result.to)}:${result.type}`)
|
|
347
359
|
.digest("hex");
|
|
348
|
-
const dedup = await qdrant.dedupCheck(result.content, hash);
|
|
360
|
+
const dedup = await qdrant.dedupCheck(result.content, hash, undefined, undefined, { destination });
|
|
349
361
|
if (dedup.action === "skip") {
|
|
350
362
|
logFn("DEBUG", `Relations: skipping duplicate ${candidate.entityA}↔${candidate.entityB}`);
|
|
351
363
|
continue;
|
|
352
364
|
}
|
|
353
|
-
await storeRelation(result.from, result.to, result.type, result.content, candidate, { evidence: result.evidence, confidence: result.confidence, inVocabulary: result.inVocabulary, judgment: result.judgment });
|
|
365
|
+
await storeRelation(result.from, result.to, result.type, result.content, candidate, destination, { evidence: result.evidence, confidence: result.confidence, inVocabulary: result.inVocabulary, judgment: result.judgment });
|
|
354
366
|
inferred++;
|
|
355
367
|
}
|
|
356
368
|
catch (e) {
|
|
@@ -2,6 +2,7 @@ import type { BikkyConfig } from "../config.js";
|
|
|
2
2
|
import type { EpisodeSummaryWriteResult } from "./episode-summary.js";
|
|
3
3
|
import type { QdrantPayload } from "./qdrant.js";
|
|
4
4
|
import { type RedactionSummary } from "../privacy/redaction.js";
|
|
5
|
+
import { type OperationOrigin } from "../provenance/origin.js";
|
|
5
6
|
export interface WorkspaceScope {
|
|
6
7
|
workspaceId?: string;
|
|
7
8
|
actorId?: string;
|
|
@@ -38,6 +39,8 @@ export declare const buildSessionIndexPayload: (input: {
|
|
|
38
39
|
enabled: boolean;
|
|
39
40
|
redactPii: boolean;
|
|
40
41
|
};
|
|
42
|
+
config?: BikkyConfig;
|
|
43
|
+
origin?: OperationOrigin;
|
|
41
44
|
}) => SessionIndexPayloadResult;
|
|
42
45
|
export declare const updateSessionIndex: (input: {
|
|
43
46
|
sessionId: string;
|
|
@@ -45,9 +48,11 @@ export declare const updateSessionIndex: (input: {
|
|
|
45
48
|
episodeResults: EpisodeSummaryWriteResult[];
|
|
46
49
|
scope: WorkspaceScope;
|
|
47
50
|
config: BikkyConfig;
|
|
51
|
+
destination?: string;
|
|
48
52
|
}) => Promise<{
|
|
49
53
|
action: "stored" | "updated" | "skipped";
|
|
50
54
|
factId?: string;
|
|
55
|
+
destination?: string;
|
|
51
56
|
reason?: string;
|
|
52
57
|
}>;
|
|
53
58
|
//# sourceMappingURL=session-index.d.ts.map
|