bikky 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -37
- package/dist/config.d.ts +15 -1
- package/dist/config.js +116 -20
- package/dist/daemon/capture-policy.d.ts +0 -1
- package/dist/daemon/capture-policy.js +0 -2
- package/dist/daemon/consolidation.d.ts +2 -1
- package/dist/daemon/consolidation.js +32 -15
- package/dist/daemon/entity-typing.js +10 -0
- package/dist/daemon/episode-summary.d.ts +4 -0
- package/dist/daemon/episode-summary.js +39 -8
- package/dist/daemon/extraction.d.ts +2 -2
- package/dist/daemon/extraction.js +65 -22
- package/dist/daemon/loop.js +8 -0
- package/dist/daemon/maintenance-state.d.ts +1 -1
- package/dist/daemon/maintenance-state.js +2 -0
- package/dist/daemon/qdrant.d.ts +32 -10
- package/dist/daemon/qdrant.js +199 -60
- package/dist/daemon/quality-rollups.d.ts +51 -0
- package/dist/daemon/quality-rollups.js +378 -0
- package/dist/daemon/relations.d.ts +3 -3
- package/dist/daemon/relations.js +28 -16
- package/dist/daemon/session-index.d.ts +5 -0
- package/dist/daemon/session-index.js +36 -9
- package/dist/daemon/session-summary.d.ts +3 -0
- package/dist/daemon/session-summary.js +48 -15
- package/dist/daemon/staleness.js +3 -3
- package/dist/daemon/transcript-sources.js +3 -2
- package/dist/daemon/watcher.js +2 -0
- package/dist/daemon/workstream-summary.d.ts +4 -0
- package/dist/daemon/workstream-summary.js +58 -16
- package/dist/install.d.ts +11 -0
- package/dist/install.js +38 -0
- package/dist/lifecycle.js +7 -5
- package/dist/llm/embedding/index.js +2 -1
- package/dist/llm/embedding/providers/openai.js +8 -2
- package/dist/llm/embedding/providers/portkey.js +9 -2
- package/dist/llm/inference/index.js +2 -1
- package/dist/llm/util.d.ts +12 -0
- package/dist/llm/util.js +18 -0
- package/dist/mcp/helpers.d.ts +8 -0
- package/dist/mcp/helpers.js +36 -3
- package/dist/mcp/taxonomy.d.ts +9 -13
- package/dist/mcp/taxonomy.js +59 -42
- package/dist/mcp/tools.js +351 -83
- package/dist/mcp/types.d.ts +35 -0
- package/dist/package-verifier.d.ts +19 -0
- package/dist/package-verifier.js +83 -0
- package/dist/prompts/brief.d.ts +2 -2
- package/dist/prompts/brief.js +0 -1
- package/dist/prompts/extraction.js +9 -11
- package/dist/provenance/origin.d.ts +57 -0
- package/dist/provenance/origin.js +254 -0
- package/dist/routing-context.d.ts +16 -0
- package/dist/routing-context.js +55 -0
- package/dist/status.d.ts +1 -0
- package/dist/status.js +7 -1
- package/docs/config/fully-hosted.md +33 -13
- package/docs/config/hosted-models.md +33 -13
- package/docs/config/hosted-qdrant-local-models.md +1 -0
- package/docs/config/local.md +1 -0
- package/docs/configuration.md +42 -17
- package/package.json +2 -2
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backend aggregation for memory quality telemetry.
|
|
3
|
+
*/
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { contentHash, computeEffectiveConfidence } from "../mcp/helpers.js";
|
|
6
|
+
import { categoryForMemorySubtype, layerForMemorySubtype } from "../mcp/taxonomy.js";
|
|
7
|
+
import { buildOperationOrigin } from "../provenance/origin.js";
|
|
8
|
+
import { readMaintenanceState, recordMaintenanceRun, shouldRunMaintenance, } from "./maintenance-state.js";
|
|
9
|
+
import * as qdrant from "./qdrant.js";
|
|
10
|
+
const SCROLL_LIMIT = 256;
|
|
11
|
+
const ROLLUP_TYPE = "latest";
|
|
12
|
+
const JOB_NAME = "memory_quality_rollups";
|
|
13
|
+
let logFn = () => { };
|
|
14
|
+
const defaultDeps = {
|
|
15
|
+
isReady: qdrant.isReady,
|
|
16
|
+
activeDestinations: qdrant.activeDestinations,
|
|
17
|
+
qdrantRequest: qdrant.qdrantRequest,
|
|
18
|
+
embed: qdrant.embed,
|
|
19
|
+
};
|
|
20
|
+
export const setLogger = (fn) => {
|
|
21
|
+
logFn = fn;
|
|
22
|
+
};
|
|
23
|
+
const nonEmptyString = (value) => {
|
|
24
|
+
if (typeof value !== "string")
|
|
25
|
+
return null;
|
|
26
|
+
const trimmed = value.trim();
|
|
27
|
+
return trimmed.length > 0 ? trimmed : null;
|
|
28
|
+
};
|
|
29
|
+
const numberValue = (value) => (typeof value === "number" && Number.isFinite(value) ? value : 0);
|
|
30
|
+
const stringArray = (value) => (Array.isArray(value)
|
|
31
|
+
? value.map((item) => nonEmptyString(item)).filter((item) => item !== null)
|
|
32
|
+
: []);
|
|
33
|
+
const stableUuid = (input) => {
|
|
34
|
+
const hash = createHash("sha256").update(input).digest("hex");
|
|
35
|
+
return [
|
|
36
|
+
hash.slice(0, 8),
|
|
37
|
+
hash.slice(8, 12),
|
|
38
|
+
hash.slice(12, 16),
|
|
39
|
+
hash.slice(16, 20),
|
|
40
|
+
hash.slice(20, 32),
|
|
41
|
+
].join("-");
|
|
42
|
+
};
|
|
43
|
+
const completePayload = (payload) => ({
|
|
44
|
+
...payload,
|
|
45
|
+
content: payload.content ?? "",
|
|
46
|
+
category: payload.category ?? "engineering",
|
|
47
|
+
domain: payload.domain ?? "software_engineering",
|
|
48
|
+
kind: payload.kind ?? "fact",
|
|
49
|
+
memory_subtype: payload.memory_subtype ?? null,
|
|
50
|
+
entities: payload.entities ?? [],
|
|
51
|
+
confidence: typeof payload.confidence === "number" ? payload.confidence : 0.7,
|
|
52
|
+
content_hash: payload.content_hash ?? "",
|
|
53
|
+
reinforcement_count: payload.reinforcement_count ?? 1,
|
|
54
|
+
last_reinforced_at: payload.last_reinforced_at ?? payload.created_at ?? "",
|
|
55
|
+
superseded_by: payload.superseded_by ?? null,
|
|
56
|
+
superseded_at: payload.superseded_at ?? null,
|
|
57
|
+
created_at: payload.created_at ?? "",
|
|
58
|
+
updated_at: payload.updated_at ?? payload.created_at ?? "",
|
|
59
|
+
});
|
|
60
|
+
const daysBetween = (now, iso) => {
|
|
61
|
+
if (!iso)
|
|
62
|
+
return 0;
|
|
63
|
+
const timestamp = Date.parse(iso);
|
|
64
|
+
if (!Number.isFinite(timestamp))
|
|
65
|
+
return 0;
|
|
66
|
+
return Math.max(0, (now.getTime() - timestamp) / 86_400_000);
|
|
67
|
+
};
|
|
68
|
+
const isFactStale = (payload, now, thresholdDays) => {
|
|
69
|
+
const activity = payload.last_verified_at ?? payload.last_reinforced_at ?? payload.created_at;
|
|
70
|
+
if (!activity)
|
|
71
|
+
return false;
|
|
72
|
+
return daysBetween(now, activity) > thresholdDays;
|
|
73
|
+
};
|
|
74
|
+
const scopesForFact = (fact) => {
|
|
75
|
+
const scopes = [{ type: "destination", value: fact.destination }];
|
|
76
|
+
const add = (type, value) => {
|
|
77
|
+
const normalized = nonEmptyString(value);
|
|
78
|
+
if (normalized)
|
|
79
|
+
scopes.push({ type, value: normalized });
|
|
80
|
+
};
|
|
81
|
+
add("repo", fact.payload.repo);
|
|
82
|
+
add("workstream_key", fact.payload.workstream_key);
|
|
83
|
+
add("task_key", fact.payload.task_key);
|
|
84
|
+
for (const entity of fact.payload.entities ?? [])
|
|
85
|
+
add("entity", entity);
|
|
86
|
+
add("origin_user", fact.payload.origin?.user?.id);
|
|
87
|
+
add("origin_agent", fact.payload.origin?.agent?.id);
|
|
88
|
+
const seen = new Set();
|
|
89
|
+
return scopes.filter((scope) => {
|
|
90
|
+
const key = `${scope.type}\0${scope.value}`;
|
|
91
|
+
if (seen.has(key))
|
|
92
|
+
return false;
|
|
93
|
+
seen.add(key);
|
|
94
|
+
return true;
|
|
95
|
+
});
|
|
96
|
+
};
|
|
97
|
+
const rollupKey = (destination, scope) => `${destination}\0${scope.type}\0${scope.value}`;
|
|
98
|
+
const getRollup = (rollups, destination, scope, generatedAt) => {
|
|
99
|
+
const key = rollupKey(destination, scope);
|
|
100
|
+
const existing = rollups.get(key);
|
|
101
|
+
if (existing)
|
|
102
|
+
return existing;
|
|
103
|
+
const created = {
|
|
104
|
+
destination,
|
|
105
|
+
scope_type: scope.type,
|
|
106
|
+
scope_value: scope.value,
|
|
107
|
+
active_fact_count: 0,
|
|
108
|
+
recall_count: 0,
|
|
109
|
+
useful_count: 0,
|
|
110
|
+
misleading_count: 0,
|
|
111
|
+
wrong_count: 0,
|
|
112
|
+
stale_count: 0,
|
|
113
|
+
low_confidence_count: 0,
|
|
114
|
+
generated_at: generatedAt,
|
|
115
|
+
sourceFactIds: new Set(),
|
|
116
|
+
sourceEventIds: new Set(),
|
|
117
|
+
};
|
|
118
|
+
rollups.set(key, created);
|
|
119
|
+
return created;
|
|
120
|
+
};
|
|
121
|
+
const addEventSignal = (rollups, scopes, destination, generatedAt, eventId, update) => {
|
|
122
|
+
for (const scope of scopes) {
|
|
123
|
+
const rollup = getRollup(rollups, destination, scope, generatedAt);
|
|
124
|
+
update(rollup);
|
|
125
|
+
rollup.sourceEventIds.add(eventId);
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
export const buildQualityRollups = (input) => {
|
|
129
|
+
const now = input.generatedAt ?? new Date();
|
|
130
|
+
const generatedAt = now.toISOString();
|
|
131
|
+
const staleThresholdDays = input.staleThresholdDays ?? 30;
|
|
132
|
+
const lowConfidenceThreshold = input.lowConfidenceThreshold ?? 0.6;
|
|
133
|
+
const factsById = new Map(input.facts.map((fact) => [fact.id, fact]));
|
|
134
|
+
const scopesByFactId = new Map(input.facts.map((fact) => [fact.id, scopesForFact(fact)]));
|
|
135
|
+
const rollups = new Map();
|
|
136
|
+
for (const fact of input.facts) {
|
|
137
|
+
const payload = completePayload(fact.payload);
|
|
138
|
+
const scopes = scopesByFactId.get(fact.id) ?? [];
|
|
139
|
+
for (const scope of scopes) {
|
|
140
|
+
const rollup = getRollup(rollups, fact.destination, scope, generatedAt);
|
|
141
|
+
rollup.active_fact_count++;
|
|
142
|
+
rollup.recall_count += numberValue(payload.recall_count);
|
|
143
|
+
rollup.useful_count += numberValue(payload.useful_count ?? payload.useful_feedback_count);
|
|
144
|
+
if (isFactStale(payload, now, staleThresholdDays))
|
|
145
|
+
rollup.stale_count++;
|
|
146
|
+
if (computeEffectiveConfidence(payload) < lowConfidenceThreshold)
|
|
147
|
+
rollup.low_confidence_count++;
|
|
148
|
+
rollup.sourceFactIds.add(fact.id);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
for (const event of input.events ?? []) {
|
|
152
|
+
const subtype = event.payload.memory_subtype;
|
|
153
|
+
if (subtype === "recall_event") {
|
|
154
|
+
for (const factId of stringArray(event.payload.returned_fact_ids)) {
|
|
155
|
+
const fact = factsById.get(factId);
|
|
156
|
+
if (!fact || typeof fact.payload.recall_count === "number")
|
|
157
|
+
continue;
|
|
158
|
+
addEventSignal(rollups, scopesByFactId.get(factId) ?? [], fact.destination, generatedAt, event.id, (rollup) => { rollup.recall_count++; });
|
|
159
|
+
}
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
const targetFactId = nonEmptyString(event.payload.target_fact_id);
|
|
163
|
+
if (!targetFactId)
|
|
164
|
+
continue;
|
|
165
|
+
const targetFact = factsById.get(targetFactId);
|
|
166
|
+
if (!targetFact)
|
|
167
|
+
continue;
|
|
168
|
+
const targetScopes = scopesByFactId.get(targetFactId) ?? [];
|
|
169
|
+
if (subtype === "feedback_event" && event.payload.feedback_kind === "useful") {
|
|
170
|
+
if (typeof targetFact.payload.useful_count === "number" || typeof targetFact.payload.useful_feedback_count === "number") {
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
addEventSignal(rollups, targetScopes, targetFact.destination, generatedAt, event.id, (rollup) => { rollup.useful_count++; });
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (subtype === "outcome_event" && (event.payload.outcome === "misleading" || event.payload.outcome === "wrong")) {
|
|
177
|
+
addEventSignal(rollups, targetScopes, targetFact.destination, generatedAt, event.id, (rollup) => {
|
|
178
|
+
if (event.payload.outcome === "misleading")
|
|
179
|
+
rollup.misleading_count++;
|
|
180
|
+
if (event.payload.outcome === "wrong")
|
|
181
|
+
rollup.wrong_count++;
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return [...rollups.values()]
|
|
186
|
+
.map((rollup) => ({
|
|
187
|
+
destination: rollup.destination,
|
|
188
|
+
scope_type: rollup.scope_type,
|
|
189
|
+
scope_value: rollup.scope_value,
|
|
190
|
+
active_fact_count: rollup.active_fact_count,
|
|
191
|
+
recall_count: rollup.recall_count,
|
|
192
|
+
useful_count: rollup.useful_count,
|
|
193
|
+
misleading_count: rollup.misleading_count,
|
|
194
|
+
wrong_count: rollup.wrong_count,
|
|
195
|
+
stale_count: rollup.stale_count,
|
|
196
|
+
low_confidence_count: rollup.low_confidence_count,
|
|
197
|
+
generated_at: rollup.generated_at,
|
|
198
|
+
source_fact_ids: [...rollup.sourceFactIds].sort().slice(0, 100),
|
|
199
|
+
source_event_ids: [...rollup.sourceEventIds].sort().slice(0, 100),
|
|
200
|
+
}))
|
|
201
|
+
.sort((a, b) => `${a.scope_type}:${a.scope_value}`.localeCompare(`${b.scope_type}:${b.scope_value}`));
|
|
202
|
+
};
|
|
203
|
+
const scrollAllPoints = async (deps, destination, filter) => {
|
|
204
|
+
const points = [];
|
|
205
|
+
let offset;
|
|
206
|
+
do {
|
|
207
|
+
const body = {
|
|
208
|
+
filter,
|
|
209
|
+
limit: SCROLL_LIMIT,
|
|
210
|
+
with_payload: true,
|
|
211
|
+
...(offset !== undefined && offset !== null ? { offset } : {}),
|
|
212
|
+
};
|
|
213
|
+
const response = await deps.qdrantRequest("POST", `/collections/${destination.collection}/points/scroll`, body, destination);
|
|
214
|
+
for (const point of response.result?.points ?? []) {
|
|
215
|
+
points.push({
|
|
216
|
+
id: point.id,
|
|
217
|
+
destination: destination.name,
|
|
218
|
+
payload: (point.payload ?? {}),
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
offset = response.result?.next_page_offset;
|
|
222
|
+
} while (offset !== undefined && offset !== null);
|
|
223
|
+
return points;
|
|
224
|
+
};
|
|
225
|
+
const fetchQualityInputs = async (deps, destination) => {
|
|
226
|
+
const facts = await scrollAllPoints(deps, destination, {
|
|
227
|
+
must: [{ is_null: { key: "superseded_by" } }],
|
|
228
|
+
must_not: [{ key: "kind", match: { any: ["telemetry", "entity_type"] } }],
|
|
229
|
+
});
|
|
230
|
+
const events = await scrollAllPoints(deps, destination, {
|
|
231
|
+
must: [
|
|
232
|
+
{ key: "kind", match: { value: "telemetry" } },
|
|
233
|
+
{ key: "memory_subtype", match: { any: ["feedback_event", "outcome_event", "recall_event"] } },
|
|
234
|
+
],
|
|
235
|
+
});
|
|
236
|
+
return { facts, events };
|
|
237
|
+
};
|
|
238
|
+
const rollupContent = (rollup) => (`Memory quality rollup for ${rollup.scope_type}:${rollup.scope_value}: ` +
|
|
239
|
+
`${rollup.active_fact_count} active facts, ${rollup.recall_count} recalls, ` +
|
|
240
|
+
`${rollup.useful_count} useful, ${rollup.misleading_count} misleading, ` +
|
|
241
|
+
`${rollup.wrong_count} wrong, ${rollup.stale_count} stale, ` +
|
|
242
|
+
`${rollup.low_confidence_count} low-confidence.`);
|
|
243
|
+
const rollupId = (rollup) => stableUuid(`aggregate_rollup:${ROLLUP_TYPE}:${rollup.destination}:${rollup.scope_type}:${rollup.scope_value}`);
|
|
244
|
+
const rollupPayload = (config, rollup) => {
|
|
245
|
+
const content = rollupContent(rollup);
|
|
246
|
+
return {
|
|
247
|
+
content,
|
|
248
|
+
category: categoryForMemorySubtype("aggregate_rollup") ?? "system",
|
|
249
|
+
domain: "software_engineering",
|
|
250
|
+
kind: "telemetry",
|
|
251
|
+
memory_subtype: "aggregate_rollup",
|
|
252
|
+
layer: layerForMemorySubtype("aggregate_rollup") ?? "workspace",
|
|
253
|
+
entities: rollup.scope_type === "entity" ? [rollup.scope_value.toLowerCase()] : [],
|
|
254
|
+
origin: buildOperationOrigin({
|
|
255
|
+
config,
|
|
256
|
+
interface: "daemon",
|
|
257
|
+
action: "aggregate",
|
|
258
|
+
subsystem: "memory_quality_rollups",
|
|
259
|
+
metadata: {
|
|
260
|
+
destination: rollup.destination,
|
|
261
|
+
scope_type: rollup.scope_type,
|
|
262
|
+
scope_value: rollup.scope_value,
|
|
263
|
+
},
|
|
264
|
+
}),
|
|
265
|
+
confidence: 1.0,
|
|
266
|
+
importance: 0.4,
|
|
267
|
+
content_hash: contentHash("aggregate_rollup", `${ROLLUP_TYPE}:${rollup.destination}:${rollup.scope_type}:${rollup.scope_value}`),
|
|
268
|
+
reinforcement_count: 1,
|
|
269
|
+
last_reinforced_at: rollup.generated_at,
|
|
270
|
+
superseded_by: null,
|
|
271
|
+
superseded_at: null,
|
|
272
|
+
created_at: rollup.generated_at,
|
|
273
|
+
updated_at: rollup.generated_at,
|
|
274
|
+
rollup_type: ROLLUP_TYPE,
|
|
275
|
+
rollup_generated_at: rollup.generated_at,
|
|
276
|
+
rollup_window_end: rollup.generated_at,
|
|
277
|
+
scope_type: rollup.scope_type,
|
|
278
|
+
scope_value: rollup.scope_value,
|
|
279
|
+
active_fact_count: rollup.active_fact_count,
|
|
280
|
+
recall_count: rollup.recall_count,
|
|
281
|
+
useful_count: rollup.useful_count,
|
|
282
|
+
misleading_count: rollup.misleading_count,
|
|
283
|
+
wrong_count: rollup.wrong_count,
|
|
284
|
+
stale_count: rollup.stale_count,
|
|
285
|
+
low_confidence_count: rollup.low_confidence_count,
|
|
286
|
+
source_fact_ids: rollup.source_fact_ids,
|
|
287
|
+
source_event_ids: rollup.source_event_ids,
|
|
288
|
+
metadata: {
|
|
289
|
+
generated_by: "memory_quality_rollups",
|
|
290
|
+
rollup_type: ROLLUP_TYPE,
|
|
291
|
+
},
|
|
292
|
+
};
|
|
293
|
+
};
|
|
294
|
+
const upsertRollup = async (config, deps, destination, rollup) => {
|
|
295
|
+
const payload = rollupPayload(config, rollup);
|
|
296
|
+
const vector = await deps.embed(String(payload.content));
|
|
297
|
+
await deps.qdrantRequest("PUT", `/collections/${destination.collection}/points`, {
|
|
298
|
+
points: [{ id: rollupId(rollup), vector, payload }],
|
|
299
|
+
}, destination);
|
|
300
|
+
};
|
|
301
|
+
export const aggregateMemoryQualitySignals = async (config, deps = defaultDeps) => {
|
|
302
|
+
const destinations = deps.activeDestinations();
|
|
303
|
+
const generatedAt = new Date();
|
|
304
|
+
const maxScopes = config.daemon.memory_quality_rollups_max_scopes_per_run ?? 100;
|
|
305
|
+
let factsSeen = 0;
|
|
306
|
+
let eventsSeen = 0;
|
|
307
|
+
let rollupsUpserted = 0;
|
|
308
|
+
let scopesCapped = false;
|
|
309
|
+
for (const destination of destinations) {
|
|
310
|
+
const { facts, events } = await fetchQualityInputs(deps, destination);
|
|
311
|
+
factsSeen += facts.length;
|
|
312
|
+
eventsSeen += events.length;
|
|
313
|
+
const rollups = buildQualityRollups({
|
|
314
|
+
facts,
|
|
315
|
+
events,
|
|
316
|
+
generatedAt,
|
|
317
|
+
staleThresholdDays: config.daemon.staleness_threshold_days,
|
|
318
|
+
lowConfidenceThreshold: config.daemon.memory_quality_rollups_low_confidence_threshold,
|
|
319
|
+
});
|
|
320
|
+
const selectedRollups = rollups.slice(0, maxScopes);
|
|
321
|
+
scopesCapped ||= rollups.length > selectedRollups.length;
|
|
322
|
+
for (const rollup of selectedRollups) {
|
|
323
|
+
await upsertRollup(config, deps, destination, rollup);
|
|
324
|
+
rollupsUpserted++;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
return {
|
|
328
|
+
destinations_seen: destinations.length,
|
|
329
|
+
facts_seen: factsSeen,
|
|
330
|
+
events_seen: eventsSeen,
|
|
331
|
+
rollups_upserted: rollupsUpserted,
|
|
332
|
+
scopes_capped: scopesCapped,
|
|
333
|
+
};
|
|
334
|
+
};
|
|
335
|
+
export const tick = async (config, deps = defaultDeps) => {
|
|
336
|
+
if (!config.daemon.memory_quality_rollups_enabled)
|
|
337
|
+
return;
|
|
338
|
+
if (!deps.isReady())
|
|
339
|
+
return;
|
|
340
|
+
const now = new Date();
|
|
341
|
+
const nowIso = now.toISOString();
|
|
342
|
+
const state = readMaintenanceState(logFn);
|
|
343
|
+
const job = state.jobs.memory_quality_rollups;
|
|
344
|
+
const intervalSec = config.daemon.memory_quality_rollups_interval_sec ?? 3600;
|
|
345
|
+
if (!shouldRunMaintenance(now, job.last_run_at, intervalSec))
|
|
346
|
+
return;
|
|
347
|
+
try {
|
|
348
|
+
const result = await aggregateMemoryQualitySignals(config, deps);
|
|
349
|
+
recordMaintenanceRun(JOB_NAME, {
|
|
350
|
+
job: JOB_NAME,
|
|
351
|
+
ran_at: nowIso,
|
|
352
|
+
status: result.facts_seen === 0 ? "skipped" : "success",
|
|
353
|
+
candidates_seen: result.facts_seen + result.events_seen,
|
|
354
|
+
llm_calls: 0,
|
|
355
|
+
accepted: result.rollups_upserted,
|
|
356
|
+
deterministic: result.rollups_upserted,
|
|
357
|
+
skipped_reason: result.facts_seen === 0
|
|
358
|
+
? "no_active_facts"
|
|
359
|
+
: result.scopes_capped
|
|
360
|
+
? "max_scopes_per_run_reached"
|
|
361
|
+
: undefined,
|
|
362
|
+
}, { cursorUpdatedAt: nowIso }, logFn);
|
|
363
|
+
}
|
|
364
|
+
catch (e) {
|
|
365
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
366
|
+
logFn("ERROR", `Memory quality rollups failed: ${message}`);
|
|
367
|
+
recordMaintenanceRun(JOB_NAME, {
|
|
368
|
+
job: JOB_NAME,
|
|
369
|
+
ran_at: nowIso,
|
|
370
|
+
status: "error",
|
|
371
|
+
candidates_seen: 0,
|
|
372
|
+
llm_calls: 0,
|
|
373
|
+
accepted: 0,
|
|
374
|
+
error: message,
|
|
375
|
+
}, {}, logFn);
|
|
376
|
+
}
|
|
377
|
+
};
|
|
378
|
+
//# sourceMappingURL=quality-rollups.js.map
|
|
@@ -44,8 +44,8 @@ declare const buildChangedCoOccurrenceCandidates: (facts: QdrantScrollResult[])
|
|
|
44
44
|
* Get the set of entity pairs that already have a system-inferred relation.
|
|
45
45
|
* Returns a Set of pairKeys.
|
|
46
46
|
*/
|
|
47
|
-
declare const getExistingRelations: () => Promise<Set<string>>;
|
|
48
|
-
declare const fetchSupportingFacts: (entityA: string, entityB: string) => Promise<RelationFact[]>;
|
|
47
|
+
declare const getExistingRelations: (destination?: string) => Promise<Set<string>>;
|
|
48
|
+
declare const fetchSupportingFacts: (entityA: string, entityB: string, destination?: string) => Promise<RelationFact[]>;
|
|
49
49
|
declare const inferRelation: (candidate: RelationCandidate) => Promise<{
|
|
50
50
|
from: string;
|
|
51
51
|
type: string;
|
|
@@ -60,7 +60,7 @@ declare const inferRelation: (candidate: RelationCandidate) => Promise<{
|
|
|
60
60
|
directionality_clarity?: string;
|
|
61
61
|
};
|
|
62
62
|
} | null>;
|
|
63
|
-
declare const storeRelation: (fromEntity: string, toEntity: string, relationType: string, content: string, candidate: RelationCandidate, extras?: {
|
|
63
|
+
declare const storeRelation: (fromEntity: string, toEntity: string, relationType: string, content: string, candidate: RelationCandidate, destination?: string, extras?: {
|
|
64
64
|
evidence?: string;
|
|
65
65
|
confidence?: number;
|
|
66
66
|
inVocabulary?: boolean;
|
package/dist/daemon/relations.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { createHash } from "node:crypto";
|
|
9
9
|
import * as qdrant from "./qdrant.js";
|
|
10
|
+
import { buildOperationOrigin } from "../provenance/origin.js";
|
|
10
11
|
import { chatCompletion } from "../llm/index.js";
|
|
11
12
|
import { relationsPrompt, RELATIONS_PROMPT_DESCRIPTOR, safeParseJson, } from "../prompts/index.js";
|
|
12
13
|
import { DEFAULT_CAPTURE_CONTEXT } from "./capture-policy.js";
|
|
@@ -72,8 +73,9 @@ const buildChangedCoOccurrenceCandidates = (facts) => {
|
|
|
72
73
|
* Get the set of entity pairs that already have a system-inferred relation.
|
|
73
74
|
* Returns a Set of pairKeys.
|
|
74
75
|
*/
|
|
75
|
-
const getExistingRelations = async () => {
|
|
76
|
+
const getExistingRelations = async (destination) => {
|
|
76
77
|
const existing = new Set();
|
|
78
|
+
const collection = qdrant.collectionForDestination(destination);
|
|
77
79
|
let offset = null;
|
|
78
80
|
for (;;) {
|
|
79
81
|
const body = {
|
|
@@ -89,7 +91,7 @@ const getExistingRelations = async () => {
|
|
|
89
91
|
};
|
|
90
92
|
if (offset)
|
|
91
93
|
body.offset = offset;
|
|
92
|
-
const result = await qdrant.qdrantRequest("POST", `/collections/${
|
|
94
|
+
const result = await qdrant.qdrantRequest("POST", `/collections/${collection}/points/scroll`, body, destination);
|
|
93
95
|
const points = result.result?.points || [];
|
|
94
96
|
if (points.length === 0)
|
|
95
97
|
break;
|
|
@@ -106,8 +108,9 @@ const getExistingRelations = async () => {
|
|
|
106
108
|
logFn("DEBUG", `Relations: ${existing.size} existing daemon-inferred relations`);
|
|
107
109
|
return existing;
|
|
108
110
|
};
|
|
109
|
-
const fetchSupportingFacts = async (entityA, entityB) => {
|
|
110
|
-
const
|
|
111
|
+
const fetchSupportingFacts = async (entityA, entityB, destination) => {
|
|
112
|
+
const collection = qdrant.collectionForDestination(destination);
|
|
113
|
+
const result = await qdrant.qdrantRequest("POST", `/collections/${collection}/points/scroll`, {
|
|
111
114
|
filter: {
|
|
112
115
|
must: [
|
|
113
116
|
{ is_null: { key: "superseded_by" } },
|
|
@@ -122,7 +125,7 @@ const fetchSupportingFacts = async (entityA, entityB) => {
|
|
|
122
125
|
order_by: { key: "updated_at", direction: "desc" },
|
|
123
126
|
limit: SUPPORTING_FACTS_LIMIT,
|
|
124
127
|
with_payload: true,
|
|
125
|
-
});
|
|
128
|
+
}, destination);
|
|
126
129
|
return (result.result?.points ?? []).map((point) => ({
|
|
127
130
|
id: point.id,
|
|
128
131
|
content: point.payload?.content ?? "",
|
|
@@ -133,8 +136,8 @@ const fetchSupportingFacts = async (entityA, entityB) => {
|
|
|
133
136
|
metadata: point.payload?.metadata ?? {},
|
|
134
137
|
}));
|
|
135
138
|
};
|
|
136
|
-
const buildRelationCandidate = async (changed) => {
|
|
137
|
-
const supportingFacts = await fetchSupportingFacts(changed.entityA, changed.entityB);
|
|
139
|
+
const buildRelationCandidate = async (changed, destination) => {
|
|
140
|
+
const supportingFacts = await fetchSupportingFacts(changed.entityA, changed.entityB, destination);
|
|
138
141
|
if (supportingFacts.length < MIN_SHARED_FACTS)
|
|
139
142
|
return null;
|
|
140
143
|
return {
|
|
@@ -224,7 +227,7 @@ const inferRelation = async (candidate) => {
|
|
|
224
227
|
judgment: parsed.judgment,
|
|
225
228
|
};
|
|
226
229
|
};
|
|
227
|
-
const storeRelation = async (fromEntity, toEntity, relationType, content, candidate, extras = {}) => {
|
|
230
|
+
const storeRelation = async (fromEntity, toEntity, relationType, content, candidate, destination, extras = {}) => {
|
|
228
231
|
const hash = createHash("sha256")
|
|
229
232
|
.update(`daemon-relation:${pairKey(fromEntity, toEntity)}:${relationType}`)
|
|
230
233
|
.digest("hex");
|
|
@@ -252,15 +255,23 @@ const storeRelation = async (fromEntity, toEntity, relationType, content, candid
|
|
|
252
255
|
}
|
|
253
256
|
const id = await qdrant.storeFact({
|
|
254
257
|
content,
|
|
255
|
-
category: "
|
|
258
|
+
category: "engineering",
|
|
256
259
|
domain: DEFAULT_CAPTURE_CONTEXT.domain,
|
|
257
260
|
kind: "relation",
|
|
258
261
|
entities: [fromEntity, toEntity],
|
|
259
|
-
source: "system",
|
|
260
262
|
confidence: extras.confidence ?? 0.7,
|
|
261
263
|
importance: 0.6,
|
|
262
264
|
content_hash: hash,
|
|
263
265
|
metadata,
|
|
266
|
+
origin: buildOperationOrigin({
|
|
267
|
+
interface: "daemon",
|
|
268
|
+
action: "create",
|
|
269
|
+
subsystem: "relations",
|
|
270
|
+
metadata: {
|
|
271
|
+
relation_type: relationType,
|
|
272
|
+
supporting_fact_count: candidate.supportingFactIds.length,
|
|
273
|
+
},
|
|
274
|
+
}),
|
|
264
275
|
source_fact_ids: candidate.supportingFactIds,
|
|
265
276
|
...(candidate.workstreamKeys.length > 0 ? { workstream_key: candidate.workstreamKeys[0] } : {}),
|
|
266
277
|
relation: {
|
|
@@ -268,7 +279,7 @@ const storeRelation = async (fromEntity, toEntity, relationType, content, candid
|
|
|
268
279
|
type: relationType,
|
|
269
280
|
to: toEntity,
|
|
270
281
|
},
|
|
271
|
-
});
|
|
282
|
+
}, destination ? { destination } : undefined);
|
|
272
283
|
logFn("INFO", `Relations: inferred ${fromEntity} —[${relationType}]→ ${toEntity} (id: ${id})`);
|
|
273
284
|
return id;
|
|
274
285
|
};
|
|
@@ -287,12 +298,13 @@ const tick = async (config) => {
|
|
|
287
298
|
const attempts = pruneRecentAttempts(job.recent_attempts, now, RELATION_ATTEMPT_BACKOFF_MS);
|
|
288
299
|
const maxPairs = config.daemon.relation_inference_max_pairs_per_run ?? 3;
|
|
289
300
|
const since = job.cursor_updated_at ?? new Date(now.getTime() - DEFAULT_LOOKBACK_MS).toISOString();
|
|
301
|
+
const destination = qdrant.resolveDestination({}).name;
|
|
290
302
|
try {
|
|
291
303
|
const changedFacts = await qdrant.scrollFacts({
|
|
292
304
|
sinceUpdated: since,
|
|
293
305
|
excludeKinds: ["relation"],
|
|
294
306
|
orderBy: { key: "updated_at", direction: "asc" },
|
|
295
|
-
}, CHANGED_FACTS_LIMIT);
|
|
307
|
+
}, CHANGED_FACTS_LIMIT, destination);
|
|
296
308
|
if (changedFacts.length === 0) {
|
|
297
309
|
recordMaintenanceRun("relation_inference", {
|
|
298
310
|
job: "relation_inference",
|
|
@@ -318,14 +330,14 @@ const tick = async (config) => {
|
|
|
318
330
|
}, { cursorUpdatedAt: changedFacts.map((fact) => fact.updated_at || fact.created_at).filter(Boolean).sort().at(-1) ?? nowIso, recentAttempts: attempts }, logFn);
|
|
319
331
|
return;
|
|
320
332
|
}
|
|
321
|
-
const existing = await getExistingRelations();
|
|
333
|
+
const existing = await getExistingRelations(destination);
|
|
322
334
|
const touchedPairs = changedPairs
|
|
323
335
|
.filter((pair) => !existing.has(pairKey(pair.entityA, pair.entityB)))
|
|
324
336
|
.filter((pair) => !isAttemptBackedOff(attempts, pairKey(pair.entityA, pair.entityB), now, RELATION_ATTEMPT_BACKOFF_MS));
|
|
325
337
|
const supportLookupLimit = Math.max(maxPairs * 5, maxPairs);
|
|
326
338
|
const relationCandidates = [];
|
|
327
339
|
for (const changed of touchedPairs.slice(0, supportLookupLimit)) {
|
|
328
|
-
const candidate = await buildRelationCandidate(changed);
|
|
340
|
+
const candidate = await buildRelationCandidate(changed, destination);
|
|
329
341
|
if (candidate)
|
|
330
342
|
relationCandidates.push(candidate);
|
|
331
343
|
if (relationCandidates.length >= maxPairs)
|
|
@@ -345,12 +357,12 @@ const tick = async (config) => {
|
|
|
345
357
|
const hash = createHash("sha256")
|
|
346
358
|
.update(`daemon-relation:${pairKey(result.from, result.to)}:${result.type}`)
|
|
347
359
|
.digest("hex");
|
|
348
|
-
const dedup = await qdrant.dedupCheck(result.content, hash);
|
|
360
|
+
const dedup = await qdrant.dedupCheck(result.content, hash, undefined, undefined, { destination });
|
|
349
361
|
if (dedup.action === "skip") {
|
|
350
362
|
logFn("DEBUG", `Relations: skipping duplicate ${candidate.entityA}↔${candidate.entityB}`);
|
|
351
363
|
continue;
|
|
352
364
|
}
|
|
353
|
-
await storeRelation(result.from, result.to, result.type, result.content, candidate, { evidence: result.evidence, confidence: result.confidence, inVocabulary: result.inVocabulary, judgment: result.judgment });
|
|
365
|
+
await storeRelation(result.from, result.to, result.type, result.content, candidate, destination, { evidence: result.evidence, confidence: result.confidence, inVocabulary: result.inVocabulary, judgment: result.judgment });
|
|
354
366
|
inferred++;
|
|
355
367
|
}
|
|
356
368
|
catch (e) {
|
|
@@ -2,6 +2,7 @@ import type { BikkyConfig } from "../config.js";
|
|
|
2
2
|
import type { EpisodeSummaryWriteResult } from "./episode-summary.js";
|
|
3
3
|
import type { QdrantPayload } from "./qdrant.js";
|
|
4
4
|
import { type RedactionSummary } from "../privacy/redaction.js";
|
|
5
|
+
import { type OperationOrigin } from "../provenance/origin.js";
|
|
5
6
|
export interface WorkspaceScope {
|
|
6
7
|
workspaceId?: string;
|
|
7
8
|
actorId?: string;
|
|
@@ -38,6 +39,8 @@ export declare const buildSessionIndexPayload: (input: {
|
|
|
38
39
|
enabled: boolean;
|
|
39
40
|
redactPii: boolean;
|
|
40
41
|
};
|
|
42
|
+
config?: BikkyConfig;
|
|
43
|
+
origin?: OperationOrigin;
|
|
41
44
|
}) => SessionIndexPayloadResult;
|
|
42
45
|
export declare const updateSessionIndex: (input: {
|
|
43
46
|
sessionId: string;
|
|
@@ -45,9 +48,11 @@ export declare const updateSessionIndex: (input: {
|
|
|
45
48
|
episodeResults: EpisodeSummaryWriteResult[];
|
|
46
49
|
scope: WorkspaceScope;
|
|
47
50
|
config: BikkyConfig;
|
|
51
|
+
destination?: string;
|
|
48
52
|
}) => Promise<{
|
|
49
53
|
action: "stored" | "updated" | "skipped";
|
|
50
54
|
factId?: string;
|
|
55
|
+
destination?: string;
|
|
51
56
|
reason?: string;
|
|
52
57
|
}>;
|
|
53
58
|
//# sourceMappingURL=session-index.d.ts.map
|
|
@@ -8,6 +8,7 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
8
8
|
import { CAPTURE_POLICY_VERSION, DEFAULT_CAPTURE_CONTEXT, PROMPT_VERSIONS } from "./capture-policy.js";
|
|
9
9
|
import * as qdrant from "./qdrant.js";
|
|
10
10
|
import { combineRedactions, redactStorageText, } from "../privacy/redaction.js";
|
|
11
|
+
import { buildOperationOrigin } from "../provenance/origin.js";
|
|
11
12
|
const contentHash = (text) => createHash("sha256").update(`session-index:${text}`).digest("hex");
|
|
12
13
|
export const buildSessionIndexDraft = (input) => {
|
|
13
14
|
const episodeIds = input.episodeResults
|
|
@@ -48,6 +49,16 @@ export const buildSessionIndexPayload = (input) => {
|
|
|
48
49
|
const redactedEntities = input.draft.entities.map((entity) => redactStorageText(entity, input.redactionOptions));
|
|
49
50
|
const redaction = combineRedactions([redactedContent, ...redactedEntities]);
|
|
50
51
|
const existingPayload = input.existing?.payload ?? {};
|
|
52
|
+
const operationOrigin = input.origin ?? buildOperationOrigin({
|
|
53
|
+
interface: "daemon",
|
|
54
|
+
action: input.existing ? "update" : "create",
|
|
55
|
+
subsystem: "session_index",
|
|
56
|
+
config: input.config,
|
|
57
|
+
metadata: {
|
|
58
|
+
session_id: input.sessionId,
|
|
59
|
+
event_count: input.eventCount,
|
|
60
|
+
},
|
|
61
|
+
});
|
|
51
62
|
const payload = {
|
|
52
63
|
...existingPayload,
|
|
53
64
|
content: redactedContent.text,
|
|
@@ -57,9 +68,9 @@ export const buildSessionIndexPayload = (input) => {
|
|
|
57
68
|
memory_subtype: "session_index",
|
|
58
69
|
layer: "episode",
|
|
59
70
|
...(input.scope.workspaceId ? { workspace_id: input.scope.workspaceId } : {}),
|
|
60
|
-
|
|
71
|
+
origin: existingPayload.origin ?? operationOrigin,
|
|
72
|
+
...(input.existing ? { last_operation_origin: operationOrigin } : {}),
|
|
61
73
|
entities: redactedEntities.map((entity) => entity.text.toLowerCase()),
|
|
62
|
-
source: "system",
|
|
63
74
|
confidence: 1.0,
|
|
64
75
|
importance: input.draft.importance,
|
|
65
76
|
content_hash: contentHash(redactedContent.text),
|
|
@@ -93,24 +104,38 @@ export const buildSessionIndexPayload = (input) => {
|
|
|
93
104
|
}
|
|
94
105
|
return { payload, redaction };
|
|
95
106
|
};
|
|
96
|
-
const findExistingSessionIndex = async (sessionId, scope) => {
|
|
97
|
-
const
|
|
107
|
+
const findExistingSessionIndex = async (sessionId, scope, destination) => {
|
|
108
|
+
const collection = qdrant.collectionForDestination(destination);
|
|
109
|
+
const result = await qdrant.qdrantRequest("POST", `/collections/${collection}/points/scroll`, {
|
|
98
110
|
filter: buildSessionIndexFilter(sessionId, scope),
|
|
99
111
|
limit: 1,
|
|
100
112
|
with_payload: true,
|
|
101
|
-
});
|
|
113
|
+
}, destination);
|
|
102
114
|
return result.result?.points?.[0] ?? null;
|
|
103
115
|
};
|
|
104
116
|
export const updateSessionIndex = async (input) => {
|
|
105
117
|
if (input.episodeResults.length === 0) {
|
|
106
118
|
return { action: "skipped", reason: "no_episode_results" };
|
|
107
119
|
}
|
|
108
|
-
const existing = await findExistingSessionIndex(input.sessionId, input.scope);
|
|
109
120
|
const draft = buildSessionIndexDraft({
|
|
110
121
|
sessionId: input.sessionId,
|
|
111
122
|
eventCount: input.eventCount,
|
|
112
123
|
episodeResults: input.episodeResults,
|
|
113
124
|
});
|
|
125
|
+
const destination = input.destination
|
|
126
|
+
?? input.episodeResults.find((result) => result.destination)?.destination
|
|
127
|
+
?? qdrant.resolveDestination({
|
|
128
|
+
content: draft.content,
|
|
129
|
+
entities: draft.entities,
|
|
130
|
+
metadata: {
|
|
131
|
+
session_id: input.sessionId,
|
|
132
|
+
memory_subtype: "session_index",
|
|
133
|
+
kind: "summary",
|
|
134
|
+
origin_interface: "daemon",
|
|
135
|
+
origin_agent_type: "daemon",
|
|
136
|
+
},
|
|
137
|
+
}).name;
|
|
138
|
+
const existing = await findExistingSessionIndex(input.sessionId, input.scope, destination);
|
|
114
139
|
const now = new Date().toISOString();
|
|
115
140
|
const { payload } = buildSessionIndexPayload({
|
|
116
141
|
draft,
|
|
@@ -119,6 +144,7 @@ export const updateSessionIndex = async (input) => {
|
|
|
119
144
|
now,
|
|
120
145
|
existing,
|
|
121
146
|
eventCount: input.eventCount,
|
|
147
|
+
config: input.config,
|
|
122
148
|
redactionOptions: {
|
|
123
149
|
enabled: true,
|
|
124
150
|
redactPii: false,
|
|
@@ -126,9 +152,10 @@ export const updateSessionIndex = async (input) => {
|
|
|
126
152
|
});
|
|
127
153
|
const vector = await qdrant.embed(String(payload.content));
|
|
128
154
|
const factId = existing?.id ?? randomUUID();
|
|
129
|
-
|
|
155
|
+
const collection = qdrant.collectionForDestination(destination);
|
|
156
|
+
await qdrant.qdrantRequest("PUT", `/collections/${collection}/points`, {
|
|
130
157
|
points: [{ id: factId, vector, payload }],
|
|
131
|
-
});
|
|
132
|
-
return { action: existing ? "updated" : "stored", factId };
|
|
158
|
+
}, destination);
|
|
159
|
+
return { action: existing ? "updated" : "stored", factId, destination };
|
|
133
160
|
};
|
|
134
161
|
//# sourceMappingURL=session-index.js.map
|