@unblocklabs/unblock-memory 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/analysis.js +36 -25
- package/dist/src/config.js +2 -17
- package/dist/src/contracts.d.ts +1 -1
- package/dist/src/extraction-chunk.d.ts +1 -0
- package/dist/src/extraction-chunk.js +1 -0
- package/dist/src/extraction-model.js +5 -4
- package/dist/src/extraction-runtime.js +2 -7
- package/dist/src/extraction-source.js +6 -2
- package/dist/src/extraction-store.d.ts +3 -0
- package/dist/src/extraction-store.js +14 -10
- package/dist/src/extraction-worker.js +31 -15
- package/dist/src/manager.d.ts +1 -1
- package/dist/src/manager.js +66 -85
- package/dist/src/memory-whisperer.js +6 -10
- package/dist/src/people-primer-config.js +1 -4
- package/dist/src/people-primer.js +6 -8
- package/dist/src/people-store.js +9 -1
- package/dist/src/people-tools.js +2 -6
- package/dist/src/plugin.js +6 -14
- package/dist/src/quality-audit.js +11 -2
- package/dist/src/query-contract.js +18 -10
- package/dist/src/response-audit.js +3 -3
- package/dist/src/response-judge.js +1 -2
- package/dist/src/response-reviews.d.ts +1 -1
- package/dist/src/response-reviews.js +4 -2
- package/dist/src/response-store.js +3 -2
- package/dist/src/retrieval-telemetry.d.ts +1 -1
- package/dist/src/review-tools.d.ts +24 -0
- package/dist/src/review-tools.js +22 -26
- package/dist/src/runtime.js +2 -1
- package/dist/src/session-projector.js +8 -4
- package/dist/src/session-sync.js +2 -1
- package/dist/src/skill-whisperer.js +51 -24
- package/dist/src/training-candidates.d.ts +12 -13
- package/dist/src/training-candidates.js +11 -81
- package/dist/src/training-gate.js +2 -2
- package/dist/src/training-queries.js +5 -2
- package/dist/src/training-retrieval.d.ts +15 -2
- package/dist/src/training-retrieval.js +63 -39
- package/dist/src/training-store.d.ts +2 -8
- package/dist/src/training-store.js +15 -20
- package/dist/src/typesafe-client.d.ts +5 -0
- package/dist/src/typesafe-client.js +4 -0
- package/dist/src/typesafe-review.js +4 -6
- package/dist/src/typesafe.d.ts +1 -9
- package/dist/src/typesafe.js +4 -10
- package/dist/src/whisperer-context.d.ts +5 -1
- package/dist/src/whisperer-context.js +12 -7
- package/dist/src/workspace-path-classifier.js +39 -32
- package/docs/configuration.md +3 -2
- package/docs/memory-training.md +8 -7
- package/docs/retrieval.md +15 -1
- package/openclaw.plugin.json +2 -2
- package/package.json +6 -4
package/dist/src/manager.js
CHANGED
|
@@ -74,16 +74,6 @@ function frontmatterValue(body, key) {
|
|
|
74
74
|
.find((value) => value !== undefined);
|
|
75
75
|
return raw?.replace(/^(?:"(.*)"|'(.*)')$/u, "$1$2").trim();
|
|
76
76
|
}
|
|
77
|
-
function embeddingText(name, description, model) {
|
|
78
|
-
return model.toLowerCase().includes("qwen3-embedding")
|
|
79
|
-
? `${name}\n${description}`
|
|
80
|
-
: `title: ${name} | text: ${description}`;
|
|
81
|
-
}
|
|
82
|
-
function queryText(query, model) {
|
|
83
|
-
return model.toLowerCase().includes("qwen3-embedding")
|
|
84
|
-
? `Instruct: Retrieve relevant documents for the given query\nQuery: ${query}`
|
|
85
|
-
: `task: search result | query: ${query}`;
|
|
86
|
-
}
|
|
87
77
|
function cosineSimilarity(left, right) {
|
|
88
78
|
if (left.length !== right.length || left.length === 0)
|
|
89
79
|
return 0;
|
|
@@ -192,22 +182,6 @@ function lineSpan(body, position, text) {
|
|
|
192
182
|
const endLine = startLine + Math.max(0, text.split("\n").length - 1);
|
|
193
183
|
return { startLine, endLine };
|
|
194
184
|
}
|
|
195
|
-
function lexicalResult(hit, corpus, session) {
|
|
196
|
-
const body = hit.body ?? hit.title;
|
|
197
|
-
const endLine = Math.max(1, body.split("\n").length);
|
|
198
|
-
return {
|
|
199
|
-
path: hit.filepath,
|
|
200
|
-
startLine: 1,
|
|
201
|
-
endLine,
|
|
202
|
-
score: hit.score,
|
|
203
|
-
textScore: hit.score,
|
|
204
|
-
snippet: body,
|
|
205
|
-
source: "memory",
|
|
206
|
-
corpus,
|
|
207
|
-
...(session ? { session } : {}),
|
|
208
|
-
citation: `${hit.displayPath}#L1-L${endLine}`,
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
185
|
function sessionAllowedPaths(metadataByPath, collection, filter) {
|
|
212
186
|
const startedFrom = filter.startedFrom === undefined ? undefined : Date.parse(filter.startedFrom);
|
|
213
187
|
const startedTo = filter.startedTo === undefined ? undefined : Date.parse(filter.startedTo);
|
|
@@ -251,6 +225,7 @@ export class QmdMemoryManager {
|
|
|
251
225
|
#watcher;
|
|
252
226
|
#watchReady;
|
|
253
227
|
#watchTimer;
|
|
228
|
+
#watchCollections = new Set();
|
|
254
229
|
#watchError;
|
|
255
230
|
#closed = false;
|
|
256
231
|
#files = 0;
|
|
@@ -360,12 +335,17 @@ export class QmdMemoryManager {
|
|
|
360
335
|
this.#skillIndex = undefined;
|
|
361
336
|
if (!matchingSources.some((source) => source.kind !== "skills"))
|
|
362
337
|
return;
|
|
338
|
+
for (const source of matchingSources)
|
|
339
|
+
if (source.kind !== "skills")
|
|
340
|
+
this.#watchCollections.add(source.collection);
|
|
363
341
|
this.#dirty = true;
|
|
364
342
|
if (this.#watchTimer)
|
|
365
343
|
clearTimeout(this.#watchTimer);
|
|
366
344
|
this.#watchTimer = setTimeout(() => {
|
|
367
345
|
this.#watchTimer = undefined;
|
|
368
|
-
|
|
346
|
+
const collections = [...this.#watchCollections];
|
|
347
|
+
this.#watchCollections.clear();
|
|
348
|
+
void this.#sync({ reason: "watch" }, collections).catch(() => undefined);
|
|
369
349
|
}, WATCH_DEBOUNCE_MS);
|
|
370
350
|
});
|
|
371
351
|
}
|
|
@@ -489,23 +469,36 @@ export class QmdMemoryManager {
|
|
|
489
469
|
const allowed = new Set(sessionAllowedPaths(new Map(records.map(r => [extractedPath(r), r.metadata])), EXTRACTED_COLLECTION, filter)[EXTRACTED_COLLECTION]);
|
|
490
470
|
return records.filter(r => allowed.has(extractedPath(r)));
|
|
491
471
|
}
|
|
492
|
-
async #updateAndEmbed(store,
|
|
472
|
+
async #updateAndEmbed(store, collections, force) {
|
|
493
473
|
// Extracted facts are DB projections, not files: never run filesystem sync on them.
|
|
494
|
-
const update = await store.update({ collections: collection ? [collection] : this.#qmdSources().map(source => source.collection) });
|
|
495
|
-
this.#cleanupRemovedDocuments?.(update.updated + update.removed);
|
|
496
474
|
const analysisStore = store;
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
475
|
+
let changedDocuments = 0;
|
|
476
|
+
try {
|
|
477
|
+
const update = await store.update({ collections: collections ? [...collections] : this.#qmdSources().map(source => source.collection) });
|
|
478
|
+
changedDocuments = update.updated + update.removed;
|
|
479
|
+
if ((update.indexed + changedDocuments > 0 || update.needsEmbedding > 0 || force) && analysisStore.internal) {
|
|
480
|
+
markMemoryAnalysisStale(analysisStore.internal.db);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
catch (error) {
|
|
484
|
+
// QMD may commit earlier collections before a later collection fails.
|
|
485
|
+
if (analysisStore.internal)
|
|
486
|
+
markMemoryAnalysisStale(analysisStore.internal.db);
|
|
487
|
+
changedDocuments = Math.max(1, changedDocuments);
|
|
488
|
+
throw error;
|
|
489
|
+
}
|
|
490
|
+
finally {
|
|
491
|
+
// A later collection failing must not leave earlier removed plaintext behind.
|
|
492
|
+
this.#cleanupRemovedDocuments?.(changedDocuments);
|
|
493
|
+
}
|
|
494
|
+
let chunksEmbedded = 0;
|
|
495
|
+
for (const collection of collections?.length ? collections : [undefined]) {
|
|
496
|
+
const embed = await store.embed({ ...(collection ? { collection } : {}), force, chunkStrategy: "semantic" });
|
|
497
|
+
this.#recordEmbedding(embed);
|
|
498
|
+
const count = completedEmbeddingCount(embed);
|
|
499
|
+
chunksEmbedded += count;
|
|
500
|
+
if (count > 0 && analysisStore.internal)
|
|
501
|
+
markMemoryAnalysisStale(analysisStore.internal.db);
|
|
509
502
|
}
|
|
510
503
|
return chunksEmbedded;
|
|
511
504
|
}
|
|
@@ -516,16 +509,14 @@ export class QmdMemoryManager {
|
|
|
516
509
|
this.#dirty = status.needsEmbedding > 0;
|
|
517
510
|
}
|
|
518
511
|
sync(params) {
|
|
512
|
+
return this.#sync(params);
|
|
513
|
+
}
|
|
514
|
+
#sync(params, affectedCollections) {
|
|
519
515
|
const run = async () => {
|
|
520
516
|
const store = await this.#getStore();
|
|
521
517
|
this.#dirty = true;
|
|
522
|
-
const collections = this.#qmdSources().filter((source) => source.kind !== "sessions");
|
|
523
|
-
|
|
524
|
-
await this.#updateAndEmbed(store, undefined, params?.force);
|
|
525
|
-
}
|
|
526
|
-
for (const source of collections) {
|
|
527
|
-
await this.#updateAndEmbed(store, source.collection, params?.force);
|
|
528
|
-
}
|
|
518
|
+
const collections = affectedCollections ?? this.#qmdSources().filter((source) => source.kind !== "sessions").map(source => source.collection);
|
|
519
|
+
await this.#updateAndEmbed(store, collections.length ? collections : undefined, params?.force);
|
|
529
520
|
await this.#refreshIndexStatus(store);
|
|
530
521
|
};
|
|
531
522
|
return this.#enqueue(run);
|
|
@@ -544,7 +535,7 @@ export class QmdMemoryManager {
|
|
|
544
535
|
index: async () => {
|
|
545
536
|
onPhase?.("indexing");
|
|
546
537
|
const store = await this.#getStore();
|
|
547
|
-
return this.#updateAndEmbed(store, sessions.collection);
|
|
538
|
+
return this.#updateAndEmbed(store, [sessions.collection]);
|
|
548
539
|
},
|
|
549
540
|
});
|
|
550
541
|
this.#sessionMetadata = sessionMetadataByPath(synced.manifest);
|
|
@@ -699,18 +690,28 @@ export class QmdMemoryManager {
|
|
|
699
690
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
700
691
|
`);
|
|
701
692
|
const curation = this.#getCuration();
|
|
693
|
+
const fingerprints = new Map();
|
|
702
694
|
for (const annotation of curation.annotations()) {
|
|
703
695
|
if (!annotation.contentFingerprint) {
|
|
704
696
|
insert.run(annotation.collection, annotation.path, null, null, annotation.eventTime, annotation.basis, 1);
|
|
705
697
|
continue;
|
|
706
698
|
}
|
|
707
|
-
const
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
699
|
+
const key = JSON.stringify([annotation.collection, annotation.path]);
|
|
700
|
+
let chunks = fingerprints.get(key);
|
|
701
|
+
if (!chunks) {
|
|
702
|
+
chunks = new Map();
|
|
703
|
+
const rows = findChunks.all(annotation.collection, annotation.path);
|
|
704
|
+
for (const row of rows) {
|
|
705
|
+
const fingerprint = chunkFingerprint(row.doc.slice(row.pos, row.pos + row.chunk_len));
|
|
706
|
+
if (!chunks.has(fingerprint))
|
|
707
|
+
chunks.set(fingerprint, row);
|
|
708
|
+
}
|
|
709
|
+
fingerprints.set(key, chunks);
|
|
710
|
+
}
|
|
711
|
+
const matched = chunks.get(annotation.contentFingerprint);
|
|
712
|
+
if (annotation.qmdHash !== (matched?.hash ?? null) || annotation.qmdSeq !== (matched?.seq ?? null)) {
|
|
713
|
+
curation.updateAnnotationLocation({ annotation, qmdHash: matched?.hash ?? null, qmdSeq: matched?.seq ?? null });
|
|
714
|
+
}
|
|
714
715
|
if (matched) {
|
|
715
716
|
insert.run(annotation.collection, annotation.path, matched.hash, matched.seq, annotation.eventTime, annotation.basis, 0);
|
|
716
717
|
}
|
|
@@ -779,7 +780,7 @@ export class QmdMemoryManager {
|
|
|
779
780
|
}
|
|
780
781
|
async search(query, opts) {
|
|
781
782
|
const started = performance.now();
|
|
782
|
-
const operation =
|
|
783
|
+
const operation = "vector";
|
|
783
784
|
let results;
|
|
784
785
|
try {
|
|
785
786
|
results = await this.#search(query, opts);
|
|
@@ -820,25 +821,6 @@ export class QmdMemoryManager {
|
|
|
820
821
|
if (collections.includes(EXTRACTED_COLLECTION)) {
|
|
821
822
|
allowedPaths = { ...allowedPaths, [EXTRACTED_COLLECTION]: currentRecords().map(extractedPath) };
|
|
822
823
|
}
|
|
823
|
-
if (opts?.lexicalOnly) {
|
|
824
|
-
const hits = await store.searchLex(query, {
|
|
825
|
-
limit: opts.maxResults ?? 5,
|
|
826
|
-
collection: collections,
|
|
827
|
-
});
|
|
828
|
-
const current = new Map(currentRecords(hits.map(hit => hit.filepath)).map(r => [`qmd://${EXTRACTED_COLLECTION}/${extractedPath(r)}`, r]));
|
|
829
|
-
return hits.flatMap((hit) => {
|
|
830
|
-
if (hit.collectionName === EXTRACTED_COLLECTION) {
|
|
831
|
-
const record = current.get(hit.filepath);
|
|
832
|
-
return record && hit.score >= (opts.minScore ?? 0) ? [extractedHit(record, hit.score)] : [];
|
|
833
|
-
}
|
|
834
|
-
const corpus = this.#sources.get(hit.collectionName)?.corpus;
|
|
835
|
-
const prefix = `qmd://${hit.collectionName}/`;
|
|
836
|
-
const session = corpus === "sessions" && hit.filepath.startsWith(prefix)
|
|
837
|
-
? this.#sessionMetadata.get(hit.filepath.slice(prefix.length))
|
|
838
|
-
: undefined;
|
|
839
|
-
return hit.score >= (opts.minScore ?? 0) && corpus ? [lexicalResult(hit, corpus, session)] : [];
|
|
840
|
-
});
|
|
841
|
-
}
|
|
842
824
|
const hits = await store.vsearch(query, {
|
|
843
825
|
collection: collections,
|
|
844
826
|
limit: opts?.maxResults ?? 5,
|
|
@@ -944,6 +926,7 @@ export class QmdMemoryManager {
|
|
|
944
926
|
if (!store.internal?.llm)
|
|
945
927
|
throw new Error("Skill Whisperer requires the QMD embedding model");
|
|
946
928
|
const llm = store.internal.llm;
|
|
929
|
+
const { formatDocForEmbedding, formatQueryForEmbedding } = await qmdModule;
|
|
947
930
|
let skillIndex = this.#skillIndex;
|
|
948
931
|
if (!skillIndex) {
|
|
949
932
|
const pending = (async () => {
|
|
@@ -968,7 +951,7 @@ export class QmdMemoryManager {
|
|
|
968
951
|
}
|
|
969
952
|
}
|
|
970
953
|
const skills = [...metadata.values()];
|
|
971
|
-
const embeddings = await llm.embedBatch(skills.map(({ candidate, description }) =>
|
|
954
|
+
const embeddings = await llm.embedBatch(skills.map(({ candidate, description }) => formatDocForEmbedding(description, candidate.name, llm.embedModelName)));
|
|
972
955
|
return skills.flatMap(({ candidate }, index) => {
|
|
973
956
|
const embedding = embeddings[index]?.embedding;
|
|
974
957
|
return embedding ? [{ ...candidate, score: 0, embedding }] : [];
|
|
@@ -981,7 +964,7 @@ export class QmdMemoryManager {
|
|
|
981
964
|
});
|
|
982
965
|
this.#skillIndex = skillIndex;
|
|
983
966
|
}
|
|
984
|
-
const queryEmbedding = await llm.embed(
|
|
967
|
+
const queryEmbedding = await llm.embed(formatQueryForEmbedding(query, llm.embedModelName), { isQuery: true });
|
|
985
968
|
if (!queryEmbedding)
|
|
986
969
|
return [];
|
|
987
970
|
const candidates = await skillIndex;
|
|
@@ -1008,15 +991,12 @@ export class QmdMemoryManager {
|
|
|
1008
991
|
}
|
|
1009
992
|
await this.#operationChain;
|
|
1010
993
|
const store = await this.#getStore();
|
|
1011
|
-
const doc = await store.get(safe.normalized);
|
|
1012
|
-
if ("error" in doc || doc.filepath !== safe.normalized) {
|
|
994
|
+
const doc = await store.get(safe.normalized, { includeBody: true });
|
|
995
|
+
if ("error" in doc || doc.filepath !== safe.normalized || doc.body === undefined) {
|
|
1013
996
|
return { status: "not_found", text: "", path: params.relPath };
|
|
1014
997
|
}
|
|
1015
|
-
const content = await store.getDocumentBody(safe.normalized);
|
|
1016
|
-
if (content === null)
|
|
1017
|
-
return { status: "not_found", text: "", path: params.relPath };
|
|
1018
998
|
return buildReadResult({
|
|
1019
|
-
content,
|
|
999
|
+
content: doc.body,
|
|
1020
1000
|
path: safe.normalized,
|
|
1021
1001
|
from: params.from,
|
|
1022
1002
|
lines: params.lines,
|
|
@@ -1064,6 +1044,7 @@ export class QmdMemoryManager {
|
|
|
1064
1044
|
if (this.#watchTimer)
|
|
1065
1045
|
clearTimeout(this.#watchTimer);
|
|
1066
1046
|
this.#watchTimer = undefined;
|
|
1047
|
+
this.#watchCollections.clear();
|
|
1067
1048
|
await this.#watcher?.close();
|
|
1068
1049
|
this.#watcher = undefined;
|
|
1069
1050
|
this.#watchReady = undefined;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { judgeMemoryPassage, memoryUsefulnessRequest } from "./typesafe.js";
|
|
3
3
|
import { resolveTypeSafeApiKey, TypeSafeRequestError } from "./typesafe-client.js";
|
|
4
4
|
import { complementaryIndices, reviewMemoryRedundancy } from "./typesafe-review.js";
|
|
5
5
|
import { ApiQueryGenerator, QueryApiError, queryConversation } from "./query-generator.js";
|
|
@@ -164,15 +164,11 @@ export function registerMemoryWhisperer(api, runtime, config, typesafe, diagnost
|
|
|
164
164
|
timeoutMs: typesafe.timeoutMs };
|
|
165
165
|
try {
|
|
166
166
|
const { hit, excerpt } = candidate;
|
|
167
|
-
const
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
dates: [...new Set(hit.sessionMessages?.flatMap(message => message.timestamp ? [message.timestamp] : [])
|
|
173
|
-
?? (hit.messageTimestamp ? [hit.messageTimestamp] : []))],
|
|
174
|
-
}],
|
|
175
|
-
});
|
|
167
|
+
const { probability } = await judgeMemoryPassage(memoryUsefulnessRequest(conversation, {
|
|
168
|
+
excerpt, corpus: hit.corpus, sourcePath: hit.path,
|
|
169
|
+
dates: [...new Set(hit.sessionMessages?.flatMap(message => message.timestamp ? [message.timestamp] : [])
|
|
170
|
+
?? (hit.messageTimestamp ? [hit.messageTimestamp] : []))],
|
|
171
|
+
}, asOf), { apiKey, timeoutMs: typesafe.timeoutMs, signal });
|
|
176
172
|
if (signal.aborted)
|
|
177
173
|
return [];
|
|
178
174
|
requestsSucceeded++;
|
|
@@ -10,12 +10,9 @@ export const peoplePrimerSchema = Type.Object({
|
|
|
10
10
|
timeoutMs: Type.Integer({ minimum: 1, maximum: 60000, default: 30000 }),
|
|
11
11
|
}, { additionalProperties: false });
|
|
12
12
|
export function resolvePeoplePrimer(value, corpora, peopleEnabled) {
|
|
13
|
-
if (value === undefined)
|
|
14
|
-
return { enabled: false, corpora: [], hitsPerQuestion: 30,
|
|
15
|
-
minScore: 0.35, minUsefulness: 0.8, maxEvidencePerQuestion: 3, timeoutMs: 30000 };
|
|
16
13
|
let config;
|
|
17
14
|
try {
|
|
18
|
-
const withDefaults = Value.Default(peoplePrimerSchema, value);
|
|
15
|
+
const withDefaults = Value.Default(peoplePrimerSchema, value === undefined ? {} : value);
|
|
19
16
|
if (!Value.Check(peoplePrimerSchema, withDefaults))
|
|
20
17
|
throw new Error("Invalid config");
|
|
21
18
|
config = withDefaults;
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { Type } from "typebox";
|
|
3
3
|
import { Value } from "typebox/value";
|
|
4
|
-
import { requestTypeSafe } from "./typesafe-client.js";
|
|
4
|
+
import { requestTypeSafe, TYPESAFE_NOUL_SCHEMA } from "./typesafe-client.js";
|
|
5
5
|
import { TYPESAFE_REVIEW_MODEL } from "./typesafe-review.js";
|
|
6
6
|
import { abortable } from "./abortable.js";
|
|
7
7
|
const VERSION = "people-primer-background-v4";
|
|
8
8
|
const MAX_EXCERPT_CHARS = 6000;
|
|
9
|
-
const
|
|
10
|
-
const answerSchema = Type.Object({ answers: Type.Record(Type.String(), noul) });
|
|
9
|
+
const answerSchema = Type.Object({ answers: Type.Record(Type.String(), TYPESAFE_NOUL_SCHEMA) });
|
|
11
10
|
function questionsFor(name, agent) {
|
|
12
11
|
return [
|
|
13
12
|
{ id: "role", question: `Who is ${name}? What is their explicitly stated role and organization?` },
|
|
@@ -115,15 +114,14 @@ export async function primePersonDossier(params) {
|
|
|
115
114
|
requests++;
|
|
116
115
|
payload = await requestTypeSafe({ apiKey: params.apiKey, timeoutMs: config.timeoutMs, signal }, state, questions);
|
|
117
116
|
signal.throwIfAborted();
|
|
118
|
-
if (!valid(payload)
|
|
117
|
+
if (!valid(payload))
|
|
119
118
|
throw new Error("Invalid primer judgments");
|
|
120
119
|
// Keep only validated numerical answers, never provider extras or echoes.
|
|
121
120
|
const answers = payload.answers;
|
|
122
|
-
|
|
123
|
-
|
|
121
|
+
store.cachePrimerJudgment(person.id, key, {
|
|
122
|
+
answers: Object.fromEntries(expectedKeys.map(k => [k, { type: "noul", noul: answers[k].noul }])),
|
|
123
|
+
});
|
|
124
124
|
}
|
|
125
|
-
if (!Value.Check(answerSchema, payload))
|
|
126
|
-
throw new Error("Invalid primer cache");
|
|
127
125
|
graded.push({ ...candidate, aboutPerson: payload.answers.aboutPerson.noul,
|
|
128
126
|
explicitBackground: payload.answers.explicitBackground.noul, enduring: payload.answers.enduring.noul,
|
|
129
127
|
recognition: payload.answers.recognition.noul,
|
package/dist/src/people-store.js
CHANGED
|
@@ -147,12 +147,15 @@ export class PeopleStore {
|
|
|
147
147
|
#db;
|
|
148
148
|
#maxOpenTodos;
|
|
149
149
|
#maxBlurbChars;
|
|
150
|
+
#primerCacheReady = false;
|
|
150
151
|
constructor(path, options) {
|
|
151
152
|
this.#db = openMemoryDatabase(path);
|
|
152
153
|
this.#maxOpenTodos = options.maxOpenTodos;
|
|
153
154
|
this.#maxBlurbChars = options.maxBlurbChars;
|
|
154
155
|
try {
|
|
155
156
|
this.#migrate();
|
|
157
|
+
this.#db.exec(`CREATE INDEX IF NOT EXISTS person_identities_person
|
|
158
|
+
ON person_identities(person_id, provider, account_scope, external_id)`);
|
|
156
159
|
}
|
|
157
160
|
catch (error) {
|
|
158
161
|
this.#db.close();
|
|
@@ -165,12 +168,17 @@ export class PeopleStore {
|
|
|
165
168
|
// Derived, bounded cache: no source text or credentials. Kept outside the
|
|
166
169
|
// authoritative dossier schema so older plugin versions can still open it.
|
|
167
170
|
#ensurePrimerCache() {
|
|
171
|
+
if (this.#primerCacheReady)
|
|
172
|
+
return;
|
|
168
173
|
this.#db.exec(`CREATE TABLE IF NOT EXISTS person_primer_judgments (
|
|
169
174
|
cache_key TEXT PRIMARY KEY,
|
|
170
175
|
person_id TEXT NOT NULL REFERENCES people(id) ON DELETE CASCADE,
|
|
171
176
|
judgment_json TEXT NOT NULL,
|
|
172
177
|
created_at TEXT NOT NULL
|
|
173
|
-
) STRICT
|
|
178
|
+
) STRICT;
|
|
179
|
+
CREATE INDEX IF NOT EXISTS person_primer_judgments_created
|
|
180
|
+
ON person_primer_judgments(created_at DESC, cache_key)`);
|
|
181
|
+
this.#primerCacheReady = true;
|
|
174
182
|
}
|
|
175
183
|
getPrimerJudgment(key) {
|
|
176
184
|
this.#ensurePrimerCache();
|
package/dist/src/people-tools.js
CHANGED
|
@@ -112,10 +112,6 @@ const syncParameters = Type.Object({
|
|
|
112
112
|
accountId: Type.String({ pattern: "\\S", maxLength: 200 }),
|
|
113
113
|
limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 200 })),
|
|
114
114
|
}, { additionalProperties: false });
|
|
115
|
-
function context(ctx) {
|
|
116
|
-
const cfg = ctx.getRuntimeConfig?.() ?? ctx.runtimeConfig ?? ctx.config;
|
|
117
|
-
return cfg && ctx.agentId ? { agentId: ctx.agentId } : undefined;
|
|
118
|
-
}
|
|
119
115
|
function personView(stores, agentId, selector, maxChars) {
|
|
120
116
|
const store = stores.get(agentId);
|
|
121
117
|
const person = "personId" in selector
|
|
@@ -138,7 +134,7 @@ function personView(stores, agentId, selector, maxChars) {
|
|
|
138
134
|
};
|
|
139
135
|
}
|
|
140
136
|
function createInspectTool(stores, config, ctx) {
|
|
141
|
-
const active =
|
|
137
|
+
const active = getContext(ctx);
|
|
142
138
|
if (!active)
|
|
143
139
|
return null;
|
|
144
140
|
return {
|
|
@@ -265,7 +261,7 @@ function createUpdateTool(stores, config, runtime, ctx) {
|
|
|
265
261
|
};
|
|
266
262
|
}
|
|
267
263
|
function createSyncTool(stores, reader, ctx) {
|
|
268
|
-
const active =
|
|
264
|
+
const active = getContext(ctx);
|
|
269
265
|
if (!active)
|
|
270
266
|
return null;
|
|
271
267
|
return {
|
package/dist/src/plugin.js
CHANGED
|
@@ -2,7 +2,6 @@ import { Type } from "typebox";
|
|
|
2
2
|
import { Value } from "typebox/value";
|
|
3
3
|
import { jsonResult } from "openclaw/plugin-sdk/agent-runtime";
|
|
4
4
|
import { resolveConfig } from "./config.js";
|
|
5
|
-
import { resolveTypeSafeApiKey } from "./typesafe-client.js";
|
|
6
5
|
import { registerPeopleHooks } from "./people-hooks.js";
|
|
7
6
|
import { PeopleStores } from "./people-store.js";
|
|
8
7
|
import { registerPeopleTools } from "./people-tools.js";
|
|
@@ -13,7 +12,7 @@ import { registerMemoryWhisperer } from "./memory-whisperer.js";
|
|
|
13
12
|
import { registerWhispererPrompt } from "./whisperer-prompt.js";
|
|
14
13
|
import { getContext } from "./tool-context.js";
|
|
15
14
|
import { WhispererDiagnostics } from "./diagnostics.js";
|
|
16
|
-
import { registerReviewTools } from "./review-tools.js";
|
|
15
|
+
import { registerReviewTools, reviewRequestContext } from "./review-tools.js";
|
|
17
16
|
import { registerResponseAudit } from "./response-runtime.js";
|
|
18
17
|
import { registerInsideOut } from "./inside-out-runtime.js";
|
|
19
18
|
import { registerMemoryTraining } from "./training-runtime.js";
|
|
@@ -263,20 +262,13 @@ function createAuditQualityTool(runtime, ctx, config) {
|
|
|
263
262
|
const options = Value.Parse(auditQualityParameters, params);
|
|
264
263
|
if (!config.qualityAudit.enabled || !config.typesafe.enabled)
|
|
265
264
|
return jsonResult({ status: "disabled" });
|
|
266
|
-
const deadline = AbortSignal.timeout(30_000);
|
|
267
|
-
const combined = signal ? AbortSignal.any([signal, deadline]) : deadline;
|
|
268
265
|
try {
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
combined.throwIfAborted();
|
|
274
|
-
const { manager } = await runtime.getMemorySearchManager(active);
|
|
275
|
-
if (!manager)
|
|
276
|
-
return jsonResult({ status: "unavailable", reason: "Memory manager unavailable" });
|
|
277
|
-
return jsonResult(await manager.auditQuality({
|
|
266
|
+
const context = await reviewRequestContext(runtime, active, config.typesafe, signal);
|
|
267
|
+
if (context.status !== "ready")
|
|
268
|
+
return jsonResult(context);
|
|
269
|
+
return jsonResult(await context.manager.auditQuality({
|
|
278
270
|
...options, corpora: config.qualityAudit.corpora, minNoise: config.qualityAudit.minNoise,
|
|
279
|
-
apiKey, timeoutMs:
|
|
271
|
+
apiKey: context.apiKey, timeoutMs: context.timeoutMs, signal: context.signal,
|
|
280
272
|
}));
|
|
281
273
|
}
|
|
282
274
|
catch {
|
|
@@ -50,7 +50,7 @@ export async function auditQualityPage(params) {
|
|
|
50
50
|
return result("ok", true);
|
|
51
51
|
const limit = Math.max(1, Math.min(20, Math.floor(params.limit ?? 10)));
|
|
52
52
|
const rows = db.prepare(`SELECT d.id AS document_id, cv.seq, d.collection, d.path,
|
|
53
|
-
d.hash, cv.pos, cv.chunk_len
|
|
53
|
+
d.hash, cv.pos, cv.chunk_len
|
|
54
54
|
FROM documents d JOIN content c ON c.hash = d.hash JOIN content_vectors cv ON cv.hash = d.hash
|
|
55
55
|
WHERE d.active = 1 AND d.collection IN (${[...sources].map(() => "?").join(",")})
|
|
56
56
|
AND (d.id > ? OR (d.id = ? AND cv.seq > ?))
|
|
@@ -59,9 +59,18 @@ export async function auditQualityPage(params) {
|
|
|
59
59
|
WHERE d.id = ? AND d.active = 1 AND d.collection = ? AND d.path = ? AND d.hash = ?
|
|
60
60
|
AND cv.seq = ? AND cv.pos = ? AND cv.chunk_len = ?`);
|
|
61
61
|
const page = rows.slice(0, limit);
|
|
62
|
+
const bodies = new Map();
|
|
63
|
+
const readBody = db.prepare("SELECT doc FROM content WHERE hash = ?");
|
|
62
64
|
try {
|
|
63
65
|
check();
|
|
64
|
-
const batch = page.map(
|
|
66
|
+
const batch = page.map(metadata => {
|
|
67
|
+
let doc = bodies.get(metadata.hash);
|
|
68
|
+
if (doc === undefined) {
|
|
69
|
+
// Content is hash-addressed; this capture does not yield before inference.
|
|
70
|
+
doc = readBody.get(metadata.hash).doc;
|
|
71
|
+
bodies.set(metadata.hash, doc);
|
|
72
|
+
}
|
|
73
|
+
const row = { ...metadata, doc };
|
|
65
74
|
const source = sources.get(row.collection);
|
|
66
75
|
const text = row.doc.slice(row.pos, row.pos + row.chunk_len);
|
|
67
76
|
const fingerprint = chunkFingerprint(text);
|
|
@@ -44,15 +44,23 @@ export function prepareQueryConversation(history, currentRequest) {
|
|
|
44
44
|
conversation.history.unshift(message);
|
|
45
45
|
bytes += added;
|
|
46
46
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
47
|
+
const fits = (start) => queryTokenIds(serializeQueryConversation({
|
|
48
|
+
history: conversation.history.slice(start), currentRequest,
|
|
49
|
+
})).length <= contract.conversationTokens;
|
|
50
|
+
if (fits(0))
|
|
51
|
+
return { conversation, contextLimited: conversation.history.length < history.length };
|
|
52
|
+
if (!fits(conversation.history.length))
|
|
53
|
+
throw new QueryInputBudgetError();
|
|
54
|
+
// Search whole-message suffixes with the exact serialized tokenizer contract;
|
|
55
|
+
// never approximate the budget by adding independently tokenized messages.
|
|
56
|
+
let tooLarge = 0, fitting = conversation.history.length;
|
|
57
|
+
while (fitting - tooLarge > 1) {
|
|
58
|
+
const middle = Math.floor((tooLarge + fitting) / 2);
|
|
59
|
+
if (fits(middle))
|
|
60
|
+
fitting = middle;
|
|
61
|
+
else
|
|
62
|
+
tooLarge = middle;
|
|
57
63
|
}
|
|
64
|
+
conversation.history = conversation.history.slice(fitting);
|
|
65
|
+
return { conversation, contextLimited: true };
|
|
58
66
|
}
|
|
@@ -142,11 +142,11 @@ export async function auditResponses(options) {
|
|
|
142
142
|
}
|
|
143
143
|
}
|
|
144
144
|
signal.throwIfAborted();
|
|
145
|
-
const fresh = reader.read(session, config.responseAudit);
|
|
146
|
-
if (!fresh
|
|
145
|
+
const fresh = reader.read(session, config.responseAudit, snapshot.revision);
|
|
146
|
+
if (!fresh || !("unchanged" in fresh) && !fresh.episodes.some(candidate => candidate.id === e.id && candidate.inputHash === e.inputHash)) {
|
|
147
147
|
coverage.stale++;
|
|
148
148
|
store.stale(cohort, e);
|
|
149
|
-
if (fresh !== undefined) {
|
|
149
|
+
if (fresh !== undefined && (!fresh || !("unchanged" in fresh))) {
|
|
150
150
|
store.observe(cohort, session, fresh?.episodes ?? []);
|
|
151
151
|
store.checkpointForget(cohort, session);
|
|
152
152
|
}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
2
|
import { Value } from "typebox/value";
|
|
3
|
-
import { requestTypeSafe } from "./typesafe-client.js";
|
|
3
|
+
import { requestTypeSafe, TYPESAFE_NOUL_SCHEMA as noul } from "./typesafe-client.js";
|
|
4
4
|
import { TYPESAFE_REVIEW_MODEL } from "./typesafe-review.js";
|
|
5
5
|
export const RESPONSE_RUBRIC_VERSION = `${TYPESAFE_REVIEW_MODEL}:response-v10`;
|
|
6
6
|
export const RESPONSE_STAGE_VERSIONS = { quality: "quality-v9", feedback: "feedback-v9", sentiment: "sentiment-v10", retrospective: "retrospective-v9", memory: "memory-v2-isolated" };
|
|
7
7
|
const probability = Type.Number({ minimum: 0, maximum: 1 });
|
|
8
|
-
const noul = Type.Object({ type: Type.Literal("noul"), noul: probability });
|
|
9
8
|
const score = Type.Object({ type: Type.Literal("score"), score: Type.Number({ minimum: 0, maximum: 3 }),
|
|
10
9
|
confidence: probability, probabilities: Type.Object({ "0": probability, "1": probability, "2": probability, "3": probability }) });
|
|
11
10
|
function choiceSchema(criteria) {
|
|
@@ -8,7 +8,7 @@ export declare class ResponseReviews {
|
|
|
8
8
|
constructor(db: DatabaseSync);
|
|
9
9
|
sync(cohort: string, episodeId: string, inputHash: string, result: ResponseResult, now: number): void;
|
|
10
10
|
refresh(cohort: string, since: number): number;
|
|
11
|
-
reconcile(cohort: string): void;
|
|
11
|
+
reconcile(cohort: string, sessionId: string): void;
|
|
12
12
|
list(cohort: string, id?: string): {
|
|
13
13
|
id: string;
|
|
14
14
|
episodeId: string;
|
|
@@ -68,11 +68,13 @@ export class ResponseReviews {
|
|
|
68
68
|
throw error;
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
-
reconcile(cohort) {
|
|
71
|
+
reconcile(cohort, sessionId) {
|
|
72
72
|
this.db.prepare(`UPDATE response_review_evidence AS e SET active=CASE WHEN EXISTS (
|
|
73
73
|
SELECT 1 FROM response_review_tasks t JOIN response_results r ON r.id=t.episode_id
|
|
74
74
|
WHERE t.id=e.task_id AND r.cohort=e.cohort AND r.input_hash=e.input_hash AND r.active=1 AND r.status='ok')
|
|
75
|
-
THEN 1 ELSE 0 END WHERE cohort=? AND active>=0
|
|
75
|
+
THEN 1 ELSE 0 END WHERE cohort=? AND active>=0 AND task_id IN (
|
|
76
|
+
SELECT t.id FROM response_results r JOIN response_review_tasks t ON t.episode_id=r.id
|
|
77
|
+
WHERE r.cohort=? AND r.session_id=?)`).run(cohort, cohort, sessionId);
|
|
76
78
|
}
|
|
77
79
|
list(cohort, id) {
|
|
78
80
|
return this.db.prepare(`SELECT t.*,e.input_hash,e.active,e.detail FROM response_review_tasks t
|
|
@@ -22,7 +22,8 @@ export class ResponseAuditStore {
|
|
|
22
22
|
attempts INTEGER NOT NULL DEFAULT 0, attempted_at INTEGER, assessed_at INTEGER, result TEXT,
|
|
23
23
|
PRIMARY KEY(cohort,id));
|
|
24
24
|
CREATE TABLE IF NOT EXISTS response_scans (cohort TEXT PRIMARY KEY, observed_at INTEGER, coverage TEXT);
|
|
25
|
-
CREATE INDEX IF NOT EXISTS response_results_time ON response_results(cohort,episode_at)
|
|
25
|
+
CREATE INDEX IF NOT EXISTS response_results_time ON response_results(cohort,episode_at);
|
|
26
|
+
CREATE INDEX IF NOT EXISTS response_results_session ON response_results(cohort,session_id,episode_at);`);
|
|
26
27
|
this.#db.exec(`CREATE TABLE IF NOT EXISTS response_checkpoints (
|
|
27
28
|
cohort TEXT NOT NULL, session_id TEXT NOT NULL, revision TEXT NOT NULL, coverage TEXT NOT NULL,
|
|
28
29
|
PRIMARY KEY(cohort,session_id));
|
|
@@ -86,7 +87,7 @@ export class ResponseAuditStore {
|
|
|
86
87
|
result=CASE WHEN input_hash=excluded.input_hash THEN result ELSE NULL END`);
|
|
87
88
|
for (const e of episodes)
|
|
88
89
|
insert.run(cohort, e.id, sessionId, e.inputHash, e.timestamp);
|
|
89
|
-
this.reviews.reconcile(cohort);
|
|
90
|
+
this.reviews.reconcile(cohort, sessionId);
|
|
90
91
|
this.#db.exec("COMMIT");
|
|
91
92
|
}
|
|
92
93
|
catch (error) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
type RetrievalOperation = "vector" | "
|
|
1
|
+
type RetrievalOperation = "vector" | "memoryWhisperer";
|
|
2
2
|
type RetrievalOutcome = "ok" | "empty" | "failed" | "cancelled" | "timed_out" | "skipped";
|
|
3
3
|
declare const fields: readonly ["elapsedMs", "retrievalMs", "judgeMs", "gateMs", "generationMs", "candidates", "eligible", "results", "contextChars"];
|
|
4
4
|
type Field = typeof fields[number];
|
|
@@ -2,4 +2,28 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
|
|
2
2
|
import type { UnblockMemoryConfig } from "./config.js";
|
|
3
3
|
import type { QmdMemoryRuntime } from "./runtime.js";
|
|
4
4
|
import type { WhispererDiagnostics } from "./diagnostics.js";
|
|
5
|
+
import { getContext } from "./tool-context.js";
|
|
6
|
+
/** Bound initialization as well as inference, without cancelling shared managers. */
|
|
7
|
+
export declare function reviewRequestContext(runtime: QmdMemoryRuntime, active: NonNullable<ReturnType<typeof getContext>>, typesafe: UnblockMemoryConfig["typesafe"], signal?: AbortSignal): Promise<{
|
|
8
|
+
readonly status: "unavailable";
|
|
9
|
+
readonly reason: "TypeSafe API key not configured";
|
|
10
|
+
readonly manager?: undefined;
|
|
11
|
+
readonly apiKey?: undefined;
|
|
12
|
+
readonly signal?: undefined;
|
|
13
|
+
readonly timeoutMs?: undefined;
|
|
14
|
+
} | {
|
|
15
|
+
readonly status: "unavailable";
|
|
16
|
+
readonly reason: "Memory manager unavailable";
|
|
17
|
+
readonly manager?: undefined;
|
|
18
|
+
readonly apiKey?: undefined;
|
|
19
|
+
readonly signal?: undefined;
|
|
20
|
+
readonly timeoutMs?: undefined;
|
|
21
|
+
} | {
|
|
22
|
+
readonly status: "ready";
|
|
23
|
+
readonly manager: import("./manager.js").QmdMemoryManager;
|
|
24
|
+
readonly apiKey: string;
|
|
25
|
+
readonly signal: AbortSignal;
|
|
26
|
+
readonly timeoutMs: number;
|
|
27
|
+
reason?: undefined;
|
|
28
|
+
}>;
|
|
5
29
|
export declare function registerReviewTools(api: OpenClawPluginApi, runtime: QmdMemoryRuntime, config: UnblockMemoryConfig, diagnostics: WhispererDiagnostics): void;
|