@unblocklabs/unblock-memory 0.7.2 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/analysis.js +25 -13
- package/dist/src/config.js +2 -17
- package/dist/src/manager.d.ts +1 -1
- package/dist/src/manager.js +3 -6
- package/dist/src/people-primer-config.js +1 -4
- package/dist/src/people-store.js +7 -1
- package/dist/src/quality-audit.js +11 -2
- package/dist/src/response-reviews.d.ts +1 -1
- package/dist/src/response-reviews.js +4 -2
- package/dist/src/response-store.js +1 -1
- package/dist/src/runtime.js +2 -1
- package/dist/src/session-sync.js +2 -1
- package/dist/src/skill-whisperer.js +8 -10
- package/dist/src/training-store.d.ts +2 -8
- package/dist/src/training-store.js +15 -20
- package/dist/src/whisperer-context.d.ts +5 -1
- package/dist/src/whisperer-context.js +12 -7
- package/dist/src/workspace-path-classifier.js +39 -32
- package/docs/memory-training.md +8 -7
- package/openclaw.plugin.json +1 -1
- package/package.json +6 -4
package/dist/src/analysis.js
CHANGED
|
@@ -260,8 +260,8 @@ function byteSlice(text, maxBytes) {
|
|
|
260
260
|
return "";
|
|
261
261
|
return bytes.subarray(0, maxBytes - 3).toString("utf8").replace(/\uFFFD$/u, "") + "…";
|
|
262
262
|
}
|
|
263
|
-
function
|
|
264
|
-
|
|
263
|
+
function representativeOrder(clusterId) {
|
|
264
|
+
return clusterId === -1
|
|
265
265
|
? "m.outlier_score DESC, m.hash, m.seq"
|
|
266
266
|
: `CASE WHEN m.representative_rank IS NULL THEN 1 ELSE 0 END,
|
|
267
267
|
m.representative_rank,
|
|
@@ -269,9 +269,11 @@ function members(db, runId, clusterId, limit, offset = 0, sort = "representative
|
|
|
269
269
|
m.outlier_score,
|
|
270
270
|
m.hash,
|
|
271
271
|
m.seq`;
|
|
272
|
+
}
|
|
273
|
+
function members(db, runId, clusterId, limit, offset = 0, sort = "representative", maxExcerptBytes = MAX_EXCERPT_BYTES, maxTotalBytes = MAX_TOTAL_EXCERPT_BYTES, maxTotalAliases = MAX_TOTAL_ALIASES, temporal = {}) {
|
|
272
274
|
const score = clusterId === -1 ? "m.outlier_score" : "m.probability";
|
|
273
275
|
const order = {
|
|
274
|
-
representative: representativeOrder,
|
|
276
|
+
representative: representativeOrder(clusterId),
|
|
275
277
|
score_desc: `${score} DESC, m.hash, m.seq`,
|
|
276
278
|
score_asc: `${score} ASC, m.hash, m.seq`,
|
|
277
279
|
date_desc: "julianday(COALESCE(m.event_time, m.source_modified_at)) DESC, m.hash, m.seq",
|
|
@@ -395,24 +397,34 @@ function readMetadata(run) {
|
|
|
395
397
|
}
|
|
396
398
|
return { stale: false, staleSince: null, analyzedAt: run.completed_at };
|
|
397
399
|
}
|
|
400
|
+
function clusterPreview(db, runId, clusterId, maxBytes, aliasLimit) {
|
|
401
|
+
const row = db.prepare(`
|
|
402
|
+
SELECT m.hash, m.seq, m.probability, m.pos, m.chunk_len, m.doc
|
|
403
|
+
FROM memory_analysis_available_memberships m
|
|
404
|
+
WHERE m.run_id = ? AND m.cluster_id = ?
|
|
405
|
+
ORDER BY ${representativeOrder(clusterId)}
|
|
406
|
+
LIMIT 1
|
|
407
|
+
`).get(runId, clusterId);
|
|
408
|
+
if (!row)
|
|
409
|
+
return undefined;
|
|
410
|
+
return {
|
|
411
|
+
hash: row.hash,
|
|
412
|
+
seq: row.seq,
|
|
413
|
+
probability: row.probability,
|
|
414
|
+
text: byteSlice(row.doc.slice(row.pos, row.pos + row.chunk_len), maxBytes),
|
|
415
|
+
sourcePaths: sourcePaths(db, row.hash, Math.min(MAX_ALIASES_PER_MEMBER, aliasLimit)),
|
|
416
|
+
};
|
|
417
|
+
}
|
|
398
418
|
function toSummary(db, run, row, includePreview, previewBytes = 600, aliasLimit = MAX_TOTAL_ALIASES) {
|
|
399
419
|
const preview = includePreview
|
|
400
|
-
?
|
|
420
|
+
? clusterPreview(db, run.id, row.cluster_id, previewBytes, aliasLimit)
|
|
401
421
|
: undefined;
|
|
402
422
|
return {
|
|
403
423
|
clusterId: clusterReference(run.id, row.cluster_id),
|
|
404
424
|
size: row.size,
|
|
405
425
|
availableSize: row.available_size,
|
|
406
426
|
meanProbability: row.mean_probability,
|
|
407
|
-
...(preview ? {
|
|
408
|
-
preview: {
|
|
409
|
-
hash: preview.hash,
|
|
410
|
-
seq: preview.seq,
|
|
411
|
-
probability: preview.probability,
|
|
412
|
-
text: preview.text,
|
|
413
|
-
sourcePaths: preview.sourcePaths,
|
|
414
|
-
},
|
|
415
|
-
} : {}),
|
|
427
|
+
...(preview ? { preview } : {}),
|
|
416
428
|
};
|
|
417
429
|
}
|
|
418
430
|
function noiseRow(db, runId) {
|
package/dist/src/config.js
CHANGED
|
@@ -291,23 +291,8 @@ function resolvePeople(value) {
|
|
|
291
291
|
};
|
|
292
292
|
}
|
|
293
293
|
export function resolveConfig(value) {
|
|
294
|
-
if (value === undefined || value === null)
|
|
295
|
-
|
|
296
|
-
corpora: DEFAULT_CORPORA,
|
|
297
|
-
extraction: resolveExtraction(undefined, DEFAULT_CORPORA),
|
|
298
|
-
keepEmbeddingModelWarm: true,
|
|
299
|
-
analysis: {},
|
|
300
|
-
typesafe: { ...DEFAULT_TYPESAFE_CONFIG },
|
|
301
|
-
qualityAudit: { ...DEFAULT_QUALITY_AUDIT },
|
|
302
|
-
evidenceReview: { enabled: false, corpora: [] },
|
|
303
|
-
responseAudit: resolveResponseAudit(undefined, DEFAULT_CORPORA),
|
|
304
|
-
insideOut: resolveInsideOut(undefined),
|
|
305
|
-
peoplePrimer: resolvePeoplePrimer(undefined, DEFAULT_CORPORA, false),
|
|
306
|
-
people: DEFAULT_PEOPLE_CONFIG,
|
|
307
|
-
skillWhisperer: DEFAULT_SKILL_WHISPERER,
|
|
308
|
-
memoryWhisperer: { ...DEFAULT_MEMORY_WHISPERER, api: resolveQueryApi(undefined) },
|
|
309
|
-
};
|
|
310
|
-
}
|
|
294
|
+
if (value === undefined || value === null)
|
|
295
|
+
value = {};
|
|
311
296
|
if (typeof value !== "object" || Array.isArray(value)) {
|
|
312
297
|
throw new Error("unblock-memory config must be an object");
|
|
313
298
|
}
|
package/dist/src/manager.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ import { reviewIndexedClaim } from "./evidence-review.js";
|
|
|
12
12
|
import { reviewClusterIngestion } from "./cluster-review.js";
|
|
13
13
|
import type { QueryPair } from "./query-contract.js";
|
|
14
14
|
export { expandSessionSearchHit } from "./memory-passage.js";
|
|
15
|
-
export type ManagerStore = Pick<QMDStore, "update" | "embed" | "getStatus" | "listCollections" | "vsearch" | "get" | "
|
|
15
|
+
export type ManagerStore = Pick<QMDStore, "update" | "embed" | "getStatus" | "listCollections" | "vsearch" | "get" | "close">;
|
|
16
16
|
export type ManagerSessionConfig = {
|
|
17
17
|
agentId: string;
|
|
18
18
|
agentName: string;
|
package/dist/src/manager.js
CHANGED
|
@@ -991,15 +991,12 @@ export class QmdMemoryManager {
|
|
|
991
991
|
}
|
|
992
992
|
await this.#operationChain;
|
|
993
993
|
const store = await this.#getStore();
|
|
994
|
-
const doc = await store.get(safe.normalized);
|
|
995
|
-
if ("error" in doc || doc.filepath !== safe.normalized) {
|
|
994
|
+
const doc = await store.get(safe.normalized, { includeBody: true });
|
|
995
|
+
if ("error" in doc || doc.filepath !== safe.normalized || doc.body === undefined) {
|
|
996
996
|
return { status: "not_found", text: "", path: params.relPath };
|
|
997
997
|
}
|
|
998
|
-
const content = await store.getDocumentBody(safe.normalized);
|
|
999
|
-
if (content === null)
|
|
1000
|
-
return { status: "not_found", text: "", path: params.relPath };
|
|
1001
998
|
return buildReadResult({
|
|
1002
|
-
content,
|
|
999
|
+
content: doc.body,
|
|
1003
1000
|
path: safe.normalized,
|
|
1004
1001
|
from: params.from,
|
|
1005
1002
|
lines: params.lines,
|
|
@@ -10,12 +10,9 @@ export const peoplePrimerSchema = Type.Object({
|
|
|
10
10
|
timeoutMs: Type.Integer({ minimum: 1, maximum: 60000, default: 30000 }),
|
|
11
11
|
}, { additionalProperties: false });
|
|
12
12
|
export function resolvePeoplePrimer(value, corpora, peopleEnabled) {
|
|
13
|
-
if (value === undefined)
|
|
14
|
-
return { enabled: false, corpora: [], hitsPerQuestion: 30,
|
|
15
|
-
minScore: 0.35, minUsefulness: 0.8, maxEvidencePerQuestion: 3, timeoutMs: 30000 };
|
|
16
13
|
let config;
|
|
17
14
|
try {
|
|
18
|
-
const withDefaults = Value.Default(peoplePrimerSchema, value);
|
|
15
|
+
const withDefaults = Value.Default(peoplePrimerSchema, value === undefined ? {} : value);
|
|
19
16
|
if (!Value.Check(peoplePrimerSchema, withDefaults))
|
|
20
17
|
throw new Error("Invalid config");
|
|
21
18
|
config = withDefaults;
|
package/dist/src/people-store.js
CHANGED
|
@@ -147,6 +147,7 @@ export class PeopleStore {
|
|
|
147
147
|
#db;
|
|
148
148
|
#maxOpenTodos;
|
|
149
149
|
#maxBlurbChars;
|
|
150
|
+
#primerCacheReady = false;
|
|
150
151
|
constructor(path, options) {
|
|
151
152
|
this.#db = openMemoryDatabase(path);
|
|
152
153
|
this.#maxOpenTodos = options.maxOpenTodos;
|
|
@@ -167,12 +168,17 @@ export class PeopleStore {
|
|
|
167
168
|
// Derived, bounded cache: no source text or credentials. Kept outside the
|
|
168
169
|
// authoritative dossier schema so older plugin versions can still open it.
|
|
169
170
|
#ensurePrimerCache() {
|
|
171
|
+
if (this.#primerCacheReady)
|
|
172
|
+
return;
|
|
170
173
|
this.#db.exec(`CREATE TABLE IF NOT EXISTS person_primer_judgments (
|
|
171
174
|
cache_key TEXT PRIMARY KEY,
|
|
172
175
|
person_id TEXT NOT NULL REFERENCES people(id) ON DELETE CASCADE,
|
|
173
176
|
judgment_json TEXT NOT NULL,
|
|
174
177
|
created_at TEXT NOT NULL
|
|
175
|
-
) STRICT
|
|
178
|
+
) STRICT;
|
|
179
|
+
CREATE INDEX IF NOT EXISTS person_primer_judgments_created
|
|
180
|
+
ON person_primer_judgments(created_at DESC, cache_key)`);
|
|
181
|
+
this.#primerCacheReady = true;
|
|
176
182
|
}
|
|
177
183
|
getPrimerJudgment(key) {
|
|
178
184
|
this.#ensurePrimerCache();
|
|
@@ -50,7 +50,7 @@ export async function auditQualityPage(params) {
|
|
|
50
50
|
return result("ok", true);
|
|
51
51
|
const limit = Math.max(1, Math.min(20, Math.floor(params.limit ?? 10)));
|
|
52
52
|
const rows = db.prepare(`SELECT d.id AS document_id, cv.seq, d.collection, d.path,
|
|
53
|
-
d.hash, cv.pos, cv.chunk_len
|
|
53
|
+
d.hash, cv.pos, cv.chunk_len
|
|
54
54
|
FROM documents d JOIN content c ON c.hash = d.hash JOIN content_vectors cv ON cv.hash = d.hash
|
|
55
55
|
WHERE d.active = 1 AND d.collection IN (${[...sources].map(() => "?").join(",")})
|
|
56
56
|
AND (d.id > ? OR (d.id = ? AND cv.seq > ?))
|
|
@@ -59,9 +59,18 @@ export async function auditQualityPage(params) {
|
|
|
59
59
|
WHERE d.id = ? AND d.active = 1 AND d.collection = ? AND d.path = ? AND d.hash = ?
|
|
60
60
|
AND cv.seq = ? AND cv.pos = ? AND cv.chunk_len = ?`);
|
|
61
61
|
const page = rows.slice(0, limit);
|
|
62
|
+
const bodies = new Map();
|
|
63
|
+
const readBody = db.prepare("SELECT doc FROM content WHERE hash = ?");
|
|
62
64
|
try {
|
|
63
65
|
check();
|
|
64
|
-
const batch = page.map(
|
|
66
|
+
const batch = page.map(metadata => {
|
|
67
|
+
let doc = bodies.get(metadata.hash);
|
|
68
|
+
if (doc === undefined) {
|
|
69
|
+
// Content is hash-addressed; this capture does not yield before inference.
|
|
70
|
+
doc = readBody.get(metadata.hash).doc;
|
|
71
|
+
bodies.set(metadata.hash, doc);
|
|
72
|
+
}
|
|
73
|
+
const row = { ...metadata, doc };
|
|
65
74
|
const source = sources.get(row.collection);
|
|
66
75
|
const text = row.doc.slice(row.pos, row.pos + row.chunk_len);
|
|
67
76
|
const fingerprint = chunkFingerprint(text);
|
|
@@ -8,7 +8,7 @@ export declare class ResponseReviews {
|
|
|
8
8
|
constructor(db: DatabaseSync);
|
|
9
9
|
sync(cohort: string, episodeId: string, inputHash: string, result: ResponseResult, now: number): void;
|
|
10
10
|
refresh(cohort: string, since: number): number;
|
|
11
|
-
reconcile(cohort: string): void;
|
|
11
|
+
reconcile(cohort: string, sessionId: string): void;
|
|
12
12
|
list(cohort: string, id?: string): {
|
|
13
13
|
id: string;
|
|
14
14
|
episodeId: string;
|
|
@@ -68,11 +68,13 @@ export class ResponseReviews {
|
|
|
68
68
|
throw error;
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
-
reconcile(cohort) {
|
|
71
|
+
reconcile(cohort, sessionId) {
|
|
72
72
|
this.db.prepare(`UPDATE response_review_evidence AS e SET active=CASE WHEN EXISTS (
|
|
73
73
|
SELECT 1 FROM response_review_tasks t JOIN response_results r ON r.id=t.episode_id
|
|
74
74
|
WHERE t.id=e.task_id AND r.cohort=e.cohort AND r.input_hash=e.input_hash AND r.active=1 AND r.status='ok')
|
|
75
|
-
THEN 1 ELSE 0 END WHERE cohort=? AND active>=0
|
|
75
|
+
THEN 1 ELSE 0 END WHERE cohort=? AND active>=0 AND task_id IN (
|
|
76
|
+
SELECT t.id FROM response_results r JOIN response_review_tasks t ON t.episode_id=r.id
|
|
77
|
+
WHERE r.cohort=? AND r.session_id=?)`).run(cohort, cohort, sessionId);
|
|
76
78
|
}
|
|
77
79
|
list(cohort, id) {
|
|
78
80
|
return this.db.prepare(`SELECT t.*,e.input_hash,e.active,e.detail FROM response_review_tasks t
|
|
@@ -87,7 +87,7 @@ export class ResponseAuditStore {
|
|
|
87
87
|
result=CASE WHEN input_hash=excluded.input_hash THEN result ELSE NULL END`);
|
|
88
88
|
for (const e of episodes)
|
|
89
89
|
insert.run(cohort, e.id, sessionId, e.inputHash, e.timestamp);
|
|
90
|
-
this.reviews.reconcile(cohort);
|
|
90
|
+
this.reviews.reconcile(cohort, sessionId);
|
|
91
91
|
this.#db.exec("COMMIT");
|
|
92
92
|
}
|
|
93
93
|
catch (error) {
|
package/dist/src/runtime.js
CHANGED
|
@@ -106,7 +106,8 @@ export class QmdMemoryRuntime {
|
|
|
106
106
|
return { manager: await pending };
|
|
107
107
|
}
|
|
108
108
|
catch (error) {
|
|
109
|
-
this.#managers.
|
|
109
|
+
if (this.#managers.get(params.agentId) === pending)
|
|
110
|
+
this.#managers.delete(params.agentId);
|
|
110
111
|
return { manager: null, error: error instanceof Error ? error.message : String(error) };
|
|
111
112
|
}
|
|
112
113
|
}
|
package/dist/src/session-sync.js
CHANGED
|
@@ -332,8 +332,9 @@ export async function syncSessionProjections(params) {
|
|
|
332
332
|
else
|
|
333
333
|
counts.unchanged += 1;
|
|
334
334
|
}
|
|
335
|
+
const liveIds = new Set(snapshot.windows.map(window => window.sessionId));
|
|
335
336
|
for (const [sessionId, session] of Object.entries(previousManifest.sessions)) {
|
|
336
|
-
if (sessions[sessionId] ||
|
|
337
|
+
if (sessions[sessionId] || liveIds.has(sessionId))
|
|
337
338
|
continue;
|
|
338
339
|
await remove(projectionPath(params.outputDir, session.documentPath));
|
|
339
340
|
counts.removed += 1;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { basename } from "node:path";
|
|
2
2
|
import { selectTypeSafeSkill } from "./typesafe.js";
|
|
3
3
|
import { resolveTypeSafeApiKey, TypeSafeRequestError } from "./typesafe-client.js";
|
|
4
|
-
import { buildSkillWhispererQuery,
|
|
4
|
+
import { buildSkillWhispererQuery, recentSkillMessages } from "./whisperer-context.js";
|
|
5
5
|
import { abortable } from "./abortable.js";
|
|
6
6
|
const CANDIDATE_LIMIT = 10;
|
|
7
7
|
const MAX_QUERY_CHARS = 12_000;
|
|
@@ -10,18 +10,15 @@ const TOTAL_TIMEOUT_MS = 3000;
|
|
|
10
10
|
function isRecord(value) {
|
|
11
11
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
12
12
|
}
|
|
13
|
-
function typeSafeConversation(prompt, messages
|
|
13
|
+
function typeSafeConversation(prompt, messages) {
|
|
14
14
|
const currentRequest = prompt.trim().slice(-MAX_QUERY_CHARS);
|
|
15
15
|
let remaining = MAX_QUERY_CHARS - currentRequest.length;
|
|
16
|
-
const available = messages.flatMap(message => {
|
|
17
|
-
const parsed = messageText(message);
|
|
18
|
-
return parsed ? [{ role: parsed.role, content: parsed.text }] : [];
|
|
19
|
-
});
|
|
20
16
|
const history = [];
|
|
21
|
-
for (
|
|
17
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
22
18
|
if (remaining <= 0)
|
|
23
19
|
break;
|
|
24
|
-
const
|
|
20
|
+
const message = messages[i];
|
|
21
|
+
const content = message.text.slice(-remaining);
|
|
25
22
|
history.unshift({ role: message.role, content });
|
|
26
23
|
remaining -= content.length;
|
|
27
24
|
}
|
|
@@ -73,7 +70,8 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
|
|
|
73
70
|
signal.throwIfAborted();
|
|
74
71
|
if (!apiKey)
|
|
75
72
|
diagnostics?.record(agentId, "skill", typesafe.enabled ? "missing_key" : "typesafe_disabled");
|
|
76
|
-
const
|
|
73
|
+
const recentMessages = recentSkillMessages(event.messages, config.historyMessages);
|
|
74
|
+
const candidates = await abortable(runtime.searchSkills(runtimeParams, buildSkillWhispererQuery(event.prompt, recentMessages), apiKey ? -1 : config.minScore, CANDIDATE_LIMIT), signal);
|
|
77
75
|
signal.throwIfAborted();
|
|
78
76
|
const resolvedCandidates = candidates.flatMap((candidate) => {
|
|
79
77
|
const canonicalPath = runtime.resolveSkillPath(runtimeParams, candidate.path);
|
|
@@ -96,7 +94,7 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
|
|
|
96
94
|
}
|
|
97
95
|
const selectedIndex = await abortable(selectTypeSafeSkill({
|
|
98
96
|
apiKey, timeoutMs: typesafe.timeoutMs, signal,
|
|
99
|
-
...typeSafeConversation(event.prompt,
|
|
97
|
+
...typeSafeConversation(event.prompt, recentMessages),
|
|
100
98
|
candidates: shortlist.map(({ candidate }) => candidate),
|
|
101
99
|
onCandidateFailure: (candidateIndex, error) => {
|
|
102
100
|
if (signal.aborted)
|
|
@@ -85,15 +85,9 @@ export declare class TrainingStore {
|
|
|
85
85
|
error: string;
|
|
86
86
|
}): void;
|
|
87
87
|
retry(includeAmbiguous: boolean, ids: readonly string[]): number;
|
|
88
|
-
|
|
89
|
-
queryExamples(threshold?: number): {
|
|
88
|
+
queryExamples(threshold?: number): (TrainingSourceExample & {
|
|
90
89
|
recallProbability: number;
|
|
91
|
-
|
|
92
|
-
inputHash: string;
|
|
93
|
-
inputJson: string;
|
|
94
|
-
sessionId: string;
|
|
95
|
-
timestamp: number;
|
|
96
|
-
}[];
|
|
90
|
+
})[];
|
|
97
91
|
step<S extends keyof TrainingStepResults>(stage: S, parameters: Record<string, unknown>): {
|
|
98
92
|
id: string;
|
|
99
93
|
stage: S;
|
|
@@ -136,20 +136,21 @@ export class TrainingStore {
|
|
|
136
136
|
this.#db.prepare("UPDATE training_examples SET active=0 WHERE session_id=?").run(sessionId);
|
|
137
137
|
const counts = { added: 0, changed: 0, unchanged: 0, retired: 0 };
|
|
138
138
|
const retained = new Set();
|
|
139
|
+
const input = this.#db.prepare("INSERT OR IGNORE INTO training_inputs VALUES (?,?,?)");
|
|
140
|
+
const source = this.#db.prepare(`INSERT INTO training_examples VALUES (?,?,?,?,?,?,1)
|
|
141
|
+
ON CONFLICT(session_id,event_seq) DO UPDATE SET timestamp=excluded.timestamp,input_hash=excluded.input_hash,
|
|
142
|
+
context_limited=excluded.context_limited,active=1`);
|
|
143
|
+
const gate = this.#db.prepare(`INSERT OR IGNORE INTO training_gates
|
|
144
|
+
(id,input_hash,prompt_version,requested_model,status) VALUES (?,?,?,?,'pending')`);
|
|
139
145
|
for (const example of examples) {
|
|
140
146
|
const old = previous.get(example.seq);
|
|
141
147
|
if (!old && (existingOnly || example.timestamp < since || example.timestamp >= until))
|
|
142
148
|
continue;
|
|
143
149
|
retained.add(example.seq);
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
this.#db.prepare(`INSERT INTO training_examples VALUES (?,?,?,?,?,?,1)
|
|
147
|
-
ON CONFLICT(session_id,event_seq) DO UPDATE SET timestamp=excluded.timestamp,input_hash=excluded.input_hash,
|
|
148
|
-
context_limited=excluded.context_limited,active=1`)
|
|
149
|
-
.run(trainingHash([this.#nodeId, this.#agentId, sessionId, example.seq]), sessionId, example.seq, example.timestamp, example.inputHash, Number(example.contextLimited));
|
|
150
|
+
input.run(example.inputHash, TRAINING_PREPARATION, JSON.stringify(example.input));
|
|
151
|
+
source.run(trainingHash([this.#nodeId, this.#agentId, sessionId, example.seq]), sessionId, example.seq, example.timestamp, example.inputHash, Number(example.contextLimited));
|
|
150
152
|
const id = trainingHash([example.inputHash, TRAINING_GATE_VERSION, TRAINING_GATE_MODEL]);
|
|
151
|
-
|
|
152
|
-
.run(id, example.inputHash, TRAINING_GATE_VERSION, TRAINING_GATE_MODEL);
|
|
153
|
+
gate.run(id, example.inputHash, TRAINING_GATE_VERSION, TRAINING_GATE_MODEL);
|
|
153
154
|
if (!old)
|
|
154
155
|
counts.added++;
|
|
155
156
|
else if (old.input_hash !== example.inputHash || old.active !== 1)
|
|
@@ -211,21 +212,15 @@ export class TrainingStore {
|
|
|
211
212
|
return count + Number(gates.changes) + Number(steps.changes);
|
|
212
213
|
}, 0));
|
|
213
214
|
}
|
|
214
|
-
activeExamples() {
|
|
215
|
-
return this.#db.prepare(`SELECT e.id,e.input_hash inputHash,i.input_json inputJson,e.session_id sessionId,e.timestamp
|
|
216
|
-
FROM training_examples e JOIN training_inputs i ON i.hash=e.input_hash
|
|
217
|
-
WHERE e.active=1 AND i.preparation=? ORDER BY e.timestamp DESC,e.id`).all(TRAINING_PREPARATION);
|
|
218
|
-
}
|
|
219
215
|
queryExamples(threshold = TRAINING_GATE_THRESHOLD) {
|
|
220
216
|
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)
|
|
221
217
|
throw new Error("Threshold must be between 0 and 1");
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
});
|
|
218
|
+
return this.#db.prepare(`SELECT e.id,e.input_hash inputHash,i.input_json inputJson,e.session_id sessionId,e.timestamp,
|
|
219
|
+
g.probability recallProbability FROM training_examples e JOIN training_inputs i ON i.hash=e.input_hash
|
|
220
|
+
JOIN training_gates g ON g.input_hash=e.input_hash
|
|
221
|
+
WHERE e.active=1 AND i.preparation=? AND g.prompt_version=? AND g.requested_model=?
|
|
222
|
+
AND g.status='complete' AND g.probability>=? ORDER BY e.timestamp DESC,e.id`)
|
|
223
|
+
.all(TRAINING_PREPARATION, TRAINING_GATE_VERSION, TRAINING_GATE_MODEL, threshold);
|
|
229
224
|
}
|
|
230
225
|
step(stage, parameters) {
|
|
231
226
|
const request = { ...parameters, recipe: TRAINING_RECIPE_VERSION };
|
|
@@ -3,4 +3,8 @@ export declare function messageText(message: unknown): {
|
|
|
3
3
|
role: "user" | "assistant";
|
|
4
4
|
text: string;
|
|
5
5
|
} | undefined;
|
|
6
|
-
export declare function
|
|
6
|
+
export declare function recentSkillMessages(messages: readonly unknown[], limit: number): {
|
|
7
|
+
role: "user" | "assistant";
|
|
8
|
+
text: string;
|
|
9
|
+
}[];
|
|
10
|
+
export declare function buildSkillWhispererQuery(prompt: string, history: readonly NonNullable<ReturnType<typeof messageText>>[]): string;
|
|
@@ -11,11 +11,16 @@ export function messageText(message) {
|
|
|
11
11
|
}).join("\n").trim() : "";
|
|
12
12
|
return text ? { role: message.role, text } : undefined;
|
|
13
13
|
}
|
|
14
|
-
export function
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
14
|
+
export function recentSkillMessages(messages, limit) {
|
|
15
|
+
const history = [];
|
|
16
|
+
for (let i = messages.length - 1; i >= 0 && history.length < limit; i--) {
|
|
17
|
+
const parsed = messageText(messages[i]);
|
|
18
|
+
if (parsed)
|
|
19
|
+
history.push(parsed);
|
|
20
|
+
}
|
|
21
|
+
return history.reverse();
|
|
22
|
+
}
|
|
23
|
+
export function buildSkillWhispererQuery(prompt, history) {
|
|
24
|
+
return [...history.map(message => `${message.role}: ${message.text}`), `user: ${prompt.trim()}`]
|
|
25
|
+
.join("\n\n").slice(-MAX_SKILL_QUERY_CHARS);
|
|
21
26
|
}
|
|
@@ -2,40 +2,47 @@ import fs from "node:fs/promises";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { isPathStrictlyInside } from "openclaw/plugin-sdk/file-access-runtime";
|
|
4
4
|
import { readMemoryArtifactProvenance } from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
|
5
|
-
export const classifyWorkspaceMemoryPaths = async (params) =>
|
|
5
|
+
export const classifyWorkspaceMemoryPaths = async (params) => {
|
|
6
|
+
if (!params.relativePaths.length)
|
|
7
|
+
return [];
|
|
6
8
|
let workspacePath;
|
|
7
|
-
let filePath;
|
|
8
9
|
try {
|
|
9
|
-
|
|
10
|
-
fs.realpath(params.workspaceDir),
|
|
11
|
-
fs.realpath(path.resolve(params.workspaceDir, relativePath)),
|
|
12
|
-
]);
|
|
10
|
+
workspacePath = await fs.realpath(params.workspaceDir);
|
|
13
11
|
}
|
|
14
12
|
catch {
|
|
15
|
-
return { relativePath, originClass: "untrusted" };
|
|
13
|
+
return params.relativePaths.map(relativePath => ({ relativePath, originClass: "untrusted" }));
|
|
16
14
|
}
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
relativePath:
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
15
|
+
return await Promise.all(params.relativePaths.map(async (relativePath) => {
|
|
16
|
+
let filePath;
|
|
17
|
+
try {
|
|
18
|
+
filePath = await fs.realpath(path.resolve(params.workspaceDir, relativePath));
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return { relativePath, originClass: "untrusted" };
|
|
22
|
+
}
|
|
23
|
+
if (!isPathStrictlyInside(workspacePath, filePath)) {
|
|
24
|
+
return { relativePath, originClass: "untrusted" };
|
|
25
|
+
}
|
|
26
|
+
const workspaceRelativePath = path.relative(workspacePath, filePath);
|
|
27
|
+
const segments = workspaceRelativePath.split(path.sep);
|
|
28
|
+
const curatedRoot = segments.length === 1 &&
|
|
29
|
+
["MEMORY.md", "memory.md", "USER.md"].includes(segments[0]);
|
|
30
|
+
if ((segments.length === 1 && ["DREAMS.md", "dreams.md"].includes(segments[0])) ||
|
|
31
|
+
(segments[0] === "memory" && ["dreaming", ".dreams"].includes(segments[1]))) {
|
|
32
|
+
return { relativePath, originClass: "system" };
|
|
33
|
+
}
|
|
34
|
+
const isWorkspaceMemory = curatedRoot ||
|
|
35
|
+
(segments[0] === "memory" && segments.at(-1)?.endsWith(".md") === true);
|
|
36
|
+
const normalizedPath = workspaceRelativePath.replaceAll(path.sep, "/");
|
|
37
|
+
const recorded = isWorkspaceMemory
|
|
38
|
+
? await readMemoryArtifactProvenance({
|
|
39
|
+
workspaceDir: params.workspaceDir,
|
|
40
|
+
relativePath: normalizedPath,
|
|
41
|
+
})
|
|
42
|
+
: undefined;
|
|
43
|
+
return {
|
|
44
|
+
relativePath,
|
|
45
|
+
originClass: recorded?.originClass ?? (isWorkspaceMemory ? "agent" : "untrusted"),
|
|
46
|
+
};
|
|
47
|
+
}));
|
|
48
|
+
};
|
package/docs/memory-training.md
CHANGED
|
@@ -70,9 +70,11 @@ continue without retrying failed requests. Storage/lease failures still stop the
|
|
|
70
70
|
In-flight work drains and persists before snapshots close. Dry runs make no inference calls.
|
|
71
71
|
|
|
72
72
|
Cached evaluations still validate the exact historical text and vector fingerprint,
|
|
73
|
-
but do not rebuild a QMD index.
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
but do not rebuild a QMD index. Each evaluation run captures immutable source text
|
|
74
|
+
and vector bytes once, then closes the source reader. Each exact cutoff derives its
|
|
75
|
+
own fingerprint and builds a private index only on its first uncached search.
|
|
76
|
+
Concurrent queries at that cutoff share the index; later runs capture fresh source
|
|
77
|
+
bytes, so changed corpus content still invalidates caches.
|
|
76
78
|
|
|
77
79
|
## Input and historical boundaries
|
|
78
80
|
|
|
@@ -99,10 +101,9 @@ unknown/future dates. Never infer dates from quoted headings. Copy existing vect
|
|
|
99
101
|
only for complete chunks within the safe prefix; BM25 sees that same prefix.
|
|
100
102
|
Validate returned passages/dates again. No reembedding or temporary transcripts.
|
|
101
103
|
|
|
102
|
-
QMD
|
|
103
|
-
A small
|
|
104
|
-
|
|
105
|
-
backend and omits query-conditioned scoring. It does not rewrite QMD. Training and
|
|
104
|
+
QMD's public `discoverCandidates` API owns tokenization, FTS-highlight chunk
|
|
105
|
+
selection and source-aware deduplication. A small adapter requests ten from the
|
|
106
|
+
selected backend and omits query-conditioned scoring. Training and
|
|
106
107
|
runtime share complete-excerpt rendering, including the 1,200-character cap;
|
|
107
108
|
oversized matches are discarded rather than truncated.
|
|
108
109
|
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "unblock-memory",
|
|
3
3
|
"name": "Unblock Memory",
|
|
4
|
-
"version": "0.7.
|
|
4
|
+
"version": "0.7.3",
|
|
5
5
|
"description": "Indexes, retrieves, and analyzes configured workspace memory with existing QMD vectors.",
|
|
6
6
|
"kind": "memory",
|
|
7
7
|
"activation": { "onStartup": true },
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@unblocklabs/unblock-memory",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.3",
|
|
4
4
|
"description": "Workspace-native memory for OpenClaw, powered by QMD",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -37,16 +37,18 @@
|
|
|
37
37
|
"build": "tsc -p tsconfig.build.json",
|
|
38
38
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
39
39
|
"knip": "knip --reporter compact",
|
|
40
|
-
"test": "
|
|
40
|
+
"test": "npm run test:node && npm run test:python",
|
|
41
|
+
"test:node": "node --import tsx --test --test-concurrency=1 tests/**/*.test.ts tests/**/*.test.mjs",
|
|
42
|
+
"test:python": "python3 -B tests/prepare-query-training.test.py",
|
|
41
43
|
"eval:retrieval": "node --import tsx eval/retrieval/lab.ts",
|
|
42
44
|
"plugin:inspect": "plugin-inspector check --config plugin-inspector.config.json --no-openclaw",
|
|
43
|
-
"plugin:inspect:runtime": "plugin-inspector check --config plugin-inspector.config.json --no-openclaw --runtime --
|
|
45
|
+
"plugin:inspect:runtime": "plugin-inspector check --config plugin-inspector.config.json --no-openclaw --runtime --real-sdk --allow-execute",
|
|
44
46
|
"release:check": "node scripts/check-release-version.mjs",
|
|
45
47
|
"preflight": "npm run knip && npm run build && npm run typecheck && npm test && npm run plugin:inspect && npm run plugin:inspect:runtime && npm pack --dry-run"
|
|
46
48
|
},
|
|
47
49
|
"dependencies": {
|
|
48
50
|
"@huggingface/tokenizers": "0.2.0",
|
|
49
|
-
"@unblocklabs/qmd": "https://github.com/unblocklabs-ai/qmd/releases/download/v2.11.
|
|
51
|
+
"@unblocklabs/qmd": "https://github.com/unblocklabs-ai/qmd/releases/download/v2.11.1/unblocklabs-qmd-2.11.1.tgz",
|
|
50
52
|
"chokidar": "5.0.0",
|
|
51
53
|
"js-tiktoken": "1.0.21",
|
|
52
54
|
"picomatch": "^4.0.5",
|