@unblocklabs/unblock-memory 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/src/config.d.ts +2 -0
- package/dist/src/config.js +4 -1
- package/dist/src/contracts.d.ts +5 -0
- package/dist/src/extraction-chunk.d.ts +29 -0
- package/dist/src/extraction-chunk.js +130 -0
- package/dist/src/extraction-config.d.ts +13 -0
- package/dist/src/extraction-config.js +39 -0
- package/dist/src/extraction-index.d.ts +9 -0
- package/dist/src/extraction-index.js +44 -0
- package/dist/src/extraction-model.d.ts +58 -0
- package/dist/src/extraction-model.js +140 -0
- package/dist/src/extraction-runtime.d.ts +4 -0
- package/dist/src/extraction-runtime.js +121 -0
- package/dist/src/extraction-source.d.ts +26 -0
- package/dist/src/extraction-source.js +66 -0
- package/dist/src/extraction-store.d.ts +52 -0
- package/dist/src/extraction-store.js +138 -0
- package/dist/src/extraction-worker.d.ts +51 -0
- package/dist/src/extraction-worker.js +128 -0
- package/dist/src/manager.d.ts +3 -0
- package/dist/src/manager.js +72 -12
- package/dist/src/plugin.js +6 -1
- package/dist/src/response-judge.d.ts +1 -1
- package/dist/src/response-stages.d.ts +4 -4
- package/dist/src/runtime.d.ts +2 -0
- package/dist/src/runtime.js +3 -0
- package/dist/src/session-projector.d.ts +10 -0
- package/dist/src/session-projector.js +1 -1
- package/docs/configuration.md +9 -0
- package/docs/extraction.md +124 -0
- package/openclaw.plugin.json +15 -1
- package/package.json +3 -1
package/dist/src/manager.js
CHANGED
|
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
|
|
3
3
|
import { mkdir, stat } from "node:fs/promises";
|
|
4
4
|
import { basename, dirname, relative, resolve, sep } from "node:path";
|
|
5
|
+
import { EXTRACTED_COLLECTION, extractedRecords, extractedPath, extractedHit, syncExtractedIndex } from "./extraction-index.js";
|
|
5
6
|
import chokidar from "chokidar";
|
|
6
7
|
import picomatch from "picomatch";
|
|
7
8
|
import { ensureMemoryAnalysisSchema, latestAnalysisCollections, latestAnalysisRunId, markMemoryAnalysisStale, readAnalysisSummary, readCluster, readClusters, runAnalysisWorker, } from "./analysis.js";
|
|
@@ -236,6 +237,7 @@ export class QmdMemoryManager {
|
|
|
236
237
|
#dbPath;
|
|
237
238
|
#workspaceDir;
|
|
238
239
|
#curationPath;
|
|
240
|
+
#extraction;
|
|
239
241
|
#sources;
|
|
240
242
|
#storeFactory;
|
|
241
243
|
#keepModelsWarm;
|
|
@@ -287,6 +289,7 @@ export class QmdMemoryManager {
|
|
|
287
289
|
constructor(params) {
|
|
288
290
|
this.#dbPath = params.dbPath;
|
|
289
291
|
this.#curationPath = params.curationPath ?? `${params.dbPath}.curation.sqlite`;
|
|
292
|
+
this.#extraction = params.extraction;
|
|
290
293
|
this.#workspaceDir = params.workspaceDir;
|
|
291
294
|
this.#sources = new Map(params.sources.map((source) => [source.collection, source]));
|
|
292
295
|
this.#storeFactory = params.storeFactory;
|
|
@@ -384,10 +387,11 @@ export class QmdMemoryManager {
|
|
|
384
387
|
dbPath: this.#dbPath,
|
|
385
388
|
keepModelsWarm: this.#keepModelsWarm,
|
|
386
389
|
config: {
|
|
387
|
-
collections: Object.fromEntries(this.#qmdSources().map((source) => [
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
390
|
+
collections: Object.fromEntries([...this.#qmdSources().map((source) => [
|
|
391
|
+
source.collection,
|
|
392
|
+
{ path: source.root, pattern: source.pattern },
|
|
393
|
+
]), ...(this.#extraction?.enabled && this.#extraction.publish
|
|
394
|
+
? [[EXTRACTED_COLLECTION, { path: dirname(this.#dbPath), pattern: ".no-extracted-files", includeByDefault: false }]] : [])]),
|
|
391
395
|
},
|
|
392
396
|
});
|
|
393
397
|
try {
|
|
@@ -395,6 +399,8 @@ export class QmdMemoryManager {
|
|
|
395
399
|
ensureMemoryAnalysisSchema(store.internal.db);
|
|
396
400
|
markStaleForAnalysisCollectionChange(store.internal.db, this.#analysisCollectionNames(), this.#skillCollectionNames().length > 0);
|
|
397
401
|
const configuredCollections = new Set(this.#qmdSources().map((source) => source.collection));
|
|
402
|
+
if (this.#extraction?.enabled && this.#extraction.publish)
|
|
403
|
+
configuredCollections.add(EXTRACTED_COLLECTION);
|
|
398
404
|
const staleCollections = (await store.getStatus()).collections
|
|
399
405
|
.map((collection) => collection.name)
|
|
400
406
|
.filter((collection) => !configuredCollections.has(collection));
|
|
@@ -441,26 +447,51 @@ export class QmdMemoryManager {
|
|
|
441
447
|
}
|
|
442
448
|
#collectionNames(corpora) {
|
|
443
449
|
const publicSources = [...this.#sources.values()].filter((source) => source.kind !== "skills");
|
|
450
|
+
const extracted = this.#extraction?.enabled && this.#extraction.publish ? [EXTRACTED_COLLECTION] : [];
|
|
444
451
|
if (corpora === undefined)
|
|
445
|
-
return publicSources.map((source) => source.collection);
|
|
452
|
+
return [...publicSources.map((source) => source.collection), ...extracted];
|
|
446
453
|
if (corpora.length === 0)
|
|
447
454
|
throw new Error("memory_search corpora must not be empty");
|
|
448
455
|
const selected = new Set(corpora);
|
|
449
456
|
if (selected.has("all")) {
|
|
450
457
|
if (selected.size > 1)
|
|
451
458
|
throw new Error('memory_search corpus "all" must be used alone');
|
|
452
|
-
return publicSources.map((source) => source.collection);
|
|
459
|
+
return [...publicSources.map((source) => source.collection), ...extracted];
|
|
453
460
|
}
|
|
454
461
|
const known = new Set(publicSources.map((source) => source.corpus));
|
|
462
|
+
if (extracted.length)
|
|
463
|
+
known.add("extracted");
|
|
455
464
|
const unknown = [...selected].find((corpus) => !known.has(corpus));
|
|
456
465
|
if (unknown)
|
|
457
466
|
throw new Error(`memory_search unknown corpus: ${unknown}`);
|
|
458
|
-
return publicSources
|
|
459
|
-
|
|
460
|
-
|
|
467
|
+
return [...publicSources
|
|
468
|
+
.filter((source) => selected.has(source.corpus))
|
|
469
|
+
.map((source) => source.collection), ...(selected.has("extracted") ? extracted : [])];
|
|
470
|
+
}
|
|
471
|
+
syncExtracted() {
|
|
472
|
+
return this.#enqueue(async () => {
|
|
473
|
+
if (!this.#extraction?.enabled || !this.#extraction.publish)
|
|
474
|
+
return;
|
|
475
|
+
const store = await this.#getAnalysisStore();
|
|
476
|
+
const records = extractedRecords(this.#curationPath).filter(r => this.#extraction.chatTypes.includes(r.metadata.chatType));
|
|
477
|
+
await syncExtractedIndex(store, records);
|
|
478
|
+
await this.#refreshIndexStatus(store);
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
#currentExtracted(paths, filter) {
|
|
482
|
+
if (!this.#extraction?.enabled || !this.#extraction.publish)
|
|
483
|
+
return [];
|
|
484
|
+
const records = extractedRecords(this.#curationPath, paths?.filter(path => path.startsWith(`qmd://${EXTRACTED_COLLECTION}/`))
|
|
485
|
+
.map(path => path.slice(`qmd://${EXTRACTED_COLLECTION}/`.length)))
|
|
486
|
+
.filter(r => this.#extraction.chatTypes.includes(r.metadata.chatType));
|
|
487
|
+
if (!filter)
|
|
488
|
+
return records;
|
|
489
|
+
const allowed = new Set(sessionAllowedPaths(new Map(records.map(r => [extractedPath(r), r.metadata])), EXTRACTED_COLLECTION, filter)[EXTRACTED_COLLECTION]);
|
|
490
|
+
return records.filter(r => allowed.has(extractedPath(r)));
|
|
461
491
|
}
|
|
462
492
|
async #updateAndEmbed(store, collection, force) {
|
|
463
|
-
|
|
493
|
+
// Extracted facts are DB projections, not files: never run filesystem sync on them.
|
|
494
|
+
const update = await store.update({ collections: collection ? [collection] : this.#qmdSources().map(source => source.collection) });
|
|
464
495
|
this.#cleanupRemovedDocuments?.(update.updated + update.removed);
|
|
465
496
|
const analysisStore = store;
|
|
466
497
|
const changed = update.indexed + update.updated + update.removed > 0 || update.needsEmbedding > 0;
|
|
@@ -776,17 +807,30 @@ export class QmdMemoryManager {
|
|
|
776
807
|
if (sessions && collections.includes(sessions.collection)) {
|
|
777
808
|
await this.#refreshSessionMetadata();
|
|
778
809
|
}
|
|
779
|
-
|
|
810
|
+
let allowedPaths = opts?.sessionFilter && sessions && collections.includes(sessions.collection)
|
|
780
811
|
? sessionAllowedPaths(this.#sessionMetadata, sessions.collection, opts.sessionFilter)
|
|
781
812
|
: undefined;
|
|
782
813
|
const store = await this.#getStore();
|
|
783
814
|
opts?.signal?.throwIfAborted();
|
|
815
|
+
const currentRecords = (paths) => {
|
|
816
|
+
if (!collections.includes(EXTRACTED_COLLECTION))
|
|
817
|
+
return [];
|
|
818
|
+
return this.#currentExtracted(paths, opts?.sessionFilter);
|
|
819
|
+
};
|
|
820
|
+
if (collections.includes(EXTRACTED_COLLECTION)) {
|
|
821
|
+
allowedPaths = { ...allowedPaths, [EXTRACTED_COLLECTION]: currentRecords().map(extractedPath) };
|
|
822
|
+
}
|
|
784
823
|
if (opts?.lexicalOnly) {
|
|
785
824
|
const hits = await store.searchLex(query, {
|
|
786
825
|
limit: opts.maxResults ?? 5,
|
|
787
826
|
collection: collections,
|
|
788
827
|
});
|
|
828
|
+
const current = new Map(currentRecords(hits.map(hit => hit.filepath)).map(r => [`qmd://${EXTRACTED_COLLECTION}/${extractedPath(r)}`, r]));
|
|
789
829
|
return hits.flatMap((hit) => {
|
|
830
|
+
if (hit.collectionName === EXTRACTED_COLLECTION) {
|
|
831
|
+
const record = current.get(hit.filepath);
|
|
832
|
+
return record && hit.score >= (opts.minScore ?? 0) ? [extractedHit(record, hit.score)] : [];
|
|
833
|
+
}
|
|
790
834
|
const corpus = this.#sources.get(hit.collectionName)?.corpus;
|
|
791
835
|
const prefix = `qmd://${hit.collectionName}/`;
|
|
792
836
|
const session = corpus === "sessions" && hit.filepath.startsWith(prefix)
|
|
@@ -833,6 +877,7 @@ export class QmdMemoryManager {
|
|
|
833
877
|
async #renderSearchHits(hits, store, opts, whisperer = false) {
|
|
834
878
|
const tokenizer = store.internal?.llm;
|
|
835
879
|
const results = [];
|
|
880
|
+
const current = new Map(this.#currentExtracted(hits.map(hit => hit.file), opts?.sessionFilter).map(r => [`qmd://${EXTRACTED_COLLECTION}/${extractedPath(r)}`, r]));
|
|
836
881
|
for (const hit of hits) {
|
|
837
882
|
opts?.signal?.throwIfAborted();
|
|
838
883
|
// Proactive hints must retain the entire matched chunk, even when expanded
|
|
@@ -840,6 +885,12 @@ export class QmdMemoryManager {
|
|
|
840
885
|
if (hit.bestChunk.length > (opts?.maxSnippetChars ?? Infinity))
|
|
841
886
|
continue;
|
|
842
887
|
const collection = /^qmd:\/\/([^/]+)\//.exec(hit.file)?.[1];
|
|
888
|
+
if (collection === EXTRACTED_COLLECTION) {
|
|
889
|
+
const record = current.get(hit.file);
|
|
890
|
+
if (record && record.text.length <= (opts?.maxSnippetChars ?? Infinity))
|
|
891
|
+
results.push(extractedHit(record, hit.score));
|
|
892
|
+
continue;
|
|
893
|
+
}
|
|
843
894
|
const corpus = collection ? this.#sources.get(collection)?.corpus : undefined;
|
|
844
895
|
if (!corpus)
|
|
845
896
|
continue;
|
|
@@ -880,7 +931,9 @@ export class QmdMemoryManager {
|
|
|
880
931
|
citation: `${hit.displayPath}#L${span.startLine}-L${span.endLine}`,
|
|
881
932
|
});
|
|
882
933
|
}
|
|
883
|
-
|
|
934
|
+
// Expansion above can yield; recheck only derived hits once more before returning.
|
|
935
|
+
const finalPaths = new Set(this.#currentExtracted(results.map(hit => hit.path), opts?.sessionFilter).map(r => `qmd://${EXTRACTED_COLLECTION}/${extractedPath(r)}`));
|
|
936
|
+
return results.filter(hit => !hit.path.startsWith(`qmd://${EXTRACTED_COLLECTION}/`) || finalPaths.has(hit.path));
|
|
884
937
|
}
|
|
885
938
|
async searchSkills(query, minScore, limit) {
|
|
886
939
|
const collections = this.#skillCollectionNames();
|
|
@@ -942,6 +995,13 @@ export class QmdMemoryManager {
|
|
|
942
995
|
.slice(0, limit);
|
|
943
996
|
}
|
|
944
997
|
async readFile(params) {
|
|
998
|
+
if (params.relPath.startsWith(`qmd://${EXTRACTED_COLLECTION}/`)) {
|
|
999
|
+
const record = this.#extraction?.enabled && this.#extraction.publish
|
|
1000
|
+
? extractedRecords(this.#curationPath, [params.relPath.slice(`qmd://${EXTRACTED_COLLECTION}/`.length)]).find(r => this.#extraction.chatTypes.includes(r.metadata.chatType) &&
|
|
1001
|
+
params.relPath === `qmd://${EXTRACTED_COLLECTION}/${extractedPath(r)}`) : undefined;
|
|
1002
|
+
return record ? buildReadResult({ content: `${record.text}\nObserved: ${new Date(record.observedAt).toISOString()}\nSource: ${extractedHit(record, 1).citation}`,
|
|
1003
|
+
path: params.relPath, from: params.from, lines: params.lines }) : { status: "not_found", text: "", path: params.relPath };
|
|
1004
|
+
}
|
|
945
1005
|
const safe = parseSafeVirtualPath(params.relPath, this.#sources);
|
|
946
1006
|
if (!safe || safe.source.kind === "skills") {
|
|
947
1007
|
return { status: "not_found", text: "", path: params.relPath };
|
package/dist/src/plugin.js
CHANGED
|
@@ -18,6 +18,7 @@ import { registerResponseAudit } from "./response-runtime.js";
|
|
|
18
18
|
import { registerInsideOut } from "./inside-out-runtime.js";
|
|
19
19
|
import { registerMemoryTraining } from "./training-runtime.js";
|
|
20
20
|
import { resolveTimezone } from "./session-projector.js";
|
|
21
|
+
import { registerExtraction } from "./extraction-runtime.js";
|
|
21
22
|
const searchParameters = Type.Object({
|
|
22
23
|
query: Type.String({ pattern: "\\S" }),
|
|
23
24
|
corpora: Type.Optional(Type.Array(Type.String({ pattern: "\\S" }), {
|
|
@@ -417,12 +418,16 @@ export function registerUnblockMemory(api) {
|
|
|
417
418
|
registerResponseAudit(api, config);
|
|
418
419
|
registerInsideOut(api, config);
|
|
419
420
|
registerMemoryTraining(api, config);
|
|
420
|
-
if (api.registrationMode === "cli-metadata")
|
|
421
|
+
if (api.registrationMode === "cli-metadata") {
|
|
422
|
+
registerExtraction(api, config);
|
|
421
423
|
return;
|
|
424
|
+
}
|
|
422
425
|
const runtime = new QmdMemoryRuntime(config.corpora, {
|
|
423
426
|
analysisExecutable: config.analysis.executable,
|
|
424
427
|
keepEmbeddingModelWarm: config.keepEmbeddingModelWarm,
|
|
428
|
+
extraction: config.extraction,
|
|
425
429
|
});
|
|
430
|
+
registerExtraction(api, config, runtime);
|
|
426
431
|
const capability = {
|
|
427
432
|
deterministicRecallToolName: "memory_search",
|
|
428
433
|
supportsPrivateTranscriptRecall: false,
|
|
@@ -70,7 +70,7 @@ declare const qualitySchema: Type.TObject<{
|
|
|
70
70
|
}>;
|
|
71
71
|
consistency: Type.TObject<{
|
|
72
72
|
type: Type.TLiteral<"choice">;
|
|
73
|
-
choice: Type.TEnum<("
|
|
73
|
+
choice: Type.TEnum<("contradicted" | "not_assessable" | "consistent")[]>;
|
|
74
74
|
confidence: Type.TNumber;
|
|
75
75
|
probabilities: Type.TObject<{
|
|
76
76
|
[k: string]: Type.TNumber;
|
|
@@ -91,29 +91,29 @@ export declare function assessResponseStages(store: ResponseAuditStore, cohort:
|
|
|
91
91
|
fulfillment: {
|
|
92
92
|
type: "score";
|
|
93
93
|
confidence: number;
|
|
94
|
-
score: number;
|
|
95
94
|
probabilities: {
|
|
96
95
|
"0": number;
|
|
97
96
|
"1": number;
|
|
98
97
|
"2": number;
|
|
99
98
|
"3": number;
|
|
100
99
|
};
|
|
100
|
+
score: number;
|
|
101
101
|
};
|
|
102
102
|
deliverableFit: {
|
|
103
103
|
type: "score";
|
|
104
104
|
confidence: number;
|
|
105
|
-
score: number;
|
|
106
105
|
probabilities: {
|
|
107
106
|
"0": number;
|
|
108
107
|
"1": number;
|
|
109
108
|
"2": number;
|
|
110
109
|
"3": number;
|
|
111
110
|
};
|
|
111
|
+
score: number;
|
|
112
112
|
};
|
|
113
113
|
consistency: {
|
|
114
114
|
type: "choice";
|
|
115
115
|
confidence: number;
|
|
116
|
-
choice: "
|
|
116
|
+
choice: "contradicted" | "not_assessable" | "consistent";
|
|
117
117
|
probabilities: {
|
|
118
118
|
[x: string]: number;
|
|
119
119
|
[x: number]: number;
|
|
@@ -172,13 +172,13 @@ export declare function assessResponseStages(store: ResponseAuditStore, cohort:
|
|
|
172
172
|
dissatisfactionIntensity: {
|
|
173
173
|
type: "score";
|
|
174
174
|
confidence: number;
|
|
175
|
-
score: number;
|
|
176
175
|
probabilities: {
|
|
177
176
|
"0": number;
|
|
178
177
|
"1": number;
|
|
179
178
|
"2": number;
|
|
180
179
|
"3": number;
|
|
181
180
|
};
|
|
181
|
+
score: number;
|
|
182
182
|
};
|
|
183
183
|
}>;
|
|
184
184
|
}>;
|
package/dist/src/runtime.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { OpenClawConfig } from "openclaw/plugin-sdk/plugin-entry";
|
|
2
2
|
import type { CorpusConfig } from "./config.js";
|
|
3
|
+
import type { ExtractionConfig } from "./extraction-config.js";
|
|
3
4
|
import type { MemoryPluginRuntimeContract } from "./contracts.js";
|
|
4
5
|
import { QmdMemoryManager } from "./manager.js";
|
|
5
6
|
import { type SessionSyncResult } from "./session-sync.js";
|
|
@@ -46,6 +47,7 @@ export declare class QmdMemoryRuntime implements MemoryPluginRuntimeContract {
|
|
|
46
47
|
analysisExecutable?: string;
|
|
47
48
|
keepEmbeddingModelWarm?: boolean;
|
|
48
49
|
stateRoot?: string;
|
|
50
|
+
extraction?: ExtractionConfig;
|
|
49
51
|
});
|
|
50
52
|
getMemorySearchManager(params: {
|
|
51
53
|
cfg: OpenClawConfig;
|
package/dist/src/runtime.js
CHANGED
|
@@ -66,6 +66,7 @@ export async function recoverInterruptedSessionSync(directory, statusPath, stale
|
|
|
66
66
|
}
|
|
67
67
|
export class QmdMemoryRuntime {
|
|
68
68
|
#corpora;
|
|
69
|
+
#extraction;
|
|
69
70
|
#analysisExecutable;
|
|
70
71
|
#keepEmbeddingModelWarm;
|
|
71
72
|
#stateRoot;
|
|
@@ -90,6 +91,7 @@ export class QmdMemoryRuntime {
|
|
|
90
91
|
}
|
|
91
92
|
constructor(corpora, options = {}) {
|
|
92
93
|
this.#corpora = corpora;
|
|
94
|
+
this.#extraction = options.extraction;
|
|
93
95
|
this.#analysisExecutable = options.analysisExecutable;
|
|
94
96
|
this.#keepEmbeddingModelWarm = options.keepEmbeddingModelWarm ?? true;
|
|
95
97
|
this.#stateRoot = options.stateRoot ?? resolveStateDir();
|
|
@@ -242,6 +244,7 @@ export class QmdMemoryRuntime {
|
|
|
242
244
|
workspaceDir,
|
|
243
245
|
dbPath: join(stateDir, "index.sqlite"),
|
|
244
246
|
curationPath: join(stateDir, MEMORY_DATABASE),
|
|
247
|
+
extraction: this.#extraction,
|
|
245
248
|
sources,
|
|
246
249
|
keepModelsWarm: this.#keepEmbeddingModelWarm,
|
|
247
250
|
analysisExecutable: this.#analysisExecutable,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ChatType } from "./config.js";
|
|
2
|
+
import { projectLoggieMessage } from "./loggie-projection.js";
|
|
2
3
|
export type SessionMetadata = {
|
|
3
4
|
sessionId: string;
|
|
4
5
|
provider?: string;
|
|
@@ -22,6 +23,13 @@ export type SessionProjectionInput = SessionMetadata & {
|
|
|
22
23
|
attachmentBudgetSkipped: number;
|
|
23
24
|
};
|
|
24
25
|
};
|
|
26
|
+
type ProjectedMessage = {
|
|
27
|
+
role: "user" | "assistant";
|
|
28
|
+
speaker: string;
|
|
29
|
+
text: string;
|
|
30
|
+
timestamp: number;
|
|
31
|
+
meeting?: ReturnType<typeof projectLoggieMessage>;
|
|
32
|
+
};
|
|
25
33
|
export type SessionSnippetMessage = {
|
|
26
34
|
type?: "user" | "assistant";
|
|
27
35
|
name?: string;
|
|
@@ -49,6 +57,7 @@ export type SessionContextSpans = {
|
|
|
49
57
|
end: number;
|
|
50
58
|
};
|
|
51
59
|
};
|
|
60
|
+
export declare function projectMessage(row: SessionProjectionInput["events"][number], input: SessionProjectionInput): ProjectedMessage | undefined;
|
|
52
61
|
export declare function projectSession(input: SessionProjectionInput): string | undefined;
|
|
53
62
|
export declare function projectSessionDocument(input: SessionProjectionInput): {
|
|
54
63
|
content: string;
|
|
@@ -67,3 +76,4 @@ export declare function sessionSnippetMessages(content: string, selected: {
|
|
|
67
76
|
}): SessionSnippetMessage[];
|
|
68
77
|
export declare function sessionDocumentPath(metadata: SessionMetadata): string;
|
|
69
78
|
export declare function resolveTimezone(configured?: string): string;
|
|
79
|
+
export {};
|
package/docs/configuration.md
CHANGED
|
@@ -176,6 +176,15 @@ Set this only after [installing the worker](retrieval.md#memory-analysis). No
|
|
|
176
176
|
clustering or curation schedule is created. Response-audit setup and a dry-run-first
|
|
177
177
|
workflow are in [its own guide](response-audit.md).
|
|
178
178
|
|
|
179
|
+
## Session memory extraction
|
|
180
|
+
|
|
181
|
+
See [the extraction guide](extraction.md) for source approval, host LLM policy,
|
|
182
|
+
single-session validation, backfill, and publication. Defaults: disabled,
|
|
183
|
+
`publish: false`, `chatTypes: []`, hourly, six overlap messages, five chunks per run.
|
|
184
|
+
`extraction.minSupport`, `minRetention`, and `minReplacement` independently accept
|
|
185
|
+
0–1 and default to 0.90. Settings are operator-controlled; reload/restart after edits.
|
|
186
|
+
Extraction always uses its own 10-second Jev deadline, not `typesafe.timeoutMs`.
|
|
187
|
+
|
|
179
188
|
## Settings reference
|
|
180
189
|
|
|
181
190
|
The tables show resolved defaults. For source-specific entries,
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Session memory extraction
|
|
2
|
+
|
|
3
|
+
Opt-in lasting-fact extraction: **Luna proposes facts → code checks exact quotes →
|
|
4
|
+
Jev checks support and retention → SQLite stores accepted facts → optional QMD search.**
|
|
5
|
+
It is independent of Inside Out: emotion scores do not select or strengthen memories.
|
|
6
|
+
It never writes `MEMORY.md`, daily Markdown, or people dossiers.
|
|
7
|
+
|
|
8
|
+
## Enable in shadow mode first
|
|
9
|
+
|
|
10
|
+
Merge this fragment under `plugins.entries.unblock-memory.config`, preserving
|
|
11
|
+
existing corpora and settings. Extraction chat types must be explicitly approved
|
|
12
|
+
and be a subset of the configured sessions corpus. This is channel agnostic.
|
|
13
|
+
|
|
14
|
+
```json
|
|
15
|
+
{
|
|
16
|
+
"corpora": [
|
|
17
|
+
{ "name": "memory", "kind": "files", "paths": ["MEMORY.md", "USER.md", "memory/**/*.md"] },
|
|
18
|
+
{ "name": "sessions", "kind": "sessions", "chatTypes": ["channel", "group"] }
|
|
19
|
+
],
|
|
20
|
+
"extraction": {
|
|
21
|
+
"enabled": true,
|
|
22
|
+
"publish": false,
|
|
23
|
+
"chatTypes": ["channel", "group"],
|
|
24
|
+
"intervalMinutes": 60,
|
|
25
|
+
"historyMessages": 6,
|
|
26
|
+
"maxBatches": 5,
|
|
27
|
+
"minSupport": 0.90,
|
|
28
|
+
"minRetention": 0.90,
|
|
29
|
+
"minReplacement": 0.90
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Configure [TypeSafe credentials](configuration.md#typesafe-and-whisperers).
|
|
35
|
+
Manual runs use the same Gateway-owned worker as scheduled runs; the Gateway must
|
|
36
|
+
be running and the CLI caller needs `operator.admin`. Reports can be read offline.
|
|
37
|
+
Requires host visible-transcript delta reads and working isolated LLM completion
|
|
38
|
+
(validated on OpenClaw 2026.9.2 with Codex 0.159.2). The native harness must be
|
|
39
|
+
installed through OpenClaw, not merely added to `plugins.load.paths`. Grant this
|
|
40
|
+
host policy **alongside `config`** in the plugin entry:
|
|
41
|
+
|
|
42
|
+
```json
|
|
43
|
+
{
|
|
44
|
+
"llm": {
|
|
45
|
+
"allowAgentIdOverride": true,
|
|
46
|
+
"allowModelOverride": true,
|
|
47
|
+
"allowedModels": ["openai/gpt-5.6-luna"],
|
|
48
|
+
"allowedCompletionModels": ["openai/gpt-5.6-luna"]
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Luna uses the host's configured agent runtime and credentials, with zero tools and
|
|
54
|
+
no alternate-model/provider fallback. Missing TypeSafe credentials prevent source
|
|
55
|
+
scans and model calls. Enabling approves sending selected conversation text to
|
|
56
|
+
Luna and Jev; tool calls/results and hidden thinking are removed first.
|
|
57
|
+
|
|
58
|
+
## Run, backfill, inspect
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
openclaw memory-extract run --agent main --since 2026-09-01 --session SESSION_ID
|
|
62
|
+
openclaw memory-extract report --agent main
|
|
63
|
+
openclaw memory-extract run --agent main --since 2026-09-01
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Without `--since`, the first enabled run establishes a live start time; historical
|
|
67
|
+
messages require an explicit backfill. Runs process at most `maxBatches` session
|
|
68
|
+
chunks; repeat the same command to continue. The approved backfill boundary and
|
|
69
|
+
cursors persist across restarts and scheduled runs. An earlier `--since` replays
|
|
70
|
+
history without deleting retained facts. Unchanged reruns make no model calls;
|
|
71
|
+
new messages resume from each session's cursor. Failed chunks do not advance it.
|
|
72
|
+
|
|
73
|
+
## Retention and thresholds
|
|
74
|
+
|
|
75
|
+
Retain explicit identities, relationships, ongoing responsibilities, stable
|
|
76
|
+
preferences, enduring project purposes, and adopted standing rules. Omit guesses,
|
|
77
|
+
hypotheticals, one-off tasks, temporary status, release/incident details, generic
|
|
78
|
+
technical knowledge, and credentials. The original transcripts remain searchable.
|
|
79
|
+
|
|
80
|
+
A blue guess followed by a human correction to red retains only the red preference.
|
|
81
|
+
Each proposal needs exact supporting quotes and new-message evidence. Jev checks
|
|
82
|
+
support and retention independently in one request; replacements also require an
|
|
83
|
+
explicit update to the same fact. All applicable thresholds must pass, and support
|
|
84
|
+
must select `supported`, even if its threshold is lowered.
|
|
85
|
+
|
|
86
|
+
The three thresholds accept 0–1 and default to 0.90. Lowering retention changes
|
|
87
|
+
certainty about category membership, not the categories themselves. Agents can
|
|
88
|
+
edit configuration when instructed, then reload/restart the Gateway; they must
|
|
89
|
+
not silently lower thresholds. Decisions store their probabilities and thresholds
|
|
90
|
+
for inspection. Changing thresholds affects future processing, not existing records;
|
|
91
|
+
rejected proposals are not kept for automatic regrading.
|
|
92
|
+
|
|
93
|
+
## Storage, search, and limits
|
|
94
|
+
|
|
95
|
+
Three tables in the existing agent `unblock-memory.sqlite` store worker state,
|
|
96
|
+
session checkpoints, and versioned facts/evidence/judgments. Facts and checkpoints
|
|
97
|
+
commit together. Deduplication/corrections are per session, not cross-session;
|
|
98
|
+
older evidence cannot overwrite a newer observation. Source resets/deletions
|
|
99
|
+
withdraw affected facts when the worker next checks them. Reports include accepted
|
|
100
|
+
facts, judgments, checkpoints, and source/proposal/validation/commit failure stages
|
|
101
|
+
without leaking provider errors or source text into error messages.
|
|
102
|
+
|
|
103
|
+
`publish: false` stores accepted facts for evaluation only. `publish: true` indexes
|
|
104
|
+
them in the existing QMD `index.sqlite`; `memory_search` includes them by default,
|
|
105
|
+
or use `corpora: ["extracted"]`. `memory_get` checks the current active revision.
|
|
106
|
+
No automatic whisperer/dossier changes are introduced. Published facts can be
|
|
107
|
+
retrieved across this agent's audiences; this is **not per-person access control**.
|
|
108
|
+
Do not approve private conversations for a shared agent unless sharing is intended.
|
|
109
|
+
|
|
110
|
+
The Gateway owns the scheduler; no cron/launchd job is added. Interval 0 means
|
|
111
|
+
manual-only; the maximum is 1,440 minutes (daily). Defaults are disabled, unpublished,
|
|
112
|
+
no approved chat types, hourly, six overlap messages, and five chunks per run.
|
|
113
|
+
|
|
114
|
+
Chunks target 24k input tokens and cap at 28k, including Luna instructions, prior
|
|
115
|
+
facts and overlap, leaving room for Jev's 32k state-plus-longest-question limit.
|
|
116
|
+
`o200k_base` is a budgeting proxy, not an exact Jev/host token count. Quotes are
|
|
117
|
+
limited to 512 characters each. Up to six overlap messages fit within 8k tokens by
|
|
118
|
+
default. Oversized messages split losslessly with resumable offsets and original
|
|
119
|
+
message-ID citations. A session above 100 retained facts or 40k characters of prior
|
|
120
|
+
facts pauses rather than silently dropping comparison context. The output limit
|
|
121
|
+
is advisory on some host runtimes; malformed/truncated JSON is retried, never saved.
|
|
122
|
+
|
|
123
|
+
This is a conservative v1, not a calibrated guarantee of memory recall or truth.
|
|
124
|
+
Start with a single-session shadow run and inspect the retained facts before publishing.
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "unblock-memory",
|
|
3
3
|
"name": "Unblock Memory",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.7.0",
|
|
5
5
|
"description": "Indexes, retrieves, and analyzes configured workspace memory with existing QMD vectors.",
|
|
6
6
|
"kind": "memory",
|
|
7
7
|
"activation": { "onStartup": true },
|
|
@@ -119,6 +119,20 @@
|
|
|
119
119
|
"maxContextTokens": { "type": "integer", "minimum": 256, "maximum": 30000, "default": 30000 }
|
|
120
120
|
}
|
|
121
121
|
},
|
|
122
|
+
"extraction": {
|
|
123
|
+
"type": "object", "additionalProperties": false,
|
|
124
|
+
"properties": {
|
|
125
|
+
"enabled": { "type": "boolean", "default": false },
|
|
126
|
+
"publish": { "type": "boolean", "default": false },
|
|
127
|
+
"intervalMinutes": { "type": "integer", "minimum": 0, "maximum": 1440, "default": 60 },
|
|
128
|
+
"chatTypes": { "type": "array", "items": { "enum": ["direct", "channel", "group"] }, "default": [] },
|
|
129
|
+
"historyMessages": { "type": "integer", "minimum": 0, "maximum": 20, "default": 6 },
|
|
130
|
+
"maxBatches": { "type": "integer", "minimum": 1, "maximum": 50, "default": 5 },
|
|
131
|
+
"minSupport": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.9 },
|
|
132
|
+
"minRetention": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.9 },
|
|
133
|
+
"minReplacement": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.9 }
|
|
134
|
+
}
|
|
135
|
+
},
|
|
122
136
|
"responseAudit": {
|
|
123
137
|
"type": "object", "additionalProperties": false,
|
|
124
138
|
"properties": {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@unblocklabs/unblock-memory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Workspace-native memory for OpenClaw, powered by QMD",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"docs/peoplesql.md",
|
|
31
31
|
"docs/response-audit.md",
|
|
32
32
|
"docs/memory-training.md",
|
|
33
|
+
"docs/extraction.md",
|
|
33
34
|
"openclaw.plugin.json"
|
|
34
35
|
],
|
|
35
36
|
"scripts": {
|
|
@@ -47,6 +48,7 @@
|
|
|
47
48
|
"@huggingface/tokenizers": "0.2.0",
|
|
48
49
|
"@unblocklabs/qmd": "https://github.com/unblocklabs-ai/qmd/releases/download/v2.10.3/unblocklabs-qmd-2.10.3.tgz",
|
|
49
50
|
"chokidar": "5.0.0",
|
|
51
|
+
"js-tiktoken": "1.0.21",
|
|
50
52
|
"picomatch": "^4.0.5",
|
|
51
53
|
"typebox": "1.3.6"
|
|
52
54
|
},
|