@totalreclaw/totalreclaw 1.6.0 → 3.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAWHUB.md +134 -0
- package/README.md +407 -64
- package/SKILL.md +1032 -0
- package/api-client.ts +5 -5
- package/claims-helper.ts +686 -0
- package/config.ts +211 -0
- package/consolidation.ts +141 -33
- package/contradiction-sync.ts +1389 -0
- package/crypto.ts +63 -261
- package/digest-sync.ts +516 -0
- package/embedding.ts +69 -46
- package/extractor.ts +1307 -84
- package/hot-cache-wrapper.ts +1 -1
- package/import-adapters/gemini-adapter.ts +243 -0
- package/import-adapters/index.ts +3 -0
- package/import-adapters/types.ts +1 -1
- package/index.ts +1887 -323
- package/llm-client.ts +106 -53
- package/lsh.ts +21 -210
- package/package.json +20 -7
- package/pin.ts +502 -0
- package/reranker.ts +96 -124
- package/skill.json +213 -0
- package/subgraph-search.ts +112 -5
- package/subgraph-store.ts +559 -275
- package/consolidation.test.ts +0 -356
- package/extractor-dedup.test.ts +0 -168
- package/import-adapters/import-adapters.test.ts +0 -1123
- package/lsh.test.ts +0 -463
- package/pocv2-e2e-test.ts +0 -917
- package/porter-stemmer.d.ts +0 -4
- package/reranker.test.ts +0 -594
- package/semantic-dedup.test.ts +0 -392
- package/setup.sh +0 -19
- package/store-dedup-wiring.test.ts +0 -186
package/index.ts
CHANGED
|
@@ -8,14 +8,17 @@
|
|
|
8
8
|
* - totalreclaw_export -- export all memories (JSON or Markdown)
|
|
9
9
|
* - totalreclaw_status -- check billing/subscription status
|
|
10
10
|
* - totalreclaw_consolidate -- scan and merge near-duplicate memories
|
|
11
|
+
* - totalreclaw_pin -- pin a memory so auto-resolution can never supersede it
|
|
12
|
+
* - totalreclaw_unpin -- remove a pin, returning the memory to active status
|
|
11
13
|
* - totalreclaw_import_from -- import memories from other tools (Mem0, MCP Memory, etc.)
|
|
12
14
|
* - totalreclaw_upgrade -- create Stripe checkout for Pro upgrade
|
|
13
15
|
* - totalreclaw_migrate -- migrate testnet memories to mainnet after Pro upgrade
|
|
16
|
+
* - totalreclaw_setup -- initialize with recovery phrase (no gateway restart needed)
|
|
14
17
|
*
|
|
15
18
|
* Also registers a `before_agent_start` hook that automatically injects
|
|
16
19
|
* relevant memories into the agent's context.
|
|
17
20
|
*
|
|
18
|
-
* All data is encrypted client-side with
|
|
21
|
+
* All data is encrypted client-side with XChaCha20-Poly1305. The server never
|
|
19
22
|
* sees plaintext.
|
|
20
23
|
*/
|
|
21
24
|
|
|
@@ -29,8 +32,24 @@ import {
|
|
|
29
32
|
generateContentFingerprint,
|
|
30
33
|
} from './crypto.js';
|
|
31
34
|
import { createApiClient, type StoreFactPayload } from './api-client.js';
|
|
32
|
-
import {
|
|
33
|
-
|
|
35
|
+
import {
|
|
36
|
+
extractFacts,
|
|
37
|
+
extractDebrief,
|
|
38
|
+
isValidMemoryType,
|
|
39
|
+
parseEntity,
|
|
40
|
+
VALID_MEMORY_TYPES,
|
|
41
|
+
LEGACY_V0_MEMORY_TYPES,
|
|
42
|
+
VALID_MEMORY_SOURCES,
|
|
43
|
+
VALID_MEMORY_SCOPES,
|
|
44
|
+
EXTRACTION_SYSTEM_PROMPT,
|
|
45
|
+
extractFactsForCompaction,
|
|
46
|
+
type ExtractedFact,
|
|
47
|
+
type ExtractedEntity,
|
|
48
|
+
type MemoryType,
|
|
49
|
+
type MemorySource,
|
|
50
|
+
type MemoryScope,
|
|
51
|
+
} from './extractor.js';
|
|
52
|
+
import { initLLMClient, resolveLLMConfig, chatCompletion, generateEmbedding, getEmbeddingDims } from './llm-client.js';
|
|
34
53
|
import { LSHHasher } from './lsh.js';
|
|
35
54
|
import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS, type RerankerCandidate } from './reranker.js';
|
|
36
55
|
import { deduplicateBatch } from './semantic-dedup.js';
|
|
@@ -43,9 +62,39 @@ import {
|
|
|
43
62
|
STORE_DEDUP_MAX_CANDIDATES,
|
|
44
63
|
type DecryptedCandidate,
|
|
45
64
|
} from './consolidation.js';
|
|
46
|
-
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, type FactPayload } from './subgraph-store.js';
|
|
47
|
-
import {
|
|
65
|
+
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4, type FactPayload } from './subgraph-store.js';
|
|
66
|
+
import {
|
|
67
|
+
DIGEST_TRAPDOOR,
|
|
68
|
+
buildCanonicalClaim,
|
|
69
|
+
computeEntityTrapdoor,
|
|
70
|
+
computeEntityTrapdoors,
|
|
71
|
+
isDigestBlob,
|
|
72
|
+
normalizeToV1Type,
|
|
73
|
+
readClaimFromBlob,
|
|
74
|
+
resolveDigestMode,
|
|
75
|
+
type DigestMode,
|
|
76
|
+
} from './claims-helper.js';
|
|
77
|
+
import {
|
|
78
|
+
maybeInjectDigest,
|
|
79
|
+
recompileDigest,
|
|
80
|
+
fetchAllActiveClaims,
|
|
81
|
+
isRecompileInProgress,
|
|
82
|
+
tryBeginRecompile,
|
|
83
|
+
endRecompile,
|
|
84
|
+
} from './digest-sync.js';
|
|
85
|
+
import {
|
|
86
|
+
detectAndResolveContradictions,
|
|
87
|
+
runWeightTuningLoop,
|
|
88
|
+
type ResolutionDecision as ContradictionDecision,
|
|
89
|
+
} from './contradiction-sync.js';
|
|
90
|
+
import { searchSubgraph, searchSubgraphBroadened, getSubgraphFactCount, fetchFactById } from './subgraph-search.js';
|
|
91
|
+
import {
|
|
92
|
+
executePinOperation,
|
|
93
|
+
validatePinArgs,
|
|
94
|
+
type PinOpDeps,
|
|
95
|
+
} from './pin.js';
|
|
48
96
|
import { PluginHotCache, type HotFact } from './hot-cache-wrapper.js';
|
|
97
|
+
import { CONFIG, setRecoveryPhraseOverride, setChainIdOverride } from './config.js';
|
|
49
98
|
import crypto from 'node:crypto';
|
|
50
99
|
import fs from 'node:fs';
|
|
51
100
|
import path from 'node:path';
|
|
@@ -68,6 +117,16 @@ interface OpenClawPluginApi {
|
|
|
68
117
|
};
|
|
69
118
|
};
|
|
70
119
|
};
|
|
120
|
+
models?: {
|
|
121
|
+
providers?: Record<string, {
|
|
122
|
+
baseUrl: string;
|
|
123
|
+
apiKey?: string;
|
|
124
|
+
api?: string;
|
|
125
|
+
models?: Array<{ id: string; [k: string]: unknown }>;
|
|
126
|
+
[k: string]: unknown;
|
|
127
|
+
}>;
|
|
128
|
+
[k: string]: unknown;
|
|
129
|
+
};
|
|
71
130
|
[key: string]: unknown;
|
|
72
131
|
};
|
|
73
132
|
pluginConfig?: Record<string, unknown>;
|
|
@@ -76,12 +135,44 @@ interface OpenClawPluginApi {
|
|
|
76
135
|
on(hookName: string, handler: (...args: unknown[]) => unknown, opts?: { priority?: number }): void;
|
|
77
136
|
}
|
|
78
137
|
|
|
138
|
+
// ---------------------------------------------------------------------------
|
|
139
|
+
// Human-friendly error messages
|
|
140
|
+
// ---------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Translate technical error messages from the on-chain submission pipeline
|
|
144
|
+
* into user-friendly messages. The original technical details are still
|
|
145
|
+
* logged via api.logger — this only affects what the agent sees.
|
|
146
|
+
*/
|
|
147
|
+
function humanizeError(rawMessage: string): string {
|
|
148
|
+
if (rawMessage.includes('AA23')) {
|
|
149
|
+
return 'Memory storage temporarily unavailable. Will retry next time.';
|
|
150
|
+
}
|
|
151
|
+
if (rawMessage.includes('AA10')) {
|
|
152
|
+
return 'Please wait a moment before storing more memories.';
|
|
153
|
+
}
|
|
154
|
+
if (rawMessage.includes('AA25')) {
|
|
155
|
+
return 'Memory storage busy. Will retry.';
|
|
156
|
+
}
|
|
157
|
+
if (rawMessage.includes('pm_sponsorUserOperation')) {
|
|
158
|
+
return 'Memory storage service temporarily unavailable.';
|
|
159
|
+
}
|
|
160
|
+
if (/Relay returned HTTP\s*404/.test(rawMessage)) {
|
|
161
|
+
return 'Memory service is temporarily offline.';
|
|
162
|
+
}
|
|
163
|
+
if (/Relay returned HTTP\s*5\d\d/.test(rawMessage)) {
|
|
164
|
+
return 'Memory service encountered a temporary error. Will retry next time.';
|
|
165
|
+
}
|
|
166
|
+
// Pass through non-technical messages as-is.
|
|
167
|
+
return rawMessage;
|
|
168
|
+
}
|
|
169
|
+
|
|
79
170
|
// ---------------------------------------------------------------------------
|
|
80
171
|
// Persistent credential storage
|
|
81
172
|
// ---------------------------------------------------------------------------
|
|
82
173
|
|
|
83
174
|
/** Path where we persist userId + salt across restarts. */
|
|
84
|
-
const CREDENTIALS_PATH =
|
|
175
|
+
const CREDENTIALS_PATH = CONFIG.credentialsPath;
|
|
85
176
|
|
|
86
177
|
// ---------------------------------------------------------------------------
|
|
87
178
|
// Cosine similarity threshold — skip injection when top result is below this
|
|
@@ -92,12 +183,10 @@ const CREDENTIALS_PATH = process.env.TOTALRECLAW_CREDENTIALS_PATH || `${process.
|
|
|
92
183
|
* memories into context. Below this threshold, the query is considered
|
|
93
184
|
* irrelevant to any stored memories and results are suppressed.
|
|
94
185
|
*
|
|
95
|
-
* Default 0.15 is tuned for
|
|
186
|
+
* Default 0.15 is tuned for local ONNX models which produce lower
|
|
96
187
|
* similarity scores than OpenAI models. Configurable via env var.
|
|
97
188
|
*/
|
|
98
|
-
const COSINE_THRESHOLD =
|
|
99
|
-
process.env.TOTALRECLAW_COSINE_THRESHOLD ?? '0.15',
|
|
100
|
-
);
|
|
189
|
+
const COSINE_THRESHOLD = CONFIG.cosineThreshold;
|
|
101
190
|
|
|
102
191
|
// ---------------------------------------------------------------------------
|
|
103
192
|
// Module-level state (persists across tool calls within a session)
|
|
@@ -123,30 +212,36 @@ let lastSearchTimestamp = 0;
|
|
|
123
212
|
let lastQueryEmbedding: number[] | null = null;
|
|
124
213
|
|
|
125
214
|
// Feature flags — configurable for A/B testing
|
|
126
|
-
const CACHE_TTL_MS =
|
|
127
|
-
const SEMANTIC_SKIP_THRESHOLD =
|
|
215
|
+
const CACHE_TTL_MS = CONFIG.cacheTtlMs;
|
|
216
|
+
const SEMANTIC_SKIP_THRESHOLD = CONFIG.semanticSkipThreshold;
|
|
128
217
|
|
|
129
218
|
// Auto-extract throttle (C3): only extract every N turns in agent_end hook
|
|
130
219
|
let turnsSinceLastExtraction = 0;
|
|
131
|
-
|
|
220
|
+
|
|
221
|
+
// BUG-2 fix: Skip agent_end extraction during import operations.
|
|
222
|
+
// Import failures previously triggered agent_end → re-extraction → re-import loops.
|
|
223
|
+
let _importInProgress = false;
|
|
224
|
+
const AUTO_EXTRACT_EVERY_TURNS_ENV = CONFIG.extractInterval;
|
|
132
225
|
|
|
133
226
|
// Hard cap on facts per extraction to prevent LLM over-extraction from dense conversations
|
|
134
227
|
const MAX_FACTS_PER_EXTRACTION = 15;
|
|
135
228
|
|
|
136
|
-
// Store-time near-duplicate detection
|
|
137
|
-
|
|
229
|
+
// Store-time near-duplicate detection is always ON in v1.
|
|
230
|
+
// The TOTALRECLAW_STORE_DEDUP env var was removed.
|
|
231
|
+
const STORE_DEDUP_ENABLED = true;
|
|
138
232
|
|
|
139
233
|
// One-time welcome-back message for returning Pro users (set during init, consumed by first before_agent_start)
|
|
140
234
|
let welcomeBackMessage: string | null = null;
|
|
141
235
|
|
|
142
|
-
// B2:
|
|
143
|
-
|
|
236
|
+
// B2: COSINE_THRESHOLD (above) is the single relevance gate for both
|
|
237
|
+
// the before_agent_start hook and the recall tool. The former "RELEVANCE_THRESHOLD"
|
|
238
|
+
// (0.3) was too aggressive and silently suppressed auto-recall at session start.
|
|
144
239
|
|
|
145
240
|
// ---------------------------------------------------------------------------
|
|
146
241
|
// Billing cache infrastructure
|
|
147
242
|
// ---------------------------------------------------------------------------
|
|
148
243
|
|
|
149
|
-
const BILLING_CACHE_PATH =
|
|
244
|
+
const BILLING_CACHE_PATH = CONFIG.billingCachePath;
|
|
150
245
|
const BILLING_CACHE_TTL = 2 * 60 * 60 * 1000; // 2 hours
|
|
151
246
|
const QUOTA_WARNING_THRESHOLD = 0.8; // 80%
|
|
152
247
|
|
|
@@ -165,11 +260,34 @@ interface BillingCache {
|
|
|
165
260
|
checked_at: number;
|
|
166
261
|
}
|
|
167
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Apply the billing tier to the runtime chain override.
|
|
265
|
+
*
|
|
266
|
+
* Pro tier → chain 100 (Gnosis mainnet). Free tier (or unknown) stays on
|
|
267
|
+
* 84532 (Base Sepolia). The relay routes Pro UserOps to Gnosis, so the
|
|
268
|
+
* client MUST sign them against chain 100 — otherwise the bundler returns
|
|
269
|
+
* AA23 (invalid signature). See MCP's equivalent path in mcp/src/index.ts.
|
|
270
|
+
*
|
|
271
|
+
* Called from `readBillingCache` and `writeBillingCache` so that every cache
|
|
272
|
+
* read or write keeps the chain override in sync with the cached tier.
|
|
273
|
+
* Idempotent — calling with the same tier is a no-op.
|
|
274
|
+
*/
|
|
275
|
+
function syncChainIdFromTier(tier: string | undefined): void {
|
|
276
|
+
if (tier === 'pro') {
|
|
277
|
+
setChainIdOverride(100);
|
|
278
|
+
} else {
|
|
279
|
+
// Free or unknown → reset to the default free-tier chain.
|
|
280
|
+
setChainIdOverride(84532);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
168
284
|
function readBillingCache(): BillingCache | null {
|
|
169
285
|
try {
|
|
170
286
|
if (!fs.existsSync(BILLING_CACHE_PATH)) return null;
|
|
171
287
|
const raw = JSON.parse(fs.readFileSync(BILLING_CACHE_PATH, 'utf-8')) as BillingCache;
|
|
172
288
|
if (!raw.checked_at || Date.now() - raw.checked_at > BILLING_CACHE_TTL) return null;
|
|
289
|
+
// Keep chain override in sync with persisted tier across process restarts.
|
|
290
|
+
syncChainIdFromTier(raw.tier);
|
|
173
291
|
return raw;
|
|
174
292
|
} catch {
|
|
175
293
|
return null;
|
|
@@ -184,18 +302,22 @@ function writeBillingCache(cache: BillingCache): void {
|
|
|
184
302
|
} catch {
|
|
185
303
|
// Best-effort — don't block on cache write failure.
|
|
186
304
|
}
|
|
305
|
+
// Sync chain override AFTER the write so in-process UserOp signing picks
|
|
306
|
+
// up the correct chain immediately, even if the disk write failed.
|
|
307
|
+
syncChainIdFromTier(cache.tier);
|
|
187
308
|
}
|
|
188
309
|
|
|
189
310
|
/**
|
|
190
|
-
* Check if LLM-guided dedup is enabled
|
|
191
|
-
*
|
|
311
|
+
* Check if LLM-guided dedup is enabled.
|
|
312
|
+
*
|
|
313
|
+
* Always returns true — LLM extraction runs client-side using the user's
|
|
314
|
+
* own API key, so there is no cost to us. The server flag is respected as
|
|
315
|
+
* a kill-switch but defaults to true for all tiers.
|
|
192
316
|
*/
|
|
193
317
|
function isLlmDedupEnabled(): boolean {
|
|
194
318
|
const cache = readBillingCache();
|
|
195
|
-
if (
|
|
196
|
-
|
|
197
|
-
if (cache.features?.llm_dedup !== undefined) return cache.features.llm_dedup;
|
|
198
|
-
return false;
|
|
319
|
+
if (cache?.features?.llm_dedup === false) return false; // Server kill-switch
|
|
320
|
+
return true;
|
|
199
321
|
}
|
|
200
322
|
|
|
201
323
|
/**
|
|
@@ -241,7 +363,7 @@ const MEMORY_HEADER = `# Memory
|
|
|
241
363
|
|
|
242
364
|
function ensureMemoryHeader(logger: OpenClawPluginApi['logger']): void {
|
|
243
365
|
try {
|
|
244
|
-
const workspace =
|
|
366
|
+
const workspace = CONFIG.openclawWorkspace;
|
|
245
367
|
const memoryMd = path.join(workspace, 'MEMORY.md');
|
|
246
368
|
|
|
247
369
|
if (fs.existsSync(memoryMd)) {
|
|
@@ -340,9 +462,8 @@ let firstRunAfterInit = true;
|
|
|
340
462
|
* register with the server if this is the first run.
|
|
341
463
|
*/
|
|
342
464
|
async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
343
|
-
const serverUrl =
|
|
344
|
-
|
|
345
|
-
const masterPassword = process.env.TOTALRECLAW_RECOVERY_PHRASE;
|
|
465
|
+
const serverUrl = CONFIG.serverUrl || 'https://api.totalreclaw.xyz';
|
|
466
|
+
const masterPassword = CONFIG.recoveryPhrase;
|
|
346
467
|
|
|
347
468
|
if (!masterPassword) {
|
|
348
469
|
needsSetup = true;
|
|
@@ -359,7 +480,14 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
|
359
480
|
try {
|
|
360
481
|
if (fs.existsSync(CREDENTIALS_PATH)) {
|
|
361
482
|
const creds = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8'));
|
|
362
|
-
|
|
483
|
+
// Salt may be stored as base64 (plugin-written) or hex (MCP setup-written).
|
|
484
|
+
// Detect format: hex strings are 64 chars of [0-9a-f], base64 uses [A-Z+/=].
|
|
485
|
+
const saltStr: string = creds.salt;
|
|
486
|
+
if (saltStr && /^[0-9a-f]{64}$/i.test(saltStr)) {
|
|
487
|
+
existingSalt = Buffer.from(saltStr, 'hex');
|
|
488
|
+
} else if (saltStr) {
|
|
489
|
+
existingSalt = Buffer.from(saltStr, 'base64');
|
|
490
|
+
}
|
|
363
491
|
existingUserId = creds.userId;
|
|
364
492
|
logger.info(`Loaded existing credentials for user ${existingUserId}`);
|
|
365
493
|
}
|
|
@@ -380,6 +508,20 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
|
380
508
|
if (existingUserId) {
|
|
381
509
|
userId = existingUserId;
|
|
382
510
|
logger.info(`Authenticated as user ${userId}`);
|
|
511
|
+
|
|
512
|
+
// Idempotent registration — ensure auth key is registered with the relay.
|
|
513
|
+
// Without this, returning users get 401 if the relay database was reset or
|
|
514
|
+
// if credentials were created by the MCP setup CLI (different process).
|
|
515
|
+
try {
|
|
516
|
+
const authHash = computeAuthKeyHash(keys.authKey);
|
|
517
|
+
const saltHex = keys.salt.toString('hex');
|
|
518
|
+
await apiClient.register(authHash, saltHex);
|
|
519
|
+
} catch {
|
|
520
|
+
// Best-effort — relay returns 200 for already-registered users.
|
|
521
|
+
// Only fails on network errors; bearer token auth still works if
|
|
522
|
+
// a prior registration succeeded.
|
|
523
|
+
logger.warn('Idempotent relay registration failed (best-effort, will retry on next start)');
|
|
524
|
+
}
|
|
383
525
|
} else {
|
|
384
526
|
// First run -- register with the server.
|
|
385
527
|
const authHash = computeAuthKeyHash(keys.authKey);
|
|
@@ -405,14 +547,20 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
|
405
547
|
userId = registeredUserId!;
|
|
406
548
|
|
|
407
549
|
// Persist credentials so we can resume later.
|
|
550
|
+
// Include the mnemonic so hot-reload works without env var.
|
|
408
551
|
const dir = path.dirname(CREDENTIALS_PATH);
|
|
409
552
|
if (!fs.existsSync(dir)) {
|
|
410
553
|
fs.mkdirSync(dir, { recursive: true });
|
|
411
554
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
555
|
+
const credsToSave: Record<string, string> = {
|
|
556
|
+
userId,
|
|
557
|
+
salt: keys.salt.toString('base64'),
|
|
558
|
+
};
|
|
559
|
+
// Only persist mnemonic if we have one (avoid writing empty string).
|
|
560
|
+
if (masterPassword) {
|
|
561
|
+
credsToSave.mnemonic = masterPassword;
|
|
562
|
+
}
|
|
563
|
+
fs.writeFileSync(CREDENTIALS_PATH, JSON.stringify(credsToSave), { mode: 0o600 });
|
|
416
564
|
|
|
417
565
|
logger.info(`Registered new user: ${userId}`);
|
|
418
566
|
}
|
|
@@ -436,7 +584,7 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
|
436
584
|
try {
|
|
437
585
|
const walletAddr = subgraphOwner || userId || '';
|
|
438
586
|
if (walletAddr) {
|
|
439
|
-
const billingUrl =
|
|
587
|
+
const billingUrl = CONFIG.serverUrl;
|
|
440
588
|
const resp = await fetch(`${billingUrl}/v1/billing/status?wallet_address=${encodeURIComponent(walletAddr)}`, {
|
|
441
589
|
method: 'GET',
|
|
442
590
|
headers: {
|
|
@@ -479,6 +627,13 @@ function isDocker(): boolean {
|
|
|
479
627
|
}
|
|
480
628
|
|
|
481
629
|
function buildSetupErrorMsg(): string {
|
|
630
|
+
return 'TotalReclaw setup required. Use the `totalreclaw_setup` tool with a 12-word BIP-39 recovery phrase.\n\n' +
|
|
631
|
+
'1. Ask the user if they have an existing recovery phrase, or generate a new one with `npx @totalreclaw/mcp-server setup`.\n' +
|
|
632
|
+
'2. Call `totalreclaw_setup` with the phrase — no gateway restart needed.\n' +
|
|
633
|
+
' (Optional: set TOTALRECLAW_SELF_HOSTED=true if using your own server instead of the managed service.)';
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function buildSetupErrorMsgLegacy(): string {
|
|
482
637
|
const base =
|
|
483
638
|
'TotalReclaw setup required:\n' +
|
|
484
639
|
'1. Set TOTALRECLAW_RECOVERY_PHRASE — ask the user if they have an existing recovery phrase or generate a new 12-word BIP-39 mnemonic.\n' +
|
|
@@ -509,12 +664,101 @@ const SETUP_ERROR_MSG = buildSetupErrorMsg();
|
|
|
509
664
|
|
|
510
665
|
/**
|
|
511
666
|
* Ensure `initialize()` has completed (runs at most once).
|
|
667
|
+
*
|
|
668
|
+
* If `needsSetup` is true after init, attempts a hot-reload from
|
|
669
|
+
* credentials.json in case the mnemonic was written there by a
|
|
670
|
+
* `totalreclaw_setup` tool call or `npx @totalreclaw/mcp-server setup`.
|
|
512
671
|
*/
|
|
513
672
|
async function ensureInitialized(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
514
673
|
if (!initPromise) {
|
|
515
674
|
initPromise = initialize(logger);
|
|
516
675
|
}
|
|
517
676
|
await initPromise;
|
|
677
|
+
|
|
678
|
+
// Hot-reload: if setup is still needed, check if credentials.json
|
|
679
|
+
// now has a mnemonic (written by totalreclaw_setup or MCP setup CLI).
|
|
680
|
+
if (needsSetup) {
|
|
681
|
+
await attemptHotReload(logger);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Attempt to hot-reload credentials from credentials.json.
|
|
687
|
+
*
|
|
688
|
+
* Called when `needsSetup` is true — checks if credentials.json contains
|
|
689
|
+
* a mnemonic (written by the `totalreclaw_setup` tool or MCP setup CLI).
|
|
690
|
+
* If found, re-derives keys and completes initialization without requiring
|
|
691
|
+
* a gateway restart.
|
|
692
|
+
*/
|
|
693
|
+
async function attemptHotReload(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
694
|
+
try {
|
|
695
|
+
if (!fs.existsSync(CREDENTIALS_PATH)) return;
|
|
696
|
+
|
|
697
|
+
const creds = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8'));
|
|
698
|
+
if (!creds.mnemonic) return;
|
|
699
|
+
|
|
700
|
+
logger.info('Hot-reloading credentials from credentials.json (no restart needed)');
|
|
701
|
+
|
|
702
|
+
// Set the runtime override so CONFIG.recoveryPhrase returns the mnemonic.
|
|
703
|
+
setRecoveryPhraseOverride(creds.mnemonic);
|
|
704
|
+
|
|
705
|
+
// Re-run initialization with the newly available mnemonic.
|
|
706
|
+
needsSetup = false;
|
|
707
|
+
initPromise = initialize(logger);
|
|
708
|
+
await initPromise;
|
|
709
|
+
} catch (err) {
|
|
710
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
711
|
+
logger.warn(`Hot-reload from credentials.json failed: ${msg}`);
|
|
712
|
+
// Leave needsSetup as true — user will see the setup prompt.
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/**
|
|
717
|
+
* Force re-initialization with a specific mnemonic.
|
|
718
|
+
*
|
|
719
|
+
* Called by the `totalreclaw_setup` tool. Clears stale credentials from
|
|
720
|
+
* disk so that `initialize()` treats this as a fresh registration and
|
|
721
|
+
* persists the NEW mnemonic + freshly derived salt/userId.
|
|
722
|
+
*
|
|
723
|
+
* Without clearing credentials.json first, `initialize()` would load the
|
|
724
|
+
* OLD salt and userId, derive keys from (new mnemonic + old salt), skip
|
|
725
|
+
* writing credentials (because existingUserId is set), and the new
|
|
726
|
+
* mnemonic would never be persisted — a critical data-loss bug.
|
|
727
|
+
*/
|
|
728
|
+
async function forceReinitialization(mnemonic: string, logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
729
|
+
// Set the runtime override so CONFIG.recoveryPhrase returns this mnemonic.
|
|
730
|
+
setRecoveryPhraseOverride(mnemonic);
|
|
731
|
+
|
|
732
|
+
// CRITICAL: Remove stale credentials so initialize() does a fresh
|
|
733
|
+
// registration with a new salt. If we leave the old file, initialize()
|
|
734
|
+
// loads the old salt + userId and never writes the new mnemonic.
|
|
735
|
+
try {
|
|
736
|
+
if (fs.existsSync(CREDENTIALS_PATH)) {
|
|
737
|
+
fs.unlinkSync(CREDENTIALS_PATH);
|
|
738
|
+
logger.info('Cleared stale credentials.json for fresh setup');
|
|
739
|
+
}
|
|
740
|
+
} catch (err) {
|
|
741
|
+
logger.warn(`Could not remove old credentials.json: ${err instanceof Error ? err.message : String(err)}`);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// Reset module state for a clean re-init.
|
|
745
|
+
needsSetup = false;
|
|
746
|
+
authKeyHex = null;
|
|
747
|
+
encryptionKey = null;
|
|
748
|
+
dedupKey = null;
|
|
749
|
+
userId = null;
|
|
750
|
+
subgraphOwner = null;
|
|
751
|
+
apiClient = null;
|
|
752
|
+
lshHasher = null;
|
|
753
|
+
lshInitFailed = false;
|
|
754
|
+
masterPasswordCache = null;
|
|
755
|
+
saltCache = null;
|
|
756
|
+
pluginHotCache = null;
|
|
757
|
+
firstRunAfterInit = true;
|
|
758
|
+
|
|
759
|
+
// Re-run initialization — will register fresh and persist new credentials.
|
|
760
|
+
initPromise = initialize(logger);
|
|
761
|
+
await initPromise;
|
|
518
762
|
}
|
|
519
763
|
|
|
520
764
|
/**
|
|
@@ -634,7 +878,8 @@ async function searchForNearDuplicates(
|
|
|
634
878
|
for (const result of results) {
|
|
635
879
|
try {
|
|
636
880
|
const docJson = decryptFromHex(result.encryptedBlob, encryptionKey);
|
|
637
|
-
|
|
881
|
+
if (isDigestBlob(docJson)) continue;
|
|
882
|
+
const doc = readClaimFromBlob(docJson);
|
|
638
883
|
|
|
639
884
|
let embedding: number[] | null = null;
|
|
640
885
|
if (result.encryptedEmbedding) {
|
|
@@ -647,9 +892,7 @@ async function searchForNearDuplicates(
|
|
|
647
892
|
id: result.id,
|
|
648
893
|
text: doc.text,
|
|
649
894
|
embedding,
|
|
650
|
-
importance: doc.
|
|
651
|
-
? Math.round((doc.metadata.importance as number) * 10)
|
|
652
|
-
: 5,
|
|
895
|
+
importance: doc.importance,
|
|
653
896
|
decayScore: 5,
|
|
654
897
|
createdAt: result.timestamp ? parseInt(result.timestamp, 10) * 1000 : Date.now(),
|
|
655
898
|
version: 1,
|
|
@@ -666,7 +909,8 @@ async function searchForNearDuplicates(
|
|
|
666
909
|
for (const candidate of candidates) {
|
|
667
910
|
try {
|
|
668
911
|
const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey);
|
|
669
|
-
|
|
912
|
+
if (isDigestBlob(docJson)) continue;
|
|
913
|
+
const doc = readClaimFromBlob(docJson);
|
|
670
914
|
|
|
671
915
|
let embedding: number[] | null = null;
|
|
672
916
|
if (candidate.encrypted_embedding) {
|
|
@@ -679,9 +923,7 @@ async function searchForNearDuplicates(
|
|
|
679
923
|
id: candidate.fact_id,
|
|
680
924
|
text: doc.text,
|
|
681
925
|
embedding,
|
|
682
|
-
importance: doc.
|
|
683
|
-
? Math.round((doc.metadata.importance as number) * 10)
|
|
684
|
-
: 5,
|
|
926
|
+
importance: doc.importance,
|
|
685
927
|
decayScore: candidate.decay_score,
|
|
686
928
|
createdAt: typeof candidate.timestamp === 'number'
|
|
687
929
|
? candidate.timestamp
|
|
@@ -720,6 +962,182 @@ function encryptToHex(plaintext: string, key: Buffer): string {
|
|
|
720
962
|
return Buffer.from(b64, 'base64').toString('hex');
|
|
721
963
|
}
|
|
722
964
|
|
|
965
|
+
// Plugin v3.0.0 removed the legacy claim-format fallback. Write path
|
|
966
|
+
// always emits Memory Taxonomy v1 JSON blobs. The logClaimFormatOnce
|
|
967
|
+
// helper is gone along with TOTALRECLAW_CLAIM_FORMAT / TOTALRECLAW_TAXONOMY_VERSION.
|
|
968
|
+
|
|
969
|
+
let _loggedDigestMode = false;
|
|
970
|
+
function logDigestModeOnce(mode: DigestMode, logger: OpenClawPluginApi['logger']): void {
|
|
971
|
+
if (_loggedDigestMode) return;
|
|
972
|
+
_loggedDigestMode = true;
|
|
973
|
+
logger.info(`TotalReclaw: digest injection mode = ${mode}`);
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* How many active facts to pull into a digest recompilation.
|
|
978
|
+
* Digest compiler itself will apply DIGEST_CLAIM_CAP for the LLM path.
|
|
979
|
+
*/
|
|
980
|
+
const DIGEST_FETCH_LIMIT = 500;
|
|
981
|
+
|
|
982
|
+
/**
|
|
983
|
+
* Schedule a background digest recompile. Fire-and-forget.
|
|
984
|
+
*
|
|
985
|
+
* The caller must check `!isRecompileInProgress()` before invoking.
|
|
986
|
+
* Errors are logged and swallowed; the guard flag is always released.
|
|
987
|
+
*/
|
|
988
|
+
function scheduleDigestRecompile(
|
|
989
|
+
previousClaimId: string | null,
|
|
990
|
+
logger: OpenClawPluginApi['logger'],
|
|
991
|
+
): void {
|
|
992
|
+
if (!isRecompileInProgress()) {
|
|
993
|
+
if (!tryBeginRecompile()) return;
|
|
994
|
+
} else {
|
|
995
|
+
return;
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
const mode = resolveDigestMode();
|
|
999
|
+
const owner = subgraphOwner || userId;
|
|
1000
|
+
const authKey = authKeyHex;
|
|
1001
|
+
const encKey = encryptionKey;
|
|
1002
|
+
const ownerForBatch = subgraphOwner ?? undefined;
|
|
1003
|
+
|
|
1004
|
+
if (!owner || !authKey || !encKey) {
|
|
1005
|
+
endRecompile();
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
// Capture llmFn from the current LLM config (cheap variant of the user's
|
|
1010
|
+
// provider, already resolved by resolveLLMConfig).
|
|
1011
|
+
const llmConfig = resolveLLMConfig();
|
|
1012
|
+
const llmFn = llmConfig
|
|
1013
|
+
? async (prompt: string): Promise<string> => {
|
|
1014
|
+
const out = await chatCompletion(
|
|
1015
|
+
llmConfig,
|
|
1016
|
+
[
|
|
1017
|
+
{ role: 'system', content: 'You return only valid JSON. No markdown fences, no commentary.' },
|
|
1018
|
+
{ role: 'user', content: prompt },
|
|
1019
|
+
],
|
|
1020
|
+
{ maxTokens: 800, temperature: 0 },
|
|
1021
|
+
);
|
|
1022
|
+
return out ?? '';
|
|
1023
|
+
}
|
|
1024
|
+
: null;
|
|
1025
|
+
|
|
1026
|
+
// Build the I/O deps closures. We capture the owner/auth/key values so the
|
|
1027
|
+
// background task doesn't race with module-level state resets.
|
|
1028
|
+
const fetchFn = () =>
|
|
1029
|
+
fetchAllActiveClaims(
|
|
1030
|
+
owner,
|
|
1031
|
+
authKey,
|
|
1032
|
+
encKey,
|
|
1033
|
+
DIGEST_FETCH_LIMIT,
|
|
1034
|
+
{
|
|
1035
|
+
searchSubgraphBroadened: async (o, n, a) => searchSubgraphBroadened(o, n, a),
|
|
1036
|
+
decryptFromHex: (hex, key) => decryptFromHex(hex, key),
|
|
1037
|
+
},
|
|
1038
|
+
logger,
|
|
1039
|
+
);
|
|
1040
|
+
|
|
1041
|
+
const storeFn = async (canonicalClaimJson: string, compiledAt: string): Promise<void> => {
|
|
1042
|
+
if (!isSubgraphMode()) {
|
|
1043
|
+
// Self-hosted mode — store via the REST API.
|
|
1044
|
+
if (!apiClient) throw new Error('apiClient not initialized');
|
|
1045
|
+
const encryptedBlob = encryptToHex(canonicalClaimJson, encKey);
|
|
1046
|
+
const contentFp = generateContentFingerprint(canonicalClaimJson, dedupKey!);
|
|
1047
|
+
const payload: StoreFactPayload = {
|
|
1048
|
+
id: crypto.randomUUID(),
|
|
1049
|
+
timestamp: compiledAt,
|
|
1050
|
+
encrypted_blob: encryptedBlob,
|
|
1051
|
+
blind_indices: [DIGEST_TRAPDOOR],
|
|
1052
|
+
decay_score: 10,
|
|
1053
|
+
source: 'openclaw-plugin-digest',
|
|
1054
|
+
content_fp: contentFp,
|
|
1055
|
+
agent_id: 'openclaw-plugin-digest',
|
|
1056
|
+
};
|
|
1057
|
+
await apiClient.store(userId!, [payload], authKey);
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
// Subgraph / managed-service mode — encrypt, encode, submit as a single-fact UserOp.
|
|
1062
|
+
const encryptedBlob = encryptToHex(canonicalClaimJson, encKey);
|
|
1063
|
+
const contentFp = generateContentFingerprint(canonicalClaimJson, dedupKey!);
|
|
1064
|
+
const protobuf = encodeFactProtobuf({
|
|
1065
|
+
id: crypto.randomUUID(),
|
|
1066
|
+
timestamp: compiledAt,
|
|
1067
|
+
owner,
|
|
1068
|
+
encryptedBlob,
|
|
1069
|
+
blindIndices: [DIGEST_TRAPDOOR],
|
|
1070
|
+
decayScore: 10,
|
|
1071
|
+
source: 'openclaw-plugin-digest',
|
|
1072
|
+
contentFp,
|
|
1073
|
+
agentId: 'openclaw-plugin-digest',
|
|
1074
|
+
version: PROTOBUF_VERSION_V4,
|
|
1075
|
+
});
|
|
1076
|
+
const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
|
|
1077
|
+
const result = await submitFactBatchOnChain([protobuf], config);
|
|
1078
|
+
if (!result.success) {
|
|
1079
|
+
throw new Error('Digest store UserOp did not succeed on-chain');
|
|
1080
|
+
}
|
|
1081
|
+
};
|
|
1082
|
+
|
|
1083
|
+
const tombstoneFn = async (claimId: string): Promise<void> => {
|
|
1084
|
+
if (!isSubgraphMode()) {
|
|
1085
|
+
if (apiClient) {
|
|
1086
|
+
try { await apiClient.deleteFact(claimId, authKey); } catch { /* best-effort */ }
|
|
1087
|
+
}
|
|
1088
|
+
return;
|
|
1089
|
+
}
|
|
1090
|
+
const tombstone: FactPayload = {
|
|
1091
|
+
id: claimId,
|
|
1092
|
+
timestamp: new Date().toISOString(),
|
|
1093
|
+
owner,
|
|
1094
|
+
encryptedBlob: '00',
|
|
1095
|
+
blindIndices: [],
|
|
1096
|
+
decayScore: 0,
|
|
1097
|
+
source: 'tombstone',
|
|
1098
|
+
contentFp: '',
|
|
1099
|
+
agentId: 'openclaw-plugin-digest',
|
|
1100
|
+
version: PROTOBUF_VERSION_V4,
|
|
1101
|
+
};
|
|
1102
|
+
const protobuf = encodeFactProtobuf(tombstone);
|
|
1103
|
+
const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
|
|
1104
|
+
const result = await submitFactBatchOnChain([protobuf], config);
|
|
1105
|
+
if (!result.success) {
|
|
1106
|
+
throw new Error('Digest tombstone UserOp did not succeed on-chain');
|
|
1107
|
+
}
|
|
1108
|
+
};
|
|
1109
|
+
|
|
1110
|
+
// Slice 2f: run the weight-tuning loop as a fire-and-forget pre-compile step.
|
|
1111
|
+
// This consumes any feedback.jsonl entries written since the last compile
|
|
1112
|
+
// and nudges ~/.totalreclaw/weights.json, so the NEXT contradiction detection
|
|
1113
|
+
// uses the adjusted weights. Rate-limited and idempotent — see
|
|
1114
|
+
// runWeightTuningLoop for details. Failures are logged, never fatal.
|
|
1115
|
+
void runWeightTuningLoop(Math.floor(Date.now() / 1000), logger).catch((err: unknown) => {
|
|
1116
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1117
|
+
logger.warn(`Digest: tuning loop threw: ${msg}`);
|
|
1118
|
+
});
|
|
1119
|
+
|
|
1120
|
+
void recompileDigest({
|
|
1121
|
+
mode,
|
|
1122
|
+
previousClaimId,
|
|
1123
|
+
nowUnixSeconds: Math.floor(Date.now() / 1000),
|
|
1124
|
+
deps: {
|
|
1125
|
+
storeDigestClaim: storeFn,
|
|
1126
|
+
tombstoneDigest: tombstoneFn,
|
|
1127
|
+
fetchAllActiveClaimsFn: fetchFn,
|
|
1128
|
+
llmFn,
|
|
1129
|
+
},
|
|
1130
|
+
logger,
|
|
1131
|
+
})
|
|
1132
|
+
.catch((err: unknown) => {
|
|
1133
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1134
|
+
logger.warn(`Digest: background recompile threw: ${msg}`);
|
|
1135
|
+
})
|
|
1136
|
+
.finally(() => {
|
|
1137
|
+
endRecompile();
|
|
1138
|
+
});
|
|
1139
|
+
}
|
|
1140
|
+
|
|
723
1141
|
/**
|
|
724
1142
|
* Decrypt a hex-encoded ciphertext blob into a UTF-8 string.
|
|
725
1143
|
*/
|
|
@@ -909,7 +1327,8 @@ async function fetchExistingMemoriesForExtraction(
|
|
|
909
1327
|
for (const r of rawResults) {
|
|
910
1328
|
try {
|
|
911
1329
|
const docJson = decryptFromHex(r.encryptedBlob, encryptionKey);
|
|
912
|
-
|
|
1330
|
+
if (isDigestBlob(docJson)) continue;
|
|
1331
|
+
const doc = readClaimFromBlob(docJson);
|
|
913
1332
|
results.push({ id: r.id, text: doc.text });
|
|
914
1333
|
} catch { /* skip undecryptable */ }
|
|
915
1334
|
}
|
|
@@ -918,7 +1337,8 @@ async function fetchExistingMemoriesForExtraction(
|
|
|
918
1337
|
for (const c of candidates) {
|
|
919
1338
|
try {
|
|
920
1339
|
const docJson = decryptFromHex(c.encrypted_blob, encryptionKey);
|
|
921
|
-
|
|
1340
|
+
if (isDigestBlob(docJson)) continue;
|
|
1341
|
+
const doc = readClaimFromBlob(docJson);
|
|
922
1342
|
results.push({ id: c.fact_id, text: doc.text });
|
|
923
1343
|
} catch { /* skip undecryptable */ }
|
|
924
1344
|
}
|
|
@@ -975,10 +1395,7 @@ function relativeTime(isoOrMs: string | number): string {
|
|
|
975
1395
|
* NOTE: This filter is ONLY applied to auto-extraction (hooks).
|
|
976
1396
|
* The explicit `totalreclaw_remember` tool always stores regardless of importance.
|
|
977
1397
|
*/
|
|
978
|
-
const MIN_IMPORTANCE_THRESHOLD =
|
|
979
|
-
1,
|
|
980
|
-
Math.min(10, Number(process.env.TOTALRECLAW_MIN_IMPORTANCE) || 3),
|
|
981
|
-
);
|
|
1398
|
+
const MIN_IMPORTANCE_THRESHOLD = CONFIG.minImportance;
|
|
982
1399
|
|
|
983
1400
|
/**
|
|
984
1401
|
* Filter extracted facts by importance threshold.
|
|
@@ -1001,10 +1418,20 @@ function filterByImportance(
|
|
|
1001
1418
|
}
|
|
1002
1419
|
}
|
|
1003
1420
|
|
|
1004
|
-
|
|
1421
|
+
// Phase 2.2.5: always log the filter outcome so the agent_end path can
|
|
1422
|
+
// distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
|
|
1423
|
+
// below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
|
|
1424
|
+
// this only logged on drops, which made empty-input invisible.
|
|
1425
|
+
if (facts.length === 0) {
|
|
1426
|
+
logger.info('Importance filter: input=0 (nothing to filter)');
|
|
1427
|
+
} else if (dropped > 0) {
|
|
1005
1428
|
logger.info(
|
|
1006
1429
|
`Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`,
|
|
1007
1430
|
);
|
|
1431
|
+
} else {
|
|
1432
|
+
logger.info(
|
|
1433
|
+
`Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`,
|
|
1434
|
+
);
|
|
1008
1435
|
}
|
|
1009
1436
|
|
|
1010
1437
|
return { kept, dropped };
|
|
@@ -1026,6 +1453,7 @@ function filterByImportance(
|
|
|
1026
1453
|
async function storeExtractedFacts(
|
|
1027
1454
|
facts: ExtractedFact[],
|
|
1028
1455
|
logger: OpenClawPluginApi['logger'],
|
|
1456
|
+
sourceOverride?: string,
|
|
1029
1457
|
): Promise<number> {
|
|
1030
1458
|
if (!encryptionKey || !dedupKey || !authKeyHex || !userId || !apiClient) return 0;
|
|
1031
1459
|
|
|
@@ -1063,18 +1491,24 @@ async function storeExtractedFacts(
|
|
|
1063
1491
|
let stored = 0;
|
|
1064
1492
|
let superseded = 0;
|
|
1065
1493
|
let skipped = 0;
|
|
1494
|
+
let failedFacts = 0;
|
|
1066
1495
|
const pendingPayloads: Buffer[] = []; // Batched subgraph payloads
|
|
1067
1496
|
let preparedForSubgraph = 0;
|
|
1068
1497
|
|
|
1498
|
+
// Plugin v3.0.0: always emit Memory Taxonomy v1 JSON blobs. The
|
|
1499
|
+
// TOTALRECLAW_TAXONOMY_VERSION opt-in and the TOTALRECLAW_CLAIM_FORMAT
|
|
1500
|
+
// legacy fallback have both been retired — v1 is the single write path.
|
|
1501
|
+
|
|
1069
1502
|
for (const fact of dedupedFacts) {
|
|
1070
1503
|
try {
|
|
1071
1504
|
const blindIndices = generateBlindIndices(fact.text);
|
|
1505
|
+
const entityTrapdoors = computeEntityTrapdoors(fact.entities);
|
|
1072
1506
|
|
|
1073
1507
|
// Use pre-computed embedding result if available.
|
|
1074
1508
|
const embeddingResult = embeddingResultMap.get(fact.text) ?? null;
|
|
1075
1509
|
const allIndices = embeddingResult
|
|
1076
|
-
? [...blindIndices, ...embeddingResult.lshBuckets]
|
|
1077
|
-
: blindIndices;
|
|
1510
|
+
? [...blindIndices, ...embeddingResult.lshBuckets, ...entityTrapdoors]
|
|
1511
|
+
: [...blindIndices, ...entityTrapdoors];
|
|
1078
1512
|
|
|
1079
1513
|
// LLM-guided dedup: handle UPDATE/DELETE/NOOP actions.
|
|
1080
1514
|
if (fact.action === 'NOOP') {
|
|
@@ -1096,6 +1530,7 @@ async function storeExtractedFacts(
|
|
|
1096
1530
|
source: 'tombstone',
|
|
1097
1531
|
contentFp: '',
|
|
1098
1532
|
agentId: 'openclaw-plugin-auto',
|
|
1533
|
+
version: PROTOBUF_VERSION_V4,
|
|
1099
1534
|
};
|
|
1100
1535
|
pendingPayloads.push(encodeFactProtobuf(tombstone));
|
|
1101
1536
|
logger.info(`LLM dedup: DELETE — queued tombstone for ${fact.existingFactId}`);
|
|
@@ -1124,6 +1559,7 @@ async function storeExtractedFacts(
|
|
|
1124
1559
|
source: 'tombstone',
|
|
1125
1560
|
contentFp: '',
|
|
1126
1561
|
agentId: 'openclaw-plugin-auto',
|
|
1562
|
+
version: PROTOBUF_VERSION_V4,
|
|
1127
1563
|
};
|
|
1128
1564
|
pendingPayloads.push(encodeFactProtobuf(tombstone));
|
|
1129
1565
|
logger.info(`LLM dedup: UPDATE — queued tombstone for ${fact.existingFactId}, storing replacement`);
|
|
@@ -1174,6 +1610,7 @@ async function storeExtractedFacts(
|
|
|
1174
1610
|
source: 'tombstone',
|
|
1175
1611
|
contentFp: '',
|
|
1176
1612
|
agentId: 'openclaw-plugin-auto',
|
|
1613
|
+
version: PROTOBUF_VERSION_V4,
|
|
1177
1614
|
};
|
|
1178
1615
|
pendingPayloads.push(encodeFactProtobuf(tombstone));
|
|
1179
1616
|
logger.info(
|
|
@@ -1196,20 +1633,133 @@ async function storeExtractedFacts(
|
|
|
1196
1633
|
}
|
|
1197
1634
|
}
|
|
1198
1635
|
|
|
1199
|
-
const
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1636
|
+
const factSource = sourceOverride || 'auto-extraction';
|
|
1637
|
+
|
|
1638
|
+
// Plugin v3.0.0: always build a Memory Taxonomy v1 JSON blob. The
|
|
1639
|
+
// blob is decryptable by `readClaimFromBlob` which prefers v1 →
|
|
1640
|
+
// falls back to v0 short-key → then plugin-legacy {text, metadata}
|
|
1641
|
+
// for pre-v3 vault entries.
|
|
1642
|
+
//
|
|
1643
|
+
// We build it BEFORE the on-chain write so Phase 2 contradiction
|
|
1644
|
+
// detection can inspect the same canonical Claim the write path will
|
|
1645
|
+
// actually store. The string is encrypted byte-identically below.
|
|
1646
|
+
//
|
|
1647
|
+
// Defensive: if the extraction hook didn't populate `fact.source`
|
|
1648
|
+
// (e.g. explicit tool path, legacy caller), default to 'user-inferred'
|
|
1649
|
+
// so v1 schema validation passes.
|
|
1650
|
+
const factForBlob: ExtractedFact = fact.source
|
|
1651
|
+
? fact
|
|
1652
|
+
: { ...fact, source: 'user-inferred' };
|
|
1653
|
+
const blobPlaintext = buildCanonicalClaim({
|
|
1654
|
+
fact: factForBlob,
|
|
1655
|
+
importance: effectiveImportance,
|
|
1656
|
+
sourceAgent: factSource,
|
|
1657
|
+
});
|
|
1658
|
+
|
|
1659
|
+
const factId = crypto.randomUUID();
|
|
1660
|
+
|
|
1661
|
+
// Phase 2 Slice 2d: contradiction detection + auto-resolution.
|
|
1662
|
+
//
|
|
1663
|
+
// Runs only when the canonical Claim format is active (legacy blobs
|
|
1664
|
+
// carry no entity refs, so there is nothing to check), only for
|
|
1665
|
+
// Subgraph / managed-service mode (self-hosted contradiction handling
|
|
1666
|
+
// can come later), and only when the new fact has entities. The helper
|
|
1667
|
+
// is a no-op in all other cases.
|
|
1668
|
+
//
|
|
1669
|
+
// Returns one decision per candidate contradicting claim:
|
|
1670
|
+
// - supersede_existing → queue a tombstone + proceed with the new write
|
|
1671
|
+
// - skip_new → do not write the new fact; record the skip reason
|
|
1672
|
+
// - empty list → no contradiction, proceed unchanged
|
|
1673
|
+
//
|
|
1674
|
+
// On any error (subgraph, decrypt, WASM), the helper returns [] and we
|
|
1675
|
+
// fall back to Phase 1 behaviour.
|
|
1676
|
+
let contradictionSkipNew = false;
|
|
1677
|
+
if (
|
|
1678
|
+
isSubgraphMode() &&
|
|
1679
|
+
fact.entities &&
|
|
1680
|
+
fact.entities.length > 0 &&
|
|
1681
|
+
embeddingResult
|
|
1682
|
+
) {
|
|
1683
|
+
const newClaimObj = JSON.parse(blobPlaintext) as Record<string, unknown>;
|
|
1684
|
+
let decisions: ContradictionDecision[] = [];
|
|
1685
|
+
try {
|
|
1686
|
+
decisions = await detectAndResolveContradictions({
|
|
1687
|
+
newClaim: newClaimObj,
|
|
1688
|
+
newClaimId: factId,
|
|
1689
|
+
newEmbedding: embeddingResult.embedding,
|
|
1690
|
+
subgraphOwner: subgraphOwner || userId!,
|
|
1691
|
+
authKeyHex: authKeyHex!,
|
|
1692
|
+
encryptionKey: encryptionKey!,
|
|
1693
|
+
deps: {
|
|
1694
|
+
searchSubgraph: (owner, trapdoors, maxCandidates, authKey) =>
|
|
1695
|
+
searchSubgraph(owner, trapdoors, maxCandidates, authKey).then((rows) =>
|
|
1696
|
+
rows.map((r) => ({
|
|
1697
|
+
id: r.id,
|
|
1698
|
+
encryptedBlob: r.encryptedBlob,
|
|
1699
|
+
encryptedEmbedding: r.encryptedEmbedding ?? null,
|
|
1700
|
+
timestamp: r.timestamp,
|
|
1701
|
+
isActive: r.isActive,
|
|
1702
|
+
})),
|
|
1703
|
+
),
|
|
1704
|
+
decryptFromHex: (hex, key) => decryptFromHex(hex, key),
|
|
1705
|
+
},
|
|
1706
|
+
logger: {
|
|
1707
|
+
info: (m) => logger.info(m),
|
|
1708
|
+
warn: (m) => logger.warn(m),
|
|
1709
|
+
},
|
|
1710
|
+
});
|
|
1711
|
+
} catch (crErr) {
|
|
1712
|
+
// detectAndResolveContradictions is supposed to never throw — if
|
|
1713
|
+
// it does, we log and continue with Phase 1 behaviour.
|
|
1714
|
+
const msg = crErr instanceof Error ? crErr.message : String(crErr);
|
|
1715
|
+
logger.warn(`Contradiction detection failed (proceeding with store): ${msg}`);
|
|
1716
|
+
decisions = [];
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
for (const decision of decisions) {
|
|
1720
|
+
if (decision.action === 'supersede_existing') {
|
|
1721
|
+
const tombstone: FactPayload = {
|
|
1722
|
+
id: decision.existingFactId,
|
|
1723
|
+
timestamp: new Date().toISOString(),
|
|
1724
|
+
owner: subgraphOwner || userId!,
|
|
1725
|
+
encryptedBlob: '00',
|
|
1726
|
+
blindIndices: [],
|
|
1727
|
+
decayScore: 0,
|
|
1728
|
+
source: 'tombstone',
|
|
1729
|
+
contentFp: '',
|
|
1730
|
+
agentId: 'openclaw-plugin-auto',
|
|
1731
|
+
version: PROTOBUF_VERSION_V4,
|
|
1732
|
+
};
|
|
1733
|
+
pendingPayloads.push(encodeFactProtobuf(tombstone));
|
|
1734
|
+
superseded++;
|
|
1735
|
+
logger.info(
|
|
1736
|
+
`Auto-resolve: queued supersede for ${decision.existingFactId.slice(0, 10)}… ` +
|
|
1737
|
+
`(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
|
|
1738
|
+
);
|
|
1739
|
+
} else if (decision.action === 'skip_new') {
|
|
1740
|
+
if (decision.reason === 'existing_pinned') {
|
|
1741
|
+
logger.warn(
|
|
1742
|
+
`Auto-resolve: skipped new write — existing claim ${decision.existingFactId.slice(0, 10)}… is pinned ` +
|
|
1743
|
+
`(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
|
|
1744
|
+
);
|
|
1745
|
+
} else {
|
|
1746
|
+
logger.info(
|
|
1747
|
+
`Auto-resolve: skipped new write — existing ${decision.existingFactId.slice(0, 10)}… wins ` +
|
|
1748
|
+
`(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
|
|
1749
|
+
);
|
|
1750
|
+
}
|
|
1751
|
+
contradictionSkipNew = true;
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1208
1755
|
|
|
1209
|
-
|
|
1756
|
+
if (contradictionSkipNew) {
|
|
1757
|
+
skipped++;
|
|
1758
|
+
continue;
|
|
1759
|
+
}
|
|
1210
1760
|
|
|
1761
|
+
const encryptedBlob = encryptToHex(blobPlaintext, encryptionKey);
|
|
1211
1762
|
const contentFp = generateContentFingerprint(fact.text, dedupKey);
|
|
1212
|
-
const factId = crypto.randomUUID();
|
|
1213
1763
|
|
|
1214
1764
|
if (isSubgraphMode()) {
|
|
1215
1765
|
const protobuf = encodeFactProtobuf({
|
|
@@ -1219,9 +1769,10 @@ async function storeExtractedFacts(
|
|
|
1219
1769
|
encryptedBlob: encryptedBlob,
|
|
1220
1770
|
blindIndices: allIndices,
|
|
1221
1771
|
decayScore: effectiveImportance,
|
|
1222
|
-
source:
|
|
1772
|
+
source: factSource,
|
|
1223
1773
|
contentFp: contentFp,
|
|
1224
1774
|
agentId: 'openclaw-plugin-auto',
|
|
1775
|
+
version: PROTOBUF_VERSION_V4,
|
|
1225
1776
|
encryptedEmbedding: embeddingResult?.encryptedEmbedding,
|
|
1226
1777
|
});
|
|
1227
1778
|
pendingPayloads.push(protobuf);
|
|
@@ -1233,7 +1784,7 @@ async function storeExtractedFacts(
|
|
|
1233
1784
|
encrypted_blob: encryptedBlob,
|
|
1234
1785
|
blind_indices: allIndices,
|
|
1235
1786
|
decay_score: effectiveImportance,
|
|
1236
|
-
source:
|
|
1787
|
+
source: factSource,
|
|
1237
1788
|
content_fp: contentFp,
|
|
1238
1789
|
agent_id: 'openclaw-plugin-auto',
|
|
1239
1790
|
encrypted_embedding: embeddingResult?.encryptedEmbedding,
|
|
@@ -1244,40 +1795,68 @@ async function storeExtractedFacts(
|
|
|
1244
1795
|
} catch (err: unknown) {
|
|
1245
1796
|
// Check for 403 / quota exceeded — invalidate billing cache so next
|
|
1246
1797
|
// before_agent_start re-fetches and warns the user.
|
|
1247
|
-
const
|
|
1248
|
-
if (
|
|
1798
|
+
const factErrMsg = err instanceof Error ? err.message : String(err);
|
|
1799
|
+
if (factErrMsg.includes('403') || factErrMsg.toLowerCase().includes('quota')) {
|
|
1249
1800
|
try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
|
|
1250
|
-
logger.warn(`Quota exceeded — billing cache invalidated. ${
|
|
1801
|
+
logger.warn(`Quota exceeded — billing cache invalidated. ${factErrMsg}`);
|
|
1251
1802
|
break; // Stop trying to store remaining facts — they'll all fail too
|
|
1252
1803
|
}
|
|
1253
|
-
// Otherwise
|
|
1804
|
+
// Otherwise log and continue — individual fact failures shouldn't block remaining facts
|
|
1805
|
+
logger.warn(`Failed to store fact "${fact.text.slice(0, 60)}…": ${factErrMsg}`);
|
|
1806
|
+
failedFacts++;
|
|
1254
1807
|
}
|
|
1255
1808
|
}
|
|
1256
1809
|
|
|
1257
|
-
//
|
|
1810
|
+
// Submit subgraph payloads one fact at a time (sequential single-call UserOps).
|
|
1811
|
+
// Batch executeBatch UserOps have persistent gas estimation issues on Base Sepolia
|
|
1812
|
+
// that cause on-chain reverts. Single-fact UserOps use the simpler submitFactOnChain
|
|
1813
|
+
// path which works reliably (same path as totalreclaw_remember). Each submission
|
|
1814
|
+
// polls for receipt (120s) before proceeding, so nonce is consumed before the next.
|
|
1815
|
+
let batchError: string | undefined;
|
|
1258
1816
|
if (pendingPayloads.length > 0 && isSubgraphMode()) {
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
const
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1817
|
+
const batchConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
1818
|
+
for (let i = 0; i < pendingPayloads.length; i++) {
|
|
1819
|
+
const slice = [pendingPayloads[i]]; // Single fact per UserOp
|
|
1820
|
+
try {
|
|
1821
|
+
const result = await submitFactBatchOnChain(slice, batchConfig);
|
|
1822
|
+
if (result.success) {
|
|
1823
|
+
stored += slice.length;
|
|
1824
|
+
logger.info(`Fact ${i + 1}/${pendingPayloads.length}: submitted on-chain (tx=${result.txHash.slice(0, 10)}…)`);
|
|
1825
|
+
} else {
|
|
1826
|
+
batchError = `On-chain batch submission failed (tx=${result.txHash.slice(0, 10)}…)`;
|
|
1827
|
+
logger.warn(batchError);
|
|
1828
|
+
break; // Stop submitting remaining batches
|
|
1829
|
+
}
|
|
1830
|
+
} catch (err: unknown) {
|
|
1831
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
1832
|
+
if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
|
|
1833
|
+
try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
|
|
1834
|
+
batchError = `Quota exceeded — billing cache invalidated. ${errMsg}`;
|
|
1835
|
+
logger.warn(batchError);
|
|
1836
|
+
break;
|
|
1837
|
+
} else {
|
|
1838
|
+
batchError = `Batch submission failed: ${errMsg}`;
|
|
1839
|
+
logger.warn(batchError);
|
|
1840
|
+
break;
|
|
1841
|
+
}
|
|
1275
1842
|
}
|
|
1276
1843
|
}
|
|
1277
1844
|
}
|
|
1278
1845
|
|
|
1279
|
-
if (stored > 0 || superseded > 0 || skipped > 0) {
|
|
1280
|
-
logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}`);
|
|
1846
|
+
if (stored > 0 || superseded > 0 || skipped > 0 || failedFacts > 0) {
|
|
1847
|
+
logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}, failed=${failedFacts}`);
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
// If ANY batch failed, throw — even if some facts were stored earlier.
|
|
1851
|
+
// A failed/timed-out UserOp may still linger in the bundler mempool as a
|
|
1852
|
+
// "nonce zombie." If we return normally, the caller's next storeExtractedFacts
|
|
1853
|
+
// call will fetch the same on-chain nonce and hit AA25 ("invalid account nonce").
|
|
1854
|
+
// Throwing forces all callers (import loops, chunk handlers) to stop submitting.
|
|
1855
|
+
if (batchError) {
|
|
1856
|
+
throw new Error(`Memory storage failed (${stored} stored before failure): ${batchError}`);
|
|
1857
|
+
}
|
|
1858
|
+
if (stored === 0 && failedFacts > 0) {
|
|
1859
|
+
throw new Error(`Memory storage failed: ${failedFacts} fact(s) failed to store`);
|
|
1281
1860
|
}
|
|
1282
1861
|
|
|
1283
1862
|
return stored;
|
|
@@ -1301,10 +1880,11 @@ async function handlePluginImportFrom(
|
|
|
1301
1880
|
params: Record<string, unknown>,
|
|
1302
1881
|
logger: OpenClawPluginApi['logger'],
|
|
1303
1882
|
): Promise<Record<string, unknown>> {
|
|
1883
|
+
_importInProgress = true;
|
|
1304
1884
|
const startTime = Date.now();
|
|
1305
1885
|
|
|
1306
1886
|
const source = params.source as string;
|
|
1307
|
-
const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'memoclaw', 'generic-json', 'generic-csv'];
|
|
1887
|
+
const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'];
|
|
1308
1888
|
|
|
1309
1889
|
if (!source || !validSources.includes(source)) {
|
|
1310
1890
|
return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
|
|
@@ -1336,18 +1916,31 @@ async function handlePluginImportFrom(
|
|
|
1336
1916
|
// Dry run: report what was parsed (chunks or facts)
|
|
1337
1917
|
if (params.dry_run) {
|
|
1338
1918
|
if (hasChunks) {
|
|
1919
|
+
const totalChunks = parseResult.chunks.length;
|
|
1920
|
+
const EXTRACTION_RATIO = 2.5; // avg facts per chunk, from empirical data
|
|
1921
|
+
const BATCH_SIZE = 25;
|
|
1922
|
+
const SECONDS_PER_BATCH = 45; // ~30s extraction + ~15s embed+store
|
|
1923
|
+
const estimatedFacts = Math.round(totalChunks * EXTRACTION_RATIO);
|
|
1924
|
+
const estimatedBatches = Math.ceil(totalChunks / BATCH_SIZE);
|
|
1925
|
+
const estimatedMinutes = Math.ceil(estimatedBatches * SECONDS_PER_BATCH / 60);
|
|
1926
|
+
|
|
1339
1927
|
return {
|
|
1340
1928
|
success: true,
|
|
1341
1929
|
dry_run: true,
|
|
1342
1930
|
source,
|
|
1343
|
-
total_chunks:
|
|
1931
|
+
total_chunks: totalChunks,
|
|
1344
1932
|
total_messages: parseResult.totalMessages,
|
|
1933
|
+
estimated_facts: estimatedFacts,
|
|
1934
|
+
estimated_batches: estimatedBatches,
|
|
1935
|
+
estimated_minutes: estimatedMinutes,
|
|
1936
|
+
batch_size: BATCH_SIZE,
|
|
1937
|
+
use_background: totalChunks > 50,
|
|
1345
1938
|
preview: parseResult.chunks.slice(0, 5).map((c) => ({
|
|
1346
1939
|
title: c.title,
|
|
1347
1940
|
messages: c.messages.length,
|
|
1348
1941
|
first_message: c.messages[0]?.text.slice(0, 100),
|
|
1349
1942
|
})),
|
|
1350
|
-
note:
|
|
1943
|
+
note: `Estimated ${estimatedFacts} facts from ${totalChunks} chunks (~${estimatedMinutes} min).${totalChunks > 50 ? ' Recommended: background import via sessions_spawn.' : ''}`,
|
|
1351
1944
|
warnings: parseResult.warnings,
|
|
1352
1945
|
};
|
|
1353
1946
|
}
|
|
@@ -1378,28 +1971,42 @@ async function handlePluginImportFrom(
|
|
|
1378
1971
|
action: 'ADD' as const,
|
|
1379
1972
|
}));
|
|
1380
1973
|
|
|
1381
|
-
// Store in batches of 50
|
|
1974
|
+
// Store in batches of 50. Stop on any batch failure to prevent
|
|
1975
|
+
// nonce zombies from blocking subsequent UserOps (AA25).
|
|
1382
1976
|
let totalStored = 0;
|
|
1977
|
+
let storeError: string | undefined;
|
|
1383
1978
|
const batchSize = 50;
|
|
1384
1979
|
|
|
1385
1980
|
for (let i = 0; i < extractedFacts.length; i += batchSize) {
|
|
1386
1981
|
const batch = extractedFacts.slice(i, i + batchSize);
|
|
1387
|
-
|
|
1388
|
-
|
|
1982
|
+
try {
|
|
1983
|
+
const stored = await storeExtractedFacts(batch, logger);
|
|
1984
|
+
totalStored += stored;
|
|
1389
1985
|
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1986
|
+
logger.info(
|
|
1987
|
+
`Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`,
|
|
1988
|
+
);
|
|
1989
|
+
} catch (err: unknown) {
|
|
1990
|
+
storeError = err instanceof Error ? err.message : String(err);
|
|
1991
|
+
logger.warn(`Import stopped at batch ${Math.floor(i / batchSize) + 1}: ${storeError}`);
|
|
1992
|
+
break; // Stop processing further batches
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
const importWarnings = [...parseResult.warnings];
|
|
1997
|
+
if (storeError) {
|
|
1998
|
+
importWarnings.push(`Import stopped early: ${storeError}`);
|
|
1393
1999
|
}
|
|
1394
2000
|
|
|
1395
2001
|
return {
|
|
1396
|
-
success:
|
|
2002
|
+
success: totalStored > 0,
|
|
1397
2003
|
source,
|
|
1398
2004
|
import_id: crypto.randomUUID(),
|
|
1399
2005
|
total_found: parseResult.facts.length,
|
|
1400
2006
|
imported: totalStored,
|
|
1401
2007
|
skipped: parseResult.facts.length - totalStored,
|
|
1402
|
-
|
|
2008
|
+
stopped_early: !!storeError,
|
|
2009
|
+
warnings: importWarnings,
|
|
1403
2010
|
duration_ms: Date.now() - startTime,
|
|
1404
2011
|
};
|
|
1405
2012
|
} catch (e) {
|
|
@@ -1409,6 +2016,343 @@ async function handlePluginImportFrom(
|
|
|
1409
2016
|
}
|
|
1410
2017
|
}
|
|
1411
2018
|
|
|
2019
|
+
// ---------------------------------------------------------------------------
|
|
2020
|
+
// Smart Import — Two-Pass Pipeline (Profile + Triage)
|
|
2021
|
+
// ---------------------------------------------------------------------------
|
|
2022
|
+
|
|
2023
|
+
// Lazy-load WASM for smart import functions (same pattern as crypto.ts / subgraph-store.ts).
|
|
2024
|
+
let _smartImportWasm: typeof import('@totalreclaw/core') | null = null;
|
|
2025
|
+
function getSmartImportWasm() {
|
|
2026
|
+
if (!_smartImportWasm) _smartImportWasm = require('@totalreclaw/core');
|
|
2027
|
+
return _smartImportWasm;
|
|
2028
|
+
}
|
|
2029
|
+
|
|
2030
|
+
/**
|
|
2031
|
+
* Check whether the @totalreclaw/core WASM module exposes smart import functions.
|
|
2032
|
+
* Returns false if the module is an older version without smart import support.
|
|
2033
|
+
*/
|
|
2034
|
+
function hasSmartImportSupport(): boolean {
|
|
2035
|
+
try {
|
|
2036
|
+
const wasm = getSmartImportWasm();
|
|
2037
|
+
return typeof wasm.chunksToSummaries === 'function' &&
|
|
2038
|
+
typeof wasm.buildProfileBatchPrompt === 'function' &&
|
|
2039
|
+
typeof wasm.parseProfileBatchResponse === 'function' &&
|
|
2040
|
+
typeof wasm.buildTriagePrompt === 'function' &&
|
|
2041
|
+
typeof wasm.parseTriageResponse === 'function' &&
|
|
2042
|
+
typeof wasm.enrichExtractionPrompt === 'function';
|
|
2043
|
+
} catch {
|
|
2044
|
+
return false;
|
|
2045
|
+
}
|
|
2046
|
+
}
|
|
2047
|
+
|
|
2048
|
+
/** Smart import result containing profile, triage decisions, and enriched system prompt. */
|
|
2049
|
+
interface SmartImportContext {
|
|
2050
|
+
/** JSON-serialized UserProfile (for WASM calls that require profile_json) */
|
|
2051
|
+
profileJson: string;
|
|
2052
|
+
/** Triage decisions indexed by chunk_index */
|
|
2053
|
+
decisions: Array<{ chunk_index: number; decision: string; reason: string }>;
|
|
2054
|
+
/** Enriched system prompt for extraction (profile context injected) */
|
|
2055
|
+
enrichedSystemPrompt: string;
|
|
2056
|
+
/** Number of chunks marked for extraction */
|
|
2057
|
+
extractCount: number;
|
|
2058
|
+
/** Number of chunks marked for skipping */
|
|
2059
|
+
skipCount: number;
|
|
2060
|
+
/** Duration of the profiling + triage pipeline in ms */
|
|
2061
|
+
durationMs: number;
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
/**
|
|
2065
|
+
* Run the smart import two-pass pipeline: profile the user from conversation
|
|
2066
|
+
* summaries, then triage chunks as EXTRACT or SKIP.
|
|
2067
|
+
*
|
|
2068
|
+
* All prompt construction and response parsing happens in @totalreclaw/core WASM.
|
|
2069
|
+
* LLM calls use the plugin's existing chatCompletion() function.
|
|
2070
|
+
*
|
|
2071
|
+
* Returns null if smart import is unavailable (old WASM, no LLM config, etc.)
|
|
2072
|
+
* so the caller can fall back to blind extraction.
|
|
2073
|
+
*/
|
|
2074
|
+
async function runSmartImportPipeline(
|
|
2075
|
+
chunks: import('./import-adapters/types.js').ConversationChunk[],
|
|
2076
|
+
logger: { info: (msg: string) => void; warn: (msg: string) => void },
|
|
2077
|
+
): Promise<SmartImportContext | null> {
|
|
2078
|
+
// Guard: WASM must have smart import functions
|
|
2079
|
+
if (!hasSmartImportSupport()) {
|
|
2080
|
+
logger.info('Smart import: WASM module does not support smart import, falling back to blind extraction');
|
|
2081
|
+
return null;
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
// Guard: LLM must be available
|
|
2085
|
+
const llmConfig = resolveLLMConfig();
|
|
2086
|
+
if (!llmConfig) {
|
|
2087
|
+
logger.info('Smart import: no LLM available, falling back to blind extraction');
|
|
2088
|
+
return null;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
const pipelineStart = Date.now();
|
|
2092
|
+
const wasm = getSmartImportWasm();
|
|
2093
|
+
|
|
2094
|
+
try {
|
|
2095
|
+
// Step 0: Convert chunks to compact summaries (first + last message)
|
|
2096
|
+
const wasmChunks = chunks.map((c, i) => ({
|
|
2097
|
+
index: i,
|
|
2098
|
+
title: c.title || 'Untitled',
|
|
2099
|
+
messages: c.messages.map((m) => ({ role: m.role, content: m.text })),
|
|
2100
|
+
timestamp: c.timestamp || null,
|
|
2101
|
+
}));
|
|
2102
|
+
const summaries = wasm.chunksToSummaries(JSON.stringify(wasmChunks));
|
|
2103
|
+
const summariesJson = JSON.stringify(summaries);
|
|
2104
|
+
|
|
2105
|
+
// Step 1: Build user profile (batch summarize -> merge)
|
|
2106
|
+
const PROFILE_BATCH_SIZE = 50;
|
|
2107
|
+
const profileStart = Date.now();
|
|
2108
|
+
const partials: unknown[] = [];
|
|
2109
|
+
|
|
2110
|
+
for (let i = 0; i < summaries.length; i += PROFILE_BATCH_SIZE) {
|
|
2111
|
+
const batch = summaries.slice(i, i + PROFILE_BATCH_SIZE);
|
|
2112
|
+
const prompt = wasm.buildProfileBatchPrompt(JSON.stringify(batch));
|
|
2113
|
+
const response = await chatCompletion(llmConfig, [
|
|
2114
|
+
{ role: 'user', content: prompt },
|
|
2115
|
+
], { maxTokens: 2048, temperature: 0 });
|
|
2116
|
+
|
|
2117
|
+
if (!response) {
|
|
2118
|
+
logger.warn(`Smart import: LLM returned empty response for profile batch ${Math.floor(i / PROFILE_BATCH_SIZE) + 1}`);
|
|
2119
|
+
continue;
|
|
2120
|
+
}
|
|
2121
|
+
|
|
2122
|
+
const partial = wasm.parseProfileBatchResponse(response);
|
|
2123
|
+
partials.push(partial);
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
if (partials.length === 0) {
|
|
2127
|
+
logger.warn('Smart import: no profile batches produced, falling back to blind extraction');
|
|
2128
|
+
return null;
|
|
2129
|
+
}
|
|
2130
|
+
|
|
2131
|
+
let profile: unknown;
|
|
2132
|
+
if (partials.length === 1) {
|
|
2133
|
+
// Single batch — skip merge, promote partial to full profile
|
|
2134
|
+
// parseProfileBatchResponse returns a PartialProfile; convert to UserProfile shape
|
|
2135
|
+
const p = partials[0] as Record<string, unknown>;
|
|
2136
|
+
profile = {
|
|
2137
|
+
identity: p.identity ?? null,
|
|
2138
|
+
themes: p.themes ?? [],
|
|
2139
|
+
projects: p.projects ?? [],
|
|
2140
|
+
stack: p.stack ?? [],
|
|
2141
|
+
decisions: p.decisions ?? [],
|
|
2142
|
+
interests: p.interests ?? [],
|
|
2143
|
+
skip_patterns: p.skip_patterns ?? [],
|
|
2144
|
+
};
|
|
2145
|
+
} else {
|
|
2146
|
+
const mergePrompt = wasm.buildProfileMergePrompt(JSON.stringify(partials));
|
|
2147
|
+
const mergeResponse = await chatCompletion(llmConfig, [
|
|
2148
|
+
{ role: 'user', content: mergePrompt },
|
|
2149
|
+
], { maxTokens: 2048, temperature: 0 });
|
|
2150
|
+
|
|
2151
|
+
if (!mergeResponse) {
|
|
2152
|
+
logger.warn('Smart import: LLM returned empty response for profile merge, falling back to blind extraction');
|
|
2153
|
+
return null;
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
profile = wasm.parseProfileResponse(mergeResponse);
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2159
|
+
const profileJson = JSON.stringify(profile);
|
|
2160
|
+
const profileDuration = Date.now() - profileStart;
|
|
2161
|
+
|
|
2162
|
+
const p = profile as Record<string, unknown>;
|
|
2163
|
+
const themeCount = Array.isArray(p.themes) ? p.themes.length : 0;
|
|
2164
|
+
const skipPatternCount = Array.isArray(p.skip_patterns) ? p.skip_patterns.length : 0;
|
|
2165
|
+
logger.info(
|
|
2166
|
+
`Smart import: profile built in ${profileDuration}ms (themes=${themeCount}, skip_patterns=${skipPatternCount})`,
|
|
2167
|
+
);
|
|
2168
|
+
|
|
2169
|
+
// Step 1.5: Chunk triage (EXTRACT or SKIP)
|
|
2170
|
+
const triageStart = Date.now();
|
|
2171
|
+
const allDecisions: Array<{ chunk_index: number; decision: string; reason: string }> = [];
|
|
2172
|
+
const TRIAGE_BATCH_SIZE = 50;
|
|
2173
|
+
|
|
2174
|
+
for (let i = 0; i < summaries.length; i += TRIAGE_BATCH_SIZE) {
|
|
2175
|
+
const batch = summaries.slice(i, i + TRIAGE_BATCH_SIZE);
|
|
2176
|
+
const triagePrompt = wasm.buildTriagePrompt(profileJson, JSON.stringify(batch));
|
|
2177
|
+
const triageResponse = await chatCompletion(llmConfig, [
|
|
2178
|
+
{ role: 'user', content: triagePrompt },
|
|
2179
|
+
], { maxTokens: 4096, temperature: 0 });
|
|
2180
|
+
|
|
2181
|
+
if (!triageResponse) {
|
|
2182
|
+
logger.warn(`Smart import: LLM returned empty response for triage batch ${Math.floor(i / TRIAGE_BATCH_SIZE) + 1}, defaulting to EXTRACT`);
|
|
2183
|
+
// Default all chunks in this batch to EXTRACT
|
|
2184
|
+
for (let j = i; j < Math.min(i + TRIAGE_BATCH_SIZE, summaries.length); j++) {
|
|
2185
|
+
allDecisions.push({ chunk_index: j, decision: 'EXTRACT', reason: 'triage LLM unavailable' });
|
|
2186
|
+
}
|
|
2187
|
+
continue;
|
|
2188
|
+
}
|
|
2189
|
+
|
|
2190
|
+
const batchDecisions = wasm.parseTriageResponse(triageResponse) as Array<{
|
|
2191
|
+
chunk_index: number;
|
|
2192
|
+
decision: string;
|
|
2193
|
+
reason: string;
|
|
2194
|
+
}>;
|
|
2195
|
+
allDecisions.push(...batchDecisions);
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
const triageDuration = Date.now() - triageStart;
|
|
2199
|
+
|
|
2200
|
+
const extractCount = allDecisions.filter((d) => d.decision !== 'SKIP').length;
|
|
2201
|
+
const skipCount = allDecisions.filter((d) => d.decision === 'SKIP').length;
|
|
2202
|
+
logger.info(
|
|
2203
|
+
`Smart import: triage complete in ${triageDuration}ms (extract=${extractCount}, skip=${skipCount}, total=${chunks.length})`,
|
|
2204
|
+
);
|
|
2205
|
+
|
|
2206
|
+
// Step 2: Build enriched system prompt for extraction
|
|
2207
|
+
const enrichedSystemPrompt = wasm.enrichExtractionPrompt(profileJson, EXTRACTION_SYSTEM_PROMPT);
|
|
2208
|
+
|
|
2209
|
+
const totalDuration = Date.now() - pipelineStart;
|
|
2210
|
+
logger.info(`Smart import: pipeline complete in ${totalDuration}ms`);
|
|
2211
|
+
|
|
2212
|
+
return {
|
|
2213
|
+
profileJson,
|
|
2214
|
+
decisions: allDecisions,
|
|
2215
|
+
enrichedSystemPrompt,
|
|
2216
|
+
extractCount,
|
|
2217
|
+
skipCount,
|
|
2218
|
+
durationMs: totalDuration,
|
|
2219
|
+
};
|
|
2220
|
+
} catch (err) {
|
|
2221
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2222
|
+
logger.warn(`Smart import: pipeline failed (${msg}), falling back to blind extraction`);
|
|
2223
|
+
return null;
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
/**
|
|
2228
|
+
* Check if a chunk should be skipped based on triage decisions.
|
|
2229
|
+
* If no decision exists for the chunk index, defaults to EXTRACT (safe default).
|
|
2230
|
+
*/
|
|
2231
|
+
function isChunkSkipped(
|
|
2232
|
+
chunkIndex: number,
|
|
2233
|
+
decisions: Array<{ chunk_index: number; decision: string }>,
|
|
2234
|
+
): { skipped: boolean; reason: string } {
|
|
2235
|
+
const decision = decisions.find((d) => d.chunk_index === chunkIndex);
|
|
2236
|
+
if (decision && decision.decision === 'SKIP') {
|
|
2237
|
+
return { skipped: true, reason: (decision as { reason?: string }).reason || 'triage: skip' };
|
|
2238
|
+
}
|
|
2239
|
+
return { skipped: false, reason: '' };
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
/**
|
|
2243
|
+
* Process a batch (slice) of conversation chunks from a file.
|
|
2244
|
+
* Called repeatedly by the agent for large imports.
|
|
2245
|
+
*/
|
|
2246
|
+
async function handleBatchImport(
|
|
2247
|
+
params: Record<string, unknown>,
|
|
2248
|
+
logger: OpenClawPluginApi['logger'],
|
|
2249
|
+
): Promise<Record<string, unknown>> {
|
|
2250
|
+
_importInProgress = true;
|
|
2251
|
+
const source = params.source as string;
|
|
2252
|
+
const filePath = params.file_path as string | undefined;
|
|
2253
|
+
const content = params.content as string | undefined;
|
|
2254
|
+
const offset = (params.offset as number) ?? 0;
|
|
2255
|
+
const batchSize = (params.batch_size as number) ?? 25;
|
|
2256
|
+
|
|
2257
|
+
const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'];
|
|
2258
|
+
if (!source || !validSources.includes(source)) {
|
|
2259
|
+
return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
const startTime = Date.now();
|
|
2263
|
+
|
|
2264
|
+
const { getAdapter } = await import('./import-adapters/index.js');
|
|
2265
|
+
const adapter = getAdapter(source as import('./import-adapters/types.js').ImportSource);
|
|
2266
|
+
|
|
2267
|
+
const parseResult = await adapter.parse({ content, file_path: filePath });
|
|
2268
|
+
|
|
2269
|
+
if (parseResult.errors.length > 0 && parseResult.chunks.length === 0) {
|
|
2270
|
+
return { success: false, error: parseResult.errors.join('; ') };
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
const totalChunks = parseResult.chunks.length;
|
|
2274
|
+
const slice = parseResult.chunks.slice(offset, offset + batchSize);
|
|
2275
|
+
const remaining = Math.max(0, totalChunks - offset - slice.length);
|
|
2276
|
+
|
|
2277
|
+
// --- Smart Import: Profile + Triage ---
|
|
2278
|
+
// Build profile from ALL chunks (not just the slice) for full context,
|
|
2279
|
+
// then triage only the current slice. For simplicity, we rebuild on every
|
|
2280
|
+
// batch call — optimization (caching) can come later.
|
|
2281
|
+
const smartCtx = await runSmartImportPipeline(parseResult.chunks, logger);
|
|
2282
|
+
let chunksSkipped = 0;
|
|
2283
|
+
|
|
2284
|
+
// Process the slice through the normal extraction + storage pipeline.
|
|
2285
|
+
// If a batch fails (nonce zombie, quota exceeded, etc.), stop immediately
|
|
2286
|
+
// to prevent subsequent UserOps from hitting AA25 nonce conflicts.
|
|
2287
|
+
let factsExtracted = 0;
|
|
2288
|
+
let factsStored = 0;
|
|
2289
|
+
let chunksProcessed = 0;
|
|
2290
|
+
let storeError: string | undefined;
|
|
2291
|
+
|
|
2292
|
+
for (let i = 0; i < slice.length; i++) {
|
|
2293
|
+
const chunk = slice[i];
|
|
2294
|
+
const globalIndex = offset + i; // Index in the full chunks array
|
|
2295
|
+
|
|
2296
|
+
// Smart import: skip chunks triaged as SKIP
|
|
2297
|
+
if (smartCtx) {
|
|
2298
|
+
const { skipped, reason } = isChunkSkipped(globalIndex, smartCtx.decisions);
|
|
2299
|
+
if (skipped) {
|
|
2300
|
+
logger.info(`Import: skipping chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}" (${reason})`);
|
|
2301
|
+
chunksSkipped++;
|
|
2302
|
+
chunksProcessed++;
|
|
2303
|
+
continue;
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
logger.info(`Import: extracting facts from chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}"`);
|
|
2308
|
+
|
|
2309
|
+
const messages = chunk.messages.map((m) => ({ role: m.role, content: m.text }));
|
|
2310
|
+
const facts = await extractFacts(
|
|
2311
|
+
messages,
|
|
2312
|
+
'full',
|
|
2313
|
+
undefined, // no existing memories for dedup during import
|
|
2314
|
+
smartCtx?.enrichedSystemPrompt, // profile-enriched extraction prompt
|
|
2315
|
+
);
|
|
2316
|
+
chunksProcessed++;
|
|
2317
|
+
|
|
2318
|
+
if (facts.length > 0) {
|
|
2319
|
+
factsExtracted += facts.length;
|
|
2320
|
+
try {
|
|
2321
|
+
const stored = await storeExtractedFacts(facts, logger);
|
|
2322
|
+
factsStored += stored;
|
|
2323
|
+
} catch (err: unknown) {
|
|
2324
|
+
storeError = err instanceof Error ? err.message : String(err);
|
|
2325
|
+
logger.warn(`Import batch stopped at chunk ${globalIndex + 1}/${totalChunks}: ${storeError}`);
|
|
2326
|
+
break; // Stop processing further chunks — a zombie UserOp may block writes
|
|
2327
|
+
}
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
|
|
2331
|
+
return {
|
|
2332
|
+
success: factsStored > 0 || (!storeError && factsExtracted === 0),
|
|
2333
|
+
batch_offset: offset,
|
|
2334
|
+
batch_size: chunksProcessed,
|
|
2335
|
+
total_chunks: totalChunks,
|
|
2336
|
+
facts_extracted: factsExtracted,
|
|
2337
|
+
facts_stored: factsStored,
|
|
2338
|
+
chunks_skipped: chunksSkipped,
|
|
2339
|
+
remaining_chunks: remaining,
|
|
2340
|
+
is_complete: remaining === 0 && !storeError,
|
|
2341
|
+
stopped_early: !!storeError,
|
|
2342
|
+
error: storeError,
|
|
2343
|
+
smart_import: smartCtx ? {
|
|
2344
|
+
profile_duration_ms: smartCtx.durationMs,
|
|
2345
|
+
extract_count: smartCtx.extractCount,
|
|
2346
|
+
skip_count: smartCtx.skipCount,
|
|
2347
|
+
} : null,
|
|
2348
|
+
// Estimation for the full import
|
|
2349
|
+
estimated_total_facts: Math.round(totalChunks * 2.5),
|
|
2350
|
+
estimated_total_userops: Math.ceil(totalChunks * 2.5 / 15),
|
|
2351
|
+
estimated_minutes: Math.ceil(Math.ceil(totalChunks / batchSize) * 45 / 60),
|
|
2352
|
+
duration_ms: Date.now() - startTime,
|
|
2353
|
+
};
|
|
2354
|
+
}
|
|
2355
|
+
|
|
1412
2356
|
/**
|
|
1413
2357
|
* Process conversation chunks through LLM extraction and store results.
|
|
1414
2358
|
*
|
|
@@ -1427,9 +2371,29 @@ async function handleChunkImport(
|
|
|
1427
2371
|
let totalExtracted = 0;
|
|
1428
2372
|
let totalStored = 0;
|
|
1429
2373
|
let chunksProcessed = 0;
|
|
2374
|
+
let chunksSkipped = 0;
|
|
2375
|
+
|
|
2376
|
+
let storeError: string | undefined;
|
|
2377
|
+
|
|
2378
|
+
// --- Smart Import: Profile + Triage ---
|
|
2379
|
+
const smartCtx = await runSmartImportPipeline(chunks, logger);
|
|
1430
2380
|
|
|
1431
|
-
for (
|
|
2381
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
2382
|
+
const chunk = chunks[i];
|
|
1432
2383
|
chunksProcessed++;
|
|
2384
|
+
|
|
2385
|
+
// Smart import: skip chunks triaged as SKIP
|
|
2386
|
+
if (smartCtx) {
|
|
2387
|
+
const { skipped, reason } = isChunkSkipped(i, smartCtx.decisions);
|
|
2388
|
+
if (skipped) {
|
|
2389
|
+
logger.info(
|
|
2390
|
+
`Import: skipping chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}" (${reason})`,
|
|
2391
|
+
);
|
|
2392
|
+
chunksSkipped++;
|
|
2393
|
+
continue;
|
|
2394
|
+
}
|
|
2395
|
+
}
|
|
2396
|
+
|
|
1433
2397
|
logger.info(
|
|
1434
2398
|
`Import: extracting facts from chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}"`,
|
|
1435
2399
|
);
|
|
@@ -1443,22 +2407,35 @@ async function handleChunkImport(
|
|
|
1443
2407
|
|
|
1444
2408
|
// Use 'full' mode to extract ALL valuable memories from the chunk
|
|
1445
2409
|
// (not just the last few messages like 'turn' mode does).
|
|
1446
|
-
|
|
2410
|
+
// Smart import: pass enriched system prompt with user profile context.
|
|
2411
|
+
const facts = await extractFacts(
|
|
2412
|
+
messages,
|
|
2413
|
+
'full',
|
|
2414
|
+
undefined, // no existing memories for dedup during import
|
|
2415
|
+
smartCtx?.enrichedSystemPrompt, // profile-enriched extraction prompt
|
|
2416
|
+
);
|
|
1447
2417
|
|
|
1448
2418
|
if (facts.length > 0) {
|
|
1449
2419
|
totalExtracted += facts.length;
|
|
1450
2420
|
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
2421
|
+
try {
|
|
2422
|
+
// Store through the normal pipeline (dedup, encrypt, store).
|
|
2423
|
+
// storeExtractedFacts throws on batch failure to prevent nonce zombies.
|
|
2424
|
+
const stored = await storeExtractedFacts(facts, logger);
|
|
2425
|
+
totalStored += stored;
|
|
1454
2426
|
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
2427
|
+
logger.info(
|
|
2428
|
+
`Import chunk ${chunksProcessed}/${chunks.length}: extracted ${facts.length} facts, stored ${stored}`,
|
|
2429
|
+
);
|
|
2430
|
+
} catch (err: unknown) {
|
|
2431
|
+
storeError = err instanceof Error ? err.message : String(err);
|
|
2432
|
+
logger.warn(`Import stopped at chunk ${chunksProcessed}/${chunks.length}: ${storeError}`);
|
|
2433
|
+
break; // Stop processing further chunks — a zombie UserOp may block writes
|
|
2434
|
+
}
|
|
1458
2435
|
}
|
|
1459
2436
|
}
|
|
1460
2437
|
|
|
1461
|
-
if (totalExtracted === 0 && chunks.length > 0) {
|
|
2438
|
+
if (totalExtracted === 0 && chunks.length > 0 && !storeError && chunksSkipped < chunks.length) {
|
|
1462
2439
|
warnings.push(
|
|
1463
2440
|
`Processed ${chunks.length} conversation chunks (${totalMessages} messages) but the LLM ` +
|
|
1464
2441
|
`did not extract any facts worth storing. This can happen if the conversations are mostly ` +
|
|
@@ -1466,15 +2443,27 @@ async function handleChunkImport(
|
|
|
1466
2443
|
);
|
|
1467
2444
|
}
|
|
1468
2445
|
|
|
2446
|
+
if (storeError) {
|
|
2447
|
+
warnings.push(`Import stopped early: ${storeError}. ${chunks.length - chunksProcessed} chunk(s) not processed.`);
|
|
2448
|
+
}
|
|
2449
|
+
|
|
1469
2450
|
return {
|
|
1470
2451
|
success: totalStored > 0 || totalExtracted > 0,
|
|
1471
2452
|
source,
|
|
1472
2453
|
import_id: crypto.randomUUID(),
|
|
1473
2454
|
total_chunks: chunks.length,
|
|
2455
|
+
chunks_processed: chunksProcessed,
|
|
2456
|
+
chunks_skipped: chunksSkipped,
|
|
1474
2457
|
total_messages: totalMessages,
|
|
1475
2458
|
facts_extracted: totalExtracted,
|
|
1476
2459
|
imported: totalStored,
|
|
1477
2460
|
skipped: totalExtracted - totalStored,
|
|
2461
|
+
stopped_early: !!storeError,
|
|
2462
|
+
smart_import: smartCtx ? {
|
|
2463
|
+
profile_duration_ms: smartCtx.durationMs,
|
|
2464
|
+
extract_count: smartCtx.extractCount,
|
|
2465
|
+
skip_count: smartCtx.skipCount,
|
|
2466
|
+
} : null,
|
|
1478
2467
|
warnings,
|
|
1479
2468
|
duration_ms: Date.now() - startTime,
|
|
1480
2469
|
};
|
|
@@ -1512,6 +2501,7 @@ const plugin = {
|
|
|
1512
2501
|
initLLMClient({
|
|
1513
2502
|
primaryModel: api.config?.agents?.defaults?.model?.primary as string | undefined,
|
|
1514
2503
|
pluginConfig: api.pluginConfig,
|
|
2504
|
+
openclawProviders: api.config?.models?.providers,
|
|
1515
2505
|
logger: api.logger,
|
|
1516
2506
|
});
|
|
1517
2507
|
|
|
@@ -1548,160 +2538,164 @@ const plugin = {
|
|
|
1548
2538
|
},
|
|
1549
2539
|
type: {
|
|
1550
2540
|
type: 'string',
|
|
1551
|
-
enum: [
|
|
1552
|
-
description:
|
|
2541
|
+
enum: [...VALID_MEMORY_TYPES, ...LEGACY_V0_MEMORY_TYPES],
|
|
2542
|
+
description:
|
|
2543
|
+
'Memory Taxonomy v1 type: claim, preference, directive, commitment, episode, summary. ' +
|
|
2544
|
+
'Use "claim" for factual assertions and decisions (populate `reasoning` with the why clause). ' +
|
|
2545
|
+
'Use "directive" for imperative rules ("always X", "never Y"), "commitment" for future intent, ' +
|
|
2546
|
+
'and "episode" for notable events. Legacy v0 tokens (fact, decision, episodic, goal, context, ' +
|
|
2547
|
+
'rule) are silently coerced to their v1 equivalents. Default: claim.',
|
|
2548
|
+
},
|
|
2549
|
+
source: {
|
|
2550
|
+
type: 'string',
|
|
2551
|
+
enum: [...VALID_MEMORY_SOURCES],
|
|
2552
|
+
description:
|
|
2553
|
+
'v1 provenance tag. "user" = user explicitly stated it, "user-inferred" = inferred from user ' +
|
|
2554
|
+
'signals, "assistant" = assistant-authored (downgrade unless user affirmed), "external" / ' +
|
|
2555
|
+
'"derived" = rare. Explicit remembers default to "user".',
|
|
2556
|
+
},
|
|
2557
|
+
scope: {
|
|
2558
|
+
type: 'string',
|
|
2559
|
+
enum: [...VALID_MEMORY_SCOPES],
|
|
2560
|
+
description:
|
|
2561
|
+
'v1 life-domain scope: work, personal, health, family, creative, finance, misc, unspecified. ' +
|
|
2562
|
+
'Default: unspecified.',
|
|
2563
|
+
},
|
|
2564
|
+
reasoning: {
|
|
2565
|
+
type: 'string',
|
|
2566
|
+
description:
|
|
2567
|
+
'For type=claim expressing a decision, the WHY clause ("because Y"). Max 256 chars. ' +
|
|
2568
|
+
'Omit for non-decision claims.',
|
|
2569
|
+
maxLength: 256,
|
|
1553
2570
|
},
|
|
1554
2571
|
importance: {
|
|
1555
2572
|
type: 'number',
|
|
1556
2573
|
minimum: 1,
|
|
1557
2574
|
maximum: 10,
|
|
1558
|
-
description: 'Importance score 1-10 (default:
|
|
2575
|
+
description: 'Importance score 1-10 (default: 8 for explicit remember)',
|
|
2576
|
+
},
|
|
2577
|
+
entities: {
|
|
2578
|
+
type: 'array',
|
|
2579
|
+
description:
|
|
2580
|
+
'Named entities this memory is about (people, projects, tools, companies, concepts, places). ' +
|
|
2581
|
+
'Supplying entities enables Phase 2 contradiction detection against existing facts about the same entity. ' +
|
|
2582
|
+
'Omit if unclear — a best-effort fallback will still store the memory.',
|
|
2583
|
+
items: {
|
|
2584
|
+
type: 'object',
|
|
2585
|
+
properties: {
|
|
2586
|
+
name: { type: 'string' },
|
|
2587
|
+
type: {
|
|
2588
|
+
type: 'string',
|
|
2589
|
+
enum: ['person', 'project', 'tool', 'company', 'concept', 'place'],
|
|
2590
|
+
},
|
|
2591
|
+
role: { type: 'string' },
|
|
2592
|
+
},
|
|
2593
|
+
required: ['name', 'type'],
|
|
2594
|
+
additionalProperties: false,
|
|
2595
|
+
},
|
|
1559
2596
|
},
|
|
1560
2597
|
},
|
|
1561
2598
|
required: ['text'],
|
|
1562
2599
|
additionalProperties: false,
|
|
1563
2600
|
},
|
|
1564
|
-
async execute(
|
|
2601
|
+
async execute(
|
|
2602
|
+
_toolCallId: string,
|
|
2603
|
+
params: {
|
|
2604
|
+
text: string;
|
|
2605
|
+
type?: string;
|
|
2606
|
+
source?: string;
|
|
2607
|
+
scope?: string;
|
|
2608
|
+
reasoning?: string;
|
|
2609
|
+
importance?: number;
|
|
2610
|
+
entities?: Array<{ name: string; type: string; role?: string }>;
|
|
2611
|
+
},
|
|
2612
|
+
) {
|
|
1565
2613
|
try {
|
|
1566
2614
|
await requireFullSetup(api.logger);
|
|
1567
2615
|
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
//
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
//
|
|
1575
|
-
//
|
|
1576
|
-
const
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
//
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
)
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
};
|
|
1612
|
-
const tombProtobuf = encodeFactProtobuf(tombstone);
|
|
1613
|
-
await submitFactOnChain(tombProtobuf, tombConfig);
|
|
1614
|
-
api.logger.info(
|
|
1615
|
-
`Remember dedup: superseded ${dupResult.match.id} on-chain (sim=${dupResult.similarity.toFixed(3)})`,
|
|
1616
|
-
);
|
|
1617
|
-
} catch (tombErr) {
|
|
1618
|
-
api.logger.warn(
|
|
1619
|
-
`Remember dedup: failed to tombstone ${dupResult.match.id}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`,
|
|
1620
|
-
);
|
|
1621
|
-
supersededId = undefined;
|
|
1622
|
-
}
|
|
1623
|
-
} else if (apiClient && authKeyHex) {
|
|
1624
|
-
try {
|
|
1625
|
-
await apiClient.deleteFact(dupResult.match.id, authKeyHex);
|
|
1626
|
-
api.logger.info(
|
|
1627
|
-
`Remember dedup: superseded ${dupResult.match.id} (sim=${dupResult.similarity.toFixed(3)})`,
|
|
1628
|
-
);
|
|
1629
|
-
} catch (delErr) {
|
|
1630
|
-
api.logger.warn(
|
|
1631
|
-
`Remember dedup: failed to delete superseded fact ${dupResult.match.id}: ${delErr instanceof Error ? delErr.message : String(delErr)}`,
|
|
1632
|
-
);
|
|
1633
|
-
supersededId = undefined; // Don't report supersession if delete failed
|
|
1634
|
-
}
|
|
1635
|
-
}
|
|
1636
|
-
}
|
|
1637
|
-
}
|
|
1638
|
-
|
|
1639
|
-
// Build the document JSON that will be encrypted.
|
|
1640
|
-
const doc = {
|
|
1641
|
-
text: params.text,
|
|
1642
|
-
metadata: {
|
|
1643
|
-
type: memoryType,
|
|
1644
|
-
importance: importance / 10, // normalise to 0-1 range
|
|
1645
|
-
source: 'explicit',
|
|
1646
|
-
created_at: new Date().toISOString(),
|
|
1647
|
-
},
|
|
1648
|
-
};
|
|
1649
|
-
|
|
1650
|
-
// Encrypt the document.
|
|
1651
|
-
const encryptedBlob = encryptToHex(JSON.stringify(doc), encryptionKey!);
|
|
1652
|
-
|
|
1653
|
-
// Generate content fingerprint for dedup.
|
|
1654
|
-
const contentFp = generateContentFingerprint(params.text, dedupKey!);
|
|
1655
|
-
|
|
1656
|
-
// Generate a unique fact ID.
|
|
1657
|
-
const factId = crypto.randomUUID();
|
|
2616
|
+
// v1 taxonomy: route explicit remembers through the same canonical
|
|
2617
|
+
// store path that auto-extraction uses (`storeExtractedFacts`). This
|
|
2618
|
+
// emits a Memory Taxonomy v1 JSON blob, generates entity trapdoors,
|
|
2619
|
+
// and runs through the Phase 2 contradiction-resolution pipeline.
|
|
2620
|
+
//
|
|
2621
|
+
// Accept legacy v0 tokens on input and coerce to v1 via
|
|
2622
|
+
// `normalizeToV1Type` so agents that still emit the pre-v3
|
|
2623
|
+
// taxonomy keep working.
|
|
2624
|
+
const rawType = typeof params.type === 'string' ? params.type.toLowerCase() : 'claim';
|
|
2625
|
+
const memoryType: MemoryType = isValidMemoryType(rawType)
|
|
2626
|
+
? rawType
|
|
2627
|
+
: normalizeToV1Type(rawType);
|
|
2628
|
+
|
|
2629
|
+
// Source defaults to 'user' for explicit remembers (the user is
|
|
2630
|
+
// the author by definition). Ignored if the caller passes an
|
|
2631
|
+
// invalid value.
|
|
2632
|
+
const rawSource = typeof params.source === 'string' ? params.source.toLowerCase() : 'user';
|
|
2633
|
+
const memorySource: MemorySource =
|
|
2634
|
+
(VALID_MEMORY_SOURCES as readonly string[]).includes(rawSource)
|
|
2635
|
+
? (rawSource as MemorySource)
|
|
2636
|
+
: 'user';
|
|
2637
|
+
|
|
2638
|
+
const rawScope = typeof params.scope === 'string' ? params.scope.toLowerCase() : 'unspecified';
|
|
2639
|
+
const memoryScope: MemoryScope =
|
|
2640
|
+
(VALID_MEMORY_SCOPES as readonly string[]).includes(rawScope)
|
|
2641
|
+
? (rawScope as MemoryScope)
|
|
2642
|
+
: 'unspecified';
|
|
2643
|
+
|
|
2644
|
+
const reasoning =
|
|
2645
|
+
typeof params.reasoning === 'string' && params.reasoning.length > 0
|
|
2646
|
+
? params.reasoning.slice(0, 256)
|
|
2647
|
+
: undefined;
|
|
2648
|
+
|
|
2649
|
+
// Explicit remember defaults to importance 8 (above auto-extraction's
|
|
2650
|
+
// typical 6-7), so store-time dedup's shouldSupersede prefers the
|
|
2651
|
+
// explicit call when it collides with an auto-extracted claim.
|
|
2652
|
+
const importance = Math.max(1, Math.min(10, params.importance ?? 8));
|
|
2653
|
+
|
|
2654
|
+
const validatedEntities: ExtractedEntity[] = Array.isArray(params.entities)
|
|
2655
|
+
? params.entities
|
|
2656
|
+
.map((e) => parseEntity(e))
|
|
2657
|
+
.filter((e): e is ExtractedEntity => e !== null)
|
|
2658
|
+
: [];
|
|
1658
2659
|
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
agent_id: 'openclaw-plugin',
|
|
1669
|
-
encrypted_embedding: embeddingResult?.encryptedEmbedding,
|
|
2660
|
+
const fact: ExtractedFact = {
|
|
2661
|
+
text: params.text.slice(0, 512),
|
|
2662
|
+
type: memoryType,
|
|
2663
|
+
source: memorySource,
|
|
2664
|
+
scope: memoryScope,
|
|
2665
|
+
reasoning,
|
|
2666
|
+
importance,
|
|
2667
|
+
action: 'ADD',
|
|
2668
|
+
confidence: 1.0, // user explicitly asked to remember — highest confidence
|
|
1670
2669
|
};
|
|
2670
|
+
if (validatedEntities.length > 0) fact.entities = validatedEntities;
|
|
1671
2671
|
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
id: factId,
|
|
1677
|
-
timestamp: new Date().toISOString(),
|
|
1678
|
-
owner: subgraphOwner || userId!,
|
|
1679
|
-
encryptedBlob: encryptedBlob,
|
|
1680
|
-
blindIndices: allIndices,
|
|
1681
|
-
decayScore: importance,
|
|
1682
|
-
source: 'explicit',
|
|
1683
|
-
contentFp: contentFp,
|
|
1684
|
-
agentId: 'openclaw-plugin',
|
|
1685
|
-
encryptedEmbedding: embeddingResult?.encryptedEmbedding,
|
|
1686
|
-
});
|
|
1687
|
-
await submitFactOnChain(protobuf, config);
|
|
1688
|
-
} else {
|
|
1689
|
-
await apiClient!.store(userId!, [factPayload], authKeyHex!);
|
|
1690
|
-
}
|
|
2672
|
+
const stored = await storeExtractedFacts([fact], api.logger, 'explicit');
|
|
2673
|
+
api.logger.info(
|
|
2674
|
+
`totalreclaw_remember: routed to storeExtractedFacts (stored=${stored}, entities=${validatedEntities.length})`,
|
|
2675
|
+
);
|
|
1691
2676
|
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
2677
|
+
if (stored === 0) {
|
|
2678
|
+
// Dedup or supersession consumed the write. Treat as success from
|
|
2679
|
+
// the user's perspective — the memory's content is already in the
|
|
2680
|
+
// vault (possibly under a different ID).
|
|
2681
|
+
return {
|
|
2682
|
+
content: [
|
|
2683
|
+
{
|
|
2684
|
+
type: 'text',
|
|
2685
|
+
text: 'Memory noted (matched existing content in vault).',
|
|
2686
|
+
},
|
|
2687
|
+
],
|
|
2688
|
+
};
|
|
2689
|
+
}
|
|
1695
2690
|
|
|
1696
2691
|
return {
|
|
1697
|
-
content: [{ type: 'text', text:
|
|
1698
|
-
details: { factId, supersededId },
|
|
2692
|
+
content: [{ type: 'text', text: 'Memory encrypted and stored.' }],
|
|
1699
2693
|
};
|
|
1700
2694
|
} catch (err: unknown) {
|
|
1701
2695
|
const message = err instanceof Error ? err.message : String(err);
|
|
1702
2696
|
api.logger.error(`totalreclaw_remember failed: ${message}`);
|
|
1703
2697
|
return {
|
|
1704
|
-
content: [{ type: 'text', text: `Failed to store memory: ${message}` }],
|
|
2698
|
+
content: [{ type: 'text', text: `Failed to store memory: ${humanizeError(message)}` }],
|
|
1705
2699
|
};
|
|
1706
2700
|
}
|
|
1707
2701
|
},
|
|
@@ -1778,12 +2772,27 @@ const plugin = {
|
|
|
1778
2772
|
// --- Subgraph search path ---
|
|
1779
2773
|
const factCount = await getSubgraphFactCount(subgraphOwner || userId!, authKeyHex!);
|
|
1780
2774
|
const pool = computeCandidatePool(factCount);
|
|
1781
|
-
|
|
2775
|
+
let subgraphResults = await searchSubgraph(subgraphOwner || userId!, allTrapdoors, pool, authKeyHex!);
|
|
2776
|
+
|
|
2777
|
+
// Always run broadened search and merge — ensures vocabulary mismatches
|
|
2778
|
+
// (e.g., "preferences" vs "prefer") don't cause recall failures.
|
|
2779
|
+
// The reranker handles scoring; extra cost is ~1 GraphQL query per recall.
|
|
2780
|
+
try {
|
|
2781
|
+
const broadenedResults = await searchSubgraphBroadened(subgraphOwner || userId!, pool, authKeyHex!);
|
|
2782
|
+
// Merge broadened results with existing (deduplicate by ID)
|
|
2783
|
+
const existingIds = new Set(subgraphResults.map(r => r.id));
|
|
2784
|
+
for (const br of broadenedResults) {
|
|
2785
|
+
if (!existingIds.has(br.id)) {
|
|
2786
|
+
subgraphResults.push(br);
|
|
2787
|
+
}
|
|
2788
|
+
}
|
|
2789
|
+
} catch { /* best-effort */ }
|
|
1782
2790
|
|
|
1783
2791
|
for (const result of subgraphResults) {
|
|
1784
2792
|
try {
|
|
1785
2793
|
const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
|
|
1786
|
-
|
|
2794
|
+
if (isDigestBlob(docJson)) continue;
|
|
2795
|
+
const doc = readClaimFromBlob(docJson);
|
|
1787
2796
|
|
|
1788
2797
|
let decryptedEmbedding: number[] | undefined;
|
|
1789
2798
|
if (result.encryptedEmbedding) {
|
|
@@ -1796,17 +2805,29 @@ const plugin = {
|
|
|
1796
2805
|
}
|
|
1797
2806
|
}
|
|
1798
2807
|
|
|
2808
|
+
if (decryptedEmbedding && decryptedEmbedding.length !== getEmbeddingDims()) {
|
|
2809
|
+
try {
|
|
2810
|
+
decryptedEmbedding = await generateEmbedding(doc.text);
|
|
2811
|
+
} catch {
|
|
2812
|
+
decryptedEmbedding = undefined;
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2815
|
+
|
|
1799
2816
|
rerankerCandidates.push({
|
|
1800
2817
|
id: result.id,
|
|
1801
2818
|
text: doc.text,
|
|
1802
2819
|
embedding: decryptedEmbedding,
|
|
1803
|
-
importance:
|
|
2820
|
+
importance: doc.importance / 10,
|
|
1804
2821
|
createdAt: result.timestamp ? parseInt(result.timestamp, 10) : undefined,
|
|
2822
|
+
// Retrieval v2 Tier 1: surface v1 source so applySourceWeights
|
|
2823
|
+
// can multiply the final RRF score by the source weight.
|
|
2824
|
+
source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
|
|
1805
2825
|
});
|
|
1806
2826
|
|
|
1807
2827
|
metaMap.set(result.id, {
|
|
1808
2828
|
metadata: doc.metadata ?? {},
|
|
1809
|
-
timestamp: Date.now(),
|
|
2829
|
+
timestamp: Date.now(),
|
|
2830
|
+
category: doc.category,
|
|
1810
2831
|
});
|
|
1811
2832
|
} catch {
|
|
1812
2833
|
// Skip candidates we cannot decrypt.
|
|
@@ -1849,7 +2870,8 @@ const plugin = {
|
|
|
1849
2870
|
for (const candidate of candidates) {
|
|
1850
2871
|
try {
|
|
1851
2872
|
const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
|
|
1852
|
-
|
|
2873
|
+
if (isDigestBlob(docJson)) continue;
|
|
2874
|
+
const doc = readClaimFromBlob(docJson);
|
|
1853
2875
|
|
|
1854
2876
|
let decryptedEmbedding: number[] | undefined;
|
|
1855
2877
|
if (candidate.encrypted_embedding) {
|
|
@@ -1862,19 +2884,29 @@ const plugin = {
|
|
|
1862
2884
|
}
|
|
1863
2885
|
}
|
|
1864
2886
|
|
|
2887
|
+
if (decryptedEmbedding && decryptedEmbedding.length !== getEmbeddingDims()) {
|
|
2888
|
+
try {
|
|
2889
|
+
decryptedEmbedding = await generateEmbedding(doc.text);
|
|
2890
|
+
} catch {
|
|
2891
|
+
decryptedEmbedding = undefined;
|
|
2892
|
+
}
|
|
2893
|
+
}
|
|
2894
|
+
|
|
1865
2895
|
rerankerCandidates.push({
|
|
1866
2896
|
id: candidate.fact_id,
|
|
1867
2897
|
text: doc.text,
|
|
1868
2898
|
embedding: decryptedEmbedding,
|
|
1869
|
-
importance:
|
|
2899
|
+
importance: doc.importance / 10,
|
|
1870
2900
|
createdAt: typeof candidate.timestamp === 'number'
|
|
1871
2901
|
? candidate.timestamp / 1000
|
|
1872
2902
|
: new Date(candidate.timestamp).getTime() / 1000,
|
|
2903
|
+
source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
|
|
1873
2904
|
});
|
|
1874
2905
|
|
|
1875
2906
|
metaMap.set(candidate.fact_id, {
|
|
1876
2907
|
metadata: doc.metadata ?? {},
|
|
1877
2908
|
timestamp: candidate.timestamp,
|
|
2909
|
+
category: doc.category,
|
|
1878
2910
|
});
|
|
1879
2911
|
} catch {
|
|
1880
2912
|
// Skip candidates we cannot decrypt (e.g. corrupted data).
|
|
@@ -1890,6 +2922,7 @@ const plugin = {
|
|
|
1890
2922
|
rerankerCandidates,
|
|
1891
2923
|
k,
|
|
1892
2924
|
INTENT_WEIGHTS[queryIntent],
|
|
2925
|
+
/* applySourceWeights (Retrieval v2 Tier 1) */ true,
|
|
1893
2926
|
);
|
|
1894
2927
|
|
|
1895
2928
|
if (reranked.length === 0) {
|
|
@@ -1921,7 +2954,8 @@ const plugin = {
|
|
|
1921
2954
|
? ` (importance: ${Math.round((meta.metadata.importance as number) * 10)}/10)`
|
|
1922
2955
|
: '';
|
|
1923
2956
|
const age = meta ? relativeTime(meta.timestamp) : '';
|
|
1924
|
-
|
|
2957
|
+
const typeTag = meta?.category ? `[${meta.category}] ` : '';
|
|
2958
|
+
return `${i + 1}. ${typeTag}${m.text}${imp} -- ${age} [ID: ${m.id}]`;
|
|
1925
2959
|
});
|
|
1926
2960
|
|
|
1927
2961
|
const formatted = lines.join('\n');
|
|
@@ -1940,7 +2974,7 @@ const plugin = {
|
|
|
1940
2974
|
const message = err instanceof Error ? err.message : String(err);
|
|
1941
2975
|
api.logger.error(`totalreclaw_recall failed: ${message}`);
|
|
1942
2976
|
return {
|
|
1943
|
-
content: [{ type: 'text', text: `Failed to search memories: ${message}` }],
|
|
2977
|
+
content: [{ type: 'text', text: `Failed to search memories: ${humanizeError(message)}` }],
|
|
1944
2978
|
};
|
|
1945
2979
|
}
|
|
1946
2980
|
},
|
|
@@ -1986,9 +3020,13 @@ const plugin = {
|
|
|
1986
3020
|
source: 'tombstone',
|
|
1987
3021
|
contentFp: '',
|
|
1988
3022
|
agentId: 'openclaw-plugin',
|
|
3023
|
+
version: PROTOBUF_VERSION_V4,
|
|
1989
3024
|
};
|
|
1990
3025
|
const protobuf = encodeFactProtobuf(tombstone);
|
|
1991
3026
|
const result = await submitFactOnChain(protobuf, config);
|
|
3027
|
+
if (!result.success) {
|
|
3028
|
+
throw new Error(`On-chain tombstone failed (tx=${result.txHash?.slice(0, 10) || 'none'}…)`);
|
|
3029
|
+
}
|
|
1992
3030
|
api.logger.info(`Tombstone written for ${params.factId}: tx=${result.txHash}`);
|
|
1993
3031
|
return {
|
|
1994
3032
|
content: [{ type: 'text', text: `Memory ${params.factId} deleted (on-chain tombstone, tx: ${result.txHash})` }],
|
|
@@ -2005,7 +3043,7 @@ const plugin = {
|
|
|
2005
3043
|
const message = err instanceof Error ? err.message : String(err);
|
|
2006
3044
|
api.logger.error(`totalreclaw_forget failed: ${message}`);
|
|
2007
3045
|
return {
|
|
2008
|
-
content: [{ type: 'text', text: `Failed to delete memory: ${message}` }],
|
|
3046
|
+
content: [{ type: 'text', text: `Failed to delete memory: ${humanizeError(message)}` }],
|
|
2009
3047
|
};
|
|
2010
3048
|
}
|
|
2011
3049
|
},
|
|
@@ -2049,16 +3087,22 @@ const plugin = {
|
|
|
2049
3087
|
}> = [];
|
|
2050
3088
|
|
|
2051
3089
|
if (isSubgraphMode()) {
|
|
2052
|
-
// Query subgraph for all active facts
|
|
3090
|
+
// Query subgraph for all active facts (cursor-based pagination via id_gt)
|
|
2053
3091
|
const config = getSubgraphConfig();
|
|
2054
3092
|
const relayUrl = config.relayUrl;
|
|
2055
3093
|
const PAGE_SIZE = 1000;
|
|
2056
|
-
let
|
|
2057
|
-
let hasMore = true;
|
|
3094
|
+
let lastId = '';
|
|
2058
3095
|
const owner = subgraphOwner || userId || '';
|
|
3096
|
+
console.error(`[TotalReclaw Export] owner=${owner} subgraphOwner=${subgraphOwner} userId=${userId} relayUrl=${relayUrl} authKey=${authKeyHex ? authKeyHex.slice(0, 8) + '...' : 'MISSING'} isSubgraph=${isSubgraphMode()}`);
|
|
2059
3097
|
|
|
2060
|
-
while (
|
|
2061
|
-
const
|
|
3098
|
+
while (true) {
|
|
3099
|
+
const hasLastId = lastId !== '';
|
|
3100
|
+
const query = hasLastId
|
|
3101
|
+
? `query($owner:Bytes!,$first:Int!,$lastId:String!){facts(where:{owner:$owner,isActive:true,id_gt:$lastId},first:$first,orderBy:id,orderDirection:asc){id encryptedBlob timestamp sequenceId}}`
|
|
3102
|
+
: `query($owner:Bytes!,$first:Int!){facts(where:{owner:$owner,isActive:true},first:$first,orderBy:id,orderDirection:asc){id encryptedBlob timestamp sequenceId}}`;
|
|
3103
|
+
const variables: Record<string, unknown> = hasLastId
|
|
3104
|
+
? { owner, first: PAGE_SIZE, lastId }
|
|
3105
|
+
: { owner, first: PAGE_SIZE };
|
|
2062
3106
|
|
|
2063
3107
|
const res = await fetch(`${relayUrl}/v1/subgraph`, {
|
|
2064
3108
|
method: 'POST',
|
|
@@ -2067,24 +3111,36 @@ const plugin = {
|
|
|
2067
3111
|
'X-TotalReclaw-Client': 'openclaw-plugin',
|
|
2068
3112
|
...(authKeyHex ? { Authorization: `Bearer ${authKeyHex}` } : {}),
|
|
2069
3113
|
},
|
|
2070
|
-
body: JSON.stringify({ query }),
|
|
3114
|
+
body: JSON.stringify({ query, variables }),
|
|
2071
3115
|
});
|
|
2072
3116
|
|
|
2073
3117
|
const json = (await res.json()) as {
|
|
2074
3118
|
data?: { facts?: Array<{ id: string; encryptedBlob: string; source: string; agentId: string; timestamp: string; sequenceId: string }> };
|
|
3119
|
+
error?: string;
|
|
3120
|
+
errors?: Array<{ message: string }>;
|
|
2075
3121
|
};
|
|
3122
|
+
// Surface relay/subgraph errors instead of silently returning empty
|
|
3123
|
+
if (json.error || json.errors) {
|
|
3124
|
+
const errMsg = json.error || json.errors?.map(e => e.message).join('; ') || 'Unknown error';
|
|
3125
|
+
api.logger.error(`Export subgraph query failed: ${errMsg} (owner=${owner}, status=${res.status})`);
|
|
3126
|
+
return {
|
|
3127
|
+
content: [{ type: 'text', text: `Export failed: ${errMsg}` }],
|
|
3128
|
+
};
|
|
3129
|
+
}
|
|
2076
3130
|
const facts = json?.data?.facts || [];
|
|
3131
|
+
if (facts.length === 0) break;
|
|
2077
3132
|
|
|
2078
3133
|
for (const fact of facts) {
|
|
2079
3134
|
try {
|
|
2080
3135
|
let hexBlob = fact.encryptedBlob;
|
|
2081
3136
|
if (hexBlob.startsWith('0x')) hexBlob = hexBlob.slice(2);
|
|
2082
3137
|
const docJson = decryptFromHex(hexBlob, encryptionKey!);
|
|
2083
|
-
|
|
3138
|
+
if (isDigestBlob(docJson)) continue;
|
|
3139
|
+
const doc = readClaimFromBlob(docJson);
|
|
2084
3140
|
allFacts.push({
|
|
2085
3141
|
id: fact.id,
|
|
2086
3142
|
text: doc.text,
|
|
2087
|
-
metadata: doc.metadata
|
|
3143
|
+
metadata: doc.metadata,
|
|
2088
3144
|
created_at: new Date(parseInt(fact.timestamp) * 1000).toISOString(),
|
|
2089
3145
|
});
|
|
2090
3146
|
} catch {
|
|
@@ -2092,8 +3148,8 @@ const plugin = {
|
|
|
2092
3148
|
}
|
|
2093
3149
|
}
|
|
2094
3150
|
|
|
2095
|
-
|
|
2096
|
-
|
|
3151
|
+
if (facts.length < PAGE_SIZE) break;
|
|
3152
|
+
lastId = facts[facts.length - 1].id;
|
|
2097
3153
|
}
|
|
2098
3154
|
} else {
|
|
2099
3155
|
// HTTP server mode — paginate through PostgreSQL facts
|
|
@@ -2106,11 +3162,12 @@ const plugin = {
|
|
|
2106
3162
|
for (const fact of page.facts) {
|
|
2107
3163
|
try {
|
|
2108
3164
|
const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey!);
|
|
2109
|
-
|
|
3165
|
+
if (isDigestBlob(docJson)) continue;
|
|
3166
|
+
const doc = readClaimFromBlob(docJson);
|
|
2110
3167
|
allFacts.push({
|
|
2111
3168
|
id: fact.id,
|
|
2112
3169
|
text: doc.text,
|
|
2113
|
-
metadata: doc.metadata
|
|
3170
|
+
metadata: doc.metadata,
|
|
2114
3171
|
created_at: fact.created_at,
|
|
2115
3172
|
});
|
|
2116
3173
|
} catch {
|
|
@@ -2152,7 +3209,7 @@ const plugin = {
|
|
|
2152
3209
|
const message = err instanceof Error ? err.message : String(err);
|
|
2153
3210
|
api.logger.error(`totalreclaw_export failed: ${message}`);
|
|
2154
3211
|
return {
|
|
2155
|
-
content: [{ type: 'text', text: `Failed to export memories: ${message}` }],
|
|
3212
|
+
content: [{ type: 'text', text: `Failed to export memories: ${humanizeError(message)}` }],
|
|
2156
3213
|
};
|
|
2157
3214
|
}
|
|
2158
3215
|
},
|
|
@@ -2185,7 +3242,7 @@ const plugin = {
|
|
|
2185
3242
|
};
|
|
2186
3243
|
}
|
|
2187
3244
|
|
|
2188
|
-
const serverUrl =
|
|
3245
|
+
const serverUrl = CONFIG.serverUrl;
|
|
2189
3246
|
const walletAddr = subgraphOwner || userId || '';
|
|
2190
3247
|
const response = await fetch(`${serverUrl}/v1/billing/status?wallet_address=${encodeURIComponent(walletAddr)}`, {
|
|
2191
3248
|
method: 'GET',
|
|
@@ -2238,7 +3295,7 @@ const plugin = {
|
|
|
2238
3295
|
const message = err instanceof Error ? err.message : String(err);
|
|
2239
3296
|
api.logger.error(`totalreclaw_status failed: ${message}`);
|
|
2240
3297
|
return {
|
|
2241
|
-
content: [{ type: 'text', text: `Failed to check status: ${message}` }],
|
|
3298
|
+
content: [{ type: 'text', text: `Failed to check status: ${humanizeError(message)}` }],
|
|
2242
3299
|
};
|
|
2243
3300
|
}
|
|
2244
3301
|
},
|
|
@@ -2255,13 +3312,13 @@ const plugin = {
|
|
|
2255
3312
|
name: 'totalreclaw_consolidate',
|
|
2256
3313
|
label: 'Consolidate',
|
|
2257
3314
|
description:
|
|
2258
|
-
'
|
|
3315
|
+
'Deduplicate and merge related memories. Self-hosted mode only.',
|
|
2259
3316
|
parameters: {
|
|
2260
3317
|
type: 'object',
|
|
2261
3318
|
properties: {
|
|
2262
3319
|
dry_run: {
|
|
2263
3320
|
type: 'boolean',
|
|
2264
|
-
description: 'Preview
|
|
3321
|
+
description: 'Preview only (default: false)',
|
|
2265
3322
|
},
|
|
2266
3323
|
},
|
|
2267
3324
|
additionalProperties: false,
|
|
@@ -2298,11 +3355,10 @@ const plugin = {
|
|
|
2298
3355
|
for (const fact of page.facts) {
|
|
2299
3356
|
try {
|
|
2300
3357
|
const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey);
|
|
2301
|
-
|
|
3358
|
+
if (isDigestBlob(docJson)) continue;
|
|
3359
|
+
const doc = readClaimFromBlob(docJson);
|
|
2302
3360
|
|
|
2303
3361
|
let embedding: number[] | null = null;
|
|
2304
|
-
// ExportedFact does not include encrypted_embedding — generate it on-the-fly.
|
|
2305
|
-
// For consolidation we need embeddings, so generate them.
|
|
2306
3362
|
try {
|
|
2307
3363
|
embedding = await generateEmbedding(doc.text);
|
|
2308
3364
|
} catch { /* skip — fact will not be clustered */ }
|
|
@@ -2311,9 +3367,7 @@ const plugin = {
|
|
|
2311
3367
|
id: fact.id,
|
|
2312
3368
|
text: doc.text,
|
|
2313
3369
|
embedding,
|
|
2314
|
-
importance: doc.
|
|
2315
|
-
? Math.round((doc.metadata.importance as number) * 10)
|
|
2316
|
-
: 5,
|
|
3370
|
+
importance: doc.importance,
|
|
2317
3371
|
decayScore: fact.decay_score,
|
|
2318
3372
|
createdAt: new Date(fact.created_at).getTime(),
|
|
2319
3373
|
version: fact.version,
|
|
@@ -2395,7 +3449,7 @@ const plugin = {
|
|
|
2395
3449
|
const message = err instanceof Error ? err.message : String(err);
|
|
2396
3450
|
api.logger.error(`totalreclaw_consolidate failed: ${message}`);
|
|
2397
3451
|
return {
|
|
2398
|
-
content: [{ type: 'text', text: `Failed to consolidate memories: ${message}` }],
|
|
3452
|
+
content: [{ type: 'text', text: `Failed to consolidate memories: ${humanizeError(message)}` }],
|
|
2399
3453
|
};
|
|
2400
3454
|
}
|
|
2401
3455
|
},
|
|
@@ -2403,6 +3457,205 @@ const plugin = {
|
|
|
2403
3457
|
{ name: 'totalreclaw_consolidate' },
|
|
2404
3458
|
);
|
|
2405
3459
|
|
|
3460
|
+
// ---------------------------------------------------------------
|
|
3461
|
+
// Helper: build PinOpDeps bound to the live plugin state
|
|
3462
|
+
// ---------------------------------------------------------------
|
|
3463
|
+
// Wires the pure pin/unpin operation to the managed-service transport +
|
|
3464
|
+
// crypto layer. Mirrors MCP's buildPinDepsFromState and Python's
|
|
3465
|
+
// _change_claim_status argument plumbing.
|
|
3466
|
+
const buildPinDeps = (): PinOpDeps => {
|
|
3467
|
+
const owner = subgraphOwner || userId || '';
|
|
3468
|
+
const config = {
|
|
3469
|
+
...getSubgraphConfig(),
|
|
3470
|
+
authKeyHex: authKeyHex!,
|
|
3471
|
+
walletAddress: subgraphOwner ?? undefined,
|
|
3472
|
+
};
|
|
3473
|
+
return {
|
|
3474
|
+
owner,
|
|
3475
|
+
sourceAgent: 'openclaw-plugin',
|
|
3476
|
+
fetchFactById: (factId: string) => fetchFactById(owner, factId, authKeyHex!),
|
|
3477
|
+
decryptBlob: (hex: string) => decryptFromHex(hex, encryptionKey!),
|
|
3478
|
+
encryptBlob: (plaintext: string) => encryptToHex(plaintext, encryptionKey!),
|
|
3479
|
+
submitBatch: async (payloads: Buffer[]) => {
|
|
3480
|
+
const result = await submitFactBatchOnChain(payloads, config);
|
|
3481
|
+
return { txHash: result.txHash, success: result.success };
|
|
3482
|
+
},
|
|
3483
|
+
generateIndices: async (text: string, entityNames: string[]) => {
|
|
3484
|
+
if (!text) return { blindIndices: [] };
|
|
3485
|
+
const wordIndices = generateBlindIndices(text);
|
|
3486
|
+
let lshIndices: string[] = [];
|
|
3487
|
+
let encryptedEmbedding: string | undefined;
|
|
3488
|
+
try {
|
|
3489
|
+
const embedding = await generateEmbedding(text);
|
|
3490
|
+
const hasher = getLSHHasher(api.logger);
|
|
3491
|
+
if (hasher) lshIndices = hasher.hash(embedding);
|
|
3492
|
+
encryptedEmbedding = encryptToHex(JSON.stringify(embedding), encryptionKey!);
|
|
3493
|
+
} catch {
|
|
3494
|
+
// Best-effort: word + entity trapdoors alone still surface the claim.
|
|
3495
|
+
}
|
|
3496
|
+
const entityTrapdoors = entityNames.map((n) => computeEntityTrapdoor(n));
|
|
3497
|
+
return {
|
|
3498
|
+
blindIndices: [...wordIndices, ...lshIndices, ...entityTrapdoors],
|
|
3499
|
+
encryptedEmbedding,
|
|
3500
|
+
};
|
|
3501
|
+
},
|
|
3502
|
+
};
|
|
3503
|
+
};
|
|
3504
|
+
|
|
3505
|
+
// ---------------------------------------------------------------
|
|
3506
|
+
// Tool: totalreclaw_pin
|
|
3507
|
+
// ---------------------------------------------------------------
|
|
3508
|
+
|
|
3509
|
+
api.registerTool(
|
|
3510
|
+
{
|
|
3511
|
+
name: 'totalreclaw_pin',
|
|
3512
|
+
label: 'Pin',
|
|
3513
|
+
description:
|
|
3514
|
+
'Pin a memory so the auto-resolution engine will never override or supersede it. ' +
|
|
3515
|
+
"Use when the user explicitly confirms a claim is still valid after you or another agent " +
|
|
3516
|
+
"tried to retract/contradict it (e.g. 'wait, I still use Vim sometimes'). " +
|
|
3517
|
+
'Takes fact_id (from a prior recall result). Pinning is idempotent — pinning an already-pinned ' +
|
|
3518
|
+
'claim is a no-op. Cross-device: the pin propagates via the on-chain supersession chain.',
|
|
3519
|
+
parameters: {
|
|
3520
|
+
type: 'object',
|
|
3521
|
+
properties: {
|
|
3522
|
+
fact_id: {
|
|
3523
|
+
type: 'string',
|
|
3524
|
+
description: 'The ID of the fact to pin (from a totalreclaw_recall result).',
|
|
3525
|
+
},
|
|
3526
|
+
reason: {
|
|
3527
|
+
type: 'string',
|
|
3528
|
+
description: 'Optional human-readable reason for pinning (logged locally for tuning).',
|
|
3529
|
+
},
|
|
3530
|
+
},
|
|
3531
|
+
required: ['fact_id'],
|
|
3532
|
+
additionalProperties: false,
|
|
3533
|
+
},
|
|
3534
|
+
async execute(_toolCallId: string, params: Record<string, unknown>) {
|
|
3535
|
+
try {
|
|
3536
|
+
await requireFullSetup(api.logger);
|
|
3537
|
+
if (!isSubgraphMode()) {
|
|
3538
|
+
return {
|
|
3539
|
+
content: [{
|
|
3540
|
+
type: 'text',
|
|
3541
|
+
text: 'Pin/unpin is only supported with the managed service. Self-hosted mode does not yet implement the status-flip supersession flow.',
|
|
3542
|
+
}],
|
|
3543
|
+
};
|
|
3544
|
+
}
|
|
3545
|
+
const validation = validatePinArgs(params);
|
|
3546
|
+
if (!validation.ok) {
|
|
3547
|
+
return { content: [{ type: 'text', text: validation.error }] };
|
|
3548
|
+
}
|
|
3549
|
+
const deps = buildPinDeps();
|
|
3550
|
+
const result = await executePinOperation(validation.factId, 'pinned', deps, validation.reason);
|
|
3551
|
+
if (result.success && result.idempotent) {
|
|
3552
|
+
api.logger.info(`totalreclaw_pin: ${result.fact_id} already pinned (no-op)`);
|
|
3553
|
+
return {
|
|
3554
|
+
content: [{ type: 'text', text: `Memory ${result.fact_id} is already pinned.` }],
|
|
3555
|
+
details: result,
|
|
3556
|
+
};
|
|
3557
|
+
}
|
|
3558
|
+
if (result.success) {
|
|
3559
|
+
api.logger.info(`totalreclaw_pin: ${result.fact_id} → ${result.new_fact_id} (tx ${result.tx_hash?.slice(0, 10)})`);
|
|
3560
|
+
return {
|
|
3561
|
+
content: [{
|
|
3562
|
+
type: 'text',
|
|
3563
|
+
text: `Pinned memory ${result.fact_id}. New fact id: ${result.new_fact_id} (tx: ${result.tx_hash}).`,
|
|
3564
|
+
}],
|
|
3565
|
+
details: result,
|
|
3566
|
+
};
|
|
3567
|
+
}
|
|
3568
|
+
api.logger.error(`totalreclaw_pin failed: ${result.error}`);
|
|
3569
|
+
return {
|
|
3570
|
+
content: [{ type: 'text', text: `Failed to pin memory: ${humanizeError(result.error ?? 'unknown error')}` }],
|
|
3571
|
+
details: result,
|
|
3572
|
+
};
|
|
3573
|
+
} catch (err: unknown) {
|
|
3574
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
3575
|
+
api.logger.error(`totalreclaw_pin failed: ${message}`);
|
|
3576
|
+
return {
|
|
3577
|
+
content: [{ type: 'text', text: `Failed to pin memory: ${humanizeError(message)}` }],
|
|
3578
|
+
};
|
|
3579
|
+
}
|
|
3580
|
+
},
|
|
3581
|
+
},
|
|
3582
|
+
{ name: 'totalreclaw_pin' },
|
|
3583
|
+
);
|
|
3584
|
+
|
|
3585
|
+
// ---------------------------------------------------------------
|
|
3586
|
+
// Tool: totalreclaw_unpin
|
|
3587
|
+
// ---------------------------------------------------------------
|
|
3588
|
+
|
|
3589
|
+
api.registerTool(
|
|
3590
|
+
{
|
|
3591
|
+
name: 'totalreclaw_unpin',
|
|
3592
|
+
label: 'Unpin',
|
|
3593
|
+
description:
|
|
3594
|
+
'Remove the pin from a previously pinned memory, returning it to active status so the ' +
|
|
3595
|
+
'auto-resolution engine can supersede or retract it again. Takes fact_id. Idempotent — ' +
|
|
3596
|
+
'unpinning a non-pinned claim is a no-op.',
|
|
3597
|
+
parameters: {
|
|
3598
|
+
type: 'object',
|
|
3599
|
+
properties: {
|
|
3600
|
+
fact_id: {
|
|
3601
|
+
type: 'string',
|
|
3602
|
+
description: 'The ID of the fact to unpin (from a totalreclaw_recall result).',
|
|
3603
|
+
},
|
|
3604
|
+
},
|
|
3605
|
+
required: ['fact_id'],
|
|
3606
|
+
additionalProperties: false,
|
|
3607
|
+
},
|
|
3608
|
+
async execute(_toolCallId: string, params: Record<string, unknown>) {
|
|
3609
|
+
try {
|
|
3610
|
+
await requireFullSetup(api.logger);
|
|
3611
|
+
if (!isSubgraphMode()) {
|
|
3612
|
+
return {
|
|
3613
|
+
content: [{
|
|
3614
|
+
type: 'text',
|
|
3615
|
+
text: 'Pin/unpin is only supported with the managed service. Self-hosted mode does not yet implement the status-flip supersession flow.',
|
|
3616
|
+
}],
|
|
3617
|
+
};
|
|
3618
|
+
}
|
|
3619
|
+
const validation = validatePinArgs(params);
|
|
3620
|
+
if (!validation.ok) {
|
|
3621
|
+
return { content: [{ type: 'text', text: validation.error }] };
|
|
3622
|
+
}
|
|
3623
|
+
const deps = buildPinDeps();
|
|
3624
|
+
const result = await executePinOperation(validation.factId, 'active', deps);
|
|
3625
|
+
if (result.success && result.idempotent) {
|
|
3626
|
+
api.logger.info(`totalreclaw_unpin: ${result.fact_id} already active (no-op)`);
|
|
3627
|
+
return {
|
|
3628
|
+
content: [{ type: 'text', text: `Memory ${result.fact_id} is not pinned.` }],
|
|
3629
|
+
details: result,
|
|
3630
|
+
};
|
|
3631
|
+
}
|
|
3632
|
+
if (result.success) {
|
|
3633
|
+
api.logger.info(`totalreclaw_unpin: ${result.fact_id} → ${result.new_fact_id} (tx ${result.tx_hash?.slice(0, 10)})`);
|
|
3634
|
+
return {
|
|
3635
|
+
content: [{
|
|
3636
|
+
type: 'text',
|
|
3637
|
+
text: `Unpinned memory ${result.fact_id}. New fact id: ${result.new_fact_id} (tx: ${result.tx_hash}).`,
|
|
3638
|
+
}],
|
|
3639
|
+
details: result,
|
|
3640
|
+
};
|
|
3641
|
+
}
|
|
3642
|
+
api.logger.error(`totalreclaw_unpin failed: ${result.error}`);
|
|
3643
|
+
return {
|
|
3644
|
+
content: [{ type: 'text', text: `Failed to unpin memory: ${humanizeError(result.error ?? 'unknown error')}` }],
|
|
3645
|
+
details: result,
|
|
3646
|
+
};
|
|
3647
|
+
} catch (err: unknown) {
|
|
3648
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
3649
|
+
api.logger.error(`totalreclaw_unpin failed: ${message}`);
|
|
3650
|
+
return {
|
|
3651
|
+
content: [{ type: 'text', text: `Failed to unpin memory: ${humanizeError(message)}` }],
|
|
3652
|
+
};
|
|
3653
|
+
}
|
|
3654
|
+
},
|
|
3655
|
+
},
|
|
3656
|
+
{ name: 'totalreclaw_unpin' },
|
|
3657
|
+
);
|
|
3658
|
+
|
|
2406
3659
|
// ---------------------------------------------------------------
|
|
2407
3660
|
// Tool: totalreclaw_import_from
|
|
2408
3661
|
// ---------------------------------------------------------------
|
|
@@ -2412,7 +3665,7 @@ const plugin = {
|
|
|
2412
3665
|
name: 'totalreclaw_import_from',
|
|
2413
3666
|
label: 'Import From',
|
|
2414
3667
|
description:
|
|
2415
|
-
'Import memories from other AI memory tools (Mem0, MCP Memory Server, ChatGPT, Claude, MemoClaw, or generic JSON/CSV). ' +
|
|
3668
|
+
'Import memories from other AI memory tools (Mem0, MCP Memory Server, ChatGPT, Claude, Gemini, MemoClaw, or generic JSON/CSV). ' +
|
|
2416
3669
|
'Provide the source name and either an API key, file content, or file path. ' +
|
|
2417
3670
|
'Use dry_run=true to preview before importing. Idempotent — safe to run multiple times.',
|
|
2418
3671
|
parameters: {
|
|
@@ -2420,8 +3673,8 @@ const plugin = {
|
|
|
2420
3673
|
properties: {
|
|
2421
3674
|
source: {
|
|
2422
3675
|
type: 'string',
|
|
2423
|
-
enum: ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'memoclaw', 'generic-json', 'generic-csv'],
|
|
2424
|
-
description: 'The source system to import from (chatgpt: conversations.json or memory text; claude: memory text)',
|
|
3676
|
+
enum: ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'],
|
|
3677
|
+
description: 'The source system to import from (gemini: Google Takeout HTML; chatgpt: conversations.json or memory text; claude: memory text)',
|
|
2425
3678
|
},
|
|
2426
3679
|
api_key: {
|
|
2427
3680
|
type: 'string',
|
|
@@ -2463,6 +3716,56 @@ const plugin = {
|
|
|
2463
3716
|
{ name: 'totalreclaw_import_from' },
|
|
2464
3717
|
);
|
|
2465
3718
|
|
|
3719
|
+
// ---------------------------------------------------------------
|
|
3720
|
+
// Tool: totalreclaw_import_batch
|
|
3721
|
+
// ---------------------------------------------------------------
|
|
3722
|
+
|
|
3723
|
+
api.registerTool(
|
|
3724
|
+
{
|
|
3725
|
+
name: 'totalreclaw_import_batch',
|
|
3726
|
+
label: 'Import Batch',
|
|
3727
|
+
description:
|
|
3728
|
+
'Process one batch of a large import. Call repeatedly with increasing offset until is_complete=true.',
|
|
3729
|
+
parameters: {
|
|
3730
|
+
type: 'object',
|
|
3731
|
+
properties: {
|
|
3732
|
+
source: {
|
|
3733
|
+
type: 'string',
|
|
3734
|
+
enum: ['gemini', 'chatgpt', 'claude'],
|
|
3735
|
+
description: 'Source format',
|
|
3736
|
+
},
|
|
3737
|
+
file_path: {
|
|
3738
|
+
type: 'string',
|
|
3739
|
+
description: 'Path to source file',
|
|
3740
|
+
},
|
|
3741
|
+
content: {
|
|
3742
|
+
type: 'string',
|
|
3743
|
+
description: 'File content (text sources)',
|
|
3744
|
+
},
|
|
3745
|
+
offset: {
|
|
3746
|
+
type: 'number',
|
|
3747
|
+
description: 'Starting chunk index (0-based)',
|
|
3748
|
+
},
|
|
3749
|
+
batch_size: {
|
|
3750
|
+
type: 'number',
|
|
3751
|
+
description: 'Chunks per call (default 25)',
|
|
3752
|
+
},
|
|
3753
|
+
},
|
|
3754
|
+
required: ['source'],
|
|
3755
|
+
},
|
|
3756
|
+
async execute(_toolCallId: string, params: Record<string, unknown>) {
|
|
3757
|
+
try {
|
|
3758
|
+
await requireFullSetup(api.logger);
|
|
3759
|
+
return handleBatchImport(params, api.logger);
|
|
3760
|
+
} catch (err: unknown) {
|
|
3761
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
3762
|
+
return { error: message };
|
|
3763
|
+
}
|
|
3764
|
+
},
|
|
3765
|
+
},
|
|
3766
|
+
{ name: 'totalreclaw_import_batch' },
|
|
3767
|
+
);
|
|
3768
|
+
|
|
2466
3769
|
// ---------------------------------------------------------------
|
|
2467
3770
|
// Tool: totalreclaw_upgrade
|
|
2468
3771
|
// ---------------------------------------------------------------
|
|
@@ -2489,7 +3792,7 @@ const plugin = {
|
|
|
2489
3792
|
};
|
|
2490
3793
|
}
|
|
2491
3794
|
|
|
2492
|
-
const serverUrl =
|
|
3795
|
+
const serverUrl = CONFIG.serverUrl;
|
|
2493
3796
|
const walletAddr = subgraphOwner || userId || '';
|
|
2494
3797
|
|
|
2495
3798
|
if (!walletAddr) {
|
|
@@ -2534,7 +3837,7 @@ const plugin = {
|
|
|
2534
3837
|
const message = err instanceof Error ? err.message : String(err);
|
|
2535
3838
|
api.logger.error(`totalreclaw_upgrade failed: ${message}`);
|
|
2536
3839
|
return {
|
|
2537
|
-
content: [{ type: 'text', text: `Failed to create checkout session: ${message}` }],
|
|
3840
|
+
content: [{ type: 'text', text: `Failed to create checkout session: ${humanizeError(message)}` }],
|
|
2538
3841
|
};
|
|
2539
3842
|
}
|
|
2540
3843
|
},
|
|
@@ -2581,7 +3884,7 @@ const plugin = {
|
|
|
2581
3884
|
}
|
|
2582
3885
|
|
|
2583
3886
|
const confirm = _params?.confirm === true;
|
|
2584
|
-
const serverUrl =
|
|
3887
|
+
const serverUrl = CONFIG.serverUrl;
|
|
2585
3888
|
|
|
2586
3889
|
// 1. Check billing tier
|
|
2587
3890
|
const billingResp = await fetch(
|
|
@@ -2668,6 +3971,7 @@ const plugin = {
|
|
|
2668
3971
|
contentFp: fact.contentFp || '',
|
|
2669
3972
|
agentId: fact.agentId || 'openclaw-plugin',
|
|
2670
3973
|
encryptedEmbedding: fact.encryptedEmbedding || undefined,
|
|
3974
|
+
version: PROTOBUF_VERSION_V4,
|
|
2671
3975
|
};
|
|
2672
3976
|
payloads.push(encodeFactProtobuf(factPayload));
|
|
2673
3977
|
}
|
|
@@ -2717,7 +4021,7 @@ const plugin = {
|
|
|
2717
4021
|
const message = err instanceof Error ? err.message : String(err);
|
|
2718
4022
|
api.logger.error(`totalreclaw_migrate failed: ${message}`);
|
|
2719
4023
|
return {
|
|
2720
|
-
content: [{ type: 'text', text: `Migration failed: ${message}` }],
|
|
4024
|
+
content: [{ type: 'text', text: `Migration failed: ${humanizeError(message)}` }],
|
|
2721
4025
|
};
|
|
2722
4026
|
}
|
|
2723
4027
|
},
|
|
@@ -2725,6 +4029,107 @@ const plugin = {
|
|
|
2725
4029
|
{ name: 'totalreclaw_migrate' },
|
|
2726
4030
|
);
|
|
2727
4031
|
|
|
4032
|
+
// ---------------------------------------------------------------
|
|
4033
|
+
// Tool: totalreclaw_setup
|
|
4034
|
+
// ---------------------------------------------------------------
|
|
4035
|
+
|
|
4036
|
+
api.registerTool(
|
|
4037
|
+
{
|
|
4038
|
+
name: 'totalreclaw_setup',
|
|
4039
|
+
label: 'Setup TotalReclaw',
|
|
4040
|
+
description:
|
|
4041
|
+
'Initialize TotalReclaw with a recovery phrase. Derives encryption keys and registers with the server. ' +
|
|
4042
|
+
'Use this during first-time setup instead of setting environment variables — no gateway restart needed.',
|
|
4043
|
+
parameters: {
|
|
4044
|
+
type: 'object',
|
|
4045
|
+
properties: {
|
|
4046
|
+
recovery_phrase: {
|
|
4047
|
+
type: 'string',
|
|
4048
|
+
description: 'Optional 12-word BIP-39 recovery phrase. If not provided, generates a new one automatically. For returning users, pass their existing phrase.',
|
|
4049
|
+
},
|
|
4050
|
+
},
|
|
4051
|
+
additionalProperties: false,
|
|
4052
|
+
},
|
|
4053
|
+
async execute(_toolCallId: string, params: { recovery_phrase?: string }) {
|
|
4054
|
+
try {
|
|
4055
|
+
let mnemonic = params.recovery_phrase?.trim() || '';
|
|
4056
|
+
|
|
4057
|
+
// Auto-generate if not provided
|
|
4058
|
+
if (!mnemonic) {
|
|
4059
|
+
const { generateMnemonic } = await import('@scure/bip39');
|
|
4060
|
+
const { wordlist } = await import('@scure/bip39/wordlists/english');
|
|
4061
|
+
mnemonic = generateMnemonic(wordlist, 128);
|
|
4062
|
+
api.logger.info('totalreclaw_setup: generated new BIP-39 mnemonic');
|
|
4063
|
+
}
|
|
4064
|
+
|
|
4065
|
+
// Guard: refuse to overwrite existing credentials with a DIFFERENT phrase
|
|
4066
|
+
// (prevents data loss when background sessions_spawn workers call setup).
|
|
4067
|
+
// Allow re-init with the SAME phrase (handles agent exec → setup flow).
|
|
4068
|
+
try {
|
|
4069
|
+
const existing = fs.readFileSync(CREDENTIALS_PATH, 'utf-8');
|
|
4070
|
+
const creds = JSON.parse(existing);
|
|
4071
|
+
if (creds.mnemonic && creds.userId && creds.mnemonic !== mnemonic) {
|
|
4072
|
+
api.logger.info('totalreclaw_setup: credentials exist with different mnemonic, refusing to overwrite');
|
|
4073
|
+
return {
|
|
4074
|
+
content: [{
|
|
4075
|
+
type: 'text',
|
|
4076
|
+
text: 'TotalReclaw is already set up with an existing recovery phrase. Your encrypted memories are tied to that phrase.\n\n' +
|
|
4077
|
+
'If you intentionally want to start fresh with a NEW phrase (this will make existing memories inaccessible), ' +
|
|
4078
|
+
'delete ~/.totalreclaw/credentials.json first, then call this tool again.',
|
|
4079
|
+
}],
|
|
4080
|
+
};
|
|
4081
|
+
}
|
|
4082
|
+
} catch { /* credentials.json doesn't exist or is corrupted — proceed with setup */ }
|
|
4083
|
+
|
|
4084
|
+
// Basic validation: must be 12 words
|
|
4085
|
+
const words = mnemonic.split(/\s+/);
|
|
4086
|
+
if (words.length !== 12) {
|
|
4087
|
+
return {
|
|
4088
|
+
content: [{
|
|
4089
|
+
type: 'text',
|
|
4090
|
+
text: `Error: Recovery phrase must be exactly 12 words (got ${words.length}). Use \`npx @totalreclaw/mcp-server setup\` to generate a valid BIP-39 mnemonic.`,
|
|
4091
|
+
}],
|
|
4092
|
+
};
|
|
4093
|
+
}
|
|
4094
|
+
|
|
4095
|
+
api.logger.info('totalreclaw_setup: initializing with provided recovery phrase');
|
|
4096
|
+
|
|
4097
|
+
// Force re-initialization with the new mnemonic.
|
|
4098
|
+
// This derives keys, registers with the server, saves credentials,
|
|
4099
|
+
// and sets up LSH/auth — all without a gateway restart.
|
|
4100
|
+
await forceReinitialization(mnemonic, api.logger);
|
|
4101
|
+
|
|
4102
|
+
if (needsSetup) {
|
|
4103
|
+
return {
|
|
4104
|
+
content: [{
|
|
4105
|
+
type: 'text',
|
|
4106
|
+
text: 'Setup failed — could not initialize with the provided recovery phrase. Check the logs for details.',
|
|
4107
|
+
}],
|
|
4108
|
+
};
|
|
4109
|
+
}
|
|
4110
|
+
|
|
4111
|
+
const wasGenerated = !params.recovery_phrase?.trim();
|
|
4112
|
+
return {
|
|
4113
|
+
content: [{
|
|
4114
|
+
type: 'text',
|
|
4115
|
+
text: 'TotalReclaw setup complete! Encryption keys derived, server registration confirmed. ' +
|
|
4116
|
+
'You can now use totalreclaw_remember, totalreclaw_recall, and all other tools immediately — no restart needed.\n\n' +
|
|
4117
|
+
(wasGenerated ? `Recovery phrase: ${mnemonic}\n\n` : '') +
|
|
4118
|
+
'From now on, I will automatically remember important things from our conversations and recall relevant context at the start of each session.',
|
|
4119
|
+
}],
|
|
4120
|
+
};
|
|
4121
|
+
} catch (err: unknown) {
|
|
4122
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4123
|
+
api.logger.error(`totalreclaw_setup failed: ${message}`);
|
|
4124
|
+
return {
|
|
4125
|
+
content: [{ type: 'text', text: `Setup failed: ${humanizeError(message)}` }],
|
|
4126
|
+
};
|
|
4127
|
+
}
|
|
4128
|
+
},
|
|
4129
|
+
},
|
|
4130
|
+
{ name: 'totalreclaw_setup' },
|
|
4131
|
+
);
|
|
4132
|
+
|
|
2728
4133
|
// ---------------------------------------------------------------
|
|
2729
4134
|
// Hook: before_agent_start
|
|
2730
4135
|
// ---------------------------------------------------------------
|
|
@@ -2752,11 +4157,12 @@ const plugin = {
|
|
|
2752
4157
|
'## TotalReclaw — First-Time Setup\n\n' +
|
|
2753
4158
|
'TotalReclaw is an end-to-end encrypted memory vault. It remembers things across conversations — preferences, decisions, project context — all encrypted so no one (not even the server) can read your memories.\n\n' +
|
|
2754
4159
|
'**To get started, ask the user:** "Do you have an existing TotalReclaw recovery phrase you\'d like to restore, or should I generate a new one?"\n\n' +
|
|
2755
|
-
'**If new user:**
|
|
4160
|
+
'**If new user:** Run `npx @totalreclaw/mcp-server setup` to generate a cryptographically valid recovery phrase. Do NOT generate the phrase yourself — LLMs cannot compute BIP-39 checksums. The command will output the phrase. Then call the `totalreclaw_setup` tool with the phrase. Display the phrase prominently and warn:\n' +
|
|
2756
4161
|
'- "This is your ONLY way to access your memories. Write it down and store it securely."\n' +
|
|
2757
4162
|
'- "NEVER use a recovery phrase from an existing crypto wallet — keep this separate from any wallet that holds funds."\n' +
|
|
2758
4163
|
'- "If you lose this phrase, your memories are gone forever — there is no recovery."\n\n' +
|
|
2759
|
-
'**If returning user:** Ask them to provide their 12-word phrase,
|
|
4164
|
+
'**If returning user:** Ask them to provide their 12-word phrase, then call `totalreclaw_setup` with that phrase.\n\n' +
|
|
4165
|
+
'**IMPORTANT:** Use the `totalreclaw_setup` tool to complete setup. This initializes encryption keys immediately — no gateway restart needed. The user can continue chatting right away.\n\n' +
|
|
2760
4166
|
'**After setup:** Explain that from now on, you will automatically remember important things from conversations and recall relevant memories at the start of each session. The user can also explicitly ask you to remember, recall, forget, or export memories.',
|
|
2761
4167
|
};
|
|
2762
4168
|
}
|
|
@@ -2783,7 +4189,7 @@ const plugin = {
|
|
|
2783
4189
|
let cache = readBillingCache();
|
|
2784
4190
|
if (!cache && authKeyHex) {
|
|
2785
4191
|
// Cache is stale or missing — fetch fresh billing status.
|
|
2786
|
-
const billingUrl =
|
|
4192
|
+
const billingUrl = CONFIG.serverUrl;
|
|
2787
4193
|
const walletParam = encodeURIComponent(subgraphOwner || userId || '');
|
|
2788
4194
|
const billingResp = await fetch(`${billingUrl}/v1/billing/status?wallet_address=${walletParam}`, {
|
|
2789
4195
|
method: 'GET',
|
|
@@ -2812,7 +4218,46 @@ const plugin = {
|
|
|
2812
4218
|
}
|
|
2813
4219
|
|
|
2814
4220
|
if (isSubgraphMode()) {
|
|
2815
|
-
// --- Subgraph mode: hot cache
|
|
4221
|
+
// --- Subgraph mode: digest fast path → hot cache → background refresh ---
|
|
4222
|
+
|
|
4223
|
+
// Digest fast path (Stage 3b). When a digest exists and the mode is
|
|
4224
|
+
// not 'off', inject its pre-compiled promptText instead of running
|
|
4225
|
+
// the per-query search. A stale digest triggers a background
|
|
4226
|
+
// recompile (non-blocking). Failures fall through to the legacy
|
|
4227
|
+
// path silently.
|
|
4228
|
+
const digestMode = resolveDigestMode();
|
|
4229
|
+
logDigestModeOnce(digestMode, api.logger);
|
|
4230
|
+
if (digestMode !== 'off' && encryptionKey && authKeyHex && (subgraphOwner || userId)) {
|
|
4231
|
+
try {
|
|
4232
|
+
const injectResult = await maybeInjectDigest({
|
|
4233
|
+
owner: subgraphOwner || userId!,
|
|
4234
|
+
authKeyHex: authKeyHex!,
|
|
4235
|
+
encryptionKey: encryptionKey!,
|
|
4236
|
+
mode: digestMode,
|
|
4237
|
+
nowMs: Date.now(),
|
|
4238
|
+
loadDeps: {
|
|
4239
|
+
searchSubgraph: async (o, tds, n, a) => searchSubgraph(o, tds, n, a),
|
|
4240
|
+
decryptFromHex: (hex, key) => decryptFromHex(hex, key),
|
|
4241
|
+
},
|
|
4242
|
+
probeDeps: {
|
|
4243
|
+
searchSubgraphBroadened: async (o, n, a) => searchSubgraphBroadened(o, n, a),
|
|
4244
|
+
},
|
|
4245
|
+
recompileFn: (prev) => scheduleDigestRecompile(prev, api.logger),
|
|
4246
|
+
logger: api.logger,
|
|
4247
|
+
});
|
|
4248
|
+
if (injectResult.promptText) {
|
|
4249
|
+
api.logger.info(`Digest injection: state=${injectResult.state}`);
|
|
4250
|
+
return {
|
|
4251
|
+
prependContext:
|
|
4252
|
+
`## Your Memory\n\n${injectResult.promptText}` + welcomeBack + billingWarning,
|
|
4253
|
+
};
|
|
4254
|
+
}
|
|
4255
|
+
} catch (err) {
|
|
4256
|
+
// Never block session start on digest failure.
|
|
4257
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
4258
|
+
api.logger.warn(`Digest fast path failed: ${msg}`);
|
|
4259
|
+
}
|
|
4260
|
+
}
|
|
2816
4261
|
|
|
2817
4262
|
// Initialize hot cache if needed.
|
|
2818
4263
|
if (!pluginHotCache && encryptionKey) {
|
|
@@ -2885,6 +4330,21 @@ const plugin = {
|
|
|
2885
4330
|
return undefined;
|
|
2886
4331
|
}
|
|
2887
4332
|
|
|
4333
|
+
// Always run broadened search and merge — ensures vocabulary mismatches
|
|
4334
|
+
// (e.g., "preferences" vs "prefer") don't cause recall failures.
|
|
4335
|
+
// The reranker handles scoring; extra cost is ~1 GraphQL query per recall.
|
|
4336
|
+
try {
|
|
4337
|
+
const broadPool = computeCandidatePool(0);
|
|
4338
|
+
const broadenedResults = await searchSubgraphBroadened(subgraphOwner || userId!, broadPool, authKeyHex!);
|
|
4339
|
+
// Merge broadened results with existing (deduplicate by ID)
|
|
4340
|
+
const existingIds = new Set(subgraphResults.map(r => r.id));
|
|
4341
|
+
for (const br of broadenedResults) {
|
|
4342
|
+
if (!existingIds.has(br.id)) {
|
|
4343
|
+
subgraphResults.push(br);
|
|
4344
|
+
}
|
|
4345
|
+
}
|
|
4346
|
+
} catch { /* best-effort */ }
|
|
4347
|
+
|
|
2888
4348
|
if (subgraphResults.length === 0 && cachedFacts.length === 0) return undefined;
|
|
2889
4349
|
|
|
2890
4350
|
// If subgraph returned no results but we have cache, use cache.
|
|
@@ -2902,7 +4362,10 @@ const plugin = {
|
|
|
2902
4362
|
for (const result of subgraphResults) {
|
|
2903
4363
|
try {
|
|
2904
4364
|
const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
|
|
2905
|
-
|
|
4365
|
+
// Filter out digest infrastructure blobs — they have no user
|
|
4366
|
+
// text and should never surface in recall results.
|
|
4367
|
+
if (isDigestBlob(docJson)) continue;
|
|
4368
|
+
const doc = readClaimFromBlob(docJson);
|
|
2906
4369
|
|
|
2907
4370
|
let decryptedEmbedding: number[] | undefined;
|
|
2908
4371
|
if (result.encryptedEmbedding) {
|
|
@@ -2915,22 +4378,20 @@ const plugin = {
|
|
|
2915
4378
|
}
|
|
2916
4379
|
}
|
|
2917
4380
|
|
|
2918
|
-
const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
|
|
2919
4381
|
const createdAtSec = result.timestamp ? parseInt(result.timestamp, 10) : undefined;
|
|
2920
4382
|
rerankerCandidates.push({
|
|
2921
4383
|
id: result.id,
|
|
2922
4384
|
text: doc.text,
|
|
2923
4385
|
embedding: decryptedEmbedding,
|
|
2924
|
-
importance:
|
|
4386
|
+
importance: doc.importance / 10,
|
|
2925
4387
|
createdAt: createdAtSec,
|
|
4388
|
+
source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
|
|
2926
4389
|
});
|
|
2927
4390
|
|
|
2928
|
-
const importance = doc.metadata?.importance
|
|
2929
|
-
? Math.round((doc.metadata.importance as number) * 10)
|
|
2930
|
-
: 5;
|
|
2931
4391
|
hookMetaMap.set(result.id, {
|
|
2932
|
-
importance,
|
|
4392
|
+
importance: doc.importance,
|
|
2933
4393
|
age: 'subgraph',
|
|
4394
|
+
category: doc.category,
|
|
2934
4395
|
});
|
|
2935
4396
|
} catch {
|
|
2936
4397
|
// Skip un-decryptable candidates.
|
|
@@ -2945,17 +4406,9 @@ const plugin = {
|
|
|
2945
4406
|
rerankerCandidates,
|
|
2946
4407
|
8,
|
|
2947
4408
|
INTENT_WEIGHTS[hookQueryIntent],
|
|
4409
|
+
/* applySourceWeights (Retrieval v2 Tier 1) */ true,
|
|
2948
4410
|
);
|
|
2949
4411
|
|
|
2950
|
-
// B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
|
|
2951
|
-
const candidatesWithEmb = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
|
|
2952
|
-
if (candidatesWithEmb.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
|
|
2953
|
-
const topCosine = Math.max(
|
|
2954
|
-
...candidatesWithEmb.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
|
|
2955
|
-
);
|
|
2956
|
-
if (topCosine < RELEVANCE_THRESHOLD) return undefined;
|
|
2957
|
-
}
|
|
2958
|
-
|
|
2959
4412
|
// Update hot cache with reranked results.
|
|
2960
4413
|
try {
|
|
2961
4414
|
if (pluginHotCache) {
|
|
@@ -2994,7 +4447,8 @@ const plugin = {
|
|
|
2994
4447
|
const meta = hookMetaMap.get(m.id);
|
|
2995
4448
|
const importance = meta?.importance ?? 5;
|
|
2996
4449
|
const age = meta?.age ?? '';
|
|
2997
|
-
|
|
4450
|
+
const typeTag = meta?.category ? `[${meta.category}] ` : '';
|
|
4451
|
+
return `${i + 1}. ${typeTag}${m.text} (importance: ${importance}/10, ${age})`;
|
|
2998
4452
|
});
|
|
2999
4453
|
const contextString = `## Relevant Memories\n\n${lines.join('\n')}`;
|
|
3000
4454
|
|
|
@@ -3042,9 +4496,10 @@ const plugin = {
|
|
|
3042
4496
|
for (const candidate of candidates) {
|
|
3043
4497
|
try {
|
|
3044
4498
|
const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
|
|
3045
|
-
|
|
4499
|
+
// Skip digest infrastructure blobs.
|
|
4500
|
+
if (isDigestBlob(docJson)) continue;
|
|
4501
|
+
const doc = readClaimFromBlob(docJson);
|
|
3046
4502
|
|
|
3047
|
-
// Decrypt embedding if present.
|
|
3048
4503
|
let decryptedEmbedding: number[] | undefined;
|
|
3049
4504
|
if (candidate.encrypted_embedding) {
|
|
3050
4505
|
try {
|
|
@@ -3056,7 +4511,6 @@ const plugin = {
|
|
|
3056
4511
|
}
|
|
3057
4512
|
}
|
|
3058
4513
|
|
|
3059
|
-
const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
|
|
3060
4514
|
const createdAtSec = typeof candidate.timestamp === 'number'
|
|
3061
4515
|
? candidate.timestamp / 1000
|
|
3062
4516
|
: new Date(candidate.timestamp).getTime() / 1000;
|
|
@@ -3064,15 +4518,13 @@ const plugin = {
|
|
|
3064
4518
|
id: candidate.fact_id,
|
|
3065
4519
|
text: doc.text,
|
|
3066
4520
|
embedding: decryptedEmbedding,
|
|
3067
|
-
importance:
|
|
4521
|
+
importance: doc.importance / 10,
|
|
3068
4522
|
createdAt: createdAtSec,
|
|
4523
|
+
source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
|
|
3069
4524
|
});
|
|
3070
4525
|
|
|
3071
|
-
const importance = doc.metadata?.importance
|
|
3072
|
-
? Math.round((doc.metadata.importance as number) * 10)
|
|
3073
|
-
: 5;
|
|
3074
4526
|
hookMetaMap.set(candidate.fact_id, {
|
|
3075
|
-
importance,
|
|
4527
|
+
importance: doc.importance,
|
|
3076
4528
|
age: relativeTime(candidate.timestamp),
|
|
3077
4529
|
});
|
|
3078
4530
|
} catch {
|
|
@@ -3088,19 +4540,23 @@ const plugin = {
|
|
|
3088
4540
|
rerankerCandidates,
|
|
3089
4541
|
8,
|
|
3090
4542
|
INTENT_WEIGHTS[srvHookIntent],
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
// B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
|
|
3094
|
-
const candidatesWithEmbSrv = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
|
|
3095
|
-
if (candidatesWithEmbSrv.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
|
|
3096
|
-
const topCosine = Math.max(
|
|
3097
|
-
...candidatesWithEmbSrv.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
|
|
4543
|
+
/* applySourceWeights (Retrieval v2 Tier 1) */ true,
|
|
3098
4544
|
);
|
|
3099
|
-
if (topCosine < RELEVANCE_THRESHOLD) return undefined;
|
|
3100
|
-
}
|
|
3101
4545
|
|
|
3102
4546
|
if (reranked.length === 0) return undefined;
|
|
3103
4547
|
|
|
4548
|
+
// Cosine similarity threshold gate — skip injection when the
|
|
4549
|
+
// best match is below the minimum relevance threshold.
|
|
4550
|
+
const srvMaxCosine = Math.max(
|
|
4551
|
+
...reranked.map((r) => r.cosineSimilarity ?? 0),
|
|
4552
|
+
);
|
|
4553
|
+
if (srvMaxCosine < COSINE_THRESHOLD) {
|
|
4554
|
+
api.logger.info(
|
|
4555
|
+
`Hook: cosine threshold gate filtered results (max=${srvMaxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD})`,
|
|
4556
|
+
);
|
|
4557
|
+
return undefined;
|
|
4558
|
+
}
|
|
4559
|
+
|
|
3104
4560
|
// 7. Build context string.
|
|
3105
4561
|
const lines = reranked.map((m, i) => {
|
|
3106
4562
|
const meta = hookMetaMap.get(m.id);
|
|
@@ -3128,21 +4584,73 @@ const plugin = {
|
|
|
3128
4584
|
api.on(
|
|
3129
4585
|
'agent_end',
|
|
3130
4586
|
async (event: unknown) => {
|
|
4587
|
+
// CRITICAL: Always return { memoryHandled: true } so OpenClaw's default
|
|
4588
|
+
// memory system does NOT fall back to writing plaintext MEMORY.md.
|
|
4589
|
+
// Losing facts on error is acceptable; leaking them in cleartext is not.
|
|
3131
4590
|
try {
|
|
4591
|
+
// Defensive: ensure MEMORY.md header is present so OpenClaw's default
|
|
4592
|
+
// memory system doesn't write sensitive data in cleartext, even if
|
|
4593
|
+
// our extraction fails below.
|
|
4594
|
+
ensureMemoryHeader(api.logger);
|
|
4595
|
+
|
|
4596
|
+
// BUG-2 fix: skip extraction if an import was in progress this turn.
|
|
4597
|
+
// Import failures were retriggering agent_end → extraction → import loops.
|
|
4598
|
+
if (_importInProgress) {
|
|
4599
|
+
_importInProgress = false; // auto-reset for next turn
|
|
4600
|
+
api.logger.info('agent_end: skipping extraction (import was in progress)');
|
|
4601
|
+
return { memoryHandled: true };
|
|
4602
|
+
}
|
|
4603
|
+
|
|
3132
4604
|
const evt = event as { messages?: unknown[]; success?: boolean } | undefined;
|
|
3133
|
-
if (!evt?.
|
|
4605
|
+
if (!evt?.messages || evt.messages.length < 2) {
|
|
4606
|
+
api.logger.info('agent_end: skipping extraction (no messages)');
|
|
4607
|
+
return { memoryHandled: true };
|
|
4608
|
+
}
|
|
4609
|
+
// Proceed with extraction even when evt.success is false or undefined.
|
|
4610
|
+
// A single LLM timeout on one turn should not prevent extraction of
|
|
4611
|
+
// facts from the (potentially many) successful turns in the message
|
|
4612
|
+
// history. The extractor processes the full message array and can
|
|
4613
|
+
// extract valuable facts from content before the failure.
|
|
4614
|
+
if (evt.success === false) {
|
|
4615
|
+
api.logger.info('agent_end: turn reported failure, but proceeding with extraction from message history');
|
|
4616
|
+
}
|
|
3134
4617
|
|
|
3135
4618
|
await ensureInitialized(api.logger);
|
|
3136
|
-
if (needsSetup) return;
|
|
4619
|
+
if (needsSetup) return { memoryHandled: true };
|
|
3137
4620
|
|
|
3138
4621
|
// C3: Throttle auto-extraction to every N turns (configurable via env).
|
|
4622
|
+
// Phase 2.2.5: every branch of the extraction pipeline now logs its
|
|
4623
|
+
// outcome. Prior to 2.2.5, only the "stored N facts" happy path
|
|
4624
|
+
// produced a log line, so silent JSON parse failures / chatCompletion
|
|
4625
|
+
// timeouts / importance-filter-drops-everything scenarios left no
|
|
4626
|
+
// trace whatsoever in the gateway log. See the investigation report
|
|
4627
|
+
// in CHANGELOG for the full failure chain we uncovered.
|
|
3139
4628
|
turnsSinceLastExtraction++;
|
|
3140
|
-
|
|
4629
|
+
const extractInterval = getExtractInterval();
|
|
4630
|
+
api.logger.info(
|
|
4631
|
+
`agent_end: turn ${turnsSinceLastExtraction}/${extractInterval} (messages=${evt.messages.length})`,
|
|
4632
|
+
);
|
|
4633
|
+
if (turnsSinceLastExtraction >= extractInterval) {
|
|
3141
4634
|
const existingMemories = isLlmDedupEnabled()
|
|
3142
4635
|
? await fetchExistingMemoriesForExtraction(api.logger, 20, evt.messages)
|
|
3143
4636
|
: [];
|
|
3144
|
-
const rawFacts = await extractFacts(
|
|
3145
|
-
|
|
4637
|
+
const rawFacts = await extractFacts(
|
|
4638
|
+
evt.messages,
|
|
4639
|
+
'turn',
|
|
4640
|
+
existingMemories,
|
|
4641
|
+
undefined,
|
|
4642
|
+
api.logger,
|
|
4643
|
+
);
|
|
4644
|
+
api.logger.info(
|
|
4645
|
+
`agent_end: extractFacts returned ${rawFacts.length} raw facts`,
|
|
4646
|
+
);
|
|
4647
|
+
const { kept: importanceFiltered, dropped } = filterByImportance(
|
|
4648
|
+
rawFacts,
|
|
4649
|
+
api.logger,
|
|
4650
|
+
);
|
|
4651
|
+
api.logger.info(
|
|
4652
|
+
`agent_end: after importance filter: kept=${importanceFiltered.length}, dropped=${dropped}`,
|
|
4653
|
+
);
|
|
3146
4654
|
const maxFacts = getMaxFactsPerExtraction();
|
|
3147
4655
|
if (importanceFiltered.length > maxFacts) {
|
|
3148
4656
|
api.logger.info(
|
|
@@ -3152,13 +4660,23 @@ const plugin = {
|
|
|
3152
4660
|
const facts = importanceFiltered.slice(0, maxFacts);
|
|
3153
4661
|
if (facts.length > 0) {
|
|
3154
4662
|
await storeExtractedFacts(facts, api.logger);
|
|
4663
|
+
api.logger.info(`agent_end: stored ${facts.length} facts to encrypted vault`);
|
|
4664
|
+
} else {
|
|
4665
|
+
// Phase 2.2.5: no longer silent when extraction produces nothing.
|
|
4666
|
+
api.logger.info(
|
|
4667
|
+
`agent_end: extraction produced 0 storable facts (raw=${rawFacts.length}, after-importance=${importanceFiltered.length})`,
|
|
4668
|
+
);
|
|
3155
4669
|
}
|
|
3156
4670
|
turnsSinceLastExtraction = 0;
|
|
3157
4671
|
}
|
|
3158
4672
|
} catch (err: unknown) {
|
|
3159
4673
|
const message = err instanceof Error ? err.message : String(err);
|
|
3160
|
-
api.logger.
|
|
4674
|
+
api.logger.error(`agent_end extraction failed: ${message}`);
|
|
4675
|
+
// Re-assert MEMORY.md header even on failure — last line of defense.
|
|
4676
|
+
ensureMemoryHeader(api.logger);
|
|
3161
4677
|
}
|
|
4678
|
+
// Always signal that memory is handled — prevent plaintext fallback.
|
|
4679
|
+
return { memoryHandled: true };
|
|
3162
4680
|
},
|
|
3163
4681
|
{ priority: 90 },
|
|
3164
4682
|
);
|
|
@@ -3178,13 +4696,13 @@ const plugin = {
|
|
|
3178
4696
|
if (needsSetup) return;
|
|
3179
4697
|
|
|
3180
4698
|
api.logger.info(
|
|
3181
|
-
`
|
|
4699
|
+
`pre_compaction: using compaction-aware extraction (importance >= 5), processing ${evt.messages.length} messages`,
|
|
3182
4700
|
);
|
|
3183
4701
|
|
|
3184
4702
|
const existingMemories = isLlmDedupEnabled()
|
|
3185
4703
|
? await fetchExistingMemoriesForExtraction(api.logger, 50, evt.messages)
|
|
3186
4704
|
: [];
|
|
3187
|
-
const rawCompactFacts = await
|
|
4705
|
+
const rawCompactFacts = await extractFactsForCompaction(evt.messages, existingMemories, api.logger);
|
|
3188
4706
|
const { kept: compactImportanceFiltered } = filterByImportance(rawCompactFacts, api.logger);
|
|
3189
4707
|
const maxFactsCompact = getMaxFactsPerExtraction();
|
|
3190
4708
|
if (compactImportanceFiltered.length > maxFactsCompact) {
|
|
@@ -3197,6 +4715,29 @@ const plugin = {
|
|
|
3197
4715
|
await storeExtractedFacts(facts, api.logger);
|
|
3198
4716
|
}
|
|
3199
4717
|
turnsSinceLastExtraction = 0; // Reset C3 counter on compaction.
|
|
4718
|
+
|
|
4719
|
+
// Session debrief — after regular extraction.
|
|
4720
|
+
// v1 mapping: DebriefItem { type: 'summary'|'context' } →
|
|
4721
|
+
// v1 type 'summary' (always, since context → claim would lose
|
|
4722
|
+
// the "this is a session summary" signal) + source 'derived'
|
|
4723
|
+
// (session debrief is a derived synthesis by definition).
|
|
4724
|
+
try {
|
|
4725
|
+
const storedTexts = facts.map((f) => f.text);
|
|
4726
|
+
const debriefItems = await extractDebrief(evt.messages, storedTexts);
|
|
4727
|
+
if (debriefItems.length > 0) {
|
|
4728
|
+
const debriefFacts: ExtractedFact[] = debriefItems.map((d) => ({
|
|
4729
|
+
text: d.text,
|
|
4730
|
+
type: 'summary' as MemoryType,
|
|
4731
|
+
source: 'derived' as MemorySource,
|
|
4732
|
+
importance: d.importance,
|
|
4733
|
+
action: 'ADD' as const,
|
|
4734
|
+
}));
|
|
4735
|
+
await storeExtractedFacts(debriefFacts, api.logger, 'openclaw_debrief');
|
|
4736
|
+
api.logger.info(`Session debrief: stored ${debriefItems.length} items`);
|
|
4737
|
+
}
|
|
4738
|
+
} catch (debriefErr: unknown) {
|
|
4739
|
+
api.logger.warn(`before_compaction debrief failed: ${debriefErr instanceof Error ? debriefErr.message : String(debriefErr)}`);
|
|
4740
|
+
}
|
|
3200
4741
|
} catch (err: unknown) {
|
|
3201
4742
|
const message = err instanceof Error ? err.message : String(err);
|
|
3202
4743
|
api.logger.warn(`before_compaction extraction failed: ${message}`);
|
|
@@ -3239,6 +4780,29 @@ const plugin = {
|
|
|
3239
4780
|
await storeExtractedFacts(facts, api.logger);
|
|
3240
4781
|
}
|
|
3241
4782
|
turnsSinceLastExtraction = 0; // Reset C3 counter on reset.
|
|
4783
|
+
|
|
4784
|
+
// Session debrief — after regular extraction.
|
|
4785
|
+
// v1 mapping: DebriefItem { type: 'summary'|'context' } →
|
|
4786
|
+
// v1 type 'summary' (always, since context → claim would lose
|
|
4787
|
+
// the "this is a session summary" signal) + source 'derived'
|
|
4788
|
+
// (session debrief is a derived synthesis by definition).
|
|
4789
|
+
try {
|
|
4790
|
+
const storedTexts = facts.map((f) => f.text);
|
|
4791
|
+
const debriefItems = await extractDebrief(evt.messages, storedTexts);
|
|
4792
|
+
if (debriefItems.length > 0) {
|
|
4793
|
+
const debriefFacts: ExtractedFact[] = debriefItems.map((d) => ({
|
|
4794
|
+
text: d.text,
|
|
4795
|
+
type: 'summary' as MemoryType,
|
|
4796
|
+
source: 'derived' as MemorySource,
|
|
4797
|
+
importance: d.importance,
|
|
4798
|
+
action: 'ADD' as const,
|
|
4799
|
+
}));
|
|
4800
|
+
await storeExtractedFacts(debriefFacts, api.logger, 'openclaw_debrief');
|
|
4801
|
+
api.logger.info(`Session debrief: stored ${debriefItems.length} items`);
|
|
4802
|
+
}
|
|
4803
|
+
} catch (debriefErr: unknown) {
|
|
4804
|
+
api.logger.warn(`before_reset debrief failed: ${debriefErr instanceof Error ? debriefErr.message : String(debriefErr)}`);
|
|
4805
|
+
}
|
|
3242
4806
|
} catch (err: unknown) {
|
|
3243
4807
|
const message = err instanceof Error ? err.message : String(err);
|
|
3244
4808
|
api.logger.warn(`before_reset extraction failed: ${message}`);
|