@totalreclaw/totalreclaw 1.6.0 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.ts CHANGED
@@ -8,14 +8,17 @@
8
8
  * - totalreclaw_export -- export all memories (JSON or Markdown)
9
9
  * - totalreclaw_status -- check billing/subscription status
10
10
  * - totalreclaw_consolidate -- scan and merge near-duplicate memories
11
+ * - totalreclaw_pin -- pin a memory so auto-resolution can never supersede it
12
+ * - totalreclaw_unpin -- remove a pin, returning the memory to active status
11
13
  * - totalreclaw_import_from -- import memories from other tools (Mem0, MCP Memory, etc.)
12
14
  * - totalreclaw_upgrade -- create Stripe checkout for Pro upgrade
13
15
  * - totalreclaw_migrate -- migrate testnet memories to mainnet after Pro upgrade
16
+ * - totalreclaw_setup -- initialize with recovery phrase (no gateway restart needed)
14
17
  *
15
18
  * Also registers a `before_agent_start` hook that automatically injects
16
19
  * relevant memories into the agent's context.
17
20
  *
18
- * All data is encrypted client-side with AES-256-GCM. The server never
21
+ * All data is encrypted client-side with XChaCha20-Poly1305. The server never
19
22
  * sees plaintext.
20
23
  */
21
24
 
@@ -29,8 +32,24 @@ import {
29
32
  generateContentFingerprint,
30
33
  } from './crypto.js';
31
34
  import { createApiClient, type StoreFactPayload } from './api-client.js';
32
- import { extractFacts, type ExtractedFact } from './extractor.js';
33
- import { initLLMClient, generateEmbedding, getEmbeddingDims } from './llm-client.js';
35
+ import {
36
+ extractFacts,
37
+ extractDebrief,
38
+ isValidMemoryType,
39
+ parseEntity,
40
+ VALID_MEMORY_TYPES,
41
+ LEGACY_V0_MEMORY_TYPES,
42
+ VALID_MEMORY_SOURCES,
43
+ VALID_MEMORY_SCOPES,
44
+ EXTRACTION_SYSTEM_PROMPT,
45
+ extractFactsForCompaction,
46
+ type ExtractedFact,
47
+ type ExtractedEntity,
48
+ type MemoryType,
49
+ type MemorySource,
50
+ type MemoryScope,
51
+ } from './extractor.js';
52
+ import { initLLMClient, resolveLLMConfig, chatCompletion, generateEmbedding, getEmbeddingDims } from './llm-client.js';
34
53
  import { LSHHasher } from './lsh.js';
35
54
  import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS, type RerankerCandidate } from './reranker.js';
36
55
  import { deduplicateBatch } from './semantic-dedup.js';
@@ -43,9 +62,39 @@ import {
43
62
  STORE_DEDUP_MAX_CANDIDATES,
44
63
  type DecryptedCandidate,
45
64
  } from './consolidation.js';
46
- import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, type FactPayload } from './subgraph-store.js';
47
- import { searchSubgraph, getSubgraphFactCount } from './subgraph-search.js';
65
+ import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4, type FactPayload } from './subgraph-store.js';
66
+ import {
67
+ DIGEST_TRAPDOOR,
68
+ buildCanonicalClaim,
69
+ computeEntityTrapdoor,
70
+ computeEntityTrapdoors,
71
+ isDigestBlob,
72
+ normalizeToV1Type,
73
+ readClaimFromBlob,
74
+ resolveDigestMode,
75
+ type DigestMode,
76
+ } from './claims-helper.js';
77
+ import {
78
+ maybeInjectDigest,
79
+ recompileDigest,
80
+ fetchAllActiveClaims,
81
+ isRecompileInProgress,
82
+ tryBeginRecompile,
83
+ endRecompile,
84
+ } from './digest-sync.js';
85
+ import {
86
+ detectAndResolveContradictions,
87
+ runWeightTuningLoop,
88
+ type ResolutionDecision as ContradictionDecision,
89
+ } from './contradiction-sync.js';
90
+ import { searchSubgraph, searchSubgraphBroadened, getSubgraphFactCount, fetchFactById } from './subgraph-search.js';
91
+ import {
92
+ executePinOperation,
93
+ validatePinArgs,
94
+ type PinOpDeps,
95
+ } from './pin.js';
48
96
  import { PluginHotCache, type HotFact } from './hot-cache-wrapper.js';
97
+ import { CONFIG, setRecoveryPhraseOverride, setChainIdOverride } from './config.js';
49
98
  import crypto from 'node:crypto';
50
99
  import fs from 'node:fs';
51
100
  import path from 'node:path';
@@ -68,6 +117,16 @@ interface OpenClawPluginApi {
68
117
  };
69
118
  };
70
119
  };
120
+ models?: {
121
+ providers?: Record<string, {
122
+ baseUrl: string;
123
+ apiKey?: string;
124
+ api?: string;
125
+ models?: Array<{ id: string; [k: string]: unknown }>;
126
+ [k: string]: unknown;
127
+ }>;
128
+ [k: string]: unknown;
129
+ };
71
130
  [key: string]: unknown;
72
131
  };
73
132
  pluginConfig?: Record<string, unknown>;
@@ -76,12 +135,44 @@ interface OpenClawPluginApi {
76
135
  on(hookName: string, handler: (...args: unknown[]) => unknown, opts?: { priority?: number }): void;
77
136
  }
78
137
 
138
+ // ---------------------------------------------------------------------------
139
+ // Human-friendly error messages
140
+ // ---------------------------------------------------------------------------
141
+
142
+ /**
143
+ * Translate technical error messages from the on-chain submission pipeline
144
+ * into user-friendly messages. The original technical details are still
145
+ * logged via api.logger — this only affects what the agent sees.
146
+ */
147
+ function humanizeError(rawMessage: string): string {
148
+ if (rawMessage.includes('AA23')) {
149
+ return 'Memory storage temporarily unavailable. Will retry next time.';
150
+ }
151
+ if (rawMessage.includes('AA10')) {
152
+ return 'Please wait a moment before storing more memories.';
153
+ }
154
+ if (rawMessage.includes('AA25')) {
155
+ return 'Memory storage busy. Will retry.';
156
+ }
157
+ if (rawMessage.includes('pm_sponsorUserOperation')) {
158
+ return 'Memory storage service temporarily unavailable.';
159
+ }
160
+ if (/Relay returned HTTP\s*404/.test(rawMessage)) {
161
+ return 'Memory service is temporarily offline.';
162
+ }
163
+ if (/Relay returned HTTP\s*5\d\d/.test(rawMessage)) {
164
+ return 'Memory service encountered a temporary error. Will retry next time.';
165
+ }
166
+ // Pass through non-technical messages as-is.
167
+ return rawMessage;
168
+ }
169
+
79
170
  // ---------------------------------------------------------------------------
80
171
  // Persistent credential storage
81
172
  // ---------------------------------------------------------------------------
82
173
 
83
174
  /** Path where we persist userId + salt across restarts. */
84
- const CREDENTIALS_PATH = process.env.TOTALRECLAW_CREDENTIALS_PATH || `${process.env.HOME ?? '/home/node'}/.totalreclaw/credentials.json`;
175
+ const CREDENTIALS_PATH = CONFIG.credentialsPath;
85
176
 
86
177
  // ---------------------------------------------------------------------------
87
178
  // Cosine similarity threshold — skip injection when top result is below this
@@ -92,12 +183,10 @@ const CREDENTIALS_PATH = process.env.TOTALRECLAW_CREDENTIALS_PATH || `${process.
92
183
  * memories into context. Below this threshold, the query is considered
93
184
  * irrelevant to any stored memories and results are suppressed.
94
185
  *
95
- * Default 0.15 is tuned for bge-small-en-v1.5 which produces lower
186
+ * Default 0.15 is tuned for local ONNX models which produce lower
96
187
  * similarity scores than OpenAI models. Configurable via env var.
97
188
  */
98
- const COSINE_THRESHOLD = parseFloat(
99
- process.env.TOTALRECLAW_COSINE_THRESHOLD ?? '0.15',
100
- );
189
+ const COSINE_THRESHOLD = CONFIG.cosineThreshold;
101
190
 
102
191
  // ---------------------------------------------------------------------------
103
192
  // Module-level state (persists across tool calls within a session)
@@ -123,30 +212,36 @@ let lastSearchTimestamp = 0;
123
212
  let lastQueryEmbedding: number[] | null = null;
124
213
 
125
214
  // Feature flags — configurable for A/B testing
126
- const CACHE_TTL_MS = parseInt(process.env.TOTALRECLAW_CACHE_TTL_MS ?? String(5 * 60 * 1000), 10);
127
- const SEMANTIC_SKIP_THRESHOLD = parseFloat(process.env.TOTALRECLAW_SEMANTIC_SKIP_THRESHOLD ?? '0.85');
215
+ const CACHE_TTL_MS = CONFIG.cacheTtlMs;
216
+ const SEMANTIC_SKIP_THRESHOLD = CONFIG.semanticSkipThreshold;
128
217
 
129
218
  // Auto-extract throttle (C3): only extract every N turns in agent_end hook
130
219
  let turnsSinceLastExtraction = 0;
131
- const AUTO_EXTRACT_EVERY_TURNS_ENV = parseInt(process.env.TOTALRECLAW_EXTRACT_EVERY_TURNS ?? '3', 10);
220
+
221
+ // BUG-2 fix: Skip agent_end extraction during import operations.
222
+ // Import failures previously triggered agent_end → re-extraction → re-import loops.
223
+ let _importInProgress = false;
224
+ const AUTO_EXTRACT_EVERY_TURNS_ENV = CONFIG.extractInterval;
132
225
 
133
226
  // Hard cap on facts per extraction to prevent LLM over-extraction from dense conversations
134
227
  const MAX_FACTS_PER_EXTRACTION = 15;
135
228
 
136
- // Store-time near-duplicate detection (consolidation module)
137
- const STORE_DEDUP_ENABLED = process.env.TOTALRECLAW_STORE_DEDUP !== 'false';
229
+ // Store-time near-duplicate detection is always ON in v1.
230
+ // The TOTALRECLAW_STORE_DEDUP env var was removed.
231
+ const STORE_DEDUP_ENABLED = true;
138
232
 
139
233
  // One-time welcome-back message for returning Pro users (set during init, consumed by first before_agent_start)
140
234
  let welcomeBackMessage: string | null = null;
141
235
 
142
- // B2: Minimum relevance threshold cosine below this means no memory injection
143
- const RELEVANCE_THRESHOLD = parseFloat(process.env.TOTALRECLAW_RELEVANCE_THRESHOLD ?? '0.3');
236
+ // B2: COSINE_THRESHOLD (above) is the single relevance gate for both
237
+ // the before_agent_start hook and the recall tool. The former "RELEVANCE_THRESHOLD"
238
+ // (0.3) was too aggressive and silently suppressed auto-recall at session start.
144
239
 
145
240
  // ---------------------------------------------------------------------------
146
241
  // Billing cache infrastructure
147
242
  // ---------------------------------------------------------------------------
148
243
 
149
- const BILLING_CACHE_PATH = path.join(process.env.HOME ?? '/home/node', '.totalreclaw', 'billing-cache.json');
244
+ const BILLING_CACHE_PATH = CONFIG.billingCachePath;
150
245
  const BILLING_CACHE_TTL = 2 * 60 * 60 * 1000; // 2 hours
151
246
  const QUOTA_WARNING_THRESHOLD = 0.8; // 80%
152
247
 
@@ -165,11 +260,34 @@ interface BillingCache {
165
260
  checked_at: number;
166
261
  }
167
262
 
263
+ /**
264
+ * Apply the billing tier to the runtime chain override.
265
+ *
266
+ * Pro tier → chain 100 (Gnosis mainnet). Free tier (or unknown) stays on
267
+ * 84532 (Base Sepolia). The relay routes Pro UserOps to Gnosis, so the
268
+ * client MUST sign them against chain 100 — otherwise the bundler returns
269
+ * AA23 (invalid signature). See MCP's equivalent path in mcp/src/index.ts.
270
+ *
271
+ * Called from `readBillingCache` and `writeBillingCache` so that every cache
272
+ * read or write keeps the chain override in sync with the cached tier.
273
+ * Idempotent — calling with the same tier is a no-op.
274
+ */
275
+ function syncChainIdFromTier(tier: string | undefined): void {
276
+ if (tier === 'pro') {
277
+ setChainIdOverride(100);
278
+ } else {
279
+ // Free or unknown → reset to the default free-tier chain.
280
+ setChainIdOverride(84532);
281
+ }
282
+ }
283
+
168
284
  function readBillingCache(): BillingCache | null {
169
285
  try {
170
286
  if (!fs.existsSync(BILLING_CACHE_PATH)) return null;
171
287
  const raw = JSON.parse(fs.readFileSync(BILLING_CACHE_PATH, 'utf-8')) as BillingCache;
172
288
  if (!raw.checked_at || Date.now() - raw.checked_at > BILLING_CACHE_TTL) return null;
289
+ // Keep chain override in sync with persisted tier across process restarts.
290
+ syncChainIdFromTier(raw.tier);
173
291
  return raw;
174
292
  } catch {
175
293
  return null;
@@ -184,18 +302,22 @@ function writeBillingCache(cache: BillingCache): void {
184
302
  } catch {
185
303
  // Best-effort — don't block on cache write failure.
186
304
  }
305
+ // Sync chain override AFTER the write so in-process UserOp signing picks
306
+ // up the correct chain immediately, even if the disk write failed.
307
+ syncChainIdFromTier(cache.tier);
187
308
  }
188
309
 
189
310
  /**
190
- * Check if LLM-guided dedup is enabled for the current tier.
191
- * Returns true for Pro users, or when no billing cache exists (fail-open for self-hosters).
311
+ * Check if LLM-guided dedup is enabled.
312
+ *
313
+ * Always returns true — LLM extraction runs client-side using the user's
314
+ * own API key, so there is no cost to us. The server flag is respected as
315
+ * a kill-switch but defaults to true for all tiers.
192
316
  */
193
317
  function isLlmDedupEnabled(): boolean {
194
318
  const cache = readBillingCache();
195
- if (!cache) return true;
196
- if (cache.tier === 'pro') return true;
197
- if (cache.features?.llm_dedup !== undefined) return cache.features.llm_dedup;
198
- return false;
319
+ if (cache?.features?.llm_dedup === false) return false; // Server kill-switch
320
+ return true;
199
321
  }
200
322
 
201
323
  /**
@@ -241,7 +363,7 @@ const MEMORY_HEADER = `# Memory
241
363
 
242
364
  function ensureMemoryHeader(logger: OpenClawPluginApi['logger']): void {
243
365
  try {
244
- const workspace = path.join(process.env.HOME ?? '/home/node', '.openclaw', 'workspace');
366
+ const workspace = CONFIG.openclawWorkspace;
245
367
  const memoryMd = path.join(workspace, 'MEMORY.md');
246
368
 
247
369
  if (fs.existsSync(memoryMd)) {
@@ -340,9 +462,8 @@ let firstRunAfterInit = true;
340
462
  * register with the server if this is the first run.
341
463
  */
342
464
  async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
343
- const serverUrl =
344
- process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz';
345
- const masterPassword = process.env.TOTALRECLAW_RECOVERY_PHRASE;
465
+ const serverUrl = CONFIG.serverUrl || 'https://api.totalreclaw.xyz';
466
+ const masterPassword = CONFIG.recoveryPhrase;
346
467
 
347
468
  if (!masterPassword) {
348
469
  needsSetup = true;
@@ -359,7 +480,14 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
359
480
  try {
360
481
  if (fs.existsSync(CREDENTIALS_PATH)) {
361
482
  const creds = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8'));
362
- existingSalt = Buffer.from(creds.salt, 'base64');
483
+ // Salt may be stored as base64 (plugin-written) or hex (MCP setup-written).
484
+ // Detect format: hex strings are 64 chars of [0-9a-f], base64 uses [A-Z+/=].
485
+ const saltStr: string = creds.salt;
486
+ if (saltStr && /^[0-9a-f]{64}$/i.test(saltStr)) {
487
+ existingSalt = Buffer.from(saltStr, 'hex');
488
+ } else if (saltStr) {
489
+ existingSalt = Buffer.from(saltStr, 'base64');
490
+ }
363
491
  existingUserId = creds.userId;
364
492
  logger.info(`Loaded existing credentials for user ${existingUserId}`);
365
493
  }
@@ -380,6 +508,20 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
380
508
  if (existingUserId) {
381
509
  userId = existingUserId;
382
510
  logger.info(`Authenticated as user ${userId}`);
511
+
512
+ // Idempotent registration — ensure auth key is registered with the relay.
513
+ // Without this, returning users get 401 if the relay database was reset or
514
+ // if credentials were created by the MCP setup CLI (different process).
515
+ try {
516
+ const authHash = computeAuthKeyHash(keys.authKey);
517
+ const saltHex = keys.salt.toString('hex');
518
+ await apiClient.register(authHash, saltHex);
519
+ } catch {
520
+ // Best-effort — relay returns 200 for already-registered users.
521
+ // Only fails on network errors; bearer token auth still works if
522
+ // a prior registration succeeded.
523
+ logger.warn('Idempotent relay registration failed (best-effort, will retry on next start)');
524
+ }
383
525
  } else {
384
526
  // First run -- register with the server.
385
527
  const authHash = computeAuthKeyHash(keys.authKey);
@@ -405,14 +547,20 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
405
547
  userId = registeredUserId!;
406
548
 
407
549
  // Persist credentials so we can resume later.
550
+ // Include the mnemonic so hot-reload works without env var.
408
551
  const dir = path.dirname(CREDENTIALS_PATH);
409
552
  if (!fs.existsSync(dir)) {
410
553
  fs.mkdirSync(dir, { recursive: true });
411
554
  }
412
- fs.writeFileSync(
413
- CREDENTIALS_PATH,
414
- JSON.stringify({ userId, salt: keys.salt.toString('base64') }),
415
- );
555
+ const credsToSave: Record<string, string> = {
556
+ userId,
557
+ salt: keys.salt.toString('base64'),
558
+ };
559
+ // Only persist mnemonic if we have one (avoid writing empty string).
560
+ if (masterPassword) {
561
+ credsToSave.mnemonic = masterPassword;
562
+ }
563
+ fs.writeFileSync(CREDENTIALS_PATH, JSON.stringify(credsToSave), { mode: 0o600 });
416
564
 
417
565
  logger.info(`Registered new user: ${userId}`);
418
566
  }
@@ -436,7 +584,7 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
436
584
  try {
437
585
  const walletAddr = subgraphOwner || userId || '';
438
586
  if (walletAddr) {
439
- const billingUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
587
+ const billingUrl = CONFIG.serverUrl;
440
588
  const resp = await fetch(`${billingUrl}/v1/billing/status?wallet_address=${encodeURIComponent(walletAddr)}`, {
441
589
  method: 'GET',
442
590
  headers: {
@@ -479,6 +627,13 @@ function isDocker(): boolean {
479
627
  }
480
628
 
481
629
  function buildSetupErrorMsg(): string {
630
+ return 'TotalReclaw setup required. Use the `totalreclaw_setup` tool with a 12-word BIP-39 recovery phrase.\n\n' +
631
+ '1. Ask the user if they have an existing recovery phrase, or generate a new one with `npx @totalreclaw/mcp-server setup`.\n' +
632
+ '2. Call `totalreclaw_setup` with the phrase — no gateway restart needed.\n' +
633
+ ' (Optional: set TOTALRECLAW_SELF_HOSTED=true if using your own server instead of the managed service.)';
634
+ }
635
+
636
+ function buildSetupErrorMsgLegacy(): string {
482
637
  const base =
483
638
  'TotalReclaw setup required:\n' +
484
639
  '1. Set TOTALRECLAW_RECOVERY_PHRASE — ask the user if they have an existing recovery phrase or generate a new 12-word BIP-39 mnemonic.\n' +
@@ -509,12 +664,101 @@ const SETUP_ERROR_MSG = buildSetupErrorMsg();
509
664
 
510
665
  /**
511
666
  * Ensure `initialize()` has completed (runs at most once).
667
+ *
668
+ * If `needsSetup` is true after init, attempts a hot-reload from
669
+ * credentials.json in case the mnemonic was written there by a
670
+ * `totalreclaw_setup` tool call or `npx @totalreclaw/mcp-server setup`.
512
671
  */
513
672
  async function ensureInitialized(logger: OpenClawPluginApi['logger']): Promise<void> {
514
673
  if (!initPromise) {
515
674
  initPromise = initialize(logger);
516
675
  }
517
676
  await initPromise;
677
+
678
+ // Hot-reload: if setup is still needed, check if credentials.json
679
+ // now has a mnemonic (written by totalreclaw_setup or MCP setup CLI).
680
+ if (needsSetup) {
681
+ await attemptHotReload(logger);
682
+ }
683
+ }
684
+
685
+ /**
686
+ * Attempt to hot-reload credentials from credentials.json.
687
+ *
688
+ * Called when `needsSetup` is true — checks if credentials.json contains
689
+ * a mnemonic (written by the `totalreclaw_setup` tool or MCP setup CLI).
690
+ * If found, re-derives keys and completes initialization without requiring
691
+ * a gateway restart.
692
+ */
693
+ async function attemptHotReload(logger: OpenClawPluginApi['logger']): Promise<void> {
694
+ try {
695
+ if (!fs.existsSync(CREDENTIALS_PATH)) return;
696
+
697
+ const creds = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8'));
698
+ if (!creds.mnemonic) return;
699
+
700
+ logger.info('Hot-reloading credentials from credentials.json (no restart needed)');
701
+
702
+ // Set the runtime override so CONFIG.recoveryPhrase returns the mnemonic.
703
+ setRecoveryPhraseOverride(creds.mnemonic);
704
+
705
+ // Re-run initialization with the newly available mnemonic.
706
+ needsSetup = false;
707
+ initPromise = initialize(logger);
708
+ await initPromise;
709
+ } catch (err) {
710
+ const msg = err instanceof Error ? err.message : String(err);
711
+ logger.warn(`Hot-reload from credentials.json failed: ${msg}`);
712
+ // Leave needsSetup as true — user will see the setup prompt.
713
+ }
714
+ }
715
+
716
+ /**
717
+ * Force re-initialization with a specific mnemonic.
718
+ *
719
+ * Called by the `totalreclaw_setup` tool. Clears stale credentials from
720
+ * disk so that `initialize()` treats this as a fresh registration and
721
+ * persists the NEW mnemonic + freshly derived salt/userId.
722
+ *
723
+ * Without clearing credentials.json first, `initialize()` would load the
724
+ * OLD salt and userId, derive keys from (new mnemonic + old salt), skip
725
+ * writing credentials (because existingUserId is set), and the new
726
+ * mnemonic would never be persisted — a critical data-loss bug.
727
+ */
728
+ async function forceReinitialization(mnemonic: string, logger: OpenClawPluginApi['logger']): Promise<void> {
729
+ // Set the runtime override so CONFIG.recoveryPhrase returns this mnemonic.
730
+ setRecoveryPhraseOverride(mnemonic);
731
+
732
+ // CRITICAL: Remove stale credentials so initialize() does a fresh
733
+ // registration with a new salt. If we leave the old file, initialize()
734
+ // loads the old salt + userId and never writes the new mnemonic.
735
+ try {
736
+ if (fs.existsSync(CREDENTIALS_PATH)) {
737
+ fs.unlinkSync(CREDENTIALS_PATH);
738
+ logger.info('Cleared stale credentials.json for fresh setup');
739
+ }
740
+ } catch (err) {
741
+ logger.warn(`Could not remove old credentials.json: ${err instanceof Error ? err.message : String(err)}`);
742
+ }
743
+
744
+ // Reset module state for a clean re-init.
745
+ needsSetup = false;
746
+ authKeyHex = null;
747
+ encryptionKey = null;
748
+ dedupKey = null;
749
+ userId = null;
750
+ subgraphOwner = null;
751
+ apiClient = null;
752
+ lshHasher = null;
753
+ lshInitFailed = false;
754
+ masterPasswordCache = null;
755
+ saltCache = null;
756
+ pluginHotCache = null;
757
+ firstRunAfterInit = true;
758
+
759
+ // Re-run initialization — will register fresh and persist new credentials.
760
+ initPromise = initialize(logger);
761
+ await initPromise;
518
762
  }
519
763
 
520
764
  /**
@@ -634,7 +878,8 @@ async function searchForNearDuplicates(
634
878
  for (const result of results) {
635
879
  try {
636
880
  const docJson = decryptFromHex(result.encryptedBlob, encryptionKey);
637
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
881
+ if (isDigestBlob(docJson)) continue;
882
+ const doc = readClaimFromBlob(docJson);
638
883
 
639
884
  let embedding: number[] | null = null;
640
885
  if (result.encryptedEmbedding) {
@@ -647,9 +892,7 @@ async function searchForNearDuplicates(
647
892
  id: result.id,
648
893
  text: doc.text,
649
894
  embedding,
650
- importance: doc.metadata?.importance
651
- ? Math.round((doc.metadata.importance as number) * 10)
652
- : 5,
895
+ importance: doc.importance,
653
896
  decayScore: 5,
654
897
  createdAt: result.timestamp ? parseInt(result.timestamp, 10) * 1000 : Date.now(),
655
898
  version: 1,
@@ -666,7 +909,8 @@ async function searchForNearDuplicates(
666
909
  for (const candidate of candidates) {
667
910
  try {
668
911
  const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey);
669
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
912
+ if (isDigestBlob(docJson)) continue;
913
+ const doc = readClaimFromBlob(docJson);
670
914
 
671
915
  let embedding: number[] | null = null;
672
916
  if (candidate.encrypted_embedding) {
@@ -679,9 +923,7 @@ async function searchForNearDuplicates(
679
923
  id: candidate.fact_id,
680
924
  text: doc.text,
681
925
  embedding,
682
- importance: doc.metadata?.importance
683
- ? Math.round((doc.metadata.importance as number) * 10)
684
- : 5,
926
+ importance: doc.importance,
685
927
  decayScore: candidate.decay_score,
686
928
  createdAt: typeof candidate.timestamp === 'number'
687
929
  ? candidate.timestamp
@@ -720,6 +962,182 @@ function encryptToHex(plaintext: string, key: Buffer): string {
720
962
  return Buffer.from(b64, 'base64').toString('hex');
721
963
  }
722
964
 
965
+ // Plugin v3.0.0 removed the legacy claim-format fallback. Write path
966
+ // always emits Memory Taxonomy v1 JSON blobs. The logClaimFormatOnce
967
+ // helper is gone along with TOTALRECLAW_CLAIM_FORMAT / TOTALRECLAW_TAXONOMY_VERSION.
968
+
969
+ let _loggedDigestMode = false;
970
+ function logDigestModeOnce(mode: DigestMode, logger: OpenClawPluginApi['logger']): void {
971
+ if (_loggedDigestMode) return;
972
+ _loggedDigestMode = true;
973
+ logger.info(`TotalReclaw: digest injection mode = ${mode}`);
974
+ }
975
+
976
+ /**
977
+ * How many active facts to pull into a digest recompilation.
978
+ * Digest compiler itself will apply DIGEST_CLAIM_CAP for the LLM path.
979
+ */
980
+ const DIGEST_FETCH_LIMIT = 500;
981
+
982
+ /**
983
+ * Schedule a background digest recompile. Fire-and-forget.
984
+ *
985
+ * The caller must check `!isRecompileInProgress()` before invoking.
986
+ * Errors are logged and swallowed; the guard flag is always released.
987
+ */
988
+ function scheduleDigestRecompile(
989
+ previousClaimId: string | null,
990
+ logger: OpenClawPluginApi['logger'],
991
+ ): void {
992
+ if (!isRecompileInProgress()) {
993
+ if (!tryBeginRecompile()) return;
994
+ } else {
995
+ return;
996
+ }
997
+
998
+ const mode = resolveDigestMode();
999
+ const owner = subgraphOwner || userId;
1000
+ const authKey = authKeyHex;
1001
+ const encKey = encryptionKey;
1002
+ const ownerForBatch = subgraphOwner ?? undefined;
1003
+
1004
+ if (!owner || !authKey || !encKey) {
1005
+ endRecompile();
1006
+ return;
1007
+ }
1008
+
1009
+ // Capture llmFn from the current LLM config (cheap variant of the user's
1010
+ // provider, already resolved by resolveLLMConfig).
1011
+ const llmConfig = resolveLLMConfig();
1012
+ const llmFn = llmConfig
1013
+ ? async (prompt: string): Promise<string> => {
1014
+ const out = await chatCompletion(
1015
+ llmConfig,
1016
+ [
1017
+ { role: 'system', content: 'You return only valid JSON. No markdown fences, no commentary.' },
1018
+ { role: 'user', content: prompt },
1019
+ ],
1020
+ { maxTokens: 800, temperature: 0 },
1021
+ );
1022
+ return out ?? '';
1023
+ }
1024
+ : null;
1025
+
1026
+ // Build the I/O deps closures. We capture the owner/auth/key values so the
1027
+ // background task doesn't race with module-level state resets.
1028
+ const fetchFn = () =>
1029
+ fetchAllActiveClaims(
1030
+ owner,
1031
+ authKey,
1032
+ encKey,
1033
+ DIGEST_FETCH_LIMIT,
1034
+ {
1035
+ searchSubgraphBroadened: async (o, n, a) => searchSubgraphBroadened(o, n, a),
1036
+ decryptFromHex: (hex, key) => decryptFromHex(hex, key),
1037
+ },
1038
+ logger,
1039
+ );
1040
+
1041
+ const storeFn = async (canonicalClaimJson: string, compiledAt: string): Promise<void> => {
1042
+ if (!isSubgraphMode()) {
1043
+ // Self-hosted mode — store via the REST API.
1044
+ if (!apiClient) throw new Error('apiClient not initialized');
1045
+ const encryptedBlob = encryptToHex(canonicalClaimJson, encKey);
1046
+ const contentFp = generateContentFingerprint(canonicalClaimJson, dedupKey!);
1047
+ const payload: StoreFactPayload = {
1048
+ id: crypto.randomUUID(),
1049
+ timestamp: compiledAt,
1050
+ encrypted_blob: encryptedBlob,
1051
+ blind_indices: [DIGEST_TRAPDOOR],
1052
+ decay_score: 10,
1053
+ source: 'openclaw-plugin-digest',
1054
+ content_fp: contentFp,
1055
+ agent_id: 'openclaw-plugin-digest',
1056
+ };
1057
+ await apiClient.store(userId!, [payload], authKey);
1058
+ return;
1059
+ }
1060
+
1061
+ // Subgraph / managed-service mode — encrypt, encode, submit as a single-fact UserOp.
1062
+ const encryptedBlob = encryptToHex(canonicalClaimJson, encKey);
1063
+ const contentFp = generateContentFingerprint(canonicalClaimJson, dedupKey!);
1064
+ const protobuf = encodeFactProtobuf({
1065
+ id: crypto.randomUUID(),
1066
+ timestamp: compiledAt,
1067
+ owner,
1068
+ encryptedBlob,
1069
+ blindIndices: [DIGEST_TRAPDOOR],
1070
+ decayScore: 10,
1071
+ source: 'openclaw-plugin-digest',
1072
+ contentFp,
1073
+ agentId: 'openclaw-plugin-digest',
1074
+ version: PROTOBUF_VERSION_V4,
1075
+ });
1076
+ const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
1077
+ const result = await submitFactBatchOnChain([protobuf], config);
1078
+ if (!result.success) {
1079
+ throw new Error('Digest store UserOp did not succeed on-chain');
1080
+ }
1081
+ };
1082
+
1083
+ const tombstoneFn = async (claimId: string): Promise<void> => {
1084
+ if (!isSubgraphMode()) {
1085
+ if (apiClient) {
1086
+ try { await apiClient.deleteFact(claimId, authKey); } catch { /* best-effort */ }
1087
+ }
1088
+ return;
1089
+ }
1090
+ const tombstone: FactPayload = {
1091
+ id: claimId,
1092
+ timestamp: new Date().toISOString(),
1093
+ owner,
1094
+ encryptedBlob: '00',
1095
+ blindIndices: [],
1096
+ decayScore: 0,
1097
+ source: 'tombstone',
1098
+ contentFp: '',
1099
+ agentId: 'openclaw-plugin-digest',
1100
+ version: PROTOBUF_VERSION_V4,
1101
+ };
1102
+ const protobuf = encodeFactProtobuf(tombstone);
1103
+ const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
1104
+ const result = await submitFactBatchOnChain([protobuf], config);
1105
+ if (!result.success) {
1106
+ throw new Error('Digest tombstone UserOp did not succeed on-chain');
1107
+ }
1108
+ };
1109
+
1110
+ // Slice 2f: run the weight-tuning loop as a fire-and-forget pre-compile step.
1111
+ // This consumes any feedback.jsonl entries written since the last compile
1112
+ // and nudges ~/.totalreclaw/weights.json, so the NEXT contradiction detection
1113
+ // uses the adjusted weights. Rate-limited and idempotent — see
1114
+ // runWeightTuningLoop for details. Failures are logged, never fatal.
1115
+ void runWeightTuningLoop(Math.floor(Date.now() / 1000), logger).catch((err: unknown) => {
1116
+ const msg = err instanceof Error ? err.message : String(err);
1117
+ logger.warn(`Digest: tuning loop threw: ${msg}`);
1118
+ });
1119
+
1120
+ void recompileDigest({
1121
+ mode,
1122
+ previousClaimId,
1123
+ nowUnixSeconds: Math.floor(Date.now() / 1000),
1124
+ deps: {
1125
+ storeDigestClaim: storeFn,
1126
+ tombstoneDigest: tombstoneFn,
1127
+ fetchAllActiveClaimsFn: fetchFn,
1128
+ llmFn,
1129
+ },
1130
+ logger,
1131
+ })
1132
+ .catch((err: unknown) => {
1133
+ const msg = err instanceof Error ? err.message : String(err);
1134
+ logger.warn(`Digest: background recompile threw: ${msg}`);
1135
+ })
1136
+ .finally(() => {
1137
+ endRecompile();
1138
+ });
1139
+ }
1140
+
723
1141
  /**
724
1142
  * Decrypt a hex-encoded ciphertext blob into a UTF-8 string.
725
1143
  */
@@ -909,7 +1327,8 @@ async function fetchExistingMemoriesForExtraction(
909
1327
  for (const r of rawResults) {
910
1328
  try {
911
1329
  const docJson = decryptFromHex(r.encryptedBlob, encryptionKey);
912
- const doc = JSON.parse(docJson) as { text: string };
1330
+ if (isDigestBlob(docJson)) continue;
1331
+ const doc = readClaimFromBlob(docJson);
913
1332
  results.push({ id: r.id, text: doc.text });
914
1333
  } catch { /* skip undecryptable */ }
915
1334
  }
@@ -918,7 +1337,8 @@ async function fetchExistingMemoriesForExtraction(
918
1337
  for (const c of candidates) {
919
1338
  try {
920
1339
  const docJson = decryptFromHex(c.encrypted_blob, encryptionKey);
921
- const doc = JSON.parse(docJson) as { text: string };
1340
+ if (isDigestBlob(docJson)) continue;
1341
+ const doc = readClaimFromBlob(docJson);
922
1342
  results.push({ id: c.fact_id, text: doc.text });
923
1343
  } catch { /* skip undecryptable */ }
924
1344
  }
@@ -975,10 +1395,7 @@ function relativeTime(isoOrMs: string | number): string {
975
1395
  * NOTE: This filter is ONLY applied to auto-extraction (hooks).
976
1396
  * The explicit `totalreclaw_remember` tool always stores regardless of importance.
977
1397
  */
978
- const MIN_IMPORTANCE_THRESHOLD = Math.max(
979
- 1,
980
- Math.min(10, Number(process.env.TOTALRECLAW_MIN_IMPORTANCE) || 3),
981
- );
1398
+ const MIN_IMPORTANCE_THRESHOLD = CONFIG.minImportance;
982
1399
 
983
1400
  /**
984
1401
  * Filter extracted facts by importance threshold.
@@ -1001,10 +1418,20 @@ function filterByImportance(
1001
1418
  }
1002
1419
  }
1003
1420
 
1004
- if (dropped > 0) {
1421
+ // Phase 2.2.5: always log the filter outcome so the agent_end path can
1422
+ // distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
1423
+ // below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
1424
+ // this only logged on drops, which made empty-input invisible.
1425
+ if (facts.length === 0) {
1426
+ logger.info('Importance filter: input=0 (nothing to filter)');
1427
+ } else if (dropped > 0) {
1005
1428
  logger.info(
1006
1429
  `Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`,
1007
1430
  );
1431
+ } else {
1432
+ logger.info(
1433
+ `Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`,
1434
+ );
1008
1435
  }
1009
1436
 
1010
1437
  return { kept, dropped };
@@ -1026,6 +1453,7 @@ function filterByImportance(
1026
1453
  async function storeExtractedFacts(
1027
1454
  facts: ExtractedFact[],
1028
1455
  logger: OpenClawPluginApi['logger'],
1456
+ sourceOverride?: string,
1029
1457
  ): Promise<number> {
1030
1458
  if (!encryptionKey || !dedupKey || !authKeyHex || !userId || !apiClient) return 0;
1031
1459
 
@@ -1063,18 +1491,24 @@ async function storeExtractedFacts(
1063
1491
  let stored = 0;
1064
1492
  let superseded = 0;
1065
1493
  let skipped = 0;
1494
+ let failedFacts = 0;
1066
1495
  const pendingPayloads: Buffer[] = []; // Batched subgraph payloads
1067
1496
  let preparedForSubgraph = 0;
1068
1497
 
1498
+ // Plugin v3.0.0: always emit Memory Taxonomy v1 JSON blobs. The
1499
+ // TOTALRECLAW_TAXONOMY_VERSION opt-in and the TOTALRECLAW_CLAIM_FORMAT
1500
+ // legacy fallback have both been retired — v1 is the single write path.
1501
+
1069
1502
  for (const fact of dedupedFacts) {
1070
1503
  try {
1071
1504
  const blindIndices = generateBlindIndices(fact.text);
1505
+ const entityTrapdoors = computeEntityTrapdoors(fact.entities);
1072
1506
 
1073
1507
  // Use pre-computed embedding result if available.
1074
1508
  const embeddingResult = embeddingResultMap.get(fact.text) ?? null;
1075
1509
  const allIndices = embeddingResult
1076
- ? [...blindIndices, ...embeddingResult.lshBuckets]
1077
- : blindIndices;
1510
+ ? [...blindIndices, ...embeddingResult.lshBuckets, ...entityTrapdoors]
1511
+ : [...blindIndices, ...entityTrapdoors];
1078
1512
 
1079
1513
  // LLM-guided dedup: handle UPDATE/DELETE/NOOP actions.
1080
1514
  if (fact.action === 'NOOP') {
@@ -1096,6 +1530,7 @@ async function storeExtractedFacts(
1096
1530
  source: 'tombstone',
1097
1531
  contentFp: '',
1098
1532
  agentId: 'openclaw-plugin-auto',
1533
+ version: PROTOBUF_VERSION_V4,
1099
1534
  };
1100
1535
  pendingPayloads.push(encodeFactProtobuf(tombstone));
1101
1536
  logger.info(`LLM dedup: DELETE — queued tombstone for ${fact.existingFactId}`);
@@ -1124,6 +1559,7 @@ async function storeExtractedFacts(
1124
1559
  source: 'tombstone',
1125
1560
  contentFp: '',
1126
1561
  agentId: 'openclaw-plugin-auto',
1562
+ version: PROTOBUF_VERSION_V4,
1127
1563
  };
1128
1564
  pendingPayloads.push(encodeFactProtobuf(tombstone));
1129
1565
  logger.info(`LLM dedup: UPDATE — queued tombstone for ${fact.existingFactId}, storing replacement`);
@@ -1174,6 +1610,7 @@ async function storeExtractedFacts(
1174
1610
  source: 'tombstone',
1175
1611
  contentFp: '',
1176
1612
  agentId: 'openclaw-plugin-auto',
1613
+ version: PROTOBUF_VERSION_V4,
1177
1614
  };
1178
1615
  pendingPayloads.push(encodeFactProtobuf(tombstone));
1179
1616
  logger.info(
@@ -1196,20 +1633,133 @@ async function storeExtractedFacts(
1196
1633
  }
1197
1634
  }
1198
1635
 
1199
- const doc = {
1200
- text: fact.text,
1201
- metadata: {
1202
- type: fact.type,
1203
- importance: effectiveImportance / 10,
1204
- source: 'auto-extraction',
1205
- created_at: new Date().toISOString(),
1206
- },
1207
- };
1636
+ const factSource = sourceOverride || 'auto-extraction';
1637
+
1638
+ // Plugin v3.0.0: always build a Memory Taxonomy v1 JSON blob. The
1639
+ // blob is decryptable by `readClaimFromBlob` which prefers v1 →
1640
+ // falls back to v0 short-key → then plugin-legacy {text, metadata}
1641
+ // for pre-v3 vault entries.
1642
+ //
1643
+ // We build it BEFORE the on-chain write so Phase 2 contradiction
1644
+ // detection can inspect the same canonical Claim the write path will
1645
+ // actually store. The string is encrypted byte-identically below.
1646
+ //
1647
+ // Defensive: if the extraction hook didn't populate `fact.source`
1648
+ // (e.g. explicit tool path, legacy caller), default to 'user-inferred'
1649
+ // so v1 schema validation passes.
1650
+ const factForBlob: ExtractedFact = fact.source
1651
+ ? fact
1652
+ : { ...fact, source: 'user-inferred' };
1653
+ const blobPlaintext = buildCanonicalClaim({
1654
+ fact: factForBlob,
1655
+ importance: effectiveImportance,
1656
+ sourceAgent: factSource,
1657
+ });
1658
+
1659
+ const factId = crypto.randomUUID();
1660
+
1661
+ // Phase 2 Slice 2d: contradiction detection + auto-resolution.
1662
+ //
1663
+ // Runs only when the canonical Claim format is active (legacy blobs
1664
+ // carry no entity refs, so there is nothing to check), only for
1665
+ // Subgraph / managed-service mode (self-hosted contradiction handling
1666
+ // can come later), and only when the new fact has entities. The helper
1667
+ // is a no-op in all other cases.
1668
+ //
1669
+ // Returns one decision per candidate contradicting claim:
1670
+ // - supersede_existing → queue a tombstone + proceed with the new write
1671
+ // - skip_new → do not write the new fact; record the skip reason
1672
+ // - empty list → no contradiction, proceed unchanged
1673
+ //
1674
+ // On any error (subgraph, decrypt, WASM), the helper returns [] and we
1675
+ // fall back to Phase 1 behaviour.
1676
+ let contradictionSkipNew = false;
1677
+ if (
1678
+ isSubgraphMode() &&
1679
+ fact.entities &&
1680
+ fact.entities.length > 0 &&
1681
+ embeddingResult
1682
+ ) {
1683
+ const newClaimObj = JSON.parse(blobPlaintext) as Record<string, unknown>;
1684
+ let decisions: ContradictionDecision[] = [];
1685
+ try {
1686
+ decisions = await detectAndResolveContradictions({
1687
+ newClaim: newClaimObj,
1688
+ newClaimId: factId,
1689
+ newEmbedding: embeddingResult.embedding,
1690
+ subgraphOwner: subgraphOwner || userId!,
1691
+ authKeyHex: authKeyHex!,
1692
+ encryptionKey: encryptionKey!,
1693
+ deps: {
1694
+ searchSubgraph: (owner, trapdoors, maxCandidates, authKey) =>
1695
+ searchSubgraph(owner, trapdoors, maxCandidates, authKey).then((rows) =>
1696
+ rows.map((r) => ({
1697
+ id: r.id,
1698
+ encryptedBlob: r.encryptedBlob,
1699
+ encryptedEmbedding: r.encryptedEmbedding ?? null,
1700
+ timestamp: r.timestamp,
1701
+ isActive: r.isActive,
1702
+ })),
1703
+ ),
1704
+ decryptFromHex: (hex, key) => decryptFromHex(hex, key),
1705
+ },
1706
+ logger: {
1707
+ info: (m) => logger.info(m),
1708
+ warn: (m) => logger.warn(m),
1709
+ },
1710
+ });
1711
+ } catch (crErr) {
1712
+ // detectAndResolveContradictions is supposed to never throw — if
1713
+ // it does, we log and continue with Phase 1 behaviour.
1714
+ const msg = crErr instanceof Error ? crErr.message : String(crErr);
1715
+ logger.warn(`Contradiction detection failed (proceeding with store): ${msg}`);
1716
+ decisions = [];
1717
+ }
1718
+
1719
+ for (const decision of decisions) {
1720
+ if (decision.action === 'supersede_existing') {
1721
+ const tombstone: FactPayload = {
1722
+ id: decision.existingFactId,
1723
+ timestamp: new Date().toISOString(),
1724
+ owner: subgraphOwner || userId!,
1725
+ encryptedBlob: '00',
1726
+ blindIndices: [],
1727
+ decayScore: 0,
1728
+ source: 'tombstone',
1729
+ contentFp: '',
1730
+ agentId: 'openclaw-plugin-auto',
1731
+ version: PROTOBUF_VERSION_V4,
1732
+ };
1733
+ pendingPayloads.push(encodeFactProtobuf(tombstone));
1734
+ superseded++;
1735
+ logger.info(
1736
+ `Auto-resolve: queued supersede for ${decision.existingFactId.slice(0, 10)}… ` +
1737
+ `(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
1738
+ );
1739
+ } else if (decision.action === 'skip_new') {
1740
+ if (decision.reason === 'existing_pinned') {
1741
+ logger.warn(
1742
+ `Auto-resolve: skipped new write — existing claim ${decision.existingFactId.slice(0, 10)}… is pinned ` +
1743
+ `(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
1744
+ );
1745
+ } else {
1746
+ logger.info(
1747
+ `Auto-resolve: skipped new write — existing ${decision.existingFactId.slice(0, 10)}… wins ` +
1748
+ `(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
1749
+ );
1750
+ }
1751
+ contradictionSkipNew = true;
1752
+ }
1753
+ }
1754
+ }
1208
1755
 
1209
- const encryptedBlob = encryptToHex(JSON.stringify(doc), encryptionKey);
1756
+ if (contradictionSkipNew) {
1757
+ skipped++;
1758
+ continue;
1759
+ }
1210
1760
 
1761
+ const encryptedBlob = encryptToHex(blobPlaintext, encryptionKey);
1211
1762
  const contentFp = generateContentFingerprint(fact.text, dedupKey);
1212
- const factId = crypto.randomUUID();
1213
1763
 
1214
1764
  if (isSubgraphMode()) {
1215
1765
  const protobuf = encodeFactProtobuf({
@@ -1219,9 +1769,10 @@ async function storeExtractedFacts(
1219
1769
  encryptedBlob: encryptedBlob,
1220
1770
  blindIndices: allIndices,
1221
1771
  decayScore: effectiveImportance,
1222
- source: 'auto-extraction',
1772
+ source: factSource,
1223
1773
  contentFp: contentFp,
1224
1774
  agentId: 'openclaw-plugin-auto',
1775
+ version: PROTOBUF_VERSION_V4,
1225
1776
  encryptedEmbedding: embeddingResult?.encryptedEmbedding,
1226
1777
  });
1227
1778
  pendingPayloads.push(protobuf);
@@ -1233,7 +1784,7 @@ async function storeExtractedFacts(
1233
1784
  encrypted_blob: encryptedBlob,
1234
1785
  blind_indices: allIndices,
1235
1786
  decay_score: effectiveImportance,
1236
- source: 'auto-extraction',
1787
+ source: factSource,
1237
1788
  content_fp: contentFp,
1238
1789
  agent_id: 'openclaw-plugin-auto',
1239
1790
  encrypted_embedding: embeddingResult?.encryptedEmbedding,
@@ -1244,40 +1795,68 @@ async function storeExtractedFacts(
1244
1795
  } catch (err: unknown) {
1245
1796
  // Check for 403 / quota exceeded — invalidate billing cache so next
1246
1797
  // before_agent_start re-fetches and warns the user.
1247
- const errMsg = err instanceof Error ? err.message : String(err);
1248
- if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
1798
+ const factErrMsg = err instanceof Error ? err.message : String(err);
1799
+ if (factErrMsg.includes('403') || factErrMsg.toLowerCase().includes('quota')) {
1249
1800
  try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
1250
- logger.warn(`Quota exceeded — billing cache invalidated. ${errMsg}`);
1801
+ logger.warn(`Quota exceeded — billing cache invalidated. ${factErrMsg}`);
1251
1802
  break; // Stop trying to store remaining facts — they'll all fail too
1252
1803
  }
1253
- // Otherwise skip failed facts (e.g., duplicates return success with duplicate_ids)
1804
+ // Otherwise log and continue individual fact failures shouldn't block remaining facts
1805
+ logger.warn(`Failed to store fact "${fact.text.slice(0, 60)}…": ${factErrMsg}`);
1806
+ failedFacts++;
1254
1807
  }
1255
1808
  }
1256
1809
 
1257
- // Batch-submit all subgraph payloads in a single UserOp (gas-efficient).
1810
+ // Submit subgraph payloads one fact at a time (sequential single-call UserOps).
1811
+ // Batch executeBatch UserOps have persistent gas estimation issues on Base Sepolia
1812
+ // that cause on-chain reverts. Single-fact UserOps use the simpler submitFactOnChain
1813
+ // path which works reliably (same path as totalreclaw_remember). Each submission
1814
+ // polls for receipt (120s) before proceeding, so nonce is consumed before the next.
1815
+ let batchError: string | undefined;
1258
1816
  if (pendingPayloads.length > 0 && isSubgraphMode()) {
1259
- try {
1260
- const batchConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1261
- const result = await submitFactBatchOnChain(pendingPayloads, batchConfig);
1262
- if (result.success) {
1263
- stored += preparedForSubgraph;
1264
- logger.info(`Batch submitted ${result.batchSize} payloads in 1 UserOp (tx=${result.txHash.slice(0, 10)}…)`);
1265
- } else {
1266
- logger.warn(`Batch UserOp failed on-chain (tx=${result.txHash.slice(0, 10)}…)`);
1267
- }
1268
- } catch (err: unknown) {
1269
- const errMsg = err instanceof Error ? err.message : String(err);
1270
- if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
1271
- try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
1272
- logger.warn(`Quota exceeded during batch submit — billing cache invalidated. ${errMsg}`);
1273
- } else {
1274
- logger.warn(`Batch submission failed: ${errMsg}`);
1817
+ const batchConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1818
+ for (let i = 0; i < pendingPayloads.length; i++) {
1819
+ const slice = [pendingPayloads[i]]; // Single fact per UserOp
1820
+ try {
1821
+ const result = await submitFactBatchOnChain(slice, batchConfig);
1822
+ if (result.success) {
1823
+ stored += slice.length;
1824
+ logger.info(`Fact ${i + 1}/${pendingPayloads.length}: submitted on-chain (tx=${result.txHash.slice(0, 10)}…)`);
1825
+ } else {
1826
+ batchError = `On-chain batch submission failed (tx=${result.txHash.slice(0, 10)}…)`;
1827
+ logger.warn(batchError);
1828
+ break; // Stop submitting remaining batches
1829
+ }
1830
+ } catch (err: unknown) {
1831
+ const errMsg = err instanceof Error ? err.message : String(err);
1832
+ if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
1833
+ try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
1834
+ batchError = `Quota exceeded — billing cache invalidated. ${errMsg}`;
1835
+ logger.warn(batchError);
1836
+ break;
1837
+ } else {
1838
+ batchError = `Batch submission failed: ${errMsg}`;
1839
+ logger.warn(batchError);
1840
+ break;
1841
+ }
1275
1842
  }
1276
1843
  }
1277
1844
  }
1278
1845
 
1279
- if (stored > 0 || superseded > 0 || skipped > 0) {
1280
- logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}`);
1846
+ if (stored > 0 || superseded > 0 || skipped > 0 || failedFacts > 0) {
1847
+ logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}, failed=${failedFacts}`);
1848
+ }
1849
+
1850
+ // If ANY batch failed, throw — even if some facts were stored earlier.
1851
+ // A failed/timed-out UserOp may still linger in the bundler mempool as a
1852
+ // "nonce zombie." If we return normally, the caller's next storeExtractedFacts
1853
+ // call will fetch the same on-chain nonce and hit AA25 ("invalid account nonce").
1854
+ // Throwing forces all callers (import loops, chunk handlers) to stop submitting.
1855
+ if (batchError) {
1856
+ throw new Error(`Memory storage failed (${stored} stored before failure): ${batchError}`);
1857
+ }
1858
+ if (stored === 0 && failedFacts > 0) {
1859
+ throw new Error(`Memory storage failed: ${failedFacts} fact(s) failed to store`);
1281
1860
  }
1282
1861
 
1283
1862
  return stored;
@@ -1301,10 +1880,11 @@ async function handlePluginImportFrom(
1301
1880
  params: Record<string, unknown>,
1302
1881
  logger: OpenClawPluginApi['logger'],
1303
1882
  ): Promise<Record<string, unknown>> {
1883
+ _importInProgress = true;
1304
1884
  const startTime = Date.now();
1305
1885
 
1306
1886
  const source = params.source as string;
1307
- const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'memoclaw', 'generic-json', 'generic-csv'];
1887
+ const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'];
1308
1888
 
1309
1889
  if (!source || !validSources.includes(source)) {
1310
1890
  return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
@@ -1336,18 +1916,31 @@ async function handlePluginImportFrom(
1336
1916
  // Dry run: report what was parsed (chunks or facts)
1337
1917
  if (params.dry_run) {
1338
1918
  if (hasChunks) {
1919
+ const totalChunks = parseResult.chunks.length;
1920
+ const EXTRACTION_RATIO = 2.5; // avg facts per chunk, from empirical data
1921
+ const BATCH_SIZE = 25;
1922
+ const SECONDS_PER_BATCH = 45; // ~30s extraction + ~15s embed+store
1923
+ const estimatedFacts = Math.round(totalChunks * EXTRACTION_RATIO);
1924
+ const estimatedBatches = Math.ceil(totalChunks / BATCH_SIZE);
1925
+ const estimatedMinutes = Math.ceil(estimatedBatches * SECONDS_PER_BATCH / 60);
1926
+
1339
1927
  return {
1340
1928
  success: true,
1341
1929
  dry_run: true,
1342
1930
  source,
1343
- total_chunks: parseResult.chunks.length,
1931
+ total_chunks: totalChunks,
1344
1932
  total_messages: parseResult.totalMessages,
1933
+ estimated_facts: estimatedFacts,
1934
+ estimated_batches: estimatedBatches,
1935
+ estimated_minutes: estimatedMinutes,
1936
+ batch_size: BATCH_SIZE,
1937
+ use_background: totalChunks > 50,
1345
1938
  preview: parseResult.chunks.slice(0, 5).map((c) => ({
1346
1939
  title: c.title,
1347
1940
  messages: c.messages.length,
1348
1941
  first_message: c.messages[0]?.text.slice(0, 100),
1349
1942
  })),
1350
- note: 'Chunks will be processed through LLM extraction (same quality as auto-extraction).',
1943
+ note: `Estimated ${estimatedFacts} facts from ${totalChunks} chunks (~${estimatedMinutes} min).${totalChunks > 50 ? ' Recommended: background import via sessions_spawn.' : ''}`,
1351
1944
  warnings: parseResult.warnings,
1352
1945
  };
1353
1946
  }
@@ -1378,28 +1971,42 @@ async function handlePluginImportFrom(
1378
1971
  action: 'ADD' as const,
1379
1972
  }));
1380
1973
 
1381
- // Store in batches of 50
1974
+ // Store in batches of 50. Stop on any batch failure to prevent
1975
+ // nonce zombies from blocking subsequent UserOps (AA25).
1382
1976
  let totalStored = 0;
1977
+ let storeError: string | undefined;
1383
1978
  const batchSize = 50;
1384
1979
 
1385
1980
  for (let i = 0; i < extractedFacts.length; i += batchSize) {
1386
1981
  const batch = extractedFacts.slice(i, i + batchSize);
1387
- const stored = await storeExtractedFacts(batch, logger);
1388
- totalStored += stored;
1982
+ try {
1983
+ const stored = await storeExtractedFacts(batch, logger);
1984
+ totalStored += stored;
1389
1985
 
1390
- logger.info(
1391
- `Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`,
1392
- );
1986
+ logger.info(
1987
+ `Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`,
1988
+ );
1989
+ } catch (err: unknown) {
1990
+ storeError = err instanceof Error ? err.message : String(err);
1991
+ logger.warn(`Import stopped at batch ${Math.floor(i / batchSize) + 1}: ${storeError}`);
1992
+ break; // Stop processing further batches
1993
+ }
1994
+ }
1995
+
1996
+ const importWarnings = [...parseResult.warnings];
1997
+ if (storeError) {
1998
+ importWarnings.push(`Import stopped early: ${storeError}`);
1393
1999
  }
1394
2000
 
1395
2001
  return {
1396
- success: true,
2002
+ success: totalStored > 0,
1397
2003
  source,
1398
2004
  import_id: crypto.randomUUID(),
1399
2005
  total_found: parseResult.facts.length,
1400
2006
  imported: totalStored,
1401
2007
  skipped: parseResult.facts.length - totalStored,
1402
- warnings: parseResult.warnings,
2008
+ stopped_early: !!storeError,
2009
+ warnings: importWarnings,
1403
2010
  duration_ms: Date.now() - startTime,
1404
2011
  };
1405
2012
  } catch (e) {
@@ -1409,6 +2016,343 @@ async function handlePluginImportFrom(
1409
2016
  }
1410
2017
  }
1411
2018
 
2019
+ // ---------------------------------------------------------------------------
2020
+ // Smart Import — Two-Pass Pipeline (Profile + Triage)
2021
+ // ---------------------------------------------------------------------------
2022
+
2023
+ // Lazy-load WASM for smart import functions (same pattern as crypto.ts / subgraph-store.ts).
2024
+ let _smartImportWasm: typeof import('@totalreclaw/core') | null = null;
2025
+ function getSmartImportWasm() {
2026
+ if (!_smartImportWasm) _smartImportWasm = require('@totalreclaw/core');
2027
+ return _smartImportWasm;
2028
+ }
2029
+
2030
+ /**
2031
+ * Check whether the @totalreclaw/core WASM module exposes smart import functions.
2032
+ * Returns false if the module is an older version without smart import support.
2033
+ */
2034
+ function hasSmartImportSupport(): boolean {
2035
+ try {
2036
+ const wasm = getSmartImportWasm();
2037
+ return typeof wasm.chunksToSummaries === 'function' &&
2038
+ typeof wasm.buildProfileBatchPrompt === 'function' &&
2039
+ typeof wasm.parseProfileBatchResponse === 'function' &&
2040
+ typeof wasm.buildTriagePrompt === 'function' &&
2041
+ typeof wasm.parseTriageResponse === 'function' &&
2042
+ typeof wasm.enrichExtractionPrompt === 'function';
2043
+ } catch {
2044
+ return false;
2045
+ }
2046
+ }
2047
+
2048
+ /** Smart import result containing profile, triage decisions, and enriched system prompt. */
2049
+ interface SmartImportContext {
2050
+ /** JSON-serialized UserProfile (for WASM calls that require profile_json) */
2051
+ profileJson: string;
2052
+ /** Triage decisions indexed by chunk_index */
2053
+ decisions: Array<{ chunk_index: number; decision: string; reason: string }>;
2054
+ /** Enriched system prompt for extraction (profile context injected) */
2055
+ enrichedSystemPrompt: string;
2056
+ /** Number of chunks marked for extraction */
2057
+ extractCount: number;
2058
+ /** Number of chunks marked for skipping */
2059
+ skipCount: number;
2060
+ /** Duration of the profiling + triage pipeline in ms */
2061
+ durationMs: number;
2062
+ }
2063
+
2064
+ /**
2065
+ * Run the smart import two-pass pipeline: profile the user from conversation
2066
+ * summaries, then triage chunks as EXTRACT or SKIP.
2067
+ *
2068
+ * All prompt construction and response parsing happens in @totalreclaw/core WASM.
2069
+ * LLM calls use the plugin's existing chatCompletion() function.
2070
+ *
2071
+ * Returns null if smart import is unavailable (old WASM, no LLM config, etc.)
2072
+ * so the caller can fall back to blind extraction.
2073
+ */
2074
+ async function runSmartImportPipeline(
2075
+ chunks: import('./import-adapters/types.js').ConversationChunk[],
2076
+ logger: { info: (msg: string) => void; warn: (msg: string) => void },
2077
+ ): Promise<SmartImportContext | null> {
2078
+ // Guard: WASM must have smart import functions
2079
+ if (!hasSmartImportSupport()) {
2080
+ logger.info('Smart import: WASM module does not support smart import, falling back to blind extraction');
2081
+ return null;
2082
+ }
2083
+
2084
+ // Guard: LLM must be available
2085
+ const llmConfig = resolveLLMConfig();
2086
+ if (!llmConfig) {
2087
+ logger.info('Smart import: no LLM available, falling back to blind extraction');
2088
+ return null;
2089
+ }
2090
+
2091
+ const pipelineStart = Date.now();
2092
+ const wasm = getSmartImportWasm();
2093
+
2094
+ try {
2095
+ // Step 0: Convert chunks to compact summaries (first + last message)
2096
+ const wasmChunks = chunks.map((c, i) => ({
2097
+ index: i,
2098
+ title: c.title || 'Untitled',
2099
+ messages: c.messages.map((m) => ({ role: m.role, content: m.text })),
2100
+ timestamp: c.timestamp || null,
2101
+ }));
2102
+ const summaries = wasm.chunksToSummaries(JSON.stringify(wasmChunks));
2103
+ const summariesJson = JSON.stringify(summaries);
2104
+
2105
+ // Step 1: Build user profile (batch summarize -> merge)
2106
+ const PROFILE_BATCH_SIZE = 50;
2107
+ const profileStart = Date.now();
2108
+ const partials: unknown[] = [];
2109
+
2110
+ for (let i = 0; i < summaries.length; i += PROFILE_BATCH_SIZE) {
2111
+ const batch = summaries.slice(i, i + PROFILE_BATCH_SIZE);
2112
+ const prompt = wasm.buildProfileBatchPrompt(JSON.stringify(batch));
2113
+ const response = await chatCompletion(llmConfig, [
2114
+ { role: 'user', content: prompt },
2115
+ ], { maxTokens: 2048, temperature: 0 });
2116
+
2117
+ if (!response) {
2118
+ logger.warn(`Smart import: LLM returned empty response for profile batch ${Math.floor(i / PROFILE_BATCH_SIZE) + 1}`);
2119
+ continue;
2120
+ }
2121
+
2122
+ const partial = wasm.parseProfileBatchResponse(response);
2123
+ partials.push(partial);
2124
+ }
2125
+
2126
+ if (partials.length === 0) {
2127
+ logger.warn('Smart import: no profile batches produced, falling back to blind extraction');
2128
+ return null;
2129
+ }
2130
+
2131
+ let profile: unknown;
2132
+ if (partials.length === 1) {
2133
+ // Single batch — skip merge, promote partial to full profile
2134
+ // parseProfileBatchResponse returns a PartialProfile; convert to UserProfile shape
2135
+ const p = partials[0] as Record<string, unknown>;
2136
+ profile = {
2137
+ identity: p.identity ?? null,
2138
+ themes: p.themes ?? [],
2139
+ projects: p.projects ?? [],
2140
+ stack: p.stack ?? [],
2141
+ decisions: p.decisions ?? [],
2142
+ interests: p.interests ?? [],
2143
+ skip_patterns: p.skip_patterns ?? [],
2144
+ };
2145
+ } else {
2146
+ const mergePrompt = wasm.buildProfileMergePrompt(JSON.stringify(partials));
2147
+ const mergeResponse = await chatCompletion(llmConfig, [
2148
+ { role: 'user', content: mergePrompt },
2149
+ ], { maxTokens: 2048, temperature: 0 });
2150
+
2151
+ if (!mergeResponse) {
2152
+ logger.warn('Smart import: LLM returned empty response for profile merge, falling back to blind extraction');
2153
+ return null;
2154
+ }
2155
+
2156
+ profile = wasm.parseProfileResponse(mergeResponse);
2157
+ }
2158
+
2159
+ const profileJson = JSON.stringify(profile);
2160
+ const profileDuration = Date.now() - profileStart;
2161
+
2162
+ const p = profile as Record<string, unknown>;
2163
+ const themeCount = Array.isArray(p.themes) ? p.themes.length : 0;
2164
+ const skipPatternCount = Array.isArray(p.skip_patterns) ? p.skip_patterns.length : 0;
2165
+ logger.info(
2166
+ `Smart import: profile built in ${profileDuration}ms (themes=${themeCount}, skip_patterns=${skipPatternCount})`,
2167
+ );
2168
+
2169
+ // Step 1.5: Chunk triage (EXTRACT or SKIP)
2170
+ const triageStart = Date.now();
2171
+ const allDecisions: Array<{ chunk_index: number; decision: string; reason: string }> = [];
2172
+ const TRIAGE_BATCH_SIZE = 50;
2173
+
2174
+ for (let i = 0; i < summaries.length; i += TRIAGE_BATCH_SIZE) {
2175
+ const batch = summaries.slice(i, i + TRIAGE_BATCH_SIZE);
2176
+ const triagePrompt = wasm.buildTriagePrompt(profileJson, JSON.stringify(batch));
2177
+ const triageResponse = await chatCompletion(llmConfig, [
2178
+ { role: 'user', content: triagePrompt },
2179
+ ], { maxTokens: 4096, temperature: 0 });
2180
+
2181
+ if (!triageResponse) {
2182
+ logger.warn(`Smart import: LLM returned empty response for triage batch ${Math.floor(i / TRIAGE_BATCH_SIZE) + 1}, defaulting to EXTRACT`);
2183
+ // Default all chunks in this batch to EXTRACT
2184
+ for (let j = i; j < Math.min(i + TRIAGE_BATCH_SIZE, summaries.length); j++) {
2185
+ allDecisions.push({ chunk_index: j, decision: 'EXTRACT', reason: 'triage LLM unavailable' });
2186
+ }
2187
+ continue;
2188
+ }
2189
+
2190
+ const batchDecisions = wasm.parseTriageResponse(triageResponse) as Array<{
2191
+ chunk_index: number;
2192
+ decision: string;
2193
+ reason: string;
2194
+ }>;
2195
+ allDecisions.push(...batchDecisions);
2196
+ }
2197
+
2198
+ const triageDuration = Date.now() - triageStart;
2199
+
2200
+ const extractCount = allDecisions.filter((d) => d.decision !== 'SKIP').length;
2201
+ const skipCount = allDecisions.filter((d) => d.decision === 'SKIP').length;
2202
+ logger.info(
2203
+ `Smart import: triage complete in ${triageDuration}ms (extract=${extractCount}, skip=${skipCount}, total=${chunks.length})`,
2204
+ );
2205
+
2206
+ // Step 2: Build enriched system prompt for extraction
2207
+ const enrichedSystemPrompt = wasm.enrichExtractionPrompt(profileJson, EXTRACTION_SYSTEM_PROMPT);
2208
+
2209
+ const totalDuration = Date.now() - pipelineStart;
2210
+ logger.info(`Smart import: pipeline complete in ${totalDuration}ms`);
2211
+
2212
+ return {
2213
+ profileJson,
2214
+ decisions: allDecisions,
2215
+ enrichedSystemPrompt,
2216
+ extractCount,
2217
+ skipCount,
2218
+ durationMs: totalDuration,
2219
+ };
2220
+ } catch (err) {
2221
+ const msg = err instanceof Error ? err.message : String(err);
2222
+ logger.warn(`Smart import: pipeline failed (${msg}), falling back to blind extraction`);
2223
+ return null;
2224
+ }
2225
+ }
2226
+
2227
+ /**
2228
+ * Check if a chunk should be skipped based on triage decisions.
2229
+ * If no decision exists for the chunk index, defaults to EXTRACT (safe default).
2230
+ */
2231
+ function isChunkSkipped(
2232
+ chunkIndex: number,
2233
+ decisions: Array<{ chunk_index: number; decision: string }>,
2234
+ ): { skipped: boolean; reason: string } {
2235
+ const decision = decisions.find((d) => d.chunk_index === chunkIndex);
2236
+ if (decision && decision.decision === 'SKIP') {
2237
+ return { skipped: true, reason: (decision as { reason?: string }).reason || 'triage: skip' };
2238
+ }
2239
+ return { skipped: false, reason: '' };
2240
+ }
2241
+
2242
+ /**
2243
+ * Process a batch (slice) of conversation chunks from a file.
2244
+ * Called repeatedly by the agent for large imports.
2245
+ */
2246
+ async function handleBatchImport(
2247
+ params: Record<string, unknown>,
2248
+ logger: OpenClawPluginApi['logger'],
2249
+ ): Promise<Record<string, unknown>> {
2250
+ _importInProgress = true;
2251
+ const source = params.source as string;
2252
+ const filePath = params.file_path as string | undefined;
2253
+ const content = params.content as string | undefined;
2254
+ const offset = (params.offset as number) ?? 0;
2255
+ const batchSize = (params.batch_size as number) ?? 25;
2256
+
2257
+ const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'];
2258
+ if (!source || !validSources.includes(source)) {
2259
+ return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
2260
+ }
2261
+
2262
+ const startTime = Date.now();
2263
+
2264
+ const { getAdapter } = await import('./import-adapters/index.js');
2265
+ const adapter = getAdapter(source as import('./import-adapters/types.js').ImportSource);
2266
+
2267
+ const parseResult = await adapter.parse({ content, file_path: filePath });
2268
+
2269
+ if (parseResult.errors.length > 0 && parseResult.chunks.length === 0) {
2270
+ return { success: false, error: parseResult.errors.join('; ') };
2271
+ }
2272
+
2273
+ const totalChunks = parseResult.chunks.length;
2274
+ const slice = parseResult.chunks.slice(offset, offset + batchSize);
2275
+ const remaining = Math.max(0, totalChunks - offset - slice.length);
2276
+
2277
+ // --- Smart Import: Profile + Triage ---
2278
+ // Build profile from ALL chunks (not just the slice) for full context,
2279
+ // then triage only the current slice. For simplicity, we rebuild on every
2280
+ // batch call — optimization (caching) can come later.
2281
+ const smartCtx = await runSmartImportPipeline(parseResult.chunks, logger);
2282
+ let chunksSkipped = 0;
2283
+
2284
+ // Process the slice through the normal extraction + storage pipeline.
2285
+ // If a batch fails (nonce zombie, quota exceeded, etc.), stop immediately
2286
+ // to prevent subsequent UserOps from hitting AA25 nonce conflicts.
2287
+ let factsExtracted = 0;
2288
+ let factsStored = 0;
2289
+ let chunksProcessed = 0;
2290
+ let storeError: string | undefined;
2291
+
2292
+ for (let i = 0; i < slice.length; i++) {
2293
+ const chunk = slice[i];
2294
+ const globalIndex = offset + i; // Index in the full chunks array
2295
+
2296
+ // Smart import: skip chunks triaged as SKIP
2297
+ if (smartCtx) {
2298
+ const { skipped, reason } = isChunkSkipped(globalIndex, smartCtx.decisions);
2299
+ if (skipped) {
2300
+ logger.info(`Import: skipping chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}" (${reason})`);
2301
+ chunksSkipped++;
2302
+ chunksProcessed++;
2303
+ continue;
2304
+ }
2305
+ }
2306
+
2307
+ logger.info(`Import: extracting facts from chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}"`);
2308
+
2309
+ const messages = chunk.messages.map((m) => ({ role: m.role, content: m.text }));
2310
+ const facts = await extractFacts(
2311
+ messages,
2312
+ 'full',
2313
+ undefined, // no existing memories for dedup during import
2314
+ smartCtx?.enrichedSystemPrompt, // profile-enriched extraction prompt
2315
+ );
2316
+ chunksProcessed++;
2317
+
2318
+ if (facts.length > 0) {
2319
+ factsExtracted += facts.length;
2320
+ try {
2321
+ const stored = await storeExtractedFacts(facts, logger);
2322
+ factsStored += stored;
2323
+ } catch (err: unknown) {
2324
+ storeError = err instanceof Error ? err.message : String(err);
2325
+ logger.warn(`Import batch stopped at chunk ${globalIndex + 1}/${totalChunks}: ${storeError}`);
2326
+ break; // Stop processing further chunks — a zombie UserOp may block writes
2327
+ }
2328
+ }
2329
+ }
2330
+
2331
+ return {
2332
+ success: factsStored > 0 || (!storeError && factsExtracted === 0),
2333
+ batch_offset: offset,
2334
+ batch_size: chunksProcessed,
2335
+ total_chunks: totalChunks,
2336
+ facts_extracted: factsExtracted,
2337
+ facts_stored: factsStored,
2338
+ chunks_skipped: chunksSkipped,
2339
+ remaining_chunks: remaining,
2340
+ is_complete: remaining === 0 && !storeError,
2341
+ stopped_early: !!storeError,
2342
+ error: storeError,
2343
+ smart_import: smartCtx ? {
2344
+ profile_duration_ms: smartCtx.durationMs,
2345
+ extract_count: smartCtx.extractCount,
2346
+ skip_count: smartCtx.skipCount,
2347
+ } : null,
2348
+ // Estimation for the full import
2349
+ estimated_total_facts: Math.round(totalChunks * 2.5),
2350
+ estimated_total_userops: Math.ceil(totalChunks * 2.5 / 15),
2351
+ estimated_minutes: Math.ceil(Math.ceil(totalChunks / batchSize) * 45 / 60),
2352
+ duration_ms: Date.now() - startTime,
2353
+ };
2354
+ }
2355
+
1412
2356
  /**
1413
2357
  * Process conversation chunks through LLM extraction and store results.
1414
2358
  *
@@ -1427,9 +2371,29 @@ async function handleChunkImport(
1427
2371
  let totalExtracted = 0;
1428
2372
  let totalStored = 0;
1429
2373
  let chunksProcessed = 0;
2374
+ let chunksSkipped = 0;
2375
+
2376
+ let storeError: string | undefined;
2377
+
2378
+ // --- Smart Import: Profile + Triage ---
2379
+ const smartCtx = await runSmartImportPipeline(chunks, logger);
1430
2380
 
1431
- for (const chunk of chunks) {
2381
+ for (let i = 0; i < chunks.length; i++) {
2382
+ const chunk = chunks[i];
1432
2383
  chunksProcessed++;
2384
+
2385
+ // Smart import: skip chunks triaged as SKIP
2386
+ if (smartCtx) {
2387
+ const { skipped, reason } = isChunkSkipped(i, smartCtx.decisions);
2388
+ if (skipped) {
2389
+ logger.info(
2390
+ `Import: skipping chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}" (${reason})`,
2391
+ );
2392
+ chunksSkipped++;
2393
+ continue;
2394
+ }
2395
+ }
2396
+
1433
2397
  logger.info(
1434
2398
  `Import: extracting facts from chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}"`,
1435
2399
  );
@@ -1443,22 +2407,35 @@ async function handleChunkImport(
1443
2407
 
1444
2408
  // Use 'full' mode to extract ALL valuable memories from the chunk
1445
2409
  // (not just the last few messages like 'turn' mode does).
1446
- const facts = await extractFacts(messages, 'full');
2410
+ // Smart import: pass enriched system prompt with user profile context.
2411
+ const facts = await extractFacts(
2412
+ messages,
2413
+ 'full',
2414
+ undefined, // no existing memories for dedup during import
2415
+ smartCtx?.enrichedSystemPrompt, // profile-enriched extraction prompt
2416
+ );
1447
2417
 
1448
2418
  if (facts.length > 0) {
1449
2419
  totalExtracted += facts.length;
1450
2420
 
1451
- // Store through the normal pipeline (dedup, encrypt, store)
1452
- const stored = await storeExtractedFacts(facts, logger);
1453
- totalStored += stored;
2421
+ try {
2422
+ // Store through the normal pipeline (dedup, encrypt, store).
2423
+ // storeExtractedFacts throws on batch failure to prevent nonce zombies.
2424
+ const stored = await storeExtractedFacts(facts, logger);
2425
+ totalStored += stored;
1454
2426
 
1455
- logger.info(
1456
- `Import chunk ${chunksProcessed}/${chunks.length}: extracted ${facts.length} facts, stored ${stored}`,
1457
- );
2427
+ logger.info(
2428
+ `Import chunk ${chunksProcessed}/${chunks.length}: extracted ${facts.length} facts, stored ${stored}`,
2429
+ );
2430
+ } catch (err: unknown) {
2431
+ storeError = err instanceof Error ? err.message : String(err);
2432
+ logger.warn(`Import stopped at chunk ${chunksProcessed}/${chunks.length}: ${storeError}`);
2433
+ break; // Stop processing further chunks — a zombie UserOp may block writes
2434
+ }
1458
2435
  }
1459
2436
  }
1460
2437
 
1461
- if (totalExtracted === 0 && chunks.length > 0) {
2438
+ if (totalExtracted === 0 && chunks.length > 0 && !storeError && chunksSkipped < chunks.length) {
1462
2439
  warnings.push(
1463
2440
  `Processed ${chunks.length} conversation chunks (${totalMessages} messages) but the LLM ` +
1464
2441
  `did not extract any facts worth storing. This can happen if the conversations are mostly ` +
@@ -1466,15 +2443,27 @@ async function handleChunkImport(
1466
2443
  );
1467
2444
  }
1468
2445
 
2446
+ if (storeError) {
2447
+ warnings.push(`Import stopped early: ${storeError}. ${chunks.length - chunksProcessed} chunk(s) not processed.`);
2448
+ }
2449
+
1469
2450
  return {
1470
2451
  success: totalStored > 0 || totalExtracted > 0,
1471
2452
  source,
1472
2453
  import_id: crypto.randomUUID(),
1473
2454
  total_chunks: chunks.length,
2455
+ chunks_processed: chunksProcessed,
2456
+ chunks_skipped: chunksSkipped,
1474
2457
  total_messages: totalMessages,
1475
2458
  facts_extracted: totalExtracted,
1476
2459
  imported: totalStored,
1477
2460
  skipped: totalExtracted - totalStored,
2461
+ stopped_early: !!storeError,
2462
+ smart_import: smartCtx ? {
2463
+ profile_duration_ms: smartCtx.durationMs,
2464
+ extract_count: smartCtx.extractCount,
2465
+ skip_count: smartCtx.skipCount,
2466
+ } : null,
1478
2467
  warnings,
1479
2468
  duration_ms: Date.now() - startTime,
1480
2469
  };
@@ -1512,6 +2501,7 @@ const plugin = {
1512
2501
  initLLMClient({
1513
2502
  primaryModel: api.config?.agents?.defaults?.model?.primary as string | undefined,
1514
2503
  pluginConfig: api.pluginConfig,
2504
+ openclawProviders: api.config?.models?.providers,
1515
2505
  logger: api.logger,
1516
2506
  });
1517
2507
 
@@ -1548,160 +2538,164 @@ const plugin = {
1548
2538
  },
1549
2539
  type: {
1550
2540
  type: 'string',
1551
- enum: ['fact', 'preference', 'decision', 'episodic', 'goal', 'context', 'summary'],
1552
- description: 'The kind of memory (default: fact)',
2541
+ enum: [...VALID_MEMORY_TYPES, ...LEGACY_V0_MEMORY_TYPES],
2542
+ description:
2543
+ 'Memory Taxonomy v1 type: claim, preference, directive, commitment, episode, summary. ' +
2544
+ 'Use "claim" for factual assertions and decisions (populate `reasoning` with the why clause). ' +
2545
+ 'Use "directive" for imperative rules ("always X", "never Y"), "commitment" for future intent, ' +
2546
+ 'and "episode" for notable events. Legacy v0 tokens (fact, decision, episodic, goal, context, ' +
2547
+ 'rule) are silently coerced to their v1 equivalents. Default: claim.',
2548
+ },
2549
+ source: {
2550
+ type: 'string',
2551
+ enum: [...VALID_MEMORY_SOURCES],
2552
+ description:
2553
+ 'v1 provenance tag. "user" = user explicitly stated it, "user-inferred" = inferred from user ' +
2554
+ 'signals, "assistant" = assistant-authored (downgrade unless user affirmed), "external" / ' +
2555
+ '"derived" = rare. Explicit remembers default to "user".',
2556
+ },
2557
+ scope: {
2558
+ type: 'string',
2559
+ enum: [...VALID_MEMORY_SCOPES],
2560
+ description:
2561
+ 'v1 life-domain scope: work, personal, health, family, creative, finance, misc, unspecified. ' +
2562
+ 'Default: unspecified.',
2563
+ },
2564
+ reasoning: {
2565
+ type: 'string',
2566
+ description:
2567
+ 'For type=claim expressing a decision, the WHY clause ("because Y"). Max 256 chars. ' +
2568
+ 'Omit for non-decision claims.',
2569
+ maxLength: 256,
1553
2570
  },
1554
2571
  importance: {
1555
2572
  type: 'number',
1556
2573
  minimum: 1,
1557
2574
  maximum: 10,
1558
- description: 'Importance score 1-10 (default: 5)',
2575
+ description: 'Importance score 1-10 (default: 8 for explicit remember)',
2576
+ },
2577
+ entities: {
2578
+ type: 'array',
2579
+ description:
2580
+ 'Named entities this memory is about (people, projects, tools, companies, concepts, places). ' +
2581
+ 'Supplying entities enables Phase 2 contradiction detection against existing facts about the same entity. ' +
2582
+ 'Omit if unclear — a best-effort fallback will still store the memory.',
2583
+ items: {
2584
+ type: 'object',
2585
+ properties: {
2586
+ name: { type: 'string' },
2587
+ type: {
2588
+ type: 'string',
2589
+ enum: ['person', 'project', 'tool', 'company', 'concept', 'place'],
2590
+ },
2591
+ role: { type: 'string' },
2592
+ },
2593
+ required: ['name', 'type'],
2594
+ additionalProperties: false,
2595
+ },
1559
2596
  },
1560
2597
  },
1561
2598
  required: ['text'],
1562
2599
  additionalProperties: false,
1563
2600
  },
1564
- async execute(_toolCallId: string, params: { text: string; type?: string; importance?: number }) {
2601
+ async execute(
2602
+ _toolCallId: string,
2603
+ params: {
2604
+ text: string;
2605
+ type?: string;
2606
+ source?: string;
2607
+ scope?: string;
2608
+ reasoning?: string;
2609
+ importance?: number;
2610
+ entities?: Array<{ name: string; type: string; role?: string }>;
2611
+ },
2612
+ ) {
1565
2613
  try {
1566
2614
  await requireFullSetup(api.logger);
1567
2615
 
1568
- const memoryType = params.type ?? 'fact';
1569
- let importance = params.importance ?? 5;
1570
-
1571
- // Generate blind indices for server-side search.
1572
- const blindIndices = generateBlindIndices(params.text);
1573
-
1574
- // Generate embedding + LSH bucket hashes (PoC v2).
1575
- // Falls back to word-only indices if embedding generation fails.
1576
- const embeddingResult = await generateEmbeddingAndLSH(params.text, api.logger);
1577
-
1578
- // Merge LSH bucket hashes into blind indices.
1579
- const allIndices = embeddingResult
1580
- ? [...blindIndices, ...embeddingResult.lshBuckets]
1581
- : blindIndices;
1582
-
1583
- // Store-time dedup: for explicit remember, ALWAYS supersede
1584
- // (user explicitly wants this stored just remove the old one).
1585
- let supersededId: string | undefined;
1586
- if (STORE_DEDUP_ENABLED && embeddingResult) {
1587
- const dupResult = await searchForNearDuplicates(
1588
- params.text,
1589
- embeddingResult.embedding,
1590
- allIndices,
1591
- api.logger,
1592
- );
1593
- if (dupResult) {
1594
- // Inherit higher importance from existing fact.
1595
- importance = Math.max(importance, dupResult.match.decayScore);
1596
- supersededId = dupResult.match.id;
1597
-
1598
- if (isSubgraphMode()) {
1599
- try {
1600
- const tombConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1601
- const tombstone: FactPayload = {
1602
- id: dupResult.match.id,
1603
- timestamp: new Date().toISOString(),
1604
- owner: subgraphOwner || userId!,
1605
- encryptedBlob: '00',
1606
- blindIndices: [],
1607
- decayScore: 0,
1608
- source: 'tombstone',
1609
- contentFp: '',
1610
- agentId: 'openclaw-plugin',
1611
- };
1612
- const tombProtobuf = encodeFactProtobuf(tombstone);
1613
- await submitFactOnChain(tombProtobuf, tombConfig);
1614
- api.logger.info(
1615
- `Remember dedup: superseded ${dupResult.match.id} on-chain (sim=${dupResult.similarity.toFixed(3)})`,
1616
- );
1617
- } catch (tombErr) {
1618
- api.logger.warn(
1619
- `Remember dedup: failed to tombstone ${dupResult.match.id}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`,
1620
- );
1621
- supersededId = undefined;
1622
- }
1623
- } else if (apiClient && authKeyHex) {
1624
- try {
1625
- await apiClient.deleteFact(dupResult.match.id, authKeyHex);
1626
- api.logger.info(
1627
- `Remember dedup: superseded ${dupResult.match.id} (sim=${dupResult.similarity.toFixed(3)})`,
1628
- );
1629
- } catch (delErr) {
1630
- api.logger.warn(
1631
- `Remember dedup: failed to delete superseded fact ${dupResult.match.id}: ${delErr instanceof Error ? delErr.message : String(delErr)}`,
1632
- );
1633
- supersededId = undefined; // Don't report supersession if delete failed
1634
- }
1635
- }
1636
- }
1637
- }
1638
-
1639
- // Build the document JSON that will be encrypted.
1640
- const doc = {
1641
- text: params.text,
1642
- metadata: {
1643
- type: memoryType,
1644
- importance: importance / 10, // normalise to 0-1 range
1645
- source: 'explicit',
1646
- created_at: new Date().toISOString(),
1647
- },
1648
- };
1649
-
1650
- // Encrypt the document.
1651
- const encryptedBlob = encryptToHex(JSON.stringify(doc), encryptionKey!);
1652
-
1653
- // Generate content fingerprint for dedup.
1654
- const contentFp = generateContentFingerprint(params.text, dedupKey!);
1655
-
1656
- // Generate a unique fact ID.
1657
- const factId = crypto.randomUUID();
2616
+ // v1 taxonomy: route explicit remembers through the same canonical
2617
+ // store path that auto-extraction uses (`storeExtractedFacts`). This
2618
+ // emits a Memory Taxonomy v1 JSON blob, generates entity trapdoors,
2619
+ // and runs through the Phase 2 contradiction-resolution pipeline.
2620
+ //
2621
+ // Accept legacy v0 tokens on input and coerce to v1 via
2622
+ // `normalizeToV1Type` so agents that still emit the pre-v3
2623
+ // taxonomy keep working.
2624
+ const rawType = typeof params.type === 'string' ? params.type.toLowerCase() : 'claim';
2625
+ const memoryType: MemoryType = isValidMemoryType(rawType)
2626
+ ? rawType
2627
+ : normalizeToV1Type(rawType);
2628
+
2629
+ // Source defaults to 'user' for explicit remembers (the user is
2630
+ // the author by definition). Ignored if the caller passes an
2631
+ // invalid value.
2632
+ const rawSource = typeof params.source === 'string' ? params.source.toLowerCase() : 'user';
2633
+ const memorySource: MemorySource =
2634
+ (VALID_MEMORY_SOURCES as readonly string[]).includes(rawSource)
2635
+ ? (rawSource as MemorySource)
2636
+ : 'user';
2637
+
2638
+ const rawScope = typeof params.scope === 'string' ? params.scope.toLowerCase() : 'unspecified';
2639
+ const memoryScope: MemoryScope =
2640
+ (VALID_MEMORY_SCOPES as readonly string[]).includes(rawScope)
2641
+ ? (rawScope as MemoryScope)
2642
+ : 'unspecified';
2643
+
2644
+ const reasoning =
2645
+ typeof params.reasoning === 'string' && params.reasoning.length > 0
2646
+ ? params.reasoning.slice(0, 256)
2647
+ : undefined;
2648
+
2649
+ // Explicit remember defaults to importance 8 (above auto-extraction's
2650
+ // typical 6-7), so store-time dedup's shouldSupersede prefers the
2651
+ // explicit call when it collides with an auto-extracted claim.
2652
+ const importance = Math.max(1, Math.min(10, params.importance ?? 8));
2653
+
2654
+ const validatedEntities: ExtractedEntity[] = Array.isArray(params.entities)
2655
+ ? params.entities
2656
+ .map((e) => parseEntity(e))
2657
+ .filter((e): e is ExtractedEntity => e !== null)
2658
+ : [];
1658
2659
 
1659
- // Build the payload matching the server's FactJSON schema.
1660
- const factPayload: StoreFactPayload = {
1661
- id: factId,
1662
- timestamp: new Date().toISOString(),
1663
- encrypted_blob: encryptedBlob,
1664
- blind_indices: allIndices,
1665
- decay_score: importance,
1666
- source: 'explicit',
1667
- content_fp: contentFp,
1668
- agent_id: 'openclaw-plugin',
1669
- encrypted_embedding: embeddingResult?.encryptedEmbedding,
2660
+ const fact: ExtractedFact = {
2661
+ text: params.text.slice(0, 512),
2662
+ type: memoryType,
2663
+ source: memorySource,
2664
+ scope: memoryScope,
2665
+ reasoning,
2666
+ importance,
2667
+ action: 'ADD',
2668
+ confidence: 1.0, // user explicitly asked to remember — highest confidence
1670
2669
  };
2670
+ if (validatedEntities.length > 0) fact.entities = validatedEntities;
1671
2671
 
1672
- if (isSubgraphMode()) {
1673
- // Subgraph mode: encode as Protobuf and submit on-chain via relay UserOp
1674
- const config = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1675
- const protobuf = encodeFactProtobuf({
1676
- id: factId,
1677
- timestamp: new Date().toISOString(),
1678
- owner: subgraphOwner || userId!,
1679
- encryptedBlob: encryptedBlob,
1680
- blindIndices: allIndices,
1681
- decayScore: importance,
1682
- source: 'explicit',
1683
- contentFp: contentFp,
1684
- agentId: 'openclaw-plugin',
1685
- encryptedEmbedding: embeddingResult?.encryptedEmbedding,
1686
- });
1687
- await submitFactOnChain(protobuf, config);
1688
- } else {
1689
- await apiClient!.store(userId!, [factPayload], authKeyHex!);
1690
- }
2672
+ const stored = await storeExtractedFacts([fact], api.logger, 'explicit');
2673
+ api.logger.info(
2674
+ `totalreclaw_remember: routed to storeExtractedFacts (stored=${stored}, entities=${validatedEntities.length})`,
2675
+ );
1691
2676
 
1692
- const statusMsg = supersededId
1693
- ? `Memory stored (ID: ${factId}). Superseded an older similar memory.`
1694
- : `Memory stored (ID: ${factId})`;
2677
+ if (stored === 0) {
2678
+ // Dedup or supersession consumed the write. Treat as success from
2679
+ // the user's perspective — the memory's content is already in the
2680
+ // vault (possibly under a different ID).
2681
+ return {
2682
+ content: [
2683
+ {
2684
+ type: 'text',
2685
+ text: 'Memory noted (matched existing content in vault).',
2686
+ },
2687
+ ],
2688
+ };
2689
+ }
1695
2690
 
1696
2691
  return {
1697
- content: [{ type: 'text', text: statusMsg }],
1698
- details: { factId, supersededId },
2692
+ content: [{ type: 'text', text: 'Memory encrypted and stored.' }],
1699
2693
  };
1700
2694
  } catch (err: unknown) {
1701
2695
  const message = err instanceof Error ? err.message : String(err);
1702
2696
  api.logger.error(`totalreclaw_remember failed: ${message}`);
1703
2697
  return {
1704
- content: [{ type: 'text', text: `Failed to store memory: ${message}` }],
2698
+ content: [{ type: 'text', text: `Failed to store memory: ${humanizeError(message)}` }],
1705
2699
  };
1706
2700
  }
1707
2701
  },
@@ -1778,12 +2772,27 @@ const plugin = {
1778
2772
  // --- Subgraph search path ---
1779
2773
  const factCount = await getSubgraphFactCount(subgraphOwner || userId!, authKeyHex!);
1780
2774
  const pool = computeCandidatePool(factCount);
1781
- const subgraphResults = await searchSubgraph(subgraphOwner || userId!, allTrapdoors, pool, authKeyHex!);
2775
+ let subgraphResults = await searchSubgraph(subgraphOwner || userId!, allTrapdoors, pool, authKeyHex!);
2776
+
2777
+ // Always run broadened search and merge — ensures vocabulary mismatches
2778
+ // (e.g., "preferences" vs "prefer") don't cause recall failures.
2779
+ // The reranker handles scoring; extra cost is ~1 GraphQL query per recall.
2780
+ try {
2781
+ const broadenedResults = await searchSubgraphBroadened(subgraphOwner || userId!, pool, authKeyHex!);
2782
+ // Merge broadened results with existing (deduplicate by ID)
2783
+ const existingIds = new Set(subgraphResults.map(r => r.id));
2784
+ for (const br of broadenedResults) {
2785
+ if (!existingIds.has(br.id)) {
2786
+ subgraphResults.push(br);
2787
+ }
2788
+ }
2789
+ } catch { /* best-effort */ }
1782
2790
 
1783
2791
  for (const result of subgraphResults) {
1784
2792
  try {
1785
2793
  const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
1786
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
2794
+ if (isDigestBlob(docJson)) continue;
2795
+ const doc = readClaimFromBlob(docJson);
1787
2796
 
1788
2797
  let decryptedEmbedding: number[] | undefined;
1789
2798
  if (result.encryptedEmbedding) {
@@ -1796,17 +2805,29 @@ const plugin = {
1796
2805
  }
1797
2806
  }
1798
2807
 
2808
+ if (decryptedEmbedding && decryptedEmbedding.length !== getEmbeddingDims()) {
2809
+ try {
2810
+ decryptedEmbedding = await generateEmbedding(doc.text);
2811
+ } catch {
2812
+ decryptedEmbedding = undefined;
2813
+ }
2814
+ }
2815
+
1799
2816
  rerankerCandidates.push({
1800
2817
  id: result.id,
1801
2818
  text: doc.text,
1802
2819
  embedding: decryptedEmbedding,
1803
- importance: (doc.metadata?.importance as number) ?? 0.5,
2820
+ importance: doc.importance / 10,
1804
2821
  createdAt: result.timestamp ? parseInt(result.timestamp, 10) : undefined,
2822
+ // Retrieval v2 Tier 1: surface v1 source so applySourceWeights
2823
+ // can multiply the final RRF score by the source weight.
2824
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
1805
2825
  });
1806
2826
 
1807
2827
  metaMap.set(result.id, {
1808
2828
  metadata: doc.metadata ?? {},
1809
- timestamp: Date.now(), // Subgraph doesn't return ms timestamp; use current
2829
+ timestamp: Date.now(),
2830
+ category: doc.category,
1810
2831
  });
1811
2832
  } catch {
1812
2833
  // Skip candidates we cannot decrypt.
@@ -1849,7 +2870,8 @@ const plugin = {
1849
2870
  for (const candidate of candidates) {
1850
2871
  try {
1851
2872
  const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
1852
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
2873
+ if (isDigestBlob(docJson)) continue;
2874
+ const doc = readClaimFromBlob(docJson);
1853
2875
 
1854
2876
  let decryptedEmbedding: number[] | undefined;
1855
2877
  if (candidate.encrypted_embedding) {
@@ -1862,19 +2884,29 @@ const plugin = {
1862
2884
  }
1863
2885
  }
1864
2886
 
2887
+ if (decryptedEmbedding && decryptedEmbedding.length !== getEmbeddingDims()) {
2888
+ try {
2889
+ decryptedEmbedding = await generateEmbedding(doc.text);
2890
+ } catch {
2891
+ decryptedEmbedding = undefined;
2892
+ }
2893
+ }
2894
+
1865
2895
  rerankerCandidates.push({
1866
2896
  id: candidate.fact_id,
1867
2897
  text: doc.text,
1868
2898
  embedding: decryptedEmbedding,
1869
- importance: (doc.metadata?.importance as number) ?? 0.5,
2899
+ importance: doc.importance / 10,
1870
2900
  createdAt: typeof candidate.timestamp === 'number'
1871
2901
  ? candidate.timestamp / 1000
1872
2902
  : new Date(candidate.timestamp).getTime() / 1000,
2903
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
1873
2904
  });
1874
2905
 
1875
2906
  metaMap.set(candidate.fact_id, {
1876
2907
  metadata: doc.metadata ?? {},
1877
2908
  timestamp: candidate.timestamp,
2909
+ category: doc.category,
1878
2910
  });
1879
2911
  } catch {
1880
2912
  // Skip candidates we cannot decrypt (e.g. corrupted data).
@@ -1890,6 +2922,7 @@ const plugin = {
1890
2922
  rerankerCandidates,
1891
2923
  k,
1892
2924
  INTENT_WEIGHTS[queryIntent],
2925
+ /* applySourceWeights (Retrieval v2 Tier 1) */ true,
1893
2926
  );
1894
2927
 
1895
2928
  if (reranked.length === 0) {
@@ -1921,7 +2954,8 @@ const plugin = {
1921
2954
  ? ` (importance: ${Math.round((meta.metadata.importance as number) * 10)}/10)`
1922
2955
  : '';
1923
2956
  const age = meta ? relativeTime(meta.timestamp) : '';
1924
- return `${i + 1}. ${m.text}${imp} -- ${age} [ID: ${m.id}]`;
2957
+ const typeTag = meta?.category ? `[${meta.category}] ` : '';
2958
+ return `${i + 1}. ${typeTag}${m.text}${imp} -- ${age} [ID: ${m.id}]`;
1925
2959
  });
1926
2960
 
1927
2961
  const formatted = lines.join('\n');
@@ -1940,7 +2974,7 @@ const plugin = {
1940
2974
  const message = err instanceof Error ? err.message : String(err);
1941
2975
  api.logger.error(`totalreclaw_recall failed: ${message}`);
1942
2976
  return {
1943
- content: [{ type: 'text', text: `Failed to search memories: ${message}` }],
2977
+ content: [{ type: 'text', text: `Failed to search memories: ${humanizeError(message)}` }],
1944
2978
  };
1945
2979
  }
1946
2980
  },
@@ -1986,9 +3020,13 @@ const plugin = {
1986
3020
  source: 'tombstone',
1987
3021
  contentFp: '',
1988
3022
  agentId: 'openclaw-plugin',
3023
+ version: PROTOBUF_VERSION_V4,
1989
3024
  };
1990
3025
  const protobuf = encodeFactProtobuf(tombstone);
1991
3026
  const result = await submitFactOnChain(protobuf, config);
3027
+ if (!result.success) {
3028
+ throw new Error(`On-chain tombstone failed (tx=${result.txHash?.slice(0, 10) || 'none'}…)`);
3029
+ }
1992
3030
  api.logger.info(`Tombstone written for ${params.factId}: tx=${result.txHash}`);
1993
3031
  return {
1994
3032
  content: [{ type: 'text', text: `Memory ${params.factId} deleted (on-chain tombstone, tx: ${result.txHash})` }],
@@ -2005,7 +3043,7 @@ const plugin = {
2005
3043
  const message = err instanceof Error ? err.message : String(err);
2006
3044
  api.logger.error(`totalreclaw_forget failed: ${message}`);
2007
3045
  return {
2008
- content: [{ type: 'text', text: `Failed to delete memory: ${message}` }],
3046
+ content: [{ type: 'text', text: `Failed to delete memory: ${humanizeError(message)}` }],
2009
3047
  };
2010
3048
  }
2011
3049
  },
@@ -2049,16 +3087,22 @@ const plugin = {
2049
3087
  }> = [];
2050
3088
 
2051
3089
  if (isSubgraphMode()) {
2052
- // Query subgraph for all active facts
3090
+ // Query subgraph for all active facts (cursor-based pagination via id_gt)
2053
3091
  const config = getSubgraphConfig();
2054
3092
  const relayUrl = config.relayUrl;
2055
3093
  const PAGE_SIZE = 1000;
2056
- let skip = 0;
2057
- let hasMore = true;
3094
+ let lastId = '';
2058
3095
  const owner = subgraphOwner || userId || '';
3096
+ console.error(`[TotalReclaw Export] owner=${owner} subgraphOwner=${subgraphOwner} userId=${userId} relayUrl=${relayUrl} authKey=${authKeyHex ? authKeyHex.slice(0, 8) + '...' : 'MISSING'} isSubgraph=${isSubgraphMode()}`);
2059
3097
 
2060
- while (hasMore) {
2061
- const query = `{ facts(where: { owner: "${owner}", isActive: true }, first: ${PAGE_SIZE}, skip: ${skip}, orderBy: sequenceId, orderDirection: asc) { id encryptedBlob source agentId timestamp sequenceId } }`;
3098
+ while (true) {
3099
+ const hasLastId = lastId !== '';
3100
+ const query = hasLastId
3101
+ ? `query($owner:Bytes!,$first:Int!,$lastId:String!){facts(where:{owner:$owner,isActive:true,id_gt:$lastId},first:$first,orderBy:id,orderDirection:asc){id encryptedBlob timestamp sequenceId}}`
3102
+ : `query($owner:Bytes!,$first:Int!){facts(where:{owner:$owner,isActive:true},first:$first,orderBy:id,orderDirection:asc){id encryptedBlob timestamp sequenceId}}`;
3103
+ const variables: Record<string, unknown> = hasLastId
3104
+ ? { owner, first: PAGE_SIZE, lastId }
3105
+ : { owner, first: PAGE_SIZE };
2062
3106
 
2063
3107
  const res = await fetch(`${relayUrl}/v1/subgraph`, {
2064
3108
  method: 'POST',
@@ -2067,24 +3111,36 @@ const plugin = {
2067
3111
  'X-TotalReclaw-Client': 'openclaw-plugin',
2068
3112
  ...(authKeyHex ? { Authorization: `Bearer ${authKeyHex}` } : {}),
2069
3113
  },
2070
- body: JSON.stringify({ query }),
3114
+ body: JSON.stringify({ query, variables }),
2071
3115
  });
2072
3116
 
2073
3117
  const json = (await res.json()) as {
2074
3118
  data?: { facts?: Array<{ id: string; encryptedBlob: string; source: string; agentId: string; timestamp: string; sequenceId: string }> };
3119
+ error?: string;
3120
+ errors?: Array<{ message: string }>;
2075
3121
  };
3122
+ // Surface relay/subgraph errors instead of silently returning empty
3123
+ if (json.error || json.errors) {
3124
+ const errMsg = json.error || json.errors?.map(e => e.message).join('; ') || 'Unknown error';
3125
+ api.logger.error(`Export subgraph query failed: ${errMsg} (owner=${owner}, status=${res.status})`);
3126
+ return {
3127
+ content: [{ type: 'text', text: `Export failed: ${errMsg}` }],
3128
+ };
3129
+ }
2076
3130
  const facts = json?.data?.facts || [];
3131
+ if (facts.length === 0) break;
2077
3132
 
2078
3133
  for (const fact of facts) {
2079
3134
  try {
2080
3135
  let hexBlob = fact.encryptedBlob;
2081
3136
  if (hexBlob.startsWith('0x')) hexBlob = hexBlob.slice(2);
2082
3137
  const docJson = decryptFromHex(hexBlob, encryptionKey!);
2083
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
3138
+ if (isDigestBlob(docJson)) continue;
3139
+ const doc = readClaimFromBlob(docJson);
2084
3140
  allFacts.push({
2085
3141
  id: fact.id,
2086
3142
  text: doc.text,
2087
- metadata: doc.metadata ?? {},
3143
+ metadata: doc.metadata,
2088
3144
  created_at: new Date(parseInt(fact.timestamp) * 1000).toISOString(),
2089
3145
  });
2090
3146
  } catch {
@@ -2092,8 +3148,8 @@ const plugin = {
2092
3148
  }
2093
3149
  }
2094
3150
 
2095
- skip += PAGE_SIZE;
2096
- hasMore = facts.length === PAGE_SIZE;
3151
+ if (facts.length < PAGE_SIZE) break;
3152
+ lastId = facts[facts.length - 1].id;
2097
3153
  }
2098
3154
  } else {
2099
3155
  // HTTP server mode — paginate through PostgreSQL facts
@@ -2106,11 +3162,12 @@ const plugin = {
2106
3162
  for (const fact of page.facts) {
2107
3163
  try {
2108
3164
  const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey!);
2109
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
3165
+ if (isDigestBlob(docJson)) continue;
3166
+ const doc = readClaimFromBlob(docJson);
2110
3167
  allFacts.push({
2111
3168
  id: fact.id,
2112
3169
  text: doc.text,
2113
- metadata: doc.metadata ?? {},
3170
+ metadata: doc.metadata,
2114
3171
  created_at: fact.created_at,
2115
3172
  });
2116
3173
  } catch {
@@ -2152,7 +3209,7 @@ const plugin = {
2152
3209
  const message = err instanceof Error ? err.message : String(err);
2153
3210
  api.logger.error(`totalreclaw_export failed: ${message}`);
2154
3211
  return {
2155
- content: [{ type: 'text', text: `Failed to export memories: ${message}` }],
3212
+ content: [{ type: 'text', text: `Failed to export memories: ${humanizeError(message)}` }],
2156
3213
  };
2157
3214
  }
2158
3215
  },
@@ -2185,7 +3242,7 @@ const plugin = {
2185
3242
  };
2186
3243
  }
2187
3244
 
2188
- const serverUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
3245
+ const serverUrl = CONFIG.serverUrl;
2189
3246
  const walletAddr = subgraphOwner || userId || '';
2190
3247
  const response = await fetch(`${serverUrl}/v1/billing/status?wallet_address=${encodeURIComponent(walletAddr)}`, {
2191
3248
  method: 'GET',
@@ -2238,7 +3295,7 @@ const plugin = {
2238
3295
  const message = err instanceof Error ? err.message : String(err);
2239
3296
  api.logger.error(`totalreclaw_status failed: ${message}`);
2240
3297
  return {
2241
- content: [{ type: 'text', text: `Failed to check status: ${message}` }],
3298
+ content: [{ type: 'text', text: `Failed to check status: ${humanizeError(message)}` }],
2242
3299
  };
2243
3300
  }
2244
3301
  },
@@ -2255,13 +3312,13 @@ const plugin = {
2255
3312
  name: 'totalreclaw_consolidate',
2256
3313
  label: 'Consolidate',
2257
3314
  description:
2258
- 'Scan all stored memories and merge near-duplicates. Keeps the most important/recent version and removes redundant copies.',
3315
+ 'Deduplicate and merge related memories. Self-hosted mode only.',
2259
3316
  parameters: {
2260
3317
  type: 'object',
2261
3318
  properties: {
2262
3319
  dry_run: {
2263
3320
  type: 'boolean',
2264
- description: 'Preview consolidation without deleting (default: false)',
3321
+ description: 'Preview only (default: false)',
2265
3322
  },
2266
3323
  },
2267
3324
  additionalProperties: false,
@@ -2298,11 +3355,10 @@ const plugin = {
2298
3355
  for (const fact of page.facts) {
2299
3356
  try {
2300
3357
  const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey);
2301
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
3358
+ if (isDigestBlob(docJson)) continue;
3359
+ const doc = readClaimFromBlob(docJson);
2302
3360
 
2303
3361
  let embedding: number[] | null = null;
2304
- // ExportedFact does not include encrypted_embedding — generate it on-the-fly.
2305
- // For consolidation we need embeddings, so generate them.
2306
3362
  try {
2307
3363
  embedding = await generateEmbedding(doc.text);
2308
3364
  } catch { /* skip — fact will not be clustered */ }
@@ -2311,9 +3367,7 @@ const plugin = {
2311
3367
  id: fact.id,
2312
3368
  text: doc.text,
2313
3369
  embedding,
2314
- importance: doc.metadata?.importance
2315
- ? Math.round((doc.metadata.importance as number) * 10)
2316
- : 5,
3370
+ importance: doc.importance,
2317
3371
  decayScore: fact.decay_score,
2318
3372
  createdAt: new Date(fact.created_at).getTime(),
2319
3373
  version: fact.version,
@@ -2395,7 +3449,7 @@ const plugin = {
2395
3449
  const message = err instanceof Error ? err.message : String(err);
2396
3450
  api.logger.error(`totalreclaw_consolidate failed: ${message}`);
2397
3451
  return {
2398
- content: [{ type: 'text', text: `Failed to consolidate memories: ${message}` }],
3452
+ content: [{ type: 'text', text: `Failed to consolidate memories: ${humanizeError(message)}` }],
2399
3453
  };
2400
3454
  }
2401
3455
  },
@@ -2403,6 +3457,205 @@ const plugin = {
2403
3457
  { name: 'totalreclaw_consolidate' },
2404
3458
  );
2405
3459
 
3460
+ // ---------------------------------------------------------------
3461
+ // Helper: build PinOpDeps bound to the live plugin state
3462
+ // ---------------------------------------------------------------
3463
+ // Wires the pure pin/unpin operation to the managed-service transport +
3464
+ // crypto layer. Mirrors MCP's buildPinDepsFromState and Python's
3465
+ // _change_claim_status argument plumbing.
3466
+ const buildPinDeps = (): PinOpDeps => {
3467
+ const owner = subgraphOwner || userId || '';
3468
+ const config = {
3469
+ ...getSubgraphConfig(),
3470
+ authKeyHex: authKeyHex!,
3471
+ walletAddress: subgraphOwner ?? undefined,
3472
+ };
3473
+ return {
3474
+ owner,
3475
+ sourceAgent: 'openclaw-plugin',
3476
+ fetchFactById: (factId: string) => fetchFactById(owner, factId, authKeyHex!),
3477
+ decryptBlob: (hex: string) => decryptFromHex(hex, encryptionKey!),
3478
+ encryptBlob: (plaintext: string) => encryptToHex(plaintext, encryptionKey!),
3479
+ submitBatch: async (payloads: Buffer[]) => {
3480
+ const result = await submitFactBatchOnChain(payloads, config);
3481
+ return { txHash: result.txHash, success: result.success };
3482
+ },
3483
+ generateIndices: async (text: string, entityNames: string[]) => {
3484
+ if (!text) return { blindIndices: [] };
3485
+ const wordIndices = generateBlindIndices(text);
3486
+ let lshIndices: string[] = [];
3487
+ let encryptedEmbedding: string | undefined;
3488
+ try {
3489
+ const embedding = await generateEmbedding(text);
3490
+ const hasher = getLSHHasher(api.logger);
3491
+ if (hasher) lshIndices = hasher.hash(embedding);
3492
+ encryptedEmbedding = encryptToHex(JSON.stringify(embedding), encryptionKey!);
3493
+ } catch {
3494
+ // Best-effort: word + entity trapdoors alone still surface the claim.
3495
+ }
3496
+ const entityTrapdoors = entityNames.map((n) => computeEntityTrapdoor(n));
3497
+ return {
3498
+ blindIndices: [...wordIndices, ...lshIndices, ...entityTrapdoors],
3499
+ encryptedEmbedding,
3500
+ };
3501
+ },
3502
+ };
3503
+ };
3504
+
3505
+ // ---------------------------------------------------------------
3506
+ // Tool: totalreclaw_pin
3507
+ // ---------------------------------------------------------------
3508
+
3509
+ api.registerTool(
3510
+ {
3511
+ name: 'totalreclaw_pin',
3512
+ label: 'Pin',
3513
+ description:
3514
+ 'Pin a memory so the auto-resolution engine will never override or supersede it. ' +
3515
+ "Use when the user explicitly confirms a claim is still valid after you or another agent " +
3516
+ "tried to retract/contradict it (e.g. 'wait, I still use Vim sometimes'). " +
3517
+ 'Takes fact_id (from a prior recall result). Pinning is idempotent — pinning an already-pinned ' +
3518
+ 'claim is a no-op. Cross-device: the pin propagates via the on-chain supersession chain.',
3519
+ parameters: {
3520
+ type: 'object',
3521
+ properties: {
3522
+ fact_id: {
3523
+ type: 'string',
3524
+ description: 'The ID of the fact to pin (from a totalreclaw_recall result).',
3525
+ },
3526
+ reason: {
3527
+ type: 'string',
3528
+ description: 'Optional human-readable reason for pinning (logged locally for tuning).',
3529
+ },
3530
+ },
3531
+ required: ['fact_id'],
3532
+ additionalProperties: false,
3533
+ },
3534
+ async execute(_toolCallId: string, params: Record<string, unknown>) {
3535
+ try {
3536
+ await requireFullSetup(api.logger);
3537
+ if (!isSubgraphMode()) {
3538
+ return {
3539
+ content: [{
3540
+ type: 'text',
3541
+ text: 'Pin/unpin is only supported with the managed service. Self-hosted mode does not yet implement the status-flip supersession flow.',
3542
+ }],
3543
+ };
3544
+ }
3545
+ const validation = validatePinArgs(params);
3546
+ if (!validation.ok) {
3547
+ return { content: [{ type: 'text', text: validation.error }] };
3548
+ }
3549
+ const deps = buildPinDeps();
3550
+ const result = await executePinOperation(validation.factId, 'pinned', deps, validation.reason);
3551
+ if (result.success && result.idempotent) {
3552
+ api.logger.info(`totalreclaw_pin: ${result.fact_id} already pinned (no-op)`);
3553
+ return {
3554
+ content: [{ type: 'text', text: `Memory ${result.fact_id} is already pinned.` }],
3555
+ details: result,
3556
+ };
3557
+ }
3558
+ if (result.success) {
3559
+ api.logger.info(`totalreclaw_pin: ${result.fact_id} → ${result.new_fact_id} (tx ${result.tx_hash?.slice(0, 10)})`);
3560
+ return {
3561
+ content: [{
3562
+ type: 'text',
3563
+ text: `Pinned memory ${result.fact_id}. New fact id: ${result.new_fact_id} (tx: ${result.tx_hash}).`,
3564
+ }],
3565
+ details: result,
3566
+ };
3567
+ }
3568
+ api.logger.error(`totalreclaw_pin failed: ${result.error}`);
3569
+ return {
3570
+ content: [{ type: 'text', text: `Failed to pin memory: ${humanizeError(result.error ?? 'unknown error')}` }],
3571
+ details: result,
3572
+ };
3573
+ } catch (err: unknown) {
3574
+ const message = err instanceof Error ? err.message : String(err);
3575
+ api.logger.error(`totalreclaw_pin failed: ${message}`);
3576
+ return {
3577
+ content: [{ type: 'text', text: `Failed to pin memory: ${humanizeError(message)}` }],
3578
+ };
3579
+ }
3580
+ },
3581
+ },
3582
+ { name: 'totalreclaw_pin' },
3583
+ );
3584
+
3585
+ // ---------------------------------------------------------------
3586
+ // Tool: totalreclaw_unpin
3587
+ // ---------------------------------------------------------------
3588
+
3589
+ api.registerTool(
3590
+ {
3591
+ name: 'totalreclaw_unpin',
3592
+ label: 'Unpin',
3593
+ description:
3594
+ 'Remove the pin from a previously pinned memory, returning it to active status so the ' +
3595
+ 'auto-resolution engine can supersede or retract it again. Takes fact_id. Idempotent — ' +
3596
+ 'unpinning a non-pinned claim is a no-op.',
3597
+ parameters: {
3598
+ type: 'object',
3599
+ properties: {
3600
+ fact_id: {
3601
+ type: 'string',
3602
+ description: 'The ID of the fact to unpin (from a totalreclaw_recall result).',
3603
+ },
3604
+ },
3605
+ required: ['fact_id'],
3606
+ additionalProperties: false,
3607
+ },
3608
+ async execute(_toolCallId: string, params: Record<string, unknown>) {
3609
+ try {
3610
+ await requireFullSetup(api.logger);
3611
+ if (!isSubgraphMode()) {
3612
+ return {
3613
+ content: [{
3614
+ type: 'text',
3615
+ text: 'Pin/unpin is only supported with the managed service. Self-hosted mode does not yet implement the status-flip supersession flow.',
3616
+ }],
3617
+ };
3618
+ }
3619
+ const validation = validatePinArgs(params);
3620
+ if (!validation.ok) {
3621
+ return { content: [{ type: 'text', text: validation.error }] };
3622
+ }
3623
+ const deps = buildPinDeps();
3624
+ const result = await executePinOperation(validation.factId, 'active', deps);
3625
+ if (result.success && result.idempotent) {
3626
+ api.logger.info(`totalreclaw_unpin: ${result.fact_id} already active (no-op)`);
3627
+ return {
3628
+ content: [{ type: 'text', text: `Memory ${result.fact_id} is not pinned.` }],
3629
+ details: result,
3630
+ };
3631
+ }
3632
+ if (result.success) {
3633
+ api.logger.info(`totalreclaw_unpin: ${result.fact_id} → ${result.new_fact_id} (tx ${result.tx_hash?.slice(0, 10)})`);
3634
+ return {
3635
+ content: [{
3636
+ type: 'text',
3637
+ text: `Unpinned memory ${result.fact_id}. New fact id: ${result.new_fact_id} (tx: ${result.tx_hash}).`,
3638
+ }],
3639
+ details: result,
3640
+ };
3641
+ }
3642
+ api.logger.error(`totalreclaw_unpin failed: ${result.error}`);
3643
+ return {
3644
+ content: [{ type: 'text', text: `Failed to unpin memory: ${humanizeError(result.error ?? 'unknown error')}` }],
3645
+ details: result,
3646
+ };
3647
+ } catch (err: unknown) {
3648
+ const message = err instanceof Error ? err.message : String(err);
3649
+ api.logger.error(`totalreclaw_unpin failed: ${message}`);
3650
+ return {
3651
+ content: [{ type: 'text', text: `Failed to unpin memory: ${humanizeError(message)}` }],
3652
+ };
3653
+ }
3654
+ },
3655
+ },
3656
+ { name: 'totalreclaw_unpin' },
3657
+ );
3658
+
2406
3659
  // ---------------------------------------------------------------
2407
3660
  // Tool: totalreclaw_import_from
2408
3661
  // ---------------------------------------------------------------
@@ -2412,7 +3665,7 @@ const plugin = {
2412
3665
  name: 'totalreclaw_import_from',
2413
3666
  label: 'Import From',
2414
3667
  description:
2415
- 'Import memories from other AI memory tools (Mem0, MCP Memory Server, ChatGPT, Claude, MemoClaw, or generic JSON/CSV). ' +
3668
+ 'Import memories from other AI memory tools (Mem0, MCP Memory Server, ChatGPT, Claude, Gemini, MemoClaw, or generic JSON/CSV). ' +
2416
3669
  'Provide the source name and either an API key, file content, or file path. ' +
2417
3670
  'Use dry_run=true to preview before importing. Idempotent — safe to run multiple times.',
2418
3671
  parameters: {
@@ -2420,8 +3673,8 @@ const plugin = {
2420
3673
  properties: {
2421
3674
  source: {
2422
3675
  type: 'string',
2423
- enum: ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'memoclaw', 'generic-json', 'generic-csv'],
2424
- description: 'The source system to import from (chatgpt: conversations.json or memory text; claude: memory text)',
3676
+ enum: ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'],
3677
+ description: 'The source system to import from (gemini: Google Takeout HTML; chatgpt: conversations.json or memory text; claude: memory text)',
2425
3678
  },
2426
3679
  api_key: {
2427
3680
  type: 'string',
@@ -2463,6 +3716,56 @@ const plugin = {
2463
3716
  { name: 'totalreclaw_import_from' },
2464
3717
  );
2465
3718
 
3719
+ // ---------------------------------------------------------------
3720
+ // Tool: totalreclaw_import_batch
3721
+ // ---------------------------------------------------------------
3722
+
3723
+ api.registerTool(
3724
+ {
3725
+ name: 'totalreclaw_import_batch',
3726
+ label: 'Import Batch',
3727
+ description:
3728
+ 'Process one batch of a large import. Call repeatedly with increasing offset until is_complete=true.',
3729
+ parameters: {
3730
+ type: 'object',
3731
+ properties: {
3732
+ source: {
3733
+ type: 'string',
3734
+ enum: ['gemini', 'chatgpt', 'claude'],
3735
+ description: 'Source format',
3736
+ },
3737
+ file_path: {
3738
+ type: 'string',
3739
+ description: 'Path to source file',
3740
+ },
3741
+ content: {
3742
+ type: 'string',
3743
+ description: 'File content (text sources)',
3744
+ },
3745
+ offset: {
3746
+ type: 'number',
3747
+ description: 'Starting chunk index (0-based)',
3748
+ },
3749
+ batch_size: {
3750
+ type: 'number',
3751
+ description: 'Chunks per call (default 25)',
3752
+ },
3753
+ },
3754
+ required: ['source'],
3755
+ },
3756
+ async execute(_toolCallId: string, params: Record<string, unknown>) {
3757
+ try {
3758
+ await requireFullSetup(api.logger);
3759
+ return handleBatchImport(params, api.logger);
3760
+ } catch (err: unknown) {
3761
+ const message = err instanceof Error ? err.message : String(err);
3762
+ return { error: message };
3763
+ }
3764
+ },
3765
+ },
3766
+ { name: 'totalreclaw_import_batch' },
3767
+ );
3768
+
2466
3769
  // ---------------------------------------------------------------
2467
3770
  // Tool: totalreclaw_upgrade
2468
3771
  // ---------------------------------------------------------------
@@ -2489,7 +3792,7 @@ const plugin = {
2489
3792
  };
2490
3793
  }
2491
3794
 
2492
- const serverUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
3795
+ const serverUrl = CONFIG.serverUrl;
2493
3796
  const walletAddr = subgraphOwner || userId || '';
2494
3797
 
2495
3798
  if (!walletAddr) {
@@ -2534,7 +3837,7 @@ const plugin = {
2534
3837
  const message = err instanceof Error ? err.message : String(err);
2535
3838
  api.logger.error(`totalreclaw_upgrade failed: ${message}`);
2536
3839
  return {
2537
- content: [{ type: 'text', text: `Failed to create checkout session: ${message}` }],
3840
+ content: [{ type: 'text', text: `Failed to create checkout session: ${humanizeError(message)}` }],
2538
3841
  };
2539
3842
  }
2540
3843
  },
@@ -2581,7 +3884,7 @@ const plugin = {
2581
3884
  }
2582
3885
 
2583
3886
  const confirm = _params?.confirm === true;
2584
- const serverUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
3887
+ const serverUrl = CONFIG.serverUrl;
2585
3888
 
2586
3889
  // 1. Check billing tier
2587
3890
  const billingResp = await fetch(
@@ -2668,6 +3971,7 @@ const plugin = {
2668
3971
  contentFp: fact.contentFp || '',
2669
3972
  agentId: fact.agentId || 'openclaw-plugin',
2670
3973
  encryptedEmbedding: fact.encryptedEmbedding || undefined,
3974
+ version: PROTOBUF_VERSION_V4,
2671
3975
  };
2672
3976
  payloads.push(encodeFactProtobuf(factPayload));
2673
3977
  }
@@ -2717,7 +4021,7 @@ const plugin = {
2717
4021
  const message = err instanceof Error ? err.message : String(err);
2718
4022
  api.logger.error(`totalreclaw_migrate failed: ${message}`);
2719
4023
  return {
2720
- content: [{ type: 'text', text: `Migration failed: ${message}` }],
4024
+ content: [{ type: 'text', text: `Migration failed: ${humanizeError(message)}` }],
2721
4025
  };
2722
4026
  }
2723
4027
  },
@@ -2725,6 +4029,107 @@ const plugin = {
2725
4029
  { name: 'totalreclaw_migrate' },
2726
4030
  );
2727
4031
 
4032
+ // ---------------------------------------------------------------
4033
+ // Tool: totalreclaw_setup
4034
+ // ---------------------------------------------------------------
4035
+
4036
+ api.registerTool(
4037
+ {
4038
+ name: 'totalreclaw_setup',
4039
+ label: 'Setup TotalReclaw',
4040
+ description:
4041
+ 'Initialize TotalReclaw with a recovery phrase. Derives encryption keys and registers with the server. ' +
4042
+ 'Use this during first-time setup instead of setting environment variables — no gateway restart needed.',
4043
+ parameters: {
4044
+ type: 'object',
4045
+ properties: {
4046
+ recovery_phrase: {
4047
+ type: 'string',
4048
+ description: 'Optional 12-word BIP-39 recovery phrase. If not provided, generates a new one automatically. For returning users, pass their existing phrase.',
4049
+ },
4050
+ },
4051
+ additionalProperties: false,
4052
+ },
4053
+ async execute(_toolCallId: string, params: { recovery_phrase?: string }) {
4054
+ try {
4055
+ let mnemonic = params.recovery_phrase?.trim() || '';
4056
+
4057
+ // Auto-generate if not provided
4058
+ if (!mnemonic) {
4059
+ const { generateMnemonic } = await import('@scure/bip39');
4060
+ const { wordlist } = await import('@scure/bip39/wordlists/english');
4061
+ mnemonic = generateMnemonic(wordlist, 128);
4062
+ api.logger.info('totalreclaw_setup: generated new BIP-39 mnemonic');
4063
+ }
4064
+
4065
+ // Guard: refuse to overwrite existing credentials with a DIFFERENT phrase
4066
+ // (prevents data loss when background sessions_spawn workers call setup).
4067
+ // Allow re-init with the SAME phrase (handles agent exec → setup flow).
4068
+ try {
4069
+ const existing = fs.readFileSync(CREDENTIALS_PATH, 'utf-8');
4070
+ const creds = JSON.parse(existing);
4071
+ if (creds.mnemonic && creds.userId && creds.mnemonic !== mnemonic) {
4072
+ api.logger.info('totalreclaw_setup: credentials exist with different mnemonic, refusing to overwrite');
4073
+ return {
4074
+ content: [{
4075
+ type: 'text',
4076
+ text: 'TotalReclaw is already set up with an existing recovery phrase. Your encrypted memories are tied to that phrase.\n\n' +
4077
+ 'If you intentionally want to start fresh with a NEW phrase (this will make existing memories inaccessible), ' +
4078
+ 'delete ~/.totalreclaw/credentials.json first, then call this tool again.',
4079
+ }],
4080
+ };
4081
+ }
4082
+ } catch { /* credentials.json doesn't exist or is corrupted — proceed with setup */ }
4083
+
4084
+ // Basic validation: must be 12 words
4085
+ const words = mnemonic.split(/\s+/);
4086
+ if (words.length !== 12) {
4087
+ return {
4088
+ content: [{
4089
+ type: 'text',
4090
+ text: `Error: Recovery phrase must be exactly 12 words (got ${words.length}). Use \`npx @totalreclaw/mcp-server setup\` to generate a valid BIP-39 mnemonic.`,
4091
+ }],
4092
+ };
4093
+ }
4094
+
4095
+ api.logger.info('totalreclaw_setup: initializing with provided recovery phrase');
4096
+
4097
+ // Force re-initialization with the new mnemonic.
4098
+ // This derives keys, registers with the server, saves credentials,
4099
+ // and sets up LSH/auth — all without a gateway restart.
4100
+ await forceReinitialization(mnemonic, api.logger);
4101
+
4102
+ if (needsSetup) {
4103
+ return {
4104
+ content: [{
4105
+ type: 'text',
4106
+ text: 'Setup failed — could not initialize with the provided recovery phrase. Check the logs for details.',
4107
+ }],
4108
+ };
4109
+ }
4110
+
4111
+ const wasGenerated = !params.recovery_phrase?.trim();
4112
+ return {
4113
+ content: [{
4114
+ type: 'text',
4115
+ text: 'TotalReclaw setup complete! Encryption keys derived, server registration confirmed. ' +
4116
+ 'You can now use totalreclaw_remember, totalreclaw_recall, and all other tools immediately — no restart needed.\n\n' +
4117
+ (wasGenerated ? `Recovery phrase: ${mnemonic}\n\n` : '') +
4118
+ 'From now on, I will automatically remember important things from our conversations and recall relevant context at the start of each session.',
4119
+ }],
4120
+ };
4121
+ } catch (err: unknown) {
4122
+ const message = err instanceof Error ? err.message : String(err);
4123
+ api.logger.error(`totalreclaw_setup failed: ${message}`);
4124
+ return {
4125
+ content: [{ type: 'text', text: `Setup failed: ${humanizeError(message)}` }],
4126
+ };
4127
+ }
4128
+ },
4129
+ },
4130
+ { name: 'totalreclaw_setup' },
4131
+ );
4132
+
2728
4133
  // ---------------------------------------------------------------
2729
4134
  // Hook: before_agent_start
2730
4135
  // ---------------------------------------------------------------
@@ -2752,11 +4157,12 @@ const plugin = {
2752
4157
  '## TotalReclaw — First-Time Setup\n\n' +
2753
4158
  'TotalReclaw is an end-to-end encrypted memory vault. It remembers things across conversations — preferences, decisions, project context — all encrypted so no one (not even the server) can read your memories.\n\n' +
2754
4159
  '**To get started, ask the user:** "Do you have an existing TotalReclaw recovery phrase you\'d like to restore, or should I generate a new one?"\n\n' +
2755
- '**If new user:** Generate a random 12-word BIP-39 mnemonic and set it as the TOTALRECLAW_RECOVERY_PHRASE environment variable. Display it prominently and warn:\n' +
4160
+ '**If new user:** Run `npx @totalreclaw/mcp-server setup` to generate a cryptographically valid recovery phrase. Do NOT generate the phrase yourself — LLMs cannot compute BIP-39 checksums. The command will output the phrase. Then call the `totalreclaw_setup` tool with the phrase. Display the phrase prominently and warn:\n' +
2756
4161
  '- "This is your ONLY way to access your memories. Write it down and store it securely."\n' +
2757
4162
  '- "NEVER use a recovery phrase from an existing crypto wallet — keep this separate from any wallet that holds funds."\n' +
2758
4163
  '- "If you lose this phrase, your memories are gone forever — there is no recovery."\n\n' +
2759
- '**If returning user:** Ask them to provide their 12-word phrase, validate it, and set it as TOTALRECLAW_RECOVERY_PHRASE.\n\n' +
4164
+ '**If returning user:** Ask them to provide their 12-word phrase, then call `totalreclaw_setup` with that phrase.\n\n' +
4165
+ '**IMPORTANT:** Use the `totalreclaw_setup` tool to complete setup. This initializes encryption keys immediately — no gateway restart needed. The user can continue chatting right away.\n\n' +
2760
4166
  '**After setup:** Explain that from now on, you will automatically remember important things from conversations and recall relevant memories at the start of each session. The user can also explicitly ask you to remember, recall, forget, or export memories.',
2761
4167
  };
2762
4168
  }
@@ -2783,7 +4189,7 @@ const plugin = {
2783
4189
  let cache = readBillingCache();
2784
4190
  if (!cache && authKeyHex) {
2785
4191
  // Cache is stale or missing — fetch fresh billing status.
2786
- const billingUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
4192
+ const billingUrl = CONFIG.serverUrl;
2787
4193
  const walletParam = encodeURIComponent(subgraphOwner || userId || '');
2788
4194
  const billingResp = await fetch(`${billingUrl}/v1/billing/status?wallet_address=${walletParam}`, {
2789
4195
  method: 'GET',
@@ -2812,7 +4218,46 @@ const plugin = {
2812
4218
  }
2813
4219
 
2814
4220
  if (isSubgraphMode()) {
2815
- // --- Subgraph mode: hot cache first, then background refresh ---
4221
+ // --- Subgraph mode: digest fast path → hot cache background refresh ---
4222
+
4223
+ // Digest fast path (Stage 3b). When a digest exists and the mode is
4224
+ // not 'off', inject its pre-compiled promptText instead of running
4225
+ // the per-query search. A stale digest triggers a background
4226
+ // recompile (non-blocking). Failures fall through to the legacy
4227
+ // path silently.
4228
+ const digestMode = resolveDigestMode();
4229
+ logDigestModeOnce(digestMode, api.logger);
4230
+ if (digestMode !== 'off' && encryptionKey && authKeyHex && (subgraphOwner || userId)) {
4231
+ try {
4232
+ const injectResult = await maybeInjectDigest({
4233
+ owner: subgraphOwner || userId!,
4234
+ authKeyHex: authKeyHex!,
4235
+ encryptionKey: encryptionKey!,
4236
+ mode: digestMode,
4237
+ nowMs: Date.now(),
4238
+ loadDeps: {
4239
+ searchSubgraph: async (o, tds, n, a) => searchSubgraph(o, tds, n, a),
4240
+ decryptFromHex: (hex, key) => decryptFromHex(hex, key),
4241
+ },
4242
+ probeDeps: {
4243
+ searchSubgraphBroadened: async (o, n, a) => searchSubgraphBroadened(o, n, a),
4244
+ },
4245
+ recompileFn: (prev) => scheduleDigestRecompile(prev, api.logger),
4246
+ logger: api.logger,
4247
+ });
4248
+ if (injectResult.promptText) {
4249
+ api.logger.info(`Digest injection: state=${injectResult.state}`);
4250
+ return {
4251
+ prependContext:
4252
+ `## Your Memory\n\n${injectResult.promptText}` + welcomeBack + billingWarning,
4253
+ };
4254
+ }
4255
+ } catch (err) {
4256
+ // Never block session start on digest failure.
4257
+ const msg = err instanceof Error ? err.message : String(err);
4258
+ api.logger.warn(`Digest fast path failed: ${msg}`);
4259
+ }
4260
+ }
2816
4261
 
2817
4262
  // Initialize hot cache if needed.
2818
4263
  if (!pluginHotCache && encryptionKey) {
@@ -2885,6 +4330,21 @@ const plugin = {
2885
4330
  return undefined;
2886
4331
  }
2887
4332
 
4333
+ // Always run broadened search and merge — ensures vocabulary mismatches
4334
+ // (e.g., "preferences" vs "prefer") don't cause recall failures.
4335
+ // The reranker handles scoring; extra cost is ~1 GraphQL query per recall.
4336
+ try {
4337
+ const broadPool = computeCandidatePool(0);
4338
+ const broadenedResults = await searchSubgraphBroadened(subgraphOwner || userId!, broadPool, authKeyHex!);
4339
+ // Merge broadened results with existing (deduplicate by ID)
4340
+ const existingIds = new Set(subgraphResults.map(r => r.id));
4341
+ for (const br of broadenedResults) {
4342
+ if (!existingIds.has(br.id)) {
4343
+ subgraphResults.push(br);
4344
+ }
4345
+ }
4346
+ } catch { /* best-effort */ }
4347
+
2888
4348
  if (subgraphResults.length === 0 && cachedFacts.length === 0) return undefined;
2889
4349
 
2890
4350
  // If subgraph returned no results but we have cache, use cache.
@@ -2902,7 +4362,10 @@ const plugin = {
2902
4362
  for (const result of subgraphResults) {
2903
4363
  try {
2904
4364
  const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
2905
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
4365
+ // Filter out digest infrastructure blobs they have no user
4366
+ // text and should never surface in recall results.
4367
+ if (isDigestBlob(docJson)) continue;
4368
+ const doc = readClaimFromBlob(docJson);
2906
4369
 
2907
4370
  let decryptedEmbedding: number[] | undefined;
2908
4371
  if (result.encryptedEmbedding) {
@@ -2915,22 +4378,20 @@ const plugin = {
2915
4378
  }
2916
4379
  }
2917
4380
 
2918
- const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
2919
4381
  const createdAtSec = result.timestamp ? parseInt(result.timestamp, 10) : undefined;
2920
4382
  rerankerCandidates.push({
2921
4383
  id: result.id,
2922
4384
  text: doc.text,
2923
4385
  embedding: decryptedEmbedding,
2924
- importance: importanceRaw,
4386
+ importance: doc.importance / 10,
2925
4387
  createdAt: createdAtSec,
4388
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
2926
4389
  });
2927
4390
 
2928
- const importance = doc.metadata?.importance
2929
- ? Math.round((doc.metadata.importance as number) * 10)
2930
- : 5;
2931
4391
  hookMetaMap.set(result.id, {
2932
- importance,
4392
+ importance: doc.importance,
2933
4393
  age: 'subgraph',
4394
+ category: doc.category,
2934
4395
  });
2935
4396
  } catch {
2936
4397
  // Skip un-decryptable candidates.
@@ -2945,17 +4406,9 @@ const plugin = {
2945
4406
  rerankerCandidates,
2946
4407
  8,
2947
4408
  INTENT_WEIGHTS[hookQueryIntent],
4409
+ /* applySourceWeights (Retrieval v2 Tier 1) */ true,
2948
4410
  );
2949
4411
 
2950
- // B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
2951
- const candidatesWithEmb = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
2952
- if (candidatesWithEmb.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
2953
- const topCosine = Math.max(
2954
- ...candidatesWithEmb.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
2955
- );
2956
- if (topCosine < RELEVANCE_THRESHOLD) return undefined;
2957
- }
2958
-
2959
4412
  // Update hot cache with reranked results.
2960
4413
  try {
2961
4414
  if (pluginHotCache) {
@@ -2994,7 +4447,8 @@ const plugin = {
2994
4447
  const meta = hookMetaMap.get(m.id);
2995
4448
  const importance = meta?.importance ?? 5;
2996
4449
  const age = meta?.age ?? '';
2997
- return `${i + 1}. ${m.text} (importance: ${importance}/10, ${age})`;
4450
+ const typeTag = meta?.category ? `[${meta.category}] ` : '';
4451
+ return `${i + 1}. ${typeTag}${m.text} (importance: ${importance}/10, ${age})`;
2998
4452
  });
2999
4453
  const contextString = `## Relevant Memories\n\n${lines.join('\n')}`;
3000
4454
 
@@ -3042,9 +4496,10 @@ const plugin = {
3042
4496
  for (const candidate of candidates) {
3043
4497
  try {
3044
4498
  const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
3045
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
4499
+ // Skip digest infrastructure blobs.
4500
+ if (isDigestBlob(docJson)) continue;
4501
+ const doc = readClaimFromBlob(docJson);
3046
4502
 
3047
- // Decrypt embedding if present.
3048
4503
  let decryptedEmbedding: number[] | undefined;
3049
4504
  if (candidate.encrypted_embedding) {
3050
4505
  try {
@@ -3056,7 +4511,6 @@ const plugin = {
3056
4511
  }
3057
4512
  }
3058
4513
 
3059
- const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
3060
4514
  const createdAtSec = typeof candidate.timestamp === 'number'
3061
4515
  ? candidate.timestamp / 1000
3062
4516
  : new Date(candidate.timestamp).getTime() / 1000;
@@ -3064,15 +4518,13 @@ const plugin = {
3064
4518
  id: candidate.fact_id,
3065
4519
  text: doc.text,
3066
4520
  embedding: decryptedEmbedding,
3067
- importance: importanceRaw,
4521
+ importance: doc.importance / 10,
3068
4522
  createdAt: createdAtSec,
4523
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
3069
4524
  });
3070
4525
 
3071
- const importance = doc.metadata?.importance
3072
- ? Math.round((doc.metadata.importance as number) * 10)
3073
- : 5;
3074
4526
  hookMetaMap.set(candidate.fact_id, {
3075
- importance,
4527
+ importance: doc.importance,
3076
4528
  age: relativeTime(candidate.timestamp),
3077
4529
  });
3078
4530
  } catch {
@@ -3088,19 +4540,23 @@ const plugin = {
3088
4540
  rerankerCandidates,
3089
4541
  8,
3090
4542
  INTENT_WEIGHTS[srvHookIntent],
3091
- );
3092
-
3093
- // B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
3094
- const candidatesWithEmbSrv = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
3095
- if (candidatesWithEmbSrv.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
3096
- const topCosine = Math.max(
3097
- ...candidatesWithEmbSrv.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
4543
+ /* applySourceWeights (Retrieval v2 Tier 1) */ true,
3098
4544
  );
3099
- if (topCosine < RELEVANCE_THRESHOLD) return undefined;
3100
- }
3101
4545
 
3102
4546
  if (reranked.length === 0) return undefined;
3103
4547
 
4548
+ // Cosine similarity threshold gate — skip injection when the
4549
+ // best match is below the minimum relevance threshold.
4550
+ const srvMaxCosine = Math.max(
4551
+ ...reranked.map((r) => r.cosineSimilarity ?? 0),
4552
+ );
4553
+ if (srvMaxCosine < COSINE_THRESHOLD) {
4554
+ api.logger.info(
4555
+ `Hook: cosine threshold gate filtered results (max=${srvMaxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD})`,
4556
+ );
4557
+ return undefined;
4558
+ }
4559
+
3104
4560
  // 7. Build context string.
3105
4561
  const lines = reranked.map((m, i) => {
3106
4562
  const meta = hookMetaMap.get(m.id);
@@ -3128,21 +4584,73 @@ const plugin = {
3128
4584
  api.on(
3129
4585
  'agent_end',
3130
4586
  async (event: unknown) => {
4587
+ // CRITICAL: Always return { memoryHandled: true } so OpenClaw's default
4588
+ // memory system does NOT fall back to writing plaintext MEMORY.md.
4589
+ // Losing facts on error is acceptable; leaking them in cleartext is not.
3131
4590
  try {
4591
+ // Defensive: ensure MEMORY.md header is present so OpenClaw's default
4592
+ // memory system doesn't write sensitive data in cleartext, even if
4593
+ // our extraction fails below.
4594
+ ensureMemoryHeader(api.logger);
4595
+
4596
+ // BUG-2 fix: skip extraction if an import was in progress this turn.
4597
+ // Import failures were retriggering agent_end → extraction → import loops.
4598
+ if (_importInProgress) {
4599
+ _importInProgress = false; // auto-reset for next turn
4600
+ api.logger.info('agent_end: skipping extraction (import was in progress)');
4601
+ return { memoryHandled: true };
4602
+ }
4603
+
3132
4604
  const evt = event as { messages?: unknown[]; success?: boolean } | undefined;
3133
- if (!evt?.success || !evt?.messages || evt.messages.length < 2) return;
4605
+ if (!evt?.messages || evt.messages.length < 2) {
4606
+ api.logger.info('agent_end: skipping extraction (no messages)');
4607
+ return { memoryHandled: true };
4608
+ }
4609
+ // Proceed with extraction even when evt.success is false or undefined.
4610
+ // A single LLM timeout on one turn should not prevent extraction of
4611
+ // facts from the (potentially many) successful turns in the message
4612
+ // history. The extractor processes the full message array and can
4613
+ // extract valuable facts from content before the failure.
4614
+ if (evt.success === false) {
4615
+ api.logger.info('agent_end: turn reported failure, but proceeding with extraction from message history');
4616
+ }
3134
4617
 
3135
4618
  await ensureInitialized(api.logger);
3136
- if (needsSetup) return;
4619
+ if (needsSetup) return { memoryHandled: true };
3137
4620
 
3138
4621
  // C3: Throttle auto-extraction to every N turns (configurable via env).
4622
+ // Phase 2.2.5: every branch of the extraction pipeline now logs its
4623
+ // outcome. Prior to 2.2.5, only the "stored N facts" happy path
4624
+ // produced a log line, so silent JSON parse failures / chatCompletion
4625
+ // timeouts / importance-filter-drops-everything scenarios left no
4626
+ // trace whatsoever in the gateway log. See the investigation report
4627
+ // in CHANGELOG for the full failure chain we uncovered.
3139
4628
  turnsSinceLastExtraction++;
3140
- if (turnsSinceLastExtraction >= getExtractInterval()) {
4629
+ const extractInterval = getExtractInterval();
4630
+ api.logger.info(
4631
+ `agent_end: turn ${turnsSinceLastExtraction}/${extractInterval} (messages=${evt.messages.length})`,
4632
+ );
4633
+ if (turnsSinceLastExtraction >= extractInterval) {
3141
4634
  const existingMemories = isLlmDedupEnabled()
3142
4635
  ? await fetchExistingMemoriesForExtraction(api.logger, 20, evt.messages)
3143
4636
  : [];
3144
- const rawFacts = await extractFacts(evt.messages, 'turn', existingMemories);
3145
- const { kept: importanceFiltered } = filterByImportance(rawFacts, api.logger);
4637
+ const rawFacts = await extractFacts(
4638
+ evt.messages,
4639
+ 'turn',
4640
+ existingMemories,
4641
+ undefined,
4642
+ api.logger,
4643
+ );
4644
+ api.logger.info(
4645
+ `agent_end: extractFacts returned ${rawFacts.length} raw facts`,
4646
+ );
4647
+ const { kept: importanceFiltered, dropped } = filterByImportance(
4648
+ rawFacts,
4649
+ api.logger,
4650
+ );
4651
+ api.logger.info(
4652
+ `agent_end: after importance filter: kept=${importanceFiltered.length}, dropped=${dropped}`,
4653
+ );
3146
4654
  const maxFacts = getMaxFactsPerExtraction();
3147
4655
  if (importanceFiltered.length > maxFacts) {
3148
4656
  api.logger.info(
@@ -3152,13 +4660,23 @@ const plugin = {
3152
4660
  const facts = importanceFiltered.slice(0, maxFacts);
3153
4661
  if (facts.length > 0) {
3154
4662
  await storeExtractedFacts(facts, api.logger);
4663
+ api.logger.info(`agent_end: stored ${facts.length} facts to encrypted vault`);
4664
+ } else {
4665
+ // Phase 2.2.5: no longer silent when extraction produces nothing.
4666
+ api.logger.info(
4667
+ `agent_end: extraction produced 0 storable facts (raw=${rawFacts.length}, after-importance=${importanceFiltered.length})`,
4668
+ );
3155
4669
  }
3156
4670
  turnsSinceLastExtraction = 0;
3157
4671
  }
3158
4672
  } catch (err: unknown) {
3159
4673
  const message = err instanceof Error ? err.message : String(err);
3160
- api.logger.warn(`agent_end extraction failed: ${message}`);
4674
+ api.logger.error(`agent_end extraction failed: ${message}`);
4675
+ // Re-assert MEMORY.md header even on failure — last line of defense.
4676
+ ensureMemoryHeader(api.logger);
3161
4677
  }
4678
+ // Always signal that memory is handled — prevent plaintext fallback.
4679
+ return { memoryHandled: true };
3162
4680
  },
3163
4681
  { priority: 90 },
3164
4682
  );
@@ -3178,13 +4696,13 @@ const plugin = {
3178
4696
  if (needsSetup) return;
3179
4697
 
3180
4698
  api.logger.info(
3181
- `Pre-compaction extraction: processing ${evt.messages.length} messages`,
4699
+ `pre_compaction: using compaction-aware extraction (importance >= 5), processing ${evt.messages.length} messages`,
3182
4700
  );
3183
4701
 
3184
4702
  const existingMemories = isLlmDedupEnabled()
3185
4703
  ? await fetchExistingMemoriesForExtraction(api.logger, 50, evt.messages)
3186
4704
  : [];
3187
- const rawCompactFacts = await extractFacts(evt.messages, 'full', existingMemories);
4705
+ const rawCompactFacts = await extractFactsForCompaction(evt.messages, existingMemories, api.logger);
3188
4706
  const { kept: compactImportanceFiltered } = filterByImportance(rawCompactFacts, api.logger);
3189
4707
  const maxFactsCompact = getMaxFactsPerExtraction();
3190
4708
  if (compactImportanceFiltered.length > maxFactsCompact) {
@@ -3197,6 +4715,29 @@ const plugin = {
3197
4715
  await storeExtractedFacts(facts, api.logger);
3198
4716
  }
3199
4717
  turnsSinceLastExtraction = 0; // Reset C3 counter on compaction.
4718
+
4719
+ // Session debrief — after regular extraction.
4720
+ // v1 mapping: DebriefItem { type: 'summary'|'context' } →
4721
+ // v1 type 'summary' (always, since context → claim would lose
4722
+ // the "this is a session summary" signal) + source 'derived'
4723
+ // (session debrief is a derived synthesis by definition).
4724
+ try {
4725
+ const storedTexts = facts.map((f) => f.text);
4726
+ const debriefItems = await extractDebrief(evt.messages, storedTexts);
4727
+ if (debriefItems.length > 0) {
4728
+ const debriefFacts: ExtractedFact[] = debriefItems.map((d) => ({
4729
+ text: d.text,
4730
+ type: 'summary' as MemoryType,
4731
+ source: 'derived' as MemorySource,
4732
+ importance: d.importance,
4733
+ action: 'ADD' as const,
4734
+ }));
4735
+ await storeExtractedFacts(debriefFacts, api.logger, 'openclaw_debrief');
4736
+ api.logger.info(`Session debrief: stored ${debriefItems.length} items`);
4737
+ }
4738
+ } catch (debriefErr: unknown) {
4739
+ api.logger.warn(`before_compaction debrief failed: ${debriefErr instanceof Error ? debriefErr.message : String(debriefErr)}`);
4740
+ }
3200
4741
  } catch (err: unknown) {
3201
4742
  const message = err instanceof Error ? err.message : String(err);
3202
4743
  api.logger.warn(`before_compaction extraction failed: ${message}`);
@@ -3239,6 +4780,29 @@ const plugin = {
3239
4780
  await storeExtractedFacts(facts, api.logger);
3240
4781
  }
3241
4782
  turnsSinceLastExtraction = 0; // Reset C3 counter on reset.
4783
+
4784
+ // Session debrief — after regular extraction.
4785
+ // v1 mapping: DebriefItem { type: 'summary'|'context' } →
4786
+ // v1 type 'summary' (always, since context → claim would lose
4787
+ // the "this is a session summary" signal) + source 'derived'
4788
+ // (session debrief is a derived synthesis by definition).
4789
+ try {
4790
+ const storedTexts = facts.map((f) => f.text);
4791
+ const debriefItems = await extractDebrief(evt.messages, storedTexts);
4792
+ if (debriefItems.length > 0) {
4793
+ const debriefFacts: ExtractedFact[] = debriefItems.map((d) => ({
4794
+ text: d.text,
4795
+ type: 'summary' as MemoryType,
4796
+ source: 'derived' as MemorySource,
4797
+ importance: d.importance,
4798
+ action: 'ADD' as const,
4799
+ }));
4800
+ await storeExtractedFacts(debriefFacts, api.logger, 'openclaw_debrief');
4801
+ api.logger.info(`Session debrief: stored ${debriefItems.length} items`);
4802
+ }
4803
+ } catch (debriefErr: unknown) {
4804
+ api.logger.warn(`before_reset debrief failed: ${debriefErr instanceof Error ? debriefErr.message : String(debriefErr)}`);
4805
+ }
3242
4806
  } catch (err: unknown) {
3243
4807
  const message = err instanceof Error ? err.message : String(err);
3244
4808
  api.logger.warn(`before_reset extraction failed: ${message}`);