@totalreclaw/totalreclaw 1.5.0 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.ts CHANGED
@@ -8,14 +8,17 @@
8
8
  * - totalreclaw_export -- export all memories (JSON or Markdown)
9
9
  * - totalreclaw_status -- check billing/subscription status
10
10
  * - totalreclaw_consolidate -- scan and merge near-duplicate memories
11
+ * - totalreclaw_pin -- pin a memory so auto-resolution can never supersede it
12
+ * - totalreclaw_unpin -- remove a pin, returning the memory to active status
11
13
  * - totalreclaw_import_from -- import memories from other tools (Mem0, MCP Memory, etc.)
12
14
  * - totalreclaw_upgrade -- create Stripe checkout for Pro upgrade
13
15
  * - totalreclaw_migrate -- migrate testnet memories to mainnet after Pro upgrade
16
+ * - totalreclaw_setup -- initialize with recovery phrase (no gateway restart needed)
14
17
  *
15
18
  * Also registers a `before_agent_start` hook that automatically injects
16
19
  * relevant memories into the agent's context.
17
20
  *
18
- * All data is encrypted client-side with AES-256-GCM. The server never
21
+ * All data is encrypted client-side with XChaCha20-Poly1305. The server never
19
22
  * sees plaintext.
20
23
  */
21
24
 
@@ -29,8 +32,24 @@ import {
29
32
  generateContentFingerprint,
30
33
  } from './crypto.js';
31
34
  import { createApiClient, type StoreFactPayload } from './api-client.js';
32
- import { extractFacts, type ExtractedFact } from './extractor.js';
33
- import { initLLMClient, generateEmbedding, getEmbeddingDims } from './llm-client.js';
35
+ import {
36
+ extractFacts,
37
+ extractDebrief,
38
+ isValidMemoryType,
39
+ parseEntity,
40
+ VALID_MEMORY_TYPES,
41
+ LEGACY_V0_MEMORY_TYPES,
42
+ VALID_MEMORY_SOURCES,
43
+ VALID_MEMORY_SCOPES,
44
+ EXTRACTION_SYSTEM_PROMPT,
45
+ extractFactsForCompaction,
46
+ type ExtractedFact,
47
+ type ExtractedEntity,
48
+ type MemoryType,
49
+ type MemorySource,
50
+ type MemoryScope,
51
+ } from './extractor.js';
52
+ import { initLLMClient, resolveLLMConfig, chatCompletion, generateEmbedding, getEmbeddingDims } from './llm-client.js';
34
53
  import { LSHHasher } from './lsh.js';
35
54
  import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS, type RerankerCandidate } from './reranker.js';
36
55
  import { deduplicateBatch } from './semantic-dedup.js';
@@ -43,9 +62,39 @@ import {
43
62
  STORE_DEDUP_MAX_CANDIDATES,
44
63
  type DecryptedCandidate,
45
64
  } from './consolidation.js';
46
- import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, type FactPayload } from './subgraph-store.js';
47
- import { searchSubgraph, getSubgraphFactCount } from './subgraph-search.js';
65
+ import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4, type FactPayload } from './subgraph-store.js';
66
+ import {
67
+ DIGEST_TRAPDOOR,
68
+ buildCanonicalClaim,
69
+ computeEntityTrapdoor,
70
+ computeEntityTrapdoors,
71
+ isDigestBlob,
72
+ normalizeToV1Type,
73
+ readClaimFromBlob,
74
+ resolveDigestMode,
75
+ type DigestMode,
76
+ } from './claims-helper.js';
77
+ import {
78
+ maybeInjectDigest,
79
+ recompileDigest,
80
+ fetchAllActiveClaims,
81
+ isRecompileInProgress,
82
+ tryBeginRecompile,
83
+ endRecompile,
84
+ } from './digest-sync.js';
85
+ import {
86
+ detectAndResolveContradictions,
87
+ runWeightTuningLoop,
88
+ type ResolutionDecision as ContradictionDecision,
89
+ } from './contradiction-sync.js';
90
+ import { searchSubgraph, searchSubgraphBroadened, getSubgraphFactCount, fetchFactById } from './subgraph-search.js';
91
+ import {
92
+ executePinOperation,
93
+ validatePinArgs,
94
+ type PinOpDeps,
95
+ } from './pin.js';
48
96
  import { PluginHotCache, type HotFact } from './hot-cache-wrapper.js';
97
+ import { CONFIG, setRecoveryPhraseOverride, setChainIdOverride } from './config.js';
49
98
  import crypto from 'node:crypto';
50
99
  import fs from 'node:fs';
51
100
  import path from 'node:path';
@@ -68,6 +117,16 @@ interface OpenClawPluginApi {
68
117
  };
69
118
  };
70
119
  };
120
+ models?: {
121
+ providers?: Record<string, {
122
+ baseUrl: string;
123
+ apiKey?: string;
124
+ api?: string;
125
+ models?: Array<{ id: string; [k: string]: unknown }>;
126
+ [k: string]: unknown;
127
+ }>;
128
+ [k: string]: unknown;
129
+ };
71
130
  [key: string]: unknown;
72
131
  };
73
132
  pluginConfig?: Record<string, unknown>;
@@ -76,12 +135,44 @@ interface OpenClawPluginApi {
76
135
  on(hookName: string, handler: (...args: unknown[]) => unknown, opts?: { priority?: number }): void;
77
136
  }
78
137
 
138
+ // ---------------------------------------------------------------------------
139
+ // Human-friendly error messages
140
+ // ---------------------------------------------------------------------------
141
+
142
+ /**
143
+ * Translate technical error messages from the on-chain submission pipeline
144
+ * into user-friendly messages. The original technical details are still
145
+ * logged via api.logger — this only affects what the agent sees.
146
+ */
147
+ function humanizeError(rawMessage: string): string {
148
+ if (rawMessage.includes('AA23')) {
149
+ return 'Memory storage temporarily unavailable. Will retry next time.';
150
+ }
151
+ if (rawMessage.includes('AA10')) {
152
+ return 'Please wait a moment before storing more memories.';
153
+ }
154
+ if (rawMessage.includes('AA25')) {
155
+ return 'Memory storage busy. Will retry.';
156
+ }
157
+ if (rawMessage.includes('pm_sponsorUserOperation')) {
158
+ return 'Memory storage service temporarily unavailable.';
159
+ }
160
+ if (/Relay returned HTTP\s*404/.test(rawMessage)) {
161
+ return 'Memory service is temporarily offline.';
162
+ }
163
+ if (/Relay returned HTTP\s*5\d\d/.test(rawMessage)) {
164
+ return 'Memory service encountered a temporary error. Will retry next time.';
165
+ }
166
+ // Pass through non-technical messages as-is.
167
+ return rawMessage;
168
+ }
169
+
79
170
  // ---------------------------------------------------------------------------
80
171
  // Persistent credential storage
81
172
  // ---------------------------------------------------------------------------
82
173
 
83
174
  /** Path where we persist userId + salt across restarts. */
84
- const CREDENTIALS_PATH = process.env.TOTALRECLAW_CREDENTIALS_PATH || `${process.env.HOME ?? '/home/node'}/.totalreclaw/credentials.json`;
175
+ const CREDENTIALS_PATH = CONFIG.credentialsPath;
85
176
 
86
177
  // ---------------------------------------------------------------------------
87
178
  // Cosine similarity threshold — skip injection when top result is below this
@@ -92,12 +183,10 @@ const CREDENTIALS_PATH = process.env.TOTALRECLAW_CREDENTIALS_PATH || `${process.
92
183
  * memories into context. Below this threshold, the query is considered
93
184
  * irrelevant to any stored memories and results are suppressed.
94
185
  *
95
- * Default 0.15 is tuned for bge-small-en-v1.5 which produces lower
186
+ * Default 0.15 is tuned for local ONNX models which produce lower
96
187
  * similarity scores than OpenAI models. Configurable via env var.
97
188
  */
98
- const COSINE_THRESHOLD = parseFloat(
99
- process.env.TOTALRECLAW_COSINE_THRESHOLD ?? '0.15',
100
- );
189
+ const COSINE_THRESHOLD = CONFIG.cosineThreshold;
101
190
 
102
191
  // ---------------------------------------------------------------------------
103
192
  // Module-level state (persists across tool calls within a session)
@@ -123,30 +212,36 @@ let lastSearchTimestamp = 0;
123
212
  let lastQueryEmbedding: number[] | null = null;
124
213
 
125
214
  // Feature flags — configurable for A/B testing
126
- const CACHE_TTL_MS = parseInt(process.env.TOTALRECLAW_CACHE_TTL_MS ?? String(5 * 60 * 1000), 10);
127
- const SEMANTIC_SKIP_THRESHOLD = parseFloat(process.env.TOTALRECLAW_SEMANTIC_SKIP_THRESHOLD ?? '0.85');
215
+ const CACHE_TTL_MS = CONFIG.cacheTtlMs;
216
+ const SEMANTIC_SKIP_THRESHOLD = CONFIG.semanticSkipThreshold;
128
217
 
129
218
  // Auto-extract throttle (C3): only extract every N turns in agent_end hook
130
219
  let turnsSinceLastExtraction = 0;
131
- const AUTO_EXTRACT_EVERY_TURNS_ENV = parseInt(process.env.TOTALRECLAW_EXTRACT_EVERY_TURNS ?? '3', 10);
220
+
221
+ // BUG-2 fix: Skip agent_end extraction during import operations.
222
+ // Import failures previously triggered agent_end → re-extraction → re-import loops.
223
+ let _importInProgress = false;
224
+ const AUTO_EXTRACT_EVERY_TURNS_ENV = CONFIG.extractInterval;
132
225
 
133
226
  // Hard cap on facts per extraction to prevent LLM over-extraction from dense conversations
134
227
  const MAX_FACTS_PER_EXTRACTION = 15;
135
228
 
136
- // Store-time near-duplicate detection (consolidation module)
137
- const STORE_DEDUP_ENABLED = process.env.TOTALRECLAW_STORE_DEDUP !== 'false';
229
+ // Store-time near-duplicate detection is always ON in v1.
230
+ // The TOTALRECLAW_STORE_DEDUP env var was removed.
231
+ const STORE_DEDUP_ENABLED = true;
138
232
 
139
233
  // One-time welcome-back message for returning Pro users (set during init, consumed by first before_agent_start)
140
234
  let welcomeBackMessage: string | null = null;
141
235
 
142
- // B2: Minimum relevance threshold cosine below this means no memory injection
143
- const RELEVANCE_THRESHOLD = parseFloat(process.env.TOTALRECLAW_RELEVANCE_THRESHOLD ?? '0.3');
236
+ // B2: COSINE_THRESHOLD (above) is the single relevance gate for both
237
+ // the before_agent_start hook and the recall tool. The former "RELEVANCE_THRESHOLD"
238
+ // (0.3) was too aggressive and silently suppressed auto-recall at session start.
144
239
 
145
240
  // ---------------------------------------------------------------------------
146
241
  // Billing cache infrastructure
147
242
  // ---------------------------------------------------------------------------
148
243
 
149
- const BILLING_CACHE_PATH = path.join(process.env.HOME ?? '/home/node', '.totalreclaw', 'billing-cache.json');
244
+ const BILLING_CACHE_PATH = CONFIG.billingCachePath;
150
245
  const BILLING_CACHE_TTL = 2 * 60 * 60 * 1000; // 2 hours
151
246
  const QUOTA_WARNING_THRESHOLD = 0.8; // 80%
152
247
 
@@ -165,11 +260,34 @@ interface BillingCache {
165
260
  checked_at: number;
166
261
  }
167
262
 
263
+ /**
264
+ * Apply the billing tier to the runtime chain override.
265
+ *
266
+ * Pro tier → chain 100 (Gnosis mainnet). Free tier (or unknown) stays on
267
+ * 84532 (Base Sepolia). The relay routes Pro UserOps to Gnosis, so the
268
+ * client MUST sign them against chain 100 — otherwise the bundler returns
269
+ * AA23 (invalid signature). See MCP's equivalent path in mcp/src/index.ts.
270
+ *
271
+ * Called from `readBillingCache` and `writeBillingCache` so that every cache
272
+ * read or write keeps the chain override in sync with the cached tier.
273
+ * Idempotent — calling with the same tier is a no-op.
274
+ */
275
+ function syncChainIdFromTier(tier: string | undefined): void {
276
+ if (tier === 'pro') {
277
+ setChainIdOverride(100);
278
+ } else {
279
+ // Free or unknown → reset to the default free-tier chain.
280
+ setChainIdOverride(84532);
281
+ }
282
+ }
283
+
168
284
  function readBillingCache(): BillingCache | null {
169
285
  try {
170
286
  if (!fs.existsSync(BILLING_CACHE_PATH)) return null;
171
287
  const raw = JSON.parse(fs.readFileSync(BILLING_CACHE_PATH, 'utf-8')) as BillingCache;
172
288
  if (!raw.checked_at || Date.now() - raw.checked_at > BILLING_CACHE_TTL) return null;
289
+ // Keep chain override in sync with persisted tier across process restarts.
290
+ syncChainIdFromTier(raw.tier);
173
291
  return raw;
174
292
  } catch {
175
293
  return null;
@@ -184,18 +302,22 @@ function writeBillingCache(cache: BillingCache): void {
184
302
  } catch {
185
303
  // Best-effort — don't block on cache write failure.
186
304
  }
305
+ // Sync chain override AFTER the write so in-process UserOp signing picks
306
+ // up the correct chain immediately, even if the disk write failed.
307
+ syncChainIdFromTier(cache.tier);
187
308
  }
188
309
 
189
310
  /**
190
- * Check if LLM-guided dedup is enabled for the current tier.
191
- * Returns true for Pro users, or when no billing cache exists (fail-open for self-hosters).
311
+ * Check if LLM-guided dedup is enabled.
312
+ *
313
+ * Always returns true — LLM extraction runs client-side using the user's
314
+ * own API key, so there is no cost to us. The server flag is respected as
315
+ * a kill-switch but defaults to true for all tiers.
192
316
  */
193
317
  function isLlmDedupEnabled(): boolean {
194
318
  const cache = readBillingCache();
195
- if (!cache) return true;
196
- if (cache.tier === 'pro') return true;
197
- if (cache.features?.llm_dedup !== undefined) return cache.features.llm_dedup;
198
- return false;
319
+ if (cache?.features?.llm_dedup === false) return false; // Server kill-switch
320
+ return true;
199
321
  }
200
322
 
201
323
  /**
@@ -241,7 +363,7 @@ const MEMORY_HEADER = `# Memory
241
363
 
242
364
  function ensureMemoryHeader(logger: OpenClawPluginApi['logger']): void {
243
365
  try {
244
- const workspace = path.join(process.env.HOME ?? '/home/node', '.openclaw', 'workspace');
366
+ const workspace = CONFIG.openclawWorkspace;
245
367
  const memoryMd = path.join(workspace, 'MEMORY.md');
246
368
 
247
369
  if (fs.existsSync(memoryMd)) {
@@ -332,14 +454,16 @@ async function getFactCount(logger: OpenClawPluginApi['logger']): Promise<number
332
454
  /** True when recovery phrase is missing — tools return setup instructions. */
333
455
  let needsSetup = false;
334
456
 
457
+ /** True on first before_agent_start after successful init — show welcome message once. */
458
+ let firstRunAfterInit = true;
459
+
335
460
  /**
336
461
  * Derive keys from the recovery phrase, load or create credentials, and
337
462
  * register with the server if this is the first run.
338
463
  */
339
464
  async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
340
- const serverUrl =
341
- process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz';
342
- const masterPassword = process.env.TOTALRECLAW_RECOVERY_PHRASE;
465
+ const serverUrl = CONFIG.serverUrl || 'https://api.totalreclaw.xyz';
466
+ const masterPassword = CONFIG.recoveryPhrase;
343
467
 
344
468
  if (!masterPassword) {
345
469
  needsSetup = true;
@@ -356,7 +480,14 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
356
480
  try {
357
481
  if (fs.existsSync(CREDENTIALS_PATH)) {
358
482
  const creds = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8'));
359
- existingSalt = Buffer.from(creds.salt, 'base64');
483
+ // Salt may be stored as base64 (plugin-written) or hex (MCP setup-written).
484
+ // Detect format: hex strings are 64 chars of [0-9a-f], base64 uses [A-Z+/=].
485
+ const saltStr: string = creds.salt;
486
+ if (saltStr && /^[0-9a-f]{64}$/i.test(saltStr)) {
487
+ existingSalt = Buffer.from(saltStr, 'hex');
488
+ } else if (saltStr) {
489
+ existingSalt = Buffer.from(saltStr, 'base64');
490
+ }
360
491
  existingUserId = creds.userId;
361
492
  logger.info(`Loaded existing credentials for user ${existingUserId}`);
362
493
  }
@@ -377,6 +508,20 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
377
508
  if (existingUserId) {
378
509
  userId = existingUserId;
379
510
  logger.info(`Authenticated as user ${userId}`);
511
+
512
+ // Idempotent registration — ensure auth key is registered with the relay.
513
+ // Without this, returning users get 401 if the relay database was reset or
514
+ // if credentials were created by the MCP setup CLI (different process).
515
+ try {
516
+ const authHash = computeAuthKeyHash(keys.authKey);
517
+ const saltHex = keys.salt.toString('hex');
518
+ await apiClient.register(authHash, saltHex);
519
+ } catch {
520
+ // Best-effort — relay returns 200 for already-registered users.
521
+ // Only fails on network errors; bearer token auth still works if
522
+ // a prior registration succeeded.
523
+ logger.warn('Idempotent relay registration failed (best-effort, will retry on next start)');
524
+ }
380
525
  } else {
381
526
  // First run -- register with the server.
382
527
  const authHash = computeAuthKeyHash(keys.authKey);
@@ -402,14 +547,20 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
402
547
  userId = registeredUserId!;
403
548
 
404
549
  // Persist credentials so we can resume later.
550
+ // Include the mnemonic so hot-reload works without env var.
405
551
  const dir = path.dirname(CREDENTIALS_PATH);
406
552
  if (!fs.existsSync(dir)) {
407
553
  fs.mkdirSync(dir, { recursive: true });
408
554
  }
409
- fs.writeFileSync(
410
- CREDENTIALS_PATH,
411
- JSON.stringify({ userId, salt: keys.salt.toString('base64') }),
412
- );
555
+ const credsToSave: Record<string, string> = {
556
+ userId,
557
+ salt: keys.salt.toString('base64'),
558
+ };
559
+ // Only persist mnemonic if we have one (avoid writing empty string).
560
+ if (masterPassword) {
561
+ credsToSave.mnemonic = masterPassword;
562
+ }
563
+ fs.writeFileSync(CREDENTIALS_PATH, JSON.stringify(credsToSave), { mode: 0o600 });
413
564
 
414
565
  logger.info(`Registered new user: ${userId}`);
415
566
  }
@@ -433,7 +584,7 @@ async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
433
584
  try {
434
585
  const walletAddr = subgraphOwner || userId || '';
435
586
  if (walletAddr) {
436
- const billingUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
587
+ const billingUrl = CONFIG.serverUrl;
437
588
  const resp = await fetch(`${billingUrl}/v1/billing/status?wallet_address=${encodeURIComponent(walletAddr)}`, {
438
589
  method: 'GET',
439
590
  headers: {
@@ -476,6 +627,13 @@ function isDocker(): boolean {
476
627
  }
477
628
 
478
629
  function buildSetupErrorMsg(): string {
630
+ return 'TotalReclaw setup required. Use the `totalreclaw_setup` tool with a 12-word BIP-39 recovery phrase.\n\n' +
631
+ '1. Ask the user if they have an existing recovery phrase, or generate a new one with `npx @totalreclaw/mcp-server setup`.\n' +
632
+ '2. Call `totalreclaw_setup` with the phrase — no gateway restart needed.\n' +
633
+ ' (Optional: set TOTALRECLAW_SELF_HOSTED=true if using your own server instead of the managed service.)';
634
+ }
635
+
636
+ function buildSetupErrorMsgLegacy(): string {
479
637
  const base =
480
638
  'TotalReclaw setup required:\n' +
481
639
  '1. Set TOTALRECLAW_RECOVERY_PHRASE — ask the user if they have an existing recovery phrase or generate a new 12-word BIP-39 mnemonic.\n' +
@@ -506,12 +664,101 @@ const SETUP_ERROR_MSG = buildSetupErrorMsg();
506
664
 
507
665
  /**
508
666
  * Ensure `initialize()` has completed (runs at most once).
667
+ *
668
+ * If `needsSetup` is true after init, attempts a hot-reload from
669
+ * credentials.json in case the mnemonic was written there by a
670
+ * `totalreclaw_setup` tool call or `npx @totalreclaw/mcp-server setup`.
509
671
  */
510
672
  async function ensureInitialized(logger: OpenClawPluginApi['logger']): Promise<void> {
511
673
  if (!initPromise) {
512
674
  initPromise = initialize(logger);
513
675
  }
514
676
  await initPromise;
677
+
678
+ // Hot-reload: if setup is still needed, check if credentials.json
679
+ // now has a mnemonic (written by totalreclaw_setup or MCP setup CLI).
680
+ if (needsSetup) {
681
+ await attemptHotReload(logger);
682
+ }
683
+ }
684
+
685
+ /**
686
+ * Attempt to hot-reload credentials from credentials.json.
687
+ *
688
+ * Called when `needsSetup` is true — checks if credentials.json contains
689
+ * a mnemonic (written by the `totalreclaw_setup` tool or MCP setup CLI).
690
+ * If found, re-derives keys and completes initialization without requiring
691
+ * a gateway restart.
692
+ */
693
+ async function attemptHotReload(logger: OpenClawPluginApi['logger']): Promise<void> {
694
+ try {
695
+ if (!fs.existsSync(CREDENTIALS_PATH)) return;
696
+
697
+ const creds = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8'));
698
+ if (!creds.mnemonic) return;
699
+
700
+ logger.info('Hot-reloading credentials from credentials.json (no restart needed)');
701
+
702
+ // Set the runtime override so CONFIG.recoveryPhrase returns the mnemonic.
703
+ setRecoveryPhraseOverride(creds.mnemonic);
704
+
705
+ // Re-run initialization with the newly available mnemonic.
706
+ needsSetup = false;
707
+ initPromise = initialize(logger);
708
+ await initPromise;
709
+ } catch (err) {
710
+ const msg = err instanceof Error ? err.message : String(err);
711
+ logger.warn(`Hot-reload from credentials.json failed: ${msg}`);
712
+ // Leave needsSetup as true — user will see the setup prompt.
713
+ }
714
+ }
715
+
716
+ /**
717
+ * Force re-initialization with a specific mnemonic.
718
+ *
719
+ * Called by the `totalreclaw_setup` tool. Clears stale credentials from
720
+ * disk so that `initialize()` treats this as a fresh registration and
721
+ * persists the NEW mnemonic + freshly derived salt/userId.
722
+ *
723
+ * Without clearing credentials.json first, `initialize()` would load the
724
+ * OLD salt and userId, derive keys from (new mnemonic + old salt), skip
725
+ * writing credentials (because existingUserId is set), and the new
726
+ * mnemonic would never be persisted — a critical data-loss bug.
727
+ */
728
+ async function forceReinitialization(mnemonic: string, logger: OpenClawPluginApi['logger']): Promise<void> {
729
+ // Set the runtime override so CONFIG.recoveryPhrase returns this mnemonic.
730
+ setRecoveryPhraseOverride(mnemonic);
731
+
732
+ // CRITICAL: Remove stale credentials so initialize() does a fresh
733
+ // registration with a new salt. If we leave the old file, initialize()
734
+ // loads the old salt + userId and never writes the new mnemonic.
735
+ try {
736
+ if (fs.existsSync(CREDENTIALS_PATH)) {
737
+ fs.unlinkSync(CREDENTIALS_PATH);
738
+ logger.info('Cleared stale credentials.json for fresh setup');
739
+ }
740
+ } catch (err) {
741
+ logger.warn(`Could not remove old credentials.json: ${err instanceof Error ? err.message : String(err)}`);
742
+ }
743
+
744
+ // Reset module state for a clean re-init.
745
+ needsSetup = false;
746
+ authKeyHex = null;
747
+ encryptionKey = null;
748
+ dedupKey = null;
749
+ userId = null;
750
+ subgraphOwner = null;
751
+ apiClient = null;
752
+ lshHasher = null;
753
+ lshInitFailed = false;
754
+ masterPasswordCache = null;
755
+ saltCache = null;
756
+ pluginHotCache = null;
757
+ firstRunAfterInit = true;
758
+
759
+ // Re-run initialization — will register fresh and persist new credentials.
760
+ initPromise = initialize(logger);
761
+ await initPromise;
515
762
  }
516
763
 
517
764
  /**
@@ -631,7 +878,8 @@ async function searchForNearDuplicates(
631
878
  for (const result of results) {
632
879
  try {
633
880
  const docJson = decryptFromHex(result.encryptedBlob, encryptionKey);
634
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
881
+ if (isDigestBlob(docJson)) continue;
882
+ const doc = readClaimFromBlob(docJson);
635
883
 
636
884
  let embedding: number[] | null = null;
637
885
  if (result.encryptedEmbedding) {
@@ -644,9 +892,7 @@ async function searchForNearDuplicates(
644
892
  id: result.id,
645
893
  text: doc.text,
646
894
  embedding,
647
- importance: doc.metadata?.importance
648
- ? Math.round((doc.metadata.importance as number) * 10)
649
- : 5,
895
+ importance: doc.importance,
650
896
  decayScore: 5,
651
897
  createdAt: result.timestamp ? parseInt(result.timestamp, 10) * 1000 : Date.now(),
652
898
  version: 1,
@@ -663,7 +909,8 @@ async function searchForNearDuplicates(
663
909
  for (const candidate of candidates) {
664
910
  try {
665
911
  const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey);
666
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
912
+ if (isDigestBlob(docJson)) continue;
913
+ const doc = readClaimFromBlob(docJson);
667
914
 
668
915
  let embedding: number[] | null = null;
669
916
  if (candidate.encrypted_embedding) {
@@ -676,9 +923,7 @@ async function searchForNearDuplicates(
676
923
  id: candidate.fact_id,
677
924
  text: doc.text,
678
925
  embedding,
679
- importance: doc.metadata?.importance
680
- ? Math.round((doc.metadata.importance as number) * 10)
681
- : 5,
926
+ importance: doc.importance,
682
927
  decayScore: candidate.decay_score,
683
928
  createdAt: typeof candidate.timestamp === 'number'
684
929
  ? candidate.timestamp
@@ -717,6 +962,182 @@ function encryptToHex(plaintext: string, key: Buffer): string {
717
962
  return Buffer.from(b64, 'base64').toString('hex');
718
963
  }
719
964
 
965
+ // Plugin v3.0.0 removed the legacy claim-format fallback. Write path
966
+ // always emits Memory Taxonomy v1 JSON blobs. The logClaimFormatOnce
967
+ // helper is gone along with TOTALRECLAW_CLAIM_FORMAT / TOTALRECLAW_TAXONOMY_VERSION.
968
+
969
+ let _loggedDigestMode = false;
970
+ function logDigestModeOnce(mode: DigestMode, logger: OpenClawPluginApi['logger']): void {
971
+ if (_loggedDigestMode) return;
972
+ _loggedDigestMode = true;
973
+ logger.info(`TotalReclaw: digest injection mode = ${mode}`);
974
+ }
975
+
976
+ /**
977
+ * How many active facts to pull into a digest recompilation.
978
+ * Digest compiler itself will apply DIGEST_CLAIM_CAP for the LLM path.
979
+ */
980
+ const DIGEST_FETCH_LIMIT = 500;
981
+
982
+ /**
983
+ * Schedule a background digest recompile. Fire-and-forget.
984
+ *
985
+ * The caller must check `!isRecompileInProgress()` before invoking.
986
+ * Errors are logged and swallowed; the guard flag is always released.
987
+ */
988
+ function scheduleDigestRecompile(
989
+ previousClaimId: string | null,
990
+ logger: OpenClawPluginApi['logger'],
991
+ ): void {
992
+ if (!isRecompileInProgress()) {
993
+ if (!tryBeginRecompile()) return;
994
+ } else {
995
+ return;
996
+ }
997
+
998
+ const mode = resolveDigestMode();
999
+ const owner = subgraphOwner || userId;
1000
+ const authKey = authKeyHex;
1001
+ const encKey = encryptionKey;
1002
+ const ownerForBatch = subgraphOwner ?? undefined;
1003
+
1004
+ if (!owner || !authKey || !encKey) {
1005
+ endRecompile();
1006
+ return;
1007
+ }
1008
+
1009
+ // Capture llmFn from the current LLM config (cheap variant of the user's
1010
+ // provider, already resolved by resolveLLMConfig).
1011
+ const llmConfig = resolveLLMConfig();
1012
+ const llmFn = llmConfig
1013
+ ? async (prompt: string): Promise<string> => {
1014
+ const out = await chatCompletion(
1015
+ llmConfig,
1016
+ [
1017
+ { role: 'system', content: 'You return only valid JSON. No markdown fences, no commentary.' },
1018
+ { role: 'user', content: prompt },
1019
+ ],
1020
+ { maxTokens: 800, temperature: 0 },
1021
+ );
1022
+ return out ?? '';
1023
+ }
1024
+ : null;
1025
+
1026
+ // Build the I/O deps closures. We capture the owner/auth/key values so the
1027
+ // background task doesn't race with module-level state resets.
1028
+ const fetchFn = () =>
1029
+ fetchAllActiveClaims(
1030
+ owner,
1031
+ authKey,
1032
+ encKey,
1033
+ DIGEST_FETCH_LIMIT,
1034
+ {
1035
+ searchSubgraphBroadened: async (o, n, a) => searchSubgraphBroadened(o, n, a),
1036
+ decryptFromHex: (hex, key) => decryptFromHex(hex, key),
1037
+ },
1038
+ logger,
1039
+ );
1040
+
1041
+ const storeFn = async (canonicalClaimJson: string, compiledAt: string): Promise<void> => {
1042
+ if (!isSubgraphMode()) {
1043
+ // Self-hosted mode — store via the REST API.
1044
+ if (!apiClient) throw new Error('apiClient not initialized');
1045
+ const encryptedBlob = encryptToHex(canonicalClaimJson, encKey);
1046
+ const contentFp = generateContentFingerprint(canonicalClaimJson, dedupKey!);
1047
+ const payload: StoreFactPayload = {
1048
+ id: crypto.randomUUID(),
1049
+ timestamp: compiledAt,
1050
+ encrypted_blob: encryptedBlob,
1051
+ blind_indices: [DIGEST_TRAPDOOR],
1052
+ decay_score: 10,
1053
+ source: 'openclaw-plugin-digest',
1054
+ content_fp: contentFp,
1055
+ agent_id: 'openclaw-plugin-digest',
1056
+ };
1057
+ await apiClient.store(userId!, [payload], authKey);
1058
+ return;
1059
+ }
1060
+
1061
+ // Subgraph / managed-service mode — encrypt, encode, submit as a single-fact UserOp.
1062
+ const encryptedBlob = encryptToHex(canonicalClaimJson, encKey);
1063
+ const contentFp = generateContentFingerprint(canonicalClaimJson, dedupKey!);
1064
+ const protobuf = encodeFactProtobuf({
1065
+ id: crypto.randomUUID(),
1066
+ timestamp: compiledAt,
1067
+ owner,
1068
+ encryptedBlob,
1069
+ blindIndices: [DIGEST_TRAPDOOR],
1070
+ decayScore: 10,
1071
+ source: 'openclaw-plugin-digest',
1072
+ contentFp,
1073
+ agentId: 'openclaw-plugin-digest',
1074
+ version: PROTOBUF_VERSION_V4,
1075
+ });
1076
+ const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
1077
+ const result = await submitFactBatchOnChain([protobuf], config);
1078
+ if (!result.success) {
1079
+ throw new Error('Digest store UserOp did not succeed on-chain');
1080
+ }
1081
+ };
1082
+
1083
+ const tombstoneFn = async (claimId: string): Promise<void> => {
1084
+ if (!isSubgraphMode()) {
1085
+ if (apiClient) {
1086
+ try { await apiClient.deleteFact(claimId, authKey); } catch { /* best-effort */ }
1087
+ }
1088
+ return;
1089
+ }
1090
+ const tombstone: FactPayload = {
1091
+ id: claimId,
1092
+ timestamp: new Date().toISOString(),
1093
+ owner,
1094
+ encryptedBlob: '00',
1095
+ blindIndices: [],
1096
+ decayScore: 0,
1097
+ source: 'tombstone',
1098
+ contentFp: '',
1099
+ agentId: 'openclaw-plugin-digest',
1100
+ version: PROTOBUF_VERSION_V4,
1101
+ };
1102
+ const protobuf = encodeFactProtobuf(tombstone);
1103
+ const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
1104
+ const result = await submitFactBatchOnChain([protobuf], config);
1105
+ if (!result.success) {
1106
+ throw new Error('Digest tombstone UserOp did not succeed on-chain');
1107
+ }
1108
+ };
1109
+
1110
+ // Slice 2f: run the weight-tuning loop as a fire-and-forget pre-compile step.
1111
+ // This consumes any feedback.jsonl entries written since the last compile
1112
+ // and nudges ~/.totalreclaw/weights.json, so the NEXT contradiction detection
1113
+ // uses the adjusted weights. Rate-limited and idempotent — see
1114
+ // runWeightTuningLoop for details. Failures are logged, never fatal.
1115
+ void runWeightTuningLoop(Math.floor(Date.now() / 1000), logger).catch((err: unknown) => {
1116
+ const msg = err instanceof Error ? err.message : String(err);
1117
+ logger.warn(`Digest: tuning loop threw: ${msg}`);
1118
+ });
1119
+
1120
+ void recompileDigest({
1121
+ mode,
1122
+ previousClaimId,
1123
+ nowUnixSeconds: Math.floor(Date.now() / 1000),
1124
+ deps: {
1125
+ storeDigestClaim: storeFn,
1126
+ tombstoneDigest: tombstoneFn,
1127
+ fetchAllActiveClaimsFn: fetchFn,
1128
+ llmFn,
1129
+ },
1130
+ logger,
1131
+ })
1132
+ .catch((err: unknown) => {
1133
+ const msg = err instanceof Error ? err.message : String(err);
1134
+ logger.warn(`Digest: background recompile threw: ${msg}`);
1135
+ })
1136
+ .finally(() => {
1137
+ endRecompile();
1138
+ });
1139
+ }
1140
+
720
1141
  /**
721
1142
  * Decrypt a hex-encoded ciphertext blob into a UTF-8 string.
722
1143
  */
@@ -906,7 +1327,8 @@ async function fetchExistingMemoriesForExtraction(
906
1327
  for (const r of rawResults) {
907
1328
  try {
908
1329
  const docJson = decryptFromHex(r.encryptedBlob, encryptionKey);
909
- const doc = JSON.parse(docJson) as { text: string };
1330
+ if (isDigestBlob(docJson)) continue;
1331
+ const doc = readClaimFromBlob(docJson);
910
1332
  results.push({ id: r.id, text: doc.text });
911
1333
  } catch { /* skip undecryptable */ }
912
1334
  }
@@ -915,7 +1337,8 @@ async function fetchExistingMemoriesForExtraction(
915
1337
  for (const c of candidates) {
916
1338
  try {
917
1339
  const docJson = decryptFromHex(c.encrypted_blob, encryptionKey);
918
- const doc = JSON.parse(docJson) as { text: string };
1340
+ if (isDigestBlob(docJson)) continue;
1341
+ const doc = readClaimFromBlob(docJson);
919
1342
  results.push({ id: c.fact_id, text: doc.text });
920
1343
  } catch { /* skip undecryptable */ }
921
1344
  }
@@ -972,10 +1395,7 @@ function relativeTime(isoOrMs: string | number): string {
972
1395
  * NOTE: This filter is ONLY applied to auto-extraction (hooks).
973
1396
  * The explicit `totalreclaw_remember` tool always stores regardless of importance.
974
1397
  */
975
- const MIN_IMPORTANCE_THRESHOLD = Math.max(
976
- 1,
977
- Math.min(10, Number(process.env.TOTALRECLAW_MIN_IMPORTANCE) || 3),
978
- );
1398
+ const MIN_IMPORTANCE_THRESHOLD = CONFIG.minImportance;
979
1399
 
980
1400
  /**
981
1401
  * Filter extracted facts by importance threshold.
@@ -998,10 +1418,20 @@ function filterByImportance(
998
1418
  }
999
1419
  }
1000
1420
 
1001
- if (dropped > 0) {
1421
+ // Phase 2.2.5: always log the filter outcome so the agent_end path can
1422
+ // distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
1423
+ // below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
1424
+ // this only logged on drops, which made empty-input invisible.
1425
+ if (facts.length === 0) {
1426
+ logger.info('Importance filter: input=0 (nothing to filter)');
1427
+ } else if (dropped > 0) {
1002
1428
  logger.info(
1003
1429
  `Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`,
1004
1430
  );
1431
+ } else {
1432
+ logger.info(
1433
+ `Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`,
1434
+ );
1005
1435
  }
1006
1436
 
1007
1437
  return { kept, dropped };
@@ -1023,6 +1453,7 @@ function filterByImportance(
1023
1453
  async function storeExtractedFacts(
1024
1454
  facts: ExtractedFact[],
1025
1455
  logger: OpenClawPluginApi['logger'],
1456
+ sourceOverride?: string,
1026
1457
  ): Promise<number> {
1027
1458
  if (!encryptionKey || !dedupKey || !authKeyHex || !userId || !apiClient) return 0;
1028
1459
 
@@ -1060,18 +1491,24 @@ async function storeExtractedFacts(
1060
1491
  let stored = 0;
1061
1492
  let superseded = 0;
1062
1493
  let skipped = 0;
1494
+ let failedFacts = 0;
1063
1495
  const pendingPayloads: Buffer[] = []; // Batched subgraph payloads
1064
1496
  let preparedForSubgraph = 0;
1065
1497
 
1498
+ // Plugin v3.0.0: always emit Memory Taxonomy v1 JSON blobs. The
1499
+ // TOTALRECLAW_TAXONOMY_VERSION opt-in and the TOTALRECLAW_CLAIM_FORMAT
1500
+ // legacy fallback have both been retired — v1 is the single write path.
1501
+
1066
1502
  for (const fact of dedupedFacts) {
1067
1503
  try {
1068
1504
  const blindIndices = generateBlindIndices(fact.text);
1505
+ const entityTrapdoors = computeEntityTrapdoors(fact.entities);
1069
1506
 
1070
1507
  // Use pre-computed embedding result if available.
1071
1508
  const embeddingResult = embeddingResultMap.get(fact.text) ?? null;
1072
1509
  const allIndices = embeddingResult
1073
- ? [...blindIndices, ...embeddingResult.lshBuckets]
1074
- : blindIndices;
1510
+ ? [...blindIndices, ...embeddingResult.lshBuckets, ...entityTrapdoors]
1511
+ : [...blindIndices, ...entityTrapdoors];
1075
1512
 
1076
1513
  // LLM-guided dedup: handle UPDATE/DELETE/NOOP actions.
1077
1514
  if (fact.action === 'NOOP') {
@@ -1093,6 +1530,7 @@ async function storeExtractedFacts(
1093
1530
  source: 'tombstone',
1094
1531
  contentFp: '',
1095
1532
  agentId: 'openclaw-plugin-auto',
1533
+ version: PROTOBUF_VERSION_V4,
1096
1534
  };
1097
1535
  pendingPayloads.push(encodeFactProtobuf(tombstone));
1098
1536
  logger.info(`LLM dedup: DELETE — queued tombstone for ${fact.existingFactId}`);
@@ -1121,6 +1559,7 @@ async function storeExtractedFacts(
1121
1559
  source: 'tombstone',
1122
1560
  contentFp: '',
1123
1561
  agentId: 'openclaw-plugin-auto',
1562
+ version: PROTOBUF_VERSION_V4,
1124
1563
  };
1125
1564
  pendingPayloads.push(encodeFactProtobuf(tombstone));
1126
1565
  logger.info(`LLM dedup: UPDATE — queued tombstone for ${fact.existingFactId}, storing replacement`);
@@ -1171,6 +1610,7 @@ async function storeExtractedFacts(
1171
1610
  source: 'tombstone',
1172
1611
  contentFp: '',
1173
1612
  agentId: 'openclaw-plugin-auto',
1613
+ version: PROTOBUF_VERSION_V4,
1174
1614
  };
1175
1615
  pendingPayloads.push(encodeFactProtobuf(tombstone));
1176
1616
  logger.info(
@@ -1193,20 +1633,133 @@ async function storeExtractedFacts(
1193
1633
  }
1194
1634
  }
1195
1635
 
1196
- const doc = {
1197
- text: fact.text,
1198
- metadata: {
1199
- type: fact.type,
1200
- importance: effectiveImportance / 10,
1201
- source: 'auto-extraction',
1202
- created_at: new Date().toISOString(),
1203
- },
1204
- };
1636
+ const factSource = sourceOverride || 'auto-extraction';
1637
+
1638
+ // Plugin v3.0.0: always build a Memory Taxonomy v1 JSON blob. The
1639
+ // blob is decryptable by `readClaimFromBlob` which prefers v1 →
1640
+ // falls back to v0 short-key → then plugin-legacy {text, metadata}
1641
+ // for pre-v3 vault entries.
1642
+ //
1643
+ // We build it BEFORE the on-chain write so Phase 2 contradiction
1644
+ // detection can inspect the same canonical Claim the write path will
1645
+ // actually store. The string is encrypted byte-identically below.
1646
+ //
1647
+ // Defensive: if the extraction hook didn't populate `fact.source`
1648
+ // (e.g. explicit tool path, legacy caller), default to 'user-inferred'
1649
+ // so v1 schema validation passes.
1650
+ const factForBlob: ExtractedFact = fact.source
1651
+ ? fact
1652
+ : { ...fact, source: 'user-inferred' };
1653
+ const blobPlaintext = buildCanonicalClaim({
1654
+ fact: factForBlob,
1655
+ importance: effectiveImportance,
1656
+ sourceAgent: factSource,
1657
+ });
1205
1658
 
1206
- const encryptedBlob = encryptToHex(JSON.stringify(doc), encryptionKey);
1659
+ const factId = crypto.randomUUID();
1660
+
1661
+ // Phase 2 Slice 2d: contradiction detection + auto-resolution.
1662
+ //
1663
+ // Runs only when the canonical Claim format is active (legacy blobs
1664
+ // carry no entity refs, so there is nothing to check), only for
1665
+ // Subgraph / managed-service mode (self-hosted contradiction handling
1666
+ // can come later), and only when the new fact has entities. The helper
1667
+ // is a no-op in all other cases.
1668
+ //
1669
+ // Returns one decision per candidate contradicting claim:
1670
+ // - supersede_existing → queue a tombstone + proceed with the new write
1671
+ // - skip_new → do not write the new fact; record the skip reason
1672
+ // - empty list → no contradiction, proceed unchanged
1673
+ //
1674
+ // On any error (subgraph, decrypt, WASM), the helper returns [] and we
1675
+ // fall back to Phase 1 behaviour.
1676
+ let contradictionSkipNew = false;
1677
+ if (
1678
+ isSubgraphMode() &&
1679
+ fact.entities &&
1680
+ fact.entities.length > 0 &&
1681
+ embeddingResult
1682
+ ) {
1683
+ const newClaimObj = JSON.parse(blobPlaintext) as Record<string, unknown>;
1684
+ let decisions: ContradictionDecision[] = [];
1685
+ try {
1686
+ decisions = await detectAndResolveContradictions({
1687
+ newClaim: newClaimObj,
1688
+ newClaimId: factId,
1689
+ newEmbedding: embeddingResult.embedding,
1690
+ subgraphOwner: subgraphOwner || userId!,
1691
+ authKeyHex: authKeyHex!,
1692
+ encryptionKey: encryptionKey!,
1693
+ deps: {
1694
+ searchSubgraph: (owner, trapdoors, maxCandidates, authKey) =>
1695
+ searchSubgraph(owner, trapdoors, maxCandidates, authKey).then((rows) =>
1696
+ rows.map((r) => ({
1697
+ id: r.id,
1698
+ encryptedBlob: r.encryptedBlob,
1699
+ encryptedEmbedding: r.encryptedEmbedding ?? null,
1700
+ timestamp: r.timestamp,
1701
+ isActive: r.isActive,
1702
+ })),
1703
+ ),
1704
+ decryptFromHex: (hex, key) => decryptFromHex(hex, key),
1705
+ },
1706
+ logger: {
1707
+ info: (m) => logger.info(m),
1708
+ warn: (m) => logger.warn(m),
1709
+ },
1710
+ });
1711
+ } catch (crErr) {
1712
+ // detectAndResolveContradictions is supposed to never throw — if
1713
+ // it does, we log and continue with Phase 1 behaviour.
1714
+ const msg = crErr instanceof Error ? crErr.message : String(crErr);
1715
+ logger.warn(`Contradiction detection failed (proceeding with store): ${msg}`);
1716
+ decisions = [];
1717
+ }
1718
+
1719
+ for (const decision of decisions) {
1720
+ if (decision.action === 'supersede_existing') {
1721
+ const tombstone: FactPayload = {
1722
+ id: decision.existingFactId,
1723
+ timestamp: new Date().toISOString(),
1724
+ owner: subgraphOwner || userId!,
1725
+ encryptedBlob: '00',
1726
+ blindIndices: [],
1727
+ decayScore: 0,
1728
+ source: 'tombstone',
1729
+ contentFp: '',
1730
+ agentId: 'openclaw-plugin-auto',
1731
+ version: PROTOBUF_VERSION_V4,
1732
+ };
1733
+ pendingPayloads.push(encodeFactProtobuf(tombstone));
1734
+ superseded++;
1735
+ logger.info(
1736
+ `Auto-resolve: queued supersede for ${decision.existingFactId.slice(0, 10)}… ` +
1737
+ `(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
1738
+ );
1739
+ } else if (decision.action === 'skip_new') {
1740
+ if (decision.reason === 'existing_pinned') {
1741
+ logger.warn(
1742
+ `Auto-resolve: skipped new write — existing claim ${decision.existingFactId.slice(0, 10)}… is pinned ` +
1743
+ `(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
1744
+ );
1745
+ } else {
1746
+ logger.info(
1747
+ `Auto-resolve: skipped new write — existing ${decision.existingFactId.slice(0, 10)}… wins ` +
1748
+ `(sim=${decision.similarity.toFixed(3)}, entity=${decision.entityId})`,
1749
+ );
1750
+ }
1751
+ contradictionSkipNew = true;
1752
+ }
1753
+ }
1754
+ }
1755
+
1756
+ if (contradictionSkipNew) {
1757
+ skipped++;
1758
+ continue;
1759
+ }
1207
1760
 
1761
+ const encryptedBlob = encryptToHex(blobPlaintext, encryptionKey);
1208
1762
  const contentFp = generateContentFingerprint(fact.text, dedupKey);
1209
- const factId = crypto.randomUUID();
1210
1763
 
1211
1764
  if (isSubgraphMode()) {
1212
1765
  const protobuf = encodeFactProtobuf({
@@ -1216,9 +1769,10 @@ async function storeExtractedFacts(
1216
1769
  encryptedBlob: encryptedBlob,
1217
1770
  blindIndices: allIndices,
1218
1771
  decayScore: effectiveImportance,
1219
- source: 'auto-extraction',
1772
+ source: factSource,
1220
1773
  contentFp: contentFp,
1221
1774
  agentId: 'openclaw-plugin-auto',
1775
+ version: PROTOBUF_VERSION_V4,
1222
1776
  encryptedEmbedding: embeddingResult?.encryptedEmbedding,
1223
1777
  });
1224
1778
  pendingPayloads.push(protobuf);
@@ -1230,7 +1784,7 @@ async function storeExtractedFacts(
1230
1784
  encrypted_blob: encryptedBlob,
1231
1785
  blind_indices: allIndices,
1232
1786
  decay_score: effectiveImportance,
1233
- source: 'auto-extraction',
1787
+ source: factSource,
1234
1788
  content_fp: contentFp,
1235
1789
  agent_id: 'openclaw-plugin-auto',
1236
1790
  encrypted_embedding: embeddingResult?.encryptedEmbedding,
@@ -1241,40 +1795,68 @@ async function storeExtractedFacts(
1241
1795
  } catch (err: unknown) {
1242
1796
  // Check for 403 / quota exceeded — invalidate billing cache so next
1243
1797
  // before_agent_start re-fetches and warns the user.
1244
- const errMsg = err instanceof Error ? err.message : String(err);
1245
- if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
1798
+ const factErrMsg = err instanceof Error ? err.message : String(err);
1799
+ if (factErrMsg.includes('403') || factErrMsg.toLowerCase().includes('quota')) {
1246
1800
  try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
1247
- logger.warn(`Quota exceeded — billing cache invalidated. ${errMsg}`);
1801
+ logger.warn(`Quota exceeded — billing cache invalidated. ${factErrMsg}`);
1248
1802
  break; // Stop trying to store remaining facts — they'll all fail too
1249
1803
  }
1250
- // Otherwise skip failed facts (e.g., duplicates return success with duplicate_ids)
1804
+ // Otherwise log and continue individual fact failures shouldn't block remaining facts
1805
+ logger.warn(`Failed to store fact "${fact.text.slice(0, 60)}…": ${factErrMsg}`);
1806
+ failedFacts++;
1251
1807
  }
1252
1808
  }
1253
1809
 
1254
- // Batch-submit all subgraph payloads in a single UserOp (gas-efficient).
1810
+ // Submit subgraph payloads one fact at a time (sequential single-call UserOps).
1811
+ // Batch executeBatch UserOps have persistent gas estimation issues on Base Sepolia
1812
+ // that cause on-chain reverts. Single-fact UserOps use the simpler submitFactOnChain
1813
+ // path which works reliably (same path as totalreclaw_remember). Each submission
1814
+ // polls for receipt (120s) before proceeding, so nonce is consumed before the next.
1815
+ let batchError: string | undefined;
1255
1816
  if (pendingPayloads.length > 0 && isSubgraphMode()) {
1256
- try {
1257
- const batchConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1258
- const result = await submitFactBatchOnChain(pendingPayloads, batchConfig);
1259
- if (result.success) {
1260
- stored += preparedForSubgraph;
1261
- logger.info(`Batch submitted ${result.batchSize} payloads in 1 UserOp (tx=${result.txHash.slice(0, 10)}…)`);
1262
- } else {
1263
- logger.warn(`Batch UserOp failed on-chain (tx=${result.txHash.slice(0, 10)}…)`);
1264
- }
1265
- } catch (err: unknown) {
1266
- const errMsg = err instanceof Error ? err.message : String(err);
1267
- if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
1268
- try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
1269
- logger.warn(`Quota exceeded during batch submit — billing cache invalidated. ${errMsg}`);
1270
- } else {
1271
- logger.warn(`Batch submission failed: ${errMsg}`);
1817
+ const batchConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1818
+ for (let i = 0; i < pendingPayloads.length; i++) {
1819
+ const slice = [pendingPayloads[i]]; // Single fact per UserOp
1820
+ try {
1821
+ const result = await submitFactBatchOnChain(slice, batchConfig);
1822
+ if (result.success) {
1823
+ stored += slice.length;
1824
+ logger.info(`Fact ${i + 1}/${pendingPayloads.length}: submitted on-chain (tx=${result.txHash.slice(0, 10)}…)`);
1825
+ } else {
1826
+ batchError = `On-chain batch submission failed (tx=${result.txHash.slice(0, 10)}…)`;
1827
+ logger.warn(batchError);
1828
+ break; // Stop submitting remaining batches
1829
+ }
1830
+ } catch (err: unknown) {
1831
+ const errMsg = err instanceof Error ? err.message : String(err);
1832
+ if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
1833
+ try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
1834
+ batchError = `Quota exceeded — billing cache invalidated. ${errMsg}`;
1835
+ logger.warn(batchError);
1836
+ break;
1837
+ } else {
1838
+ batchError = `Batch submission failed: ${errMsg}`;
1839
+ logger.warn(batchError);
1840
+ break;
1841
+ }
1272
1842
  }
1273
1843
  }
1274
1844
  }
1275
1845
 
1276
- if (stored > 0 || superseded > 0 || skipped > 0) {
1277
- logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}`);
1846
+ if (stored > 0 || superseded > 0 || skipped > 0 || failedFacts > 0) {
1847
+ logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}, failed=${failedFacts}`);
1848
+ }
1849
+
1850
+ // If ANY batch failed, throw — even if some facts were stored earlier.
1851
+ // A failed/timed-out UserOp may still linger in the bundler mempool as a
1852
+ // "nonce zombie." If we return normally, the caller's next storeExtractedFacts
1853
+ // call will fetch the same on-chain nonce and hit AA25 ("invalid account nonce").
1854
+ // Throwing forces all callers (import loops, chunk handlers) to stop submitting.
1855
+ if (batchError) {
1856
+ throw new Error(`Memory storage failed (${stored} stored before failure): ${batchError}`);
1857
+ }
1858
+ if (stored === 0 && failedFacts > 0) {
1859
+ throw new Error(`Memory storage failed: ${failedFacts} fact(s) failed to store`);
1278
1860
  }
1279
1861
 
1280
1862
  return stored;
@@ -1287,16 +1869,22 @@ async function storeExtractedFacts(
1287
1869
  /**
1288
1870
  * Handle import_from tool calls in the plugin context.
1289
1871
  *
1290
- * Uses the shared adapters to parse, then stores via storeExtractedFacts().
1872
+ * Two paths:
1873
+ * 1. Pre-structured sources (Mem0, MCP Memory) — adapter returns facts directly,
1874
+ * stored via storeExtractedFacts().
1875
+ * 2. Conversation-based sources (ChatGPT, Claude) — adapter returns conversation
1876
+ * chunks, each chunk is passed through extractFacts() (the same LLM extraction
1877
+ * pipeline used for auto-extraction), then stored via storeExtractedFacts().
1291
1878
  */
1292
1879
  async function handlePluginImportFrom(
1293
1880
  params: Record<string, unknown>,
1294
1881
  logger: OpenClawPluginApi['logger'],
1295
1882
  ): Promise<Record<string, unknown>> {
1883
+ _importInProgress = true;
1296
1884
  const startTime = Date.now();
1297
1885
 
1298
1886
  const source = params.source as string;
1299
- const validSources = ['mem0', 'mcp-memory', 'memoclaw', 'generic-json', 'generic-csv'];
1887
+ const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'];
1300
1888
 
1301
1889
  if (!source || !validSources.includes(source)) {
1302
1890
  return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
@@ -1314,7 +1902,10 @@ async function handlePluginImportFrom(
1314
1902
  file_path: params.file_path as string | undefined,
1315
1903
  });
1316
1904
 
1317
- if (parseResult.errors.length > 0 && parseResult.facts.length === 0) {
1905
+ const hasChunks = parseResult.chunks && parseResult.chunks.length > 0;
1906
+ const hasFacts = parseResult.facts && parseResult.facts.length > 0;
1907
+
1908
+ if (parseResult.errors.length > 0 && !hasFacts && !hasChunks) {
1318
1909
  return {
1319
1910
  success: false,
1320
1911
  error: `Failed to parse ${adapter.displayName} data`,
@@ -1322,7 +1913,37 @@ async function handlePluginImportFrom(
1322
1913
  };
1323
1914
  }
1324
1915
 
1916
+ // Dry run: report what was parsed (chunks or facts)
1325
1917
  if (params.dry_run) {
1918
+ if (hasChunks) {
1919
+ const totalChunks = parseResult.chunks.length;
1920
+ const EXTRACTION_RATIO = 2.5; // avg facts per chunk, from empirical data
1921
+ const BATCH_SIZE = 25;
1922
+ const SECONDS_PER_BATCH = 45; // ~30s extraction + ~15s embed+store
1923
+ const estimatedFacts = Math.round(totalChunks * EXTRACTION_RATIO);
1924
+ const estimatedBatches = Math.ceil(totalChunks / BATCH_SIZE);
1925
+ const estimatedMinutes = Math.ceil(estimatedBatches * SECONDS_PER_BATCH / 60);
1926
+
1927
+ return {
1928
+ success: true,
1929
+ dry_run: true,
1930
+ source,
1931
+ total_chunks: totalChunks,
1932
+ total_messages: parseResult.totalMessages,
1933
+ estimated_facts: estimatedFacts,
1934
+ estimated_batches: estimatedBatches,
1935
+ estimated_minutes: estimatedMinutes,
1936
+ batch_size: BATCH_SIZE,
1937
+ use_background: totalChunks > 50,
1938
+ preview: parseResult.chunks.slice(0, 5).map((c) => ({
1939
+ title: c.title,
1940
+ messages: c.messages.length,
1941
+ first_message: c.messages[0]?.text.slice(0, 100),
1942
+ })),
1943
+ note: `Estimated ${estimatedFacts} facts from ${totalChunks} chunks (~${estimatedMinutes} min).${totalChunks > 50 ? ' Recommended: background import via sessions_spawn.' : ''}`,
1944
+ warnings: parseResult.warnings,
1945
+ };
1946
+ }
1326
1947
  return {
1327
1948
  success: true,
1328
1949
  dry_run: true,
@@ -1337,7 +1958,12 @@ async function handlePluginImportFrom(
1337
1958
  };
1338
1959
  }
1339
1960
 
1340
- // Convert NormalizedFact[] to ExtractedFact[] for storeExtractedFacts()
1961
+ // ── Path 1: Conversation chunks (ChatGPT, Claude) — LLM extraction ──
1962
+ if (hasChunks) {
1963
+ return handleChunkImport(parseResult.chunks, parseResult.totalMessages, source, logger, startTime, parseResult.warnings);
1964
+ }
1965
+
1966
+ // ── Path 2: Pre-structured facts (Mem0, MCP Memory) — direct store ──
1341
1967
  const extractedFacts: ExtractedFact[] = parseResult.facts.map((f) => ({
1342
1968
  text: f.text,
1343
1969
  type: f.type,
@@ -1345,28 +1971,42 @@ async function handlePluginImportFrom(
1345
1971
  action: 'ADD' as const,
1346
1972
  }));
1347
1973
 
1348
- // Store in batches of 50
1974
+ // Store in batches of 50. Stop on any batch failure to prevent
1975
+ // nonce zombies from blocking subsequent UserOps (AA25).
1349
1976
  let totalStored = 0;
1977
+ let storeError: string | undefined;
1350
1978
  const batchSize = 50;
1351
1979
 
1352
1980
  for (let i = 0; i < extractedFacts.length; i += batchSize) {
1353
1981
  const batch = extractedFacts.slice(i, i + batchSize);
1354
- const stored = await storeExtractedFacts(batch, logger);
1355
- totalStored += stored;
1982
+ try {
1983
+ const stored = await storeExtractedFacts(batch, logger);
1984
+ totalStored += stored;
1356
1985
 
1357
- logger.info(
1358
- `Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`,
1359
- );
1986
+ logger.info(
1987
+ `Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`,
1988
+ );
1989
+ } catch (err: unknown) {
1990
+ storeError = err instanceof Error ? err.message : String(err);
1991
+ logger.warn(`Import stopped at batch ${Math.floor(i / batchSize) + 1}: ${storeError}`);
1992
+ break; // Stop processing further batches
1993
+ }
1994
+ }
1995
+
1996
+ const importWarnings = [...parseResult.warnings];
1997
+ if (storeError) {
1998
+ importWarnings.push(`Import stopped early: ${storeError}`);
1360
1999
  }
1361
2000
 
1362
2001
  return {
1363
- success: true,
2002
+ success: totalStored > 0,
1364
2003
  source,
1365
2004
  import_id: crypto.randomUUID(),
1366
2005
  total_found: parseResult.facts.length,
1367
2006
  imported: totalStored,
1368
2007
  skipped: parseResult.facts.length - totalStored,
1369
- warnings: parseResult.warnings,
2008
+ stopped_early: !!storeError,
2009
+ warnings: importWarnings,
1370
2010
  duration_ms: Date.now() - startTime,
1371
2011
  };
1372
2012
  } catch (e) {
@@ -1376,6 +2016,459 @@ async function handlePluginImportFrom(
1376
2016
  }
1377
2017
  }
1378
2018
 
2019
+ // ---------------------------------------------------------------------------
2020
+ // Smart Import — Two-Pass Pipeline (Profile + Triage)
2021
+ // ---------------------------------------------------------------------------
2022
+
2023
+ // Lazy-load WASM for smart import functions (same pattern as crypto.ts / subgraph-store.ts).
2024
+ let _smartImportWasm: typeof import('@totalreclaw/core') | null = null;
2025
+ function getSmartImportWasm() {
2026
+ if (!_smartImportWasm) _smartImportWasm = require('@totalreclaw/core');
2027
+ return _smartImportWasm;
2028
+ }
2029
+
2030
+ /**
2031
+ * Check whether the @totalreclaw/core WASM module exposes smart import functions.
2032
+ * Returns false if the module is an older version without smart import support.
2033
+ */
2034
+ function hasSmartImportSupport(): boolean {
2035
+ try {
2036
+ const wasm = getSmartImportWasm();
2037
+ return typeof wasm.chunksToSummaries === 'function' &&
2038
+ typeof wasm.buildProfileBatchPrompt === 'function' &&
2039
+ typeof wasm.parseProfileBatchResponse === 'function' &&
2040
+ typeof wasm.buildTriagePrompt === 'function' &&
2041
+ typeof wasm.parseTriageResponse === 'function' &&
2042
+ typeof wasm.enrichExtractionPrompt === 'function';
2043
+ } catch {
2044
+ return false;
2045
+ }
2046
+ }
2047
+
2048
+ /** Smart import result containing profile, triage decisions, and enriched system prompt. */
2049
+ interface SmartImportContext {
2050
+ /** JSON-serialized UserProfile (for WASM calls that require profile_json) */
2051
+ profileJson: string;
2052
+ /** Triage decisions indexed by chunk_index */
2053
+ decisions: Array<{ chunk_index: number; decision: string; reason: string }>;
2054
+ /** Enriched system prompt for extraction (profile context injected) */
2055
+ enrichedSystemPrompt: string;
2056
+ /** Number of chunks marked for extraction */
2057
+ extractCount: number;
2058
+ /** Number of chunks marked for skipping */
2059
+ skipCount: number;
2060
+ /** Duration of the profiling + triage pipeline in ms */
2061
+ durationMs: number;
2062
+ }
2063
+
2064
+ /**
2065
+ * Run the smart import two-pass pipeline: profile the user from conversation
2066
+ * summaries, then triage chunks as EXTRACT or SKIP.
2067
+ *
2068
+ * All prompt construction and response parsing happens in @totalreclaw/core WASM.
2069
+ * LLM calls use the plugin's existing chatCompletion() function.
2070
+ *
2071
+ * Returns null if smart import is unavailable (old WASM, no LLM config, etc.)
2072
+ * so the caller can fall back to blind extraction.
2073
+ */
2074
+ async function runSmartImportPipeline(
2075
+ chunks: import('./import-adapters/types.js').ConversationChunk[],
2076
+ logger: { info: (msg: string) => void; warn: (msg: string) => void },
2077
+ ): Promise<SmartImportContext | null> {
2078
+ // Guard: WASM must have smart import functions
2079
+ if (!hasSmartImportSupport()) {
2080
+ logger.info('Smart import: WASM module does not support smart import, falling back to blind extraction');
2081
+ return null;
2082
+ }
2083
+
2084
+ // Guard: LLM must be available
2085
+ const llmConfig = resolveLLMConfig();
2086
+ if (!llmConfig) {
2087
+ logger.info('Smart import: no LLM available, falling back to blind extraction');
2088
+ return null;
2089
+ }
2090
+
2091
+ const pipelineStart = Date.now();
2092
+ const wasm = getSmartImportWasm();
2093
+
2094
+ try {
2095
+ // Step 0: Convert chunks to compact summaries (first + last message)
2096
+ const wasmChunks = chunks.map((c, i) => ({
2097
+ index: i,
2098
+ title: c.title || 'Untitled',
2099
+ messages: c.messages.map((m) => ({ role: m.role, content: m.text })),
2100
+ timestamp: c.timestamp || null,
2101
+ }));
2102
+ const summaries = wasm.chunksToSummaries(JSON.stringify(wasmChunks));
2103
+ const summariesJson = JSON.stringify(summaries);
2104
+
2105
+ // Step 1: Build user profile (batch summarize -> merge)
2106
+ const PROFILE_BATCH_SIZE = 50;
2107
+ const profileStart = Date.now();
2108
+ const partials: unknown[] = [];
2109
+
2110
+ for (let i = 0; i < summaries.length; i += PROFILE_BATCH_SIZE) {
2111
+ const batch = summaries.slice(i, i + PROFILE_BATCH_SIZE);
2112
+ const prompt = wasm.buildProfileBatchPrompt(JSON.stringify(batch));
2113
+ const response = await chatCompletion(llmConfig, [
2114
+ { role: 'user', content: prompt },
2115
+ ], { maxTokens: 2048, temperature: 0 });
2116
+
2117
+ if (!response) {
2118
+ logger.warn(`Smart import: LLM returned empty response for profile batch ${Math.floor(i / PROFILE_BATCH_SIZE) + 1}`);
2119
+ continue;
2120
+ }
2121
+
2122
+ const partial = wasm.parseProfileBatchResponse(response);
2123
+ partials.push(partial);
2124
+ }
2125
+
2126
+ if (partials.length === 0) {
2127
+ logger.warn('Smart import: no profile batches produced, falling back to blind extraction');
2128
+ return null;
2129
+ }
2130
+
2131
+ let profile: unknown;
2132
+ if (partials.length === 1) {
2133
+ // Single batch — skip merge, promote partial to full profile
2134
+ // parseProfileBatchResponse returns a PartialProfile; convert to UserProfile shape
2135
+ const p = partials[0] as Record<string, unknown>;
2136
+ profile = {
2137
+ identity: p.identity ?? null,
2138
+ themes: p.themes ?? [],
2139
+ projects: p.projects ?? [],
2140
+ stack: p.stack ?? [],
2141
+ decisions: p.decisions ?? [],
2142
+ interests: p.interests ?? [],
2143
+ skip_patterns: p.skip_patterns ?? [],
2144
+ };
2145
+ } else {
2146
+ const mergePrompt = wasm.buildProfileMergePrompt(JSON.stringify(partials));
2147
+ const mergeResponse = await chatCompletion(llmConfig, [
2148
+ { role: 'user', content: mergePrompt },
2149
+ ], { maxTokens: 2048, temperature: 0 });
2150
+
2151
+ if (!mergeResponse) {
2152
+ logger.warn('Smart import: LLM returned empty response for profile merge, falling back to blind extraction');
2153
+ return null;
2154
+ }
2155
+
2156
+ profile = wasm.parseProfileResponse(mergeResponse);
2157
+ }
2158
+
2159
+ const profileJson = JSON.stringify(profile);
2160
+ const profileDuration = Date.now() - profileStart;
2161
+
2162
+ const p = profile as Record<string, unknown>;
2163
+ const themeCount = Array.isArray(p.themes) ? p.themes.length : 0;
2164
+ const skipPatternCount = Array.isArray(p.skip_patterns) ? p.skip_patterns.length : 0;
2165
+ logger.info(
2166
+ `Smart import: profile built in ${profileDuration}ms (themes=${themeCount}, skip_patterns=${skipPatternCount})`,
2167
+ );
2168
+
2169
+ // Step 1.5: Chunk triage (EXTRACT or SKIP)
2170
+ const triageStart = Date.now();
2171
+ const allDecisions: Array<{ chunk_index: number; decision: string; reason: string }> = [];
2172
+ const TRIAGE_BATCH_SIZE = 50;
2173
+
2174
+ for (let i = 0; i < summaries.length; i += TRIAGE_BATCH_SIZE) {
2175
+ const batch = summaries.slice(i, i + TRIAGE_BATCH_SIZE);
2176
+ const triagePrompt = wasm.buildTriagePrompt(profileJson, JSON.stringify(batch));
2177
+ const triageResponse = await chatCompletion(llmConfig, [
2178
+ { role: 'user', content: triagePrompt },
2179
+ ], { maxTokens: 4096, temperature: 0 });
2180
+
2181
+ if (!triageResponse) {
2182
+ logger.warn(`Smart import: LLM returned empty response for triage batch ${Math.floor(i / TRIAGE_BATCH_SIZE) + 1}, defaulting to EXTRACT`);
2183
+ // Default all chunks in this batch to EXTRACT
2184
+ for (let j = i; j < Math.min(i + TRIAGE_BATCH_SIZE, summaries.length); j++) {
2185
+ allDecisions.push({ chunk_index: j, decision: 'EXTRACT', reason: 'triage LLM unavailable' });
2186
+ }
2187
+ continue;
2188
+ }
2189
+
2190
+ const batchDecisions = wasm.parseTriageResponse(triageResponse) as Array<{
2191
+ chunk_index: number;
2192
+ decision: string;
2193
+ reason: string;
2194
+ }>;
2195
+ allDecisions.push(...batchDecisions);
2196
+ }
2197
+
2198
+ const triageDuration = Date.now() - triageStart;
2199
+
2200
+ const extractCount = allDecisions.filter((d) => d.decision !== 'SKIP').length;
2201
+ const skipCount = allDecisions.filter((d) => d.decision === 'SKIP').length;
2202
+ logger.info(
2203
+ `Smart import: triage complete in ${triageDuration}ms (extract=${extractCount}, skip=${skipCount}, total=${chunks.length})`,
2204
+ );
2205
+
2206
+ // Step 2: Build enriched system prompt for extraction
2207
+ const enrichedSystemPrompt = wasm.enrichExtractionPrompt(profileJson, EXTRACTION_SYSTEM_PROMPT);
2208
+
2209
+ const totalDuration = Date.now() - pipelineStart;
2210
+ logger.info(`Smart import: pipeline complete in ${totalDuration}ms`);
2211
+
2212
+ return {
2213
+ profileJson,
2214
+ decisions: allDecisions,
2215
+ enrichedSystemPrompt,
2216
+ extractCount,
2217
+ skipCount,
2218
+ durationMs: totalDuration,
2219
+ };
2220
+ } catch (err) {
2221
+ const msg = err instanceof Error ? err.message : String(err);
2222
+ logger.warn(`Smart import: pipeline failed (${msg}), falling back to blind extraction`);
2223
+ return null;
2224
+ }
2225
+ }
2226
+
2227
+ /**
2228
+ * Check if a chunk should be skipped based on triage decisions.
2229
+ * If no decision exists for the chunk index, defaults to EXTRACT (safe default).
2230
+ */
2231
+ function isChunkSkipped(
2232
+ chunkIndex: number,
2233
+ decisions: Array<{ chunk_index: number; decision: string }>,
2234
+ ): { skipped: boolean; reason: string } {
2235
+ const decision = decisions.find((d) => d.chunk_index === chunkIndex);
2236
+ if (decision && decision.decision === 'SKIP') {
2237
+ return { skipped: true, reason: (decision as { reason?: string }).reason || 'triage: skip' };
2238
+ }
2239
+ return { skipped: false, reason: '' };
2240
+ }
2241
+
2242
+ /**
2243
+ * Process a batch (slice) of conversation chunks from a file.
2244
+ * Called repeatedly by the agent for large imports.
2245
+ */
2246
+ async function handleBatchImport(
2247
+ params: Record<string, unknown>,
2248
+ logger: OpenClawPluginApi['logger'],
2249
+ ): Promise<Record<string, unknown>> {
2250
+ _importInProgress = true;
2251
+ const source = params.source as string;
2252
+ const filePath = params.file_path as string | undefined;
2253
+ const content = params.content as string | undefined;
2254
+ const offset = (params.offset as number) ?? 0;
2255
+ const batchSize = (params.batch_size as number) ?? 25;
2256
+
2257
+ const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'];
2258
+ if (!source || !validSources.includes(source)) {
2259
+ return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
2260
+ }
2261
+
2262
+ const startTime = Date.now();
2263
+
2264
+ const { getAdapter } = await import('./import-adapters/index.js');
2265
+ const adapter = getAdapter(source as import('./import-adapters/types.js').ImportSource);
2266
+
2267
+ const parseResult = await adapter.parse({ content, file_path: filePath });
2268
+
2269
+ if (parseResult.errors.length > 0 && parseResult.chunks.length === 0) {
2270
+ return { success: false, error: parseResult.errors.join('; ') };
2271
+ }
2272
+
2273
+ const totalChunks = parseResult.chunks.length;
2274
+ const slice = parseResult.chunks.slice(offset, offset + batchSize);
2275
+ const remaining = Math.max(0, totalChunks - offset - slice.length);
2276
+
2277
+ // --- Smart Import: Profile + Triage ---
2278
+ // Build profile from ALL chunks (not just the slice) for full context,
2279
+ // then triage only the current slice. For simplicity, we rebuild on every
2280
+ // batch call — optimization (caching) can come later.
2281
+ const smartCtx = await runSmartImportPipeline(parseResult.chunks, logger);
2282
+ let chunksSkipped = 0;
2283
+
2284
+ // Process the slice through the normal extraction + storage pipeline.
2285
+ // If a batch fails (nonce zombie, quota exceeded, etc.), stop immediately
2286
+ // to prevent subsequent UserOps from hitting AA25 nonce conflicts.
2287
+ let factsExtracted = 0;
2288
+ let factsStored = 0;
2289
+ let chunksProcessed = 0;
2290
+ let storeError: string | undefined;
2291
+
2292
+ for (let i = 0; i < slice.length; i++) {
2293
+ const chunk = slice[i];
2294
+ const globalIndex = offset + i; // Index in the full chunks array
2295
+
2296
+ // Smart import: skip chunks triaged as SKIP
2297
+ if (smartCtx) {
2298
+ const { skipped, reason } = isChunkSkipped(globalIndex, smartCtx.decisions);
2299
+ if (skipped) {
2300
+ logger.info(`Import: skipping chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}" (${reason})`);
2301
+ chunksSkipped++;
2302
+ chunksProcessed++;
2303
+ continue;
2304
+ }
2305
+ }
2306
+
2307
+ logger.info(`Import: extracting facts from chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}"`);
2308
+
2309
+ const messages = chunk.messages.map((m) => ({ role: m.role, content: m.text }));
2310
+ const facts = await extractFacts(
2311
+ messages,
2312
+ 'full',
2313
+ undefined, // no existing memories for dedup during import
2314
+ smartCtx?.enrichedSystemPrompt, // profile-enriched extraction prompt
2315
+ );
2316
+ chunksProcessed++;
2317
+
2318
+ if (facts.length > 0) {
2319
+ factsExtracted += facts.length;
2320
+ try {
2321
+ const stored = await storeExtractedFacts(facts, logger);
2322
+ factsStored += stored;
2323
+ } catch (err: unknown) {
2324
+ storeError = err instanceof Error ? err.message : String(err);
2325
+ logger.warn(`Import batch stopped at chunk ${globalIndex + 1}/${totalChunks}: ${storeError}`);
2326
+ break; // Stop processing further chunks — a zombie UserOp may block writes
2327
+ }
2328
+ }
2329
+ }
2330
+
2331
+ return {
2332
+ success: factsStored > 0 || (!storeError && factsExtracted === 0),
2333
+ batch_offset: offset,
2334
+ batch_size: chunksProcessed,
2335
+ total_chunks: totalChunks,
2336
+ facts_extracted: factsExtracted,
2337
+ facts_stored: factsStored,
2338
+ chunks_skipped: chunksSkipped,
2339
+ remaining_chunks: remaining,
2340
+ is_complete: remaining === 0 && !storeError,
2341
+ stopped_early: !!storeError,
2342
+ error: storeError,
2343
+ smart_import: smartCtx ? {
2344
+ profile_duration_ms: smartCtx.durationMs,
2345
+ extract_count: smartCtx.extractCount,
2346
+ skip_count: smartCtx.skipCount,
2347
+ } : null,
2348
+ // Estimation for the full import
2349
+ estimated_total_facts: Math.round(totalChunks * 2.5),
2350
+ estimated_total_userops: Math.ceil(totalChunks * 2.5 / 15),
2351
+ estimated_minutes: Math.ceil(Math.ceil(totalChunks / batchSize) * 45 / 60),
2352
+ duration_ms: Date.now() - startTime,
2353
+ };
2354
+ }
2355
+
2356
+ /**
2357
+ * Process conversation chunks through LLM extraction and store results.
2358
+ *
2359
+ * Each chunk is passed to extractFacts() — the same extraction pipeline used
2360
+ * for auto-extraction during live conversations. This ensures import quality
2361
+ * matches conversation extraction quality.
2362
+ */
2363
+ async function handleChunkImport(
2364
+ chunks: import('./import-adapters/types.js').ConversationChunk[],
2365
+ totalMessages: number,
2366
+ source: string,
2367
+ logger: OpenClawPluginApi['logger'],
2368
+ startTime: number,
2369
+ warnings: string[],
2370
+ ): Promise<Record<string, unknown>> {
2371
+ let totalExtracted = 0;
2372
+ let totalStored = 0;
2373
+ let chunksProcessed = 0;
2374
+ let chunksSkipped = 0;
2375
+
2376
+ let storeError: string | undefined;
2377
+
2378
+ // --- Smart Import: Profile + Triage ---
2379
+ const smartCtx = await runSmartImportPipeline(chunks, logger);
2380
+
2381
+ for (let i = 0; i < chunks.length; i++) {
2382
+ const chunk = chunks[i];
2383
+ chunksProcessed++;
2384
+
2385
+ // Smart import: skip chunks triaged as SKIP
2386
+ if (smartCtx) {
2387
+ const { skipped, reason } = isChunkSkipped(i, smartCtx.decisions);
2388
+ if (skipped) {
2389
+ logger.info(
2390
+ `Import: skipping chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}" (${reason})`,
2391
+ );
2392
+ chunksSkipped++;
2393
+ continue;
2394
+ }
2395
+ }
2396
+
2397
+ logger.info(
2398
+ `Import: extracting facts from chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}"`,
2399
+ );
2400
+
2401
+ // Convert chunk messages to the format extractFacts() expects.
2402
+ // extractFacts() takes an array of message-like objects with { role, content }.
2403
+ const messages = chunk.messages.map((m) => ({
2404
+ role: m.role,
2405
+ content: m.text,
2406
+ }));
2407
+
2408
+ // Use 'full' mode to extract ALL valuable memories from the chunk
2409
+ // (not just the last few messages like 'turn' mode does).
2410
+ // Smart import: pass enriched system prompt with user profile context.
2411
+ const facts = await extractFacts(
2412
+ messages,
2413
+ 'full',
2414
+ undefined, // no existing memories for dedup during import
2415
+ smartCtx?.enrichedSystemPrompt, // profile-enriched extraction prompt
2416
+ );
2417
+
2418
+ if (facts.length > 0) {
2419
+ totalExtracted += facts.length;
2420
+
2421
+ try {
2422
+ // Store through the normal pipeline (dedup, encrypt, store).
2423
+ // storeExtractedFacts throws on batch failure to prevent nonce zombies.
2424
+ const stored = await storeExtractedFacts(facts, logger);
2425
+ totalStored += stored;
2426
+
2427
+ logger.info(
2428
+ `Import chunk ${chunksProcessed}/${chunks.length}: extracted ${facts.length} facts, stored ${stored}`,
2429
+ );
2430
+ } catch (err: unknown) {
2431
+ storeError = err instanceof Error ? err.message : String(err);
2432
+ logger.warn(`Import stopped at chunk ${chunksProcessed}/${chunks.length}: ${storeError}`);
2433
+ break; // Stop processing further chunks — a zombie UserOp may block writes
2434
+ }
2435
+ }
2436
+ }
2437
+
2438
+ if (totalExtracted === 0 && chunks.length > 0 && !storeError && chunksSkipped < chunks.length) {
2439
+ warnings.push(
2440
+ `Processed ${chunks.length} conversation chunks (${totalMessages} messages) but the LLM ` +
2441
+ `did not extract any facts worth storing. This can happen if the conversations are mostly ` +
2442
+ `generic/ephemeral content without personal facts, preferences, or decisions.`,
2443
+ );
2444
+ }
2445
+
2446
+ if (storeError) {
2447
+ warnings.push(`Import stopped early: ${storeError}. ${chunks.length - chunksProcessed} chunk(s) not processed.`);
2448
+ }
2449
+
2450
+ return {
2451
+ success: totalStored > 0 || totalExtracted > 0,
2452
+ source,
2453
+ import_id: crypto.randomUUID(),
2454
+ total_chunks: chunks.length,
2455
+ chunks_processed: chunksProcessed,
2456
+ chunks_skipped: chunksSkipped,
2457
+ total_messages: totalMessages,
2458
+ facts_extracted: totalExtracted,
2459
+ imported: totalStored,
2460
+ skipped: totalExtracted - totalStored,
2461
+ stopped_early: !!storeError,
2462
+ smart_import: smartCtx ? {
2463
+ profile_duration_ms: smartCtx.durationMs,
2464
+ extract_count: smartCtx.extractCount,
2465
+ skip_count: smartCtx.skipCount,
2466
+ } : null,
2467
+ warnings,
2468
+ duration_ms: Date.now() - startTime,
2469
+ };
2470
+ }
2471
+
1379
2472
  // ---------------------------------------------------------------------------
1380
2473
  // Plugin definition
1381
2474
  // ---------------------------------------------------------------------------
@@ -1408,6 +2501,7 @@ const plugin = {
1408
2501
  initLLMClient({
1409
2502
  primaryModel: api.config?.agents?.defaults?.model?.primary as string | undefined,
1410
2503
  pluginConfig: api.pluginConfig,
2504
+ openclawProviders: api.config?.models?.providers,
1411
2505
  logger: api.logger,
1412
2506
  });
1413
2507
 
@@ -1444,160 +2538,164 @@ const plugin = {
1444
2538
  },
1445
2539
  type: {
1446
2540
  type: 'string',
1447
- enum: ['fact', 'preference', 'decision', 'episodic', 'goal', 'context', 'summary'],
1448
- description: 'The kind of memory (default: fact)',
2541
+ enum: [...VALID_MEMORY_TYPES, ...LEGACY_V0_MEMORY_TYPES],
2542
+ description:
2543
+ 'Memory Taxonomy v1 type: claim, preference, directive, commitment, episode, summary. ' +
2544
+ 'Use "claim" for factual assertions and decisions (populate `reasoning` with the why clause). ' +
2545
+ 'Use "directive" for imperative rules ("always X", "never Y"), "commitment" for future intent, ' +
2546
+ 'and "episode" for notable events. Legacy v0 tokens (fact, decision, episodic, goal, context, ' +
2547
+ 'rule) are silently coerced to their v1 equivalents. Default: claim.',
2548
+ },
2549
+ source: {
2550
+ type: 'string',
2551
+ enum: [...VALID_MEMORY_SOURCES],
2552
+ description:
2553
+ 'v1 provenance tag. "user" = user explicitly stated it, "user-inferred" = inferred from user ' +
2554
+ 'signals, "assistant" = assistant-authored (downgrade unless user affirmed), "external" / ' +
2555
+ '"derived" = rare. Explicit remembers default to "user".',
2556
+ },
2557
+ scope: {
2558
+ type: 'string',
2559
+ enum: [...VALID_MEMORY_SCOPES],
2560
+ description:
2561
+ 'v1 life-domain scope: work, personal, health, family, creative, finance, misc, unspecified. ' +
2562
+ 'Default: unspecified.',
2563
+ },
2564
+ reasoning: {
2565
+ type: 'string',
2566
+ description:
2567
+ 'For type=claim expressing a decision, the WHY clause ("because Y"). Max 256 chars. ' +
2568
+ 'Omit for non-decision claims.',
2569
+ maxLength: 256,
1449
2570
  },
1450
2571
  importance: {
1451
2572
  type: 'number',
1452
2573
  minimum: 1,
1453
2574
  maximum: 10,
1454
- description: 'Importance score 1-10 (default: 5)',
2575
+ description: 'Importance score 1-10 (default: 8 for explicit remember)',
2576
+ },
2577
+ entities: {
2578
+ type: 'array',
2579
+ description:
2580
+ 'Named entities this memory is about (people, projects, tools, companies, concepts, places). ' +
2581
+ 'Supplying entities enables Phase 2 contradiction detection against existing facts about the same entity. ' +
2582
+ 'Omit if unclear — a best-effort fallback will still store the memory.',
2583
+ items: {
2584
+ type: 'object',
2585
+ properties: {
2586
+ name: { type: 'string' },
2587
+ type: {
2588
+ type: 'string',
2589
+ enum: ['person', 'project', 'tool', 'company', 'concept', 'place'],
2590
+ },
2591
+ role: { type: 'string' },
2592
+ },
2593
+ required: ['name', 'type'],
2594
+ additionalProperties: false,
2595
+ },
1455
2596
  },
1456
2597
  },
1457
2598
  required: ['text'],
1458
2599
  additionalProperties: false,
1459
2600
  },
1460
- async execute(_toolCallId: string, params: { text: string; type?: string; importance?: number }) {
2601
+ async execute(
2602
+ _toolCallId: string,
2603
+ params: {
2604
+ text: string;
2605
+ type?: string;
2606
+ source?: string;
2607
+ scope?: string;
2608
+ reasoning?: string;
2609
+ importance?: number;
2610
+ entities?: Array<{ name: string; type: string; role?: string }>;
2611
+ },
2612
+ ) {
1461
2613
  try {
1462
2614
  await requireFullSetup(api.logger);
1463
2615
 
1464
- const memoryType = params.type ?? 'fact';
1465
- let importance = params.importance ?? 5;
1466
-
1467
- // Generate blind indices for server-side search.
1468
- const blindIndices = generateBlindIndices(params.text);
1469
-
1470
- // Generate embedding + LSH bucket hashes (PoC v2).
1471
- // Falls back to word-only indices if embedding generation fails.
1472
- const embeddingResult = await generateEmbeddingAndLSH(params.text, api.logger);
1473
-
1474
- // Merge LSH bucket hashes into blind indices.
1475
- const allIndices = embeddingResult
1476
- ? [...blindIndices, ...embeddingResult.lshBuckets]
1477
- : blindIndices;
1478
-
1479
- // Store-time dedup: for explicit remember, ALWAYS supersede
1480
- // (user explicitly wants this stored just remove the old one).
1481
- let supersededId: string | undefined;
1482
- if (STORE_DEDUP_ENABLED && embeddingResult) {
1483
- const dupResult = await searchForNearDuplicates(
1484
- params.text,
1485
- embeddingResult.embedding,
1486
- allIndices,
1487
- api.logger,
1488
- );
1489
- if (dupResult) {
1490
- // Inherit higher importance from existing fact.
1491
- importance = Math.max(importance, dupResult.match.decayScore);
1492
- supersededId = dupResult.match.id;
1493
-
1494
- if (isSubgraphMode()) {
1495
- try {
1496
- const tombConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1497
- const tombstone: FactPayload = {
1498
- id: dupResult.match.id,
1499
- timestamp: new Date().toISOString(),
1500
- owner: subgraphOwner || userId!,
1501
- encryptedBlob: '00',
1502
- blindIndices: [],
1503
- decayScore: 0,
1504
- source: 'tombstone',
1505
- contentFp: '',
1506
- agentId: 'openclaw-plugin',
1507
- };
1508
- const tombProtobuf = encodeFactProtobuf(tombstone);
1509
- await submitFactOnChain(tombProtobuf, tombConfig);
1510
- api.logger.info(
1511
- `Remember dedup: superseded ${dupResult.match.id} on-chain (sim=${dupResult.similarity.toFixed(3)})`,
1512
- );
1513
- } catch (tombErr) {
1514
- api.logger.warn(
1515
- `Remember dedup: failed to tombstone ${dupResult.match.id}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`,
1516
- );
1517
- supersededId = undefined;
1518
- }
1519
- } else if (apiClient && authKeyHex) {
1520
- try {
1521
- await apiClient.deleteFact(dupResult.match.id, authKeyHex);
1522
- api.logger.info(
1523
- `Remember dedup: superseded ${dupResult.match.id} (sim=${dupResult.similarity.toFixed(3)})`,
1524
- );
1525
- } catch (delErr) {
1526
- api.logger.warn(
1527
- `Remember dedup: failed to delete superseded fact ${dupResult.match.id}: ${delErr instanceof Error ? delErr.message : String(delErr)}`,
1528
- );
1529
- supersededId = undefined; // Don't report supersession if delete failed
1530
- }
1531
- }
1532
- }
1533
- }
1534
-
1535
- // Build the document JSON that will be encrypted.
1536
- const doc = {
1537
- text: params.text,
1538
- metadata: {
1539
- type: memoryType,
1540
- importance: importance / 10, // normalise to 0-1 range
1541
- source: 'explicit',
1542
- created_at: new Date().toISOString(),
1543
- },
1544
- };
1545
-
1546
- // Encrypt the document.
1547
- const encryptedBlob = encryptToHex(JSON.stringify(doc), encryptionKey!);
1548
-
1549
- // Generate content fingerprint for dedup.
1550
- const contentFp = generateContentFingerprint(params.text, dedupKey!);
1551
-
1552
- // Generate a unique fact ID.
1553
- const factId = crypto.randomUUID();
2616
+ // v1 taxonomy: route explicit remembers through the same canonical
2617
+ // store path that auto-extraction uses (`storeExtractedFacts`). This
2618
+ // emits a Memory Taxonomy v1 JSON blob, generates entity trapdoors,
2619
+ // and runs through the Phase 2 contradiction-resolution pipeline.
2620
+ //
2621
+ // Accept legacy v0 tokens on input and coerce to v1 via
2622
+ // `normalizeToV1Type` so agents that still emit the pre-v3
2623
+ // taxonomy keep working.
2624
+ const rawType = typeof params.type === 'string' ? params.type.toLowerCase() : 'claim';
2625
+ const memoryType: MemoryType = isValidMemoryType(rawType)
2626
+ ? rawType
2627
+ : normalizeToV1Type(rawType);
2628
+
2629
+ // Source defaults to 'user' for explicit remembers (the user is
2630
+ // the author by definition). Ignored if the caller passes an
2631
+ // invalid value.
2632
+ const rawSource = typeof params.source === 'string' ? params.source.toLowerCase() : 'user';
2633
+ const memorySource: MemorySource =
2634
+ (VALID_MEMORY_SOURCES as readonly string[]).includes(rawSource)
2635
+ ? (rawSource as MemorySource)
2636
+ : 'user';
2637
+
2638
+ const rawScope = typeof params.scope === 'string' ? params.scope.toLowerCase() : 'unspecified';
2639
+ const memoryScope: MemoryScope =
2640
+ (VALID_MEMORY_SCOPES as readonly string[]).includes(rawScope)
2641
+ ? (rawScope as MemoryScope)
2642
+ : 'unspecified';
2643
+
2644
+ const reasoning =
2645
+ typeof params.reasoning === 'string' && params.reasoning.length > 0
2646
+ ? params.reasoning.slice(0, 256)
2647
+ : undefined;
2648
+
2649
+ // Explicit remember defaults to importance 8 (above auto-extraction's
2650
+ // typical 6-7), so store-time dedup's shouldSupersede prefers the
2651
+ // explicit call when it collides with an auto-extracted claim.
2652
+ const importance = Math.max(1, Math.min(10, params.importance ?? 8));
2653
+
2654
+ const validatedEntities: ExtractedEntity[] = Array.isArray(params.entities)
2655
+ ? params.entities
2656
+ .map((e) => parseEntity(e))
2657
+ .filter((e): e is ExtractedEntity => e !== null)
2658
+ : [];
1554
2659
 
1555
- // Build the payload matching the server's FactJSON schema.
1556
- const factPayload: StoreFactPayload = {
1557
- id: factId,
1558
- timestamp: new Date().toISOString(),
1559
- encrypted_blob: encryptedBlob,
1560
- blind_indices: allIndices,
1561
- decay_score: importance,
1562
- source: 'explicit',
1563
- content_fp: contentFp,
1564
- agent_id: 'openclaw-plugin',
1565
- encrypted_embedding: embeddingResult?.encryptedEmbedding,
2660
+ const fact: ExtractedFact = {
2661
+ text: params.text.slice(0, 512),
2662
+ type: memoryType,
2663
+ source: memorySource,
2664
+ scope: memoryScope,
2665
+ reasoning,
2666
+ importance,
2667
+ action: 'ADD',
2668
+ confidence: 1.0, // user explicitly asked to remember — highest confidence
1566
2669
  };
2670
+ if (validatedEntities.length > 0) fact.entities = validatedEntities;
1567
2671
 
1568
- if (isSubgraphMode()) {
1569
- // Subgraph mode: encode as Protobuf and submit on-chain via relay UserOp
1570
- const config = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
1571
- const protobuf = encodeFactProtobuf({
1572
- id: factId,
1573
- timestamp: new Date().toISOString(),
1574
- owner: subgraphOwner || userId!,
1575
- encryptedBlob: encryptedBlob,
1576
- blindIndices: allIndices,
1577
- decayScore: importance,
1578
- source: 'explicit',
1579
- contentFp: contentFp,
1580
- agentId: 'openclaw-plugin',
1581
- encryptedEmbedding: embeddingResult?.encryptedEmbedding,
1582
- });
1583
- await submitFactOnChain(protobuf, config);
1584
- } else {
1585
- await apiClient!.store(userId!, [factPayload], authKeyHex!);
1586
- }
2672
+ const stored = await storeExtractedFacts([fact], api.logger, 'explicit');
2673
+ api.logger.info(
2674
+ `totalreclaw_remember: routed to storeExtractedFacts (stored=${stored}, entities=${validatedEntities.length})`,
2675
+ );
1587
2676
 
1588
- const statusMsg = supersededId
1589
- ? `Memory stored (ID: ${factId}). Superseded an older similar memory.`
1590
- : `Memory stored (ID: ${factId})`;
2677
+ if (stored === 0) {
2678
+ // Dedup or supersession consumed the write. Treat as success from
2679
+ // the user's perspective — the memory's content is already in the
2680
+ // vault (possibly under a different ID).
2681
+ return {
2682
+ content: [
2683
+ {
2684
+ type: 'text',
2685
+ text: 'Memory noted (matched existing content in vault).',
2686
+ },
2687
+ ],
2688
+ };
2689
+ }
1591
2690
 
1592
2691
  return {
1593
- content: [{ type: 'text', text: statusMsg }],
1594
- details: { factId, supersededId },
2692
+ content: [{ type: 'text', text: 'Memory encrypted and stored.' }],
1595
2693
  };
1596
2694
  } catch (err: unknown) {
1597
2695
  const message = err instanceof Error ? err.message : String(err);
1598
2696
  api.logger.error(`totalreclaw_remember failed: ${message}`);
1599
2697
  return {
1600
- content: [{ type: 'text', text: `Failed to store memory: ${message}` }],
2698
+ content: [{ type: 'text', text: `Failed to store memory: ${humanizeError(message)}` }],
1601
2699
  };
1602
2700
  }
1603
2701
  },
@@ -1674,12 +2772,27 @@ const plugin = {
1674
2772
  // --- Subgraph search path ---
1675
2773
  const factCount = await getSubgraphFactCount(subgraphOwner || userId!, authKeyHex!);
1676
2774
  const pool = computeCandidatePool(factCount);
1677
- const subgraphResults = await searchSubgraph(subgraphOwner || userId!, allTrapdoors, pool, authKeyHex!);
2775
+ let subgraphResults = await searchSubgraph(subgraphOwner || userId!, allTrapdoors, pool, authKeyHex!);
2776
+
2777
+ // Always run broadened search and merge — ensures vocabulary mismatches
2778
+ // (e.g., "preferences" vs "prefer") don't cause recall failures.
2779
+ // The reranker handles scoring; extra cost is ~1 GraphQL query per recall.
2780
+ try {
2781
+ const broadenedResults = await searchSubgraphBroadened(subgraphOwner || userId!, pool, authKeyHex!);
2782
+ // Merge broadened results with existing (deduplicate by ID)
2783
+ const existingIds = new Set(subgraphResults.map(r => r.id));
2784
+ for (const br of broadenedResults) {
2785
+ if (!existingIds.has(br.id)) {
2786
+ subgraphResults.push(br);
2787
+ }
2788
+ }
2789
+ } catch { /* best-effort */ }
1678
2790
 
1679
2791
  for (const result of subgraphResults) {
1680
2792
  try {
1681
2793
  const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
1682
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
2794
+ if (isDigestBlob(docJson)) continue;
2795
+ const doc = readClaimFromBlob(docJson);
1683
2796
 
1684
2797
  let decryptedEmbedding: number[] | undefined;
1685
2798
  if (result.encryptedEmbedding) {
@@ -1692,17 +2805,29 @@ const plugin = {
1692
2805
  }
1693
2806
  }
1694
2807
 
2808
+ if (decryptedEmbedding && decryptedEmbedding.length !== getEmbeddingDims()) {
2809
+ try {
2810
+ decryptedEmbedding = await generateEmbedding(doc.text);
2811
+ } catch {
2812
+ decryptedEmbedding = undefined;
2813
+ }
2814
+ }
2815
+
1695
2816
  rerankerCandidates.push({
1696
2817
  id: result.id,
1697
2818
  text: doc.text,
1698
2819
  embedding: decryptedEmbedding,
1699
- importance: (doc.metadata?.importance as number) ?? 0.5,
2820
+ importance: doc.importance / 10,
1700
2821
  createdAt: result.timestamp ? parseInt(result.timestamp, 10) : undefined,
2822
+ // Retrieval v2 Tier 1: surface v1 source so applySourceWeights
2823
+ // can multiply the final RRF score by the source weight.
2824
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
1701
2825
  });
1702
2826
 
1703
2827
  metaMap.set(result.id, {
1704
2828
  metadata: doc.metadata ?? {},
1705
- timestamp: Date.now(), // Subgraph doesn't return ms timestamp; use current
2829
+ timestamp: Date.now(),
2830
+ category: doc.category,
1706
2831
  });
1707
2832
  } catch {
1708
2833
  // Skip candidates we cannot decrypt.
@@ -1745,7 +2870,8 @@ const plugin = {
1745
2870
  for (const candidate of candidates) {
1746
2871
  try {
1747
2872
  const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
1748
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
2873
+ if (isDigestBlob(docJson)) continue;
2874
+ const doc = readClaimFromBlob(docJson);
1749
2875
 
1750
2876
  let decryptedEmbedding: number[] | undefined;
1751
2877
  if (candidate.encrypted_embedding) {
@@ -1758,19 +2884,29 @@ const plugin = {
1758
2884
  }
1759
2885
  }
1760
2886
 
2887
+ if (decryptedEmbedding && decryptedEmbedding.length !== getEmbeddingDims()) {
2888
+ try {
2889
+ decryptedEmbedding = await generateEmbedding(doc.text);
2890
+ } catch {
2891
+ decryptedEmbedding = undefined;
2892
+ }
2893
+ }
2894
+
1761
2895
  rerankerCandidates.push({
1762
2896
  id: candidate.fact_id,
1763
2897
  text: doc.text,
1764
2898
  embedding: decryptedEmbedding,
1765
- importance: (doc.metadata?.importance as number) ?? 0.5,
2899
+ importance: doc.importance / 10,
1766
2900
  createdAt: typeof candidate.timestamp === 'number'
1767
2901
  ? candidate.timestamp / 1000
1768
2902
  : new Date(candidate.timestamp).getTime() / 1000,
2903
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
1769
2904
  });
1770
2905
 
1771
2906
  metaMap.set(candidate.fact_id, {
1772
2907
  metadata: doc.metadata ?? {},
1773
2908
  timestamp: candidate.timestamp,
2909
+ category: doc.category,
1774
2910
  });
1775
2911
  } catch {
1776
2912
  // Skip candidates we cannot decrypt (e.g. corrupted data).
@@ -1786,6 +2922,7 @@ const plugin = {
1786
2922
  rerankerCandidates,
1787
2923
  k,
1788
2924
  INTENT_WEIGHTS[queryIntent],
2925
+ /* applySourceWeights (Retrieval v2 Tier 1) */ true,
1789
2926
  );
1790
2927
 
1791
2928
  if (reranked.length === 0) {
@@ -1817,7 +2954,8 @@ const plugin = {
1817
2954
  ? ` (importance: ${Math.round((meta.metadata.importance as number) * 10)}/10)`
1818
2955
  : '';
1819
2956
  const age = meta ? relativeTime(meta.timestamp) : '';
1820
- return `${i + 1}. ${m.text}${imp} -- ${age} [ID: ${m.id}]`;
2957
+ const typeTag = meta?.category ? `[${meta.category}] ` : '';
2958
+ return `${i + 1}. ${typeTag}${m.text}${imp} -- ${age} [ID: ${m.id}]`;
1821
2959
  });
1822
2960
 
1823
2961
  const formatted = lines.join('\n');
@@ -1836,7 +2974,7 @@ const plugin = {
1836
2974
  const message = err instanceof Error ? err.message : String(err);
1837
2975
  api.logger.error(`totalreclaw_recall failed: ${message}`);
1838
2976
  return {
1839
- content: [{ type: 'text', text: `Failed to search memories: ${message}` }],
2977
+ content: [{ type: 'text', text: `Failed to search memories: ${humanizeError(message)}` }],
1840
2978
  };
1841
2979
  }
1842
2980
  },
@@ -1882,9 +3020,13 @@ const plugin = {
1882
3020
  source: 'tombstone',
1883
3021
  contentFp: '',
1884
3022
  agentId: 'openclaw-plugin',
3023
+ version: PROTOBUF_VERSION_V4,
1885
3024
  };
1886
3025
  const protobuf = encodeFactProtobuf(tombstone);
1887
3026
  const result = await submitFactOnChain(protobuf, config);
3027
+ if (!result.success) {
3028
+ throw new Error(`On-chain tombstone failed (tx=${result.txHash?.slice(0, 10) || 'none'}…)`);
3029
+ }
1888
3030
  api.logger.info(`Tombstone written for ${params.factId}: tx=${result.txHash}`);
1889
3031
  return {
1890
3032
  content: [{ type: 'text', text: `Memory ${params.factId} deleted (on-chain tombstone, tx: ${result.txHash})` }],
@@ -1901,7 +3043,7 @@ const plugin = {
1901
3043
  const message = err instanceof Error ? err.message : String(err);
1902
3044
  api.logger.error(`totalreclaw_forget failed: ${message}`);
1903
3045
  return {
1904
- content: [{ type: 'text', text: `Failed to delete memory: ${message}` }],
3046
+ content: [{ type: 'text', text: `Failed to delete memory: ${humanizeError(message)}` }],
1905
3047
  };
1906
3048
  }
1907
3049
  },
@@ -1945,16 +3087,22 @@ const plugin = {
1945
3087
  }> = [];
1946
3088
 
1947
3089
  if (isSubgraphMode()) {
1948
- // Query subgraph for all active facts
3090
+ // Query subgraph for all active facts (cursor-based pagination via id_gt)
1949
3091
  const config = getSubgraphConfig();
1950
3092
  const relayUrl = config.relayUrl;
1951
3093
  const PAGE_SIZE = 1000;
1952
- let skip = 0;
1953
- let hasMore = true;
3094
+ let lastId = '';
1954
3095
  const owner = subgraphOwner || userId || '';
3096
+ console.error(`[TotalReclaw Export] owner=${owner} subgraphOwner=${subgraphOwner} userId=${userId} relayUrl=${relayUrl} authKey=${authKeyHex ? authKeyHex.slice(0, 8) + '...' : 'MISSING'} isSubgraph=${isSubgraphMode()}`);
1955
3097
 
1956
- while (hasMore) {
1957
- const query = `{ facts(where: { owner: "${owner}", isActive: true }, first: ${PAGE_SIZE}, skip: ${skip}, orderBy: sequenceId, orderDirection: asc) { id encryptedBlob source agentId timestamp sequenceId } }`;
3098
+ while (true) {
3099
+ const hasLastId = lastId !== '';
3100
+ const query = hasLastId
3101
+ ? `query($owner:Bytes!,$first:Int!,$lastId:String!){facts(where:{owner:$owner,isActive:true,id_gt:$lastId},first:$first,orderBy:id,orderDirection:asc){id encryptedBlob timestamp sequenceId}}`
3102
+ : `query($owner:Bytes!,$first:Int!){facts(where:{owner:$owner,isActive:true},first:$first,orderBy:id,orderDirection:asc){id encryptedBlob timestamp sequenceId}}`;
3103
+ const variables: Record<string, unknown> = hasLastId
3104
+ ? { owner, first: PAGE_SIZE, lastId }
3105
+ : { owner, first: PAGE_SIZE };
1958
3106
 
1959
3107
  const res = await fetch(`${relayUrl}/v1/subgraph`, {
1960
3108
  method: 'POST',
@@ -1963,24 +3111,36 @@ const plugin = {
1963
3111
  'X-TotalReclaw-Client': 'openclaw-plugin',
1964
3112
  ...(authKeyHex ? { Authorization: `Bearer ${authKeyHex}` } : {}),
1965
3113
  },
1966
- body: JSON.stringify({ query }),
3114
+ body: JSON.stringify({ query, variables }),
1967
3115
  });
1968
3116
 
1969
3117
  const json = (await res.json()) as {
1970
3118
  data?: { facts?: Array<{ id: string; encryptedBlob: string; source: string; agentId: string; timestamp: string; sequenceId: string }> };
3119
+ error?: string;
3120
+ errors?: Array<{ message: string }>;
1971
3121
  };
3122
+ // Surface relay/subgraph errors instead of silently returning empty
3123
+ if (json.error || json.errors) {
3124
+ const errMsg = json.error || json.errors?.map(e => e.message).join('; ') || 'Unknown error';
3125
+ api.logger.error(`Export subgraph query failed: ${errMsg} (owner=${owner}, status=${res.status})`);
3126
+ return {
3127
+ content: [{ type: 'text', text: `Export failed: ${errMsg}` }],
3128
+ };
3129
+ }
1972
3130
  const facts = json?.data?.facts || [];
3131
+ if (facts.length === 0) break;
1973
3132
 
1974
3133
  for (const fact of facts) {
1975
3134
  try {
1976
3135
  let hexBlob = fact.encryptedBlob;
1977
3136
  if (hexBlob.startsWith('0x')) hexBlob = hexBlob.slice(2);
1978
3137
  const docJson = decryptFromHex(hexBlob, encryptionKey!);
1979
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
3138
+ if (isDigestBlob(docJson)) continue;
3139
+ const doc = readClaimFromBlob(docJson);
1980
3140
  allFacts.push({
1981
3141
  id: fact.id,
1982
3142
  text: doc.text,
1983
- metadata: doc.metadata ?? {},
3143
+ metadata: doc.metadata,
1984
3144
  created_at: new Date(parseInt(fact.timestamp) * 1000).toISOString(),
1985
3145
  });
1986
3146
  } catch {
@@ -1988,8 +3148,8 @@ const plugin = {
1988
3148
  }
1989
3149
  }
1990
3150
 
1991
- skip += PAGE_SIZE;
1992
- hasMore = facts.length === PAGE_SIZE;
3151
+ if (facts.length < PAGE_SIZE) break;
3152
+ lastId = facts[facts.length - 1].id;
1993
3153
  }
1994
3154
  } else {
1995
3155
  // HTTP server mode — paginate through PostgreSQL facts
@@ -2002,11 +3162,12 @@ const plugin = {
2002
3162
  for (const fact of page.facts) {
2003
3163
  try {
2004
3164
  const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey!);
2005
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
3165
+ if (isDigestBlob(docJson)) continue;
3166
+ const doc = readClaimFromBlob(docJson);
2006
3167
  allFacts.push({
2007
3168
  id: fact.id,
2008
3169
  text: doc.text,
2009
- metadata: doc.metadata ?? {},
3170
+ metadata: doc.metadata,
2010
3171
  created_at: fact.created_at,
2011
3172
  });
2012
3173
  } catch {
@@ -2048,7 +3209,7 @@ const plugin = {
2048
3209
  const message = err instanceof Error ? err.message : String(err);
2049
3210
  api.logger.error(`totalreclaw_export failed: ${message}`);
2050
3211
  return {
2051
- content: [{ type: 'text', text: `Failed to export memories: ${message}` }],
3212
+ content: [{ type: 'text', text: `Failed to export memories: ${humanizeError(message)}` }],
2052
3213
  };
2053
3214
  }
2054
3215
  },
@@ -2081,7 +3242,7 @@ const plugin = {
2081
3242
  };
2082
3243
  }
2083
3244
 
2084
- const serverUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
3245
+ const serverUrl = CONFIG.serverUrl;
2085
3246
  const walletAddr = subgraphOwner || userId || '';
2086
3247
  const response = await fetch(`${serverUrl}/v1/billing/status?wallet_address=${encodeURIComponent(walletAddr)}`, {
2087
3248
  method: 'GET',
@@ -2134,7 +3295,7 @@ const plugin = {
2134
3295
  const message = err instanceof Error ? err.message : String(err);
2135
3296
  api.logger.error(`totalreclaw_status failed: ${message}`);
2136
3297
  return {
2137
- content: [{ type: 'text', text: `Failed to check status: ${message}` }],
3298
+ content: [{ type: 'text', text: `Failed to check status: ${humanizeError(message)}` }],
2138
3299
  };
2139
3300
  }
2140
3301
  },
@@ -2151,13 +3312,13 @@ const plugin = {
2151
3312
  name: 'totalreclaw_consolidate',
2152
3313
  label: 'Consolidate',
2153
3314
  description:
2154
- 'Scan all stored memories and merge near-duplicates. Keeps the most important/recent version and removes redundant copies.',
3315
+ 'Deduplicate and merge related memories. Self-hosted mode only.',
2155
3316
  parameters: {
2156
3317
  type: 'object',
2157
3318
  properties: {
2158
3319
  dry_run: {
2159
3320
  type: 'boolean',
2160
- description: 'Preview consolidation without deleting (default: false)',
3321
+ description: 'Preview only (default: false)',
2161
3322
  },
2162
3323
  },
2163
3324
  additionalProperties: false,
@@ -2194,11 +3355,10 @@ const plugin = {
2194
3355
  for (const fact of page.facts) {
2195
3356
  try {
2196
3357
  const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey);
2197
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
3358
+ if (isDigestBlob(docJson)) continue;
3359
+ const doc = readClaimFromBlob(docJson);
2198
3360
 
2199
3361
  let embedding: number[] | null = null;
2200
- // ExportedFact does not include encrypted_embedding — generate it on-the-fly.
2201
- // For consolidation we need embeddings, so generate them.
2202
3362
  try {
2203
3363
  embedding = await generateEmbedding(doc.text);
2204
3364
  } catch { /* skip — fact will not be clustered */ }
@@ -2207,9 +3367,7 @@ const plugin = {
2207
3367
  id: fact.id,
2208
3368
  text: doc.text,
2209
3369
  embedding,
2210
- importance: doc.metadata?.importance
2211
- ? Math.round((doc.metadata.importance as number) * 10)
2212
- : 5,
3370
+ importance: doc.importance,
2213
3371
  decayScore: fact.decay_score,
2214
3372
  createdAt: new Date(fact.created_at).getTime(),
2215
3373
  version: fact.version,
@@ -2291,7 +3449,7 @@ const plugin = {
2291
3449
  const message = err instanceof Error ? err.message : String(err);
2292
3450
  api.logger.error(`totalreclaw_consolidate failed: ${message}`);
2293
3451
  return {
2294
- content: [{ type: 'text', text: `Failed to consolidate memories: ${message}` }],
3452
+ content: [{ type: 'text', text: `Failed to consolidate memories: ${humanizeError(message)}` }],
2295
3453
  };
2296
3454
  }
2297
3455
  },
@@ -2299,6 +3457,205 @@ const plugin = {
2299
3457
  { name: 'totalreclaw_consolidate' },
2300
3458
  );
2301
3459
 
3460
+ // ---------------------------------------------------------------
3461
+ // Helper: build PinOpDeps bound to the live plugin state
3462
+ // ---------------------------------------------------------------
3463
+ // Wires the pure pin/unpin operation to the managed-service transport +
3464
+ // crypto layer. Mirrors MCP's buildPinDepsFromState and Python's
3465
+ // _change_claim_status argument plumbing.
3466
+ const buildPinDeps = (): PinOpDeps => {
3467
+ const owner = subgraphOwner || userId || '';
3468
+ const config = {
3469
+ ...getSubgraphConfig(),
3470
+ authKeyHex: authKeyHex!,
3471
+ walletAddress: subgraphOwner ?? undefined,
3472
+ };
3473
+ return {
3474
+ owner,
3475
+ sourceAgent: 'openclaw-plugin',
3476
+ fetchFactById: (factId: string) => fetchFactById(owner, factId, authKeyHex!),
3477
+ decryptBlob: (hex: string) => decryptFromHex(hex, encryptionKey!),
3478
+ encryptBlob: (plaintext: string) => encryptToHex(plaintext, encryptionKey!),
3479
+ submitBatch: async (payloads: Buffer[]) => {
3480
+ const result = await submitFactBatchOnChain(payloads, config);
3481
+ return { txHash: result.txHash, success: result.success };
3482
+ },
3483
+ generateIndices: async (text: string, entityNames: string[]) => {
3484
+ if (!text) return { blindIndices: [] };
3485
+ const wordIndices = generateBlindIndices(text);
3486
+ let lshIndices: string[] = [];
3487
+ let encryptedEmbedding: string | undefined;
3488
+ try {
3489
+ const embedding = await generateEmbedding(text);
3490
+ const hasher = getLSHHasher(api.logger);
3491
+ if (hasher) lshIndices = hasher.hash(embedding);
3492
+ encryptedEmbedding = encryptToHex(JSON.stringify(embedding), encryptionKey!);
3493
+ } catch {
3494
+ // Best-effort: word + entity trapdoors alone still surface the claim.
3495
+ }
3496
+ const entityTrapdoors = entityNames.map((n) => computeEntityTrapdoor(n));
3497
+ return {
3498
+ blindIndices: [...wordIndices, ...lshIndices, ...entityTrapdoors],
3499
+ encryptedEmbedding,
3500
+ };
3501
+ },
3502
+ };
3503
+ };
3504
+
3505
+ // ---------------------------------------------------------------
3506
+ // Tool: totalreclaw_pin
3507
+ // ---------------------------------------------------------------
3508
+
3509
+ api.registerTool(
3510
+ {
3511
+ name: 'totalreclaw_pin',
3512
+ label: 'Pin',
3513
+ description:
3514
+ 'Pin a memory so the auto-resolution engine will never override or supersede it. ' +
3515
+ "Use when the user explicitly confirms a claim is still valid after you or another agent " +
3516
+ "tried to retract/contradict it (e.g. 'wait, I still use Vim sometimes'). " +
3517
+ 'Takes fact_id (from a prior recall result). Pinning is idempotent — pinning an already-pinned ' +
3518
+ 'claim is a no-op. Cross-device: the pin propagates via the on-chain supersession chain.',
3519
+ parameters: {
3520
+ type: 'object',
3521
+ properties: {
3522
+ fact_id: {
3523
+ type: 'string',
3524
+ description: 'The ID of the fact to pin (from a totalreclaw_recall result).',
3525
+ },
3526
+ reason: {
3527
+ type: 'string',
3528
+ description: 'Optional human-readable reason for pinning (logged locally for tuning).',
3529
+ },
3530
+ },
3531
+ required: ['fact_id'],
3532
+ additionalProperties: false,
3533
+ },
3534
+ async execute(_toolCallId: string, params: Record<string, unknown>) {
3535
+ try {
3536
+ await requireFullSetup(api.logger);
3537
+ if (!isSubgraphMode()) {
3538
+ return {
3539
+ content: [{
3540
+ type: 'text',
3541
+ text: 'Pin/unpin is only supported with the managed service. Self-hosted mode does not yet implement the status-flip supersession flow.',
3542
+ }],
3543
+ };
3544
+ }
3545
+ const validation = validatePinArgs(params);
3546
+ if (!validation.ok) {
3547
+ return { content: [{ type: 'text', text: validation.error }] };
3548
+ }
3549
+ const deps = buildPinDeps();
3550
+ const result = await executePinOperation(validation.factId, 'pinned', deps, validation.reason);
3551
+ if (result.success && result.idempotent) {
3552
+ api.logger.info(`totalreclaw_pin: ${result.fact_id} already pinned (no-op)`);
3553
+ return {
3554
+ content: [{ type: 'text', text: `Memory ${result.fact_id} is already pinned.` }],
3555
+ details: result,
3556
+ };
3557
+ }
3558
+ if (result.success) {
3559
+ api.logger.info(`totalreclaw_pin: ${result.fact_id} → ${result.new_fact_id} (tx ${result.tx_hash?.slice(0, 10)})`);
3560
+ return {
3561
+ content: [{
3562
+ type: 'text',
3563
+ text: `Pinned memory ${result.fact_id}. New fact id: ${result.new_fact_id} (tx: ${result.tx_hash}).`,
3564
+ }],
3565
+ details: result,
3566
+ };
3567
+ }
3568
+ api.logger.error(`totalreclaw_pin failed: ${result.error}`);
3569
+ return {
3570
+ content: [{ type: 'text', text: `Failed to pin memory: ${humanizeError(result.error ?? 'unknown error')}` }],
3571
+ details: result,
3572
+ };
3573
+ } catch (err: unknown) {
3574
+ const message = err instanceof Error ? err.message : String(err);
3575
+ api.logger.error(`totalreclaw_pin failed: ${message}`);
3576
+ return {
3577
+ content: [{ type: 'text', text: `Failed to pin memory: ${humanizeError(message)}` }],
3578
+ };
3579
+ }
3580
+ },
3581
+ },
3582
+ { name: 'totalreclaw_pin' },
3583
+ );
3584
+
3585
+ // ---------------------------------------------------------------
3586
+ // Tool: totalreclaw_unpin
3587
+ // ---------------------------------------------------------------
3588
+
3589
+ api.registerTool(
3590
+ {
3591
+ name: 'totalreclaw_unpin',
3592
+ label: 'Unpin',
3593
+ description:
3594
+ 'Remove the pin from a previously pinned memory, returning it to active status so the ' +
3595
+ 'auto-resolution engine can supersede or retract it again. Takes fact_id. Idempotent — ' +
3596
+ 'unpinning a non-pinned claim is a no-op.',
3597
+ parameters: {
3598
+ type: 'object',
3599
+ properties: {
3600
+ fact_id: {
3601
+ type: 'string',
3602
+ description: 'The ID of the fact to unpin (from a totalreclaw_recall result).',
3603
+ },
3604
+ },
3605
+ required: ['fact_id'],
3606
+ additionalProperties: false,
3607
+ },
3608
+ async execute(_toolCallId: string, params: Record<string, unknown>) {
3609
+ try {
3610
+ await requireFullSetup(api.logger);
3611
+ if (!isSubgraphMode()) {
3612
+ return {
3613
+ content: [{
3614
+ type: 'text',
3615
+ text: 'Pin/unpin is only supported with the managed service. Self-hosted mode does not yet implement the status-flip supersession flow.',
3616
+ }],
3617
+ };
3618
+ }
3619
+ const validation = validatePinArgs(params);
3620
+ if (!validation.ok) {
3621
+ return { content: [{ type: 'text', text: validation.error }] };
3622
+ }
3623
+ const deps = buildPinDeps();
3624
+ const result = await executePinOperation(validation.factId, 'active', deps);
3625
+ if (result.success && result.idempotent) {
3626
+ api.logger.info(`totalreclaw_unpin: ${result.fact_id} already active (no-op)`);
3627
+ return {
3628
+ content: [{ type: 'text', text: `Memory ${result.fact_id} is not pinned.` }],
3629
+ details: result,
3630
+ };
3631
+ }
3632
+ if (result.success) {
3633
+ api.logger.info(`totalreclaw_unpin: ${result.fact_id} → ${result.new_fact_id} (tx ${result.tx_hash?.slice(0, 10)})`);
3634
+ return {
3635
+ content: [{
3636
+ type: 'text',
3637
+ text: `Unpinned memory ${result.fact_id}. New fact id: ${result.new_fact_id} (tx: ${result.tx_hash}).`,
3638
+ }],
3639
+ details: result,
3640
+ };
3641
+ }
3642
+ api.logger.error(`totalreclaw_unpin failed: ${result.error}`);
3643
+ return {
3644
+ content: [{ type: 'text', text: `Failed to unpin memory: ${humanizeError(result.error ?? 'unknown error')}` }],
3645
+ details: result,
3646
+ };
3647
+ } catch (err: unknown) {
3648
+ const message = err instanceof Error ? err.message : String(err);
3649
+ api.logger.error(`totalreclaw_unpin failed: ${message}`);
3650
+ return {
3651
+ content: [{ type: 'text', text: `Failed to unpin memory: ${humanizeError(message)}` }],
3652
+ };
3653
+ }
3654
+ },
3655
+ },
3656
+ { name: 'totalreclaw_unpin' },
3657
+ );
3658
+
2302
3659
  // ---------------------------------------------------------------
2303
3660
  // Tool: totalreclaw_import_from
2304
3661
  // ---------------------------------------------------------------
@@ -2308,16 +3665,16 @@ const plugin = {
2308
3665
  name: 'totalreclaw_import_from',
2309
3666
  label: 'Import From',
2310
3667
  description:
2311
- 'Import memories from other AI memory tools (Mem0, MCP Memory Server, MemoClaw, or generic JSON/CSV). ' +
2312
- 'Provide the source name and either an API key or file content. ' +
3668
+ 'Import memories from other AI memory tools (Mem0, MCP Memory Server, ChatGPT, Claude, Gemini, MemoClaw, or generic JSON/CSV). ' +
3669
+ 'Provide the source name and either an API key, file content, or file path. ' +
2313
3670
  'Use dry_run=true to preview before importing. Idempotent — safe to run multiple times.',
2314
3671
  parameters: {
2315
3672
  type: 'object',
2316
3673
  properties: {
2317
3674
  source: {
2318
3675
  type: 'string',
2319
- enum: ['mem0', 'mcp-memory', 'memoclaw', 'generic-json', 'generic-csv'],
2320
- description: 'The source system to import from',
3676
+ enum: ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini', 'memoclaw', 'generic-json', 'generic-csv'],
3677
+ description: 'The source system to import from (gemini: Google Takeout HTML; chatgpt: conversations.json or memory text; claude: memory text)',
2321
3678
  },
2322
3679
  api_key: {
2323
3680
  type: 'string',
@@ -2359,6 +3716,56 @@ const plugin = {
2359
3716
  { name: 'totalreclaw_import_from' },
2360
3717
  );
2361
3718
 
3719
+ // ---------------------------------------------------------------
3720
+ // Tool: totalreclaw_import_batch
3721
+ // ---------------------------------------------------------------
3722
+
3723
+ api.registerTool(
3724
+ {
3725
+ name: 'totalreclaw_import_batch',
3726
+ label: 'Import Batch',
3727
+ description:
3728
+ 'Process one batch of a large import. Call repeatedly with increasing offset until is_complete=true.',
3729
+ parameters: {
3730
+ type: 'object',
3731
+ properties: {
3732
+ source: {
3733
+ type: 'string',
3734
+ enum: ['gemini', 'chatgpt', 'claude'],
3735
+ description: 'Source format',
3736
+ },
3737
+ file_path: {
3738
+ type: 'string',
3739
+ description: 'Path to source file',
3740
+ },
3741
+ content: {
3742
+ type: 'string',
3743
+ description: 'File content (text sources)',
3744
+ },
3745
+ offset: {
3746
+ type: 'number',
3747
+ description: 'Starting chunk index (0-based)',
3748
+ },
3749
+ batch_size: {
3750
+ type: 'number',
3751
+ description: 'Chunks per call (default 25)',
3752
+ },
3753
+ },
3754
+ required: ['source'],
3755
+ },
3756
+ async execute(_toolCallId: string, params: Record<string, unknown>) {
3757
+ try {
3758
+ await requireFullSetup(api.logger);
3759
+ return handleBatchImport(params, api.logger);
3760
+ } catch (err: unknown) {
3761
+ const message = err instanceof Error ? err.message : String(err);
3762
+ return { error: message };
3763
+ }
3764
+ },
3765
+ },
3766
+ { name: 'totalreclaw_import_batch' },
3767
+ );
3768
+
2362
3769
  // ---------------------------------------------------------------
2363
3770
  // Tool: totalreclaw_upgrade
2364
3771
  // ---------------------------------------------------------------
@@ -2385,7 +3792,7 @@ const plugin = {
2385
3792
  };
2386
3793
  }
2387
3794
 
2388
- const serverUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
3795
+ const serverUrl = CONFIG.serverUrl;
2389
3796
  const walletAddr = subgraphOwner || userId || '';
2390
3797
 
2391
3798
  if (!walletAddr) {
@@ -2430,7 +3837,7 @@ const plugin = {
2430
3837
  const message = err instanceof Error ? err.message : String(err);
2431
3838
  api.logger.error(`totalreclaw_upgrade failed: ${message}`);
2432
3839
  return {
2433
- content: [{ type: 'text', text: `Failed to create checkout session: ${message}` }],
3840
+ content: [{ type: 'text', text: `Failed to create checkout session: ${humanizeError(message)}` }],
2434
3841
  };
2435
3842
  }
2436
3843
  },
@@ -2477,7 +3884,7 @@ const plugin = {
2477
3884
  }
2478
3885
 
2479
3886
  const confirm = _params?.confirm === true;
2480
- const serverUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
3887
+ const serverUrl = CONFIG.serverUrl;
2481
3888
 
2482
3889
  // 1. Check billing tier
2483
3890
  const billingResp = await fetch(
@@ -2564,6 +3971,7 @@ const plugin = {
2564
3971
  contentFp: fact.contentFp || '',
2565
3972
  agentId: fact.agentId || 'openclaw-plugin',
2566
3973
  encryptedEmbedding: fact.encryptedEmbedding || undefined,
3974
+ version: PROTOBUF_VERSION_V4,
2567
3975
  };
2568
3976
  payloads.push(encodeFactProtobuf(factPayload));
2569
3977
  }
@@ -2613,7 +4021,7 @@ const plugin = {
2613
4021
  const message = err instanceof Error ? err.message : String(err);
2614
4022
  api.logger.error(`totalreclaw_migrate failed: ${message}`);
2615
4023
  return {
2616
- content: [{ type: 'text', text: `Migration failed: ${message}` }],
4024
+ content: [{ type: 'text', text: `Migration failed: ${humanizeError(message)}` }],
2617
4025
  };
2618
4026
  }
2619
4027
  },
@@ -2621,6 +4029,107 @@ const plugin = {
2621
4029
  { name: 'totalreclaw_migrate' },
2622
4030
  );
2623
4031
 
4032
+ // ---------------------------------------------------------------
4033
+ // Tool: totalreclaw_setup
4034
+ // ---------------------------------------------------------------
4035
+
4036
+ api.registerTool(
4037
+ {
4038
+ name: 'totalreclaw_setup',
4039
+ label: 'Setup TotalReclaw',
4040
+ description:
4041
+ 'Initialize TotalReclaw with a recovery phrase. Derives encryption keys and registers with the server. ' +
4042
+ 'Use this during first-time setup instead of setting environment variables — no gateway restart needed.',
4043
+ parameters: {
4044
+ type: 'object',
4045
+ properties: {
4046
+ recovery_phrase: {
4047
+ type: 'string',
4048
+ description: 'Optional 12-word BIP-39 recovery phrase. If not provided, generates a new one automatically. For returning users, pass their existing phrase.',
4049
+ },
4050
+ },
4051
+ additionalProperties: false,
4052
+ },
4053
+ async execute(_toolCallId: string, params: { recovery_phrase?: string }) {
4054
+ try {
4055
+ let mnemonic = params.recovery_phrase?.trim() || '';
4056
+
4057
+ // Auto-generate if not provided
4058
+ if (!mnemonic) {
4059
+ const { generateMnemonic } = await import('@scure/bip39');
4060
+ const { wordlist } = await import('@scure/bip39/wordlists/english');
4061
+ mnemonic = generateMnemonic(wordlist, 128);
4062
+ api.logger.info('totalreclaw_setup: generated new BIP-39 mnemonic');
4063
+ }
4064
+
4065
+ // Guard: refuse to overwrite existing credentials with a DIFFERENT phrase
4066
+ // (prevents data loss when background sessions_spawn workers call setup).
4067
+ // Allow re-init with the SAME phrase (handles agent exec → setup flow).
4068
+ try {
4069
+ const existing = fs.readFileSync(CREDENTIALS_PATH, 'utf-8');
4070
+ const creds = JSON.parse(existing);
4071
+ if (creds.mnemonic && creds.userId && creds.mnemonic !== mnemonic) {
4072
+ api.logger.info('totalreclaw_setup: credentials exist with different mnemonic, refusing to overwrite');
4073
+ return {
4074
+ content: [{
4075
+ type: 'text',
4076
+ text: 'TotalReclaw is already set up with an existing recovery phrase. Your encrypted memories are tied to that phrase.\n\n' +
4077
+ 'If you intentionally want to start fresh with a NEW phrase (this will make existing memories inaccessible), ' +
4078
+ 'delete ~/.totalreclaw/credentials.json first, then call this tool again.',
4079
+ }],
4080
+ };
4081
+ }
4082
+ } catch { /* credentials.json doesn't exist or is corrupted — proceed with setup */ }
4083
+
4084
+ // Basic validation: must be 12 words
4085
+ const words = mnemonic.split(/\s+/);
4086
+ if (words.length !== 12) {
4087
+ return {
4088
+ content: [{
4089
+ type: 'text',
4090
+ text: `Error: Recovery phrase must be exactly 12 words (got ${words.length}). Use \`npx @totalreclaw/mcp-server setup\` to generate a valid BIP-39 mnemonic.`,
4091
+ }],
4092
+ };
4093
+ }
4094
+
4095
+ api.logger.info('totalreclaw_setup: initializing with provided recovery phrase');
4096
+
4097
+ // Force re-initialization with the new mnemonic.
4098
+ // This derives keys, registers with the server, saves credentials,
4099
+ // and sets up LSH/auth — all without a gateway restart.
4100
+ await forceReinitialization(mnemonic, api.logger);
4101
+
4102
+ if (needsSetup) {
4103
+ return {
4104
+ content: [{
4105
+ type: 'text',
4106
+ text: 'Setup failed — could not initialize with the provided recovery phrase. Check the logs for details.',
4107
+ }],
4108
+ };
4109
+ }
4110
+
4111
+ const wasGenerated = !params.recovery_phrase?.trim();
4112
+ return {
4113
+ content: [{
4114
+ type: 'text',
4115
+ text: 'TotalReclaw setup complete! Encryption keys derived, server registration confirmed. ' +
4116
+ 'You can now use totalreclaw_remember, totalreclaw_recall, and all other tools immediately — no restart needed.\n\n' +
4117
+ (wasGenerated ? `Recovery phrase: ${mnemonic}\n\n` : '') +
4118
+ 'From now on, I will automatically remember important things from our conversations and recall relevant context at the start of each session.',
4119
+ }],
4120
+ };
4121
+ } catch (err: unknown) {
4122
+ const message = err instanceof Error ? err.message : String(err);
4123
+ api.logger.error(`totalreclaw_setup failed: ${message}`);
4124
+ return {
4125
+ content: [{ type: 'text', text: `Setup failed: ${humanizeError(message)}` }],
4126
+ };
4127
+ }
4128
+ },
4129
+ },
4130
+ { name: 'totalreclaw_setup' },
4131
+ );
4132
+
2624
4133
  // ---------------------------------------------------------------
2625
4134
  // Hook: before_agent_start
2626
4135
  // ---------------------------------------------------------------
@@ -2641,21 +4150,37 @@ const plugin = {
2641
4150
 
2642
4151
  await ensureInitialized(api.logger);
2643
4152
 
2644
- // If setup is still needed after init, inject setup instructions
4153
+ // If setup is still needed after init, inject guided onboarding
2645
4154
  if (needsSetup) {
2646
4155
  return {
2647
4156
  prependContext:
2648
- '## TotalReclaw Setup Required\n\n' +
2649
- 'TotalReclaw is installed but needs configuration. Follow the "Post-Install Setup" steps in SKILL.md to complete setup.\n' +
2650
- 'Ask the user: "Do you have an existing TotalReclaw recovery phrase, or should I generate a new one?"',
4157
+ '## TotalReclaw — First-Time Setup\n\n' +
4158
+ 'TotalReclaw is an end-to-end encrypted memory vault. It remembers things across conversations preferences, decisions, project context — all encrypted so no one (not even the server) can read your memories.\n\n' +
4159
+ '**To get started, ask the user:** "Do you have an existing TotalReclaw recovery phrase you\'d like to restore, or should I generate a new one?"\n\n' +
4160
+ '**If new user:** Run `npx @totalreclaw/mcp-server setup` to generate a cryptographically valid recovery phrase. Do NOT generate the phrase yourself — LLMs cannot compute BIP-39 checksums. The command will output the phrase. Then call the `totalreclaw_setup` tool with the phrase. Display the phrase prominently and warn:\n' +
4161
+ '- "This is your ONLY way to access your memories. Write it down and store it securely."\n' +
4162
+ '- "NEVER use a recovery phrase from an existing crypto wallet — keep this separate from any wallet that holds funds."\n' +
4163
+ '- "If you lose this phrase, your memories are gone forever — there is no recovery."\n\n' +
4164
+ '**If returning user:** Ask them to provide their 12-word phrase, then call `totalreclaw_setup` with that phrase.\n\n' +
4165
+ '**IMPORTANT:** Use the `totalreclaw_setup` tool to complete setup. This initializes encryption keys immediately — no gateway restart needed. The user can continue chatting right away.\n\n' +
4166
+ '**After setup:** Explain that from now on, you will automatically remember important things from conversations and recall relevant memories at the start of each session. The user can also explicitly ask you to remember, recall, forget, or export memories.',
2651
4167
  };
2652
4168
  }
2653
4169
 
2654
- // One-time welcome-back message for returning Pro users.
4170
+ // One-time welcome message (first conversation after setup or returning user)
2655
4171
  let welcomeBack = '';
2656
4172
  if (welcomeBackMessage) {
2657
4173
  welcomeBack = `\n\n${welcomeBackMessage}`;
2658
4174
  welcomeBackMessage = null; // Consume — only show once
4175
+ } else if (firstRunAfterInit) {
4176
+ // First conversation with a configured user — explain what's happening
4177
+ firstRunAfterInit = false;
4178
+ const cache = readBillingCache();
4179
+ const tier = cache?.tier || 'free';
4180
+ const tierInfo = tier === 'pro'
4181
+ ? 'You are on the **Pro** tier — unlimited memories, permanently stored on Gnosis mainnet.'
4182
+ : 'You are on the **Free** tier — memories stored on testnet. Use the totalreclaw_upgrade tool to upgrade to Pro for permanent on-chain storage.';
4183
+ welcomeBack = `\n\nTotalReclaw is active. I will automatically remember important things from our conversations and recall relevant context at the start of each session. ${tierInfo}`;
2659
4184
  }
2660
4185
 
2661
4186
  // Billing cache check — warn if quota is approaching limit.
@@ -2664,7 +4189,7 @@ const plugin = {
2664
4189
  let cache = readBillingCache();
2665
4190
  if (!cache && authKeyHex) {
2666
4191
  // Cache is stale or missing — fetch fresh billing status.
2667
- const billingUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
4192
+ const billingUrl = CONFIG.serverUrl;
2668
4193
  const walletParam = encodeURIComponent(subgraphOwner || userId || '');
2669
4194
  const billingResp = await fetch(`${billingUrl}/v1/billing/status?wallet_address=${walletParam}`, {
2670
4195
  method: 'GET',
@@ -2693,7 +4218,46 @@ const plugin = {
2693
4218
  }
2694
4219
 
2695
4220
  if (isSubgraphMode()) {
2696
- // --- Subgraph mode: hot cache first, then background refresh ---
4221
+ // --- Subgraph mode: digest fast path → hot cache background refresh ---
4222
+
4223
+ // Digest fast path (Stage 3b). When a digest exists and the mode is
4224
+ // not 'off', inject its pre-compiled promptText instead of running
4225
+ // the per-query search. A stale digest triggers a background
4226
+ // recompile (non-blocking). Failures fall through to the legacy
4227
+ // path silently.
4228
+ const digestMode = resolveDigestMode();
4229
+ logDigestModeOnce(digestMode, api.logger);
4230
+ if (digestMode !== 'off' && encryptionKey && authKeyHex && (subgraphOwner || userId)) {
4231
+ try {
4232
+ const injectResult = await maybeInjectDigest({
4233
+ owner: subgraphOwner || userId!,
4234
+ authKeyHex: authKeyHex!,
4235
+ encryptionKey: encryptionKey!,
4236
+ mode: digestMode,
4237
+ nowMs: Date.now(),
4238
+ loadDeps: {
4239
+ searchSubgraph: async (o, tds, n, a) => searchSubgraph(o, tds, n, a),
4240
+ decryptFromHex: (hex, key) => decryptFromHex(hex, key),
4241
+ },
4242
+ probeDeps: {
4243
+ searchSubgraphBroadened: async (o, n, a) => searchSubgraphBroadened(o, n, a),
4244
+ },
4245
+ recompileFn: (prev) => scheduleDigestRecompile(prev, api.logger),
4246
+ logger: api.logger,
4247
+ });
4248
+ if (injectResult.promptText) {
4249
+ api.logger.info(`Digest injection: state=${injectResult.state}`);
4250
+ return {
4251
+ prependContext:
4252
+ `## Your Memory\n\n${injectResult.promptText}` + welcomeBack + billingWarning,
4253
+ };
4254
+ }
4255
+ } catch (err) {
4256
+ // Never block session start on digest failure.
4257
+ const msg = err instanceof Error ? err.message : String(err);
4258
+ api.logger.warn(`Digest fast path failed: ${msg}`);
4259
+ }
4260
+ }
2697
4261
 
2698
4262
  // Initialize hot cache if needed.
2699
4263
  if (!pluginHotCache && encryptionKey) {
@@ -2766,6 +4330,21 @@ const plugin = {
2766
4330
  return undefined;
2767
4331
  }
2768
4332
 
4333
+ // Always run broadened search and merge — ensures vocabulary mismatches
4334
+ // (e.g., "preferences" vs "prefer") don't cause recall failures.
4335
+ // The reranker handles scoring; extra cost is ~1 GraphQL query per recall.
4336
+ try {
4337
+ const broadPool = computeCandidatePool(0);
4338
+ const broadenedResults = await searchSubgraphBroadened(subgraphOwner || userId!, broadPool, authKeyHex!);
4339
+ // Merge broadened results with existing (deduplicate by ID)
4340
+ const existingIds = new Set(subgraphResults.map(r => r.id));
4341
+ for (const br of broadenedResults) {
4342
+ if (!existingIds.has(br.id)) {
4343
+ subgraphResults.push(br);
4344
+ }
4345
+ }
4346
+ } catch { /* best-effort */ }
4347
+
2769
4348
  if (subgraphResults.length === 0 && cachedFacts.length === 0) return undefined;
2770
4349
 
2771
4350
  // If subgraph returned no results but we have cache, use cache.
@@ -2783,7 +4362,10 @@ const plugin = {
2783
4362
  for (const result of subgraphResults) {
2784
4363
  try {
2785
4364
  const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
2786
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
4365
+ // Filter out digest infrastructure blobs they have no user
4366
+ // text and should never surface in recall results.
4367
+ if (isDigestBlob(docJson)) continue;
4368
+ const doc = readClaimFromBlob(docJson);
2787
4369
 
2788
4370
  let decryptedEmbedding: number[] | undefined;
2789
4371
  if (result.encryptedEmbedding) {
@@ -2796,22 +4378,20 @@ const plugin = {
2796
4378
  }
2797
4379
  }
2798
4380
 
2799
- const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
2800
4381
  const createdAtSec = result.timestamp ? parseInt(result.timestamp, 10) : undefined;
2801
4382
  rerankerCandidates.push({
2802
4383
  id: result.id,
2803
4384
  text: doc.text,
2804
4385
  embedding: decryptedEmbedding,
2805
- importance: importanceRaw,
4386
+ importance: doc.importance / 10,
2806
4387
  createdAt: createdAtSec,
4388
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
2807
4389
  });
2808
4390
 
2809
- const importance = doc.metadata?.importance
2810
- ? Math.round((doc.metadata.importance as number) * 10)
2811
- : 5;
2812
4391
  hookMetaMap.set(result.id, {
2813
- importance,
4392
+ importance: doc.importance,
2814
4393
  age: 'subgraph',
4394
+ category: doc.category,
2815
4395
  });
2816
4396
  } catch {
2817
4397
  // Skip un-decryptable candidates.
@@ -2826,17 +4406,9 @@ const plugin = {
2826
4406
  rerankerCandidates,
2827
4407
  8,
2828
4408
  INTENT_WEIGHTS[hookQueryIntent],
4409
+ /* applySourceWeights (Retrieval v2 Tier 1) */ true,
2829
4410
  );
2830
4411
 
2831
- // B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
2832
- const candidatesWithEmb = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
2833
- if (candidatesWithEmb.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
2834
- const topCosine = Math.max(
2835
- ...candidatesWithEmb.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
2836
- );
2837
- if (topCosine < RELEVANCE_THRESHOLD) return undefined;
2838
- }
2839
-
2840
4412
  // Update hot cache with reranked results.
2841
4413
  try {
2842
4414
  if (pluginHotCache) {
@@ -2875,7 +4447,8 @@ const plugin = {
2875
4447
  const meta = hookMetaMap.get(m.id);
2876
4448
  const importance = meta?.importance ?? 5;
2877
4449
  const age = meta?.age ?? '';
2878
- return `${i + 1}. ${m.text} (importance: ${importance}/10, ${age})`;
4450
+ const typeTag = meta?.category ? `[${meta.category}] ` : '';
4451
+ return `${i + 1}. ${typeTag}${m.text} (importance: ${importance}/10, ${age})`;
2879
4452
  });
2880
4453
  const contextString = `## Relevant Memories\n\n${lines.join('\n')}`;
2881
4454
 
@@ -2923,9 +4496,10 @@ const plugin = {
2923
4496
  for (const candidate of candidates) {
2924
4497
  try {
2925
4498
  const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
2926
- const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
4499
+ // Skip digest infrastructure blobs.
4500
+ if (isDigestBlob(docJson)) continue;
4501
+ const doc = readClaimFromBlob(docJson);
2927
4502
 
2928
- // Decrypt embedding if present.
2929
4503
  let decryptedEmbedding: number[] | undefined;
2930
4504
  if (candidate.encrypted_embedding) {
2931
4505
  try {
@@ -2937,7 +4511,6 @@ const plugin = {
2937
4511
  }
2938
4512
  }
2939
4513
 
2940
- const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
2941
4514
  const createdAtSec = typeof candidate.timestamp === 'number'
2942
4515
  ? candidate.timestamp / 1000
2943
4516
  : new Date(candidate.timestamp).getTime() / 1000;
@@ -2945,15 +4518,13 @@ const plugin = {
2945
4518
  id: candidate.fact_id,
2946
4519
  text: doc.text,
2947
4520
  embedding: decryptedEmbedding,
2948
- importance: importanceRaw,
4521
+ importance: doc.importance / 10,
2949
4522
  createdAt: createdAtSec,
4523
+ source: typeof doc.metadata?.source === 'string' ? doc.metadata.source : undefined,
2950
4524
  });
2951
4525
 
2952
- const importance = doc.metadata?.importance
2953
- ? Math.round((doc.metadata.importance as number) * 10)
2954
- : 5;
2955
4526
  hookMetaMap.set(candidate.fact_id, {
2956
- importance,
4527
+ importance: doc.importance,
2957
4528
  age: relativeTime(candidate.timestamp),
2958
4529
  });
2959
4530
  } catch {
@@ -2969,19 +4540,23 @@ const plugin = {
2969
4540
  rerankerCandidates,
2970
4541
  8,
2971
4542
  INTENT_WEIGHTS[srvHookIntent],
2972
- );
2973
-
2974
- // B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
2975
- const candidatesWithEmbSrv = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
2976
- if (candidatesWithEmbSrv.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
2977
- const topCosine = Math.max(
2978
- ...candidatesWithEmbSrv.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
4543
+ /* applySourceWeights (Retrieval v2 Tier 1) */ true,
2979
4544
  );
2980
- if (topCosine < RELEVANCE_THRESHOLD) return undefined;
2981
- }
2982
4545
 
2983
4546
  if (reranked.length === 0) return undefined;
2984
4547
 
4548
+ // Cosine similarity threshold gate — skip injection when the
4549
+ // best match is below the minimum relevance threshold.
4550
+ const srvMaxCosine = Math.max(
4551
+ ...reranked.map((r) => r.cosineSimilarity ?? 0),
4552
+ );
4553
+ if (srvMaxCosine < COSINE_THRESHOLD) {
4554
+ api.logger.info(
4555
+ `Hook: cosine threshold gate filtered results (max=${srvMaxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD})`,
4556
+ );
4557
+ return undefined;
4558
+ }
4559
+
2985
4560
  // 7. Build context string.
2986
4561
  const lines = reranked.map((m, i) => {
2987
4562
  const meta = hookMetaMap.get(m.id);
@@ -3009,21 +4584,73 @@ const plugin = {
3009
4584
  api.on(
3010
4585
  'agent_end',
3011
4586
  async (event: unknown) => {
4587
+ // CRITICAL: Always return { memoryHandled: true } so OpenClaw's default
4588
+ // memory system does NOT fall back to writing plaintext MEMORY.md.
4589
+ // Losing facts on error is acceptable; leaking them in cleartext is not.
3012
4590
  try {
4591
+ // Defensive: ensure MEMORY.md header is present so OpenClaw's default
4592
+ // memory system doesn't write sensitive data in cleartext, even if
4593
+ // our extraction fails below.
4594
+ ensureMemoryHeader(api.logger);
4595
+
4596
+ // BUG-2 fix: skip extraction if an import was in progress this turn.
4597
+ // Import failures were retriggering agent_end → extraction → import loops.
4598
+ if (_importInProgress) {
4599
+ _importInProgress = false; // auto-reset for next turn
4600
+ api.logger.info('agent_end: skipping extraction (import was in progress)');
4601
+ return { memoryHandled: true };
4602
+ }
4603
+
3013
4604
  const evt = event as { messages?: unknown[]; success?: boolean } | undefined;
3014
- if (!evt?.success || !evt?.messages || evt.messages.length < 2) return;
4605
+ if (!evt?.messages || evt.messages.length < 2) {
4606
+ api.logger.info('agent_end: skipping extraction (no messages)');
4607
+ return { memoryHandled: true };
4608
+ }
4609
+ // Proceed with extraction even when evt.success is false or undefined.
4610
+ // A single LLM timeout on one turn should not prevent extraction of
4611
+ // facts from the (potentially many) successful turns in the message
4612
+ // history. The extractor processes the full message array and can
4613
+ // extract valuable facts from content before the failure.
4614
+ if (evt.success === false) {
4615
+ api.logger.info('agent_end: turn reported failure, but proceeding with extraction from message history');
4616
+ }
3015
4617
 
3016
4618
  await ensureInitialized(api.logger);
3017
- if (needsSetup) return;
4619
+ if (needsSetup) return { memoryHandled: true };
3018
4620
 
3019
4621
  // C3: Throttle auto-extraction to every N turns (configurable via env).
4622
+ // Phase 2.2.5: every branch of the extraction pipeline now logs its
4623
+ // outcome. Prior to 2.2.5, only the "stored N facts" happy path
4624
+ // produced a log line, so silent JSON parse failures / chatCompletion
4625
+ // timeouts / importance-filter-drops-everything scenarios left no
4626
+ // trace whatsoever in the gateway log. See the investigation report
4627
+ // in CHANGELOG for the full failure chain we uncovered.
3020
4628
  turnsSinceLastExtraction++;
3021
- if (turnsSinceLastExtraction >= getExtractInterval()) {
4629
+ const extractInterval = getExtractInterval();
4630
+ api.logger.info(
4631
+ `agent_end: turn ${turnsSinceLastExtraction}/${extractInterval} (messages=${evt.messages.length})`,
4632
+ );
4633
+ if (turnsSinceLastExtraction >= extractInterval) {
3022
4634
  const existingMemories = isLlmDedupEnabled()
3023
4635
  ? await fetchExistingMemoriesForExtraction(api.logger, 20, evt.messages)
3024
4636
  : [];
3025
- const rawFacts = await extractFacts(evt.messages, 'turn', existingMemories);
3026
- const { kept: importanceFiltered } = filterByImportance(rawFacts, api.logger);
4637
+ const rawFacts = await extractFacts(
4638
+ evt.messages,
4639
+ 'turn',
4640
+ existingMemories,
4641
+ undefined,
4642
+ api.logger,
4643
+ );
4644
+ api.logger.info(
4645
+ `agent_end: extractFacts returned ${rawFacts.length} raw facts`,
4646
+ );
4647
+ const { kept: importanceFiltered, dropped } = filterByImportance(
4648
+ rawFacts,
4649
+ api.logger,
4650
+ );
4651
+ api.logger.info(
4652
+ `agent_end: after importance filter: kept=${importanceFiltered.length}, dropped=${dropped}`,
4653
+ );
3027
4654
  const maxFacts = getMaxFactsPerExtraction();
3028
4655
  if (importanceFiltered.length > maxFacts) {
3029
4656
  api.logger.info(
@@ -3033,13 +4660,23 @@ const plugin = {
3033
4660
  const facts = importanceFiltered.slice(0, maxFacts);
3034
4661
  if (facts.length > 0) {
3035
4662
  await storeExtractedFacts(facts, api.logger);
4663
+ api.logger.info(`agent_end: stored ${facts.length} facts to encrypted vault`);
4664
+ } else {
4665
+ // Phase 2.2.5: no longer silent when extraction produces nothing.
4666
+ api.logger.info(
4667
+ `agent_end: extraction produced 0 storable facts (raw=${rawFacts.length}, after-importance=${importanceFiltered.length})`,
4668
+ );
3036
4669
  }
3037
4670
  turnsSinceLastExtraction = 0;
3038
4671
  }
3039
4672
  } catch (err: unknown) {
3040
4673
  const message = err instanceof Error ? err.message : String(err);
3041
- api.logger.warn(`agent_end extraction failed: ${message}`);
4674
+ api.logger.error(`agent_end extraction failed: ${message}`);
4675
+ // Re-assert MEMORY.md header even on failure — last line of defense.
4676
+ ensureMemoryHeader(api.logger);
3042
4677
  }
4678
+ // Always signal that memory is handled — prevent plaintext fallback.
4679
+ return { memoryHandled: true };
3043
4680
  },
3044
4681
  { priority: 90 },
3045
4682
  );
@@ -3059,13 +4696,13 @@ const plugin = {
3059
4696
  if (needsSetup) return;
3060
4697
 
3061
4698
  api.logger.info(
3062
- `Pre-compaction extraction: processing ${evt.messages.length} messages`,
4699
+ `pre_compaction: using compaction-aware extraction (importance >= 5), processing ${evt.messages.length} messages`,
3063
4700
  );
3064
4701
 
3065
4702
  const existingMemories = isLlmDedupEnabled()
3066
4703
  ? await fetchExistingMemoriesForExtraction(api.logger, 50, evt.messages)
3067
4704
  : [];
3068
- const rawCompactFacts = await extractFacts(evt.messages, 'full', existingMemories);
4705
+ const rawCompactFacts = await extractFactsForCompaction(evt.messages, existingMemories, api.logger);
3069
4706
  const { kept: compactImportanceFiltered } = filterByImportance(rawCompactFacts, api.logger);
3070
4707
  const maxFactsCompact = getMaxFactsPerExtraction();
3071
4708
  if (compactImportanceFiltered.length > maxFactsCompact) {
@@ -3078,6 +4715,29 @@ const plugin = {
3078
4715
  await storeExtractedFacts(facts, api.logger);
3079
4716
  }
3080
4717
  turnsSinceLastExtraction = 0; // Reset C3 counter on compaction.
4718
+
4719
+ // Session debrief — after regular extraction.
4720
+ // v1 mapping: DebriefItem { type: 'summary'|'context' } →
4721
+ // v1 type 'summary' (always, since context → claim would lose
4722
+ // the "this is a session summary" signal) + source 'derived'
4723
+ // (session debrief is a derived synthesis by definition).
4724
+ try {
4725
+ const storedTexts = facts.map((f) => f.text);
4726
+ const debriefItems = await extractDebrief(evt.messages, storedTexts);
4727
+ if (debriefItems.length > 0) {
4728
+ const debriefFacts: ExtractedFact[] = debriefItems.map((d) => ({
4729
+ text: d.text,
4730
+ type: 'summary' as MemoryType,
4731
+ source: 'derived' as MemorySource,
4732
+ importance: d.importance,
4733
+ action: 'ADD' as const,
4734
+ }));
4735
+ await storeExtractedFacts(debriefFacts, api.logger, 'openclaw_debrief');
4736
+ api.logger.info(`Session debrief: stored ${debriefItems.length} items`);
4737
+ }
4738
+ } catch (debriefErr: unknown) {
4739
+ api.logger.warn(`before_compaction debrief failed: ${debriefErr instanceof Error ? debriefErr.message : String(debriefErr)}`);
4740
+ }
3081
4741
  } catch (err: unknown) {
3082
4742
  const message = err instanceof Error ? err.message : String(err);
3083
4743
  api.logger.warn(`before_compaction extraction failed: ${message}`);
@@ -3120,6 +4780,29 @@ const plugin = {
3120
4780
  await storeExtractedFacts(facts, api.logger);
3121
4781
  }
3122
4782
  turnsSinceLastExtraction = 0; // Reset C3 counter on reset.
4783
+
4784
+ // Session debrief — after regular extraction.
4785
+ // v1 mapping: DebriefItem { type: 'summary'|'context' } →
4786
+ // v1 type 'summary' (always, since context → claim would lose
4787
+ // the "this is a session summary" signal) + source 'derived'
4788
+ // (session debrief is a derived synthesis by definition).
4789
+ try {
4790
+ const storedTexts = facts.map((f) => f.text);
4791
+ const debriefItems = await extractDebrief(evt.messages, storedTexts);
4792
+ if (debriefItems.length > 0) {
4793
+ const debriefFacts: ExtractedFact[] = debriefItems.map((d) => ({
4794
+ text: d.text,
4795
+ type: 'summary' as MemoryType,
4796
+ source: 'derived' as MemorySource,
4797
+ importance: d.importance,
4798
+ action: 'ADD' as const,
4799
+ }));
4800
+ await storeExtractedFacts(debriefFacts, api.logger, 'openclaw_debrief');
4801
+ api.logger.info(`Session debrief: stored ${debriefItems.length} items`);
4802
+ }
4803
+ } catch (debriefErr: unknown) {
4804
+ api.logger.warn(`before_reset debrief failed: ${debriefErr instanceof Error ? debriefErr.message : String(debriefErr)}`);
4805
+ }
3123
4806
  } catch (err: unknown) {
3124
4807
  const message = err instanceof Error ? err.message : String(err);
3125
4808
  api.logger.warn(`before_reset extraction failed: ${message}`);