@totalreclaw/totalreclaw 1.0.4 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -67
- package/api-client.ts +328 -0
- package/consolidation.test.ts +356 -0
- package/consolidation.ts +227 -0
- package/crypto.ts +351 -0
- package/embedding.ts +84 -0
- package/extractor-dedup.test.ts +168 -0
- package/extractor.ts +237 -0
- package/generate-mnemonic.ts +14 -0
- package/hot-cache-wrapper.ts +126 -0
- package/import-adapters/base-adapter.ts +93 -0
- package/import-adapters/import-adapters.test.ts +595 -0
- package/import-adapters/index.ts +22 -0
- package/import-adapters/mcp-memory-adapter.ts +274 -0
- package/import-adapters/mem0-adapter.ts +233 -0
- package/import-adapters/types.ts +89 -0
- package/index.ts +2661 -0
- package/llm-client.ts +418 -0
- package/lsh.test.ts +463 -0
- package/lsh.ts +257 -0
- package/package.json +18 -33
- package/pocv2-e2e-test.ts +917 -0
- package/reranker.test.ts +594 -0
- package/reranker.ts +537 -0
- package/semantic-dedup.test.ts +392 -0
- package/semantic-dedup.ts +100 -0
- package/setup.sh +19 -0
- package/store-dedup-wiring.test.ts +186 -0
- package/subgraph-search.ts +282 -0
- package/subgraph-store.ts +346 -0
- package/SKILL.md +0 -709
- package/dist/index.js +0 -32154
package/index.ts
ADDED
|
@@ -0,0 +1,2661 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TotalReclaw Plugin for OpenClaw
|
|
3
|
+
*
|
|
4
|
+
* Registers runtime tools so OpenClaw can execute TotalReclaw operations:
|
|
5
|
+
* - totalreclaw_remember -- store an encrypted memory
|
|
6
|
+
* - totalreclaw_recall -- search and decrypt memories
|
|
7
|
+
* - totalreclaw_forget -- soft-delete a memory
|
|
8
|
+
* - totalreclaw_export -- export all memories (JSON or Markdown)
|
|
9
|
+
* - totalreclaw_status -- check billing/subscription status
|
|
10
|
+
* - totalreclaw_consolidate -- scan and merge near-duplicate memories
|
|
11
|
+
* - totalreclaw_import_from -- import memories from other tools (Mem0, MCP Memory, etc.)
|
|
12
|
+
*
|
|
13
|
+
* Also registers a `before_agent_start` hook that automatically injects
|
|
14
|
+
* relevant memories into the agent's context.
|
|
15
|
+
*
|
|
16
|
+
* All data is encrypted client-side with AES-256-GCM. The server never
|
|
17
|
+
* sees plaintext.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import {
|
|
21
|
+
deriveKeys,
|
|
22
|
+
deriveLshSeed,
|
|
23
|
+
computeAuthKeyHash,
|
|
24
|
+
encrypt,
|
|
25
|
+
decrypt,
|
|
26
|
+
generateBlindIndices,
|
|
27
|
+
generateContentFingerprint,
|
|
28
|
+
} from './crypto.js';
|
|
29
|
+
import { createApiClient, type StoreFactPayload } from './api-client.js';
|
|
30
|
+
import { extractFacts, type ExtractedFact } from './extractor.js';
|
|
31
|
+
import { initLLMClient, generateEmbedding, getEmbeddingDims } from './llm-client.js';
|
|
32
|
+
import { LSHHasher } from './lsh.js';
|
|
33
|
+
import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS, type RerankerCandidate } from './reranker.js';
|
|
34
|
+
import { deduplicateBatch } from './semantic-dedup.js';
|
|
35
|
+
import {
|
|
36
|
+
findNearDuplicate,
|
|
37
|
+
shouldSupersede,
|
|
38
|
+
clusterFacts,
|
|
39
|
+
getStoreDedupThreshold,
|
|
40
|
+
getConsolidationThreshold,
|
|
41
|
+
STORE_DEDUP_MAX_CANDIDATES,
|
|
42
|
+
type DecryptedCandidate,
|
|
43
|
+
} from './consolidation.js';
|
|
44
|
+
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, deriveSmartAccountAddress, type FactPayload } from './subgraph-store.js';
|
|
45
|
+
import { searchSubgraph, getSubgraphFactCount } from './subgraph-search.js';
|
|
46
|
+
import { PluginHotCache, type HotFact } from './hot-cache-wrapper.js';
|
|
47
|
+
import crypto from 'node:crypto';
|
|
48
|
+
import fs from 'node:fs';
|
|
49
|
+
import path from 'node:path';
|
|
50
|
+
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// OpenClaw Plugin API type (defined locally to avoid SDK dependency)
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
interface OpenClawPluginApi {
|
|
56
|
+
logger: {
|
|
57
|
+
info(...args: unknown[]): void;
|
|
58
|
+
warn(...args: unknown[]): void;
|
|
59
|
+
error(...args: unknown[]): void;
|
|
60
|
+
};
|
|
61
|
+
config?: {
|
|
62
|
+
agents?: {
|
|
63
|
+
defaults?: {
|
|
64
|
+
model?: {
|
|
65
|
+
primary?: string;
|
|
66
|
+
};
|
|
67
|
+
};
|
|
68
|
+
};
|
|
69
|
+
[key: string]: unknown;
|
|
70
|
+
};
|
|
71
|
+
pluginConfig?: Record<string, unknown>;
|
|
72
|
+
registerTool(tool: unknown, opts?: { name?: string; names?: string[] }): void;
|
|
73
|
+
registerService(service: { id: string; start(): void; stop?(): void }): void;
|
|
74
|
+
on(hookName: string, handler: (...args: unknown[]) => unknown, opts?: { priority?: number }): void;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// ---------------------------------------------------------------------------
|
|
78
|
+
// Persistent credential storage
|
|
79
|
+
// ---------------------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
/** Path where we persist userId + salt across restarts. */
|
|
82
|
+
const CREDENTIALS_PATH = process.env.TOTALRECLAW_CREDENTIALS_PATH || `${process.env.HOME ?? '/home/node'}/.totalreclaw/credentials.json`;
|
|
83
|
+
|
|
84
|
+
// ---------------------------------------------------------------------------
|
|
85
|
+
// Cosine similarity threshold — skip injection when top result is below this
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Minimum cosine similarity of the top reranked result required to inject
|
|
90
|
+
* memories into context. Below this threshold, the query is considered
|
|
91
|
+
* irrelevant to any stored memories and results are suppressed.
|
|
92
|
+
*
|
|
93
|
+
* Default 0.15 is tuned for bge-small-en-v1.5 which produces lower
|
|
94
|
+
* similarity scores than OpenAI models. Configurable via env var.
|
|
95
|
+
*/
|
|
96
|
+
const COSINE_THRESHOLD = parseFloat(
|
|
97
|
+
process.env.TOTALRECLAW_COSINE_THRESHOLD ?? '0.15',
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
// ---------------------------------------------------------------------------
|
|
101
|
+
// Module-level state (persists across tool calls within a session)
|
|
102
|
+
// ---------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
let authKeyHex: string | null = null;
|
|
105
|
+
let encryptionKey: Buffer | null = null;
|
|
106
|
+
let dedupKey: Buffer | null = null;
|
|
107
|
+
let userId: string | null = null;
|
|
108
|
+
let subgraphOwner: string | null = null; // Smart Account address for subgraph queries
|
|
109
|
+
let apiClient: ReturnType<typeof createApiClient> | null = null;
|
|
110
|
+
let initPromise: Promise<void> | null = null;
|
|
111
|
+
|
|
112
|
+
// LSH hasher — lazily initialized on first use (needs credentials + embedding dims)
|
|
113
|
+
let lshHasher: LSHHasher | null = null;
|
|
114
|
+
let lshInitFailed = false; // If true, skip LSH on future calls (provider doesn't support embeddings)
|
|
115
|
+
|
|
116
|
+
// Hot cache for managed service (subgraph mode) — lazily initialized
|
|
117
|
+
let pluginHotCache: PluginHotCache | null = null;
|
|
118
|
+
|
|
119
|
+
// Two-tier search state (C1): skip redundant searches when query is semantically similar
|
|
120
|
+
let lastSearchTimestamp = 0;
|
|
121
|
+
let lastQueryEmbedding: number[] | null = null;
|
|
122
|
+
|
|
123
|
+
// Feature flags — configurable for A/B testing
|
|
124
|
+
const CACHE_TTL_MS = parseInt(process.env.TOTALRECLAW_CACHE_TTL_MS ?? String(5 * 60 * 1000), 10);
|
|
125
|
+
const SEMANTIC_SKIP_THRESHOLD = parseFloat(process.env.TOTALRECLAW_SEMANTIC_SKIP_THRESHOLD ?? '0.85');
|
|
126
|
+
|
|
127
|
+
// Auto-extract throttle (C3): only extract every N turns in agent_end hook
|
|
128
|
+
let turnsSinceLastExtraction = 0;
|
|
129
|
+
const AUTO_EXTRACT_EVERY_TURNS_ENV = parseInt(process.env.TOTALRECLAW_EXTRACT_EVERY_TURNS ?? '5', 10);
|
|
130
|
+
|
|
131
|
+
// Store-time near-duplicate detection (consolidation module)
|
|
132
|
+
const STORE_DEDUP_ENABLED = process.env.TOTALRECLAW_STORE_DEDUP !== 'false';
|
|
133
|
+
|
|
134
|
+
// B2: Minimum relevance threshold — cosine below this means no memory injection
|
|
135
|
+
const RELEVANCE_THRESHOLD = parseFloat(process.env.TOTALRECLAW_RELEVANCE_THRESHOLD ?? '0.3');
|
|
136
|
+
|
|
137
|
+
// ---------------------------------------------------------------------------
|
|
138
|
+
// Billing cache infrastructure
|
|
139
|
+
// ---------------------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
const BILLING_CACHE_PATH = path.join(process.env.HOME ?? '/home/node', '.totalreclaw', 'billing-cache.json');
|
|
142
|
+
const BILLING_CACHE_TTL = 12 * 60 * 60 * 1000; // 12 hours
|
|
143
|
+
const QUOTA_WARNING_THRESHOLD = 0.8; // 80%
|
|
144
|
+
|
|
145
|
+
interface BillingCache {
|
|
146
|
+
tier: string;
|
|
147
|
+
free_writes_used: number;
|
|
148
|
+
free_writes_limit: number;
|
|
149
|
+
features?: {
|
|
150
|
+
llm_dedup?: boolean;
|
|
151
|
+
custom_extract_interval?: boolean;
|
|
152
|
+
min_extract_interval?: number;
|
|
153
|
+
};
|
|
154
|
+
checked_at: number;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function readBillingCache(): BillingCache | null {
|
|
158
|
+
try {
|
|
159
|
+
if (!fs.existsSync(BILLING_CACHE_PATH)) return null;
|
|
160
|
+
const raw = JSON.parse(fs.readFileSync(BILLING_CACHE_PATH, 'utf-8')) as BillingCache;
|
|
161
|
+
if (!raw.checked_at || Date.now() - raw.checked_at > BILLING_CACHE_TTL) return null;
|
|
162
|
+
return raw;
|
|
163
|
+
} catch {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function writeBillingCache(cache: BillingCache): void {
|
|
169
|
+
try {
|
|
170
|
+
const dir = path.dirname(BILLING_CACHE_PATH);
|
|
171
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
172
|
+
fs.writeFileSync(BILLING_CACHE_PATH, JSON.stringify(cache));
|
|
173
|
+
} catch {
|
|
174
|
+
// Best-effort — don't block on cache write failure.
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Check if LLM-guided dedup is enabled for the current tier.
|
|
180
|
+
* Returns true for Pro users, or when no billing cache exists (fail-open for self-hosters).
|
|
181
|
+
*/
|
|
182
|
+
function isLlmDedupEnabled(): boolean {
|
|
183
|
+
const cache = readBillingCache();
|
|
184
|
+
if (!cache) return true;
|
|
185
|
+
if (cache.tier === 'pro') return true;
|
|
186
|
+
if (cache.features?.llm_dedup !== undefined) return cache.features.llm_dedup;
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Get the effective extraction interval based on tier.
|
|
192
|
+
* Pro users can set interval as low as 2 via env; Free users are clamped to minimum 5.
|
|
193
|
+
*/
|
|
194
|
+
function getExtractInterval(): number {
|
|
195
|
+
const cache = readBillingCache();
|
|
196
|
+
const minInterval = cache?.features?.min_extract_interval ?? 5;
|
|
197
|
+
return Math.max(AUTO_EXTRACT_EVERY_TURNS_ENV, minInterval);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Ensure MEMORY.md has a TotalReclaw header so the agent knows encrypted
|
|
202
|
+
* memories are injected automatically via the before_agent_start hook.
|
|
203
|
+
*
|
|
204
|
+
* Option 3 approach: don't delete or stub MEMORY.md — let the agent use it
|
|
205
|
+
* for workspace-level notes, but make clear that user facts/preferences are
|
|
206
|
+
* handled by TotalReclaw's E2EE pipeline. This avoids confusing the agent
|
|
207
|
+
* (OpenClaw's system prompt still tells it to search MEMORY.md) while
|
|
208
|
+
* guiding it away from writing sensitive data in cleartext.
|
|
209
|
+
*/
|
|
210
|
+
const MEMORY_HEADER = `# Memory
|
|
211
|
+
|
|
212
|
+
> **TotalReclaw is active.** Your encrypted memories are loaded automatically
|
|
213
|
+
> at the start of each conversation — no need to search this file for them.
|
|
214
|
+
> Use \`totalreclaw_remember\` to store new memories and \`totalreclaw_recall\`
|
|
215
|
+
> to search. Do NOT write user facts, preferences, or decisions to this file.
|
|
216
|
+
> This file is for workspace-level notes only (non-sensitive).
|
|
217
|
+
|
|
218
|
+
`;
|
|
219
|
+
|
|
220
|
+
function ensureMemoryHeader(logger: OpenClawPluginApi['logger']): void {
|
|
221
|
+
try {
|
|
222
|
+
const workspace = path.join(process.env.HOME ?? '/home/node', '.openclaw', 'workspace');
|
|
223
|
+
const memoryMd = path.join(workspace, 'MEMORY.md');
|
|
224
|
+
|
|
225
|
+
if (fs.existsSync(memoryMd)) {
|
|
226
|
+
const content = fs.readFileSync(memoryMd, 'utf-8');
|
|
227
|
+
if (!content.includes('TotalReclaw is active')) {
|
|
228
|
+
fs.writeFileSync(memoryMd, MEMORY_HEADER + content);
|
|
229
|
+
logger.info('Added TotalReclaw header to MEMORY.md');
|
|
230
|
+
}
|
|
231
|
+
} else {
|
|
232
|
+
// Create MEMORY.md with the header so the agent doesn't get ENOENT
|
|
233
|
+
const dir = path.dirname(memoryMd);
|
|
234
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
235
|
+
fs.writeFileSync(memoryMd, MEMORY_HEADER);
|
|
236
|
+
logger.info('Created MEMORY.md with TotalReclaw header');
|
|
237
|
+
}
|
|
238
|
+
} catch {
|
|
239
|
+
// Best-effort — don't block the hook
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// ---------------------------------------------------------------------------
|
|
244
|
+
// Dynamic candidate pool sizing
|
|
245
|
+
// ---------------------------------------------------------------------------
|
|
246
|
+
|
|
247
|
+
/** Cached fact count for dynamic candidate pool sizing. */
|
|
248
|
+
let cachedFactCount: number | null = null;
|
|
249
|
+
/** Timestamp of last fact count fetch (ms). */
|
|
250
|
+
let lastFactCountFetch: number = 0;
|
|
251
|
+
/** Cache TTL for fact count: 5 minutes. */
|
|
252
|
+
const FACT_COUNT_CACHE_TTL = 5 * 60 * 1000;
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Compute the candidate pool size from a fact count.
|
|
256
|
+
*
|
|
257
|
+
* Formula: pool = min(max(factCount * 3, 400), 5000)
|
|
258
|
+
* - At least 400 candidates (even for tiny vaults)
|
|
259
|
+
* - At most 5000 candidates (to bound decryption + reranking cost)
|
|
260
|
+
* - 3x fact count in between
|
|
261
|
+
*/
|
|
262
|
+
function computeCandidatePool(factCount: number): number {
|
|
263
|
+
return Math.min(Math.max(factCount * 3, 400), 5000);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Fetch the user's fact count from the server, with caching.
|
|
268
|
+
*
|
|
269
|
+
* Uses the /v1/export endpoint with limit=1 to get `total_count` without
|
|
270
|
+
* downloading all facts. Falls back to 400 (which gives pool=1200) if
|
|
271
|
+
* the server is unreachable or returns no count.
|
|
272
|
+
*/
|
|
273
|
+
async function getFactCount(logger: OpenClawPluginApi['logger']): Promise<number> {
|
|
274
|
+
const now = Date.now();
|
|
275
|
+
|
|
276
|
+
// Return cached value if fresh.
|
|
277
|
+
if (cachedFactCount !== null && (now - lastFactCountFetch) < FACT_COUNT_CACHE_TTL) {
|
|
278
|
+
return cachedFactCount;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
try {
|
|
282
|
+
if (!apiClient || !authKeyHex) {
|
|
283
|
+
return cachedFactCount ?? 400; // Not initialized yet, use default
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
const page = await apiClient.exportFacts(authKeyHex, 1);
|
|
287
|
+
const count = page.total_count ?? page.facts.length;
|
|
288
|
+
|
|
289
|
+
cachedFactCount = count;
|
|
290
|
+
lastFactCountFetch = now;
|
|
291
|
+
logger.info(`Fact count updated: ${count} (candidate pool: ${computeCandidatePool(count)})`);
|
|
292
|
+
return count;
|
|
293
|
+
} catch (err) {
|
|
294
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
295
|
+
logger.warn(`Failed to fetch fact count (using ${cachedFactCount ?? 400}): ${msg}`);
|
|
296
|
+
return cachedFactCount ?? 400; // Fall back to cached or default
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// ---------------------------------------------------------------------------
|
|
301
|
+
// Initialisation
|
|
302
|
+
// ---------------------------------------------------------------------------
|
|
303
|
+
|
|
304
|
+
/** True when master password is missing — tools return setup instructions. */
|
|
305
|
+
let needsSetup = false;
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Derive keys from the master password, load or create credentials, and
|
|
309
|
+
* register with the server if this is the first run.
|
|
310
|
+
*/
|
|
311
|
+
async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
312
|
+
const serverUrl =
|
|
313
|
+
process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz';
|
|
314
|
+
const masterPassword = process.env.TOTALRECLAW_MASTER_PASSWORD;
|
|
315
|
+
|
|
316
|
+
if (!masterPassword) {
|
|
317
|
+
needsSetup = true;
|
|
318
|
+
logger.info('TOTALRECLAW_MASTER_PASSWORD not set — setup required (see SKILL.md Post-Install Setup)');
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
apiClient = createApiClient(serverUrl);
|
|
323
|
+
|
|
324
|
+
// --- Attempt to load existing credentials ---
|
|
325
|
+
let existingSalt: Buffer | undefined;
|
|
326
|
+
let existingUserId: string | undefined;
|
|
327
|
+
|
|
328
|
+
try {
|
|
329
|
+
if (fs.existsSync(CREDENTIALS_PATH)) {
|
|
330
|
+
const creds = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8'));
|
|
331
|
+
existingSalt = Buffer.from(creds.salt, 'base64');
|
|
332
|
+
existingUserId = creds.userId;
|
|
333
|
+
logger.info(`Loaded existing credentials for user ${existingUserId}`);
|
|
334
|
+
}
|
|
335
|
+
} catch (e) {
|
|
336
|
+
logger.warn('Failed to load credentials, will register new account');
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// --- Derive keys ---
|
|
340
|
+
const keys = deriveKeys(masterPassword, existingSalt);
|
|
341
|
+
authKeyHex = keys.authKey.toString('hex');
|
|
342
|
+
encryptionKey = keys.encryptionKey;
|
|
343
|
+
dedupKey = keys.dedupKey;
|
|
344
|
+
|
|
345
|
+
// Cache credentials for lazy LSH seed derivation
|
|
346
|
+
masterPasswordCache = masterPassword;
|
|
347
|
+
saltCache = keys.salt;
|
|
348
|
+
|
|
349
|
+
if (existingUserId) {
|
|
350
|
+
userId = existingUserId;
|
|
351
|
+
logger.info(`Authenticated as user ${userId}`);
|
|
352
|
+
} else {
|
|
353
|
+
// First run -- register with the server.
|
|
354
|
+
const authHash = computeAuthKeyHash(keys.authKey);
|
|
355
|
+
const saltHex = keys.salt.toString('hex');
|
|
356
|
+
|
|
357
|
+
let registeredUserId: string | undefined;
|
|
358
|
+
try {
|
|
359
|
+
const result = await apiClient.register(authHash, saltHex);
|
|
360
|
+
registeredUserId = result.user_id;
|
|
361
|
+
} catch (err: unknown) {
|
|
362
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
363
|
+
if (msg.includes('USER_EXISTS') && isSubgraphMode()) {
|
|
364
|
+
// In managed mode, derive a deterministic userId from the auth key
|
|
365
|
+
// hash. The server is only a relay proxy — userId is used as the
|
|
366
|
+
// subgraph owner field and must be consistent between store/search.
|
|
367
|
+
registeredUserId = authHash.slice(0, 32);
|
|
368
|
+
logger.info(`Using derived userId for managed mode (server returned USER_EXISTS)`);
|
|
369
|
+
} else {
|
|
370
|
+
throw err;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
userId = registeredUserId!;
|
|
375
|
+
|
|
376
|
+
// Persist credentials so we can resume later.
|
|
377
|
+
const dir = path.dirname(CREDENTIALS_PATH);
|
|
378
|
+
if (!fs.existsSync(dir)) {
|
|
379
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
380
|
+
}
|
|
381
|
+
fs.writeFileSync(
|
|
382
|
+
CREDENTIALS_PATH,
|
|
383
|
+
JSON.stringify({ userId, salt: keys.salt.toString('base64') }),
|
|
384
|
+
);
|
|
385
|
+
|
|
386
|
+
logger.info(`Registered new user: ${userId}`);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// Derive Smart Account address for subgraph queries (on-chain owner identity).
|
|
390
|
+
if (isSubgraphMode()) {
|
|
391
|
+
try {
|
|
392
|
+
const config = getSubgraphConfig();
|
|
393
|
+
subgraphOwner = await deriveSmartAccountAddress(config.mnemonic, config.chainId);
|
|
394
|
+
logger.info(`Subgraph owner (Smart Account): ${subgraphOwner}`);
|
|
395
|
+
} catch (err) {
|
|
396
|
+
logger.warn(`Failed to derive Smart Account address: ${err instanceof Error ? err.message : String(err)}`);
|
|
397
|
+
// Fall back to userId — won't match subgraph Bytes format, but better than null
|
|
398
|
+
subgraphOwner = userId;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
function isDocker(): boolean {
|
|
404
|
+
try {
|
|
405
|
+
return fs.existsSync('/.dockerenv') ||
|
|
406
|
+
(fs.existsSync('/proc/1/cgroup') &&
|
|
407
|
+
fs.readFileSync('/proc/1/cgroup', 'utf8').includes('docker'));
|
|
408
|
+
} catch { return false; }
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function buildSetupErrorMsg(): string {
|
|
412
|
+
const base =
|
|
413
|
+
'TotalReclaw setup required:\n' +
|
|
414
|
+
'1. Set TOTALRECLAW_MASTER_PASSWORD — ask the user if they have an existing recovery phrase or generate a new 12-word BIP-39 mnemonic.\n' +
|
|
415
|
+
'2. Restart the gateway to apply changes.\n' +
|
|
416
|
+
' (Optional: set TOTALRECLAW_SELF_HOSTED=true if using your own server instead of the managed service.)\n\n';
|
|
417
|
+
|
|
418
|
+
if (isDocker()) {
|
|
419
|
+
return base +
|
|
420
|
+
'Running in Docker — pass env vars via `-e` flags or your compose file:\n' +
|
|
421
|
+
' -e TOTALRECLAW_MASTER_PASSWORD="word1 word2 ..."';
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
if (process.platform === 'darwin') {
|
|
425
|
+
return base +
|
|
426
|
+
'Running on macOS — add env vars to the LaunchAgent plist at\n' +
|
|
427
|
+
'~/Library/LaunchAgents/ai.openclaw.gateway.plist under <key>EnvironmentVariables</key>:\n' +
|
|
428
|
+
' <key>TOTALRECLAW_MASTER_PASSWORD</key><string>word1 word2 ...</string>\n' +
|
|
429
|
+
'Then run: openclaw gateway restart';
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
return base +
|
|
433
|
+
'Running on Linux — add env vars to the systemd unit override or your shell profile:\n' +
|
|
434
|
+
' export TOTALRECLAW_MASTER_PASSWORD="word1 word2 ..."\n' +
|
|
435
|
+
'Then run: openclaw gateway restart';
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
const SETUP_ERROR_MSG = buildSetupErrorMsg();
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Ensure `initialize()` has completed (runs at most once).
|
|
442
|
+
*/
|
|
443
|
+
async function ensureInitialized(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
444
|
+
if (!initPromise) {
|
|
445
|
+
initPromise = initialize(logger);
|
|
446
|
+
}
|
|
447
|
+
await initPromise;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Like ensureInitialized, but throws if setup is still needed.
|
|
452
|
+
* Use in tool handlers where we need a fully configured plugin.
|
|
453
|
+
*/
|
|
454
|
+
async function requireFullSetup(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
455
|
+
await ensureInitialized(logger);
|
|
456
|
+
if (needsSetup) {
|
|
457
|
+
throw new Error(SETUP_ERROR_MSG);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// ---------------------------------------------------------------------------
|
|
462
|
+
// LSH + Embedding helpers
|
|
463
|
+
// ---------------------------------------------------------------------------
|
|
464
|
+
|
|
465
|
+
/** Master password cached for LSH seed derivation (set during initialize()). */
|
|
466
|
+
let masterPasswordCache: string | null = null;
|
|
467
|
+
/** Salt cached for LSH seed derivation (set during initialize()). */
|
|
468
|
+
let saltCache: Buffer | null = null;
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* Get or initialize the LSH hasher.
|
|
472
|
+
*
|
|
473
|
+
* The hasher is created lazily because it needs:
|
|
474
|
+
* 1. The master password + salt (available after initialize())
|
|
475
|
+
* 2. The embedding dimensions (available after initLLMClient())
|
|
476
|
+
*
|
|
477
|
+
* If the provider doesn't support embeddings, this returns null and
|
|
478
|
+
* sets `lshInitFailed` to avoid retrying.
|
|
479
|
+
*/
|
|
480
|
+
function getLSHHasher(logger: OpenClawPluginApi['logger']): LSHHasher | null {
|
|
481
|
+
if (lshHasher) return lshHasher;
|
|
482
|
+
if (lshInitFailed) return null;
|
|
483
|
+
|
|
484
|
+
try {
|
|
485
|
+
if (!masterPasswordCache || !saltCache) {
|
|
486
|
+
logger.warn('LSH hasher: credentials not available yet');
|
|
487
|
+
return null;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const dims = getEmbeddingDims();
|
|
491
|
+
const lshSeed = deriveLshSeed(masterPasswordCache, saltCache);
|
|
492
|
+
lshHasher = new LSHHasher(lshSeed, dims);
|
|
493
|
+
logger.info(`LSH hasher initialized (dims=${dims}, tables=${lshHasher.tables}, bits=${lshHasher.bits})`);
|
|
494
|
+
return lshHasher;
|
|
495
|
+
} catch (err) {
|
|
496
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
497
|
+
logger.warn(`LSH hasher initialization failed (will use word-only indices): ${msg}`);
|
|
498
|
+
lshInitFailed = true;
|
|
499
|
+
return null;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Generate an embedding for the given text and compute LSH bucket hashes.
|
|
505
|
+
*
|
|
506
|
+
* Returns null if embedding generation fails (provider doesn't support it,
|
|
507
|
+
* network error, etc.). In that case, the caller should fall back to
|
|
508
|
+
* word-only blind indices.
|
|
509
|
+
*/
|
|
510
|
+
async function generateEmbeddingAndLSH(
|
|
511
|
+
text: string,
|
|
512
|
+
logger: OpenClawPluginApi['logger'],
|
|
513
|
+
): Promise<{ embedding: number[]; lshBuckets: string[]; encryptedEmbedding: string } | null> {
|
|
514
|
+
try {
|
|
515
|
+
const embedding = await generateEmbedding(text);
|
|
516
|
+
|
|
517
|
+
const hasher = getLSHHasher(logger);
|
|
518
|
+
const lshBuckets = hasher ? hasher.hash(embedding) : [];
|
|
519
|
+
|
|
520
|
+
// Encrypt the embedding (JSON array of numbers) for zero-knowledge storage
|
|
521
|
+
const encryptedEmbedding = encryptToHex(JSON.stringify(embedding), encryptionKey!);
|
|
522
|
+
|
|
523
|
+
return { embedding, lshBuckets, encryptedEmbedding };
|
|
524
|
+
} catch (err) {
|
|
525
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
526
|
+
logger.warn(`Embedding/LSH generation failed (falling back to word-only indices): ${msg}`);
|
|
527
|
+
return null;
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// ---------------------------------------------------------------------------
|
|
532
|
+
// Store-time near-duplicate search helper
|
|
533
|
+
// ---------------------------------------------------------------------------
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* Search the vault for near-duplicates of a fact about to be stored.
|
|
537
|
+
*
|
|
538
|
+
* Uses the fact's blind indices as trapdoors to fetch candidates, decrypts
|
|
539
|
+
* them, extracts embeddings, and calls `findNearDuplicate()` from the
|
|
540
|
+
* consolidation module.
|
|
541
|
+
*
|
|
542
|
+
* Returns null on any failure (fail-open: we'd rather store a duplicate than
|
|
543
|
+
* lose a fact).
|
|
544
|
+
*/
|
|
545
|
+
async function searchForNearDuplicates(
|
|
546
|
+
factText: string,
|
|
547
|
+
factEmbedding: number[],
|
|
548
|
+
allIndices: string[],
|
|
549
|
+
logger: OpenClawPluginApi['logger'],
|
|
550
|
+
): Promise<{ match: DecryptedCandidate; similarity: number } | null> {
|
|
551
|
+
try {
|
|
552
|
+
if (!encryptionKey || !authKeyHex || !userId) return null;
|
|
553
|
+
|
|
554
|
+
// Fetch candidates from the vault using the fact's blind indices as trapdoors.
|
|
555
|
+
let decryptedCandidates: DecryptedCandidate[] = [];
|
|
556
|
+
|
|
557
|
+
if (isSubgraphMode()) {
|
|
558
|
+
const results = await searchSubgraph(
|
|
559
|
+
subgraphOwner || userId,
|
|
560
|
+
allIndices,
|
|
561
|
+
STORE_DEDUP_MAX_CANDIDATES,
|
|
562
|
+
authKeyHex,
|
|
563
|
+
);
|
|
564
|
+
for (const result of results) {
|
|
565
|
+
try {
|
|
566
|
+
const docJson = decryptFromHex(result.encryptedBlob, encryptionKey);
|
|
567
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
568
|
+
|
|
569
|
+
let embedding: number[] | null = null;
|
|
570
|
+
if (result.encryptedEmbedding) {
|
|
571
|
+
try {
|
|
572
|
+
embedding = JSON.parse(decryptFromHex(result.encryptedEmbedding, encryptionKey));
|
|
573
|
+
} catch { /* skip */ }
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
decryptedCandidates.push({
|
|
577
|
+
id: result.id,
|
|
578
|
+
text: doc.text,
|
|
579
|
+
embedding,
|
|
580
|
+
importance: doc.metadata?.importance
|
|
581
|
+
? Math.round((doc.metadata.importance as number) * 10)
|
|
582
|
+
: 5,
|
|
583
|
+
decayScore: 5,
|
|
584
|
+
createdAt: result.timestamp ? parseInt(result.timestamp, 10) * 1000 : Date.now(),
|
|
585
|
+
version: 1,
|
|
586
|
+
});
|
|
587
|
+
} catch { /* skip undecryptable */ }
|
|
588
|
+
}
|
|
589
|
+
} else if (apiClient) {
|
|
590
|
+
const candidates = await apiClient.search(
|
|
591
|
+
userId,
|
|
592
|
+
allIndices,
|
|
593
|
+
STORE_DEDUP_MAX_CANDIDATES,
|
|
594
|
+
authKeyHex,
|
|
595
|
+
);
|
|
596
|
+
for (const candidate of candidates) {
|
|
597
|
+
try {
|
|
598
|
+
const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey);
|
|
599
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
600
|
+
|
|
601
|
+
let embedding: number[] | null = null;
|
|
602
|
+
if (candidate.encrypted_embedding) {
|
|
603
|
+
try {
|
|
604
|
+
embedding = JSON.parse(decryptFromHex(candidate.encrypted_embedding, encryptionKey));
|
|
605
|
+
} catch { /* skip */ }
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
decryptedCandidates.push({
|
|
609
|
+
id: candidate.fact_id,
|
|
610
|
+
text: doc.text,
|
|
611
|
+
embedding,
|
|
612
|
+
importance: doc.metadata?.importance
|
|
613
|
+
? Math.round((doc.metadata.importance as number) * 10)
|
|
614
|
+
: 5,
|
|
615
|
+
decayScore: candidate.decay_score,
|
|
616
|
+
createdAt: typeof candidate.timestamp === 'number'
|
|
617
|
+
? candidate.timestamp
|
|
618
|
+
: new Date(candidate.timestamp).getTime(),
|
|
619
|
+
version: candidate.version,
|
|
620
|
+
});
|
|
621
|
+
} catch { /* skip undecryptable */ }
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
if (decryptedCandidates.length === 0) return null;
|
|
626
|
+
|
|
627
|
+
const result = findNearDuplicate(factEmbedding, decryptedCandidates, getStoreDedupThreshold());
|
|
628
|
+
if (!result) return null;
|
|
629
|
+
|
|
630
|
+
return { match: result.existingFact, similarity: result.similarity };
|
|
631
|
+
} catch (err) {
|
|
632
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
633
|
+
logger.warn(`Store-time dedup search failed (proceeding with store): ${msg}`);
|
|
634
|
+
return null;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// ---------------------------------------------------------------------------
|
|
639
|
+
// Utility helpers
|
|
640
|
+
// ---------------------------------------------------------------------------
|
|
641
|
+
|
|
642
|
+
/**
|
|
643
|
+
* Encrypt a plaintext document string and return its hex-encoded ciphertext.
|
|
644
|
+
*
|
|
645
|
+
* The server stores blobs as hex (not base64), so we convert the base64
|
|
646
|
+
* output of `encrypt()` into hex.
|
|
647
|
+
*/
|
|
648
|
+
function encryptToHex(plaintext: string, key: Buffer): string {
|
|
649
|
+
const b64 = encrypt(plaintext, key);
|
|
650
|
+
return Buffer.from(b64, 'base64').toString('hex');
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/**
|
|
654
|
+
* Decrypt a hex-encoded ciphertext blob into a UTF-8 string.
|
|
655
|
+
*/
|
|
656
|
+
function decryptFromHex(hexBlob: string, key: Buffer): string {
|
|
657
|
+
const hex = hexBlob.startsWith('0x') ? hexBlob.slice(2) : hexBlob;
|
|
658
|
+
const b64 = Buffer.from(hex, 'hex').toString('base64');
|
|
659
|
+
return decrypt(b64, key);
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* Fetch existing memories from the vault to provide dedup context for extraction.
|
|
664
|
+
* Returns a lightweight list of {id, text} pairs for the LLM prompt.
|
|
665
|
+
* Fails silently — returns empty array on any error.
|
|
666
|
+
*/
|
|
667
|
+
async function fetchExistingMemoriesForExtraction(
|
|
668
|
+
logger: { warn: (msg: string) => void },
|
|
669
|
+
limit: number = 30,
|
|
670
|
+
rawMessages: unknown[] = [],
|
|
671
|
+
): Promise<Array<{ id: string; text: string }>> {
|
|
672
|
+
try {
|
|
673
|
+
if (!encryptionKey || !authKeyHex || !userId) return [];
|
|
674
|
+
|
|
675
|
+
// Extract key terms from the last few messages to generate meaningful trapdoors.
|
|
676
|
+
// Using '*' would produce zero trapdoors (stripped as punctuation), so we pull
|
|
677
|
+
// text from the conversation to find memories relevant to the current context.
|
|
678
|
+
const recentMessages = rawMessages.slice(-4);
|
|
679
|
+
const textChunks: string[] = [];
|
|
680
|
+
for (const msg of recentMessages) {
|
|
681
|
+
const m = msg as { content?: string | Array<{ text?: string }>; text?: string };
|
|
682
|
+
if (typeof m.content === 'string') {
|
|
683
|
+
textChunks.push(m.content);
|
|
684
|
+
} else if (Array.isArray(m.content)) {
|
|
685
|
+
for (const block of m.content) {
|
|
686
|
+
if (block.text) textChunks.push(block.text);
|
|
687
|
+
}
|
|
688
|
+
} else if (typeof m.text === 'string') {
|
|
689
|
+
textChunks.push(m.text);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
const queryText = textChunks.join(' ').slice(0, 500); // cap to avoid giant trapdoor sets
|
|
693
|
+
if (!queryText.trim()) return [];
|
|
694
|
+
|
|
695
|
+
const trapdoors = generateBlindIndices(queryText);
|
|
696
|
+
if (trapdoors.length === 0) return [];
|
|
697
|
+
|
|
698
|
+
const results: Array<{ id: string; text: string }> = [];
|
|
699
|
+
|
|
700
|
+
if (isSubgraphMode()) {
|
|
701
|
+
const rawResults = await searchSubgraph(
|
|
702
|
+
subgraphOwner || userId,
|
|
703
|
+
trapdoors,
|
|
704
|
+
limit,
|
|
705
|
+
authKeyHex,
|
|
706
|
+
);
|
|
707
|
+
for (const r of rawResults) {
|
|
708
|
+
try {
|
|
709
|
+
const docJson = decryptFromHex(r.encryptedBlob, encryptionKey);
|
|
710
|
+
const doc = JSON.parse(docJson) as { text: string };
|
|
711
|
+
results.push({ id: r.id, text: doc.text });
|
|
712
|
+
} catch { /* skip undecryptable */ }
|
|
713
|
+
}
|
|
714
|
+
} else if (apiClient) {
|
|
715
|
+
const candidates = await apiClient.search(userId, trapdoors, limit, authKeyHex);
|
|
716
|
+
for (const c of candidates) {
|
|
717
|
+
try {
|
|
718
|
+
const docJson = decryptFromHex(c.encrypted_blob, encryptionKey);
|
|
719
|
+
const doc = JSON.parse(docJson) as { text: string };
|
|
720
|
+
results.push({ id: c.fact_id, text: doc.text });
|
|
721
|
+
} catch { /* skip undecryptable */ }
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
return results;
|
|
726
|
+
} catch (err) {
|
|
727
|
+
logger.warn(`Failed to fetch existing memories for extraction context: ${err instanceof Error ? err.message : String(err)}`);
|
|
728
|
+
return [];
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Simple text-overlap scoring between a query and a candidate document.
|
|
734
|
+
* Returns the number of overlapping lowercase words.
|
|
735
|
+
*/
|
|
736
|
+
function textScore(query: string, docText: string): number {
|
|
737
|
+
const queryWords = new Set(
|
|
738
|
+
query.toLowerCase().split(/\s+/).filter((w) => w.length >= 2),
|
|
739
|
+
);
|
|
740
|
+
const docWords = docText.toLowerCase().split(/\s+/);
|
|
741
|
+
let score = 0;
|
|
742
|
+
for (const word of docWords) {
|
|
743
|
+
if (queryWords.has(word)) score++;
|
|
744
|
+
}
|
|
745
|
+
return score;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/**
|
|
749
|
+
* Format a relative time string (e.g. "2 hours ago").
|
|
750
|
+
*/
|
|
751
|
+
function relativeTime(isoOrMs: string | number): string {
|
|
752
|
+
const ms = typeof isoOrMs === 'number' ? isoOrMs : new Date(isoOrMs).getTime();
|
|
753
|
+
const diffMs = Date.now() - ms;
|
|
754
|
+
const seconds = Math.floor(diffMs / 1000);
|
|
755
|
+
if (seconds < 60) return 'just now';
|
|
756
|
+
const minutes = Math.floor(seconds / 60);
|
|
757
|
+
if (minutes < 60) return `${minutes}m ago`;
|
|
758
|
+
const hours = Math.floor(minutes / 60);
|
|
759
|
+
if (hours < 24) return `${hours}h ago`;
|
|
760
|
+
const days = Math.floor(hours / 24);
|
|
761
|
+
return `${days}d ago`;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// ---------------------------------------------------------------------------
|
|
765
|
+
// Importance filter for auto-extraction
|
|
766
|
+
// ---------------------------------------------------------------------------
|
|
767
|
+
|
|
768
|
+
/**
|
|
769
|
+
* Minimum importance score (1-10) for auto-extracted facts to be stored.
|
|
770
|
+
* Facts below this threshold are silently dropped to save storage and gas.
|
|
771
|
+
* Configurable via TOTALRECLAW_MIN_IMPORTANCE env var (default: 3).
|
|
772
|
+
*
|
|
773
|
+
* NOTE: This filter is ONLY applied to auto-extraction (hooks).
|
|
774
|
+
* The explicit `totalreclaw_remember` tool always stores regardless of importance.
|
|
775
|
+
*/
|
|
776
|
+
const MIN_IMPORTANCE_THRESHOLD = Math.max(
|
|
777
|
+
1,
|
|
778
|
+
Math.min(10, Number(process.env.TOTALRECLAW_MIN_IMPORTANCE) || 3),
|
|
779
|
+
);
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Filter extracted facts by importance threshold.
|
|
783
|
+
* Facts with importance < MIN_IMPORTANCE_THRESHOLD are dropped.
|
|
784
|
+
* Facts with missing/undefined importance are treated as importance=5 (kept).
|
|
785
|
+
*/
|
|
786
|
+
function filterByImportance(
|
|
787
|
+
facts: ExtractedFact[],
|
|
788
|
+
logger: OpenClawPluginApi['logger'],
|
|
789
|
+
): { kept: ExtractedFact[]; dropped: number } {
|
|
790
|
+
const kept: ExtractedFact[] = [];
|
|
791
|
+
let dropped = 0;
|
|
792
|
+
|
|
793
|
+
for (const fact of facts) {
|
|
794
|
+
const importance = fact.importance ?? 5;
|
|
795
|
+
if (importance >= MIN_IMPORTANCE_THRESHOLD) {
|
|
796
|
+
kept.push(fact);
|
|
797
|
+
} else {
|
|
798
|
+
dropped++;
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
if (dropped > 0) {
|
|
803
|
+
logger.info(
|
|
804
|
+
`Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`,
|
|
805
|
+
);
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
return { kept, dropped };
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// ---------------------------------------------------------------------------
|
|
812
|
+
// Auto-extraction helper
|
|
813
|
+
// ---------------------------------------------------------------------------
|
|
814
|
+
|
|
815
|
+
/**
|
|
816
|
+
* Store extracted facts in the TotalReclaw server.
|
|
817
|
+
* Encrypts each fact, generates blind indices and fingerprint, stores via API.
|
|
818
|
+
* Silently skips duplicates.
|
|
819
|
+
*
|
|
820
|
+
* Before storing, performs semantic near-duplicate detection within the batch:
|
|
821
|
+
* facts whose embeddings have cosine similarity >= threshold (default 0.9)
|
|
822
|
+
* against an already-accepted fact in the same batch are skipped.
|
|
823
|
+
*/
|
|
824
|
+
async function storeExtractedFacts(
|
|
825
|
+
facts: ExtractedFact[],
|
|
826
|
+
logger: OpenClawPluginApi['logger'],
|
|
827
|
+
): Promise<number> {
|
|
828
|
+
if (!encryptionKey || !dedupKey || !authKeyHex || !userId || !apiClient) return 0;
|
|
829
|
+
|
|
830
|
+
// Phase 1: Generate embeddings for all facts (needed for dedup + storage).
|
|
831
|
+
const embeddingMap = new Map<string, number[]>();
|
|
832
|
+
const embeddingResultMap = new Map<
|
|
833
|
+
string,
|
|
834
|
+
{ embedding: number[]; lshBuckets: string[]; encryptedEmbedding: string }
|
|
835
|
+
>();
|
|
836
|
+
|
|
837
|
+
for (const fact of facts) {
|
|
838
|
+
try {
|
|
839
|
+
const result = await generateEmbeddingAndLSH(fact.text, logger);
|
|
840
|
+
if (result) {
|
|
841
|
+
embeddingMap.set(fact.text, result.embedding);
|
|
842
|
+
embeddingResultMap.set(fact.text, result);
|
|
843
|
+
}
|
|
844
|
+
} catch {
|
|
845
|
+
// Embedding generation failed for this fact -- proceed without it.
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
// Phase 2: Semantic batch dedup.
|
|
850
|
+
const dedupedFacts = deduplicateBatch(facts, embeddingMap, logger);
|
|
851
|
+
|
|
852
|
+
if (dedupedFacts.length < facts.length) {
|
|
853
|
+
logger.info(
|
|
854
|
+
`Semantic dedup: ${facts.length - dedupedFacts.length} near-duplicate(s) removed from batch of ${facts.length}`,
|
|
855
|
+
);
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
// Phase 3: Store the deduplicated facts (with optional store-time dedup).
|
|
859
|
+
let stored = 0;
|
|
860
|
+
let superseded = 0;
|
|
861
|
+
let skipped = 0;
|
|
862
|
+
|
|
863
|
+
for (const fact of dedupedFacts) {
|
|
864
|
+
try {
|
|
865
|
+
const blindIndices = generateBlindIndices(fact.text);
|
|
866
|
+
|
|
867
|
+
// Use pre-computed embedding result if available.
|
|
868
|
+
const embeddingResult = embeddingResultMap.get(fact.text) ?? null;
|
|
869
|
+
const allIndices = embeddingResult
|
|
870
|
+
? [...blindIndices, ...embeddingResult.lshBuckets]
|
|
871
|
+
: blindIndices;
|
|
872
|
+
|
|
873
|
+
// LLM-guided dedup: handle UPDATE/DELETE/NOOP actions.
|
|
874
|
+
if (fact.action === 'NOOP') {
|
|
875
|
+
logger.info(`LLM dedup: NOOP — skipping "${fact.text.slice(0, 60)}…"`);
|
|
876
|
+
skipped++;
|
|
877
|
+
continue;
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
if (fact.action === 'DELETE' && fact.existingFactId) {
|
|
881
|
+
// Tombstone the old fact, don't store anything new.
|
|
882
|
+
if (isSubgraphMode()) {
|
|
883
|
+
try {
|
|
884
|
+
const tombConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
885
|
+
const tombstone: FactPayload = {
|
|
886
|
+
id: fact.existingFactId,
|
|
887
|
+
timestamp: new Date().toISOString(),
|
|
888
|
+
owner: subgraphOwner || userId!,
|
|
889
|
+
encryptedBlob: '00',
|
|
890
|
+
blindIndices: [],
|
|
891
|
+
decayScore: 0,
|
|
892
|
+
source: 'tombstone',
|
|
893
|
+
contentFp: '',
|
|
894
|
+
agentId: 'openclaw-plugin-auto',
|
|
895
|
+
};
|
|
896
|
+
await submitFactOnChain(encodeFactProtobuf(tombstone), tombConfig);
|
|
897
|
+
logger.info(`LLM dedup: DELETE — tombstoned ${fact.existingFactId} on-chain`);
|
|
898
|
+
} catch (tombErr) {
|
|
899
|
+
logger.warn(`LLM dedup: DELETE failed for ${fact.existingFactId}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`);
|
|
900
|
+
}
|
|
901
|
+
} else if (apiClient && authKeyHex) {
|
|
902
|
+
try {
|
|
903
|
+
await apiClient.deleteFact(fact.existingFactId, authKeyHex);
|
|
904
|
+
logger.info(`LLM dedup: DELETE — removed ${fact.existingFactId}`);
|
|
905
|
+
} catch (delErr) {
|
|
906
|
+
logger.warn(`LLM dedup: DELETE failed for ${fact.existingFactId}: ${delErr instanceof Error ? delErr.message : String(delErr)}`);
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
superseded++;
|
|
910
|
+
continue;
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
if (fact.action === 'UPDATE' && fact.existingFactId) {
|
|
914
|
+
// Tombstone the old fact, then fall through to store the new version.
|
|
915
|
+
if (isSubgraphMode()) {
|
|
916
|
+
try {
|
|
917
|
+
const tombConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
918
|
+
const tombstone: FactPayload = {
|
|
919
|
+
id: fact.existingFactId,
|
|
920
|
+
timestamp: new Date().toISOString(),
|
|
921
|
+
owner: subgraphOwner || userId!,
|
|
922
|
+
encryptedBlob: '00',
|
|
923
|
+
blindIndices: [],
|
|
924
|
+
decayScore: 0,
|
|
925
|
+
source: 'tombstone',
|
|
926
|
+
contentFp: '',
|
|
927
|
+
agentId: 'openclaw-plugin-auto',
|
|
928
|
+
};
|
|
929
|
+
await submitFactOnChain(encodeFactProtobuf(tombstone), tombConfig);
|
|
930
|
+
logger.info(`LLM dedup: UPDATE — tombstoned ${fact.existingFactId} on-chain, storing replacement`);
|
|
931
|
+
} catch (tombErr) {
|
|
932
|
+
logger.warn(`LLM dedup: UPDATE tombstone failed for ${fact.existingFactId}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`);
|
|
933
|
+
}
|
|
934
|
+
} else if (apiClient && authKeyHex) {
|
|
935
|
+
try {
|
|
936
|
+
await apiClient.deleteFact(fact.existingFactId, authKeyHex);
|
|
937
|
+
logger.info(`LLM dedup: UPDATE — deleted ${fact.existingFactId}, storing replacement`);
|
|
938
|
+
} catch (delErr) {
|
|
939
|
+
logger.warn(`LLM dedup: UPDATE delete failed for ${fact.existingFactId}: ${delErr instanceof Error ? delErr.message : String(delErr)}`);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
superseded++;
|
|
943
|
+
// Fall through to store the new replacement fact below.
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// ADD (default) or UPDATE (after tombstoning old) — proceed to store.
|
|
947
|
+
// The cosine-based store-time dedup below provides an additional safety net.
|
|
948
|
+
|
|
949
|
+
// Store-time near-duplicate check: search vault before writing.
|
|
950
|
+
let effectiveImportance = fact.importance;
|
|
951
|
+
|
|
952
|
+
if (STORE_DEDUP_ENABLED && embeddingResult) {
|
|
953
|
+
const dupResult = await searchForNearDuplicates(
|
|
954
|
+
fact.text,
|
|
955
|
+
embeddingResult.embedding,
|
|
956
|
+
allIndices,
|
|
957
|
+
logger,
|
|
958
|
+
);
|
|
959
|
+
|
|
960
|
+
if (dupResult) {
|
|
961
|
+
const action = shouldSupersede(fact.importance, dupResult.match);
|
|
962
|
+
if (action === 'skip') {
|
|
963
|
+
logger.info(
|
|
964
|
+
`Store-time dedup: skipping "${fact.text.slice(0, 60)}…" (sim=${dupResult.similarity.toFixed(3)}, existing ID=${dupResult.match.id})`,
|
|
965
|
+
);
|
|
966
|
+
skipped++;
|
|
967
|
+
continue;
|
|
968
|
+
}
|
|
969
|
+
// action === 'supersede': delete old fact, inherit higher importance
|
|
970
|
+
if (isSubgraphMode()) {
|
|
971
|
+
try {
|
|
972
|
+
const tombConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
973
|
+
const tombstone: FactPayload = {
|
|
974
|
+
id: dupResult.match.id,
|
|
975
|
+
timestamp: new Date().toISOString(),
|
|
976
|
+
owner: subgraphOwner || userId!,
|
|
977
|
+
encryptedBlob: '00',
|
|
978
|
+
blindIndices: [],
|
|
979
|
+
decayScore: 0,
|
|
980
|
+
source: 'tombstone',
|
|
981
|
+
contentFp: '',
|
|
982
|
+
agentId: 'openclaw-plugin-auto',
|
|
983
|
+
};
|
|
984
|
+
const tombProtobuf = encodeFactProtobuf(tombstone);
|
|
985
|
+
await submitFactOnChain(tombProtobuf, tombConfig);
|
|
986
|
+
logger.info(
|
|
987
|
+
`Store-time dedup: superseded ${dupResult.match.id} on-chain (sim=${dupResult.similarity.toFixed(3)})`,
|
|
988
|
+
);
|
|
989
|
+
} catch (tombErr) {
|
|
990
|
+
logger.warn(
|
|
991
|
+
`Store-time dedup: failed to tombstone ${dupResult.match.id}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`,
|
|
992
|
+
);
|
|
993
|
+
}
|
|
994
|
+
} else if (apiClient && authKeyHex) {
|
|
995
|
+
try {
|
|
996
|
+
await apiClient.deleteFact(dupResult.match.id, authKeyHex);
|
|
997
|
+
logger.info(
|
|
998
|
+
`Store-time dedup: superseding ${dupResult.match.id} (sim=${dupResult.similarity.toFixed(3)})`,
|
|
999
|
+
);
|
|
1000
|
+
} catch (delErr) {
|
|
1001
|
+
logger.warn(
|
|
1002
|
+
`Store-time dedup: failed to delete superseded fact ${dupResult.match.id}: ${delErr instanceof Error ? delErr.message : String(delErr)}`,
|
|
1003
|
+
);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
effectiveImportance = Math.max(fact.importance, dupResult.match.decayScore);
|
|
1007
|
+
superseded++;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
const doc = {
|
|
1012
|
+
text: fact.text,
|
|
1013
|
+
metadata: {
|
|
1014
|
+
type: fact.type,
|
|
1015
|
+
importance: effectiveImportance / 10,
|
|
1016
|
+
source: 'auto-extraction',
|
|
1017
|
+
created_at: new Date().toISOString(),
|
|
1018
|
+
},
|
|
1019
|
+
};
|
|
1020
|
+
|
|
1021
|
+
const encryptedBlob = encryptToHex(JSON.stringify(doc), encryptionKey);
|
|
1022
|
+
|
|
1023
|
+
const contentFp = generateContentFingerprint(fact.text, dedupKey);
|
|
1024
|
+
const factId = crypto.randomUUID();
|
|
1025
|
+
|
|
1026
|
+
const payload: StoreFactPayload = {
|
|
1027
|
+
id: factId,
|
|
1028
|
+
timestamp: new Date().toISOString(),
|
|
1029
|
+
encrypted_blob: encryptedBlob,
|
|
1030
|
+
blind_indices: allIndices,
|
|
1031
|
+
decay_score: effectiveImportance,
|
|
1032
|
+
source: 'auto-extraction',
|
|
1033
|
+
content_fp: contentFp,
|
|
1034
|
+
agent_id: 'openclaw-plugin-auto',
|
|
1035
|
+
encrypted_embedding: embeddingResult?.encryptedEmbedding,
|
|
1036
|
+
};
|
|
1037
|
+
|
|
1038
|
+
if (isSubgraphMode()) {
|
|
1039
|
+
const config = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
1040
|
+
const protobuf = encodeFactProtobuf({
|
|
1041
|
+
id: factId,
|
|
1042
|
+
timestamp: new Date().toISOString(),
|
|
1043
|
+
owner: subgraphOwner || userId!,
|
|
1044
|
+
encryptedBlob: encryptedBlob,
|
|
1045
|
+
blindIndices: allIndices,
|
|
1046
|
+
decayScore: effectiveImportance,
|
|
1047
|
+
source: 'auto-extraction',
|
|
1048
|
+
contentFp: contentFp,
|
|
1049
|
+
agentId: 'openclaw-plugin-auto',
|
|
1050
|
+
encryptedEmbedding: embeddingResult?.encryptedEmbedding,
|
|
1051
|
+
});
|
|
1052
|
+
await submitFactOnChain(protobuf, config);
|
|
1053
|
+
} else {
|
|
1054
|
+
await apiClient.store(userId, [payload], authKeyHex);
|
|
1055
|
+
}
|
|
1056
|
+
stored++;
|
|
1057
|
+
} catch (err: unknown) {
|
|
1058
|
+
// Check for 403 / quota exceeded — invalidate billing cache so next
|
|
1059
|
+
// before_agent_start re-fetches and warns the user.
|
|
1060
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
1061
|
+
if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
|
|
1062
|
+
try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
|
|
1063
|
+
logger.warn(`Quota exceeded — billing cache invalidated. ${errMsg}`);
|
|
1064
|
+
break; // Stop trying to store remaining facts — they'll all fail too
|
|
1065
|
+
}
|
|
1066
|
+
// Otherwise skip failed facts (e.g., duplicates return success with duplicate_ids)
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
if (stored > 0 || superseded > 0 || skipped > 0) {
|
|
1071
|
+
logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}`);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
return stored;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
// ---------------------------------------------------------------------------
|
|
1078
|
+
// Import handler (for totalreclaw_import_from tool)
|
|
1079
|
+
// ---------------------------------------------------------------------------
|
|
1080
|
+
|
|
1081
|
+
/**
|
|
1082
|
+
* Handle import_from tool calls in the plugin context.
|
|
1083
|
+
*
|
|
1084
|
+
* Uses the shared adapters to parse, then stores via storeExtractedFacts().
|
|
1085
|
+
*/
|
|
1086
|
+
async function handlePluginImportFrom(
|
|
1087
|
+
params: Record<string, unknown>,
|
|
1088
|
+
logger: OpenClawPluginApi['logger'],
|
|
1089
|
+
): Promise<Record<string, unknown>> {
|
|
1090
|
+
const startTime = Date.now();
|
|
1091
|
+
|
|
1092
|
+
const source = params.source as string;
|
|
1093
|
+
const validSources = ['mem0', 'mcp-memory', 'memoclaw', 'generic-json', 'generic-csv'];
|
|
1094
|
+
|
|
1095
|
+
if (!source || !validSources.includes(source)) {
|
|
1096
|
+
return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
try {
|
|
1100
|
+
const { getAdapter } = await import('./import-adapters/index.js');
|
|
1101
|
+
const adapter = getAdapter(source as import('./import-adapters/types.js').ImportSource);
|
|
1102
|
+
|
|
1103
|
+
const parseResult = await adapter.parse({
|
|
1104
|
+
content: params.content as string | undefined,
|
|
1105
|
+
api_key: params.api_key as string | undefined,
|
|
1106
|
+
source_user_id: params.source_user_id as string | undefined,
|
|
1107
|
+
api_url: params.api_url as string | undefined,
|
|
1108
|
+
file_path: params.file_path as string | undefined,
|
|
1109
|
+
});
|
|
1110
|
+
|
|
1111
|
+
if (parseResult.errors.length > 0 && parseResult.facts.length === 0) {
|
|
1112
|
+
return {
|
|
1113
|
+
success: false,
|
|
1114
|
+
error: `Failed to parse ${adapter.displayName} data`,
|
|
1115
|
+
details: parseResult.errors,
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
if (params.dry_run) {
|
|
1120
|
+
return {
|
|
1121
|
+
success: true,
|
|
1122
|
+
dry_run: true,
|
|
1123
|
+
source,
|
|
1124
|
+
total_found: parseResult.facts.length,
|
|
1125
|
+
preview: parseResult.facts.slice(0, 10).map((f) => ({
|
|
1126
|
+
type: f.type,
|
|
1127
|
+
text: f.text.slice(0, 100),
|
|
1128
|
+
importance: f.importance,
|
|
1129
|
+
})),
|
|
1130
|
+
warnings: parseResult.warnings,
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
// Convert NormalizedFact[] to ExtractedFact[] for storeExtractedFacts()
|
|
1135
|
+
const extractedFacts: ExtractedFact[] = parseResult.facts.map((f) => ({
|
|
1136
|
+
text: f.text,
|
|
1137
|
+
type: f.type,
|
|
1138
|
+
importance: f.importance,
|
|
1139
|
+
action: 'ADD' as const,
|
|
1140
|
+
}));
|
|
1141
|
+
|
|
1142
|
+
// Store in batches of 50
|
|
1143
|
+
let totalStored = 0;
|
|
1144
|
+
const batchSize = 50;
|
|
1145
|
+
|
|
1146
|
+
for (let i = 0; i < extractedFacts.length; i += batchSize) {
|
|
1147
|
+
const batch = extractedFacts.slice(i, i + batchSize);
|
|
1148
|
+
const stored = await storeExtractedFacts(batch, logger);
|
|
1149
|
+
totalStored += stored;
|
|
1150
|
+
|
|
1151
|
+
logger.info(
|
|
1152
|
+
`Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`,
|
|
1153
|
+
);
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
return {
|
|
1157
|
+
success: true,
|
|
1158
|
+
source,
|
|
1159
|
+
import_id: crypto.randomUUID(),
|
|
1160
|
+
total_found: parseResult.facts.length,
|
|
1161
|
+
imported: totalStored,
|
|
1162
|
+
skipped: parseResult.facts.length - totalStored,
|
|
1163
|
+
warnings: parseResult.warnings,
|
|
1164
|
+
duration_ms: Date.now() - startTime,
|
|
1165
|
+
};
|
|
1166
|
+
} catch (e) {
|
|
1167
|
+
const msg = e instanceof Error ? e.message : 'Unknown error';
|
|
1168
|
+
logger.error(`Import failed: ${msg}`);
|
|
1169
|
+
return { success: false, error: `Import failed: ${msg}` };
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
// ---------------------------------------------------------------------------
|
|
1174
|
+
// Plugin definition
|
|
1175
|
+
// ---------------------------------------------------------------------------
|
|
1176
|
+
|
|
1177
|
+
const plugin = {
|
|
1178
|
+
id: 'totalreclaw',
|
|
1179
|
+
name: 'TotalReclaw',
|
|
1180
|
+
description: 'Zero-knowledge encrypted memory vault for AI agents',
|
|
1181
|
+
kind: 'memory' as const,
|
|
1182
|
+
configSchema: {
|
|
1183
|
+
type: 'object',
|
|
1184
|
+
additionalProperties: false,
|
|
1185
|
+
properties: {
|
|
1186
|
+
extraction: {
|
|
1187
|
+
type: 'object',
|
|
1188
|
+
properties: {
|
|
1189
|
+
model: { type: 'string', description: "Override the extraction model (e.g., 'glm-4.5-flash', 'gpt-4.1-mini')" },
|
|
1190
|
+
enabled: { type: 'boolean', description: 'Enable/disable auto-extraction (default: true)' },
|
|
1191
|
+
},
|
|
1192
|
+
additionalProperties: false,
|
|
1193
|
+
},
|
|
1194
|
+
},
|
|
1195
|
+
},
|
|
1196
|
+
|
|
1197
|
+
register(api: OpenClawPluginApi) {
|
|
1198
|
+
// ---------------------------------------------------------------
|
|
1199
|
+
// LLM client initialization (auto-detect provider from OpenClaw config)
|
|
1200
|
+
// ---------------------------------------------------------------
|
|
1201
|
+
|
|
1202
|
+
initLLMClient({
|
|
1203
|
+
primaryModel: api.config?.agents?.defaults?.model?.primary as string | undefined,
|
|
1204
|
+
pluginConfig: api.pluginConfig,
|
|
1205
|
+
logger: api.logger,
|
|
1206
|
+
});
|
|
1207
|
+
|
|
1208
|
+
// ---------------------------------------------------------------
|
|
1209
|
+
// Service registration (lifecycle logging)
|
|
1210
|
+
// ---------------------------------------------------------------
|
|
1211
|
+
|
|
1212
|
+
api.registerService({
|
|
1213
|
+
id: 'totalreclaw',
|
|
1214
|
+
start: () => {
|
|
1215
|
+
api.logger.info('TotalReclaw plugin loaded');
|
|
1216
|
+
},
|
|
1217
|
+
stop: () => {
|
|
1218
|
+
api.logger.info('TotalReclaw plugin stopped');
|
|
1219
|
+
},
|
|
1220
|
+
});
|
|
1221
|
+
|
|
1222
|
+
// ---------------------------------------------------------------
|
|
1223
|
+
// Tool: totalreclaw_remember
|
|
1224
|
+
// ---------------------------------------------------------------
|
|
1225
|
+
|
|
1226
|
+
api.registerTool(
|
|
1227
|
+
{
|
|
1228
|
+
name: 'totalreclaw_remember',
|
|
1229
|
+
label: 'Remember',
|
|
1230
|
+
description:
|
|
1231
|
+
'Store a memory in the encrypted vault. Use this when the user shares important information worth remembering.',
|
|
1232
|
+
parameters: {
|
|
1233
|
+
type: 'object',
|
|
1234
|
+
properties: {
|
|
1235
|
+
text: {
|
|
1236
|
+
type: 'string',
|
|
1237
|
+
description: 'The memory text to store',
|
|
1238
|
+
},
|
|
1239
|
+
type: {
|
|
1240
|
+
type: 'string',
|
|
1241
|
+
enum: ['fact', 'preference', 'decision', 'episodic', 'goal'],
|
|
1242
|
+
description: 'The kind of memory (default: fact)',
|
|
1243
|
+
},
|
|
1244
|
+
importance: {
|
|
1245
|
+
type: 'number',
|
|
1246
|
+
minimum: 1,
|
|
1247
|
+
maximum: 10,
|
|
1248
|
+
description: 'Importance score 1-10 (default: 5)',
|
|
1249
|
+
},
|
|
1250
|
+
},
|
|
1251
|
+
required: ['text'],
|
|
1252
|
+
additionalProperties: false,
|
|
1253
|
+
},
|
|
1254
|
+
async execute(_toolCallId: string, params: { text: string; type?: string; importance?: number }) {
|
|
1255
|
+
try {
|
|
1256
|
+
await requireFullSetup(api.logger);
|
|
1257
|
+
|
|
1258
|
+
const memoryType = params.type ?? 'fact';
|
|
1259
|
+
let importance = params.importance ?? 5;
|
|
1260
|
+
|
|
1261
|
+
// Generate blind indices for server-side search.
|
|
1262
|
+
const blindIndices = generateBlindIndices(params.text);
|
|
1263
|
+
|
|
1264
|
+
// Generate embedding + LSH bucket hashes (PoC v2).
|
|
1265
|
+
// Falls back to word-only indices if embedding generation fails.
|
|
1266
|
+
const embeddingResult = await generateEmbeddingAndLSH(params.text, api.logger);
|
|
1267
|
+
|
|
1268
|
+
// Merge LSH bucket hashes into blind indices.
|
|
1269
|
+
const allIndices = embeddingResult
|
|
1270
|
+
? [...blindIndices, ...embeddingResult.lshBuckets]
|
|
1271
|
+
: blindIndices;
|
|
1272
|
+
|
|
1273
|
+
// Store-time dedup: for explicit remember, ALWAYS supersede
|
|
1274
|
+
// (user explicitly wants this stored — just remove the old one).
|
|
1275
|
+
let supersededId: string | undefined;
|
|
1276
|
+
if (STORE_DEDUP_ENABLED && embeddingResult) {
|
|
1277
|
+
const dupResult = await searchForNearDuplicates(
|
|
1278
|
+
params.text,
|
|
1279
|
+
embeddingResult.embedding,
|
|
1280
|
+
allIndices,
|
|
1281
|
+
api.logger,
|
|
1282
|
+
);
|
|
1283
|
+
if (dupResult) {
|
|
1284
|
+
// Inherit higher importance from existing fact.
|
|
1285
|
+
importance = Math.max(importance, dupResult.match.decayScore);
|
|
1286
|
+
supersededId = dupResult.match.id;
|
|
1287
|
+
|
|
1288
|
+
if (isSubgraphMode()) {
|
|
1289
|
+
try {
|
|
1290
|
+
const tombConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
1291
|
+
const tombstone: FactPayload = {
|
|
1292
|
+
id: dupResult.match.id,
|
|
1293
|
+
timestamp: new Date().toISOString(),
|
|
1294
|
+
owner: subgraphOwner || userId!,
|
|
1295
|
+
encryptedBlob: '00',
|
|
1296
|
+
blindIndices: [],
|
|
1297
|
+
decayScore: 0,
|
|
1298
|
+
source: 'tombstone',
|
|
1299
|
+
contentFp: '',
|
|
1300
|
+
agentId: 'openclaw-plugin',
|
|
1301
|
+
};
|
|
1302
|
+
const tombProtobuf = encodeFactProtobuf(tombstone);
|
|
1303
|
+
await submitFactOnChain(tombProtobuf, tombConfig);
|
|
1304
|
+
api.logger.info(
|
|
1305
|
+
`Remember dedup: superseded ${dupResult.match.id} on-chain (sim=${dupResult.similarity.toFixed(3)})`,
|
|
1306
|
+
);
|
|
1307
|
+
} catch (tombErr) {
|
|
1308
|
+
api.logger.warn(
|
|
1309
|
+
`Remember dedup: failed to tombstone ${dupResult.match.id}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`,
|
|
1310
|
+
);
|
|
1311
|
+
supersededId = undefined;
|
|
1312
|
+
}
|
|
1313
|
+
} else if (apiClient && authKeyHex) {
|
|
1314
|
+
try {
|
|
1315
|
+
await apiClient.deleteFact(dupResult.match.id, authKeyHex);
|
|
1316
|
+
api.logger.info(
|
|
1317
|
+
`Remember dedup: superseded ${dupResult.match.id} (sim=${dupResult.similarity.toFixed(3)})`,
|
|
1318
|
+
);
|
|
1319
|
+
} catch (delErr) {
|
|
1320
|
+
api.logger.warn(
|
|
1321
|
+
`Remember dedup: failed to delete superseded fact ${dupResult.match.id}: ${delErr instanceof Error ? delErr.message : String(delErr)}`,
|
|
1322
|
+
);
|
|
1323
|
+
supersededId = undefined; // Don't report supersession if delete failed
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// Build the document JSON that will be encrypted.
|
|
1330
|
+
const doc = {
|
|
1331
|
+
text: params.text,
|
|
1332
|
+
metadata: {
|
|
1333
|
+
type: memoryType,
|
|
1334
|
+
importance: importance / 10, // normalise to 0-1 range
|
|
1335
|
+
source: 'explicit',
|
|
1336
|
+
created_at: new Date().toISOString(),
|
|
1337
|
+
},
|
|
1338
|
+
};
|
|
1339
|
+
|
|
1340
|
+
// Encrypt the document.
|
|
1341
|
+
const encryptedBlob = encryptToHex(JSON.stringify(doc), encryptionKey!);
|
|
1342
|
+
|
|
1343
|
+
// Generate content fingerprint for dedup.
|
|
1344
|
+
const contentFp = generateContentFingerprint(params.text, dedupKey!);
|
|
1345
|
+
|
|
1346
|
+
// Generate a unique fact ID.
|
|
1347
|
+
const factId = crypto.randomUUID();
|
|
1348
|
+
|
|
1349
|
+
// Build the payload matching the server's FactJSON schema.
|
|
1350
|
+
const factPayload: StoreFactPayload = {
|
|
1351
|
+
id: factId,
|
|
1352
|
+
timestamp: new Date().toISOString(),
|
|
1353
|
+
encrypted_blob: encryptedBlob,
|
|
1354
|
+
blind_indices: allIndices,
|
|
1355
|
+
decay_score: importance,
|
|
1356
|
+
source: 'explicit',
|
|
1357
|
+
content_fp: contentFp,
|
|
1358
|
+
agent_id: 'openclaw-plugin',
|
|
1359
|
+
encrypted_embedding: embeddingResult?.encryptedEmbedding,
|
|
1360
|
+
};
|
|
1361
|
+
|
|
1362
|
+
if (isSubgraphMode()) {
|
|
1363
|
+
// Subgraph mode: encode as Protobuf and submit on-chain via relay UserOp
|
|
1364
|
+
const config = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
1365
|
+
const protobuf = encodeFactProtobuf({
|
|
1366
|
+
id: factId,
|
|
1367
|
+
timestamp: new Date().toISOString(),
|
|
1368
|
+
owner: subgraphOwner || userId!,
|
|
1369
|
+
encryptedBlob: encryptedBlob,
|
|
1370
|
+
blindIndices: allIndices,
|
|
1371
|
+
decayScore: importance,
|
|
1372
|
+
source: 'explicit',
|
|
1373
|
+
contentFp: contentFp,
|
|
1374
|
+
agentId: 'openclaw-plugin',
|
|
1375
|
+
encryptedEmbedding: embeddingResult?.encryptedEmbedding,
|
|
1376
|
+
});
|
|
1377
|
+
await submitFactOnChain(protobuf, config);
|
|
1378
|
+
} else {
|
|
1379
|
+
await apiClient!.store(userId!, [factPayload], authKeyHex!);
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
const statusMsg = supersededId
|
|
1383
|
+
? `Memory stored (ID: ${factId}). Superseded an older similar memory.`
|
|
1384
|
+
: `Memory stored (ID: ${factId})`;
|
|
1385
|
+
|
|
1386
|
+
return {
|
|
1387
|
+
content: [{ type: 'text', text: statusMsg }],
|
|
1388
|
+
details: { factId, supersededId },
|
|
1389
|
+
};
|
|
1390
|
+
} catch (err: unknown) {
|
|
1391
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1392
|
+
api.logger.error(`totalreclaw_remember failed: ${message}`);
|
|
1393
|
+
return {
|
|
1394
|
+
content: [{ type: 'text', text: `Failed to store memory: ${message}` }],
|
|
1395
|
+
};
|
|
1396
|
+
}
|
|
1397
|
+
},
|
|
1398
|
+
},
|
|
1399
|
+
{ name: 'totalreclaw_remember' },
|
|
1400
|
+
);
|
|
1401
|
+
|
|
1402
|
+
// ---------------------------------------------------------------
|
|
1403
|
+
// Tool: totalreclaw_recall
|
|
1404
|
+
// ---------------------------------------------------------------
|
|
1405
|
+
|
|
1406
|
+
api.registerTool(
|
|
1407
|
+
{
|
|
1408
|
+
name: 'totalreclaw_recall',
|
|
1409
|
+
label: 'Recall',
|
|
1410
|
+
description:
|
|
1411
|
+
'Search the encrypted memory vault. Returns the most relevant memories matching the query.',
|
|
1412
|
+
parameters: {
|
|
1413
|
+
type: 'object',
|
|
1414
|
+
properties: {
|
|
1415
|
+
query: {
|
|
1416
|
+
type: 'string',
|
|
1417
|
+
description: 'Search query text',
|
|
1418
|
+
},
|
|
1419
|
+
k: {
|
|
1420
|
+
type: 'number',
|
|
1421
|
+
minimum: 1,
|
|
1422
|
+
maximum: 20,
|
|
1423
|
+
description: 'Number of results to return (default: 8)',
|
|
1424
|
+
},
|
|
1425
|
+
},
|
|
1426
|
+
required: ['query'],
|
|
1427
|
+
additionalProperties: false,
|
|
1428
|
+
},
|
|
1429
|
+
async execute(_toolCallId: string, params: { query: string; k?: number }) {
|
|
1430
|
+
try {
|
|
1431
|
+
await requireFullSetup(api.logger);
|
|
1432
|
+
|
|
1433
|
+
const k = Math.min(params.k ?? 8, 20);
|
|
1434
|
+
|
|
1435
|
+
// 1. Generate word trapdoors (blind indices for the query).
|
|
1436
|
+
const wordTrapdoors = generateBlindIndices(params.query);
|
|
1437
|
+
|
|
1438
|
+
// 2. Generate query embedding + LSH trapdoors (may fail gracefully).
|
|
1439
|
+
let queryEmbedding: number[] | null = null;
|
|
1440
|
+
let lshTrapdoors: string[] = [];
|
|
1441
|
+
try {
|
|
1442
|
+
queryEmbedding = await generateEmbedding(params.query, { isQuery: true });
|
|
1443
|
+
const hasher = getLSHHasher(api.logger);
|
|
1444
|
+
if (hasher && queryEmbedding) {
|
|
1445
|
+
lshTrapdoors = hasher.hash(queryEmbedding);
|
|
1446
|
+
}
|
|
1447
|
+
} catch (err) {
|
|
1448
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1449
|
+
api.logger.warn(`Recall: embedding/LSH generation failed (using word-only trapdoors): ${msg}`);
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
// 3. Merge word trapdoors + LSH trapdoors.
|
|
1453
|
+
const allTrapdoors = [...wordTrapdoors, ...lshTrapdoors];
|
|
1454
|
+
|
|
1455
|
+
if (allTrapdoors.length === 0) {
|
|
1456
|
+
return {
|
|
1457
|
+
content: [{ type: 'text', text: 'No searchable terms in query.' }],
|
|
1458
|
+
details: { count: 0, memories: [] },
|
|
1459
|
+
};
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
// 4. Request more candidates than needed so we can re-rank client-side.
|
|
1463
|
+
// 5. Decrypt candidates (text + embeddings) and build reranker input.
|
|
1464
|
+
const rerankerCandidates: RerankerCandidate[] = [];
|
|
1465
|
+
const metaMap = new Map<string, { metadata: Record<string, unknown>; timestamp: number }>();
|
|
1466
|
+
|
|
1467
|
+
if (isSubgraphMode()) {
|
|
1468
|
+
// --- Subgraph search path ---
|
|
1469
|
+
const factCount = await getSubgraphFactCount(subgraphOwner || userId!, authKeyHex!);
|
|
1470
|
+
const pool = computeCandidatePool(factCount);
|
|
1471
|
+
const subgraphResults = await searchSubgraph(subgraphOwner || userId!, allTrapdoors, pool, authKeyHex!);
|
|
1472
|
+
|
|
1473
|
+
for (const result of subgraphResults) {
|
|
1474
|
+
try {
|
|
1475
|
+
const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
|
|
1476
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
1477
|
+
|
|
1478
|
+
let decryptedEmbedding: number[] | undefined;
|
|
1479
|
+
if (result.encryptedEmbedding) {
|
|
1480
|
+
try {
|
|
1481
|
+
decryptedEmbedding = JSON.parse(
|
|
1482
|
+
decryptFromHex(result.encryptedEmbedding, encryptionKey!),
|
|
1483
|
+
);
|
|
1484
|
+
} catch {
|
|
1485
|
+
// Embedding decryption failed -- proceed without it.
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
rerankerCandidates.push({
|
|
1490
|
+
id: result.id,
|
|
1491
|
+
text: doc.text,
|
|
1492
|
+
embedding: decryptedEmbedding,
|
|
1493
|
+
importance: (doc.metadata?.importance as number) ?? 0.5,
|
|
1494
|
+
createdAt: result.timestamp ? parseInt(result.timestamp, 10) : undefined,
|
|
1495
|
+
});
|
|
1496
|
+
|
|
1497
|
+
metaMap.set(result.id, {
|
|
1498
|
+
metadata: doc.metadata ?? {},
|
|
1499
|
+
timestamp: Date.now(), // Subgraph doesn't return ms timestamp; use current
|
|
1500
|
+
});
|
|
1501
|
+
} catch {
|
|
1502
|
+
// Skip candidates we cannot decrypt.
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
// Update hot cache with top results for instant auto-recall.
|
|
1507
|
+
try {
|
|
1508
|
+
if (!pluginHotCache && encryptionKey) {
|
|
1509
|
+
const config = getSubgraphConfig();
|
|
1510
|
+
pluginHotCache = new PluginHotCache(config.cachePath, encryptionKey.toString('hex'));
|
|
1511
|
+
pluginHotCache.load();
|
|
1512
|
+
}
|
|
1513
|
+
if (pluginHotCache) {
|
|
1514
|
+
const hotFacts: HotFact[] = rerankerCandidates.map((c) => {
|
|
1515
|
+
const meta = metaMap.get(c.id);
|
|
1516
|
+
const importance = meta?.metadata.importance
|
|
1517
|
+
? Math.round((meta.metadata.importance as number) * 10)
|
|
1518
|
+
: 5;
|
|
1519
|
+
return { id: c.id, text: c.text, importance };
|
|
1520
|
+
});
|
|
1521
|
+
pluginHotCache.setHotFacts(hotFacts);
|
|
1522
|
+
pluginHotCache.setFactCount(rerankerCandidates.length);
|
|
1523
|
+
pluginHotCache.flush();
|
|
1524
|
+
}
|
|
1525
|
+
} catch {
|
|
1526
|
+
// Hot cache update is best-effort -- don't fail the recall.
|
|
1527
|
+
}
|
|
1528
|
+
} else {
|
|
1529
|
+
// --- Server search path (existing behavior) ---
|
|
1530
|
+
const factCount = await getFactCount(api.logger);
|
|
1531
|
+
const pool = computeCandidatePool(factCount);
|
|
1532
|
+
const candidates = await apiClient!.search(
|
|
1533
|
+
userId!,
|
|
1534
|
+
allTrapdoors,
|
|
1535
|
+
pool,
|
|
1536
|
+
authKeyHex!,
|
|
1537
|
+
);
|
|
1538
|
+
|
|
1539
|
+
for (const candidate of candidates) {
|
|
1540
|
+
try {
|
|
1541
|
+
const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
|
|
1542
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
1543
|
+
|
|
1544
|
+
let decryptedEmbedding: number[] | undefined;
|
|
1545
|
+
if (candidate.encrypted_embedding) {
|
|
1546
|
+
try {
|
|
1547
|
+
decryptedEmbedding = JSON.parse(
|
|
1548
|
+
decryptFromHex(candidate.encrypted_embedding, encryptionKey!),
|
|
1549
|
+
);
|
|
1550
|
+
} catch {
|
|
1551
|
+
// Embedding decryption failed -- proceed without it.
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
rerankerCandidates.push({
|
|
1556
|
+
id: candidate.fact_id,
|
|
1557
|
+
text: doc.text,
|
|
1558
|
+
embedding: decryptedEmbedding,
|
|
1559
|
+
importance: (doc.metadata?.importance as number) ?? 0.5,
|
|
1560
|
+
createdAt: typeof candidate.timestamp === 'number'
|
|
1561
|
+
? candidate.timestamp / 1000
|
|
1562
|
+
: new Date(candidate.timestamp).getTime() / 1000,
|
|
1563
|
+
});
|
|
1564
|
+
|
|
1565
|
+
metaMap.set(candidate.fact_id, {
|
|
1566
|
+
metadata: doc.metadata ?? {},
|
|
1567
|
+
timestamp: candidate.timestamp,
|
|
1568
|
+
});
|
|
1569
|
+
} catch {
|
|
1570
|
+
// Skip candidates we cannot decrypt (e.g. corrupted data).
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
// 6. Re-rank with BM25 + cosine + intent-weighted RRF fusion.
|
|
1576
|
+
const queryIntent = detectQueryIntent(params.query);
|
|
1577
|
+
const reranked = rerank(
|
|
1578
|
+
params.query,
|
|
1579
|
+
queryEmbedding ?? [],
|
|
1580
|
+
rerankerCandidates,
|
|
1581
|
+
k,
|
|
1582
|
+
INTENT_WEIGHTS[queryIntent],
|
|
1583
|
+
);
|
|
1584
|
+
|
|
1585
|
+
if (reranked.length === 0) {
|
|
1586
|
+
return {
|
|
1587
|
+
content: [{ type: 'text', text: 'No memories found matching your query.' }],
|
|
1588
|
+
details: { count: 0, memories: [] },
|
|
1589
|
+
};
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
// 6b. Cosine similarity threshold gate — skip results when the
|
|
1593
|
+
// best match is below the minimum relevance threshold.
|
|
1594
|
+
const maxCosine = Math.max(
|
|
1595
|
+
...reranked.map((r) => r.cosineSimilarity ?? 0),
|
|
1596
|
+
);
|
|
1597
|
+
if (maxCosine < COSINE_THRESHOLD) {
|
|
1598
|
+
api.logger.info(
|
|
1599
|
+
`Recall: cosine threshold gate filtered results (max=${maxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD})`,
|
|
1600
|
+
);
|
|
1601
|
+
return {
|
|
1602
|
+
content: [{ type: 'text', text: 'No relevant memories found for this query.' }],
|
|
1603
|
+
details: { count: 0, memories: [] },
|
|
1604
|
+
};
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
// 7. Format results.
|
|
1608
|
+
const lines = reranked.map((m, i) => {
|
|
1609
|
+
const meta = metaMap.get(m.id);
|
|
1610
|
+
const imp = meta?.metadata.importance
|
|
1611
|
+
? ` (importance: ${Math.round((meta.metadata.importance as number) * 10)}/10)`
|
|
1612
|
+
: '';
|
|
1613
|
+
const age = meta ? relativeTime(meta.timestamp) : '';
|
|
1614
|
+
return `${i + 1}. ${m.text}${imp} -- ${age} [ID: ${m.id}]`;
|
|
1615
|
+
});
|
|
1616
|
+
|
|
1617
|
+
const formatted = lines.join('\n');
|
|
1618
|
+
|
|
1619
|
+
return {
|
|
1620
|
+
content: [{ type: 'text', text: formatted }],
|
|
1621
|
+
details: {
|
|
1622
|
+
count: reranked.length,
|
|
1623
|
+
memories: reranked.map((m) => ({
|
|
1624
|
+
factId: m.id,
|
|
1625
|
+
text: m.text,
|
|
1626
|
+
})),
|
|
1627
|
+
},
|
|
1628
|
+
};
|
|
1629
|
+
} catch (err: unknown) {
|
|
1630
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1631
|
+
api.logger.error(`totalreclaw_recall failed: ${message}`);
|
|
1632
|
+
return {
|
|
1633
|
+
content: [{ type: 'text', text: `Failed to search memories: ${message}` }],
|
|
1634
|
+
};
|
|
1635
|
+
}
|
|
1636
|
+
},
|
|
1637
|
+
},
|
|
1638
|
+
{ name: 'totalreclaw_recall' },
|
|
1639
|
+
);
|
|
1640
|
+
|
|
1641
|
+
// ---------------------------------------------------------------
|
|
1642
|
+
// Tool: totalreclaw_forget
|
|
1643
|
+
// ---------------------------------------------------------------
|
|
1644
|
+
|
|
1645
|
+
api.registerTool(
|
|
1646
|
+
{
|
|
1647
|
+
name: 'totalreclaw_forget',
|
|
1648
|
+
label: 'Forget',
|
|
1649
|
+
description: 'Delete a specific memory by its ID.',
|
|
1650
|
+
parameters: {
|
|
1651
|
+
type: 'object',
|
|
1652
|
+
properties: {
|
|
1653
|
+
factId: {
|
|
1654
|
+
type: 'string',
|
|
1655
|
+
description: 'The UUID of the memory to delete',
|
|
1656
|
+
},
|
|
1657
|
+
},
|
|
1658
|
+
required: ['factId'],
|
|
1659
|
+
additionalProperties: false,
|
|
1660
|
+
},
|
|
1661
|
+
async execute(_toolCallId: string, params: { factId: string }) {
|
|
1662
|
+
try {
|
|
1663
|
+
await requireFullSetup(api.logger);
|
|
1664
|
+
|
|
1665
|
+
if (isSubgraphMode()) {
|
|
1666
|
+
// On-chain tombstone: write a minimal protobuf with decayScore=0
|
|
1667
|
+
// The subgraph will overwrite the fact and set isActive=false
|
|
1668
|
+
const config = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
1669
|
+
const tombstone: FactPayload = {
|
|
1670
|
+
id: params.factId,
|
|
1671
|
+
timestamp: new Date().toISOString(),
|
|
1672
|
+
owner: subgraphOwner || userId!,
|
|
1673
|
+
encryptedBlob: '00', // minimal 1-byte placeholder
|
|
1674
|
+
blindIndices: [],
|
|
1675
|
+
decayScore: 0,
|
|
1676
|
+
source: 'tombstone',
|
|
1677
|
+
contentFp: '',
|
|
1678
|
+
agentId: 'openclaw-plugin',
|
|
1679
|
+
};
|
|
1680
|
+
const protobuf = encodeFactProtobuf(tombstone);
|
|
1681
|
+
const result = await submitFactOnChain(protobuf, config);
|
|
1682
|
+
api.logger.info(`Tombstone written for ${params.factId}: tx=${result.txHash}`);
|
|
1683
|
+
return {
|
|
1684
|
+
content: [{ type: 'text', text: `Memory ${params.factId} deleted (on-chain tombstone, tx: ${result.txHash})` }],
|
|
1685
|
+
details: { deleted: true, txHash: result.txHash },
|
|
1686
|
+
};
|
|
1687
|
+
} else {
|
|
1688
|
+
await apiClient!.deleteFact(params.factId, authKeyHex!);
|
|
1689
|
+
return {
|
|
1690
|
+
content: [{ type: 'text', text: `Memory ${params.factId} deleted` }],
|
|
1691
|
+
details: { deleted: true },
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
} catch (err: unknown) {
|
|
1695
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1696
|
+
api.logger.error(`totalreclaw_forget failed: ${message}`);
|
|
1697
|
+
return {
|
|
1698
|
+
content: [{ type: 'text', text: `Failed to delete memory: ${message}` }],
|
|
1699
|
+
};
|
|
1700
|
+
}
|
|
1701
|
+
},
|
|
1702
|
+
},
|
|
1703
|
+
{ name: 'totalreclaw_forget' },
|
|
1704
|
+
);
|
|
1705
|
+
|
|
1706
|
+
// ---------------------------------------------------------------
|
|
1707
|
+
// Tool: totalreclaw_export
|
|
1708
|
+
// ---------------------------------------------------------------
|
|
1709
|
+
|
|
1710
|
+
api.registerTool(
|
|
1711
|
+
{
|
|
1712
|
+
name: 'totalreclaw_export',
|
|
1713
|
+
label: 'Export',
|
|
1714
|
+
description:
|
|
1715
|
+
'Export all stored memories. Decrypts every memory and returns them as JSON or Markdown.',
|
|
1716
|
+
parameters: {
|
|
1717
|
+
type: 'object',
|
|
1718
|
+
properties: {
|
|
1719
|
+
format: {
|
|
1720
|
+
type: 'string',
|
|
1721
|
+
enum: ['json', 'markdown'],
|
|
1722
|
+
description: 'Output format (default: json)',
|
|
1723
|
+
},
|
|
1724
|
+
},
|
|
1725
|
+
additionalProperties: false,
|
|
1726
|
+
},
|
|
1727
|
+
async execute(_toolCallId: string, params: { format?: string }) {
|
|
1728
|
+
try {
|
|
1729
|
+
await requireFullSetup(api.logger);
|
|
1730
|
+
|
|
1731
|
+
const format = params.format ?? 'json';
|
|
1732
|
+
|
|
1733
|
+
// Paginate through all facts.
|
|
1734
|
+
const allFacts: Array<{
|
|
1735
|
+
id: string;
|
|
1736
|
+
text: string;
|
|
1737
|
+
metadata: Record<string, unknown>;
|
|
1738
|
+
created_at: string;
|
|
1739
|
+
}> = [];
|
|
1740
|
+
|
|
1741
|
+
if (isSubgraphMode()) {
|
|
1742
|
+
// Query subgraph for all active facts
|
|
1743
|
+
const config = getSubgraphConfig();
|
|
1744
|
+
const relayUrl = config.relayUrl;
|
|
1745
|
+
const PAGE_SIZE = 1000;
|
|
1746
|
+
let skip = 0;
|
|
1747
|
+
let hasMore = true;
|
|
1748
|
+
const owner = subgraphOwner || userId || '';
|
|
1749
|
+
|
|
1750
|
+
while (hasMore) {
|
|
1751
|
+
const query = `{ facts(where: { owner: "${owner}", isActive: true }, first: ${PAGE_SIZE}, skip: ${skip}, orderBy: sequenceId, orderDirection: asc) { id encryptedBlob source agentId timestamp sequenceId } }`;
|
|
1752
|
+
|
|
1753
|
+
const res = await fetch(`${relayUrl}/v1/subgraph`, {
|
|
1754
|
+
method: 'POST',
|
|
1755
|
+
headers: {
|
|
1756
|
+
'Content-Type': 'application/json',
|
|
1757
|
+
'X-TotalReclaw-Client': 'openclaw-plugin',
|
|
1758
|
+
...(authKeyHex ? { Authorization: `Bearer ${authKeyHex}` } : {}),
|
|
1759
|
+
},
|
|
1760
|
+
body: JSON.stringify({ query }),
|
|
1761
|
+
});
|
|
1762
|
+
|
|
1763
|
+
const json = (await res.json()) as {
|
|
1764
|
+
data?: { facts?: Array<{ id: string; encryptedBlob: string; source: string; agentId: string; timestamp: string; sequenceId: string }> };
|
|
1765
|
+
};
|
|
1766
|
+
const facts = json?.data?.facts || [];
|
|
1767
|
+
|
|
1768
|
+
for (const fact of facts) {
|
|
1769
|
+
try {
|
|
1770
|
+
let hexBlob = fact.encryptedBlob;
|
|
1771
|
+
if (hexBlob.startsWith('0x')) hexBlob = hexBlob.slice(2);
|
|
1772
|
+
const docJson = decryptFromHex(hexBlob, encryptionKey!);
|
|
1773
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
1774
|
+
allFacts.push({
|
|
1775
|
+
id: fact.id,
|
|
1776
|
+
text: doc.text,
|
|
1777
|
+
metadata: doc.metadata ?? {},
|
|
1778
|
+
created_at: new Date(parseInt(fact.timestamp) * 1000).toISOString(),
|
|
1779
|
+
});
|
|
1780
|
+
} catch {
|
|
1781
|
+
// Skip facts we cannot decrypt
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
skip += PAGE_SIZE;
|
|
1786
|
+
hasMore = facts.length === PAGE_SIZE;
|
|
1787
|
+
}
|
|
1788
|
+
} else {
|
|
1789
|
+
// HTTP server mode — paginate through PostgreSQL facts
|
|
1790
|
+
let cursor: string | undefined;
|
|
1791
|
+
let hasMore = true;
|
|
1792
|
+
|
|
1793
|
+
while (hasMore) {
|
|
1794
|
+
const page = await apiClient!.exportFacts(authKeyHex!, 1000, cursor);
|
|
1795
|
+
|
|
1796
|
+
for (const fact of page.facts) {
|
|
1797
|
+
try {
|
|
1798
|
+
const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey!);
|
|
1799
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
1800
|
+
allFacts.push({
|
|
1801
|
+
id: fact.id,
|
|
1802
|
+
text: doc.text,
|
|
1803
|
+
metadata: doc.metadata ?? {},
|
|
1804
|
+
created_at: fact.created_at,
|
|
1805
|
+
});
|
|
1806
|
+
} catch {
|
|
1807
|
+
// Skip facts we cannot decrypt.
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
cursor = page.cursor ?? undefined;
|
|
1812
|
+
hasMore = page.has_more;
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
// Format output.
|
|
1817
|
+
let formatted: string;
|
|
1818
|
+
|
|
1819
|
+
if (format === 'markdown') {
|
|
1820
|
+
if (allFacts.length === 0) {
|
|
1821
|
+
formatted = '*No memories stored.*';
|
|
1822
|
+
} else {
|
|
1823
|
+
const lines = allFacts.map((f, i) => {
|
|
1824
|
+
const meta = f.metadata;
|
|
1825
|
+
const type = (meta.type as string) ?? 'fact';
|
|
1826
|
+
const imp = meta.importance
|
|
1827
|
+
? ` (importance: ${Math.round((meta.importance as number) * 10)}/10)`
|
|
1828
|
+
: '';
|
|
1829
|
+
return `${i + 1}. **[${type}]** ${f.text}${imp} \n _ID: ${f.id} | Created: ${f.created_at}_`;
|
|
1830
|
+
});
|
|
1831
|
+
formatted = `# Exported Memories (${allFacts.length})\n\n${lines.join('\n')}`;
|
|
1832
|
+
}
|
|
1833
|
+
} else {
|
|
1834
|
+
formatted = JSON.stringify(allFacts, null, 2);
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
return {
|
|
1838
|
+
content: [{ type: 'text', text: formatted }],
|
|
1839
|
+
details: { count: allFacts.length },
|
|
1840
|
+
};
|
|
1841
|
+
} catch (err: unknown) {
|
|
1842
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1843
|
+
api.logger.error(`totalreclaw_export failed: ${message}`);
|
|
1844
|
+
return {
|
|
1845
|
+
content: [{ type: 'text', text: `Failed to export memories: ${message}` }],
|
|
1846
|
+
};
|
|
1847
|
+
}
|
|
1848
|
+
},
|
|
1849
|
+
},
|
|
1850
|
+
{ name: 'totalreclaw_export' },
|
|
1851
|
+
);
|
|
1852
|
+
|
|
1853
|
+
// ---------------------------------------------------------------
|
|
1854
|
+
// Tool: totalreclaw_status
|
|
1855
|
+
// ---------------------------------------------------------------
|
|
1856
|
+
|
|
1857
|
+
api.registerTool(
|
|
1858
|
+
{
|
|
1859
|
+
name: 'totalreclaw_status',
|
|
1860
|
+
label: 'Status',
|
|
1861
|
+
description:
|
|
1862
|
+
'Check TotalReclaw billing and subscription status — tier, writes used, reset date.',
|
|
1863
|
+
parameters: {
|
|
1864
|
+
type: 'object',
|
|
1865
|
+
properties: {},
|
|
1866
|
+
additionalProperties: false,
|
|
1867
|
+
},
|
|
1868
|
+
async execute() {
|
|
1869
|
+
try {
|
|
1870
|
+
await requireFullSetup(api.logger);
|
|
1871
|
+
|
|
1872
|
+
if (!authKeyHex) {
|
|
1873
|
+
return {
|
|
1874
|
+
content: [{ type: 'text', text: 'Auth credentials are not available. Please initialize first.' }],
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
const serverUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
|
|
1879
|
+
const walletAddr = subgraphOwner || userId || '';
|
|
1880
|
+
const response = await fetch(`${serverUrl}/v1/billing/status?wallet_address=${encodeURIComponent(walletAddr)}`, {
|
|
1881
|
+
method: 'GET',
|
|
1882
|
+
headers: {
|
|
1883
|
+
'Authorization': `Bearer ${authKeyHex}`,
|
|
1884
|
+
'Accept': 'application/json',
|
|
1885
|
+
'X-TotalReclaw-Client': 'openclaw-plugin',
|
|
1886
|
+
},
|
|
1887
|
+
});
|
|
1888
|
+
|
|
1889
|
+
if (!response.ok) {
|
|
1890
|
+
const body = await response.text().catch(() => '');
|
|
1891
|
+
return {
|
|
1892
|
+
content: [{ type: 'text', text: `Failed to fetch billing status (HTTP ${response.status}): ${body || response.statusText}` }],
|
|
1893
|
+
};
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
const data = await response.json() as Record<string, unknown>;
|
|
1897
|
+
const tier = (data.tier as string) || 'free';
|
|
1898
|
+
const freeWritesUsed = (data.free_writes_used as number) ?? 0;
|
|
1899
|
+
const freeWritesLimit = (data.free_writes_limit as number) ?? 0;
|
|
1900
|
+
const freeWritesResetAt = data.free_writes_reset_at as string | undefined;
|
|
1901
|
+
|
|
1902
|
+
// Update billing cache on success.
|
|
1903
|
+
writeBillingCache({
|
|
1904
|
+
tier,
|
|
1905
|
+
free_writes_used: freeWritesUsed,
|
|
1906
|
+
free_writes_limit: freeWritesLimit,
|
|
1907
|
+
features: data.features as BillingCache['features'] | undefined,
|
|
1908
|
+
checked_at: Date.now(),
|
|
1909
|
+
});
|
|
1910
|
+
|
|
1911
|
+
const tierLabel = tier === 'pro' ? 'Pro' : 'Free';
|
|
1912
|
+
const lines: string[] = [
|
|
1913
|
+
`Tier: ${tierLabel}`,
|
|
1914
|
+
`Writes: ${freeWritesUsed}/${freeWritesLimit} used this month`,
|
|
1915
|
+
];
|
|
1916
|
+
if (freeWritesResetAt) {
|
|
1917
|
+
lines.push(`Resets: ${new Date(freeWritesResetAt).toLocaleDateString()}`);
|
|
1918
|
+
}
|
|
1919
|
+
if (tier !== 'pro') {
|
|
1920
|
+
lines.push(`Pricing: https://totalreclaw.xyz/pricing`);
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1923
|
+
return {
|
|
1924
|
+
content: [{ type: 'text', text: lines.join('\n') }],
|
|
1925
|
+
details: { tier, free_writes_used: freeWritesUsed, free_writes_limit: freeWritesLimit },
|
|
1926
|
+
};
|
|
1927
|
+
} catch (err: unknown) {
|
|
1928
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1929
|
+
api.logger.error(`totalreclaw_status failed: ${message}`);
|
|
1930
|
+
return {
|
|
1931
|
+
content: [{ type: 'text', text: `Failed to check status: ${message}` }],
|
|
1932
|
+
};
|
|
1933
|
+
}
|
|
1934
|
+
},
|
|
1935
|
+
},
|
|
1936
|
+
{ name: 'totalreclaw_status' },
|
|
1937
|
+
);
|
|
1938
|
+
|
|
1939
|
+
// ---------------------------------------------------------------
|
|
1940
|
+
// Tool: totalreclaw_consolidate
|
|
1941
|
+
// ---------------------------------------------------------------
|
|
1942
|
+
|
|
1943
|
+
api.registerTool(
|
|
1944
|
+
{
|
|
1945
|
+
name: 'totalreclaw_consolidate',
|
|
1946
|
+
label: 'Consolidate',
|
|
1947
|
+
description:
|
|
1948
|
+
'Scan all stored memories and merge near-duplicates. Keeps the most important/recent version and removes redundant copies.',
|
|
1949
|
+
parameters: {
|
|
1950
|
+
type: 'object',
|
|
1951
|
+
properties: {
|
|
1952
|
+
dry_run: {
|
|
1953
|
+
type: 'boolean',
|
|
1954
|
+
description: 'Preview consolidation without deleting (default: false)',
|
|
1955
|
+
},
|
|
1956
|
+
},
|
|
1957
|
+
additionalProperties: false,
|
|
1958
|
+
},
|
|
1959
|
+
async execute(_toolCallId: string, params: { dry_run?: boolean }) {
|
|
1960
|
+
try {
|
|
1961
|
+
await requireFullSetup(api.logger);
|
|
1962
|
+
|
|
1963
|
+
const dryRun = params.dry_run ?? false;
|
|
1964
|
+
|
|
1965
|
+
// Consolidation is only available in centralized (HTTP server) mode.
|
|
1966
|
+
if (isSubgraphMode()) {
|
|
1967
|
+
return {
|
|
1968
|
+
content: [{ type: 'text', text: 'Consolidation is currently only available in centralized mode.' }],
|
|
1969
|
+
};
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
if (!apiClient || !authKeyHex || !encryptionKey) {
|
|
1973
|
+
return {
|
|
1974
|
+
content: [{ type: 'text', text: 'Plugin not fully initialized. Cannot consolidate.' }],
|
|
1975
|
+
};
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
// 1. Export all facts (paginated, max 10 pages of 1000).
|
|
1979
|
+
const allDecrypted: DecryptedCandidate[] = [];
|
|
1980
|
+
let cursor: string | undefined;
|
|
1981
|
+
let hasMore = true;
|
|
1982
|
+
let pageCount = 0;
|
|
1983
|
+
const MAX_PAGES = 10;
|
|
1984
|
+
|
|
1985
|
+
while (hasMore && pageCount < MAX_PAGES) {
|
|
1986
|
+
const page = await apiClient.exportFacts(authKeyHex, 1000, cursor);
|
|
1987
|
+
|
|
1988
|
+
for (const fact of page.facts) {
|
|
1989
|
+
try {
|
|
1990
|
+
const docJson = decryptFromHex(fact.encrypted_blob, encryptionKey);
|
|
1991
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
1992
|
+
|
|
1993
|
+
let embedding: number[] | null = null;
|
|
1994
|
+
// ExportedFact does not include encrypted_embedding — generate it on-the-fly.
|
|
1995
|
+
// For consolidation we need embeddings, so generate them.
|
|
1996
|
+
try {
|
|
1997
|
+
embedding = await generateEmbedding(doc.text);
|
|
1998
|
+
} catch { /* skip — fact will not be clustered */ }
|
|
1999
|
+
|
|
2000
|
+
allDecrypted.push({
|
|
2001
|
+
id: fact.id,
|
|
2002
|
+
text: doc.text,
|
|
2003
|
+
embedding,
|
|
2004
|
+
importance: doc.metadata?.importance
|
|
2005
|
+
? Math.round((doc.metadata.importance as number) * 10)
|
|
2006
|
+
: 5,
|
|
2007
|
+
decayScore: fact.decay_score,
|
|
2008
|
+
createdAt: new Date(fact.created_at).getTime(),
|
|
2009
|
+
version: fact.version,
|
|
2010
|
+
});
|
|
2011
|
+
} catch {
|
|
2012
|
+
// Skip undecryptable facts.
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
cursor = page.cursor ?? undefined;
|
|
2017
|
+
hasMore = page.has_more;
|
|
2018
|
+
pageCount++;
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
if (allDecrypted.length === 0) {
|
|
2022
|
+
return {
|
|
2023
|
+
content: [{ type: 'text', text: 'No memories found to consolidate.' }],
|
|
2024
|
+
};
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
// 2. Cluster by cosine similarity.
|
|
2028
|
+
const clusters = clusterFacts(allDecrypted, getConsolidationThreshold());
|
|
2029
|
+
|
|
2030
|
+
if (clusters.length === 0) {
|
|
2031
|
+
return {
|
|
2032
|
+
content: [{ type: 'text', text: `Scanned ${allDecrypted.length} memories — no near-duplicates found.` }],
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
// 3. Build report.
|
|
2037
|
+
const totalDuplicates = clusters.reduce((sum, c) => sum + c.duplicates.length, 0);
|
|
2038
|
+
const reportLines: string[] = [
|
|
2039
|
+
`Scanned ${allDecrypted.length} memories.`,
|
|
2040
|
+
`Found ${clusters.length} cluster(s) with ${totalDuplicates} duplicate(s).`,
|
|
2041
|
+
'',
|
|
2042
|
+
];
|
|
2043
|
+
|
|
2044
|
+
const displayClusters = clusters.slice(0, 10);
|
|
2045
|
+
for (let i = 0; i < displayClusters.length; i++) {
|
|
2046
|
+
const cluster = displayClusters[i];
|
|
2047
|
+
reportLines.push(`Cluster ${i + 1}: KEEP "${cluster.representative.text.slice(0, 80)}…"`);
|
|
2048
|
+
for (const dup of cluster.duplicates) {
|
|
2049
|
+
reportLines.push(` - REMOVE "${dup.text.slice(0, 80)}…" (ID: ${dup.id})`);
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
if (clusters.length > 10) {
|
|
2053
|
+
reportLines.push(`... and ${clusters.length - 10} more cluster(s).`);
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
// 4. If not dry_run, batch-delete duplicates.
|
|
2057
|
+
if (!dryRun) {
|
|
2058
|
+
const idsToDelete = clusters.flatMap((c) => c.duplicates.map((d) => d.id));
|
|
2059
|
+
const BATCH_SIZE = 500;
|
|
2060
|
+
let totalDeleted = 0;
|
|
2061
|
+
|
|
2062
|
+
for (let i = 0; i < idsToDelete.length; i += BATCH_SIZE) {
|
|
2063
|
+
const batch = idsToDelete.slice(i, i + BATCH_SIZE);
|
|
2064
|
+
const deleted = await apiClient.batchDelete(batch, authKeyHex);
|
|
2065
|
+
totalDeleted += deleted;
|
|
2066
|
+
}
|
|
2067
|
+
|
|
2068
|
+
reportLines.push('');
|
|
2069
|
+
reportLines.push(`Deleted ${totalDeleted} duplicate memories.`);
|
|
2070
|
+
} else {
|
|
2071
|
+
reportLines.push('');
|
|
2072
|
+
reportLines.push('DRY RUN — no memories were deleted. Run without dry_run to apply.');
|
|
2073
|
+
}
|
|
2074
|
+
|
|
2075
|
+
return {
|
|
2076
|
+
content: [{ type: 'text', text: reportLines.join('\n') }],
|
|
2077
|
+
details: {
|
|
2078
|
+
scanned: allDecrypted.length,
|
|
2079
|
+
clusters: clusters.length,
|
|
2080
|
+
duplicates: totalDuplicates,
|
|
2081
|
+
dry_run: dryRun,
|
|
2082
|
+
},
|
|
2083
|
+
};
|
|
2084
|
+
} catch (err: unknown) {
|
|
2085
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2086
|
+
api.logger.error(`totalreclaw_consolidate failed: ${message}`);
|
|
2087
|
+
return {
|
|
2088
|
+
content: [{ type: 'text', text: `Failed to consolidate memories: ${message}` }],
|
|
2089
|
+
};
|
|
2090
|
+
}
|
|
2091
|
+
},
|
|
2092
|
+
},
|
|
2093
|
+
{ name: 'totalreclaw_consolidate' },
|
|
2094
|
+
);
|
|
2095
|
+
|
|
2096
|
+
// ---------------------------------------------------------------
|
|
2097
|
+
// Tool: totalreclaw_import_from
|
|
2098
|
+
// ---------------------------------------------------------------
|
|
2099
|
+
|
|
2100
|
+
api.registerTool(
|
|
2101
|
+
{
|
|
2102
|
+
name: 'totalreclaw_import_from',
|
|
2103
|
+
label: 'Import From',
|
|
2104
|
+
description:
|
|
2105
|
+
'Import memories from other AI memory tools (Mem0, MCP Memory Server, MemoClaw, or generic JSON/CSV). ' +
|
|
2106
|
+
'Provide the source name and either an API key or file content. ' +
|
|
2107
|
+
'Use dry_run=true to preview before importing. Idempotent — safe to run multiple times.',
|
|
2108
|
+
parameters: {
|
|
2109
|
+
type: 'object',
|
|
2110
|
+
properties: {
|
|
2111
|
+
source: {
|
|
2112
|
+
type: 'string',
|
|
2113
|
+
enum: ['mem0', 'mcp-memory', 'memoclaw', 'generic-json', 'generic-csv'],
|
|
2114
|
+
description: 'The source system to import from',
|
|
2115
|
+
},
|
|
2116
|
+
api_key: {
|
|
2117
|
+
type: 'string',
|
|
2118
|
+
description: 'API key for the source system (used once, never stored)',
|
|
2119
|
+
},
|
|
2120
|
+
source_user_id: {
|
|
2121
|
+
type: 'string',
|
|
2122
|
+
description: 'User or agent ID in the source system',
|
|
2123
|
+
},
|
|
2124
|
+
content: {
|
|
2125
|
+
type: 'string',
|
|
2126
|
+
description: 'File content (JSON, JSONL, or CSV)',
|
|
2127
|
+
},
|
|
2128
|
+
file_path: {
|
|
2129
|
+
type: 'string',
|
|
2130
|
+
description: 'Path to the file on disk',
|
|
2131
|
+
},
|
|
2132
|
+
namespace: {
|
|
2133
|
+
type: 'string',
|
|
2134
|
+
description: 'Target namespace (default: "imported")',
|
|
2135
|
+
},
|
|
2136
|
+
dry_run: {
|
|
2137
|
+
type: 'boolean',
|
|
2138
|
+
description: 'Preview without importing',
|
|
2139
|
+
},
|
|
2140
|
+
},
|
|
2141
|
+
required: ['source'],
|
|
2142
|
+
},
|
|
2143
|
+
async execute(_toolCallId: string, params: Record<string, unknown>) {
|
|
2144
|
+
try {
|
|
2145
|
+
await requireFullSetup(api.logger);
|
|
2146
|
+
return handlePluginImportFrom(params, api.logger);
|
|
2147
|
+
} catch (err: unknown) {
|
|
2148
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2149
|
+
return { error: message };
|
|
2150
|
+
}
|
|
2151
|
+
},
|
|
2152
|
+
},
|
|
2153
|
+
{ name: 'totalreclaw_import_from' },
|
|
2154
|
+
);
|
|
2155
|
+
|
|
2156
|
+
// ---------------------------------------------------------------
|
|
2157
|
+
// Hook: before_agent_start
|
|
2158
|
+
// ---------------------------------------------------------------
|
|
2159
|
+
|
|
2160
|
+
api.on(
|
|
2161
|
+
'before_agent_start',
|
|
2162
|
+
async (event: unknown) => {
|
|
2163
|
+
try {
|
|
2164
|
+
// Prevent cleartext leakage from OpenClaw's native memory system.
|
|
2165
|
+
ensureMemoryHeader(api.logger);
|
|
2166
|
+
|
|
2167
|
+
const evt = event as { prompt?: string } | undefined;
|
|
2168
|
+
|
|
2169
|
+
// Skip trivial or missing prompts.
|
|
2170
|
+
if (!evt?.prompt || evt.prompt.length < 5) {
|
|
2171
|
+
return undefined;
|
|
2172
|
+
}
|
|
2173
|
+
|
|
2174
|
+
await ensureInitialized(api.logger);
|
|
2175
|
+
|
|
2176
|
+
// If setup is still needed after init, inject setup instructions
|
|
2177
|
+
if (needsSetup) {
|
|
2178
|
+
return {
|
|
2179
|
+
prependContext:
|
|
2180
|
+
'## TotalReclaw Setup Required\n\n' +
|
|
2181
|
+
'TotalReclaw is installed but needs configuration. Follow the "Post-Install Setup" steps in SKILL.md to complete setup.\n' +
|
|
2182
|
+
'Ask the user: "Do you have an existing TotalReclaw recovery phrase, or should I generate a new one?"',
|
|
2183
|
+
};
|
|
2184
|
+
}
|
|
2185
|
+
|
|
2186
|
+
// Billing cache check — warn if quota is approaching limit.
|
|
2187
|
+
let billingWarning = '';
|
|
2188
|
+
try {
|
|
2189
|
+
let cache = readBillingCache();
|
|
2190
|
+
if (!cache && authKeyHex) {
|
|
2191
|
+
// Cache is stale or missing — fetch fresh billing status.
|
|
2192
|
+
const billingUrl = (process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz').replace(/\/+$/, '');
|
|
2193
|
+
const walletParam = encodeURIComponent(subgraphOwner || userId || '');
|
|
2194
|
+
const billingResp = await fetch(`${billingUrl}/v1/billing/status?wallet_address=${walletParam}`, {
|
|
2195
|
+
method: 'GET',
|
|
2196
|
+
headers: { 'Authorization': `Bearer ${authKeyHex}`, 'Accept': 'application/json', 'X-TotalReclaw-Client': 'openclaw-plugin' },
|
|
2197
|
+
});
|
|
2198
|
+
if (billingResp.ok) {
|
|
2199
|
+
const billingData = await billingResp.json() as Record<string, unknown>;
|
|
2200
|
+
cache = {
|
|
2201
|
+
tier: (billingData.tier as string) || 'free',
|
|
2202
|
+
free_writes_used: (billingData.free_writes_used as number) ?? 0,
|
|
2203
|
+
free_writes_limit: (billingData.free_writes_limit as number) ?? 0,
|
|
2204
|
+
features: billingData.features as BillingCache['features'] | undefined,
|
|
2205
|
+
checked_at: Date.now(),
|
|
2206
|
+
};
|
|
2207
|
+
writeBillingCache(cache);
|
|
2208
|
+
}
|
|
2209
|
+
}
|
|
2210
|
+
if (cache && cache.free_writes_limit > 0) {
|
|
2211
|
+
const usageRatio = cache.free_writes_used / cache.free_writes_limit;
|
|
2212
|
+
if (usageRatio >= QUOTA_WARNING_THRESHOLD) {
|
|
2213
|
+
billingWarning = `\n\nTotalReclaw quota warning: ${cache.free_writes_used}/${cache.free_writes_limit} writes used this month (${Math.round(usageRatio * 100)}%). Visit https://totalreclaw.xyz/pricing to upgrade.`;
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
} catch {
|
|
2217
|
+
// Best-effort — don't block on billing check failure.
|
|
2218
|
+
}
|
|
2219
|
+
|
|
2220
|
+
if (isSubgraphMode()) {
|
|
2221
|
+
// --- Subgraph mode: hot cache first, then background refresh ---
|
|
2222
|
+
|
|
2223
|
+
// Initialize hot cache if needed.
|
|
2224
|
+
if (!pluginHotCache && encryptionKey) {
|
|
2225
|
+
const config = getSubgraphConfig();
|
|
2226
|
+
pluginHotCache = new PluginHotCache(config.cachePath, encryptionKey.toString('hex'));
|
|
2227
|
+
pluginHotCache.load();
|
|
2228
|
+
}
|
|
2229
|
+
|
|
2230
|
+
// Try to return cached facts instantly.
|
|
2231
|
+
const cachedFacts = pluginHotCache?.getHotFacts() ?? [];
|
|
2232
|
+
|
|
2233
|
+
// Query subgraph in parallel for fresh results.
|
|
2234
|
+
// 1. Generate word trapdoors from the user prompt.
|
|
2235
|
+
const wordTrapdoors = generateBlindIndices(evt.prompt);
|
|
2236
|
+
|
|
2237
|
+
// 2. Generate query embedding + LSH trapdoors (may fail gracefully).
|
|
2238
|
+
let queryEmbedding: number[] | null = null;
|
|
2239
|
+
let lshTrapdoors: string[] = [];
|
|
2240
|
+
try {
|
|
2241
|
+
queryEmbedding = await generateEmbedding(evt.prompt, { isQuery: true });
|
|
2242
|
+
const hasher = getLSHHasher(api.logger);
|
|
2243
|
+
if (hasher && queryEmbedding) {
|
|
2244
|
+
lshTrapdoors = hasher.hash(queryEmbedding);
|
|
2245
|
+
}
|
|
2246
|
+
} catch {
|
|
2247
|
+
// Embedding/LSH failed -- proceed with word-only trapdoors.
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
// Two-tier search (C1): if cache is fresh AND query is semantically similar, return cached
|
|
2251
|
+
const now = Date.now();
|
|
2252
|
+
const cacheAge = now - lastSearchTimestamp;
|
|
2253
|
+
if (cacheAge < CACHE_TTL_MS && cachedFacts.length > 0 && queryEmbedding && lastQueryEmbedding) {
|
|
2254
|
+
const querySimilarity = cosineSimilarity(queryEmbedding, lastQueryEmbedding);
|
|
2255
|
+
if (querySimilarity > SEMANTIC_SKIP_THRESHOLD) {
|
|
2256
|
+
const lines = cachedFacts.slice(0, 8).map((f, i) =>
|
|
2257
|
+
`${i + 1}. ${f.text} (importance: ${f.importance}/10, cached)`,
|
|
2258
|
+
);
|
|
2259
|
+
return { prependContext: `## Relevant Memories\n\n${lines.join('\n')}` + billingWarning };
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
// 3. Merge trapdoors — always include word trapdoors for small-dataset coverage.
|
|
2264
|
+
// LSH alone has low collision probability on <100 facts, causing 0 matches.
|
|
2265
|
+
const allTrapdoors = [...wordTrapdoors, ...lshTrapdoors];
|
|
2266
|
+
|
|
2267
|
+
// If we have cached facts and no trapdoors, return cached facts.
|
|
2268
|
+
if (allTrapdoors.length === 0 && cachedFacts.length > 0) {
|
|
2269
|
+
const lines = cachedFacts.slice(0, 8).map((f, i) =>
|
|
2270
|
+
`${i + 1}. ${f.text} (importance: ${f.importance}/10, cached)`,
|
|
2271
|
+
);
|
|
2272
|
+
return { prependContext: `## Relevant Memories\n\n${lines.join('\n')}` + billingWarning };
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
if (allTrapdoors.length === 0) return undefined;
|
|
2276
|
+
|
|
2277
|
+
// 4. Query subgraph for fresh results.
|
|
2278
|
+
let subgraphResults: Awaited<ReturnType<typeof searchSubgraph>> = [];
|
|
2279
|
+
try {
|
|
2280
|
+
const factCount = await getSubgraphFactCount(subgraphOwner || userId!, authKeyHex!);
|
|
2281
|
+
const pool = computeCandidatePool(factCount);
|
|
2282
|
+
subgraphResults = await searchSubgraph(subgraphOwner || userId!, allTrapdoors, pool, authKeyHex!);
|
|
2283
|
+
} catch {
|
|
2284
|
+
// Subgraph query failed -- fall back to cached facts if available.
|
|
2285
|
+
if (cachedFacts.length > 0) {
|
|
2286
|
+
const lines = cachedFacts.slice(0, 8).map((f, i) =>
|
|
2287
|
+
`${i + 1}. ${f.text} (importance: ${f.importance}/10, cached)`,
|
|
2288
|
+
);
|
|
2289
|
+
return { prependContext: `## Relevant Memories\n\n${lines.join('\n')}` + billingWarning };
|
|
2290
|
+
}
|
|
2291
|
+
return undefined;
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
if (subgraphResults.length === 0 && cachedFacts.length === 0) return undefined;
|
|
2295
|
+
|
|
2296
|
+
// If subgraph returned no results but we have cache, use cache.
|
|
2297
|
+
if (subgraphResults.length === 0) {
|
|
2298
|
+
const lines = cachedFacts.slice(0, 8).map((f, i) =>
|
|
2299
|
+
`${i + 1}. ${f.text} (importance: ${f.importance}/10, cached)`,
|
|
2300
|
+
);
|
|
2301
|
+
return { prependContext: `## Relevant Memories\n\n${lines.join('\n')}` + billingWarning };
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
// 5. Decrypt subgraph results and build reranker input.
|
|
2305
|
+
const rerankerCandidates: RerankerCandidate[] = [];
|
|
2306
|
+
const hookMetaMap = new Map<string, { importance: number; age: string }>();
|
|
2307
|
+
|
|
2308
|
+
for (const result of subgraphResults) {
|
|
2309
|
+
try {
|
|
2310
|
+
const docJson = decryptFromHex(result.encryptedBlob, encryptionKey!);
|
|
2311
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
2312
|
+
|
|
2313
|
+
let decryptedEmbedding: number[] | undefined;
|
|
2314
|
+
if (result.encryptedEmbedding) {
|
|
2315
|
+
try {
|
|
2316
|
+
decryptedEmbedding = JSON.parse(
|
|
2317
|
+
decryptFromHex(result.encryptedEmbedding, encryptionKey!),
|
|
2318
|
+
);
|
|
2319
|
+
} catch {
|
|
2320
|
+
// Embedding decryption failed -- proceed without it.
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2323
|
+
|
|
2324
|
+
const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
|
|
2325
|
+
const createdAtSec = result.timestamp ? parseInt(result.timestamp, 10) : undefined;
|
|
2326
|
+
rerankerCandidates.push({
|
|
2327
|
+
id: result.id,
|
|
2328
|
+
text: doc.text,
|
|
2329
|
+
embedding: decryptedEmbedding,
|
|
2330
|
+
importance: importanceRaw,
|
|
2331
|
+
createdAt: createdAtSec,
|
|
2332
|
+
});
|
|
2333
|
+
|
|
2334
|
+
const importance = doc.metadata?.importance
|
|
2335
|
+
? Math.round((doc.metadata.importance as number) * 10)
|
|
2336
|
+
: 5;
|
|
2337
|
+
hookMetaMap.set(result.id, {
|
|
2338
|
+
importance,
|
|
2339
|
+
age: 'subgraph',
|
|
2340
|
+
});
|
|
2341
|
+
} catch {
|
|
2342
|
+
// Skip un-decryptable candidates.
|
|
2343
|
+
}
|
|
2344
|
+
}
|
|
2345
|
+
|
|
2346
|
+
// 6. Re-rank with BM25 + cosine + intent-weighted RRF fusion.
|
|
2347
|
+
const hookQueryIntent = detectQueryIntent(evt.prompt);
|
|
2348
|
+
const reranked = rerank(
|
|
2349
|
+
evt.prompt,
|
|
2350
|
+
queryEmbedding ?? [],
|
|
2351
|
+
rerankerCandidates,
|
|
2352
|
+
8,
|
|
2353
|
+
INTENT_WEIGHTS[hookQueryIntent],
|
|
2354
|
+
);
|
|
2355
|
+
|
|
2356
|
+
// B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
|
|
2357
|
+
const candidatesWithEmb = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
|
|
2358
|
+
if (candidatesWithEmb.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
|
|
2359
|
+
const topCosine = Math.max(
|
|
2360
|
+
...candidatesWithEmb.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
|
|
2361
|
+
);
|
|
2362
|
+
if (topCosine < RELEVANCE_THRESHOLD) return undefined;
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
// Update hot cache with reranked results.
|
|
2366
|
+
try {
|
|
2367
|
+
if (pluginHotCache) {
|
|
2368
|
+
const hotFacts: HotFact[] = rerankerCandidates.map((c) => {
|
|
2369
|
+
const meta = hookMetaMap.get(c.id);
|
|
2370
|
+
return { id: c.id, text: c.text, importance: meta?.importance ?? 5 };
|
|
2371
|
+
});
|
|
2372
|
+
pluginHotCache.setHotFacts(hotFacts);
|
|
2373
|
+
pluginHotCache.setLastQueryEmbedding(queryEmbedding);
|
|
2374
|
+
pluginHotCache.flush();
|
|
2375
|
+
}
|
|
2376
|
+
} catch {
|
|
2377
|
+
// Hot cache update is best-effort.
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
// Record search state for two-tier cache (C1).
|
|
2381
|
+
lastSearchTimestamp = Date.now();
|
|
2382
|
+
lastQueryEmbedding = queryEmbedding;
|
|
2383
|
+
|
|
2384
|
+
if (reranked.length === 0) return undefined;
|
|
2385
|
+
|
|
2386
|
+
// 6b. Cosine similarity threshold gate — skip injection when the
|
|
2387
|
+
// best match is below the minimum relevance threshold.
|
|
2388
|
+
const hookMaxCosine = Math.max(
|
|
2389
|
+
...reranked.map((r) => r.cosineSimilarity ?? 0),
|
|
2390
|
+
);
|
|
2391
|
+
if (hookMaxCosine < COSINE_THRESHOLD) {
|
|
2392
|
+
api.logger.info(
|
|
2393
|
+
`Hook: cosine threshold gate filtered results (max=${hookMaxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD})`,
|
|
2394
|
+
);
|
|
2395
|
+
return undefined;
|
|
2396
|
+
}
|
|
2397
|
+
|
|
2398
|
+
// 7. Build context string.
|
|
2399
|
+
const lines = reranked.map((m, i) => {
|
|
2400
|
+
const meta = hookMetaMap.get(m.id);
|
|
2401
|
+
const importance = meta?.importance ?? 5;
|
|
2402
|
+
const age = meta?.age ?? '';
|
|
2403
|
+
return `${i + 1}. ${m.text} (importance: ${importance}/10, ${age})`;
|
|
2404
|
+
});
|
|
2405
|
+
const contextString = `## Relevant Memories\n\n${lines.join('\n')}`;
|
|
2406
|
+
|
|
2407
|
+
return { prependContext: contextString + billingWarning };
|
|
2408
|
+
}
|
|
2409
|
+
|
|
2410
|
+
// --- Server mode (existing behavior) ---
|
|
2411
|
+
|
|
2412
|
+
// 1. Generate word trapdoors from the user prompt.
|
|
2413
|
+
const wordTrapdoors = generateBlindIndices(evt.prompt);
|
|
2414
|
+
|
|
2415
|
+
// 2. Generate query embedding + LSH trapdoors (may fail gracefully).
|
|
2416
|
+
let queryEmbedding: number[] | null = null;
|
|
2417
|
+
let lshTrapdoors: string[] = [];
|
|
2418
|
+
try {
|
|
2419
|
+
queryEmbedding = await generateEmbedding(evt.prompt, { isQuery: true });
|
|
2420
|
+
const hasher = getLSHHasher(api.logger);
|
|
2421
|
+
if (hasher && queryEmbedding) {
|
|
2422
|
+
lshTrapdoors = hasher.hash(queryEmbedding);
|
|
2423
|
+
}
|
|
2424
|
+
} catch {
|
|
2425
|
+
// Embedding/LSH failed -- proceed with word-only trapdoors.
|
|
2426
|
+
}
|
|
2427
|
+
|
|
2428
|
+
// 3. Merge word + LSH trapdoors.
|
|
2429
|
+
const allTrapdoors = [...wordTrapdoors, ...lshTrapdoors];
|
|
2430
|
+
if (allTrapdoors.length === 0) return undefined;
|
|
2431
|
+
|
|
2432
|
+
// 4. Fetch candidates from the server (dynamic pool sizing).
|
|
2433
|
+
const factCount = await getFactCount(api.logger);
|
|
2434
|
+
const pool = computeCandidatePool(factCount);
|
|
2435
|
+
const candidates = await apiClient!.search(
|
|
2436
|
+
userId!,
|
|
2437
|
+
allTrapdoors,
|
|
2438
|
+
pool,
|
|
2439
|
+
authKeyHex!,
|
|
2440
|
+
);
|
|
2441
|
+
|
|
2442
|
+
if (candidates.length === 0) return undefined;
|
|
2443
|
+
|
|
2444
|
+
// 5. Decrypt candidates (text + embeddings) and build reranker input.
|
|
2445
|
+
const rerankerCandidates: RerankerCandidate[] = [];
|
|
2446
|
+
const hookMetaMap = new Map<string, { importance: number; age: string }>();
|
|
2447
|
+
|
|
2448
|
+
for (const candidate of candidates) {
|
|
2449
|
+
try {
|
|
2450
|
+
const docJson = decryptFromHex(candidate.encrypted_blob, encryptionKey!);
|
|
2451
|
+
const doc = JSON.parse(docJson) as { text: string; metadata?: Record<string, unknown> };
|
|
2452
|
+
|
|
2453
|
+
// Decrypt embedding if present.
|
|
2454
|
+
let decryptedEmbedding: number[] | undefined;
|
|
2455
|
+
if (candidate.encrypted_embedding) {
|
|
2456
|
+
try {
|
|
2457
|
+
decryptedEmbedding = JSON.parse(
|
|
2458
|
+
decryptFromHex(candidate.encrypted_embedding, encryptionKey!),
|
|
2459
|
+
);
|
|
2460
|
+
} catch {
|
|
2461
|
+
// Embedding decryption failed -- proceed without it.
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
const importanceRaw = (doc.metadata?.importance as number) ?? 0.5;
|
|
2466
|
+
const createdAtSec = typeof candidate.timestamp === 'number'
|
|
2467
|
+
? candidate.timestamp / 1000
|
|
2468
|
+
: new Date(candidate.timestamp).getTime() / 1000;
|
|
2469
|
+
rerankerCandidates.push({
|
|
2470
|
+
id: candidate.fact_id,
|
|
2471
|
+
text: doc.text,
|
|
2472
|
+
embedding: decryptedEmbedding,
|
|
2473
|
+
importance: importanceRaw,
|
|
2474
|
+
createdAt: createdAtSec,
|
|
2475
|
+
});
|
|
2476
|
+
|
|
2477
|
+
const importance = doc.metadata?.importance
|
|
2478
|
+
? Math.round((doc.metadata.importance as number) * 10)
|
|
2479
|
+
: 5;
|
|
2480
|
+
hookMetaMap.set(candidate.fact_id, {
|
|
2481
|
+
importance,
|
|
2482
|
+
age: relativeTime(candidate.timestamp),
|
|
2483
|
+
});
|
|
2484
|
+
} catch {
|
|
2485
|
+
// Skip un-decryptable candidates.
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
// 6. Re-rank with BM25 + cosine + RRF fusion (intent-weighted).
|
|
2490
|
+
const srvHookIntent = detectQueryIntent(evt.prompt);
|
|
2491
|
+
const reranked = rerank(
|
|
2492
|
+
evt.prompt,
|
|
2493
|
+
queryEmbedding ?? [],
|
|
2494
|
+
rerankerCandidates,
|
|
2495
|
+
8,
|
|
2496
|
+
INTENT_WEIGHTS[srvHookIntent],
|
|
2497
|
+
);
|
|
2498
|
+
|
|
2499
|
+
// B2: Minimum relevance threshold — skip noise injection for irrelevant turns.
|
|
2500
|
+
const candidatesWithEmbSrv = rerankerCandidates.filter(c => c.embedding && c.embedding.length > 0);
|
|
2501
|
+
if (candidatesWithEmbSrv.length > 0 && queryEmbedding && queryEmbedding.length > 0) {
|
|
2502
|
+
const topCosine = Math.max(
|
|
2503
|
+
...candidatesWithEmbSrv.map(c => cosineSimilarity(queryEmbedding!, c.embedding!))
|
|
2504
|
+
);
|
|
2505
|
+
if (topCosine < RELEVANCE_THRESHOLD) return undefined;
|
|
2506
|
+
}
|
|
2507
|
+
|
|
2508
|
+
if (reranked.length === 0) return undefined;
|
|
2509
|
+
|
|
2510
|
+
// 7. Build context string.
|
|
2511
|
+
const lines = reranked.map((m, i) => {
|
|
2512
|
+
const meta = hookMetaMap.get(m.id);
|
|
2513
|
+
const importance = meta?.importance ?? 5;
|
|
2514
|
+
const age = meta?.age ?? '';
|
|
2515
|
+
return `${i + 1}. ${m.text} (importance: ${importance}/10, ${age})`;
|
|
2516
|
+
});
|
|
2517
|
+
const contextString = `## Relevant Memories\n\n${lines.join('\n')}`;
|
|
2518
|
+
|
|
2519
|
+
return { prependContext: contextString + billingWarning };
|
|
2520
|
+
} catch (err: unknown) {
|
|
2521
|
+
// The hook must NEVER throw -- log and return undefined.
|
|
2522
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2523
|
+
api.logger.warn(`before_agent_start hook failed: ${message}`);
|
|
2524
|
+
return undefined;
|
|
2525
|
+
}
|
|
2526
|
+
},
|
|
2527
|
+
{ priority: 10 },
|
|
2528
|
+
);
|
|
2529
|
+
|
|
2530
|
+
// ---------------------------------------------------------------
|
|
2531
|
+
// Hook: agent_end — auto-extract facts after each conversation turn
|
|
2532
|
+
// ---------------------------------------------------------------
|
|
2533
|
+
|
|
2534
|
+
api.on(
|
|
2535
|
+
'agent_end',
|
|
2536
|
+
async (event: unknown) => {
|
|
2537
|
+
try {
|
|
2538
|
+
const evt = event as { messages?: unknown[]; success?: boolean } | undefined;
|
|
2539
|
+
if (!evt?.success || !evt?.messages || evt.messages.length < 2) return;
|
|
2540
|
+
|
|
2541
|
+
await ensureInitialized(api.logger);
|
|
2542
|
+
if (needsSetup) return;
|
|
2543
|
+
|
|
2544
|
+
// C3: Throttle auto-extraction to every N turns (configurable via env).
|
|
2545
|
+
turnsSinceLastExtraction++;
|
|
2546
|
+
if (turnsSinceLastExtraction >= getExtractInterval()) {
|
|
2547
|
+
const existingMemories = isLlmDedupEnabled()
|
|
2548
|
+
? await fetchExistingMemoriesForExtraction(api.logger, 20, evt.messages)
|
|
2549
|
+
: [];
|
|
2550
|
+
const rawFacts = await extractFacts(evt.messages, 'turn', existingMemories);
|
|
2551
|
+
const { kept: facts } = filterByImportance(rawFacts, api.logger);
|
|
2552
|
+
if (facts.length > 0) {
|
|
2553
|
+
await storeExtractedFacts(facts, api.logger);
|
|
2554
|
+
}
|
|
2555
|
+
turnsSinceLastExtraction = 0;
|
|
2556
|
+
}
|
|
2557
|
+
} catch (err: unknown) {
|
|
2558
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2559
|
+
api.logger.warn(`agent_end extraction failed: ${message}`);
|
|
2560
|
+
}
|
|
2561
|
+
},
|
|
2562
|
+
{ priority: 90 },
|
|
2563
|
+
);
|
|
2564
|
+
|
|
2565
|
+
// ---------------------------------------------------------------
|
|
2566
|
+
// Hook: before_compaction — extract ALL facts before context is lost
|
|
2567
|
+
// ---------------------------------------------------------------
|
|
2568
|
+
|
|
2569
|
+
api.on(
|
|
2570
|
+
'before_compaction',
|
|
2571
|
+
async (event: unknown) => {
|
|
2572
|
+
try {
|
|
2573
|
+
const evt = event as { messages?: unknown[]; messageCount?: number } | undefined;
|
|
2574
|
+
if (!evt?.messages || evt.messages.length < 2) return;
|
|
2575
|
+
|
|
2576
|
+
await ensureInitialized(api.logger);
|
|
2577
|
+
if (needsSetup) return;
|
|
2578
|
+
|
|
2579
|
+
api.logger.info(
|
|
2580
|
+
`Pre-compaction extraction: processing ${evt.messages.length} messages`,
|
|
2581
|
+
);
|
|
2582
|
+
|
|
2583
|
+
const existingMemories = isLlmDedupEnabled()
|
|
2584
|
+
? await fetchExistingMemoriesForExtraction(api.logger, 50, evt.messages)
|
|
2585
|
+
: [];
|
|
2586
|
+
const rawCompactFacts = await extractFacts(evt.messages, 'full', existingMemories);
|
|
2587
|
+
const { kept: facts } = filterByImportance(rawCompactFacts, api.logger);
|
|
2588
|
+
if (facts.length > 0) {
|
|
2589
|
+
await storeExtractedFacts(facts, api.logger);
|
|
2590
|
+
}
|
|
2591
|
+
turnsSinceLastExtraction = 0; // Reset C3 counter on compaction.
|
|
2592
|
+
} catch (err: unknown) {
|
|
2593
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2594
|
+
api.logger.warn(`before_compaction extraction failed: ${message}`);
|
|
2595
|
+
}
|
|
2596
|
+
},
|
|
2597
|
+
{ priority: 5 },
|
|
2598
|
+
);
|
|
2599
|
+
|
|
2600
|
+
// ---------------------------------------------------------------
|
|
2601
|
+
// Hook: before_reset — final extraction before session is cleared
|
|
2602
|
+
// ---------------------------------------------------------------
|
|
2603
|
+
|
|
2604
|
+
api.on(
|
|
2605
|
+
'before_reset',
|
|
2606
|
+
async (event: unknown) => {
|
|
2607
|
+
try {
|
|
2608
|
+
const evt = event as { messages?: unknown[]; reason?: string } | undefined;
|
|
2609
|
+
if (!evt?.messages || evt.messages.length < 2) return;
|
|
2610
|
+
|
|
2611
|
+
await ensureInitialized(api.logger);
|
|
2612
|
+
if (needsSetup) return;
|
|
2613
|
+
|
|
2614
|
+
api.logger.info(
|
|
2615
|
+
`Pre-reset extraction (${evt.reason ?? 'unknown'}): processing ${evt.messages.length} messages`,
|
|
2616
|
+
);
|
|
2617
|
+
|
|
2618
|
+
const existingMemories = isLlmDedupEnabled()
|
|
2619
|
+
? await fetchExistingMemoriesForExtraction(api.logger, 50, evt.messages)
|
|
2620
|
+
: [];
|
|
2621
|
+
const rawResetFacts = await extractFacts(evt.messages, 'full', existingMemories);
|
|
2622
|
+
const { kept: facts } = filterByImportance(rawResetFacts, api.logger);
|
|
2623
|
+
if (facts.length > 0) {
|
|
2624
|
+
await storeExtractedFacts(facts, api.logger);
|
|
2625
|
+
}
|
|
2626
|
+
turnsSinceLastExtraction = 0; // Reset C3 counter on reset.
|
|
2627
|
+
} catch (err: unknown) {
|
|
2628
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2629
|
+
api.logger.warn(`before_reset extraction failed: ${message}`);
|
|
2630
|
+
}
|
|
2631
|
+
},
|
|
2632
|
+
{ priority: 5 },
|
|
2633
|
+
);
|
|
2634
|
+
},
|
|
2635
|
+
};
|
|
2636
|
+
|
|
2637
|
+
export default plugin;
|
|
2638
|
+
|
|
2639
|
+
/**
|
|
2640
|
+
* Reset all module-level state for test isolation.
|
|
2641
|
+
* ONLY call this from test code — never in production.
|
|
2642
|
+
*/
|
|
2643
|
+
export function __resetForTesting(): void {
|
|
2644
|
+
authKeyHex = null;
|
|
2645
|
+
encryptionKey = null;
|
|
2646
|
+
dedupKey = null;
|
|
2647
|
+
userId = null;
|
|
2648
|
+
subgraphOwner = null;
|
|
2649
|
+
apiClient = null;
|
|
2650
|
+
initPromise = null;
|
|
2651
|
+
lshHasher = null;
|
|
2652
|
+
lshInitFailed = false;
|
|
2653
|
+
masterPasswordCache = null;
|
|
2654
|
+
saltCache = null;
|
|
2655
|
+
cachedFactCount = null;
|
|
2656
|
+
lastFactCountFetch = 0;
|
|
2657
|
+
pluginHotCache = null;
|
|
2658
|
+
lastSearchTimestamp = 0;
|
|
2659
|
+
lastQueryEmbedding = null;
|
|
2660
|
+
turnsSinceLastExtraction = 0;
|
|
2661
|
+
}
|