@totalreclaw/totalreclaw 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crypto.ts +1 -1
- package/extractor-dedup.test.ts +1 -1
- package/extractor.ts +26 -18
- package/generate-mnemonic.ts +3 -3
- package/import-adapters/base-adapter.ts +1 -1
- package/import-adapters/import-adapters.test.ts +1 -1
- package/import-adapters/types.ts +1 -1
- package/index.ts +130 -98
- package/package.json +1 -1
- package/setup.sh +1 -1
- package/subgraph-store.ts +102 -3
package/crypto.ts
CHANGED
|
@@ -89,7 +89,7 @@ function deriveKeysFromMnemonic(
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
/**
|
|
92
|
-
* Derive auth, encryption, and dedup keys from a
|
|
92
|
+
* Derive auth, encryption, and dedup keys from a recovery phrase.
|
|
93
93
|
*
|
|
94
94
|
* If the password is a valid BIP-39 mnemonic (12 or 24 words), keys are
|
|
95
95
|
* derived from the 512-bit BIP-39 seed via HKDF. Otherwise, the legacy
|
package/extractor-dedup.test.ts
CHANGED
|
@@ -57,7 +57,7 @@ function parseFactsResponse(response: string): ExtractedFact[] {
|
|
|
57
57
|
: 'ADD';
|
|
58
58
|
return {
|
|
59
59
|
text: String(fact.text).slice(0, 512),
|
|
60
|
-
type: (['fact', 'preference', 'decision', 'episodic', 'goal'].includes(String(fact.type))
|
|
60
|
+
type: (['fact', 'preference', 'decision', 'episodic', 'goal', 'context', 'summary'].includes(String(fact.type))
|
|
61
61
|
? String(fact.type)
|
|
62
62
|
: 'fact') as ExtractedFact['type'],
|
|
63
63
|
importance: Math.max(1, Math.min(10, Number(fact.importance) || 5)),
|
package/extractor.ts
CHANGED
|
@@ -15,7 +15,7 @@ export type ExtractionAction = 'ADD' | 'UPDATE' | 'DELETE' | 'NOOP';
|
|
|
15
15
|
|
|
16
16
|
export interface ExtractedFact {
|
|
17
17
|
text: string;
|
|
18
|
-
type: 'fact' | 'preference' | 'decision' | 'episodic' | 'goal';
|
|
18
|
+
type: 'fact' | 'preference' | 'decision' | 'episodic' | 'goal' | 'context' | 'summary';
|
|
19
19
|
importance: number; // 1-10
|
|
20
20
|
action: ExtractionAction;
|
|
21
21
|
existingFactId?: string;
|
|
@@ -37,28 +37,36 @@ interface ConversationMessage {
|
|
|
37
37
|
// Extraction Prompt
|
|
38
38
|
// ---------------------------------------------------------------------------
|
|
39
39
|
|
|
40
|
-
const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction engine. Analyze the conversation and extract
|
|
40
|
+
const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction engine. Analyze the conversation and extract valuable long-term memories.
|
|
41
41
|
|
|
42
42
|
Rules:
|
|
43
|
-
1. Each
|
|
44
|
-
2. Focus on user-specific information
|
|
45
|
-
3. Skip generic knowledge, greetings, and
|
|
46
|
-
4.
|
|
47
|
-
5.
|
|
48
|
-
6. Only extract facts with importance >= 6
|
|
43
|
+
1. Each memory must be a single, self-contained piece of information
|
|
44
|
+
2. Focus on user-specific information that would be useful in future conversations
|
|
45
|
+
3. Skip generic knowledge, greetings, small talk, and ephemeral task coordination
|
|
46
|
+
4. Score importance 1-10 (6+ = worth storing)
|
|
47
|
+
5. Only extract memories with importance >= 6
|
|
49
48
|
|
|
50
49
|
Types:
|
|
51
|
-
- fact: Objective information about the user
|
|
52
|
-
- preference: Likes, dislikes, or preferences
|
|
53
|
-
- decision: Choices
|
|
54
|
-
- episodic:
|
|
55
|
-
- goal: Objectives or
|
|
50
|
+
- fact: Objective information about the user (name, location, job, relationships)
|
|
51
|
+
- preference: Likes, dislikes, or preferences ("prefers dark mode", "allergic to peanuts")
|
|
52
|
+
- decision: Choices WITH reasoning ("chose PostgreSQL because data is relational and needs ACID")
|
|
53
|
+
- episodic: Notable events or experiences ("deployed v1.0 to production on March 15")
|
|
54
|
+
- goal: Objectives, targets, or plans ("wants to launch public beta by end of Q1")
|
|
55
|
+
- context: Active project/task context ("working on TotalReclaw v1.2, staging on Base Sepolia")
|
|
56
|
+
- summary: Key outcome or conclusion from a discussion ("agreed to use phased rollout for migration")
|
|
57
|
+
|
|
58
|
+
Extraction guidance:
|
|
59
|
+
- For decisions: ALWAYS include the reasoning. "Chose X" is weak. "Chose X because Y" is strong.
|
|
60
|
+
- For context: Capture what the user is actively working on, including versions, environments, and status.
|
|
61
|
+
- For summaries: Only extract when a conversation reaches a clear conclusion or agreement.
|
|
62
|
+
- For facts: Prefer specific over vague. "Lives in Lisbon" beats "lives in Europe".
|
|
63
|
+
- Decisions and context should be importance >= 7 (they are high-value for future conversations).
|
|
56
64
|
|
|
57
65
|
Actions (compare against existing memories if provided):
|
|
58
|
-
- ADD: New
|
|
59
|
-
- UPDATE:
|
|
60
|
-
- DELETE: Contradicts an existing memory
|
|
61
|
-
- NOOP: Already captured
|
|
66
|
+
- ADD: New memory, no conflict with existing
|
|
67
|
+
- UPDATE: Refines or corrects an existing memory (provide existingFactId)
|
|
68
|
+
- DELETE: Contradicts an existing memory -- the old one is now wrong (provide existingFactId)
|
|
69
|
+
- NOOP: Already captured or not worth storing
|
|
62
70
|
|
|
63
71
|
Return a JSON array (no markdown, no code fences):
|
|
64
72
|
[{"text": "...", "type": "...", "importance": N, "action": "ADD|UPDATE|DELETE|NOOP", "existingFactId": "..."}, ...]
|
|
@@ -158,7 +166,7 @@ function parseFactsResponse(response: string): ExtractedFact[] {
|
|
|
158
166
|
: 'ADD'; // Default to ADD for backward compatibility
|
|
159
167
|
return {
|
|
160
168
|
text: String(fact.text).slice(0, 512),
|
|
161
|
-
type: (['fact', 'preference', 'decision', 'episodic', 'goal'].includes(String(fact.type))
|
|
169
|
+
type: (['fact', 'preference', 'decision', 'episodic', 'goal', 'context', 'summary'].includes(String(fact.type))
|
|
162
170
|
? String(fact.type)
|
|
163
171
|
: 'fact') as ExtractedFact['type'],
|
|
164
172
|
importance: Math.max(1, Math.min(10, Number(fact.importance) || 5)),
|
package/generate-mnemonic.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env npx tsx
|
|
2
2
|
/**
|
|
3
|
-
* Generate a BIP-39 12-word mnemonic for use as
|
|
3
|
+
* Generate a BIP-39 12-word mnemonic for use as TOTALRECLAW_RECOVERY_PHRASE.
|
|
4
4
|
*
|
|
5
5
|
* Usage: npx tsx generate-mnemonic.ts
|
|
6
6
|
*/
|
|
@@ -8,7 +8,7 @@ import { generateMnemonic } from '@scure/bip39';
|
|
|
8
8
|
import { wordlist } from '@scure/bip39/wordlists/english.js';
|
|
9
9
|
|
|
10
10
|
const mnemonic = generateMnemonic(wordlist, 128);
|
|
11
|
-
console.log('\n Your TotalReclaw
|
|
11
|
+
console.log('\n Your TotalReclaw recovery phrase (12 words):\n');
|
|
12
12
|
console.log(` ${mnemonic}\n`);
|
|
13
13
|
console.log(' WRITE THIS DOWN. If you lose it, your memories are unrecoverable.');
|
|
14
|
-
console.log(' Set it as
|
|
14
|
+
console.log(' Set it as TOTALRECLAW_RECOVERY_PHRASE in your .env file.\n');
|
|
@@ -44,7 +44,7 @@ export abstract class BaseImportAdapter {
|
|
|
44
44
|
const text = fact.text.trim().slice(0, 512);
|
|
45
45
|
|
|
46
46
|
// Normalize type
|
|
47
|
-
const validTypes = ['fact', 'preference', 'decision', 'episodic', 'goal'] as const;
|
|
47
|
+
const validTypes = ['fact', 'preference', 'decision', 'episodic', 'goal', 'context', 'summary'] as const;
|
|
48
48
|
const type = validTypes.includes(fact.type as typeof validTypes[number])
|
|
49
49
|
? (fact.type as NormalizedFact['type'])
|
|
50
50
|
: 'fact';
|
|
@@ -453,7 +453,7 @@ async function runTests(): Promise<void> {
|
|
|
453
453
|
|
|
454
454
|
// --- type normalization ---
|
|
455
455
|
{
|
|
456
|
-
const validTypes = ['fact', 'preference', 'decision', 'episodic', 'goal'] as const;
|
|
456
|
+
const validTypes = ['fact', 'preference', 'decision', 'episodic', 'goal', 'context', 'summary'] as const;
|
|
457
457
|
|
|
458
458
|
for (const t of validTypes) {
|
|
459
459
|
const result = testAdapter.testValidateFact({ text: 'test fact', type: t });
|
package/import-adapters/types.ts
CHANGED
|
@@ -6,7 +6,7 @@ export interface NormalizedFact {
|
|
|
6
6
|
/** The atomic fact text (max 512 chars) */
|
|
7
7
|
text: string;
|
|
8
8
|
/** Fact type matching TotalReclaw's taxonomy */
|
|
9
|
-
type: 'fact' | 'preference' | 'decision' | 'episodic' | 'goal';
|
|
9
|
+
type: 'fact' | 'preference' | 'decision' | 'episodic' | 'goal' | 'context' | 'summary';
|
|
10
10
|
/** Importance score 1-10 */
|
|
11
11
|
importance: number;
|
|
12
12
|
/** Original source system */
|
package/index.ts
CHANGED
|
@@ -41,7 +41,7 @@ import {
|
|
|
41
41
|
STORE_DEDUP_MAX_CANDIDATES,
|
|
42
42
|
type DecryptedCandidate,
|
|
43
43
|
} from './consolidation.js';
|
|
44
|
-
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, deriveSmartAccountAddress, type FactPayload } from './subgraph-store.js';
|
|
44
|
+
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, type FactPayload } from './subgraph-store.js';
|
|
45
45
|
import { searchSubgraph, getSubgraphFactCount } from './subgraph-search.js';
|
|
46
46
|
import { PluginHotCache, type HotFact } from './hot-cache-wrapper.js';
|
|
47
47
|
import crypto from 'node:crypto';
|
|
@@ -142,7 +142,7 @@ const RELEVANCE_THRESHOLD = parseFloat(process.env.TOTALRECLAW_RELEVANCE_THRESHO
|
|
|
142
142
|
// ---------------------------------------------------------------------------
|
|
143
143
|
|
|
144
144
|
const BILLING_CACHE_PATH = path.join(process.env.HOME ?? '/home/node', '.totalreclaw', 'billing-cache.json');
|
|
145
|
-
const BILLING_CACHE_TTL =
|
|
145
|
+
const BILLING_CACHE_TTL = 2 * 60 * 60 * 1000; // 2 hours
|
|
146
146
|
const QUOTA_WARNING_THRESHOLD = 0.8; // 80%
|
|
147
147
|
|
|
148
148
|
interface BillingCache {
|
|
@@ -153,6 +153,9 @@ interface BillingCache {
|
|
|
153
153
|
llm_dedup?: boolean;
|
|
154
154
|
custom_extract_interval?: boolean;
|
|
155
155
|
min_extract_interval?: number;
|
|
156
|
+
extraction_interval?: number;
|
|
157
|
+
max_facts_per_extraction?: number;
|
|
158
|
+
max_candidate_pool?: number;
|
|
156
159
|
};
|
|
157
160
|
checked_at: number;
|
|
158
161
|
}
|
|
@@ -192,12 +195,25 @@ function isLlmDedupEnabled(): boolean {
|
|
|
192
195
|
|
|
193
196
|
/**
|
|
194
197
|
* Get the effective extraction interval.
|
|
195
|
-
*
|
|
198
|
+
* Server-side config takes priority (from billing cache), then env var fallback.
|
|
199
|
+
* This allows the relay admin to tune extraction without an npm publish.
|
|
196
200
|
*/
|
|
197
201
|
function getExtractInterval(): number {
|
|
202
|
+
const cache = readBillingCache();
|
|
203
|
+
if (cache?.features?.extraction_interval != null) return cache.features.extraction_interval;
|
|
198
204
|
return AUTO_EXTRACT_EVERY_TURNS_ENV;
|
|
199
205
|
}
|
|
200
206
|
|
|
207
|
+
/**
|
|
208
|
+
* Get the max facts per extraction cycle.
|
|
209
|
+
* Server-side config takes priority (from billing cache), then env var / constant fallback.
|
|
210
|
+
*/
|
|
211
|
+
function getMaxFactsPerExtraction(): number {
|
|
212
|
+
const cache = readBillingCache();
|
|
213
|
+
if (cache?.features?.max_facts_per_extraction != null) return cache.features.max_facts_per_extraction;
|
|
214
|
+
return MAX_FACTS_PER_EXTRACTION;
|
|
215
|
+
}
|
|
216
|
+
|
|
201
217
|
/**
|
|
202
218
|
* Ensure MEMORY.md has a TotalReclaw header so the agent knows encrypted
|
|
203
219
|
* memories are injected automatically via the before_agent_start hook.
|
|
@@ -255,12 +271,18 @@ const FACT_COUNT_CACHE_TTL = 5 * 60 * 1000;
|
|
|
255
271
|
/**
|
|
256
272
|
* Compute the candidate pool size from a fact count.
|
|
257
273
|
*
|
|
258
|
-
*
|
|
274
|
+
* Server-side config takes priority (from billing cache), then local fallback.
|
|
275
|
+
* The server computes the optimal pool based on vault size and tier caps.
|
|
276
|
+
*
|
|
277
|
+
* Local fallback formula: pool = min(max(factCount * 3, 400), 5000)
|
|
259
278
|
* - At least 400 candidates (even for tiny vaults)
|
|
260
279
|
* - At most 5000 candidates (to bound decryption + reranking cost)
|
|
261
280
|
* - 3x fact count in between
|
|
262
281
|
*/
|
|
263
282
|
function computeCandidatePool(factCount: number): number {
|
|
283
|
+
const cache = readBillingCache();
|
|
284
|
+
if (cache?.features?.max_candidate_pool != null) return cache.features.max_candidate_pool;
|
|
285
|
+
// Fallback to local formula if no server config
|
|
264
286
|
return Math.min(Math.max(factCount * 3, 400), 5000);
|
|
265
287
|
}
|
|
266
288
|
|
|
@@ -302,21 +324,21 @@ async function getFactCount(logger: OpenClawPluginApi['logger']): Promise<number
|
|
|
302
324
|
// Initialisation
|
|
303
325
|
// ---------------------------------------------------------------------------
|
|
304
326
|
|
|
305
|
-
/** True when
|
|
327
|
+
/** True when recovery phrase is missing — tools return setup instructions. */
|
|
306
328
|
let needsSetup = false;
|
|
307
329
|
|
|
308
330
|
/**
|
|
309
|
-
* Derive keys from the
|
|
331
|
+
* Derive keys from the recovery phrase, load or create credentials, and
|
|
310
332
|
* register with the server if this is the first run.
|
|
311
333
|
*/
|
|
312
334
|
async function initialize(logger: OpenClawPluginApi['logger']): Promise<void> {
|
|
313
335
|
const serverUrl =
|
|
314
336
|
process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz';
|
|
315
|
-
const masterPassword = process.env.
|
|
337
|
+
const masterPassword = process.env.TOTALRECLAW_RECOVERY_PHRASE;
|
|
316
338
|
|
|
317
339
|
if (!masterPassword) {
|
|
318
340
|
needsSetup = true;
|
|
319
|
-
logger.info('
|
|
341
|
+
logger.info('TOTALRECLAW_RECOVERY_PHRASE not set — setup required (see SKILL.md Post-Install Setup)');
|
|
320
342
|
return;
|
|
321
343
|
}
|
|
322
344
|
|
|
@@ -412,27 +434,27 @@ function isDocker(): boolean {
|
|
|
412
434
|
function buildSetupErrorMsg(): string {
|
|
413
435
|
const base =
|
|
414
436
|
'TotalReclaw setup required:\n' +
|
|
415
|
-
'1. Set
|
|
437
|
+
'1. Set TOTALRECLAW_RECOVERY_PHRASE — ask the user if they have an existing recovery phrase or generate a new 12-word BIP-39 mnemonic.\n' +
|
|
416
438
|
'2. Restart the gateway to apply changes.\n' +
|
|
417
439
|
' (Optional: set TOTALRECLAW_SELF_HOSTED=true if using your own server instead of the managed service.)\n\n';
|
|
418
440
|
|
|
419
441
|
if (isDocker()) {
|
|
420
442
|
return base +
|
|
421
443
|
'Running in Docker — pass env vars via `-e` flags or your compose file:\n' +
|
|
422
|
-
' -e
|
|
444
|
+
' -e TOTALRECLAW_RECOVERY_PHRASE="word1 word2 ..."';
|
|
423
445
|
}
|
|
424
446
|
|
|
425
447
|
if (process.platform === 'darwin') {
|
|
426
448
|
return base +
|
|
427
449
|
'Running on macOS — add env vars to the LaunchAgent plist at\n' +
|
|
428
450
|
'~/Library/LaunchAgents/ai.openclaw.gateway.plist under <key>EnvironmentVariables</key>:\n' +
|
|
429
|
-
' <key>
|
|
451
|
+
' <key>TOTALRECLAW_RECOVERY_PHRASE</key><string>word1 word2 ...</string>\n' +
|
|
430
452
|
'Then run: openclaw gateway restart';
|
|
431
453
|
}
|
|
432
454
|
|
|
433
455
|
return base +
|
|
434
456
|
'Running on Linux — add env vars to the systemd unit override or your shell profile:\n' +
|
|
435
|
-
' export
|
|
457
|
+
' export TOTALRECLAW_RECOVERY_PHRASE="word1 word2 ..."\n' +
|
|
436
458
|
'Then run: openclaw gateway restart';
|
|
437
459
|
}
|
|
438
460
|
|
|
@@ -463,7 +485,7 @@ async function requireFullSetup(logger: OpenClawPluginApi['logger']): Promise<vo
|
|
|
463
485
|
// LSH + Embedding helpers
|
|
464
486
|
// ---------------------------------------------------------------------------
|
|
465
487
|
|
|
466
|
-
/**
|
|
488
|
+
/** Recovery phrase cached for LSH seed derivation (set during initialize()). */
|
|
467
489
|
let masterPasswordCache: string | null = null;
|
|
468
490
|
/** Salt cached for LSH seed derivation (set during initialize()). */
|
|
469
491
|
let saltCache: Buffer | null = null;
|
|
@@ -472,7 +494,7 @@ let saltCache: Buffer | null = null;
|
|
|
472
494
|
* Get or initialize the LSH hasher.
|
|
473
495
|
*
|
|
474
496
|
* The hasher is created lazily because it needs:
|
|
475
|
-
* 1. The
|
|
497
|
+
* 1. The recovery phrase + salt (available after initialize())
|
|
476
498
|
* 2. The embedding dimensions (available after initLLMClient())
|
|
477
499
|
*
|
|
478
500
|
* If the provider doesn't support embeddings, this returns null and
|
|
@@ -857,9 +879,13 @@ async function storeExtractedFacts(
|
|
|
857
879
|
}
|
|
858
880
|
|
|
859
881
|
// Phase 3: Store the deduplicated facts (with optional store-time dedup).
|
|
882
|
+
// In subgraph mode, collect all protobuf payloads (tombstones + new facts)
|
|
883
|
+
// and submit them in a single batched UserOp for gas efficiency.
|
|
860
884
|
let stored = 0;
|
|
861
885
|
let superseded = 0;
|
|
862
886
|
let skipped = 0;
|
|
887
|
+
const pendingPayloads: Buffer[] = []; // Batched subgraph payloads
|
|
888
|
+
let preparedForSubgraph = 0;
|
|
863
889
|
|
|
864
890
|
for (const fact of dedupedFacts) {
|
|
865
891
|
try {
|
|
@@ -881,24 +907,19 @@ async function storeExtractedFacts(
|
|
|
881
907
|
if (fact.action === 'DELETE' && fact.existingFactId) {
|
|
882
908
|
// Tombstone the old fact, don't store anything new.
|
|
883
909
|
if (isSubgraphMode()) {
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
await submitFactOnChain(encodeFactProtobuf(tombstone), tombConfig);
|
|
898
|
-
logger.info(`LLM dedup: DELETE — tombstoned ${fact.existingFactId} on-chain`);
|
|
899
|
-
} catch (tombErr) {
|
|
900
|
-
logger.warn(`LLM dedup: DELETE failed for ${fact.existingFactId}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`);
|
|
901
|
-
}
|
|
910
|
+
const tombstone: FactPayload = {
|
|
911
|
+
id: fact.existingFactId,
|
|
912
|
+
timestamp: new Date().toISOString(),
|
|
913
|
+
owner: subgraphOwner || userId!,
|
|
914
|
+
encryptedBlob: '00',
|
|
915
|
+
blindIndices: [],
|
|
916
|
+
decayScore: 0,
|
|
917
|
+
source: 'tombstone',
|
|
918
|
+
contentFp: '',
|
|
919
|
+
agentId: 'openclaw-plugin-auto',
|
|
920
|
+
};
|
|
921
|
+
pendingPayloads.push(encodeFactProtobuf(tombstone));
|
|
922
|
+
logger.info(`LLM dedup: DELETE — queued tombstone for ${fact.existingFactId}`);
|
|
902
923
|
} else if (apiClient && authKeyHex) {
|
|
903
924
|
try {
|
|
904
925
|
await apiClient.deleteFact(fact.existingFactId, authKeyHex);
|
|
@@ -914,24 +935,19 @@ async function storeExtractedFacts(
|
|
|
914
935
|
if (fact.action === 'UPDATE' && fact.existingFactId) {
|
|
915
936
|
// Tombstone the old fact, then fall through to store the new version.
|
|
916
937
|
if (isSubgraphMode()) {
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
await submitFactOnChain(encodeFactProtobuf(tombstone), tombConfig);
|
|
931
|
-
logger.info(`LLM dedup: UPDATE — tombstoned ${fact.existingFactId} on-chain, storing replacement`);
|
|
932
|
-
} catch (tombErr) {
|
|
933
|
-
logger.warn(`LLM dedup: UPDATE tombstone failed for ${fact.existingFactId}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`);
|
|
934
|
-
}
|
|
938
|
+
const tombstone: FactPayload = {
|
|
939
|
+
id: fact.existingFactId,
|
|
940
|
+
timestamp: new Date().toISOString(),
|
|
941
|
+
owner: subgraphOwner || userId!,
|
|
942
|
+
encryptedBlob: '00',
|
|
943
|
+
blindIndices: [],
|
|
944
|
+
decayScore: 0,
|
|
945
|
+
source: 'tombstone',
|
|
946
|
+
contentFp: '',
|
|
947
|
+
agentId: 'openclaw-plugin-auto',
|
|
948
|
+
};
|
|
949
|
+
pendingPayloads.push(encodeFactProtobuf(tombstone));
|
|
950
|
+
logger.info(`LLM dedup: UPDATE — queued tombstone for ${fact.existingFactId}, storing replacement`);
|
|
935
951
|
} else if (apiClient && authKeyHex) {
|
|
936
952
|
try {
|
|
937
953
|
await apiClient.deleteFact(fact.existingFactId, authKeyHex);
|
|
@@ -969,29 +985,21 @@ async function storeExtractedFacts(
|
|
|
969
985
|
}
|
|
970
986
|
// action === 'supersede': delete old fact, inherit higher importance
|
|
971
987
|
if (isSubgraphMode()) {
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
logger.info(
|
|
988
|
-
`Store-time dedup: superseded ${dupResult.match.id} on-chain (sim=${dupResult.similarity.toFixed(3)})`,
|
|
989
|
-
);
|
|
990
|
-
} catch (tombErr) {
|
|
991
|
-
logger.warn(
|
|
992
|
-
`Store-time dedup: failed to tombstone ${dupResult.match.id}: ${tombErr instanceof Error ? tombErr.message : String(tombErr)}`,
|
|
993
|
-
);
|
|
994
|
-
}
|
|
988
|
+
const tombstone: FactPayload = {
|
|
989
|
+
id: dupResult.match.id,
|
|
990
|
+
timestamp: new Date().toISOString(),
|
|
991
|
+
owner: subgraphOwner || userId!,
|
|
992
|
+
encryptedBlob: '00',
|
|
993
|
+
blindIndices: [],
|
|
994
|
+
decayScore: 0,
|
|
995
|
+
source: 'tombstone',
|
|
996
|
+
contentFp: '',
|
|
997
|
+
agentId: 'openclaw-plugin-auto',
|
|
998
|
+
};
|
|
999
|
+
pendingPayloads.push(encodeFactProtobuf(tombstone));
|
|
1000
|
+
logger.info(
|
|
1001
|
+
`Store-time dedup: queued supersede for ${dupResult.match.id} (sim=${dupResult.similarity.toFixed(3)})`,
|
|
1002
|
+
);
|
|
995
1003
|
} else if (apiClient && authKeyHex) {
|
|
996
1004
|
try {
|
|
997
1005
|
await apiClient.deleteFact(dupResult.match.id, authKeyHex);
|
|
@@ -1024,20 +1032,7 @@ async function storeExtractedFacts(
|
|
|
1024
1032
|
const contentFp = generateContentFingerprint(fact.text, dedupKey);
|
|
1025
1033
|
const factId = crypto.randomUUID();
|
|
1026
1034
|
|
|
1027
|
-
const payload: StoreFactPayload = {
|
|
1028
|
-
id: factId,
|
|
1029
|
-
timestamp: new Date().toISOString(),
|
|
1030
|
-
encrypted_blob: encryptedBlob,
|
|
1031
|
-
blind_indices: allIndices,
|
|
1032
|
-
decay_score: effectiveImportance,
|
|
1033
|
-
source: 'auto-extraction',
|
|
1034
|
-
content_fp: contentFp,
|
|
1035
|
-
agent_id: 'openclaw-plugin-auto',
|
|
1036
|
-
encrypted_embedding: embeddingResult?.encryptedEmbedding,
|
|
1037
|
-
};
|
|
1038
|
-
|
|
1039
1035
|
if (isSubgraphMode()) {
|
|
1040
|
-
const config = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
1041
1036
|
const protobuf = encodeFactProtobuf({
|
|
1042
1037
|
id: factId,
|
|
1043
1038
|
timestamp: new Date().toISOString(),
|
|
@@ -1050,11 +1045,23 @@ async function storeExtractedFacts(
|
|
|
1050
1045
|
agentId: 'openclaw-plugin-auto',
|
|
1051
1046
|
encryptedEmbedding: embeddingResult?.encryptedEmbedding,
|
|
1052
1047
|
});
|
|
1053
|
-
|
|
1048
|
+
pendingPayloads.push(protobuf);
|
|
1049
|
+
preparedForSubgraph++;
|
|
1054
1050
|
} else {
|
|
1051
|
+
const payload: StoreFactPayload = {
|
|
1052
|
+
id: factId,
|
|
1053
|
+
timestamp: new Date().toISOString(),
|
|
1054
|
+
encrypted_blob: encryptedBlob,
|
|
1055
|
+
blind_indices: allIndices,
|
|
1056
|
+
decay_score: effectiveImportance,
|
|
1057
|
+
source: 'auto-extraction',
|
|
1058
|
+
content_fp: contentFp,
|
|
1059
|
+
agent_id: 'openclaw-plugin-auto',
|
|
1060
|
+
encrypted_embedding: embeddingResult?.encryptedEmbedding,
|
|
1061
|
+
};
|
|
1055
1062
|
await apiClient.store(userId, [payload], authKeyHex);
|
|
1063
|
+
stored++;
|
|
1056
1064
|
}
|
|
1057
|
-
stored++;
|
|
1058
1065
|
} catch (err: unknown) {
|
|
1059
1066
|
// Check for 403 / quota exceeded — invalidate billing cache so next
|
|
1060
1067
|
// before_agent_start re-fetches and warns the user.
|
|
@@ -1068,6 +1075,28 @@ async function storeExtractedFacts(
|
|
|
1068
1075
|
}
|
|
1069
1076
|
}
|
|
1070
1077
|
|
|
1078
|
+
// Batch-submit all subgraph payloads in a single UserOp (gas-efficient).
|
|
1079
|
+
if (pendingPayloads.length > 0 && isSubgraphMode()) {
|
|
1080
|
+
try {
|
|
1081
|
+
const batchConfig = { ...getSubgraphConfig(), authKeyHex: authKeyHex!, walletAddress: subgraphOwner ?? undefined };
|
|
1082
|
+
const result = await submitFactBatchOnChain(pendingPayloads, batchConfig);
|
|
1083
|
+
if (result.success) {
|
|
1084
|
+
stored += preparedForSubgraph;
|
|
1085
|
+
logger.info(`Batch submitted ${result.batchSize} payloads in 1 UserOp (tx=${result.txHash.slice(0, 10)}…)`);
|
|
1086
|
+
} else {
|
|
1087
|
+
logger.warn(`Batch UserOp failed on-chain (tx=${result.txHash.slice(0, 10)}…)`);
|
|
1088
|
+
}
|
|
1089
|
+
} catch (err: unknown) {
|
|
1090
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
1091
|
+
if (errMsg.includes('403') || errMsg.toLowerCase().includes('quota')) {
|
|
1092
|
+
try { fs.unlinkSync(BILLING_CACHE_PATH); } catch { /* ignore */ }
|
|
1093
|
+
logger.warn(`Quota exceeded during batch submit — billing cache invalidated. ${errMsg}`);
|
|
1094
|
+
} else {
|
|
1095
|
+
logger.warn(`Batch submission failed: ${errMsg}`);
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1071
1100
|
if (stored > 0 || superseded > 0 || skipped > 0) {
|
|
1072
1101
|
logger.info(`Auto-extraction results: stored=${stored}, superseded=${superseded}, skipped=${skipped}`);
|
|
1073
1102
|
}
|
|
@@ -1239,7 +1268,7 @@ const plugin = {
|
|
|
1239
1268
|
},
|
|
1240
1269
|
type: {
|
|
1241
1270
|
type: 'string',
|
|
1242
|
-
enum: ['fact', 'preference', 'decision', 'episodic', 'goal'],
|
|
1271
|
+
enum: ['fact', 'preference', 'decision', 'episodic', 'goal', 'context', 'summary'],
|
|
1243
1272
|
description: 'The kind of memory (default: fact)',
|
|
1244
1273
|
},
|
|
1245
1274
|
importance: {
|
|
@@ -2550,12 +2579,13 @@ const plugin = {
|
|
|
2550
2579
|
: [];
|
|
2551
2580
|
const rawFacts = await extractFacts(evt.messages, 'turn', existingMemories);
|
|
2552
2581
|
const { kept: importanceFiltered } = filterByImportance(rawFacts, api.logger);
|
|
2553
|
-
|
|
2582
|
+
const maxFacts = getMaxFactsPerExtraction();
|
|
2583
|
+
if (importanceFiltered.length > maxFacts) {
|
|
2554
2584
|
api.logger.info(
|
|
2555
|
-
`Capped extraction from ${importanceFiltered.length} to ${
|
|
2585
|
+
`Capped extraction from ${importanceFiltered.length} to ${maxFacts} facts`,
|
|
2556
2586
|
);
|
|
2557
2587
|
}
|
|
2558
|
-
const facts = importanceFiltered.slice(0,
|
|
2588
|
+
const facts = importanceFiltered.slice(0, maxFacts);
|
|
2559
2589
|
if (facts.length > 0) {
|
|
2560
2590
|
await storeExtractedFacts(facts, api.logger);
|
|
2561
2591
|
}
|
|
@@ -2592,12 +2622,13 @@ const plugin = {
|
|
|
2592
2622
|
: [];
|
|
2593
2623
|
const rawCompactFacts = await extractFacts(evt.messages, 'full', existingMemories);
|
|
2594
2624
|
const { kept: compactImportanceFiltered } = filterByImportance(rawCompactFacts, api.logger);
|
|
2595
|
-
|
|
2625
|
+
const maxFactsCompact = getMaxFactsPerExtraction();
|
|
2626
|
+
if (compactImportanceFiltered.length > maxFactsCompact) {
|
|
2596
2627
|
api.logger.info(
|
|
2597
|
-
`Capped compaction extraction from ${compactImportanceFiltered.length} to ${
|
|
2628
|
+
`Capped compaction extraction from ${compactImportanceFiltered.length} to ${maxFactsCompact} facts`,
|
|
2598
2629
|
);
|
|
2599
2630
|
}
|
|
2600
|
-
const facts = compactImportanceFiltered.slice(0,
|
|
2631
|
+
const facts = compactImportanceFiltered.slice(0, maxFactsCompact);
|
|
2601
2632
|
if (facts.length > 0) {
|
|
2602
2633
|
await storeExtractedFacts(facts, api.logger);
|
|
2603
2634
|
}
|
|
@@ -2633,12 +2664,13 @@ const plugin = {
|
|
|
2633
2664
|
: [];
|
|
2634
2665
|
const rawResetFacts = await extractFacts(evt.messages, 'full', existingMemories);
|
|
2635
2666
|
const { kept: resetImportanceFiltered } = filterByImportance(rawResetFacts, api.logger);
|
|
2636
|
-
|
|
2667
|
+
const maxFactsReset = getMaxFactsPerExtraction();
|
|
2668
|
+
if (resetImportanceFiltered.length > maxFactsReset) {
|
|
2637
2669
|
api.logger.info(
|
|
2638
|
-
`Capped reset extraction from ${resetImportanceFiltered.length} to ${
|
|
2670
|
+
`Capped reset extraction from ${resetImportanceFiltered.length} to ${maxFactsReset} facts`,
|
|
2639
2671
|
);
|
|
2640
2672
|
}
|
|
2641
|
-
const facts = resetImportanceFiltered.slice(0,
|
|
2673
|
+
const facts = resetImportanceFiltered.slice(0, maxFactsReset);
|
|
2642
2674
|
if (facts.length > 0) {
|
|
2643
2675
|
await storeExtractedFacts(facts, api.logger);
|
|
2644
2676
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@totalreclaw/totalreclaw",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "End-to-end encrypted memory for AI agents — portable, yours forever. Automatic extraction, semantic search, and on-chain storage",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"keywords": [
|
package/setup.sh
CHANGED
|
@@ -16,4 +16,4 @@ echo " cd testbed/functional-test"
|
|
|
16
16
|
echo " docker compose -f docker-compose.functional-test.yml up -d"
|
|
17
17
|
echo ""
|
|
18
18
|
echo "The plugin will auto-register on first use."
|
|
19
|
-
echo "Set
|
|
19
|
+
echo "Set TOTALRECLAW_RECOVERY_PHRASE in your .env file."
|
package/subgraph-store.ts
CHANGED
|
@@ -189,7 +189,7 @@ export async function submitFactOnChain(
|
|
|
189
189
|
}
|
|
190
190
|
|
|
191
191
|
if (!config.mnemonic) {
|
|
192
|
-
throw new Error('Mnemonic (
|
|
192
|
+
throw new Error('Mnemonic (TOTALRECLAW_RECOVERY_PHRASE) is required for on-chain submission');
|
|
193
193
|
}
|
|
194
194
|
|
|
195
195
|
const chain = getChainFromId(config.chainId);
|
|
@@ -281,6 +281,105 @@ export async function submitFactOnChain(
|
|
|
281
281
|
};
|
|
282
282
|
}
|
|
283
283
|
|
|
284
|
+
/**
|
|
285
|
+
* Submit multiple facts on-chain in a single ERC-4337 UserOp (batched).
|
|
286
|
+
*
|
|
287
|
+
* Each protobuf payload becomes one call in a multi-call UserOp. The
|
|
288
|
+
* DataEdge contract emits a separate Log(bytes) event per call, and the
|
|
289
|
+
* subgraph indexes each event independently (by txHash + logIndex).
|
|
290
|
+
*
|
|
291
|
+
* Falls back to single-fact path for batches of 1 (no multicall overhead).
|
|
292
|
+
*/
|
|
293
|
+
export async function submitFactBatchOnChain(
|
|
294
|
+
protobufPayloads: Buffer[],
|
|
295
|
+
config: SubgraphStoreConfig,
|
|
296
|
+
): Promise<{ txHash: string; userOpHash: string; success: boolean; batchSize: number }> {
|
|
297
|
+
if (!protobufPayloads.length) {
|
|
298
|
+
return { txHash: '', userOpHash: '', success: true, batchSize: 0 };
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// Single fact — use standard path (avoids multicall overhead)
|
|
302
|
+
if (protobufPayloads.length === 1) {
|
|
303
|
+
const result = await submitFactOnChain(protobufPayloads[0], config);
|
|
304
|
+
return { ...result, batchSize: 1 };
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
if (!config.relayUrl) {
|
|
308
|
+
throw new Error('Relay URL (TOTALRECLAW_SERVER_URL) is required for on-chain submission');
|
|
309
|
+
}
|
|
310
|
+
if (!config.mnemonic) {
|
|
311
|
+
throw new Error('Mnemonic (TOTALRECLAW_RECOVERY_PHRASE) is required for on-chain submission');
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const chain = getChainFromId(config.chainId);
|
|
315
|
+
const bundlerRpcUrl = getRelayBundlerUrl(config.relayUrl);
|
|
316
|
+
const dataEdgeAddress = config.dataEdgeAddress as Address;
|
|
317
|
+
const entryPointAddr = (config.entryPointAddress || entryPoint07Address) as Address;
|
|
318
|
+
|
|
319
|
+
const headers: Record<string, string> = {
|
|
320
|
+
'X-TotalReclaw-Client': 'openclaw-plugin',
|
|
321
|
+
};
|
|
322
|
+
if (config.authKeyHex) headers['Authorization'] = `Bearer ${config.authKeyHex}`;
|
|
323
|
+
if (config.walletAddress) headers['X-Wallet-Address'] = config.walletAddress;
|
|
324
|
+
|
|
325
|
+
const authTransport = Object.keys(headers).length > 0
|
|
326
|
+
? http(bundlerRpcUrl, { fetchOptions: { headers } })
|
|
327
|
+
: http(bundlerRpcUrl);
|
|
328
|
+
|
|
329
|
+
const ownerAccount = mnemonicToAccount(config.mnemonic);
|
|
330
|
+
const publicClient = createPublicClient({
|
|
331
|
+
chain,
|
|
332
|
+
transport: config.rpcUrl ? http(config.rpcUrl) : http(),
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
const pimlicoClient = createPimlicoClient({
|
|
336
|
+
chain,
|
|
337
|
+
transport: authTransport,
|
|
338
|
+
entryPoint: {
|
|
339
|
+
address: entryPointAddr,
|
|
340
|
+
version: '0.7',
|
|
341
|
+
},
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
const smartAccount = await toSimpleSmartAccount({
|
|
345
|
+
client: publicClient,
|
|
346
|
+
owner: ownerAccount,
|
|
347
|
+
entryPoint: {
|
|
348
|
+
address: entryPointAddr,
|
|
349
|
+
version: '0.7',
|
|
350
|
+
},
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
const smartAccountClient = createSmartAccountClient({
|
|
354
|
+
account: smartAccount,
|
|
355
|
+
chain,
|
|
356
|
+
bundlerTransport: authTransport,
|
|
357
|
+
paymaster: pimlicoClient,
|
|
358
|
+
userOperation: {
|
|
359
|
+
estimateFeesPerGas: async () => {
|
|
360
|
+
return (await pimlicoClient.getUserOperationGasPrice()).fast;
|
|
361
|
+
},
|
|
362
|
+
},
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
// Build multi-call batch: each payload → one call to DataEdge fallback()
|
|
366
|
+
const calls = protobufPayloads.map(payload => ({
|
|
367
|
+
to: dataEdgeAddress,
|
|
368
|
+
value: 0n,
|
|
369
|
+
data: `0x${payload.toString('hex')}` as Hex,
|
|
370
|
+
}));
|
|
371
|
+
|
|
372
|
+
const userOpHash = await smartAccountClient.sendUserOperation({ calls });
|
|
373
|
+
const receipt = await pimlicoClient.waitForUserOperationReceipt({ hash: userOpHash });
|
|
374
|
+
|
|
375
|
+
return {
|
|
376
|
+
txHash: receipt.receipt.transactionHash,
|
|
377
|
+
userOpHash,
|
|
378
|
+
success: receipt.success,
|
|
379
|
+
batchSize: protobufPayloads.length,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
|
|
284
383
|
// ---------------------------------------------------------------------------
|
|
285
384
|
// Configuration
|
|
286
385
|
// ---------------------------------------------------------------------------
|
|
@@ -299,7 +398,7 @@ export function isSubgraphMode(): boolean {
|
|
|
299
398
|
* Get subgraph configuration from environment variables.
|
|
300
399
|
*
|
|
301
400
|
* After the relay refactor, clients only need:
|
|
302
|
-
* -
|
|
401
|
+
* - TOTALRECLAW_RECOVERY_PHRASE -- BIP-39 mnemonic
|
|
303
402
|
* - TOTALRECLAW_SERVER_URL -- relay server URL (default: https://api.totalreclaw.xyz)
|
|
304
403
|
* - TOTALRECLAW_SELF_HOSTED -- set "true" to use self-hosted server (default: managed service)
|
|
305
404
|
* - TOTALRECLAW_CHAIN_ID -- optional, defaults to 100 (Gnosis mainnet)
|
|
@@ -338,7 +437,7 @@ export async function deriveSmartAccountAddress(mnemonic: string, chainId?: numb
|
|
|
338
437
|
export function getSubgraphConfig(): SubgraphStoreConfig {
|
|
339
438
|
return {
|
|
340
439
|
relayUrl: process.env.TOTALRECLAW_SERVER_URL || 'https://api.totalreclaw.xyz',
|
|
341
|
-
mnemonic: process.env.
|
|
440
|
+
mnemonic: process.env.TOTALRECLAW_RECOVERY_PHRASE || '',
|
|
342
441
|
cachePath: process.env.TOTALRECLAW_CACHE_PATH || `${process.env.HOME}/.totalreclaw/cache.enc`,
|
|
343
442
|
chainId: parseInt(process.env.TOTALRECLAW_CHAIN_ID || '100'),
|
|
344
443
|
dataEdgeAddress: process.env.TOTALRECLAW_DATA_EDGE_ADDRESS || DEFAULT_DATA_EDGE_ADDRESS,
|