@totalreclaw/totalreclaw 3.3.12 → 3.3.13-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/SKILL.md +25 -8
  2. package/{billing-cache.ts → billing/billing-cache.ts} +1 -1
  3. package/{relay-headers.ts → billing/relay-headers.ts} +1 -1
  4. package/{tr-cli-export-helper.ts → cli/tr-cli-export-helper.ts} +5 -5
  5. package/{tr-cli.ts → cli/tr-cli.ts} +20 -20
  6. package/{contradiction-sync.ts → contradiction/contradiction-sync.ts} +13 -13
  7. package/{digest-sync.ts → digest/digest-sync.ts} +32 -2
  8. package/dist/{billing-cache.js → billing/billing-cache.js} +1 -1
  9. package/dist/{relay-headers.js → billing/relay-headers.js} +1 -1
  10. package/dist/{tr-cli-export-helper.js → cli/tr-cli-export-helper.js} +5 -5
  11. package/dist/{tr-cli.js → cli/tr-cli.js} +19 -19
  12. package/dist/{contradiction-sync.js → contradiction/contradiction-sync.js} +13 -13
  13. package/dist/{digest-sync.js → digest/digest-sync.js} +32 -2
  14. package/dist/{download-ux.js → embedding/download-ux.js} +1 -1
  15. package/dist/{reranker.js → embedding/reranker.js} +13 -4
  16. package/dist/{claims-helper.js → extraction/claims-helper.js} +54 -58
  17. package/dist/extraction/consolidation.js +170 -0
  18. package/dist/{extractor.js → extraction/extractor.js} +17 -17
  19. package/dist/extraction/importance-filter.js +38 -0
  20. package/dist/{semantic-dedup.js → extraction/semantic-dedup.js} +2 -2
  21. package/dist/{batch-gate.js → import/batch-gate.js} +1 -1
  22. package/dist/import/import-runtime.js +692 -0
  23. package/dist/import-adapters/chatgpt-adapter.js +7 -7
  24. package/dist/import-adapters/gemini-adapter.js +13 -13
  25. package/dist/import-adapters/mem0-adapter.js +10 -10
  26. package/dist/index.js +111 -1094
  27. package/dist/{llm-client.js → llm/llm-client.js} +2 -2
  28. package/dist/{hot-cache-wrapper.js → memory/hot-cache-wrapper.js} +5 -5
  29. package/dist/{memory-runtime.js → memory/memory-runtime.js} +1 -1
  30. package/dist/{pin.js → memory/pin.js} +18 -18
  31. package/dist/{retype-setscope.js → memory/retype-setscope.js} +19 -19
  32. package/dist/{credential-provider.js → pairing/credential-provider.js} +2 -2
  33. package/dist/{first-run.js → pairing/first-run.js} +1 -1
  34. package/dist/{onboarding-cli.js → pairing/onboarding-cli.js} +20 -20
  35. package/dist/{pair-cli-relay.js → pairing/pair-cli-relay.js} +10 -10
  36. package/dist/{pair-crypto.js → pairing/pair-crypto.js} +1 -1
  37. package/dist/{pair-http.js → pairing/pair-http.js} +3 -3
  38. package/dist/{pair-remote-client.js → pairing/pair-remote-client.js} +10 -10
  39. package/dist/{pair-session-store.js → pairing/pair-session-store.js} +4 -4
  40. package/dist/runtime/config-schema.js +59 -0
  41. package/dist/runtime/format-helpers.js +257 -0
  42. package/dist/runtime/types.js +11 -0
  43. package/dist/{confirm-indexed.js → subgraph/confirm-indexed.js} +1 -1
  44. package/dist/{subgraph-search.js → subgraph/subgraph-search.js} +14 -14
  45. package/dist/{subgraph-store.js → subgraph/subgraph-store.js} +6 -6
  46. package/{download-ux.ts → embedding/download-ux.ts} +1 -1
  47. package/{reranker.ts → embedding/reranker.ts} +13 -4
  48. package/{claims-helper.ts → extraction/claims-helper.ts} +52 -60
  49. package/{consolidation.ts → extraction/consolidation.ts} +63 -154
  50. package/{extractor.ts → extraction/extractor.ts} +18 -17
  51. package/extraction/importance-filter.ts +50 -0
  52. package/{semantic-dedup.ts → extraction/semantic-dedup.ts} +2 -2
  53. package/{batch-gate.ts → import/batch-gate.ts} +1 -1
  54. package/import/import-runtime.ts +853 -0
  55. package/import-adapters/chatgpt-adapter.ts +7 -7
  56. package/import-adapters/gemini-adapter.ts +13 -13
  57. package/import-adapters/mem0-adapter.ts +10 -10
  58. package/index.ts +129 -1377
  59. package/{llm-client.ts → llm/llm-client.ts} +2 -2
  60. package/{hot-cache-wrapper.ts → memory/hot-cache-wrapper.ts} +5 -5
  61. package/{memory-runtime.ts → memory/memory-runtime.ts} +1 -1
  62. package/{pin.ts → memory/pin.ts} +20 -20
  63. package/{retype-setscope.ts → memory/retype-setscope.ts} +21 -21
  64. package/{tool-gating.ts → memory/tool-gating.ts} +1 -1
  65. package/package.json +6 -6
  66. package/{credential-provider.ts → pairing/credential-provider.ts} +2 -2
  67. package/{first-run.ts → pairing/first-run.ts} +1 -1
  68. package/{onboarding-cli.ts → pairing/onboarding-cli.ts} +20 -20
  69. package/{pair-cli-relay.ts → pairing/pair-cli-relay.ts} +10 -10
  70. package/{pair-crypto.ts → pairing/pair-crypto.ts} +1 -1
  71. package/{pair-http.ts → pairing/pair-http.ts} +3 -3
  72. package/{pair-remote-client.ts → pairing/pair-remote-client.ts} +10 -10
  73. package/{pair-session-store.ts → pairing/pair-session-store.ts} +4 -4
  74. package/runtime/config-schema.ts +64 -0
  75. package/runtime/format-helpers.ts +293 -0
  76. package/runtime/types.ts +118 -0
  77. package/skill.json +44 -10
  78. package/{confirm-indexed.ts → subgraph/confirm-indexed.ts} +1 -1
  79. package/{subgraph-search.ts → subgraph/subgraph-search.ts} +14 -14
  80. package/{subgraph-store.ts → subgraph/subgraph-store.ts} +10 -10
  81. package/dist/consolidation.js +0 -249
  82. /package/{api-client.ts → billing/api-client.ts} +0 -0
  83. /package/{gateway-url.ts → billing/gateway-url.ts} +0 -0
  84. /package/{inbound-user-tracker.ts → billing/inbound-user-tracker.ts} +0 -0
  85. /package/{relay.ts → billing/relay.ts} +0 -0
  86. /package/{crypto.ts → crypto/crypto.ts} +0 -0
  87. /package/{vault-crypto.ts → crypto/vault-crypto.ts} +0 -0
  88. /package/dist/{api-client.js → billing/api-client.js} +0 -0
  89. /package/dist/{gateway-url.js → billing/gateway-url.js} +0 -0
  90. /package/dist/{inbound-user-tracker.js → billing/inbound-user-tracker.js} +0 -0
  91. /package/dist/{relay.js → billing/relay.js} +0 -0
  92. /package/dist/{crypto.js → crypto/crypto.js} +0 -0
  93. /package/dist/{vault-crypto.js → crypto/vault-crypto.js} +0 -0
  94. /package/dist/{embedder-cache.js → embedding/embedder-cache.js} +0 -0
  95. /package/dist/{embedder-loader.js → embedding/embedder-loader.js} +0 -0
  96. /package/dist/{embedder-network.js → embedding/embedder-network.js} +0 -0
  97. /package/dist/{embedding.js → embedding/embedding.js} +0 -0
  98. /package/dist/{lsh.js → embedding/lsh.js} +0 -0
  99. /package/dist/{import-state-manager.js → import/import-state-manager.js} +0 -0
  100. /package/dist/{llm-profile-reader.js → llm/llm-profile-reader.js} +0 -0
  101. /package/dist/{native-memory.js → memory/native-memory.js} +0 -0
  102. /package/dist/{tool-gating.js → memory/tool-gating.js} +0 -0
  103. /package/dist/{tools.js → memory/tools.js} +0 -0
  104. /package/dist/{generate-mnemonic.js → pairing/generate-mnemonic.js} +0 -0
  105. /package/dist/{pair-cli.js → pairing/pair-cli.js} +0 -0
  106. /package/dist/{pair-page.js → pairing/pair-page.js} +0 -0
  107. /package/dist/{pair-qr.js → pairing/pair-qr.js} +0 -0
  108. /package/dist/{restart-auth.js → pairing/restart-auth.js} +0 -0
  109. /package/dist/{qa-bug-report.js → setup/qa-bug-report.js} +0 -0
  110. /package/dist/{skill-register.js → setup/skill-register.js} +0 -0
  111. /package/dist/{trajectory-poller.js → subgraph/trajectory-poller.js} +0 -0
  112. /package/{embedder-cache.ts → embedding/embedder-cache.ts} +0 -0
  113. /package/{embedder-loader.ts → embedding/embedder-loader.ts} +0 -0
  114. /package/{embedder-network.ts → embedding/embedder-network.ts} +0 -0
  115. /package/{embedding.ts → embedding/embedding.ts} +0 -0
  116. /package/{lsh.ts → embedding/lsh.ts} +0 -0
  117. /package/{import-state-manager.ts → import/import-state-manager.ts} +0 -0
  118. /package/{llm-profile-reader.ts → llm/llm-profile-reader.ts} +0 -0
  119. /package/{native-memory.ts → memory/native-memory.ts} +0 -0
  120. /package/{tools.ts → memory/tools.ts} +0 -0
  121. /package/{generate-mnemonic.ts → pairing/generate-mnemonic.ts} +0 -0
  122. /package/{pair-cli.ts → pairing/pair-cli.ts} +0 -0
  123. /package/{pair-page.ts → pairing/pair-page.ts} +0 -0
  124. /package/{pair-qr.ts → pairing/pair-qr.ts} +0 -0
  125. /package/{restart-auth.ts → pairing/restart-auth.ts} +0 -0
  126. /package/{qa-bug-report.ts → setup/qa-bug-report.ts} +0 -0
  127. /package/{skill-register.ts → setup/skill-register.ts} +0 -0
  128. /package/{trajectory-poller.ts → subgraph/trajectory-poller.ts} +0 -0
@@ -10,8 +10,8 @@
10
10
  */
11
11
  import crypto from 'node:crypto';
12
12
  import { createRequire } from 'node:module';
13
- import { isValidMemoryType, isValidMemoryTypeV1, V0_TO_V1_TYPE, VALID_MEMORY_SCOPES, VALID_MEMORY_SOURCES, VALID_MEMORY_VOLATILITIES, VALID_MEMORY_TYPES_V1, } from './extractor.js';
14
- import { envStringLower } from './entry.js';
13
+ import { isValidMemoryType, V0_TO_V1_TYPE, VALID_MEMORY_SCOPES, VALID_MEMORY_SOURCES, VALID_MEMORY_VOLATILITIES, } from './extractor.js';
14
+ import { envStringLower } from '../entry.js';
15
15
  // Lazy-load WASM. We use createRequire so this module loads cleanly under
16
16
  // both the OpenClaw runtime (CJS-ish tsx) and bare Node ESM (used by tests).
17
17
  const requireWasm = createRequire(import.meta.url);
@@ -285,28 +285,28 @@ export function isV1Blob(decrypted) {
285
285
  }
286
286
  export function readV1Blob(decrypted) {
287
287
  try {
288
- const obj = JSON.parse(decrypted);
289
- if (typeof obj.schema_version !== 'string' || !obj.schema_version.startsWith('1.')) {
288
+ const decodedBlob = JSON.parse(decrypted);
289
+ if (typeof decodedBlob.schema_version !== 'string' || !decodedBlob.schema_version.startsWith('1.')) {
290
290
  return null;
291
291
  }
292
- const text = typeof obj.text === 'string' ? obj.text : '';
293
- const rawType = typeof obj.type === 'string' ? obj.type : 'claim';
294
- const type = isValidMemoryTypeV1(rawType) ? rawType : 'claim';
295
- const rawSource = typeof obj.source === 'string' ? obj.source : 'user-inferred';
292
+ const text = typeof decodedBlob.text === 'string' ? decodedBlob.text : '';
293
+ const rawType = typeof decodedBlob.type === 'string' ? decodedBlob.type : 'claim';
294
+ const type = isValidMemoryType(rawType) ? rawType : 'claim';
295
+ const rawSource = typeof decodedBlob.source === 'string' ? decodedBlob.source : 'user-inferred';
296
296
  const source = VALID_MEMORY_SOURCES.includes(rawSource)
297
297
  ? rawSource
298
298
  : 'user-inferred';
299
- const rawScope = typeof obj.scope === 'string' ? obj.scope : 'unspecified';
299
+ const rawScope = typeof decodedBlob.scope === 'string' ? decodedBlob.scope : 'unspecified';
300
300
  const scope = VALID_MEMORY_SCOPES.includes(rawScope)
301
301
  ? rawScope
302
302
  : 'unspecified';
303
- const rawVolatility = typeof obj.volatility === 'string' ? obj.volatility : 'updatable';
303
+ const rawVolatility = typeof decodedBlob.volatility === 'string' ? decodedBlob.volatility : 'updatable';
304
304
  const volatility = VALID_MEMORY_VOLATILITIES.includes(rawVolatility)
305
305
  ? rawVolatility
306
306
  : 'updatable';
307
- const impRaw = typeof obj.importance === 'number' ? obj.importance : 5;
307
+ const impRaw = typeof decodedBlob.importance === 'number' ? decodedBlob.importance : 5;
308
308
  const importance = Math.max(1, Math.min(10, Math.round(impRaw)));
309
- const confRaw = typeof obj.confidence === 'number' ? obj.confidence : 0.85;
309
+ const confRaw = typeof decodedBlob.confidence === 'number' ? decodedBlob.confidence : 0.85;
310
310
  const confidence = Math.max(0, Math.min(1, confRaw));
311
311
  const result = {
312
312
  text,
@@ -316,25 +316,25 @@ export function readV1Blob(decrypted) {
316
316
  volatility,
317
317
  importance,
318
318
  confidence,
319
- createdAt: typeof obj.created_at === 'string' ? obj.created_at : '',
319
+ createdAt: typeof decodedBlob.created_at === 'string' ? decodedBlob.created_at : '',
320
320
  };
321
- if (typeof obj.reasoning === 'string' && obj.reasoning.length > 0) {
322
- result.reasoning = obj.reasoning;
321
+ if (typeof decodedBlob.reasoning === 'string' && decodedBlob.reasoning.length > 0) {
322
+ result.reasoning = decodedBlob.reasoning;
323
323
  }
324
- if (Array.isArray(obj.entities)) {
325
- result.entities = obj.entities.filter((e) => !!e &&
324
+ if (Array.isArray(decodedBlob.entities)) {
325
+ result.entities = decodedBlob.entities.filter((e) => !!e &&
326
326
  typeof e === 'object' &&
327
327
  typeof e.name === 'string' &&
328
328
  typeof e.type === 'string');
329
329
  }
330
- if (typeof obj.expires_at === 'string')
331
- result.expiresAt = obj.expires_at;
332
- if (typeof obj.superseded_by === 'string')
333
- result.supersededBy = obj.superseded_by;
334
- if (typeof obj.id === 'string')
335
- result.id = obj.id;
336
- if (typeof obj.pin_status === 'string') {
337
- const ps = obj.pin_status;
330
+ if (typeof decodedBlob.expires_at === 'string')
331
+ result.expiresAt = decodedBlob.expires_at;
332
+ if (typeof decodedBlob.superseded_by === 'string')
333
+ result.supersededBy = decodedBlob.superseded_by;
334
+ if (typeof decodedBlob.id === 'string')
335
+ result.id = decodedBlob.id;
336
+ if (typeof decodedBlob.pin_status === 'string') {
337
+ const ps = decodedBlob.pin_status;
338
338
  if (ps === 'pinned' || ps === 'unpinned') {
339
339
  result.pinStatus = ps;
340
340
  }
@@ -342,8 +342,8 @@ export function readV1Blob(decrypted) {
342
342
  // 3.3.1-rc.22 — pull the embedder identity tag through. Plugin-only
343
343
  // field added by `buildCanonicalClaimV1` / `buildV1ClaimBlob` after
344
344
  // core validation.
345
- if (typeof obj.embedding_model_id === 'string' && obj.embedding_model_id.length > 0) {
346
- result.embeddingModelId = obj.embedding_model_id;
345
+ if (typeof decodedBlob.embedding_model_id === 'string' && decodedBlob.embedding_model_id.length > 0) {
346
+ result.embeddingModelId = decodedBlob.embedding_model_id;
347
347
  }
348
348
  return result;
349
349
  }
@@ -351,10 +351,6 @@ export function readV1Blob(decrypted) {
351
351
  return null;
352
352
  }
353
353
  }
354
- // Suppress unused-import lint warnings for VALID_MEMORY_TYPES_V1 — it is
355
- // exported from extractor.ts for downstream clients and kept in scope here
356
- // so future v1 helpers can reuse it without re-importing.
357
- void VALID_MEMORY_TYPES_V1;
358
354
  // ---------------------------------------------------------------------------
359
355
  // Back-compat alias: buildCanonicalClaimRouted
360
356
  //
@@ -421,56 +417,56 @@ export function resolveAutoResolveMode() {
421
417
  }
422
418
  export function readClaimFromBlob(decryptedJson) {
423
419
  try {
424
- const obj = JSON.parse(decryptedJson);
420
+ const decodedBlob = JSON.parse(decryptedJson);
425
421
  // v1 payload: long-form fields + schema_version "1.x"
426
- if (typeof obj.text === 'string' &&
427
- typeof obj.type === 'string' &&
428
- typeof obj.schema_version === 'string' &&
429
- obj.schema_version.startsWith('1.')) {
430
- const importance = typeof obj.importance === 'number'
431
- ? Math.max(1, Math.min(10, Math.round(obj.importance)))
422
+ if (typeof decodedBlob.text === 'string' &&
423
+ typeof decodedBlob.type === 'string' &&
424
+ typeof decodedBlob.schema_version === 'string' &&
425
+ decodedBlob.schema_version.startsWith('1.')) {
426
+ const importance = typeof decodedBlob.importance === 'number'
427
+ ? Math.max(1, Math.min(10, Math.round(decodedBlob.importance)))
432
428
  : 5;
433
429
  return {
434
- text: obj.text,
430
+ text: decodedBlob.text,
435
431
  importance,
436
- category: mapTypeToCategory(obj.type),
432
+ category: mapTypeToCategory(decodedBlob.type),
437
433
  metadata: {
438
- type: obj.type,
439
- source: typeof obj.source === 'string' ? obj.source : 'user-inferred',
440
- scope: typeof obj.scope === 'string' ? obj.scope : 'unspecified',
441
- volatility: typeof obj.volatility === 'string' ? obj.volatility : 'updatable',
442
- reasoning: typeof obj.reasoning === 'string' ? obj.reasoning : undefined,
434
+ type: decodedBlob.type,
435
+ source: typeof decodedBlob.source === 'string' ? decodedBlob.source : 'user-inferred',
436
+ scope: typeof decodedBlob.scope === 'string' ? decodedBlob.scope : 'unspecified',
437
+ volatility: typeof decodedBlob.volatility === 'string' ? decodedBlob.volatility : 'updatable',
438
+ reasoning: typeof decodedBlob.reasoning === 'string' ? decodedBlob.reasoning : undefined,
443
439
  // v1.1: surface pin_status verbatim for downstream (recall display +
444
440
  // export). Absent ⇒ undefined (receivers treat as "unpinned").
445
- pin_status: typeof obj.pin_status === 'string' ? obj.pin_status : undefined,
441
+ pin_status: typeof decodedBlob.pin_status === 'string' ? decodedBlob.pin_status : undefined,
446
442
  importance: importance / 10,
447
- created_at: typeof obj.created_at === 'string' ? obj.created_at : '',
448
- schema_version: obj.schema_version,
443
+ created_at: typeof decodedBlob.created_at === 'string' ? decodedBlob.created_at : '',
444
+ schema_version: decodedBlob.schema_version,
449
445
  },
450
446
  };
451
447
  }
452
448
  // New canonical Claim format: short keys
453
- if (typeof obj.t === 'string' && typeof obj.c === 'string') {
454
- const importance = typeof obj.i === 'number' ? Math.max(1, Math.min(10, Math.round(obj.i))) : 5;
449
+ if (typeof decodedBlob.t === 'string' && typeof decodedBlob.c === 'string') {
450
+ const importance = typeof decodedBlob.i === 'number' ? Math.max(1, Math.min(10, Math.round(decodedBlob.i))) : 5;
455
451
  return {
456
- text: obj.t,
452
+ text: decodedBlob.t,
457
453
  importance,
458
- category: obj.c,
454
+ category: decodedBlob.c,
459
455
  metadata: {
460
- type: obj.c,
456
+ type: decodedBlob.c,
461
457
  importance: importance / 10,
462
- source: typeof obj.sa === 'string' ? obj.sa : 'auto-extraction',
463
- created_at: typeof obj.ea === 'string' ? obj.ea : '',
458
+ source: typeof decodedBlob.sa === 'string' ? decodedBlob.sa : 'auto-extraction',
459
+ created_at: typeof decodedBlob.ea === 'string' ? decodedBlob.ea : '',
464
460
  },
465
461
  };
466
462
  }
467
463
  // Legacy plugin {text, metadata: {importance: 0-1}} format
468
- if (typeof obj.text === 'string') {
469
- const meta = obj.metadata ?? {};
464
+ if (typeof decodedBlob.text === 'string') {
465
+ const meta = decodedBlob.metadata ?? {};
470
466
  const impFloat = typeof meta.importance === 'number' ? meta.importance : 0.5;
471
467
  const importance = Math.max(1, Math.min(10, Math.round(impFloat * 10)));
472
468
  return {
473
- text: obj.text,
469
+ text: decodedBlob.text,
474
470
  importance,
475
471
  category: typeof meta.type === 'string' ? meta.type : 'fact',
476
472
  metadata: meta,
@@ -0,0 +1,170 @@
1
+ /**
2
+ * TotalReclaw Plugin - Memory Consolidation & Near-Duplicate Detection
3
+ *
4
+ * Provides cross-session / cross-vault deduplication of stored facts using
5
+ * cosine similarity on their embeddings. Unlike semantic-dedup.ts (which
6
+ * handles within-batch dedup at threshold 0.9), this module handles:
7
+ *
8
+ * 1. Store-time dedup — before writing a new fact, check whether a
9
+ * near-duplicate already exists in the vault (findNearDuplicate).
10
+ * 2. Supersede logic — when a near-duplicate is found, decide whether
11
+ * the new fact should replace or be skipped (shouldSupersede).
12
+ * 3. Bulk consolidation — cluster all facts in the vault and identify
13
+ * groups of near-duplicates for cleanup (clusterFacts).
14
+ *
15
+ * Delegates core computation to `@totalreclaw/core` Rust WASM module:
16
+ * `findNearDuplicate`, `shouldSupersede`, and `clusterFacts` all call the
17
+ * corresponding core functions directly.
18
+ *
19
+ * Threshold helpers remain local; their env reads are centralized in
20
+ * entry.ts (env-reading seam, Task 1.3 of the OpenClaw native
21
+ * integration plan, 2026-06-21).
22
+ */
23
+ import { createRequire } from 'node:module';
24
+ import { envNumber } from '../entry.js';
25
+ // ---------------------------------------------------------------------------
26
+ // Lazy-load WASM core (mirrors claims-helper.ts / contradiction-sync.ts
27
+ // pattern — plays nicely under both the OpenClaw runtime (CJS-ish tsx) and
28
+ // bare Node ESM used by tests).
29
+ // ---------------------------------------------------------------------------
30
+ const requireWasm = createRequire(import.meta.url);
31
+ let _wasm = null;
32
+ function getWasm() {
33
+ if (!_wasm)
34
+ _wasm = requireWasm('@totalreclaw/core');
35
+ return _wasm;
36
+ }
37
+ // ---------------------------------------------------------------------------
38
+ // Configuration
39
+ // ---------------------------------------------------------------------------
40
+ /**
41
+ * Get the cosine similarity threshold for store-time dedup.
42
+ *
43
+ * Configurable via TOTALRECLAW_STORE_DEDUP_THRESHOLD env var.
44
+ * Must be a number in [0, 1]. Falls back to 0.85 if invalid or unset.
45
+ */
46
+ export function getStoreDedupThreshold() {
47
+ return envNumber('TOTALRECLAW_STORE_DEDUP_THRESHOLD', 0.85, { min: 0, max: 1 });
48
+ }
49
+ /**
50
+ * Get the cosine similarity threshold for bulk consolidation clustering.
51
+ *
52
+ * Configurable via TOTALRECLAW_CONSOLIDATION_THRESHOLD env var.
53
+ * Must be a number in [0, 1]. Falls back to 0.88 if invalid or unset.
54
+ */
55
+ export function getConsolidationThreshold() {
56
+ return envNumber('TOTALRECLAW_CONSOLIDATION_THRESHOLD', 0.88, { min: 0, max: 1 });
57
+ }
58
+ /** Maximum candidates to compare against during store-time dedup. */
59
+ export const STORE_DEDUP_MAX_CANDIDATES = 200;
60
+ // ---------------------------------------------------------------------------
61
+ // Store-time dedup
62
+ // ---------------------------------------------------------------------------
63
+ /**
64
+ * Find the best near-duplicate match for a new fact among existing candidates.
65
+ *
66
+ * Compares the new fact's embedding against all candidates using cosine
67
+ * similarity. Returns the candidate with the highest similarity above the
68
+ * threshold, or null if no match is found.
69
+ *
70
+ * Candidates without embeddings are skipped (fail-safe).
71
+ *
72
+ * @param newFactEmbedding - Embedding vector for the new fact
73
+ * @param candidates - Existing facts to compare against
74
+ * @param threshold - Cosine similarity threshold (e.g. 0.85)
75
+ * @returns - Best match above threshold, or null
76
+ */
77
+ export function findNearDuplicate(newFactEmbedding, candidates, threshold) {
78
+ const wasm = getWasm();
79
+ const existing = candidates
80
+ .filter((c) => c.embedding && c.embedding.length > 0)
81
+ .map((c) => ({ id: c.id, embedding: c.embedding }));
82
+ if (existing.length === 0)
83
+ return null;
84
+ const bestMatchJs = wasm.findBestNearDuplicate(JSON.stringify(newFactEmbedding), JSON.stringify(existing), threshold);
85
+ if (bestMatchJs == null)
86
+ return null;
87
+ const bestMatch = typeof bestMatchJs === 'string' ? JSON.parse(bestMatchJs) : bestMatchJs;
88
+ const matched = candidates.find((c) => c.id === bestMatch.fact_id);
89
+ if (!matched)
90
+ return null;
91
+ return { existingFact: matched, similarity: bestMatch.similarity };
92
+ }
93
+ // ---------------------------------------------------------------------------
94
+ // Supersede logic
95
+ // ---------------------------------------------------------------------------
96
+ /**
97
+ * Decide whether a new fact should supersede an existing near-duplicate.
98
+ *
99
+ * - Higher importance wins.
100
+ * - Equal importance: new fact supersedes (newer is preferred).
101
+ *
102
+ * Delegates to `@totalreclaw/core` WASM `shouldSupersede`.
103
+ *
104
+ * @param newImportance - Importance score of the new fact
105
+ * @param existingFact - The existing near-duplicate candidate
106
+ * @returns - 'supersede' if new fact should replace, 'skip' otherwise
107
+ */
108
+ export function shouldSupersede(newImportance, existingFact) {
109
+ const wasm = getWasm();
110
+ return wasm.shouldSupersede(newImportance, existingFact.importance) ? 'supersede' : 'skip';
111
+ }
112
+ // ---------------------------------------------------------------------------
113
+ // Bulk consolidation
114
+ // ---------------------------------------------------------------------------
115
+ /**
116
+ * Cluster facts by semantic similarity using greedy single-pass clustering.
117
+ *
118
+ * Delegates to `@totalreclaw/core` WASM `clusterFacts` which performs the
119
+ * same greedy single-pass algorithm and representative selection. The WASM
120
+ * function returns ID-only clusters; this wrapper maps IDs back to full
121
+ * `DecryptedCandidate` objects for callers.
122
+ *
123
+ * Only returns clusters that have duplicates (i.e. more than one member).
124
+ * Facts without embeddings are not clustered.
125
+ *
126
+ * @param facts - All facts to cluster
127
+ * @param threshold - Cosine similarity threshold (e.g. 0.88)
128
+ * @returns - Clusters with duplicates (representative + duplicates)
129
+ */
130
+ export function clusterFacts(facts, threshold) {
131
+ const wasm = getWasm();
132
+ // Build ConsolidationCandidate JSON for WASM (snake_case fields).
133
+ const wasmCandidates = facts
134
+ .filter((f) => f.embedding && f.embedding.length > 0)
135
+ .map((f) => ({
136
+ id: f.id,
137
+ text: f.text,
138
+ embedding: f.embedding,
139
+ importance: f.importance,
140
+ decay_score: f.decayScore,
141
+ created_at: f.createdAt,
142
+ version: f.version,
143
+ }));
144
+ if (wasmCandidates.length === 0)
145
+ return [];
146
+ const clustersJs = wasm.clusterFacts(JSON.stringify(wasmCandidates), threshold);
147
+ // WASM returns a JSON string: [{ representative: string, duplicates: string[] }]
148
+ const wasmClusters = typeof clustersJs === 'string' ? JSON.parse(clustersJs) : clustersJs;
149
+ // Build a lookup map for fast ID -> DecryptedCandidate resolution.
150
+ const byId = new Map();
151
+ for (const f of facts)
152
+ byId.set(f.id, f);
153
+ // Map ID-only clusters back to full DecryptedCandidate objects.
154
+ // Filter out singleton clusters (no duplicates) to match the pre-WASM
155
+ // plugin contract — callers rely on `clusters.length === 0` when nothing
156
+ // duplicates anything.
157
+ const clusters = [];
158
+ for (const wc of wasmClusters) {
159
+ const rep = byId.get(wc.representative);
160
+ if (!rep)
161
+ continue;
162
+ const dups = wc.duplicates
163
+ .map((id) => byId.get(id))
164
+ .filter((d) => d !== undefined);
165
+ if (dups.length > 0) {
166
+ clusters.push({ representative: rep, duplicates: dups });
167
+ }
168
+ }
169
+ return clusters;
170
+ }
@@ -4,7 +4,7 @@
4
4
  * Uses LLM calls to extract atomic facts from conversation messages.
5
5
  * Matches the extraction prompts described in SKILL.md.
6
6
  */
7
- import { chatCompletion, resolveLLMConfig } from './llm-client.js';
7
+ import { chatCompletion, resolveLLMConfig } from '../llm/llm-client.js';
8
8
  // ---------------------------------------------------------------------------
9
9
  // Memory Taxonomy v1 — the 6 canonical memory types. Single source of truth.
10
10
  //
@@ -461,7 +461,7 @@ export function parseFactsResponseForCompaction(response, logger) {
461
461
  }
462
462
  }
463
463
  if (recoveryUsed === 'bracket-scan') {
464
- logger?.info?.(`parseFactsResponseForCompaction: recovered JSON via bracket-scan fallback`);
464
+ logger?.debug?.(`parseFactsResponseForCompaction: recovered JSON via bracket-scan fallback`);
465
465
  }
466
466
  if (!parsed || typeof parsed !== 'object') {
467
467
  logger?.warn?.(`parseFactsResponseForCompaction: could not parse LLM output as JSON object. Preview: ${JSON.stringify(originalPreview)}`);
@@ -514,7 +514,7 @@ export function parseFactsResponseForCompaction(response, logger) {
514
514
  if (valid.length > 0)
515
515
  entities = valid;
516
516
  }
517
- const result = {
517
+ const extractedFact = {
518
518
  text: String(f.text).slice(0, 512),
519
519
  type,
520
520
  source,
@@ -526,8 +526,8 @@ export function parseFactsResponseForCompaction(response, logger) {
526
526
  confidence: normalizeConfidence(f.confidence),
527
527
  };
528
528
  if (entities)
529
- result.entities = entities;
530
- return result;
529
+ extractedFact.entities = entities;
530
+ return extractedFact;
531
531
  })
532
532
  // Reject illegal type:summary + source:user
533
533
  .filter((f) => !(f.type === 'summary' && f.source === 'user'))
@@ -601,7 +601,7 @@ export async function extractFactsForCompaction(rawMessages, existingMemories, l
601
601
  logger?.info?.('extractFactsForCompaction: chatCompletion returned null/empty response');
602
602
  return [];
603
603
  }
604
- logger?.info?.(`extractFactsForCompaction: LLM returned ${response.length} chars; handing to parseFactsResponseForCompaction`);
604
+ logger?.debug?.(`extractFactsForCompaction: LLM returned ${response.length} chars; handing to parseFactsResponseForCompaction`);
605
605
  let facts = parseFactsResponseForCompaction(response, logger);
606
606
  // v1 provenance filter (tag-don't-drop). Uses importance >= 5 floor because
607
607
  // the filter's own floor is 5 in lax mode, matching compaction semantics.
@@ -617,7 +617,7 @@ export async function extractFactsForCompaction(rawMessages, existingMemories, l
617
617
  const oldImportance = f.importance;
618
618
  const effectiveBump = f.importance >= 8 ? Math.min(bump, 1) : bump;
619
619
  f.importance = Math.min(10, f.importance + effectiveBump);
620
- logger?.info?.(`extractFactsForCompaction: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`);
620
+ logger?.debug?.(`extractFactsForCompaction: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`);
621
621
  }
622
622
  }
623
623
  return facts;
@@ -1020,7 +1020,7 @@ export function parseMergedResponseV1(response, logger) {
1020
1020
  }
1021
1021
  }
1022
1022
  if (recoveryUsed === 'bracket-scan') {
1023
- logger?.info?.(`parseFactsResponse: recovered JSON via bracket-scan fallback`);
1023
+ logger?.debug?.(`parseFactsResponse: recovered JSON via bracket-scan fallback`);
1024
1024
  }
1025
1025
  if (!parsed || typeof parsed !== 'object') {
1026
1026
  logger?.warn?.(`parseFactsResponse: could not parse LLM output as JSON. Preview: ${JSON.stringify(originalPreview)}`);
@@ -1030,25 +1030,25 @@ export function parseMergedResponseV1(response, logger) {
1030
1030
  // a bare JSON array of fact objects (legacy / test fixture shape). The
1031
1031
  // bare array is wrapped as { topics: [], facts: [...] } so the downstream
1032
1032
  // logic stays uniform. A single fact object (no wrapper) is also wrapped.
1033
- let obj;
1033
+ let wrapper;
1034
1034
  if (Array.isArray(parsed)) {
1035
- obj = { topics: [], facts: parsed };
1035
+ wrapper = { topics: [], facts: parsed };
1036
1036
  }
1037
1037
  else if (typeof parsed.facts === 'undefined' &&
1038
1038
  typeof parsed.text === 'string') {
1039
1039
  // Single fact object, not a merged wrapper.
1040
- obj = { topics: [], facts: [parsed] };
1040
+ wrapper = { topics: [], facts: [parsed] };
1041
1041
  }
1042
1042
  else {
1043
- obj = parsed;
1043
+ wrapper = parsed;
1044
1044
  }
1045
- const rawTopics = obj.topics;
1045
+ const rawTopics = wrapper.topics;
1046
1046
  const topics = Array.isArray(rawTopics)
1047
1047
  ? rawTopics
1048
1048
  .filter((t) => typeof t === 'string' && t.length > 0)
1049
1049
  .slice(0, 3)
1050
1050
  : [];
1051
- const rawFacts = obj.facts;
1051
+ const rawFacts = wrapper.facts;
1052
1052
  if (!Array.isArray(rawFacts))
1053
1053
  return { topics, facts: [] };
1054
1054
  const validActions = ['ADD', 'UPDATE', 'DELETE', 'NOOP'];
@@ -1334,10 +1334,10 @@ export async function extractFacts(rawMessages, mode, existingMemories, profileC
1334
1334
  logger?.info?.('extractFacts: chatCompletion returned null/empty response');
1335
1335
  return [];
1336
1336
  }
1337
- logger?.info?.(`extractFacts: LLM returned ${response.length} chars; parsing merged response`);
1337
+ logger?.debug?.(`extractFacts: LLM returned ${response.length} chars; parsing merged response`);
1338
1338
  const { topics, facts: rawFacts } = parseMergedResponseV1(response, logger);
1339
1339
  if (topics.length > 0) {
1340
- logger?.info?.(`extractFacts: topics = ${JSON.stringify(topics)}`);
1340
+ logger?.debug?.(`extractFacts: topics = ${JSON.stringify(topics)}`);
1341
1341
  }
1342
1342
  // Provenance filter (tag-don't-drop)
1343
1343
  let facts = applyProvenanceFilterLax(rawFacts, conversationText);
@@ -1352,7 +1352,7 @@ export async function extractFacts(rawMessages, mode, existingMemories, profileC
1352
1352
  const oldImportance = f.importance;
1353
1353
  const effectiveBump = f.importance >= 8 ? Math.min(bump, 1) : bump;
1354
1354
  f.importance = Math.min(10, f.importance + effectiveBump);
1355
- logger?.info?.(`extractFacts: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`);
1355
+ logger?.debug?.(`extractFacts: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`);
1356
1356
  }
1357
1357
  }
1358
1358
  return facts;
@@ -0,0 +1,38 @@
1
+ // ---------------------------------------------------------------------------
2
+ // Importance filter for auto-extraction. Extracted from index.ts.
3
+ // Pure over its inputs (threshold derives from CONFIG); no plugin session state.
4
+ // ---------------------------------------------------------------------------
5
+ import { CONFIG } from '../config.js';
6
+ const MIN_IMPORTANCE_THRESHOLD = CONFIG.minImportance;
7
+ /**
8
+ * Filter extracted facts by importance threshold.
9
+ * Facts with importance < MIN_IMPORTANCE_THRESHOLD are dropped.
10
+ * Facts with missing/undefined importance are treated as importance=5 (kept).
11
+ */
12
+ export function filterByImportance(facts, logger) {
13
+ const kept = [];
14
+ let dropped = 0;
15
+ for (const fact of facts) {
16
+ const importance = fact.importance ?? 5;
17
+ if (importance >= MIN_IMPORTANCE_THRESHOLD) {
18
+ kept.push(fact);
19
+ }
20
+ else {
21
+ dropped++;
22
+ }
23
+ }
24
+ // Phase 2.2.5: always log the filter outcome so the agent_end path can
25
+ // distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
26
+ // below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
27
+ // this only logged on drops, which made empty-input invisible.
28
+ if (facts.length === 0) {
29
+ logger.info('Importance filter: input=0 (nothing to filter)');
30
+ }
31
+ else if (dropped > 0) {
32
+ logger.info(`Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`);
33
+ }
34
+ else {
35
+ logger.info(`Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`);
36
+ }
37
+ return { kept, dropped };
38
+ }
@@ -10,8 +10,8 @@
10
10
  * cosineSimilarity and extractor for the ExtractedFact type) so it can
11
11
  * be tested without pulling in the full plugin dependency graph.
12
12
  */
13
- import { cosineSimilarity } from './reranker.js';
14
- import { envNumber } from './entry.js';
13
+ import { cosineSimilarity } from '../embedding/reranker.js';
14
+ import { envNumber } from '../entry.js';
15
15
  // ---------------------------------------------------------------------------
16
16
  // Configuration
17
17
  // ---------------------------------------------------------------------------
@@ -23,7 +23,7 @@
23
23
  * Env read is centralized in entry.ts (env-reading seam, Task 1.3 of the
24
24
  * OpenClaw native integration plan, 2026-06-21).
25
25
  */
26
- import { isGnosisBatchEnabledAtBoot } from './entry.js';
26
+ import { isGnosisBatchEnabledAtBoot } from '../entry.js';
27
27
  const GNOSIS_CHAIN_ID = 100;
28
28
  export function shouldBatchOnChain(chainId) {
29
29
  if (!isGnosisBatchEnabledAtBoot())