@totalreclaw/totalreclaw 3.3.12 → 3.3.13-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +25 -8
- package/{billing-cache.ts → billing/billing-cache.ts} +1 -1
- package/{relay-headers.ts → billing/relay-headers.ts} +1 -1
- package/{tr-cli-export-helper.ts → cli/tr-cli-export-helper.ts} +5 -5
- package/{tr-cli.ts → cli/tr-cli.ts} +20 -20
- package/{contradiction-sync.ts → contradiction/contradiction-sync.ts} +13 -13
- package/{digest-sync.ts → digest/digest-sync.ts} +32 -2
- package/dist/{billing-cache.js → billing/billing-cache.js} +1 -1
- package/dist/{relay-headers.js → billing/relay-headers.js} +1 -1
- package/dist/{tr-cli-export-helper.js → cli/tr-cli-export-helper.js} +5 -5
- package/dist/{tr-cli.js → cli/tr-cli.js} +19 -19
- package/dist/{contradiction-sync.js → contradiction/contradiction-sync.js} +13 -13
- package/dist/{digest-sync.js → digest/digest-sync.js} +32 -2
- package/dist/{download-ux.js → embedding/download-ux.js} +1 -1
- package/dist/{reranker.js → embedding/reranker.js} +13 -4
- package/dist/{claims-helper.js → extraction/claims-helper.js} +54 -58
- package/dist/extraction/consolidation.js +170 -0
- package/dist/{extractor.js → extraction/extractor.js} +17 -17
- package/dist/extraction/importance-filter.js +38 -0
- package/dist/{semantic-dedup.js → extraction/semantic-dedup.js} +2 -2
- package/dist/{batch-gate.js → import/batch-gate.js} +1 -1
- package/dist/import/import-runtime.js +692 -0
- package/dist/import-adapters/chatgpt-adapter.js +7 -7
- package/dist/import-adapters/gemini-adapter.js +13 -13
- package/dist/import-adapters/mem0-adapter.js +10 -10
- package/dist/index.js +111 -1094
- package/dist/{llm-client.js → llm/llm-client.js} +2 -2
- package/dist/{hot-cache-wrapper.js → memory/hot-cache-wrapper.js} +5 -5
- package/dist/{memory-runtime.js → memory/memory-runtime.js} +1 -1
- package/dist/{pin.js → memory/pin.js} +18 -18
- package/dist/{retype-setscope.js → memory/retype-setscope.js} +19 -19
- package/dist/{credential-provider.js → pairing/credential-provider.js} +2 -2
- package/dist/{first-run.js → pairing/first-run.js} +1 -1
- package/dist/{onboarding-cli.js → pairing/onboarding-cli.js} +20 -20
- package/dist/{pair-cli-relay.js → pairing/pair-cli-relay.js} +10 -10
- package/dist/{pair-crypto.js → pairing/pair-crypto.js} +1 -1
- package/dist/{pair-http.js → pairing/pair-http.js} +3 -3
- package/dist/{pair-remote-client.js → pairing/pair-remote-client.js} +10 -10
- package/dist/{pair-session-store.js → pairing/pair-session-store.js} +4 -4
- package/dist/runtime/config-schema.js +59 -0
- package/dist/runtime/format-helpers.js +257 -0
- package/dist/runtime/types.js +11 -0
- package/dist/{confirm-indexed.js → subgraph/confirm-indexed.js} +1 -1
- package/dist/{subgraph-search.js → subgraph/subgraph-search.js} +14 -14
- package/dist/{subgraph-store.js → subgraph/subgraph-store.js} +6 -6
- package/{download-ux.ts → embedding/download-ux.ts} +1 -1
- package/{reranker.ts → embedding/reranker.ts} +13 -4
- package/{claims-helper.ts → extraction/claims-helper.ts} +52 -60
- package/{consolidation.ts → extraction/consolidation.ts} +63 -154
- package/{extractor.ts → extraction/extractor.ts} +18 -17
- package/extraction/importance-filter.ts +50 -0
- package/{semantic-dedup.ts → extraction/semantic-dedup.ts} +2 -2
- package/{batch-gate.ts → import/batch-gate.ts} +1 -1
- package/import/import-runtime.ts +853 -0
- package/import-adapters/chatgpt-adapter.ts +7 -7
- package/import-adapters/gemini-adapter.ts +13 -13
- package/import-adapters/mem0-adapter.ts +10 -10
- package/index.ts +129 -1377
- package/{llm-client.ts → llm/llm-client.ts} +2 -2
- package/{hot-cache-wrapper.ts → memory/hot-cache-wrapper.ts} +5 -5
- package/{memory-runtime.ts → memory/memory-runtime.ts} +1 -1
- package/{pin.ts → memory/pin.ts} +20 -20
- package/{retype-setscope.ts → memory/retype-setscope.ts} +21 -21
- package/{tool-gating.ts → memory/tool-gating.ts} +1 -1
- package/package.json +6 -6
- package/{credential-provider.ts → pairing/credential-provider.ts} +2 -2
- package/{first-run.ts → pairing/first-run.ts} +1 -1
- package/{onboarding-cli.ts → pairing/onboarding-cli.ts} +20 -20
- package/{pair-cli-relay.ts → pairing/pair-cli-relay.ts} +10 -10
- package/{pair-crypto.ts → pairing/pair-crypto.ts} +1 -1
- package/{pair-http.ts → pairing/pair-http.ts} +3 -3
- package/{pair-remote-client.ts → pairing/pair-remote-client.ts} +10 -10
- package/{pair-session-store.ts → pairing/pair-session-store.ts} +4 -4
- package/runtime/config-schema.ts +64 -0
- package/runtime/format-helpers.ts +293 -0
- package/runtime/types.ts +118 -0
- package/skill.json +44 -10
- package/{confirm-indexed.ts → subgraph/confirm-indexed.ts} +1 -1
- package/{subgraph-search.ts → subgraph/subgraph-search.ts} +14 -14
- package/{subgraph-store.ts → subgraph/subgraph-store.ts} +10 -10
- package/dist/consolidation.js +0 -249
- /package/{api-client.ts → billing/api-client.ts} +0 -0
- /package/{gateway-url.ts → billing/gateway-url.ts} +0 -0
- /package/{inbound-user-tracker.ts → billing/inbound-user-tracker.ts} +0 -0
- /package/{relay.ts → billing/relay.ts} +0 -0
- /package/{crypto.ts → crypto/crypto.ts} +0 -0
- /package/{vault-crypto.ts → crypto/vault-crypto.ts} +0 -0
- /package/dist/{api-client.js → billing/api-client.js} +0 -0
- /package/dist/{gateway-url.js → billing/gateway-url.js} +0 -0
- /package/dist/{inbound-user-tracker.js → billing/inbound-user-tracker.js} +0 -0
- /package/dist/{relay.js → billing/relay.js} +0 -0
- /package/dist/{crypto.js → crypto/crypto.js} +0 -0
- /package/dist/{vault-crypto.js → crypto/vault-crypto.js} +0 -0
- /package/dist/{embedder-cache.js → embedding/embedder-cache.js} +0 -0
- /package/dist/{embedder-loader.js → embedding/embedder-loader.js} +0 -0
- /package/dist/{embedder-network.js → embedding/embedder-network.js} +0 -0
- /package/dist/{embedding.js → embedding/embedding.js} +0 -0
- /package/dist/{lsh.js → embedding/lsh.js} +0 -0
- /package/dist/{import-state-manager.js → import/import-state-manager.js} +0 -0
- /package/dist/{llm-profile-reader.js → llm/llm-profile-reader.js} +0 -0
- /package/dist/{native-memory.js → memory/native-memory.js} +0 -0
- /package/dist/{tool-gating.js → memory/tool-gating.js} +0 -0
- /package/dist/{tools.js → memory/tools.js} +0 -0
- /package/dist/{generate-mnemonic.js → pairing/generate-mnemonic.js} +0 -0
- /package/dist/{pair-cli.js → pairing/pair-cli.js} +0 -0
- /package/dist/{pair-page.js → pairing/pair-page.js} +0 -0
- /package/dist/{pair-qr.js → pairing/pair-qr.js} +0 -0
- /package/dist/{restart-auth.js → pairing/restart-auth.js} +0 -0
- /package/dist/{qa-bug-report.js → setup/qa-bug-report.js} +0 -0
- /package/dist/{skill-register.js → setup/skill-register.js} +0 -0
- /package/dist/{trajectory-poller.js → subgraph/trajectory-poller.js} +0 -0
- /package/{embedder-cache.ts → embedding/embedder-cache.ts} +0 -0
- /package/{embedder-loader.ts → embedding/embedder-loader.ts} +0 -0
- /package/{embedder-network.ts → embedding/embedder-network.ts} +0 -0
- /package/{embedding.ts → embedding/embedding.ts} +0 -0
- /package/{lsh.ts → embedding/lsh.ts} +0 -0
- /package/{import-state-manager.ts → import/import-state-manager.ts} +0 -0
- /package/{llm-profile-reader.ts → llm/llm-profile-reader.ts} +0 -0
- /package/{native-memory.ts → memory/native-memory.ts} +0 -0
- /package/{tools.ts → memory/tools.ts} +0 -0
- /package/{generate-mnemonic.ts → pairing/generate-mnemonic.ts} +0 -0
- /package/{pair-cli.ts → pairing/pair-cli.ts} +0 -0
- /package/{pair-page.ts → pairing/pair-page.ts} +0 -0
- /package/{pair-qr.ts → pairing/pair-qr.ts} +0 -0
- /package/{restart-auth.ts → pairing/restart-auth.ts} +0 -0
- /package/{qa-bug-report.ts → setup/qa-bug-report.ts} +0 -0
- /package/{skill-register.ts → setup/skill-register.ts} +0 -0
- /package/{trajectory-poller.ts → subgraph/trajectory-poller.ts} +0 -0
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import crypto from 'node:crypto';
|
|
12
12
|
import { createRequire } from 'node:module';
|
|
13
|
-
import { isValidMemoryType,
|
|
14
|
-
import { envStringLower } from '
|
|
13
|
+
import { isValidMemoryType, V0_TO_V1_TYPE, VALID_MEMORY_SCOPES, VALID_MEMORY_SOURCES, VALID_MEMORY_VOLATILITIES, } from './extractor.js';
|
|
14
|
+
import { envStringLower } from '../entry.js';
|
|
15
15
|
// Lazy-load WASM. We use createRequire so this module loads cleanly under
|
|
16
16
|
// both the OpenClaw runtime (CJS-ish tsx) and bare Node ESM (used by tests).
|
|
17
17
|
const requireWasm = createRequire(import.meta.url);
|
|
@@ -285,28 +285,28 @@ export function isV1Blob(decrypted) {
|
|
|
285
285
|
}
|
|
286
286
|
export function readV1Blob(decrypted) {
|
|
287
287
|
try {
|
|
288
|
-
const
|
|
289
|
-
if (typeof
|
|
288
|
+
const decodedBlob = JSON.parse(decrypted);
|
|
289
|
+
if (typeof decodedBlob.schema_version !== 'string' || !decodedBlob.schema_version.startsWith('1.')) {
|
|
290
290
|
return null;
|
|
291
291
|
}
|
|
292
|
-
const text = typeof
|
|
293
|
-
const rawType = typeof
|
|
294
|
-
const type =
|
|
295
|
-
const rawSource = typeof
|
|
292
|
+
const text = typeof decodedBlob.text === 'string' ? decodedBlob.text : '';
|
|
293
|
+
const rawType = typeof decodedBlob.type === 'string' ? decodedBlob.type : 'claim';
|
|
294
|
+
const type = isValidMemoryType(rawType) ? rawType : 'claim';
|
|
295
|
+
const rawSource = typeof decodedBlob.source === 'string' ? decodedBlob.source : 'user-inferred';
|
|
296
296
|
const source = VALID_MEMORY_SOURCES.includes(rawSource)
|
|
297
297
|
? rawSource
|
|
298
298
|
: 'user-inferred';
|
|
299
|
-
const rawScope = typeof
|
|
299
|
+
const rawScope = typeof decodedBlob.scope === 'string' ? decodedBlob.scope : 'unspecified';
|
|
300
300
|
const scope = VALID_MEMORY_SCOPES.includes(rawScope)
|
|
301
301
|
? rawScope
|
|
302
302
|
: 'unspecified';
|
|
303
|
-
const rawVolatility = typeof
|
|
303
|
+
const rawVolatility = typeof decodedBlob.volatility === 'string' ? decodedBlob.volatility : 'updatable';
|
|
304
304
|
const volatility = VALID_MEMORY_VOLATILITIES.includes(rawVolatility)
|
|
305
305
|
? rawVolatility
|
|
306
306
|
: 'updatable';
|
|
307
|
-
const impRaw = typeof
|
|
307
|
+
const impRaw = typeof decodedBlob.importance === 'number' ? decodedBlob.importance : 5;
|
|
308
308
|
const importance = Math.max(1, Math.min(10, Math.round(impRaw)));
|
|
309
|
-
const confRaw = typeof
|
|
309
|
+
const confRaw = typeof decodedBlob.confidence === 'number' ? decodedBlob.confidence : 0.85;
|
|
310
310
|
const confidence = Math.max(0, Math.min(1, confRaw));
|
|
311
311
|
const result = {
|
|
312
312
|
text,
|
|
@@ -316,25 +316,25 @@ export function readV1Blob(decrypted) {
|
|
|
316
316
|
volatility,
|
|
317
317
|
importance,
|
|
318
318
|
confidence,
|
|
319
|
-
createdAt: typeof
|
|
319
|
+
createdAt: typeof decodedBlob.created_at === 'string' ? decodedBlob.created_at : '',
|
|
320
320
|
};
|
|
321
|
-
if (typeof
|
|
322
|
-
result.reasoning =
|
|
321
|
+
if (typeof decodedBlob.reasoning === 'string' && decodedBlob.reasoning.length > 0) {
|
|
322
|
+
result.reasoning = decodedBlob.reasoning;
|
|
323
323
|
}
|
|
324
|
-
if (Array.isArray(
|
|
325
|
-
result.entities =
|
|
324
|
+
if (Array.isArray(decodedBlob.entities)) {
|
|
325
|
+
result.entities = decodedBlob.entities.filter((e) => !!e &&
|
|
326
326
|
typeof e === 'object' &&
|
|
327
327
|
typeof e.name === 'string' &&
|
|
328
328
|
typeof e.type === 'string');
|
|
329
329
|
}
|
|
330
|
-
if (typeof
|
|
331
|
-
result.expiresAt =
|
|
332
|
-
if (typeof
|
|
333
|
-
result.supersededBy =
|
|
334
|
-
if (typeof
|
|
335
|
-
result.id =
|
|
336
|
-
if (typeof
|
|
337
|
-
const ps =
|
|
330
|
+
if (typeof decodedBlob.expires_at === 'string')
|
|
331
|
+
result.expiresAt = decodedBlob.expires_at;
|
|
332
|
+
if (typeof decodedBlob.superseded_by === 'string')
|
|
333
|
+
result.supersededBy = decodedBlob.superseded_by;
|
|
334
|
+
if (typeof decodedBlob.id === 'string')
|
|
335
|
+
result.id = decodedBlob.id;
|
|
336
|
+
if (typeof decodedBlob.pin_status === 'string') {
|
|
337
|
+
const ps = decodedBlob.pin_status;
|
|
338
338
|
if (ps === 'pinned' || ps === 'unpinned') {
|
|
339
339
|
result.pinStatus = ps;
|
|
340
340
|
}
|
|
@@ -342,8 +342,8 @@ export function readV1Blob(decrypted) {
|
|
|
342
342
|
// 3.3.1-rc.22 — pull the embedder identity tag through. Plugin-only
|
|
343
343
|
// field added by `buildCanonicalClaimV1` / `buildV1ClaimBlob` after
|
|
344
344
|
// core validation.
|
|
345
|
-
if (typeof
|
|
346
|
-
result.embeddingModelId =
|
|
345
|
+
if (typeof decodedBlob.embedding_model_id === 'string' && decodedBlob.embedding_model_id.length > 0) {
|
|
346
|
+
result.embeddingModelId = decodedBlob.embedding_model_id;
|
|
347
347
|
}
|
|
348
348
|
return result;
|
|
349
349
|
}
|
|
@@ -351,10 +351,6 @@ export function readV1Blob(decrypted) {
|
|
|
351
351
|
return null;
|
|
352
352
|
}
|
|
353
353
|
}
|
|
354
|
-
// Suppress unused-import lint warnings for VALID_MEMORY_TYPES_V1 — it is
|
|
355
|
-
// exported from extractor.ts for downstream clients and kept in scope here
|
|
356
|
-
// so future v1 helpers can reuse it without re-importing.
|
|
357
|
-
void VALID_MEMORY_TYPES_V1;
|
|
358
354
|
// ---------------------------------------------------------------------------
|
|
359
355
|
// Back-compat alias: buildCanonicalClaimRouted
|
|
360
356
|
//
|
|
@@ -421,56 +417,56 @@ export function resolveAutoResolveMode() {
|
|
|
421
417
|
}
|
|
422
418
|
export function readClaimFromBlob(decryptedJson) {
|
|
423
419
|
try {
|
|
424
|
-
const
|
|
420
|
+
const decodedBlob = JSON.parse(decryptedJson);
|
|
425
421
|
// v1 payload: long-form fields + schema_version "1.x"
|
|
426
|
-
if (typeof
|
|
427
|
-
typeof
|
|
428
|
-
typeof
|
|
429
|
-
|
|
430
|
-
const importance = typeof
|
|
431
|
-
? Math.max(1, Math.min(10, Math.round(
|
|
422
|
+
if (typeof decodedBlob.text === 'string' &&
|
|
423
|
+
typeof decodedBlob.type === 'string' &&
|
|
424
|
+
typeof decodedBlob.schema_version === 'string' &&
|
|
425
|
+
decodedBlob.schema_version.startsWith('1.')) {
|
|
426
|
+
const importance = typeof decodedBlob.importance === 'number'
|
|
427
|
+
? Math.max(1, Math.min(10, Math.round(decodedBlob.importance)))
|
|
432
428
|
: 5;
|
|
433
429
|
return {
|
|
434
|
-
text:
|
|
430
|
+
text: decodedBlob.text,
|
|
435
431
|
importance,
|
|
436
|
-
category: mapTypeToCategory(
|
|
432
|
+
category: mapTypeToCategory(decodedBlob.type),
|
|
437
433
|
metadata: {
|
|
438
|
-
type:
|
|
439
|
-
source: typeof
|
|
440
|
-
scope: typeof
|
|
441
|
-
volatility: typeof
|
|
442
|
-
reasoning: typeof
|
|
434
|
+
type: decodedBlob.type,
|
|
435
|
+
source: typeof decodedBlob.source === 'string' ? decodedBlob.source : 'user-inferred',
|
|
436
|
+
scope: typeof decodedBlob.scope === 'string' ? decodedBlob.scope : 'unspecified',
|
|
437
|
+
volatility: typeof decodedBlob.volatility === 'string' ? decodedBlob.volatility : 'updatable',
|
|
438
|
+
reasoning: typeof decodedBlob.reasoning === 'string' ? decodedBlob.reasoning : undefined,
|
|
443
439
|
// v1.1: surface pin_status verbatim for downstream (recall display +
|
|
444
440
|
// export). Absent ⇒ undefined (receivers treat as "unpinned").
|
|
445
|
-
pin_status: typeof
|
|
441
|
+
pin_status: typeof decodedBlob.pin_status === 'string' ? decodedBlob.pin_status : undefined,
|
|
446
442
|
importance: importance / 10,
|
|
447
|
-
created_at: typeof
|
|
448
|
-
schema_version:
|
|
443
|
+
created_at: typeof decodedBlob.created_at === 'string' ? decodedBlob.created_at : '',
|
|
444
|
+
schema_version: decodedBlob.schema_version,
|
|
449
445
|
},
|
|
450
446
|
};
|
|
451
447
|
}
|
|
452
448
|
// New canonical Claim format: short keys
|
|
453
|
-
if (typeof
|
|
454
|
-
const importance = typeof
|
|
449
|
+
if (typeof decodedBlob.t === 'string' && typeof decodedBlob.c === 'string') {
|
|
450
|
+
const importance = typeof decodedBlob.i === 'number' ? Math.max(1, Math.min(10, Math.round(decodedBlob.i))) : 5;
|
|
455
451
|
return {
|
|
456
|
-
text:
|
|
452
|
+
text: decodedBlob.t,
|
|
457
453
|
importance,
|
|
458
|
-
category:
|
|
454
|
+
category: decodedBlob.c,
|
|
459
455
|
metadata: {
|
|
460
|
-
type:
|
|
456
|
+
type: decodedBlob.c,
|
|
461
457
|
importance: importance / 10,
|
|
462
|
-
source: typeof
|
|
463
|
-
created_at: typeof
|
|
458
|
+
source: typeof decodedBlob.sa === 'string' ? decodedBlob.sa : 'auto-extraction',
|
|
459
|
+
created_at: typeof decodedBlob.ea === 'string' ? decodedBlob.ea : '',
|
|
464
460
|
},
|
|
465
461
|
};
|
|
466
462
|
}
|
|
467
463
|
// Legacy plugin {text, metadata: {importance: 0-1}} format
|
|
468
|
-
if (typeof
|
|
469
|
-
const meta =
|
|
464
|
+
if (typeof decodedBlob.text === 'string') {
|
|
465
|
+
const meta = decodedBlob.metadata ?? {};
|
|
470
466
|
const impFloat = typeof meta.importance === 'number' ? meta.importance : 0.5;
|
|
471
467
|
const importance = Math.max(1, Math.min(10, Math.round(impFloat * 10)));
|
|
472
468
|
return {
|
|
473
|
-
text:
|
|
469
|
+
text: decodedBlob.text,
|
|
474
470
|
importance,
|
|
475
471
|
category: typeof meta.type === 'string' ? meta.type : 'fact',
|
|
476
472
|
metadata: meta,
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TotalReclaw Plugin - Memory Consolidation & Near-Duplicate Detection
|
|
3
|
+
*
|
|
4
|
+
* Provides cross-session / cross-vault deduplication of stored facts using
|
|
5
|
+
* cosine similarity on their embeddings. Unlike semantic-dedup.ts (which
|
|
6
|
+
* handles within-batch dedup at threshold 0.9), this module handles:
|
|
7
|
+
*
|
|
8
|
+
* 1. Store-time dedup — before writing a new fact, check whether a
|
|
9
|
+
* near-duplicate already exists in the vault (findNearDuplicate).
|
|
10
|
+
* 2. Supersede logic — when a near-duplicate is found, decide whether
|
|
11
|
+
* the new fact should replace or be skipped (shouldSupersede).
|
|
12
|
+
* 3. Bulk consolidation — cluster all facts in the vault and identify
|
|
13
|
+
* groups of near-duplicates for cleanup (clusterFacts).
|
|
14
|
+
*
|
|
15
|
+
* Delegates core computation to `@totalreclaw/core` Rust WASM module:
|
|
16
|
+
* `findNearDuplicate`, `shouldSupersede`, and `clusterFacts` all call the
|
|
17
|
+
* corresponding core functions directly.
|
|
18
|
+
*
|
|
19
|
+
* Threshold helpers remain local; their env reads are centralized in
|
|
20
|
+
* entry.ts (env-reading seam, Task 1.3 of the OpenClaw native
|
|
21
|
+
* integration plan, 2026-06-21).
|
|
22
|
+
*/
|
|
23
|
+
import { createRequire } from 'node:module';
|
|
24
|
+
import { envNumber } from '../entry.js';
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Lazy-load WASM core (mirrors claims-helper.ts / contradiction-sync.ts
|
|
27
|
+
// pattern — plays nicely under both the OpenClaw runtime (CJS-ish tsx) and
|
|
28
|
+
// bare Node ESM used by tests).
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
const requireWasm = createRequire(import.meta.url);
|
|
31
|
+
let _wasm = null;
|
|
32
|
+
function getWasm() {
|
|
33
|
+
if (!_wasm)
|
|
34
|
+
_wasm = requireWasm('@totalreclaw/core');
|
|
35
|
+
return _wasm;
|
|
36
|
+
}
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// Configuration
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
/**
|
|
41
|
+
* Get the cosine similarity threshold for store-time dedup.
|
|
42
|
+
*
|
|
43
|
+
* Configurable via TOTALRECLAW_STORE_DEDUP_THRESHOLD env var.
|
|
44
|
+
* Must be a number in [0, 1]. Falls back to 0.85 if invalid or unset.
|
|
45
|
+
*/
|
|
46
|
+
export function getStoreDedupThreshold() {
|
|
47
|
+
return envNumber('TOTALRECLAW_STORE_DEDUP_THRESHOLD', 0.85, { min: 0, max: 1 });
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Get the cosine similarity threshold for bulk consolidation clustering.
|
|
51
|
+
*
|
|
52
|
+
* Configurable via TOTALRECLAW_CONSOLIDATION_THRESHOLD env var.
|
|
53
|
+
* Must be a number in [0, 1]. Falls back to 0.88 if invalid or unset.
|
|
54
|
+
*/
|
|
55
|
+
export function getConsolidationThreshold() {
|
|
56
|
+
return envNumber('TOTALRECLAW_CONSOLIDATION_THRESHOLD', 0.88, { min: 0, max: 1 });
|
|
57
|
+
}
|
|
58
|
+
/** Maximum candidates to compare against during store-time dedup. */
|
|
59
|
+
export const STORE_DEDUP_MAX_CANDIDATES = 200;
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
// Store-time dedup
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
/**
|
|
64
|
+
* Find the best near-duplicate match for a new fact among existing candidates.
|
|
65
|
+
*
|
|
66
|
+
* Compares the new fact's embedding against all candidates using cosine
|
|
67
|
+
* similarity. Returns the candidate with the highest similarity above the
|
|
68
|
+
* threshold, or null if no match is found.
|
|
69
|
+
*
|
|
70
|
+
* Candidates without embeddings are skipped (fail-safe).
|
|
71
|
+
*
|
|
72
|
+
* @param newFactEmbedding - Embedding vector for the new fact
|
|
73
|
+
* @param candidates - Existing facts to compare against
|
|
74
|
+
* @param threshold - Cosine similarity threshold (e.g. 0.85)
|
|
75
|
+
* @returns - Best match above threshold, or null
|
|
76
|
+
*/
|
|
77
|
+
export function findNearDuplicate(newFactEmbedding, candidates, threshold) {
|
|
78
|
+
const wasm = getWasm();
|
|
79
|
+
const existing = candidates
|
|
80
|
+
.filter((c) => c.embedding && c.embedding.length > 0)
|
|
81
|
+
.map((c) => ({ id: c.id, embedding: c.embedding }));
|
|
82
|
+
if (existing.length === 0)
|
|
83
|
+
return null;
|
|
84
|
+
const bestMatchJs = wasm.findBestNearDuplicate(JSON.stringify(newFactEmbedding), JSON.stringify(existing), threshold);
|
|
85
|
+
if (bestMatchJs == null)
|
|
86
|
+
return null;
|
|
87
|
+
const bestMatch = typeof bestMatchJs === 'string' ? JSON.parse(bestMatchJs) : bestMatchJs;
|
|
88
|
+
const matched = candidates.find((c) => c.id === bestMatch.fact_id);
|
|
89
|
+
if (!matched)
|
|
90
|
+
return null;
|
|
91
|
+
return { existingFact: matched, similarity: bestMatch.similarity };
|
|
92
|
+
}
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
// Supersede logic
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
/**
|
|
97
|
+
* Decide whether a new fact should supersede an existing near-duplicate.
|
|
98
|
+
*
|
|
99
|
+
* - Higher importance wins.
|
|
100
|
+
* - Equal importance: new fact supersedes (newer is preferred).
|
|
101
|
+
*
|
|
102
|
+
* Delegates to `@totalreclaw/core` WASM `shouldSupersede`.
|
|
103
|
+
*
|
|
104
|
+
* @param newImportance - Importance score of the new fact
|
|
105
|
+
* @param existingFact - The existing near-duplicate candidate
|
|
106
|
+
* @returns - 'supersede' if new fact should replace, 'skip' otherwise
|
|
107
|
+
*/
|
|
108
|
+
export function shouldSupersede(newImportance, existingFact) {
|
|
109
|
+
const wasm = getWasm();
|
|
110
|
+
return wasm.shouldSupersede(newImportance, existingFact.importance) ? 'supersede' : 'skip';
|
|
111
|
+
}
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
// Bulk consolidation
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
/**
|
|
116
|
+
* Cluster facts by semantic similarity using greedy single-pass clustering.
|
|
117
|
+
*
|
|
118
|
+
* Delegates to `@totalreclaw/core` WASM `clusterFacts` which performs the
|
|
119
|
+
* same greedy single-pass algorithm and representative selection. The WASM
|
|
120
|
+
* function returns ID-only clusters; this wrapper maps IDs back to full
|
|
121
|
+
* `DecryptedCandidate` objects for callers.
|
|
122
|
+
*
|
|
123
|
+
* Only returns clusters that have duplicates (i.e. more than one member).
|
|
124
|
+
* Facts without embeddings are not clustered.
|
|
125
|
+
*
|
|
126
|
+
* @param facts - All facts to cluster
|
|
127
|
+
* @param threshold - Cosine similarity threshold (e.g. 0.88)
|
|
128
|
+
* @returns - Clusters with duplicates (representative + duplicates)
|
|
129
|
+
*/
|
|
130
|
+
export function clusterFacts(facts, threshold) {
|
|
131
|
+
const wasm = getWasm();
|
|
132
|
+
// Build ConsolidationCandidate JSON for WASM (snake_case fields).
|
|
133
|
+
const wasmCandidates = facts
|
|
134
|
+
.filter((f) => f.embedding && f.embedding.length > 0)
|
|
135
|
+
.map((f) => ({
|
|
136
|
+
id: f.id,
|
|
137
|
+
text: f.text,
|
|
138
|
+
embedding: f.embedding,
|
|
139
|
+
importance: f.importance,
|
|
140
|
+
decay_score: f.decayScore,
|
|
141
|
+
created_at: f.createdAt,
|
|
142
|
+
version: f.version,
|
|
143
|
+
}));
|
|
144
|
+
if (wasmCandidates.length === 0)
|
|
145
|
+
return [];
|
|
146
|
+
const clustersJs = wasm.clusterFacts(JSON.stringify(wasmCandidates), threshold);
|
|
147
|
+
// WASM returns a JSON string: [{ representative: string, duplicates: string[] }]
|
|
148
|
+
const wasmClusters = typeof clustersJs === 'string' ? JSON.parse(clustersJs) : clustersJs;
|
|
149
|
+
// Build a lookup map for fast ID -> DecryptedCandidate resolution.
|
|
150
|
+
const byId = new Map();
|
|
151
|
+
for (const f of facts)
|
|
152
|
+
byId.set(f.id, f);
|
|
153
|
+
// Map ID-only clusters back to full DecryptedCandidate objects.
|
|
154
|
+
// Filter out singleton clusters (no duplicates) to match the pre-WASM
|
|
155
|
+
// plugin contract — callers rely on `clusters.length === 0` when nothing
|
|
156
|
+
// duplicates anything.
|
|
157
|
+
const clusters = [];
|
|
158
|
+
for (const wc of wasmClusters) {
|
|
159
|
+
const rep = byId.get(wc.representative);
|
|
160
|
+
if (!rep)
|
|
161
|
+
continue;
|
|
162
|
+
const dups = wc.duplicates
|
|
163
|
+
.map((id) => byId.get(id))
|
|
164
|
+
.filter((d) => d !== undefined);
|
|
165
|
+
if (dups.length > 0) {
|
|
166
|
+
clusters.push({ representative: rep, duplicates: dups });
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return clusters;
|
|
170
|
+
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Uses LLM calls to extract atomic facts from conversation messages.
|
|
5
5
|
* Matches the extraction prompts described in SKILL.md.
|
|
6
6
|
*/
|
|
7
|
-
import { chatCompletion, resolveLLMConfig } from '
|
|
7
|
+
import { chatCompletion, resolveLLMConfig } from '../llm/llm-client.js';
|
|
8
8
|
// ---------------------------------------------------------------------------
|
|
9
9
|
// Memory Taxonomy v1 — the 6 canonical memory types. Single source of truth.
|
|
10
10
|
//
|
|
@@ -461,7 +461,7 @@ export function parseFactsResponseForCompaction(response, logger) {
|
|
|
461
461
|
}
|
|
462
462
|
}
|
|
463
463
|
if (recoveryUsed === 'bracket-scan') {
|
|
464
|
-
logger?.
|
|
464
|
+
logger?.debug?.(`parseFactsResponseForCompaction: recovered JSON via bracket-scan fallback`);
|
|
465
465
|
}
|
|
466
466
|
if (!parsed || typeof parsed !== 'object') {
|
|
467
467
|
logger?.warn?.(`parseFactsResponseForCompaction: could not parse LLM output as JSON object. Preview: ${JSON.stringify(originalPreview)}`);
|
|
@@ -514,7 +514,7 @@ export function parseFactsResponseForCompaction(response, logger) {
|
|
|
514
514
|
if (valid.length > 0)
|
|
515
515
|
entities = valid;
|
|
516
516
|
}
|
|
517
|
-
const
|
|
517
|
+
const extractedFact = {
|
|
518
518
|
text: String(f.text).slice(0, 512),
|
|
519
519
|
type,
|
|
520
520
|
source,
|
|
@@ -526,8 +526,8 @@ export function parseFactsResponseForCompaction(response, logger) {
|
|
|
526
526
|
confidence: normalizeConfidence(f.confidence),
|
|
527
527
|
};
|
|
528
528
|
if (entities)
|
|
529
|
-
|
|
530
|
-
return
|
|
529
|
+
extractedFact.entities = entities;
|
|
530
|
+
return extractedFact;
|
|
531
531
|
})
|
|
532
532
|
// Reject illegal type:summary + source:user
|
|
533
533
|
.filter((f) => !(f.type === 'summary' && f.source === 'user'))
|
|
@@ -601,7 +601,7 @@ export async function extractFactsForCompaction(rawMessages, existingMemories, l
|
|
|
601
601
|
logger?.info?.('extractFactsForCompaction: chatCompletion returned null/empty response');
|
|
602
602
|
return [];
|
|
603
603
|
}
|
|
604
|
-
logger?.
|
|
604
|
+
logger?.debug?.(`extractFactsForCompaction: LLM returned ${response.length} chars; handing to parseFactsResponseForCompaction`);
|
|
605
605
|
let facts = parseFactsResponseForCompaction(response, logger);
|
|
606
606
|
// v1 provenance filter (tag-don't-drop). Uses importance >= 5 floor because
|
|
607
607
|
// the filter's own floor is 5 in lax mode, matching compaction semantics.
|
|
@@ -617,7 +617,7 @@ export async function extractFactsForCompaction(rawMessages, existingMemories, l
|
|
|
617
617
|
const oldImportance = f.importance;
|
|
618
618
|
const effectiveBump = f.importance >= 8 ? Math.min(bump, 1) : bump;
|
|
619
619
|
f.importance = Math.min(10, f.importance + effectiveBump);
|
|
620
|
-
logger?.
|
|
620
|
+
logger?.debug?.(`extractFactsForCompaction: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`);
|
|
621
621
|
}
|
|
622
622
|
}
|
|
623
623
|
return facts;
|
|
@@ -1020,7 +1020,7 @@ export function parseMergedResponseV1(response, logger) {
|
|
|
1020
1020
|
}
|
|
1021
1021
|
}
|
|
1022
1022
|
if (recoveryUsed === 'bracket-scan') {
|
|
1023
|
-
logger?.
|
|
1023
|
+
logger?.debug?.(`parseFactsResponse: recovered JSON via bracket-scan fallback`);
|
|
1024
1024
|
}
|
|
1025
1025
|
if (!parsed || typeof parsed !== 'object') {
|
|
1026
1026
|
logger?.warn?.(`parseFactsResponse: could not parse LLM output as JSON. Preview: ${JSON.stringify(originalPreview)}`);
|
|
@@ -1030,25 +1030,25 @@ export function parseMergedResponseV1(response, logger) {
|
|
|
1030
1030
|
// a bare JSON array of fact objects (legacy / test fixture shape). The
|
|
1031
1031
|
// bare array is wrapped as { topics: [], facts: [...] } so the downstream
|
|
1032
1032
|
// logic stays uniform. A single fact object (no wrapper) is also wrapped.
|
|
1033
|
-
let
|
|
1033
|
+
let wrapper;
|
|
1034
1034
|
if (Array.isArray(parsed)) {
|
|
1035
|
-
|
|
1035
|
+
wrapper = { topics: [], facts: parsed };
|
|
1036
1036
|
}
|
|
1037
1037
|
else if (typeof parsed.facts === 'undefined' &&
|
|
1038
1038
|
typeof parsed.text === 'string') {
|
|
1039
1039
|
// Single fact object, not a merged wrapper.
|
|
1040
|
-
|
|
1040
|
+
wrapper = { topics: [], facts: [parsed] };
|
|
1041
1041
|
}
|
|
1042
1042
|
else {
|
|
1043
|
-
|
|
1043
|
+
wrapper = parsed;
|
|
1044
1044
|
}
|
|
1045
|
-
const rawTopics =
|
|
1045
|
+
const rawTopics = wrapper.topics;
|
|
1046
1046
|
const topics = Array.isArray(rawTopics)
|
|
1047
1047
|
? rawTopics
|
|
1048
1048
|
.filter((t) => typeof t === 'string' && t.length > 0)
|
|
1049
1049
|
.slice(0, 3)
|
|
1050
1050
|
: [];
|
|
1051
|
-
const rawFacts =
|
|
1051
|
+
const rawFacts = wrapper.facts;
|
|
1052
1052
|
if (!Array.isArray(rawFacts))
|
|
1053
1053
|
return { topics, facts: [] };
|
|
1054
1054
|
const validActions = ['ADD', 'UPDATE', 'DELETE', 'NOOP'];
|
|
@@ -1334,10 +1334,10 @@ export async function extractFacts(rawMessages, mode, existingMemories, profileC
|
|
|
1334
1334
|
logger?.info?.('extractFacts: chatCompletion returned null/empty response');
|
|
1335
1335
|
return [];
|
|
1336
1336
|
}
|
|
1337
|
-
logger?.
|
|
1337
|
+
logger?.debug?.(`extractFacts: LLM returned ${response.length} chars; parsing merged response`);
|
|
1338
1338
|
const { topics, facts: rawFacts } = parseMergedResponseV1(response, logger);
|
|
1339
1339
|
if (topics.length > 0) {
|
|
1340
|
-
logger?.
|
|
1340
|
+
logger?.debug?.(`extractFacts: topics = ${JSON.stringify(topics)}`);
|
|
1341
1341
|
}
|
|
1342
1342
|
// Provenance filter (tag-don't-drop)
|
|
1343
1343
|
let facts = applyProvenanceFilterLax(rawFacts, conversationText);
|
|
@@ -1352,7 +1352,7 @@ export async function extractFacts(rawMessages, mode, existingMemories, profileC
|
|
|
1352
1352
|
const oldImportance = f.importance;
|
|
1353
1353
|
const effectiveBump = f.importance >= 8 ? Math.min(bump, 1) : bump;
|
|
1354
1354
|
f.importance = Math.min(10, f.importance + effectiveBump);
|
|
1355
|
-
logger?.
|
|
1355
|
+
logger?.debug?.(`extractFacts: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`);
|
|
1356
1356
|
}
|
|
1357
1357
|
}
|
|
1358
1358
|
return facts;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// Importance filter for auto-extraction. Extracted from index.ts.
|
|
3
|
+
// Pure over its inputs (threshold derives from CONFIG); no plugin session state.
|
|
4
|
+
// ---------------------------------------------------------------------------
|
|
5
|
+
import { CONFIG } from '../config.js';
|
|
6
|
+
const MIN_IMPORTANCE_THRESHOLD = CONFIG.minImportance;
|
|
7
|
+
/**
|
|
8
|
+
* Filter extracted facts by importance threshold.
|
|
9
|
+
* Facts with importance < MIN_IMPORTANCE_THRESHOLD are dropped.
|
|
10
|
+
* Facts with missing/undefined importance are treated as importance=5 (kept).
|
|
11
|
+
*/
|
|
12
|
+
export function filterByImportance(facts, logger) {
|
|
13
|
+
const kept = [];
|
|
14
|
+
let dropped = 0;
|
|
15
|
+
for (const fact of facts) {
|
|
16
|
+
const importance = fact.importance ?? 5;
|
|
17
|
+
if (importance >= MIN_IMPORTANCE_THRESHOLD) {
|
|
18
|
+
kept.push(fact);
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
dropped++;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// Phase 2.2.5: always log the filter outcome so the agent_end path can
|
|
25
|
+
// distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
|
|
26
|
+
// below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
|
|
27
|
+
// this only logged on drops, which made empty-input invisible.
|
|
28
|
+
if (facts.length === 0) {
|
|
29
|
+
logger.info('Importance filter: input=0 (nothing to filter)');
|
|
30
|
+
}
|
|
31
|
+
else if (dropped > 0) {
|
|
32
|
+
logger.info(`Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
logger.info(`Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`);
|
|
36
|
+
}
|
|
37
|
+
return { kept, dropped };
|
|
38
|
+
}
|
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
* cosineSimilarity and extractor for the ExtractedFact type) so it can
|
|
11
11
|
* be tested without pulling in the full plugin dependency graph.
|
|
12
12
|
*/
|
|
13
|
-
import { cosineSimilarity } from '
|
|
14
|
-
import { envNumber } from '
|
|
13
|
+
import { cosineSimilarity } from '../embedding/reranker.js';
|
|
14
|
+
import { envNumber } from '../entry.js';
|
|
15
15
|
// ---------------------------------------------------------------------------
|
|
16
16
|
// Configuration
|
|
17
17
|
// ---------------------------------------------------------------------------
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* Env read is centralized in entry.ts (env-reading seam, Task 1.3 of the
|
|
24
24
|
* OpenClaw native integration plan, 2026-06-21).
|
|
25
25
|
*/
|
|
26
|
-
import { isGnosisBatchEnabledAtBoot } from '
|
|
26
|
+
import { isGnosisBatchEnabledAtBoot } from '../entry.js';
|
|
27
27
|
const GNOSIS_CHAIN_ID = 100;
|
|
28
28
|
export function shouldBatchOnChain(chainId) {
|
|
29
29
|
if (!isGnosisBatchEnabledAtBoot())
|