@totalreclaw/totalreclaw 3.3.12 → 3.3.13-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +25 -8
- package/{billing-cache.ts → billing/billing-cache.ts} +1 -1
- package/{relay-headers.ts → billing/relay-headers.ts} +1 -1
- package/{tr-cli-export-helper.ts → cli/tr-cli-export-helper.ts} +5 -5
- package/{tr-cli.ts → cli/tr-cli.ts} +20 -20
- package/{contradiction-sync.ts → contradiction/contradiction-sync.ts} +13 -13
- package/{digest-sync.ts → digest/digest-sync.ts} +32 -2
- package/dist/{billing-cache.js → billing/billing-cache.js} +1 -1
- package/dist/{relay-headers.js → billing/relay-headers.js} +1 -1
- package/dist/{tr-cli-export-helper.js → cli/tr-cli-export-helper.js} +5 -5
- package/dist/{tr-cli.js → cli/tr-cli.js} +19 -19
- package/dist/{contradiction-sync.js → contradiction/contradiction-sync.js} +13 -13
- package/dist/{digest-sync.js → digest/digest-sync.js} +32 -2
- package/dist/{download-ux.js → embedding/download-ux.js} +1 -1
- package/dist/{reranker.js → embedding/reranker.js} +13 -4
- package/dist/{claims-helper.js → extraction/claims-helper.js} +54 -58
- package/dist/extraction/consolidation.js +170 -0
- package/dist/{extractor.js → extraction/extractor.js} +17 -17
- package/dist/extraction/importance-filter.js +38 -0
- package/dist/{semantic-dedup.js → extraction/semantic-dedup.js} +2 -2
- package/dist/{batch-gate.js → import/batch-gate.js} +1 -1
- package/dist/import/import-runtime.js +692 -0
- package/dist/import-adapters/chatgpt-adapter.js +7 -7
- package/dist/import-adapters/gemini-adapter.js +13 -13
- package/dist/import-adapters/mem0-adapter.js +10 -10
- package/dist/index.js +111 -1094
- package/dist/{llm-client.js → llm/llm-client.js} +2 -2
- package/dist/{hot-cache-wrapper.js → memory/hot-cache-wrapper.js} +5 -5
- package/dist/{memory-runtime.js → memory/memory-runtime.js} +1 -1
- package/dist/{pin.js → memory/pin.js} +18 -18
- package/dist/{retype-setscope.js → memory/retype-setscope.js} +19 -19
- package/dist/{credential-provider.js → pairing/credential-provider.js} +2 -2
- package/dist/{first-run.js → pairing/first-run.js} +1 -1
- package/dist/{onboarding-cli.js → pairing/onboarding-cli.js} +20 -20
- package/dist/{pair-cli-relay.js → pairing/pair-cli-relay.js} +10 -10
- package/dist/{pair-crypto.js → pairing/pair-crypto.js} +1 -1
- package/dist/{pair-http.js → pairing/pair-http.js} +3 -3
- package/dist/{pair-remote-client.js → pairing/pair-remote-client.js} +10 -10
- package/dist/{pair-session-store.js → pairing/pair-session-store.js} +4 -4
- package/dist/runtime/config-schema.js +59 -0
- package/dist/runtime/format-helpers.js +257 -0
- package/dist/runtime/types.js +11 -0
- package/dist/{confirm-indexed.js → subgraph/confirm-indexed.js} +1 -1
- package/dist/{subgraph-search.js → subgraph/subgraph-search.js} +14 -14
- package/dist/{subgraph-store.js → subgraph/subgraph-store.js} +6 -6
- package/{download-ux.ts → embedding/download-ux.ts} +1 -1
- package/{reranker.ts → embedding/reranker.ts} +13 -4
- package/{claims-helper.ts → extraction/claims-helper.ts} +52 -60
- package/{consolidation.ts → extraction/consolidation.ts} +63 -154
- package/{extractor.ts → extraction/extractor.ts} +18 -17
- package/extraction/importance-filter.ts +50 -0
- package/{semantic-dedup.ts → extraction/semantic-dedup.ts} +2 -2
- package/{batch-gate.ts → import/batch-gate.ts} +1 -1
- package/import/import-runtime.ts +853 -0
- package/import-adapters/chatgpt-adapter.ts +7 -7
- package/import-adapters/gemini-adapter.ts +13 -13
- package/import-adapters/mem0-adapter.ts +10 -10
- package/index.ts +129 -1377
- package/{llm-client.ts → llm/llm-client.ts} +2 -2
- package/{hot-cache-wrapper.ts → memory/hot-cache-wrapper.ts} +5 -5
- package/{memory-runtime.ts → memory/memory-runtime.ts} +1 -1
- package/{pin.ts → memory/pin.ts} +20 -20
- package/{retype-setscope.ts → memory/retype-setscope.ts} +21 -21
- package/{tool-gating.ts → memory/tool-gating.ts} +1 -1
- package/package.json +6 -6
- package/{credential-provider.ts → pairing/credential-provider.ts} +2 -2
- package/{first-run.ts → pairing/first-run.ts} +1 -1
- package/{onboarding-cli.ts → pairing/onboarding-cli.ts} +20 -20
- package/{pair-cli-relay.ts → pairing/pair-cli-relay.ts} +10 -10
- package/{pair-crypto.ts → pairing/pair-crypto.ts} +1 -1
- package/{pair-http.ts → pairing/pair-http.ts} +3 -3
- package/{pair-remote-client.ts → pairing/pair-remote-client.ts} +10 -10
- package/{pair-session-store.ts → pairing/pair-session-store.ts} +4 -4
- package/runtime/config-schema.ts +64 -0
- package/runtime/format-helpers.ts +293 -0
- package/runtime/types.ts +118 -0
- package/skill.json +44 -10
- package/{confirm-indexed.ts → subgraph/confirm-indexed.ts} +1 -1
- package/{subgraph-search.ts → subgraph/subgraph-search.ts} +14 -14
- package/{subgraph-store.ts → subgraph/subgraph-store.ts} +10 -10
- package/dist/consolidation.js +0 -249
- /package/{api-client.ts → billing/api-client.ts} +0 -0
- /package/{gateway-url.ts → billing/gateway-url.ts} +0 -0
- /package/{inbound-user-tracker.ts → billing/inbound-user-tracker.ts} +0 -0
- /package/{relay.ts → billing/relay.ts} +0 -0
- /package/{crypto.ts → crypto/crypto.ts} +0 -0
- /package/{vault-crypto.ts → crypto/vault-crypto.ts} +0 -0
- /package/dist/{api-client.js → billing/api-client.js} +0 -0
- /package/dist/{gateway-url.js → billing/gateway-url.js} +0 -0
- /package/dist/{inbound-user-tracker.js → billing/inbound-user-tracker.js} +0 -0
- /package/dist/{relay.js → billing/relay.js} +0 -0
- /package/dist/{crypto.js → crypto/crypto.js} +0 -0
- /package/dist/{vault-crypto.js → crypto/vault-crypto.js} +0 -0
- /package/dist/{embedder-cache.js → embedding/embedder-cache.js} +0 -0
- /package/dist/{embedder-loader.js → embedding/embedder-loader.js} +0 -0
- /package/dist/{embedder-network.js → embedding/embedder-network.js} +0 -0
- /package/dist/{embedding.js → embedding/embedding.js} +0 -0
- /package/dist/{lsh.js → embedding/lsh.js} +0 -0
- /package/dist/{import-state-manager.js → import/import-state-manager.js} +0 -0
- /package/dist/{llm-profile-reader.js → llm/llm-profile-reader.js} +0 -0
- /package/dist/{native-memory.js → memory/native-memory.js} +0 -0
- /package/dist/{tool-gating.js → memory/tool-gating.js} +0 -0
- /package/dist/{tools.js → memory/tools.js} +0 -0
- /package/dist/{generate-mnemonic.js → pairing/generate-mnemonic.js} +0 -0
- /package/dist/{pair-cli.js → pairing/pair-cli.js} +0 -0
- /package/dist/{pair-page.js → pairing/pair-page.js} +0 -0
- /package/dist/{pair-qr.js → pairing/pair-qr.js} +0 -0
- /package/dist/{restart-auth.js → pairing/restart-auth.js} +0 -0
- /package/dist/{qa-bug-report.js → setup/qa-bug-report.js} +0 -0
- /package/dist/{skill-register.js → setup/skill-register.js} +0 -0
- /package/dist/{trajectory-poller.js → subgraph/trajectory-poller.js} +0 -0
- /package/{embedder-cache.ts → embedding/embedder-cache.ts} +0 -0
- /package/{embedder-loader.ts → embedding/embedder-loader.ts} +0 -0
- /package/{embedder-network.ts → embedding/embedder-network.ts} +0 -0
- /package/{embedding.ts → embedding/embedding.ts} +0 -0
- /package/{lsh.ts → embedding/lsh.ts} +0 -0
- /package/{import-state-manager.ts → import/import-state-manager.ts} +0 -0
- /package/{llm-profile-reader.ts → llm/llm-profile-reader.ts} +0 -0
- /package/{native-memory.ts → memory/native-memory.ts} +0 -0
- /package/{tools.ts → memory/tools.ts} +0 -0
- /package/{generate-mnemonic.ts → pairing/generate-mnemonic.ts} +0 -0
- /package/{pair-cli.ts → pairing/pair-cli.ts} +0 -0
- /package/{pair-page.ts → pairing/pair-page.ts} +0 -0
- /package/{pair-qr.ts → pairing/pair-qr.ts} +0 -0
- /package/{restart-auth.ts → pairing/restart-auth.ts} +0 -0
- /package/{qa-bug-report.ts → setup/qa-bug-report.ts} +0 -0
- /package/{skill-register.ts → setup/skill-register.ts} +0 -0
- /package/{trajectory-poller.ts → subgraph/trajectory-poller.ts} +0 -0
|
@@ -16,21 +16,18 @@ import type {
|
|
|
16
16
|
ExtractedFact,
|
|
17
17
|
MemoryType,
|
|
18
18
|
MemoryTypeV0,
|
|
19
|
-
MemoryTypeV1,
|
|
20
19
|
MemoryScope,
|
|
21
20
|
MemorySource,
|
|
22
21
|
MemoryVolatility,
|
|
23
22
|
} from './extractor.js';
|
|
24
23
|
import {
|
|
25
24
|
isValidMemoryType,
|
|
26
|
-
isValidMemoryTypeV1,
|
|
27
25
|
V0_TO_V1_TYPE,
|
|
28
26
|
VALID_MEMORY_SCOPES,
|
|
29
27
|
VALID_MEMORY_SOURCES,
|
|
30
28
|
VALID_MEMORY_VOLATILITIES,
|
|
31
|
-
VALID_MEMORY_TYPES_V1,
|
|
32
29
|
} from './extractor.js';
|
|
33
|
-
import { envStringLower } from '
|
|
30
|
+
import { envStringLower } from '../entry.js';
|
|
34
31
|
|
|
35
32
|
// Lazy-load WASM. We use createRequire so this module loads cleanly under
|
|
36
33
|
// both the OpenClaw runtime (CJS-ish tsx) and bare Node ESM (used by tests).
|
|
@@ -163,7 +160,7 @@ export const V1_SCHEMA_VERSION = '1.0' as const;
|
|
|
163
160
|
export type PinStatus = 'pinned' | 'unpinned';
|
|
164
161
|
|
|
165
162
|
export interface BuildClaimV1Input {
|
|
166
|
-
/** The extracted fact in v1 shape. Must have `type` as a
|
|
163
|
+
/** The extracted fact in v1 shape. Must have `type` as a MemoryType token. */
|
|
167
164
|
fact: ExtractedFact;
|
|
168
165
|
/** Final importance after any store-time dedup adjustment. 1-10. */
|
|
169
166
|
importance: number;
|
|
@@ -453,7 +450,7 @@ export function isV1Blob(decrypted: string): boolean {
|
|
|
453
450
|
*/
|
|
454
451
|
export interface V1BlobReadResult {
|
|
455
452
|
text: string;
|
|
456
|
-
type:
|
|
453
|
+
type: MemoryType;
|
|
457
454
|
source: MemorySource;
|
|
458
455
|
scope: MemoryScope;
|
|
459
456
|
volatility: MemoryVolatility;
|
|
@@ -481,34 +478,34 @@ export interface V1BlobReadResult {
|
|
|
481
478
|
|
|
482
479
|
export function readV1Blob(decrypted: string): V1BlobReadResult | null {
|
|
483
480
|
try {
|
|
484
|
-
const
|
|
485
|
-
if (typeof
|
|
481
|
+
const decodedBlob = JSON.parse(decrypted) as Record<string, unknown>;
|
|
482
|
+
if (typeof decodedBlob.schema_version !== 'string' || !decodedBlob.schema_version.startsWith('1.')) {
|
|
486
483
|
return null;
|
|
487
484
|
}
|
|
488
485
|
|
|
489
|
-
const text = typeof
|
|
490
|
-
const rawType = typeof
|
|
491
|
-
const type:
|
|
486
|
+
const text = typeof decodedBlob.text === 'string' ? decodedBlob.text : '';
|
|
487
|
+
const rawType = typeof decodedBlob.type === 'string' ? decodedBlob.type : 'claim';
|
|
488
|
+
const type: MemoryType = isValidMemoryType(rawType) ? rawType : 'claim';
|
|
492
489
|
|
|
493
|
-
const rawSource = typeof
|
|
490
|
+
const rawSource = typeof decodedBlob.source === 'string' ? decodedBlob.source : 'user-inferred';
|
|
494
491
|
const source: MemorySource = (VALID_MEMORY_SOURCES as readonly string[]).includes(rawSource)
|
|
495
492
|
? (rawSource as MemorySource)
|
|
496
493
|
: 'user-inferred';
|
|
497
494
|
|
|
498
|
-
const rawScope = typeof
|
|
495
|
+
const rawScope = typeof decodedBlob.scope === 'string' ? decodedBlob.scope : 'unspecified';
|
|
499
496
|
const scope: MemoryScope = (VALID_MEMORY_SCOPES as readonly string[]).includes(rawScope)
|
|
500
497
|
? (rawScope as MemoryScope)
|
|
501
498
|
: 'unspecified';
|
|
502
499
|
|
|
503
|
-
const rawVolatility = typeof
|
|
500
|
+
const rawVolatility = typeof decodedBlob.volatility === 'string' ? decodedBlob.volatility : 'updatable';
|
|
504
501
|
const volatility: MemoryVolatility = (VALID_MEMORY_VOLATILITIES as readonly string[]).includes(rawVolatility)
|
|
505
502
|
? (rawVolatility as MemoryVolatility)
|
|
506
503
|
: 'updatable';
|
|
507
504
|
|
|
508
|
-
const impRaw = typeof
|
|
505
|
+
const impRaw = typeof decodedBlob.importance === 'number' ? decodedBlob.importance : 5;
|
|
509
506
|
const importance = Math.max(1, Math.min(10, Math.round(impRaw)));
|
|
510
507
|
|
|
511
|
-
const confRaw = typeof
|
|
508
|
+
const confRaw = typeof decodedBlob.confidence === 'number' ? decodedBlob.confidence : 0.85;
|
|
512
509
|
const confidence = Math.max(0, Math.min(1, confRaw));
|
|
513
510
|
|
|
514
511
|
const result: V1BlobReadResult = {
|
|
@@ -519,14 +516,14 @@ export function readV1Blob(decrypted: string): V1BlobReadResult | null {
|
|
|
519
516
|
volatility,
|
|
520
517
|
importance,
|
|
521
518
|
confidence,
|
|
522
|
-
createdAt: typeof
|
|
519
|
+
createdAt: typeof decodedBlob.created_at === 'string' ? decodedBlob.created_at : '',
|
|
523
520
|
};
|
|
524
521
|
|
|
525
|
-
if (typeof
|
|
526
|
-
result.reasoning =
|
|
522
|
+
if (typeof decodedBlob.reasoning === 'string' && decodedBlob.reasoning.length > 0) {
|
|
523
|
+
result.reasoning = decodedBlob.reasoning;
|
|
527
524
|
}
|
|
528
|
-
if (Array.isArray(
|
|
529
|
-
result.entities = (
|
|
525
|
+
if (Array.isArray(decodedBlob.entities)) {
|
|
526
|
+
result.entities = (decodedBlob.entities as unknown[]).filter(
|
|
530
527
|
(e): e is { name: string; type: string; role?: string } =>
|
|
531
528
|
!!e &&
|
|
532
529
|
typeof e === 'object' &&
|
|
@@ -534,11 +531,11 @@ export function readV1Blob(decrypted: string): V1BlobReadResult | null {
|
|
|
534
531
|
typeof (e as { type?: unknown }).type === 'string',
|
|
535
532
|
) as Array<{ name: string; type: string; role?: string }>;
|
|
536
533
|
}
|
|
537
|
-
if (typeof
|
|
538
|
-
if (typeof
|
|
539
|
-
if (typeof
|
|
540
|
-
if (typeof
|
|
541
|
-
const ps =
|
|
534
|
+
if (typeof decodedBlob.expires_at === 'string') result.expiresAt = decodedBlob.expires_at;
|
|
535
|
+
if (typeof decodedBlob.superseded_by === 'string') result.supersededBy = decodedBlob.superseded_by;
|
|
536
|
+
if (typeof decodedBlob.id === 'string') result.id = decodedBlob.id;
|
|
537
|
+
if (typeof decodedBlob.pin_status === 'string') {
|
|
538
|
+
const ps = decodedBlob.pin_status;
|
|
542
539
|
if (ps === 'pinned' || ps === 'unpinned') {
|
|
543
540
|
result.pinStatus = ps;
|
|
544
541
|
}
|
|
@@ -546,8 +543,8 @@ export function readV1Blob(decrypted: string): V1BlobReadResult | null {
|
|
|
546
543
|
// 3.3.1-rc.22 — pull the embedder identity tag through. Plugin-only
|
|
547
544
|
// field added by `buildCanonicalClaimV1` / `buildV1ClaimBlob` after
|
|
548
545
|
// core validation.
|
|
549
|
-
if (typeof
|
|
550
|
-
result.embeddingModelId =
|
|
546
|
+
if (typeof decodedBlob.embedding_model_id === 'string' && decodedBlob.embedding_model_id.length > 0) {
|
|
547
|
+
result.embeddingModelId = decodedBlob.embedding_model_id;
|
|
551
548
|
}
|
|
552
549
|
|
|
553
550
|
return result;
|
|
@@ -556,11 +553,6 @@ export function readV1Blob(decrypted: string): V1BlobReadResult | null {
|
|
|
556
553
|
}
|
|
557
554
|
}
|
|
558
555
|
|
|
559
|
-
// Suppress unused-import lint warnings for VALID_MEMORY_TYPES_V1 — it is
|
|
560
|
-
// exported from extractor.ts for downstream clients and kept in scope here
|
|
561
|
-
// so future v1 helpers can reuse it without re-importing.
|
|
562
|
-
void VALID_MEMORY_TYPES_V1;
|
|
563
|
-
|
|
564
556
|
// ---------------------------------------------------------------------------
|
|
565
557
|
// Back-compat alias: buildCanonicalClaimRouted
|
|
566
558
|
//
|
|
@@ -662,60 +654,60 @@ export interface BlobReadResult {
|
|
|
662
654
|
|
|
663
655
|
export function readClaimFromBlob(decryptedJson: string): BlobReadResult {
|
|
664
656
|
try {
|
|
665
|
-
const
|
|
657
|
+
const decodedBlob = JSON.parse(decryptedJson) as Record<string, unknown>;
|
|
666
658
|
|
|
667
659
|
// v1 payload: long-form fields + schema_version "1.x"
|
|
668
660
|
if (
|
|
669
|
-
typeof
|
|
670
|
-
typeof
|
|
671
|
-
typeof
|
|
672
|
-
|
|
661
|
+
typeof decodedBlob.text === 'string' &&
|
|
662
|
+
typeof decodedBlob.type === 'string' &&
|
|
663
|
+
typeof decodedBlob.schema_version === 'string' &&
|
|
664
|
+
decodedBlob.schema_version.startsWith('1.')
|
|
673
665
|
) {
|
|
674
|
-
const importance = typeof
|
|
675
|
-
? Math.max(1, Math.min(10, Math.round(
|
|
666
|
+
const importance = typeof decodedBlob.importance === 'number'
|
|
667
|
+
? Math.max(1, Math.min(10, Math.round(decodedBlob.importance)))
|
|
676
668
|
: 5;
|
|
677
669
|
return {
|
|
678
|
-
text:
|
|
670
|
+
text: decodedBlob.text,
|
|
679
671
|
importance,
|
|
680
|
-
category: mapTypeToCategory(
|
|
672
|
+
category: mapTypeToCategory(decodedBlob.type as MemoryType),
|
|
681
673
|
metadata: {
|
|
682
|
-
type:
|
|
683
|
-
source: typeof
|
|
684
|
-
scope: typeof
|
|
685
|
-
volatility: typeof
|
|
686
|
-
reasoning: typeof
|
|
674
|
+
type: decodedBlob.type,
|
|
675
|
+
source: typeof decodedBlob.source === 'string' ? decodedBlob.source : 'user-inferred',
|
|
676
|
+
scope: typeof decodedBlob.scope === 'string' ? decodedBlob.scope : 'unspecified',
|
|
677
|
+
volatility: typeof decodedBlob.volatility === 'string' ? decodedBlob.volatility : 'updatable',
|
|
678
|
+
reasoning: typeof decodedBlob.reasoning === 'string' ? decodedBlob.reasoning : undefined,
|
|
687
679
|
// v1.1: surface pin_status verbatim for downstream (recall display +
|
|
688
680
|
// export). Absent ⇒ undefined (receivers treat as "unpinned").
|
|
689
|
-
pin_status: typeof
|
|
681
|
+
pin_status: typeof decodedBlob.pin_status === 'string' ? decodedBlob.pin_status : undefined,
|
|
690
682
|
importance: importance / 10,
|
|
691
|
-
created_at: typeof
|
|
692
|
-
schema_version:
|
|
683
|
+
created_at: typeof decodedBlob.created_at === 'string' ? decodedBlob.created_at : '',
|
|
684
|
+
schema_version: decodedBlob.schema_version,
|
|
693
685
|
},
|
|
694
686
|
};
|
|
695
687
|
}
|
|
696
688
|
|
|
697
689
|
// New canonical Claim format: short keys
|
|
698
|
-
if (typeof
|
|
699
|
-
const importance = typeof
|
|
690
|
+
if (typeof decodedBlob.t === 'string' && typeof decodedBlob.c === 'string') {
|
|
691
|
+
const importance = typeof decodedBlob.i === 'number' ? Math.max(1, Math.min(10, Math.round(decodedBlob.i))) : 5;
|
|
700
692
|
return {
|
|
701
|
-
text:
|
|
693
|
+
text: decodedBlob.t,
|
|
702
694
|
importance,
|
|
703
|
-
category:
|
|
695
|
+
category: decodedBlob.c,
|
|
704
696
|
metadata: {
|
|
705
|
-
type:
|
|
697
|
+
type: decodedBlob.c,
|
|
706
698
|
importance: importance / 10,
|
|
707
|
-
source: typeof
|
|
708
|
-
created_at: typeof
|
|
699
|
+
source: typeof decodedBlob.sa === 'string' ? decodedBlob.sa : 'auto-extraction',
|
|
700
|
+
created_at: typeof decodedBlob.ea === 'string' ? decodedBlob.ea : '',
|
|
709
701
|
},
|
|
710
702
|
};
|
|
711
703
|
}
|
|
712
704
|
// Legacy plugin {text, metadata: {importance: 0-1}} format
|
|
713
|
-
if (typeof
|
|
714
|
-
const meta = (
|
|
705
|
+
if (typeof decodedBlob.text === 'string') {
|
|
706
|
+
const meta = (decodedBlob.metadata as Record<string, unknown>) ?? {};
|
|
715
707
|
const impFloat = typeof meta.importance === 'number' ? meta.importance : 0.5;
|
|
716
708
|
const importance = Math.max(1, Math.min(10, Math.round(impFloat * 10)));
|
|
717
709
|
return {
|
|
718
|
-
text:
|
|
710
|
+
text: decodedBlob.text,
|
|
719
711
|
importance,
|
|
720
712
|
category: typeof meta.type === 'string' ? meta.type : 'fact',
|
|
721
713
|
metadata: meta,
|
|
@@ -12,11 +12,9 @@
|
|
|
12
12
|
* 3. Bulk consolidation — cluster all facts in the vault and identify
|
|
13
13
|
* groups of near-duplicates for cleanup (clusterFacts).
|
|
14
14
|
*
|
|
15
|
-
* Delegates core computation to `@totalreclaw/core` Rust WASM module
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* and `clusterFacts` WASM functions when available, falling back to local
|
|
19
|
-
* implementations that use WASM-backed `cosineSimilarity`.
|
|
15
|
+
* Delegates core computation to `@totalreclaw/core` Rust WASM module:
|
|
16
|
+
* `findNearDuplicate`, `shouldSupersede`, and `clusterFacts` all call the
|
|
17
|
+
* corresponding core functions directly.
|
|
20
18
|
*
|
|
21
19
|
* Threshold helpers remain local; their env reads are centralized in
|
|
22
20
|
* entry.ts (env-reading seam, Task 1.3 of the OpenClaw native
|
|
@@ -24,8 +22,7 @@
|
|
|
24
22
|
*/
|
|
25
23
|
|
|
26
24
|
import { createRequire } from 'node:module';
|
|
27
|
-
import {
|
|
28
|
-
import { envNumber } from './entry.js';
|
|
25
|
+
import { envNumber } from '../entry.js';
|
|
29
26
|
|
|
30
27
|
// ---------------------------------------------------------------------------
|
|
31
28
|
// Lazy-load WASM core (mirrors claims-helper.ts / contradiction-sync.ts
|
|
@@ -118,48 +115,27 @@ export function findNearDuplicate(
|
|
|
118
115
|
): NearDuplicateMatch | null {
|
|
119
116
|
const wasm = getWasm();
|
|
120
117
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
const existing = candidates
|
|
125
|
-
.filter((c) => c.embedding && c.embedding.length > 0)
|
|
126
|
-
.map((c) => ({ id: c.id, embedding: c.embedding! }));
|
|
118
|
+
const existing = candidates
|
|
119
|
+
.filter((c) => c.embedding && c.embedding.length > 0)
|
|
120
|
+
.map((c) => ({ id: c.id, embedding: c.embedding! }));
|
|
127
121
|
|
|
128
|
-
|
|
122
|
+
if (existing.length === 0) return null;
|
|
129
123
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
124
|
+
const bestMatchJs = (wasm as any).findBestNearDuplicate(
|
|
125
|
+
JSON.stringify(newFactEmbedding),
|
|
126
|
+
JSON.stringify(existing),
|
|
127
|
+
threshold,
|
|
128
|
+
);
|
|
135
129
|
|
|
136
|
-
|
|
130
|
+
if (bestMatchJs == null) return null;
|
|
137
131
|
|
|
138
|
-
|
|
139
|
-
|
|
132
|
+
const bestMatch: { fact_id: string; similarity: number } =
|
|
133
|
+
typeof bestMatchJs === 'string' ? JSON.parse(bestMatchJs) : bestMatchJs;
|
|
140
134
|
|
|
141
|
-
|
|
142
|
-
|
|
135
|
+
const matched = candidates.find((c) => c.id === bestMatch.fact_id);
|
|
136
|
+
if (!matched) return null;
|
|
143
137
|
|
|
144
|
-
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
// Fallback: local loop using WASM-backed cosineSimilarity. Defensive only
|
|
148
|
-
// — core >=2.0.0 always exposes findBestNearDuplicate.
|
|
149
|
-
let bestMatch: NearDuplicateMatch | null = null;
|
|
150
|
-
|
|
151
|
-
for (const candidate of candidates) {
|
|
152
|
-
if (!candidate.embedding || candidate.embedding.length === 0) continue;
|
|
153
|
-
|
|
154
|
-
const similarity = cosineSimilarity(newFactEmbedding, candidate.embedding);
|
|
155
|
-
if (similarity >= threshold) {
|
|
156
|
-
if (!bestMatch || similarity > bestMatch.similarity) {
|
|
157
|
-
bestMatch = { existingFact: candidate, similarity };
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
return bestMatch;
|
|
138
|
+
return { existingFact: matched, similarity: bestMatch.similarity };
|
|
163
139
|
}
|
|
164
140
|
|
|
165
141
|
// ---------------------------------------------------------------------------
|
|
@@ -211,118 +187,51 @@ export function clusterFacts(
|
|
|
211
187
|
): ConsolidationCluster[] {
|
|
212
188
|
const wasm = getWasm();
|
|
213
189
|
|
|
214
|
-
//
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
.
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
if (dups.length > 0) {
|
|
259
|
-
result.push({ representative: rep, duplicates: dups });
|
|
260
|
-
}
|
|
190
|
+
// Build ConsolidationCandidate JSON for WASM (snake_case fields).
|
|
191
|
+
const wasmCandidates = facts
|
|
192
|
+
.filter((f) => f.embedding && f.embedding.length > 0)
|
|
193
|
+
.map((f) => ({
|
|
194
|
+
id: f.id,
|
|
195
|
+
text: f.text,
|
|
196
|
+
embedding: f.embedding!,
|
|
197
|
+
importance: f.importance,
|
|
198
|
+
decay_score: f.decayScore,
|
|
199
|
+
created_at: f.createdAt,
|
|
200
|
+
version: f.version,
|
|
201
|
+
}));
|
|
202
|
+
|
|
203
|
+
if (wasmCandidates.length === 0) return [];
|
|
204
|
+
|
|
205
|
+
const clustersJs = (wasm as any).clusterFacts(
|
|
206
|
+
JSON.stringify(wasmCandidates),
|
|
207
|
+
threshold,
|
|
208
|
+
);
|
|
209
|
+
|
|
210
|
+
// WASM returns a JSON string: [{ representative: string, duplicates: string[] }]
|
|
211
|
+
const wasmClusters: { representative: string; duplicates: string[] }[] =
|
|
212
|
+
typeof clustersJs === 'string' ? JSON.parse(clustersJs) : clustersJs;
|
|
213
|
+
|
|
214
|
+
// Build a lookup map for fast ID -> DecryptedCandidate resolution.
|
|
215
|
+
const byId = new Map<string, DecryptedCandidate>();
|
|
216
|
+
for (const f of facts) byId.set(f.id, f);
|
|
217
|
+
|
|
218
|
+
// Map ID-only clusters back to full DecryptedCandidate objects.
|
|
219
|
+
// Filter out singleton clusters (no duplicates) to match the pre-WASM
|
|
220
|
+
// plugin contract — callers rely on `clusters.length === 0` when nothing
|
|
221
|
+
// duplicates anything.
|
|
222
|
+
const clusters: ConsolidationCluster[] = [];
|
|
223
|
+
for (const wc of wasmClusters) {
|
|
224
|
+
const rep = byId.get(wc.representative);
|
|
225
|
+
if (!rep) continue;
|
|
226
|
+
|
|
227
|
+
const dups = wc.duplicates
|
|
228
|
+
.map((id) => byId.get(id))
|
|
229
|
+
.filter((d): d is DecryptedCandidate => d !== undefined);
|
|
230
|
+
|
|
231
|
+
if (dups.length > 0) {
|
|
232
|
+
clusters.push({ representative: rep, duplicates: dups });
|
|
261
233
|
}
|
|
262
|
-
|
|
263
|
-
return result;
|
|
264
234
|
}
|
|
265
235
|
|
|
266
|
-
|
|
267
|
-
// cosineSimilarity. Defensive only — core >=2.0.0 always exposes clusterFacts.
|
|
268
|
-
const clusters: { members: DecryptedCandidate[] }[] = [];
|
|
269
|
-
|
|
270
|
-
for (const fact of facts) {
|
|
271
|
-
if (!fact.embedding || fact.embedding.length === 0) continue;
|
|
272
|
-
|
|
273
|
-
let assigned = false;
|
|
274
|
-
for (const cluster of clusters) {
|
|
275
|
-
const seed = cluster.members[0];
|
|
276
|
-
if (!seed.embedding) continue;
|
|
277
|
-
|
|
278
|
-
const similarity = cosineSimilarity(fact.embedding, seed.embedding);
|
|
279
|
-
if (similarity >= threshold) {
|
|
280
|
-
cluster.members.push(fact);
|
|
281
|
-
assigned = true;
|
|
282
|
-
break;
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
if (!assigned) {
|
|
287
|
-
clusters.push({ members: [fact] });
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
const result: ConsolidationCluster[] = [];
|
|
292
|
-
for (const cluster of clusters) {
|
|
293
|
-
if (cluster.members.length < 2) continue;
|
|
294
|
-
|
|
295
|
-
const representative = pickRepresentative(cluster.members);
|
|
296
|
-
const duplicates = cluster.members.filter((m) => m !== representative);
|
|
297
|
-
result.push({ representative, duplicates });
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
return result;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
// ---------------------------------------------------------------------------
|
|
304
|
-
// Local helpers (used only in fallback paths)
|
|
305
|
-
// ---------------------------------------------------------------------------
|
|
306
|
-
|
|
307
|
-
/**
|
|
308
|
-
* Pick the best representative from a group of near-duplicate facts.
|
|
309
|
-
*
|
|
310
|
-
* Tiebreak order:
|
|
311
|
-
* 1. Highest decayScore
|
|
312
|
-
* 2. Most recent (highest createdAt)
|
|
313
|
-
* 3. Longest text
|
|
314
|
-
*/
|
|
315
|
-
function pickRepresentative(facts: DecryptedCandidate[]): DecryptedCandidate {
|
|
316
|
-
let best = facts[0];
|
|
317
|
-
for (let i = 1; i < facts.length; i++) {
|
|
318
|
-
const f = facts[i];
|
|
319
|
-
if (
|
|
320
|
-
f.decayScore > best.decayScore ||
|
|
321
|
-
(f.decayScore === best.decayScore && f.createdAt > best.createdAt) ||
|
|
322
|
-
(f.decayScore === best.decayScore && f.createdAt === best.createdAt && f.text.length > best.text.length)
|
|
323
|
-
) {
|
|
324
|
-
best = f;
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
return best;
|
|
236
|
+
return clusters;
|
|
328
237
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Matches the extraction prompts described in SKILL.md.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { chatCompletion, resolveLLMConfig } from '
|
|
8
|
+
import { chatCompletion, resolveLLMConfig } from '../llm/llm-client.js';
|
|
9
9
|
|
|
10
10
|
// ---------------------------------------------------------------------------
|
|
11
11
|
// Types
|
|
@@ -348,6 +348,7 @@ export function normalizeConfidence(raw: unknown): number {
|
|
|
348
348
|
export interface ExtractorLogger {
|
|
349
349
|
info?: (msg: string) => void;
|
|
350
350
|
warn?: (msg: string) => void;
|
|
351
|
+
debug?: (msg: string) => void;
|
|
351
352
|
}
|
|
352
353
|
|
|
353
354
|
|
|
@@ -617,7 +618,7 @@ export function parseFactsResponseForCompaction(
|
|
|
617
618
|
}
|
|
618
619
|
}
|
|
619
620
|
if (recoveryUsed === 'bracket-scan') {
|
|
620
|
-
logger?.
|
|
621
|
+
logger?.debug?.(
|
|
621
622
|
`parseFactsResponseForCompaction: recovered JSON via bracket-scan fallback`,
|
|
622
623
|
);
|
|
623
624
|
}
|
|
@@ -687,7 +688,7 @@ export function parseFactsResponseForCompaction(
|
|
|
687
688
|
if (valid.length > 0) entities = valid;
|
|
688
689
|
}
|
|
689
690
|
|
|
690
|
-
const
|
|
691
|
+
const extractedFact: ExtractedFact = {
|
|
691
692
|
text: String(f.text).slice(0, 512),
|
|
692
693
|
type,
|
|
693
694
|
source,
|
|
@@ -698,8 +699,8 @@ export function parseFactsResponseForCompaction(
|
|
|
698
699
|
existingFactId: typeof f.existingFactId === 'string' ? f.existingFactId : undefined,
|
|
699
700
|
confidence: normalizeConfidence(f.confidence),
|
|
700
701
|
};
|
|
701
|
-
if (entities)
|
|
702
|
-
return
|
|
702
|
+
if (entities) extractedFact.entities = entities;
|
|
703
|
+
return extractedFact;
|
|
703
704
|
})
|
|
704
705
|
// Reject illegal type:summary + source:user
|
|
705
706
|
.filter((f) => !(f.type === 'summary' && f.source === 'user'))
|
|
@@ -789,7 +790,7 @@ export async function extractFactsForCompaction(
|
|
|
789
790
|
return [];
|
|
790
791
|
}
|
|
791
792
|
|
|
792
|
-
logger?.
|
|
793
|
+
logger?.debug?.(
|
|
793
794
|
`extractFactsForCompaction: LLM returned ${response.length} chars; handing to parseFactsResponseForCompaction`,
|
|
794
795
|
);
|
|
795
796
|
let facts = parseFactsResponseForCompaction(response, logger);
|
|
@@ -810,7 +811,7 @@ export async function extractFactsForCompaction(
|
|
|
810
811
|
const oldImportance = f.importance;
|
|
811
812
|
const effectiveBump = f.importance >= 8 ? Math.min(bump, 1) : bump;
|
|
812
813
|
f.importance = Math.min(10, f.importance + effectiveBump);
|
|
813
|
-
logger?.
|
|
814
|
+
logger?.debug?.(
|
|
814
815
|
`extractFactsForCompaction: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`,
|
|
815
816
|
);
|
|
816
817
|
}
|
|
@@ -1251,7 +1252,7 @@ export function parseMergedResponseV1(
|
|
|
1251
1252
|
}
|
|
1252
1253
|
}
|
|
1253
1254
|
if (recoveryUsed === 'bracket-scan') {
|
|
1254
|
-
logger?.
|
|
1255
|
+
logger?.debug?.(
|
|
1255
1256
|
`parseFactsResponse: recovered JSON via bracket-scan fallback`,
|
|
1256
1257
|
);
|
|
1257
1258
|
}
|
|
@@ -1267,27 +1268,27 @@ export function parseMergedResponseV1(
|
|
|
1267
1268
|
// a bare JSON array of fact objects (legacy / test fixture shape). The
|
|
1268
1269
|
// bare array is wrapped as { topics: [], facts: [...] } so the downstream
|
|
1269
1270
|
// logic stays uniform. A single fact object (no wrapper) is also wrapped.
|
|
1270
|
-
let
|
|
1271
|
+
let wrapper: Record<string, unknown>;
|
|
1271
1272
|
if (Array.isArray(parsed)) {
|
|
1272
|
-
|
|
1273
|
+
wrapper = { topics: [], facts: parsed };
|
|
1273
1274
|
} else if (
|
|
1274
1275
|
typeof (parsed as Record<string, unknown>).facts === 'undefined' &&
|
|
1275
1276
|
typeof (parsed as Record<string, unknown>).text === 'string'
|
|
1276
1277
|
) {
|
|
1277
1278
|
// Single fact object, not a merged wrapper.
|
|
1278
|
-
|
|
1279
|
+
wrapper = { topics: [], facts: [parsed] };
|
|
1279
1280
|
} else {
|
|
1280
|
-
|
|
1281
|
+
wrapper = parsed as Record<string, unknown>;
|
|
1281
1282
|
}
|
|
1282
1283
|
|
|
1283
|
-
const rawTopics =
|
|
1284
|
+
const rawTopics = wrapper.topics;
|
|
1284
1285
|
const topics = Array.isArray(rawTopics)
|
|
1285
1286
|
? (rawTopics as unknown[])
|
|
1286
1287
|
.filter((t): t is string => typeof t === 'string' && t.length > 0)
|
|
1287
1288
|
.slice(0, 3)
|
|
1288
1289
|
: [];
|
|
1289
1290
|
|
|
1290
|
-
const rawFacts =
|
|
1291
|
+
const rawFacts = wrapper.facts;
|
|
1291
1292
|
if (!Array.isArray(rawFacts)) return { topics, facts: [] };
|
|
1292
1293
|
|
|
1293
1294
|
const validActions: ExtractionAction[] = ['ADD', 'UPDATE', 'DELETE', 'NOOP'];
|
|
@@ -1625,12 +1626,12 @@ export async function extractFacts(
|
|
|
1625
1626
|
return [];
|
|
1626
1627
|
}
|
|
1627
1628
|
|
|
1628
|
-
logger?.
|
|
1629
|
+
logger?.debug?.(
|
|
1629
1630
|
`extractFacts: LLM returned ${response.length} chars; parsing merged response`,
|
|
1630
1631
|
);
|
|
1631
1632
|
const { topics, facts: rawFacts } = parseMergedResponseV1(response, logger);
|
|
1632
1633
|
if (topics.length > 0) {
|
|
1633
|
-
logger?.
|
|
1634
|
+
logger?.debug?.(`extractFacts: topics = ${JSON.stringify(topics)}`);
|
|
1634
1635
|
}
|
|
1635
1636
|
|
|
1636
1637
|
// Provenance filter (tag-don't-drop)
|
|
@@ -1649,7 +1650,7 @@ export async function extractFacts(
|
|
|
1649
1650
|
const oldImportance = f.importance;
|
|
1650
1651
|
const effectiveBump = f.importance >= 8 ? Math.min(bump, 1) : bump;
|
|
1651
1652
|
f.importance = Math.min(10, f.importance + effectiveBump);
|
|
1652
|
-
logger?.
|
|
1653
|
+
logger?.debug?.(
|
|
1653
1654
|
`extractFacts: lexical bump +${bump} for "${f.text.slice(0, 60)}..." (${oldImportance} → ${f.importance})`,
|
|
1654
1655
|
);
|
|
1655
1656
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// Importance filter for auto-extraction. Extracted from index.ts.
|
|
3
|
+
// Pure over its inputs (threshold derives from CONFIG); no plugin session state.
|
|
4
|
+
// ---------------------------------------------------------------------------
|
|
5
|
+
|
|
6
|
+
import { CONFIG } from '../config.js';
|
|
7
|
+
import type { ExtractedFact } from './extractor.js';
|
|
8
|
+
import type { OpenClawPluginApi } from '../runtime/types.js';
|
|
9
|
+
|
|
10
|
+
const MIN_IMPORTANCE_THRESHOLD = CONFIG.minImportance;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Filter extracted facts by importance threshold.
|
|
14
|
+
* Facts with importance < MIN_IMPORTANCE_THRESHOLD are dropped.
|
|
15
|
+
* Facts with missing/undefined importance are treated as importance=5 (kept).
|
|
16
|
+
*/
|
|
17
|
+
export function filterByImportance(
|
|
18
|
+
facts: ExtractedFact[],
|
|
19
|
+
logger: OpenClawPluginApi['logger'],
|
|
20
|
+
): { kept: ExtractedFact[]; dropped: number } {
|
|
21
|
+
const kept: ExtractedFact[] = [];
|
|
22
|
+
let dropped = 0;
|
|
23
|
+
|
|
24
|
+
for (const fact of facts) {
|
|
25
|
+
const importance = fact.importance ?? 5;
|
|
26
|
+
if (importance >= MIN_IMPORTANCE_THRESHOLD) {
|
|
27
|
+
kept.push(fact);
|
|
28
|
+
} else {
|
|
29
|
+
dropped++;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Phase 2.2.5: always log the filter outcome so the agent_end path can
|
|
34
|
+
// distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
|
|
35
|
+
// below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
|
|
36
|
+
// this only logged on drops, which made empty-input invisible.
|
|
37
|
+
if (facts.length === 0) {
|
|
38
|
+
logger.info('Importance filter: input=0 (nothing to filter)');
|
|
39
|
+
} else if (dropped > 0) {
|
|
40
|
+
logger.info(
|
|
41
|
+
`Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`,
|
|
42
|
+
);
|
|
43
|
+
} else {
|
|
44
|
+
logger.info(
|
|
45
|
+
`Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`,
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return { kept, dropped };
|
|
50
|
+
}
|