@totalreclaw/totalreclaw 3.3.1-rc.21 → 3.3.1-rc.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -1
- package/SKILL.md +1 -1
- package/claims-helper.ts +47 -1
- package/config.ts +22 -1
- package/confirm-indexed.ts +191 -0
- package/dist/claims-helper.js +19 -1
- package/dist/config.js +18 -1
- package/dist/confirm-indexed.js +127 -0
- package/dist/embedder-cache.js +185 -0
- package/dist/embedder-loader.js +121 -0
- package/dist/embedder-network.js +301 -0
- package/dist/embedding.js +88 -33
- package/dist/fs-helpers.js +162 -0
- package/dist/index.js +75 -35
- package/dist/llm-client.js +4 -3
- package/dist/pin.js +15 -1
- package/dist/reranker.js +19 -52
- package/dist/retype-setscope.js +25 -5
- package/embedder-cache.ts +230 -0
- package/embedder-loader.ts +189 -0
- package/embedder-network.ts +350 -0
- package/embedding.ts +119 -51
- package/fs-helpers.ts +185 -0
- package/index.ts +86 -47
- package/llm-client.ts +4 -3
- package/package.json +10 -5
- package/pin.ts +31 -0
- package/reranker.ts +19 -52
- package/retype-setscope.ts +57 -8
- package/skill.json +1 -1
package/fs-helpers.ts
CHANGED
|
@@ -344,6 +344,191 @@ export function cleanupInstallStagingDirs(
|
|
|
344
344
|
return removed;
|
|
345
345
|
}
|
|
346
346
|
|
|
347
|
+
// ---------------------------------------------------------------------------
|
|
348
|
+
// Partial-install detection (rc.22 finding #5)
|
|
349
|
+
// ---------------------------------------------------------------------------
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Marker filename written into the plugin directory at register-time. Its
|
|
353
|
+
* presence means a prior install was interrupted before the plugin successfully
|
|
354
|
+
* loaded — a confirmed-broken half-state that the next `openclaw plugins
|
|
355
|
+
* install` retry can detect and clean.
|
|
356
|
+
*
|
|
357
|
+
* Conceptually the marker is dropped BEFORE npm install completes (the
|
|
358
|
+
* complementary npm script removes it on success) and additionally
|
|
359
|
+
* re-asserted at register-time as a second-line check. If you see this file
|
|
360
|
+
* in `<extensionsDir>/totalreclaw/`, the install never reached register()
|
|
361
|
+
* AND the marker drop wasn't undone.
|
|
362
|
+
*
|
|
363
|
+
* Constants are exported so the npm preinstall/cleanup scripts in
|
|
364
|
+
* `package.json` use the same name as the runtime detector.
|
|
365
|
+
*/
|
|
366
|
+
export const PARTIAL_INSTALL_MARKER = '.tr-partial-install';
|
|
367
|
+
|
|
368
|
+
/** Package name we own — used to confirm a directory is OUR plugin, not a stray. */
|
|
369
|
+
export const PLUGIN_PACKAGE_NAME = '@totalreclaw/totalreclaw';
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Outcome of `detectPartialInstall`.
|
|
373
|
+
* - `'clean'` — the dir is a fully-installed plugin (package.json claims our
|
|
374
|
+
* name AND `dist/index.js` exists AND no marker present).
|
|
375
|
+
* - `'partial'` — the dir is OUR plugin but in a corrupt half-state. Caller
|
|
376
|
+
* should wipe + retry. Returned reasons include:
|
|
377
|
+
* * marker file present (preinstall fired, postinstall did not)
|
|
378
|
+
* * dist/index.js missing (build never finished)
|
|
379
|
+
* - `'foreign'` — package.json missing or claims a different name. Helper
|
|
380
|
+
* refuses to act so we never delete an unrelated plugin.
|
|
381
|
+
* - `'absent'` — dir does not exist at all.
|
|
382
|
+
*/
|
|
383
|
+
export type PartialInstallStatus = 'clean' | 'partial' | 'foreign' | 'absent';
|
|
384
|
+
|
|
385
|
+
export interface PartialInstallResult {
|
|
386
|
+
status: PartialInstallStatus;
|
|
387
|
+
/** Why the caller decided this is partial — surfaces in error messages. */
|
|
388
|
+
reasons: string[];
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Inspect a plugin install directory to decide whether it is fully installed,
|
|
393
|
+
* a corrupted half-state from an interrupted install, or someone else's
|
|
394
|
+
* plugin. Pure filesystem inspection; never deletes anything.
|
|
395
|
+
*
|
|
396
|
+
* Background — rc.22 finding #5
|
|
397
|
+
* ------------------------------
|
|
398
|
+
* After a partial `openclaw plugins install @totalreclaw/totalreclaw` (e.g.
|
|
399
|
+
* the auto-gateway-restart kills npm mid-build), `extensions/totalreclaw/`
|
|
400
|
+
* survives with a populated package.json but a missing or empty `dist/`. The
|
|
401
|
+
* agent's recovery retry then fires another install; OpenClaw's plugin
|
|
402
|
+
* loader scans `extensions/` and tries to register the half-state as a "hook
|
|
403
|
+
* pack", failing with the cryptic "package.json missing openclaw.hooks". The
|
|
404
|
+
* fix: detect the partial state up-front so the retry can wipe + reinstall
|
|
405
|
+
* instead of cargo-culting a confused error.
|
|
406
|
+
*
|
|
407
|
+
* Decision rules
|
|
408
|
+
* --------------
|
|
409
|
+
* 1. `pluginRootDir` does not exist → `'absent'`.
|
|
410
|
+
* 2. package.json missing or unparsable → `'foreign'` (don't touch).
|
|
411
|
+
* 3. package.json `name !== '@totalreclaw/totalreclaw'` → `'foreign'`.
|
|
412
|
+
* 4. `<root>/.tr-partial-install` exists → `'partial'` (the canonical signal).
|
|
413
|
+
* 5. `<root>/dist/index.js` missing → `'partial'` (build never finished).
|
|
414
|
+
* 6. otherwise → `'clean'`.
|
|
415
|
+
*
|
|
416
|
+
* The function is intentionally conservative: it returns `'foreign'` on any
|
|
417
|
+
* ambiguous read. Callers should NEVER auto-wipe a `'foreign'` directory.
|
|
418
|
+
*
|
|
419
|
+
* @param pluginRootDir Absolute path to the suspect plugin dir, e.g.
|
|
420
|
+
* `~/.openclaw/extensions/totalreclaw`.
|
|
421
|
+
*/
|
|
422
|
+
export function detectPartialInstall(pluginRootDir: string): PartialInstallResult {
|
|
423
|
+
const reasons: string[] = [];
|
|
424
|
+
|
|
425
|
+
// Rule 1 — absent dir.
|
|
426
|
+
let rootStat: fs.Stats;
|
|
427
|
+
try {
|
|
428
|
+
rootStat = fs.statSync(pluginRootDir);
|
|
429
|
+
} catch {
|
|
430
|
+
return { status: 'absent', reasons: ['directory does not exist'] };
|
|
431
|
+
}
|
|
432
|
+
if (!rootStat.isDirectory()) {
|
|
433
|
+
return { status: 'foreign', reasons: ['path exists but is not a directory'] };
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// Rules 2-3 — package.json must claim our name.
|
|
437
|
+
const pkgJsonPath = path.join(pluginRootDir, 'package.json');
|
|
438
|
+
let pkgRaw: string;
|
|
439
|
+
try {
|
|
440
|
+
pkgRaw = fs.readFileSync(pkgJsonPath, 'utf-8');
|
|
441
|
+
} catch {
|
|
442
|
+
return { status: 'foreign', reasons: ['package.json missing or unreadable'] };
|
|
443
|
+
}
|
|
444
|
+
let parsed: { name?: unknown };
|
|
445
|
+
try {
|
|
446
|
+
parsed = JSON.parse(pkgRaw) as { name?: unknown };
|
|
447
|
+
} catch {
|
|
448
|
+
return { status: 'foreign', reasons: ['package.json is not valid JSON'] };
|
|
449
|
+
}
|
|
450
|
+
if (parsed.name !== PLUGIN_PACKAGE_NAME) {
|
|
451
|
+
return {
|
|
452
|
+
status: 'foreign',
|
|
453
|
+
reasons: [`package.json declares "${String(parsed.name)}" not "${PLUGIN_PACKAGE_NAME}"`],
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Rule 4 — explicit partial marker wins.
|
|
458
|
+
const markerPath = path.join(pluginRootDir, PARTIAL_INSTALL_MARKER);
|
|
459
|
+
if (fs.existsSync(markerPath)) {
|
|
460
|
+
reasons.push(`${PARTIAL_INSTALL_MARKER} marker present (preinstall fired, postinstall did not)`);
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// Rule 5 — dist/index.js must exist for the loader to register.
|
|
464
|
+
const distIndex = path.join(pluginRootDir, 'dist', 'index.js');
|
|
465
|
+
if (!fs.existsSync(distIndex)) {
|
|
466
|
+
reasons.push('dist/index.js missing (build artifact absent)');
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (reasons.length > 0) {
|
|
470
|
+
return { status: 'partial', reasons };
|
|
471
|
+
}
|
|
472
|
+
return { status: 'clean', reasons: [] };
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Wipe a partial-install directory so the next `openclaw plugins install`
|
|
477
|
+
* starts from a blank slate. Only acts when `detectPartialInstall(...)`
|
|
478
|
+
* returns `'partial'` — `'foreign'` and `'clean'` are no-ops by design.
|
|
479
|
+
*
|
|
480
|
+
* Returns `true` if the directory was wiped, `false` otherwise.
|
|
481
|
+
*
|
|
482
|
+
* SAFETY: this helper is the only place that recursively deletes a plugin
|
|
483
|
+
* dir. It refuses to act on `'foreign'` and `'clean'` results so a
|
|
484
|
+
* misconfigured caller can never wipe a healthy install or someone else's
|
|
485
|
+
* plugin.
|
|
486
|
+
*/
|
|
487
|
+
export function wipePartialInstall(pluginRootDir: string): boolean {
|
|
488
|
+
const detection = detectPartialInstall(pluginRootDir);
|
|
489
|
+
if (detection.status !== 'partial') return false;
|
|
490
|
+
try {
|
|
491
|
+
fs.rmSync(pluginRootDir, { recursive: true, force: true });
|
|
492
|
+
return true;
|
|
493
|
+
} catch {
|
|
494
|
+
return false;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Drop the `.tr-partial-install` marker into `pluginRootDir`. Idempotent
|
|
500
|
+
* (overwrites any existing marker) and best-effort — returns `true` on
|
|
501
|
+
* success, `false` if the dir doesn't exist or write fails. Used by the
|
|
502
|
+
* `preinstall` npm script and (defensively) by the runtime if the npm
|
|
503
|
+
* preinstall/cleanup script pair did not fire.
|
|
504
|
+
*/
|
|
505
|
+
export function writePartialInstallMarker(pluginRootDir: string): boolean {
|
|
506
|
+
try {
|
|
507
|
+
if (!fs.existsSync(pluginRootDir)) return false;
|
|
508
|
+
fs.writeFileSync(path.join(pluginRootDir, PARTIAL_INSTALL_MARKER), '');
|
|
509
|
+
return true;
|
|
510
|
+
} catch {
|
|
511
|
+
return false;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* Remove the partial-install marker. Called by the `postinstall` script and
|
|
517
|
+
* (defensively) at register-time once we've confirmed the load succeeded.
|
|
518
|
+
* Returns `true` if a marker was removed, `false` if there was nothing to
|
|
519
|
+
* remove.
|
|
520
|
+
*/
|
|
521
|
+
export function clearPartialInstallMarker(pluginRootDir: string): boolean {
|
|
522
|
+
try {
|
|
523
|
+
const markerPath = path.join(pluginRootDir, PARTIAL_INSTALL_MARKER);
|
|
524
|
+
if (!fs.existsSync(markerPath)) return false;
|
|
525
|
+
fs.unlinkSync(markerPath);
|
|
526
|
+
return true;
|
|
527
|
+
} catch {
|
|
528
|
+
return false;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
347
532
|
// ---------------------------------------------------------------------------
|
|
348
533
|
// Auto-bootstrap of credentials.json (3.1.0 first-run UX)
|
|
349
534
|
// ---------------------------------------------------------------------------
|
package/index.ts
CHANGED
|
@@ -73,14 +73,22 @@ import {
|
|
|
73
73
|
type MemorySource,
|
|
74
74
|
type MemoryScope,
|
|
75
75
|
} from './extractor.js';
|
|
76
|
-
import {
|
|
76
|
+
import {
|
|
77
|
+
initLLMClient,
|
|
78
|
+
resolveLLMConfig,
|
|
79
|
+
chatCompletion,
|
|
80
|
+
generateEmbedding,
|
|
81
|
+
getEmbeddingDims,
|
|
82
|
+
getEmbeddingModelId,
|
|
83
|
+
configureEmbedder,
|
|
84
|
+
} from './llm-client.js';
|
|
77
85
|
import {
|
|
78
86
|
defaultAuthProfilesRoot,
|
|
79
87
|
readAllProfileKeys,
|
|
80
88
|
dedupeByProvider,
|
|
81
89
|
} from './llm-profile-reader.js';
|
|
82
90
|
import { LSHHasher } from './lsh.js';
|
|
83
|
-
import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS,
|
|
91
|
+
import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS, type RerankerCandidate } from './reranker.js';
|
|
84
92
|
import { deduplicateBatch } from './semantic-dedup.js';
|
|
85
93
|
import {
|
|
86
94
|
findNearDuplicate,
|
|
@@ -92,6 +100,7 @@ import {
|
|
|
92
100
|
type DecryptedCandidate,
|
|
93
101
|
} from './consolidation.js';
|
|
94
102
|
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactOnChain, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4, type FactPayload } from './subgraph-store.js';
|
|
103
|
+
import { confirmIndexed } from './confirm-indexed.js';
|
|
95
104
|
import {
|
|
96
105
|
DIGEST_TRAPDOOR,
|
|
97
106
|
buildCanonicalClaim,
|
|
@@ -153,6 +162,8 @@ import {
|
|
|
153
162
|
writeOnboardingState,
|
|
154
163
|
readPluginVersion,
|
|
155
164
|
cleanupInstallStagingDirs,
|
|
165
|
+
detectPartialInstall,
|
|
166
|
+
clearPartialInstallMarker,
|
|
156
167
|
type OnboardingState,
|
|
157
168
|
} from './fs-helpers.js';
|
|
158
169
|
import { isRcBuild } from './qa-bug-report.js';
|
|
@@ -1946,6 +1957,11 @@ async function storeExtractedFacts(
|
|
|
1946
1957
|
fact: factForBlob,
|
|
1947
1958
|
importance: effectiveImportance,
|
|
1948
1959
|
sourceAgent: factSource,
|
|
1960
|
+
// 3.3.1-rc.22 — tag every new claim with the active embedder id
|
|
1961
|
+
// so future distillation can rescore selectively. Plugin-only
|
|
1962
|
+
// field; survives the core validator strip via re-attach in
|
|
1963
|
+
// `buildCanonicalClaimV1`.
|
|
1964
|
+
embeddingModelId: getEmbeddingModelId(),
|
|
1949
1965
|
});
|
|
1950
1966
|
|
|
1951
1967
|
const factId = crypto.randomUUID();
|
|
@@ -2888,6 +2904,49 @@ const plugin = {
|
|
|
2888
2904
|
// Best-effort — already swallowed inside the helper, but keep this
|
|
2889
2905
|
// outer try as belt-and-braces against future helper changes.
|
|
2890
2906
|
}
|
|
2907
|
+
|
|
2908
|
+
// 3.3.1-rc.22 — wire the lazy-embedder runtime config so the first
|
|
2909
|
+
// `generateEmbedding()` call knows where to cache the bundle and
|
|
2910
|
+
// which RC's GitHub Release to fetch from. `pluginVersion` may be
|
|
2911
|
+
// `null` if package.json is unreadable; the embedder defaults to
|
|
2912
|
+
// a "0.0.0-dev" tag in that case.
|
|
2913
|
+
try {
|
|
2914
|
+
configureEmbedder({
|
|
2915
|
+
cacheRoot: CONFIG.embedderCachePath,
|
|
2916
|
+
rcTag: pluginVersion ?? '0.0.0-dev',
|
|
2917
|
+
});
|
|
2918
|
+
} catch (err) {
|
|
2919
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2920
|
+
api.logger.warn(`TotalReclaw: configureEmbedder failed (will use defaults): ${msg}`);
|
|
2921
|
+
}
|
|
2922
|
+
|
|
2923
|
+
// 3.3.1-rc.22 (rc.21 finding #5): self-heal partial-install marker.
|
|
2924
|
+
// The `preinstall` npm script writes `.tr-partial-install`; the
|
|
2925
|
+
// `postinstall` script removes it on a successful install. If we
|
|
2926
|
+
// have gotten this far the loader did register us — meaning the
|
|
2927
|
+
// install succeeded enough to be useful — so any lingering marker
|
|
2928
|
+
// (e.g. npm ran preinstall but postinstall misfired) is stale.
|
|
2929
|
+
// Clear it so the next retry's detector does not see a false positive.
|
|
2930
|
+
//
|
|
2931
|
+
// 3.3.1-rc.22 (rc.21 finding #6) — gateway/reload upstream caveat:
|
|
2932
|
+
// OpenClaw's config-watcher fires `gateway/reload` when
|
|
2933
|
+
// `plugins.entries.totalreclaw` mutates (e.g. mid-install). In-flight
|
|
2934
|
+
// CLI clients see `1006 abnormal closure` and start a 600-second wait.
|
|
2935
|
+
// Proper fix is upstream OpenClaw FR. Plugin-side mitigation = these
|
|
2936
|
+
// helper calls MUST be idempotent under repeated register() calls
|
|
2937
|
+
// triggered by reload chatter. Asserted by
|
|
2938
|
+
// `install-reload-idempotency.test.ts`.
|
|
2939
|
+
try {
|
|
2940
|
+
const pluginRoot = nodePath.resolve(pluginDir, '..');
|
|
2941
|
+
const cleared = clearPartialInstallMarker(pluginRoot);
|
|
2942
|
+
if (cleared) {
|
|
2943
|
+
api.logger.info(
|
|
2944
|
+
`TotalReclaw: cleared stale .tr-partial-install marker (rc.22 finding #5)`,
|
|
2945
|
+
);
|
|
2946
|
+
}
|
|
2947
|
+
} catch {
|
|
2948
|
+
// Best-effort. Helper logs internally and never throws.
|
|
2949
|
+
}
|
|
2891
2950
|
} catch {
|
|
2892
2951
|
rcMode = false;
|
|
2893
2952
|
}
|
|
@@ -3622,25 +3681,10 @@ const plugin = {
|
|
|
3622
3681
|
};
|
|
3623
3682
|
}
|
|
3624
3683
|
|
|
3625
|
-
// 6b. Relevance gate
|
|
3626
|
-
//
|
|
3627
|
-
//
|
|
3628
|
-
//
|
|
3629
|
-
// "favorite color" produce embeddings with low cosine sim
|
|
3630
|
-
// against the local Harrier-OSS-270m model even when the
|
|
3631
|
-
// stored fact text contains every query token.
|
|
3632
|
-
if (!passesRelevanceGate(params.query, reranked, COSINE_THRESHOLD)) {
|
|
3633
|
-
const maxCosine = Math.max(
|
|
3634
|
-
...reranked.map((r) => r.cosineSimilarity ?? 0),
|
|
3635
|
-
);
|
|
3636
|
-
api.logger.info(
|
|
3637
|
-
`Recall: relevance gate filtered results (max cosine=${maxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD}, no lexical override)`,
|
|
3638
|
-
);
|
|
3639
|
-
return {
|
|
3640
|
-
content: [{ type: 'text', text: 'No relevant memories found for this query.' }],
|
|
3641
|
-
details: { count: 0, memories: [] },
|
|
3642
|
-
};
|
|
3643
|
-
}
|
|
3684
|
+
// 6b. Relevance gate removed in rc.22 -- core's intent-weighted
|
|
3685
|
+
// RRF + Tier 1 source weighting handles short queries via the
|
|
3686
|
+
// BM25 component, making the rc.18 cosine + lexical-override
|
|
3687
|
+
// band-aid (issue #116) redundant.
|
|
3644
3688
|
|
|
3645
3689
|
// 7. Format results.
|
|
3646
3690
|
const lines = reranked.map((m, i) => {
|
|
@@ -3779,14 +3823,29 @@ const plugin = {
|
|
|
3779
3823
|
throw new Error(`On-chain tombstone failed (tx=${result.txHash?.slice(0, 10) || 'none'}…)`);
|
|
3780
3824
|
}
|
|
3781
3825
|
api.logger.info(`Tombstone written for ${factId}: tx=${result.txHash}`);
|
|
3826
|
+
// Read-after-write: poll the subgraph until the original fact id
|
|
3827
|
+
// is no longer active (forget flips isActive=false). On timeout
|
|
3828
|
+
// surface `partial: true` so the agent can explain the chain
|
|
3829
|
+
// write succeeded but the subgraph is still propagating.
|
|
3830
|
+
const confirm = await confirmIndexed(factId, {
|
|
3831
|
+
expect: 'inactive',
|
|
3832
|
+
authKeyHex: authKeyHex!,
|
|
3833
|
+
});
|
|
3782
3834
|
return {
|
|
3783
3835
|
content: [{
|
|
3784
3836
|
type: 'text',
|
|
3785
|
-
text:
|
|
3786
|
-
`Memory ${factId} deleted on-chain (tx: ${result.txHash})
|
|
3787
|
-
|
|
3837
|
+
text: confirm.indexed
|
|
3838
|
+
? `Memory ${factId} deleted on-chain and confirmed by the subgraph (tx: ${result.txHash}).`
|
|
3839
|
+
: `Memory ${factId} deleted on-chain (tx: ${result.txHash}). ` +
|
|
3840
|
+
'The subgraph indexer is still propagating the change — ' +
|
|
3841
|
+
'recall/export may briefly show the memory as still active.',
|
|
3788
3842
|
}],
|
|
3789
|
-
details: {
|
|
3843
|
+
details: {
|
|
3844
|
+
deleted: true,
|
|
3845
|
+
txHash: result.txHash,
|
|
3846
|
+
factId,
|
|
3847
|
+
...(confirm.indexed ? {} : { partial: true }),
|
|
3848
|
+
},
|
|
3790
3849
|
};
|
|
3791
3850
|
} else {
|
|
3792
3851
|
await apiClient!.deleteFact(factId, authKeyHex!);
|
|
@@ -5872,17 +5931,7 @@ const plugin = {
|
|
|
5872
5931
|
|
|
5873
5932
|
if (reranked.length === 0) return undefined;
|
|
5874
5933
|
|
|
5875
|
-
//
|
|
5876
|
-
// lexical-override rule (issue #116).
|
|
5877
|
-
if (!passesRelevanceGate(evt.prompt, reranked, COSINE_THRESHOLD)) {
|
|
5878
|
-
const hookMaxCosine = Math.max(
|
|
5879
|
-
...reranked.map((r) => r.cosineSimilarity ?? 0),
|
|
5880
|
-
);
|
|
5881
|
-
api.logger.info(
|
|
5882
|
-
`Hook: relevance gate filtered results (max cosine=${hookMaxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD}, no lexical override)`,
|
|
5883
|
-
);
|
|
5884
|
-
return undefined;
|
|
5885
|
-
}
|
|
5934
|
+
// Relevance gate removed in rc.22 (see recall tool comment).
|
|
5886
5935
|
|
|
5887
5936
|
// 7. Build context string.
|
|
5888
5937
|
const lines = reranked.map((m, i) => {
|
|
@@ -5987,17 +6036,7 @@ const plugin = {
|
|
|
5987
6036
|
|
|
5988
6037
|
if (reranked.length === 0) return undefined;
|
|
5989
6038
|
|
|
5990
|
-
// Relevance gate
|
|
5991
|
-
// rule (issue #116).
|
|
5992
|
-
if (!passesRelevanceGate(evt.prompt, reranked, COSINE_THRESHOLD)) {
|
|
5993
|
-
const srvMaxCosine = Math.max(
|
|
5994
|
-
...reranked.map((r) => r.cosineSimilarity ?? 0),
|
|
5995
|
-
);
|
|
5996
|
-
api.logger.info(
|
|
5997
|
-
`Hook: relevance gate filtered results (max cosine=${srvMaxCosine.toFixed(3)}, threshold=${COSINE_THRESHOLD}, no lexical override)`,
|
|
5998
|
-
);
|
|
5999
|
-
return undefined;
|
|
6000
|
-
}
|
|
6039
|
+
// Relevance gate removed in rc.22 (see recall tool comment).
|
|
6001
6040
|
|
|
6002
6041
|
// 7. Build context string.
|
|
6003
6042
|
const lines = reranked.map((m, i) => {
|
package/llm-client.ts
CHANGED
|
@@ -908,6 +908,7 @@ async function chatCompletionAnthropic(
|
|
|
908
908
|
// ---------------------------------------------------------------------------
|
|
909
909
|
|
|
910
910
|
// Embeddings are now generated locally via @huggingface/transformers
|
|
911
|
-
// (Harrier-OSS-v1-270M ONNX model). No API key needed.
|
|
912
|
-
//
|
|
913
|
-
|
|
911
|
+
// (Harrier-OSS-v1-270M ONNX model). No API key needed. The native deps +
|
|
912
|
+
// model are lazy-fetched from a pinned GitHub Release on first call —
|
|
913
|
+
// see embedding.ts + embedder-loader.ts.
|
|
914
|
+
export { generateEmbedding, getEmbeddingDims, getEmbeddingModelId, configureEmbedder } from './embedding.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@totalreclaw/totalreclaw",
|
|
3
|
-
"version": "3.3.1-rc.
|
|
3
|
+
"version": "3.3.1-rc.22",
|
|
4
4
|
"description": "End-to-end encrypted, agent-portable memory for OpenClaw and any LLM-agent runtime. XChaCha20-Poly1305 with protobuf v4 + on-chain Memory Taxonomy v1 (claim / preference / directive / commitment / episode / summary).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"keywords": [
|
|
@@ -31,17 +31,18 @@
|
|
|
31
31
|
"author": "TotalReclaw Team",
|
|
32
32
|
"license": "MIT",
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@huggingface/transformers": "^4.0.1",
|
|
35
34
|
"@totalreclaw/client": "^1.2.0",
|
|
36
35
|
"@totalreclaw/core": "^2.1.1",
|
|
37
36
|
"@types/qrcode": "^1.5.6",
|
|
38
37
|
"@types/ws": "^8.5.12",
|
|
39
|
-
"onnxruntime-node": "^1.24.0",
|
|
40
38
|
"qrcode": "^1.5.4",
|
|
41
39
|
"qrcode-terminal": "^0.12.0",
|
|
42
40
|
"ws": "^8.18.3"
|
|
43
41
|
},
|
|
42
|
+
"//": "@huggingface/transformers + onnxruntime-node deliberately omitted from runtime deps. They are heavy native bundles (~700 MB peak install RAM) that OOM-killed the OpenClaw gateway on small VPS during `openclaw plugins install` in rc.21 (issue: 3.7 GB Hetzner host). rc.22 ships them via a lazy GitHub-Releases bundle (`embedder-v1.tar.gz`) downloaded on first call to embed(). See `embedder-network.ts` + `scripts/build-embedder-bundle.mjs`. The dev-deps below are for type-checking + bundle generation only; npm install of the plugin tarball never installs them.",
|
|
44
43
|
"devDependencies": {
|
|
44
|
+
"@huggingface/transformers": "^4.0.1",
|
|
45
|
+
"onnxruntime-node": "^1.24.0",
|
|
45
46
|
"typescript": "^5.5.0"
|
|
46
47
|
},
|
|
47
48
|
"main": "./dist/index.js",
|
|
@@ -62,11 +63,15 @@
|
|
|
62
63
|
"scripts": {
|
|
63
64
|
"build": "rm -rf dist && tsc -p tsconfig.json --noCheck",
|
|
64
65
|
"verify-tarball": "node ../scripts/verify-tarball.mjs",
|
|
65
|
-
"test": "npx tsx manifest-shape.test.ts && npx tsx config-schema.test.ts && npx tsx config.test.ts && npx tsx relay-headers.test.ts && npx tsx scope-address-visible.test.ts && npx tsx llm-profile-reader.test.ts && npx tsx llm-client.test.ts && npx tsx llm-client-retry.test.ts && npx tsx gateway-url.test.ts && npx tsx retype-setscope.test.ts && npx tsx tool-gating.test.ts && npx tsx onboarding-noninteractive.test.ts && npx tsx pair-cli-json.test.ts && npx tsx pair-qr.test.ts && npx tsx pair-remote-client.test.ts && npx tsx qa-bug-report.test.ts && npx tsx nonce-serialization.test.ts && npx tsx phrase-safety-registry.test.ts && npx tsx test_issue_92_onnx_download_ux.test.ts && npx tsx onboard-pair-only.test.ts && npx tsx import-time-smoke.test.ts && npx tsx
|
|
66
|
+
"test": "npx tsx manifest-shape.test.ts && npx tsx config-schema.test.ts && npx tsx config.test.ts && npx tsx relay-headers.test.ts && npx tsx scope-address-visible.test.ts && npx tsx llm-profile-reader.test.ts && npx tsx llm-client.test.ts && npx tsx llm-client-retry.test.ts && npx tsx gateway-url.test.ts && npx tsx retype-setscope.test.ts && npx tsx tool-gating.test.ts && npx tsx onboarding-noninteractive.test.ts && npx tsx pair-cli-json.test.ts && npx tsx pair-qr.test.ts && npx tsx pair-remote-client.test.ts && npx tsx qa-bug-report.test.ts && npx tsx nonce-serialization.test.ts && npx tsx phrase-safety-registry.test.ts && npx tsx test_issue_92_onnx_download_ux.test.ts && npx tsx onboard-pair-only.test.ts && npx tsx import-time-smoke.test.ts && npx tsx install-staging-cleanup.test.ts && npx tsx partial-install-detection.test.ts && npx tsx install-reload-idempotency.test.ts && npx tsx json-stdout-cleanliness.test.ts",
|
|
66
67
|
"smoke:dist": "npx tsx dist-esm-smoke.test.ts",
|
|
67
68
|
"check-scanner": "node ../scripts/check-scanner.mjs",
|
|
69
|
+
"check-version-drift": "node ../scripts/check-version-drift.mjs",
|
|
70
|
+
"sync-version": "node ../scripts/sync-version.mjs",
|
|
71
|
+
"preinstall": "node -e \"try{require('fs').writeFileSync('.tr-partial-install','');}catch{}\"",
|
|
72
|
+
"postinstall": "node -e \"try{require('fs').unlinkSync('.tr-partial-install');}catch{}\"",
|
|
68
73
|
"prepack": "npm run build",
|
|
69
|
-
"prepublishOnly": "node ../scripts/check-scanner.mjs"
|
|
74
|
+
"prepublishOnly": "node ../scripts/check-scanner.mjs && node ../scripts/check-version-drift.mjs"
|
|
70
75
|
},
|
|
71
76
|
"openclaw": {
|
|
72
77
|
"extensions": [
|
package/pin.ts
CHANGED
|
@@ -26,6 +26,7 @@ import { isValidMemoryType, V0_TO_V1_TYPE } from './extractor.js';
|
|
|
26
26
|
import type { MemoryType, MemorySource, MemoryScope, MemoryVolatility } from './extractor.js';
|
|
27
27
|
import { PROTOBUF_VERSION_V4 } from './subgraph-store.js';
|
|
28
28
|
import type { SubgraphSearchFact } from './subgraph-search.js';
|
|
29
|
+
import { confirmIndexed, type ConfirmIndexedOptions } from './confirm-indexed.js';
|
|
29
30
|
|
|
30
31
|
// Lazy-load WASM core (mirrors claims-helper.ts pattern — plays nicely under
|
|
31
32
|
// both the OpenClaw runtime (CJS-ish tsx) and bare Node ESM used by tests).
|
|
@@ -114,6 +115,12 @@ export interface V1PinBlob {
|
|
|
114
115
|
id?: string;
|
|
115
116
|
/** Previously-stored pin_status on the blob (v1.1). */
|
|
116
117
|
pinStatus?: PinStatus;
|
|
118
|
+
/**
|
|
119
|
+
* 3.3.1-rc.22 — preserved when round-tripping a v1 blob through pin
|
|
120
|
+
* mutation. We keep the SOURCE blob's tag so distillation backfill
|
|
121
|
+
* never loses track of which embedder produced the original vector.
|
|
122
|
+
*/
|
|
123
|
+
embeddingModelId?: string;
|
|
117
124
|
}
|
|
118
125
|
|
|
119
126
|
/** Shape of a v0 (short-key) blob or a legacy {text, metadata} blob. */
|
|
@@ -186,6 +193,7 @@ export function parseBlobForPin(decrypted: string): ParsedBlob {
|
|
|
186
193
|
expiresAt: v1.expiresAt,
|
|
187
194
|
id: v1.id,
|
|
188
195
|
pinStatus: v1.pinStatus,
|
|
196
|
+
embeddingModelId: v1.embeddingModelId,
|
|
189
197
|
},
|
|
190
198
|
claim: shortProjection,
|
|
191
199
|
currentStatus: human,
|
|
@@ -314,6 +322,8 @@ interface V1Projection {
|
|
|
314
322
|
entities?: Array<{ name: string; type: string; role?: string }>;
|
|
315
323
|
importance: number;
|
|
316
324
|
confidence: number;
|
|
325
|
+
/** 3.3.1-rc.22 — carried through pin/retype rewrites for forward-compat. */
|
|
326
|
+
embeddingModelId?: string;
|
|
317
327
|
}
|
|
318
328
|
|
|
319
329
|
/**
|
|
@@ -336,6 +346,7 @@ function projectToV1(src: V1PinBlob | V0PinBlob, defaultSourceAgent: string): V1
|
|
|
336
346
|
entities: src.entities,
|
|
337
347
|
importance: src.importance,
|
|
338
348
|
confidence: src.confidence,
|
|
349
|
+
embeddingModelId: src.embeddingModelId,
|
|
339
350
|
};
|
|
340
351
|
}
|
|
341
352
|
|
|
@@ -445,6 +456,14 @@ export interface PinOpResult {
|
|
|
445
456
|
tx_hash?: string;
|
|
446
457
|
reason?: string;
|
|
447
458
|
error?: string;
|
|
459
|
+
/**
|
|
460
|
+
* On-chain batch submitted but subgraph indexer did not confirm the new
|
|
461
|
+
* fact id within the timeout window (default 30s). The pin/unpin IS
|
|
462
|
+
* on-chain — `tx_hash` is observable on the explorer — but a follow-up
|
|
463
|
+
* `recall`/`export` may briefly surface stale state. Resolves once the
|
|
464
|
+
* indexer catches up. See `confirm-indexed.ts`.
|
|
465
|
+
*/
|
|
466
|
+
partial?: boolean;
|
|
448
467
|
}
|
|
449
468
|
|
|
450
469
|
/**
|
|
@@ -461,6 +480,7 @@ export async function executePinOperation(
|
|
|
461
480
|
targetStatus: 'pinned' | 'active',
|
|
462
481
|
deps: PinOpDeps,
|
|
463
482
|
reason?: string,
|
|
483
|
+
confirmOpts?: ConfirmIndexedOptions,
|
|
464
484
|
): Promise<PinOpResult> {
|
|
465
485
|
// 1. Fetch the existing fact
|
|
466
486
|
const existing = await deps.fetchFactById(factId);
|
|
@@ -570,6 +590,11 @@ export async function executePinOperation(
|
|
|
570
590
|
createdAt: new Date().toISOString(),
|
|
571
591
|
supersededBy: factId,
|
|
572
592
|
pinStatus,
|
|
593
|
+
// 3.3.1-rc.22 — preserve the source claim's embedder tag through
|
|
594
|
+
// pin mutation. The new fact reuses the same encrypted embedding
|
|
595
|
+
// as the original (re-indexed via deps.regenerateBlindIndices),
|
|
596
|
+
// so the embedder identity must round-trip too.
|
|
597
|
+
embeddingModelId: v1View.embeddingModelId,
|
|
573
598
|
});
|
|
574
599
|
} catch (err) {
|
|
575
600
|
return {
|
|
@@ -672,6 +697,11 @@ export async function executePinOperation(
|
|
|
672
697
|
tx_hash: txHash,
|
|
673
698
|
};
|
|
674
699
|
}
|
|
700
|
+
// Read-after-write: poll the subgraph until the new (pinned/unpinned)
|
|
701
|
+
// fact id is indexed and active. On timeout, surface `partial: true`
|
|
702
|
+
// so a follow-up recall/export that races against indexer lag can
|
|
703
|
+
// surface a clear "still propagating" hint rather than apparent staleness.
|
|
704
|
+
const confirm = await confirmIndexed(newFactId, confirmOpts);
|
|
675
705
|
return {
|
|
676
706
|
success: true,
|
|
677
707
|
fact_id: factId,
|
|
@@ -680,6 +710,7 @@ export async function executePinOperation(
|
|
|
680
710
|
new_status: targetStatus,
|
|
681
711
|
tx_hash: txHash,
|
|
682
712
|
reason,
|
|
713
|
+
...(confirm.indexed ? {} : { partial: true }),
|
|
683
714
|
};
|
|
684
715
|
} catch (err) {
|
|
685
716
|
return {
|
package/reranker.ts
CHANGED
|
@@ -509,74 +509,41 @@ export function rerank(
|
|
|
509
509
|
}
|
|
510
510
|
|
|
511
511
|
// ---------------------------------------------------------------------------
|
|
512
|
-
// Relevance gate
|
|
512
|
+
// Relevance gate
|
|
513
513
|
// ---------------------------------------------------------------------------
|
|
514
514
|
|
|
515
515
|
/**
|
|
516
516
|
* Decide whether reranked results clear the relevance gate for surfacing to
|
|
517
517
|
* the user (recall tool) or auto-injecting into agent context (hooks).
|
|
518
518
|
*
|
|
519
|
-
*
|
|
519
|
+
* Plain cosine cut-off: at least one reranked result has cosine similarity
|
|
520
|
+
* with the query embedding >= `cosineThreshold`.
|
|
520
521
|
*
|
|
521
|
-
*
|
|
522
|
-
*
|
|
523
|
-
*
|
|
522
|
+
* History (rc.18 → rc.22):
|
|
523
|
+
* rc.18 issue #116 surfaced as a recall miss for short queries against
|
|
524
|
+
* the local Harrier-OSS-270m model — cosine alone produced false-negatives
|
|
525
|
+
* even when every query token literally appeared in the candidate. rc.18
|
|
526
|
+
* shipped a defensive "lexical-override" fallback (every meaningful query
|
|
527
|
+
* token had to appear as a 4-char-prefix substring in the top result).
|
|
528
|
+
* The override was always intended as a band-aid until the
|
|
529
|
+
* source-weighted reranker hoisted from `totalreclaw-core` produced
|
|
530
|
+
* honest cosine signals for short queries. rc.22 hoists that reranker
|
|
531
|
+
* and drops the band-aid: the gate is now back to a single-signal cosine
|
|
532
|
+
* cut-off, matching Hermes (Python client) and the canonical reranker
|
|
533
|
+
* spec.
|
|
524
534
|
*
|
|
525
|
-
*
|
|
526
|
-
*
|
|
527
|
-
* with low cosine sim regardless of topical match), the gate ALSO
|
|
528
|
-
* passes when every meaningful query token (post stop-word removal)
|
|
529
|
-
* appears as a stem-prefix substring in the top reranked result's
|
|
530
|
-
* text. This is strong lexical evidence that the user is asking
|
|
531
|
-
* about a fact already stored, even when embedding sim is weak.
|
|
532
|
-
*
|
|
533
|
-
* Without (2), short queries like `"favorite color"` against the stored
|
|
534
|
-
* fact `"User's favorite color is cobalt blue"` were silently filtered
|
|
535
|
-
* even though every query token was present in the candidate. Hermes
|
|
536
|
-
* (Python client) does not apply any cosine gate, which is why it
|
|
537
|
-
* recalled the same fact for the same Smart Account in rc.18 QA.
|
|
538
|
-
*
|
|
539
|
-
* The lexical override is intentionally conservative:
|
|
540
|
-
* - Requires ALL non-stop-word query tokens to be present (any-of would
|
|
541
|
-
* over-trigger).
|
|
542
|
-
* - Uses 4-char-prefix substring match to be stem-tolerant ("favorite"
|
|
543
|
-
* stems to "favorit" in the stored fact's blind index, but the raw
|
|
544
|
-
* fact text contains the unstemmed word; the prefix check absorbs
|
|
545
|
-
* light morphology).
|
|
546
|
-
* - Token count must be >= 1 — empty/all-stop-word queries fall back
|
|
547
|
-
* to cosine path.
|
|
548
|
-
*
|
|
549
|
-
* @param query - the user's search query (raw string)
|
|
535
|
+
* @param query - the user's search query (raw string) — accepted for ABI
|
|
536
|
+
* stability, no longer consulted post rc.22.
|
|
550
537
|
* @param reranked - reranked results (top first)
|
|
551
538
|
* @param cosineThreshold - the configured cosine cutoff (typically 0.15)
|
|
552
539
|
* @returns true if results should be surfaced; false to suppress
|
|
553
540
|
*/
|
|
554
541
|
export function passesRelevanceGate(
|
|
555
|
-
|
|
542
|
+
_query: string,
|
|
556
543
|
reranked: RerankerResult[],
|
|
557
544
|
cosineThreshold: number,
|
|
558
545
|
): boolean {
|
|
559
546
|
if (reranked.length === 0) return false;
|
|
560
|
-
|
|
561
|
-
// Path 1: cosine clears threshold.
|
|
562
547
|
const maxCosine = Math.max(...reranked.map((r) => r.cosineSimilarity ?? 0));
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
// Path 2: lexical override — every meaningful query token appears in
|
|
566
|
-
// the top reranked result's text.
|
|
567
|
-
const queryTokens = tokenize(query, /* removeStopWords */ true);
|
|
568
|
-
if (queryTokens.length === 0) return false;
|
|
569
|
-
|
|
570
|
-
const topText = (reranked[0]?.text ?? '').toLowerCase();
|
|
571
|
-
if (topText.length === 0) return false;
|
|
572
|
-
|
|
573
|
-
// 4-char prefix substring match: tolerates light stemming ("favorite"
|
|
574
|
-
// matches a fact text containing "favorite", "favorites", "favoring",
|
|
575
|
-
// etc., without re-running the WASM Porter stemmer client-side).
|
|
576
|
-
const PREFIX_LEN = 4;
|
|
577
|
-
for (const token of queryTokens) {
|
|
578
|
-
const probe = token.length >= PREFIX_LEN ? token.slice(0, PREFIX_LEN) : token;
|
|
579
|
-
if (!topText.includes(probe)) return false;
|
|
580
|
-
}
|
|
581
|
-
return true;
|
|
548
|
+
return maxCosine >= cosineThreshold;
|
|
582
549
|
}
|