@totalreclaw/totalreclaw 3.3.12 → 3.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +25 -8
- package/{billing-cache.ts → billing/billing-cache.ts} +1 -1
- package/{relay-headers.ts → billing/relay-headers.ts} +1 -1
- package/{tr-cli-export-helper.ts → cli/tr-cli-export-helper.ts} +5 -5
- package/{tr-cli.ts → cli/tr-cli.ts} +20 -20
- package/{contradiction-sync.ts → contradiction/contradiction-sync.ts} +13 -13
- package/{digest-sync.ts → digest/digest-sync.ts} +32 -2
- package/dist/{billing-cache.js → billing/billing-cache.js} +1 -1
- package/dist/{relay-headers.js → billing/relay-headers.js} +1 -1
- package/dist/{tr-cli-export-helper.js → cli/tr-cli-export-helper.js} +5 -5
- package/dist/{tr-cli.js → cli/tr-cli.js} +19 -19
- package/dist/{contradiction-sync.js → contradiction/contradiction-sync.js} +13 -13
- package/dist/{digest-sync.js → digest/digest-sync.js} +32 -2
- package/dist/{download-ux.js → embedding/download-ux.js} +1 -1
- package/dist/{reranker.js → embedding/reranker.js} +13 -4
- package/dist/{claims-helper.js → extraction/claims-helper.js} +54 -58
- package/dist/extraction/consolidation.js +170 -0
- package/dist/{extractor.js → extraction/extractor.js} +17 -17
- package/dist/extraction/importance-filter.js +38 -0
- package/dist/{semantic-dedup.js → extraction/semantic-dedup.js} +2 -2
- package/dist/{batch-gate.js → import/batch-gate.js} +1 -1
- package/dist/import/import-runtime.js +692 -0
- package/dist/import-adapters/chatgpt-adapter.js +7 -7
- package/dist/import-adapters/gemini-adapter.js +13 -13
- package/dist/import-adapters/mem0-adapter.js +10 -10
- package/dist/index.js +111 -1094
- package/dist/{llm-client.js → llm/llm-client.js} +2 -2
- package/dist/{hot-cache-wrapper.js → memory/hot-cache-wrapper.js} +5 -5
- package/dist/{memory-runtime.js → memory/memory-runtime.js} +1 -1
- package/dist/{pin.js → memory/pin.js} +18 -18
- package/dist/{retype-setscope.js → memory/retype-setscope.js} +19 -19
- package/dist/{credential-provider.js → pairing/credential-provider.js} +2 -2
- package/dist/{first-run.js → pairing/first-run.js} +1 -1
- package/dist/{onboarding-cli.js → pairing/onboarding-cli.js} +20 -20
- package/dist/{pair-cli-relay.js → pairing/pair-cli-relay.js} +10 -10
- package/dist/{pair-crypto.js → pairing/pair-crypto.js} +1 -1
- package/dist/{pair-http.js → pairing/pair-http.js} +3 -3
- package/dist/{pair-remote-client.js → pairing/pair-remote-client.js} +10 -10
- package/dist/{pair-session-store.js → pairing/pair-session-store.js} +4 -4
- package/dist/runtime/config-schema.js +59 -0
- package/dist/runtime/format-helpers.js +257 -0
- package/dist/runtime/types.js +11 -0
- package/dist/{confirm-indexed.js → subgraph/confirm-indexed.js} +1 -1
- package/dist/{subgraph-search.js → subgraph/subgraph-search.js} +14 -14
- package/dist/{subgraph-store.js → subgraph/subgraph-store.js} +6 -6
- package/{download-ux.ts → embedding/download-ux.ts} +1 -1
- package/{reranker.ts → embedding/reranker.ts} +13 -4
- package/{claims-helper.ts → extraction/claims-helper.ts} +52 -60
- package/{consolidation.ts → extraction/consolidation.ts} +63 -154
- package/{extractor.ts → extraction/extractor.ts} +18 -17
- package/extraction/importance-filter.ts +50 -0
- package/{semantic-dedup.ts → extraction/semantic-dedup.ts} +2 -2
- package/{batch-gate.ts → import/batch-gate.ts} +1 -1
- package/import/import-runtime.ts +853 -0
- package/import-adapters/chatgpt-adapter.ts +7 -7
- package/import-adapters/gemini-adapter.ts +13 -13
- package/import-adapters/mem0-adapter.ts +10 -10
- package/index.ts +129 -1377
- package/{llm-client.ts → llm/llm-client.ts} +2 -2
- package/{hot-cache-wrapper.ts → memory/hot-cache-wrapper.ts} +5 -5
- package/{memory-runtime.ts → memory/memory-runtime.ts} +1 -1
- package/{pin.ts → memory/pin.ts} +20 -20
- package/{retype-setscope.ts → memory/retype-setscope.ts} +21 -21
- package/{tool-gating.ts → memory/tool-gating.ts} +1 -1
- package/package.json +6 -6
- package/{credential-provider.ts → pairing/credential-provider.ts} +2 -2
- package/{first-run.ts → pairing/first-run.ts} +1 -1
- package/{onboarding-cli.ts → pairing/onboarding-cli.ts} +20 -20
- package/{pair-cli-relay.ts → pairing/pair-cli-relay.ts} +10 -10
- package/{pair-crypto.ts → pairing/pair-crypto.ts} +1 -1
- package/{pair-http.ts → pairing/pair-http.ts} +3 -3
- package/{pair-remote-client.ts → pairing/pair-remote-client.ts} +10 -10
- package/{pair-session-store.ts → pairing/pair-session-store.ts} +4 -4
- package/runtime/config-schema.ts +64 -0
- package/runtime/format-helpers.ts +293 -0
- package/runtime/types.ts +118 -0
- package/skill.json +44 -10
- package/{confirm-indexed.ts → subgraph/confirm-indexed.ts} +1 -1
- package/{subgraph-search.ts → subgraph/subgraph-search.ts} +14 -14
- package/{subgraph-store.ts → subgraph/subgraph-store.ts} +10 -10
- package/dist/consolidation.js +0 -249
- /package/{api-client.ts → billing/api-client.ts} +0 -0
- /package/{gateway-url.ts → billing/gateway-url.ts} +0 -0
- /package/{inbound-user-tracker.ts → billing/inbound-user-tracker.ts} +0 -0
- /package/{relay.ts → billing/relay.ts} +0 -0
- /package/{crypto.ts → crypto/crypto.ts} +0 -0
- /package/{vault-crypto.ts → crypto/vault-crypto.ts} +0 -0
- /package/dist/{api-client.js → billing/api-client.js} +0 -0
- /package/dist/{gateway-url.js → billing/gateway-url.js} +0 -0
- /package/dist/{inbound-user-tracker.js → billing/inbound-user-tracker.js} +0 -0
- /package/dist/{relay.js → billing/relay.js} +0 -0
- /package/dist/{crypto.js → crypto/crypto.js} +0 -0
- /package/dist/{vault-crypto.js → crypto/vault-crypto.js} +0 -0
- /package/dist/{embedder-cache.js → embedding/embedder-cache.js} +0 -0
- /package/dist/{embedder-loader.js → embedding/embedder-loader.js} +0 -0
- /package/dist/{embedder-network.js → embedding/embedder-network.js} +0 -0
- /package/dist/{embedding.js → embedding/embedding.js} +0 -0
- /package/dist/{lsh.js → embedding/lsh.js} +0 -0
- /package/dist/{import-state-manager.js → import/import-state-manager.js} +0 -0
- /package/dist/{llm-profile-reader.js → llm/llm-profile-reader.js} +0 -0
- /package/dist/{native-memory.js → memory/native-memory.js} +0 -0
- /package/dist/{tool-gating.js → memory/tool-gating.js} +0 -0
- /package/dist/{tools.js → memory/tools.js} +0 -0
- /package/dist/{generate-mnemonic.js → pairing/generate-mnemonic.js} +0 -0
- /package/dist/{pair-cli.js → pairing/pair-cli.js} +0 -0
- /package/dist/{pair-page.js → pairing/pair-page.js} +0 -0
- /package/dist/{pair-qr.js → pairing/pair-qr.js} +0 -0
- /package/dist/{restart-auth.js → pairing/restart-auth.js} +0 -0
- /package/dist/{qa-bug-report.js → setup/qa-bug-report.js} +0 -0
- /package/dist/{skill-register.js → setup/skill-register.js} +0 -0
- /package/dist/{trajectory-poller.js → subgraph/trajectory-poller.js} +0 -0
- /package/{embedder-cache.ts → embedding/embedder-cache.ts} +0 -0
- /package/{embedder-loader.ts → embedding/embedder-loader.ts} +0 -0
- /package/{embedder-network.ts → embedding/embedder-network.ts} +0 -0
- /package/{embedding.ts → embedding/embedding.ts} +0 -0
- /package/{lsh.ts → embedding/lsh.ts} +0 -0
- /package/{import-state-manager.ts → import/import-state-manager.ts} +0 -0
- /package/{llm-profile-reader.ts → llm/llm-profile-reader.ts} +0 -0
- /package/{native-memory.ts → memory/native-memory.ts} +0 -0
- /package/{tools.ts → memory/tools.ts} +0 -0
- /package/{generate-mnemonic.ts → pairing/generate-mnemonic.ts} +0 -0
- /package/{pair-cli.ts → pairing/pair-cli.ts} +0 -0
- /package/{pair-page.ts → pairing/pair-page.ts} +0 -0
- /package/{pair-qr.ts → pairing/pair-qr.ts} +0 -0
- /package/{restart-auth.ts → pairing/restart-auth.ts} +0 -0
- /package/{qa-bug-report.ts → setup/qa-bug-report.ts} +0 -0
- /package/{skill-register.ts → setup/skill-register.ts} +0 -0
- /package/{trajectory-poller.ts → subgraph/trajectory-poller.ts} +0 -0
package/dist/index.js
CHANGED
|
@@ -65,42 +65,45 @@
|
|
|
65
65
|
* All data is encrypted client-side with XChaCha20-Poly1305. The server never
|
|
66
66
|
* sees plaintext.
|
|
67
67
|
*/
|
|
68
|
-
import { deriveKeys, deriveLshSeed, computeAuthKeyHash,
|
|
69
|
-
import { createApiClient } from './api-client.js';
|
|
70
|
-
import { extractFacts, extractCrystal,
|
|
71
|
-
import { initLLMClient, resolveLLMConfig, chatCompletion, generateEmbedding, getEmbeddingDims, getEmbeddingModelId, configureEmbedder, prefetchEmbedderBundle, } from './llm-client.js';
|
|
72
|
-
import { defaultAuthProfilesRoot, readAllProfileKeys, dedupeByProvider, } from './llm-profile-reader.js';
|
|
73
|
-
import { LSHHasher } from './lsh.js';
|
|
74
|
-
import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS } from './reranker.js';
|
|
75
|
-
import { deduplicateBatch } from './semantic-dedup.js';
|
|
76
|
-
import { startTrajectoryPoller } from './trajectory-poller.js';
|
|
77
|
-
import { findNearDuplicate, shouldSupersede, getStoreDedupThreshold, STORE_DEDUP_MAX_CANDIDATES, } from './consolidation.js';
|
|
78
|
-
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4 } from './subgraph-store.js';
|
|
79
|
-
import { DIGEST_TRAPDOOR, buildCanonicalClaim, computeEntityTrapdoors, isDigestBlob, readClaimFromBlob, resolveDigestMode, } from './claims-helper.js';
|
|
80
|
-
import { maybeInjectDigest, recompileDigest, fetchAllActiveClaims, isRecompileInProgress, tryBeginRecompile, endRecompile, } from './digest-sync.js';
|
|
81
|
-
import { detectAndResolveContradictions, runWeightTuningLoop, } from './contradiction-sync.js';
|
|
82
|
-
import { searchSubgraph, searchSubgraphBroadened, getSubgraphFactCount, fetchFactById } from './subgraph-search.js';
|
|
83
|
-
import { PluginHotCache } from './hot-cache-wrapper.js';
|
|
68
|
+
import { deriveKeys, deriveLshSeed, computeAuthKeyHash, generateBlindIndices, generateContentFingerprint, } from './crypto/crypto.js';
|
|
69
|
+
import { createApiClient } from './billing/api-client.js';
|
|
70
|
+
import { extractFacts, extractCrystal, extractFactsForCompaction, } from './extraction/extractor.js';
|
|
71
|
+
import { initLLMClient, resolveLLMConfig, chatCompletion, generateEmbedding, getEmbeddingDims, getEmbeddingModelId, configureEmbedder, prefetchEmbedderBundle, } from './llm/llm-client.js';
|
|
72
|
+
import { defaultAuthProfilesRoot, readAllProfileKeys, dedupeByProvider, } from './llm/llm-profile-reader.js';
|
|
73
|
+
import { LSHHasher } from './embedding/lsh.js';
|
|
74
|
+
import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS } from './embedding/reranker.js';
|
|
75
|
+
import { deduplicateBatch } from './extraction/semantic-dedup.js';
|
|
76
|
+
import { startTrajectoryPoller } from './subgraph/trajectory-poller.js';
|
|
77
|
+
import { findNearDuplicate, shouldSupersede, getStoreDedupThreshold, STORE_DEDUP_MAX_CANDIDATES, } from './extraction/consolidation.js';
|
|
78
|
+
import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4 } from './subgraph/subgraph-store.js';
|
|
79
|
+
import { DIGEST_TRAPDOOR, buildCanonicalClaim, computeEntityTrapdoors, isDigestBlob, readClaimFromBlob, resolveDigestMode, } from './extraction/claims-helper.js';
|
|
80
|
+
import { maybeInjectDigest, recompileDigest, fetchAllActiveClaims, isRecompileInProgress, tryBeginRecompile, endRecompile, } from './digest/digest-sync.js';
|
|
81
|
+
import { detectAndResolveContradictions, runWeightTuningLoop, } from './contradiction/contradiction-sync.js';
|
|
82
|
+
import { searchSubgraph, searchSubgraphBroadened, getSubgraphFactCount, fetchFactById } from './subgraph/subgraph-search.js';
|
|
83
|
+
import { PluginHotCache } from './memory/hot-cache-wrapper.js';
|
|
84
84
|
import { CONFIG, setRecoveryPhraseOverride } from './config.js';
|
|
85
|
-
import { buildRelayHeaders } from './relay-headers.js';
|
|
86
|
-
import { readBillingCache, writeBillingCache, BILLING_CACHE_PATH, } from './billing-cache.js';
|
|
87
|
-
import { ensureMemoryHeaderFile, loadCredentialsJson, writeCredentialsJson, deleteCredentialsFile,
|
|
88
|
-
import { isRcBuild } from './qa-bug-report.js';
|
|
89
|
-
import { decideToolGate, isGatedToolName } from './tool-gating.js';
|
|
90
|
-
import { resolveRestartAuth, rejectMessageFor, } from './restart-auth.js';
|
|
91
|
-
import { recordInboundUser, getDistinctInboundUserCount, resolveTrackerPath, } from './inbound-user-tracker.js';
|
|
92
|
-
import { detectFirstRun, buildWelcomePrepend } from './first-run.js';
|
|
93
|
-
import { buildPairRoutes } from './pair-http.js';
|
|
94
|
-
import {
|
|
95
|
-
import {
|
|
96
|
-
import {
|
|
85
|
+
import { buildRelayHeaders } from './billing/relay-headers.js';
|
|
86
|
+
import { readBillingCache, writeBillingCache, BILLING_CACHE_PATH, } from './billing/billing-cache.js';
|
|
87
|
+
import { ensureMemoryHeaderFile, loadCredentialsJson, writeCredentialsJson, deleteCredentialsFile, deleteFileIfExists, resolveOnboardingState, writeOnboardingState, readPluginVersion, cleanupInstallStagingDirs, clearPartialInstallMarker, patchOpenClawConfig, checkCredentialsFileMode, } from './fs-helpers.js';
|
|
88
|
+
import { isRcBuild } from './setup/qa-bug-report.js';
|
|
89
|
+
import { decideToolGate, isGatedToolName } from './memory/tool-gating.js';
|
|
90
|
+
import { resolveRestartAuth, rejectMessageFor, } from './pairing/restart-auth.js';
|
|
91
|
+
import { recordInboundUser, getDistinctInboundUserCount, resolveTrackerPath, } from './billing/inbound-user-tracker.js';
|
|
92
|
+
import { detectFirstRun, buildWelcomePrepend } from './pairing/first-run.js';
|
|
93
|
+
import { buildPairRoutes } from './pairing/pair-http.js';
|
|
94
|
+
import { registerNativeMemory } from './memory/native-memory.js';
|
|
95
|
+
import { ensureSkillRegistered } from './setup/skill-register.js';
|
|
96
|
+
import { humanizeError, buildPairingUrl, resolveGatewayMode, computeCandidatePool, encryptToHex, decryptFromHex, relativeTime, } from './runtime/format-helpers.js';
|
|
97
|
+
import { CONFIG_SCHEMA } from './runtime/config-schema.js';
|
|
98
|
+
import { filterByImportance } from './extraction/importance-filter.js';
|
|
99
|
+
import { handlePluginImportFrom, handleImportStatus, handleImportAbort, configureImportRuntime, isImportInProgress, setImportInProgress, } from './import/import-runtime.js';
|
|
97
100
|
import { validateMnemonic } from '@scure/bip39';
|
|
98
101
|
import { wordlist } from '@scure/bip39/wordlists/english.js';
|
|
99
102
|
import crypto from 'node:crypto';
|
|
100
103
|
import { createRequire } from 'node:module';
|
|
101
104
|
import { fileURLToPath } from 'node:url';
|
|
102
105
|
import * as nodePath from 'node:path';
|
|
103
|
-
import {
|
|
106
|
+
import { isImportStale, readMostRecentActiveImport, } from './import/import-state-manager.js';
|
|
104
107
|
// CJS-style require for the @totalreclaw/core WASM module. We keep this
|
|
105
108
|
// load path lazy (only inside getSmartImportWasm() below) so a partial
|
|
106
109
|
// install of the dependency tree doesn't crash module init. Bare
|
|
@@ -111,35 +114,8 @@ import { writeImportState, readImportState, isImportStale, readMostRecentActiveI
|
|
|
111
114
|
// subgraph-store.ts / claims-helper.ts.
|
|
112
115
|
const __cjsRequire = createRequire(import.meta.url);
|
|
113
116
|
// ---------------------------------------------------------------------------
|
|
114
|
-
//
|
|
117
|
+
// OpenClaw Plugin API type — extracted to ./runtime/types.ts (imported above).
|
|
115
118
|
// ---------------------------------------------------------------------------
|
|
116
|
-
/**
|
|
117
|
-
* Translate technical error messages from the on-chain submission pipeline
|
|
118
|
-
* into user-friendly messages. The original technical details are still
|
|
119
|
-
* logged via api.logger — this only affects what the agent sees.
|
|
120
|
-
*/
|
|
121
|
-
function humanizeError(rawMessage) {
|
|
122
|
-
if (rawMessage.includes('AA23')) {
|
|
123
|
-
return 'Memory storage temporarily unavailable. Will retry next time.';
|
|
124
|
-
}
|
|
125
|
-
if (rawMessage.includes('AA10')) {
|
|
126
|
-
return 'Please wait a moment before storing more memories.';
|
|
127
|
-
}
|
|
128
|
-
if (rawMessage.includes('AA25')) {
|
|
129
|
-
return 'Memory storage busy. Will retry.';
|
|
130
|
-
}
|
|
131
|
-
if (rawMessage.includes('pm_sponsorUserOperation')) {
|
|
132
|
-
return 'Memory storage service temporarily unavailable.';
|
|
133
|
-
}
|
|
134
|
-
if (/Relay returned HTTP\s*404/.test(rawMessage)) {
|
|
135
|
-
return 'Memory service is temporarily offline.';
|
|
136
|
-
}
|
|
137
|
-
if (/Relay returned HTTP\s*5\d\d/.test(rawMessage)) {
|
|
138
|
-
return 'Memory service encountered a temporary error. Will retry next time.';
|
|
139
|
-
}
|
|
140
|
-
// Pass through non-technical messages as-is.
|
|
141
|
-
return rawMessage;
|
|
142
|
-
}
|
|
143
119
|
// ---------------------------------------------------------------------------
|
|
144
120
|
// Persistent credential storage
|
|
145
121
|
// ---------------------------------------------------------------------------
|
|
@@ -148,153 +124,7 @@ const CREDENTIALS_PATH = CONFIG.credentialsPath;
|
|
|
148
124
|
// ---------------------------------------------------------------------------
|
|
149
125
|
// 3.3.0 — pairing URL resolution
|
|
150
126
|
// ---------------------------------------------------------------------------
|
|
151
|
-
|
|
152
|
-
* Build the full pairing URL (including `#pk=` fragment) for a fresh
|
|
153
|
-
* pairing session. Pulls gateway config from `api.config.gateway`.
|
|
154
|
-
*
|
|
155
|
-
* Resolution order (3.3.1 — six-layer cascade):
|
|
156
|
-
* 1. `plugins.entries.totalreclaw.config.publicUrl` — explicit override
|
|
157
|
-
* 2. `gateway.remote.url` — OpenClaw's own remote-gateway URL
|
|
158
|
-
* 3. `gateway.bind === 'custom'` + `gateway.customBindHost` + port
|
|
159
|
-
* 4. Tailscale auto-detect — `tailscale status --json` → `https://<MagicDNS>`
|
|
160
|
-
* (assumes `tailscale serve` proxies to the gateway port on 443)
|
|
161
|
-
* 5. LAN auto-detect — first non-loopback, non-virtual IPv4 interface.
|
|
162
|
-
* Emits a warning: "only works on the same network".
|
|
163
|
-
* 6. Fallback `http://localhost:<port>` — warns with a pointer to
|
|
164
|
-
* configure `plugins.entries.totalreclaw.config.publicUrl`.
|
|
165
|
-
*
|
|
166
|
-
* Always returns a working URL string; never throws. The caller (CLI or
|
|
167
|
-
* JSON output) prints whatever we give it.
|
|
168
|
-
*/
|
|
169
|
-
function buildPairingUrl(api, session) {
|
|
170
|
-
const cfg = api.config;
|
|
171
|
-
const pluginCfg = (api.pluginConfig ?? {});
|
|
172
|
-
const tlsEnabled = cfg?.gateway?.tls?.enabled === true;
|
|
173
|
-
const scheme = tlsEnabled ? 'https' : 'http';
|
|
174
|
-
const port = cfg?.gateway?.port ?? 18789;
|
|
175
|
-
let base;
|
|
176
|
-
// Layer 1 — explicit user override
|
|
177
|
-
if (typeof pluginCfg.publicUrl === 'string' && pluginCfg.publicUrl.trim()) {
|
|
178
|
-
base = pluginCfg.publicUrl.replace(/\/+$/, '');
|
|
179
|
-
base = base.replace(/^wss:\/\//i, 'https://').replace(/^ws:\/\//i, 'http://');
|
|
180
|
-
}
|
|
181
|
-
// Layer 2 — OpenClaw gateway remote URL
|
|
182
|
-
else if (typeof cfg?.gateway?.remote?.url === 'string' && cfg.gateway.remote.url.trim()) {
|
|
183
|
-
base = cfg.gateway.remote.url.trim().replace(/\/+$/, '');
|
|
184
|
-
base = base.replace(/^wss:\/\//i, 'https://').replace(/^ws:\/\//i, 'http://');
|
|
185
|
-
}
|
|
186
|
-
// Layer 3 — gateway.bind = custom + explicit customBindHost
|
|
187
|
-
else if (cfg?.gateway?.bind === 'custom' && cfg.gateway.customBindHost) {
|
|
188
|
-
base = `${scheme}://${cfg.gateway.customBindHost}:${port}`;
|
|
189
|
-
}
|
|
190
|
-
// Layers 4 + 5 — auto-detect via gateway-url helper (Tailscale CGNAT, then LAN)
|
|
191
|
-
else {
|
|
192
|
-
let detected = null;
|
|
193
|
-
// issue #110 fix 4 — pass `isDocker` so LAN detection skips
|
|
194
|
-
// 172.16/12 bridge IPs that no external browser can reach.
|
|
195
|
-
let isDocker = false;
|
|
196
|
-
try {
|
|
197
|
-
isDocker = isRunningInDocker();
|
|
198
|
-
}
|
|
199
|
-
catch {
|
|
200
|
-
// Defensive: never block URL building on Docker sniff errors.
|
|
201
|
-
isDocker = false;
|
|
202
|
-
}
|
|
203
|
-
try {
|
|
204
|
-
detected = detectGatewayHost({ isDocker });
|
|
205
|
-
}
|
|
206
|
-
catch (err) {
|
|
207
|
-
api.logger.warn(`TotalReclaw: host autodetect crashed: ${err instanceof Error ? err.message : String(err)} — falling back to localhost`);
|
|
208
|
-
}
|
|
209
|
-
if (detected?.kind === 'tailscale') {
|
|
210
|
-
// 3.3.1-rc.2: we surface the raw Tailscale CGNAT IP because passive
|
|
211
|
-
// NIC detection (no subprocess) cannot resolve the MagicDNS name.
|
|
212
|
-
// Caller can override via `publicUrl` for a proper https://<magicdns>.
|
|
213
|
-
// The IP + port URL still works inside the tailnet (peers can reach
|
|
214
|
-
// each other by CGNAT IP directly). TLS defaults to the gateway's
|
|
215
|
-
// own config because we no longer assume `tailscale serve`.
|
|
216
|
-
base = `${scheme}://${detected.host}:${port}`;
|
|
217
|
-
api.logger.warn(`TotalReclaw: pairing URL using Tailscale CGNAT IP ${detected.host}:${port} — ` +
|
|
218
|
-
detected.note);
|
|
219
|
-
}
|
|
220
|
-
else if (detected?.kind === 'lan') {
|
|
221
|
-
base = `${scheme}://${detected.host}:${port}`;
|
|
222
|
-
api.logger.warn(`TotalReclaw: pairing URL using LAN host ${detected.host}:${port} — ` +
|
|
223
|
-
`this URL only works from the same network. ` +
|
|
224
|
-
`Set plugins.entries.totalreclaw.config.publicUrl for remote access.`);
|
|
225
|
-
}
|
|
226
|
-
else {
|
|
227
|
-
// Layer 6 — localhost fallback (or Docker-aware relay-pointer warning)
|
|
228
|
-
const bind = cfg?.gateway?.bind;
|
|
229
|
-
if (isDocker) {
|
|
230
|
-
// issue #110 fix 4: inside Docker the LAN IP is container-internal
|
|
231
|
-
// and useless. Loopback localhost only works for `docker exec`
|
|
232
|
-
// tests. The CORRECT pair URL for Docker is the relay-brokered
|
|
233
|
-
// path served by `tr pair` / the `/plugin/totalreclaw/pair/*` HTTP
|
|
234
|
-
// routes (CONFIG.pairMode === 'relay' since rc.11). The CLI-only
|
|
235
|
-
// path here cannot mint a relay session synchronously (the relay
|
|
236
|
-
// handshake needs a WS round-trip), so we emit the loopback URL
|
|
237
|
-
// with a LOUD warning pointing the operator at the pair CLI /
|
|
238
|
-
// publicUrl override.
|
|
239
|
-
api.logger.warn(`TotalReclaw: Docker container detected — pairing URL falling back to ` +
|
|
240
|
-
`http://localhost:${port}, which is unreachable from the host browser. ` +
|
|
241
|
-
`Run \`tr pair --url-pin\` (or \`openclaw totalreclaw pair generate --url-pin-only\`) ` +
|
|
242
|
-
`on the gateway host to mint a relay-brokered pair URL that reaches the host browser, ` +
|
|
243
|
-
`OR set plugins.entries.totalreclaw.config.publicUrl ` +
|
|
244
|
-
`to your gateway's host-reachable URL (e.g. http://<host-ip>:${port} when the ` +
|
|
245
|
-
`Docker port is published). Setting TOTALRECLAW_PAIR_MODE=relay is the default; ` +
|
|
246
|
-
`air-gapped operators on TOTALRECLAW_PAIR_MODE=local must publish a port + set publicUrl.`);
|
|
247
|
-
}
|
|
248
|
-
else if (bind === 'lan' || bind === 'tailnet') {
|
|
249
|
-
api.logger.warn(`TotalReclaw: pairing URL falling back to localhost because gateway.bind=${bind} could not be autodetected. ` +
|
|
250
|
-
'Set plugins.entries.totalreclaw.config.publicUrl to override.');
|
|
251
|
-
}
|
|
252
|
-
else {
|
|
253
|
-
api.logger.warn(`TotalReclaw: pairing URL fell back to http://localhost:${port} — this URL only works on this machine. ` +
|
|
254
|
-
`Configure plugins.entries.totalreclaw.config.publicUrl for remote access.`);
|
|
255
|
-
}
|
|
256
|
-
base = `${scheme}://localhost:${port}`;
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
return `${base}/plugin/totalreclaw/pair/finish?sid=${encodeURIComponent(session.sid)}#pk=${encodeURIComponent(session.pkGatewayB64)}`;
|
|
260
|
-
}
|
|
261
|
-
/**
|
|
262
|
-
* Resolve whether this plugin is running on a `local` or `remote` gateway.
|
|
263
|
-
*
|
|
264
|
-
* Follows the same config surface `buildPairingUrl` uses:
|
|
265
|
-
* - `pluginConfig.publicUrl` set + non-localhost → remote
|
|
266
|
-
* - `gateway.remote.url` set + non-localhost → remote
|
|
267
|
-
* - `gateway.bind === 'lan' | 'tailnet' | 'custom'` → remote
|
|
268
|
-
* - anything else → local
|
|
269
|
-
*
|
|
270
|
-
* We treat a `publicUrl` or `remote.url` that points at `localhost` /
|
|
271
|
-
* `127.*` as local because that is what a dev-loopback override looks like;
|
|
272
|
-
* no one publishes a remote QR pairing for localhost.
|
|
273
|
-
*/
|
|
274
|
-
function resolveGatewayMode(api) {
|
|
275
|
-
const cfg = api.config;
|
|
276
|
-
const pluginCfg = (api.pluginConfig ?? {});
|
|
277
|
-
const looksLocal = (url) => {
|
|
278
|
-
if (!url)
|
|
279
|
-
return true;
|
|
280
|
-
const u = url.trim().toLowerCase();
|
|
281
|
-
if (u === '')
|
|
282
|
-
return true;
|
|
283
|
-
return /^(?:wss?:\/\/|https?:\/\/)?(?:localhost|127\.|0\.0\.0\.0)/.test(u);
|
|
284
|
-
};
|
|
285
|
-
if (typeof pluginCfg.publicUrl === 'string' && !looksLocal(pluginCfg.publicUrl)) {
|
|
286
|
-
return 'remote';
|
|
287
|
-
}
|
|
288
|
-
const remoteUrl = cfg?.gateway?.remote?.url;
|
|
289
|
-
if (typeof remoteUrl === 'string' && !looksLocal(remoteUrl)) {
|
|
290
|
-
return 'remote';
|
|
291
|
-
}
|
|
292
|
-
const bind = cfg?.gateway?.bind;
|
|
293
|
-
if (bind === 'lan' || bind === 'tailnet' || bind === 'custom') {
|
|
294
|
-
return 'remote';
|
|
295
|
-
}
|
|
296
|
-
return 'local';
|
|
297
|
-
}
|
|
127
|
+
// buildPairingUrl / resolveGatewayMode extracted to ./runtime/format-helpers.ts.
|
|
298
128
|
// ---------------------------------------------------------------------------
|
|
299
129
|
// Cosine similarity threshold — skip injection when top result is below this
|
|
300
130
|
// ---------------------------------------------------------------------------
|
|
@@ -330,9 +160,6 @@ const CACHE_TTL_MS = CONFIG.cacheTtlMs;
|
|
|
330
160
|
const SEMANTIC_SKIP_THRESHOLD = CONFIG.semanticSkipThreshold;
|
|
331
161
|
// Auto-extract throttle (C3): only extract every N turns in agent_end hook
|
|
332
162
|
let turnsSinceLastExtraction = 0;
|
|
333
|
-
// BUG-2 fix: Skip agent_end extraction during import operations.
|
|
334
|
-
// Import failures previously triggered agent_end → re-extraction → re-import loops.
|
|
335
|
-
let _importInProgress = false;
|
|
336
163
|
const AUTO_EXTRACT_EVERY_TURNS_ENV = CONFIG.extractInterval;
|
|
337
164
|
// Hard cap on facts per extraction to prevent LLM over-extraction from dense conversations
|
|
338
165
|
const MAX_FACTS_PER_EXTRACTION = 15;
|
|
@@ -469,24 +296,7 @@ let cachedFactCount = null;
|
|
|
469
296
|
let lastFactCountFetch = 0;
|
|
470
297
|
/** Cache TTL for fact count: 5 minutes. */
|
|
471
298
|
const FACT_COUNT_CACHE_TTL = 5 * 60 * 1000;
|
|
472
|
-
|
|
473
|
-
* Compute the candidate pool size from a fact count.
|
|
474
|
-
*
|
|
475
|
-
* Server-side config takes priority (from billing cache), then local fallback.
|
|
476
|
-
* The server computes the optimal pool based on vault size and tier caps.
|
|
477
|
-
*
|
|
478
|
-
* Local fallback formula: pool = min(max(factCount * 3, 400), 5000)
|
|
479
|
-
* - At least 400 candidates (even for tiny vaults)
|
|
480
|
-
* - At most 5000 candidates (to bound decryption + reranking cost)
|
|
481
|
-
* - 3x fact count in between
|
|
482
|
-
*/
|
|
483
|
-
function computeCandidatePool(factCount) {
|
|
484
|
-
const cache = readBillingCache();
|
|
485
|
-
if (cache?.features?.max_candidate_pool != null)
|
|
486
|
-
return cache.features.max_candidate_pool;
|
|
487
|
-
// Fallback to local formula if no server config
|
|
488
|
-
return Math.min(Math.max(factCount * 3, 400), 5000);
|
|
489
|
-
}
|
|
299
|
+
// computeCandidatePool extracted to ./runtime/format-helpers.ts.
|
|
490
300
|
/**
|
|
491
301
|
* Fetch the user's fact count from the server, with caching.
|
|
492
302
|
*
|
|
@@ -773,9 +583,6 @@ async function initialize(logger) {
|
|
|
773
583
|
}
|
|
774
584
|
}
|
|
775
585
|
}
|
|
776
|
-
function isDocker() {
|
|
777
|
-
return isRunningInDocker();
|
|
778
|
-
}
|
|
779
586
|
function buildSetupErrorMsg() {
|
|
780
587
|
// NOTE: the legacy `totalreclaw_setup` agent tool was retired in 3.2.0
|
|
781
588
|
// (phrase-safety: the agent must never accept or relay a recovery phrase).
|
|
@@ -788,28 +595,6 @@ function buildSetupErrorMsg() {
|
|
|
788
595
|
'and hand the user the returned `url` and `pin`. The user opens the URL in a browser to complete pairing. ' +
|
|
789
596
|
'Do NOT ask the user for a recovery phrase and do NOT attempt to generate or relay one yourself.';
|
|
790
597
|
}
|
|
791
|
-
function buildSetupErrorMsgLegacy() {
|
|
792
|
-
const base = 'TotalReclaw setup required:\n' +
|
|
793
|
-
'1. Set TOTALRECLAW_RECOVERY_PHRASE — ask the user if they have an existing recovery phrase or generate a new 12-word recovery phrase.\n' +
|
|
794
|
-
'2. Restart the gateway to apply changes.\n' +
|
|
795
|
-
' (Optional: set TOTALRECLAW_SELF_HOSTED=true if using your own server instead of the managed service.)\n\n';
|
|
796
|
-
if (isDocker()) {
|
|
797
|
-
return base +
|
|
798
|
-
'Running in Docker — pass env vars via `-e` flags or your compose file:\n' +
|
|
799
|
-
' -e TOTALRECLAW_RECOVERY_PHRASE="word1 word2 ..."';
|
|
800
|
-
}
|
|
801
|
-
if (process.platform === 'darwin') {
|
|
802
|
-
return base +
|
|
803
|
-
'Running on macOS — add env vars to the LaunchAgent plist at\n' +
|
|
804
|
-
'~/Library/LaunchAgents/ai.openclaw.gateway.plist under <key>EnvironmentVariables</key>:\n' +
|
|
805
|
-
' <key>TOTALRECLAW_RECOVERY_PHRASE</key><string>word1 word2 ...</string>\n' +
|
|
806
|
-
'Then run: openclaw gateway restart';
|
|
807
|
-
}
|
|
808
|
-
return base +
|
|
809
|
-
'Running on Linux — add env vars to the systemd unit override or your shell profile:\n' +
|
|
810
|
-
' export TOTALRECLAW_RECOVERY_PHRASE="word1 word2 ..."\n' +
|
|
811
|
-
'Then run: openclaw gateway restart';
|
|
812
|
-
}
|
|
813
598
|
const SETUP_ERROR_MSG = buildSetupErrorMsg();
|
|
814
599
|
/**
|
|
815
600
|
* Ensure `initialize()` has completed (runs at most once).
|
|
@@ -1056,10 +841,10 @@ async function searchForNearDuplicates(factText, factEmbedding, allIndices, logg
|
|
|
1056
841
|
}
|
|
1057
842
|
if (decryptedCandidates.length === 0)
|
|
1058
843
|
return null;
|
|
1059
|
-
const
|
|
1060
|
-
if (!
|
|
844
|
+
const nearDup = findNearDuplicate(factEmbedding, decryptedCandidates, getStoreDedupThreshold());
|
|
845
|
+
if (!nearDup)
|
|
1061
846
|
return null;
|
|
1062
|
-
return { match:
|
|
847
|
+
return { match: nearDup.existingFact, similarity: nearDup.similarity };
|
|
1063
848
|
}
|
|
1064
849
|
catch (err) {
|
|
1065
850
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -1070,16 +855,7 @@ async function searchForNearDuplicates(factText, factEmbedding, allIndices, logg
|
|
|
1070
855
|
// ---------------------------------------------------------------------------
|
|
1071
856
|
// Utility helpers
|
|
1072
857
|
// ---------------------------------------------------------------------------
|
|
1073
|
-
|
|
1074
|
-
* Encrypt a plaintext document string and return its hex-encoded ciphertext.
|
|
1075
|
-
*
|
|
1076
|
-
* The server stores blobs as hex (not base64), so we convert the base64
|
|
1077
|
-
* output of `encrypt()` into hex.
|
|
1078
|
-
*/
|
|
1079
|
-
function encryptToHex(plaintext, key) {
|
|
1080
|
-
const b64 = encrypt(plaintext, key);
|
|
1081
|
-
return Buffer.from(b64, 'base64').toString('hex');
|
|
1082
|
-
}
|
|
858
|
+
// encryptToHex extracted to ./runtime/format-helpers.ts.
|
|
1083
859
|
// Plugin v3.0.0 removed the legacy claim-format fallback. Write path
|
|
1084
860
|
// always emits Memory Taxonomy v1 JSON blobs. The logClaimFormatOnce
|
|
1085
861
|
// helper is gone along with TOTALRECLAW_CLAIM_FORMAT / TOTALRECLAW_TAXONOMY_VERSION.
|
|
@@ -1172,8 +948,8 @@ function scheduleDigestRecompile(previousClaimId, logger) {
|
|
|
1172
948
|
version: PROTOBUF_VERSION_V4,
|
|
1173
949
|
});
|
|
1174
950
|
const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
|
|
1175
|
-
const
|
|
1176
|
-
if (!
|
|
951
|
+
const batchResult = await submitFactBatchOnChain([protobuf], config);
|
|
952
|
+
if (!batchResult.success) {
|
|
1177
953
|
throw new Error('Digest store UserOp did not succeed on-chain');
|
|
1178
954
|
}
|
|
1179
955
|
};
|
|
@@ -1201,8 +977,8 @@ function scheduleDigestRecompile(previousClaimId, logger) {
|
|
|
1201
977
|
};
|
|
1202
978
|
const protobuf = encodeFactProtobuf(tombstone);
|
|
1203
979
|
const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
|
|
1204
|
-
const
|
|
1205
|
-
if (!
|
|
980
|
+
const batchResult = await submitFactBatchOnChain([protobuf], config);
|
|
981
|
+
if (!batchResult.success) {
|
|
1206
982
|
throw new Error('Digest tombstone UserOp did not succeed on-chain');
|
|
1207
983
|
}
|
|
1208
984
|
};
|
|
@@ -1235,14 +1011,11 @@ function scheduleDigestRecompile(previousClaimId, logger) {
|
|
|
1235
1011
|
endRecompile();
|
|
1236
1012
|
});
|
|
1237
1013
|
}
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
const b64 = Buffer.from(hex, 'hex').toString('base64');
|
|
1244
|
-
return decrypt(b64, key);
|
|
1245
|
-
}
|
|
1014
|
+
// decryptFromHex extracted to ./runtime/format-helpers.ts.
|
|
1015
|
+
// ---------------------------------------------------------------------------
|
|
1016
|
+
// Migration GraphQL helpers
|
|
1017
|
+
// ---------------------------------------------------------------------------
|
|
1018
|
+
// MigrationFact — extracted to ./runtime/types.ts (imported above).
|
|
1246
1019
|
const MIGRATION_PAGE_SIZE = 1000;
|
|
1247
1020
|
/** Execute a GraphQL query against a subgraph endpoint. Returns null on error. */
|
|
1248
1021
|
async function migrationGqlQuery(endpoint, query, variables, authKey) {
|
|
@@ -1279,8 +1052,8 @@ async function fetchAllFactsByOwner(subgraphUrl, owner, authKey) {
|
|
|
1279
1052
|
const vars = hasLastId
|
|
1280
1053
|
? { owner, first: MIGRATION_PAGE_SIZE, lastId }
|
|
1281
1054
|
: { owner, first: MIGRATION_PAGE_SIZE };
|
|
1282
|
-
const
|
|
1283
|
-
const facts =
|
|
1055
|
+
const subgraphResponse = await migrationGqlQuery(subgraphUrl, query, vars, authKey);
|
|
1056
|
+
const facts = subgraphResponse?.facts ?? [];
|
|
1284
1057
|
if (facts.length === 0)
|
|
1285
1058
|
break;
|
|
1286
1059
|
allFacts.push(...facts);
|
|
@@ -1302,8 +1075,8 @@ async function fetchContentFingerprintsByOwner(subgraphUrl, owner, authKey) {
|
|
|
1302
1075
|
const vars = hasLastId
|
|
1303
1076
|
? { owner, first: MIGRATION_PAGE_SIZE, lastId }
|
|
1304
1077
|
: { owner, first: MIGRATION_PAGE_SIZE };
|
|
1305
|
-
const
|
|
1306
|
-
const facts =
|
|
1078
|
+
const subgraphResponse = await migrationGqlQuery(subgraphUrl, query, vars, authKey);
|
|
1079
|
+
const facts = subgraphResponse?.facts ?? [];
|
|
1307
1080
|
if (facts.length === 0)
|
|
1308
1081
|
break;
|
|
1309
1082
|
for (const f of facts) {
|
|
@@ -1318,19 +1091,19 @@ async function fetchContentFingerprintsByOwner(subgraphUrl, owner, authKey) {
|
|
|
1318
1091
|
}
|
|
1319
1092
|
/** Fetch blind index hashes for given fact IDs. */
|
|
1320
1093
|
async function fetchBlindIndicesByFactIds(subgraphUrl, factIds, authKey) {
|
|
1321
|
-
const
|
|
1094
|
+
const hashesByFactId = new Map();
|
|
1322
1095
|
const CHUNK = 50;
|
|
1323
1096
|
for (let i = 0; i < factIds.length; i += CHUNK) {
|
|
1324
1097
|
const chunk = factIds.slice(i, i + CHUNK);
|
|
1325
1098
|
const query = `query($factIds:[String!]!,$first:Int!){blindIndexes(where:{fact_in:$factIds},first:$first){hash fact{id}}}`;
|
|
1326
|
-
const
|
|
1327
|
-
for (const entry of
|
|
1328
|
-
const existing =
|
|
1099
|
+
const subgraphResponse = await migrationGqlQuery(subgraphUrl, query, { factIds: chunk, first: 1000 }, authKey);
|
|
1100
|
+
for (const entry of subgraphResponse?.blindIndexes ?? []) {
|
|
1101
|
+
const existing = hashesByFactId.get(entry.fact.id) || [];
|
|
1329
1102
|
existing.push(entry.hash);
|
|
1330
|
-
|
|
1103
|
+
hashesByFactId.set(entry.fact.id, existing);
|
|
1331
1104
|
}
|
|
1332
1105
|
}
|
|
1333
|
-
return
|
|
1106
|
+
return hashesByFactId;
|
|
1334
1107
|
}
|
|
1335
1108
|
/**
|
|
1336
1109
|
* Fetch existing memories from the vault to provide dedup context for extraction.
|
|
@@ -1405,34 +1178,11 @@ async function fetchExistingMemoriesForExtraction(logger, limit = 30, rawMessage
|
|
|
1405
1178
|
* Simple text-overlap scoring between a query and a candidate document.
|
|
1406
1179
|
* Returns the number of overlapping lowercase words.
|
|
1407
1180
|
*/
|
|
1408
|
-
|
|
1409
|
-
const queryWords = new Set(query.toLowerCase().split(/\s+/).filter((w) => w.length >= 2));
|
|
1410
|
-
const docWords = docText.toLowerCase().split(/\s+/);
|
|
1411
|
-
let score = 0;
|
|
1412
|
-
for (const word of docWords) {
|
|
1413
|
-
if (queryWords.has(word))
|
|
1414
|
-
score++;
|
|
1415
|
-
}
|
|
1416
|
-
return score;
|
|
1417
|
-
}
|
|
1181
|
+
// textScore extracted to ./runtime/format-helpers.ts.
|
|
1418
1182
|
/**
|
|
1419
1183
|
* Format a relative time string (e.g. "2 hours ago").
|
|
1420
1184
|
*/
|
|
1421
|
-
|
|
1422
|
-
const ms = typeof isoOrMs === 'number' ? isoOrMs : new Date(isoOrMs).getTime();
|
|
1423
|
-
const diffMs = Date.now() - ms;
|
|
1424
|
-
const seconds = Math.floor(diffMs / 1000);
|
|
1425
|
-
if (seconds < 60)
|
|
1426
|
-
return 'just now';
|
|
1427
|
-
const minutes = Math.floor(seconds / 60);
|
|
1428
|
-
if (minutes < 60)
|
|
1429
|
-
return `${minutes}m ago`;
|
|
1430
|
-
const hours = Math.floor(minutes / 60);
|
|
1431
|
-
if (hours < 24)
|
|
1432
|
-
return `${hours}h ago`;
|
|
1433
|
-
const days = Math.floor(hours / 24);
|
|
1434
|
-
return `${days}d ago`;
|
|
1435
|
-
}
|
|
1185
|
+
// relativeTime extracted to ./runtime/format-helpers.ts.
|
|
1436
1186
|
// ---------------------------------------------------------------------------
|
|
1437
1187
|
// Importance filter for auto-extraction
|
|
1438
1188
|
// ---------------------------------------------------------------------------
|
|
@@ -1444,39 +1194,7 @@ function relativeTime(isoOrMs) {
|
|
|
1444
1194
|
* NOTE: This filter is ONLY applied to auto-extraction (hooks).
|
|
1445
1195
|
* The explicit `tr remember` CLI always stores regardless of importance.
|
|
1446
1196
|
*/
|
|
1447
|
-
|
|
1448
|
-
/**
|
|
1449
|
-
* Filter extracted facts by importance threshold.
|
|
1450
|
-
* Facts with importance < MIN_IMPORTANCE_THRESHOLD are dropped.
|
|
1451
|
-
* Facts with missing/undefined importance are treated as importance=5 (kept).
|
|
1452
|
-
*/
|
|
1453
|
-
function filterByImportance(facts, logger) {
|
|
1454
|
-
const kept = [];
|
|
1455
|
-
let dropped = 0;
|
|
1456
|
-
for (const fact of facts) {
|
|
1457
|
-
const importance = fact.importance ?? 5;
|
|
1458
|
-
if (importance >= MIN_IMPORTANCE_THRESHOLD) {
|
|
1459
|
-
kept.push(fact);
|
|
1460
|
-
}
|
|
1461
|
-
else {
|
|
1462
|
-
dropped++;
|
|
1463
|
-
}
|
|
1464
|
-
}
|
|
1465
|
-
// Phase 2.2.5: always log the filter outcome so the agent_end path can
|
|
1466
|
-
// distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
|
|
1467
|
-
// below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
|
|
1468
|
-
// this only logged on drops, which made empty-input invisible.
|
|
1469
|
-
if (facts.length === 0) {
|
|
1470
|
-
logger.info('Importance filter: input=0 (nothing to filter)');
|
|
1471
|
-
}
|
|
1472
|
-
else if (dropped > 0) {
|
|
1473
|
-
logger.info(`Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`);
|
|
1474
|
-
}
|
|
1475
|
-
else {
|
|
1476
|
-
logger.info(`Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`);
|
|
1477
|
-
}
|
|
1478
|
-
return { kept, dropped };
|
|
1479
|
-
}
|
|
1197
|
+
// filterByImportance + MIN_IMPORTANCE_THRESHOLD extracted to ./extraction/importance-filter.ts.
|
|
1480
1198
|
// ---------------------------------------------------------------------------
|
|
1481
1199
|
// Auto-extraction helper
|
|
1482
1200
|
// ---------------------------------------------------------------------------
|
|
@@ -1497,10 +1215,10 @@ async function storeExtractedFacts(facts, logger, sourceOverride) {
|
|
|
1497
1215
|
const embeddingResultMap = new Map();
|
|
1498
1216
|
for (const fact of facts) {
|
|
1499
1217
|
try {
|
|
1500
|
-
const
|
|
1501
|
-
if (
|
|
1502
|
-
embeddingMap.set(fact.text,
|
|
1503
|
-
embeddingResultMap.set(fact.text,
|
|
1218
|
+
const embeddingResult = await generateEmbeddingAndLSH(fact.text, logger);
|
|
1219
|
+
if (embeddingResult) {
|
|
1220
|
+
embeddingMap.set(fact.text, embeddingResult.embedding);
|
|
1221
|
+
embeddingResultMap.set(fact.text, embeddingResult);
|
|
1504
1222
|
}
|
|
1505
1223
|
}
|
|
1506
1224
|
catch {
|
|
@@ -1818,13 +1536,13 @@ async function storeExtractedFacts(facts, logger, sourceOverride) {
|
|
|
1818
1536
|
for (let i = 0; i < pendingPayloads.length; i++) {
|
|
1819
1537
|
const slice = [pendingPayloads[i]]; // Single fact per UserOp
|
|
1820
1538
|
try {
|
|
1821
|
-
const
|
|
1822
|
-
if (
|
|
1539
|
+
const submitResult = await submitFactBatchOnChain(slice, batchConfig);
|
|
1540
|
+
if (submitResult.success) {
|
|
1823
1541
|
stored += slice.length;
|
|
1824
|
-
logger.info(`Fact ${i + 1}/${pendingPayloads.length}: submitted on-chain (tx=${
|
|
1542
|
+
logger.info(`Fact ${i + 1}/${pendingPayloads.length}: submitted on-chain (tx=${submitResult.txHash.slice(0, 10)}…)`);
|
|
1825
1543
|
}
|
|
1826
1544
|
else {
|
|
1827
|
-
batchError = `On-chain batch submission failed (tx=${
|
|
1545
|
+
batchError = `On-chain batch submission failed (tx=${submitResult.txHash.slice(0, 10)}…)`;
|
|
1828
1546
|
logger.warn(batchError);
|
|
1829
1547
|
break; // Stop submitting remaining batches
|
|
1830
1548
|
}
|
|
@@ -1862,657 +1580,10 @@ async function storeExtractedFacts(facts, logger, sourceOverride) {
|
|
|
1862
1580
|
return stored;
|
|
1863
1581
|
}
|
|
1864
1582
|
// ---------------------------------------------------------------------------
|
|
1865
|
-
// Import
|
|
1866
|
-
//
|
|
1867
|
-
/**
|
|
1868
|
-
* Handle import_from calls (CLI subcommand path; was the totalreclaw_import_from
|
|
1869
|
-
* agent tool before Phase 3.2 retired the agent tools).
|
|
1870
|
-
*
|
|
1871
|
-
* Two paths:
|
|
1872
|
-
* 1. Pre-structured sources (Mem0, MCP Memory) — adapter returns facts directly,
|
|
1873
|
-
* stored via storeExtractedFacts().
|
|
1874
|
-
* 2. Conversation-based sources (ChatGPT, Claude) — adapter returns conversation
|
|
1875
|
-
* chunks, each chunk is passed through extractFacts() (the same LLM extraction
|
|
1876
|
-
* pipeline used for auto-extraction), then stored via storeExtractedFacts().
|
|
1877
|
-
*/
|
|
1878
|
-
async function handlePluginImportFrom(params, logger) {
|
|
1879
|
-
_importInProgress = true;
|
|
1880
|
-
const startTime = Date.now();
|
|
1881
|
-
const source = params.source;
|
|
1882
|
-
const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini'];
|
|
1883
|
-
if (!source || !validSources.includes(source)) {
|
|
1884
|
-
return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
|
|
1885
|
-
}
|
|
1886
|
-
// Generate import_id up front so dry-run responses and background tasks share it.
|
|
1887
|
-
const importId = params.resume_id ?? crypto.randomUUID();
|
|
1888
|
-
try {
|
|
1889
|
-
const { getAdapter } = await import('./import-adapters/index.js');
|
|
1890
|
-
const adapter = getAdapter(source);
|
|
1891
|
-
const parseResult = await adapter.parse({
|
|
1892
|
-
content: params.content,
|
|
1893
|
-
api_key: params.api_key,
|
|
1894
|
-
source_user_id: params.source_user_id,
|
|
1895
|
-
api_url: params.api_url,
|
|
1896
|
-
file_path: params.file_path,
|
|
1897
|
-
});
|
|
1898
|
-
const hasChunks = parseResult.chunks && parseResult.chunks.length > 0;
|
|
1899
|
-
const hasFacts = parseResult.facts && parseResult.facts.length > 0;
|
|
1900
|
-
if (parseResult.errors.length > 0 && !hasFacts && !hasChunks) {
|
|
1901
|
-
return {
|
|
1902
|
-
success: false,
|
|
1903
|
-
error: `Failed to parse ${adapter.displayName} data`,
|
|
1904
|
-
details: parseResult.errors,
|
|
1905
|
-
};
|
|
1906
|
-
}
|
|
1907
|
-
// Dry run: report what was parsed (chunks or facts)
|
|
1908
|
-
if (params.dry_run) {
|
|
1909
|
-
if (hasChunks) {
|
|
1910
|
-
const totalChunks = parseResult.chunks.length;
|
|
1911
|
-
const EXTRACTION_RATIO = 2.5; // avg facts per chunk, from empirical data
|
|
1912
|
-
const BATCH_SIZE = 25;
|
|
1913
|
-
const SECONDS_PER_BATCH = 45; // ~30s extraction + ~15s embed+store
|
|
1914
|
-
const estimatedFacts = Math.round(totalChunks * EXTRACTION_RATIO);
|
|
1915
|
-
const estimatedBatches = Math.ceil(totalChunks / BATCH_SIZE);
|
|
1916
|
-
const estimatedMinutes = Math.ceil(estimatedBatches * SECONDS_PER_BATCH / 60);
|
|
1917
|
-
return {
|
|
1918
|
-
success: true,
|
|
1919
|
-
dry_run: true,
|
|
1920
|
-
import_id: importId,
|
|
1921
|
-
source,
|
|
1922
|
-
total_chunks: totalChunks,
|
|
1923
|
-
total_messages: parseResult.totalMessages,
|
|
1924
|
-
estimated_facts: estimatedFacts,
|
|
1925
|
-
estimated_batches: estimatedBatches,
|
|
1926
|
-
estimated_minutes: estimatedMinutes,
|
|
1927
|
-
batch_size: BATCH_SIZE,
|
|
1928
|
-
preview: parseResult.chunks.slice(0, 5).map((c) => ({
|
|
1929
|
-
title: c.title,
|
|
1930
|
-
messages: c.messages.length,
|
|
1931
|
-
first_message: c.messages[0]?.text.slice(0, 100),
|
|
1932
|
-
})),
|
|
1933
|
-
note: `Estimated ${estimatedFacts} facts from ${totalChunks} chunks (~${estimatedMinutes} min). Confirm to start background import.`,
|
|
1934
|
-
warnings: parseResult.warnings,
|
|
1935
|
-
};
|
|
1936
|
-
}
|
|
1937
|
-
return {
|
|
1938
|
-
success: true,
|
|
1939
|
-
dry_run: true,
|
|
1940
|
-
import_id: importId,
|
|
1941
|
-
source,
|
|
1942
|
-
total_found: parseResult.facts.length,
|
|
1943
|
-
preview: parseResult.facts.slice(0, 10).map((f) => ({
|
|
1944
|
-
type: f.type,
|
|
1945
|
-
text: f.text.slice(0, 100),
|
|
1946
|
-
importance: f.importance,
|
|
1947
|
-
})),
|
|
1948
|
-
warnings: parseResult.warnings,
|
|
1949
|
-
};
|
|
1950
|
-
}
|
|
1951
|
-
// ── Path 1: Conversation chunks (ChatGPT, Claude, Gemini) — background execution ──
|
|
1952
|
-
if (hasChunks) {
|
|
1953
|
-
const totalChunks = parseResult.chunks.length;
|
|
1954
|
-
const BATCH_SIZE = 25;
|
|
1955
|
-
const SECONDS_PER_BATCH = 45;
|
|
1956
|
-
const estimatedBatches = Math.ceil(totalChunks / BATCH_SIZE);
|
|
1957
|
-
const estimatedMinutes = Math.ceil(estimatedBatches * SECONDS_PER_BATCH / 60);
|
|
1958
|
-
const estimatedTotalFacts = Math.round(totalChunks * 2.5);
|
|
1959
|
-
const now = new Date();
|
|
1960
|
-
const initialState = {
|
|
1961
|
-
import_id: importId,
|
|
1962
|
-
source,
|
|
1963
|
-
status: 'running',
|
|
1964
|
-
started_at: now.toISOString(),
|
|
1965
|
-
last_updated: now.toISOString(),
|
|
1966
|
-
total_chunks: totalChunks,
|
|
1967
|
-
total_messages: parseResult.totalMessages,
|
|
1968
|
-
batch_done: 0,
|
|
1969
|
-
batch_total: estimatedBatches,
|
|
1970
|
-
facts_stored: 0,
|
|
1971
|
-
facts_extracted: 0,
|
|
1972
|
-
dups_skipped: 0,
|
|
1973
|
-
errors: [],
|
|
1974
|
-
file_path: params.file_path,
|
|
1975
|
-
estimated_total_facts: estimatedTotalFacts,
|
|
1976
|
-
estimated_minutes: estimatedMinutes,
|
|
1977
|
-
estimated_completion_iso: new Date(now.getTime() + estimatedBatches * SECONDS_PER_BATCH * 1000).toISOString(),
|
|
1978
|
-
disclosure_confirmed: !!(params.disclosure_confirmed),
|
|
1979
|
-
};
|
|
1980
|
-
writeImportState(initialState);
|
|
1981
|
-
logger.info(`Import ${importId}: background task started (${totalChunks} chunks, ~${estimatedMinutes}min)`);
|
|
1982
|
-
void handleChunkImport(parseResult.chunks, parseResult.totalMessages, source, logger, startTime, parseResult.warnings, importId).catch((err) => {
|
|
1983
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
1984
|
-
logger.warn(`Import ${importId}: background task failed: ${msg}`);
|
|
1985
|
-
const failedState = readImportState(importId);
|
|
1986
|
-
if (failedState && failedState.status === 'running') {
|
|
1987
|
-
writeImportState({ ...failedState, status: 'failed', errors: [...failedState.errors, msg] });
|
|
1988
|
-
}
|
|
1989
|
-
});
|
|
1990
|
-
return {
|
|
1991
|
-
import_id: importId,
|
|
1992
|
-
status: 'running',
|
|
1993
|
-
source,
|
|
1994
|
-
total_chunks: totalChunks,
|
|
1995
|
-
estimated_batches: estimatedBatches,
|
|
1996
|
-
estimated_minutes: estimatedMinutes,
|
|
1997
|
-
estimated_completion_iso: initialState.estimated_completion_iso,
|
|
1998
|
-
message: `Import started in background. ~${estimatedMinutes} min for ${totalChunks} chunks. Check progress with \`openclaw totalreclaw import status\` on the gateway host (or \`import status --id ${importId} --json\` from an agent shell).`,
|
|
1999
|
-
warnings: parseResult.warnings,
|
|
2000
|
-
};
|
|
2001
|
-
}
|
|
2002
|
-
// ── Path 2: Pre-structured facts (Mem0, MCP Memory) — direct store ──
|
|
2003
|
-
const extractedFacts = parseResult.facts.map((f) => ({
|
|
2004
|
-
text: f.text,
|
|
2005
|
-
type: f.type,
|
|
2006
|
-
importance: f.importance,
|
|
2007
|
-
action: 'ADD',
|
|
2008
|
-
}));
|
|
2009
|
-
// Store in batches of 50. Stop on any batch failure to prevent
|
|
2010
|
-
// nonce zombies from blocking subsequent UserOps (AA25).
|
|
2011
|
-
let totalStored = 0;
|
|
2012
|
-
let storeError;
|
|
2013
|
-
const batchSize = 50;
|
|
2014
|
-
for (let i = 0; i < extractedFacts.length; i += batchSize) {
|
|
2015
|
-
const batch = extractedFacts.slice(i, i + batchSize);
|
|
2016
|
-
try {
|
|
2017
|
-
const stored = await storeExtractedFacts(batch, logger);
|
|
2018
|
-
totalStored += stored;
|
|
2019
|
-
logger.info(`Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`);
|
|
2020
|
-
}
|
|
2021
|
-
catch (err) {
|
|
2022
|
-
storeError = err instanceof Error ? err.message : String(err);
|
|
2023
|
-
logger.warn(`Import stopped at batch ${Math.floor(i / batchSize) + 1}: ${storeError}`);
|
|
2024
|
-
break; // Stop processing further batches
|
|
2025
|
-
}
|
|
2026
|
-
}
|
|
2027
|
-
const importWarnings = [...parseResult.warnings];
|
|
2028
|
-
if (storeError) {
|
|
2029
|
-
importWarnings.push(`Import stopped early: ${storeError}`);
|
|
2030
|
-
}
|
|
2031
|
-
return {
|
|
2032
|
-
success: totalStored > 0,
|
|
2033
|
-
source,
|
|
2034
|
-
import_id: importId,
|
|
2035
|
-
total_found: parseResult.facts.length,
|
|
2036
|
-
imported: totalStored,
|
|
2037
|
-
skipped: parseResult.facts.length - totalStored,
|
|
2038
|
-
stopped_early: !!storeError,
|
|
2039
|
-
warnings: importWarnings,
|
|
2040
|
-
duration_ms: Date.now() - startTime,
|
|
2041
|
-
};
|
|
2042
|
-
}
|
|
2043
|
-
catch (e) {
|
|
2044
|
-
const msg = e instanceof Error ? e.message : 'Unknown error';
|
|
2045
|
-
logger.error(`Import failed: ${msg}`);
|
|
2046
|
-
return { success: false, error: `Import failed: ${msg}` };
|
|
2047
|
-
}
|
|
2048
|
-
}
|
|
2049
|
-
// ---------------------------------------------------------------------------
|
|
2050
|
-
// Smart Import — Two-Pass Pipeline (Profile + Triage)
|
|
1583
|
+
// Import subsystem extracted to ./import/import-runtime.ts.
|
|
1584
|
+
// storeExtractedFacts is injected below (it closes over plugin session state).
|
|
2051
1585
|
// ---------------------------------------------------------------------------
|
|
2052
|
-
|
|
2053
|
-
// subgraph-store.ts). Goes through __cjsRequire (createRequire(import.meta.url))
|
|
2054
|
-
// declared at the top of the file — bare `require()` is undefined under
|
|
2055
|
-
// pure-ESM Node, see issue #124.
|
|
2056
|
-
let _smartImportWasm = null;
|
|
2057
|
-
function getSmartImportWasm() {
|
|
2058
|
-
if (!_smartImportWasm)
|
|
2059
|
-
_smartImportWasm = __cjsRequire('@totalreclaw/core');
|
|
2060
|
-
return _smartImportWasm;
|
|
2061
|
-
}
|
|
2062
|
-
/**
|
|
2063
|
-
* Check whether the @totalreclaw/core WASM module exposes smart import functions.
|
|
2064
|
-
* Returns false if the module is an older version without smart import support.
|
|
2065
|
-
*/
|
|
2066
|
-
function hasSmartImportSupport() {
|
|
2067
|
-
try {
|
|
2068
|
-
const wasm = getSmartImportWasm();
|
|
2069
|
-
return typeof wasm.chunksToSummaries === 'function' &&
|
|
2070
|
-
typeof wasm.buildProfileBatchPrompt === 'function' &&
|
|
2071
|
-
typeof wasm.parseProfileBatchResponse === 'function' &&
|
|
2072
|
-
typeof wasm.buildTriagePrompt === 'function' &&
|
|
2073
|
-
typeof wasm.parseTriageResponse === 'function' &&
|
|
2074
|
-
typeof wasm.enrichExtractionPrompt === 'function';
|
|
2075
|
-
}
|
|
2076
|
-
catch {
|
|
2077
|
-
return false;
|
|
2078
|
-
}
|
|
2079
|
-
}
|
|
2080
|
-
/**
|
|
2081
|
-
* Run the smart import two-pass pipeline: profile the user from conversation
|
|
2082
|
-
* summaries, then triage chunks as EXTRACT or SKIP.
|
|
2083
|
-
*
|
|
2084
|
-
* All prompt construction and response parsing happens in @totalreclaw/core WASM.
|
|
2085
|
-
* LLM calls use the plugin's existing chatCompletion() function.
|
|
2086
|
-
*
|
|
2087
|
-
* Returns null if smart import is unavailable (old WASM, no LLM config, etc.)
|
|
2088
|
-
* so the caller can fall back to blind extraction.
|
|
2089
|
-
*/
|
|
2090
|
-
async function runSmartImportPipeline(chunks, logger) {
|
|
2091
|
-
// Guard: WASM must have smart import functions
|
|
2092
|
-
if (!hasSmartImportSupport()) {
|
|
2093
|
-
logger.info('Smart import: WASM module does not support smart import, falling back to blind extraction');
|
|
2094
|
-
return null;
|
|
2095
|
-
}
|
|
2096
|
-
// Guard: LLM must be available
|
|
2097
|
-
const llmConfig = resolveLLMConfig();
|
|
2098
|
-
if (!llmConfig) {
|
|
2099
|
-
logger.info('Smart import: no LLM available, falling back to blind extraction');
|
|
2100
|
-
return null;
|
|
2101
|
-
}
|
|
2102
|
-
const pipelineStart = Date.now();
|
|
2103
|
-
const wasm = getSmartImportWasm();
|
|
2104
|
-
try {
|
|
2105
|
-
// Step 0: Convert chunks to compact summaries (first + last message)
|
|
2106
|
-
const wasmChunks = chunks.map((c, i) => ({
|
|
2107
|
-
index: i,
|
|
2108
|
-
title: c.title || 'Untitled',
|
|
2109
|
-
messages: c.messages.map((m) => ({ role: m.role, content: m.text })),
|
|
2110
|
-
timestamp: c.timestamp || null,
|
|
2111
|
-
}));
|
|
2112
|
-
const summaries = wasm.chunksToSummaries(JSON.stringify(wasmChunks));
|
|
2113
|
-
const summariesJson = JSON.stringify(summaries);
|
|
2114
|
-
// Step 1: Build user profile (batch summarize -> merge)
|
|
2115
|
-
const PROFILE_BATCH_SIZE = 50;
|
|
2116
|
-
const profileStart = Date.now();
|
|
2117
|
-
const partials = [];
|
|
2118
|
-
for (let i = 0; i < summaries.length; i += PROFILE_BATCH_SIZE) {
|
|
2119
|
-
const batch = summaries.slice(i, i + PROFILE_BATCH_SIZE);
|
|
2120
|
-
const prompt = wasm.buildProfileBatchPrompt(JSON.stringify(batch));
|
|
2121
|
-
const response = await chatCompletion(llmConfig, [
|
|
2122
|
-
{ role: 'user', content: prompt },
|
|
2123
|
-
], { maxTokens: 2048, temperature: 0 });
|
|
2124
|
-
if (!response) {
|
|
2125
|
-
logger.warn(`Smart import: LLM returned empty response for profile batch ${Math.floor(i / PROFILE_BATCH_SIZE) + 1}`);
|
|
2126
|
-
continue;
|
|
2127
|
-
}
|
|
2128
|
-
const partial = wasm.parseProfileBatchResponse(response);
|
|
2129
|
-
partials.push(partial);
|
|
2130
|
-
}
|
|
2131
|
-
if (partials.length === 0) {
|
|
2132
|
-
logger.warn('Smart import: no profile batches produced, falling back to blind extraction');
|
|
2133
|
-
return null;
|
|
2134
|
-
}
|
|
2135
|
-
let profile;
|
|
2136
|
-
if (partials.length === 1) {
|
|
2137
|
-
// Single batch — skip merge, promote partial to full profile
|
|
2138
|
-
// parseProfileBatchResponse returns a PartialProfile; convert to UserProfile shape
|
|
2139
|
-
const p = partials[0];
|
|
2140
|
-
profile = {
|
|
2141
|
-
identity: p.identity ?? null,
|
|
2142
|
-
themes: p.themes ?? [],
|
|
2143
|
-
projects: p.projects ?? [],
|
|
2144
|
-
stack: p.stack ?? [],
|
|
2145
|
-
decisions: p.decisions ?? [],
|
|
2146
|
-
interests: p.interests ?? [],
|
|
2147
|
-
skip_patterns: p.skip_patterns ?? [],
|
|
2148
|
-
};
|
|
2149
|
-
}
|
|
2150
|
-
else {
|
|
2151
|
-
const mergePrompt = wasm.buildProfileMergePrompt(JSON.stringify(partials));
|
|
2152
|
-
const mergeResponse = await chatCompletion(llmConfig, [
|
|
2153
|
-
{ role: 'user', content: mergePrompt },
|
|
2154
|
-
], { maxTokens: 2048, temperature: 0 });
|
|
2155
|
-
if (!mergeResponse) {
|
|
2156
|
-
logger.warn('Smart import: LLM returned empty response for profile merge, falling back to blind extraction');
|
|
2157
|
-
return null;
|
|
2158
|
-
}
|
|
2159
|
-
profile = wasm.parseProfileResponse(mergeResponse);
|
|
2160
|
-
}
|
|
2161
|
-
const profileJson = JSON.stringify(profile);
|
|
2162
|
-
const profileDuration = Date.now() - profileStart;
|
|
2163
|
-
const p = profile;
|
|
2164
|
-
const themeCount = Array.isArray(p.themes) ? p.themes.length : 0;
|
|
2165
|
-
const skipPatternCount = Array.isArray(p.skip_patterns) ? p.skip_patterns.length : 0;
|
|
2166
|
-
logger.info(`Smart import: profile built in ${profileDuration}ms (themes=${themeCount}, skip_patterns=${skipPatternCount})`);
|
|
2167
|
-
// Step 1.5: Chunk triage (EXTRACT or SKIP)
|
|
2168
|
-
const triageStart = Date.now();
|
|
2169
|
-
const allDecisions = [];
|
|
2170
|
-
const TRIAGE_BATCH_SIZE = 50;
|
|
2171
|
-
for (let i = 0; i < summaries.length; i += TRIAGE_BATCH_SIZE) {
|
|
2172
|
-
const batch = summaries.slice(i, i + TRIAGE_BATCH_SIZE);
|
|
2173
|
-
const triagePrompt = wasm.buildTriagePrompt(profileJson, JSON.stringify(batch));
|
|
2174
|
-
const triageResponse = await chatCompletion(llmConfig, [
|
|
2175
|
-
{ role: 'user', content: triagePrompt },
|
|
2176
|
-
], { maxTokens: 4096, temperature: 0 });
|
|
2177
|
-
if (!triageResponse) {
|
|
2178
|
-
logger.warn(`Smart import: LLM returned empty response for triage batch ${Math.floor(i / TRIAGE_BATCH_SIZE) + 1}, defaulting to EXTRACT`);
|
|
2179
|
-
// Default all chunks in this batch to EXTRACT
|
|
2180
|
-
for (let j = i; j < Math.min(i + TRIAGE_BATCH_SIZE, summaries.length); j++) {
|
|
2181
|
-
allDecisions.push({ chunk_index: j, decision: 'EXTRACT', reason: 'triage LLM unavailable' });
|
|
2182
|
-
}
|
|
2183
|
-
continue;
|
|
2184
|
-
}
|
|
2185
|
-
const batchDecisions = wasm.parseTriageResponse(triageResponse);
|
|
2186
|
-
allDecisions.push(...batchDecisions);
|
|
2187
|
-
}
|
|
2188
|
-
const triageDuration = Date.now() - triageStart;
|
|
2189
|
-
const extractCount = allDecisions.filter((d) => d.decision !== 'SKIP').length;
|
|
2190
|
-
const skipCount = allDecisions.filter((d) => d.decision === 'SKIP').length;
|
|
2191
|
-
logger.info(`Smart import: triage complete in ${triageDuration}ms (extract=${extractCount}, skip=${skipCount}, total=${chunks.length})`);
|
|
2192
|
-
// Step 2: Build enriched system prompt for extraction
|
|
2193
|
-
const enrichedSystemPrompt = wasm.enrichExtractionPrompt(profileJson, EXTRACTION_SYSTEM_PROMPT);
|
|
2194
|
-
const totalDuration = Date.now() - pipelineStart;
|
|
2195
|
-
logger.info(`Smart import: pipeline complete in ${totalDuration}ms`);
|
|
2196
|
-
return {
|
|
2197
|
-
profileJson,
|
|
2198
|
-
decisions: allDecisions,
|
|
2199
|
-
enrichedSystemPrompt,
|
|
2200
|
-
extractCount,
|
|
2201
|
-
skipCount,
|
|
2202
|
-
durationMs: totalDuration,
|
|
2203
|
-
};
|
|
2204
|
-
}
|
|
2205
|
-
catch (err) {
|
|
2206
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
2207
|
-
logger.warn(`Smart import: pipeline failed (${msg}), falling back to blind extraction`);
|
|
2208
|
-
return null;
|
|
2209
|
-
}
|
|
2210
|
-
}
|
|
2211
|
-
/**
|
|
2212
|
-
* Check if a chunk should be skipped based on triage decisions.
|
|
2213
|
-
* If no decision exists for the chunk index, defaults to EXTRACT (safe default).
|
|
2214
|
-
*/
|
|
2215
|
-
function isChunkSkipped(chunkIndex, decisions) {
|
|
2216
|
-
const decision = decisions.find((d) => d.chunk_index === chunkIndex);
|
|
2217
|
-
if (decision && decision.decision === 'SKIP') {
|
|
2218
|
-
return { skipped: true, reason: decision.reason || 'triage: skip' };
|
|
2219
|
-
}
|
|
2220
|
-
return { skipped: false, reason: '' };
|
|
2221
|
-
}
|
|
2222
|
-
/**
|
|
2223
|
-
* Process a batch (slice) of conversation chunks from a file.
|
|
2224
|
-
* Called repeatedly by the agent for large imports.
|
|
2225
|
-
*/
|
|
2226
|
-
async function handleBatchImport(params, logger) {
|
|
2227
|
-
_importInProgress = true;
|
|
2228
|
-
const source = params.source;
|
|
2229
|
-
const filePath = params.file_path;
|
|
2230
|
-
const content = params.content;
|
|
2231
|
-
const offset = params.offset ?? 0;
|
|
2232
|
-
const batchSize = params.batch_size ?? 25;
|
|
2233
|
-
const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini'];
|
|
2234
|
-
if (!source || !validSources.includes(source)) {
|
|
2235
|
-
return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
|
|
2236
|
-
}
|
|
2237
|
-
const startTime = Date.now();
|
|
2238
|
-
const { getAdapter } = await import('./import-adapters/index.js');
|
|
2239
|
-
const adapter = getAdapter(source);
|
|
2240
|
-
const parseResult = await adapter.parse({ content, file_path: filePath });
|
|
2241
|
-
if (parseResult.errors.length > 0 && parseResult.chunks.length === 0) {
|
|
2242
|
-
return { success: false, error: parseResult.errors.join('; ') };
|
|
2243
|
-
}
|
|
2244
|
-
const totalChunks = parseResult.chunks.length;
|
|
2245
|
-
const slice = parseResult.chunks.slice(offset, offset + batchSize);
|
|
2246
|
-
const remaining = Math.max(0, totalChunks - offset - slice.length);
|
|
2247
|
-
// --- Smart Import: Profile + Triage ---
|
|
2248
|
-
// Build profile from ALL chunks (not just the slice) for full context,
|
|
2249
|
-
// then triage only the current slice. For simplicity, we rebuild on every
|
|
2250
|
-
// batch call — optimization (caching) can come later.
|
|
2251
|
-
const smartCtx = await runSmartImportPipeline(parseResult.chunks, logger);
|
|
2252
|
-
let chunksSkipped = 0;
|
|
2253
|
-
// Process the slice through the normal extraction + storage pipeline.
|
|
2254
|
-
// If a batch fails (nonce zombie, quota exceeded, etc.), stop immediately
|
|
2255
|
-
// to prevent subsequent UserOps from hitting AA25 nonce conflicts.
|
|
2256
|
-
let factsExtracted = 0;
|
|
2257
|
-
let factsStored = 0;
|
|
2258
|
-
let chunksProcessed = 0;
|
|
2259
|
-
let storeError;
|
|
2260
|
-
for (let i = 0; i < slice.length; i++) {
|
|
2261
|
-
const chunk = slice[i];
|
|
2262
|
-
const globalIndex = offset + i; // Index in the full chunks array
|
|
2263
|
-
// Smart import: skip chunks triaged as SKIP
|
|
2264
|
-
if (smartCtx) {
|
|
2265
|
-
const { skipped, reason } = isChunkSkipped(globalIndex, smartCtx.decisions);
|
|
2266
|
-
if (skipped) {
|
|
2267
|
-
logger.info(`Import: skipping chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}" (${reason})`);
|
|
2268
|
-
chunksSkipped++;
|
|
2269
|
-
chunksProcessed++;
|
|
2270
|
-
continue;
|
|
2271
|
-
}
|
|
2272
|
-
}
|
|
2273
|
-
logger.info(`Import: extracting facts from chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}"`);
|
|
2274
|
-
const messages = chunk.messages.map((m) => ({ role: m.role, content: m.text }));
|
|
2275
|
-
const facts = await extractFacts(messages, 'full', undefined, // no existing memories for dedup during import
|
|
2276
|
-
smartCtx?.enrichedSystemPrompt);
|
|
2277
|
-
chunksProcessed++;
|
|
2278
|
-
if (facts.length > 0) {
|
|
2279
|
-
factsExtracted += facts.length;
|
|
2280
|
-
try {
|
|
2281
|
-
const stored = await storeExtractedFacts(facts, logger);
|
|
2282
|
-
factsStored += stored;
|
|
2283
|
-
}
|
|
2284
|
-
catch (err) {
|
|
2285
|
-
storeError = err instanceof Error ? err.message : String(err);
|
|
2286
|
-
logger.warn(`Import batch stopped at chunk ${globalIndex + 1}/${totalChunks}: ${storeError}`);
|
|
2287
|
-
break; // Stop processing further chunks — a zombie UserOp may block writes
|
|
2288
|
-
}
|
|
2289
|
-
}
|
|
2290
|
-
}
|
|
2291
|
-
return {
|
|
2292
|
-
success: factsStored > 0 || (!storeError && factsExtracted === 0),
|
|
2293
|
-
batch_offset: offset,
|
|
2294
|
-
batch_size: chunksProcessed,
|
|
2295
|
-
total_chunks: totalChunks,
|
|
2296
|
-
facts_extracted: factsExtracted,
|
|
2297
|
-
facts_stored: factsStored,
|
|
2298
|
-
chunks_skipped: chunksSkipped,
|
|
2299
|
-
remaining_chunks: remaining,
|
|
2300
|
-
is_complete: remaining === 0 && !storeError,
|
|
2301
|
-
stopped_early: !!storeError,
|
|
2302
|
-
error: storeError,
|
|
2303
|
-
smart_import: smartCtx ? {
|
|
2304
|
-
profile_duration_ms: smartCtx.durationMs,
|
|
2305
|
-
extract_count: smartCtx.extractCount,
|
|
2306
|
-
skip_count: smartCtx.skipCount,
|
|
2307
|
-
} : null,
|
|
2308
|
-
// Estimation for the full import
|
|
2309
|
-
estimated_total_facts: Math.round(totalChunks * 2.5),
|
|
2310
|
-
estimated_total_userops: Math.ceil(totalChunks * 2.5 / 15),
|
|
2311
|
-
estimated_minutes: Math.ceil(Math.ceil(totalChunks / batchSize) * 45 / 60),
|
|
2312
|
-
duration_ms: Date.now() - startTime,
|
|
2313
|
-
};
|
|
2314
|
-
}
|
|
2315
|
-
/**
|
|
2316
|
-
* Process conversation chunks through LLM extraction and store results.
|
|
2317
|
-
*
|
|
2318
|
-
* Each chunk is passed to extractFacts() — the same extraction pipeline used
|
|
2319
|
-
* for auto-extraction during live conversations. This ensures import quality
|
|
2320
|
-
* matches conversation extraction quality.
|
|
2321
|
-
*/
|
|
2322
|
-
async function handleChunkImport(chunks, totalMessages, source, logger, startTime, warnings, importId) {
|
|
2323
|
-
let totalExtracted = 0;
|
|
2324
|
-
let totalStored = 0;
|
|
2325
|
-
let chunksProcessed = 0;
|
|
2326
|
-
let chunksSkipped = 0;
|
|
2327
|
-
const resolvedImportId = importId ?? crypto.randomUUID();
|
|
2328
|
-
let storeError;
|
|
2329
|
-
// --- Smart Import: Profile + Triage ---
|
|
2330
|
-
const smartCtx = await runSmartImportPipeline(chunks, logger);
|
|
2331
|
-
const CHECKPOINT_EVERY = 25; // write state file every N chunks
|
|
2332
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
2333
|
-
// Check abort flag from state file before each chunk (background task may be cancelled).
|
|
2334
|
-
if (importId) {
|
|
2335
|
-
const currentState = readImportState(importId);
|
|
2336
|
-
if (currentState?.status === 'aborted') {
|
|
2337
|
-
logger.info(`Import ${importId}: abort flag detected at chunk ${i + 1}/${chunks.length}, stopping`);
|
|
2338
|
-
break;
|
|
2339
|
-
}
|
|
2340
|
-
}
|
|
2341
|
-
const chunk = chunks[i];
|
|
2342
|
-
chunksProcessed++;
|
|
2343
|
-
// Smart import: skip chunks triaged as SKIP
|
|
2344
|
-
if (smartCtx) {
|
|
2345
|
-
const { skipped, reason } = isChunkSkipped(i, smartCtx.decisions);
|
|
2346
|
-
if (skipped) {
|
|
2347
|
-
logger.info(`Import: skipping chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}" (${reason})`);
|
|
2348
|
-
chunksSkipped++;
|
|
2349
|
-
continue;
|
|
2350
|
-
}
|
|
2351
|
-
}
|
|
2352
|
-
logger.info(`Import: extracting facts from chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}"`);
|
|
2353
|
-
// Convert chunk messages to the format extractFacts() expects.
|
|
2354
|
-
// extractFacts() takes an array of message-like objects with { role, content }.
|
|
2355
|
-
const messages = chunk.messages.map((m) => ({
|
|
2356
|
-
role: m.role,
|
|
2357
|
-
content: m.text,
|
|
2358
|
-
}));
|
|
2359
|
-
// Use 'full' mode to extract ALL valuable memories from the chunk
|
|
2360
|
-
// (not just the last few messages like 'turn' mode does).
|
|
2361
|
-
// Smart import: pass enriched system prompt with user profile context.
|
|
2362
|
-
const facts = await extractFacts(messages, 'full', undefined, // no existing memories for dedup during import
|
|
2363
|
-
smartCtx?.enrichedSystemPrompt);
|
|
2364
|
-
if (facts.length > 0) {
|
|
2365
|
-
totalExtracted += facts.length;
|
|
2366
|
-
try {
|
|
2367
|
-
// Store through the normal pipeline (dedup, encrypt, store).
|
|
2368
|
-
// storeExtractedFacts throws on batch failure to prevent nonce zombies.
|
|
2369
|
-
const stored = await storeExtractedFacts(facts, logger);
|
|
2370
|
-
totalStored += stored;
|
|
2371
|
-
logger.info(`Import chunk ${chunksProcessed}/${chunks.length}: extracted ${facts.length} facts, stored ${stored}`);
|
|
2372
|
-
}
|
|
2373
|
-
catch (err) {
|
|
2374
|
-
storeError = err instanceof Error ? err.message : String(err);
|
|
2375
|
-
logger.warn(`Import stopped at chunk ${chunksProcessed}/${chunks.length}: ${storeError}`);
|
|
2376
|
-
break; // Stop processing further chunks — a zombie UserOp may block writes
|
|
2377
|
-
}
|
|
2378
|
-
}
|
|
2379
|
-
// Checkpoint state file periodically so _import_status reflects live progress.
|
|
2380
|
-
if (importId && chunksProcessed % CHECKPOINT_EVERY === 0) {
|
|
2381
|
-
const liveState = readImportState(importId);
|
|
2382
|
-
if (liveState) {
|
|
2383
|
-
const estimatedBatches = liveState.batch_total || 1;
|
|
2384
|
-
const doneBatches = Math.floor(chunksProcessed / CHECKPOINT_EVERY);
|
|
2385
|
-
const elapsed = Date.now() - new Date(liveState.started_at).getTime();
|
|
2386
|
-
const secPerBatch = doneBatches > 0 ? elapsed / 1000 / doneBatches : 45;
|
|
2387
|
-
const remaining = estimatedBatches - doneBatches;
|
|
2388
|
-
const etaMs = remaining * secPerBatch * 1000;
|
|
2389
|
-
writeImportState({
|
|
2390
|
-
...liveState,
|
|
2391
|
-
batch_done: doneBatches,
|
|
2392
|
-
facts_stored: totalStored,
|
|
2393
|
-
facts_extracted: totalExtracted,
|
|
2394
|
-
estimated_completion_iso: new Date(Date.now() + etaMs).toISOString(),
|
|
2395
|
-
});
|
|
2396
|
-
}
|
|
2397
|
-
}
|
|
2398
|
-
}
|
|
2399
|
-
if (totalExtracted === 0 && chunks.length > 0 && !storeError && chunksSkipped < chunks.length) {
|
|
2400
|
-
warnings.push(`Processed ${chunks.length} conversation chunks (${totalMessages} messages) but the LLM ` +
|
|
2401
|
-
`did not extract any facts worth storing. This can happen if the conversations are mostly ` +
|
|
2402
|
-
`generic/ephemeral content without personal facts, preferences, or decisions.`);
|
|
2403
|
-
}
|
|
2404
|
-
if (storeError) {
|
|
2405
|
-
warnings.push(`Import stopped early: ${storeError}. ${chunks.length - chunksProcessed} chunk(s) not processed.`);
|
|
2406
|
-
}
|
|
2407
|
-
// Final state file write for background imports.
|
|
2408
|
-
if (importId) {
|
|
2409
|
-
const finalState = readImportState(importId);
|
|
2410
|
-
if (finalState) {
|
|
2411
|
-
const finalStatus = storeError ? 'failed' : (finalState.status === 'aborted' ? 'aborted' : 'completed');
|
|
2412
|
-
writeImportState({
|
|
2413
|
-
...finalState,
|
|
2414
|
-
status: finalStatus,
|
|
2415
|
-
batch_done: finalState.batch_total,
|
|
2416
|
-
facts_stored: totalStored,
|
|
2417
|
-
facts_extracted: totalExtracted,
|
|
2418
|
-
errors: storeError ? [...finalState.errors, storeError] : finalState.errors,
|
|
2419
|
-
});
|
|
2420
|
-
}
|
|
2421
|
-
_importInProgress = false;
|
|
2422
|
-
}
|
|
2423
|
-
return {
|
|
2424
|
-
success: totalStored > 0 || totalExtracted > 0,
|
|
2425
|
-
source,
|
|
2426
|
-
import_id: resolvedImportId,
|
|
2427
|
-
total_chunks: chunks.length,
|
|
2428
|
-
chunks_processed: chunksProcessed,
|
|
2429
|
-
chunks_skipped: chunksSkipped,
|
|
2430
|
-
total_messages: totalMessages,
|
|
2431
|
-
facts_extracted: totalExtracted,
|
|
2432
|
-
imported: totalStored,
|
|
2433
|
-
skipped: totalExtracted - totalStored,
|
|
2434
|
-
stopped_early: !!storeError,
|
|
2435
|
-
smart_import: smartCtx ? {
|
|
2436
|
-
profile_duration_ms: smartCtx.durationMs,
|
|
2437
|
-
extract_count: smartCtx.extractCount,
|
|
2438
|
-
skip_count: smartCtx.skipCount,
|
|
2439
|
-
} : null,
|
|
2440
|
-
warnings,
|
|
2441
|
-
duration_ms: Date.now() - startTime,
|
|
2442
|
-
};
|
|
2443
|
-
}
|
|
2444
|
-
// ---------------------------------------------------------------------------
|
|
2445
|
-
// Import status + abort handlers
|
|
2446
|
-
// ---------------------------------------------------------------------------
|
|
2447
|
-
async function handleImportStatus(params, logger) {
|
|
2448
|
-
const importId = params.import_id;
|
|
2449
|
-
let state;
|
|
2450
|
-
if (importId) {
|
|
2451
|
-
state = readImportState(importId);
|
|
2452
|
-
if (!state)
|
|
2453
|
-
return { error: `No import found with id: ${importId}` };
|
|
2454
|
-
}
|
|
2455
|
-
else {
|
|
2456
|
-
state = readMostRecentActiveImport();
|
|
2457
|
-
if (!state)
|
|
2458
|
-
return { status: 'no_active_import', message: 'No active import found. Start one with `openclaw totalreclaw import from <source>` on the gateway host. (Auto-resume still picks up running imports on gateway restart.)' };
|
|
2459
|
-
}
|
|
2460
|
-
// 1h freshness guard: mark stale imports as failed and prompt user to resume.
|
|
2461
|
-
if (state.status === 'running' && isImportStale(state)) {
|
|
2462
|
-
writeImportState({ ...state, status: 'failed', errors: [...state.errors, 'Stale: no progress in 1h'] });
|
|
2463
|
-
logger.info(`Import ${state.import_id}: marked stale (no progress in 1h)`);
|
|
2464
|
-
return {
|
|
2465
|
-
import_id: state.import_id,
|
|
2466
|
-
status: 'failed',
|
|
2467
|
-
stale: true,
|
|
2468
|
-
facts_stored: state.facts_stored,
|
|
2469
|
-
message: 'Import appears stale — no progress in 1 hour. Resume it with `openclaw totalreclaw import from <source> --file <path> --resume ' + state.import_id + '` on the gateway host, or restart the gateway to trigger auto-resume.',
|
|
2470
|
-
resume_id: state.import_id,
|
|
2471
|
-
};
|
|
2472
|
-
}
|
|
2473
|
-
const now = Date.now();
|
|
2474
|
-
const elapsedMs = now - new Date(state.started_at).getTime();
|
|
2475
|
-
const secPerBatch = state.batch_done > 0 ? elapsedMs / 1000 / state.batch_done : 45;
|
|
2476
|
-
const remaining = Math.max(0, state.batch_total - state.batch_done);
|
|
2477
|
-
const etaSeconds = state.status === 'running' ? Math.round(remaining * secPerBatch) : 0;
|
|
2478
|
-
return {
|
|
2479
|
-
import_id: state.import_id,
|
|
2480
|
-
status: state.status,
|
|
2481
|
-
batch_done: state.batch_done,
|
|
2482
|
-
batch_total: state.batch_total,
|
|
2483
|
-
facts_stored: state.facts_stored,
|
|
2484
|
-
dups_skipped: state.dups_skipped,
|
|
2485
|
-
eta_seconds: etaSeconds,
|
|
2486
|
-
completion_iso: state.status === 'running'
|
|
2487
|
-
? new Date(now + etaSeconds * 1000).toISOString()
|
|
2488
|
-
: state.last_updated,
|
|
2489
|
-
source: state.source,
|
|
2490
|
-
started_at: state.started_at,
|
|
2491
|
-
errors: state.errors,
|
|
2492
|
-
};
|
|
2493
|
-
}
|
|
2494
|
-
async function handleImportAbort(params, logger) {
|
|
2495
|
-
const importId = params.import_id;
|
|
2496
|
-
if (!importId)
|
|
2497
|
-
return { error: 'import_id is required' };
|
|
2498
|
-
const state = readImportState(importId);
|
|
2499
|
-
if (!state)
|
|
2500
|
-
return { error: `No import found with id: ${importId}` };
|
|
2501
|
-
if (state.status === 'aborted') {
|
|
2502
|
-
return { aborted: true, idempotent: true, import_id: importId, facts_already_stored: state.facts_stored };
|
|
2503
|
-
}
|
|
2504
|
-
if (state.status === 'completed') {
|
|
2505
|
-
return { error: 'Import already completed — nothing to abort', import_id: importId, facts_stored: state.facts_stored };
|
|
2506
|
-
}
|
|
2507
|
-
writeImportState({ ...state, status: 'aborted' });
|
|
2508
|
-
logger.info(`Import ${importId}: abort requested (${state.facts_stored} facts already stored)`);
|
|
2509
|
-
return {
|
|
2510
|
-
aborted: true,
|
|
2511
|
-
import_id: importId,
|
|
2512
|
-
facts_already_stored: state.facts_stored,
|
|
2513
|
-
message: 'Import abort requested. The background task will stop at the next chunk boundary. Already-stored facts are kept.',
|
|
2514
|
-
};
|
|
2515
|
-
}
|
|
1586
|
+
configureImportRuntime({ storeExtractedFacts });
|
|
2516
1587
|
// ---------------------------------------------------------------------------
|
|
2517
1588
|
// buildRecallDeps — bind the real recall pipeline into the closures the
|
|
2518
1589
|
// native MemoryPluginCapability wiring helper (Task 2.7) consumes.
|
|
@@ -2767,10 +1838,10 @@ function buildRecallDeps(logger) {
|
|
|
2767
1838
|
if (!subgraphOwner)
|
|
2768
1839
|
return null;
|
|
2769
1840
|
try {
|
|
2770
|
-
const
|
|
2771
|
-
if (!
|
|
1841
|
+
const fetchedFact = await fetchFactById(subgraphOwner, id, authKeyHex);
|
|
1842
|
+
if (!fetchedFact)
|
|
2772
1843
|
return null;
|
|
2773
|
-
const docJson = decryptFromHex(
|
|
1844
|
+
const docJson = decryptFromHex(fetchedFact.encryptedBlob, encryptionKey);
|
|
2774
1845
|
if (isDigestBlob(docJson))
|
|
2775
1846
|
return null;
|
|
2776
1847
|
const doc = readClaimFromBlob(docJson);
|
|
@@ -2844,61 +1915,7 @@ const plugin = {
|
|
|
2844
1915
|
// `invalid config: must NOT have additional properties`, which blocked
|
|
2845
1916
|
// the documented remote-pairing setup (publicUrl) and made it impossible
|
|
2846
1917
|
// for a user to hand-pick an extraction model (extraction.llm.*).
|
|
2847
|
-
configSchema:
|
|
2848
|
-
type: 'object',
|
|
2849
|
-
additionalProperties: false,
|
|
2850
|
-
properties: {
|
|
2851
|
-
publicUrl: {
|
|
2852
|
-
type: 'string',
|
|
2853
|
-
description: "Public gateway URL for QR pairing (e.g. 'https://gateway.example.com:18789'). Overrides the auto-resolution cascade in buildPairingUrl.",
|
|
2854
|
-
},
|
|
2855
|
-
extraction: {
|
|
2856
|
-
type: 'object',
|
|
2857
|
-
additionalProperties: false,
|
|
2858
|
-
properties: {
|
|
2859
|
-
enabled: {
|
|
2860
|
-
type: 'boolean',
|
|
2861
|
-
description: 'Enable/disable auto-extraction (default: true)',
|
|
2862
|
-
},
|
|
2863
|
-
model: {
|
|
2864
|
-
type: 'string',
|
|
2865
|
-
description: "Shorthand: override just the extraction model (e.g., 'glm-4.5-flash', 'gpt-4.1-mini'). For a full provider override use extraction.llm.",
|
|
2866
|
-
},
|
|
2867
|
-
interval: {
|
|
2868
|
-
type: 'number',
|
|
2869
|
-
description: 'Number of turns between automatic extractions (default: 3)',
|
|
2870
|
-
},
|
|
2871
|
-
maxFactsPerExtraction: {
|
|
2872
|
-
type: 'number',
|
|
2873
|
-
description: 'Hard cap on facts extracted per turn (default: 15)',
|
|
2874
|
-
},
|
|
2875
|
-
llm: {
|
|
2876
|
-
type: 'object',
|
|
2877
|
-
additionalProperties: false,
|
|
2878
|
-
description: 'Explicit LLM override block. Highest-priority tier in the extraction-provider cascade. Any subset of provider+apiKey is enough to pin a provider.',
|
|
2879
|
-
properties: {
|
|
2880
|
-
provider: {
|
|
2881
|
-
type: 'string',
|
|
2882
|
-
description: "Provider name: zai | openai | anthropic | gemini | google | mistral | groq | deepseek | openrouter | xai | together | cerebras.",
|
|
2883
|
-
},
|
|
2884
|
-
model: {
|
|
2885
|
-
type: 'string',
|
|
2886
|
-
description: 'Explicit model id. If omitted, deriveCheapModel(provider) picks a sensible default.',
|
|
2887
|
-
},
|
|
2888
|
-
apiKey: {
|
|
2889
|
-
type: 'string',
|
|
2890
|
-
description: 'API key for the selected provider. Required for the override to take effect.',
|
|
2891
|
-
},
|
|
2892
|
-
baseUrl: {
|
|
2893
|
-
type: 'string',
|
|
2894
|
-
description: 'Override the provider base URL (self-hosted / custom gateway setups).',
|
|
2895
|
-
},
|
|
2896
|
-
},
|
|
2897
|
-
},
|
|
2898
|
-
},
|
|
2899
|
-
},
|
|
2900
|
-
},
|
|
2901
|
-
},
|
|
1918
|
+
configSchema: CONFIG_SCHEMA,
|
|
2902
1919
|
register(api) {
|
|
2903
1920
|
// NOTE: the body of register() below is intentionally NOT re-indent
|
|
2904
1921
|
// under this `try` block — re-indenting would touch every line in a
|
|
@@ -3215,7 +2232,7 @@ const plugin = {
|
|
|
3215
2232
|
// CLI subcommand actually fires.
|
|
3216
2233
|
if (typeof api.registerCli === 'function') {
|
|
3217
2234
|
api.registerCli(async ({ program }) => {
|
|
3218
|
-
const { registerOnboardingCli } = await import('./onboarding-cli.js');
|
|
2235
|
+
const { registerOnboardingCli } = await import('./pairing/onboarding-cli.js');
|
|
3219
2236
|
registerOnboardingCli(program, {
|
|
3220
2237
|
credentialsPath: CREDENTIALS_PATH,
|
|
3221
2238
|
statePath: CONFIG.onboardingStatePath,
|
|
@@ -3250,13 +2267,13 @@ const plugin = {
|
|
|
3250
2267
|
// same relay-brokered URL surface the agent tool uses. The local
|
|
3251
2268
|
// (gateway-loopback) flow is still available via `--local`. See
|
|
3252
2269
|
// pair-cli.ts header for the rationale.
|
|
3253
|
-
const { registerPairCli } = await import('./pair-cli.js');
|
|
2270
|
+
const { registerPairCli } = await import('./pairing/pair-cli.js');
|
|
3254
2271
|
registerPairCli(program, {
|
|
3255
2272
|
sessionsPath: CONFIG.pairSessionsPath,
|
|
3256
2273
|
renderPairingUrl: (session) => buildPairingUrl(api, session),
|
|
3257
2274
|
logger: api.logger,
|
|
3258
2275
|
runRelayPairCli: async (cliMode, runOpts) => {
|
|
3259
|
-
const { runRelayPairCli } = await import('./pair-cli-relay.js');
|
|
2276
|
+
const { runRelayPairCli } = await import('./pairing/pair-cli-relay.js');
|
|
3260
2277
|
return runRelayPairCli(cliMode, {
|
|
3261
2278
|
relayBaseUrl: CONFIG.pairRelayUrl,
|
|
3262
2279
|
credentialsPath: CREDENTIALS_PATH,
|
|
@@ -3339,7 +2356,7 @@ const plugin = {
|
|
|
3339
2356
|
.action(async (source, opts) => {
|
|
3340
2357
|
try {
|
|
3341
2358
|
await requireFullSetup(api.logger);
|
|
3342
|
-
const
|
|
2359
|
+
const importResult = await handlePluginImportFrom({
|
|
3343
2360
|
source,
|
|
3344
2361
|
file_path: opts.file,
|
|
3345
2362
|
content: opts.content,
|
|
@@ -3351,28 +2368,28 @@ const plugin = {
|
|
|
3351
2368
|
disclosure_confirmed: true,
|
|
3352
2369
|
}, api.logger);
|
|
3353
2370
|
if (opts.json) {
|
|
3354
|
-
process.stdout.write(JSON.stringify(
|
|
2371
|
+
process.stdout.write(JSON.stringify(importResult) + '\n');
|
|
3355
2372
|
}
|
|
3356
2373
|
else {
|
|
3357
2374
|
// Human-readable summary. The handler already returns a
|
|
3358
2375
|
// `message` for chunked (background) imports; for direct
|
|
3359
2376
|
// stores + dry runs, synthesize a short summary.
|
|
3360
|
-
if (
|
|
3361
|
-
const chunks =
|
|
2377
|
+
if (importResult.dry_run) {
|
|
2378
|
+
const chunks = importResult.total_chunks;
|
|
3362
2379
|
if (chunks !== undefined) {
|
|
3363
|
-
process.stdout.write(`Dry run: ~${
|
|
3364
|
-
`(~${
|
|
2380
|
+
process.stdout.write(`Dry run: ~${importResult.estimated_facts} facts from ${chunks} chunks ` +
|
|
2381
|
+
`(~${importResult.estimated_minutes} min). Confirm without --dry-run to start.\n`);
|
|
3365
2382
|
}
|
|
3366
2383
|
else {
|
|
3367
|
-
process.stdout.write(`Dry run: found ${
|
|
2384
|
+
process.stdout.write(`Dry run: found ${importResult.total_found} facts. Confirm without --dry-run to import.\n`);
|
|
3368
2385
|
}
|
|
3369
2386
|
}
|
|
3370
|
-
else if (
|
|
3371
|
-
process.stdout.write(`${
|
|
2387
|
+
else if (importResult.import_id && importResult.status === 'running') {
|
|
2388
|
+
process.stdout.write(`${importResult.message}\nImport id: ${importResult.import_id}\n`);
|
|
3372
2389
|
}
|
|
3373
2390
|
else {
|
|
3374
|
-
const stored =
|
|
3375
|
-
const total =
|
|
2391
|
+
const stored = importResult.imported;
|
|
2392
|
+
const total = importResult.total_found;
|
|
3376
2393
|
process.stdout.write(`Imported ${stored ?? 0}/${total ?? stored ?? 0} facts from ${source}.\n`);
|
|
3377
2394
|
}
|
|
3378
2395
|
}
|
|
@@ -3396,25 +2413,25 @@ const plugin = {
|
|
|
3396
2413
|
.action(async (opts) => {
|
|
3397
2414
|
try {
|
|
3398
2415
|
await requireFullSetup(api.logger);
|
|
3399
|
-
const
|
|
2416
|
+
const statusResult = await handleImportStatus({ import_id: opts.id }, api.logger);
|
|
3400
2417
|
if (opts.json) {
|
|
3401
|
-
process.stdout.write(JSON.stringify(
|
|
2418
|
+
process.stdout.write(JSON.stringify(statusResult) + '\n');
|
|
3402
2419
|
}
|
|
3403
2420
|
else {
|
|
3404
|
-
const status =
|
|
3405
|
-
const stored =
|
|
3406
|
-
const batchDone =
|
|
3407
|
-
const batchTotal =
|
|
2421
|
+
const status = statusResult.status;
|
|
2422
|
+
const stored = statusResult.facts_stored;
|
|
2423
|
+
const batchDone = statusResult.batch_done;
|
|
2424
|
+
const batchTotal = statusResult.batch_total;
|
|
3408
2425
|
if (status === 'no_active_import') {
|
|
3409
2426
|
process.stdout.write('No active import. Start one with `openclaw totalreclaw import from <source>`.\n');
|
|
3410
2427
|
}
|
|
3411
2428
|
else if (status === 'running') {
|
|
3412
|
-
process.stdout.write(`Import ${
|
|
2429
|
+
process.stdout.write(`Import ${statusResult.import_id}: running — ${stored} facts stored, ` +
|
|
3413
2430
|
`batch ${batchDone}/${batchTotal}` +
|
|
3414
|
-
(
|
|
2431
|
+
(statusResult.completion_iso ? `, ETA ${statusResult.completion_iso}` : '') + '.\n');
|
|
3415
2432
|
}
|
|
3416
2433
|
else {
|
|
3417
|
-
process.stdout.write(`Import ${
|
|
2434
|
+
process.stdout.write(`Import ${statusResult.import_id}: ${status} — ${stored ?? 0} facts stored.\n`);
|
|
3418
2435
|
}
|
|
3419
2436
|
}
|
|
3420
2437
|
}
|
|
@@ -3437,16 +2454,16 @@ const plugin = {
|
|
|
3437
2454
|
.action(async (importId, opts) => {
|
|
3438
2455
|
try {
|
|
3439
2456
|
await requireFullSetup(api.logger);
|
|
3440
|
-
const
|
|
2457
|
+
const abortResult = await handleImportAbort({ import_id: importId }, api.logger);
|
|
3441
2458
|
if (opts.json) {
|
|
3442
|
-
process.stdout.write(JSON.stringify(
|
|
2459
|
+
process.stdout.write(JSON.stringify(abortResult) + '\n');
|
|
3443
2460
|
}
|
|
3444
2461
|
else {
|
|
3445
|
-
if (
|
|
3446
|
-
process.stdout.write(`Import ${importId}: abort requested. ${
|
|
2462
|
+
if (abortResult.aborted) {
|
|
2463
|
+
process.stdout.write(`Import ${importId}: abort requested. ${abortResult.facts_already_stored ?? 0} facts already stored (kept).\n`);
|
|
3447
2464
|
}
|
|
3448
2465
|
else {
|
|
3449
|
-
process.stdout.write(`Import ${importId}: ${
|
|
2466
|
+
process.stdout.write(`Import ${importId}: ${abortResult.error ?? 'abort failed'}\n`);
|
|
3450
2467
|
}
|
|
3451
2468
|
}
|
|
3452
2469
|
}
|
|
@@ -3489,15 +2506,15 @@ const plugin = {
|
|
|
3489
2506
|
const body = await response.text().catch(() => '');
|
|
3490
2507
|
throw new Error(`checkout session failed (HTTP ${response.status}): ${body || response.statusText}`);
|
|
3491
2508
|
}
|
|
3492
|
-
const
|
|
3493
|
-
if (!
|
|
2509
|
+
const checkoutJson = await response.json();
|
|
2510
|
+
if (!checkoutJson.checkout_url) {
|
|
3494
2511
|
throw new Error('no checkout URL returned by the relay');
|
|
3495
2512
|
}
|
|
3496
2513
|
if (opts.json) {
|
|
3497
|
-
process.stdout.write(JSON.stringify({ checkout_url:
|
|
2514
|
+
process.stdout.write(JSON.stringify({ checkout_url: checkoutJson.checkout_url }) + '\n');
|
|
3498
2515
|
}
|
|
3499
2516
|
else {
|
|
3500
|
-
process.stdout.write(`Open this URL to upgrade to Pro: ${
|
|
2517
|
+
process.stdout.write(`Open this URL to upgrade to Pro: ${checkoutJson.checkout_url}\n`);
|
|
3501
2518
|
}
|
|
3502
2519
|
}
|
|
3503
2520
|
catch (err) {
|
|
@@ -4402,8 +3419,8 @@ const plugin = {
|
|
|
4402
3419
|
ensureMemoryHeader(api.logger);
|
|
4403
3420
|
// BUG-2 fix: skip extraction if an import was in progress this turn.
|
|
4404
3421
|
// Import failures were retriggering agent_end → extraction → import loops.
|
|
4405
|
-
if (
|
|
4406
|
-
|
|
3422
|
+
if (isImportInProgress()) {
|
|
3423
|
+
setImportInProgress(false); // auto-reset for next turn
|
|
4407
3424
|
api.logger.info('agent_end: skipping extraction (import was in progress)');
|
|
4408
3425
|
return { memoryHandled: true };
|
|
4409
3426
|
}
|
|
@@ -4488,7 +3505,7 @@ const plugin = {
|
|
|
4488
3505
|
logger: api.logger,
|
|
4489
3506
|
ensureInitialized: () => ensureInitialized(api.logger),
|
|
4490
3507
|
isPairingPending: () => needsSetup,
|
|
4491
|
-
isImportActive: () =>
|
|
3508
|
+
isImportActive: () => isImportInProgress(),
|
|
4492
3509
|
getExtractInterval,
|
|
4493
3510
|
getMaxFactsPerExtraction,
|
|
4494
3511
|
isDedupEnabled: isLlmDedupEnabled,
|