@totalreclaw/totalreclaw 3.3.12 → 3.3.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/SKILL.md +25 -8
  2. package/{billing-cache.ts → billing/billing-cache.ts} +1 -1
  3. package/{relay-headers.ts → billing/relay-headers.ts} +1 -1
  4. package/{tr-cli-export-helper.ts → cli/tr-cli-export-helper.ts} +5 -5
  5. package/{tr-cli.ts → cli/tr-cli.ts} +20 -20
  6. package/{contradiction-sync.ts → contradiction/contradiction-sync.ts} +13 -13
  7. package/{digest-sync.ts → digest/digest-sync.ts} +32 -2
  8. package/dist/{billing-cache.js → billing/billing-cache.js} +1 -1
  9. package/dist/{relay-headers.js → billing/relay-headers.js} +1 -1
  10. package/dist/{tr-cli-export-helper.js → cli/tr-cli-export-helper.js} +5 -5
  11. package/dist/{tr-cli.js → cli/tr-cli.js} +19 -19
  12. package/dist/{contradiction-sync.js → contradiction/contradiction-sync.js} +13 -13
  13. package/dist/{digest-sync.js → digest/digest-sync.js} +32 -2
  14. package/dist/{download-ux.js → embedding/download-ux.js} +1 -1
  15. package/dist/{reranker.js → embedding/reranker.js} +13 -4
  16. package/dist/{claims-helper.js → extraction/claims-helper.js} +54 -58
  17. package/dist/extraction/consolidation.js +170 -0
  18. package/dist/{extractor.js → extraction/extractor.js} +17 -17
  19. package/dist/extraction/importance-filter.js +38 -0
  20. package/dist/{semantic-dedup.js → extraction/semantic-dedup.js} +2 -2
  21. package/dist/{batch-gate.js → import/batch-gate.js} +1 -1
  22. package/dist/import/import-runtime.js +692 -0
  23. package/dist/import-adapters/chatgpt-adapter.js +7 -7
  24. package/dist/import-adapters/gemini-adapter.js +13 -13
  25. package/dist/import-adapters/mem0-adapter.js +10 -10
  26. package/dist/index.js +111 -1094
  27. package/dist/{llm-client.js → llm/llm-client.js} +2 -2
  28. package/dist/{hot-cache-wrapper.js → memory/hot-cache-wrapper.js} +5 -5
  29. package/dist/{memory-runtime.js → memory/memory-runtime.js} +1 -1
  30. package/dist/{pin.js → memory/pin.js} +18 -18
  31. package/dist/{retype-setscope.js → memory/retype-setscope.js} +19 -19
  32. package/dist/{credential-provider.js → pairing/credential-provider.js} +2 -2
  33. package/dist/{first-run.js → pairing/first-run.js} +1 -1
  34. package/dist/{onboarding-cli.js → pairing/onboarding-cli.js} +20 -20
  35. package/dist/{pair-cli-relay.js → pairing/pair-cli-relay.js} +10 -10
  36. package/dist/{pair-crypto.js → pairing/pair-crypto.js} +1 -1
  37. package/dist/{pair-http.js → pairing/pair-http.js} +3 -3
  38. package/dist/{pair-remote-client.js → pairing/pair-remote-client.js} +10 -10
  39. package/dist/{pair-session-store.js → pairing/pair-session-store.js} +4 -4
  40. package/dist/runtime/config-schema.js +59 -0
  41. package/dist/runtime/format-helpers.js +257 -0
  42. package/dist/runtime/types.js +11 -0
  43. package/dist/{confirm-indexed.js → subgraph/confirm-indexed.js} +1 -1
  44. package/dist/{subgraph-search.js → subgraph/subgraph-search.js} +14 -14
  45. package/dist/{subgraph-store.js → subgraph/subgraph-store.js} +6 -6
  46. package/{download-ux.ts → embedding/download-ux.ts} +1 -1
  47. package/{reranker.ts → embedding/reranker.ts} +13 -4
  48. package/{claims-helper.ts → extraction/claims-helper.ts} +52 -60
  49. package/{consolidation.ts → extraction/consolidation.ts} +63 -154
  50. package/{extractor.ts → extraction/extractor.ts} +18 -17
  51. package/extraction/importance-filter.ts +50 -0
  52. package/{semantic-dedup.ts → extraction/semantic-dedup.ts} +2 -2
  53. package/{batch-gate.ts → import/batch-gate.ts} +1 -1
  54. package/import/import-runtime.ts +853 -0
  55. package/import-adapters/chatgpt-adapter.ts +7 -7
  56. package/import-adapters/gemini-adapter.ts +13 -13
  57. package/import-adapters/mem0-adapter.ts +10 -10
  58. package/index.ts +129 -1377
  59. package/{llm-client.ts → llm/llm-client.ts} +2 -2
  60. package/{hot-cache-wrapper.ts → memory/hot-cache-wrapper.ts} +5 -5
  61. package/{memory-runtime.ts → memory/memory-runtime.ts} +1 -1
  62. package/{pin.ts → memory/pin.ts} +20 -20
  63. package/{retype-setscope.ts → memory/retype-setscope.ts} +21 -21
  64. package/{tool-gating.ts → memory/tool-gating.ts} +1 -1
  65. package/package.json +6 -6
  66. package/{credential-provider.ts → pairing/credential-provider.ts} +2 -2
  67. package/{first-run.ts → pairing/first-run.ts} +1 -1
  68. package/{onboarding-cli.ts → pairing/onboarding-cli.ts} +20 -20
  69. package/{pair-cli-relay.ts → pairing/pair-cli-relay.ts} +10 -10
  70. package/{pair-crypto.ts → pairing/pair-crypto.ts} +1 -1
  71. package/{pair-http.ts → pairing/pair-http.ts} +3 -3
  72. package/{pair-remote-client.ts → pairing/pair-remote-client.ts} +10 -10
  73. package/{pair-session-store.ts → pairing/pair-session-store.ts} +4 -4
  74. package/runtime/config-schema.ts +64 -0
  75. package/runtime/format-helpers.ts +293 -0
  76. package/runtime/types.ts +118 -0
  77. package/skill.json +44 -10
  78. package/{confirm-indexed.ts → subgraph/confirm-indexed.ts} +1 -1
  79. package/{subgraph-search.ts → subgraph/subgraph-search.ts} +14 -14
  80. package/{subgraph-store.ts → subgraph/subgraph-store.ts} +10 -10
  81. package/dist/consolidation.js +0 -249
  82. /package/{api-client.ts → billing/api-client.ts} +0 -0
  83. /package/{gateway-url.ts → billing/gateway-url.ts} +0 -0
  84. /package/{inbound-user-tracker.ts → billing/inbound-user-tracker.ts} +0 -0
  85. /package/{relay.ts → billing/relay.ts} +0 -0
  86. /package/{crypto.ts → crypto/crypto.ts} +0 -0
  87. /package/{vault-crypto.ts → crypto/vault-crypto.ts} +0 -0
  88. /package/dist/{api-client.js → billing/api-client.js} +0 -0
  89. /package/dist/{gateway-url.js → billing/gateway-url.js} +0 -0
  90. /package/dist/{inbound-user-tracker.js → billing/inbound-user-tracker.js} +0 -0
  91. /package/dist/{relay.js → billing/relay.js} +0 -0
  92. /package/dist/{crypto.js → crypto/crypto.js} +0 -0
  93. /package/dist/{vault-crypto.js → crypto/vault-crypto.js} +0 -0
  94. /package/dist/{embedder-cache.js → embedding/embedder-cache.js} +0 -0
  95. /package/dist/{embedder-loader.js → embedding/embedder-loader.js} +0 -0
  96. /package/dist/{embedder-network.js → embedding/embedder-network.js} +0 -0
  97. /package/dist/{embedding.js → embedding/embedding.js} +0 -0
  98. /package/dist/{lsh.js → embedding/lsh.js} +0 -0
  99. /package/dist/{import-state-manager.js → import/import-state-manager.js} +0 -0
  100. /package/dist/{llm-profile-reader.js → llm/llm-profile-reader.js} +0 -0
  101. /package/dist/{native-memory.js → memory/native-memory.js} +0 -0
  102. /package/dist/{tool-gating.js → memory/tool-gating.js} +0 -0
  103. /package/dist/{tools.js → memory/tools.js} +0 -0
  104. /package/dist/{generate-mnemonic.js → pairing/generate-mnemonic.js} +0 -0
  105. /package/dist/{pair-cli.js → pairing/pair-cli.js} +0 -0
  106. /package/dist/{pair-page.js → pairing/pair-page.js} +0 -0
  107. /package/dist/{pair-qr.js → pairing/pair-qr.js} +0 -0
  108. /package/dist/{restart-auth.js → pairing/restart-auth.js} +0 -0
  109. /package/dist/{qa-bug-report.js → setup/qa-bug-report.js} +0 -0
  110. /package/dist/{skill-register.js → setup/skill-register.js} +0 -0
  111. /package/dist/{trajectory-poller.js → subgraph/trajectory-poller.js} +0 -0
  112. /package/{embedder-cache.ts → embedding/embedder-cache.ts} +0 -0
  113. /package/{embedder-loader.ts → embedding/embedder-loader.ts} +0 -0
  114. /package/{embedder-network.ts → embedding/embedder-network.ts} +0 -0
  115. /package/{embedding.ts → embedding/embedding.ts} +0 -0
  116. /package/{lsh.ts → embedding/lsh.ts} +0 -0
  117. /package/{import-state-manager.ts → import/import-state-manager.ts} +0 -0
  118. /package/{llm-profile-reader.ts → llm/llm-profile-reader.ts} +0 -0
  119. /package/{native-memory.ts → memory/native-memory.ts} +0 -0
  120. /package/{tools.ts → memory/tools.ts} +0 -0
  121. /package/{generate-mnemonic.ts → pairing/generate-mnemonic.ts} +0 -0
  122. /package/{pair-cli.ts → pairing/pair-cli.ts} +0 -0
  123. /package/{pair-page.ts → pairing/pair-page.ts} +0 -0
  124. /package/{pair-qr.ts → pairing/pair-qr.ts} +0 -0
  125. /package/{restart-auth.ts → pairing/restart-auth.ts} +0 -0
  126. /package/{qa-bug-report.ts → setup/qa-bug-report.ts} +0 -0
  127. /package/{skill-register.ts → setup/skill-register.ts} +0 -0
  128. /package/{trajectory-poller.ts → subgraph/trajectory-poller.ts} +0 -0
package/dist/index.js CHANGED
@@ -65,42 +65,45 @@
65
65
  * All data is encrypted client-side with XChaCha20-Poly1305. The server never
66
66
  * sees plaintext.
67
67
  */
68
- import { deriveKeys, deriveLshSeed, computeAuthKeyHash, encrypt, decrypt, generateBlindIndices, generateContentFingerprint, } from './crypto.js';
69
- import { createApiClient } from './api-client.js';
70
- import { extractFacts, extractCrystal, EXTRACTION_SYSTEM_PROMPT, extractFactsForCompaction, } from './extractor.js';
71
- import { initLLMClient, resolveLLMConfig, chatCompletion, generateEmbedding, getEmbeddingDims, getEmbeddingModelId, configureEmbedder, prefetchEmbedderBundle, } from './llm-client.js';
72
- import { defaultAuthProfilesRoot, readAllProfileKeys, dedupeByProvider, } from './llm-profile-reader.js';
73
- import { LSHHasher } from './lsh.js';
74
- import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS } from './reranker.js';
75
- import { deduplicateBatch } from './semantic-dedup.js';
76
- import { startTrajectoryPoller } from './trajectory-poller.js';
77
- import { findNearDuplicate, shouldSupersede, getStoreDedupThreshold, STORE_DEDUP_MAX_CANDIDATES, } from './consolidation.js';
78
- import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4 } from './subgraph-store.js';
79
- import { DIGEST_TRAPDOOR, buildCanonicalClaim, computeEntityTrapdoors, isDigestBlob, readClaimFromBlob, resolveDigestMode, } from './claims-helper.js';
80
- import { maybeInjectDigest, recompileDigest, fetchAllActiveClaims, isRecompileInProgress, tryBeginRecompile, endRecompile, } from './digest-sync.js';
81
- import { detectAndResolveContradictions, runWeightTuningLoop, } from './contradiction-sync.js';
82
- import { searchSubgraph, searchSubgraphBroadened, getSubgraphFactCount, fetchFactById } from './subgraph-search.js';
83
- import { PluginHotCache } from './hot-cache-wrapper.js';
68
+ import { deriveKeys, deriveLshSeed, computeAuthKeyHash, generateBlindIndices, generateContentFingerprint, } from './crypto/crypto.js';
69
+ import { createApiClient } from './billing/api-client.js';
70
+ import { extractFacts, extractCrystal, extractFactsForCompaction, } from './extraction/extractor.js';
71
+ import { initLLMClient, resolveLLMConfig, chatCompletion, generateEmbedding, getEmbeddingDims, getEmbeddingModelId, configureEmbedder, prefetchEmbedderBundle, } from './llm/llm-client.js';
72
+ import { defaultAuthProfilesRoot, readAllProfileKeys, dedupeByProvider, } from './llm/llm-profile-reader.js';
73
+ import { LSHHasher } from './embedding/lsh.js';
74
+ import { rerank, cosineSimilarity, detectQueryIntent, INTENT_WEIGHTS } from './embedding/reranker.js';
75
+ import { deduplicateBatch } from './extraction/semantic-dedup.js';
76
+ import { startTrajectoryPoller } from './subgraph/trajectory-poller.js';
77
+ import { findNearDuplicate, shouldSupersede, getStoreDedupThreshold, STORE_DEDUP_MAX_CANDIDATES, } from './extraction/consolidation.js';
78
+ import { isSubgraphMode, getSubgraphConfig, encodeFactProtobuf, submitFactBatchOnChain, deriveSmartAccountAddress, PROTOBUF_VERSION_V4 } from './subgraph/subgraph-store.js';
79
+ import { DIGEST_TRAPDOOR, buildCanonicalClaim, computeEntityTrapdoors, isDigestBlob, readClaimFromBlob, resolveDigestMode, } from './extraction/claims-helper.js';
80
+ import { maybeInjectDigest, recompileDigest, fetchAllActiveClaims, isRecompileInProgress, tryBeginRecompile, endRecompile, } from './digest/digest-sync.js';
81
+ import { detectAndResolveContradictions, runWeightTuningLoop, } from './contradiction/contradiction-sync.js';
82
+ import { searchSubgraph, searchSubgraphBroadened, getSubgraphFactCount, fetchFactById } from './subgraph/subgraph-search.js';
83
+ import { PluginHotCache } from './memory/hot-cache-wrapper.js';
84
84
  import { CONFIG, setRecoveryPhraseOverride } from './config.js';
85
- import { buildRelayHeaders } from './relay-headers.js';
86
- import { readBillingCache, writeBillingCache, BILLING_CACHE_PATH, } from './billing-cache.js';
87
- import { ensureMemoryHeaderFile, loadCredentialsJson, writeCredentialsJson, deleteCredentialsFile, isRunningInDocker, deleteFileIfExists, resolveOnboardingState, writeOnboardingState, readPluginVersion, cleanupInstallStagingDirs, clearPartialInstallMarker, patchOpenClawConfig, checkCredentialsFileMode, } from './fs-helpers.js';
88
- import { isRcBuild } from './qa-bug-report.js';
89
- import { decideToolGate, isGatedToolName } from './tool-gating.js';
90
- import { resolveRestartAuth, rejectMessageFor, } from './restart-auth.js';
91
- import { recordInboundUser, getDistinctInboundUserCount, resolveTrackerPath, } from './inbound-user-tracker.js';
92
- import { detectFirstRun, buildWelcomePrepend } from './first-run.js';
93
- import { buildPairRoutes } from './pair-http.js';
94
- import { detectGatewayHost } from './gateway-url.js';
95
- import { registerNativeMemory } from './native-memory.js';
96
- import { ensureSkillRegistered } from './skill-register.js';
85
+ import { buildRelayHeaders } from './billing/relay-headers.js';
86
+ import { readBillingCache, writeBillingCache, BILLING_CACHE_PATH, } from './billing/billing-cache.js';
87
+ import { ensureMemoryHeaderFile, loadCredentialsJson, writeCredentialsJson, deleteCredentialsFile, deleteFileIfExists, resolveOnboardingState, writeOnboardingState, readPluginVersion, cleanupInstallStagingDirs, clearPartialInstallMarker, patchOpenClawConfig, checkCredentialsFileMode, } from './fs-helpers.js';
88
+ import { isRcBuild } from './setup/qa-bug-report.js';
89
+ import { decideToolGate, isGatedToolName } from './memory/tool-gating.js';
90
+ import { resolveRestartAuth, rejectMessageFor, } from './pairing/restart-auth.js';
91
+ import { recordInboundUser, getDistinctInboundUserCount, resolveTrackerPath, } from './billing/inbound-user-tracker.js';
92
+ import { detectFirstRun, buildWelcomePrepend } from './pairing/first-run.js';
93
+ import { buildPairRoutes } from './pairing/pair-http.js';
94
+ import { registerNativeMemory } from './memory/native-memory.js';
95
+ import { ensureSkillRegistered } from './setup/skill-register.js';
96
+ import { humanizeError, buildPairingUrl, resolveGatewayMode, computeCandidatePool, encryptToHex, decryptFromHex, relativeTime, } from './runtime/format-helpers.js';
97
+ import { CONFIG_SCHEMA } from './runtime/config-schema.js';
98
+ import { filterByImportance } from './extraction/importance-filter.js';
99
+ import { handlePluginImportFrom, handleImportStatus, handleImportAbort, configureImportRuntime, isImportInProgress, setImportInProgress, } from './import/import-runtime.js';
97
100
  import { validateMnemonic } from '@scure/bip39';
98
101
  import { wordlist } from '@scure/bip39/wordlists/english.js';
99
102
  import crypto from 'node:crypto';
100
103
  import { createRequire } from 'node:module';
101
104
  import { fileURLToPath } from 'node:url';
102
105
  import * as nodePath from 'node:path';
103
- import { writeImportState, readImportState, isImportStale, readMostRecentActiveImport, } from './import-state-manager.js';
106
+ import { isImportStale, readMostRecentActiveImport, } from './import/import-state-manager.js';
104
107
  // CJS-style require for the @totalreclaw/core WASM module. We keep this
105
108
  // load path lazy (only inside getSmartImportWasm() below) so a partial
106
109
  // install of the dependency tree doesn't crash module init. Bare
@@ -111,35 +114,8 @@ import { writeImportState, readImportState, isImportStale, readMostRecentActiveI
111
114
  // subgraph-store.ts / claims-helper.ts.
112
115
  const __cjsRequire = createRequire(import.meta.url);
113
116
  // ---------------------------------------------------------------------------
114
- // Human-friendly error messages
117
+ // OpenClaw Plugin API type — extracted to ./runtime/types.ts (imported above).
115
118
  // ---------------------------------------------------------------------------
116
- /**
117
- * Translate technical error messages from the on-chain submission pipeline
118
- * into user-friendly messages. The original technical details are still
119
- * logged via api.logger — this only affects what the agent sees.
120
- */
121
- function humanizeError(rawMessage) {
122
- if (rawMessage.includes('AA23')) {
123
- return 'Memory storage temporarily unavailable. Will retry next time.';
124
- }
125
- if (rawMessage.includes('AA10')) {
126
- return 'Please wait a moment before storing more memories.';
127
- }
128
- if (rawMessage.includes('AA25')) {
129
- return 'Memory storage busy. Will retry.';
130
- }
131
- if (rawMessage.includes('pm_sponsorUserOperation')) {
132
- return 'Memory storage service temporarily unavailable.';
133
- }
134
- if (/Relay returned HTTP\s*404/.test(rawMessage)) {
135
- return 'Memory service is temporarily offline.';
136
- }
137
- if (/Relay returned HTTP\s*5\d\d/.test(rawMessage)) {
138
- return 'Memory service encountered a temporary error. Will retry next time.';
139
- }
140
- // Pass through non-technical messages as-is.
141
- return rawMessage;
142
- }
143
119
  // ---------------------------------------------------------------------------
144
120
  // Persistent credential storage
145
121
  // ---------------------------------------------------------------------------
@@ -148,153 +124,7 @@ const CREDENTIALS_PATH = CONFIG.credentialsPath;
148
124
  // ---------------------------------------------------------------------------
149
125
  // 3.3.0 — pairing URL resolution
150
126
  // ---------------------------------------------------------------------------
151
- /**
152
- * Build the full pairing URL (including `#pk=` fragment) for a fresh
153
- * pairing session. Pulls gateway config from `api.config.gateway`.
154
- *
155
- * Resolution order (3.3.1 — six-layer cascade):
156
- * 1. `plugins.entries.totalreclaw.config.publicUrl` — explicit override
157
- * 2. `gateway.remote.url` — OpenClaw's own remote-gateway URL
158
- * 3. `gateway.bind === 'custom'` + `gateway.customBindHost` + port
159
- * 4. Tailscale auto-detect — `tailscale status --json` → `https://<MagicDNS>`
160
- * (assumes `tailscale serve` proxies to the gateway port on 443)
161
- * 5. LAN auto-detect — first non-loopback, non-virtual IPv4 interface.
162
- * Emits a warning: "only works on the same network".
163
- * 6. Fallback `http://localhost:<port>` — warns with a pointer to
164
- * configure `plugins.entries.totalreclaw.config.publicUrl`.
165
- *
166
- * Always returns a working URL string; never throws. The caller (CLI or
167
- * JSON output) prints whatever we give it.
168
- */
169
- function buildPairingUrl(api, session) {
170
- const cfg = api.config;
171
- const pluginCfg = (api.pluginConfig ?? {});
172
- const tlsEnabled = cfg?.gateway?.tls?.enabled === true;
173
- const scheme = tlsEnabled ? 'https' : 'http';
174
- const port = cfg?.gateway?.port ?? 18789;
175
- let base;
176
- // Layer 1 — explicit user override
177
- if (typeof pluginCfg.publicUrl === 'string' && pluginCfg.publicUrl.trim()) {
178
- base = pluginCfg.publicUrl.replace(/\/+$/, '');
179
- base = base.replace(/^wss:\/\//i, 'https://').replace(/^ws:\/\//i, 'http://');
180
- }
181
- // Layer 2 — OpenClaw gateway remote URL
182
- else if (typeof cfg?.gateway?.remote?.url === 'string' && cfg.gateway.remote.url.trim()) {
183
- base = cfg.gateway.remote.url.trim().replace(/\/+$/, '');
184
- base = base.replace(/^wss:\/\//i, 'https://').replace(/^ws:\/\//i, 'http://');
185
- }
186
- // Layer 3 — gateway.bind = custom + explicit customBindHost
187
- else if (cfg?.gateway?.bind === 'custom' && cfg.gateway.customBindHost) {
188
- base = `${scheme}://${cfg.gateway.customBindHost}:${port}`;
189
- }
190
- // Layers 4 + 5 — auto-detect via gateway-url helper (Tailscale CGNAT, then LAN)
191
- else {
192
- let detected = null;
193
- // issue #110 fix 4 — pass `isDocker` so LAN detection skips
194
- // 172.16/12 bridge IPs that no external browser can reach.
195
- let isDocker = false;
196
- try {
197
- isDocker = isRunningInDocker();
198
- }
199
- catch {
200
- // Defensive: never block URL building on Docker sniff errors.
201
- isDocker = false;
202
- }
203
- try {
204
- detected = detectGatewayHost({ isDocker });
205
- }
206
- catch (err) {
207
- api.logger.warn(`TotalReclaw: host autodetect crashed: ${err instanceof Error ? err.message : String(err)} — falling back to localhost`);
208
- }
209
- if (detected?.kind === 'tailscale') {
210
- // 3.3.1-rc.2: we surface the raw Tailscale CGNAT IP because passive
211
- // NIC detection (no subprocess) cannot resolve the MagicDNS name.
212
- // Caller can override via `publicUrl` for a proper https://<magicdns>.
213
- // The IP + port URL still works inside the tailnet (peers can reach
214
- // each other by CGNAT IP directly). TLS defaults to the gateway's
215
- // own config because we no longer assume `tailscale serve`.
216
- base = `${scheme}://${detected.host}:${port}`;
217
- api.logger.warn(`TotalReclaw: pairing URL using Tailscale CGNAT IP ${detected.host}:${port} — ` +
218
- detected.note);
219
- }
220
- else if (detected?.kind === 'lan') {
221
- base = `${scheme}://${detected.host}:${port}`;
222
- api.logger.warn(`TotalReclaw: pairing URL using LAN host ${detected.host}:${port} — ` +
223
- `this URL only works from the same network. ` +
224
- `Set plugins.entries.totalreclaw.config.publicUrl for remote access.`);
225
- }
226
- else {
227
- // Layer 6 — localhost fallback (or Docker-aware relay-pointer warning)
228
- const bind = cfg?.gateway?.bind;
229
- if (isDocker) {
230
- // issue #110 fix 4: inside Docker the LAN IP is container-internal
231
- // and useless. Loopback localhost only works for `docker exec`
232
- // tests. The CORRECT pair URL for Docker is the relay-brokered
233
- // path served by `tr pair` / the `/plugin/totalreclaw/pair/*` HTTP
234
- // routes (CONFIG.pairMode === 'relay' since rc.11). The CLI-only
235
- // path here cannot mint a relay session synchronously (the relay
236
- // handshake needs a WS round-trip), so we emit the loopback URL
237
- // with a LOUD warning pointing the operator at the pair CLI /
238
- // publicUrl override.
239
- api.logger.warn(`TotalReclaw: Docker container detected — pairing URL falling back to ` +
240
- `http://localhost:${port}, which is unreachable from the host browser. ` +
241
- `Run \`tr pair --url-pin\` (or \`openclaw totalreclaw pair generate --url-pin-only\`) ` +
242
- `on the gateway host to mint a relay-brokered pair URL that reaches the host browser, ` +
243
- `OR set plugins.entries.totalreclaw.config.publicUrl ` +
244
- `to your gateway's host-reachable URL (e.g. http://<host-ip>:${port} when the ` +
245
- `Docker port is published). Setting TOTALRECLAW_PAIR_MODE=relay is the default; ` +
246
- `air-gapped operators on TOTALRECLAW_PAIR_MODE=local must publish a port + set publicUrl.`);
247
- }
248
- else if (bind === 'lan' || bind === 'tailnet') {
249
- api.logger.warn(`TotalReclaw: pairing URL falling back to localhost because gateway.bind=${bind} could not be autodetected. ` +
250
- 'Set plugins.entries.totalreclaw.config.publicUrl to override.');
251
- }
252
- else {
253
- api.logger.warn(`TotalReclaw: pairing URL fell back to http://localhost:${port} — this URL only works on this machine. ` +
254
- `Configure plugins.entries.totalreclaw.config.publicUrl for remote access.`);
255
- }
256
- base = `${scheme}://localhost:${port}`;
257
- }
258
- }
259
- return `${base}/plugin/totalreclaw/pair/finish?sid=${encodeURIComponent(session.sid)}#pk=${encodeURIComponent(session.pkGatewayB64)}`;
260
- }
261
- /**
262
- * Resolve whether this plugin is running on a `local` or `remote` gateway.
263
- *
264
- * Follows the same config surface `buildPairingUrl` uses:
265
- * - `pluginConfig.publicUrl` set + non-localhost → remote
266
- * - `gateway.remote.url` set + non-localhost → remote
267
- * - `gateway.bind === 'lan' | 'tailnet' | 'custom'` → remote
268
- * - anything else → local
269
- *
270
- * We treat a `publicUrl` or `remote.url` that points at `localhost` /
271
- * `127.*` as local because that is what a dev-loopback override looks like;
272
- * no one publishes a remote QR pairing for localhost.
273
- */
274
- function resolveGatewayMode(api) {
275
- const cfg = api.config;
276
- const pluginCfg = (api.pluginConfig ?? {});
277
- const looksLocal = (url) => {
278
- if (!url)
279
- return true;
280
- const u = url.trim().toLowerCase();
281
- if (u === '')
282
- return true;
283
- return /^(?:wss?:\/\/|https?:\/\/)?(?:localhost|127\.|0\.0\.0\.0)/.test(u);
284
- };
285
- if (typeof pluginCfg.publicUrl === 'string' && !looksLocal(pluginCfg.publicUrl)) {
286
- return 'remote';
287
- }
288
- const remoteUrl = cfg?.gateway?.remote?.url;
289
- if (typeof remoteUrl === 'string' && !looksLocal(remoteUrl)) {
290
- return 'remote';
291
- }
292
- const bind = cfg?.gateway?.bind;
293
- if (bind === 'lan' || bind === 'tailnet' || bind === 'custom') {
294
- return 'remote';
295
- }
296
- return 'local';
297
- }
127
+ // buildPairingUrl / resolveGatewayMode extracted to ./runtime/format-helpers.ts.
298
128
  // ---------------------------------------------------------------------------
299
129
  // Cosine similarity threshold — skip injection when top result is below this
300
130
  // ---------------------------------------------------------------------------
@@ -330,9 +160,6 @@ const CACHE_TTL_MS = CONFIG.cacheTtlMs;
330
160
  const SEMANTIC_SKIP_THRESHOLD = CONFIG.semanticSkipThreshold;
331
161
  // Auto-extract throttle (C3): only extract every N turns in agent_end hook
332
162
  let turnsSinceLastExtraction = 0;
333
- // BUG-2 fix: Skip agent_end extraction during import operations.
334
- // Import failures previously triggered agent_end → re-extraction → re-import loops.
335
- let _importInProgress = false;
336
163
  const AUTO_EXTRACT_EVERY_TURNS_ENV = CONFIG.extractInterval;
337
164
  // Hard cap on facts per extraction to prevent LLM over-extraction from dense conversations
338
165
  const MAX_FACTS_PER_EXTRACTION = 15;
@@ -469,24 +296,7 @@ let cachedFactCount = null;
469
296
  let lastFactCountFetch = 0;
470
297
  /** Cache TTL for fact count: 5 minutes. */
471
298
  const FACT_COUNT_CACHE_TTL = 5 * 60 * 1000;
472
- /**
473
- * Compute the candidate pool size from a fact count.
474
- *
475
- * Server-side config takes priority (from billing cache), then local fallback.
476
- * The server computes the optimal pool based on vault size and tier caps.
477
- *
478
- * Local fallback formula: pool = min(max(factCount * 3, 400), 5000)
479
- * - At least 400 candidates (even for tiny vaults)
480
- * - At most 5000 candidates (to bound decryption + reranking cost)
481
- * - 3x fact count in between
482
- */
483
- function computeCandidatePool(factCount) {
484
- const cache = readBillingCache();
485
- if (cache?.features?.max_candidate_pool != null)
486
- return cache.features.max_candidate_pool;
487
- // Fallback to local formula if no server config
488
- return Math.min(Math.max(factCount * 3, 400), 5000);
489
- }
299
+ // computeCandidatePool extracted to ./runtime/format-helpers.ts.
490
300
  /**
491
301
  * Fetch the user's fact count from the server, with caching.
492
302
  *
@@ -773,9 +583,6 @@ async function initialize(logger) {
773
583
  }
774
584
  }
775
585
  }
776
- function isDocker() {
777
- return isRunningInDocker();
778
- }
779
586
  function buildSetupErrorMsg() {
780
587
  // NOTE: the legacy `totalreclaw_setup` agent tool was retired in 3.2.0
781
588
  // (phrase-safety: the agent must never accept or relay a recovery phrase).
@@ -788,28 +595,6 @@ function buildSetupErrorMsg() {
788
595
  'and hand the user the returned `url` and `pin`. The user opens the URL in a browser to complete pairing. ' +
789
596
  'Do NOT ask the user for a recovery phrase and do NOT attempt to generate or relay one yourself.';
790
597
  }
791
- function buildSetupErrorMsgLegacy() {
792
- const base = 'TotalReclaw setup required:\n' +
793
- '1. Set TOTALRECLAW_RECOVERY_PHRASE — ask the user if they have an existing recovery phrase or generate a new 12-word recovery phrase.\n' +
794
- '2. Restart the gateway to apply changes.\n' +
795
- ' (Optional: set TOTALRECLAW_SELF_HOSTED=true if using your own server instead of the managed service.)\n\n';
796
- if (isDocker()) {
797
- return base +
798
- 'Running in Docker — pass env vars via `-e` flags or your compose file:\n' +
799
- ' -e TOTALRECLAW_RECOVERY_PHRASE="word1 word2 ..."';
800
- }
801
- if (process.platform === 'darwin') {
802
- return base +
803
- 'Running on macOS — add env vars to the LaunchAgent plist at\n' +
804
- '~/Library/LaunchAgents/ai.openclaw.gateway.plist under <key>EnvironmentVariables</key>:\n' +
805
- ' <key>TOTALRECLAW_RECOVERY_PHRASE</key><string>word1 word2 ...</string>\n' +
806
- 'Then run: openclaw gateway restart';
807
- }
808
- return base +
809
- 'Running on Linux — add env vars to the systemd unit override or your shell profile:\n' +
810
- ' export TOTALRECLAW_RECOVERY_PHRASE="word1 word2 ..."\n' +
811
- 'Then run: openclaw gateway restart';
812
- }
813
598
  const SETUP_ERROR_MSG = buildSetupErrorMsg();
814
599
  /**
815
600
  * Ensure `initialize()` has completed (runs at most once).
@@ -1056,10 +841,10 @@ async function searchForNearDuplicates(factText, factEmbedding, allIndices, logg
1056
841
  }
1057
842
  if (decryptedCandidates.length === 0)
1058
843
  return null;
1059
- const result = findNearDuplicate(factEmbedding, decryptedCandidates, getStoreDedupThreshold());
1060
- if (!result)
844
+ const nearDup = findNearDuplicate(factEmbedding, decryptedCandidates, getStoreDedupThreshold());
845
+ if (!nearDup)
1061
846
  return null;
1062
- return { match: result.existingFact, similarity: result.similarity };
847
+ return { match: nearDup.existingFact, similarity: nearDup.similarity };
1063
848
  }
1064
849
  catch (err) {
1065
850
  const msg = err instanceof Error ? err.message : String(err);
@@ -1070,16 +855,7 @@ async function searchForNearDuplicates(factText, factEmbedding, allIndices, logg
1070
855
  // ---------------------------------------------------------------------------
1071
856
  // Utility helpers
1072
857
  // ---------------------------------------------------------------------------
1073
- /**
1074
- * Encrypt a plaintext document string and return its hex-encoded ciphertext.
1075
- *
1076
- * The server stores blobs as hex (not base64), so we convert the base64
1077
- * output of `encrypt()` into hex.
1078
- */
1079
- function encryptToHex(plaintext, key) {
1080
- const b64 = encrypt(plaintext, key);
1081
- return Buffer.from(b64, 'base64').toString('hex');
1082
- }
858
+ // encryptToHex extracted to ./runtime/format-helpers.ts.
1083
859
  // Plugin v3.0.0 removed the legacy claim-format fallback. Write path
1084
860
  // always emits Memory Taxonomy v1 JSON blobs. The logClaimFormatOnce
1085
861
  // helper is gone along with TOTALRECLAW_CLAIM_FORMAT / TOTALRECLAW_TAXONOMY_VERSION.
@@ -1172,8 +948,8 @@ function scheduleDigestRecompile(previousClaimId, logger) {
1172
948
  version: PROTOBUF_VERSION_V4,
1173
949
  });
1174
950
  const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
1175
- const result = await submitFactBatchOnChain([protobuf], config);
1176
- if (!result.success) {
951
+ const batchResult = await submitFactBatchOnChain([protobuf], config);
952
+ if (!batchResult.success) {
1177
953
  throw new Error('Digest store UserOp did not succeed on-chain');
1178
954
  }
1179
955
  };
@@ -1201,8 +977,8 @@ function scheduleDigestRecompile(previousClaimId, logger) {
1201
977
  };
1202
978
  const protobuf = encodeFactProtobuf(tombstone);
1203
979
  const config = { ...getSubgraphConfig(), authKeyHex: authKey, walletAddress: ownerForBatch };
1204
- const result = await submitFactBatchOnChain([protobuf], config);
1205
- if (!result.success) {
980
+ const batchResult = await submitFactBatchOnChain([protobuf], config);
981
+ if (!batchResult.success) {
1206
982
  throw new Error('Digest tombstone UserOp did not succeed on-chain');
1207
983
  }
1208
984
  };
@@ -1235,14 +1011,11 @@ function scheduleDigestRecompile(previousClaimId, logger) {
1235
1011
  endRecompile();
1236
1012
  });
1237
1013
  }
1238
- /**
1239
- * Decrypt a hex-encoded ciphertext blob into a UTF-8 string.
1240
- */
1241
- function decryptFromHex(hexBlob, key) {
1242
- const hex = hexBlob.startsWith('0x') ? hexBlob.slice(2) : hexBlob;
1243
- const b64 = Buffer.from(hex, 'hex').toString('base64');
1244
- return decrypt(b64, key);
1245
- }
1014
+ // decryptFromHex extracted to ./runtime/format-helpers.ts.
1015
+ // ---------------------------------------------------------------------------
1016
+ // Migration GraphQL helpers
1017
+ // ---------------------------------------------------------------------------
1018
+ // MigrationFact extracted to ./runtime/types.ts (imported above).
1246
1019
  const MIGRATION_PAGE_SIZE = 1000;
1247
1020
  /** Execute a GraphQL query against a subgraph endpoint. Returns null on error. */
1248
1021
  async function migrationGqlQuery(endpoint, query, variables, authKey) {
@@ -1279,8 +1052,8 @@ async function fetchAllFactsByOwner(subgraphUrl, owner, authKey) {
1279
1052
  const vars = hasLastId
1280
1053
  ? { owner, first: MIGRATION_PAGE_SIZE, lastId }
1281
1054
  : { owner, first: MIGRATION_PAGE_SIZE };
1282
- const data = await migrationGqlQuery(subgraphUrl, query, vars, authKey);
1283
- const facts = data?.facts ?? [];
1055
+ const subgraphResponse = await migrationGqlQuery(subgraphUrl, query, vars, authKey);
1056
+ const facts = subgraphResponse?.facts ?? [];
1284
1057
  if (facts.length === 0)
1285
1058
  break;
1286
1059
  allFacts.push(...facts);
@@ -1302,8 +1075,8 @@ async function fetchContentFingerprintsByOwner(subgraphUrl, owner, authKey) {
1302
1075
  const vars = hasLastId
1303
1076
  ? { owner, first: MIGRATION_PAGE_SIZE, lastId }
1304
1077
  : { owner, first: MIGRATION_PAGE_SIZE };
1305
- const data = await migrationGqlQuery(subgraphUrl, query, vars, authKey);
1306
- const facts = data?.facts ?? [];
1078
+ const subgraphResponse = await migrationGqlQuery(subgraphUrl, query, vars, authKey);
1079
+ const facts = subgraphResponse?.facts ?? [];
1307
1080
  if (facts.length === 0)
1308
1081
  break;
1309
1082
  for (const f of facts) {
@@ -1318,19 +1091,19 @@ async function fetchContentFingerprintsByOwner(subgraphUrl, owner, authKey) {
1318
1091
  }
1319
1092
  /** Fetch blind index hashes for given fact IDs. */
1320
1093
  async function fetchBlindIndicesByFactIds(subgraphUrl, factIds, authKey) {
1321
- const result = new Map();
1094
+ const hashesByFactId = new Map();
1322
1095
  const CHUNK = 50;
1323
1096
  for (let i = 0; i < factIds.length; i += CHUNK) {
1324
1097
  const chunk = factIds.slice(i, i + CHUNK);
1325
1098
  const query = `query($factIds:[String!]!,$first:Int!){blindIndexes(where:{fact_in:$factIds},first:$first){hash fact{id}}}`;
1326
- const data = await migrationGqlQuery(subgraphUrl, query, { factIds: chunk, first: 1000 }, authKey);
1327
- for (const entry of data?.blindIndexes ?? []) {
1328
- const existing = result.get(entry.fact.id) || [];
1099
+ const subgraphResponse = await migrationGqlQuery(subgraphUrl, query, { factIds: chunk, first: 1000 }, authKey);
1100
+ for (const entry of subgraphResponse?.blindIndexes ?? []) {
1101
+ const existing = hashesByFactId.get(entry.fact.id) || [];
1329
1102
  existing.push(entry.hash);
1330
- result.set(entry.fact.id, existing);
1103
+ hashesByFactId.set(entry.fact.id, existing);
1331
1104
  }
1332
1105
  }
1333
- return result;
1106
+ return hashesByFactId;
1334
1107
  }
1335
1108
  /**
1336
1109
  * Fetch existing memories from the vault to provide dedup context for extraction.
@@ -1405,34 +1178,11 @@ async function fetchExistingMemoriesForExtraction(logger, limit = 30, rawMessage
1405
1178
  * Simple text-overlap scoring between a query and a candidate document.
1406
1179
  * Returns the number of overlapping lowercase words.
1407
1180
  */
1408
- function textScore(query, docText) {
1409
- const queryWords = new Set(query.toLowerCase().split(/\s+/).filter((w) => w.length >= 2));
1410
- const docWords = docText.toLowerCase().split(/\s+/);
1411
- let score = 0;
1412
- for (const word of docWords) {
1413
- if (queryWords.has(word))
1414
- score++;
1415
- }
1416
- return score;
1417
- }
1181
+ // textScore extracted to ./runtime/format-helpers.ts.
1418
1182
  /**
1419
1183
  * Format a relative time string (e.g. "2 hours ago").
1420
1184
  */
1421
- function relativeTime(isoOrMs) {
1422
- const ms = typeof isoOrMs === 'number' ? isoOrMs : new Date(isoOrMs).getTime();
1423
- const diffMs = Date.now() - ms;
1424
- const seconds = Math.floor(diffMs / 1000);
1425
- if (seconds < 60)
1426
- return 'just now';
1427
- const minutes = Math.floor(seconds / 60);
1428
- if (minutes < 60)
1429
- return `${minutes}m ago`;
1430
- const hours = Math.floor(minutes / 60);
1431
- if (hours < 24)
1432
- return `${hours}h ago`;
1433
- const days = Math.floor(hours / 24);
1434
- return `${days}d ago`;
1435
- }
1185
+ // relativeTime extracted to ./runtime/format-helpers.ts.
1436
1186
  // ---------------------------------------------------------------------------
1437
1187
  // Importance filter for auto-extraction
1438
1188
  // ---------------------------------------------------------------------------
@@ -1444,39 +1194,7 @@ function relativeTime(isoOrMs) {
1444
1194
  * NOTE: This filter is ONLY applied to auto-extraction (hooks).
1445
1195
  * The explicit `tr remember` CLI always stores regardless of importance.
1446
1196
  */
1447
- const MIN_IMPORTANCE_THRESHOLD = CONFIG.minImportance;
1448
- /**
1449
- * Filter extracted facts by importance threshold.
1450
- * Facts with importance < MIN_IMPORTANCE_THRESHOLD are dropped.
1451
- * Facts with missing/undefined importance are treated as importance=5 (kept).
1452
- */
1453
- function filterByImportance(facts, logger) {
1454
- const kept = [];
1455
- let dropped = 0;
1456
- for (const fact of facts) {
1457
- const importance = fact.importance ?? 5;
1458
- if (importance >= MIN_IMPORTANCE_THRESHOLD) {
1459
- kept.push(fact);
1460
- }
1461
- else {
1462
- dropped++;
1463
- }
1464
- }
1465
- // Phase 2.2.5: always log the filter outcome so the agent_end path can
1466
- // distinguish "LLM returned 0 facts" from "LLM returned N facts all dropped
1467
- // below threshold" from "LLM returned N facts, all kept". Prior to 2.2.5
1468
- // this only logged on drops, which made empty-input invisible.
1469
- if (facts.length === 0) {
1470
- logger.info('Importance filter: input=0 (nothing to filter)');
1471
- }
1472
- else if (dropped > 0) {
1473
- logger.info(`Importance filter: dropped ${dropped}/${facts.length} facts below threshold ${MIN_IMPORTANCE_THRESHOLD}`);
1474
- }
1475
- else {
1476
- logger.info(`Importance filter: kept all ${facts.length} facts (threshold ${MIN_IMPORTANCE_THRESHOLD})`);
1477
- }
1478
- return { kept, dropped };
1479
- }
1197
+ // filterByImportance + MIN_IMPORTANCE_THRESHOLD extracted to ./extraction/importance-filter.ts.
1480
1198
  // ---------------------------------------------------------------------------
1481
1199
  // Auto-extraction helper
1482
1200
  // ---------------------------------------------------------------------------
@@ -1497,10 +1215,10 @@ async function storeExtractedFacts(facts, logger, sourceOverride) {
1497
1215
  const embeddingResultMap = new Map();
1498
1216
  for (const fact of facts) {
1499
1217
  try {
1500
- const result = await generateEmbeddingAndLSH(fact.text, logger);
1501
- if (result) {
1502
- embeddingMap.set(fact.text, result.embedding);
1503
- embeddingResultMap.set(fact.text, result);
1218
+ const embeddingResult = await generateEmbeddingAndLSH(fact.text, logger);
1219
+ if (embeddingResult) {
1220
+ embeddingMap.set(fact.text, embeddingResult.embedding);
1221
+ embeddingResultMap.set(fact.text, embeddingResult);
1504
1222
  }
1505
1223
  }
1506
1224
  catch {
@@ -1818,13 +1536,13 @@ async function storeExtractedFacts(facts, logger, sourceOverride) {
1818
1536
  for (let i = 0; i < pendingPayloads.length; i++) {
1819
1537
  const slice = [pendingPayloads[i]]; // Single fact per UserOp
1820
1538
  try {
1821
- const result = await submitFactBatchOnChain(slice, batchConfig);
1822
- if (result.success) {
1539
+ const submitResult = await submitFactBatchOnChain(slice, batchConfig);
1540
+ if (submitResult.success) {
1823
1541
  stored += slice.length;
1824
- logger.info(`Fact ${i + 1}/${pendingPayloads.length}: submitted on-chain (tx=${result.txHash.slice(0, 10)}…)`);
1542
+ logger.info(`Fact ${i + 1}/${pendingPayloads.length}: submitted on-chain (tx=${submitResult.txHash.slice(0, 10)}…)`);
1825
1543
  }
1826
1544
  else {
1827
- batchError = `On-chain batch submission failed (tx=${result.txHash.slice(0, 10)}…)`;
1545
+ batchError = `On-chain batch submission failed (tx=${submitResult.txHash.slice(0, 10)}…)`;
1828
1546
  logger.warn(batchError);
1829
1547
  break; // Stop submitting remaining batches
1830
1548
  }
@@ -1862,657 +1580,10 @@ async function storeExtractedFacts(facts, logger, sourceOverride) {
1862
1580
  return stored;
1863
1581
  }
1864
1582
  // ---------------------------------------------------------------------------
1865
- // Import handler (for the registerCli `openclaw totalreclaw import-from` surface)
1866
- // ---------------------------------------------------------------------------
1867
- /**
1868
- * Handle import_from calls (CLI subcommand path; was the totalreclaw_import_from
1869
- * agent tool before Phase 3.2 retired the agent tools).
1870
- *
1871
- * Two paths:
1872
- * 1. Pre-structured sources (Mem0, MCP Memory) — adapter returns facts directly,
1873
- * stored via storeExtractedFacts().
1874
- * 2. Conversation-based sources (ChatGPT, Claude) — adapter returns conversation
1875
- * chunks, each chunk is passed through extractFacts() (the same LLM extraction
1876
- * pipeline used for auto-extraction), then stored via storeExtractedFacts().
1877
- */
1878
- async function handlePluginImportFrom(params, logger) {
1879
- _importInProgress = true;
1880
- const startTime = Date.now();
1881
- const source = params.source;
1882
- const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini'];
1883
- if (!source || !validSources.includes(source)) {
1884
- return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
1885
- }
1886
- // Generate import_id up front so dry-run responses and background tasks share it.
1887
- const importId = params.resume_id ?? crypto.randomUUID();
1888
- try {
1889
- const { getAdapter } = await import('./import-adapters/index.js');
1890
- const adapter = getAdapter(source);
1891
- const parseResult = await adapter.parse({
1892
- content: params.content,
1893
- api_key: params.api_key,
1894
- source_user_id: params.source_user_id,
1895
- api_url: params.api_url,
1896
- file_path: params.file_path,
1897
- });
1898
- const hasChunks = parseResult.chunks && parseResult.chunks.length > 0;
1899
- const hasFacts = parseResult.facts && parseResult.facts.length > 0;
1900
- if (parseResult.errors.length > 0 && !hasFacts && !hasChunks) {
1901
- return {
1902
- success: false,
1903
- error: `Failed to parse ${adapter.displayName} data`,
1904
- details: parseResult.errors,
1905
- };
1906
- }
1907
- // Dry run: report what was parsed (chunks or facts)
1908
- if (params.dry_run) {
1909
- if (hasChunks) {
1910
- const totalChunks = parseResult.chunks.length;
1911
- const EXTRACTION_RATIO = 2.5; // avg facts per chunk, from empirical data
1912
- const BATCH_SIZE = 25;
1913
- const SECONDS_PER_BATCH = 45; // ~30s extraction + ~15s embed+store
1914
- const estimatedFacts = Math.round(totalChunks * EXTRACTION_RATIO);
1915
- const estimatedBatches = Math.ceil(totalChunks / BATCH_SIZE);
1916
- const estimatedMinutes = Math.ceil(estimatedBatches * SECONDS_PER_BATCH / 60);
1917
- return {
1918
- success: true,
1919
- dry_run: true,
1920
- import_id: importId,
1921
- source,
1922
- total_chunks: totalChunks,
1923
- total_messages: parseResult.totalMessages,
1924
- estimated_facts: estimatedFacts,
1925
- estimated_batches: estimatedBatches,
1926
- estimated_minutes: estimatedMinutes,
1927
- batch_size: BATCH_SIZE,
1928
- preview: parseResult.chunks.slice(0, 5).map((c) => ({
1929
- title: c.title,
1930
- messages: c.messages.length,
1931
- first_message: c.messages[0]?.text.slice(0, 100),
1932
- })),
1933
- note: `Estimated ${estimatedFacts} facts from ${totalChunks} chunks (~${estimatedMinutes} min). Confirm to start background import.`,
1934
- warnings: parseResult.warnings,
1935
- };
1936
- }
1937
- return {
1938
- success: true,
1939
- dry_run: true,
1940
- import_id: importId,
1941
- source,
1942
- total_found: parseResult.facts.length,
1943
- preview: parseResult.facts.slice(0, 10).map((f) => ({
1944
- type: f.type,
1945
- text: f.text.slice(0, 100),
1946
- importance: f.importance,
1947
- })),
1948
- warnings: parseResult.warnings,
1949
- };
1950
- }
1951
- // ── Path 1: Conversation chunks (ChatGPT, Claude, Gemini) — background execution ──
1952
- if (hasChunks) {
1953
- const totalChunks = parseResult.chunks.length;
1954
- const BATCH_SIZE = 25;
1955
- const SECONDS_PER_BATCH = 45;
1956
- const estimatedBatches = Math.ceil(totalChunks / BATCH_SIZE);
1957
- const estimatedMinutes = Math.ceil(estimatedBatches * SECONDS_PER_BATCH / 60);
1958
- const estimatedTotalFacts = Math.round(totalChunks * 2.5);
1959
- const now = new Date();
1960
- const initialState = {
1961
- import_id: importId,
1962
- source,
1963
- status: 'running',
1964
- started_at: now.toISOString(),
1965
- last_updated: now.toISOString(),
1966
- total_chunks: totalChunks,
1967
- total_messages: parseResult.totalMessages,
1968
- batch_done: 0,
1969
- batch_total: estimatedBatches,
1970
- facts_stored: 0,
1971
- facts_extracted: 0,
1972
- dups_skipped: 0,
1973
- errors: [],
1974
- file_path: params.file_path,
1975
- estimated_total_facts: estimatedTotalFacts,
1976
- estimated_minutes: estimatedMinutes,
1977
- estimated_completion_iso: new Date(now.getTime() + estimatedBatches * SECONDS_PER_BATCH * 1000).toISOString(),
1978
- disclosure_confirmed: !!(params.disclosure_confirmed),
1979
- };
1980
- writeImportState(initialState);
1981
- logger.info(`Import ${importId}: background task started (${totalChunks} chunks, ~${estimatedMinutes}min)`);
1982
- void handleChunkImport(parseResult.chunks, parseResult.totalMessages, source, logger, startTime, parseResult.warnings, importId).catch((err) => {
1983
- const msg = err instanceof Error ? err.message : String(err);
1984
- logger.warn(`Import ${importId}: background task failed: ${msg}`);
1985
- const failedState = readImportState(importId);
1986
- if (failedState && failedState.status === 'running') {
1987
- writeImportState({ ...failedState, status: 'failed', errors: [...failedState.errors, msg] });
1988
- }
1989
- });
1990
- return {
1991
- import_id: importId,
1992
- status: 'running',
1993
- source,
1994
- total_chunks: totalChunks,
1995
- estimated_batches: estimatedBatches,
1996
- estimated_minutes: estimatedMinutes,
1997
- estimated_completion_iso: initialState.estimated_completion_iso,
1998
- message: `Import started in background. ~${estimatedMinutes} min for ${totalChunks} chunks. Check progress with \`openclaw totalreclaw import status\` on the gateway host (or \`import status --id ${importId} --json\` from an agent shell).`,
1999
- warnings: parseResult.warnings,
2000
- };
2001
- }
2002
- // ── Path 2: Pre-structured facts (Mem0, MCP Memory) — direct store ──
2003
- const extractedFacts = parseResult.facts.map((f) => ({
2004
- text: f.text,
2005
- type: f.type,
2006
- importance: f.importance,
2007
- action: 'ADD',
2008
- }));
2009
- // Store in batches of 50. Stop on any batch failure to prevent
2010
- // nonce zombies from blocking subsequent UserOps (AA25).
2011
- let totalStored = 0;
2012
- let storeError;
2013
- const batchSize = 50;
2014
- for (let i = 0; i < extractedFacts.length; i += batchSize) {
2015
- const batch = extractedFacts.slice(i, i + batchSize);
2016
- try {
2017
- const stored = await storeExtractedFacts(batch, logger);
2018
- totalStored += stored;
2019
- logger.info(`Import progress: ${Math.min(i + batchSize, extractedFacts.length)}/${extractedFacts.length} processed, ${totalStored} stored`);
2020
- }
2021
- catch (err) {
2022
- storeError = err instanceof Error ? err.message : String(err);
2023
- logger.warn(`Import stopped at batch ${Math.floor(i / batchSize) + 1}: ${storeError}`);
2024
- break; // Stop processing further batches
2025
- }
2026
- }
2027
- const importWarnings = [...parseResult.warnings];
2028
- if (storeError) {
2029
- importWarnings.push(`Import stopped early: ${storeError}`);
2030
- }
2031
- return {
2032
- success: totalStored > 0,
2033
- source,
2034
- import_id: importId,
2035
- total_found: parseResult.facts.length,
2036
- imported: totalStored,
2037
- skipped: parseResult.facts.length - totalStored,
2038
- stopped_early: !!storeError,
2039
- warnings: importWarnings,
2040
- duration_ms: Date.now() - startTime,
2041
- };
2042
- }
2043
- catch (e) {
2044
- const msg = e instanceof Error ? e.message : 'Unknown error';
2045
- logger.error(`Import failed: ${msg}`);
2046
- return { success: false, error: `Import failed: ${msg}` };
2047
- }
2048
- }
2049
- // ---------------------------------------------------------------------------
2050
- // Smart Import — Two-Pass Pipeline (Profile + Triage)
1583
+ // Import subsystem extracted to ./import/import-runtime.ts.
1584
+ // storeExtractedFacts is injected below (it closes over plugin session state).
2051
1585
  // ---------------------------------------------------------------------------
2052
- // Lazy-load WASM for smart import functions (same pattern as crypto.ts /
2053
- // subgraph-store.ts). Goes through __cjsRequire (createRequire(import.meta.url))
2054
- // declared at the top of the file — bare `require()` is undefined under
2055
- // pure-ESM Node, see issue #124.
2056
- let _smartImportWasm = null;
2057
- function getSmartImportWasm() {
2058
- if (!_smartImportWasm)
2059
- _smartImportWasm = __cjsRequire('@totalreclaw/core');
2060
- return _smartImportWasm;
2061
- }
2062
- /**
2063
- * Check whether the @totalreclaw/core WASM module exposes smart import functions.
2064
- * Returns false if the module is an older version without smart import support.
2065
- */
2066
- function hasSmartImportSupport() {
2067
- try {
2068
- const wasm = getSmartImportWasm();
2069
- return typeof wasm.chunksToSummaries === 'function' &&
2070
- typeof wasm.buildProfileBatchPrompt === 'function' &&
2071
- typeof wasm.parseProfileBatchResponse === 'function' &&
2072
- typeof wasm.buildTriagePrompt === 'function' &&
2073
- typeof wasm.parseTriageResponse === 'function' &&
2074
- typeof wasm.enrichExtractionPrompt === 'function';
2075
- }
2076
- catch {
2077
- return false;
2078
- }
2079
- }
2080
- /**
2081
- * Run the smart import two-pass pipeline: profile the user from conversation
2082
- * summaries, then triage chunks as EXTRACT or SKIP.
2083
- *
2084
- * All prompt construction and response parsing happens in @totalreclaw/core WASM.
2085
- * LLM calls use the plugin's existing chatCompletion() function.
2086
- *
2087
- * Returns null if smart import is unavailable (old WASM, no LLM config, etc.)
2088
- * so the caller can fall back to blind extraction.
2089
- */
2090
- async function runSmartImportPipeline(chunks, logger) {
2091
- // Guard: WASM must have smart import functions
2092
- if (!hasSmartImportSupport()) {
2093
- logger.info('Smart import: WASM module does not support smart import, falling back to blind extraction');
2094
- return null;
2095
- }
2096
- // Guard: LLM must be available
2097
- const llmConfig = resolveLLMConfig();
2098
- if (!llmConfig) {
2099
- logger.info('Smart import: no LLM available, falling back to blind extraction');
2100
- return null;
2101
- }
2102
- const pipelineStart = Date.now();
2103
- const wasm = getSmartImportWasm();
2104
- try {
2105
- // Step 0: Convert chunks to compact summaries (first + last message)
2106
- const wasmChunks = chunks.map((c, i) => ({
2107
- index: i,
2108
- title: c.title || 'Untitled',
2109
- messages: c.messages.map((m) => ({ role: m.role, content: m.text })),
2110
- timestamp: c.timestamp || null,
2111
- }));
2112
- const summaries = wasm.chunksToSummaries(JSON.stringify(wasmChunks));
2113
- const summariesJson = JSON.stringify(summaries);
2114
- // Step 1: Build user profile (batch summarize -> merge)
2115
- const PROFILE_BATCH_SIZE = 50;
2116
- const profileStart = Date.now();
2117
- const partials = [];
2118
- for (let i = 0; i < summaries.length; i += PROFILE_BATCH_SIZE) {
2119
- const batch = summaries.slice(i, i + PROFILE_BATCH_SIZE);
2120
- const prompt = wasm.buildProfileBatchPrompt(JSON.stringify(batch));
2121
- const response = await chatCompletion(llmConfig, [
2122
- { role: 'user', content: prompt },
2123
- ], { maxTokens: 2048, temperature: 0 });
2124
- if (!response) {
2125
- logger.warn(`Smart import: LLM returned empty response for profile batch ${Math.floor(i / PROFILE_BATCH_SIZE) + 1}`);
2126
- continue;
2127
- }
2128
- const partial = wasm.parseProfileBatchResponse(response);
2129
- partials.push(partial);
2130
- }
2131
- if (partials.length === 0) {
2132
- logger.warn('Smart import: no profile batches produced, falling back to blind extraction');
2133
- return null;
2134
- }
2135
- let profile;
2136
- if (partials.length === 1) {
2137
- // Single batch — skip merge, promote partial to full profile
2138
- // parseProfileBatchResponse returns a PartialProfile; convert to UserProfile shape
2139
- const p = partials[0];
2140
- profile = {
2141
- identity: p.identity ?? null,
2142
- themes: p.themes ?? [],
2143
- projects: p.projects ?? [],
2144
- stack: p.stack ?? [],
2145
- decisions: p.decisions ?? [],
2146
- interests: p.interests ?? [],
2147
- skip_patterns: p.skip_patterns ?? [],
2148
- };
2149
- }
2150
- else {
2151
- const mergePrompt = wasm.buildProfileMergePrompt(JSON.stringify(partials));
2152
- const mergeResponse = await chatCompletion(llmConfig, [
2153
- { role: 'user', content: mergePrompt },
2154
- ], { maxTokens: 2048, temperature: 0 });
2155
- if (!mergeResponse) {
2156
- logger.warn('Smart import: LLM returned empty response for profile merge, falling back to blind extraction');
2157
- return null;
2158
- }
2159
- profile = wasm.parseProfileResponse(mergeResponse);
2160
- }
2161
- const profileJson = JSON.stringify(profile);
2162
- const profileDuration = Date.now() - profileStart;
2163
- const p = profile;
2164
- const themeCount = Array.isArray(p.themes) ? p.themes.length : 0;
2165
- const skipPatternCount = Array.isArray(p.skip_patterns) ? p.skip_patterns.length : 0;
2166
- logger.info(`Smart import: profile built in ${profileDuration}ms (themes=${themeCount}, skip_patterns=${skipPatternCount})`);
2167
- // Step 1.5: Chunk triage (EXTRACT or SKIP)
2168
- const triageStart = Date.now();
2169
- const allDecisions = [];
2170
- const TRIAGE_BATCH_SIZE = 50;
2171
- for (let i = 0; i < summaries.length; i += TRIAGE_BATCH_SIZE) {
2172
- const batch = summaries.slice(i, i + TRIAGE_BATCH_SIZE);
2173
- const triagePrompt = wasm.buildTriagePrompt(profileJson, JSON.stringify(batch));
2174
- const triageResponse = await chatCompletion(llmConfig, [
2175
- { role: 'user', content: triagePrompt },
2176
- ], { maxTokens: 4096, temperature: 0 });
2177
- if (!triageResponse) {
2178
- logger.warn(`Smart import: LLM returned empty response for triage batch ${Math.floor(i / TRIAGE_BATCH_SIZE) + 1}, defaulting to EXTRACT`);
2179
- // Default all chunks in this batch to EXTRACT
2180
- for (let j = i; j < Math.min(i + TRIAGE_BATCH_SIZE, summaries.length); j++) {
2181
- allDecisions.push({ chunk_index: j, decision: 'EXTRACT', reason: 'triage LLM unavailable' });
2182
- }
2183
- continue;
2184
- }
2185
- const batchDecisions = wasm.parseTriageResponse(triageResponse);
2186
- allDecisions.push(...batchDecisions);
2187
- }
2188
- const triageDuration = Date.now() - triageStart;
2189
- const extractCount = allDecisions.filter((d) => d.decision !== 'SKIP').length;
2190
- const skipCount = allDecisions.filter((d) => d.decision === 'SKIP').length;
2191
- logger.info(`Smart import: triage complete in ${triageDuration}ms (extract=${extractCount}, skip=${skipCount}, total=${chunks.length})`);
2192
- // Step 2: Build enriched system prompt for extraction
2193
- const enrichedSystemPrompt = wasm.enrichExtractionPrompt(profileJson, EXTRACTION_SYSTEM_PROMPT);
2194
- const totalDuration = Date.now() - pipelineStart;
2195
- logger.info(`Smart import: pipeline complete in ${totalDuration}ms`);
2196
- return {
2197
- profileJson,
2198
- decisions: allDecisions,
2199
- enrichedSystemPrompt,
2200
- extractCount,
2201
- skipCount,
2202
- durationMs: totalDuration,
2203
- };
2204
- }
2205
- catch (err) {
2206
- const msg = err instanceof Error ? err.message : String(err);
2207
- logger.warn(`Smart import: pipeline failed (${msg}), falling back to blind extraction`);
2208
- return null;
2209
- }
2210
- }
2211
- /**
2212
- * Check if a chunk should be skipped based on triage decisions.
2213
- * If no decision exists for the chunk index, defaults to EXTRACT (safe default).
2214
- */
2215
- function isChunkSkipped(chunkIndex, decisions) {
2216
- const decision = decisions.find((d) => d.chunk_index === chunkIndex);
2217
- if (decision && decision.decision === 'SKIP') {
2218
- return { skipped: true, reason: decision.reason || 'triage: skip' };
2219
- }
2220
- return { skipped: false, reason: '' };
2221
- }
2222
- /**
2223
- * Process a batch (slice) of conversation chunks from a file.
2224
- * Called repeatedly by the agent for large imports.
2225
- */
2226
- async function handleBatchImport(params, logger) {
2227
- _importInProgress = true;
2228
- const source = params.source;
2229
- const filePath = params.file_path;
2230
- const content = params.content;
2231
- const offset = params.offset ?? 0;
2232
- const batchSize = params.batch_size ?? 25;
2233
- const validSources = ['mem0', 'mcp-memory', 'chatgpt', 'claude', 'gemini'];
2234
- if (!source || !validSources.includes(source)) {
2235
- return { success: false, error: `Invalid source. Must be one of: ${validSources.join(', ')}` };
2236
- }
2237
- const startTime = Date.now();
2238
- const { getAdapter } = await import('./import-adapters/index.js');
2239
- const adapter = getAdapter(source);
2240
- const parseResult = await adapter.parse({ content, file_path: filePath });
2241
- if (parseResult.errors.length > 0 && parseResult.chunks.length === 0) {
2242
- return { success: false, error: parseResult.errors.join('; ') };
2243
- }
2244
- const totalChunks = parseResult.chunks.length;
2245
- const slice = parseResult.chunks.slice(offset, offset + batchSize);
2246
- const remaining = Math.max(0, totalChunks - offset - slice.length);
2247
- // --- Smart Import: Profile + Triage ---
2248
- // Build profile from ALL chunks (not just the slice) for full context,
2249
- // then triage only the current slice. For simplicity, we rebuild on every
2250
- // batch call — optimization (caching) can come later.
2251
- const smartCtx = await runSmartImportPipeline(parseResult.chunks, logger);
2252
- let chunksSkipped = 0;
2253
- // Process the slice through the normal extraction + storage pipeline.
2254
- // If a batch fails (nonce zombie, quota exceeded, etc.), stop immediately
2255
- // to prevent subsequent UserOps from hitting AA25 nonce conflicts.
2256
- let factsExtracted = 0;
2257
- let factsStored = 0;
2258
- let chunksProcessed = 0;
2259
- let storeError;
2260
- for (let i = 0; i < slice.length; i++) {
2261
- const chunk = slice[i];
2262
- const globalIndex = offset + i; // Index in the full chunks array
2263
- // Smart import: skip chunks triaged as SKIP
2264
- if (smartCtx) {
2265
- const { skipped, reason } = isChunkSkipped(globalIndex, smartCtx.decisions);
2266
- if (skipped) {
2267
- logger.info(`Import: skipping chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}" (${reason})`);
2268
- chunksSkipped++;
2269
- chunksProcessed++;
2270
- continue;
2271
- }
2272
- }
2273
- logger.info(`Import: extracting facts from chunk ${globalIndex + 1}/${totalChunks}: "${chunk.title}"`);
2274
- const messages = chunk.messages.map((m) => ({ role: m.role, content: m.text }));
2275
- const facts = await extractFacts(messages, 'full', undefined, // no existing memories for dedup during import
2276
- smartCtx?.enrichedSystemPrompt);
2277
- chunksProcessed++;
2278
- if (facts.length > 0) {
2279
- factsExtracted += facts.length;
2280
- try {
2281
- const stored = await storeExtractedFacts(facts, logger);
2282
- factsStored += stored;
2283
- }
2284
- catch (err) {
2285
- storeError = err instanceof Error ? err.message : String(err);
2286
- logger.warn(`Import batch stopped at chunk ${globalIndex + 1}/${totalChunks}: ${storeError}`);
2287
- break; // Stop processing further chunks — a zombie UserOp may block writes
2288
- }
2289
- }
2290
- }
2291
- return {
2292
- success: factsStored > 0 || (!storeError && factsExtracted === 0),
2293
- batch_offset: offset,
2294
- batch_size: chunksProcessed,
2295
- total_chunks: totalChunks,
2296
- facts_extracted: factsExtracted,
2297
- facts_stored: factsStored,
2298
- chunks_skipped: chunksSkipped,
2299
- remaining_chunks: remaining,
2300
- is_complete: remaining === 0 && !storeError,
2301
- stopped_early: !!storeError,
2302
- error: storeError,
2303
- smart_import: smartCtx ? {
2304
- profile_duration_ms: smartCtx.durationMs,
2305
- extract_count: smartCtx.extractCount,
2306
- skip_count: smartCtx.skipCount,
2307
- } : null,
2308
- // Estimation for the full import
2309
- estimated_total_facts: Math.round(totalChunks * 2.5),
2310
- estimated_total_userops: Math.ceil(totalChunks * 2.5 / 15),
2311
- estimated_minutes: Math.ceil(Math.ceil(totalChunks / batchSize) * 45 / 60),
2312
- duration_ms: Date.now() - startTime,
2313
- };
2314
- }
2315
- /**
2316
- * Process conversation chunks through LLM extraction and store results.
2317
- *
2318
- * Each chunk is passed to extractFacts() — the same extraction pipeline used
2319
- * for auto-extraction during live conversations. This ensures import quality
2320
- * matches conversation extraction quality.
2321
- */
2322
- async function handleChunkImport(chunks, totalMessages, source, logger, startTime, warnings, importId) {
2323
- let totalExtracted = 0;
2324
- let totalStored = 0;
2325
- let chunksProcessed = 0;
2326
- let chunksSkipped = 0;
2327
- const resolvedImportId = importId ?? crypto.randomUUID();
2328
- let storeError;
2329
- // --- Smart Import: Profile + Triage ---
2330
- const smartCtx = await runSmartImportPipeline(chunks, logger);
2331
- const CHECKPOINT_EVERY = 25; // write state file every N chunks
2332
- for (let i = 0; i < chunks.length; i++) {
2333
- // Check abort flag from state file before each chunk (background task may be cancelled).
2334
- if (importId) {
2335
- const currentState = readImportState(importId);
2336
- if (currentState?.status === 'aborted') {
2337
- logger.info(`Import ${importId}: abort flag detected at chunk ${i + 1}/${chunks.length}, stopping`);
2338
- break;
2339
- }
2340
- }
2341
- const chunk = chunks[i];
2342
- chunksProcessed++;
2343
- // Smart import: skip chunks triaged as SKIP
2344
- if (smartCtx) {
2345
- const { skipped, reason } = isChunkSkipped(i, smartCtx.decisions);
2346
- if (skipped) {
2347
- logger.info(`Import: skipping chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}" (${reason})`);
2348
- chunksSkipped++;
2349
- continue;
2350
- }
2351
- }
2352
- logger.info(`Import: extracting facts from chunk ${chunksProcessed}/${chunks.length}: "${chunk.title}"`);
2353
- // Convert chunk messages to the format extractFacts() expects.
2354
- // extractFacts() takes an array of message-like objects with { role, content }.
2355
- const messages = chunk.messages.map((m) => ({
2356
- role: m.role,
2357
- content: m.text,
2358
- }));
2359
- // Use 'full' mode to extract ALL valuable memories from the chunk
2360
- // (not just the last few messages like 'turn' mode does).
2361
- // Smart import: pass enriched system prompt with user profile context.
2362
- const facts = await extractFacts(messages, 'full', undefined, // no existing memories for dedup during import
2363
- smartCtx?.enrichedSystemPrompt);
2364
- if (facts.length > 0) {
2365
- totalExtracted += facts.length;
2366
- try {
2367
- // Store through the normal pipeline (dedup, encrypt, store).
2368
- // storeExtractedFacts throws on batch failure to prevent nonce zombies.
2369
- const stored = await storeExtractedFacts(facts, logger);
2370
- totalStored += stored;
2371
- logger.info(`Import chunk ${chunksProcessed}/${chunks.length}: extracted ${facts.length} facts, stored ${stored}`);
2372
- }
2373
- catch (err) {
2374
- storeError = err instanceof Error ? err.message : String(err);
2375
- logger.warn(`Import stopped at chunk ${chunksProcessed}/${chunks.length}: ${storeError}`);
2376
- break; // Stop processing further chunks — a zombie UserOp may block writes
2377
- }
2378
- }
2379
- // Checkpoint state file periodically so _import_status reflects live progress.
2380
- if (importId && chunksProcessed % CHECKPOINT_EVERY === 0) {
2381
- const liveState = readImportState(importId);
2382
- if (liveState) {
2383
- const estimatedBatches = liveState.batch_total || 1;
2384
- const doneBatches = Math.floor(chunksProcessed / CHECKPOINT_EVERY);
2385
- const elapsed = Date.now() - new Date(liveState.started_at).getTime();
2386
- const secPerBatch = doneBatches > 0 ? elapsed / 1000 / doneBatches : 45;
2387
- const remaining = estimatedBatches - doneBatches;
2388
- const etaMs = remaining * secPerBatch * 1000;
2389
- writeImportState({
2390
- ...liveState,
2391
- batch_done: doneBatches,
2392
- facts_stored: totalStored,
2393
- facts_extracted: totalExtracted,
2394
- estimated_completion_iso: new Date(Date.now() + etaMs).toISOString(),
2395
- });
2396
- }
2397
- }
2398
- }
2399
- if (totalExtracted === 0 && chunks.length > 0 && !storeError && chunksSkipped < chunks.length) {
2400
- warnings.push(`Processed ${chunks.length} conversation chunks (${totalMessages} messages) but the LLM ` +
2401
- `did not extract any facts worth storing. This can happen if the conversations are mostly ` +
2402
- `generic/ephemeral content without personal facts, preferences, or decisions.`);
2403
- }
2404
- if (storeError) {
2405
- warnings.push(`Import stopped early: ${storeError}. ${chunks.length - chunksProcessed} chunk(s) not processed.`);
2406
- }
2407
- // Final state file write for background imports.
2408
- if (importId) {
2409
- const finalState = readImportState(importId);
2410
- if (finalState) {
2411
- const finalStatus = storeError ? 'failed' : (finalState.status === 'aborted' ? 'aborted' : 'completed');
2412
- writeImportState({
2413
- ...finalState,
2414
- status: finalStatus,
2415
- batch_done: finalState.batch_total,
2416
- facts_stored: totalStored,
2417
- facts_extracted: totalExtracted,
2418
- errors: storeError ? [...finalState.errors, storeError] : finalState.errors,
2419
- });
2420
- }
2421
- _importInProgress = false;
2422
- }
2423
- return {
2424
- success: totalStored > 0 || totalExtracted > 0,
2425
- source,
2426
- import_id: resolvedImportId,
2427
- total_chunks: chunks.length,
2428
- chunks_processed: chunksProcessed,
2429
- chunks_skipped: chunksSkipped,
2430
- total_messages: totalMessages,
2431
- facts_extracted: totalExtracted,
2432
- imported: totalStored,
2433
- skipped: totalExtracted - totalStored,
2434
- stopped_early: !!storeError,
2435
- smart_import: smartCtx ? {
2436
- profile_duration_ms: smartCtx.durationMs,
2437
- extract_count: smartCtx.extractCount,
2438
- skip_count: smartCtx.skipCount,
2439
- } : null,
2440
- warnings,
2441
- duration_ms: Date.now() - startTime,
2442
- };
2443
- }
2444
- // ---------------------------------------------------------------------------
2445
- // Import status + abort handlers
2446
- // ---------------------------------------------------------------------------
2447
- async function handleImportStatus(params, logger) {
2448
- const importId = params.import_id;
2449
- let state;
2450
- if (importId) {
2451
- state = readImportState(importId);
2452
- if (!state)
2453
- return { error: `No import found with id: ${importId}` };
2454
- }
2455
- else {
2456
- state = readMostRecentActiveImport();
2457
- if (!state)
2458
- return { status: 'no_active_import', message: 'No active import found. Start one with `openclaw totalreclaw import from <source>` on the gateway host. (Auto-resume still picks up running imports on gateway restart.)' };
2459
- }
2460
- // 1h freshness guard: mark stale imports as failed and prompt user to resume.
2461
- if (state.status === 'running' && isImportStale(state)) {
2462
- writeImportState({ ...state, status: 'failed', errors: [...state.errors, 'Stale: no progress in 1h'] });
2463
- logger.info(`Import ${state.import_id}: marked stale (no progress in 1h)`);
2464
- return {
2465
- import_id: state.import_id,
2466
- status: 'failed',
2467
- stale: true,
2468
- facts_stored: state.facts_stored,
2469
- message: 'Import appears stale — no progress in 1 hour. Resume it with `openclaw totalreclaw import from <source> --file <path> --resume ' + state.import_id + '` on the gateway host, or restart the gateway to trigger auto-resume.',
2470
- resume_id: state.import_id,
2471
- };
2472
- }
2473
- const now = Date.now();
2474
- const elapsedMs = now - new Date(state.started_at).getTime();
2475
- const secPerBatch = state.batch_done > 0 ? elapsedMs / 1000 / state.batch_done : 45;
2476
- const remaining = Math.max(0, state.batch_total - state.batch_done);
2477
- const etaSeconds = state.status === 'running' ? Math.round(remaining * secPerBatch) : 0;
2478
- return {
2479
- import_id: state.import_id,
2480
- status: state.status,
2481
- batch_done: state.batch_done,
2482
- batch_total: state.batch_total,
2483
- facts_stored: state.facts_stored,
2484
- dups_skipped: state.dups_skipped,
2485
- eta_seconds: etaSeconds,
2486
- completion_iso: state.status === 'running'
2487
- ? new Date(now + etaSeconds * 1000).toISOString()
2488
- : state.last_updated,
2489
- source: state.source,
2490
- started_at: state.started_at,
2491
- errors: state.errors,
2492
- };
2493
- }
2494
- async function handleImportAbort(params, logger) {
2495
- const importId = params.import_id;
2496
- if (!importId)
2497
- return { error: 'import_id is required' };
2498
- const state = readImportState(importId);
2499
- if (!state)
2500
- return { error: `No import found with id: ${importId}` };
2501
- if (state.status === 'aborted') {
2502
- return { aborted: true, idempotent: true, import_id: importId, facts_already_stored: state.facts_stored };
2503
- }
2504
- if (state.status === 'completed') {
2505
- return { error: 'Import already completed — nothing to abort', import_id: importId, facts_stored: state.facts_stored };
2506
- }
2507
- writeImportState({ ...state, status: 'aborted' });
2508
- logger.info(`Import ${importId}: abort requested (${state.facts_stored} facts already stored)`);
2509
- return {
2510
- aborted: true,
2511
- import_id: importId,
2512
- facts_already_stored: state.facts_stored,
2513
- message: 'Import abort requested. The background task will stop at the next chunk boundary. Already-stored facts are kept.',
2514
- };
2515
- }
1586
+ configureImportRuntime({ storeExtractedFacts });
2516
1587
  // ---------------------------------------------------------------------------
2517
1588
  // buildRecallDeps — bind the real recall pipeline into the closures the
2518
1589
  // native MemoryPluginCapability wiring helper (Task 2.7) consumes.
@@ -2767,10 +1838,10 @@ function buildRecallDeps(logger) {
2767
1838
  if (!subgraphOwner)
2768
1839
  return null;
2769
1840
  try {
2770
- const result = await fetchFactById(subgraphOwner, id, authKeyHex);
2771
- if (!result)
1841
+ const fetchedFact = await fetchFactById(subgraphOwner, id, authKeyHex);
1842
+ if (!fetchedFact)
2772
1843
  return null;
2773
- const docJson = decryptFromHex(result.encryptedBlob, encryptionKey);
1844
+ const docJson = decryptFromHex(fetchedFact.encryptedBlob, encryptionKey);
2774
1845
  if (isDigestBlob(docJson))
2775
1846
  return null;
2776
1847
  const doc = readClaimFromBlob(docJson);
@@ -2844,61 +1915,7 @@ const plugin = {
2844
1915
  // `invalid config: must NOT have additional properties`, which blocked
2845
1916
  // the documented remote-pairing setup (publicUrl) and made it impossible
2846
1917
  // for a user to hand-pick an extraction model (extraction.llm.*).
2847
- configSchema: {
2848
- type: 'object',
2849
- additionalProperties: false,
2850
- properties: {
2851
- publicUrl: {
2852
- type: 'string',
2853
- description: "Public gateway URL for QR pairing (e.g. 'https://gateway.example.com:18789'). Overrides the auto-resolution cascade in buildPairingUrl.",
2854
- },
2855
- extraction: {
2856
- type: 'object',
2857
- additionalProperties: false,
2858
- properties: {
2859
- enabled: {
2860
- type: 'boolean',
2861
- description: 'Enable/disable auto-extraction (default: true)',
2862
- },
2863
- model: {
2864
- type: 'string',
2865
- description: "Shorthand: override just the extraction model (e.g., 'glm-4.5-flash', 'gpt-4.1-mini'). For a full provider override use extraction.llm.",
2866
- },
2867
- interval: {
2868
- type: 'number',
2869
- description: 'Number of turns between automatic extractions (default: 3)',
2870
- },
2871
- maxFactsPerExtraction: {
2872
- type: 'number',
2873
- description: 'Hard cap on facts extracted per turn (default: 15)',
2874
- },
2875
- llm: {
2876
- type: 'object',
2877
- additionalProperties: false,
2878
- description: 'Explicit LLM override block. Highest-priority tier in the extraction-provider cascade. Any subset of provider+apiKey is enough to pin a provider.',
2879
- properties: {
2880
- provider: {
2881
- type: 'string',
2882
- description: "Provider name: zai | openai | anthropic | gemini | google | mistral | groq | deepseek | openrouter | xai | together | cerebras.",
2883
- },
2884
- model: {
2885
- type: 'string',
2886
- description: 'Explicit model id. If omitted, deriveCheapModel(provider) picks a sensible default.',
2887
- },
2888
- apiKey: {
2889
- type: 'string',
2890
- description: 'API key for the selected provider. Required for the override to take effect.',
2891
- },
2892
- baseUrl: {
2893
- type: 'string',
2894
- description: 'Override the provider base URL (self-hosted / custom gateway setups).',
2895
- },
2896
- },
2897
- },
2898
- },
2899
- },
2900
- },
2901
- },
1918
+ configSchema: CONFIG_SCHEMA,
2902
1919
  register(api) {
2903
1920
  // NOTE: the body of register() below is intentionally NOT re-indent
2904
1921
  // under this `try` block — re-indenting would touch every line in a
@@ -3215,7 +2232,7 @@ const plugin = {
3215
2232
  // CLI subcommand actually fires.
3216
2233
  if (typeof api.registerCli === 'function') {
3217
2234
  api.registerCli(async ({ program }) => {
3218
- const { registerOnboardingCli } = await import('./onboarding-cli.js');
2235
+ const { registerOnboardingCli } = await import('./pairing/onboarding-cli.js');
3219
2236
  registerOnboardingCli(program, {
3220
2237
  credentialsPath: CREDENTIALS_PATH,
3221
2238
  statePath: CONFIG.onboardingStatePath,
@@ -3250,13 +2267,13 @@ const plugin = {
3250
2267
  // same relay-brokered URL surface the agent tool uses. The local
3251
2268
  // (gateway-loopback) flow is still available via `--local`. See
3252
2269
  // pair-cli.ts header for the rationale.
3253
- const { registerPairCli } = await import('./pair-cli.js');
2270
+ const { registerPairCli } = await import('./pairing/pair-cli.js');
3254
2271
  registerPairCli(program, {
3255
2272
  sessionsPath: CONFIG.pairSessionsPath,
3256
2273
  renderPairingUrl: (session) => buildPairingUrl(api, session),
3257
2274
  logger: api.logger,
3258
2275
  runRelayPairCli: async (cliMode, runOpts) => {
3259
- const { runRelayPairCli } = await import('./pair-cli-relay.js');
2276
+ const { runRelayPairCli } = await import('./pairing/pair-cli-relay.js');
3260
2277
  return runRelayPairCli(cliMode, {
3261
2278
  relayBaseUrl: CONFIG.pairRelayUrl,
3262
2279
  credentialsPath: CREDENTIALS_PATH,
@@ -3339,7 +2356,7 @@ const plugin = {
3339
2356
  .action(async (source, opts) => {
3340
2357
  try {
3341
2358
  await requireFullSetup(api.logger);
3342
- const result = await handlePluginImportFrom({
2359
+ const importResult = await handlePluginImportFrom({
3343
2360
  source,
3344
2361
  file_path: opts.file,
3345
2362
  content: opts.content,
@@ -3351,28 +2368,28 @@ const plugin = {
3351
2368
  disclosure_confirmed: true,
3352
2369
  }, api.logger);
3353
2370
  if (opts.json) {
3354
- process.stdout.write(JSON.stringify(result) + '\n');
2371
+ process.stdout.write(JSON.stringify(importResult) + '\n');
3355
2372
  }
3356
2373
  else {
3357
2374
  // Human-readable summary. The handler already returns a
3358
2375
  // `message` for chunked (background) imports; for direct
3359
2376
  // stores + dry runs, synthesize a short summary.
3360
- if (result.dry_run) {
3361
- const chunks = result.total_chunks;
2377
+ if (importResult.dry_run) {
2378
+ const chunks = importResult.total_chunks;
3362
2379
  if (chunks !== undefined) {
3363
- process.stdout.write(`Dry run: ~${result.estimated_facts} facts from ${chunks} chunks ` +
3364
- `(~${result.estimated_minutes} min). Confirm without --dry-run to start.\n`);
2380
+ process.stdout.write(`Dry run: ~${importResult.estimated_facts} facts from ${chunks} chunks ` +
2381
+ `(~${importResult.estimated_minutes} min). Confirm without --dry-run to start.\n`);
3365
2382
  }
3366
2383
  else {
3367
- process.stdout.write(`Dry run: found ${result.total_found} facts. Confirm without --dry-run to import.\n`);
2384
+ process.stdout.write(`Dry run: found ${importResult.total_found} facts. Confirm without --dry-run to import.\n`);
3368
2385
  }
3369
2386
  }
3370
- else if (result.import_id && result.status === 'running') {
3371
- process.stdout.write(`${result.message}\nImport id: ${result.import_id}\n`);
2387
+ else if (importResult.import_id && importResult.status === 'running') {
2388
+ process.stdout.write(`${importResult.message}\nImport id: ${importResult.import_id}\n`);
3372
2389
  }
3373
2390
  else {
3374
- const stored = result.imported;
3375
- const total = result.total_found;
2391
+ const stored = importResult.imported;
2392
+ const total = importResult.total_found;
3376
2393
  process.stdout.write(`Imported ${stored ?? 0}/${total ?? stored ?? 0} facts from ${source}.\n`);
3377
2394
  }
3378
2395
  }
@@ -3396,25 +2413,25 @@ const plugin = {
3396
2413
  .action(async (opts) => {
3397
2414
  try {
3398
2415
  await requireFullSetup(api.logger);
3399
- const result = await handleImportStatus({ import_id: opts.id }, api.logger);
2416
+ const statusResult = await handleImportStatus({ import_id: opts.id }, api.logger);
3400
2417
  if (opts.json) {
3401
- process.stdout.write(JSON.stringify(result) + '\n');
2418
+ process.stdout.write(JSON.stringify(statusResult) + '\n');
3402
2419
  }
3403
2420
  else {
3404
- const status = result.status;
3405
- const stored = result.facts_stored;
3406
- const batchDone = result.batch_done;
3407
- const batchTotal = result.batch_total;
2421
+ const status = statusResult.status;
2422
+ const stored = statusResult.facts_stored;
2423
+ const batchDone = statusResult.batch_done;
2424
+ const batchTotal = statusResult.batch_total;
3408
2425
  if (status === 'no_active_import') {
3409
2426
  process.stdout.write('No active import. Start one with `openclaw totalreclaw import from <source>`.\n');
3410
2427
  }
3411
2428
  else if (status === 'running') {
3412
- process.stdout.write(`Import ${result.import_id}: running — ${stored} facts stored, ` +
2429
+ process.stdout.write(`Import ${statusResult.import_id}: running — ${stored} facts stored, ` +
3413
2430
  `batch ${batchDone}/${batchTotal}` +
3414
- (result.completion_iso ? `, ETA ${result.completion_iso}` : '') + '.\n');
2431
+ (statusResult.completion_iso ? `, ETA ${statusResult.completion_iso}` : '') + '.\n');
3415
2432
  }
3416
2433
  else {
3417
- process.stdout.write(`Import ${result.import_id}: ${status} — ${stored ?? 0} facts stored.\n`);
2434
+ process.stdout.write(`Import ${statusResult.import_id}: ${status} — ${stored ?? 0} facts stored.\n`);
3418
2435
  }
3419
2436
  }
3420
2437
  }
@@ -3437,16 +2454,16 @@ const plugin = {
3437
2454
  .action(async (importId, opts) => {
3438
2455
  try {
3439
2456
  await requireFullSetup(api.logger);
3440
- const result = await handleImportAbort({ import_id: importId }, api.logger);
2457
+ const abortResult = await handleImportAbort({ import_id: importId }, api.logger);
3441
2458
  if (opts.json) {
3442
- process.stdout.write(JSON.stringify(result) + '\n');
2459
+ process.stdout.write(JSON.stringify(abortResult) + '\n');
3443
2460
  }
3444
2461
  else {
3445
- if (result.aborted) {
3446
- process.stdout.write(`Import ${importId}: abort requested. ${result.facts_already_stored ?? 0} facts already stored (kept).\n`);
2462
+ if (abortResult.aborted) {
2463
+ process.stdout.write(`Import ${importId}: abort requested. ${abortResult.facts_already_stored ?? 0} facts already stored (kept).\n`);
3447
2464
  }
3448
2465
  else {
3449
- process.stdout.write(`Import ${importId}: ${result.error ?? 'abort failed'}\n`);
2466
+ process.stdout.write(`Import ${importId}: ${abortResult.error ?? 'abort failed'}\n`);
3450
2467
  }
3451
2468
  }
3452
2469
  }
@@ -3489,15 +2506,15 @@ const plugin = {
3489
2506
  const body = await response.text().catch(() => '');
3490
2507
  throw new Error(`checkout session failed (HTTP ${response.status}): ${body || response.statusText}`);
3491
2508
  }
3492
- const data = await response.json();
3493
- if (!data.checkout_url) {
2509
+ const checkoutJson = await response.json();
2510
+ if (!checkoutJson.checkout_url) {
3494
2511
  throw new Error('no checkout URL returned by the relay');
3495
2512
  }
3496
2513
  if (opts.json) {
3497
- process.stdout.write(JSON.stringify({ checkout_url: data.checkout_url }) + '\n');
2514
+ process.stdout.write(JSON.stringify({ checkout_url: checkoutJson.checkout_url }) + '\n');
3498
2515
  }
3499
2516
  else {
3500
- process.stdout.write(`Open this URL to upgrade to Pro: ${data.checkout_url}\n`);
2517
+ process.stdout.write(`Open this URL to upgrade to Pro: ${checkoutJson.checkout_url}\n`);
3501
2518
  }
3502
2519
  }
3503
2520
  catch (err) {
@@ -4402,8 +3419,8 @@ const plugin = {
4402
3419
  ensureMemoryHeader(api.logger);
4403
3420
  // BUG-2 fix: skip extraction if an import was in progress this turn.
4404
3421
  // Import failures were retriggering agent_end → extraction → import loops.
4405
- if (_importInProgress) {
4406
- _importInProgress = false; // auto-reset for next turn
3422
+ if (isImportInProgress()) {
3423
+ setImportInProgress(false); // auto-reset for next turn
4407
3424
  api.logger.info('agent_end: skipping extraction (import was in progress)');
4408
3425
  return { memoryHandled: true };
4409
3426
  }
@@ -4488,7 +3505,7 @@ const plugin = {
4488
3505
  logger: api.logger,
4489
3506
  ensureInitialized: () => ensureInitialized(api.logger),
4490
3507
  isPairingPending: () => needsSetup,
4491
- isImportActive: () => _importInProgress,
3508
+ isImportActive: () => isImportInProgress(),
4492
3509
  getExtractInterval,
4493
3510
  getMaxFactsPerExtraction,
4494
3511
  isDedupEnabled: isLlmDedupEnabled,