openclaw-hybrid-memory 2026.5.310 → 2026.6.10

package/dist/cli/commands/manage/register-storage-maintenance.js

@@ -1,19 +1,22 @@
 import { getEnv } from "../../../utils/env-manager.js";
 import { capturePluginError } from "../../../services/error-reporter.js";
-import { is500OrWrapped } from "../../../services/chat.js";
+import { is403QuotaOrRateLimitLike, parseRetryAfterMs } from "../../../services/llm-rate-limit-headers.js";
+import { is429OrWrapped, is500OrWrapped, isConnectionErrorLike } from "../../../services/chat.js";
 import { AllEmbeddingProvidersFailed } from "../../../services/embeddings/types.js";
 import "../../../services/embeddings.js";
 import { recordMaintenanceTimestamp } from "../../../services/maintenance-timestamp.js";
 import { deleteVectorsForFactIds } from "../../../services/vector-maintenance.js";
 import { approxIntervalMs, withExit } from "../../shared.js";
 import { appendVectorLifecycleAuditEvent } from "../../../services/vector-lifecycle-audit.js";
+import { computeAdaptivePressureDelayMs } from "../../../services/adaptive-catch-up-pacing.js";
+import { buildVectorlessSloRepairRecommendation } from "../../../services/entity-enrichment-adaptive.js";
 import { readHybridMemVerbose } from "../../global-verbose.js";
 import { runMaintenanceHeartbeat } from "./maintenance-heartbeat.js";
 import { countPendingReviewBacklogs } from "../../../services/pending-review-digest.js";
+import { isPreStoreGuardBlocked } from "../../../backends/facts-db/crud.js";
 import { migrateEmbeddings } from "../../../services/embedding-migration.js";
 import { collectVectorBackendObservability } from "../../../services/vector-backend-observability.js";
 import { embedCallWithTimeoutAndRetry } from "../../../utils/embed-call.js";
-import { isPreStoreGuardBlocked } from "../../../backends/facts-db/crud.js";
 import { countImplicitFeedbackTrajectorySignals, defaultReindexCheckpointPath, parseBoundedFloatOption, parseBoundedIntOption, readReindexCheckpoint, recordStorageGrowthSample, writeReindexCheckpoint } from "./storage-stats-helpers.js";
 import { existsSync, unlinkSync } from "node:fs";
 //#region cli/commands/manage/register-storage-maintenance.ts
@@ -46,6 +49,33 @@ function isEmbeddingProviderServerError(err) {
 	if (err instanceof AllEmbeddingProvidersFailed) return err.causes.some((cause) => is500OrWrapped(cause));
 	return false;
 }
+function isEmbeddingProviderRateLimitError(err) {
+	if (!(err instanceof Error)) return false;
+	if (is429OrWrapped(err) || is403QuotaOrRateLimitLike(err)) return true;
+	if (err instanceof AllEmbeddingProvidersFailed) return err.causes.some((cause) => cause instanceof Error && (is429OrWrapped(cause) || is403QuotaOrRateLimitLike(cause)));
+	return false;
+}
+function isEmbeddingProviderTransientError(err) {
+	if (!(err instanceof Error)) return false;
+	if (isEmbeddingProviderRateLimitError(err) || is500OrWrapped(err) || isConnectionErrorLike(err)) return true;
+	if (err instanceof AllEmbeddingProvidersFailed) return err.causes.some((cause) => cause instanceof Error && (is429OrWrapped(cause) || is403QuotaOrRateLimitLike(cause) || is500OrWrapped(cause) || isConnectionErrorLike(cause)));
+	return false;
+}
+function parseEmbeddingRetryAfterMs(err) {
+	if (err instanceof Error) {
+		const fromTop = parseRetryAfterMs(err);
+		if (fromTop !== void 0) return fromTop;
+	}
+	if (err instanceof AllEmbeddingProvidersFailed) for (const cause of err.causes) {
+		if (!(cause instanceof Error)) continue;
+		const retryAfterMs = parseRetryAfterMs(cause);
+		if (retryAfterMs !== void 0) return retryAfterMs;
+	}
+}
+function sleep(ms) {
+	if (ms <= 0) return Promise.resolve();
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
 function registerManageStorageMaintenance(mem, b) {
 	const { factsDb, vectorDb, aliasDb, versionInfo, embeddings, mergeResults: merge, getMemoryCategories, cfg, runCompaction, tieringEnabled, ctx, listCommands, auditStore, runExport } = b;
 	const tierCompactCmd = mem.command("tier-compact").description("Tier compaction: move facts between hot/warm/cold/structural (does NOT shrink LanceDB — see vectordb-optimize)").option("--dry-run", "Preview tier changes without mutating facts");
@@ -377,11 +407,21 @@ function registerManageStorageMaintenance(mem, b) {
 			}
 		}
 	}));
-	mem.command("reembed-vectorless").description("Embed active non-kv facts that lack canonical embeddings (dry-run by default)").option("--limit <n>", "Maximum vectorless facts to process", "100").option("--source <s>", "Only process facts from this source").option("--apply", "Actually embed and write LanceDB/fact_embeddings rows; default is dry-run").option("--batch-size <n>", "Embedding batch size", "40").option("--max-embed-failures <n>", "Circuit-break after this many consecutive embed failures and exit 1; prevents indefinite stall on provider outage (default: 5)", "5").option("-v, --verbose", "Emit periodic progress heartbeat for long runs").option("--json", "Emit JSON").action(withExit(async (opts, cmd) => {
+	mem.command("reembed-vectorless").description("Embed active non-kv facts that lack canonical embeddings (dry-run by default)").option("--limit <n>", "Maximum vectorless facts to process", "100").option("--source <s>", "Only process facts from this source").option("--apply", "Actually embed and write LanceDB/fact_embeddings rows; default is dry-run").option("--batch-size <n>", "Embedding batch size", "40").option("--batch-delay-ms <n>", "Delay between adaptive batches in ms (default 150)", "150").option("--adaptive-catch-up", "Enable adaptive pacing (ramps up on healthy batches; backs off on provider/rate-limit/store pressure)").option("--max-embed-failures <n>", "Circuit-break after this many consecutive embed failures and exit 1; prevents indefinite stall on provider outage (default: 5)", "5").option("-v, --verbose", "Emit periodic progress heartbeat for long runs").option("--json", "Emit JSON").action(withExit(async (opts, cmd) => {
 		const limit = Number.parseInt(opts?.limit ?? "100", 10);
 		const batchSize = Number.parseInt(opts?.batchSize ?? "40", 10);
+		const batchDelayMs = Number.parseInt(opts?.batchDelayMs ?? "150", 10);
+		const adaptiveCatchUp = opts?.adaptiveCatchUp === true;
 		const maxEmbedFailures = Number.parseInt(opts?.maxEmbedFailures ?? "5", 10);
 		const verbose = !!opts?.verbose || readHybridMemVerbose(cmd);
+		const adaptiveMinBatchSize = 1;
+		const adaptiveMaxBatchSize = 200;
+		const adaptiveMinDelayMs = 0;
+		const adaptiveMaxDelayMs = 5e3;
+		const adaptiveBackoffMinDelayMs = 50;
+		const adaptiveBatchSizeStep = 5;
+		const adaptiveDelayStepMs = 25;
+		const adaptiveSuccessStreakForRampUp = 2;
 		if (!Number.isFinite(limit) || limit < 1) {
 			console.error("error: --limit must be a positive integer");
 			process.exitCode = 1;
@@ -392,12 +432,18 @@ function registerManageStorageMaintenance(mem, b) {
 			process.exitCode = 1;
 			return;
 		}
+		if (!Number.isFinite(batchDelayMs) || batchDelayMs < 0) {
+			console.error("error: --batch-delay-ms must be a non-negative integer");
+			process.exitCode = 1;
+			return;
+		}
 		if (!Number.isFinite(maxEmbedFailures) || maxEmbedFailures < 1) {
 			console.error("error: --max-embed-failures must be a positive integer");
 			process.exitCode = 1;
 			return;
 		}
 		const before = factsDb.countVectorlessActiveFacts(opts?.source);
+		const globalVectorlessBefore = factsDb.countVectorlessActiveFacts();
 		const candidates = factsDb.listVectorlessActiveFacts({
 			limit,
 			source: opts?.source
@@ -407,7 +453,17 @@ function registerManageStorageMaintenance(mem, b) {
 		let skipped = 0;
 		let storeFailures = 0;
 		let embedFailures = 0;
-		const totalBatches = batchSize > 0 ? Math.ceil(candidates.length / batchSize) : 0;
+		const startedAtMs = Date.now();
+		const baselineBatchSize = batchSize;
+		const baselineDelayMs = batchDelayMs;
+		let effectiveBatchSize = batchSize;
+		let effectiveDelayMs = batchDelayMs;
+		if (adaptiveCatchUp) {
+			effectiveBatchSize = Math.max(adaptiveMinBatchSize, Math.min(adaptiveMaxBatchSize, batchSize));
+			effectiveDelayMs = Math.max(adaptiveMinDelayMs, Math.min(adaptiveMaxDelayMs, batchDelayMs));
+		}
+		const adaptiveAdjustments = [];
+		let successStreak = 0;
 		let processed = 0;
 		let batchNumber = 0;
 		let providerCircuitBreak = false;
@@ -416,15 +472,28 @@ function registerManageStorageMaintenance(mem, b) {
 		let aborted = false;
 		if (opts?.apply) await runMaintenanceHeartbeat("reembed-vectorless", verbose, async () => {
 			await vectorDb.runWithAutoOptimizePaused(async () => {
-				for (let offset = 0; offset < candidates.length; offset += batchSize) {
+				let offset = 0;
+				while (offset < candidates.length) {
 					if (providerCircuitBreak) break;
-					batchNumber = Math.floor(offset / batchSize) + 1;
-					const batch = candidates.slice(offset, offset + batchSize);
+					batchNumber++;
+					const batch = candidates.slice(offset, offset + effectiveBatchSize);
 					let vectors;
+					let batchPressureSignals = 0;
+					let batchRateLimited = 0;
+					let batchTransientFailures = 0;
+					let batchStoreFailures = 0;
+					let batchEmbedFailures = 0;
+					let batchRetryAfterMs;
 					try {
 						vectors = await embedCallWithTimeoutAndRetry(() => embeddings.embedBatch(batch.map((fact) => fact.text)), `reembed-vectorless:batch-${batchNumber}`);
 						consecutiveEmbedFailures = 0;
 					} catch (batchErr) {
+						const batchRateLimitedErr = isEmbeddingProviderRateLimitError(batchErr);
+						const batchTransientErr = isEmbeddingProviderTransientError(batchErr);
+						batchRetryAfterMs = parseEmbeddingRetryAfterMs(batchErr);
+						if (batchRateLimitedErr) batchRateLimited++;
+						if (batchTransientErr) batchTransientFailures++;
+						if (batchRateLimitedErr || batchTransientErr || batchRetryAfterMs !== void 0) batchPressureSignals++;
 						if (isEmbeddingProviderServerError(batchErr)) {
 							providerCircuitBreak = true;
 							providerCircuitBreakCause = "provider_5xx";
@@ -439,7 +508,15 @@ function registerManageStorageMaintenance(mem, b) {
 						} catch (singleErr) {
 							errors.push(`fact ${fact.id}: embed failed — ${String(singleErr)}`);
 							embedFailures++;
+							batchEmbedFailures++;
 							vectors.push(null);
+							const rateLimited = isEmbeddingProviderRateLimitError(singleErr);
+							const transient = isEmbeddingProviderTransientError(singleErr);
+							const retryAfterMs = parseEmbeddingRetryAfterMs(singleErr);
+							if (rateLimited) batchRateLimited++;
+							if (transient) batchTransientFailures++;
+							if (rateLimited || transient || retryAfterMs !== void 0) batchPressureSignals++;
+							if (retryAfterMs !== void 0) batchRetryAfterMs = Math.max(batchRetryAfterMs ?? 0, retryAfterMs);
 							if (isEmbeddingProviderServerError(singleErr)) {
 								providerCircuitBreak = true;
 								providerCircuitBreakCause = "provider_5xx";
@@ -481,22 +558,63 @@ function registerManageStorageMaintenance(mem, b) {
 						} catch (err) {
 							errors.push(`fact ${fact.id}: store failed — ${String(err)}`);
 							storeFailures++;
+							batchStoreFailures++;
 							skipped++;
 						} finally {
 							processed++;
 						}
 					}
+					if (adaptiveCatchUp) {
+						const hadPressure = batchPressureSignals > 0 || batchStoreFailures > 0 || batchEmbedFailures > 0;
+						const previousBatchSize = effectiveBatchSize;
+						const previousDelayMs = effectiveDelayMs;
+						if (hadPressure) {
+							successStreak = 0;
+							effectiveBatchSize = Math.max(adaptiveMinBatchSize, Math.floor(effectiveBatchSize / 2));
+							effectiveDelayMs = computeAdaptivePressureDelayMs({
+								currentDelayMs: effectiveDelayMs,
+								batchRetryAfterMs,
+								maxAdaptiveDelayMs: adaptiveMaxDelayMs,
+								backoffMinDelayMs: adaptiveBackoffMinDelayMs
+							});
+						} else {
+							successStreak++;
+							if (successStreak >= adaptiveSuccessStreakForRampUp) {
+								successStreak = 0;
+								effectiveBatchSize = Math.min(adaptiveMaxBatchSize, effectiveBatchSize + adaptiveBatchSizeStep);
+								effectiveDelayMs = Math.max(adaptiveMinDelayMs, effectiveDelayMs - adaptiveDelayStepMs);
+							}
+						}
+						if (previousBatchSize !== effectiveBatchSize || previousDelayMs !== effectiveDelayMs) adaptiveAdjustments.push({
+							reason: hadPressure ? "pressure" : "ramp-up",
+							previousBatchSize,
+							previousDelayMs,
+							batchSize: effectiveBatchSize,
+							delayMs: effectiveDelayMs,
+							batchPressureSignals,
+							batchRateLimited,
+							batchTransientFailures,
+							batchStoreFailures,
+							batchEmbedFailures,
+							retryAfterMs: batchRetryAfterMs,
+							batchNumber
+						});
+					}
+					offset += batch.length;
+					if (adaptiveCatchUp && offset < candidates.length && !providerCircuitBreak) await sleep(effectiveDelayMs);
 				}
 			});
 		}, {
-			progressSupplier: () => `stage=embed-and-store; processed=${processed}/${candidates.length}; embedded=${embedded}; skipped=${skipped}; embedFailures=${embedFailures}; storeFailures=${storeFailures}; batch=${batchNumber}/${totalBatches}`,
+			progressSupplier: () => `stage=embed-and-store; processed=${processed}/${candidates.length}; remaining=${Math.max(0, candidates.length - processed)}; embedded=${embedded}; skipped=${skipped}; embedFailures=${embedFailures}; storeFailures=${storeFailures}; batch=${batchNumber}; batchSize=${effectiveBatchSize}; delayMs=${effectiveDelayMs}`,
 			jsonMode: opts?.json === true
 		});
 		const after = opts?.apply ? factsDb.countVectorlessActiveFacts(opts?.source) : before;
+		const durationMs = Date.now() - startedAtMs;
 		const report = {
 			apply: opts?.apply === true,
 			source: opts?.source ?? null,
 			before,
+			candidates: candidates.length,
 			considered: candidates.length,
 			embedded,
 			skipped,
@@ -505,8 +623,30 @@ function registerManageStorageMaintenance(mem, b) {
 			errors,
 			after,
 			processed,
+			remaining: after,
+			durationMs,
 			aborted
 		};
+		if (adaptiveCatchUp) report.adaptive = {
+			enabled: true,
+			baselineBatchSize,
+			baselineDelayMs,
+			effectiveBatchSize,
+			effectiveDelayMs,
+			adjustments: adaptiveAdjustments
+		};
+		const globalVectorlessAfter = opts?.apply ? factsDb.countVectorlessActiveFacts() : globalVectorlessBefore;
+		report.vectorSloRepair = buildVectorlessSloRepairRecommendation({
+			activeFacts: factsDb.getCount(),
+			vectorlessBefore: globalVectorlessBefore,
+			vectorlessAfter: globalVectorlessAfter,
+			embeddedThisRun: embedded,
+			runLimit: limit,
+			effectiveBatchSize: adaptiveCatchUp ? effectiveBatchSize : batchSize,
+			scopedSource: opts?.source,
+			scopedVectorlessBefore: opts?.source ? before : void 0,
+			scopedVectorlessAfter: opts?.source ? after : void 0
+		});
 		if (providerCircuitBreak) {
 			report.failedReason = providerCircuitBreakCause === "provider_5xx" ? "failed_embedding_provider_5xx" : "failed_embedding_provider";
 			process.exitCode = 1;
@@ -515,7 +655,10 @@ function registerManageStorageMaintenance(mem, b) {
 			console.log(JSON.stringify(report, null, 2));
 			return;
 		}
-		console.log(`Reembed vectorless ${report.apply ? "applied" : "dry-run"}: before ${before}, candidates ${candidates.length}, embedded ${embedded}, skipped ${skipped}, storeFailures ${storeFailures}, after ${after}`);
+		console.log(`Reembed vectorless ${report.apply ? "applied" : "dry-run"}: before ${before}, candidates ${candidates.length}, embedded ${embedded}, skipped ${skipped}, storeFailures ${storeFailures}, after ${after}, remaining ${after}, durationMs ${durationMs}`);
+		if (adaptiveCatchUp) console.log(`Adaptive pacing: batchSize ${baselineBatchSize}->${effectiveBatchSize}, delayMs ${baselineDelayMs}->${effectiveDelayMs}, adjustments ${adaptiveAdjustments.length}`);
+		const slo = report.vectorSloRepair;
+		console.log(`Vectorless SLO repair: ratio ${(slo.vectorlessRatioAfter * 100).toFixed(2)}% (target ${(slo.targetVectorlessRatio * 100).toFixed(0)}%), clear ${slo.vectorlessToClearForSlo} more, ~${slo.estimatedRunsToReachSlo} run(s) at limit ${slo.recommendedLimitNextRun} batch ${slo.recommendedBatchSizeNextRun}`);
 		if (candidates.length > 0 && !opts?.apply) {
 			console.log("Examples:");
 			for (const fact of candidates.slice(0, 10)) console.log(`  ${fact.id}  ${fact.source}  ${fact.text.slice(0, 80).replace(/\s+/g, " ")}`);